1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright 2020, The University of Queensland 14 * Copyright (c) 2018, Joyent, Inc. 15 * Copyright 2020 RackTop Systems, Inc. 16 */ 17 18 /* 19 * Controls the management of commands that are issues to and from the HCA 20 * command queue. 21 */ 22 23 #include <mlxcx.h> 24 25 #include <sys/debug.h> 26 #include <sys/sysmacros.h> 27 28 /* 29 * When we start up the command queue, it will undergo some internal 30 * initialization after we set the command queue address. These values allow us 31 * to control how much time we should wait for that to occur. 32 */ 33 clock_t mlxcx_cmd_init_delay = 1000 * 10; /* 10 ms in us */ 34 uint_t mlxcx_cmd_init_trys = 100; /* Wait at most 1s */ 35 36 clock_t mlxcx_cmd_delay = 1000 * 1; /* 1 ms in us */ 37 uint_t mlxcx_cmd_tries = 5000; /* Wait at most 1s */ 38 39 /* 40 * This macro is used to identify that we care about our own function that we're 41 * communicating with. We always use this function. 42 */ 43 #define MLXCX_FUNCTION_SELF (to_be16(0)) 44 45 static const char * 46 mlxcx_cmd_response_string(mlxcx_cmd_ret_t ret) 47 { 48 switch (ret) { 49 case MLXCX_CMD_R_OK: 50 return ("MLXCX_CMD_R_OK"); 51 case MLXCX_CMD_R_INTERNAL_ERR: 52 return ("MLXCX_CMD_R_INTERNAL_ERR"); 53 case MLXCX_CMD_R_BAD_OP: 54 return ("MLXCX_CMD_R_BAD_OP"); 55 case MLXCX_CMD_R_BAD_PARAM: 56 return ("MLXCX_CMD_R_BAD_PARAM"); 57 case MLXCX_CMD_R_BAD_SYS_STATE: 58 return ("MLXCX_CMD_R_BAD_SYS_STATE"); 59 case MLXCX_CMD_R_BAD_RESOURCE: 60 return ("MLXCX_CMD_R_BAD_RESOURCE"); 61 case MLXCX_CMD_R_RESOURCE_BUSY: 62 return ("MLXCX_CMD_R_RESOURCE_BUSY"); 63 case MLXCX_CMD_R_EXCEED_LIM: 64 return ("MLXCX_CMD_R_EXCEED_LIM"); 65 case MLXCX_CMD_R_BAD_RES_STATE: 66 return ("MLXCX_CMD_R_BAD_RES_STATE"); 67 case MLXCX_CMD_R_BAD_INDEX: 68 return ("MLXCX_CMD_R_BAD_INDEX"); 69 case MLXCX_CMD_R_NO_RESOURCES: 70 return ("MLXCX_CMD_R_NO_RESOURCES"); 71 case MLXCX_CMD_R_BAD_INPUT_LEN: 72 return ("MLXCX_CMD_R_BAD_INPUT_LEN"); 73 case MLXCX_CMD_R_BAD_OUTPUT_LEN: 74 return ("MLXCX_CMD_R_BAD_OUTPUT_LEN"); 75 case MLXCX_CMD_R_BAD_RESOURCE_STATE: 76 return ("MLXCX_CMD_R_BAD_RESOURCE_STATE"); 77 case MLXCX_CMD_R_BAD_PKT: 78 return ("MLXCX_CMD_R_BAD_PKT"); 79 case MLXCX_CMD_R_BAD_SIZE: 80 return ("MLXCX_CMD_R_BAD_SIZE"); 81 default: 82 return ("Unknown command"); 83 } 84 } 85 86 static const char * 87 mlxcx_cmd_opcode_string(mlxcx_cmd_op_t op) 88 { 89 switch (op) { 90 case MLXCX_OP_QUERY_HCA_CAP: 91 return ("MLXCX_OP_QUERY_HCA_CAP"); 92 case MLXCX_OP_QUERY_ADAPTER: 93 return ("MLXCX_OP_QUERY_ADAPTER"); 94 case MLXCX_OP_INIT_HCA: 95 return ("MLXCX_OP_INIT_HCA"); 96 case MLXCX_OP_TEARDOWN_HCA: 97 return ("MLXCX_OP_TEARDOWN_HCA"); 98 case MLXCX_OP_ENABLE_HCA: 99 return ("MLXCX_OP_ENABLE_HCA"); 100 case MLXCX_OP_DISABLE_HCA: 101 return ("MLXCX_OP_DISABLE_HCA"); 102 case MLXCX_OP_QUERY_PAGES: 103 return ("MLXCX_OP_QUERY_PAGES"); 104 case MLXCX_OP_MANAGE_PAGES: 105 return ("MLXCX_OP_MANAGE_PAGES"); 106 case MLXCX_OP_SET_HCA_CAP: 107 return ("MLXCX_OP_SET_HCA_CAP"); 108 case MLXCX_OP_QUERY_ISSI: 109 return ("MLXCX_OP_QUERY_ISSI"); 110 case MLXCX_OP_SET_ISSI: 111 return ("MLXCX_OP_SET_ISSI"); 112 case MLXCX_OP_SET_DRIVER_VERSION: 113 return ("MLXCX_OP_SET_DRIVER_VERSION"); 114 case MLXCX_OP_QUERY_OTHER_HCA_CAP: 115 return ("MLXCX_OP_QUERY_OTHER_HCA_CAP"); 116 case MLXCX_OP_MODIFY_OTHER_HCA_CAP: 117 return ("MLXCX_OP_MODIFY_OTHER_HCA_CAP"); 118 case MLXCX_OP_SET_TUNNELED_OPERATIONS: 119 return ("MLXCX_OP_SET_TUNNELED_OPERATIONS"); 120 case MLXCX_OP_CREATE_MKEY: 121 return ("MLXCX_OP_CREATE_MKEY"); 122 case MLXCX_OP_QUERY_MKEY: 123 return ("MLXCX_OP_QUERY_MKEY"); 124 case MLXCX_OP_DESTROY_MKEY: 125 return ("MLXCX_OP_DESTROY_MKEY"); 126 case MLXCX_OP_QUERY_SPECIAL_CONTEXTS: 127 return ("MLXCX_OP_QUERY_SPECIAL_CONTEXTS"); 128 case MLXCX_OP_PAGE_FAULT_RESUME: 129 return ("MLXCX_OP_PAGE_FAULT_RESUME"); 130 case MLXCX_OP_CREATE_EQ: 131 return ("MLXCX_OP_CREATE_EQ"); 132 case MLXCX_OP_DESTROY_EQ: 133 return ("MLXCX_OP_DESTROY_EQ"); 134 case MLXCX_OP_QUERY_EQ: 135 return ("MLXCX_OP_QUERY_EQ"); 136 case MLXCX_OP_GEN_EQE: 137 return ("MLXCX_OP_GEN_EQE"); 138 case MLXCX_OP_CREATE_CQ: 139 return ("MLXCX_OP_CREATE_CQ"); 140 case MLXCX_OP_DESTROY_CQ: 141 return ("MLXCX_OP_DESTROY_CQ"); 142 case MLXCX_OP_QUERY_CQ: 143 return ("MLXCX_OP_QUERY_CQ"); 144 case MLXCX_OP_MODIFY_CQ: 145 return ("MLXCX_OP_MODIFY_CQ"); 146 case MLXCX_OP_CREATE_QP: 147 return ("MLXCX_OP_CREATE_QP"); 148 case MLXCX_OP_DESTROY_QP: 149 return ("MLXCX_OP_DESTROY_QP"); 150 case MLXCX_OP_RST2INIT_QP: 151 return ("MLXCX_OP_RST2INIT_QP"); 152 case MLXCX_OP_INIT2RTR_QP: 153 return ("MLXCX_OP_INIT2RTR_QP"); 154 case MLXCX_OP_RTR2RTS_QP: 155 return ("MLXCX_OP_RTR2RTS_QP"); 156 case MLXCX_OP_RTS2RTS_QP: 157 return ("MLXCX_OP_RTS2RTS_QP"); 158 case MLXCX_OP_SQERR2RTS_QP: 159 return ("MLXCX_OP_SQERR2RTS_QP"); 160 case MLXCX_OP__2ERR_QP: 161 return ("MLXCX_OP__2ERR_QP"); 162 case MLXCX_OP__2RST_QP: 163 return ("MLXCX_OP__2RST_QP"); 164 case MLXCX_OP_QUERY_QP: 165 return ("MLXCX_OP_QUERY_QP"); 166 case MLXCX_OP_SQD_RTS_QP: 167 return ("MLXCX_OP_SQD_RTS_QP"); 168 case MLXCX_OP_INIT2INIT_QP: 169 return ("MLXCX_OP_INIT2INIT_QP"); 170 case MLXCX_OP_CREATE_PSV: 171 return ("MLXCX_OP_CREATE_PSV"); 172 case MLXCX_OP_DESTROY_PSV: 173 return ("MLXCX_OP_DESTROY_PSV"); 174 case MLXCX_OP_CREATE_SRQ: 175 return ("MLXCX_OP_CREATE_SRQ"); 176 case MLXCX_OP_DESTROY_SRQ: 177 return ("MLXCX_OP_DESTROY_SRQ"); 178 case MLXCX_OP_QUERY_SRQ: 179 return ("MLXCX_OP_QUERY_SRQ"); 180 case MLXCX_OP_ARM_RQ: 181 return ("MLXCX_OP_ARM_RQ"); 182 case MLXCX_OP_CREATE_XRC_SRQ: 183 return ("MLXCX_OP_CREATE_XRC_SRQ"); 184 case MLXCX_OP_DESTROY_XRC_SRQ: 185 return ("MLXCX_OP_DESTROY_XRC_SRQ"); 186 case MLXCX_OP_QUERY_XRC_SRQ: 187 return ("MLXCX_OP_QUERY_XRC_SRQ"); 188 case MLXCX_OP_ARM_XRC_SRQ: 189 return ("MLXCX_OP_ARM_XRC_SRQ"); 190 case MLXCX_OP_CREATE_DCT: 191 return ("MLXCX_OP_CREATE_DCT"); 192 case MLXCX_OP_DESTROY_DCT: 193 return ("MLXCX_OP_DESTROY_DCT"); 194 case MLXCX_OP_DRAIN_DCT: 195 return ("MLXCX_OP_DRAIN_DCT"); 196 case MLXCX_OP_QUERY_DCT: 197 return ("MLXCX_OP_QUERY_DCT"); 198 case MLXCX_OP_ARM_DCT_FOR_KEY_VIOLATION: 199 return ("MLXCX_OP_ARM_DCT_FOR_KEY_VIOLATION"); 200 case MLXCX_OP_CREATE_XRQ: 201 return ("MLXCX_OP_CREATE_XRQ"); 202 case MLXCX_OP_DESTROY_XRQ: 203 return ("MLXCX_OP_DESTROY_XRQ"); 204 case MLXCX_OP_QUERY_XRQ: 205 return ("MLXCX_OP_QUERY_XRQ"); 206 case MLXCX_OP_CREATE_NVMF_BACKEND_CONTROLLER: 207 return ("MLXCX_OP_CREATE_NVMF_BACKEND_CONTROLLER"); 208 case MLXCX_OP_DESTROY_NVMF_BACKEND_CONTROLLER: 209 return ("MLXCX_OP_DESTROY_NVMF_BACKEND_CONTROLLER"); 210 case MLXCX_OP_QUERY_NVMF_BACKEND_CONTROLLER: 211 return ("MLXCX_OP_QUERY_NVMF_BACKEND_CONTROLLER"); 212 case MLXCX_OP_ATTACH_NVMF_NAMESPACE: 213 return ("MLXCX_OP_ATTACH_NVMF_NAMESPACE"); 214 case MLXCX_OP_DETACH_NVMF_NAMESPACE: 215 return ("MLXCX_OP_DETACH_NVMF_NAMESPACE"); 216 case MLXCX_OP_QUERY_XRQ_DC_PARAMS_ENTRY: 217 return ("MLXCX_OP_QUERY_XRQ_DC_PARAMS_ENTRY"); 218 case MLXCX_OP_SET_XRQ_DC_PARAMS_ENTRY: 219 return ("MLXCX_OP_SET_XRQ_DC_PARAMS_ENTRY"); 220 case MLXCX_OP_QUERY_XRQ_ERROR_PARAMS: 221 return ("MLXCX_OP_QUERY_XRQ_ERROR_PARAMS"); 222 case MLXCX_OP_QUERY_VPORT_STATE: 223 return ("MLXCX_OP_QUERY_VPORT_STATE"); 224 case MLXCX_OP_MODIFY_VPORT_STATE: 225 return ("MLXCX_OP_MODIFY_VPORT_STATE"); 226 case MLXCX_OP_QUERY_ESW_VPORT_CONTEXT: 227 return ("MLXCX_OP_QUERY_ESW_VPORT_CONTEXT"); 228 case MLXCX_OP_MODIFY_ESW_VPORT_CONTEXT: 229 return ("MLXCX_OP_MODIFY_ESW_VPORT_CONTEXT"); 230 case MLXCX_OP_QUERY_NIC_VPORT_CONTEXT: 231 return ("MLXCX_OP_QUERY_NIC_VPORT_CONTEXT"); 232 case MLXCX_OP_MODIFY_NIC_VPORT_CONTEXT: 233 return ("MLXCX_OP_MODIFY_NIC_VPORT_CONTEXT"); 234 case MLXCX_OP_QUERY_ROCE_ADDRESS: 235 return ("MLXCX_OP_QUERY_ROCE_ADDRESS"); 236 case MLXCX_OP_SET_ROCE_ADDRESS: 237 return ("MLXCX_OP_SET_ROCE_ADDRESS"); 238 case MLXCX_OP_QUERY_HCA_VPORT_CONTEXT: 239 return ("MLXCX_OP_QUERY_HCA_VPORT_CONTEXT"); 240 case MLXCX_OP_MODIFY_HCA_VPORT_CONTEXT: 241 return ("MLXCX_OP_MODIFY_HCA_VPORT_CONTEXT"); 242 case MLXCX_OP_QUERY_HCA_VPORT_GID: 243 return ("MLXCX_OP_QUERY_HCA_VPORT_GID"); 244 case MLXCX_OP_QUERY_HCA_VPORT_PKEY: 245 return ("MLXCX_OP_QUERY_HCA_VPORT_PKEY"); 246 case MLXCX_OP_QUERY_VPORT_COUNTER: 247 return ("MLXCX_OP_QUERY_VPORT_COUNTER"); 248 case MLXCX_OP_ALLOC_Q_COUNTER: 249 return ("MLXCX_OP_ALLOC_Q_COUNTER"); 250 case MLXCX_OP_DEALLOC_Q_COUNTER: 251 return ("MLXCX_OP_DEALLOC_Q_COUNTER"); 252 case MLXCX_OP_QUERY_Q_COUNTER: 253 return ("MLXCX_OP_QUERY_Q_COUNTER"); 254 case MLXCX_OP_SET_PP_RATE_LIMIT: 255 return ("MLXCX_OP_SET_PP_RATE_LIMIT"); 256 case MLXCX_OP_QUERY_PP_RATE_LIMIT: 257 return ("MLXCX_OP_QUERY_PP_RATE_LIMIT"); 258 case MLXCX_OP_ALLOC_PD: 259 return ("MLXCX_OP_ALLOC_PD"); 260 case MLXCX_OP_DEALLOC_PD: 261 return ("MLXCX_OP_DEALLOC_PD"); 262 case MLXCX_OP_ALLOC_UAR: 263 return ("MLXCX_OP_ALLOC_UAR"); 264 case MLXCX_OP_DEALLOC_UAR: 265 return ("MLXCX_OP_DEALLOC_UAR"); 266 case MLXCX_OP_CONFIG_INT_MODERATION: 267 return ("MLXCX_OP_CONFIG_INT_MODERATION"); 268 case MLXCX_OP_ACCESS_REG: 269 return ("MLXCX_OP_ACCESS_REG"); 270 case MLXCX_OP_ATTACH_TO_MCG: 271 return ("MLXCX_OP_ATTACH_TO_MCG"); 272 case MLXCX_OP_DETACH_FROM_MCG: 273 return ("MLXCX_OP_DETACH_FROM_MCG"); 274 case MLXCX_OP_MAD_IFC: 275 return ("MLXCX_OP_MAD_IFC"); 276 case MLXCX_OP_QUERY_MAD_DEMUX: 277 return ("MLXCX_OP_QUERY_MAD_DEMUX"); 278 case MLXCX_OP_SET_MAD_DEMUX: 279 return ("MLXCX_OP_SET_MAD_DEMUX"); 280 case MLXCX_OP_NOP: 281 return ("MLXCX_OP_NOP"); 282 case MLXCX_OP_ALLOC_XRCD: 283 return ("MLXCX_OP_ALLOC_XRCD"); 284 case MLXCX_OP_DEALLOC_XRCD: 285 return ("MLXCX_OP_DEALLOC_XRCD"); 286 case MLXCX_OP_ALLOC_TRANSPORT_DOMAIN: 287 return ("MLXCX_OP_ALLOC_TRANSPORT_DOMAIN"); 288 case MLXCX_OP_DEALLOC_TRANSPORT_DOMAIN: 289 return ("MLXCX_OP_DEALLOC_TRANSPORT_DOMAIN"); 290 case MLXCX_OP_QUERY_CONG_STATUS: 291 return ("MLXCX_OP_QUERY_CONG_STATUS"); 292 case MLXCX_OP_MODIFY_CONG_STATUS: 293 return ("MLXCX_OP_MODIFY_CONG_STATUS"); 294 case MLXCX_OP_QUERY_CONG_PARAMS: 295 return ("MLXCX_OP_QUERY_CONG_PARAMS"); 296 case MLXCX_OP_MODIFY_CONG_PARAMS: 297 return ("MLXCX_OP_MODIFY_CONG_PARAMS"); 298 case MLXCX_OP_QUERY_CONG_STATISTICS: 299 return ("MLXCX_OP_QUERY_CONG_STATISTICS"); 300 case MLXCX_OP_ADD_VXLAN_UDP_DPORT: 301 return ("MLXCX_OP_ADD_VXLAN_UDP_DPORT"); 302 case MLXCX_OP_DELETE_VXLAN_UDP_DPORT: 303 return ("MLXCX_OP_DELETE_VXLAN_UDP_DPORT"); 304 case MLXCX_OP_SET_L2_TABLE_ENTRY: 305 return ("MLXCX_OP_SET_L2_TABLE_ENTRY"); 306 case MLXCX_OP_QUERY_L2_TABLE_ENTRY: 307 return ("MLXCX_OP_QUERY_L2_TABLE_ENTRY"); 308 case MLXCX_OP_DELETE_L2_TABLE_ENTRY: 309 return ("MLXCX_OP_DELETE_L2_TABLE_ENTRY"); 310 case MLXCX_OP_SET_WOL_ROL: 311 return ("MLXCX_OP_SET_WOL_ROL"); 312 case MLXCX_OP_QUERY_WOL_ROL: 313 return ("MLXCX_OP_QUERY_WOL_ROL"); 314 case MLXCX_OP_CREATE_TIR: 315 return ("MLXCX_OP_CREATE_TIR"); 316 case MLXCX_OP_MODIFY_TIR: 317 return ("MLXCX_OP_MODIFY_TIR"); 318 case MLXCX_OP_DESTROY_TIR: 319 return ("MLXCX_OP_DESTROY_TIR"); 320 case MLXCX_OP_QUERY_TIR: 321 return ("MLXCX_OP_QUERY_TIR"); 322 case MLXCX_OP_CREATE_SQ: 323 return ("MLXCX_OP_CREATE_SQ"); 324 case MLXCX_OP_MODIFY_SQ: 325 return ("MLXCX_OP_MODIFY_SQ"); 326 case MLXCX_OP_DESTROY_SQ: 327 return ("MLXCX_OP_DESTROY_SQ"); 328 case MLXCX_OP_QUERY_SQ: 329 return ("MLXCX_OP_QUERY_SQ"); 330 case MLXCX_OP_CREATE_RQ: 331 return ("MLXCX_OP_CREATE_RQ"); 332 case MLXCX_OP_MODIFY_RQ: 333 return ("MLXCX_OP_MODIFY_RQ"); 334 case MLXCX_OP_DESTROY_RQ: 335 return ("MLXCX_OP_DESTROY_RQ"); 336 case MLXCX_OP_QUERY_RQ: 337 return ("MLXCX_OP_QUERY_RQ"); 338 case MLXCX_OP_CREATE_RMP: 339 return ("MLXCX_OP_CREATE_RMP"); 340 case MLXCX_OP_MODIFY_RMP: 341 return ("MLXCX_OP_MODIFY_RMP"); 342 case MLXCX_OP_DESTROY_RMP: 343 return ("MLXCX_OP_DESTROY_RMP"); 344 case MLXCX_OP_QUERY_RMP: 345 return ("MLXCX_OP_QUERY_RMP"); 346 case MLXCX_OP_CREATE_TIS: 347 return ("MLXCX_OP_CREATE_TIS"); 348 case MLXCX_OP_MODIFY_TIS: 349 return ("MLXCX_OP_MODIFY_TIS"); 350 case MLXCX_OP_DESTROY_TIS: 351 return ("MLXCX_OP_DESTROY_TIS"); 352 case MLXCX_OP_QUERY_TIS: 353 return ("MLXCX_OP_QUERY_TIS"); 354 case MLXCX_OP_CREATE_RQT: 355 return ("MLXCX_OP_CREATE_RQT"); 356 case MLXCX_OP_MODIFY_RQT: 357 return ("MLXCX_OP_MODIFY_RQT"); 358 case MLXCX_OP_DESTROY_RQT: 359 return ("MLXCX_OP_DESTROY_RQT"); 360 case MLXCX_OP_QUERY_RQT: 361 return ("MLXCX_OP_QUERY_RQT"); 362 case MLXCX_OP_SET_FLOW_TABLE_ROOT: 363 return ("MLXCX_OP_SET_FLOW_TABLE_ROOT"); 364 case MLXCX_OP_CREATE_FLOW_TABLE: 365 return ("MLXCX_OP_CREATE_FLOW_TABLE"); 366 case MLXCX_OP_DESTROY_FLOW_TABLE: 367 return ("MLXCX_OP_DESTROY_FLOW_TABLE"); 368 case MLXCX_OP_QUERY_FLOW_TABLE: 369 return ("MLXCX_OP_QUERY_FLOW_TABLE"); 370 case MLXCX_OP_CREATE_FLOW_GROUP: 371 return ("MLXCX_OP_CREATE_FLOW_GROUP"); 372 case MLXCX_OP_DESTROY_FLOW_GROUP: 373 return ("MLXCX_OP_DESTROY_FLOW_GROUP"); 374 case MLXCX_OP_QUERY_FLOW_GROUP: 375 return ("MLXCX_OP_QUERY_FLOW_GROUP"); 376 case MLXCX_OP_SET_FLOW_TABLE_ENTRY: 377 return ("MLXCX_OP_SET_FLOW_TABLE_ENTRY"); 378 case MLXCX_OP_QUERY_FLOW_TABLE_ENTRY: 379 return ("MLXCX_OP_QUERY_FLOW_TABLE_ENTRY"); 380 case MLXCX_OP_DELETE_FLOW_TABLE_ENTRY: 381 return ("MLXCX_OP_DELETE_FLOW_TABLE_ENTRY"); 382 case MLXCX_OP_ALLOC_FLOW_COUNTER: 383 return ("MLXCX_OP_ALLOC_FLOW_COUNTER"); 384 case MLXCX_OP_DEALLOC_FLOW_COUNTER: 385 return ("MLXCX_OP_DEALLOC_FLOW_COUNTER"); 386 case MLXCX_OP_QUERY_FLOW_COUNTER: 387 return ("MLXCX_OP_QUERY_FLOW_COUNTER"); 388 case MLXCX_OP_MODIFY_FLOW_TABLE: 389 return ("MLXCX_OP_MODIFY_FLOW_TABLE"); 390 case MLXCX_OP_ALLOC_ENCAP_HEADER: 391 return ("MLXCX_OP_ALLOC_ENCAP_HEADER"); 392 case MLXCX_OP_DEALLOC_ENCAP_HEADER: 393 return ("MLXCX_OP_DEALLOC_ENCAP_HEADER"); 394 case MLXCX_OP_QUERY_ENCAP_HEADER: 395 return ("MLXCX_OP_QUERY_ENCAP_HEADER"); 396 default: 397 return ("Unknown Opcode"); 398 } 399 } 400 401 const char * 402 mlxcx_port_status_string(mlxcx_port_status_t st) 403 { 404 switch (st) { 405 case MLXCX_PORT_STATUS_UP: 406 return ("UP"); 407 case MLXCX_PORT_STATUS_DOWN: 408 return ("DOWN"); 409 case MLXCX_PORT_STATUS_UP_ONCE: 410 return ("UP_ONCE"); 411 case MLXCX_PORT_STATUS_DISABLED: 412 return ("DISABLED"); 413 default: 414 return ("UNKNOWN"); 415 } 416 } 417 418 void 419 mlxcx_eth_proto_to_string(mlxcx_eth_proto_t p, char *buf, size_t size) 420 { 421 if (p & MLXCX_PROTO_SGMII) 422 (void) strlcat(buf, "SGMII|", size); 423 if (p & MLXCX_PROTO_1000BASE_KX) 424 (void) strlcat(buf, "1000BASE_KX|", size); 425 if (p & MLXCX_PROTO_10GBASE_CX4) 426 (void) strlcat(buf, "10GBASE_CX4|", size); 427 if (p & MLXCX_PROTO_10GBASE_KX4) 428 (void) strlcat(buf, "10GBASE_KX4|", size); 429 if (p & MLXCX_PROTO_10GBASE_KR) 430 (void) strlcat(buf, "10GBASE_KR|", size); 431 if (p & MLXCX_PROTO_40GBASE_CR4) 432 (void) strlcat(buf, "40GBASE_CR4|", size); 433 if (p & MLXCX_PROTO_40GBASE_KR4) 434 (void) strlcat(buf, "40GBASE_KR4|", size); 435 if (p & MLXCX_PROTO_SGMII_100BASE) 436 (void) strlcat(buf, "SGMII_100BASE|", size); 437 if (p & MLXCX_PROTO_10GBASE_CR) 438 (void) strlcat(buf, "10GBASE_CR|", size); 439 if (p & MLXCX_PROTO_10GBASE_SR) 440 (void) strlcat(buf, "10GBASE_SR|", size); 441 if (p & MLXCX_PROTO_10GBASE_ER_LR) 442 (void) strlcat(buf, "10GBASE_ER_LR|", size); 443 if (p & MLXCX_PROTO_40GBASE_SR4) 444 (void) strlcat(buf, "40GBASE_SR4|", size); 445 if (p & MLXCX_PROTO_40GBASE_LR4_ER4) 446 (void) strlcat(buf, "40GBASE_LR4_ER4|", size); 447 if (p & MLXCX_PROTO_50GBASE_SR2) 448 (void) strlcat(buf, "50GBASE_SR2|", size); 449 if (p & MLXCX_PROTO_100GBASE_CR4) 450 (void) strlcat(buf, "100GBASE_CR4|", size); 451 if (p & MLXCX_PROTO_100GBASE_SR4) 452 (void) strlcat(buf, "100GBASE_SR4|", size); 453 if (p & MLXCX_PROTO_100GBASE_KR4) 454 (void) strlcat(buf, "100GBASE_KR4|", size); 455 if (p & MLXCX_PROTO_25GBASE_CR) 456 (void) strlcat(buf, "25GBASE_CR|", size); 457 if (p & MLXCX_PROTO_25GBASE_KR) 458 (void) strlcat(buf, "25GBASE_KR|", size); 459 if (p & MLXCX_PROTO_25GBASE_SR) 460 (void) strlcat(buf, "25GBASE_SR|", size); 461 if (p & MLXCX_PROTO_50GBASE_CR2) 462 (void) strlcat(buf, "50GBASE_CR2|", size); 463 /* Chop off the trailing '|' */ 464 if (strlen(buf) > 0) 465 buf[strlen(buf) - 1] = '\0'; 466 } 467 468 void 469 mlxcx_cmd_queue_fini(mlxcx_t *mlxp) 470 { 471 mlxcx_cmd_queue_t *cmd = &mlxp->mlx_cmd; 472 473 if (cmd->mcmd_tokens != NULL) { 474 id_space_destroy(cmd->mcmd_tokens); 475 cmd->mcmd_tokens = NULL; 476 } 477 478 if (cmd->mcmd_taskq != NULL) { 479 ddi_taskq_destroy(cmd->mcmd_taskq); 480 cmd->mcmd_taskq = NULL; 481 } 482 483 cv_destroy(&cmd->mcmd_cv); 484 mutex_destroy(&cmd->mcmd_lock); 485 486 mlxcx_dma_free(&cmd->mcmd_dma); 487 } 488 489 boolean_t 490 mlxcx_cmd_queue_init(mlxcx_t *mlxp) 491 { 492 uint32_t tmp, cmd_low, cmd_high, i; 493 mlxcx_cmd_queue_t *cmd = &mlxp->mlx_cmd; 494 char buf[32]; 495 char tq_name[TASKQ_NAMELEN]; 496 const ddi_dma_cookie_t *ck; 497 498 ddi_device_acc_attr_t acc; 499 ddi_dma_attr_t attr; 500 501 tmp = mlxcx_get32(mlxp, MLXCX_ISS_FIRMWARE); 502 mlxp->mlx_fw_maj = MLXCX_ISS_FW_MAJOR(tmp); 503 mlxp->mlx_fw_min = MLXCX_ISS_FW_MINOR(tmp); 504 505 tmp = mlxcx_get32(mlxp, MLXCX_ISS_FW_CMD); 506 mlxp->mlx_fw_rev = MLXCX_ISS_FW_REV(tmp); 507 mlxp->mlx_cmd_rev = MLXCX_ISS_CMD_REV(tmp); 508 509 if (mlxp->mlx_cmd_rev != MLXCX_CMD_REVISION) { 510 mlxcx_warn(mlxp, "found unsupported command revision: %u, " 511 "expected %u", mlxp->mlx_cmd_rev, MLXCX_CMD_REVISION); 512 return (B_FALSE); 513 } 514 515 cmd_low = mlxcx_get32(mlxp, MLXCX_ISS_CMD_LOW); 516 cmd->mcmd_size_l2 = MLXCX_ISS_CMDQ_SIZE(cmd_low); 517 cmd->mcmd_stride_l2 = MLXCX_ISS_CMDQ_STRIDE(cmd_low); 518 cmd->mcmd_size = 1U << cmd->mcmd_size_l2; 519 520 if (cmd->mcmd_size > MLXCX_CMD_MAX) { 521 mlxcx_warn(mlxp, "command queue size %u is too " 522 "large. Maximum is %u", cmd->mcmd_size, MLXCX_CMD_MAX); 523 return (B_FALSE); 524 } 525 526 cmd->mcmd_mask = (uint32_t)((1ULL << cmd->mcmd_size) - 1); 527 528 mutex_init(&cmd->mcmd_lock, NULL, MUTEX_DRIVER, NULL); 529 cv_init(&cmd->mcmd_cv, NULL, CV_DRIVER, NULL); 530 531 (void) snprintf(buf, sizeof (buf), "mlxcx_tokens_%d", mlxp->mlx_inst); 532 if ((cmd->mcmd_tokens = id_space_create(buf, 1, UINT8_MAX)) == NULL) { 533 mlxcx_warn(mlxp, "failed to allocate token id space"); 534 mlxcx_cmd_queue_fini(mlxp); 535 return (B_FALSE); 536 } 537 538 (void) snprintf(tq_name, sizeof (tq_name), "cmdq_%d", mlxp->mlx_inst); 539 if ((cmd->mcmd_taskq = ddi_taskq_create(mlxp->mlx_dip, tq_name, 1, 540 TASKQ_DEFAULTPRI, 0)) == NULL) { 541 mlxcx_warn(mlxp, "failed to create command queue task queue"); 542 mlxcx_cmd_queue_fini(mlxp); 543 return (B_FALSE); 544 } 545 546 mlxcx_dma_acc_attr(mlxp, &acc); 547 mlxcx_dma_page_attr(mlxp, &attr); 548 549 if (!mlxcx_dma_alloc(mlxp, &cmd->mcmd_dma, &attr, &acc, B_TRUE, 550 MLXCX_CMD_DMA_PAGE_SIZE, B_TRUE)) { 551 mlxcx_warn(mlxp, "failed to allocate command dma buffer"); 552 mlxcx_cmd_queue_fini(mlxp); 553 return (B_FALSE); 554 } 555 556 ck = mlxcx_dma_cookie_one(&cmd->mcmd_dma); 557 cmd_high = (uint32_t)(ck->dmac_laddress >> 32); 558 cmd_low = (uint32_t)(ck->dmac_laddress & UINT32_MAX); 559 560 mlxcx_put32(mlxp, MLXCX_ISS_CMD_HIGH, cmd_high); 561 mlxcx_put32(mlxp, MLXCX_ISS_CMD_LOW, cmd_low); 562 563 /* 564 * Before this is ready, the initializing bit must become zero. 565 */ 566 for (i = 0; i < mlxcx_cmd_init_trys; i++) { 567 uint32_t init = mlxcx_get32(mlxp, MLXCX_ISS_INIT); 568 569 if (MLXCX_ISS_INITIALIZING(init) == 0) 570 break; 571 delay(drv_usectohz(mlxcx_cmd_init_delay)); 572 } 573 if (i == mlxcx_cmd_init_trys) { 574 mlxcx_warn(mlxp, "timed out initializing command queue"); 575 mlxcx_cmd_queue_fini(mlxp); 576 return (B_FALSE); 577 } 578 579 /* 580 * start in polling mode. 581 */ 582 mlxcx_cmd_eq_disable(mlxp); 583 584 return (B_TRUE); 585 } 586 587 void 588 mlxcx_cmd_eq_enable(mlxcx_t *mlxp) 589 { 590 mlxp->mlx_cmd.mcmd_polled = B_FALSE; 591 } 592 593 void 594 mlxcx_cmd_eq_disable(mlxcx_t *mlxp) 595 { 596 mlxp->mlx_cmd.mcmd_polled = B_TRUE; 597 } 598 599 static void 600 mlxcx_cmd_in_header_init(mlxcx_cmd_t *cmd, mlxcx_cmd_in_t *in, 601 mlxcx_cmd_op_t op, uint16_t mod) 602 { 603 ASSERT3U(op, <=, UINT16_MAX); 604 in->mci_opcode = to_be16(op); 605 in->mci_op_mod = to_be16(mod); 606 cmd->mlcmd_op = op; 607 } 608 609 static boolean_t 610 mlxcx_cmd_mbox_alloc(mlxcx_t *mlxp, list_t *listp, uint8_t nblocks) 611 { 612 uint8_t i; 613 ddi_device_acc_attr_t acc; 614 ddi_dma_attr_t attr; 615 616 mlxcx_dma_acc_attr(mlxp, &acc); 617 mlxcx_dma_page_attr(mlxp, &attr); 618 619 for (i = 0; i < nblocks; i++) { 620 mlxcx_cmd_mbox_t *mbox; 621 622 mbox = kmem_zalloc(sizeof (*mbox), KM_SLEEP); 623 if (!mlxcx_dma_alloc(mlxp, &mbox->mlbox_dma, &attr, &acc, 624 B_TRUE, sizeof (mlxcx_cmd_mailbox_t), B_TRUE)) { 625 mlxcx_warn(mlxp, "failed to allocate mailbox dma " 626 "buffer"); 627 kmem_free(mbox, sizeof (*mbox)); 628 /* 629 * mlxcx_cmd_fini will clean up any mboxes that we 630 * already placed onto listp. 631 */ 632 return (B_FALSE); 633 } 634 mbox->mlbox_data = (void *)mbox->mlbox_dma.mxdb_va; 635 list_insert_tail(listp, mbox); 636 } 637 638 return (B_TRUE); 639 } 640 641 static void 642 mlxcx_cmd_mbox_free(mlxcx_cmd_mbox_t *mbox) 643 { 644 mlxcx_dma_free(&mbox->mlbox_dma); 645 kmem_free(mbox, sizeof (mlxcx_cmd_mbox_t)); 646 } 647 648 static void 649 mlxcx_cmd_fini(mlxcx_t *mlxp, mlxcx_cmd_t *cmd) 650 { 651 mlxcx_cmd_mbox_t *mbox; 652 653 while ((mbox = list_remove_head(&cmd->mlcmd_mbox_out)) != NULL) { 654 mlxcx_cmd_mbox_free(mbox); 655 } 656 list_destroy(&cmd->mlcmd_mbox_out); 657 while ((mbox = list_remove_head(&cmd->mlcmd_mbox_in)) != NULL) { 658 mlxcx_cmd_mbox_free(mbox); 659 } 660 list_destroy(&cmd->mlcmd_mbox_in); 661 id_free(mlxp->mlx_cmd.mcmd_tokens, cmd->mlcmd_token); 662 cv_destroy(&cmd->mlcmd_cv); 663 mutex_destroy(&cmd->mlcmd_lock); 664 } 665 666 static void 667 mlxcx_cmd_init(mlxcx_t *mlxp, mlxcx_cmd_t *cmd) 668 { 669 bzero(cmd, sizeof (*cmd)); 670 mutex_init(&cmd->mlcmd_lock, NULL, MUTEX_DRIVER, 671 DDI_INTR_PRI(mlxp->mlx_async_intr_pri)); 672 cv_init(&cmd->mlcmd_cv, NULL, CV_DRIVER, NULL); 673 cmd->mlcmd_token = id_alloc(mlxp->mlx_cmd.mcmd_tokens); 674 cmd->mlcmd_poll = mlxp->mlx_cmd.mcmd_polled; 675 list_create(&cmd->mlcmd_mbox_in, sizeof (mlxcx_cmd_mbox_t), 676 offsetof(mlxcx_cmd_mbox_t, mlbox_node)); 677 list_create(&cmd->mlcmd_mbox_out, sizeof (mlxcx_cmd_mbox_t), 678 offsetof(mlxcx_cmd_mbox_t, mlbox_node)); 679 } 680 681 static void 682 mlxcx_cmd_prep_input(mlxcx_cmd_ent_t *ent, mlxcx_cmd_t *cmd) 683 { 684 uint32_t rem = cmd->mlcmd_inlen; 685 uint8_t i; 686 const void *in = cmd->mlcmd_in; 687 uint32_t copy; 688 mlxcx_cmd_mbox_t *mbox; 689 const ddi_dma_cookie_t *ck; 690 691 copy = MIN(MLXCX_CMD_INLINE_INPUT_LEN, rem); 692 bcopy(in, ent->mce_input, copy); 693 694 rem -= copy; 695 in += copy; 696 697 if (rem == 0) { 698 ent->mce_in_mbox = to_be64(0); 699 VERIFY3U(cmd->mlcmd_nboxes_in, ==, 0); 700 return; 701 } 702 703 mbox = list_head(&cmd->mlcmd_mbox_in); 704 ck = mlxcx_dma_cookie_one(&mbox->mlbox_dma); 705 ent->mce_in_mbox = to_be64(ck->dmac_laddress); 706 for (i = 0; mbox != NULL; 707 mbox = list_next(&cmd->mlcmd_mbox_in, mbox), i++) { 708 mlxcx_cmd_mbox_t *next; 709 mlxcx_cmd_mailbox_t *mp = mbox->mlbox_data; 710 711 copy = MIN(MLXCX_CMD_MAILBOX_LEN, rem); 712 bcopy(in, mp->mlxb_data, copy); 713 rem -= copy; 714 in += copy; 715 716 mp->mlxb_token = cmd->mlcmd_token; 717 mp->mlxb_blockno = to_be32(i); 718 719 next = list_next(&cmd->mlcmd_mbox_in, mbox); 720 if (next == NULL) { 721 mp->mlxb_nextp = to_be64(0); 722 } else { 723 ck = mlxcx_dma_cookie_one(&next->mlbox_dma); 724 mp->mlxb_nextp = to_be64(ck->dmac_laddress); 725 } 726 MLXCX_DMA_SYNC(mbox->mlbox_dma, DDI_DMA_SYNC_FORDEV); 727 } 728 VERIFY3U(i, ==, cmd->mlcmd_nboxes_in); 729 VERIFY0(rem); 730 } 731 732 static void 733 mlxcx_cmd_prep_output(mlxcx_cmd_ent_t *ent, mlxcx_cmd_t *cmd) 734 { 735 uint8_t i; 736 mlxcx_cmd_mbox_t *mbox; 737 const ddi_dma_cookie_t *ck; 738 739 if (cmd->mlcmd_nboxes_out == 0) { 740 ent->mce_out_mbox = to_be64(0); 741 return; 742 } 743 744 mbox = list_head(&cmd->mlcmd_mbox_out); 745 ck = mlxcx_dma_cookie_one(&mbox->mlbox_dma); 746 ent->mce_out_mbox = to_be64(ck->dmac_laddress); 747 for (i = 0, mbox = list_head(&cmd->mlcmd_mbox_out); mbox != NULL; 748 mbox = list_next(&cmd->mlcmd_mbox_out, mbox), i++) { 749 mlxcx_cmd_mbox_t *next; 750 mlxcx_cmd_mailbox_t *mp = mbox->mlbox_data; 751 752 mp->mlxb_token = cmd->mlcmd_token; 753 mp->mlxb_blockno = to_be32(i); 754 755 next = list_next(&cmd->mlcmd_mbox_out, mbox); 756 if (next == NULL) { 757 mp->mlxb_nextp = to_be64(0); 758 } else { 759 ck = mlxcx_dma_cookie_one(&next->mlbox_dma); 760 mp->mlxb_nextp = to_be64(ck->dmac_laddress); 761 } 762 MLXCX_DMA_SYNC(mbox->mlbox_dma, DDI_DMA_SYNC_FORDEV); 763 } 764 VERIFY3U(i, ==, cmd->mlcmd_nboxes_out); 765 } 766 767 static void 768 mlxcx_cmd_copy_output(mlxcx_cmd_ent_t *ent, mlxcx_cmd_t *cmd) 769 { 770 void *out = cmd->mlcmd_out; 771 uint32_t rem = cmd->mlcmd_outlen; 772 uint32_t copy; 773 mlxcx_cmd_mbox_t *mbox; 774 775 copy = MIN(rem, MLXCX_CMD_INLINE_OUTPUT_LEN); 776 bcopy(ent->mce_output, out, copy); 777 out += copy; 778 rem -= copy; 779 780 if (rem == 0) { 781 VERIFY0(cmd->mlcmd_nboxes_out); 782 return; 783 } 784 785 for (mbox = list_head(&cmd->mlcmd_mbox_out); mbox != NULL; 786 mbox = list_next(&cmd->mlcmd_mbox_out, mbox)) { 787 MLXCX_DMA_SYNC(mbox->mlbox_dma, DDI_DMA_SYNC_FORKERNEL); 788 copy = MIN(MLXCX_CMD_MAILBOX_LEN, rem); 789 bcopy(mbox->mlbox_data->mlxb_data, out, copy); 790 out += copy; 791 rem -= copy; 792 } 793 VERIFY0(rem); 794 } 795 796 static uint_t 797 mlxcx_cmd_reserve_slot(mlxcx_cmd_queue_t *cmdq) 798 { 799 uint_t slot; 800 801 mutex_enter(&cmdq->mcmd_lock); 802 slot = ddi_ffs(cmdq->mcmd_mask); 803 while (slot == 0) { 804 cv_wait(&cmdq->mcmd_cv, &cmdq->mcmd_lock); 805 slot = ddi_ffs(cmdq->mcmd_mask); 806 } 807 808 cmdq->mcmd_mask &= ~(1U << --slot); 809 810 ASSERT3P(cmdq->mcmd_active[slot], ==, NULL); 811 812 mutex_exit(&cmdq->mcmd_lock); 813 814 return (slot); 815 } 816 817 static void 818 mlxcx_cmd_release_slot(mlxcx_cmd_queue_t *cmdq, uint_t slot) 819 { 820 mutex_enter(&cmdq->mcmd_lock); 821 cmdq->mcmd_mask |= 1U << slot; 822 cv_broadcast(&cmdq->mcmd_cv); 823 mutex_exit(&cmdq->mcmd_lock); 824 } 825 826 static void 827 mlxcx_cmd_done(mlxcx_cmd_t *cmd, uint_t slot) 828 { 829 mlxcx_t *mlxp = cmd->mlcmd_mlxp; 830 mlxcx_cmd_queue_t *cmdq = &mlxp->mlx_cmd; 831 mlxcx_cmd_ent_t *ent; 832 833 /* 834 * Command is done. Save relevant data. Once we broadcast on the CV and 835 * drop the lock, we must not touch it again. 836 */ 837 MLXCX_DMA_SYNC(cmdq->mcmd_dma, DDI_DMA_SYNC_FORKERNEL); 838 839 ent = (mlxcx_cmd_ent_t *)(cmdq->mcmd_dma.mxdb_va + 840 (slot << cmdq->mcmd_stride_l2)); 841 842 mutex_enter(&cmd->mlcmd_lock); 843 cmd->mlcmd_status = MLXCX_CMD_STATUS(ent->mce_status); 844 if (cmd->mlcmd_status == 0) 845 mlxcx_cmd_copy_output(ent, cmd); 846 847 cmd->mlcmd_state = MLXCX_CMD_S_DONE; 848 cv_broadcast(&cmd->mlcmd_cv); 849 mutex_exit(&cmd->mlcmd_lock); 850 851 cmdq->mcmd_active[slot] = NULL; 852 mlxcx_cmd_release_slot(cmdq, slot); 853 } 854 855 static void 856 mlxcx_cmd_taskq(void *arg) 857 { 858 mlxcx_cmd_t *cmd = arg; 859 mlxcx_t *mlxp = cmd->mlcmd_mlxp; 860 mlxcx_cmd_queue_t *cmdq = &mlxp->mlx_cmd; 861 mlxcx_cmd_ent_t *ent; 862 uint_t poll, slot; 863 864 ASSERT3S(cmd->mlcmd_op, !=, 0); 865 866 slot = mlxcx_cmd_reserve_slot(cmdq); 867 ent = (mlxcx_cmd_ent_t *)(cmdq->mcmd_dma.mxdb_va + 868 (slot << cmdq->mcmd_stride_l2)); 869 870 cmdq->mcmd_active[slot] = cmd; 871 872 /* 873 * Command queue is currently ours as we set busy. 874 */ 875 bzero(ent, sizeof (*ent)); 876 ent->mce_type = MLXCX_CMD_TRANSPORT_PCI; 877 ent->mce_in_length = to_be32(cmd->mlcmd_inlen); 878 ent->mce_out_length = to_be32(cmd->mlcmd_outlen); 879 ent->mce_token = cmd->mlcmd_token; 880 ent->mce_sig = 0; 881 ent->mce_status = MLXCX_CMD_HW_OWNED; 882 mlxcx_cmd_prep_input(ent, cmd); 883 mlxcx_cmd_prep_output(ent, cmd); 884 MLXCX_DMA_SYNC(cmdq->mcmd_dma, DDI_DMA_SYNC_FORDEV); 885 886 mlxcx_put32(mlxp, MLXCX_ISS_CMD_DOORBELL, 1 << slot); 887 888 if (!cmd->mlcmd_poll) 889 return; 890 891 for (poll = 0; poll < mlxcx_cmd_tries; poll++) { 892 delay(drv_usectohz(mlxcx_cmd_delay)); 893 MLXCX_DMA_SYNC(cmdq->mcmd_dma, DDI_DMA_SYNC_FORKERNEL); 894 if ((ent->mce_status & MLXCX_CMD_HW_OWNED) == 0) 895 break; 896 } 897 898 /* 899 * Command is done (or timed out). Save relevant data. Once we broadcast 900 * on the CV and drop the lock, we must not touch the cmd again. 901 */ 902 903 if (poll == mlxcx_cmd_tries) { 904 mutex_enter(&cmd->mlcmd_lock); 905 cmd->mlcmd_status = MLXCX_CMD_R_TIMEOUT; 906 cmd->mlcmd_state = MLXCX_CMD_S_ERROR; 907 cv_broadcast(&cmd->mlcmd_cv); 908 mutex_exit(&cmd->mlcmd_lock); 909 910 mlxcx_fm_ereport(mlxp, DDI_FM_DEVICE_NO_RESPONSE); 911 912 cmdq->mcmd_active[slot] = NULL; 913 mlxcx_cmd_release_slot(cmdq, slot); 914 915 return; 916 } 917 918 mlxcx_cmd_done(cmd, slot); 919 } 920 921 void 922 mlxcx_cmd_completion(mlxcx_t *mlxp, mlxcx_eventq_ent_t *ent) 923 { 924 mlxcx_cmd_queue_t *cmdq = &mlxp->mlx_cmd; 925 mlxcx_evdata_cmd_completion_t *eqe_cmd = &ent->mleqe_cmd_completion; 926 mlxcx_cmd_t *cmd; 927 uint32_t comp_vec = from_be32(eqe_cmd->mled_cmd_completion_vec); 928 uint_t slot; 929 930 DTRACE_PROBE2(cmd_event, mlxcx_t *, mlxp, 931 mlxcx_evdata_cmd_completion_t *, eqe_cmd); 932 933 while ((slot = ddi_ffs(comp_vec)) != 0) { 934 comp_vec &= ~(1U << --slot); 935 936 cmd = cmdq->mcmd_active[slot]; 937 if (cmd->mlcmd_poll) 938 continue; 939 940 mlxcx_cmd_done(cmd, slot); 941 } 942 } 943 944 static boolean_t 945 mlxcx_cmd_send(mlxcx_t *mlxp, mlxcx_cmd_t *cmd, const void *in, uint32_t inlen, 946 void *out, uint32_t outlen) 947 { 948 if (inlen > MLXCX_CMD_INLINE_INPUT_LEN) { 949 uint32_t need = inlen - MLXCX_CMD_INLINE_INPUT_LEN; 950 uint8_t nblocks; 951 952 if (need / MLXCX_CMD_MAILBOX_LEN + 1 > UINT8_MAX) { 953 mlxcx_warn(mlxp, "requested too many input blocks for " 954 "%u byte input len", inlen); 955 return (B_FALSE); 956 } 957 958 nblocks = need / MLXCX_CMD_MAILBOX_LEN + 1; 959 if (!mlxcx_cmd_mbox_alloc(mlxp, &cmd->mlcmd_mbox_in, nblocks)) { 960 mlxcx_warn(mlxp, "failed to allocate %u blocks of " 961 "input mailbox", nblocks); 962 return (B_FALSE); 963 } 964 cmd->mlcmd_nboxes_in = nblocks; 965 } 966 967 if (outlen > MLXCX_CMD_INLINE_OUTPUT_LEN) { 968 uint32_t need = outlen - MLXCX_CMD_INLINE_OUTPUT_LEN; 969 uint8_t nblocks; 970 971 if (need / MLXCX_CMD_MAILBOX_LEN + 1 > UINT8_MAX) { 972 mlxcx_warn(mlxp, "requested too many output blocks for " 973 "%u byte output len", outlen); 974 return (B_FALSE); 975 } 976 977 nblocks = need / MLXCX_CMD_MAILBOX_LEN + 1; 978 if (!mlxcx_cmd_mbox_alloc(mlxp, &cmd->mlcmd_mbox_out, 979 nblocks)) { 980 mlxcx_warn(mlxp, "failed to allocate %u blocks of " 981 "output mailbox", nblocks); 982 return (B_FALSE); 983 } 984 cmd->mlcmd_nboxes_out = nblocks; 985 } 986 987 cmd->mlcmd_in = in; 988 cmd->mlcmd_inlen = inlen; 989 cmd->mlcmd_out = out; 990 cmd->mlcmd_outlen = outlen; 991 cmd->mlcmd_mlxp = mlxp; 992 993 /* 994 * Now that all allocations have been done, all that remains is for us 995 * to dispatch the request to process this to the taskq for it to be 996 * processed. 997 */ 998 if (ddi_taskq_dispatch(mlxp->mlx_cmd.mcmd_taskq, mlxcx_cmd_taskq, cmd, 999 DDI_SLEEP) != DDI_SUCCESS) { 1000 mlxcx_warn(mlxp, "failed to submit command to taskq"); 1001 return (B_FALSE); 1002 } 1003 1004 return (B_TRUE); 1005 } 1006 1007 static void 1008 mlxcx_cmd_wait(mlxcx_cmd_t *cmd) 1009 { 1010 mutex_enter(&cmd->mlcmd_lock); 1011 while (cmd->mlcmd_state == 0) { 1012 cv_wait(&cmd->mlcmd_cv, &cmd->mlcmd_lock); 1013 } 1014 mutex_exit(&cmd->mlcmd_lock); 1015 } 1016 1017 static boolean_t 1018 mlxcx_cmd_evaluate(mlxcx_t *mlxp, mlxcx_cmd_t *cmd) 1019 { 1020 mlxcx_cmd_out_t *out; 1021 1022 if ((cmd->mlcmd_state & MLXCX_CMD_S_ERROR) != 0) { 1023 mlxcx_warn(mlxp, "command %s (0x%x) failed due to an internal " 1024 "driver error", 1025 mlxcx_cmd_opcode_string(cmd->mlcmd_op), 1026 cmd->mlcmd_op); 1027 return (B_FALSE); 1028 } 1029 1030 if (cmd->mlcmd_status != 0) { 1031 mlxcx_warn(mlxp, "command %s (0x%x) failed with command queue " 1032 "error 0x%x", 1033 mlxcx_cmd_opcode_string(cmd->mlcmd_op), 1034 cmd->mlcmd_op, cmd->mlcmd_status); 1035 return (B_FALSE); 1036 } 1037 1038 out = cmd->mlcmd_out; 1039 if (out->mco_status != MLXCX_CMD_R_OK) { 1040 mlxcx_warn(mlxp, "command %s 0x%x failed with status code %s " 1041 "(0x%x)", mlxcx_cmd_opcode_string(cmd->mlcmd_op), 1042 cmd->mlcmd_op, mlxcx_cmd_response_string(out->mco_status), 1043 out->mco_status); 1044 return (B_FALSE); 1045 } 1046 1047 return (B_TRUE); 1048 } 1049 1050 boolean_t 1051 mlxcx_cmd_disable_hca(mlxcx_t *mlxp) 1052 { 1053 mlxcx_cmd_t cmd; 1054 mlxcx_cmd_disable_hca_in_t in; 1055 mlxcx_cmd_disable_hca_out_t out; 1056 boolean_t ret; 1057 1058 bzero(&in, sizeof (in)); 1059 bzero(&out, sizeof (out)); 1060 1061 mlxcx_cmd_init(mlxp, &cmd); 1062 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_disable_hca_head, 1063 MLXCX_OP_DISABLE_HCA, 0); 1064 in.mlxi_disable_hca_func = MLXCX_FUNCTION_SELF; 1065 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1066 mlxcx_cmd_fini(mlxp, &cmd); 1067 return (B_FALSE); 1068 } 1069 mlxcx_cmd_wait(&cmd); 1070 1071 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1072 mlxcx_cmd_fini(mlxp, &cmd); 1073 return (ret); 1074 } 1075 1076 boolean_t 1077 mlxcx_cmd_enable_hca(mlxcx_t *mlxp) 1078 { 1079 mlxcx_cmd_t cmd; 1080 mlxcx_cmd_enable_hca_in_t in; 1081 mlxcx_cmd_enable_hca_out_t out; 1082 boolean_t ret; 1083 1084 bzero(&in, sizeof (in)); 1085 bzero(&out, sizeof (out)); 1086 1087 mlxcx_cmd_init(mlxp, &cmd); 1088 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_enable_hca_head, 1089 MLXCX_OP_ENABLE_HCA, 0); 1090 in.mlxi_enable_hca_func = MLXCX_FUNCTION_SELF; 1091 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1092 mlxcx_cmd_fini(mlxp, &cmd); 1093 return (B_FALSE); 1094 } 1095 mlxcx_cmd_wait(&cmd); 1096 1097 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1098 mlxcx_cmd_fini(mlxp, &cmd); 1099 return (ret); 1100 } 1101 1102 boolean_t 1103 mlxcx_cmd_query_issi(mlxcx_t *mlxp, uint32_t *issip) 1104 { 1105 mlxcx_cmd_t cmd; 1106 mlxcx_cmd_query_issi_in_t in; 1107 mlxcx_cmd_query_issi_out_t out; 1108 boolean_t ret; 1109 1110 bzero(&in, sizeof (in)); 1111 bzero(&out, sizeof (out)); 1112 1113 mlxcx_cmd_init(mlxp, &cmd); 1114 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_issi_head, 1115 MLXCX_OP_QUERY_ISSI, 0); 1116 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1117 mlxcx_cmd_fini(mlxp, &cmd); 1118 return (B_FALSE); 1119 } 1120 mlxcx_cmd_wait(&cmd); 1121 1122 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1123 if (ret) { 1124 *issip = out.mlxo_supported_issi; 1125 } else if (cmd.mlcmd_status == 0 && 1126 out.mlxo_query_issi_head.mco_status == MLXCX_CMD_R_BAD_OP) { 1127 /* 1128 * The PRM says that if we get a bad operation, that means this 1129 * command isn't supported so it only supports version 1 of the 1130 * ISSI, which means bit zero should be set. 1131 */ 1132 ret = B_TRUE; 1133 *issip = 1; 1134 } 1135 mlxcx_cmd_fini(mlxp, &cmd); 1136 return (ret); 1137 } 1138 1139 boolean_t 1140 mlxcx_cmd_set_issi(mlxcx_t *mlxp, uint16_t issi) 1141 { 1142 mlxcx_cmd_t cmd; 1143 mlxcx_cmd_set_issi_in_t in; 1144 mlxcx_cmd_set_issi_out_t out; 1145 boolean_t ret; 1146 1147 bzero(&in, sizeof (in)); 1148 bzero(&out, sizeof (out)); 1149 1150 mlxcx_cmd_init(mlxp, &cmd); 1151 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_set_issi_head, 1152 MLXCX_OP_SET_ISSI, 0); 1153 in.mlxi_set_issi_current = to_be16(issi); 1154 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1155 mlxcx_cmd_fini(mlxp, &cmd); 1156 return (B_FALSE); 1157 } 1158 mlxcx_cmd_wait(&cmd); 1159 1160 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1161 mlxcx_cmd_fini(mlxp, &cmd); 1162 return (ret); 1163 } 1164 1165 boolean_t 1166 mlxcx_cmd_query_pages(mlxcx_t *mlxp, uint_t type, int32_t *npages) 1167 { 1168 mlxcx_cmd_t cmd; 1169 mlxcx_cmd_query_pages_in_t in; 1170 mlxcx_cmd_query_pages_out_t out; 1171 boolean_t ret; 1172 1173 switch (type) { 1174 case MLXCX_QUERY_PAGES_OPMOD_BOOT: 1175 case MLXCX_QUERY_PAGES_OPMOD_INIT: 1176 case MLXCX_QUERY_PAGES_OPMOD_REGULAR: 1177 break; 1178 default: 1179 mlxcx_warn(mlxp, "!passed invalid type to query pages: %u", 1180 type); 1181 return (B_FALSE); 1182 } 1183 1184 bzero(&in, sizeof (in)); 1185 bzero(&out, sizeof (out)); 1186 1187 mlxcx_cmd_init(mlxp, &cmd); 1188 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_pages_head, 1189 MLXCX_OP_QUERY_PAGES, type); 1190 in.mlxi_query_pages_func = MLXCX_FUNCTION_SELF; 1191 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1192 mlxcx_cmd_fini(mlxp, &cmd); 1193 return (B_FALSE); 1194 } 1195 mlxcx_cmd_wait(&cmd); 1196 1197 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1198 if (ret) { 1199 *npages = from_be32(out.mlxo_query_pages_npages); 1200 } 1201 mlxcx_cmd_fini(mlxp, &cmd); 1202 1203 return (ret); 1204 } 1205 1206 boolean_t 1207 mlxcx_cmd_give_pages(mlxcx_t *mlxp, uint_t type, int32_t npages, 1208 mlxcx_dev_page_t **pages) 1209 { 1210 mlxcx_cmd_t cmd; 1211 mlxcx_cmd_manage_pages_in_t *in; 1212 mlxcx_cmd_manage_pages_out_t out; 1213 size_t insize, outsize; 1214 boolean_t ret; 1215 uint32_t i; 1216 uint64_t pa; 1217 const ddi_dma_cookie_t *ck; 1218 1219 switch (type) { 1220 case MLXCX_MANAGE_PAGES_OPMOD_ALLOC_FAIL: 1221 if (npages != 0) { 1222 mlxcx_warn(mlxp, "passed non-zero number of pages (%d) " 1223 "but asked to fail page allocation", npages); 1224 return (B_FALSE); 1225 } 1226 break; 1227 case MLXCX_MANAGE_PAGES_OPMOD_GIVE_PAGES: 1228 ASSERT3S(npages, <=, MLXCX_MANAGE_PAGES_MAX_PAGES); 1229 if (npages <= 0) { 1230 mlxcx_warn(mlxp, "passed invalid number of pages (%d) " 1231 "to give pages", npages); 1232 return (B_FALSE); 1233 } 1234 break; 1235 default: 1236 mlxcx_warn(mlxp, "!passed invalid type to give pages: %u", 1237 type); 1238 return (B_FALSE); 1239 } 1240 1241 insize = offsetof(mlxcx_cmd_manage_pages_in_t, mlxi_manage_pages_pas) + 1242 npages * sizeof (uint64_t); 1243 outsize = offsetof(mlxcx_cmd_manage_pages_out_t, mlxo_manage_pages_pas); 1244 1245 in = kmem_zalloc(insize, KM_SLEEP); 1246 bzero(&out, sizeof (out)); 1247 1248 mlxcx_cmd_init(mlxp, &cmd); 1249 mlxcx_cmd_in_header_init(&cmd, &in->mlxi_manage_pages_head, 1250 MLXCX_OP_MANAGE_PAGES, type); 1251 in->mlxi_manage_pages_func = MLXCX_FUNCTION_SELF; 1252 in->mlxi_manage_pages_npages = to_be32(npages); 1253 for (i = 0; i < npages; i++) { 1254 ck = mlxcx_dma_cookie_one(&pages[i]->mxdp_dma); 1255 pa = ck->dmac_laddress; 1256 ASSERT3U(pa & 0xfff, ==, 0); 1257 ASSERT3U(ck->dmac_size, ==, MLXCX_HW_PAGE_SIZE); 1258 in->mlxi_manage_pages_pas[i] = to_be64(pa); 1259 } 1260 1261 if ((ret = mlxcx_cmd_send(mlxp, &cmd, in, insize, &out, outsize))) { 1262 mlxcx_cmd_wait(&cmd); 1263 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1264 } 1265 1266 mlxcx_cmd_fini(mlxp, &cmd); 1267 1268 kmem_free(in, insize); 1269 return (ret); 1270 } 1271 1272 boolean_t 1273 mlxcx_cmd_return_pages(mlxcx_t *mlxp, int32_t nreq, uint64_t *pas, 1274 int32_t *nret) 1275 { 1276 mlxcx_cmd_t cmd; 1277 mlxcx_cmd_manage_pages_in_t in; 1278 mlxcx_cmd_manage_pages_out_t *out; 1279 size_t insize, outsize; 1280 boolean_t ret; 1281 uint32_t i; 1282 1283 if (nreq <= 0) { 1284 mlxcx_warn(mlxp, "passed invalid number of pages (%d) " 1285 "to return pages", nreq); 1286 return (B_FALSE); 1287 } 1288 1289 insize = offsetof(mlxcx_cmd_manage_pages_in_t, mlxi_manage_pages_pas); 1290 outsize = offsetof(mlxcx_cmd_manage_pages_out_t, 1291 mlxo_manage_pages_pas) + nreq * sizeof (uint64_t); 1292 1293 bzero(&in, sizeof (in)); 1294 out = kmem_alloc(outsize, KM_SLEEP); 1295 1296 mlxcx_cmd_init(mlxp, &cmd); 1297 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_manage_pages_head, 1298 MLXCX_OP_MANAGE_PAGES, MLXCX_MANAGE_PAGES_OPMOD_RETURN_PAGES); 1299 in.mlxi_manage_pages_func = MLXCX_FUNCTION_SELF; 1300 in.mlxi_manage_pages_npages = to_be32(nreq); 1301 1302 if ((ret = mlxcx_cmd_send(mlxp, &cmd, &in, insize, out, outsize))) { 1303 mlxcx_cmd_wait(&cmd); 1304 1305 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1306 if (ret) { 1307 *nret = from_be32(out->mlxo_manage_pages_npages); 1308 for (i = 0; i < *nret; i++) { 1309 pas[i] = 1310 from_be64(out->mlxo_manage_pages_pas[i]); 1311 } 1312 } 1313 } 1314 1315 mlxcx_cmd_fini(mlxp, &cmd); 1316 1317 kmem_free(out, outsize); 1318 return (ret); 1319 } 1320 1321 boolean_t 1322 mlxcx_cmd_query_hca_cap(mlxcx_t *mlxp, mlxcx_hca_cap_type_t type, 1323 mlxcx_hca_cap_mode_t mode, mlxcx_hca_cap_t *capp) 1324 { 1325 mlxcx_cmd_t cmd; 1326 mlxcx_cmd_query_hca_cap_in_t in; 1327 mlxcx_cmd_query_hca_cap_out_t *out; 1328 boolean_t ret; 1329 uint16_t opmode; 1330 1331 bzero(&in, sizeof (in)); 1332 out = kmem_zalloc(sizeof (mlxcx_cmd_query_hca_cap_out_t), KM_SLEEP); 1333 mlxcx_cmd_init(mlxp, &cmd); 1334 1335 opmode = type << 1 | mode; 1336 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_hca_cap_head, 1337 MLXCX_OP_QUERY_HCA_CAP, opmode); 1338 1339 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), out, sizeof (*out))) { 1340 mlxcx_cmd_fini(mlxp, &cmd); 1341 kmem_free(out, sizeof (mlxcx_cmd_query_hca_cap_out_t)); 1342 return (B_FALSE); 1343 } 1344 mlxcx_cmd_wait(&cmd); 1345 1346 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1347 if (ret) { 1348 capp->mhc_mode = mode; 1349 capp->mhc_type = type; 1350 ASSERT3U(sizeof (out->mlxo_query_hca_cap_data), ==, 1351 sizeof (capp->mhc_bulk)); 1352 bcopy(out->mlxo_query_hca_cap_data, capp->mhc_bulk, 1353 sizeof (capp->mhc_bulk)); 1354 } 1355 mlxcx_cmd_fini(mlxp, &cmd); 1356 1357 kmem_free(out, sizeof (mlxcx_cmd_query_hca_cap_out_t)); 1358 return (B_TRUE); 1359 } 1360 1361 boolean_t 1362 mlxcx_cmd_init_hca(mlxcx_t *mlxp) 1363 { 1364 mlxcx_cmd_t cmd; 1365 mlxcx_cmd_init_hca_in_t in; 1366 mlxcx_cmd_init_hca_out_t out; 1367 boolean_t ret; 1368 1369 bzero(&in, sizeof (in)); 1370 bzero(&out, sizeof (out)); 1371 1372 mlxcx_cmd_init(mlxp, &cmd); 1373 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_init_hca_head, 1374 MLXCX_OP_INIT_HCA, 0); 1375 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1376 mlxcx_cmd_fini(mlxp, &cmd); 1377 return (B_FALSE); 1378 } 1379 mlxcx_cmd_wait(&cmd); 1380 1381 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1382 mlxcx_cmd_fini(mlxp, &cmd); 1383 return (ret); 1384 } 1385 1386 boolean_t 1387 mlxcx_cmd_set_driver_version(mlxcx_t *mlxp, const char *version) 1388 { 1389 mlxcx_cmd_t cmd; 1390 mlxcx_cmd_set_driver_version_in_t in; 1391 mlxcx_cmd_set_driver_version_out_t out; 1392 boolean_t ret; 1393 1394 bzero(&in, sizeof (in)); 1395 bzero(&out, sizeof (out)); 1396 1397 mlxcx_cmd_init(mlxp, &cmd); 1398 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_set_driver_version_head, 1399 MLXCX_OP_SET_DRIVER_VERSION, 0); 1400 VERIFY3U(strlcpy(in.mlxi_set_driver_version_version, version, 1401 sizeof (in.mlxi_set_driver_version_version)), <=, 1402 sizeof (in.mlxi_set_driver_version_version)); 1403 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1404 mlxcx_cmd_fini(mlxp, &cmd); 1405 return (B_FALSE); 1406 } 1407 mlxcx_cmd_wait(&cmd); 1408 1409 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1410 mlxcx_cmd_fini(mlxp, &cmd); 1411 return (ret); 1412 } 1413 1414 boolean_t 1415 mlxcx_cmd_alloc_uar(mlxcx_t *mlxp, mlxcx_uar_t *mlup) 1416 { 1417 mlxcx_cmd_t cmd; 1418 mlxcx_cmd_alloc_uar_in_t in; 1419 mlxcx_cmd_alloc_uar_out_t out; 1420 boolean_t ret; 1421 size_t i; 1422 1423 bzero(&in, sizeof (in)); 1424 bzero(&out, sizeof (out)); 1425 1426 mlxcx_cmd_init(mlxp, &cmd); 1427 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_alloc_uar_head, 1428 MLXCX_OP_ALLOC_UAR, 0); 1429 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1430 mlxcx_cmd_fini(mlxp, &cmd); 1431 return (B_FALSE); 1432 } 1433 mlxcx_cmd_wait(&cmd); 1434 1435 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1436 if (ret) { 1437 mlup->mlu_allocated = B_TRUE; 1438 mlup->mlu_num = from_be24(out.mlxo_alloc_uar_uar); 1439 VERIFY3U(mlup->mlu_num, >, 0); 1440 mlup->mlu_base = mlup->mlu_num * MLXCX_HW_PAGE_SIZE; 1441 1442 for (i = 0; i < MLXCX_BF_PER_UAR; ++i) { 1443 mlup->mlu_bf[i].mbf_even = mlup->mlu_base + 1444 MLXCX_BF_BASE + MLXCX_BF_SIZE * 2 * i; 1445 mlup->mlu_bf[i].mbf_odd = mlup->mlu_bf[i].mbf_even + 1446 MLXCX_BF_SIZE; 1447 } 1448 } 1449 mlxcx_cmd_fini(mlxp, &cmd); 1450 return (ret); 1451 } 1452 1453 boolean_t 1454 mlxcx_cmd_dealloc_uar(mlxcx_t *mlxp, mlxcx_uar_t *mlup) 1455 { 1456 mlxcx_cmd_t cmd; 1457 mlxcx_cmd_dealloc_uar_in_t in; 1458 mlxcx_cmd_dealloc_uar_out_t out; 1459 boolean_t ret; 1460 1461 bzero(&in, sizeof (in)); 1462 bzero(&out, sizeof (out)); 1463 1464 mlxcx_cmd_init(mlxp, &cmd); 1465 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_dealloc_uar_head, 1466 MLXCX_OP_DEALLOC_UAR, 0); 1467 VERIFY(mlup->mlu_allocated); 1468 in.mlxi_dealloc_uar_uar = to_be24(mlup->mlu_num); 1469 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1470 mlxcx_cmd_fini(mlxp, &cmd); 1471 return (B_FALSE); 1472 } 1473 mlxcx_cmd_wait(&cmd); 1474 1475 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1476 if (ret) { 1477 mlup->mlu_allocated = B_FALSE; 1478 mlup->mlu_num = 0; 1479 } 1480 mlxcx_cmd_fini(mlxp, &cmd); 1481 return (ret); 1482 } 1483 1484 boolean_t 1485 mlxcx_cmd_alloc_pd(mlxcx_t *mlxp, mlxcx_pd_t *mlpd) 1486 { 1487 mlxcx_cmd_t cmd; 1488 mlxcx_cmd_alloc_pd_in_t in; 1489 mlxcx_cmd_alloc_pd_out_t out; 1490 boolean_t ret; 1491 1492 bzero(&in, sizeof (in)); 1493 bzero(&out, sizeof (out)); 1494 1495 mlxcx_cmd_init(mlxp, &cmd); 1496 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_alloc_pd_head, 1497 MLXCX_OP_ALLOC_PD, 0); 1498 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1499 mlxcx_cmd_fini(mlxp, &cmd); 1500 return (B_FALSE); 1501 } 1502 mlxcx_cmd_wait(&cmd); 1503 1504 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1505 if (ret) { 1506 mlpd->mlpd_allocated = B_TRUE; 1507 mlpd->mlpd_num = from_be24(out.mlxo_alloc_pd_pdn); 1508 } 1509 mlxcx_cmd_fini(mlxp, &cmd); 1510 return (ret); 1511 } 1512 1513 boolean_t 1514 mlxcx_cmd_dealloc_pd(mlxcx_t *mlxp, mlxcx_pd_t *mlpd) 1515 { 1516 mlxcx_cmd_t cmd; 1517 mlxcx_cmd_dealloc_pd_in_t in; 1518 mlxcx_cmd_dealloc_pd_out_t out; 1519 boolean_t ret; 1520 1521 bzero(&in, sizeof (in)); 1522 bzero(&out, sizeof (out)); 1523 1524 mlxcx_cmd_init(mlxp, &cmd); 1525 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_dealloc_pd_head, 1526 MLXCX_OP_DEALLOC_PD, 0); 1527 VERIFY(mlpd->mlpd_allocated); 1528 in.mlxi_dealloc_pd_pdn = to_be24(mlpd->mlpd_num); 1529 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1530 mlxcx_cmd_fini(mlxp, &cmd); 1531 return (B_FALSE); 1532 } 1533 mlxcx_cmd_wait(&cmd); 1534 1535 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1536 if (ret) { 1537 mlpd->mlpd_allocated = B_FALSE; 1538 mlpd->mlpd_num = 0; 1539 } 1540 mlxcx_cmd_fini(mlxp, &cmd); 1541 return (ret); 1542 } 1543 1544 boolean_t 1545 mlxcx_cmd_alloc_tdom(mlxcx_t *mlxp, mlxcx_tdom_t *mltd) 1546 { 1547 mlxcx_cmd_t cmd; 1548 mlxcx_cmd_alloc_tdom_in_t in; 1549 mlxcx_cmd_alloc_tdom_out_t out; 1550 boolean_t ret; 1551 1552 bzero(&in, sizeof (in)); 1553 bzero(&out, sizeof (out)); 1554 1555 mlxcx_cmd_init(mlxp, &cmd); 1556 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_alloc_tdom_head, 1557 MLXCX_OP_ALLOC_TRANSPORT_DOMAIN, 0); 1558 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1559 mlxcx_cmd_fini(mlxp, &cmd); 1560 return (B_FALSE); 1561 } 1562 mlxcx_cmd_wait(&cmd); 1563 1564 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1565 if (ret) { 1566 mltd->mltd_allocated = B_TRUE; 1567 mltd->mltd_num = from_be24(out.mlxo_alloc_tdom_tdomn); 1568 } 1569 mlxcx_cmd_fini(mlxp, &cmd); 1570 return (ret); 1571 } 1572 1573 boolean_t 1574 mlxcx_cmd_dealloc_tdom(mlxcx_t *mlxp, mlxcx_tdom_t *mltd) 1575 { 1576 mlxcx_cmd_t cmd; 1577 mlxcx_cmd_dealloc_tdom_in_t in; 1578 mlxcx_cmd_dealloc_tdom_out_t out; 1579 boolean_t ret; 1580 1581 bzero(&in, sizeof (in)); 1582 bzero(&out, sizeof (out)); 1583 1584 mlxcx_cmd_init(mlxp, &cmd); 1585 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_dealloc_tdom_head, 1586 MLXCX_OP_DEALLOC_TRANSPORT_DOMAIN, 0); 1587 VERIFY(mltd->mltd_allocated); 1588 in.mlxi_dealloc_tdom_tdomn = to_be24(mltd->mltd_num); 1589 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1590 mlxcx_cmd_fini(mlxp, &cmd); 1591 return (B_FALSE); 1592 } 1593 mlxcx_cmd_wait(&cmd); 1594 1595 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1596 if (ret) { 1597 mltd->mltd_allocated = B_FALSE; 1598 mltd->mltd_num = 0; 1599 } 1600 mlxcx_cmd_fini(mlxp, &cmd); 1601 return (ret); 1602 } 1603 1604 boolean_t 1605 mlxcx_cmd_teardown_hca(mlxcx_t *mlxp) 1606 { 1607 mlxcx_cmd_t cmd; 1608 mlxcx_cmd_teardown_hca_in_t in; 1609 mlxcx_cmd_teardown_hca_out_t out; 1610 boolean_t ret; 1611 1612 bzero(&in, sizeof (in)); 1613 bzero(&out, sizeof (out)); 1614 1615 mlxcx_cmd_init(mlxp, &cmd); 1616 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_teardown_hca_head, 1617 MLXCX_OP_TEARDOWN_HCA, 0); 1618 in.mlxi_teardown_hca_profile = to_be16(MLXCX_TEARDOWN_HCA_GRACEFUL); 1619 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1620 mlxcx_cmd_fini(mlxp, &cmd); 1621 return (B_FALSE); 1622 } 1623 mlxcx_cmd_wait(&cmd); 1624 1625 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1626 mlxcx_cmd_fini(mlxp, &cmd); 1627 return (ret); 1628 } 1629 1630 boolean_t 1631 mlxcx_cmd_query_nic_vport_ctx(mlxcx_t *mlxp, mlxcx_port_t *mlp) 1632 { 1633 mlxcx_cmd_t cmd; 1634 mlxcx_cmd_query_nic_vport_ctx_in_t in; 1635 mlxcx_cmd_query_nic_vport_ctx_out_t out; 1636 boolean_t ret; 1637 const mlxcx_nic_vport_ctx_t *ctx; 1638 1639 bzero(&in, sizeof (in)); 1640 bzero(&out, sizeof (out)); 1641 1642 ASSERT(mutex_owned(&mlp->mlp_mtx)); 1643 mlxcx_cmd_init(mlxp, &cmd); 1644 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_nic_vport_ctx_head, 1645 MLXCX_OP_QUERY_NIC_VPORT_CONTEXT, MLXCX_VPORT_TYPE_VNIC); 1646 1647 in.mlxi_query_nic_vport_ctx_vport_number = to_be16(mlp->mlp_num); 1648 1649 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1650 mlxcx_cmd_fini(mlxp, &cmd); 1651 return (B_FALSE); 1652 } 1653 mlxcx_cmd_wait(&cmd); 1654 1655 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1656 if (ret) { 1657 ctx = &out.mlxo_query_nic_vport_ctx_context; 1658 mlp->mlp_guid = from_be64(ctx->mlnvc_port_guid); 1659 mlp->mlp_mtu = from_be16(ctx->mlnvc_mtu); 1660 bcopy(ctx->mlnvc_permanent_address, mlp->mlp_mac_address, 1661 sizeof (mlp->mlp_mac_address)); 1662 mlp->mlp_wqe_min_inline = get_bits64(ctx->mlnvc_flags, 1663 MLXCX_VPORT_CTX_MIN_WQE_INLINE); 1664 } 1665 mlxcx_cmd_fini(mlxp, &cmd); 1666 return (ret); 1667 } 1668 1669 static const char * 1670 mlxcx_reg_name(mlxcx_register_id_t rid) 1671 { 1672 switch (rid) { 1673 case MLXCX_REG_PMTU: 1674 return ("PMTU"); 1675 case MLXCX_REG_PAOS: 1676 return ("PAOS"); 1677 case MLXCX_REG_PTYS: 1678 return ("PTYS"); 1679 case MLXCX_REG_MSGI: 1680 return ("MSGI"); 1681 case MLXCX_REG_PMAOS: 1682 return ("PMAOS"); 1683 case MLXCX_REG_MLCR: 1684 return ("MLCR"); 1685 case MLXCX_REG_MCIA: 1686 return ("MCIA"); 1687 case MLXCX_REG_PPCNT: 1688 return ("PPCNT"); 1689 case MLXCX_REG_PPLM: 1690 return ("PPLM"); 1691 case MLXCX_REG_MTCAP: 1692 return ("MTCAP"); 1693 case MLXCX_REG_MTMP: 1694 return ("MTMP"); 1695 default: 1696 return ("???"); 1697 } 1698 } 1699 1700 boolean_t 1701 mlxcx_cmd_access_register(mlxcx_t *mlxp, mlxcx_cmd_reg_opmod_t opmod, 1702 mlxcx_register_id_t rid, mlxcx_register_data_t *data) 1703 { 1704 mlxcx_cmd_t cmd; 1705 mlxcx_cmd_access_register_in_t in; 1706 mlxcx_cmd_access_register_out_t out; 1707 boolean_t ret; 1708 size_t dsize, insize, outsize; 1709 1710 bzero(&in, sizeof (in)); 1711 bzero(&out, sizeof (out)); 1712 1713 mlxcx_cmd_init(mlxp, &cmd); 1714 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_access_register_head, 1715 MLXCX_OP_ACCESS_REG, opmod); 1716 1717 in.mlxi_access_register_register_id = to_be16(rid); 1718 1719 switch (rid) { 1720 case MLXCX_REG_PMTU: 1721 dsize = sizeof (mlxcx_reg_pmtu_t); 1722 break; 1723 case MLXCX_REG_PAOS: 1724 dsize = sizeof (mlxcx_reg_paos_t); 1725 break; 1726 case MLXCX_REG_PTYS: 1727 dsize = sizeof (mlxcx_reg_ptys_t); 1728 break; 1729 case MLXCX_REG_MLCR: 1730 dsize = sizeof (mlxcx_reg_mlcr_t); 1731 break; 1732 case MLXCX_REG_PMAOS: 1733 dsize = sizeof (mlxcx_reg_pmaos_t); 1734 break; 1735 case MLXCX_REG_MCIA: 1736 dsize = sizeof (mlxcx_reg_mcia_t); 1737 break; 1738 case MLXCX_REG_PPCNT: 1739 dsize = sizeof (mlxcx_reg_ppcnt_t); 1740 break; 1741 case MLXCX_REG_PPLM: 1742 dsize = sizeof (mlxcx_reg_pplm_t); 1743 break; 1744 case MLXCX_REG_MTCAP: 1745 dsize = sizeof (mlxcx_reg_mtcap_t); 1746 break; 1747 case MLXCX_REG_MTMP: 1748 dsize = sizeof (mlxcx_reg_mtmp_t); 1749 break; 1750 default: 1751 dsize = 0; 1752 VERIFY(0); 1753 return (B_FALSE); 1754 } 1755 insize = dsize + offsetof(mlxcx_cmd_access_register_in_t, 1756 mlxi_access_register_data); 1757 outsize = dsize + offsetof(mlxcx_cmd_access_register_out_t, 1758 mlxo_access_register_data); 1759 1760 bcopy(data, &in.mlxi_access_register_data, dsize); 1761 1762 if (!mlxcx_cmd_send(mlxp, &cmd, &in, insize, &out, outsize)) { 1763 mlxcx_cmd_fini(mlxp, &cmd); 1764 return (B_FALSE); 1765 } 1766 mlxcx_cmd_wait(&cmd); 1767 1768 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1769 if (ret) { 1770 bcopy(&out.mlxo_access_register_data, data, dsize); 1771 } else { 1772 mlxcx_warn(mlxp, "failed OP_ACCESS_REG was for register " 1773 "%04x (%s)", rid, mlxcx_reg_name(rid)); 1774 } 1775 mlxcx_cmd_fini(mlxp, &cmd); 1776 return (ret); 1777 } 1778 1779 boolean_t 1780 mlxcx_cmd_query_port_mtu(mlxcx_t *mlxp, mlxcx_port_t *mlp) 1781 { 1782 mlxcx_register_data_t data; 1783 boolean_t ret; 1784 1785 /* 1786 * Since we modify the port here we require that the caller is holding 1787 * the port mutex. 1788 */ 1789 ASSERT(mutex_owned(&mlp->mlp_mtx)); 1790 bzero(&data, sizeof (data)); 1791 data.mlrd_pmtu.mlrd_pmtu_local_port = mlp->mlp_num + 1; 1792 1793 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_READ, 1794 MLXCX_REG_PMTU, &data); 1795 1796 if (ret) { 1797 mlp->mlp_mtu = from_be16(data.mlrd_pmtu.mlrd_pmtu_admin_mtu); 1798 mlp->mlp_max_mtu = from_be16(data.mlrd_pmtu.mlrd_pmtu_max_mtu); 1799 } 1800 1801 return (ret); 1802 } 1803 1804 boolean_t 1805 mlxcx_cmd_query_module_status(mlxcx_t *mlxp, uint_t id, 1806 mlxcx_module_status_t *pstatus, mlxcx_module_error_type_t *perr) 1807 { 1808 mlxcx_register_data_t data; 1809 boolean_t ret; 1810 1811 bzero(&data, sizeof (data)); 1812 ASSERT3U(id, <, 0xff); 1813 data.mlrd_pmaos.mlrd_pmaos_module = (uint8_t)id; 1814 1815 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_READ, 1816 MLXCX_REG_PMAOS, &data); 1817 1818 if (ret) { 1819 if (pstatus != NULL) 1820 *pstatus = data.mlrd_pmaos.mlrd_pmaos_oper_status; 1821 if (perr != NULL) 1822 *perr = data.mlrd_pmaos.mlrd_pmaos_error_type; 1823 } 1824 1825 return (ret); 1826 } 1827 1828 boolean_t 1829 mlxcx_cmd_set_port_mtu(mlxcx_t *mlxp, mlxcx_port_t *mlp) 1830 { 1831 mlxcx_register_data_t data; 1832 boolean_t ret; 1833 1834 ASSERT(mutex_owned(&mlp->mlp_mtx)); 1835 bzero(&data, sizeof (data)); 1836 data.mlrd_pmtu.mlrd_pmtu_local_port = mlp->mlp_num + 1; 1837 data.mlrd_pmtu.mlrd_pmtu_admin_mtu = to_be16(mlp->mlp_mtu); 1838 1839 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_WRITE, 1840 MLXCX_REG_PMTU, &data); 1841 1842 return (ret); 1843 } 1844 1845 boolean_t 1846 mlxcx_cmd_set_port_led(mlxcx_t *mlxp, mlxcx_port_t *mlp, uint16_t sec) 1847 { 1848 mlxcx_register_data_t data; 1849 boolean_t ret; 1850 1851 ASSERT(mutex_owned(&mlp->mlp_mtx)); 1852 bzero(&data, sizeof (data)); 1853 data.mlrd_mlcr.mlrd_mlcr_local_port = mlp->mlp_num + 1; 1854 set_bits8(&data.mlrd_mlcr.mlrd_mlcr_flags, MLXCX_MLCR_LED_TYPE, 1855 MLXCX_LED_TYPE_PORT); 1856 data.mlrd_mlcr.mlrd_mlcr_beacon_duration = to_be16(sec); 1857 1858 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_WRITE, 1859 MLXCX_REG_MLCR, &data); 1860 1861 return (ret); 1862 } 1863 1864 boolean_t 1865 mlxcx_cmd_query_port_status(mlxcx_t *mlxp, mlxcx_port_t *mlp) 1866 { 1867 mlxcx_register_data_t data; 1868 boolean_t ret; 1869 1870 ASSERT(mutex_owned(&mlp->mlp_mtx)); 1871 bzero(&data, sizeof (data)); 1872 data.mlrd_paos.mlrd_paos_local_port = mlp->mlp_num + 1; 1873 1874 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_READ, 1875 MLXCX_REG_PAOS, &data); 1876 1877 if (ret) { 1878 mlp->mlp_admin_status = data.mlrd_paos.mlrd_paos_admin_status; 1879 mlp->mlp_oper_status = data.mlrd_paos.mlrd_paos_oper_status; 1880 } 1881 1882 return (ret); 1883 } 1884 1885 boolean_t 1886 mlxcx_cmd_modify_port_status(mlxcx_t *mlxp, mlxcx_port_t *mlp, 1887 mlxcx_port_status_t status) 1888 { 1889 mlxcx_register_data_t data; 1890 boolean_t ret; 1891 1892 ASSERT(mutex_owned(&mlp->mlp_mtx)); 1893 bzero(&data, sizeof (data)); 1894 data.mlrd_paos.mlrd_paos_local_port = mlp->mlp_num + 1; 1895 data.mlrd_paos.mlrd_paos_admin_status = status; 1896 set_bit32(&data.mlrd_paos.mlrd_paos_flags, MLXCX_PAOS_ADMIN_ST_EN); 1897 1898 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_WRITE, 1899 MLXCX_REG_PAOS, &data); 1900 1901 return (ret); 1902 } 1903 1904 boolean_t 1905 mlxcx_cmd_query_port_speed(mlxcx_t *mlxp, mlxcx_port_t *mlp) 1906 { 1907 mlxcx_register_data_t data; 1908 boolean_t ret; 1909 1910 ASSERT(mutex_owned(&mlp->mlp_mtx)); 1911 bzero(&data, sizeof (data)); 1912 data.mlrd_ptys.mlrd_ptys_local_port = mlp->mlp_num + 1; 1913 set_bit8(&data.mlrd_ptys.mlrd_ptys_proto_mask, 1914 MLXCX_PTYS_PROTO_MASK_ETH); 1915 1916 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_READ, 1917 MLXCX_REG_PTYS, &data); 1918 1919 if (ret) { 1920 if (get_bit8(data.mlrd_ptys.mlrd_ptys_autoneg_flags, 1921 MLXCX_AUTONEG_DISABLE)) { 1922 mlp->mlp_autoneg = B_FALSE; 1923 } else { 1924 mlp->mlp_autoneg = B_TRUE; 1925 } 1926 mlp->mlp_max_proto = 1927 from_bits32(data.mlrd_ptys.mlrd_ptys_proto_cap); 1928 mlp->mlp_admin_proto = 1929 from_bits32(data.mlrd_ptys.mlrd_ptys_proto_admin); 1930 mlp->mlp_oper_proto = 1931 from_bits32(data.mlrd_ptys.mlrd_ptys_proto_oper); 1932 } 1933 1934 return (ret); 1935 } 1936 1937 boolean_t 1938 mlxcx_cmd_query_port_fec(mlxcx_t *mlxp, mlxcx_port_t *mlp) 1939 { 1940 mlxcx_register_data_t data; 1941 boolean_t ret; 1942 1943 ASSERT(mutex_owned(&mlp->mlp_mtx)); 1944 bzero(&data, sizeof (data)); 1945 data.mlrd_pplm.mlrd_pplm_local_port = mlp->mlp_num + 1; 1946 1947 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_READ, 1948 MLXCX_REG_PPLM, &data); 1949 1950 if (ret) { 1951 mlp->mlp_fec_active = 1952 from_be24(data.mlrd_pplm.mlrd_pplm_fec_mode_active); 1953 } 1954 1955 return (ret); 1956 } 1957 1958 boolean_t 1959 mlxcx_cmd_modify_port_fec(mlxcx_t *mlxp, mlxcx_port_t *mlp, 1960 mlxcx_pplm_fec_caps_t fec) 1961 { 1962 mlxcx_register_data_t data_in, data_out; 1963 mlxcx_pplm_fec_caps_t caps; 1964 mlxcx_reg_pplm_t *pplm_in, *pplm_out; 1965 boolean_t ret; 1966 1967 ASSERT(mutex_owned(&mlp->mlp_mtx)); 1968 bzero(&data_in, sizeof (data_in)); 1969 pplm_in = &data_in.mlrd_pplm; 1970 pplm_in->mlrd_pplm_local_port = mlp->mlp_num + 1; 1971 1972 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_READ, 1973 MLXCX_REG_PPLM, &data_in); 1974 1975 if (!ret) 1976 return (B_FALSE); 1977 1978 bzero(&data_out, sizeof (data_out)); 1979 pplm_out = &data_out.mlrd_pplm; 1980 pplm_out->mlrd_pplm_local_port = mlp->mlp_num + 1; 1981 1982 caps = get_bits32(pplm_in->mlrd_pplm_fec_override_cap, 1983 MLXCX_PPLM_CAP_56G); 1984 set_bits32(&pplm_out->mlrd_pplm_fec_override_admin, 1985 MLXCX_PPLM_CAP_56G, fec & caps); 1986 1987 caps = get_bits32(pplm_in->mlrd_pplm_fec_override_cap, 1988 MLXCX_PPLM_CAP_100G); 1989 set_bits32(&pplm_out->mlrd_pplm_fec_override_admin, 1990 MLXCX_PPLM_CAP_100G, fec & caps); 1991 1992 caps = get_bits32(pplm_in->mlrd_pplm_fec_override_cap, 1993 MLXCX_PPLM_CAP_50G); 1994 set_bits32(&pplm_out->mlrd_pplm_fec_override_admin, 1995 MLXCX_PPLM_CAP_50G, fec & caps); 1996 1997 caps = get_bits32(pplm_in->mlrd_pplm_fec_override_cap, 1998 MLXCX_PPLM_CAP_25G); 1999 set_bits32(&pplm_out->mlrd_pplm_fec_override_admin, 2000 MLXCX_PPLM_CAP_25G, fec & caps); 2001 2002 caps = get_bits32(pplm_in->mlrd_pplm_fec_override_cap, 2003 MLXCX_PPLM_CAP_10_40G); 2004 set_bits32(&pplm_out->mlrd_pplm_fec_override_admin, 2005 MLXCX_PPLM_CAP_10_40G, fec & caps); 2006 2007 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_WRITE, 2008 MLXCX_REG_PPLM, &data_out); 2009 2010 return (ret); 2011 } 2012 2013 boolean_t 2014 mlxcx_cmd_modify_nic_vport_ctx(mlxcx_t *mlxp, mlxcx_port_t *mlp, 2015 mlxcx_modify_nic_vport_ctx_fields_t fields) 2016 { 2017 mlxcx_cmd_t cmd; 2018 mlxcx_cmd_modify_nic_vport_ctx_in_t in; 2019 mlxcx_cmd_modify_nic_vport_ctx_out_t out; 2020 boolean_t ret; 2021 mlxcx_nic_vport_ctx_t *ctx; 2022 2023 ASSERT(mutex_owned(&mlp->mlp_mtx)); 2024 bzero(&in, sizeof (in)); 2025 bzero(&out, sizeof (out)); 2026 2027 mlxcx_cmd_init(mlxp, &cmd); 2028 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_modify_nic_vport_ctx_head, 2029 MLXCX_OP_MODIFY_NIC_VPORT_CONTEXT, MLXCX_VPORT_TYPE_VNIC); 2030 2031 in.mlxi_modify_nic_vport_ctx_vport_number = to_be16(mlp->mlp_num); 2032 in.mlxi_modify_nic_vport_ctx_field_select = to_be32(fields); 2033 2034 ctx = &in.mlxi_modify_nic_vport_ctx_context; 2035 if (fields & MLXCX_MODIFY_NIC_VPORT_CTX_PROMISC) { 2036 set_bit16(&ctx->mlnvc_promisc_list_type, 2037 MLXCX_VPORT_PROMISC_ALL); 2038 } 2039 if (fields & MLXCX_MODIFY_NIC_VPORT_CTX_MTU) { 2040 ctx->mlnvc_mtu = to_be16(mlp->mlp_mtu); 2041 } 2042 2043 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2044 mlxcx_cmd_fini(mlxp, &cmd); 2045 return (B_FALSE); 2046 } 2047 mlxcx_cmd_wait(&cmd); 2048 2049 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2050 if (ret) { 2051 if (fields & MLXCX_MODIFY_NIC_VPORT_CTX_PROMISC) { 2052 mlp->mlp_flags |= MLXCX_PORT_VPORT_PROMISC; 2053 } 2054 } 2055 mlxcx_cmd_fini(mlxp, &cmd); 2056 return (ret); 2057 } 2058 2059 boolean_t 2060 mlxcx_cmd_create_eq(mlxcx_t *mlxp, mlxcx_event_queue_t *mleq) 2061 { 2062 mlxcx_cmd_t cmd; 2063 mlxcx_cmd_create_eq_in_t in; 2064 mlxcx_cmd_create_eq_out_t out; 2065 boolean_t ret; 2066 mlxcx_eventq_ctx_t *ctx; 2067 size_t rem, insize; 2068 const ddi_dma_cookie_t *c; 2069 uint64_t pa, npages; 2070 2071 bzero(&in, sizeof (in)); 2072 bzero(&out, sizeof (out)); 2073 2074 ASSERT(mutex_owned(&mleq->mleq_mtx)); 2075 VERIFY(mleq->mleq_state & MLXCX_EQ_ALLOC); 2076 VERIFY0(mleq->mleq_state & MLXCX_EQ_CREATED); 2077 2078 mlxcx_cmd_init(mlxp, &cmd); 2079 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_eq_head, 2080 MLXCX_OP_CREATE_EQ, 0); 2081 2082 ctx = &in.mlxi_create_eq_context; 2083 ctx->mleqc_uar_page = to_be24(mleq->mleq_uar->mlu_num); 2084 ctx->mleqc_log_eq_size = mleq->mleq_entshift; 2085 ctx->mleqc_intr = mleq->mleq_intr_index; 2086 2087 in.mlxi_create_eq_event_bitmask = to_be64(mleq->mleq_events); 2088 2089 npages = 0; 2090 c = NULL; 2091 while ((c = mlxcx_dma_cookie_iter(&mleq->mleq_dma, c)) != NULL) { 2092 pa = c->dmac_laddress; 2093 rem = c->dmac_size; 2094 while (rem > 0) { 2095 ASSERT3U(pa & 0xfff, ==, 0); 2096 ASSERT3U(rem, >=, MLXCX_HW_PAGE_SIZE); 2097 in.mlxi_create_eq_pas[npages++] = to_be64(pa); 2098 rem -= MLXCX_HW_PAGE_SIZE; 2099 pa += MLXCX_HW_PAGE_SIZE; 2100 } 2101 } 2102 ASSERT3U(npages, <=, MLXCX_CREATE_QUEUE_MAX_PAGES); 2103 2104 insize = offsetof(mlxcx_cmd_create_eq_in_t, mlxi_create_eq_pas) + 2105 sizeof (uint64_t) * npages; 2106 2107 if (!mlxcx_cmd_send(mlxp, &cmd, &in, insize, &out, sizeof (out))) { 2108 mlxcx_cmd_fini(mlxp, &cmd); 2109 return (B_FALSE); 2110 } 2111 mlxcx_cmd_wait(&cmd); 2112 2113 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2114 if (ret) { 2115 mleq->mleq_state |= MLXCX_EQ_CREATED; 2116 mleq->mleq_num = out.mlxo_create_eq_eqn; 2117 } 2118 mlxcx_cmd_fini(mlxp, &cmd); 2119 return (ret); 2120 } 2121 2122 boolean_t 2123 mlxcx_cmd_query_eq(mlxcx_t *mlxp, mlxcx_event_queue_t *mleq, 2124 mlxcx_eventq_ctx_t *ctxp) 2125 { 2126 mlxcx_cmd_t cmd; 2127 mlxcx_cmd_query_eq_in_t in; 2128 mlxcx_cmd_query_eq_out_t out; 2129 boolean_t ret; 2130 2131 bzero(&in, sizeof (in)); 2132 bzero(&out, sizeof (out)); 2133 2134 VERIFY(mleq->mleq_state & MLXCX_EQ_ALLOC); 2135 VERIFY(mleq->mleq_state & MLXCX_EQ_CREATED); 2136 2137 mlxcx_cmd_init(mlxp, &cmd); 2138 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_eq_head, 2139 MLXCX_OP_QUERY_EQ, 0); 2140 2141 in.mlxi_query_eq_eqn = mleq->mleq_num; 2142 2143 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2144 mlxcx_cmd_fini(mlxp, &cmd); 2145 return (B_FALSE); 2146 } 2147 mlxcx_cmd_wait(&cmd); 2148 2149 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2150 if (ret) { 2151 bcopy(&out.mlxo_query_eq_context, ctxp, 2152 sizeof (mlxcx_eventq_ctx_t)); 2153 } 2154 mlxcx_cmd_fini(mlxp, &cmd); 2155 return (ret); 2156 } 2157 2158 boolean_t 2159 mlxcx_cmd_destroy_eq(mlxcx_t *mlxp, mlxcx_event_queue_t *mleq) 2160 { 2161 mlxcx_cmd_t cmd; 2162 mlxcx_cmd_destroy_eq_in_t in; 2163 mlxcx_cmd_destroy_eq_out_t out; 2164 boolean_t ret; 2165 2166 bzero(&in, sizeof (in)); 2167 bzero(&out, sizeof (out)); 2168 2169 ASSERT(mutex_owned(&mleq->mleq_mtx)); 2170 VERIFY(mleq->mleq_state & MLXCX_EQ_ALLOC); 2171 VERIFY(mleq->mleq_state & MLXCX_EQ_CREATED); 2172 2173 mlxcx_cmd_init(mlxp, &cmd); 2174 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_eq_head, 2175 MLXCX_OP_DESTROY_EQ, 0); 2176 2177 in.mlxi_destroy_eq_eqn = mleq->mleq_num; 2178 2179 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2180 mlxcx_cmd_fini(mlxp, &cmd); 2181 return (B_FALSE); 2182 } 2183 mlxcx_cmd_wait(&cmd); 2184 2185 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2186 if (ret) { 2187 mleq->mleq_state |= MLXCX_EQ_DESTROYED; 2188 } 2189 mlxcx_cmd_fini(mlxp, &cmd); 2190 return (ret); 2191 } 2192 2193 boolean_t 2194 mlxcx_cmd_query_special_ctxs(mlxcx_t *mlxp) 2195 { 2196 mlxcx_cmd_t cmd; 2197 mlxcx_cmd_query_special_ctxs_in_t in; 2198 mlxcx_cmd_query_special_ctxs_out_t out; 2199 boolean_t ret; 2200 2201 bzero(&in, sizeof (in)); 2202 bzero(&out, sizeof (out)); 2203 2204 mlxcx_cmd_init(mlxp, &cmd); 2205 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_special_ctxs_head, 2206 MLXCX_OP_QUERY_SPECIAL_CONTEXTS, 0); 2207 2208 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2209 mlxcx_cmd_fini(mlxp, &cmd); 2210 return (B_FALSE); 2211 } 2212 mlxcx_cmd_wait(&cmd); 2213 2214 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2215 if (ret) { 2216 mlxp->mlx_rsvd_lkey = from_be32( 2217 out.mlxo_query_special_ctxs_resd_lkey); 2218 } 2219 mlxcx_cmd_fini(mlxp, &cmd); 2220 return (ret); 2221 } 2222 2223 boolean_t 2224 mlxcx_cmd_create_cq(mlxcx_t *mlxp, mlxcx_completion_queue_t *mlcq) 2225 { 2226 mlxcx_cmd_t cmd; 2227 mlxcx_cmd_create_cq_in_t in; 2228 mlxcx_cmd_create_cq_out_t out; 2229 boolean_t ret; 2230 mlxcx_completionq_ctx_t *ctx; 2231 size_t rem, insize; 2232 const ddi_dma_cookie_t *c; 2233 uint64_t pa, npages; 2234 2235 bzero(&in, sizeof (in)); 2236 bzero(&out, sizeof (out)); 2237 2238 ASSERT(mutex_owned(&mlcq->mlcq_mtx)); 2239 VERIFY(mlcq->mlcq_state & MLXCX_CQ_ALLOC); 2240 VERIFY0(mlcq->mlcq_state & MLXCX_CQ_CREATED); 2241 2242 mlxcx_cmd_init(mlxp, &cmd); 2243 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_cq_head, 2244 MLXCX_OP_CREATE_CQ, 0); 2245 2246 ctx = &in.mlxi_create_cq_context; 2247 ctx->mlcqc_uar_page = to_be24(mlcq->mlcq_uar->mlu_num); 2248 ctx->mlcqc_log_cq_size = mlcq->mlcq_entshift; 2249 ctx->mlcqc_eqn = mlcq->mlcq_eq->mleq_num; 2250 ctx->mlcqc_cq_period = to_be16(mlcq->mlcq_cqemod_period_usec); 2251 ctx->mlcqc_cq_max_count = to_be16(mlcq->mlcq_cqemod_count); 2252 2253 c = mlxcx_dma_cookie_one(&mlcq->mlcq_doorbell_dma); 2254 ctx->mlcqc_dbr_addr = to_be64(c->dmac_laddress); 2255 ASSERT3U(c->dmac_size, >=, sizeof (mlxcx_completionq_doorbell_t)); 2256 2257 npages = 0; 2258 c = NULL; 2259 while ((c = mlxcx_dma_cookie_iter(&mlcq->mlcq_dma, c)) != NULL) { 2260 pa = c->dmac_laddress; 2261 rem = c->dmac_size; 2262 while (rem > 0) { 2263 ASSERT3U(pa & 0xfff, ==, 0); 2264 ASSERT3U(rem, >=, MLXCX_HW_PAGE_SIZE); 2265 in.mlxi_create_cq_pas[npages++] = to_be64(pa); 2266 rem -= MLXCX_HW_PAGE_SIZE; 2267 pa += MLXCX_HW_PAGE_SIZE; 2268 } 2269 } 2270 ASSERT3U(npages, <=, MLXCX_CREATE_QUEUE_MAX_PAGES); 2271 2272 insize = offsetof(mlxcx_cmd_create_cq_in_t, mlxi_create_cq_pas) + 2273 sizeof (uint64_t) * npages; 2274 2275 if (!mlxcx_cmd_send(mlxp, &cmd, &in, insize, &out, sizeof (out))) { 2276 mlxcx_cmd_fini(mlxp, &cmd); 2277 return (B_FALSE); 2278 } 2279 mlxcx_cmd_wait(&cmd); 2280 2281 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2282 if (ret) { 2283 atomic_or_uint(&mlcq->mlcq_state, MLXCX_CQ_CREATED); 2284 mlcq->mlcq_num = from_be24(out.mlxo_create_cq_cqn); 2285 } 2286 mlxcx_cmd_fini(mlxp, &cmd); 2287 return (ret); 2288 } 2289 2290 boolean_t 2291 mlxcx_cmd_query_rq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq, 2292 mlxcx_rq_ctx_t *ctxp) 2293 { 2294 mlxcx_cmd_t cmd; 2295 mlxcx_cmd_query_rq_in_t in; 2296 mlxcx_cmd_query_rq_out_t out; 2297 boolean_t ret; 2298 2299 bzero(&in, sizeof (in)); 2300 bzero(&out, sizeof (out)); 2301 2302 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 2303 VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED); 2304 ASSERT3S(mlwq->mlwq_type, ==, MLXCX_WQ_TYPE_RECVQ); 2305 2306 mlxcx_cmd_init(mlxp, &cmd); 2307 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_rq_head, 2308 MLXCX_OP_QUERY_RQ, 0); 2309 2310 in.mlxi_query_rq_rqn = to_be24(mlwq->mlwq_num); 2311 2312 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2313 mlxcx_cmd_fini(mlxp, &cmd); 2314 return (B_FALSE); 2315 } 2316 mlxcx_cmd_wait(&cmd); 2317 2318 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2319 if (ret) { 2320 bcopy(&out.mlxo_query_rq_context, ctxp, 2321 sizeof (mlxcx_rq_ctx_t)); 2322 } 2323 mlxcx_cmd_fini(mlxp, &cmd); 2324 return (ret); 2325 } 2326 2327 boolean_t 2328 mlxcx_cmd_query_sq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq, 2329 mlxcx_sq_ctx_t *ctxp) 2330 { 2331 mlxcx_cmd_t cmd; 2332 mlxcx_cmd_query_sq_in_t in; 2333 mlxcx_cmd_query_sq_out_t out; 2334 boolean_t ret; 2335 2336 bzero(&in, sizeof (in)); 2337 bzero(&out, sizeof (out)); 2338 2339 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 2340 VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED); 2341 ASSERT3S(mlwq->mlwq_type, ==, MLXCX_WQ_TYPE_SENDQ); 2342 2343 mlxcx_cmd_init(mlxp, &cmd); 2344 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_sq_head, 2345 MLXCX_OP_QUERY_SQ, 0); 2346 2347 in.mlxi_query_sq_sqn = to_be24(mlwq->mlwq_num); 2348 2349 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2350 mlxcx_cmd_fini(mlxp, &cmd); 2351 return (B_FALSE); 2352 } 2353 mlxcx_cmd_wait(&cmd); 2354 2355 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2356 if (ret) { 2357 bcopy(&out.mlxo_query_sq_context, ctxp, 2358 sizeof (mlxcx_sq_ctx_t)); 2359 } 2360 mlxcx_cmd_fini(mlxp, &cmd); 2361 return (ret); 2362 } 2363 2364 boolean_t 2365 mlxcx_cmd_query_cq(mlxcx_t *mlxp, mlxcx_completion_queue_t *mlcq, 2366 mlxcx_completionq_ctx_t *ctxp) 2367 { 2368 mlxcx_cmd_t cmd; 2369 mlxcx_cmd_query_cq_in_t in; 2370 mlxcx_cmd_query_cq_out_t out; 2371 boolean_t ret; 2372 2373 bzero(&in, sizeof (in)); 2374 bzero(&out, sizeof (out)); 2375 2376 VERIFY(mlcq->mlcq_state & MLXCX_CQ_ALLOC); 2377 VERIFY(mlcq->mlcq_state & MLXCX_CQ_CREATED); 2378 2379 mlxcx_cmd_init(mlxp, &cmd); 2380 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_cq_head, 2381 MLXCX_OP_QUERY_CQ, 0); 2382 2383 in.mlxi_query_cq_cqn = to_be24(mlcq->mlcq_num); 2384 2385 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2386 mlxcx_cmd_fini(mlxp, &cmd); 2387 return (B_FALSE); 2388 } 2389 mlxcx_cmd_wait(&cmd); 2390 2391 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2392 if (ret) { 2393 bcopy(&out.mlxo_query_cq_context, ctxp, 2394 sizeof (mlxcx_completionq_ctx_t)); 2395 } 2396 mlxcx_cmd_fini(mlxp, &cmd); 2397 return (ret); 2398 } 2399 2400 boolean_t 2401 mlxcx_cmd_destroy_cq(mlxcx_t *mlxp, mlxcx_completion_queue_t *mlcq) 2402 { 2403 mlxcx_cmd_t cmd; 2404 mlxcx_cmd_destroy_cq_in_t in; 2405 mlxcx_cmd_destroy_cq_out_t out; 2406 boolean_t ret; 2407 2408 bzero(&in, sizeof (in)); 2409 bzero(&out, sizeof (out)); 2410 2411 ASSERT(mutex_owned(&mlcq->mlcq_mtx)); 2412 VERIFY(mlcq->mlcq_state & MLXCX_CQ_ALLOC); 2413 VERIFY(mlcq->mlcq_state & MLXCX_CQ_CREATED); 2414 2415 mlxcx_cmd_init(mlxp, &cmd); 2416 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_cq_head, 2417 MLXCX_OP_DESTROY_CQ, 0); 2418 2419 in.mlxi_destroy_cq_cqn = to_be24(mlcq->mlcq_num); 2420 2421 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2422 mlxcx_cmd_fini(mlxp, &cmd); 2423 return (B_FALSE); 2424 } 2425 mlxcx_cmd_wait(&cmd); 2426 2427 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2428 if (ret) { 2429 atomic_or_uint(&mlcq->mlcq_state, MLXCX_CQ_DESTROYED); 2430 } 2431 mlxcx_cmd_fini(mlxp, &cmd); 2432 return (ret); 2433 } 2434 2435 boolean_t 2436 mlxcx_cmd_create_rq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq) 2437 { 2438 mlxcx_cmd_t cmd; 2439 mlxcx_cmd_create_rq_in_t in; 2440 mlxcx_cmd_create_rq_out_t out; 2441 boolean_t ret; 2442 mlxcx_rq_ctx_t *ctx; 2443 size_t rem, insize; 2444 const ddi_dma_cookie_t *c; 2445 uint64_t pa, npages; 2446 2447 bzero(&in, sizeof (in)); 2448 bzero(&out, sizeof (out)); 2449 2450 ASSERT(mutex_owned(&mlwq->mlwq_mtx)); 2451 VERIFY3U(mlwq->mlwq_type, ==, MLXCX_WQ_TYPE_RECVQ); 2452 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 2453 VERIFY0(mlwq->mlwq_state & MLXCX_WQ_CREATED); 2454 2455 mlxcx_cmd_init(mlxp, &cmd); 2456 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_rq_head, 2457 MLXCX_OP_CREATE_RQ, 0); 2458 2459 ctx = &in.mlxi_create_rq_context; 2460 2461 set_bit32(&ctx->mlrqc_flags, MLXCX_RQ_FLAGS_RLKEY); 2462 set_bit32(&ctx->mlrqc_flags, MLXCX_RQ_FLAGS_FLUSH_IN_ERROR); 2463 set_bit32(&ctx->mlrqc_flags, MLXCX_RQ_FLAGS_VLAN_STRIP_DISABLE); 2464 ctx->mlrqc_cqn = to_be24(mlwq->mlwq_cq->mlcq_num); 2465 2466 set_bits32(&ctx->mlrqc_wq.mlwqc_flags, MLXCX_WORKQ_CTX_TYPE, 2467 MLXCX_WORKQ_TYPE_CYCLIC); 2468 ctx->mlrqc_wq.mlwqc_pd = to_be24(mlwq->mlwq_pd->mlpd_num); 2469 ctx->mlrqc_wq.mlwqc_log_wq_sz = mlwq->mlwq_entshift; 2470 ctx->mlrqc_wq.mlwqc_log_wq_stride = MLXCX_RECVQ_STRIDE_SHIFT; 2471 2472 c = mlxcx_dma_cookie_one(&mlwq->mlwq_doorbell_dma); 2473 ctx->mlrqc_wq.mlwqc_dbr_addr = to_be64(c->dmac_laddress); 2474 ASSERT3U(c->dmac_size, >=, sizeof (mlxcx_workq_doorbell_t)); 2475 2476 npages = 0; 2477 c = NULL; 2478 while ((c = mlxcx_dma_cookie_iter(&mlwq->mlwq_dma, c)) != NULL) { 2479 pa = c->dmac_laddress; 2480 rem = c->dmac_size; 2481 while (rem > 0) { 2482 ASSERT3U(pa & 0xfff, ==, 0); 2483 ASSERT3U(rem, >=, MLXCX_HW_PAGE_SIZE); 2484 ctx->mlrqc_wq.mlwqc_pas[npages++] = to_be64(pa); 2485 rem -= MLXCX_HW_PAGE_SIZE; 2486 pa += MLXCX_HW_PAGE_SIZE; 2487 } 2488 } 2489 ASSERT3U(npages, <=, MLXCX_WORKQ_CTX_MAX_ADDRESSES); 2490 2491 insize = offsetof(mlxcx_cmd_create_rq_in_t, mlxi_create_rq_context) + 2492 offsetof(mlxcx_rq_ctx_t, mlrqc_wq) + 2493 offsetof(mlxcx_workq_ctx_t, mlwqc_pas) + 2494 sizeof (uint64_t) * npages; 2495 2496 if (!mlxcx_cmd_send(mlxp, &cmd, &in, insize, &out, sizeof (out))) { 2497 mlxcx_cmd_fini(mlxp, &cmd); 2498 return (B_FALSE); 2499 } 2500 mlxcx_cmd_wait(&cmd); 2501 2502 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2503 if (ret) { 2504 mlwq->mlwq_state |= MLXCX_WQ_CREATED; 2505 mlwq->mlwq_num = from_be24(out.mlxo_create_rq_rqn); 2506 } 2507 mlxcx_cmd_fini(mlxp, &cmd); 2508 return (ret); 2509 } 2510 2511 boolean_t 2512 mlxcx_cmd_start_rq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq) 2513 { 2514 mlxcx_cmd_t cmd; 2515 mlxcx_cmd_modify_rq_in_t in; 2516 mlxcx_cmd_modify_rq_out_t out; 2517 boolean_t ret; 2518 ddi_fm_error_t err; 2519 2520 bzero(&in, sizeof (in)); 2521 bzero(&out, sizeof (out)); 2522 2523 ASSERT(mutex_owned(&mlwq->mlwq_mtx)); 2524 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 2525 VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED); 2526 VERIFY0(mlwq->mlwq_state & MLXCX_WQ_STARTED); 2527 2528 /* 2529 * Before starting the queue, we have to be sure that it is 2530 * empty and the doorbell and counters are set to 0. 2531 */ 2532 ASSERT(mutex_owned(&mlwq->mlwq_cq->mlcq_mtx)); 2533 ASSERT(list_is_empty(&mlwq->mlwq_cq->mlcq_buffers)); 2534 ASSERT(list_is_empty(&mlwq->mlwq_cq->mlcq_buffers_b)); 2535 2536 mlwq->mlwq_doorbell->mlwqd_recv_counter = to_be16(0); 2537 MLXCX_DMA_SYNC(mlwq->mlwq_doorbell_dma, DDI_DMA_SYNC_FORDEV); 2538 ddi_fm_dma_err_get(mlwq->mlwq_doorbell_dma.mxdb_dma_handle, &err, 2539 DDI_FME_VERSION); 2540 if (err.fme_status != DDI_FM_OK) 2541 return (B_FALSE); 2542 mlwq->mlwq_pc = 0; 2543 2544 mlxcx_cmd_init(mlxp, &cmd); 2545 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_modify_rq_head, 2546 MLXCX_OP_MODIFY_RQ, 0); 2547 2548 in.mlxi_modify_rq_rqn = to_be24(mlwq->mlwq_num); 2549 2550 /* From state */ 2551 set_bits8(&in.mlxi_modify_rq_state, MLXCX_CMD_MODIFY_RQ_STATE, 2552 MLXCX_RQ_STATE_RST); 2553 /* To state */ 2554 set_bits32(&in.mlxi_modify_rq_context.mlrqc_flags, MLXCX_RQ_STATE, 2555 MLXCX_RQ_STATE_RDY); 2556 2557 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2558 mlxcx_cmd_fini(mlxp, &cmd); 2559 return (B_FALSE); 2560 } 2561 mlxcx_cmd_wait(&cmd); 2562 2563 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2564 if (ret) { 2565 mlwq->mlwq_state |= MLXCX_WQ_STARTED; 2566 } 2567 mlxcx_cmd_fini(mlxp, &cmd); 2568 return (ret); 2569 } 2570 2571 boolean_t 2572 mlxcx_cmd_stop_rq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq) 2573 { 2574 mlxcx_cmd_t cmd; 2575 mlxcx_cmd_modify_rq_in_t in; 2576 mlxcx_cmd_modify_rq_out_t out; 2577 boolean_t ret; 2578 2579 bzero(&in, sizeof (in)); 2580 bzero(&out, sizeof (out)); 2581 2582 ASSERT(mutex_owned(&mlwq->mlwq_mtx)); 2583 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 2584 VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED); 2585 VERIFY(mlwq->mlwq_state & MLXCX_WQ_STARTED); 2586 2587 mlxcx_cmd_init(mlxp, &cmd); 2588 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_modify_rq_head, 2589 MLXCX_OP_MODIFY_RQ, 0); 2590 2591 in.mlxi_modify_rq_rqn = to_be24(mlwq->mlwq_num); 2592 2593 /* From state */ 2594 set_bits8(&in.mlxi_modify_rq_state, MLXCX_CMD_MODIFY_RQ_STATE, 2595 MLXCX_RQ_STATE_RDY); 2596 /* To state */ 2597 set_bits32(&in.mlxi_modify_rq_context.mlrqc_flags, MLXCX_RQ_STATE, 2598 MLXCX_RQ_STATE_RST); 2599 2600 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2601 mlxcx_cmd_fini(mlxp, &cmd); 2602 return (B_FALSE); 2603 } 2604 mlxcx_cmd_wait(&cmd); 2605 2606 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2607 if (ret) { 2608 mlwq->mlwq_state &= ~MLXCX_WQ_STARTED; 2609 } 2610 mlxcx_cmd_fini(mlxp, &cmd); 2611 return (ret); 2612 } 2613 2614 boolean_t 2615 mlxcx_cmd_destroy_rq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq) 2616 { 2617 mlxcx_cmd_t cmd; 2618 mlxcx_cmd_destroy_rq_in_t in; 2619 mlxcx_cmd_destroy_rq_out_t out; 2620 boolean_t ret; 2621 2622 bzero(&in, sizeof (in)); 2623 bzero(&out, sizeof (out)); 2624 2625 ASSERT(mutex_owned(&mlwq->mlwq_mtx)); 2626 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 2627 VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED); 2628 VERIFY0(mlwq->mlwq_state & MLXCX_WQ_STARTED); 2629 2630 mlxcx_cmd_init(mlxp, &cmd); 2631 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_rq_head, 2632 MLXCX_OP_DESTROY_RQ, 0); 2633 2634 in.mlxi_destroy_rq_rqn = to_be24(mlwq->mlwq_num); 2635 2636 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2637 mlxcx_cmd_fini(mlxp, &cmd); 2638 return (B_FALSE); 2639 } 2640 mlxcx_cmd_wait(&cmd); 2641 2642 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2643 if (ret) { 2644 mlwq->mlwq_state |= MLXCX_WQ_DESTROYED; 2645 } 2646 mlxcx_cmd_fini(mlxp, &cmd); 2647 return (ret); 2648 } 2649 2650 boolean_t 2651 mlxcx_cmd_create_tir(mlxcx_t *mlxp, mlxcx_tir_t *mltir) 2652 { 2653 mlxcx_cmd_t cmd; 2654 mlxcx_cmd_create_tir_in_t in; 2655 mlxcx_cmd_create_tir_out_t out; 2656 mlxcx_tir_ctx_t *ctx; 2657 boolean_t ret; 2658 2659 bzero(&in, sizeof (in)); 2660 bzero(&out, sizeof (out)); 2661 2662 VERIFY0(mltir->mltir_state & MLXCX_TIR_CREATED); 2663 2664 mlxcx_cmd_init(mlxp, &cmd); 2665 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_tir_head, 2666 MLXCX_OP_CREATE_TIR, 0); 2667 2668 ctx = &in.mlxi_create_tir_context; 2669 ctx->mltirc_transport_domain = to_be24(mltir->mltir_tdom->mltd_num); 2670 set_bits8(&ctx->mltirc_disp_type, MLXCX_TIR_CTX_DISP_TYPE, 2671 mltir->mltir_type); 2672 switch (mltir->mltir_type) { 2673 case MLXCX_TIR_INDIRECT: 2674 VERIFY(mltir->mltir_rqtable != NULL); 2675 VERIFY(mltir->mltir_rqtable->mlrqt_state & MLXCX_RQT_CREATED); 2676 ctx->mltirc_indirect_table = 2677 to_be24(mltir->mltir_rqtable->mlrqt_num); 2678 set_bits8(&ctx->mltirc_hash_lb, MLXCX_TIR_RX_HASH_FN, 2679 mltir->mltir_hash_fn); 2680 bcopy(mltir->mltir_toeplitz_key, 2681 ctx->mltirc_rx_hash_toeplitz_key, 2682 sizeof (ctx->mltirc_rx_hash_toeplitz_key)); 2683 set_bits32(&ctx->mltirc_rx_hash_fields_outer, 2684 MLXCX_RX_HASH_L3_TYPE, mltir->mltir_l3_type); 2685 set_bits32(&ctx->mltirc_rx_hash_fields_outer, 2686 MLXCX_RX_HASH_L4_TYPE, mltir->mltir_l4_type); 2687 set_bits32(&ctx->mltirc_rx_hash_fields_outer, 2688 MLXCX_RX_HASH_FIELDS, mltir->mltir_hash_fields); 2689 break; 2690 case MLXCX_TIR_DIRECT: 2691 VERIFY(mltir->mltir_rq != NULL); 2692 VERIFY(mltir->mltir_rq->mlwq_state & MLXCX_WQ_CREATED); 2693 ctx->mltirc_inline_rqn = to_be24(mltir->mltir_rq->mlwq_num); 2694 break; 2695 default: 2696 VERIFY(0); 2697 } 2698 2699 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2700 mlxcx_cmd_fini(mlxp, &cmd); 2701 return (B_FALSE); 2702 } 2703 mlxcx_cmd_wait(&cmd); 2704 2705 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2706 if (ret) { 2707 mltir->mltir_state |= MLXCX_TIR_CREATED; 2708 mltir->mltir_num = from_be24(out.mlxo_create_tir_tirn); 2709 } 2710 mlxcx_cmd_fini(mlxp, &cmd); 2711 return (ret); 2712 } 2713 2714 boolean_t 2715 mlxcx_cmd_destroy_tir(mlxcx_t *mlxp, mlxcx_tir_t *mltir) 2716 { 2717 mlxcx_cmd_t cmd; 2718 mlxcx_cmd_destroy_tir_in_t in; 2719 mlxcx_cmd_destroy_tir_out_t out; 2720 boolean_t ret; 2721 2722 bzero(&in, sizeof (in)); 2723 bzero(&out, sizeof (out)); 2724 2725 VERIFY(mltir->mltir_state & MLXCX_TIR_CREATED); 2726 VERIFY0(mltir->mltir_state & MLXCX_TIR_DESTROYED); 2727 2728 mlxcx_cmd_init(mlxp, &cmd); 2729 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_tir_head, 2730 MLXCX_OP_DESTROY_TIR, 0); 2731 2732 in.mlxi_destroy_tir_tirn = to_be24(mltir->mltir_num); 2733 2734 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2735 mlxcx_cmd_fini(mlxp, &cmd); 2736 return (B_FALSE); 2737 } 2738 mlxcx_cmd_wait(&cmd); 2739 2740 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2741 if (ret) { 2742 mltir->mltir_state |= MLXCX_TIR_DESTROYED; 2743 } 2744 mlxcx_cmd_fini(mlxp, &cmd); 2745 return (ret); 2746 } 2747 2748 boolean_t 2749 mlxcx_cmd_create_tis(mlxcx_t *mlxp, mlxcx_tis_t *mltis) 2750 { 2751 mlxcx_cmd_t cmd; 2752 mlxcx_cmd_create_tis_in_t in; 2753 mlxcx_cmd_create_tis_out_t out; 2754 mlxcx_tis_ctx_t *ctx; 2755 boolean_t ret; 2756 2757 bzero(&in, sizeof (in)); 2758 bzero(&out, sizeof (out)); 2759 2760 VERIFY0(mltis->mltis_state & MLXCX_TIS_CREATED); 2761 2762 mlxcx_cmd_init(mlxp, &cmd); 2763 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_tis_head, 2764 MLXCX_OP_CREATE_TIS, 0); 2765 2766 ctx = &in.mlxi_create_tis_context; 2767 ctx->mltisc_transport_domain = to_be24(mltis->mltis_tdom->mltd_num); 2768 2769 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2770 mlxcx_cmd_fini(mlxp, &cmd); 2771 return (B_FALSE); 2772 } 2773 mlxcx_cmd_wait(&cmd); 2774 2775 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2776 if (ret) { 2777 mltis->mltis_state |= MLXCX_TIS_CREATED; 2778 mltis->mltis_num = from_be24(out.mlxo_create_tis_tisn); 2779 } 2780 mlxcx_cmd_fini(mlxp, &cmd); 2781 return (ret); 2782 } 2783 2784 boolean_t 2785 mlxcx_cmd_destroy_tis(mlxcx_t *mlxp, mlxcx_tis_t *mltis) 2786 { 2787 mlxcx_cmd_t cmd; 2788 mlxcx_cmd_destroy_tis_in_t in; 2789 mlxcx_cmd_destroy_tis_out_t out; 2790 boolean_t ret; 2791 2792 bzero(&in, sizeof (in)); 2793 bzero(&out, sizeof (out)); 2794 2795 VERIFY(mltis->mltis_state & MLXCX_TIR_CREATED); 2796 VERIFY0(mltis->mltis_state & MLXCX_TIR_DESTROYED); 2797 2798 mlxcx_cmd_init(mlxp, &cmd); 2799 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_tis_head, 2800 MLXCX_OP_DESTROY_TIS, 0); 2801 2802 in.mlxi_destroy_tis_tisn = to_be24(mltis->mltis_num); 2803 2804 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2805 mlxcx_cmd_fini(mlxp, &cmd); 2806 return (B_FALSE); 2807 } 2808 mlxcx_cmd_wait(&cmd); 2809 2810 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2811 if (ret) { 2812 mltis->mltis_state |= MLXCX_TIS_DESTROYED; 2813 } 2814 mlxcx_cmd_fini(mlxp, &cmd); 2815 return (ret); 2816 } 2817 2818 boolean_t 2819 mlxcx_cmd_create_flow_table(mlxcx_t *mlxp, mlxcx_flow_table_t *mlft) 2820 { 2821 mlxcx_cmd_t cmd; 2822 mlxcx_cmd_create_flow_table_in_t in; 2823 mlxcx_cmd_create_flow_table_out_t out; 2824 mlxcx_flow_table_ctx_t *ctx; 2825 boolean_t ret; 2826 2827 bzero(&in, sizeof (in)); 2828 bzero(&out, sizeof (out)); 2829 2830 ASSERT(mutex_owned(&mlft->mlft_mtx)); 2831 VERIFY0(mlft->mlft_state & MLXCX_FLOW_TABLE_CREATED); 2832 2833 mlxcx_cmd_init(mlxp, &cmd); 2834 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_flow_table_head, 2835 MLXCX_OP_CREATE_FLOW_TABLE, 0); 2836 2837 in.mlxi_create_flow_table_vport_number = 2838 to_be16(mlft->mlft_port->mlp_num); 2839 in.mlxi_create_flow_table_table_type = mlft->mlft_type; 2840 ctx = &in.mlxi_create_flow_table_context; 2841 ctx->mlftc_log_size = mlft->mlft_entshift; 2842 ctx->mlftc_level = mlft->mlft_level; 2843 2844 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2845 mlxcx_cmd_fini(mlxp, &cmd); 2846 return (B_FALSE); 2847 } 2848 mlxcx_cmd_wait(&cmd); 2849 2850 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2851 if (ret) { 2852 mlft->mlft_num = from_be24(out.mlxo_create_flow_table_table_id); 2853 mlft->mlft_state |= MLXCX_FLOW_TABLE_CREATED; 2854 } 2855 mlxcx_cmd_fini(mlxp, &cmd); 2856 return (ret); 2857 } 2858 2859 boolean_t 2860 mlxcx_cmd_destroy_flow_table(mlxcx_t *mlxp, mlxcx_flow_table_t *mlft) 2861 { 2862 mlxcx_cmd_t cmd; 2863 mlxcx_cmd_destroy_flow_table_in_t in; 2864 mlxcx_cmd_destroy_flow_table_out_t out; 2865 boolean_t ret; 2866 2867 bzero(&in, sizeof (in)); 2868 bzero(&out, sizeof (out)); 2869 2870 ASSERT(mutex_owned(&mlft->mlft_mtx)); 2871 VERIFY(mlft->mlft_state & MLXCX_FLOW_TABLE_CREATED); 2872 VERIFY0(mlft->mlft_state & MLXCX_FLOW_TABLE_DESTROYED); 2873 2874 mlxcx_cmd_init(mlxp, &cmd); 2875 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_flow_table_head, 2876 MLXCX_OP_DESTROY_FLOW_TABLE, 0); 2877 2878 in.mlxi_destroy_flow_table_vport_number = 2879 to_be16(mlft->mlft_port->mlp_num); 2880 in.mlxi_destroy_flow_table_table_type = mlft->mlft_type; 2881 in.mlxi_destroy_flow_table_table_id = to_be24(mlft->mlft_num); 2882 2883 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2884 mlxcx_cmd_fini(mlxp, &cmd); 2885 return (B_FALSE); 2886 } 2887 mlxcx_cmd_wait(&cmd); 2888 2889 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2890 if (ret) { 2891 mlft->mlft_state |= MLXCX_FLOW_TABLE_DESTROYED; 2892 } 2893 mlxcx_cmd_fini(mlxp, &cmd); 2894 return (ret); 2895 } 2896 2897 boolean_t 2898 mlxcx_cmd_set_flow_table_root(mlxcx_t *mlxp, mlxcx_flow_table_t *mlft) 2899 { 2900 mlxcx_cmd_t cmd; 2901 mlxcx_cmd_set_flow_table_root_in_t in; 2902 mlxcx_cmd_set_flow_table_root_out_t out; 2903 boolean_t ret; 2904 2905 bzero(&in, sizeof (in)); 2906 bzero(&out, sizeof (out)); 2907 2908 ASSERT(mutex_owned(&mlft->mlft_mtx)); 2909 VERIFY(mlft->mlft_state & MLXCX_FLOW_TABLE_CREATED); 2910 VERIFY0(mlft->mlft_state & MLXCX_FLOW_TABLE_DESTROYED); 2911 2912 mlxcx_cmd_init(mlxp, &cmd); 2913 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_set_flow_table_root_head, 2914 MLXCX_OP_SET_FLOW_TABLE_ROOT, 0); 2915 2916 in.mlxi_set_flow_table_root_vport_number = 2917 to_be16(mlft->mlft_port->mlp_num); 2918 in.mlxi_set_flow_table_root_table_type = mlft->mlft_type; 2919 in.mlxi_set_flow_table_root_table_id = to_be24(mlft->mlft_num); 2920 2921 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2922 mlxcx_cmd_fini(mlxp, &cmd); 2923 return (B_FALSE); 2924 } 2925 mlxcx_cmd_wait(&cmd); 2926 2927 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2928 if (ret) { 2929 mlft->mlft_state |= MLXCX_FLOW_TABLE_ROOT; 2930 } 2931 mlxcx_cmd_fini(mlxp, &cmd); 2932 return (ret); 2933 } 2934 2935 boolean_t 2936 mlxcx_cmd_create_flow_group(mlxcx_t *mlxp, mlxcx_flow_group_t *mlfg) 2937 { 2938 mlxcx_cmd_t cmd; 2939 mlxcx_cmd_create_flow_group_in_t in; 2940 mlxcx_cmd_create_flow_group_out_t out; 2941 boolean_t ret; 2942 const mlxcx_flow_table_t *mlft; 2943 mlxcx_flow_header_match_t *hdrs; 2944 mlxcx_flow_params_match_t *params; 2945 2946 bzero(&in, sizeof (in)); 2947 bzero(&out, sizeof (out)); 2948 2949 mlft = mlfg->mlfg_table; 2950 ASSERT(mutex_owned(&mlft->mlft_mtx)); 2951 VERIFY(mlft->mlft_state & MLXCX_FLOW_TABLE_CREATED); 2952 VERIFY0(mlft->mlft_state & MLXCX_FLOW_TABLE_DESTROYED); 2953 VERIFY0(mlfg->mlfg_state & MLXCX_FLOW_GROUP_CREATED); 2954 2955 mlxcx_cmd_init(mlxp, &cmd); 2956 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_flow_group_head, 2957 MLXCX_OP_CREATE_FLOW_GROUP, 0); 2958 2959 in.mlxi_create_flow_group_vport_number = 2960 to_be16(mlft->mlft_port->mlp_num); 2961 in.mlxi_create_flow_group_table_type = mlft->mlft_type; 2962 in.mlxi_create_flow_group_table_id = to_be24(mlft->mlft_num); 2963 in.mlxi_create_flow_group_start_flow_index = 2964 to_be32(mlfg->mlfg_start_idx); 2965 in.mlxi_create_flow_group_end_flow_index = 2966 to_be32(mlfg->mlfg_start_idx + (mlfg->mlfg_size - 1)); 2967 2968 hdrs = &in.mlxi_create_flow_group_match_criteria.mlfm_outer_headers; 2969 params = &in.mlxi_create_flow_group_match_criteria.mlfm_misc_parameters; 2970 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_SMAC) { 2971 in.mlxi_create_flow_group_match_criteria_en |= 2972 MLXCX_FLOW_GROUP_MATCH_OUTER_HDRS; 2973 (void) memset(&hdrs->mlfh_smac, 0xff, sizeof (hdrs->mlfh_smac)); 2974 } 2975 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_DMAC) { 2976 in.mlxi_create_flow_group_match_criteria_en |= 2977 MLXCX_FLOW_GROUP_MATCH_OUTER_HDRS; 2978 (void) memset(&hdrs->mlfh_dmac, 0xff, sizeof (hdrs->mlfh_dmac)); 2979 } 2980 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VLAN) { 2981 in.mlxi_create_flow_group_match_criteria_en |= 2982 MLXCX_FLOW_GROUP_MATCH_OUTER_HDRS; 2983 set_bit24(&hdrs->mlfh_tcp_ip_flags, MLXCX_FLOW_HDR_CVLAN_TAG); 2984 set_bit24(&hdrs->mlfh_tcp_ip_flags, MLXCX_FLOW_HDR_SVLAN_TAG); 2985 } 2986 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VID) { 2987 ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VLAN); 2988 set_bits16(&hdrs->mlfh_first_vid_flags, 2989 MLXCX_FLOW_HDR_FIRST_VID, UINT16_MAX); 2990 } 2991 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER) { 2992 in.mlxi_create_flow_group_match_criteria_en |= 2993 MLXCX_FLOW_GROUP_MATCH_OUTER_HDRS; 2994 set_bits24(&hdrs->mlfh_tcp_ip_flags, MLXCX_FLOW_HDR_IP_VERSION, 2995 UINT32_MAX); 2996 } 2997 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_SRCIP) { 2998 ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER); 2999 (void) memset(&hdrs->mlfh_src_ip, 0xff, 3000 sizeof (hdrs->mlfh_src_ip)); 3001 } 3002 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_DSTIP) { 3003 ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER); 3004 (void) memset(&hdrs->mlfh_src_ip, 0xff, 3005 sizeof (hdrs->mlfh_dst_ip)); 3006 } 3007 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_PROTO) { 3008 in.mlxi_create_flow_group_match_criteria_en |= 3009 MLXCX_FLOW_GROUP_MATCH_OUTER_HDRS; 3010 hdrs->mlfh_ip_protocol = UINT8_MAX; 3011 } 3012 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_SRCIP) { 3013 ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER); 3014 (void) memset(&hdrs->mlfh_src_ip, 0xff, 3015 sizeof (hdrs->mlfh_src_ip)); 3016 } 3017 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_DSTIP) { 3018 ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER); 3019 (void) memset(&hdrs->mlfh_src_ip, 0xff, 3020 sizeof (hdrs->mlfh_dst_ip)); 3021 } 3022 3023 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_SQN) { 3024 in.mlxi_create_flow_group_match_criteria_en |= 3025 MLXCX_FLOW_GROUP_MATCH_MISC_PARAMS; 3026 params->mlfp_source_sqn = to_be24(UINT32_MAX); 3027 } 3028 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VXLAN) { 3029 in.mlxi_create_flow_group_match_criteria_en |= 3030 MLXCX_FLOW_GROUP_MATCH_MISC_PARAMS; 3031 params->mlfp_vxlan_vni = to_be24(UINT32_MAX); 3032 } 3033 3034 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3035 mlxcx_cmd_fini(mlxp, &cmd); 3036 return (B_FALSE); 3037 } 3038 mlxcx_cmd_wait(&cmd); 3039 3040 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3041 if (ret) { 3042 mlfg->mlfg_state |= MLXCX_FLOW_GROUP_CREATED; 3043 mlfg->mlfg_num = from_be24(out.mlxo_create_flow_group_group_id); 3044 } 3045 mlxcx_cmd_fini(mlxp, &cmd); 3046 return (ret); 3047 } 3048 3049 boolean_t 3050 mlxcx_cmd_destroy_flow_group(mlxcx_t *mlxp, mlxcx_flow_group_t *mlfg) 3051 { 3052 mlxcx_cmd_t cmd; 3053 mlxcx_cmd_destroy_flow_group_in_t in; 3054 mlxcx_cmd_destroy_flow_group_out_t out; 3055 boolean_t ret; 3056 const mlxcx_flow_table_t *mlft; 3057 3058 bzero(&in, sizeof (in)); 3059 bzero(&out, sizeof (out)); 3060 3061 mlft = mlfg->mlfg_table; 3062 ASSERT(mutex_owned(&mlft->mlft_mtx)); 3063 VERIFY(mlft->mlft_state & MLXCX_FLOW_TABLE_CREATED); 3064 VERIFY0(mlft->mlft_state & MLXCX_FLOW_TABLE_DESTROYED); 3065 VERIFY(mlfg->mlfg_state & MLXCX_FLOW_GROUP_CREATED); 3066 VERIFY0(mlfg->mlfg_state & MLXCX_FLOW_GROUP_DESTROYED); 3067 3068 mlxcx_cmd_init(mlxp, &cmd); 3069 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_flow_group_head, 3070 MLXCX_OP_DESTROY_FLOW_GROUP, 0); 3071 3072 in.mlxi_destroy_flow_group_vport_number = 3073 to_be16(mlft->mlft_port->mlp_num); 3074 in.mlxi_destroy_flow_group_table_type = mlft->mlft_type; 3075 in.mlxi_destroy_flow_group_table_id = to_be24(mlft->mlft_num); 3076 in.mlxi_destroy_flow_group_group_id = to_be32(mlfg->mlfg_num); 3077 3078 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3079 mlxcx_cmd_fini(mlxp, &cmd); 3080 return (B_FALSE); 3081 } 3082 mlxcx_cmd_wait(&cmd); 3083 3084 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3085 if (ret) { 3086 mlfg->mlfg_state |= MLXCX_FLOW_GROUP_DESTROYED; 3087 } 3088 mlxcx_cmd_fini(mlxp, &cmd); 3089 return (ret); 3090 } 3091 3092 boolean_t 3093 mlxcx_cmd_set_flow_table_entry(mlxcx_t *mlxp, mlxcx_flow_entry_t *mlfe) 3094 { 3095 mlxcx_cmd_t cmd; 3096 mlxcx_cmd_set_flow_table_entry_in_t in; 3097 mlxcx_cmd_set_flow_table_entry_out_t out; 3098 boolean_t ret; 3099 size_t insize; 3100 mlxcx_flow_entry_ctx_t *ctx; 3101 const mlxcx_flow_table_t *mlft; 3102 mlxcx_flow_group_t *mlfg; 3103 mlxcx_flow_dest_t *d; 3104 uint_t i; 3105 mlxcx_flow_header_match_t *hdrs; 3106 mlxcx_flow_params_match_t *params; 3107 mlxcx_cmd_set_flow_table_entry_opmod_t opmod; 3108 3109 bzero(&in, sizeof (in)); 3110 bzero(&out, sizeof (out)); 3111 3112 mlft = mlfe->mlfe_table; 3113 ASSERT(mutex_owned(&mlft->mlft_mtx)); 3114 VERIFY(mlft->mlft_state & MLXCX_FLOW_TABLE_CREATED); 3115 VERIFY0(mlft->mlft_state & MLXCX_FLOW_TABLE_DESTROYED); 3116 3117 mlfg = mlfe->mlfe_group; 3118 VERIFY(mlfg->mlfg_state & MLXCX_FLOW_GROUP_CREATED); 3119 VERIFY0(mlfg->mlfg_state & MLXCX_FLOW_GROUP_DESTROYED); 3120 3121 opmod = MLXCX_CMD_FLOW_ENTRY_SET_NEW; 3122 if (mlfe->mlfe_state & MLXCX_FLOW_ENTRY_CREATED) { 3123 ASSERT(mlfe->mlfe_state & MLXCX_FLOW_ENTRY_DIRTY); 3124 opmod = MLXCX_CMD_FLOW_ENTRY_MODIFY; 3125 } 3126 3127 mlxcx_cmd_init(mlxp, &cmd); 3128 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_set_flow_table_entry_head, 3129 MLXCX_OP_SET_FLOW_TABLE_ENTRY, opmod); 3130 3131 in.mlxi_set_flow_table_entry_vport_number = 3132 to_be16(mlft->mlft_port->mlp_num); 3133 in.mlxi_set_flow_table_entry_table_type = mlft->mlft_type; 3134 in.mlxi_set_flow_table_entry_table_id = to_be24(mlft->mlft_num); 3135 in.mlxi_set_flow_table_entry_flow_index = to_be32(mlfe->mlfe_index); 3136 3137 if (mlfe->mlfe_state & MLXCX_FLOW_ENTRY_CREATED) { 3138 set_bit8(&in.mlxi_set_flow_table_entry_modify_bitmask, 3139 MLXCX_CMD_FLOW_ENTRY_SET_ACTION); 3140 set_bit8(&in.mlxi_set_flow_table_entry_modify_bitmask, 3141 MLXCX_CMD_FLOW_ENTRY_SET_DESTINATION); 3142 } 3143 3144 ctx = &in.mlxi_set_flow_table_entry_context; 3145 ctx->mlfec_group_id = to_be32(mlfg->mlfg_num); 3146 3147 insize = offsetof(mlxcx_cmd_set_flow_table_entry_in_t, 3148 mlxi_set_flow_table_entry_context) + 3149 offsetof(mlxcx_flow_entry_ctx_t, mlfec_destination); 3150 3151 ctx->mlfec_action = to_be16(mlfe->mlfe_action); 3152 3153 switch (mlfe->mlfe_action) { 3154 case MLXCX_FLOW_ACTION_ALLOW: 3155 case MLXCX_FLOW_ACTION_DROP: 3156 break; 3157 case MLXCX_FLOW_ACTION_FORWARD: 3158 ASSERT3U(mlfe->mlfe_ndest, <=, MLXCX_FLOW_MAX_DESTINATIONS); 3159 ASSERT3U(mlfe->mlfe_ndest, <=, 3160 mlxp->mlx_caps->mlc_max_rx_fe_dest); 3161 ctx->mlfec_destination_list_size = to_be24(mlfe->mlfe_ndest); 3162 for (i = 0; i < mlfe->mlfe_ndest; ++i) { 3163 insize += sizeof (mlxcx_flow_dest_t); 3164 d = &ctx->mlfec_destination[i]; 3165 if (mlfe->mlfe_dest[i].mlfed_tir != NULL) { 3166 d->mlfd_destination_type = MLXCX_FLOW_DEST_TIR; 3167 d->mlfd_destination_id = to_be24( 3168 mlfe->mlfe_dest[i].mlfed_tir->mltir_num); 3169 } else if (mlfe->mlfe_dest[i].mlfed_flow != NULL) { 3170 d->mlfd_destination_type = 3171 MLXCX_FLOW_DEST_FLOW_TABLE; 3172 d->mlfd_destination_id = to_be24( 3173 mlfe->mlfe_dest[i].mlfed_flow->mlft_num); 3174 } else { 3175 /* Invalid flow entry destination */ 3176 VERIFY(0); 3177 } 3178 } 3179 break; 3180 case MLXCX_FLOW_ACTION_COUNT: 3181 /* We don't support count actions yet. */ 3182 VERIFY(0); 3183 break; 3184 case MLXCX_FLOW_ACTION_ENCAP: 3185 case MLXCX_FLOW_ACTION_DECAP: 3186 /* We don't support encap/decap actions yet. */ 3187 VERIFY(0); 3188 break; 3189 } 3190 3191 hdrs = &ctx->mlfec_match_value.mlfm_outer_headers; 3192 params = &ctx->mlfec_match_value.mlfm_misc_parameters; 3193 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_SMAC) { 3194 bcopy(mlfe->mlfe_smac, hdrs->mlfh_smac, 3195 sizeof (hdrs->mlfh_smac)); 3196 } 3197 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_DMAC) { 3198 bcopy(mlfe->mlfe_dmac, hdrs->mlfh_dmac, 3199 sizeof (hdrs->mlfh_dmac)); 3200 } 3201 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VLAN) { 3202 switch (mlfe->mlfe_vlan_type) { 3203 case MLXCX_VLAN_TYPE_CVLAN: 3204 set_bit24(&hdrs->mlfh_tcp_ip_flags, 3205 MLXCX_FLOW_HDR_CVLAN_TAG); 3206 break; 3207 case MLXCX_VLAN_TYPE_SVLAN: 3208 set_bit24(&hdrs->mlfh_tcp_ip_flags, 3209 MLXCX_FLOW_HDR_SVLAN_TAG); 3210 break; 3211 default: 3212 break; 3213 } 3214 } 3215 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VID) { 3216 ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VLAN); 3217 set_bits16(&hdrs->mlfh_first_vid_flags, 3218 MLXCX_FLOW_HDR_FIRST_VID, mlfe->mlfe_vid); 3219 } 3220 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER) { 3221 set_bits24(&hdrs->mlfh_tcp_ip_flags, MLXCX_FLOW_HDR_IP_VERSION, 3222 mlfe->mlfe_ip_version); 3223 } 3224 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_SRCIP) { 3225 ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER); 3226 bcopy(mlfe->mlfe_srcip, hdrs->mlfh_src_ip, 3227 sizeof (hdrs->mlfh_src_ip)); 3228 } 3229 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_DSTIP) { 3230 ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER); 3231 bcopy(mlfe->mlfe_dstip, hdrs->mlfh_src_ip, 3232 sizeof (hdrs->mlfh_dst_ip)); 3233 } 3234 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_PROTO) { 3235 hdrs->mlfh_ip_protocol = mlfe->mlfe_ip_proto; 3236 } 3237 3238 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_SQN) { 3239 params->mlfp_source_sqn = to_be24(mlfe->mlfe_sqn); 3240 } 3241 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VXLAN) { 3242 params->mlfp_vxlan_vni = to_be24(mlfe->mlfe_vxlan_vni); 3243 } 3244 3245 if (!mlxcx_cmd_send(mlxp, &cmd, &in, insize, &out, sizeof (out))) { 3246 mlxcx_cmd_fini(mlxp, &cmd); 3247 return (B_FALSE); 3248 } 3249 mlxcx_cmd_wait(&cmd); 3250 3251 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3252 if (ret) { 3253 mlfe->mlfe_state |= MLXCX_FLOW_ENTRY_CREATED; 3254 mlfe->mlfe_state &= ~MLXCX_FLOW_ENTRY_DIRTY; 3255 mlfg->mlfg_state |= MLXCX_FLOW_GROUP_BUSY; 3256 } 3257 mlxcx_cmd_fini(mlxp, &cmd); 3258 return (ret); 3259 } 3260 3261 boolean_t 3262 mlxcx_cmd_delete_flow_table_entry(mlxcx_t *mlxp, mlxcx_flow_entry_t *mlfe) 3263 { 3264 mlxcx_cmd_t cmd; 3265 mlxcx_cmd_delete_flow_table_entry_in_t in; 3266 mlxcx_cmd_delete_flow_table_entry_out_t out; 3267 boolean_t ret; 3268 const mlxcx_flow_table_t *mlft; 3269 3270 bzero(&in, sizeof (in)); 3271 bzero(&out, sizeof (out)); 3272 3273 mlft = mlfe->mlfe_table; 3274 ASSERT(mutex_owned(&mlft->mlft_mtx)); 3275 VERIFY(mlft->mlft_state & MLXCX_FLOW_TABLE_CREATED); 3276 VERIFY0(mlft->mlft_state & MLXCX_FLOW_TABLE_DESTROYED); 3277 3278 mlxcx_cmd_init(mlxp, &cmd); 3279 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_delete_flow_table_entry_head, 3280 MLXCX_OP_DELETE_FLOW_TABLE_ENTRY, 0); 3281 3282 in.mlxi_delete_flow_table_entry_vport_number = 3283 to_be16(mlft->mlft_port->mlp_num); 3284 in.mlxi_delete_flow_table_entry_table_type = mlft->mlft_type; 3285 in.mlxi_delete_flow_table_entry_table_id = to_be24(mlft->mlft_num); 3286 in.mlxi_delete_flow_table_entry_flow_index = to_be32(mlfe->mlfe_index); 3287 3288 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3289 mlxcx_cmd_fini(mlxp, &cmd); 3290 return (B_FALSE); 3291 } 3292 mlxcx_cmd_wait(&cmd); 3293 3294 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3295 if (ret) { 3296 /* 3297 * Note that flow entries have a different lifecycle to most 3298 * other things we create -- we have to be able to re-use them 3299 * after they have been deleted, since they exist at a fixed 3300 * position in their flow table. 3301 * 3302 * So we clear the CREATED bit here for them to let us call 3303 * create_flow_table_entry() on the same entry again later. 3304 */ 3305 mlfe->mlfe_state &= ~MLXCX_FLOW_ENTRY_CREATED; 3306 mlfe->mlfe_state |= MLXCX_FLOW_ENTRY_DELETED; 3307 } 3308 mlxcx_cmd_fini(mlxp, &cmd); 3309 return (ret); 3310 } 3311 3312 boolean_t 3313 mlxcx_cmd_create_sq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq) 3314 { 3315 mlxcx_cmd_t cmd; 3316 mlxcx_cmd_create_sq_in_t in; 3317 mlxcx_cmd_create_sq_out_t out; 3318 boolean_t ret; 3319 mlxcx_sq_ctx_t *ctx; 3320 size_t rem, insize; 3321 const ddi_dma_cookie_t *c; 3322 uint64_t pa, npages; 3323 3324 bzero(&in, sizeof (in)); 3325 bzero(&out, sizeof (out)); 3326 3327 ASSERT(mutex_owned(&mlwq->mlwq_mtx)); 3328 VERIFY3U(mlwq->mlwq_type, ==, MLXCX_WQ_TYPE_SENDQ); 3329 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 3330 VERIFY0(mlwq->mlwq_state & MLXCX_WQ_CREATED); 3331 3332 mlxcx_cmd_init(mlxp, &cmd); 3333 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_sq_head, 3334 MLXCX_OP_CREATE_SQ, 0); 3335 3336 ctx = &in.mlxi_create_sq_context; 3337 3338 set_bit32(&ctx->mlsqc_flags, MLXCX_SQ_FLAGS_RLKEY); 3339 set_bit32(&ctx->mlsqc_flags, MLXCX_SQ_FLAGS_FLUSH_IN_ERROR); 3340 set_bits32(&ctx->mlsqc_flags, MLXCX_SQ_MIN_WQE_INLINE, 3341 mlwq->mlwq_inline_mode); 3342 ctx->mlsqc_cqn = to_be24(mlwq->mlwq_cq->mlcq_num); 3343 3344 VERIFY(mlwq->mlwq_tis != NULL); 3345 ctx->mlsqc_tis_lst_sz = to_be16(1); 3346 ctx->mlsqc_tis_num = to_be24(mlwq->mlwq_tis->mltis_num); 3347 3348 set_bits32(&ctx->mlsqc_wq.mlwqc_flags, MLXCX_WORKQ_CTX_TYPE, 3349 MLXCX_WORKQ_TYPE_CYCLIC); 3350 ctx->mlsqc_wq.mlwqc_pd = to_be24(mlwq->mlwq_pd->mlpd_num); 3351 ctx->mlsqc_wq.mlwqc_uar_page = to_be24(mlwq->mlwq_uar->mlu_num); 3352 ctx->mlsqc_wq.mlwqc_log_wq_sz = mlwq->mlwq_entshift; 3353 ctx->mlsqc_wq.mlwqc_log_wq_stride = MLXCX_SENDQ_STRIDE_SHIFT; 3354 3355 c = mlxcx_dma_cookie_one(&mlwq->mlwq_doorbell_dma); 3356 ctx->mlsqc_wq.mlwqc_dbr_addr = to_be64(c->dmac_laddress); 3357 ASSERT3U(c->dmac_size, >=, sizeof (mlxcx_workq_doorbell_t)); 3358 3359 npages = 0; 3360 c = NULL; 3361 while ((c = mlxcx_dma_cookie_iter(&mlwq->mlwq_dma, c)) != NULL) { 3362 pa = c->dmac_laddress; 3363 rem = c->dmac_size; 3364 while (rem > 0) { 3365 ASSERT3U(pa & 0xfff, ==, 0); 3366 ASSERT3U(rem, >=, MLXCX_HW_PAGE_SIZE); 3367 ctx->mlsqc_wq.mlwqc_pas[npages++] = to_be64(pa); 3368 rem -= MLXCX_HW_PAGE_SIZE; 3369 pa += MLXCX_HW_PAGE_SIZE; 3370 } 3371 } 3372 ASSERT3U(npages, <=, MLXCX_WORKQ_CTX_MAX_ADDRESSES); 3373 3374 insize = offsetof(mlxcx_cmd_create_sq_in_t, mlxi_create_sq_context) + 3375 offsetof(mlxcx_sq_ctx_t, mlsqc_wq) + 3376 offsetof(mlxcx_workq_ctx_t, mlwqc_pas) + 3377 sizeof (uint64_t) * npages; 3378 3379 if (!mlxcx_cmd_send(mlxp, &cmd, &in, insize, &out, sizeof (out))) { 3380 mlxcx_cmd_fini(mlxp, &cmd); 3381 return (B_FALSE); 3382 } 3383 mlxcx_cmd_wait(&cmd); 3384 3385 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3386 if (ret) { 3387 mlwq->mlwq_state |= MLXCX_WQ_CREATED; 3388 mlwq->mlwq_num = from_be24(out.mlxo_create_sq_sqn); 3389 } 3390 mlxcx_cmd_fini(mlxp, &cmd); 3391 return (ret); 3392 } 3393 3394 boolean_t 3395 mlxcx_cmd_start_sq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq) 3396 { 3397 mlxcx_cmd_t cmd; 3398 mlxcx_cmd_modify_sq_in_t in; 3399 mlxcx_cmd_modify_sq_out_t out; 3400 boolean_t ret; 3401 ddi_fm_error_t err; 3402 3403 bzero(&in, sizeof (in)); 3404 bzero(&out, sizeof (out)); 3405 3406 ASSERT(mutex_owned(&mlwq->mlwq_mtx)); 3407 ASSERT(mlwq->mlwq_cq != NULL); 3408 3409 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 3410 VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED); 3411 VERIFY0(mlwq->mlwq_state & MLXCX_WQ_STARTED); 3412 3413 /* 3414 * Before starting the queue, we have to be sure that it is 3415 * empty and the doorbell and counters are set to 0. 3416 */ 3417 ASSERT(mutex_owned(&mlwq->mlwq_cq->mlcq_mtx)); 3418 ASSERT(list_is_empty(&mlwq->mlwq_cq->mlcq_buffers)); 3419 ASSERT(list_is_empty(&mlwq->mlwq_cq->mlcq_buffers_b)); 3420 3421 mlwq->mlwq_doorbell->mlwqd_recv_counter = to_be16(0); 3422 MLXCX_DMA_SYNC(mlwq->mlwq_doorbell_dma, DDI_DMA_SYNC_FORDEV); 3423 ddi_fm_dma_err_get(mlwq->mlwq_doorbell_dma.mxdb_dma_handle, &err, 3424 DDI_FME_VERSION); 3425 if (err.fme_status != DDI_FM_OK) 3426 return (B_FALSE); 3427 mlwq->mlwq_pc = 0; 3428 3429 mlxcx_cmd_init(mlxp, &cmd); 3430 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_modify_sq_head, 3431 MLXCX_OP_MODIFY_SQ, 0); 3432 3433 in.mlxi_modify_sq_sqn = to_be24(mlwq->mlwq_num); 3434 3435 /* From state */ 3436 set_bits8(&in.mlxi_modify_sq_state, MLXCX_CMD_MODIFY_SQ_STATE, 3437 MLXCX_SQ_STATE_RST); 3438 /* To state */ 3439 set_bits32(&in.mlxi_modify_sq_context.mlsqc_flags, MLXCX_SQ_STATE, 3440 MLXCX_SQ_STATE_RDY); 3441 3442 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3443 mlxcx_cmd_fini(mlxp, &cmd); 3444 return (B_FALSE); 3445 } 3446 mlxcx_cmd_wait(&cmd); 3447 3448 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3449 if (ret) { 3450 mlwq->mlwq_state |= MLXCX_WQ_STARTED; 3451 } 3452 mlxcx_cmd_fini(mlxp, &cmd); 3453 return (ret); 3454 } 3455 3456 boolean_t 3457 mlxcx_cmd_stop_sq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq) 3458 { 3459 mlxcx_cmd_t cmd; 3460 mlxcx_cmd_modify_sq_in_t in; 3461 mlxcx_cmd_modify_sq_out_t out; 3462 boolean_t ret; 3463 3464 bzero(&in, sizeof (in)); 3465 bzero(&out, sizeof (out)); 3466 3467 ASSERT(mutex_owned(&mlwq->mlwq_mtx)); 3468 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 3469 VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED); 3470 VERIFY(mlwq->mlwq_state & MLXCX_WQ_STARTED); 3471 3472 mlxcx_cmd_init(mlxp, &cmd); 3473 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_modify_sq_head, 3474 MLXCX_OP_MODIFY_SQ, 0); 3475 3476 in.mlxi_modify_sq_sqn = to_be24(mlwq->mlwq_num); 3477 3478 /* From state */ 3479 set_bits8(&in.mlxi_modify_sq_state, MLXCX_CMD_MODIFY_SQ_STATE, 3480 MLXCX_SQ_STATE_RDY); 3481 /* To state */ 3482 set_bits32(&in.mlxi_modify_sq_context.mlsqc_flags, MLXCX_SQ_STATE, 3483 MLXCX_SQ_STATE_RST); 3484 3485 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3486 mlxcx_cmd_fini(mlxp, &cmd); 3487 return (B_FALSE); 3488 } 3489 mlxcx_cmd_wait(&cmd); 3490 3491 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3492 if (ret) { 3493 mlwq->mlwq_state &= ~MLXCX_WQ_STARTED; 3494 } 3495 mlxcx_cmd_fini(mlxp, &cmd); 3496 return (ret); 3497 } 3498 3499 boolean_t 3500 mlxcx_cmd_destroy_sq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq) 3501 { 3502 mlxcx_cmd_t cmd; 3503 mlxcx_cmd_destroy_sq_in_t in; 3504 mlxcx_cmd_destroy_sq_out_t out; 3505 boolean_t ret; 3506 3507 bzero(&in, sizeof (in)); 3508 bzero(&out, sizeof (out)); 3509 3510 ASSERT(mutex_owned(&mlwq->mlwq_mtx)); 3511 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 3512 VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED); 3513 VERIFY0(mlwq->mlwq_state & MLXCX_WQ_STARTED); 3514 3515 mlxcx_cmd_init(mlxp, &cmd); 3516 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_sq_head, 3517 MLXCX_OP_DESTROY_SQ, 0); 3518 3519 in.mlxi_destroy_sq_sqn = to_be24(mlwq->mlwq_num); 3520 3521 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3522 mlxcx_cmd_fini(mlxp, &cmd); 3523 return (B_FALSE); 3524 } 3525 mlxcx_cmd_wait(&cmd); 3526 3527 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3528 if (ret) { 3529 mlwq->mlwq_state |= MLXCX_WQ_DESTROYED; 3530 } 3531 mlxcx_cmd_fini(mlxp, &cmd); 3532 return (ret); 3533 } 3534 3535 boolean_t 3536 mlxcx_cmd_create_rqt(mlxcx_t *mlxp, mlxcx_rqtable_t *mlrqt) 3537 { 3538 mlxcx_cmd_t cmd; 3539 mlxcx_cmd_create_rqt_in_t in; 3540 mlxcx_cmd_create_rqt_out_t out; 3541 mlxcx_rqtable_ctx_t *ctx; 3542 boolean_t ret; 3543 uint_t i; 3544 3545 bzero(&in, sizeof (in)); 3546 bzero(&out, sizeof (out)); 3547 3548 VERIFY0(mlrqt->mlrqt_state & MLXCX_RQT_CREATED); 3549 3550 mlxcx_cmd_init(mlxp, &cmd); 3551 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_rqt_head, 3552 MLXCX_OP_CREATE_RQT, 0); 3553 3554 ctx = &in.mlxi_create_rqt_context; 3555 ASSERT3U(mlrqt->mlrqt_max, <=, MLXCX_RQT_MAX_RQ_REFS); 3556 ASSERT3U(mlrqt->mlrqt_max, <=, mlxp->mlx_caps->mlc_max_rqt_size); 3557 ctx->mlrqtc_max_size = to_be16(mlrqt->mlrqt_max); 3558 ctx->mlrqtc_actual_size = to_be16(mlrqt->mlrqt_used); 3559 for (i = 0; i < mlrqt->mlrqt_used; ++i) { 3560 ctx->mlrqtc_rqref[i].mlrqtr_rqn = to_be24( 3561 mlrqt->mlrqt_rq[i]->mlwq_num); 3562 } 3563 3564 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3565 mlxcx_cmd_fini(mlxp, &cmd); 3566 return (B_FALSE); 3567 } 3568 mlxcx_cmd_wait(&cmd); 3569 3570 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3571 if (ret) { 3572 mlrqt->mlrqt_num = from_be24(out.mlxo_create_rqt_rqtn); 3573 mlrqt->mlrqt_state |= MLXCX_RQT_CREATED; 3574 mlrqt->mlrqt_state &= ~MLXCX_RQT_DIRTY; 3575 } 3576 mlxcx_cmd_fini(mlxp, &cmd); 3577 return (ret); 3578 } 3579 3580 boolean_t 3581 mlxcx_cmd_destroy_rqt(mlxcx_t *mlxp, mlxcx_rqtable_t *mlrqt) 3582 { 3583 mlxcx_cmd_t cmd; 3584 mlxcx_cmd_destroy_rqt_in_t in; 3585 mlxcx_cmd_destroy_rqt_out_t out; 3586 boolean_t ret; 3587 3588 bzero(&in, sizeof (in)); 3589 bzero(&out, sizeof (out)); 3590 3591 VERIFY(mlrqt->mlrqt_state & MLXCX_RQT_CREATED); 3592 VERIFY0(mlrqt->mlrqt_state & MLXCX_RQT_DESTROYED); 3593 3594 mlxcx_cmd_init(mlxp, &cmd); 3595 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_rqt_head, 3596 MLXCX_OP_DESTROY_RQT, 0); 3597 3598 in.mlxi_destroy_rqt_rqtn = to_be24(mlrqt->mlrqt_num); 3599 3600 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3601 mlxcx_cmd_fini(mlxp, &cmd); 3602 return (B_FALSE); 3603 } 3604 mlxcx_cmd_wait(&cmd); 3605 3606 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3607 if (ret) { 3608 mlrqt->mlrqt_state |= MLXCX_RQT_DESTROYED; 3609 } 3610 mlxcx_cmd_fini(mlxp, &cmd); 3611 return (ret); 3612 } 3613 3614 boolean_t 3615 mlxcx_cmd_set_int_mod(mlxcx_t *mlxp, uint_t intr, uint_t min_delay) 3616 { 3617 mlxcx_cmd_t cmd; 3618 mlxcx_cmd_config_int_mod_in_t in; 3619 mlxcx_cmd_config_int_mod_out_t out; 3620 boolean_t ret; 3621 3622 bzero(&in, sizeof (in)); 3623 bzero(&out, sizeof (out)); 3624 3625 mlxcx_cmd_init(mlxp, &cmd); 3626 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_config_int_mod_head, 3627 MLXCX_OP_CONFIG_INT_MODERATION, MLXCX_CMD_CONFIG_INT_MOD_WRITE); 3628 3629 in.mlxi_config_int_mod_int_vector = to_be16(intr); 3630 in.mlxi_config_int_mod_min_delay = to_be16(min_delay); 3631 3632 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3633 mlxcx_cmd_fini(mlxp, &cmd); 3634 return (B_FALSE); 3635 } 3636 mlxcx_cmd_wait(&cmd); 3637 3638 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3639 mlxcx_cmd_fini(mlxp, &cmd); 3640 return (ret); 3641 } 3642 3643 /* 3644 * CTASSERTs here are for the structs in mlxcx_reg.h, to check they match 3645 * against offsets from the PRM. 3646 * 3647 * They're not in the header file, to avoid them being used by multiple .c 3648 * files. 3649 */ 3650 3651 CTASSERT(offsetof(mlxcx_eventq_ent_t, mleqe_unknown_data) == 0x20); 3652 CTASSERT(offsetof(mlxcx_eventq_ent_t, mleqe_signature) == 0x3c + 2); 3653 CTASSERT(sizeof (mlxcx_eventq_ent_t) == 64); 3654 3655 CTASSERT(offsetof(mlxcx_completionq_error_ent_t, mlcqee_byte_cnt) == 0x2C); 3656 CTASSERT(offsetof(mlxcx_completionq_error_ent_t, mlcqee_wqe_opcode) == 0x38); 3657 3658 CTASSERT(sizeof (mlxcx_completionq_error_ent_t) == 3659 sizeof (mlxcx_completionq_ent_t)); 3660 CTASSERT(sizeof (mlxcx_wqe_control_seg_t) == (1 << 4)); 3661 3662 CTASSERT(offsetof(mlxcx_wqe_eth_seg_t, mles_inline_headers) == 0x0e); 3663 CTASSERT(sizeof (mlxcx_wqe_eth_seg_t) == (1 << 5)); 3664 3665 CTASSERT(sizeof (mlxcx_wqe_data_seg_t) == (1 << 4)); 3666 3667 CTASSERT(sizeof (mlxcx_sendq_ent_t) == (1 << MLXCX_SENDQ_STRIDE_SHIFT)); 3668 3669 CTASSERT(sizeof (mlxcx_sendq_bf_t) == (1 << MLXCX_SENDQ_STRIDE_SHIFT)); 3670 3671 CTASSERT(sizeof (mlxcx_sendq_extra_ent_t) == (1 << MLXCX_SENDQ_STRIDE_SHIFT)); 3672 3673 CTASSERT(sizeof (mlxcx_recvq_ent_t) == (1 << MLXCX_RECVQ_STRIDE_SHIFT)); 3674 3675 CTASSERT(offsetof(mlxcx_workq_ctx_t, mlwqc_dbr_addr) == 0x10); 3676 CTASSERT(offsetof(mlxcx_workq_ctx_t, mlwqc_pas) == 0xc0); 3677 3678 CTASSERT(offsetof(mlxcx_rq_ctx_t, mlrqc_cqn) == 0x09); 3679 CTASSERT(offsetof(mlxcx_rq_ctx_t, mlrqc_wq) == 0x30); 3680 3681 CTASSERT(offsetof(mlxcx_sq_ctx_t, mlsqc_cqn) == 0x09); 3682 CTASSERT(offsetof(mlxcx_sq_ctx_t, mlsqc_tis_lst_sz) == 0x20); 3683 CTASSERT(offsetof(mlxcx_sq_ctx_t, mlsqc_tis_num) == 0x2d); 3684 CTASSERT(offsetof(mlxcx_sq_ctx_t, mlsqc_wq) == 0x30); 3685 3686 CTASSERT(sizeof (mlxcx_tis_ctx_t) == 0xa0); 3687 CTASSERT(offsetof(mlxcx_tis_ctx_t, mltisc_transport_domain) == 0x25); 3688 3689 CTASSERT(offsetof(mlxcx_rqtable_ctx_t, mlrqtc_max_size) == 0x16); 3690 CTASSERT(offsetof(mlxcx_rqtable_ctx_t, mlrqtc_rqref) == 0xF0); 3691 3692 CTASSERT(offsetof(mlxcx_cmd_create_eq_in_t, mlxi_create_eq_event_bitmask) == 3693 0x58); 3694 CTASSERT(offsetof(mlxcx_cmd_create_eq_in_t, mlxi_create_eq_pas) == 0x110); 3695 CTASSERT(offsetof(mlxcx_cmd_create_eq_in_t, mlxi_create_eq_context) == 0x10); 3696 3697 CTASSERT(offsetof(mlxcx_cmd_create_tir_in_t, mlxi_create_tir_context) == 0x20); 3698 3699 CTASSERT(offsetof(mlxcx_cmd_create_tis_in_t, mlxi_create_tis_context) == 0x20); 3700 3701 CTASSERT(offsetof(mlxcx_cmd_query_special_ctxs_out_t, 3702 mlxo_query_special_ctxs_resd_lkey) == 0x0c); 3703 3704 CTASSERT(offsetof(mlxcx_cmd_query_cq_out_t, mlxo_query_cq_context) == 0x10); 3705 CTASSERT(offsetof(mlxcx_cmd_query_cq_out_t, mlxo_query_cq_pas) == 0x110); 3706 3707 CTASSERT(offsetof(mlxcx_cmd_query_rq_out_t, mlxo_query_rq_context) == 0x20); 3708 3709 CTASSERT(offsetof(mlxcx_cmd_create_sq_in_t, mlxi_create_sq_context) == 0x20); 3710 3711 CTASSERT(offsetof(mlxcx_cmd_modify_sq_in_t, mlxi_modify_sq_context) == 0x20); 3712 3713 CTASSERT(offsetof(mlxcx_cmd_query_sq_out_t, mlxo_query_sq_context) == 0x20); 3714 3715 CTASSERT(offsetof(mlxcx_cmd_create_rqt_in_t, mlxi_create_rqt_context) == 0x20); 3716 3717 CTASSERT(offsetof(mlxcx_reg_pmtu_t, mlrd_pmtu_oper_mtu) == 0x0C); 3718 3719 CTASSERT(sizeof (mlxcx_reg_ptys_t) == 64); 3720 CTASSERT(offsetof(mlxcx_reg_ptys_t, mlrd_ptys_proto_cap) == 0x0c); 3721 CTASSERT(offsetof(mlxcx_reg_ptys_t, mlrd_ptys_proto_admin) == 0x18); 3722 CTASSERT(offsetof(mlxcx_reg_ptys_t, mlrd_ptys_proto_partner_advert) == 0x30); 3723 3724 CTASSERT(offsetof(mlxcx_reg_mcia_t, mlrd_mcia_data) == 0x10); 3725 3726 CTASSERT(offsetof(mlxcx_ppcnt_ieee_802_3_t, 3727 mlppc_ieee_802_3_in_range_len_err) == 0x50); 3728 CTASSERT(offsetof(mlxcx_ppcnt_ieee_802_3_t, 3729 mlppc_ieee_802_3_pause_tx) == 0x90); 3730 3731 CTASSERT(sizeof (mlxcx_reg_ppcnt_t) == 256); 3732 CTASSERT(offsetof(mlxcx_reg_ppcnt_t, mlrd_ppcnt_data) == 0x08); 3733 3734 CTASSERT(offsetof(mlxcx_cmd_access_register_in_t, 3735 mlxi_access_register_argument) == 0x0C); 3736 CTASSERT(offsetof(mlxcx_cmd_access_register_in_t, 3737 mlxi_access_register_data) == 0x10); 3738 3739 CTASSERT(offsetof(mlxcx_cmd_access_register_out_t, 3740 mlxo_access_register_data) == 0x10); 3741 3742 CTASSERT(sizeof (mlxcx_cmd_set_flow_table_root_in_t) == 64); 3743 CTASSERT(offsetof(mlxcx_cmd_set_flow_table_root_in_t, 3744 mlxi_set_flow_table_root_table_id) == 0x15); 3745 CTASSERT(offsetof(mlxcx_cmd_set_flow_table_root_in_t, 3746 mlxi_set_flow_table_root_esw_owner_vhca_id_valid) == 0x1C); 3747