1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright 2020, The University of Queensland 14 * Copyright (c) 2018, Joyent, Inc. 15 * Copyright 2020 RackTop Systems, Inc. 16 * Copyright 2023 MNX Cloud, Inc. 17 */ 18 19 /* 20 * Controls the management of commands that are issues to and from the HCA 21 * command queue. 22 */ 23 24 #include <mlxcx.h> 25 26 #include <sys/debug.h> 27 #include <sys/sysmacros.h> 28 29 /* 30 * When we start up the command queue, it will undergo some internal 31 * initialization after we set the command queue address. These values allow us 32 * to control how much time we should wait for that to occur. 33 */ 34 clock_t mlxcx_cmd_init_delay = 1000 * 10; /* 10 ms in us */ 35 uint_t mlxcx_cmd_init_trys = 100; /* Wait at most 1s */ 36 37 clock_t mlxcx_cmd_delay = 1000 * 1; /* 1 ms in us */ 38 uint_t mlxcx_cmd_tries = 5000; /* Wait at most 1s */ 39 40 /* 41 * This macro is used to identify that we care about our own function that we're 42 * communicating with. We always use this function. 43 */ 44 #define MLXCX_FUNCTION_SELF (to_be16(0)) 45 46 static const char * 47 mlxcx_cmd_response_string(mlxcx_cmd_ret_t ret) 48 { 49 switch (ret) { 50 case MLXCX_CMD_R_OK: 51 return ("MLXCX_CMD_R_OK"); 52 case MLXCX_CMD_R_INTERNAL_ERR: 53 return ("MLXCX_CMD_R_INTERNAL_ERR"); 54 case MLXCX_CMD_R_BAD_OP: 55 return ("MLXCX_CMD_R_BAD_OP"); 56 case MLXCX_CMD_R_BAD_PARAM: 57 return ("MLXCX_CMD_R_BAD_PARAM"); 58 case MLXCX_CMD_R_BAD_SYS_STATE: 59 return ("MLXCX_CMD_R_BAD_SYS_STATE"); 60 case MLXCX_CMD_R_BAD_RESOURCE: 61 return ("MLXCX_CMD_R_BAD_RESOURCE"); 62 case MLXCX_CMD_R_RESOURCE_BUSY: 63 return ("MLXCX_CMD_R_RESOURCE_BUSY"); 64 case MLXCX_CMD_R_EXCEED_LIM: 65 return ("MLXCX_CMD_R_EXCEED_LIM"); 66 case MLXCX_CMD_R_BAD_RES_STATE: 67 return ("MLXCX_CMD_R_BAD_RES_STATE"); 68 case MLXCX_CMD_R_BAD_INDEX: 69 return ("MLXCX_CMD_R_BAD_INDEX"); 70 case MLXCX_CMD_R_NO_RESOURCES: 71 return ("MLXCX_CMD_R_NO_RESOURCES"); 72 case MLXCX_CMD_R_BAD_INPUT_LEN: 73 return ("MLXCX_CMD_R_BAD_INPUT_LEN"); 74 case MLXCX_CMD_R_BAD_OUTPUT_LEN: 75 return ("MLXCX_CMD_R_BAD_OUTPUT_LEN"); 76 case MLXCX_CMD_R_BAD_RESOURCE_STATE: 77 return ("MLXCX_CMD_R_BAD_RESOURCE_STATE"); 78 case MLXCX_CMD_R_BAD_PKT: 79 return ("MLXCX_CMD_R_BAD_PKT"); 80 case MLXCX_CMD_R_BAD_SIZE: 81 return ("MLXCX_CMD_R_BAD_SIZE"); 82 default: 83 return ("Unknown command"); 84 } 85 } 86 87 static const char * 88 mlxcx_cmd_opcode_string(mlxcx_cmd_op_t op) 89 { 90 switch (op) { 91 case MLXCX_OP_QUERY_HCA_CAP: 92 return ("MLXCX_OP_QUERY_HCA_CAP"); 93 case MLXCX_OP_QUERY_ADAPTER: 94 return ("MLXCX_OP_QUERY_ADAPTER"); 95 case MLXCX_OP_INIT_HCA: 96 return ("MLXCX_OP_INIT_HCA"); 97 case MLXCX_OP_TEARDOWN_HCA: 98 return ("MLXCX_OP_TEARDOWN_HCA"); 99 case MLXCX_OP_ENABLE_HCA: 100 return ("MLXCX_OP_ENABLE_HCA"); 101 case MLXCX_OP_DISABLE_HCA: 102 return ("MLXCX_OP_DISABLE_HCA"); 103 case MLXCX_OP_QUERY_PAGES: 104 return ("MLXCX_OP_QUERY_PAGES"); 105 case MLXCX_OP_MANAGE_PAGES: 106 return ("MLXCX_OP_MANAGE_PAGES"); 107 case MLXCX_OP_SET_HCA_CAP: 108 return ("MLXCX_OP_SET_HCA_CAP"); 109 case MLXCX_OP_QUERY_ISSI: 110 return ("MLXCX_OP_QUERY_ISSI"); 111 case MLXCX_OP_SET_ISSI: 112 return ("MLXCX_OP_SET_ISSI"); 113 case MLXCX_OP_SET_DRIVER_VERSION: 114 return ("MLXCX_OP_SET_DRIVER_VERSION"); 115 case MLXCX_OP_QUERY_OTHER_HCA_CAP: 116 return ("MLXCX_OP_QUERY_OTHER_HCA_CAP"); 117 case MLXCX_OP_MODIFY_OTHER_HCA_CAP: 118 return ("MLXCX_OP_MODIFY_OTHER_HCA_CAP"); 119 case MLXCX_OP_SET_TUNNELED_OPERATIONS: 120 return ("MLXCX_OP_SET_TUNNELED_OPERATIONS"); 121 case MLXCX_OP_CREATE_MKEY: 122 return ("MLXCX_OP_CREATE_MKEY"); 123 case MLXCX_OP_QUERY_MKEY: 124 return ("MLXCX_OP_QUERY_MKEY"); 125 case MLXCX_OP_DESTROY_MKEY: 126 return ("MLXCX_OP_DESTROY_MKEY"); 127 case MLXCX_OP_QUERY_SPECIAL_CONTEXTS: 128 return ("MLXCX_OP_QUERY_SPECIAL_CONTEXTS"); 129 case MLXCX_OP_PAGE_FAULT_RESUME: 130 return ("MLXCX_OP_PAGE_FAULT_RESUME"); 131 case MLXCX_OP_CREATE_EQ: 132 return ("MLXCX_OP_CREATE_EQ"); 133 case MLXCX_OP_DESTROY_EQ: 134 return ("MLXCX_OP_DESTROY_EQ"); 135 case MLXCX_OP_QUERY_EQ: 136 return ("MLXCX_OP_QUERY_EQ"); 137 case MLXCX_OP_GEN_EQE: 138 return ("MLXCX_OP_GEN_EQE"); 139 case MLXCX_OP_CREATE_CQ: 140 return ("MLXCX_OP_CREATE_CQ"); 141 case MLXCX_OP_DESTROY_CQ: 142 return ("MLXCX_OP_DESTROY_CQ"); 143 case MLXCX_OP_QUERY_CQ: 144 return ("MLXCX_OP_QUERY_CQ"); 145 case MLXCX_OP_MODIFY_CQ: 146 return ("MLXCX_OP_MODIFY_CQ"); 147 case MLXCX_OP_CREATE_QP: 148 return ("MLXCX_OP_CREATE_QP"); 149 case MLXCX_OP_DESTROY_QP: 150 return ("MLXCX_OP_DESTROY_QP"); 151 case MLXCX_OP_RST2INIT_QP: 152 return ("MLXCX_OP_RST2INIT_QP"); 153 case MLXCX_OP_INIT2RTR_QP: 154 return ("MLXCX_OP_INIT2RTR_QP"); 155 case MLXCX_OP_RTR2RTS_QP: 156 return ("MLXCX_OP_RTR2RTS_QP"); 157 case MLXCX_OP_RTS2RTS_QP: 158 return ("MLXCX_OP_RTS2RTS_QP"); 159 case MLXCX_OP_SQERR2RTS_QP: 160 return ("MLXCX_OP_SQERR2RTS_QP"); 161 case MLXCX_OP__2ERR_QP: 162 return ("MLXCX_OP__2ERR_QP"); 163 case MLXCX_OP__2RST_QP: 164 return ("MLXCX_OP__2RST_QP"); 165 case MLXCX_OP_QUERY_QP: 166 return ("MLXCX_OP_QUERY_QP"); 167 case MLXCX_OP_SQD_RTS_QP: 168 return ("MLXCX_OP_SQD_RTS_QP"); 169 case MLXCX_OP_INIT2INIT_QP: 170 return ("MLXCX_OP_INIT2INIT_QP"); 171 case MLXCX_OP_CREATE_PSV: 172 return ("MLXCX_OP_CREATE_PSV"); 173 case MLXCX_OP_DESTROY_PSV: 174 return ("MLXCX_OP_DESTROY_PSV"); 175 case MLXCX_OP_CREATE_SRQ: 176 return ("MLXCX_OP_CREATE_SRQ"); 177 case MLXCX_OP_DESTROY_SRQ: 178 return ("MLXCX_OP_DESTROY_SRQ"); 179 case MLXCX_OP_QUERY_SRQ: 180 return ("MLXCX_OP_QUERY_SRQ"); 181 case MLXCX_OP_ARM_RQ: 182 return ("MLXCX_OP_ARM_RQ"); 183 case MLXCX_OP_CREATE_XRC_SRQ: 184 return ("MLXCX_OP_CREATE_XRC_SRQ"); 185 case MLXCX_OP_DESTROY_XRC_SRQ: 186 return ("MLXCX_OP_DESTROY_XRC_SRQ"); 187 case MLXCX_OP_QUERY_XRC_SRQ: 188 return ("MLXCX_OP_QUERY_XRC_SRQ"); 189 case MLXCX_OP_ARM_XRC_SRQ: 190 return ("MLXCX_OP_ARM_XRC_SRQ"); 191 case MLXCX_OP_CREATE_DCT: 192 return ("MLXCX_OP_CREATE_DCT"); 193 case MLXCX_OP_DESTROY_DCT: 194 return ("MLXCX_OP_DESTROY_DCT"); 195 case MLXCX_OP_DRAIN_DCT: 196 return ("MLXCX_OP_DRAIN_DCT"); 197 case MLXCX_OP_QUERY_DCT: 198 return ("MLXCX_OP_QUERY_DCT"); 199 case MLXCX_OP_ARM_DCT_FOR_KEY_VIOLATION: 200 return ("MLXCX_OP_ARM_DCT_FOR_KEY_VIOLATION"); 201 case MLXCX_OP_CREATE_XRQ: 202 return ("MLXCX_OP_CREATE_XRQ"); 203 case MLXCX_OP_DESTROY_XRQ: 204 return ("MLXCX_OP_DESTROY_XRQ"); 205 case MLXCX_OP_QUERY_XRQ: 206 return ("MLXCX_OP_QUERY_XRQ"); 207 case MLXCX_OP_CREATE_NVMF_BACKEND_CONTROLLER: 208 return ("MLXCX_OP_CREATE_NVMF_BACKEND_CONTROLLER"); 209 case MLXCX_OP_DESTROY_NVMF_BACKEND_CONTROLLER: 210 return ("MLXCX_OP_DESTROY_NVMF_BACKEND_CONTROLLER"); 211 case MLXCX_OP_QUERY_NVMF_BACKEND_CONTROLLER: 212 return ("MLXCX_OP_QUERY_NVMF_BACKEND_CONTROLLER"); 213 case MLXCX_OP_ATTACH_NVMF_NAMESPACE: 214 return ("MLXCX_OP_ATTACH_NVMF_NAMESPACE"); 215 case MLXCX_OP_DETACH_NVMF_NAMESPACE: 216 return ("MLXCX_OP_DETACH_NVMF_NAMESPACE"); 217 case MLXCX_OP_QUERY_XRQ_DC_PARAMS_ENTRY: 218 return ("MLXCX_OP_QUERY_XRQ_DC_PARAMS_ENTRY"); 219 case MLXCX_OP_SET_XRQ_DC_PARAMS_ENTRY: 220 return ("MLXCX_OP_SET_XRQ_DC_PARAMS_ENTRY"); 221 case MLXCX_OP_QUERY_XRQ_ERROR_PARAMS: 222 return ("MLXCX_OP_QUERY_XRQ_ERROR_PARAMS"); 223 case MLXCX_OP_QUERY_VPORT_STATE: 224 return ("MLXCX_OP_QUERY_VPORT_STATE"); 225 case MLXCX_OP_MODIFY_VPORT_STATE: 226 return ("MLXCX_OP_MODIFY_VPORT_STATE"); 227 case MLXCX_OP_QUERY_ESW_VPORT_CONTEXT: 228 return ("MLXCX_OP_QUERY_ESW_VPORT_CONTEXT"); 229 case MLXCX_OP_MODIFY_ESW_VPORT_CONTEXT: 230 return ("MLXCX_OP_MODIFY_ESW_VPORT_CONTEXT"); 231 case MLXCX_OP_QUERY_NIC_VPORT_CONTEXT: 232 return ("MLXCX_OP_QUERY_NIC_VPORT_CONTEXT"); 233 case MLXCX_OP_MODIFY_NIC_VPORT_CONTEXT: 234 return ("MLXCX_OP_MODIFY_NIC_VPORT_CONTEXT"); 235 case MLXCX_OP_QUERY_ROCE_ADDRESS: 236 return ("MLXCX_OP_QUERY_ROCE_ADDRESS"); 237 case MLXCX_OP_SET_ROCE_ADDRESS: 238 return ("MLXCX_OP_SET_ROCE_ADDRESS"); 239 case MLXCX_OP_QUERY_HCA_VPORT_CONTEXT: 240 return ("MLXCX_OP_QUERY_HCA_VPORT_CONTEXT"); 241 case MLXCX_OP_MODIFY_HCA_VPORT_CONTEXT: 242 return ("MLXCX_OP_MODIFY_HCA_VPORT_CONTEXT"); 243 case MLXCX_OP_QUERY_HCA_VPORT_GID: 244 return ("MLXCX_OP_QUERY_HCA_VPORT_GID"); 245 case MLXCX_OP_QUERY_HCA_VPORT_PKEY: 246 return ("MLXCX_OP_QUERY_HCA_VPORT_PKEY"); 247 case MLXCX_OP_QUERY_VPORT_COUNTER: 248 return ("MLXCX_OP_QUERY_VPORT_COUNTER"); 249 case MLXCX_OP_ALLOC_Q_COUNTER: 250 return ("MLXCX_OP_ALLOC_Q_COUNTER"); 251 case MLXCX_OP_DEALLOC_Q_COUNTER: 252 return ("MLXCX_OP_DEALLOC_Q_COUNTER"); 253 case MLXCX_OP_QUERY_Q_COUNTER: 254 return ("MLXCX_OP_QUERY_Q_COUNTER"); 255 case MLXCX_OP_SET_PP_RATE_LIMIT: 256 return ("MLXCX_OP_SET_PP_RATE_LIMIT"); 257 case MLXCX_OP_QUERY_PP_RATE_LIMIT: 258 return ("MLXCX_OP_QUERY_PP_RATE_LIMIT"); 259 case MLXCX_OP_ALLOC_PD: 260 return ("MLXCX_OP_ALLOC_PD"); 261 case MLXCX_OP_DEALLOC_PD: 262 return ("MLXCX_OP_DEALLOC_PD"); 263 case MLXCX_OP_ALLOC_UAR: 264 return ("MLXCX_OP_ALLOC_UAR"); 265 case MLXCX_OP_DEALLOC_UAR: 266 return ("MLXCX_OP_DEALLOC_UAR"); 267 case MLXCX_OP_CONFIG_INT_MODERATION: 268 return ("MLXCX_OP_CONFIG_INT_MODERATION"); 269 case MLXCX_OP_ACCESS_REG: 270 return ("MLXCX_OP_ACCESS_REG"); 271 case MLXCX_OP_ATTACH_TO_MCG: 272 return ("MLXCX_OP_ATTACH_TO_MCG"); 273 case MLXCX_OP_DETACH_FROM_MCG: 274 return ("MLXCX_OP_DETACH_FROM_MCG"); 275 case MLXCX_OP_MAD_IFC: 276 return ("MLXCX_OP_MAD_IFC"); 277 case MLXCX_OP_QUERY_MAD_DEMUX: 278 return ("MLXCX_OP_QUERY_MAD_DEMUX"); 279 case MLXCX_OP_SET_MAD_DEMUX: 280 return ("MLXCX_OP_SET_MAD_DEMUX"); 281 case MLXCX_OP_NOP: 282 return ("MLXCX_OP_NOP"); 283 case MLXCX_OP_ALLOC_XRCD: 284 return ("MLXCX_OP_ALLOC_XRCD"); 285 case MLXCX_OP_DEALLOC_XRCD: 286 return ("MLXCX_OP_DEALLOC_XRCD"); 287 case MLXCX_OP_ALLOC_TRANSPORT_DOMAIN: 288 return ("MLXCX_OP_ALLOC_TRANSPORT_DOMAIN"); 289 case MLXCX_OP_DEALLOC_TRANSPORT_DOMAIN: 290 return ("MLXCX_OP_DEALLOC_TRANSPORT_DOMAIN"); 291 case MLXCX_OP_QUERY_CONG_STATUS: 292 return ("MLXCX_OP_QUERY_CONG_STATUS"); 293 case MLXCX_OP_MODIFY_CONG_STATUS: 294 return ("MLXCX_OP_MODIFY_CONG_STATUS"); 295 case MLXCX_OP_QUERY_CONG_PARAMS: 296 return ("MLXCX_OP_QUERY_CONG_PARAMS"); 297 case MLXCX_OP_MODIFY_CONG_PARAMS: 298 return ("MLXCX_OP_MODIFY_CONG_PARAMS"); 299 case MLXCX_OP_QUERY_CONG_STATISTICS: 300 return ("MLXCX_OP_QUERY_CONG_STATISTICS"); 301 case MLXCX_OP_ADD_VXLAN_UDP_DPORT: 302 return ("MLXCX_OP_ADD_VXLAN_UDP_DPORT"); 303 case MLXCX_OP_DELETE_VXLAN_UDP_DPORT: 304 return ("MLXCX_OP_DELETE_VXLAN_UDP_DPORT"); 305 case MLXCX_OP_SET_L2_TABLE_ENTRY: 306 return ("MLXCX_OP_SET_L2_TABLE_ENTRY"); 307 case MLXCX_OP_QUERY_L2_TABLE_ENTRY: 308 return ("MLXCX_OP_QUERY_L2_TABLE_ENTRY"); 309 case MLXCX_OP_DELETE_L2_TABLE_ENTRY: 310 return ("MLXCX_OP_DELETE_L2_TABLE_ENTRY"); 311 case MLXCX_OP_SET_WOL_ROL: 312 return ("MLXCX_OP_SET_WOL_ROL"); 313 case MLXCX_OP_QUERY_WOL_ROL: 314 return ("MLXCX_OP_QUERY_WOL_ROL"); 315 case MLXCX_OP_CREATE_TIR: 316 return ("MLXCX_OP_CREATE_TIR"); 317 case MLXCX_OP_MODIFY_TIR: 318 return ("MLXCX_OP_MODIFY_TIR"); 319 case MLXCX_OP_DESTROY_TIR: 320 return ("MLXCX_OP_DESTROY_TIR"); 321 case MLXCX_OP_QUERY_TIR: 322 return ("MLXCX_OP_QUERY_TIR"); 323 case MLXCX_OP_CREATE_SQ: 324 return ("MLXCX_OP_CREATE_SQ"); 325 case MLXCX_OP_MODIFY_SQ: 326 return ("MLXCX_OP_MODIFY_SQ"); 327 case MLXCX_OP_DESTROY_SQ: 328 return ("MLXCX_OP_DESTROY_SQ"); 329 case MLXCX_OP_QUERY_SQ: 330 return ("MLXCX_OP_QUERY_SQ"); 331 case MLXCX_OP_CREATE_RQ: 332 return ("MLXCX_OP_CREATE_RQ"); 333 case MLXCX_OP_MODIFY_RQ: 334 return ("MLXCX_OP_MODIFY_RQ"); 335 case MLXCX_OP_DESTROY_RQ: 336 return ("MLXCX_OP_DESTROY_RQ"); 337 case MLXCX_OP_QUERY_RQ: 338 return ("MLXCX_OP_QUERY_RQ"); 339 case MLXCX_OP_CREATE_RMP: 340 return ("MLXCX_OP_CREATE_RMP"); 341 case MLXCX_OP_MODIFY_RMP: 342 return ("MLXCX_OP_MODIFY_RMP"); 343 case MLXCX_OP_DESTROY_RMP: 344 return ("MLXCX_OP_DESTROY_RMP"); 345 case MLXCX_OP_QUERY_RMP: 346 return ("MLXCX_OP_QUERY_RMP"); 347 case MLXCX_OP_CREATE_TIS: 348 return ("MLXCX_OP_CREATE_TIS"); 349 case MLXCX_OP_MODIFY_TIS: 350 return ("MLXCX_OP_MODIFY_TIS"); 351 case MLXCX_OP_DESTROY_TIS: 352 return ("MLXCX_OP_DESTROY_TIS"); 353 case MLXCX_OP_QUERY_TIS: 354 return ("MLXCX_OP_QUERY_TIS"); 355 case MLXCX_OP_CREATE_RQT: 356 return ("MLXCX_OP_CREATE_RQT"); 357 case MLXCX_OP_MODIFY_RQT: 358 return ("MLXCX_OP_MODIFY_RQT"); 359 case MLXCX_OP_DESTROY_RQT: 360 return ("MLXCX_OP_DESTROY_RQT"); 361 case MLXCX_OP_QUERY_RQT: 362 return ("MLXCX_OP_QUERY_RQT"); 363 case MLXCX_OP_SET_FLOW_TABLE_ROOT: 364 return ("MLXCX_OP_SET_FLOW_TABLE_ROOT"); 365 case MLXCX_OP_CREATE_FLOW_TABLE: 366 return ("MLXCX_OP_CREATE_FLOW_TABLE"); 367 case MLXCX_OP_DESTROY_FLOW_TABLE: 368 return ("MLXCX_OP_DESTROY_FLOW_TABLE"); 369 case MLXCX_OP_QUERY_FLOW_TABLE: 370 return ("MLXCX_OP_QUERY_FLOW_TABLE"); 371 case MLXCX_OP_CREATE_FLOW_GROUP: 372 return ("MLXCX_OP_CREATE_FLOW_GROUP"); 373 case MLXCX_OP_DESTROY_FLOW_GROUP: 374 return ("MLXCX_OP_DESTROY_FLOW_GROUP"); 375 case MLXCX_OP_QUERY_FLOW_GROUP: 376 return ("MLXCX_OP_QUERY_FLOW_GROUP"); 377 case MLXCX_OP_SET_FLOW_TABLE_ENTRY: 378 return ("MLXCX_OP_SET_FLOW_TABLE_ENTRY"); 379 case MLXCX_OP_QUERY_FLOW_TABLE_ENTRY: 380 return ("MLXCX_OP_QUERY_FLOW_TABLE_ENTRY"); 381 case MLXCX_OP_DELETE_FLOW_TABLE_ENTRY: 382 return ("MLXCX_OP_DELETE_FLOW_TABLE_ENTRY"); 383 case MLXCX_OP_ALLOC_FLOW_COUNTER: 384 return ("MLXCX_OP_ALLOC_FLOW_COUNTER"); 385 case MLXCX_OP_DEALLOC_FLOW_COUNTER: 386 return ("MLXCX_OP_DEALLOC_FLOW_COUNTER"); 387 case MLXCX_OP_QUERY_FLOW_COUNTER: 388 return ("MLXCX_OP_QUERY_FLOW_COUNTER"); 389 case MLXCX_OP_MODIFY_FLOW_TABLE: 390 return ("MLXCX_OP_MODIFY_FLOW_TABLE"); 391 case MLXCX_OP_ALLOC_ENCAP_HEADER: 392 return ("MLXCX_OP_ALLOC_ENCAP_HEADER"); 393 case MLXCX_OP_DEALLOC_ENCAP_HEADER: 394 return ("MLXCX_OP_DEALLOC_ENCAP_HEADER"); 395 case MLXCX_OP_QUERY_ENCAP_HEADER: 396 return ("MLXCX_OP_QUERY_ENCAP_HEADER"); 397 default: 398 return ("Unknown Opcode"); 399 } 400 } 401 402 const char * 403 mlxcx_port_status_string(mlxcx_port_status_t st) 404 { 405 switch (st) { 406 case MLXCX_PORT_STATUS_UP: 407 return ("UP"); 408 case MLXCX_PORT_STATUS_DOWN: 409 return ("DOWN"); 410 case MLXCX_PORT_STATUS_UP_ONCE: 411 return ("UP_ONCE"); 412 case MLXCX_PORT_STATUS_DISABLED: 413 return ("DISABLED"); 414 default: 415 return ("UNKNOWN"); 416 } 417 } 418 419 void 420 mlxcx_eth_proto_to_string(mlxcx_eth_proto_t p, mlxcx_ext_eth_proto_t ep, 421 char *buf, size_t size) 422 { 423 if (p & MLXCX_PROTO_SGMII) 424 (void) strlcat(buf, "SGMII|", size); 425 if (p & MLXCX_PROTO_1000BASE_KX) 426 (void) strlcat(buf, "1000BASE_KX|", size); 427 if (p & MLXCX_PROTO_10GBASE_CX4) 428 (void) strlcat(buf, "10GBASE_CX4|", size); 429 if (p & MLXCX_PROTO_10GBASE_KX4) 430 (void) strlcat(buf, "10GBASE_KX4|", size); 431 if (p & MLXCX_PROTO_10GBASE_KR) 432 (void) strlcat(buf, "10GBASE_KR|", size); 433 if (p & MLXCX_PROTO_40GBASE_CR4) 434 (void) strlcat(buf, "40GBASE_CR4|", size); 435 if (p & MLXCX_PROTO_40GBASE_KR4) 436 (void) strlcat(buf, "40GBASE_KR4|", size); 437 if (p & MLXCX_PROTO_SGMII_100BASE) 438 (void) strlcat(buf, "SGMII_100BASE|", size); 439 if (p & MLXCX_PROTO_10GBASE_CR) 440 (void) strlcat(buf, "10GBASE_CR|", size); 441 if (p & MLXCX_PROTO_10GBASE_SR) 442 (void) strlcat(buf, "10GBASE_SR|", size); 443 if (p & MLXCX_PROTO_10GBASE_ER_LR) 444 (void) strlcat(buf, "10GBASE_ER_LR|", size); 445 if (p & MLXCX_PROTO_40GBASE_SR4) 446 (void) strlcat(buf, "40GBASE_SR4|", size); 447 if (p & MLXCX_PROTO_40GBASE_LR4_ER4) 448 (void) strlcat(buf, "40GBASE_LR4_ER4|", size); 449 if (p & MLXCX_PROTO_50GBASE_SR2) 450 (void) strlcat(buf, "50GBASE_SR2|", size); 451 if (p & MLXCX_PROTO_100GBASE_CR4) 452 (void) strlcat(buf, "100GBASE_CR4|", size); 453 if (p & MLXCX_PROTO_100GBASE_SR4) 454 (void) strlcat(buf, "100GBASE_SR4|", size); 455 if (p & MLXCX_PROTO_100GBASE_KR4) 456 (void) strlcat(buf, "100GBASE_KR4|", size); 457 if (p & MLXCX_PROTO_100GBASE_LR4_ER4) 458 (void) strlcat(buf, "100GBASE_LR4_ER4|", size); 459 if (p & MLXCX_PROTO_100BASE_TX) 460 (void) strlcat(buf, "100BASE_TX|", size); 461 if (p & MLXCX_PROTO_1000BASE_T) 462 (void) strlcat(buf, "1000BASE_T|", size); 463 if (p & MLXCX_PROTO_10GBASE_T) 464 (void) strlcat(buf, "10GBASE_T|", size); 465 if (p & MLXCX_PROTO_25GBASE_CR) 466 (void) strlcat(buf, "25GBASE_CR|", size); 467 if (p & MLXCX_PROTO_25GBASE_KR) 468 (void) strlcat(buf, "25GBASE_KR|", size); 469 if (p & MLXCX_PROTO_25GBASE_SR) 470 (void) strlcat(buf, "25GBASE_SR|", size); 471 if (p & MLXCX_PROTO_50GBASE_CR2) 472 (void) strlcat(buf, "50GBASE_CR2|", size); 473 if (p & MLXCX_PROTO_50GBASE_KR2) 474 (void) strlcat(buf, "50GBASE_KR2|", size); 475 476 /* Now, for the extended bits... */ 477 if (ep & MLXCX_EXTPROTO_SGMII_100BASE) 478 (void) strlcat(buf, "SGMII_100BASE|", size); 479 if (ep & MLXCX_EXTPROTO_1000BASE_X_SGMII) 480 (void) strlcat(buf, "1000BASE_X_SGMII|", size); 481 if (ep & MLXCX_EXTPROTO_5GBASE_R) 482 (void) strlcat(buf, "5GBASE_R|", size); 483 if (ep & MLXCX_EXTPROTO_10GBASE_XFI_XAUI_1) 484 (void) strlcat(buf, "10GBASE_XFI_XAUI_1|", size); 485 if (ep & MLXCX_EXTPROTO_40GBASE_XLAUI_4_XLPPI_4) 486 (void) strlcat(buf, "40GBASE_XLAUI_4_XLPPI_4|", size); 487 if (ep & MLXCX_EXTPROTO_25GAUI_1_25GBASE_CR_KR) 488 (void) strlcat(buf, "25GAUI_1_25GBASE_CR_KR|", size); 489 if (ep & MLXCX_EXTPROTO_50GAUI_2_LAUI_2_50GBASE_CR2_KR2) 490 (void) strlcat(buf, "50GAUI_2_LAUI_2_50GBASE_CR2_KR2|", size); 491 if (ep & MLXCX_EXTPROTO_50GAUI_1_LAUI_1_50GBASE_CR_KR) 492 (void) strlcat(buf, "50GAUI_1_LAUI_1_50GBASE_CR_KR|", size); 493 if (ep & MLXCX_EXTPROTO_CAUI_4_100GBASE_CR4_KR4) 494 (void) strlcat(buf, "CAUI_4_100GBASE_CR4_KR4|", size); 495 if (ep & MLXCX_EXTPROTO_100GAUI_2_100GBASE_CR2_KR2) 496 (void) strlcat(buf, "100GAUI_2_100GBASE_CR2_KR2|", size); 497 if (ep & MLXCX_EXTPROTO_100GAUI_1_100GBASE_CR_KR) 498 (void) strlcat(buf, "100GAUI_1_100GBASE_CR_KR|", size); 499 /* Print these if we need 'em for debugging... */ 500 if (ep & MLXCX_EXTPROTO_200GAUI_4_200GBASE_CR4_KR4) 501 (void) strlcat(buf, "200GAUI_4_200GBASE_CR4_KR4|", size); 502 if (ep & MLXCX_EXTPROTO_200GAUI_2_200GBASE_CR2_KR2) 503 (void) strlcat(buf, "200GAUI_2_200GBASE_CR2_KR2|", size); 504 if (ep & MLXCX_EXTPROTO_400GAUI_8_400GBASE_CR8) 505 (void) strlcat(buf, "400GAUI_8_400GBASE_CR8|", size); 506 if (ep & MLXCX_EXTPROTO_400GAUI_4_400GBASE_CR4) 507 (void) strlcat(buf, "400GAUI_4_400GBASE_CR4|", size); 508 509 /* Chop off the trailing '|' */ 510 if (strlen(buf) > 0) 511 buf[strlen(buf) - 1] = '\0'; 512 } 513 514 void 515 mlxcx_cmd_queue_fini(mlxcx_t *mlxp) 516 { 517 mlxcx_cmd_queue_t *cmd = &mlxp->mlx_cmd; 518 519 if (cmd->mcmd_tokens != NULL) { 520 id_space_destroy(cmd->mcmd_tokens); 521 cmd->mcmd_tokens = NULL; 522 } 523 524 if (cmd->mcmd_taskq != NULL) { 525 ddi_taskq_destroy(cmd->mcmd_taskq); 526 cmd->mcmd_taskq = NULL; 527 } 528 529 cv_destroy(&cmd->mcmd_cv); 530 mutex_destroy(&cmd->mcmd_lock); 531 532 mlxcx_dma_free(&cmd->mcmd_dma); 533 } 534 535 boolean_t 536 mlxcx_cmd_queue_init(mlxcx_t *mlxp) 537 { 538 uint32_t tmp, cmd_low, cmd_high, i; 539 mlxcx_cmd_queue_t *cmd = &mlxp->mlx_cmd; 540 char buf[32]; 541 char tq_name[TASKQ_NAMELEN]; 542 const ddi_dma_cookie_t *ck; 543 544 ddi_device_acc_attr_t acc; 545 ddi_dma_attr_t attr; 546 547 tmp = mlxcx_get32(mlxp, MLXCX_ISS_FIRMWARE); 548 mlxp->mlx_fw_maj = MLXCX_ISS_FW_MAJOR(tmp); 549 mlxp->mlx_fw_min = MLXCX_ISS_FW_MINOR(tmp); 550 551 tmp = mlxcx_get32(mlxp, MLXCX_ISS_FW_CMD); 552 mlxp->mlx_fw_rev = MLXCX_ISS_FW_REV(tmp); 553 mlxp->mlx_cmd_rev = MLXCX_ISS_CMD_REV(tmp); 554 555 if (mlxp->mlx_cmd_rev != MLXCX_CMD_REVISION) { 556 mlxcx_warn(mlxp, "found unsupported command revision: %u, " 557 "expected %u", mlxp->mlx_cmd_rev, MLXCX_CMD_REVISION); 558 return (B_FALSE); 559 } 560 561 cmd_low = mlxcx_get32(mlxp, MLXCX_ISS_CMD_LOW); 562 cmd->mcmd_size_l2 = MLXCX_ISS_CMDQ_SIZE(cmd_low); 563 cmd->mcmd_stride_l2 = MLXCX_ISS_CMDQ_STRIDE(cmd_low); 564 cmd->mcmd_size = 1U << cmd->mcmd_size_l2; 565 566 if (cmd->mcmd_size > MLXCX_CMD_MAX) { 567 mlxcx_warn(mlxp, "command queue size %u is too " 568 "large. Maximum is %u", cmd->mcmd_size, MLXCX_CMD_MAX); 569 return (B_FALSE); 570 } 571 572 cmd->mcmd_mask = (uint32_t)((1ULL << cmd->mcmd_size) - 1); 573 574 mutex_init(&cmd->mcmd_lock, NULL, MUTEX_DRIVER, NULL); 575 cv_init(&cmd->mcmd_cv, NULL, CV_DRIVER, NULL); 576 577 (void) snprintf(buf, sizeof (buf), "mlxcx_tokens_%d", mlxp->mlx_inst); 578 if ((cmd->mcmd_tokens = id_space_create(buf, 1, UINT8_MAX)) == NULL) { 579 mlxcx_warn(mlxp, "failed to allocate token id space"); 580 mlxcx_cmd_queue_fini(mlxp); 581 return (B_FALSE); 582 } 583 584 (void) snprintf(tq_name, sizeof (tq_name), "cmdq_%d", mlxp->mlx_inst); 585 if ((cmd->mcmd_taskq = ddi_taskq_create(mlxp->mlx_dip, tq_name, 1, 586 TASKQ_DEFAULTPRI, 0)) == NULL) { 587 mlxcx_warn(mlxp, "failed to create command queue task queue"); 588 mlxcx_cmd_queue_fini(mlxp); 589 return (B_FALSE); 590 } 591 592 mlxcx_dma_acc_attr(mlxp, &acc); 593 mlxcx_dma_page_attr(mlxp, &attr); 594 595 if (!mlxcx_dma_alloc(mlxp, &cmd->mcmd_dma, &attr, &acc, B_TRUE, 596 MLXCX_CMD_DMA_PAGE_SIZE, B_TRUE)) { 597 mlxcx_warn(mlxp, "failed to allocate command dma buffer"); 598 mlxcx_cmd_queue_fini(mlxp); 599 return (B_FALSE); 600 } 601 602 ck = mlxcx_dma_cookie_one(&cmd->mcmd_dma); 603 cmd_high = (uint32_t)(ck->dmac_laddress >> 32); 604 cmd_low = (uint32_t)(ck->dmac_laddress & UINT32_MAX); 605 606 mlxcx_put32(mlxp, MLXCX_ISS_CMD_HIGH, cmd_high); 607 mlxcx_put32(mlxp, MLXCX_ISS_CMD_LOW, cmd_low); 608 609 /* 610 * Before this is ready, the initializing bit must become zero. 611 */ 612 for (i = 0; i < mlxcx_cmd_init_trys; i++) { 613 uint32_t init = mlxcx_get32(mlxp, MLXCX_ISS_INIT); 614 615 if (MLXCX_ISS_INITIALIZING(init) == 0) 616 break; 617 delay(drv_usectohz(mlxcx_cmd_init_delay)); 618 } 619 if (i == mlxcx_cmd_init_trys) { 620 mlxcx_warn(mlxp, "timed out initializing command queue"); 621 mlxcx_cmd_queue_fini(mlxp); 622 return (B_FALSE); 623 } 624 625 /* 626 * start in polling mode. 627 */ 628 mlxcx_cmd_eq_disable(mlxp); 629 630 return (B_TRUE); 631 } 632 633 void 634 mlxcx_cmd_eq_enable(mlxcx_t *mlxp) 635 { 636 mlxp->mlx_cmd.mcmd_polled = B_FALSE; 637 } 638 639 void 640 mlxcx_cmd_eq_disable(mlxcx_t *mlxp) 641 { 642 mlxp->mlx_cmd.mcmd_polled = B_TRUE; 643 } 644 645 static void 646 mlxcx_cmd_in_header_init(mlxcx_cmd_t *cmd, mlxcx_cmd_in_t *in, 647 mlxcx_cmd_op_t op, uint16_t mod) 648 { 649 ASSERT3U(op, <=, UINT16_MAX); 650 in->mci_opcode = to_be16(op); 651 in->mci_op_mod = to_be16(mod); 652 cmd->mlcmd_op = op; 653 } 654 655 static boolean_t 656 mlxcx_cmd_mbox_alloc(mlxcx_t *mlxp, list_t *listp, uint8_t nblocks) 657 { 658 uint8_t i; 659 ddi_device_acc_attr_t acc; 660 ddi_dma_attr_t attr; 661 662 mlxcx_dma_acc_attr(mlxp, &acc); 663 mlxcx_dma_page_attr(mlxp, &attr); 664 665 for (i = 0; i < nblocks; i++) { 666 mlxcx_cmd_mbox_t *mbox; 667 668 mbox = kmem_zalloc(sizeof (*mbox), KM_SLEEP); 669 if (!mlxcx_dma_alloc(mlxp, &mbox->mlbox_dma, &attr, &acc, 670 B_TRUE, sizeof (mlxcx_cmd_mailbox_t), B_TRUE)) { 671 mlxcx_warn(mlxp, "failed to allocate mailbox dma " 672 "buffer"); 673 kmem_free(mbox, sizeof (*mbox)); 674 /* 675 * mlxcx_cmd_fini will clean up any mboxes that we 676 * already placed onto listp. 677 */ 678 return (B_FALSE); 679 } 680 mbox->mlbox_data = (void *)mbox->mlbox_dma.mxdb_va; 681 list_insert_tail(listp, mbox); 682 } 683 684 return (B_TRUE); 685 } 686 687 static void 688 mlxcx_cmd_mbox_free(mlxcx_cmd_mbox_t *mbox) 689 { 690 mlxcx_dma_free(&mbox->mlbox_dma); 691 kmem_free(mbox, sizeof (mlxcx_cmd_mbox_t)); 692 } 693 694 static void 695 mlxcx_cmd_fini(mlxcx_t *mlxp, mlxcx_cmd_t *cmd) 696 { 697 mlxcx_cmd_mbox_t *mbox; 698 699 while ((mbox = list_remove_head(&cmd->mlcmd_mbox_out)) != NULL) { 700 mlxcx_cmd_mbox_free(mbox); 701 } 702 list_destroy(&cmd->mlcmd_mbox_out); 703 while ((mbox = list_remove_head(&cmd->mlcmd_mbox_in)) != NULL) { 704 mlxcx_cmd_mbox_free(mbox); 705 } 706 list_destroy(&cmd->mlcmd_mbox_in); 707 id_free(mlxp->mlx_cmd.mcmd_tokens, cmd->mlcmd_token); 708 cv_destroy(&cmd->mlcmd_cv); 709 mutex_destroy(&cmd->mlcmd_lock); 710 } 711 712 static void 713 mlxcx_cmd_init(mlxcx_t *mlxp, mlxcx_cmd_t *cmd) 714 { 715 bzero(cmd, sizeof (*cmd)); 716 mutex_init(&cmd->mlcmd_lock, NULL, MUTEX_DRIVER, 717 DDI_INTR_PRI(mlxp->mlx_async_intr_pri)); 718 cv_init(&cmd->mlcmd_cv, NULL, CV_DRIVER, NULL); 719 cmd->mlcmd_token = id_alloc(mlxp->mlx_cmd.mcmd_tokens); 720 cmd->mlcmd_poll = mlxp->mlx_cmd.mcmd_polled; 721 list_create(&cmd->mlcmd_mbox_in, sizeof (mlxcx_cmd_mbox_t), 722 offsetof(mlxcx_cmd_mbox_t, mlbox_node)); 723 list_create(&cmd->mlcmd_mbox_out, sizeof (mlxcx_cmd_mbox_t), 724 offsetof(mlxcx_cmd_mbox_t, mlbox_node)); 725 } 726 727 static void 728 mlxcx_cmd_prep_input(mlxcx_cmd_ent_t *ent, mlxcx_cmd_t *cmd) 729 { 730 uint32_t rem = cmd->mlcmd_inlen; 731 uint8_t i; 732 const void *in = cmd->mlcmd_in; 733 uint32_t copy; 734 mlxcx_cmd_mbox_t *mbox; 735 const ddi_dma_cookie_t *ck; 736 737 copy = MIN(MLXCX_CMD_INLINE_INPUT_LEN, rem); 738 bcopy(in, ent->mce_input, copy); 739 740 rem -= copy; 741 in += copy; 742 743 if (rem == 0) { 744 ent->mce_in_mbox = to_be64(0); 745 VERIFY3U(cmd->mlcmd_nboxes_in, ==, 0); 746 return; 747 } 748 749 mbox = list_head(&cmd->mlcmd_mbox_in); 750 ck = mlxcx_dma_cookie_one(&mbox->mlbox_dma); 751 ent->mce_in_mbox = to_be64(ck->dmac_laddress); 752 for (i = 0; mbox != NULL; 753 mbox = list_next(&cmd->mlcmd_mbox_in, mbox), i++) { 754 mlxcx_cmd_mbox_t *next; 755 mlxcx_cmd_mailbox_t *mp = mbox->mlbox_data; 756 757 copy = MIN(MLXCX_CMD_MAILBOX_LEN, rem); 758 bcopy(in, mp->mlxb_data, copy); 759 rem -= copy; 760 in += copy; 761 762 mp->mlxb_token = cmd->mlcmd_token; 763 mp->mlxb_blockno = to_be32(i); 764 765 next = list_next(&cmd->mlcmd_mbox_in, mbox); 766 if (next == NULL) { 767 mp->mlxb_nextp = to_be64(0); 768 } else { 769 ck = mlxcx_dma_cookie_one(&next->mlbox_dma); 770 mp->mlxb_nextp = to_be64(ck->dmac_laddress); 771 } 772 MLXCX_DMA_SYNC(mbox->mlbox_dma, DDI_DMA_SYNC_FORDEV); 773 } 774 VERIFY3U(i, ==, cmd->mlcmd_nboxes_in); 775 VERIFY0(rem); 776 } 777 778 static void 779 mlxcx_cmd_prep_output(mlxcx_cmd_ent_t *ent, mlxcx_cmd_t *cmd) 780 { 781 uint8_t i; 782 mlxcx_cmd_mbox_t *mbox; 783 const ddi_dma_cookie_t *ck; 784 785 if (cmd->mlcmd_nboxes_out == 0) { 786 ent->mce_out_mbox = to_be64(0); 787 return; 788 } 789 790 mbox = list_head(&cmd->mlcmd_mbox_out); 791 ck = mlxcx_dma_cookie_one(&mbox->mlbox_dma); 792 ent->mce_out_mbox = to_be64(ck->dmac_laddress); 793 for (i = 0, mbox = list_head(&cmd->mlcmd_mbox_out); mbox != NULL; 794 mbox = list_next(&cmd->mlcmd_mbox_out, mbox), i++) { 795 mlxcx_cmd_mbox_t *next; 796 mlxcx_cmd_mailbox_t *mp = mbox->mlbox_data; 797 798 mp->mlxb_token = cmd->mlcmd_token; 799 mp->mlxb_blockno = to_be32(i); 800 801 next = list_next(&cmd->mlcmd_mbox_out, mbox); 802 if (next == NULL) { 803 mp->mlxb_nextp = to_be64(0); 804 } else { 805 ck = mlxcx_dma_cookie_one(&next->mlbox_dma); 806 mp->mlxb_nextp = to_be64(ck->dmac_laddress); 807 } 808 MLXCX_DMA_SYNC(mbox->mlbox_dma, DDI_DMA_SYNC_FORDEV); 809 } 810 VERIFY3U(i, ==, cmd->mlcmd_nboxes_out); 811 } 812 813 static void 814 mlxcx_cmd_copy_output(mlxcx_cmd_ent_t *ent, mlxcx_cmd_t *cmd) 815 { 816 void *out = cmd->mlcmd_out; 817 uint32_t rem = cmd->mlcmd_outlen; 818 uint32_t copy; 819 mlxcx_cmd_mbox_t *mbox; 820 821 copy = MIN(rem, MLXCX_CMD_INLINE_OUTPUT_LEN); 822 bcopy(ent->mce_output, out, copy); 823 out += copy; 824 rem -= copy; 825 826 if (rem == 0) { 827 VERIFY0(cmd->mlcmd_nboxes_out); 828 return; 829 } 830 831 for (mbox = list_head(&cmd->mlcmd_mbox_out); mbox != NULL; 832 mbox = list_next(&cmd->mlcmd_mbox_out, mbox)) { 833 MLXCX_DMA_SYNC(mbox->mlbox_dma, DDI_DMA_SYNC_FORKERNEL); 834 copy = MIN(MLXCX_CMD_MAILBOX_LEN, rem); 835 bcopy(mbox->mlbox_data->mlxb_data, out, copy); 836 out += copy; 837 rem -= copy; 838 } 839 VERIFY0(rem); 840 } 841 842 static uint_t 843 mlxcx_cmd_reserve_slot(mlxcx_cmd_queue_t *cmdq) 844 { 845 uint_t slot; 846 847 mutex_enter(&cmdq->mcmd_lock); 848 slot = ddi_ffs(cmdq->mcmd_mask); 849 while (slot == 0) { 850 cv_wait(&cmdq->mcmd_cv, &cmdq->mcmd_lock); 851 slot = ddi_ffs(cmdq->mcmd_mask); 852 } 853 854 cmdq->mcmd_mask &= ~(1U << --slot); 855 856 ASSERT3P(cmdq->mcmd_active[slot], ==, NULL); 857 858 mutex_exit(&cmdq->mcmd_lock); 859 860 return (slot); 861 } 862 863 static void 864 mlxcx_cmd_release_slot(mlxcx_cmd_queue_t *cmdq, uint_t slot) 865 { 866 mutex_enter(&cmdq->mcmd_lock); 867 cmdq->mcmd_mask |= 1U << slot; 868 cv_broadcast(&cmdq->mcmd_cv); 869 mutex_exit(&cmdq->mcmd_lock); 870 } 871 872 static void 873 mlxcx_cmd_done(mlxcx_cmd_t *cmd, uint_t slot) 874 { 875 mlxcx_t *mlxp = cmd->mlcmd_mlxp; 876 mlxcx_cmd_queue_t *cmdq = &mlxp->mlx_cmd; 877 mlxcx_cmd_ent_t *ent; 878 879 /* 880 * Command is done. Save relevant data. Once we broadcast on the CV and 881 * drop the lock, we must not touch it again. 882 */ 883 MLXCX_DMA_SYNC(cmdq->mcmd_dma, DDI_DMA_SYNC_FORKERNEL); 884 885 ent = (mlxcx_cmd_ent_t *)(cmdq->mcmd_dma.mxdb_va + 886 (slot << cmdq->mcmd_stride_l2)); 887 888 mutex_enter(&cmd->mlcmd_lock); 889 cmd->mlcmd_status = MLXCX_CMD_STATUS(ent->mce_status); 890 if (cmd->mlcmd_status == 0) 891 mlxcx_cmd_copy_output(ent, cmd); 892 893 cmd->mlcmd_state = MLXCX_CMD_S_DONE; 894 cv_broadcast(&cmd->mlcmd_cv); 895 mutex_exit(&cmd->mlcmd_lock); 896 897 cmdq->mcmd_active[slot] = NULL; 898 mlxcx_cmd_release_slot(cmdq, slot); 899 } 900 901 static void 902 mlxcx_cmd_taskq(void *arg) 903 { 904 mlxcx_cmd_t *cmd = arg; 905 mlxcx_t *mlxp = cmd->mlcmd_mlxp; 906 mlxcx_cmd_queue_t *cmdq = &mlxp->mlx_cmd; 907 mlxcx_cmd_ent_t *ent; 908 uint_t poll, slot; 909 910 ASSERT3S(cmd->mlcmd_op, !=, 0); 911 912 slot = mlxcx_cmd_reserve_slot(cmdq); 913 ent = (mlxcx_cmd_ent_t *)(cmdq->mcmd_dma.mxdb_va + 914 (slot << cmdq->mcmd_stride_l2)); 915 916 cmdq->mcmd_active[slot] = cmd; 917 918 /* 919 * Command queue is currently ours as we set busy. 920 */ 921 bzero(ent, sizeof (*ent)); 922 ent->mce_type = MLXCX_CMD_TRANSPORT_PCI; 923 ent->mce_in_length = to_be32(cmd->mlcmd_inlen); 924 ent->mce_out_length = to_be32(cmd->mlcmd_outlen); 925 ent->mce_token = cmd->mlcmd_token; 926 ent->mce_sig = 0; 927 ent->mce_status = MLXCX_CMD_HW_OWNED; 928 mlxcx_cmd_prep_input(ent, cmd); 929 mlxcx_cmd_prep_output(ent, cmd); 930 MLXCX_DMA_SYNC(cmdq->mcmd_dma, DDI_DMA_SYNC_FORDEV); 931 932 mlxcx_put32(mlxp, MLXCX_ISS_CMD_DOORBELL, 1 << slot); 933 934 if (!cmd->mlcmd_poll) 935 return; 936 937 for (poll = 0; poll < mlxcx_cmd_tries; poll++) { 938 delay(drv_usectohz(mlxcx_cmd_delay)); 939 MLXCX_DMA_SYNC(cmdq->mcmd_dma, DDI_DMA_SYNC_FORKERNEL); 940 if ((ent->mce_status & MLXCX_CMD_HW_OWNED) == 0) 941 break; 942 } 943 944 /* 945 * Command is done (or timed out). Save relevant data. Once we broadcast 946 * on the CV and drop the lock, we must not touch the cmd again. 947 */ 948 949 if (poll == mlxcx_cmd_tries) { 950 mutex_enter(&cmd->mlcmd_lock); 951 cmd->mlcmd_status = MLXCX_CMD_R_TIMEOUT; 952 cmd->mlcmd_state = MLXCX_CMD_S_ERROR; 953 cv_broadcast(&cmd->mlcmd_cv); 954 mutex_exit(&cmd->mlcmd_lock); 955 956 mlxcx_fm_ereport(mlxp, DDI_FM_DEVICE_NO_RESPONSE); 957 958 cmdq->mcmd_active[slot] = NULL; 959 mlxcx_cmd_release_slot(cmdq, slot); 960 961 return; 962 } 963 964 mlxcx_cmd_done(cmd, slot); 965 } 966 967 void 968 mlxcx_cmd_completion(mlxcx_t *mlxp, mlxcx_eventq_ent_t *ent) 969 { 970 mlxcx_cmd_queue_t *cmdq = &mlxp->mlx_cmd; 971 mlxcx_evdata_cmd_completion_t *eqe_cmd = &ent->mleqe_cmd_completion; 972 mlxcx_cmd_t *cmd; 973 uint32_t comp_vec = from_be32(eqe_cmd->mled_cmd_completion_vec); 974 uint_t slot; 975 976 DTRACE_PROBE2(cmd_event, mlxcx_t *, mlxp, 977 mlxcx_evdata_cmd_completion_t *, eqe_cmd); 978 979 while ((slot = ddi_ffs(comp_vec)) != 0) { 980 comp_vec &= ~(1U << --slot); 981 982 cmd = cmdq->mcmd_active[slot]; 983 if (cmd->mlcmd_poll) 984 continue; 985 986 mlxcx_cmd_done(cmd, slot); 987 } 988 } 989 990 static boolean_t 991 mlxcx_cmd_send(mlxcx_t *mlxp, mlxcx_cmd_t *cmd, const void *in, uint32_t inlen, 992 void *out, uint32_t outlen) 993 { 994 if (inlen > MLXCX_CMD_INLINE_INPUT_LEN) { 995 uint32_t need = inlen - MLXCX_CMD_INLINE_INPUT_LEN; 996 uint8_t nblocks; 997 998 if (need / MLXCX_CMD_MAILBOX_LEN + 1 > UINT8_MAX) { 999 mlxcx_warn(mlxp, "requested too many input blocks for " 1000 "%u byte input len", inlen); 1001 return (B_FALSE); 1002 } 1003 1004 nblocks = need / MLXCX_CMD_MAILBOX_LEN + 1; 1005 if (!mlxcx_cmd_mbox_alloc(mlxp, &cmd->mlcmd_mbox_in, nblocks)) { 1006 mlxcx_warn(mlxp, "failed to allocate %u blocks of " 1007 "input mailbox", nblocks); 1008 return (B_FALSE); 1009 } 1010 cmd->mlcmd_nboxes_in = nblocks; 1011 } 1012 1013 if (outlen > MLXCX_CMD_INLINE_OUTPUT_LEN) { 1014 uint32_t need = outlen - MLXCX_CMD_INLINE_OUTPUT_LEN; 1015 uint8_t nblocks; 1016 1017 if (need / MLXCX_CMD_MAILBOX_LEN + 1 > UINT8_MAX) { 1018 mlxcx_warn(mlxp, "requested too many output blocks for " 1019 "%u byte output len", outlen); 1020 return (B_FALSE); 1021 } 1022 1023 nblocks = need / MLXCX_CMD_MAILBOX_LEN + 1; 1024 if (!mlxcx_cmd_mbox_alloc(mlxp, &cmd->mlcmd_mbox_out, 1025 nblocks)) { 1026 mlxcx_warn(mlxp, "failed to allocate %u blocks of " 1027 "output mailbox", nblocks); 1028 return (B_FALSE); 1029 } 1030 cmd->mlcmd_nboxes_out = nblocks; 1031 } 1032 1033 cmd->mlcmd_in = in; 1034 cmd->mlcmd_inlen = inlen; 1035 cmd->mlcmd_out = out; 1036 cmd->mlcmd_outlen = outlen; 1037 cmd->mlcmd_mlxp = mlxp; 1038 1039 /* 1040 * Now that all allocations have been done, all that remains is for us 1041 * to dispatch the request to process this to the taskq for it to be 1042 * processed. 1043 */ 1044 if (ddi_taskq_dispatch(mlxp->mlx_cmd.mcmd_taskq, mlxcx_cmd_taskq, cmd, 1045 DDI_SLEEP) != DDI_SUCCESS) { 1046 mlxcx_warn(mlxp, "failed to submit command to taskq"); 1047 return (B_FALSE); 1048 } 1049 1050 return (B_TRUE); 1051 } 1052 1053 static void 1054 mlxcx_cmd_wait(mlxcx_cmd_t *cmd) 1055 { 1056 mutex_enter(&cmd->mlcmd_lock); 1057 while (cmd->mlcmd_state == 0) { 1058 cv_wait(&cmd->mlcmd_cv, &cmd->mlcmd_lock); 1059 } 1060 mutex_exit(&cmd->mlcmd_lock); 1061 } 1062 1063 static boolean_t 1064 mlxcx_cmd_evaluate(mlxcx_t *mlxp, mlxcx_cmd_t *cmd) 1065 { 1066 mlxcx_cmd_out_t *out; 1067 1068 if ((cmd->mlcmd_state & MLXCX_CMD_S_ERROR) != 0) { 1069 mlxcx_warn(mlxp, "command %s (0x%x) failed due to an internal " 1070 "driver error", 1071 mlxcx_cmd_opcode_string(cmd->mlcmd_op), 1072 cmd->mlcmd_op); 1073 return (B_FALSE); 1074 } 1075 1076 if (cmd->mlcmd_status != 0) { 1077 mlxcx_warn(mlxp, "command %s (0x%x) failed with command queue " 1078 "error 0x%x", 1079 mlxcx_cmd_opcode_string(cmd->mlcmd_op), 1080 cmd->mlcmd_op, cmd->mlcmd_status); 1081 return (B_FALSE); 1082 } 1083 1084 out = cmd->mlcmd_out; 1085 if (out->mco_status != MLXCX_CMD_R_OK) { 1086 mlxcx_warn(mlxp, "command %s 0x%x failed with status code %s " 1087 "(0x%x)", mlxcx_cmd_opcode_string(cmd->mlcmd_op), 1088 cmd->mlcmd_op, mlxcx_cmd_response_string(out->mco_status), 1089 out->mco_status); 1090 return (B_FALSE); 1091 } 1092 1093 return (B_TRUE); 1094 } 1095 1096 boolean_t 1097 mlxcx_cmd_disable_hca(mlxcx_t *mlxp) 1098 { 1099 mlxcx_cmd_t cmd; 1100 mlxcx_cmd_disable_hca_in_t in; 1101 mlxcx_cmd_disable_hca_out_t out; 1102 boolean_t ret; 1103 1104 bzero(&in, sizeof (in)); 1105 bzero(&out, sizeof (out)); 1106 1107 mlxcx_cmd_init(mlxp, &cmd); 1108 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_disable_hca_head, 1109 MLXCX_OP_DISABLE_HCA, 0); 1110 in.mlxi_disable_hca_func = MLXCX_FUNCTION_SELF; 1111 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1112 mlxcx_cmd_fini(mlxp, &cmd); 1113 return (B_FALSE); 1114 } 1115 mlxcx_cmd_wait(&cmd); 1116 1117 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1118 mlxcx_cmd_fini(mlxp, &cmd); 1119 return (ret); 1120 } 1121 1122 boolean_t 1123 mlxcx_cmd_enable_hca(mlxcx_t *mlxp) 1124 { 1125 mlxcx_cmd_t cmd; 1126 mlxcx_cmd_enable_hca_in_t in; 1127 mlxcx_cmd_enable_hca_out_t out; 1128 boolean_t ret; 1129 1130 bzero(&in, sizeof (in)); 1131 bzero(&out, sizeof (out)); 1132 1133 mlxcx_cmd_init(mlxp, &cmd); 1134 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_enable_hca_head, 1135 MLXCX_OP_ENABLE_HCA, 0); 1136 in.mlxi_enable_hca_func = MLXCX_FUNCTION_SELF; 1137 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1138 mlxcx_cmd_fini(mlxp, &cmd); 1139 return (B_FALSE); 1140 } 1141 mlxcx_cmd_wait(&cmd); 1142 1143 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1144 mlxcx_cmd_fini(mlxp, &cmd); 1145 return (ret); 1146 } 1147 1148 boolean_t 1149 mlxcx_cmd_query_issi(mlxcx_t *mlxp, uint32_t *issip) 1150 { 1151 mlxcx_cmd_t cmd; 1152 mlxcx_cmd_query_issi_in_t in; 1153 mlxcx_cmd_query_issi_out_t out; 1154 boolean_t ret; 1155 1156 bzero(&in, sizeof (in)); 1157 bzero(&out, sizeof (out)); 1158 1159 mlxcx_cmd_init(mlxp, &cmd); 1160 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_issi_head, 1161 MLXCX_OP_QUERY_ISSI, 0); 1162 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1163 mlxcx_cmd_fini(mlxp, &cmd); 1164 return (B_FALSE); 1165 } 1166 mlxcx_cmd_wait(&cmd); 1167 1168 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1169 if (ret) { 1170 *issip = out.mlxo_supported_issi; 1171 } else if (cmd.mlcmd_status == 0 && 1172 out.mlxo_query_issi_head.mco_status == MLXCX_CMD_R_BAD_OP) { 1173 /* 1174 * The PRM says that if we get a bad operation, that means this 1175 * command isn't supported so it only supports version 1 of the 1176 * ISSI, which means bit zero should be set. 1177 */ 1178 ret = B_TRUE; 1179 *issip = 1; 1180 } 1181 mlxcx_cmd_fini(mlxp, &cmd); 1182 return (ret); 1183 } 1184 1185 boolean_t 1186 mlxcx_cmd_set_issi(mlxcx_t *mlxp, uint16_t issi) 1187 { 1188 mlxcx_cmd_t cmd; 1189 mlxcx_cmd_set_issi_in_t in; 1190 mlxcx_cmd_set_issi_out_t out; 1191 boolean_t ret; 1192 1193 bzero(&in, sizeof (in)); 1194 bzero(&out, sizeof (out)); 1195 1196 mlxcx_cmd_init(mlxp, &cmd); 1197 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_set_issi_head, 1198 MLXCX_OP_SET_ISSI, 0); 1199 in.mlxi_set_issi_current = to_be16(issi); 1200 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1201 mlxcx_cmd_fini(mlxp, &cmd); 1202 return (B_FALSE); 1203 } 1204 mlxcx_cmd_wait(&cmd); 1205 1206 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1207 mlxcx_cmd_fini(mlxp, &cmd); 1208 return (ret); 1209 } 1210 1211 boolean_t 1212 mlxcx_cmd_query_pages(mlxcx_t *mlxp, uint_t type, int32_t *npages) 1213 { 1214 mlxcx_cmd_t cmd; 1215 mlxcx_cmd_query_pages_in_t in; 1216 mlxcx_cmd_query_pages_out_t out; 1217 boolean_t ret; 1218 1219 switch (type) { 1220 case MLXCX_QUERY_PAGES_OPMOD_BOOT: 1221 case MLXCX_QUERY_PAGES_OPMOD_INIT: 1222 case MLXCX_QUERY_PAGES_OPMOD_REGULAR: 1223 break; 1224 default: 1225 mlxcx_warn(mlxp, "!passed invalid type to query pages: %u", 1226 type); 1227 return (B_FALSE); 1228 } 1229 1230 bzero(&in, sizeof (in)); 1231 bzero(&out, sizeof (out)); 1232 1233 mlxcx_cmd_init(mlxp, &cmd); 1234 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_pages_head, 1235 MLXCX_OP_QUERY_PAGES, type); 1236 in.mlxi_query_pages_func = MLXCX_FUNCTION_SELF; 1237 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1238 mlxcx_cmd_fini(mlxp, &cmd); 1239 return (B_FALSE); 1240 } 1241 mlxcx_cmd_wait(&cmd); 1242 1243 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1244 if (ret) { 1245 *npages = from_be32(out.mlxo_query_pages_npages); 1246 } 1247 mlxcx_cmd_fini(mlxp, &cmd); 1248 1249 return (ret); 1250 } 1251 1252 boolean_t 1253 mlxcx_cmd_give_pages(mlxcx_t *mlxp, uint_t type, int32_t npages, 1254 mlxcx_dev_page_t **pages) 1255 { 1256 mlxcx_cmd_t cmd; 1257 mlxcx_cmd_manage_pages_in_t *in; 1258 mlxcx_cmd_manage_pages_out_t out; 1259 size_t insize, outsize; 1260 boolean_t ret; 1261 uint32_t i; 1262 uint64_t pa; 1263 const ddi_dma_cookie_t *ck; 1264 1265 switch (type) { 1266 case MLXCX_MANAGE_PAGES_OPMOD_ALLOC_FAIL: 1267 if (npages != 0) { 1268 mlxcx_warn(mlxp, "passed non-zero number of pages (%d) " 1269 "but asked to fail page allocation", npages); 1270 return (B_FALSE); 1271 } 1272 break; 1273 case MLXCX_MANAGE_PAGES_OPMOD_GIVE_PAGES: 1274 ASSERT3S(npages, <=, MLXCX_MANAGE_PAGES_MAX_PAGES); 1275 if (npages <= 0) { 1276 mlxcx_warn(mlxp, "passed invalid number of pages (%d) " 1277 "to give pages", npages); 1278 return (B_FALSE); 1279 } 1280 break; 1281 default: 1282 mlxcx_warn(mlxp, "!passed invalid type to give pages: %u", 1283 type); 1284 return (B_FALSE); 1285 } 1286 1287 insize = offsetof(mlxcx_cmd_manage_pages_in_t, mlxi_manage_pages_pas) + 1288 npages * sizeof (uint64_t); 1289 outsize = offsetof(mlxcx_cmd_manage_pages_out_t, mlxo_manage_pages_pas); 1290 1291 in = kmem_zalloc(insize, KM_SLEEP); 1292 bzero(&out, sizeof (out)); 1293 1294 mlxcx_cmd_init(mlxp, &cmd); 1295 mlxcx_cmd_in_header_init(&cmd, &in->mlxi_manage_pages_head, 1296 MLXCX_OP_MANAGE_PAGES, type); 1297 in->mlxi_manage_pages_func = MLXCX_FUNCTION_SELF; 1298 in->mlxi_manage_pages_npages = to_be32(npages); 1299 for (i = 0; i < npages; i++) { 1300 ck = mlxcx_dma_cookie_one(&pages[i]->mxdp_dma); 1301 pa = ck->dmac_laddress; 1302 ASSERT3U(pa & 0xfff, ==, 0); 1303 ASSERT3U(ck->dmac_size, ==, MLXCX_HW_PAGE_SIZE); 1304 in->mlxi_manage_pages_pas[i] = to_be64(pa); 1305 } 1306 1307 if ((ret = mlxcx_cmd_send(mlxp, &cmd, in, insize, &out, outsize))) { 1308 mlxcx_cmd_wait(&cmd); 1309 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1310 } 1311 1312 mlxcx_cmd_fini(mlxp, &cmd); 1313 1314 kmem_free(in, insize); 1315 return (ret); 1316 } 1317 1318 boolean_t 1319 mlxcx_cmd_return_pages(mlxcx_t *mlxp, int32_t nreq, uint64_t *pas, 1320 int32_t *nret) 1321 { 1322 mlxcx_cmd_t cmd; 1323 mlxcx_cmd_manage_pages_in_t in; 1324 mlxcx_cmd_manage_pages_out_t *out; 1325 size_t insize, outsize; 1326 boolean_t ret; 1327 uint32_t i; 1328 1329 if (nreq <= 0) { 1330 mlxcx_warn(mlxp, "passed invalid number of pages (%d) " 1331 "to return pages", nreq); 1332 return (B_FALSE); 1333 } 1334 1335 insize = offsetof(mlxcx_cmd_manage_pages_in_t, mlxi_manage_pages_pas); 1336 outsize = offsetof(mlxcx_cmd_manage_pages_out_t, 1337 mlxo_manage_pages_pas) + nreq * sizeof (uint64_t); 1338 1339 bzero(&in, sizeof (in)); 1340 out = kmem_alloc(outsize, KM_SLEEP); 1341 1342 mlxcx_cmd_init(mlxp, &cmd); 1343 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_manage_pages_head, 1344 MLXCX_OP_MANAGE_PAGES, MLXCX_MANAGE_PAGES_OPMOD_RETURN_PAGES); 1345 in.mlxi_manage_pages_func = MLXCX_FUNCTION_SELF; 1346 in.mlxi_manage_pages_npages = to_be32(nreq); 1347 1348 if ((ret = mlxcx_cmd_send(mlxp, &cmd, &in, insize, out, outsize))) { 1349 mlxcx_cmd_wait(&cmd); 1350 1351 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1352 if (ret) { 1353 *nret = from_be32(out->mlxo_manage_pages_npages); 1354 for (i = 0; i < *nret; i++) { 1355 pas[i] = 1356 from_be64(out->mlxo_manage_pages_pas[i]); 1357 } 1358 } 1359 } 1360 1361 mlxcx_cmd_fini(mlxp, &cmd); 1362 1363 kmem_free(out, outsize); 1364 return (ret); 1365 } 1366 1367 boolean_t 1368 mlxcx_cmd_query_hca_cap(mlxcx_t *mlxp, mlxcx_hca_cap_type_t type, 1369 mlxcx_hca_cap_mode_t mode, mlxcx_hca_cap_t *capp) 1370 { 1371 mlxcx_cmd_t cmd; 1372 mlxcx_cmd_query_hca_cap_in_t in; 1373 mlxcx_cmd_query_hca_cap_out_t *out; 1374 boolean_t ret; 1375 uint16_t opmode; 1376 1377 bzero(&in, sizeof (in)); 1378 out = kmem_zalloc(sizeof (mlxcx_cmd_query_hca_cap_out_t), KM_SLEEP); 1379 mlxcx_cmd_init(mlxp, &cmd); 1380 1381 opmode = type << 1 | mode; 1382 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_hca_cap_head, 1383 MLXCX_OP_QUERY_HCA_CAP, opmode); 1384 1385 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), out, sizeof (*out))) { 1386 mlxcx_cmd_fini(mlxp, &cmd); 1387 kmem_free(out, sizeof (mlxcx_cmd_query_hca_cap_out_t)); 1388 return (B_FALSE); 1389 } 1390 mlxcx_cmd_wait(&cmd); 1391 1392 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1393 if (ret) { 1394 capp->mhc_mode = mode; 1395 capp->mhc_type = type; 1396 ASSERT3U(sizeof (out->mlxo_query_hca_cap_data), ==, 1397 sizeof (capp->mhc_bulk)); 1398 bcopy(out->mlxo_query_hca_cap_data, capp->mhc_bulk, 1399 sizeof (capp->mhc_bulk)); 1400 } 1401 mlxcx_cmd_fini(mlxp, &cmd); 1402 1403 kmem_free(out, sizeof (mlxcx_cmd_query_hca_cap_out_t)); 1404 return (B_TRUE); 1405 } 1406 1407 boolean_t 1408 mlxcx_cmd_init_hca(mlxcx_t *mlxp) 1409 { 1410 mlxcx_cmd_t cmd; 1411 mlxcx_cmd_init_hca_in_t in; 1412 mlxcx_cmd_init_hca_out_t out; 1413 boolean_t ret; 1414 1415 bzero(&in, sizeof (in)); 1416 bzero(&out, sizeof (out)); 1417 1418 mlxcx_cmd_init(mlxp, &cmd); 1419 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_init_hca_head, 1420 MLXCX_OP_INIT_HCA, 0); 1421 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1422 mlxcx_cmd_fini(mlxp, &cmd); 1423 return (B_FALSE); 1424 } 1425 mlxcx_cmd_wait(&cmd); 1426 1427 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1428 mlxcx_cmd_fini(mlxp, &cmd); 1429 return (ret); 1430 } 1431 1432 boolean_t 1433 mlxcx_cmd_set_driver_version(mlxcx_t *mlxp, const char *version) 1434 { 1435 mlxcx_cmd_t cmd; 1436 mlxcx_cmd_set_driver_version_in_t in; 1437 mlxcx_cmd_set_driver_version_out_t out; 1438 boolean_t ret; 1439 1440 bzero(&in, sizeof (in)); 1441 bzero(&out, sizeof (out)); 1442 1443 mlxcx_cmd_init(mlxp, &cmd); 1444 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_set_driver_version_head, 1445 MLXCX_OP_SET_DRIVER_VERSION, 0); 1446 VERIFY3U(strlcpy(in.mlxi_set_driver_version_version, version, 1447 sizeof (in.mlxi_set_driver_version_version)), <=, 1448 sizeof (in.mlxi_set_driver_version_version)); 1449 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1450 mlxcx_cmd_fini(mlxp, &cmd); 1451 return (B_FALSE); 1452 } 1453 mlxcx_cmd_wait(&cmd); 1454 1455 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1456 mlxcx_cmd_fini(mlxp, &cmd); 1457 return (ret); 1458 } 1459 1460 boolean_t 1461 mlxcx_cmd_alloc_uar(mlxcx_t *mlxp, mlxcx_uar_t *mlup) 1462 { 1463 mlxcx_cmd_t cmd; 1464 mlxcx_cmd_alloc_uar_in_t in; 1465 mlxcx_cmd_alloc_uar_out_t out; 1466 boolean_t ret; 1467 size_t i; 1468 1469 bzero(&in, sizeof (in)); 1470 bzero(&out, sizeof (out)); 1471 1472 mlxcx_cmd_init(mlxp, &cmd); 1473 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_alloc_uar_head, 1474 MLXCX_OP_ALLOC_UAR, 0); 1475 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1476 mlxcx_cmd_fini(mlxp, &cmd); 1477 return (B_FALSE); 1478 } 1479 mlxcx_cmd_wait(&cmd); 1480 1481 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1482 if (ret) { 1483 mlup->mlu_allocated = B_TRUE; 1484 mlup->mlu_num = from_be24(out.mlxo_alloc_uar_uar); 1485 VERIFY3U(mlup->mlu_num, >, 0); 1486 mlup->mlu_base = mlup->mlu_num * MLXCX_HW_PAGE_SIZE; 1487 1488 for (i = 0; i < MLXCX_BF_PER_UAR; ++i) { 1489 mlup->mlu_bf[i].mbf_even = mlup->mlu_base + 1490 MLXCX_BF_BASE + MLXCX_BF_SIZE * 2 * i; 1491 mlup->mlu_bf[i].mbf_odd = mlup->mlu_bf[i].mbf_even + 1492 MLXCX_BF_SIZE; 1493 } 1494 } 1495 mlxcx_cmd_fini(mlxp, &cmd); 1496 return (ret); 1497 } 1498 1499 boolean_t 1500 mlxcx_cmd_dealloc_uar(mlxcx_t *mlxp, mlxcx_uar_t *mlup) 1501 { 1502 mlxcx_cmd_t cmd; 1503 mlxcx_cmd_dealloc_uar_in_t in; 1504 mlxcx_cmd_dealloc_uar_out_t out; 1505 boolean_t ret; 1506 1507 bzero(&in, sizeof (in)); 1508 bzero(&out, sizeof (out)); 1509 1510 mlxcx_cmd_init(mlxp, &cmd); 1511 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_dealloc_uar_head, 1512 MLXCX_OP_DEALLOC_UAR, 0); 1513 VERIFY(mlup->mlu_allocated); 1514 in.mlxi_dealloc_uar_uar = to_be24(mlup->mlu_num); 1515 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1516 mlxcx_cmd_fini(mlxp, &cmd); 1517 return (B_FALSE); 1518 } 1519 mlxcx_cmd_wait(&cmd); 1520 1521 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1522 if (ret) { 1523 mlup->mlu_allocated = B_FALSE; 1524 mlup->mlu_num = 0; 1525 } 1526 mlxcx_cmd_fini(mlxp, &cmd); 1527 return (ret); 1528 } 1529 1530 boolean_t 1531 mlxcx_cmd_alloc_pd(mlxcx_t *mlxp, mlxcx_pd_t *mlpd) 1532 { 1533 mlxcx_cmd_t cmd; 1534 mlxcx_cmd_alloc_pd_in_t in; 1535 mlxcx_cmd_alloc_pd_out_t out; 1536 boolean_t ret; 1537 1538 bzero(&in, sizeof (in)); 1539 bzero(&out, sizeof (out)); 1540 1541 mlxcx_cmd_init(mlxp, &cmd); 1542 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_alloc_pd_head, 1543 MLXCX_OP_ALLOC_PD, 0); 1544 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1545 mlxcx_cmd_fini(mlxp, &cmd); 1546 return (B_FALSE); 1547 } 1548 mlxcx_cmd_wait(&cmd); 1549 1550 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1551 if (ret) { 1552 mlpd->mlpd_allocated = B_TRUE; 1553 mlpd->mlpd_num = from_be24(out.mlxo_alloc_pd_pdn); 1554 } 1555 mlxcx_cmd_fini(mlxp, &cmd); 1556 return (ret); 1557 } 1558 1559 boolean_t 1560 mlxcx_cmd_dealloc_pd(mlxcx_t *mlxp, mlxcx_pd_t *mlpd) 1561 { 1562 mlxcx_cmd_t cmd; 1563 mlxcx_cmd_dealloc_pd_in_t in; 1564 mlxcx_cmd_dealloc_pd_out_t out; 1565 boolean_t ret; 1566 1567 bzero(&in, sizeof (in)); 1568 bzero(&out, sizeof (out)); 1569 1570 mlxcx_cmd_init(mlxp, &cmd); 1571 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_dealloc_pd_head, 1572 MLXCX_OP_DEALLOC_PD, 0); 1573 VERIFY(mlpd->mlpd_allocated); 1574 in.mlxi_dealloc_pd_pdn = to_be24(mlpd->mlpd_num); 1575 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1576 mlxcx_cmd_fini(mlxp, &cmd); 1577 return (B_FALSE); 1578 } 1579 mlxcx_cmd_wait(&cmd); 1580 1581 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1582 if (ret) { 1583 mlpd->mlpd_allocated = B_FALSE; 1584 mlpd->mlpd_num = 0; 1585 } 1586 mlxcx_cmd_fini(mlxp, &cmd); 1587 return (ret); 1588 } 1589 1590 boolean_t 1591 mlxcx_cmd_alloc_tdom(mlxcx_t *mlxp, mlxcx_tdom_t *mltd) 1592 { 1593 mlxcx_cmd_t cmd; 1594 mlxcx_cmd_alloc_tdom_in_t in; 1595 mlxcx_cmd_alloc_tdom_out_t out; 1596 boolean_t ret; 1597 1598 bzero(&in, sizeof (in)); 1599 bzero(&out, sizeof (out)); 1600 1601 mlxcx_cmd_init(mlxp, &cmd); 1602 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_alloc_tdom_head, 1603 MLXCX_OP_ALLOC_TRANSPORT_DOMAIN, 0); 1604 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1605 mlxcx_cmd_fini(mlxp, &cmd); 1606 return (B_FALSE); 1607 } 1608 mlxcx_cmd_wait(&cmd); 1609 1610 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1611 if (ret) { 1612 mltd->mltd_allocated = B_TRUE; 1613 mltd->mltd_num = from_be24(out.mlxo_alloc_tdom_tdomn); 1614 } 1615 mlxcx_cmd_fini(mlxp, &cmd); 1616 return (ret); 1617 } 1618 1619 boolean_t 1620 mlxcx_cmd_dealloc_tdom(mlxcx_t *mlxp, mlxcx_tdom_t *mltd) 1621 { 1622 mlxcx_cmd_t cmd; 1623 mlxcx_cmd_dealloc_tdom_in_t in; 1624 mlxcx_cmd_dealloc_tdom_out_t out; 1625 boolean_t ret; 1626 1627 bzero(&in, sizeof (in)); 1628 bzero(&out, sizeof (out)); 1629 1630 mlxcx_cmd_init(mlxp, &cmd); 1631 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_dealloc_tdom_head, 1632 MLXCX_OP_DEALLOC_TRANSPORT_DOMAIN, 0); 1633 VERIFY(mltd->mltd_allocated); 1634 in.mlxi_dealloc_tdom_tdomn = to_be24(mltd->mltd_num); 1635 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1636 mlxcx_cmd_fini(mlxp, &cmd); 1637 return (B_FALSE); 1638 } 1639 mlxcx_cmd_wait(&cmd); 1640 1641 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1642 if (ret) { 1643 mltd->mltd_allocated = B_FALSE; 1644 mltd->mltd_num = 0; 1645 } 1646 mlxcx_cmd_fini(mlxp, &cmd); 1647 return (ret); 1648 } 1649 1650 boolean_t 1651 mlxcx_cmd_teardown_hca(mlxcx_t *mlxp) 1652 { 1653 mlxcx_cmd_t cmd; 1654 mlxcx_cmd_teardown_hca_in_t in; 1655 mlxcx_cmd_teardown_hca_out_t out; 1656 boolean_t ret; 1657 1658 bzero(&in, sizeof (in)); 1659 bzero(&out, sizeof (out)); 1660 1661 mlxcx_cmd_init(mlxp, &cmd); 1662 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_teardown_hca_head, 1663 MLXCX_OP_TEARDOWN_HCA, 0); 1664 in.mlxi_teardown_hca_profile = to_be16(MLXCX_TEARDOWN_HCA_GRACEFUL); 1665 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1666 mlxcx_cmd_fini(mlxp, &cmd); 1667 return (B_FALSE); 1668 } 1669 mlxcx_cmd_wait(&cmd); 1670 1671 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1672 mlxcx_cmd_fini(mlxp, &cmd); 1673 return (ret); 1674 } 1675 1676 boolean_t 1677 mlxcx_cmd_query_nic_vport_ctx(mlxcx_t *mlxp, mlxcx_port_t *mlp) 1678 { 1679 mlxcx_cmd_t cmd; 1680 mlxcx_cmd_query_nic_vport_ctx_in_t in; 1681 mlxcx_cmd_query_nic_vport_ctx_out_t out; 1682 boolean_t ret; 1683 const mlxcx_nic_vport_ctx_t *ctx; 1684 1685 bzero(&in, sizeof (in)); 1686 bzero(&out, sizeof (out)); 1687 1688 ASSERT(mutex_owned(&mlp->mlp_mtx)); 1689 mlxcx_cmd_init(mlxp, &cmd); 1690 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_nic_vport_ctx_head, 1691 MLXCX_OP_QUERY_NIC_VPORT_CONTEXT, MLXCX_VPORT_TYPE_VNIC); 1692 1693 in.mlxi_query_nic_vport_ctx_vport_number = to_be16(mlp->mlp_num); 1694 1695 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 1696 mlxcx_cmd_fini(mlxp, &cmd); 1697 return (B_FALSE); 1698 } 1699 mlxcx_cmd_wait(&cmd); 1700 1701 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1702 if (ret) { 1703 ctx = &out.mlxo_query_nic_vport_ctx_context; 1704 mlp->mlp_guid = from_be64(ctx->mlnvc_port_guid); 1705 mlp->mlp_mtu = from_be16(ctx->mlnvc_mtu); 1706 bcopy(ctx->mlnvc_permanent_address, mlp->mlp_mac_address, 1707 sizeof (mlp->mlp_mac_address)); 1708 mlp->mlp_wqe_min_inline = get_bits64(ctx->mlnvc_flags, 1709 MLXCX_VPORT_CTX_MIN_WQE_INLINE); 1710 } 1711 mlxcx_cmd_fini(mlxp, &cmd); 1712 return (ret); 1713 } 1714 1715 static const char * 1716 mlxcx_reg_name(mlxcx_register_id_t rid) 1717 { 1718 switch (rid) { 1719 case MLXCX_REG_PMTU: 1720 return ("PMTU"); 1721 case MLXCX_REG_PAOS: 1722 return ("PAOS"); 1723 case MLXCX_REG_PCAM: 1724 return ("PCAM"); 1725 case MLXCX_REG_PTYS: 1726 return ("PTYS"); 1727 case MLXCX_REG_MSGI: 1728 return ("MSGI"); 1729 case MLXCX_REG_PMAOS: 1730 return ("PMAOS"); 1731 case MLXCX_REG_MLCR: 1732 return ("MLCR"); 1733 case MLXCX_REG_MCIA: 1734 return ("MCIA"); 1735 case MLXCX_REG_PPCNT: 1736 return ("PPCNT"); 1737 case MLXCX_REG_PPLM: 1738 return ("PPLM"); 1739 case MLXCX_REG_MTCAP: 1740 return ("MTCAP"); 1741 case MLXCX_REG_MTMP: 1742 return ("MTMP"); 1743 default: 1744 return ("???"); 1745 } 1746 } 1747 1748 boolean_t 1749 mlxcx_cmd_access_register(mlxcx_t *mlxp, mlxcx_cmd_reg_opmod_t opmod, 1750 mlxcx_register_id_t rid, mlxcx_register_data_t *data) 1751 { 1752 mlxcx_cmd_t cmd; 1753 mlxcx_cmd_access_register_in_t in; 1754 mlxcx_cmd_access_register_out_t out; 1755 boolean_t ret; 1756 size_t dsize, insize, outsize; 1757 1758 bzero(&in, sizeof (in)); 1759 bzero(&out, sizeof (out)); 1760 1761 mlxcx_cmd_init(mlxp, &cmd); 1762 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_access_register_head, 1763 MLXCX_OP_ACCESS_REG, opmod); 1764 1765 in.mlxi_access_register_register_id = to_be16(rid); 1766 1767 switch (rid) { 1768 case MLXCX_REG_PMTU: 1769 dsize = sizeof (mlxcx_reg_pmtu_t); 1770 break; 1771 case MLXCX_REG_PAOS: 1772 dsize = sizeof (mlxcx_reg_paos_t); 1773 break; 1774 case MLXCX_REG_PCAM: 1775 dsize = sizeof (mlxcx_reg_pcam_t); 1776 break; 1777 case MLXCX_REG_PTYS: 1778 dsize = sizeof (mlxcx_reg_ptys_t); 1779 break; 1780 case MLXCX_REG_MLCR: 1781 dsize = sizeof (mlxcx_reg_mlcr_t); 1782 break; 1783 case MLXCX_REG_PMAOS: 1784 dsize = sizeof (mlxcx_reg_pmaos_t); 1785 break; 1786 case MLXCX_REG_MCIA: 1787 dsize = sizeof (mlxcx_reg_mcia_t); 1788 break; 1789 case MLXCX_REG_PPCNT: 1790 dsize = sizeof (mlxcx_reg_ppcnt_t); 1791 break; 1792 case MLXCX_REG_PPLM: 1793 dsize = sizeof (mlxcx_reg_pplm_t); 1794 break; 1795 case MLXCX_REG_MTCAP: 1796 dsize = sizeof (mlxcx_reg_mtcap_t); 1797 break; 1798 case MLXCX_REG_MTMP: 1799 dsize = sizeof (mlxcx_reg_mtmp_t); 1800 break; 1801 default: 1802 dsize = 0; 1803 VERIFY(0); 1804 return (B_FALSE); 1805 } 1806 insize = dsize + offsetof(mlxcx_cmd_access_register_in_t, 1807 mlxi_access_register_data); 1808 outsize = dsize + offsetof(mlxcx_cmd_access_register_out_t, 1809 mlxo_access_register_data); 1810 1811 bcopy(data, &in.mlxi_access_register_data, dsize); 1812 1813 if (!mlxcx_cmd_send(mlxp, &cmd, &in, insize, &out, outsize)) { 1814 mlxcx_cmd_fini(mlxp, &cmd); 1815 return (B_FALSE); 1816 } 1817 mlxcx_cmd_wait(&cmd); 1818 1819 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 1820 if (ret) { 1821 bcopy(&out.mlxo_access_register_data, data, dsize); 1822 } else { 1823 mlxcx_warn(mlxp, "failed OP_ACCESS_REG was for register " 1824 "%04x (%s)", rid, mlxcx_reg_name(rid)); 1825 } 1826 mlxcx_cmd_fini(mlxp, &cmd); 1827 return (ret); 1828 } 1829 1830 boolean_t 1831 mlxcx_cmd_query_port_mtu(mlxcx_t *mlxp, mlxcx_port_t *mlp) 1832 { 1833 mlxcx_register_data_t data; 1834 boolean_t ret; 1835 1836 /* 1837 * Since we modify the port here we require that the caller is holding 1838 * the port mutex. 1839 */ 1840 ASSERT(mutex_owned(&mlp->mlp_mtx)); 1841 bzero(&data, sizeof (data)); 1842 data.mlrd_pmtu.mlrd_pmtu_local_port = mlp->mlp_num + 1; 1843 1844 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_READ, 1845 MLXCX_REG_PMTU, &data); 1846 1847 if (ret) { 1848 mlp->mlp_mtu = from_be16(data.mlrd_pmtu.mlrd_pmtu_admin_mtu); 1849 mlp->mlp_max_mtu = from_be16(data.mlrd_pmtu.mlrd_pmtu_max_mtu); 1850 } 1851 1852 return (ret); 1853 } 1854 1855 boolean_t 1856 mlxcx_cmd_query_module_status(mlxcx_t *mlxp, uint_t id, 1857 mlxcx_module_status_t *pstatus, mlxcx_module_error_type_t *perr) 1858 { 1859 mlxcx_register_data_t data; 1860 boolean_t ret; 1861 1862 bzero(&data, sizeof (data)); 1863 ASSERT3U(id, <, 0xff); 1864 data.mlrd_pmaos.mlrd_pmaos_module = (uint8_t)id; 1865 1866 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_READ, 1867 MLXCX_REG_PMAOS, &data); 1868 1869 if (ret) { 1870 if (pstatus != NULL) 1871 *pstatus = data.mlrd_pmaos.mlrd_pmaos_oper_status; 1872 if (perr != NULL) 1873 *perr = data.mlrd_pmaos.mlrd_pmaos_error_type; 1874 } 1875 1876 return (ret); 1877 } 1878 1879 boolean_t 1880 mlxcx_cmd_set_port_mtu(mlxcx_t *mlxp, mlxcx_port_t *mlp) 1881 { 1882 mlxcx_register_data_t data; 1883 boolean_t ret; 1884 1885 ASSERT(mutex_owned(&mlp->mlp_mtx)); 1886 bzero(&data, sizeof (data)); 1887 data.mlrd_pmtu.mlrd_pmtu_local_port = mlp->mlp_num + 1; 1888 data.mlrd_pmtu.mlrd_pmtu_admin_mtu = to_be16(mlp->mlp_mtu); 1889 1890 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_WRITE, 1891 MLXCX_REG_PMTU, &data); 1892 1893 return (ret); 1894 } 1895 1896 boolean_t 1897 mlxcx_cmd_set_port_led(mlxcx_t *mlxp, mlxcx_port_t *mlp, uint16_t sec) 1898 { 1899 mlxcx_register_data_t data; 1900 boolean_t ret; 1901 1902 ASSERT(mutex_owned(&mlp->mlp_mtx)); 1903 bzero(&data, sizeof (data)); 1904 data.mlrd_mlcr.mlrd_mlcr_local_port = mlp->mlp_num + 1; 1905 set_bits8(&data.mlrd_mlcr.mlrd_mlcr_flags, MLXCX_MLCR_LED_TYPE, 1906 MLXCX_LED_TYPE_PORT); 1907 data.mlrd_mlcr.mlrd_mlcr_beacon_duration = to_be16(sec); 1908 1909 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_WRITE, 1910 MLXCX_REG_MLCR, &data); 1911 1912 return (ret); 1913 } 1914 1915 boolean_t 1916 mlxcx_cmd_query_port_status(mlxcx_t *mlxp, mlxcx_port_t *mlp) 1917 { 1918 mlxcx_register_data_t data; 1919 boolean_t ret; 1920 1921 ASSERT(mutex_owned(&mlp->mlp_mtx)); 1922 bzero(&data, sizeof (data)); 1923 data.mlrd_paos.mlrd_paos_local_port = mlp->mlp_num + 1; 1924 1925 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_READ, 1926 MLXCX_REG_PAOS, &data); 1927 1928 if (ret) { 1929 mlp->mlp_admin_status = data.mlrd_paos.mlrd_paos_admin_status; 1930 mlp->mlp_oper_status = data.mlrd_paos.mlrd_paos_oper_status; 1931 } 1932 1933 return (ret); 1934 } 1935 1936 boolean_t 1937 mlxcx_cmd_modify_port_status(mlxcx_t *mlxp, mlxcx_port_t *mlp, 1938 mlxcx_port_status_t status) 1939 { 1940 mlxcx_register_data_t data; 1941 boolean_t ret; 1942 1943 ASSERT(mutex_owned(&mlp->mlp_mtx)); 1944 bzero(&data, sizeof (data)); 1945 data.mlrd_paos.mlrd_paos_local_port = mlp->mlp_num + 1; 1946 data.mlrd_paos.mlrd_paos_admin_status = status; 1947 set_bit32(&data.mlrd_paos.mlrd_paos_flags, MLXCX_PAOS_ADMIN_ST_EN); 1948 1949 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_WRITE, 1950 MLXCX_REG_PAOS, &data); 1951 1952 return (ret); 1953 } 1954 1955 boolean_t 1956 mlxcx_cmd_query_port_speed(mlxcx_t *mlxp, mlxcx_port_t *mlp) 1957 { 1958 mlxcx_register_data_t data; 1959 boolean_t ret; 1960 1961 ASSERT(mutex_owned(&mlp->mlp_mtx)); 1962 bzero(&data, sizeof (data)); 1963 data.mlrd_ptys.mlrd_ptys_local_port = mlp->mlp_num + 1; 1964 set_bit8(&data.mlrd_ptys.mlrd_ptys_proto_mask, 1965 MLXCX_PTYS_PROTO_MASK_ETH); 1966 1967 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_READ, 1968 MLXCX_REG_PTYS, &data); 1969 1970 if (ret) { 1971 if (get_bit8(data.mlrd_ptys.mlrd_ptys_autoneg_flags, 1972 MLXCX_AUTONEG_DISABLE)) { 1973 mlp->mlp_autoneg = B_FALSE; 1974 } else { 1975 mlp->mlp_autoneg = B_TRUE; 1976 } 1977 mlp->mlp_max_proto = 1978 from_bits32(data.mlrd_ptys.mlrd_ptys_proto_cap); 1979 mlp->mlp_admin_proto = 1980 from_bits32(data.mlrd_ptys.mlrd_ptys_proto_admin); 1981 mlp->mlp_oper_proto = 1982 from_bits32(data.mlrd_ptys.mlrd_ptys_proto_oper); 1983 if (mlxp->mlx_caps->mlc_ext_ptys) { 1984 /* 1985 * Populate these bits only if we know the HW 1986 * supports them. Otherwise keep them zeroed 1987 * per the above bzero() and use that zero-ness to 1988 * skip over them as need be. 1989 */ 1990 mlp->mlp_ext_max_proto = from_bits32( 1991 data.mlrd_ptys.mlrd_ptys_ext_proto_cap); 1992 mlp->mlp_ext_admin_proto = from_bits32( 1993 data.mlrd_ptys.mlrd_ptys_ext_proto_admin); 1994 mlp->mlp_ext_oper_proto = from_bits32( 1995 data.mlrd_ptys.mlrd_ptys_ext_proto_oper); 1996 } 1997 } 1998 1999 return (ret); 2000 } 2001 2002 boolean_t 2003 mlxcx_cmd_query_port_fec(mlxcx_t *mlxp, mlxcx_port_t *mlp) 2004 { 2005 mlxcx_register_data_t data; 2006 boolean_t ret; 2007 2008 ASSERT(mutex_owned(&mlp->mlp_mtx)); 2009 bzero(&data, sizeof (data)); 2010 data.mlrd_pplm.mlrd_pplm_local_port = mlp->mlp_num + 1; 2011 2012 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_READ, 2013 MLXCX_REG_PPLM, &data); 2014 2015 if (ret) { 2016 mlp->mlp_fec_active = 2017 from_be24(data.mlrd_pplm.mlrd_pplm_fec_mode_active); 2018 } 2019 2020 return (ret); 2021 } 2022 2023 boolean_t 2024 mlxcx_cmd_modify_port_fec(mlxcx_t *mlxp, mlxcx_port_t *mlp, 2025 mlxcx_pplm_fec_caps_t fec) 2026 { 2027 mlxcx_register_data_t data_in, data_out; 2028 mlxcx_pplm_fec_caps_t caps; 2029 mlxcx_reg_pplm_t *pplm_in, *pplm_out; 2030 boolean_t ret; 2031 2032 ASSERT(mutex_owned(&mlp->mlp_mtx)); 2033 bzero(&data_in, sizeof (data_in)); 2034 pplm_in = &data_in.mlrd_pplm; 2035 pplm_in->mlrd_pplm_local_port = mlp->mlp_num + 1; 2036 2037 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_READ, 2038 MLXCX_REG_PPLM, &data_in); 2039 2040 if (!ret) 2041 return (B_FALSE); 2042 2043 bzero(&data_out, sizeof (data_out)); 2044 pplm_out = &data_out.mlrd_pplm; 2045 pplm_out->mlrd_pplm_local_port = mlp->mlp_num + 1; 2046 2047 caps = get_bits32(pplm_in->mlrd_pplm_fec_override_cap, 2048 MLXCX_PPLM_CAP_56G); 2049 set_bits32(&pplm_out->mlrd_pplm_fec_override_admin, 2050 MLXCX_PPLM_CAP_56G, fec & caps); 2051 2052 caps = get_bits32(pplm_in->mlrd_pplm_fec_override_cap, 2053 MLXCX_PPLM_CAP_100G); 2054 set_bits32(&pplm_out->mlrd_pplm_fec_override_admin, 2055 MLXCX_PPLM_CAP_100G, fec & caps); 2056 2057 caps = get_bits32(pplm_in->mlrd_pplm_fec_override_cap, 2058 MLXCX_PPLM_CAP_50G); 2059 set_bits32(&pplm_out->mlrd_pplm_fec_override_admin, 2060 MLXCX_PPLM_CAP_50G, fec & caps); 2061 2062 caps = get_bits32(pplm_in->mlrd_pplm_fec_override_cap, 2063 MLXCX_PPLM_CAP_25G); 2064 set_bits32(&pplm_out->mlrd_pplm_fec_override_admin, 2065 MLXCX_PPLM_CAP_25G, fec & caps); 2066 2067 caps = get_bits32(pplm_in->mlrd_pplm_fec_override_cap, 2068 MLXCX_PPLM_CAP_10_40G); 2069 set_bits32(&pplm_out->mlrd_pplm_fec_override_admin, 2070 MLXCX_PPLM_CAP_10_40G, fec & caps); 2071 2072 ret = mlxcx_cmd_access_register(mlxp, MLXCX_CMD_ACCESS_REGISTER_WRITE, 2073 MLXCX_REG_PPLM, &data_out); 2074 2075 return (ret); 2076 } 2077 2078 boolean_t 2079 mlxcx_cmd_modify_nic_vport_ctx(mlxcx_t *mlxp, mlxcx_port_t *mlp, 2080 mlxcx_modify_nic_vport_ctx_fields_t fields) 2081 { 2082 mlxcx_cmd_t cmd; 2083 mlxcx_cmd_modify_nic_vport_ctx_in_t in; 2084 mlxcx_cmd_modify_nic_vport_ctx_out_t out; 2085 boolean_t ret; 2086 mlxcx_nic_vport_ctx_t *ctx; 2087 2088 ASSERT(mutex_owned(&mlp->mlp_mtx)); 2089 bzero(&in, sizeof (in)); 2090 bzero(&out, sizeof (out)); 2091 2092 mlxcx_cmd_init(mlxp, &cmd); 2093 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_modify_nic_vport_ctx_head, 2094 MLXCX_OP_MODIFY_NIC_VPORT_CONTEXT, MLXCX_VPORT_TYPE_VNIC); 2095 2096 in.mlxi_modify_nic_vport_ctx_vport_number = to_be16(mlp->mlp_num); 2097 in.mlxi_modify_nic_vport_ctx_field_select = to_be32(fields); 2098 2099 ctx = &in.mlxi_modify_nic_vport_ctx_context; 2100 if (fields & MLXCX_MODIFY_NIC_VPORT_CTX_PROMISC) { 2101 set_bit16(&ctx->mlnvc_promisc_list_type, 2102 MLXCX_VPORT_PROMISC_ALL); 2103 } 2104 if (fields & MLXCX_MODIFY_NIC_VPORT_CTX_MTU) { 2105 ctx->mlnvc_mtu = to_be16(mlp->mlp_mtu); 2106 } 2107 2108 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2109 mlxcx_cmd_fini(mlxp, &cmd); 2110 return (B_FALSE); 2111 } 2112 mlxcx_cmd_wait(&cmd); 2113 2114 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2115 if (ret) { 2116 if (fields & MLXCX_MODIFY_NIC_VPORT_CTX_PROMISC) { 2117 mlp->mlp_flags |= MLXCX_PORT_VPORT_PROMISC; 2118 } 2119 } 2120 mlxcx_cmd_fini(mlxp, &cmd); 2121 return (ret); 2122 } 2123 2124 boolean_t 2125 mlxcx_cmd_create_eq(mlxcx_t *mlxp, mlxcx_event_queue_t *mleq) 2126 { 2127 mlxcx_cmd_t cmd; 2128 mlxcx_cmd_create_eq_in_t in; 2129 mlxcx_cmd_create_eq_out_t out; 2130 boolean_t ret; 2131 mlxcx_eventq_ctx_t *ctx; 2132 size_t rem, insize; 2133 const ddi_dma_cookie_t *c; 2134 uint64_t pa, npages; 2135 2136 bzero(&in, sizeof (in)); 2137 bzero(&out, sizeof (out)); 2138 2139 ASSERT(mutex_owned(&mleq->mleq_mtx)); 2140 VERIFY(mleq->mleq_state & MLXCX_EQ_ALLOC); 2141 VERIFY0(mleq->mleq_state & MLXCX_EQ_CREATED); 2142 2143 mlxcx_cmd_init(mlxp, &cmd); 2144 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_eq_head, 2145 MLXCX_OP_CREATE_EQ, 0); 2146 2147 ctx = &in.mlxi_create_eq_context; 2148 ctx->mleqc_uar_page = to_be24(mleq->mleq_uar->mlu_num); 2149 ctx->mleqc_log_eq_size = mleq->mleq_entshift; 2150 ctx->mleqc_intr = mleq->mleq_intr_index; 2151 2152 in.mlxi_create_eq_event_bitmask = to_be64(mleq->mleq_events); 2153 2154 npages = 0; 2155 c = NULL; 2156 while ((c = mlxcx_dma_cookie_iter(&mleq->mleq_dma, c)) != NULL) { 2157 pa = c->dmac_laddress; 2158 rem = c->dmac_size; 2159 while (rem > 0) { 2160 ASSERT3U(pa & 0xfff, ==, 0); 2161 ASSERT3U(rem, >=, MLXCX_HW_PAGE_SIZE); 2162 in.mlxi_create_eq_pas[npages++] = to_be64(pa); 2163 rem -= MLXCX_HW_PAGE_SIZE; 2164 pa += MLXCX_HW_PAGE_SIZE; 2165 } 2166 } 2167 ASSERT3U(npages, <=, MLXCX_CREATE_QUEUE_MAX_PAGES); 2168 2169 insize = offsetof(mlxcx_cmd_create_eq_in_t, mlxi_create_eq_pas) + 2170 sizeof (uint64_t) * npages; 2171 2172 if (!mlxcx_cmd_send(mlxp, &cmd, &in, insize, &out, sizeof (out))) { 2173 mlxcx_cmd_fini(mlxp, &cmd); 2174 return (B_FALSE); 2175 } 2176 mlxcx_cmd_wait(&cmd); 2177 2178 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2179 if (ret) { 2180 mleq->mleq_state |= MLXCX_EQ_CREATED; 2181 mleq->mleq_num = out.mlxo_create_eq_eqn; 2182 } 2183 mlxcx_cmd_fini(mlxp, &cmd); 2184 return (ret); 2185 } 2186 2187 boolean_t 2188 mlxcx_cmd_query_eq(mlxcx_t *mlxp, mlxcx_event_queue_t *mleq, 2189 mlxcx_eventq_ctx_t *ctxp) 2190 { 2191 mlxcx_cmd_t cmd; 2192 mlxcx_cmd_query_eq_in_t in; 2193 mlxcx_cmd_query_eq_out_t out; 2194 boolean_t ret; 2195 2196 bzero(&in, sizeof (in)); 2197 bzero(&out, sizeof (out)); 2198 2199 VERIFY(mleq->mleq_state & MLXCX_EQ_ALLOC); 2200 VERIFY(mleq->mleq_state & MLXCX_EQ_CREATED); 2201 2202 mlxcx_cmd_init(mlxp, &cmd); 2203 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_eq_head, 2204 MLXCX_OP_QUERY_EQ, 0); 2205 2206 in.mlxi_query_eq_eqn = mleq->mleq_num; 2207 2208 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2209 mlxcx_cmd_fini(mlxp, &cmd); 2210 return (B_FALSE); 2211 } 2212 mlxcx_cmd_wait(&cmd); 2213 2214 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2215 if (ret) { 2216 bcopy(&out.mlxo_query_eq_context, ctxp, 2217 sizeof (mlxcx_eventq_ctx_t)); 2218 } 2219 mlxcx_cmd_fini(mlxp, &cmd); 2220 return (ret); 2221 } 2222 2223 boolean_t 2224 mlxcx_cmd_destroy_eq(mlxcx_t *mlxp, mlxcx_event_queue_t *mleq) 2225 { 2226 mlxcx_cmd_t cmd; 2227 mlxcx_cmd_destroy_eq_in_t in; 2228 mlxcx_cmd_destroy_eq_out_t out; 2229 boolean_t ret; 2230 2231 bzero(&in, sizeof (in)); 2232 bzero(&out, sizeof (out)); 2233 2234 ASSERT(mutex_owned(&mleq->mleq_mtx)); 2235 VERIFY(mleq->mleq_state & MLXCX_EQ_ALLOC); 2236 VERIFY(mleq->mleq_state & MLXCX_EQ_CREATED); 2237 2238 mlxcx_cmd_init(mlxp, &cmd); 2239 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_eq_head, 2240 MLXCX_OP_DESTROY_EQ, 0); 2241 2242 in.mlxi_destroy_eq_eqn = mleq->mleq_num; 2243 2244 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2245 mlxcx_cmd_fini(mlxp, &cmd); 2246 return (B_FALSE); 2247 } 2248 mlxcx_cmd_wait(&cmd); 2249 2250 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2251 if (ret) { 2252 mleq->mleq_state |= MLXCX_EQ_DESTROYED; 2253 } 2254 mlxcx_cmd_fini(mlxp, &cmd); 2255 return (ret); 2256 } 2257 2258 boolean_t 2259 mlxcx_cmd_query_special_ctxs(mlxcx_t *mlxp) 2260 { 2261 mlxcx_cmd_t cmd; 2262 mlxcx_cmd_query_special_ctxs_in_t in; 2263 mlxcx_cmd_query_special_ctxs_out_t out; 2264 boolean_t ret; 2265 2266 bzero(&in, sizeof (in)); 2267 bzero(&out, sizeof (out)); 2268 2269 mlxcx_cmd_init(mlxp, &cmd); 2270 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_special_ctxs_head, 2271 MLXCX_OP_QUERY_SPECIAL_CONTEXTS, 0); 2272 2273 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2274 mlxcx_cmd_fini(mlxp, &cmd); 2275 return (B_FALSE); 2276 } 2277 mlxcx_cmd_wait(&cmd); 2278 2279 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2280 if (ret) { 2281 mlxp->mlx_rsvd_lkey = from_be32( 2282 out.mlxo_query_special_ctxs_resd_lkey); 2283 } 2284 mlxcx_cmd_fini(mlxp, &cmd); 2285 return (ret); 2286 } 2287 2288 boolean_t 2289 mlxcx_cmd_create_cq(mlxcx_t *mlxp, mlxcx_completion_queue_t *mlcq) 2290 { 2291 mlxcx_cmd_t cmd; 2292 mlxcx_cmd_create_cq_in_t in; 2293 mlxcx_cmd_create_cq_out_t out; 2294 boolean_t ret; 2295 mlxcx_completionq_ctx_t *ctx; 2296 size_t rem, insize; 2297 const ddi_dma_cookie_t *c; 2298 uint64_t pa, npages; 2299 2300 bzero(&in, sizeof (in)); 2301 bzero(&out, sizeof (out)); 2302 2303 ASSERT(mutex_owned(&mlcq->mlcq_mtx)); 2304 VERIFY(mlcq->mlcq_state & MLXCX_CQ_ALLOC); 2305 VERIFY0(mlcq->mlcq_state & MLXCX_CQ_CREATED); 2306 2307 mlxcx_cmd_init(mlxp, &cmd); 2308 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_cq_head, 2309 MLXCX_OP_CREATE_CQ, 0); 2310 2311 ctx = &in.mlxi_create_cq_context; 2312 ctx->mlcqc_uar_page = to_be24(mlcq->mlcq_uar->mlu_num); 2313 ctx->mlcqc_log_cq_size = mlcq->mlcq_entshift; 2314 ctx->mlcqc_eqn = mlcq->mlcq_eq->mleq_num; 2315 ctx->mlcqc_cq_period = to_be16(mlcq->mlcq_cqemod_period_usec); 2316 ctx->mlcqc_cq_max_count = to_be16(mlcq->mlcq_cqemod_count); 2317 2318 c = mlxcx_dma_cookie_one(&mlcq->mlcq_doorbell_dma); 2319 ctx->mlcqc_dbr_addr = to_be64(c->dmac_laddress); 2320 ASSERT3U(c->dmac_size, >=, sizeof (mlxcx_completionq_doorbell_t)); 2321 2322 npages = 0; 2323 c = NULL; 2324 while ((c = mlxcx_dma_cookie_iter(&mlcq->mlcq_dma, c)) != NULL) { 2325 pa = c->dmac_laddress; 2326 rem = c->dmac_size; 2327 while (rem > 0) { 2328 ASSERT3U(pa & 0xfff, ==, 0); 2329 ASSERT3U(rem, >=, MLXCX_HW_PAGE_SIZE); 2330 in.mlxi_create_cq_pas[npages++] = to_be64(pa); 2331 rem -= MLXCX_HW_PAGE_SIZE; 2332 pa += MLXCX_HW_PAGE_SIZE; 2333 } 2334 } 2335 ASSERT3U(npages, <=, MLXCX_CREATE_QUEUE_MAX_PAGES); 2336 2337 insize = offsetof(mlxcx_cmd_create_cq_in_t, mlxi_create_cq_pas) + 2338 sizeof (uint64_t) * npages; 2339 2340 if (!mlxcx_cmd_send(mlxp, &cmd, &in, insize, &out, sizeof (out))) { 2341 mlxcx_cmd_fini(mlxp, &cmd); 2342 return (B_FALSE); 2343 } 2344 mlxcx_cmd_wait(&cmd); 2345 2346 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2347 if (ret) { 2348 atomic_or_uint(&mlcq->mlcq_state, MLXCX_CQ_CREATED); 2349 mlcq->mlcq_num = from_be24(out.mlxo_create_cq_cqn); 2350 } 2351 mlxcx_cmd_fini(mlxp, &cmd); 2352 return (ret); 2353 } 2354 2355 boolean_t 2356 mlxcx_cmd_query_rq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq, 2357 mlxcx_rq_ctx_t *ctxp) 2358 { 2359 mlxcx_cmd_t cmd; 2360 mlxcx_cmd_query_rq_in_t in; 2361 mlxcx_cmd_query_rq_out_t out; 2362 boolean_t ret; 2363 2364 bzero(&in, sizeof (in)); 2365 bzero(&out, sizeof (out)); 2366 2367 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 2368 VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED); 2369 ASSERT3S(mlwq->mlwq_type, ==, MLXCX_WQ_TYPE_RECVQ); 2370 2371 mlxcx_cmd_init(mlxp, &cmd); 2372 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_rq_head, 2373 MLXCX_OP_QUERY_RQ, 0); 2374 2375 in.mlxi_query_rq_rqn = to_be24(mlwq->mlwq_num); 2376 2377 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2378 mlxcx_cmd_fini(mlxp, &cmd); 2379 return (B_FALSE); 2380 } 2381 mlxcx_cmd_wait(&cmd); 2382 2383 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2384 if (ret) { 2385 bcopy(&out.mlxo_query_rq_context, ctxp, 2386 sizeof (mlxcx_rq_ctx_t)); 2387 } 2388 mlxcx_cmd_fini(mlxp, &cmd); 2389 return (ret); 2390 } 2391 2392 boolean_t 2393 mlxcx_cmd_query_sq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq, 2394 mlxcx_sq_ctx_t *ctxp) 2395 { 2396 mlxcx_cmd_t cmd; 2397 mlxcx_cmd_query_sq_in_t in; 2398 mlxcx_cmd_query_sq_out_t out; 2399 boolean_t ret; 2400 2401 bzero(&in, sizeof (in)); 2402 bzero(&out, sizeof (out)); 2403 2404 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 2405 VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED); 2406 ASSERT3S(mlwq->mlwq_type, ==, MLXCX_WQ_TYPE_SENDQ); 2407 2408 mlxcx_cmd_init(mlxp, &cmd); 2409 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_sq_head, 2410 MLXCX_OP_QUERY_SQ, 0); 2411 2412 in.mlxi_query_sq_sqn = to_be24(mlwq->mlwq_num); 2413 2414 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2415 mlxcx_cmd_fini(mlxp, &cmd); 2416 return (B_FALSE); 2417 } 2418 mlxcx_cmd_wait(&cmd); 2419 2420 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2421 if (ret) { 2422 bcopy(&out.mlxo_query_sq_context, ctxp, 2423 sizeof (mlxcx_sq_ctx_t)); 2424 } 2425 mlxcx_cmd_fini(mlxp, &cmd); 2426 return (ret); 2427 } 2428 2429 boolean_t 2430 mlxcx_cmd_query_cq(mlxcx_t *mlxp, mlxcx_completion_queue_t *mlcq, 2431 mlxcx_completionq_ctx_t *ctxp) 2432 { 2433 mlxcx_cmd_t cmd; 2434 mlxcx_cmd_query_cq_in_t in; 2435 mlxcx_cmd_query_cq_out_t out; 2436 boolean_t ret; 2437 2438 bzero(&in, sizeof (in)); 2439 bzero(&out, sizeof (out)); 2440 2441 VERIFY(mlcq->mlcq_state & MLXCX_CQ_ALLOC); 2442 VERIFY(mlcq->mlcq_state & MLXCX_CQ_CREATED); 2443 2444 mlxcx_cmd_init(mlxp, &cmd); 2445 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_query_cq_head, 2446 MLXCX_OP_QUERY_CQ, 0); 2447 2448 in.mlxi_query_cq_cqn = to_be24(mlcq->mlcq_num); 2449 2450 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2451 mlxcx_cmd_fini(mlxp, &cmd); 2452 return (B_FALSE); 2453 } 2454 mlxcx_cmd_wait(&cmd); 2455 2456 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2457 if (ret) { 2458 bcopy(&out.mlxo_query_cq_context, ctxp, 2459 sizeof (mlxcx_completionq_ctx_t)); 2460 } 2461 mlxcx_cmd_fini(mlxp, &cmd); 2462 return (ret); 2463 } 2464 2465 boolean_t 2466 mlxcx_cmd_destroy_cq(mlxcx_t *mlxp, mlxcx_completion_queue_t *mlcq) 2467 { 2468 mlxcx_cmd_t cmd; 2469 mlxcx_cmd_destroy_cq_in_t in; 2470 mlxcx_cmd_destroy_cq_out_t out; 2471 boolean_t ret; 2472 2473 bzero(&in, sizeof (in)); 2474 bzero(&out, sizeof (out)); 2475 2476 ASSERT(mutex_owned(&mlcq->mlcq_mtx)); 2477 VERIFY(mlcq->mlcq_state & MLXCX_CQ_ALLOC); 2478 VERIFY(mlcq->mlcq_state & MLXCX_CQ_CREATED); 2479 2480 mlxcx_cmd_init(mlxp, &cmd); 2481 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_cq_head, 2482 MLXCX_OP_DESTROY_CQ, 0); 2483 2484 in.mlxi_destroy_cq_cqn = to_be24(mlcq->mlcq_num); 2485 2486 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2487 mlxcx_cmd_fini(mlxp, &cmd); 2488 return (B_FALSE); 2489 } 2490 mlxcx_cmd_wait(&cmd); 2491 2492 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2493 if (ret) { 2494 atomic_or_uint(&mlcq->mlcq_state, MLXCX_CQ_DESTROYED); 2495 } 2496 mlxcx_cmd_fini(mlxp, &cmd); 2497 return (ret); 2498 } 2499 2500 boolean_t 2501 mlxcx_cmd_create_rq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq) 2502 { 2503 mlxcx_cmd_t cmd; 2504 mlxcx_cmd_create_rq_in_t in; 2505 mlxcx_cmd_create_rq_out_t out; 2506 boolean_t ret; 2507 mlxcx_rq_ctx_t *ctx; 2508 size_t rem, insize; 2509 const ddi_dma_cookie_t *c; 2510 uint64_t pa, npages; 2511 2512 bzero(&in, sizeof (in)); 2513 bzero(&out, sizeof (out)); 2514 2515 ASSERT(mutex_owned(&mlwq->mlwq_mtx)); 2516 VERIFY3U(mlwq->mlwq_type, ==, MLXCX_WQ_TYPE_RECVQ); 2517 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 2518 VERIFY0(mlwq->mlwq_state & MLXCX_WQ_CREATED); 2519 2520 mlxcx_cmd_init(mlxp, &cmd); 2521 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_rq_head, 2522 MLXCX_OP_CREATE_RQ, 0); 2523 2524 ctx = &in.mlxi_create_rq_context; 2525 2526 set_bit32(&ctx->mlrqc_flags, MLXCX_RQ_FLAGS_RLKEY); 2527 set_bit32(&ctx->mlrqc_flags, MLXCX_RQ_FLAGS_FLUSH_IN_ERROR); 2528 set_bit32(&ctx->mlrqc_flags, MLXCX_RQ_FLAGS_VLAN_STRIP_DISABLE); 2529 ctx->mlrqc_cqn = to_be24(mlwq->mlwq_cq->mlcq_num); 2530 2531 set_bits32(&ctx->mlrqc_wq.mlwqc_flags, MLXCX_WORKQ_CTX_TYPE, 2532 MLXCX_WORKQ_TYPE_CYCLIC); 2533 ctx->mlrqc_wq.mlwqc_pd = to_be24(mlwq->mlwq_pd->mlpd_num); 2534 ctx->mlrqc_wq.mlwqc_log_wq_sz = mlwq->mlwq_entshift; 2535 ctx->mlrqc_wq.mlwqc_log_wq_stride = MLXCX_RECVQ_STRIDE_SHIFT; 2536 2537 c = mlxcx_dma_cookie_one(&mlwq->mlwq_doorbell_dma); 2538 ctx->mlrqc_wq.mlwqc_dbr_addr = to_be64(c->dmac_laddress); 2539 ASSERT3U(c->dmac_size, >=, sizeof (mlxcx_workq_doorbell_t)); 2540 2541 npages = 0; 2542 c = NULL; 2543 while ((c = mlxcx_dma_cookie_iter(&mlwq->mlwq_dma, c)) != NULL) { 2544 pa = c->dmac_laddress; 2545 rem = c->dmac_size; 2546 while (rem > 0) { 2547 ASSERT3U(pa & 0xfff, ==, 0); 2548 ASSERT3U(rem, >=, MLXCX_HW_PAGE_SIZE); 2549 ctx->mlrqc_wq.mlwqc_pas[npages++] = to_be64(pa); 2550 rem -= MLXCX_HW_PAGE_SIZE; 2551 pa += MLXCX_HW_PAGE_SIZE; 2552 } 2553 } 2554 ASSERT3U(npages, <=, MLXCX_WORKQ_CTX_MAX_ADDRESSES); 2555 2556 insize = offsetof(mlxcx_cmd_create_rq_in_t, mlxi_create_rq_context) + 2557 offsetof(mlxcx_rq_ctx_t, mlrqc_wq) + 2558 offsetof(mlxcx_workq_ctx_t, mlwqc_pas) + 2559 sizeof (uint64_t) * npages; 2560 2561 if (!mlxcx_cmd_send(mlxp, &cmd, &in, insize, &out, sizeof (out))) { 2562 mlxcx_cmd_fini(mlxp, &cmd); 2563 return (B_FALSE); 2564 } 2565 mlxcx_cmd_wait(&cmd); 2566 2567 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2568 if (ret) { 2569 mlwq->mlwq_state |= MLXCX_WQ_CREATED; 2570 mlwq->mlwq_num = from_be24(out.mlxo_create_rq_rqn); 2571 } 2572 mlxcx_cmd_fini(mlxp, &cmd); 2573 return (ret); 2574 } 2575 2576 boolean_t 2577 mlxcx_cmd_start_rq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq) 2578 { 2579 mlxcx_cmd_t cmd; 2580 mlxcx_cmd_modify_rq_in_t in; 2581 mlxcx_cmd_modify_rq_out_t out; 2582 boolean_t ret; 2583 ddi_fm_error_t err; 2584 2585 bzero(&in, sizeof (in)); 2586 bzero(&out, sizeof (out)); 2587 2588 ASSERT(mutex_owned(&mlwq->mlwq_mtx)); 2589 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 2590 VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED); 2591 VERIFY0(mlwq->mlwq_state & MLXCX_WQ_STARTED); 2592 2593 /* 2594 * Before starting the queue, we have to be sure that it is 2595 * empty and the doorbell and counters are set to 0. 2596 */ 2597 ASSERT(mutex_owned(&mlwq->mlwq_cq->mlcq_mtx)); 2598 ASSERT(list_is_empty(&mlwq->mlwq_cq->mlcq_buffers)); 2599 ASSERT(list_is_empty(&mlwq->mlwq_cq->mlcq_buffers_b)); 2600 2601 mlwq->mlwq_doorbell->mlwqd_recv_counter = to_be16(0); 2602 MLXCX_DMA_SYNC(mlwq->mlwq_doorbell_dma, DDI_DMA_SYNC_FORDEV); 2603 ddi_fm_dma_err_get(mlwq->mlwq_doorbell_dma.mxdb_dma_handle, &err, 2604 DDI_FME_VERSION); 2605 if (err.fme_status != DDI_FM_OK) 2606 return (B_FALSE); 2607 mlwq->mlwq_pc = 0; 2608 2609 mlxcx_cmd_init(mlxp, &cmd); 2610 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_modify_rq_head, 2611 MLXCX_OP_MODIFY_RQ, 0); 2612 2613 in.mlxi_modify_rq_rqn = to_be24(mlwq->mlwq_num); 2614 2615 /* From state */ 2616 set_bits8(&in.mlxi_modify_rq_state, MLXCX_CMD_MODIFY_RQ_STATE, 2617 MLXCX_RQ_STATE_RST); 2618 /* To state */ 2619 set_bits32(&in.mlxi_modify_rq_context.mlrqc_flags, MLXCX_RQ_STATE, 2620 MLXCX_RQ_STATE_RDY); 2621 2622 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2623 mlxcx_cmd_fini(mlxp, &cmd); 2624 return (B_FALSE); 2625 } 2626 mlxcx_cmd_wait(&cmd); 2627 2628 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2629 if (ret) { 2630 mlwq->mlwq_state |= MLXCX_WQ_STARTED; 2631 } 2632 mlxcx_cmd_fini(mlxp, &cmd); 2633 return (ret); 2634 } 2635 2636 boolean_t 2637 mlxcx_cmd_stop_rq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq) 2638 { 2639 mlxcx_cmd_t cmd; 2640 mlxcx_cmd_modify_rq_in_t in; 2641 mlxcx_cmd_modify_rq_out_t out; 2642 boolean_t ret; 2643 2644 bzero(&in, sizeof (in)); 2645 bzero(&out, sizeof (out)); 2646 2647 ASSERT(mutex_owned(&mlwq->mlwq_mtx)); 2648 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 2649 VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED); 2650 VERIFY(mlwq->mlwq_state & MLXCX_WQ_STARTED); 2651 2652 mlxcx_cmd_init(mlxp, &cmd); 2653 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_modify_rq_head, 2654 MLXCX_OP_MODIFY_RQ, 0); 2655 2656 in.mlxi_modify_rq_rqn = to_be24(mlwq->mlwq_num); 2657 2658 /* From state */ 2659 set_bits8(&in.mlxi_modify_rq_state, MLXCX_CMD_MODIFY_RQ_STATE, 2660 MLXCX_RQ_STATE_RDY); 2661 /* To state */ 2662 set_bits32(&in.mlxi_modify_rq_context.mlrqc_flags, MLXCX_RQ_STATE, 2663 MLXCX_RQ_STATE_RST); 2664 2665 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2666 mlxcx_cmd_fini(mlxp, &cmd); 2667 return (B_FALSE); 2668 } 2669 mlxcx_cmd_wait(&cmd); 2670 2671 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2672 if (ret) { 2673 mlwq->mlwq_state &= ~MLXCX_WQ_STARTED; 2674 } 2675 mlxcx_cmd_fini(mlxp, &cmd); 2676 return (ret); 2677 } 2678 2679 boolean_t 2680 mlxcx_cmd_destroy_rq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq) 2681 { 2682 mlxcx_cmd_t cmd; 2683 mlxcx_cmd_destroy_rq_in_t in; 2684 mlxcx_cmd_destroy_rq_out_t out; 2685 boolean_t ret; 2686 2687 bzero(&in, sizeof (in)); 2688 bzero(&out, sizeof (out)); 2689 2690 ASSERT(mutex_owned(&mlwq->mlwq_mtx)); 2691 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 2692 VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED); 2693 VERIFY0(mlwq->mlwq_state & MLXCX_WQ_STARTED); 2694 2695 mlxcx_cmd_init(mlxp, &cmd); 2696 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_rq_head, 2697 MLXCX_OP_DESTROY_RQ, 0); 2698 2699 in.mlxi_destroy_rq_rqn = to_be24(mlwq->mlwq_num); 2700 2701 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2702 mlxcx_cmd_fini(mlxp, &cmd); 2703 return (B_FALSE); 2704 } 2705 mlxcx_cmd_wait(&cmd); 2706 2707 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2708 if (ret) { 2709 mlwq->mlwq_state |= MLXCX_WQ_DESTROYED; 2710 } 2711 mlxcx_cmd_fini(mlxp, &cmd); 2712 return (ret); 2713 } 2714 2715 boolean_t 2716 mlxcx_cmd_create_tir(mlxcx_t *mlxp, mlxcx_tir_t *mltir) 2717 { 2718 mlxcx_cmd_t cmd; 2719 mlxcx_cmd_create_tir_in_t in; 2720 mlxcx_cmd_create_tir_out_t out; 2721 mlxcx_tir_ctx_t *ctx; 2722 boolean_t ret; 2723 2724 bzero(&in, sizeof (in)); 2725 bzero(&out, sizeof (out)); 2726 2727 VERIFY0(mltir->mltir_state & MLXCX_TIR_CREATED); 2728 2729 mlxcx_cmd_init(mlxp, &cmd); 2730 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_tir_head, 2731 MLXCX_OP_CREATE_TIR, 0); 2732 2733 ctx = &in.mlxi_create_tir_context; 2734 ctx->mltirc_transport_domain = to_be24(mltir->mltir_tdom->mltd_num); 2735 set_bits8(&ctx->mltirc_disp_type, MLXCX_TIR_CTX_DISP_TYPE, 2736 mltir->mltir_type); 2737 switch (mltir->mltir_type) { 2738 case MLXCX_TIR_INDIRECT: 2739 VERIFY(mltir->mltir_rqtable != NULL); 2740 VERIFY(mltir->mltir_rqtable->mlrqt_state & MLXCX_RQT_CREATED); 2741 ctx->mltirc_indirect_table = 2742 to_be24(mltir->mltir_rqtable->mlrqt_num); 2743 set_bits8(&ctx->mltirc_hash_lb, MLXCX_TIR_RX_HASH_FN, 2744 mltir->mltir_hash_fn); 2745 bcopy(mltir->mltir_toeplitz_key, 2746 ctx->mltirc_rx_hash_toeplitz_key, 2747 sizeof (ctx->mltirc_rx_hash_toeplitz_key)); 2748 set_bits32(&ctx->mltirc_rx_hash_fields_outer, 2749 MLXCX_RX_HASH_L3_TYPE, mltir->mltir_l3_type); 2750 set_bits32(&ctx->mltirc_rx_hash_fields_outer, 2751 MLXCX_RX_HASH_L4_TYPE, mltir->mltir_l4_type); 2752 set_bits32(&ctx->mltirc_rx_hash_fields_outer, 2753 MLXCX_RX_HASH_FIELDS, mltir->mltir_hash_fields); 2754 break; 2755 case MLXCX_TIR_DIRECT: 2756 VERIFY(mltir->mltir_rq != NULL); 2757 VERIFY(mltir->mltir_rq->mlwq_state & MLXCX_WQ_CREATED); 2758 ctx->mltirc_inline_rqn = to_be24(mltir->mltir_rq->mlwq_num); 2759 break; 2760 default: 2761 VERIFY(0); 2762 } 2763 2764 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2765 mlxcx_cmd_fini(mlxp, &cmd); 2766 return (B_FALSE); 2767 } 2768 mlxcx_cmd_wait(&cmd); 2769 2770 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2771 if (ret) { 2772 mltir->mltir_state |= MLXCX_TIR_CREATED; 2773 mltir->mltir_num = from_be24(out.mlxo_create_tir_tirn); 2774 } 2775 mlxcx_cmd_fini(mlxp, &cmd); 2776 return (ret); 2777 } 2778 2779 boolean_t 2780 mlxcx_cmd_destroy_tir(mlxcx_t *mlxp, mlxcx_tir_t *mltir) 2781 { 2782 mlxcx_cmd_t cmd; 2783 mlxcx_cmd_destroy_tir_in_t in; 2784 mlxcx_cmd_destroy_tir_out_t out; 2785 boolean_t ret; 2786 2787 bzero(&in, sizeof (in)); 2788 bzero(&out, sizeof (out)); 2789 2790 VERIFY(mltir->mltir_state & MLXCX_TIR_CREATED); 2791 VERIFY0(mltir->mltir_state & MLXCX_TIR_DESTROYED); 2792 2793 mlxcx_cmd_init(mlxp, &cmd); 2794 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_tir_head, 2795 MLXCX_OP_DESTROY_TIR, 0); 2796 2797 in.mlxi_destroy_tir_tirn = to_be24(mltir->mltir_num); 2798 2799 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2800 mlxcx_cmd_fini(mlxp, &cmd); 2801 return (B_FALSE); 2802 } 2803 mlxcx_cmd_wait(&cmd); 2804 2805 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2806 if (ret) { 2807 mltir->mltir_state |= MLXCX_TIR_DESTROYED; 2808 } 2809 mlxcx_cmd_fini(mlxp, &cmd); 2810 return (ret); 2811 } 2812 2813 boolean_t 2814 mlxcx_cmd_create_tis(mlxcx_t *mlxp, mlxcx_tis_t *mltis) 2815 { 2816 mlxcx_cmd_t cmd; 2817 mlxcx_cmd_create_tis_in_t in; 2818 mlxcx_cmd_create_tis_out_t out; 2819 mlxcx_tis_ctx_t *ctx; 2820 boolean_t ret; 2821 2822 bzero(&in, sizeof (in)); 2823 bzero(&out, sizeof (out)); 2824 2825 VERIFY0(mltis->mltis_state & MLXCX_TIS_CREATED); 2826 2827 mlxcx_cmd_init(mlxp, &cmd); 2828 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_tis_head, 2829 MLXCX_OP_CREATE_TIS, 0); 2830 2831 ctx = &in.mlxi_create_tis_context; 2832 ctx->mltisc_transport_domain = to_be24(mltis->mltis_tdom->mltd_num); 2833 2834 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2835 mlxcx_cmd_fini(mlxp, &cmd); 2836 return (B_FALSE); 2837 } 2838 mlxcx_cmd_wait(&cmd); 2839 2840 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2841 if (ret) { 2842 mltis->mltis_state |= MLXCX_TIS_CREATED; 2843 mltis->mltis_num = from_be24(out.mlxo_create_tis_tisn); 2844 } 2845 mlxcx_cmd_fini(mlxp, &cmd); 2846 return (ret); 2847 } 2848 2849 boolean_t 2850 mlxcx_cmd_destroy_tis(mlxcx_t *mlxp, mlxcx_tis_t *mltis) 2851 { 2852 mlxcx_cmd_t cmd; 2853 mlxcx_cmd_destroy_tis_in_t in; 2854 mlxcx_cmd_destroy_tis_out_t out; 2855 boolean_t ret; 2856 2857 bzero(&in, sizeof (in)); 2858 bzero(&out, sizeof (out)); 2859 2860 VERIFY(mltis->mltis_state & MLXCX_TIR_CREATED); 2861 VERIFY0(mltis->mltis_state & MLXCX_TIR_DESTROYED); 2862 2863 mlxcx_cmd_init(mlxp, &cmd); 2864 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_tis_head, 2865 MLXCX_OP_DESTROY_TIS, 0); 2866 2867 in.mlxi_destroy_tis_tisn = to_be24(mltis->mltis_num); 2868 2869 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2870 mlxcx_cmd_fini(mlxp, &cmd); 2871 return (B_FALSE); 2872 } 2873 mlxcx_cmd_wait(&cmd); 2874 2875 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2876 if (ret) { 2877 mltis->mltis_state |= MLXCX_TIS_DESTROYED; 2878 } 2879 mlxcx_cmd_fini(mlxp, &cmd); 2880 return (ret); 2881 } 2882 2883 boolean_t 2884 mlxcx_cmd_create_flow_table(mlxcx_t *mlxp, mlxcx_flow_table_t *mlft) 2885 { 2886 mlxcx_cmd_t cmd; 2887 mlxcx_cmd_create_flow_table_in_t in; 2888 mlxcx_cmd_create_flow_table_out_t out; 2889 mlxcx_flow_table_ctx_t *ctx; 2890 boolean_t ret; 2891 2892 bzero(&in, sizeof (in)); 2893 bzero(&out, sizeof (out)); 2894 2895 ASSERT(mutex_owned(&mlft->mlft_mtx)); 2896 VERIFY0(mlft->mlft_state & MLXCX_FLOW_TABLE_CREATED); 2897 2898 mlxcx_cmd_init(mlxp, &cmd); 2899 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_flow_table_head, 2900 MLXCX_OP_CREATE_FLOW_TABLE, 0); 2901 2902 in.mlxi_create_flow_table_vport_number = 2903 to_be16(mlft->mlft_port->mlp_num); 2904 in.mlxi_create_flow_table_table_type = mlft->mlft_type; 2905 ctx = &in.mlxi_create_flow_table_context; 2906 ctx->mlftc_log_size = mlft->mlft_entshift; 2907 ctx->mlftc_level = mlft->mlft_level; 2908 2909 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2910 mlxcx_cmd_fini(mlxp, &cmd); 2911 return (B_FALSE); 2912 } 2913 mlxcx_cmd_wait(&cmd); 2914 2915 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2916 if (ret) { 2917 mlft->mlft_num = from_be24(out.mlxo_create_flow_table_table_id); 2918 mlft->mlft_state |= MLXCX_FLOW_TABLE_CREATED; 2919 } 2920 mlxcx_cmd_fini(mlxp, &cmd); 2921 return (ret); 2922 } 2923 2924 boolean_t 2925 mlxcx_cmd_destroy_flow_table(mlxcx_t *mlxp, mlxcx_flow_table_t *mlft) 2926 { 2927 mlxcx_cmd_t cmd; 2928 mlxcx_cmd_destroy_flow_table_in_t in; 2929 mlxcx_cmd_destroy_flow_table_out_t out; 2930 boolean_t ret; 2931 2932 bzero(&in, sizeof (in)); 2933 bzero(&out, sizeof (out)); 2934 2935 ASSERT(mutex_owned(&mlft->mlft_mtx)); 2936 VERIFY(mlft->mlft_state & MLXCX_FLOW_TABLE_CREATED); 2937 VERIFY0(mlft->mlft_state & MLXCX_FLOW_TABLE_DESTROYED); 2938 2939 mlxcx_cmd_init(mlxp, &cmd); 2940 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_flow_table_head, 2941 MLXCX_OP_DESTROY_FLOW_TABLE, 0); 2942 2943 in.mlxi_destroy_flow_table_vport_number = 2944 to_be16(mlft->mlft_port->mlp_num); 2945 in.mlxi_destroy_flow_table_table_type = mlft->mlft_type; 2946 in.mlxi_destroy_flow_table_table_id = to_be24(mlft->mlft_num); 2947 2948 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2949 mlxcx_cmd_fini(mlxp, &cmd); 2950 return (B_FALSE); 2951 } 2952 mlxcx_cmd_wait(&cmd); 2953 2954 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2955 if (ret) { 2956 mlft->mlft_state |= MLXCX_FLOW_TABLE_DESTROYED; 2957 } 2958 mlxcx_cmd_fini(mlxp, &cmd); 2959 return (ret); 2960 } 2961 2962 boolean_t 2963 mlxcx_cmd_set_flow_table_root(mlxcx_t *mlxp, mlxcx_flow_table_t *mlft) 2964 { 2965 mlxcx_cmd_t cmd; 2966 mlxcx_cmd_set_flow_table_root_in_t in; 2967 mlxcx_cmd_set_flow_table_root_out_t out; 2968 boolean_t ret; 2969 2970 bzero(&in, sizeof (in)); 2971 bzero(&out, sizeof (out)); 2972 2973 ASSERT(mutex_owned(&mlft->mlft_mtx)); 2974 VERIFY(mlft->mlft_state & MLXCX_FLOW_TABLE_CREATED); 2975 VERIFY0(mlft->mlft_state & MLXCX_FLOW_TABLE_DESTROYED); 2976 2977 mlxcx_cmd_init(mlxp, &cmd); 2978 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_set_flow_table_root_head, 2979 MLXCX_OP_SET_FLOW_TABLE_ROOT, 0); 2980 2981 in.mlxi_set_flow_table_root_vport_number = 2982 to_be16(mlft->mlft_port->mlp_num); 2983 in.mlxi_set_flow_table_root_table_type = mlft->mlft_type; 2984 in.mlxi_set_flow_table_root_table_id = to_be24(mlft->mlft_num); 2985 2986 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 2987 mlxcx_cmd_fini(mlxp, &cmd); 2988 return (B_FALSE); 2989 } 2990 mlxcx_cmd_wait(&cmd); 2991 2992 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 2993 if (ret) { 2994 mlft->mlft_state |= MLXCX_FLOW_TABLE_ROOT; 2995 } 2996 mlxcx_cmd_fini(mlxp, &cmd); 2997 return (ret); 2998 } 2999 3000 boolean_t 3001 mlxcx_cmd_create_flow_group(mlxcx_t *mlxp, mlxcx_flow_group_t *mlfg) 3002 { 3003 mlxcx_cmd_t cmd; 3004 mlxcx_cmd_create_flow_group_in_t in; 3005 mlxcx_cmd_create_flow_group_out_t out; 3006 boolean_t ret; 3007 const mlxcx_flow_table_t *mlft; 3008 mlxcx_flow_header_match_t *hdrs; 3009 mlxcx_flow_params_match_t *params; 3010 3011 bzero(&in, sizeof (in)); 3012 bzero(&out, sizeof (out)); 3013 3014 mlft = mlfg->mlfg_table; 3015 ASSERT(mutex_owned(&mlft->mlft_mtx)); 3016 VERIFY(mlft->mlft_state & MLXCX_FLOW_TABLE_CREATED); 3017 VERIFY0(mlft->mlft_state & MLXCX_FLOW_TABLE_DESTROYED); 3018 VERIFY0(mlfg->mlfg_state & MLXCX_FLOW_GROUP_CREATED); 3019 3020 mlxcx_cmd_init(mlxp, &cmd); 3021 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_flow_group_head, 3022 MLXCX_OP_CREATE_FLOW_GROUP, 0); 3023 3024 in.mlxi_create_flow_group_vport_number = 3025 to_be16(mlft->mlft_port->mlp_num); 3026 in.mlxi_create_flow_group_table_type = mlft->mlft_type; 3027 in.mlxi_create_flow_group_table_id = to_be24(mlft->mlft_num); 3028 in.mlxi_create_flow_group_start_flow_index = 3029 to_be32(mlfg->mlfg_start_idx); 3030 in.mlxi_create_flow_group_end_flow_index = 3031 to_be32(mlfg->mlfg_start_idx + (mlfg->mlfg_size - 1)); 3032 3033 hdrs = &in.mlxi_create_flow_group_match_criteria.mlfm_outer_headers; 3034 params = &in.mlxi_create_flow_group_match_criteria.mlfm_misc_parameters; 3035 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_SMAC) { 3036 in.mlxi_create_flow_group_match_criteria_en |= 3037 MLXCX_FLOW_GROUP_MATCH_OUTER_HDRS; 3038 (void) memset(&hdrs->mlfh_smac, 0xff, sizeof (hdrs->mlfh_smac)); 3039 } 3040 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_DMAC) { 3041 in.mlxi_create_flow_group_match_criteria_en |= 3042 MLXCX_FLOW_GROUP_MATCH_OUTER_HDRS; 3043 (void) memset(&hdrs->mlfh_dmac, 0xff, sizeof (hdrs->mlfh_dmac)); 3044 } 3045 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VLAN) { 3046 in.mlxi_create_flow_group_match_criteria_en |= 3047 MLXCX_FLOW_GROUP_MATCH_OUTER_HDRS; 3048 set_bit24(&hdrs->mlfh_tcp_ip_flags, MLXCX_FLOW_HDR_CVLAN_TAG); 3049 set_bit24(&hdrs->mlfh_tcp_ip_flags, MLXCX_FLOW_HDR_SVLAN_TAG); 3050 } 3051 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VID) { 3052 ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VLAN); 3053 set_bits16(&hdrs->mlfh_first_vid_flags, 3054 MLXCX_FLOW_HDR_FIRST_VID, UINT16_MAX); 3055 } 3056 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER) { 3057 in.mlxi_create_flow_group_match_criteria_en |= 3058 MLXCX_FLOW_GROUP_MATCH_OUTER_HDRS; 3059 set_bits24(&hdrs->mlfh_tcp_ip_flags, MLXCX_FLOW_HDR_IP_VERSION, 3060 UINT32_MAX); 3061 } 3062 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_SRCIP) { 3063 ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER); 3064 (void) memset(&hdrs->mlfh_src_ip, 0xff, 3065 sizeof (hdrs->mlfh_src_ip)); 3066 } 3067 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_DSTIP) { 3068 ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER); 3069 (void) memset(&hdrs->mlfh_src_ip, 0xff, 3070 sizeof (hdrs->mlfh_dst_ip)); 3071 } 3072 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_PROTO) { 3073 in.mlxi_create_flow_group_match_criteria_en |= 3074 MLXCX_FLOW_GROUP_MATCH_OUTER_HDRS; 3075 hdrs->mlfh_ip_protocol = UINT8_MAX; 3076 } 3077 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_SRCIP) { 3078 ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER); 3079 (void) memset(&hdrs->mlfh_src_ip, 0xff, 3080 sizeof (hdrs->mlfh_src_ip)); 3081 } 3082 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_DSTIP) { 3083 ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER); 3084 (void) memset(&hdrs->mlfh_src_ip, 0xff, 3085 sizeof (hdrs->mlfh_dst_ip)); 3086 } 3087 3088 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_SQN) { 3089 in.mlxi_create_flow_group_match_criteria_en |= 3090 MLXCX_FLOW_GROUP_MATCH_MISC_PARAMS; 3091 params->mlfp_source_sqn = to_be24(UINT32_MAX); 3092 } 3093 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VXLAN) { 3094 in.mlxi_create_flow_group_match_criteria_en |= 3095 MLXCX_FLOW_GROUP_MATCH_MISC_PARAMS; 3096 params->mlfp_vxlan_vni = to_be24(UINT32_MAX); 3097 } 3098 3099 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3100 mlxcx_cmd_fini(mlxp, &cmd); 3101 return (B_FALSE); 3102 } 3103 mlxcx_cmd_wait(&cmd); 3104 3105 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3106 if (ret) { 3107 mlfg->mlfg_state |= MLXCX_FLOW_GROUP_CREATED; 3108 mlfg->mlfg_num = from_be24(out.mlxo_create_flow_group_group_id); 3109 } 3110 mlxcx_cmd_fini(mlxp, &cmd); 3111 return (ret); 3112 } 3113 3114 boolean_t 3115 mlxcx_cmd_destroy_flow_group(mlxcx_t *mlxp, mlxcx_flow_group_t *mlfg) 3116 { 3117 mlxcx_cmd_t cmd; 3118 mlxcx_cmd_destroy_flow_group_in_t in; 3119 mlxcx_cmd_destroy_flow_group_out_t out; 3120 boolean_t ret; 3121 const mlxcx_flow_table_t *mlft; 3122 3123 bzero(&in, sizeof (in)); 3124 bzero(&out, sizeof (out)); 3125 3126 mlft = mlfg->mlfg_table; 3127 ASSERT(mutex_owned(&mlft->mlft_mtx)); 3128 VERIFY(mlft->mlft_state & MLXCX_FLOW_TABLE_CREATED); 3129 VERIFY0(mlft->mlft_state & MLXCX_FLOW_TABLE_DESTROYED); 3130 VERIFY(mlfg->mlfg_state & MLXCX_FLOW_GROUP_CREATED); 3131 VERIFY0(mlfg->mlfg_state & MLXCX_FLOW_GROUP_DESTROYED); 3132 3133 mlxcx_cmd_init(mlxp, &cmd); 3134 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_flow_group_head, 3135 MLXCX_OP_DESTROY_FLOW_GROUP, 0); 3136 3137 in.mlxi_destroy_flow_group_vport_number = 3138 to_be16(mlft->mlft_port->mlp_num); 3139 in.mlxi_destroy_flow_group_table_type = mlft->mlft_type; 3140 in.mlxi_destroy_flow_group_table_id = to_be24(mlft->mlft_num); 3141 in.mlxi_destroy_flow_group_group_id = to_be32(mlfg->mlfg_num); 3142 3143 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3144 mlxcx_cmd_fini(mlxp, &cmd); 3145 return (B_FALSE); 3146 } 3147 mlxcx_cmd_wait(&cmd); 3148 3149 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3150 if (ret) { 3151 mlfg->mlfg_state |= MLXCX_FLOW_GROUP_DESTROYED; 3152 } 3153 mlxcx_cmd_fini(mlxp, &cmd); 3154 return (ret); 3155 } 3156 3157 boolean_t 3158 mlxcx_cmd_set_flow_table_entry(mlxcx_t *mlxp, mlxcx_flow_entry_t *mlfe) 3159 { 3160 mlxcx_cmd_t cmd; 3161 mlxcx_cmd_set_flow_table_entry_in_t in; 3162 mlxcx_cmd_set_flow_table_entry_out_t out; 3163 boolean_t ret; 3164 size_t insize; 3165 mlxcx_flow_entry_ctx_t *ctx; 3166 const mlxcx_flow_table_t *mlft; 3167 mlxcx_flow_group_t *mlfg; 3168 mlxcx_flow_dest_t *d; 3169 uint_t i; 3170 mlxcx_flow_header_match_t *hdrs; 3171 mlxcx_flow_params_match_t *params; 3172 mlxcx_cmd_set_flow_table_entry_opmod_t opmod; 3173 3174 bzero(&in, sizeof (in)); 3175 bzero(&out, sizeof (out)); 3176 3177 mlft = mlfe->mlfe_table; 3178 ASSERT(mutex_owned(&mlft->mlft_mtx)); 3179 VERIFY(mlft->mlft_state & MLXCX_FLOW_TABLE_CREATED); 3180 VERIFY0(mlft->mlft_state & MLXCX_FLOW_TABLE_DESTROYED); 3181 3182 mlfg = mlfe->mlfe_group; 3183 VERIFY(mlfg->mlfg_state & MLXCX_FLOW_GROUP_CREATED); 3184 VERIFY0(mlfg->mlfg_state & MLXCX_FLOW_GROUP_DESTROYED); 3185 3186 opmod = MLXCX_CMD_FLOW_ENTRY_SET_NEW; 3187 if (mlfe->mlfe_state & MLXCX_FLOW_ENTRY_CREATED) { 3188 ASSERT(mlfe->mlfe_state & MLXCX_FLOW_ENTRY_DIRTY); 3189 opmod = MLXCX_CMD_FLOW_ENTRY_MODIFY; 3190 } 3191 3192 mlxcx_cmd_init(mlxp, &cmd); 3193 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_set_flow_table_entry_head, 3194 MLXCX_OP_SET_FLOW_TABLE_ENTRY, opmod); 3195 3196 in.mlxi_set_flow_table_entry_vport_number = 3197 to_be16(mlft->mlft_port->mlp_num); 3198 in.mlxi_set_flow_table_entry_table_type = mlft->mlft_type; 3199 in.mlxi_set_flow_table_entry_table_id = to_be24(mlft->mlft_num); 3200 in.mlxi_set_flow_table_entry_flow_index = to_be32(mlfe->mlfe_index); 3201 3202 if (mlfe->mlfe_state & MLXCX_FLOW_ENTRY_CREATED) { 3203 set_bit8(&in.mlxi_set_flow_table_entry_modify_bitmask, 3204 MLXCX_CMD_FLOW_ENTRY_SET_ACTION); 3205 set_bit8(&in.mlxi_set_flow_table_entry_modify_bitmask, 3206 MLXCX_CMD_FLOW_ENTRY_SET_DESTINATION); 3207 } 3208 3209 ctx = &in.mlxi_set_flow_table_entry_context; 3210 ctx->mlfec_group_id = to_be32(mlfg->mlfg_num); 3211 3212 insize = offsetof(mlxcx_cmd_set_flow_table_entry_in_t, 3213 mlxi_set_flow_table_entry_context) + 3214 offsetof(mlxcx_flow_entry_ctx_t, mlfec_destination); 3215 3216 ctx->mlfec_action = to_be16(mlfe->mlfe_action); 3217 3218 switch (mlfe->mlfe_action) { 3219 case MLXCX_FLOW_ACTION_ALLOW: 3220 case MLXCX_FLOW_ACTION_DROP: 3221 break; 3222 case MLXCX_FLOW_ACTION_FORWARD: 3223 ASSERT3U(mlfe->mlfe_ndest, <=, MLXCX_FLOW_MAX_DESTINATIONS); 3224 ASSERT3U(mlfe->mlfe_ndest, <=, 3225 mlxp->mlx_caps->mlc_max_rx_fe_dest); 3226 ctx->mlfec_destination_list_size = to_be24(mlfe->mlfe_ndest); 3227 for (i = 0; i < mlfe->mlfe_ndest; ++i) { 3228 insize += sizeof (mlxcx_flow_dest_t); 3229 d = &ctx->mlfec_destination[i]; 3230 if (mlfe->mlfe_dest[i].mlfed_tir != NULL) { 3231 d->mlfd_destination_type = MLXCX_FLOW_DEST_TIR; 3232 d->mlfd_destination_id = to_be24( 3233 mlfe->mlfe_dest[i].mlfed_tir->mltir_num); 3234 } else if (mlfe->mlfe_dest[i].mlfed_flow != NULL) { 3235 d->mlfd_destination_type = 3236 MLXCX_FLOW_DEST_FLOW_TABLE; 3237 d->mlfd_destination_id = to_be24( 3238 mlfe->mlfe_dest[i].mlfed_flow->mlft_num); 3239 } else { 3240 /* Invalid flow entry destination */ 3241 VERIFY(0); 3242 } 3243 } 3244 break; 3245 case MLXCX_FLOW_ACTION_COUNT: 3246 /* We don't support count actions yet. */ 3247 VERIFY(0); 3248 break; 3249 case MLXCX_FLOW_ACTION_ENCAP: 3250 case MLXCX_FLOW_ACTION_DECAP: 3251 /* We don't support encap/decap actions yet. */ 3252 VERIFY(0); 3253 break; 3254 } 3255 3256 hdrs = &ctx->mlfec_match_value.mlfm_outer_headers; 3257 params = &ctx->mlfec_match_value.mlfm_misc_parameters; 3258 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_SMAC) { 3259 bcopy(mlfe->mlfe_smac, hdrs->mlfh_smac, 3260 sizeof (hdrs->mlfh_smac)); 3261 } 3262 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_DMAC) { 3263 bcopy(mlfe->mlfe_dmac, hdrs->mlfh_dmac, 3264 sizeof (hdrs->mlfh_dmac)); 3265 } 3266 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VLAN) { 3267 switch (mlfe->mlfe_vlan_type) { 3268 case MLXCX_VLAN_TYPE_CVLAN: 3269 set_bit24(&hdrs->mlfh_tcp_ip_flags, 3270 MLXCX_FLOW_HDR_CVLAN_TAG); 3271 break; 3272 case MLXCX_VLAN_TYPE_SVLAN: 3273 set_bit24(&hdrs->mlfh_tcp_ip_flags, 3274 MLXCX_FLOW_HDR_SVLAN_TAG); 3275 break; 3276 default: 3277 break; 3278 } 3279 } 3280 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VID) { 3281 ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VLAN); 3282 set_bits16(&hdrs->mlfh_first_vid_flags, 3283 MLXCX_FLOW_HDR_FIRST_VID, mlfe->mlfe_vid); 3284 } 3285 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER) { 3286 set_bits24(&hdrs->mlfh_tcp_ip_flags, MLXCX_FLOW_HDR_IP_VERSION, 3287 mlfe->mlfe_ip_version); 3288 } 3289 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_SRCIP) { 3290 ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER); 3291 bcopy(mlfe->mlfe_srcip, hdrs->mlfh_src_ip, 3292 sizeof (hdrs->mlfh_src_ip)); 3293 } 3294 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_DSTIP) { 3295 ASSERT(mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_VER); 3296 bcopy(mlfe->mlfe_dstip, hdrs->mlfh_src_ip, 3297 sizeof (hdrs->mlfh_dst_ip)); 3298 } 3299 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_IP_PROTO) { 3300 hdrs->mlfh_ip_protocol = mlfe->mlfe_ip_proto; 3301 } 3302 3303 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_SQN) { 3304 params->mlfp_source_sqn = to_be24(mlfe->mlfe_sqn); 3305 } 3306 if (mlfg->mlfg_mask & MLXCX_FLOW_MATCH_VXLAN) { 3307 params->mlfp_vxlan_vni = to_be24(mlfe->mlfe_vxlan_vni); 3308 } 3309 3310 if (!mlxcx_cmd_send(mlxp, &cmd, &in, insize, &out, sizeof (out))) { 3311 mlxcx_cmd_fini(mlxp, &cmd); 3312 return (B_FALSE); 3313 } 3314 mlxcx_cmd_wait(&cmd); 3315 3316 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3317 if (ret) { 3318 mlfe->mlfe_state |= MLXCX_FLOW_ENTRY_CREATED; 3319 mlfe->mlfe_state &= ~MLXCX_FLOW_ENTRY_DIRTY; 3320 mlfg->mlfg_state |= MLXCX_FLOW_GROUP_BUSY; 3321 } 3322 mlxcx_cmd_fini(mlxp, &cmd); 3323 return (ret); 3324 } 3325 3326 boolean_t 3327 mlxcx_cmd_delete_flow_table_entry(mlxcx_t *mlxp, mlxcx_flow_entry_t *mlfe) 3328 { 3329 mlxcx_cmd_t cmd; 3330 mlxcx_cmd_delete_flow_table_entry_in_t in; 3331 mlxcx_cmd_delete_flow_table_entry_out_t out; 3332 boolean_t ret; 3333 const mlxcx_flow_table_t *mlft; 3334 3335 bzero(&in, sizeof (in)); 3336 bzero(&out, sizeof (out)); 3337 3338 mlft = mlfe->mlfe_table; 3339 ASSERT(mutex_owned(&mlft->mlft_mtx)); 3340 VERIFY(mlft->mlft_state & MLXCX_FLOW_TABLE_CREATED); 3341 VERIFY0(mlft->mlft_state & MLXCX_FLOW_TABLE_DESTROYED); 3342 3343 mlxcx_cmd_init(mlxp, &cmd); 3344 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_delete_flow_table_entry_head, 3345 MLXCX_OP_DELETE_FLOW_TABLE_ENTRY, 0); 3346 3347 in.mlxi_delete_flow_table_entry_vport_number = 3348 to_be16(mlft->mlft_port->mlp_num); 3349 in.mlxi_delete_flow_table_entry_table_type = mlft->mlft_type; 3350 in.mlxi_delete_flow_table_entry_table_id = to_be24(mlft->mlft_num); 3351 in.mlxi_delete_flow_table_entry_flow_index = to_be32(mlfe->mlfe_index); 3352 3353 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3354 mlxcx_cmd_fini(mlxp, &cmd); 3355 return (B_FALSE); 3356 } 3357 mlxcx_cmd_wait(&cmd); 3358 3359 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3360 if (ret) { 3361 /* 3362 * Note that flow entries have a different lifecycle to most 3363 * other things we create -- we have to be able to re-use them 3364 * after they have been deleted, since they exist at a fixed 3365 * position in their flow table. 3366 * 3367 * So we clear the CREATED bit here for them to let us call 3368 * create_flow_table_entry() on the same entry again later. 3369 */ 3370 mlfe->mlfe_state &= ~MLXCX_FLOW_ENTRY_CREATED; 3371 mlfe->mlfe_state |= MLXCX_FLOW_ENTRY_DELETED; 3372 } 3373 mlxcx_cmd_fini(mlxp, &cmd); 3374 return (ret); 3375 } 3376 3377 boolean_t 3378 mlxcx_cmd_create_sq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq) 3379 { 3380 mlxcx_cmd_t cmd; 3381 mlxcx_cmd_create_sq_in_t in; 3382 mlxcx_cmd_create_sq_out_t out; 3383 boolean_t ret; 3384 mlxcx_sq_ctx_t *ctx; 3385 size_t rem, insize; 3386 const ddi_dma_cookie_t *c; 3387 uint64_t pa, npages; 3388 3389 bzero(&in, sizeof (in)); 3390 bzero(&out, sizeof (out)); 3391 3392 ASSERT(mutex_owned(&mlwq->mlwq_mtx)); 3393 VERIFY3U(mlwq->mlwq_type, ==, MLXCX_WQ_TYPE_SENDQ); 3394 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 3395 VERIFY0(mlwq->mlwq_state & MLXCX_WQ_CREATED); 3396 3397 mlxcx_cmd_init(mlxp, &cmd); 3398 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_sq_head, 3399 MLXCX_OP_CREATE_SQ, 0); 3400 3401 ctx = &in.mlxi_create_sq_context; 3402 3403 set_bit32(&ctx->mlsqc_flags, MLXCX_SQ_FLAGS_RLKEY); 3404 set_bit32(&ctx->mlsqc_flags, MLXCX_SQ_FLAGS_FLUSH_IN_ERROR); 3405 set_bits32(&ctx->mlsqc_flags, MLXCX_SQ_MIN_WQE_INLINE, 3406 mlwq->mlwq_inline_mode); 3407 ctx->mlsqc_cqn = to_be24(mlwq->mlwq_cq->mlcq_num); 3408 3409 VERIFY(mlwq->mlwq_tis != NULL); 3410 ctx->mlsqc_tis_lst_sz = to_be16(1); 3411 ctx->mlsqc_tis_num = to_be24(mlwq->mlwq_tis->mltis_num); 3412 3413 set_bits32(&ctx->mlsqc_wq.mlwqc_flags, MLXCX_WORKQ_CTX_TYPE, 3414 MLXCX_WORKQ_TYPE_CYCLIC); 3415 ctx->mlsqc_wq.mlwqc_pd = to_be24(mlwq->mlwq_pd->mlpd_num); 3416 ctx->mlsqc_wq.mlwqc_uar_page = to_be24(mlwq->mlwq_uar->mlu_num); 3417 ctx->mlsqc_wq.mlwqc_log_wq_sz = mlwq->mlwq_entshift; 3418 ctx->mlsqc_wq.mlwqc_log_wq_stride = MLXCX_SENDQ_STRIDE_SHIFT; 3419 3420 c = mlxcx_dma_cookie_one(&mlwq->mlwq_doorbell_dma); 3421 ctx->mlsqc_wq.mlwqc_dbr_addr = to_be64(c->dmac_laddress); 3422 ASSERT3U(c->dmac_size, >=, sizeof (mlxcx_workq_doorbell_t)); 3423 3424 npages = 0; 3425 c = NULL; 3426 while ((c = mlxcx_dma_cookie_iter(&mlwq->mlwq_dma, c)) != NULL) { 3427 pa = c->dmac_laddress; 3428 rem = c->dmac_size; 3429 while (rem > 0) { 3430 ASSERT3U(pa & 0xfff, ==, 0); 3431 ASSERT3U(rem, >=, MLXCX_HW_PAGE_SIZE); 3432 ctx->mlsqc_wq.mlwqc_pas[npages++] = to_be64(pa); 3433 rem -= MLXCX_HW_PAGE_SIZE; 3434 pa += MLXCX_HW_PAGE_SIZE; 3435 } 3436 } 3437 ASSERT3U(npages, <=, MLXCX_WORKQ_CTX_MAX_ADDRESSES); 3438 3439 insize = offsetof(mlxcx_cmd_create_sq_in_t, mlxi_create_sq_context) + 3440 offsetof(mlxcx_sq_ctx_t, mlsqc_wq) + 3441 offsetof(mlxcx_workq_ctx_t, mlwqc_pas) + 3442 sizeof (uint64_t) * npages; 3443 3444 if (!mlxcx_cmd_send(mlxp, &cmd, &in, insize, &out, sizeof (out))) { 3445 mlxcx_cmd_fini(mlxp, &cmd); 3446 return (B_FALSE); 3447 } 3448 mlxcx_cmd_wait(&cmd); 3449 3450 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3451 if (ret) { 3452 mlwq->mlwq_state |= MLXCX_WQ_CREATED; 3453 mlwq->mlwq_num = from_be24(out.mlxo_create_sq_sqn); 3454 } 3455 mlxcx_cmd_fini(mlxp, &cmd); 3456 return (ret); 3457 } 3458 3459 boolean_t 3460 mlxcx_cmd_start_sq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq) 3461 { 3462 mlxcx_cmd_t cmd; 3463 mlxcx_cmd_modify_sq_in_t in; 3464 mlxcx_cmd_modify_sq_out_t out; 3465 boolean_t ret; 3466 ddi_fm_error_t err; 3467 3468 bzero(&in, sizeof (in)); 3469 bzero(&out, sizeof (out)); 3470 3471 ASSERT(mutex_owned(&mlwq->mlwq_mtx)); 3472 ASSERT(mlwq->mlwq_cq != NULL); 3473 3474 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 3475 VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED); 3476 VERIFY0(mlwq->mlwq_state & MLXCX_WQ_STARTED); 3477 3478 /* 3479 * Before starting the queue, we have to be sure that it is 3480 * empty and the doorbell and counters are set to 0. 3481 */ 3482 ASSERT(mutex_owned(&mlwq->mlwq_cq->mlcq_mtx)); 3483 ASSERT(list_is_empty(&mlwq->mlwq_cq->mlcq_buffers)); 3484 ASSERT(list_is_empty(&mlwq->mlwq_cq->mlcq_buffers_b)); 3485 3486 mlwq->mlwq_doorbell->mlwqd_recv_counter = to_be16(0); 3487 MLXCX_DMA_SYNC(mlwq->mlwq_doorbell_dma, DDI_DMA_SYNC_FORDEV); 3488 ddi_fm_dma_err_get(mlwq->mlwq_doorbell_dma.mxdb_dma_handle, &err, 3489 DDI_FME_VERSION); 3490 if (err.fme_status != DDI_FM_OK) 3491 return (B_FALSE); 3492 mlwq->mlwq_pc = 0; 3493 3494 mlxcx_cmd_init(mlxp, &cmd); 3495 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_modify_sq_head, 3496 MLXCX_OP_MODIFY_SQ, 0); 3497 3498 in.mlxi_modify_sq_sqn = to_be24(mlwq->mlwq_num); 3499 3500 /* From state */ 3501 set_bits8(&in.mlxi_modify_sq_state, MLXCX_CMD_MODIFY_SQ_STATE, 3502 MLXCX_SQ_STATE_RST); 3503 /* To state */ 3504 set_bits32(&in.mlxi_modify_sq_context.mlsqc_flags, MLXCX_SQ_STATE, 3505 MLXCX_SQ_STATE_RDY); 3506 3507 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3508 mlxcx_cmd_fini(mlxp, &cmd); 3509 return (B_FALSE); 3510 } 3511 mlxcx_cmd_wait(&cmd); 3512 3513 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3514 if (ret) { 3515 mlwq->mlwq_state |= MLXCX_WQ_STARTED; 3516 } 3517 mlxcx_cmd_fini(mlxp, &cmd); 3518 return (ret); 3519 } 3520 3521 boolean_t 3522 mlxcx_cmd_stop_sq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq) 3523 { 3524 mlxcx_cmd_t cmd; 3525 mlxcx_cmd_modify_sq_in_t in; 3526 mlxcx_cmd_modify_sq_out_t out; 3527 boolean_t ret; 3528 3529 bzero(&in, sizeof (in)); 3530 bzero(&out, sizeof (out)); 3531 3532 ASSERT(mutex_owned(&mlwq->mlwq_mtx)); 3533 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 3534 VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED); 3535 VERIFY(mlwq->mlwq_state & MLXCX_WQ_STARTED); 3536 3537 mlxcx_cmd_init(mlxp, &cmd); 3538 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_modify_sq_head, 3539 MLXCX_OP_MODIFY_SQ, 0); 3540 3541 in.mlxi_modify_sq_sqn = to_be24(mlwq->mlwq_num); 3542 3543 /* From state */ 3544 set_bits8(&in.mlxi_modify_sq_state, MLXCX_CMD_MODIFY_SQ_STATE, 3545 MLXCX_SQ_STATE_RDY); 3546 /* To state */ 3547 set_bits32(&in.mlxi_modify_sq_context.mlsqc_flags, MLXCX_SQ_STATE, 3548 MLXCX_SQ_STATE_RST); 3549 3550 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3551 mlxcx_cmd_fini(mlxp, &cmd); 3552 return (B_FALSE); 3553 } 3554 mlxcx_cmd_wait(&cmd); 3555 3556 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3557 if (ret) { 3558 mlwq->mlwq_state &= ~MLXCX_WQ_STARTED; 3559 } 3560 mlxcx_cmd_fini(mlxp, &cmd); 3561 return (ret); 3562 } 3563 3564 boolean_t 3565 mlxcx_cmd_destroy_sq(mlxcx_t *mlxp, mlxcx_work_queue_t *mlwq) 3566 { 3567 mlxcx_cmd_t cmd; 3568 mlxcx_cmd_destroy_sq_in_t in; 3569 mlxcx_cmd_destroy_sq_out_t out; 3570 boolean_t ret; 3571 3572 bzero(&in, sizeof (in)); 3573 bzero(&out, sizeof (out)); 3574 3575 ASSERT(mutex_owned(&mlwq->mlwq_mtx)); 3576 VERIFY(mlwq->mlwq_state & MLXCX_WQ_ALLOC); 3577 VERIFY(mlwq->mlwq_state & MLXCX_WQ_CREATED); 3578 VERIFY0(mlwq->mlwq_state & MLXCX_WQ_STARTED); 3579 3580 mlxcx_cmd_init(mlxp, &cmd); 3581 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_sq_head, 3582 MLXCX_OP_DESTROY_SQ, 0); 3583 3584 in.mlxi_destroy_sq_sqn = to_be24(mlwq->mlwq_num); 3585 3586 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3587 mlxcx_cmd_fini(mlxp, &cmd); 3588 return (B_FALSE); 3589 } 3590 mlxcx_cmd_wait(&cmd); 3591 3592 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3593 if (ret) { 3594 mlwq->mlwq_state |= MLXCX_WQ_DESTROYED; 3595 } 3596 mlxcx_cmd_fini(mlxp, &cmd); 3597 return (ret); 3598 } 3599 3600 boolean_t 3601 mlxcx_cmd_create_rqt(mlxcx_t *mlxp, mlxcx_rqtable_t *mlrqt) 3602 { 3603 mlxcx_cmd_t cmd; 3604 mlxcx_cmd_create_rqt_in_t in; 3605 mlxcx_cmd_create_rqt_out_t out; 3606 mlxcx_rqtable_ctx_t *ctx; 3607 boolean_t ret; 3608 uint_t i; 3609 3610 bzero(&in, sizeof (in)); 3611 bzero(&out, sizeof (out)); 3612 3613 VERIFY0(mlrqt->mlrqt_state & MLXCX_RQT_CREATED); 3614 3615 mlxcx_cmd_init(mlxp, &cmd); 3616 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_create_rqt_head, 3617 MLXCX_OP_CREATE_RQT, 0); 3618 3619 ctx = &in.mlxi_create_rqt_context; 3620 ASSERT3U(mlrqt->mlrqt_max, <=, MLXCX_RQT_MAX_RQ_REFS); 3621 ASSERT3U(mlrqt->mlrqt_max, <=, mlxp->mlx_caps->mlc_max_rqt_size); 3622 ctx->mlrqtc_max_size = to_be16(mlrqt->mlrqt_max); 3623 ctx->mlrqtc_actual_size = to_be16(mlrqt->mlrqt_used); 3624 for (i = 0; i < mlrqt->mlrqt_used; ++i) { 3625 ctx->mlrqtc_rqref[i].mlrqtr_rqn = to_be24( 3626 mlrqt->mlrqt_rq[i]->mlwq_num); 3627 } 3628 3629 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3630 mlxcx_cmd_fini(mlxp, &cmd); 3631 return (B_FALSE); 3632 } 3633 mlxcx_cmd_wait(&cmd); 3634 3635 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3636 if (ret) { 3637 mlrqt->mlrqt_num = from_be24(out.mlxo_create_rqt_rqtn); 3638 mlrqt->mlrqt_state |= MLXCX_RQT_CREATED; 3639 mlrqt->mlrqt_state &= ~MLXCX_RQT_DIRTY; 3640 } 3641 mlxcx_cmd_fini(mlxp, &cmd); 3642 return (ret); 3643 } 3644 3645 boolean_t 3646 mlxcx_cmd_destroy_rqt(mlxcx_t *mlxp, mlxcx_rqtable_t *mlrqt) 3647 { 3648 mlxcx_cmd_t cmd; 3649 mlxcx_cmd_destroy_rqt_in_t in; 3650 mlxcx_cmd_destroy_rqt_out_t out; 3651 boolean_t ret; 3652 3653 bzero(&in, sizeof (in)); 3654 bzero(&out, sizeof (out)); 3655 3656 VERIFY(mlrqt->mlrqt_state & MLXCX_RQT_CREATED); 3657 VERIFY0(mlrqt->mlrqt_state & MLXCX_RQT_DESTROYED); 3658 3659 mlxcx_cmd_init(mlxp, &cmd); 3660 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_destroy_rqt_head, 3661 MLXCX_OP_DESTROY_RQT, 0); 3662 3663 in.mlxi_destroy_rqt_rqtn = to_be24(mlrqt->mlrqt_num); 3664 3665 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3666 mlxcx_cmd_fini(mlxp, &cmd); 3667 return (B_FALSE); 3668 } 3669 mlxcx_cmd_wait(&cmd); 3670 3671 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3672 if (ret) { 3673 mlrqt->mlrqt_state |= MLXCX_RQT_DESTROYED; 3674 } 3675 mlxcx_cmd_fini(mlxp, &cmd); 3676 return (ret); 3677 } 3678 3679 boolean_t 3680 mlxcx_cmd_set_int_mod(mlxcx_t *mlxp, uint_t intr, uint_t min_delay) 3681 { 3682 mlxcx_cmd_t cmd; 3683 mlxcx_cmd_config_int_mod_in_t in; 3684 mlxcx_cmd_config_int_mod_out_t out; 3685 boolean_t ret; 3686 3687 bzero(&in, sizeof (in)); 3688 bzero(&out, sizeof (out)); 3689 3690 mlxcx_cmd_init(mlxp, &cmd); 3691 mlxcx_cmd_in_header_init(&cmd, &in.mlxi_config_int_mod_head, 3692 MLXCX_OP_CONFIG_INT_MODERATION, MLXCX_CMD_CONFIG_INT_MOD_WRITE); 3693 3694 in.mlxi_config_int_mod_int_vector = to_be16(intr); 3695 in.mlxi_config_int_mod_min_delay = to_be16(min_delay); 3696 3697 if (!mlxcx_cmd_send(mlxp, &cmd, &in, sizeof (in), &out, sizeof (out))) { 3698 mlxcx_cmd_fini(mlxp, &cmd); 3699 return (B_FALSE); 3700 } 3701 mlxcx_cmd_wait(&cmd); 3702 3703 ret = mlxcx_cmd_evaluate(mlxp, &cmd); 3704 mlxcx_cmd_fini(mlxp, &cmd); 3705 return (ret); 3706 } 3707 3708 /* 3709 * CTASSERTs here are for the structs in mlxcx_reg.h, to check they match 3710 * against offsets from the PRM. 3711 * 3712 * They're not in the header file, to avoid them being used by multiple .c 3713 * files. 3714 */ 3715 3716 CTASSERT(offsetof(mlxcx_eventq_ent_t, mleqe_unknown_data) == 0x20); 3717 CTASSERT(offsetof(mlxcx_eventq_ent_t, mleqe_signature) == 0x3c + 2); 3718 CTASSERT(sizeof (mlxcx_eventq_ent_t) == 64); 3719 3720 CTASSERT(offsetof(mlxcx_completionq_error_ent_t, mlcqee_byte_cnt) == 0x2C); 3721 CTASSERT(offsetof(mlxcx_completionq_error_ent_t, mlcqee_wqe_opcode) == 0x38); 3722 3723 CTASSERT(sizeof (mlxcx_completionq_error_ent_t) == 3724 sizeof (mlxcx_completionq_ent_t)); 3725 CTASSERT(sizeof (mlxcx_wqe_control_seg_t) == (1 << 4)); 3726 3727 CTASSERT(offsetof(mlxcx_wqe_eth_seg_t, mles_inline_headers) == 0x0e); 3728 CTASSERT(sizeof (mlxcx_wqe_eth_seg_t) == (1 << 5)); 3729 3730 CTASSERT(sizeof (mlxcx_wqe_data_seg_t) == (1 << 4)); 3731 3732 CTASSERT(sizeof (mlxcx_sendq_ent_t) == (1 << MLXCX_SENDQ_STRIDE_SHIFT)); 3733 3734 CTASSERT(sizeof (mlxcx_sendq_bf_t) == (1 << MLXCX_SENDQ_STRIDE_SHIFT)); 3735 3736 CTASSERT(sizeof (mlxcx_sendq_extra_ent_t) == (1 << MLXCX_SENDQ_STRIDE_SHIFT)); 3737 3738 CTASSERT(sizeof (mlxcx_recvq_ent_t) == (1 << MLXCX_RECVQ_STRIDE_SHIFT)); 3739 3740 CTASSERT(offsetof(mlxcx_workq_ctx_t, mlwqc_dbr_addr) == 0x10); 3741 CTASSERT(offsetof(mlxcx_workq_ctx_t, mlwqc_pas) == 0xc0); 3742 3743 CTASSERT(offsetof(mlxcx_rq_ctx_t, mlrqc_cqn) == 0x09); 3744 CTASSERT(offsetof(mlxcx_rq_ctx_t, mlrqc_wq) == 0x30); 3745 3746 CTASSERT(offsetof(mlxcx_sq_ctx_t, mlsqc_cqn) == 0x09); 3747 CTASSERT(offsetof(mlxcx_sq_ctx_t, mlsqc_tis_lst_sz) == 0x20); 3748 CTASSERT(offsetof(mlxcx_sq_ctx_t, mlsqc_tis_num) == 0x2d); 3749 CTASSERT(offsetof(mlxcx_sq_ctx_t, mlsqc_wq) == 0x30); 3750 3751 CTASSERT(sizeof (mlxcx_tis_ctx_t) == 0xa0); 3752 CTASSERT(offsetof(mlxcx_tis_ctx_t, mltisc_transport_domain) == 0x25); 3753 3754 CTASSERT(offsetof(mlxcx_rqtable_ctx_t, mlrqtc_max_size) == 0x16); 3755 CTASSERT(offsetof(mlxcx_rqtable_ctx_t, mlrqtc_rqref) == 0xF0); 3756 3757 CTASSERT(offsetof(mlxcx_cmd_create_eq_in_t, mlxi_create_eq_event_bitmask) == 3758 0x58); 3759 CTASSERT(offsetof(mlxcx_cmd_create_eq_in_t, mlxi_create_eq_pas) == 0x110); 3760 CTASSERT(offsetof(mlxcx_cmd_create_eq_in_t, mlxi_create_eq_context) == 0x10); 3761 3762 CTASSERT(offsetof(mlxcx_cmd_create_tir_in_t, mlxi_create_tir_context) == 0x20); 3763 3764 CTASSERT(offsetof(mlxcx_cmd_create_tis_in_t, mlxi_create_tis_context) == 0x20); 3765 3766 CTASSERT(offsetof(mlxcx_cmd_query_special_ctxs_out_t, 3767 mlxo_query_special_ctxs_resd_lkey) == 0x0c); 3768 3769 CTASSERT(offsetof(mlxcx_cmd_query_cq_out_t, mlxo_query_cq_context) == 0x10); 3770 CTASSERT(offsetof(mlxcx_cmd_query_cq_out_t, mlxo_query_cq_pas) == 0x110); 3771 3772 CTASSERT(offsetof(mlxcx_cmd_query_rq_out_t, mlxo_query_rq_context) == 0x20); 3773 3774 CTASSERT(offsetof(mlxcx_cmd_create_sq_in_t, mlxi_create_sq_context) == 0x20); 3775 3776 CTASSERT(offsetof(mlxcx_cmd_modify_sq_in_t, mlxi_modify_sq_context) == 0x20); 3777 3778 CTASSERT(offsetof(mlxcx_cmd_query_sq_out_t, mlxo_query_sq_context) == 0x20); 3779 3780 CTASSERT(offsetof(mlxcx_cmd_create_rqt_in_t, mlxi_create_rqt_context) == 0x20); 3781 3782 CTASSERT(offsetof(mlxcx_reg_pmtu_t, mlrd_pmtu_oper_mtu) == 0x0C); 3783 3784 CTASSERT(sizeof (mlxcx_reg_ptys_t) == 64); 3785 CTASSERT(offsetof(mlxcx_reg_ptys_t, mlrd_ptys_ext_proto_cap) == 0x08); 3786 CTASSERT(offsetof(mlxcx_reg_ptys_t, mlrd_ptys_proto_cap) == 0x0c); 3787 CTASSERT(offsetof(mlxcx_reg_ptys_t, mlrd_ptys_ext_proto_admin) == 0x14); 3788 CTASSERT(offsetof(mlxcx_reg_ptys_t, mlrd_ptys_proto_admin) == 0x18); 3789 CTASSERT(offsetof(mlxcx_reg_ptys_t, mlrd_ptys_proto_partner_advert) == 0x30); 3790 3791 CTASSERT(offsetof(mlxcx_reg_mcia_t, mlrd_mcia_data) == 0x10); 3792 3793 CTASSERT(offsetof(mlxcx_ppcnt_ieee_802_3_t, 3794 mlppc_ieee_802_3_in_range_len_err) == 0x50); 3795 CTASSERT(offsetof(mlxcx_ppcnt_ieee_802_3_t, 3796 mlppc_ieee_802_3_pause_tx) == 0x90); 3797 3798 CTASSERT(sizeof (mlxcx_reg_ppcnt_t) == 256); 3799 CTASSERT(offsetof(mlxcx_reg_ppcnt_t, mlrd_ppcnt_data) == 0x08); 3800 3801 CTASSERT(offsetof(mlxcx_cmd_access_register_in_t, 3802 mlxi_access_register_argument) == 0x0C); 3803 CTASSERT(offsetof(mlxcx_cmd_access_register_in_t, 3804 mlxi_access_register_data) == 0x10); 3805 3806 CTASSERT(offsetof(mlxcx_cmd_access_register_out_t, 3807 mlxo_access_register_data) == 0x10); 3808 3809 CTASSERT(sizeof (mlxcx_cmd_set_flow_table_root_in_t) == 64); 3810 CTASSERT(offsetof(mlxcx_cmd_set_flow_table_root_in_t, 3811 mlxi_set_flow_table_root_table_id) == 0x15); 3812 CTASSERT(offsetof(mlxcx_cmd_set_flow_table_root_in_t, 3813 mlxi_set_flow_table_root_esw_owner_vhca_id_valid) == 0x1C); 3814