1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved. 24 */ 25 26 /* 27 * hermon_fcoib.c 28 * Hermon Fibre Channel over IB routines 29 * 30 * Implements all the routines necessary for setting up, using, and 31 * (later) tearing down all the FCoIB state. 32 */ 33 34 #include <sys/ib/adapters/hermon/hermon.h> 35 36 /* 37 * hermon_fcoib_enable() 38 * Context: user or kernel context 39 */ 40 static int 41 hermon_fcoib_enable(hermon_state_t *state, int port) 42 { 43 hermon_fcoib_t *fcoib; 44 hermon_hw_config_fc_basic_t config_fc_basic; 45 int status; 46 47 port--; /* passed in as 1 or 2, used as 0 or 1 */ 48 ASSERT(port >= 0 && port < HERMON_MAX_PORTS); 49 fcoib = &state->hs_fcoib; 50 51 /* Configure FCoIB on the port */ 52 bzero(&config_fc_basic, sizeof (config_fc_basic)); 53 config_fc_basic.fexch_base_hi = fcoib->hfc_fexch_base[port] >> 16; 54 config_fc_basic.fx_base_mpt_hi = fcoib->hfc_mpt_base[port] >> 17; 55 config_fc_basic.fx_base_mpt_lo = 0; 56 config_fc_basic.log2_num_rfci = 57 state->hs_ibtfinfo.hca_attr->hca_rfci_max_log2_qp; 58 config_fc_basic.rfci_base = fcoib->hfc_rfci_qps_per_port * port + 59 fcoib->hfc_rfci_rsrc->hr_indx; 60 #if 1 61 status = hermon_config_fc_cmd_post(state, &config_fc_basic, 1, 62 HERMON_HW_FC_CONF_BASIC, 0, port + 1, HERMON_CMD_NOSLEEP_SPIN); 63 #else 64 status = hermon_config_fc_cmd_post(state, &config_fc_basic, 1, 65 HERMON_HW_FC_CONF_BASIC, 0, 0, HERMON_CMD_NOSLEEP_SPIN); 66 #endif 67 if (status != HERMON_CMD_SUCCESS) { 68 cmn_err(CE_CONT, "fcoib_enable failed: status 0x%x\n", status); 69 HERMON_WARNING(state, "fcoib_enable failed"); 70 return (DDI_FAILURE); 71 } 72 fcoib->hfc_port_enabled[port] = 1; 73 state->hs_fcoib_may_be_running = B_TRUE; 74 return (DDI_SUCCESS); 75 } 76 77 /* 78 * hermon_fcoib_set_id() 79 * Context: user or kernel context 80 */ 81 int 82 hermon_fcoib_set_id(hermon_state_t *state, int port, uint32_t rfci_qpn, 83 uint32_t src_id) 84 { 85 hermon_fcoib_t *fcoib; 86 int status; 87 int offset; 88 uint32_t *n_port_ids; 89 90 port--; /* passed in as 1 or 2, used as 0 or 1 */ 91 ASSERT(port >= 0 && port < HERMON_MAX_PORTS); 92 fcoib = &state->hs_fcoib; 93 mutex_enter(&fcoib->hfc_lock); 94 95 if (fcoib->hfc_port_enabled[port] == 0) { 96 if (hermon_fcoib_enable(state, port + 1) != DDI_SUCCESS) { 97 mutex_exit(&fcoib->hfc_lock); 98 return (DDI_FAILURE); 99 } 100 } 101 102 n_port_ids = fcoib->hfc_n_port_ids[port]; 103 offset = rfci_qpn - fcoib->hfc_rfci_base[port]; 104 ASSERT(offset >= 0 && offset < fcoib->hfc_rfci_qps_per_port); 105 n_port_ids[offset] = src_id; 106 107 status = hermon_config_fc_cmd_post(state, n_port_ids, 1, 108 HERMON_HW_FC_CONF_NPORT, fcoib->hfc_rfci_qps_per_port, 109 port + 1, HERMON_CMD_NOSLEEP_SPIN); 110 if (status != HERMON_CMD_SUCCESS) { 111 HERMON_WARNING(state, "fcoib_set_id failed"); 112 mutex_exit(&fcoib->hfc_lock); 113 return (DDI_FAILURE); 114 } 115 mutex_exit(&fcoib->hfc_lock); 116 return (DDI_SUCCESS); 117 } 118 119 /* 120 * hermon_fcoib_get_id_idx() 121 * Context: user or kernel context 122 */ 123 int 124 hermon_fcoib_get_id_idx(hermon_state_t *state, int port, ibt_fc_attr_t *fcp) 125 { 126 hermon_fcoib_t *fcoib; 127 int idx; 128 129 port--; /* passed in as 1 or 2, used as 0 or 1 */ 130 ASSERT(port >= 0 && port < HERMON_MAX_PORTS); 131 fcoib = &state->hs_fcoib; 132 133 idx = fcp->fc_rfci_qpn - fcoib->hfc_rfci_base[port]; 134 if (idx < 0 || idx >= fcoib->hfc_rfci_qps_per_port) 135 idx = -1; 136 137 return (idx); 138 } 139 140 /* 141 * hermon_fcoib_get_exch_base() 142 * Context: user or kernel context 143 */ 144 int 145 hermon_fcoib_check_exch_base_off(hermon_state_t *state, int port, 146 ibt_fc_attr_t *fcp) 147 { 148 hermon_fcoib_t *fcoib; 149 int exch_base_off; 150 151 port--; /* passed in as 1 or 2, used as 0 or 1 */ 152 ASSERT(port >= 0 && port < HERMON_MAX_PORTS); 153 fcoib = &state->hs_fcoib; 154 155 exch_base_off = fcp->fc_exch_base_off; 156 if (exch_base_off >= fcoib->hfc_fexch_qps_per_port) 157 exch_base_off = -1; 158 159 return (exch_base_off); 160 } 161 162 /* 163 * hermon_fcoib_qpnum_from_fexch() 164 * Context: user, kernel, or interrupt context 165 */ 166 int 167 hermon_fcoib_is_fexch_qpn(hermon_state_t *state, uint_t qpnum) 168 { 169 hermon_fcoib_t *fcoib; 170 171 fcoib = &state->hs_fcoib; 172 qpnum -= fcoib->hfc_fexch_rsrc->hr_indx; 173 return (qpnum < fcoib->hfc_nports * fcoib->hfc_fexch_qps_per_port); 174 } 175 176 /* 177 * hermon_fcoib_qpnum_from_fexch() 178 * Context: user, kernel, or interrupt context 179 */ 180 uint_t 181 hermon_fcoib_qpnum_from_fexch(hermon_state_t *state, int port, 182 uint16_t fexch) 183 { 184 hermon_fcoib_t *fcoib; 185 uint_t qpnum; 186 187 port--; /* passed in as 1 or 2, used as 0 or 1 */ 188 ASSERT(port >= 0 && port < HERMON_MAX_PORTS); 189 fcoib = &state->hs_fcoib; 190 qpnum = fexch + fcoib->hfc_fexch_base[port]; 191 return (qpnum); 192 } 193 194 /* 195 * hermon_fcoib_qpn_to_mkey 196 * Context: user or kernel context 197 */ 198 uint32_t 199 hermon_fcoib_qpn_to_mkey(hermon_state_t *state, uint_t qpnum) 200 { 201 int i; 202 hermon_fcoib_t *fcoib; 203 uint32_t qp_indx; 204 205 fcoib = &state->hs_fcoib; 206 for (i = 0; i < fcoib->hfc_nports; i++) { 207 qp_indx = qpnum - fcoib->hfc_fexch_base[i]; 208 if (qp_indx < fcoib->hfc_fexch_qps_per_port) 209 return ((qp_indx + fcoib->hfc_mpt_base[i]) << 8); 210 } 211 return ((uint32_t)-1); /* cannot get here with valid qpnum argument */ 212 } 213 214 /* 215 * hermon_fcoib_fexch_relative_qpn() 216 * Context: user or kernel context 217 */ 218 uint32_t 219 hermon_fcoib_fexch_relative_qpn(hermon_state_t *state, uint8_t port, 220 uint32_t qp_indx) 221 { 222 port--; 223 ASSERT(port < HERMON_MAX_PORTS); 224 qp_indx -= state->hs_fcoib.hfc_fexch_base[port]; 225 return (qp_indx); 226 } 227 228 /* 229 * hermon_fcoib_fexch_mkey_init() 230 * Context: user or kernel context 231 */ 232 int 233 hermon_fcoib_fexch_mkey_init(hermon_state_t *state, hermon_pdhdl_t pd, 234 uint8_t port, uint32_t qp_indx, uint_t sleep) 235 { 236 int status; 237 uint32_t mpt_indx; 238 uint_t nummtt; 239 uint64_t mtt_addr; 240 hermon_fcoib_t *fcoib; 241 242 port--; 243 ASSERT(port < HERMON_MAX_PORTS); 244 fcoib = &state->hs_fcoib; 245 qp_indx -= fcoib->hfc_fexch_base[port]; /* relative to FEXCH base */ 246 if (qp_indx > fcoib->hfc_fexch_qps_per_port) 247 return (IBT_INVALID_PARAM); 248 mpt_indx = qp_indx + fcoib->hfc_mpt_base[port]; 249 nummtt = fcoib->hfc_mtts_per_mpt; 250 mtt_addr = ((uint64_t)qp_indx * nummtt + fcoib->hfc_mtt_base[port]) << 251 HERMON_MTT_SIZE_SHIFT; 252 253 status = hermon_mr_fexch_mpt_init(state, pd, mpt_indx, 254 nummtt, mtt_addr, sleep); 255 return (status); 256 } 257 258 /* 259 * hermon_fcoib_fexch_mkey_fini() 260 * Context: user or kernel context 261 */ 262 int 263 hermon_fcoib_fexch_mkey_fini(hermon_state_t *state, hermon_pdhdl_t pd, 264 uint32_t qpnum, uint_t sleep) 265 { 266 int status; 267 uint8_t port; 268 uint32_t qp_indx; 269 uint32_t mpt_indx; 270 hermon_fcoib_t *fcoib; 271 272 fcoib = &state->hs_fcoib; 273 for (port = 0; port < fcoib->hfc_nports; port++) { 274 qp_indx = qpnum - fcoib->hfc_fexch_base[port]; 275 if (qp_indx < fcoib->hfc_fexch_qps_per_port) 276 goto found; 277 } 278 return (IBT_INVALID_PARAM); 279 found: 280 /* qp_indx relative to FEXCH base */ 281 mpt_indx = qp_indx + fcoib->hfc_mpt_base[port]; 282 283 status = hermon_mr_fexch_mpt_fini(state, pd, mpt_indx, sleep); 284 return (status); 285 } 286 287 /* 288 * hermon_fcoib_query_fc() 289 * Context: user or kernel context 290 */ 291 void 292 hermon_fcoib_query_fc(hermon_state_t *state, hermon_fcoib_t *fcoib) 293 { 294 int status; 295 struct hermon_hw_query_fc_s query_fc; 296 297 status = hermon_cmn_query_cmd_post(state, QUERY_FC, 0, 0, &query_fc, 298 sizeof (query_fc), HERMON_CMD_NOSLEEP_SPIN); 299 if (status == HERMON_CMD_SUCCESS) { 300 fcoib->hfc_log2_max_port_ids_queried = query_fc.log2_max_nports; 301 fcoib->hfc_log2_max_fexch_queried = query_fc.log2_max_fexch; 302 fcoib->hfc_log2_max_rfci_queried = query_fc.log2_max_rfci; 303 } else 304 cmn_err(CE_CONT, "!query_fc status 0x%x\n", status); 305 } 306 307 /* 308 * hermon_fcoib_init() 309 * Context: Only called from attach() path context 310 */ 311 int 312 hermon_fcoib_init(hermon_state_t *state) 313 { 314 hermon_fcoib_t *fcoib; 315 uint_t numports; 316 char string[128]; 317 int i; 318 uintptr_t vmemstart = (uintptr_t)0x10000000; 319 320 /* used for fast checking for FCoIB during cqe_consume */ 321 state->hs_fcoib_may_be_running = B_FALSE; 322 323 if ((state->hs_ibtfinfo.hca_attr->hca_flags2 & IBT_HCA2_FC) == 0) 324 return (DDI_SUCCESS); 325 326 fcoib = &state->hs_fcoib; 327 bzero(fcoib, sizeof (*fcoib)); 328 329 hermon_fcoib_query_fc(state, fcoib); 330 331 mutex_init(&fcoib->hfc_lock, NULL, MUTEX_DRIVER, NULL); 332 mutex_enter(&fcoib->hfc_lock); 333 334 /* use a ROUND value that works on both 32 and 64-bit kernels */ 335 fcoib->hfc_vmemstart = vmemstart; 336 337 fcoib->hfc_nports = numports = state->hs_cfg_profile->cp_num_ports; 338 fcoib->hfc_fexch_qps_per_port = 339 1 << state->hs_ibtfinfo.hca_attr->hca_fexch_max_log2_qp; 340 fcoib->hfc_mpts_per_port = fcoib->hfc_fexch_qps_per_port * 2; 341 fcoib->hfc_mtts_per_mpt = 342 (1 << state->hs_ibtfinfo.hca_attr->hca_fexch_max_log2_mem) >> 343 PAGESHIFT; 344 fcoib->hfc_rfci_qps_per_port = 345 1 << state->hs_ibtfinfo.hca_attr->hca_rfci_max_log2_qp; 346 347 if (hermon_rsrc_reserve(state, HERMON_DMPT, numports * 348 fcoib->hfc_mpts_per_port, HERMON_SLEEP, 349 &fcoib->hfc_mpt_rsrc) != DDI_SUCCESS) { 350 mutex_exit(&fcoib->hfc_lock); 351 hermon_fcoib_fini(state); 352 return (DDI_FAILURE); 353 } 354 355 /* 356 * Only reserve MTTs for the Primary MPTs (first half of the 357 * range for each port). 358 */ 359 if (hermon_rsrc_reserve(state, HERMON_MTT, numports * 360 fcoib->hfc_mpts_per_port * fcoib->hfc_mtts_per_mpt / 2, 361 HERMON_SLEEP, &fcoib->hfc_mtt_rsrc) != DDI_SUCCESS) { 362 mutex_exit(&fcoib->hfc_lock); 363 hermon_fcoib_fini(state); 364 return (DDI_FAILURE); 365 } 366 if (hermon_rsrc_reserve(state, HERMON_QPC, numports * 367 fcoib->hfc_fexch_qps_per_port, HERMON_SLEEP, 368 &fcoib->hfc_fexch_rsrc) != DDI_SUCCESS) { 369 mutex_exit(&fcoib->hfc_lock); 370 hermon_fcoib_fini(state); 371 return (DDI_FAILURE); 372 } 373 if (hermon_rsrc_reserve(state, HERMON_QPC, numports * 374 fcoib->hfc_rfci_qps_per_port, HERMON_SLEEP, 375 &fcoib->hfc_rfci_rsrc) != DDI_SUCCESS) { 376 mutex_exit(&fcoib->hfc_lock); 377 hermon_fcoib_fini(state); 378 return (DDI_FAILURE); 379 } 380 381 for (i = 0; i < numports; i++) { 382 fcoib->hfc_port_enabled[i] = 0; 383 fcoib->hfc_n_port_ids[i] = kmem_zalloc(sizeof (uint32_t) * 384 fcoib->hfc_rfci_qps_per_port, KM_SLEEP); 385 386 fcoib->hfc_mpt_base[i] = i * fcoib->hfc_mpts_per_port + 387 fcoib->hfc_mpt_rsrc->hr_indx; 388 /* "/ 2" is for Secondary MKEYs never used on Client side */ 389 fcoib->hfc_mtt_base[i] = (i * fcoib->hfc_mpts_per_port * 390 fcoib->hfc_mtts_per_mpt / 2) + fcoib->hfc_mtt_rsrc->hr_indx; 391 fcoib->hfc_fexch_base[i] = i * fcoib->hfc_fexch_qps_per_port + 392 fcoib->hfc_fexch_rsrc->hr_indx; 393 fcoib->hfc_rfci_base[i] = i * fcoib->hfc_rfci_qps_per_port + 394 fcoib->hfc_rfci_rsrc->hr_indx; 395 396 /* init FEXCH QP rsrc pool */ 397 (void) sprintf(string, "hermon%d_port%d_fexch_vmem", 398 state->hs_instance, i + 1); 399 fcoib->hfc_fexch_vmemp[i] = vmem_create(string, 400 (void *)vmemstart, fcoib->hfc_fexch_qps_per_port, 401 1, NULL, NULL, NULL, 0, VM_SLEEP); 402 403 /* init RFCI QP rsrc pool */ 404 (void) sprintf(string, "hermon%d_port%d_rfci_vmem", 405 state->hs_instance, i + 1); 406 fcoib->hfc_rfci_vmemp[i] = vmem_create(string, 407 (void *)vmemstart, fcoib->hfc_rfci_qps_per_port, 408 1, NULL, NULL, NULL, 0, VM_SLEEP); 409 } 410 411 mutex_exit(&fcoib->hfc_lock); 412 413 return (DDI_SUCCESS); 414 } 415 416 417 /* 418 * hermon_fcoib_fini() 419 * Context: Only called from attach() and/or detach() path contexts 420 */ 421 void 422 hermon_fcoib_fini(hermon_state_t *state) 423 { 424 hermon_fcoib_t *fcoib; 425 uint_t numports; 426 int i; 427 428 if ((state->hs_ibtfinfo.hca_attr->hca_flags2 & IBT_HCA2_FC) == 0) 429 return; 430 431 fcoib = &state->hs_fcoib; 432 433 mutex_enter(&fcoib->hfc_lock); 434 435 numports = fcoib->hfc_nports; 436 437 for (i = 0; i < numports; i++) { 438 if (fcoib->hfc_rfci_vmemp[i]) 439 vmem_destroy(fcoib->hfc_rfci_vmemp[i]); 440 if (fcoib->hfc_fexch_vmemp[i]) 441 vmem_destroy(fcoib->hfc_fexch_vmemp[i]); 442 if (fcoib->hfc_n_port_ids[i]) 443 kmem_free(fcoib->hfc_n_port_ids[i], sizeof (uint32_t) * 444 fcoib->hfc_rfci_qps_per_port); 445 446 /* XXX --- should we issue HERMON_HW_FC_CONF_BASIC disable? */ 447 fcoib->hfc_port_enabled[i] = 0; 448 } 449 if (fcoib->hfc_rfci_rsrc) 450 hermon_rsrc_free(state, &fcoib->hfc_rfci_rsrc); 451 if (fcoib->hfc_fexch_rsrc) 452 hermon_rsrc_free(state, &fcoib->hfc_fexch_rsrc); 453 if (fcoib->hfc_mpt_rsrc) 454 hermon_rsrc_free(state, &fcoib->hfc_mpt_rsrc); 455 if (fcoib->hfc_mtt_rsrc) 456 hermon_rsrc_free(state, &fcoib->hfc_mtt_rsrc); 457 458 mutex_exit(&fcoib->hfc_lock); 459 mutex_destroy(&fcoib->hfc_lock); 460 461 bzero(fcoib, sizeof (*fcoib)); 462 } 463