1 /* 2 * Copyright (c) 2006-2016 Chelsio, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 #include <config.h> 33 34 #include <stdio.h> 35 #include <stdlib.h> 36 #include <unistd.h> 37 #include <errno.h> 38 #include <sys/mman.h> 39 #include <pthread.h> 40 #include <string.h> 41 #include <signal.h> 42 #include <stdbool.h> 43 44 #include "libcxgb4.h" 45 #include "cxgb4-abi.h" 46 47 #define PCI_VENDOR_ID_CHELSIO 0x1425 48 49 /* 50 * Macros needed to support the PCI Device ID Table ... 51 */ 52 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN \ 53 static struct { \ 54 unsigned vendor; \ 55 unsigned device; \ 56 } hca_table[] = { 57 58 #define CH_PCI_DEVICE_ID_FUNCTION \ 59 0x4 60 61 #define CH_PCI_ID_TABLE_ENTRY(__DeviceID) \ 62 { \ 63 .vendor = PCI_VENDOR_ID_CHELSIO, \ 64 .device = (__DeviceID), \ 65 } 66 67 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_END \ 68 } 69 70 #include "t4_chip_type.h" 71 #include "t4_pci_id_tbl.h" 72 73 unsigned long c4iw_page_size; 74 unsigned long c4iw_page_shift; 75 unsigned long c4iw_page_mask; 76 int ma_wr; 77 int t5_en_wc = 1; 78 79 static TAILQ_HEAD(,c4iw_dev) devices = TAILQ_HEAD_INITIALIZER(devices); 80 81 static struct ibv_context_ops c4iw_ctx_ops = { 82 .query_device = c4iw_query_device, 83 .query_port = c4iw_query_port, 84 .alloc_pd = c4iw_alloc_pd, 85 .dealloc_pd = c4iw_free_pd, 86 .reg_mr = c4iw_reg_mr, 87 .dereg_mr = c4iw_dereg_mr, 88 .create_cq = c4iw_create_cq, 89 .resize_cq = c4iw_resize_cq, 90 .destroy_cq = c4iw_destroy_cq, 91 .create_srq = c4iw_create_srq, 92 .modify_srq = c4iw_modify_srq, 93 .destroy_srq = c4iw_destroy_srq, 94 .create_qp = c4iw_create_qp, 95 .modify_qp = c4iw_modify_qp, 96 .destroy_qp = c4iw_destroy_qp, 97 .query_qp = c4iw_query_qp, 98 .create_ah = c4iw_create_ah, 99 .destroy_ah = c4iw_destroy_ah, 100 .attach_mcast = c4iw_attach_mcast, 101 .detach_mcast = c4iw_detach_mcast, 102 .post_srq_recv = c4iw_post_srq_recv, 103 .req_notify_cq = c4iw_arm_cq, 104 }; 105 106 static struct ibv_context *c4iw_alloc_context(struct ibv_device *ibdev, 107 int cmd_fd) 108 { 109 struct c4iw_context *context; 110 struct ibv_get_context cmd; 111 struct c4iw_alloc_ucontext_resp resp; 112 struct c4iw_dev *rhp = to_c4iw_dev(ibdev); 113 struct ibv_query_device qcmd; 114 uint64_t raw_fw_ver; 115 struct ibv_device_attr attr; 116 117 context = malloc(sizeof *context); 118 if (!context) 119 return NULL; 120 121 memset(context, 0, sizeof *context); 122 context->ibv_ctx.cmd_fd = cmd_fd; 123 124 resp.status_page_size = 0; 125 resp.reserved = 0; 126 if (ibv_cmd_get_context(&context->ibv_ctx, &cmd, sizeof cmd, 127 &resp.ibv_resp, sizeof resp)) 128 goto err_free; 129 130 if (resp.reserved) 131 PDBG("%s c4iw_alloc_ucontext_resp reserved field modified by kernel\n", 132 __FUNCTION__); 133 134 context->status_page_size = resp.status_page_size; 135 if (resp.status_page_size) { 136 context->status_page = mmap(NULL, resp.status_page_size, 137 PROT_READ, MAP_SHARED, cmd_fd, 138 resp.status_page_key); 139 if (context->status_page == MAP_FAILED) 140 goto err_free; 141 } 142 143 context->ibv_ctx.device = ibdev; 144 context->ibv_ctx.ops = c4iw_ctx_ops; 145 146 switch (rhp->chip_version) { 147 case CHELSIO_T7: 148 PDBG("%s T7/T6/T5/T4 device\n", __FUNCTION__); 149 case CHELSIO_T6: 150 PDBG("%s T6/T5/T4 device\n", __FUNCTION__); 151 case CHELSIO_T5: 152 PDBG("%s T5/T4 device\n", __FUNCTION__); 153 case CHELSIO_T4: 154 PDBG("%s T4 device\n", __FUNCTION__); 155 context->ibv_ctx.ops.async_event = c4iw_async_event; 156 context->ibv_ctx.ops.post_send = c4iw_post_send; 157 context->ibv_ctx.ops.post_recv = c4iw_post_receive; 158 context->ibv_ctx.ops.poll_cq = c4iw_poll_cq; 159 context->ibv_ctx.ops.req_notify_cq = c4iw_arm_cq; 160 break; 161 default: 162 PDBG("%s unknown hca type %d\n", __FUNCTION__, 163 rhp->chip_version); 164 goto err_unmap; 165 break; 166 } 167 168 if (!rhp->mmid2ptr) { 169 int ret; 170 171 ret = ibv_cmd_query_device(&context->ibv_ctx, &attr, &raw_fw_ver, &qcmd, 172 sizeof qcmd); 173 if (ret) 174 goto err_unmap; 175 rhp->max_mr = attr.max_mr; 176 rhp->mmid2ptr = calloc(attr.max_mr, sizeof(void *)); 177 if (!rhp->mmid2ptr) { 178 goto err_unmap; 179 } 180 if (rhp->abi_version < 3) { 181 fprintf(stderr, "Warning: iw_cxgb4 driver is of older version" 182 " than libcxgb4:: %d\n", rhp->abi_version); 183 rhp->max_qp = T4_QID_BASE + attr.max_qp; 184 } else { 185 rhp->max_qp = context->status_page->qp_start + 186 context->status_page->qp_size; 187 } 188 rhp->qpid2ptr = calloc(rhp->max_qp, sizeof(void *)); 189 if (!rhp->qpid2ptr) { 190 goto err_unmap; 191 } 192 if (rhp->abi_version < 3) 193 rhp->max_cq = T4_QID_BASE + attr.max_cq; 194 else 195 rhp->max_cq = context->status_page->cq_start + 196 context->status_page->cq_size; 197 rhp->cqid2ptr = calloc(rhp->max_cq, sizeof(void *)); 198 if (!rhp->cqid2ptr) 199 goto err_unmap; 200 201 /* Disable userspace WC if architecture/adapter does not 202 * support WC. 203 * Note: To forcefully disable WC in kernel driver use the 204 * loader tunable "hw.cxl.write_combine=0" 205 */ 206 if (t5_en_wc && !context->status_page->wc_supported) { 207 t5_en_wc = 0; 208 } 209 rhp->write_cmpl_supported = context->status_page->write_cmpl_supported; 210 } 211 212 return &context->ibv_ctx; 213 214 err_unmap: 215 munmap(context->status_page, context->status_page_size); 216 err_free: 217 if (rhp->cqid2ptr) 218 free(rhp->cqid2ptr); 219 if (rhp->qpid2ptr) 220 free(rhp->qpid2ptr); 221 if (rhp->mmid2ptr) 222 free(rhp->mmid2ptr); 223 free(context); 224 return NULL; 225 } 226 227 static void c4iw_free_context(struct ibv_context *ibctx) 228 { 229 struct c4iw_context *context = to_c4iw_context(ibctx); 230 231 if (context->status_page_size) 232 munmap(context->status_page, context->status_page_size); 233 free(context); 234 } 235 236 static struct verbs_device_ops c4iw_dev_ops = { 237 .alloc_context = c4iw_alloc_context, 238 .free_context = c4iw_free_context 239 }; 240 241 #ifdef STALL_DETECTION 242 243 int stall_to; 244 245 static void dump_cq(struct c4iw_cq *chp) 246 { 247 int i; 248 249 fprintf(stderr, 250 "CQ: %p id %u queue %p cidx 0x%08x sw_queue %p sw_cidx %d sw_pidx %d sw_in_use %d depth %u error %u gen %d " 251 "cidx_inc %d bits_type_ts %016" PRIx64 " notempty %d\n", chp, 252 chp->cq.cqid, chp->cq.queue, chp->cq.cidx, 253 chp->cq.sw_queue, chp->cq.sw_cidx, chp->cq.sw_pidx, chp->cq.sw_in_use, 254 chp->cq.size, chp->cq.error, chp->cq.gen, chp->cq.cidx_inc, be64toh(chp->cq.bits_type_ts), 255 t4_cq_notempty(&chp->cq)); 256 257 for (i=0; i < chp->cq.size; i++) { 258 u64 *p = (u64 *)(chp->cq.queue + i); 259 260 fprintf(stderr, "%02x: %016" PRIx64 " %016" PRIx64, i, be64toh(p[0]), be64toh(p[1])); 261 if (i == chp->cq.cidx) 262 fprintf(stderr, " <-- cidx\n"); 263 else 264 fprintf(stderr, "\n"); 265 p+= 2; 266 fprintf(stderr, "%02x: %016" PRIx64 " %016" PRIx64 "\n", i, be64toh(p[0]), be64toh(p[1])); 267 p+= 2; 268 fprintf(stderr, "%02x: %016" PRIx64 " %016" PRIx64 "\n", i, be64toh(p[0]), be64toh(p[1])); 269 p+= 2; 270 fprintf(stderr, "%02x: %016" PRIx64 " %016" PRIx64 "\n", i, be64toh(p[0]), be64toh(p[1])); 271 p+= 2; 272 } 273 } 274 275 static void dump_qp(struct c4iw_qp *qhp) 276 { 277 int i; 278 int j; 279 struct t4_swsqe *swsqe; 280 struct t4_swrqe *swrqe; 281 u16 cidx, pidx; 282 u64 *p; 283 284 fprintf(stderr, 285 "QP: %p id %u error %d flushed %d qid_mask 0x%x\n" 286 " SQ: id %u queue %p sw_queue %p cidx %u pidx %u in_use %u wq_pidx %u depth %u flags 0x%x flush_cidx %d\n" 287 " RQ: id %u queue %p sw_queue %p cidx %u pidx %u in_use %u depth %u\n", 288 qhp, 289 qhp->wq.sq.qid, 290 qhp->wq.error, 291 qhp->wq.flushed, 292 qhp->wq.qid_mask, 293 qhp->wq.sq.qid, 294 qhp->wq.sq.queue, 295 qhp->wq.sq.sw_sq, 296 qhp->wq.sq.cidx, 297 qhp->wq.sq.pidx, 298 qhp->wq.sq.in_use, 299 qhp->wq.sq.wq_pidx, 300 qhp->wq.sq.size, 301 qhp->wq.sq.flags, 302 qhp->wq.sq.flush_cidx, 303 qhp->wq.rq.qid, 304 qhp->wq.rq.queue, 305 qhp->wq.rq.sw_rq, 306 qhp->wq.rq.cidx, 307 qhp->wq.rq.pidx, 308 qhp->wq.rq.in_use, 309 qhp->wq.rq.size); 310 cidx = qhp->wq.sq.cidx; 311 pidx = qhp->wq.sq.pidx; 312 if (cidx != pidx) 313 fprintf(stderr, "SQ: \n"); 314 while (cidx != pidx) { 315 swsqe = &qhp->wq.sq.sw_sq[cidx]; 316 fprintf(stderr, "%04u: wr_id %016" PRIx64 317 " sq_wptr %08x read_len %u opcode 0x%x " 318 "complete %u signaled %u cqe %016" PRIx64 " %016" PRIx64 " %016" PRIx64 " %016" PRIx64 "\n", 319 cidx, 320 swsqe->wr_id, 321 swsqe->idx, 322 swsqe->read_len, 323 swsqe->opcode, 324 swsqe->complete, 325 swsqe->signaled, 326 htobe64(((uint64_t *)&swsqe->cqe)[0]), 327 htobe64(((uint64_t *)&swsqe->cqe)[1]), 328 htobe64(((uint64_t *)&swsqe->cqe)[2]), 329 htobe64(((uint64_t *)&swsqe->cqe)[3])); 330 if (++cidx == qhp->wq.sq.size) 331 cidx = 0; 332 } 333 334 fprintf(stderr, "SQ WQ: \n"); 335 p = (u64 *)qhp->wq.sq.queue; 336 for (i=0; i < qhp->wq.sq.size * T4_SQ_NUM_SLOTS; i++) { 337 for (j=0; j < T4_EQ_ENTRY_SIZE / 16; j++) { 338 fprintf(stderr, "%04u %016" PRIx64 " %016" PRIx64 " ", 339 i, be64toh(p[0]), be64toh(p[1])); 340 if (j == 0 && i == qhp->wq.sq.wq_pidx) 341 fprintf(stderr, " <-- pidx"); 342 fprintf(stderr, "\n"); 343 p += 2; 344 } 345 } 346 cidx = qhp->wq.rq.cidx; 347 pidx = qhp->wq.rq.pidx; 348 if (cidx != pidx) 349 fprintf(stderr, "RQ: \n"); 350 while (cidx != pidx) { 351 swrqe = &qhp->wq.rq.sw_rq[cidx]; 352 fprintf(stderr, "%04u: wr_id %016" PRIx64 "\n", 353 cidx, 354 swrqe->wr_id ); 355 if (++cidx == qhp->wq.rq.size) 356 cidx = 0; 357 } 358 359 fprintf(stderr, "RQ WQ: \n"); 360 p = (u64 *)qhp->wq.rq.queue; 361 for (i=0; i < qhp->wq.rq.size * T4_RQ_NUM_SLOTS; i++) { 362 for (j=0; j < T4_EQ_ENTRY_SIZE / 16; j++) { 363 fprintf(stderr, "%04u %016" PRIx64 " %016" PRIx64 " ", 364 i, be64toh(p[0]), be64toh(p[1])); 365 if (j == 0 && i == qhp->wq.rq.pidx) 366 fprintf(stderr, " <-- pidx"); 367 if (j == 0 && i == qhp->wq.rq.cidx) 368 fprintf(stderr, " <-- cidx"); 369 fprintf(stderr, "\n"); 370 p+=2; 371 } 372 } 373 } 374 375 void dump_state(void) 376 { 377 struct c4iw_dev *dev; 378 int i; 379 380 fprintf(stderr, "STALL DETECTED:\n"); 381 TAILQ_FOREACH(dev, &devices, list) { 382 //pthread_spin_lock(&dev->lock); 383 fprintf(stderr, "Device %s\n", dev->ibv_dev.name); 384 for (i=0; i < dev->max_cq; i++) { 385 if (dev->cqid2ptr[i]) { 386 struct c4iw_cq *chp = dev->cqid2ptr[i]; 387 //pthread_spin_lock(&chp->lock); 388 dump_cq(chp); 389 //pthread_spin_unlock(&chp->lock); 390 } 391 } 392 for (i=0; i < dev->max_qp; i++) { 393 if (dev->qpid2ptr[i]) { 394 struct c4iw_qp *qhp = dev->qpid2ptr[i]; 395 //pthread_spin_lock(&qhp->lock); 396 dump_qp(qhp); 397 //pthread_spin_unlock(&qhp->lock); 398 } 399 } 400 //pthread_spin_unlock(&dev->lock); 401 } 402 fprintf(stderr, "DUMP COMPLETE:\n"); 403 fflush(stderr); 404 } 405 #endif /* end of STALL_DETECTION */ 406 407 /* 408 * c4iw_abi_version is used to store ABI for iw_cxgb4 so the user mode library 409 * can know if the driver supports the kernel mode db ringing. 410 */ 411 int c4iw_abi_version = 1; 412 413 static struct verbs_device *cxgb4_driver_init(const char *uverbs_sys_path, 414 int abi_version) 415 { 416 char devstr[IBV_SYSFS_PATH_MAX], ibdev[16], value[128], *cp; 417 char dev_str[IBV_SYSFS_PATH_MAX]; 418 struct c4iw_dev *dev; 419 unsigned vendor, device, fw_maj, fw_min; 420 int i; 421 char devnum; 422 char ib_param[16]; 423 424 #ifndef __linux__ 425 if (ibv_read_sysfs_file(uverbs_sys_path, "ibdev", 426 ibdev, sizeof ibdev) < 0) 427 return NULL; 428 429 devnum = atoi(&ibdev[5]); 430 431 if (ibdev[0] == 't' && ibdev[1] >= '4' && ibdev[1] <= '6' && 432 strstr(&ibdev[2], "nex") && devnum >= 0) { 433 snprintf(dev_str, sizeof(dev_str), "/dev/t%cnex/%d", ibdev[1], 434 devnum); 435 } else if (strstr(&ibdev[0], "chnex") && devnum >= 0) { 436 snprintf(dev_str, sizeof(dev_str), "/dev/chnex/%d", devnum); 437 } else 438 return NULL; 439 440 if (ibv_read_sysfs_file(dev_str, "\%pnpinfo", value, sizeof value) < 0) 441 return NULL; 442 else { 443 if (strstr(value, "vendor=")) { 444 strncpy(ib_param, strstr(value, "vendor=") + 445 strlen("vendor="), 6); 446 sscanf(ib_param, "%i", &vendor); 447 } 448 449 if (strstr(value, "device=")) { 450 strncpy(ib_param, strstr(value, "device=") + 451 strlen("device="), 6); 452 sscanf(ib_param, "%i", &device); 453 } 454 } 455 #else 456 if (ibv_read_sysfs_file(uverbs_sys_path, "device/vendor", 457 value, sizeof value) < 0) 458 return NULL; 459 sscanf(value, "%i", &vendor); 460 461 if (ibv_read_sysfs_file(uverbs_sys_path, "device/device", 462 value, sizeof value) < 0) 463 return NULL; 464 sscanf(value, "%i", &device); 465 #endif 466 467 for (i = 0; i < sizeof hca_table / sizeof hca_table[0]; ++i) 468 if (vendor == hca_table[i].vendor && 469 device == hca_table[i].device) 470 goto found; 471 472 return NULL; 473 474 found: 475 c4iw_abi_version = abi_version; 476 477 #ifndef __linux__ 478 if (ibv_read_sysfs_file(dev_str, "firmware_version", 479 value, sizeof value) < 0) 480 return NULL; 481 #else 482 /* 483 * Verify that the firmware major number matches. Major number 484 * mismatches are fatal. Minor number mismatches are tolerated. 485 */ 486 if (ibv_read_sysfs_file(uverbs_sys_path, "ibdev", 487 ibdev, sizeof ibdev) < 0) 488 return NULL; 489 490 memset(devstr, 0, sizeof devstr); 491 snprintf(devstr, sizeof devstr, "%s/class/infiniband/%s", 492 ibv_get_sysfs_path(), ibdev); 493 if (ibv_read_sysfs_file(devstr, "fw_ver", value, sizeof value) < 0) 494 return NULL; 495 #endif 496 497 cp = strtok(value+1, "."); 498 sscanf(cp, "%i", &fw_maj); 499 cp = strtok(NULL, "."); 500 sscanf(cp, "%i", &fw_min); 501 502 if ((signed int)fw_maj < FW_MAJ) { 503 fprintf(stderr, "libcxgb4: Fatal firmware version mismatch. " 504 "Firmware major number is %u and libcxgb4 needs %u.\n", 505 fw_maj, FW_MAJ); 506 fflush(stderr); 507 return NULL; 508 } 509 510 DBGLOG("libcxgb4"); 511 512 if ((signed int)fw_min < FW_MIN) { 513 PDBG("libcxgb4: non-fatal firmware version mismatch. " 514 "Firmware minor number is %u and libcxgb4 needs %u.\n", 515 fw_min, FW_MIN); 516 fflush(stderr); 517 } 518 519 PDBG("%s found vendor %d device %d type %d\n", 520 __FUNCTION__, vendor, device, CHELSIO_CHIP_VERSION(hca_table[i].device >> 8)); 521 522 dev = calloc(1, sizeof *dev); 523 if (!dev) { 524 return NULL; 525 } 526 527 if (pthread_spin_init(&dev->lock, PTHREAD_PROCESS_PRIVATE)) 528 goto err; 529 530 dev->ibv_dev.ops = &c4iw_dev_ops; 531 if (hca_table[i].device == 0xd000) 532 dev->chip_version = CHELSIO_T7; 533 else 534 dev->chip_version = CHELSIO_CHIP_VERSION(hca_table[i].device >> 8); 535 dev->abi_version = abi_version; 536 537 PDBG("%s device claimed\n", __FUNCTION__); 538 TAILQ_INSERT_TAIL(&devices, dev, list); 539 #ifdef STALL_DETECTION 540 { 541 char *c = getenv("CXGB4_STALL_TIMEOUT"); 542 if (c) { 543 stall_to = strtol(c, NULL, 0); 544 if (errno || stall_to < 0) 545 stall_to = 0; 546 } 547 } 548 #endif 549 { 550 char *c = getenv("CXGB4_MA_WR"); 551 if (c) { 552 ma_wr = strtol(c, NULL, 0); 553 if (ma_wr != 1) 554 ma_wr = 0; 555 } 556 } 557 { 558 char *c = getenv("T5_ENABLE_WC"); 559 if (c) { 560 t5_en_wc = strtol(c, NULL, 0); 561 if (t5_en_wc != 1) 562 t5_en_wc = 0; 563 } 564 } 565 566 return &dev->ibv_dev; 567 568 err: 569 free(dev); 570 571 return NULL; 572 } 573 574 static __attribute__((constructor)) void cxgb4_register_driver(void) 575 { 576 c4iw_page_size = sysconf(_SC_PAGESIZE); 577 c4iw_page_shift = long_log2(c4iw_page_size); 578 c4iw_page_mask = ~(c4iw_page_size - 1); 579 verbs_register_driver("cxgb4", cxgb4_driver_init); 580 } 581 582 #ifdef STATS 583 void __attribute__ ((destructor)) cs_fini(void); 584 void __attribute__ ((destructor)) cs_fini(void) 585 { 586 syslog(LOG_NOTICE, "cxgb4 stats - sends %lu recv %lu read %lu " 587 "write %lu arm %lu cqe %lu mr %lu qp %lu cq %lu\n", 588 c4iw_stats.send, c4iw_stats.recv, c4iw_stats.read, 589 c4iw_stats.write, c4iw_stats.arm, c4iw_stats.cqe, 590 c4iw_stats.mr, c4iw_stats.qp, c4iw_stats.cq); 591 } 592 #endif 593