1 /*
2 * Copyright (c) 2006-2016 Chelsio, Inc. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32 #include <config.h>
33
34 #include <stdio.h>
35 #include <stdlib.h>
36 #include <unistd.h>
37 #include <errno.h>
38 #include <sys/mman.h>
39 #include <pthread.h>
40 #include <string.h>
41 #include <signal.h>
42 #include <stdbool.h>
43
44 #include "libcxgb4.h"
45 #include "cxgb4-abi.h"
46
47 #define PCI_VENDOR_ID_CHELSIO 0x1425
48
49 /*
50 * Macros needed to support the PCI Device ID Table ...
51 */
52 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN \
53 static struct { \
54 unsigned vendor; \
55 unsigned device; \
56 } hca_table[] = {
57
58 #define CH_PCI_DEVICE_ID_FUNCTION \
59 0x4
60
61 #define CH_PCI_ID_TABLE_ENTRY(__DeviceID) \
62 { \
63 .vendor = PCI_VENDOR_ID_CHELSIO, \
64 .device = (__DeviceID), \
65 }
66
67 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_END \
68 }
69
70 #include "t4_chip_type.h"
71 #include "t4_pci_id_tbl.h"
72
73 unsigned long c4iw_page_size;
74 unsigned long c4iw_page_shift;
75 unsigned long c4iw_page_mask;
76 int ma_wr;
77 int t5_en_wc = 1;
78
79 static TAILQ_HEAD(,c4iw_dev) devices = TAILQ_HEAD_INITIALIZER(devices);
80
81 static struct ibv_context_ops c4iw_ctx_ops = {
82 .query_device = c4iw_query_device,
83 .query_port = c4iw_query_port,
84 .alloc_pd = c4iw_alloc_pd,
85 .dealloc_pd = c4iw_free_pd,
86 .reg_mr = c4iw_reg_mr,
87 .dereg_mr = c4iw_dereg_mr,
88 .create_cq = c4iw_create_cq,
89 .resize_cq = c4iw_resize_cq,
90 .destroy_cq = c4iw_destroy_cq,
91 .create_srq = c4iw_create_srq,
92 .modify_srq = c4iw_modify_srq,
93 .destroy_srq = c4iw_destroy_srq,
94 .create_qp = c4iw_create_qp,
95 .modify_qp = c4iw_modify_qp,
96 .destroy_qp = c4iw_destroy_qp,
97 .query_qp = c4iw_query_qp,
98 .create_ah = c4iw_create_ah,
99 .destroy_ah = c4iw_destroy_ah,
100 .attach_mcast = c4iw_attach_mcast,
101 .detach_mcast = c4iw_detach_mcast,
102 .post_srq_recv = c4iw_post_srq_recv,
103 .req_notify_cq = c4iw_arm_cq,
104 };
105
c4iw_alloc_context(struct ibv_device * ibdev,int cmd_fd)106 static struct ibv_context *c4iw_alloc_context(struct ibv_device *ibdev,
107 int cmd_fd)
108 {
109 struct c4iw_context *context;
110 struct ibv_get_context cmd;
111 struct c4iw_alloc_ucontext_resp resp;
112 struct c4iw_dev *rhp = to_c4iw_dev(ibdev);
113 struct ibv_query_device qcmd;
114 uint64_t raw_fw_ver;
115 struct ibv_device_attr attr;
116
117 context = malloc(sizeof *context);
118 if (!context)
119 return NULL;
120
121 memset(context, 0, sizeof *context);
122 context->ibv_ctx.cmd_fd = cmd_fd;
123
124 resp.status_page_size = 0;
125 resp.reserved = 0;
126 if (ibv_cmd_get_context(&context->ibv_ctx, &cmd, sizeof cmd,
127 &resp.ibv_resp, sizeof resp))
128 goto err_free;
129
130 if (resp.reserved)
131 PDBG("%s c4iw_alloc_ucontext_resp reserved field modified by kernel\n",
132 __FUNCTION__);
133
134 context->status_page_size = resp.status_page_size;
135 if (resp.status_page_size) {
136 context->status_page = mmap(NULL, resp.status_page_size,
137 PROT_READ, MAP_SHARED, cmd_fd,
138 resp.status_page_key);
139 if (context->status_page == MAP_FAILED)
140 goto err_free;
141 }
142
143 context->ibv_ctx.device = ibdev;
144 context->ibv_ctx.ops = c4iw_ctx_ops;
145
146 switch (rhp->chip_version) {
147 case CHELSIO_T7:
148 PDBG("%s T7/T6/T5/T4 device\n", __FUNCTION__);
149 case CHELSIO_T6:
150 PDBG("%s T6/T5/T4 device\n", __FUNCTION__);
151 case CHELSIO_T5:
152 PDBG("%s T5/T4 device\n", __FUNCTION__);
153 case CHELSIO_T4:
154 PDBG("%s T4 device\n", __FUNCTION__);
155 context->ibv_ctx.ops.async_event = c4iw_async_event;
156 context->ibv_ctx.ops.post_send = c4iw_post_send;
157 context->ibv_ctx.ops.post_recv = c4iw_post_receive;
158 context->ibv_ctx.ops.poll_cq = c4iw_poll_cq;
159 context->ibv_ctx.ops.req_notify_cq = c4iw_arm_cq;
160 break;
161 default:
162 PDBG("%s unknown hca type %d\n", __FUNCTION__,
163 rhp->chip_version);
164 goto err_unmap;
165 break;
166 }
167
168 if (!rhp->mmid2ptr) {
169 int ret;
170
171 ret = ibv_cmd_query_device(&context->ibv_ctx, &attr, &raw_fw_ver, &qcmd,
172 sizeof qcmd);
173 if (ret)
174 goto err_unmap;
175 rhp->max_mr = attr.max_mr;
176 rhp->mmid2ptr = calloc(attr.max_mr, sizeof(void *));
177 if (!rhp->mmid2ptr) {
178 goto err_unmap;
179 }
180 if (rhp->abi_version < 3) {
181 fprintf(stderr, "Warning: iw_cxgb4 driver is of older version"
182 " than libcxgb4:: %d\n", rhp->abi_version);
183 rhp->max_qp = T4_QID_BASE + attr.max_qp;
184 } else {
185 rhp->max_qp = context->status_page->qp_start +
186 context->status_page->qp_size;
187 }
188 rhp->qpid2ptr = calloc(rhp->max_qp, sizeof(void *));
189 if (!rhp->qpid2ptr) {
190 goto err_unmap;
191 }
192 if (rhp->abi_version < 3)
193 rhp->max_cq = T4_QID_BASE + attr.max_cq;
194 else
195 rhp->max_cq = context->status_page->cq_start +
196 context->status_page->cq_size;
197 rhp->cqid2ptr = calloc(rhp->max_cq, sizeof(void *));
198 if (!rhp->cqid2ptr)
199 goto err_unmap;
200
201 /* Disable userspace WC if architecture/adapter does not
202 * support WC.
203 * Note: To forcefully disable WC in kernel driver use the
204 * loader tunable "hw.cxl.write_combine=0"
205 */
206 if (t5_en_wc && !context->status_page->wc_supported) {
207 t5_en_wc = 0;
208 }
209 rhp->write_cmpl_supported = context->status_page->write_cmpl_supported;
210 }
211
212 return &context->ibv_ctx;
213
214 err_unmap:
215 munmap(context->status_page, context->status_page_size);
216 err_free:
217 if (rhp->cqid2ptr)
218 free(rhp->cqid2ptr);
219 if (rhp->qpid2ptr)
220 free(rhp->qpid2ptr);
221 if (rhp->mmid2ptr)
222 free(rhp->mmid2ptr);
223 free(context);
224 return NULL;
225 }
226
c4iw_free_context(struct ibv_context * ibctx)227 static void c4iw_free_context(struct ibv_context *ibctx)
228 {
229 struct c4iw_context *context = to_c4iw_context(ibctx);
230
231 if (context->status_page_size)
232 munmap(context->status_page, context->status_page_size);
233 free(context);
234 }
235
236 static struct verbs_device_ops c4iw_dev_ops = {
237 .alloc_context = c4iw_alloc_context,
238 .free_context = c4iw_free_context
239 };
240
241 #ifdef STALL_DETECTION
242
243 int stall_to;
244
dump_cq(struct c4iw_cq * chp)245 static void dump_cq(struct c4iw_cq *chp)
246 {
247 int i;
248
249 fprintf(stderr,
250 "CQ: %p id %u queue %p cidx 0x%08x sw_queue %p sw_cidx %d sw_pidx %d sw_in_use %d depth %u error %u gen %d "
251 "cidx_inc %d bits_type_ts %016" PRIx64 " notempty %d\n", chp,
252 chp->cq.cqid, chp->cq.queue, chp->cq.cidx,
253 chp->cq.sw_queue, chp->cq.sw_cidx, chp->cq.sw_pidx, chp->cq.sw_in_use,
254 chp->cq.size, chp->cq.error, chp->cq.gen, chp->cq.cidx_inc, be64toh(chp->cq.bits_type_ts),
255 t4_cq_notempty(&chp->cq));
256
257 for (i=0; i < chp->cq.size; i++) {
258 u64 *p = (u64 *)(chp->cq.queue + i);
259
260 fprintf(stderr, "%02x: %016" PRIx64 " %016" PRIx64, i, be64toh(p[0]), be64toh(p[1]));
261 if (i == chp->cq.cidx)
262 fprintf(stderr, " <-- cidx\n");
263 else
264 fprintf(stderr, "\n");
265 p+= 2;
266 fprintf(stderr, "%02x: %016" PRIx64 " %016" PRIx64 "\n", i, be64toh(p[0]), be64toh(p[1]));
267 p+= 2;
268 fprintf(stderr, "%02x: %016" PRIx64 " %016" PRIx64 "\n", i, be64toh(p[0]), be64toh(p[1]));
269 p+= 2;
270 fprintf(stderr, "%02x: %016" PRIx64 " %016" PRIx64 "\n", i, be64toh(p[0]), be64toh(p[1]));
271 p+= 2;
272 }
273 }
274
dump_qp(struct c4iw_qp * qhp)275 static void dump_qp(struct c4iw_qp *qhp)
276 {
277 int i;
278 int j;
279 struct t4_swsqe *swsqe;
280 struct t4_swrqe *swrqe;
281 u16 cidx, pidx;
282 u64 *p;
283
284 fprintf(stderr,
285 "QP: %p id %u error %d flushed %d qid_mask 0x%x\n"
286 " SQ: id %u queue %p sw_queue %p cidx %u pidx %u in_use %u wq_pidx %u depth %u flags 0x%x flush_cidx %d\n"
287 " RQ: id %u queue %p sw_queue %p cidx %u pidx %u in_use %u depth %u\n",
288 qhp,
289 qhp->wq.sq.qid,
290 qhp->wq.error,
291 qhp->wq.flushed,
292 qhp->wq.qid_mask,
293 qhp->wq.sq.qid,
294 qhp->wq.sq.queue,
295 qhp->wq.sq.sw_sq,
296 qhp->wq.sq.cidx,
297 qhp->wq.sq.pidx,
298 qhp->wq.sq.in_use,
299 qhp->wq.sq.wq_pidx,
300 qhp->wq.sq.size,
301 qhp->wq.sq.flags,
302 qhp->wq.sq.flush_cidx,
303 qhp->wq.rq.qid,
304 qhp->wq.rq.queue,
305 qhp->wq.rq.sw_rq,
306 qhp->wq.rq.cidx,
307 qhp->wq.rq.pidx,
308 qhp->wq.rq.in_use,
309 qhp->wq.rq.size);
310 cidx = qhp->wq.sq.cidx;
311 pidx = qhp->wq.sq.pidx;
312 if (cidx != pidx)
313 fprintf(stderr, "SQ: \n");
314 while (cidx != pidx) {
315 swsqe = &qhp->wq.sq.sw_sq[cidx];
316 fprintf(stderr, "%04u: wr_id %016" PRIx64
317 " sq_wptr %08x read_len %u opcode 0x%x "
318 "complete %u signaled %u cqe %016" PRIx64 " %016" PRIx64 " %016" PRIx64 " %016" PRIx64 "\n",
319 cidx,
320 swsqe->wr_id,
321 swsqe->idx,
322 swsqe->read_len,
323 swsqe->opcode,
324 swsqe->complete,
325 swsqe->signaled,
326 htobe64(((uint64_t *)&swsqe->cqe)[0]),
327 htobe64(((uint64_t *)&swsqe->cqe)[1]),
328 htobe64(((uint64_t *)&swsqe->cqe)[2]),
329 htobe64(((uint64_t *)&swsqe->cqe)[3]));
330 if (++cidx == qhp->wq.sq.size)
331 cidx = 0;
332 }
333
334 fprintf(stderr, "SQ WQ: \n");
335 p = (u64 *)qhp->wq.sq.queue;
336 for (i=0; i < qhp->wq.sq.size * T4_SQ_NUM_SLOTS; i++) {
337 for (j=0; j < T4_EQ_ENTRY_SIZE / 16; j++) {
338 fprintf(stderr, "%04u %016" PRIx64 " %016" PRIx64 " ",
339 i, be64toh(p[0]), be64toh(p[1]));
340 if (j == 0 && i == qhp->wq.sq.wq_pidx)
341 fprintf(stderr, " <-- pidx");
342 fprintf(stderr, "\n");
343 p += 2;
344 }
345 }
346 cidx = qhp->wq.rq.cidx;
347 pidx = qhp->wq.rq.pidx;
348 if (cidx != pidx)
349 fprintf(stderr, "RQ: \n");
350 while (cidx != pidx) {
351 swrqe = &qhp->wq.rq.sw_rq[cidx];
352 fprintf(stderr, "%04u: wr_id %016" PRIx64 "\n",
353 cidx,
354 swrqe->wr_id );
355 if (++cidx == qhp->wq.rq.size)
356 cidx = 0;
357 }
358
359 fprintf(stderr, "RQ WQ: \n");
360 p = (u64 *)qhp->wq.rq.queue;
361 for (i=0; i < qhp->wq.rq.size * T4_RQ_NUM_SLOTS; i++) {
362 for (j=0; j < T4_EQ_ENTRY_SIZE / 16; j++) {
363 fprintf(stderr, "%04u %016" PRIx64 " %016" PRIx64 " ",
364 i, be64toh(p[0]), be64toh(p[1]));
365 if (j == 0 && i == qhp->wq.rq.pidx)
366 fprintf(stderr, " <-- pidx");
367 if (j == 0 && i == qhp->wq.rq.cidx)
368 fprintf(stderr, " <-- cidx");
369 fprintf(stderr, "\n");
370 p+=2;
371 }
372 }
373 }
374
dump_state(void)375 void dump_state(void)
376 {
377 struct c4iw_dev *dev;
378 int i;
379
380 fprintf(stderr, "STALL DETECTED:\n");
381 TAILQ_FOREACH(dev, &devices, list) {
382 //pthread_spin_lock(&dev->lock);
383 fprintf(stderr, "Device %s\n", dev->ibv_dev.name);
384 for (i=0; i < dev->max_cq; i++) {
385 if (dev->cqid2ptr[i]) {
386 struct c4iw_cq *chp = dev->cqid2ptr[i];
387 //pthread_spin_lock(&chp->lock);
388 dump_cq(chp);
389 //pthread_spin_unlock(&chp->lock);
390 }
391 }
392 for (i=0; i < dev->max_qp; i++) {
393 if (dev->qpid2ptr[i]) {
394 struct c4iw_qp *qhp = dev->qpid2ptr[i];
395 //pthread_spin_lock(&qhp->lock);
396 dump_qp(qhp);
397 //pthread_spin_unlock(&qhp->lock);
398 }
399 }
400 //pthread_spin_unlock(&dev->lock);
401 }
402 fprintf(stderr, "DUMP COMPLETE:\n");
403 fflush(stderr);
404 }
405 #endif /* end of STALL_DETECTION */
406
407 /*
408 * c4iw_abi_version is used to store ABI for iw_cxgb4 so the user mode library
409 * can know if the driver supports the kernel mode db ringing.
410 */
411 int c4iw_abi_version = 1;
412
cxgb4_driver_init(const char * uverbs_sys_path,int abi_version)413 static struct verbs_device *cxgb4_driver_init(const char *uverbs_sys_path,
414 int abi_version)
415 {
416 char devstr[IBV_SYSFS_PATH_MAX], ibdev[16], value[128], *cp;
417 char dev_str[IBV_SYSFS_PATH_MAX];
418 struct c4iw_dev *dev;
419 unsigned vendor, device, fw_maj, fw_min;
420 int i;
421 char devnum;
422 char ib_param[16];
423
424 #ifndef __linux__
425 if (ibv_read_sysfs_file(uverbs_sys_path, "ibdev",
426 ibdev, sizeof ibdev) < 0)
427 return NULL;
428
429 devnum = atoi(&ibdev[5]);
430
431 if (ibdev[0] == 't' && ibdev[1] >= '4' && ibdev[1] <= '6' &&
432 strstr(&ibdev[2], "nex") && devnum >= 0) {
433 snprintf(dev_str, sizeof(dev_str), "/dev/t%cnex/%d", ibdev[1],
434 devnum);
435 } else if (strstr(&ibdev[0], "chnex") && devnum >= 0) {
436 snprintf(dev_str, sizeof(dev_str), "/dev/chnex/%d", devnum);
437 } else
438 return NULL;
439
440 if (ibv_read_sysfs_file(dev_str, "\%pnpinfo", value, sizeof value) < 0)
441 return NULL;
442 else {
443 if (strstr(value, "vendor=")) {
444 strncpy(ib_param, strstr(value, "vendor=") +
445 strlen("vendor="), 6);
446 sscanf(ib_param, "%i", &vendor);
447 }
448
449 if (strstr(value, "device=")) {
450 strncpy(ib_param, strstr(value, "device=") +
451 strlen("device="), 6);
452 sscanf(ib_param, "%i", &device);
453 }
454 }
455 #else
456 if (ibv_read_sysfs_file(uverbs_sys_path, "device/vendor",
457 value, sizeof value) < 0)
458 return NULL;
459 sscanf(value, "%i", &vendor);
460
461 if (ibv_read_sysfs_file(uverbs_sys_path, "device/device",
462 value, sizeof value) < 0)
463 return NULL;
464 sscanf(value, "%i", &device);
465 #endif
466
467 for (i = 0; i < sizeof hca_table / sizeof hca_table[0]; ++i)
468 if (vendor == hca_table[i].vendor &&
469 device == hca_table[i].device)
470 goto found;
471
472 return NULL;
473
474 found:
475 c4iw_abi_version = abi_version;
476
477 #ifndef __linux__
478 if (ibv_read_sysfs_file(dev_str, "firmware_version",
479 value, sizeof value) < 0)
480 return NULL;
481 #else
482 /*
483 * Verify that the firmware major number matches. Major number
484 * mismatches are fatal. Minor number mismatches are tolerated.
485 */
486 if (ibv_read_sysfs_file(uverbs_sys_path, "ibdev",
487 ibdev, sizeof ibdev) < 0)
488 return NULL;
489
490 memset(devstr, 0, sizeof devstr);
491 snprintf(devstr, sizeof devstr, "%s/class/infiniband/%s",
492 ibv_get_sysfs_path(), ibdev);
493 if (ibv_read_sysfs_file(devstr, "fw_ver", value, sizeof value) < 0)
494 return NULL;
495 #endif
496
497 cp = strtok(value+1, ".");
498 sscanf(cp, "%i", &fw_maj);
499 cp = strtok(NULL, ".");
500 sscanf(cp, "%i", &fw_min);
501
502 if ((signed int)fw_maj < FW_MAJ) {
503 fprintf(stderr, "libcxgb4: Fatal firmware version mismatch. "
504 "Firmware major number is %u and libcxgb4 needs %u.\n",
505 fw_maj, FW_MAJ);
506 fflush(stderr);
507 return NULL;
508 }
509
510 DBGLOG("libcxgb4");
511
512 if ((signed int)fw_min < FW_MIN) {
513 PDBG("libcxgb4: non-fatal firmware version mismatch. "
514 "Firmware minor number is %u and libcxgb4 needs %u.\n",
515 fw_min, FW_MIN);
516 fflush(stderr);
517 }
518
519 PDBG("%s found vendor %d device %d type %d\n",
520 __FUNCTION__, vendor, device, CHELSIO_CHIP_VERSION(hca_table[i].device >> 8));
521
522 dev = calloc(1, sizeof *dev);
523 if (!dev) {
524 return NULL;
525 }
526
527 if (pthread_spin_init(&dev->lock, PTHREAD_PROCESS_PRIVATE))
528 goto err;
529
530 dev->ibv_dev.ops = &c4iw_dev_ops;
531 if (hca_table[i].device == 0xd000)
532 dev->chip_version = CHELSIO_T7;
533 else
534 dev->chip_version = CHELSIO_CHIP_VERSION(hca_table[i].device >> 8);
535 dev->abi_version = abi_version;
536
537 PDBG("%s device claimed\n", __FUNCTION__);
538 TAILQ_INSERT_TAIL(&devices, dev, list);
539 #ifdef STALL_DETECTION
540 {
541 char *c = getenv("CXGB4_STALL_TIMEOUT");
542 if (c) {
543 stall_to = strtol(c, NULL, 0);
544 if (errno || stall_to < 0)
545 stall_to = 0;
546 }
547 }
548 #endif
549 {
550 char *c = getenv("CXGB4_MA_WR");
551 if (c) {
552 ma_wr = strtol(c, NULL, 0);
553 if (ma_wr != 1)
554 ma_wr = 0;
555 }
556 }
557 {
558 char *c = getenv("T5_ENABLE_WC");
559 if (c) {
560 t5_en_wc = strtol(c, NULL, 0);
561 if (t5_en_wc != 1)
562 t5_en_wc = 0;
563 }
564 }
565
566 return &dev->ibv_dev;
567
568 err:
569 free(dev);
570
571 return NULL;
572 }
573
cxgb4_register_driver(void)574 static __attribute__((constructor)) void cxgb4_register_driver(void)
575 {
576 c4iw_page_size = sysconf(_SC_PAGESIZE);
577 c4iw_page_shift = long_log2(c4iw_page_size);
578 c4iw_page_mask = ~(c4iw_page_size - 1);
579 verbs_register_driver("cxgb4", cxgb4_driver_init);
580 }
581
582 #ifdef STATS
583 void __attribute__ ((destructor)) cs_fini(void);
cs_fini(void)584 void __attribute__ ((destructor)) cs_fini(void)
585 {
586 syslog(LOG_NOTICE, "cxgb4 stats - sends %lu recv %lu read %lu "
587 "write %lu arm %lu cqe %lu mr %lu qp %lu cq %lu\n",
588 c4iw_stats.send, c4iw_stats.recv, c4iw_stats.read,
589 c4iw_stats.write, c4iw_stats.arm, c4iw_stats.cqe,
590 c4iw_stats.mr, c4iw_stats.qp, c4iw_stats.cq);
591 }
592 #endif
593