1 /* 2 * Copyright (c) 2017 Pure Storage, Inc. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 3. The name of the author may not be used to endorse or promote 15 * products derived from this software without specific prior written 16 * permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 */ 30 31 #ifdef HAVE_CONFIG_H 32 #include "config.h" 33 #endif 34 35 #include "pcap-int.h" 36 #include "pcap-rdmasniff.h" 37 38 #include <infiniband/verbs.h> 39 #include <stdlib.h> 40 #include <string.h> 41 #include <limits.h> /* for INT_MAX */ 42 #include <sys/time.h> 43 44 #if !defined(IBV_FLOW_ATTR_SNIFFER) 45 #define IBV_FLOW_ATTR_SNIFFER 3 46 #endif 47 48 static const int RDMASNIFF_NUM_RECEIVES = 128; 49 static const int RDMASNIFF_RECEIVE_SIZE = 10000; 50 51 struct pcap_rdmasniff { 52 struct ibv_device * rdma_device; 53 struct ibv_context * context; 54 struct ibv_comp_channel * channel; 55 struct ibv_pd * pd; 56 struct ibv_cq * cq; 57 struct ibv_qp * qp; 58 struct ibv_flow * flow; 59 struct ibv_mr * mr; 60 u_char * oneshot_buffer; 61 unsigned long port_num; 62 int cq_event; 63 u_int packets_recv; 64 }; 65 66 static int 67 rdmasniff_stats(pcap_t *handle, struct pcap_stat *stat) 68 { 69 struct pcap_rdmasniff *priv = handle->priv; 70 71 stat->ps_recv = priv->packets_recv; 72 stat->ps_drop = 0; 73 stat->ps_ifdrop = 0; 74 75 return 0; 76 } 77 78 static void 79 rdmasniff_cleanup(pcap_t *handle) 80 { 81 struct pcap_rdmasniff *priv = handle->priv; 82 83 ibv_dereg_mr(priv->mr); 84 ibv_destroy_flow(priv->flow); 85 ibv_destroy_qp(priv->qp); 86 ibv_destroy_cq(priv->cq); 87 ibv_dealloc_pd(priv->pd); 88 ibv_destroy_comp_channel(priv->channel); 89 ibv_close_device(priv->context); 90 free(priv->oneshot_buffer); 91 92 pcap_cleanup_live_common(handle); 93 } 94 95 static void 96 rdmasniff_post_recv(pcap_t *handle, uint64_t wr_id) 97 { 98 struct pcap_rdmasniff *priv = handle->priv; 99 struct ibv_sge sg_entry; 100 struct ibv_recv_wr wr, *bad_wr; 101 102 sg_entry.length = RDMASNIFF_RECEIVE_SIZE; 103 sg_entry.addr = (uintptr_t) handle->buffer + RDMASNIFF_RECEIVE_SIZE * wr_id; 104 sg_entry.lkey = priv->mr->lkey; 105 106 wr.wr_id = wr_id; 107 wr.num_sge = 1; 108 wr.sg_list = &sg_entry; 109 wr.next = NULL; 110 111 ibv_post_recv(priv->qp, &wr, &bad_wr); 112 } 113 114 static int 115 rdmasniff_read(pcap_t *handle, int max_packets, pcap_handler callback, u_char *user) 116 { 117 struct pcap_rdmasniff *priv = handle->priv; 118 struct ibv_cq *ev_cq; 119 void *ev_ctx; 120 struct ibv_wc wc; 121 struct pcap_pkthdr pkth; 122 u_char *pktd; 123 int count = 0; 124 125 if (!priv->cq_event) { 126 while (ibv_get_cq_event(priv->channel, &ev_cq, &ev_ctx) < 0) { 127 if (errno != EINTR) { 128 return PCAP_ERROR; 129 } 130 if (handle->break_loop) { 131 handle->break_loop = 0; 132 return PCAP_ERROR_BREAK; 133 } 134 } 135 ibv_ack_cq_events(priv->cq, 1); 136 ibv_req_notify_cq(priv->cq, 0); 137 priv->cq_event = 1; 138 } 139 140 /* 141 * This can conceivably process more than INT_MAX packets, 142 * which would overflow the packet count, causing it either 143 * to look like a negative number, and thus cause us to 144 * return a value that looks like an error, or overflow 145 * back into positive territory, and thus cause us to 146 * return a too-low count. 147 * 148 * Therefore, if the packet count is unlimited, we clip 149 * it at INT_MAX; this routine is not expected to 150 * process packets indefinitely, so that's not an issue. 151 */ 152 if (PACKET_COUNT_IS_UNLIMITED(max_packets)) 153 max_packets = INT_MAX; 154 155 while (count < max_packets) { 156 if (ibv_poll_cq(priv->cq, 1, &wc) != 1) { 157 priv->cq_event = 0; 158 break; 159 } 160 161 if (wc.status != IBV_WC_SUCCESS) { 162 fprintf(stderr, "failed WC wr_id %" PRIu64 " status %d/%s\n", 163 wc.wr_id, 164 wc.status, ibv_wc_status_str(wc.status)); 165 continue; 166 } 167 168 pkth.len = wc.byte_len; 169 pkth.caplen = min(pkth.len, (u_int)handle->snapshot); 170 gettimeofday(&pkth.ts, NULL); 171 172 pktd = (u_char *) handle->buffer + wc.wr_id * RDMASNIFF_RECEIVE_SIZE; 173 174 if (handle->fcode.bf_insns == NULL || 175 pcap_filter(handle->fcode.bf_insns, pktd, pkth.len, pkth.caplen)) { 176 callback(user, &pkth, pktd); 177 ++priv->packets_recv; 178 ++count; 179 } 180 181 rdmasniff_post_recv(handle, wc.wr_id); 182 183 if (handle->break_loop) { 184 handle->break_loop = 0; 185 return PCAP_ERROR_BREAK; 186 } 187 } 188 189 return count; 190 } 191 192 static void 193 rdmasniff_oneshot(u_char *user, const struct pcap_pkthdr *h, const u_char *bytes) 194 { 195 struct oneshot_userdata *sp = (struct oneshot_userdata *) user; 196 pcap_t *handle = sp->pd; 197 struct pcap_rdmasniff *priv = handle->priv; 198 199 *sp->hdr = *h; 200 memcpy(priv->oneshot_buffer, bytes, h->caplen); 201 *sp->pkt = priv->oneshot_buffer; 202 } 203 204 static int 205 rdmasniff_activate(pcap_t *handle) 206 { 207 struct pcap_rdmasniff *priv = handle->priv; 208 struct ibv_qp_init_attr qp_init_attr; 209 struct ibv_qp_attr qp_attr; 210 struct ibv_flow_attr flow_attr; 211 struct ibv_port_attr port_attr; 212 int i; 213 214 priv->context = ibv_open_device(priv->rdma_device); 215 if (!priv->context) { 216 snprintf(handle->errbuf, PCAP_ERRBUF_SIZE, 217 "Failed to open device %s", handle->opt.device); 218 goto error; 219 } 220 221 priv->pd = ibv_alloc_pd(priv->context); 222 if (!priv->pd) { 223 snprintf(handle->errbuf, PCAP_ERRBUF_SIZE, 224 "Failed to alloc PD for device %s", handle->opt.device); 225 goto error; 226 } 227 228 priv->channel = ibv_create_comp_channel(priv->context); 229 if (!priv->channel) { 230 snprintf(handle->errbuf, PCAP_ERRBUF_SIZE, 231 "Failed to create comp channel for device %s", handle->opt.device); 232 goto error; 233 } 234 235 priv->cq = ibv_create_cq(priv->context, RDMASNIFF_NUM_RECEIVES, 236 NULL, priv->channel, 0); 237 if (!priv->cq) { 238 snprintf(handle->errbuf, PCAP_ERRBUF_SIZE, 239 "Failed to create CQ for device %s", handle->opt.device); 240 goto error; 241 } 242 243 ibv_req_notify_cq(priv->cq, 0); 244 245 memset(&qp_init_attr, 0, sizeof qp_init_attr); 246 qp_init_attr.send_cq = qp_init_attr.recv_cq = priv->cq; 247 qp_init_attr.cap.max_recv_wr = RDMASNIFF_NUM_RECEIVES; 248 qp_init_attr.cap.max_recv_sge = 1; 249 qp_init_attr.qp_type = IBV_QPT_RAW_PACKET; 250 priv->qp = ibv_create_qp(priv->pd, &qp_init_attr); 251 if (!priv->qp) { 252 snprintf(handle->errbuf, PCAP_ERRBUF_SIZE, 253 "Failed to create QP for device %s", handle->opt.device); 254 goto error; 255 } 256 257 memset(&qp_attr, 0, sizeof qp_attr); 258 qp_attr.qp_state = IBV_QPS_INIT; 259 qp_attr.port_num = priv->port_num; 260 if (ibv_modify_qp(priv->qp, &qp_attr, IBV_QP_STATE | IBV_QP_PORT)) { 261 snprintf(handle->errbuf, PCAP_ERRBUF_SIZE, 262 "Failed to modify QP to INIT for device %s", handle->opt.device); 263 goto error; 264 } 265 266 memset(&qp_attr, 0, sizeof qp_attr); 267 qp_attr.qp_state = IBV_QPS_RTR; 268 if (ibv_modify_qp(priv->qp, &qp_attr, IBV_QP_STATE)) { 269 snprintf(handle->errbuf, PCAP_ERRBUF_SIZE, 270 "Failed to modify QP to RTR for device %s", handle->opt.device); 271 goto error; 272 } 273 274 memset(&flow_attr, 0, sizeof flow_attr); 275 flow_attr.type = IBV_FLOW_ATTR_SNIFFER; 276 flow_attr.size = sizeof flow_attr; 277 flow_attr.port = priv->port_num; 278 priv->flow = ibv_create_flow(priv->qp, &flow_attr); 279 if (!priv->flow) { 280 snprintf(handle->errbuf, PCAP_ERRBUF_SIZE, 281 "Failed to create flow for device %s", handle->opt.device); 282 goto error; 283 } 284 285 handle->bufsize = RDMASNIFF_NUM_RECEIVES * RDMASNIFF_RECEIVE_SIZE; 286 handle->buffer = malloc(handle->bufsize); 287 if (!handle->buffer) { 288 snprintf(handle->errbuf, PCAP_ERRBUF_SIZE, 289 "Failed to allocate receive buffer for device %s", handle->opt.device); 290 goto error; 291 } 292 293 priv->oneshot_buffer = malloc(RDMASNIFF_RECEIVE_SIZE); 294 if (!priv->oneshot_buffer) { 295 snprintf(handle->errbuf, PCAP_ERRBUF_SIZE, 296 "Failed to allocate oneshot buffer for device %s", handle->opt.device); 297 goto error; 298 } 299 300 priv->mr = ibv_reg_mr(priv->pd, handle->buffer, handle->bufsize, IBV_ACCESS_LOCAL_WRITE); 301 if (!priv->mr) { 302 snprintf(handle->errbuf, PCAP_ERRBUF_SIZE, 303 "Failed to register MR for device %s", handle->opt.device); 304 goto error; 305 } 306 307 308 for (i = 0; i < RDMASNIFF_NUM_RECEIVES; ++i) { 309 rdmasniff_post_recv(handle, i); 310 } 311 312 if (!ibv_query_port(priv->context, priv->port_num, &port_attr) && 313 port_attr.link_layer == IBV_LINK_LAYER_INFINIBAND) { 314 handle->linktype = DLT_INFINIBAND; 315 } else { 316 handle->linktype = DLT_EN10MB; 317 } 318 319 if (handle->snapshot <= 0 || handle->snapshot > RDMASNIFF_RECEIVE_SIZE) 320 handle->snapshot = RDMASNIFF_RECEIVE_SIZE; 321 322 handle->offset = 0; 323 handle->read_op = rdmasniff_read; 324 handle->stats_op = rdmasniff_stats; 325 handle->cleanup_op = rdmasniff_cleanup; 326 handle->setfilter_op = install_bpf_program; 327 handle->setdirection_op = NULL; 328 handle->set_datalink_op = NULL; 329 handle->getnonblock_op = pcap_getnonblock_fd; 330 handle->setnonblock_op = pcap_setnonblock_fd; 331 handle->oneshot_callback = rdmasniff_oneshot; 332 handle->selectable_fd = priv->channel->fd; 333 334 return 0; 335 336 error: 337 if (priv->mr) { 338 ibv_dereg_mr(priv->mr); 339 } 340 341 if (priv->flow) { 342 ibv_destroy_flow(priv->flow); 343 } 344 345 if (priv->qp) { 346 ibv_destroy_qp(priv->qp); 347 } 348 349 if (priv->cq) { 350 ibv_destroy_cq(priv->cq); 351 } 352 353 if (priv->channel) { 354 ibv_destroy_comp_channel(priv->channel); 355 } 356 357 if (priv->pd) { 358 ibv_dealloc_pd(priv->pd); 359 } 360 361 if (priv->context) { 362 ibv_close_device(priv->context); 363 } 364 365 if (priv->oneshot_buffer) { 366 free(priv->oneshot_buffer); 367 } 368 369 return PCAP_ERROR; 370 } 371 372 pcap_t * 373 rdmasniff_create(const char *device, char *ebuf, int *is_ours) 374 { 375 struct pcap_rdmasniff *priv; 376 struct ibv_device **dev_list; 377 int numdev; 378 size_t namelen; 379 const char *port; 380 unsigned long port_num; 381 int i; 382 pcap_t *p = NULL; 383 384 *is_ours = 0; 385 386 dev_list = ibv_get_device_list(&numdev); 387 if (!dev_list) { 388 return NULL; 389 } 390 if (!numdev) { 391 ibv_free_device_list(dev_list); 392 return NULL; 393 } 394 395 namelen = strlen(device); 396 397 port = strchr(device, ':'); 398 if (port) { 399 port_num = strtoul(port + 1, NULL, 10); 400 if (port_num > 0) { 401 namelen = port - device; 402 } else { 403 port_num = 1; 404 } 405 } else { 406 port_num = 1; 407 } 408 409 for (i = 0; i < numdev; ++i) { 410 if (strlen(dev_list[i]->name) == namelen && 411 !strncmp(device, dev_list[i]->name, namelen)) { 412 *is_ours = 1; 413 414 p = PCAP_CREATE_COMMON(ebuf, struct pcap_rdmasniff); 415 if (p) { 416 p->activate_op = rdmasniff_activate; 417 priv = p->priv; 418 priv->rdma_device = dev_list[i]; 419 priv->port_num = port_num; 420 } 421 break; 422 } 423 } 424 425 ibv_free_device_list(dev_list); 426 return p; 427 } 428 429 int 430 rdmasniff_findalldevs(pcap_if_list_t *devlistp, char *err_str) 431 { 432 struct ibv_device **dev_list; 433 int numdev; 434 int i; 435 int ret = 0; 436 437 dev_list = ibv_get_device_list(&numdev); 438 if (!dev_list) { 439 return 0; 440 } 441 442 for (i = 0; i < numdev; ++i) { 443 /* 444 * XXX - do the notions of "up", "running", or 445 * "connected" apply here? 446 */ 447 if (!add_dev(devlistp, dev_list[i]->name, 0, "RDMA sniffer", err_str)) { 448 ret = -1; 449 break; 450 } 451 } 452 453 ibv_free_device_list(dev_list); 454 return ret; 455 } 456