Lines Matching refs:cb
249 struct krping_cb *cb = cma_id->context; in krping_cma_event_handler() local
252 (cma_id == cb->cm_id) ? "parent" : "child"); in krping_cma_event_handler()
256 cb->state = ADDR_RESOLVED; in krping_cma_event_handler()
261 wake_up_interruptible(&cb->sem); in krping_cma_event_handler()
266 cb->state = ROUTE_RESOLVED; in krping_cma_event_handler()
267 wake_up_interruptible(&cb->sem); in krping_cma_event_handler()
271 cb->state = CONNECT_REQUEST; in krping_cma_event_handler()
272 cb->child_cm_id = cma_id; in krping_cma_event_handler()
273 DEBUG_LOG("child cma %p\n", cb->child_cm_id); in krping_cma_event_handler()
274 wake_up_interruptible(&cb->sem); in krping_cma_event_handler()
279 if (!cb->server) { in krping_cma_event_handler()
280 cb->state = CONNECTED; in krping_cma_event_handler()
282 wake_up_interruptible(&cb->sem); in krping_cma_event_handler()
292 cb->state = ERROR; in krping_cma_event_handler()
293 wake_up_interruptible(&cb->sem); in krping_cma_event_handler()
298 cb->state = ERROR; in krping_cma_event_handler()
299 wake_up_interruptible(&cb->sem); in krping_cma_event_handler()
304 cb->state = ERROR; in krping_cma_event_handler()
305 wake_up_interruptible(&cb->sem); in krping_cma_event_handler()
310 wake_up_interruptible(&cb->sem); in krping_cma_event_handler()
316 static int server_recv(struct krping_cb *cb, struct ib_wc *wc) in server_recv() argument
318 if (wc->byte_len != sizeof(cb->recv_buf)) { in server_recv()
324 cb->remote_rkey = ntohl(cb->recv_buf.rkey); in server_recv()
325 cb->remote_addr = ntohll(cb->recv_buf.buf); in server_recv()
326 cb->remote_len = ntohl(cb->recv_buf.size); in server_recv()
328 cb->remote_rkey, (unsigned long long)cb->remote_addr, in server_recv()
329 cb->remote_len); in server_recv()
331 if (cb->state <= CONNECTED || cb->state == RDMA_WRITE_COMPLETE) in server_recv()
332 cb->state = RDMA_READ_ADV; in server_recv()
334 cb->state = RDMA_WRITE_ADV; in server_recv()
339 static int client_recv(struct krping_cb *cb, struct ib_wc *wc) in client_recv() argument
341 if (wc->byte_len != sizeof(cb->recv_buf)) { in client_recv()
347 if (cb->state == RDMA_READ_ADV) in client_recv()
348 cb->state = RDMA_WRITE_ADV; in client_recv()
350 cb->state = RDMA_WRITE_COMPLETE; in client_recv()
357 struct krping_cb *cb = ctx; in krping_cq_event_handler() local
362 BUG_ON(cb->cq != cq); in krping_cq_event_handler()
363 if (cb->frtest) { in krping_cq_event_handler()
367 if (!cb->wlat && !cb->rlat && !cb->bw) in krping_cq_event_handler()
368 ib_req_notify_cq(cb->cq, IB_CQ_NEXT_COMP); in krping_cq_event_handler()
369 while ((ret = ib_poll_cq(cb->cq, 1, &wc)) == 1) { in krping_cq_event_handler()
381 if (cb->state == ERROR) { in krping_cq_event_handler()
388 cb->stats.send_bytes += cb->send_sgl.length; in krping_cq_event_handler()
389 cb->stats.send_msgs++; in krping_cq_event_handler()
394 cb->stats.write_bytes += cb->rdma_sq_wr.wr.sg_list->length; in krping_cq_event_handler()
395 cb->stats.write_msgs++; in krping_cq_event_handler()
396 cb->state = RDMA_WRITE_COMPLETE; in krping_cq_event_handler()
397 wake_up_interruptible(&cb->sem); in krping_cq_event_handler()
402 cb->stats.read_bytes += cb->rdma_sq_wr.wr.sg_list->length; in krping_cq_event_handler()
403 cb->stats.read_msgs++; in krping_cq_event_handler()
404 cb->state = RDMA_READ_COMPLETE; in krping_cq_event_handler()
405 wake_up_interruptible(&cb->sem); in krping_cq_event_handler()
410 cb->stats.recv_bytes += sizeof(cb->recv_buf); in krping_cq_event_handler()
411 cb->stats.recv_msgs++; in krping_cq_event_handler()
412 if (cb->wlat || cb->rlat || cb->bw) in krping_cq_event_handler()
413 ret = server_recv(cb, &wc); in krping_cq_event_handler()
415 ret = cb->server ? server_recv(cb, &wc) : in krping_cq_event_handler()
416 client_recv(cb, &wc); in krping_cq_event_handler()
422 ret = ib_post_recv(cb->qp, &cb->rq_wr, &bad_wr); in krping_cq_event_handler()
428 wake_up_interruptible(&cb->sem); in krping_cq_event_handler()
444 cb->state = ERROR; in krping_cq_event_handler()
445 wake_up_interruptible(&cb->sem); in krping_cq_event_handler()
448 static int krping_accept(struct krping_cb *cb) in krping_accept() argument
459 ret = rdma_accept(cb->child_cm_id, &conn_param); in krping_accept()
465 if (!cb->wlat && !cb->rlat && !cb->bw) { in krping_accept()
466 wait_event_interruptible(cb->sem, cb->state >= CONNECTED); in krping_accept()
467 if (cb->state == ERROR) { in krping_accept()
469 cb->state); in krping_accept()
476 static void krping_setup_wr(struct krping_cb *cb) in krping_setup_wr() argument
478 cb->recv_sgl.addr = cb->recv_dma_addr; in krping_setup_wr()
479 cb->recv_sgl.length = sizeof cb->recv_buf; in krping_setup_wr()
480 cb->recv_sgl.lkey = cb->pd->local_dma_lkey; in krping_setup_wr()
481 cb->rq_wr.sg_list = &cb->recv_sgl; in krping_setup_wr()
482 cb->rq_wr.num_sge = 1; in krping_setup_wr()
484 cb->send_sgl.addr = cb->send_dma_addr; in krping_setup_wr()
485 cb->send_sgl.length = sizeof cb->send_buf; in krping_setup_wr()
486 cb->send_sgl.lkey = cb->pd->local_dma_lkey; in krping_setup_wr()
488 cb->sq_wr.opcode = IB_WR_SEND; in krping_setup_wr()
489 cb->sq_wr.send_flags = IB_SEND_SIGNALED; in krping_setup_wr()
490 cb->sq_wr.sg_list = &cb->send_sgl; in krping_setup_wr()
491 cb->sq_wr.num_sge = 1; in krping_setup_wr()
493 if (cb->server || cb->wlat || cb->rlat || cb->bw) { in krping_setup_wr()
494 cb->rdma_sgl.addr = cb->rdma_dma_addr; in krping_setup_wr()
495 cb->rdma_sq_wr.wr.send_flags = IB_SEND_SIGNALED; in krping_setup_wr()
496 cb->rdma_sq_wr.wr.sg_list = &cb->rdma_sgl; in krping_setup_wr()
497 cb->rdma_sq_wr.wr.num_sge = 1; in krping_setup_wr()
505 cb->reg_mr_wr.wr.opcode = IB_WR_REG_MR; in krping_setup_wr()
506 cb->reg_mr_wr.mr = cb->reg_mr; in krping_setup_wr()
508 cb->invalidate_wr.next = &cb->reg_mr_wr.wr; in krping_setup_wr()
509 cb->invalidate_wr.opcode = IB_WR_LOCAL_INV; in krping_setup_wr()
512 static int krping_setup_buffers(struct krping_cb *cb) in krping_setup_buffers() argument
516 DEBUG_LOG(PFX "krping_setup_buffers called on cb %p\n", cb); in krping_setup_buffers()
518 cb->recv_dma_addr = ib_dma_map_single(cb->pd->device, in krping_setup_buffers()
519 &cb->recv_buf, in krping_setup_buffers()
520 sizeof(cb->recv_buf), DMA_BIDIRECTIONAL); in krping_setup_buffers()
521 pci_unmap_addr_set(cb, recv_mapping, cb->recv_dma_addr); in krping_setup_buffers()
522 cb->send_dma_addr = ib_dma_map_single(cb->pd->device, in krping_setup_buffers()
523 &cb->send_buf, sizeof(cb->send_buf), in krping_setup_buffers()
525 pci_unmap_addr_set(cb, send_mapping, cb->send_dma_addr); in krping_setup_buffers()
527 cb->rdma_buf = ib_dma_alloc_coherent(cb->pd->device, cb->size, in krping_setup_buffers()
528 &cb->rdma_dma_addr, in krping_setup_buffers()
530 if (!cb->rdma_buf) { in krping_setup_buffers()
535 pci_unmap_addr_set(cb, rdma_mapping, cb->rdma_dma_addr); in krping_setup_buffers()
536 cb->page_list_len = (((cb->size - 1) & PAGE_MASK) + PAGE_SIZE) in krping_setup_buffers()
538 cb->reg_mr = ib_alloc_mr(cb->pd, IB_MR_TYPE_MEM_REG, in krping_setup_buffers()
539 cb->page_list_len); in krping_setup_buffers()
540 if (IS_ERR(cb->reg_mr)) { in krping_setup_buffers()
541 ret = PTR_ERR(cb->reg_mr); in krping_setup_buffers()
546 cb->reg_mr->rkey, cb->page_list_len); in krping_setup_buffers()
548 if (!cb->server || cb->wlat || cb->rlat || cb->bw) { in krping_setup_buffers()
550 cb->start_buf = ib_dma_alloc_coherent(cb->pd->device, cb->size, in krping_setup_buffers()
551 &cb->start_dma_addr, in krping_setup_buffers()
553 if (!cb->start_buf) { in krping_setup_buffers()
558 pci_unmap_addr_set(cb, start_mapping, cb->start_dma_addr); in krping_setup_buffers()
561 krping_setup_wr(cb); in krping_setup_buffers()
565 if (cb->reg_mr && !IS_ERR(cb->reg_mr)) in krping_setup_buffers()
566 ib_dereg_mr(cb->reg_mr); in krping_setup_buffers()
567 if (cb->rdma_mr && !IS_ERR(cb->rdma_mr)) in krping_setup_buffers()
568 ib_dereg_mr(cb->rdma_mr); in krping_setup_buffers()
569 if (cb->dma_mr && !IS_ERR(cb->dma_mr)) in krping_setup_buffers()
570 ib_dereg_mr(cb->dma_mr); in krping_setup_buffers()
571 if (cb->rdma_buf) { in krping_setup_buffers()
572 ib_dma_free_coherent(cb->pd->device, cb->size, cb->rdma_buf, in krping_setup_buffers()
573 cb->rdma_dma_addr); in krping_setup_buffers()
575 if (cb->start_buf) { in krping_setup_buffers()
576 ib_dma_free_coherent(cb->pd->device, cb->size, cb->start_buf, in krping_setup_buffers()
577 cb->start_dma_addr); in krping_setup_buffers()
582 static void krping_free_buffers(struct krping_cb *cb) in krping_free_buffers() argument
584 DEBUG_LOG("krping_free_buffers called on cb %p\n", cb); in krping_free_buffers()
586 if (cb->dma_mr) in krping_free_buffers()
587 ib_dereg_mr(cb->dma_mr); in krping_free_buffers()
588 if (cb->rdma_mr) in krping_free_buffers()
589 ib_dereg_mr(cb->rdma_mr); in krping_free_buffers()
590 if (cb->start_mr) in krping_free_buffers()
591 ib_dereg_mr(cb->start_mr); in krping_free_buffers()
592 if (cb->reg_mr) in krping_free_buffers()
593 ib_dereg_mr(cb->reg_mr); in krping_free_buffers()
595 dma_unmap_single(cb->pd->device->dma_device, in krping_free_buffers()
596 pci_unmap_addr(cb, recv_mapping), in krping_free_buffers()
597 sizeof(cb->recv_buf), DMA_BIDIRECTIONAL); in krping_free_buffers()
598 dma_unmap_single(cb->pd->device->dma_device, in krping_free_buffers()
599 pci_unmap_addr(cb, send_mapping), in krping_free_buffers()
600 sizeof(cb->send_buf), DMA_BIDIRECTIONAL); in krping_free_buffers()
602 ib_dma_free_coherent(cb->pd->device, cb->size, cb->rdma_buf, in krping_free_buffers()
603 cb->rdma_dma_addr); in krping_free_buffers()
605 if (cb->start_buf) { in krping_free_buffers()
606 ib_dma_free_coherent(cb->pd->device, cb->size, cb->start_buf, in krping_free_buffers()
607 cb->start_dma_addr); in krping_free_buffers()
611 static int krping_create_qp(struct krping_cb *cb) in krping_create_qp() argument
617 init_attr.cap.max_send_wr = cb->txdepth; in krping_create_qp()
627 init_attr.send_cq = cb->cq; in krping_create_qp()
628 init_attr.recv_cq = cb->cq; in krping_create_qp()
631 if (cb->server) { in krping_create_qp()
632 ret = rdma_create_qp(cb->child_cm_id, cb->pd, &init_attr); in krping_create_qp()
634 cb->qp = cb->child_cm_id->qp; in krping_create_qp()
636 ret = rdma_create_qp(cb->cm_id, cb->pd, &init_attr); in krping_create_qp()
638 cb->qp = cb->cm_id->qp; in krping_create_qp()
644 static void krping_free_qp(struct krping_cb *cb) in krping_free_qp() argument
646 ib_destroy_qp(cb->qp); in krping_free_qp()
647 ib_destroy_cq(cb->cq); in krping_free_qp()
648 ib_dealloc_pd(cb->pd); in krping_free_qp()
651 static int krping_setup_qp(struct krping_cb *cb, struct rdma_cm_id *cm_id) in krping_setup_qp() argument
656 cb->pd = ib_alloc_pd(cm_id->device, 0); in krping_setup_qp()
657 if (IS_ERR(cb->pd)) { in krping_setup_qp()
659 return PTR_ERR(cb->pd); in krping_setup_qp()
661 DEBUG_LOG("created pd %p\n", cb->pd); in krping_setup_qp()
663 strlcpy(cb->stats.name, cb->pd->device->name, sizeof(cb->stats.name)); in krping_setup_qp()
665 attr.cqe = cb->txdepth * 2; in krping_setup_qp()
667 cb->cq = ib_create_cq(cm_id->device, krping_cq_event_handler, NULL, in krping_setup_qp()
668 cb, &attr); in krping_setup_qp()
669 if (IS_ERR(cb->cq)) { in krping_setup_qp()
671 ret = PTR_ERR(cb->cq); in krping_setup_qp()
674 DEBUG_LOG("created cq %p\n", cb->cq); in krping_setup_qp()
676 if (!cb->wlat && !cb->rlat && !cb->bw && !cb->frtest) { in krping_setup_qp()
677 ret = ib_req_notify_cq(cb->cq, IB_CQ_NEXT_COMP); in krping_setup_qp()
684 ret = krping_create_qp(cb); in krping_setup_qp()
689 DEBUG_LOG("created qp %p\n", cb->qp); in krping_setup_qp()
692 ib_destroy_cq(cb->cq); in krping_setup_qp()
694 ib_dealloc_pd(cb->pd); in krping_setup_qp()
703 static u32 krping_rdma_rkey(struct krping_cb *cb, u64 buf, int post_inv) in krping_rdma_rkey() argument
710 cb->invalidate_wr.ex.invalidate_rkey = cb->reg_mr->rkey; in krping_rdma_rkey()
715 ib_update_fast_reg_key(cb->reg_mr, ++cb->key); in krping_rdma_rkey()
716 cb->reg_mr_wr.key = cb->reg_mr->rkey; in krping_rdma_rkey()
721 if (buf == (u64)cb->start_dma_addr) in krping_rdma_rkey()
722 cb->reg_mr_wr.access = IB_ACCESS_REMOTE_READ; in krping_rdma_rkey()
724 cb->reg_mr_wr.access = IB_ACCESS_REMOTE_WRITE | IB_ACCESS_LOCAL_WRITE; in krping_rdma_rkey()
726 sg_dma_len(&sg) = cb->size; in krping_rdma_rkey()
728 ret = ib_map_mr_sg(cb->reg_mr, &sg, 1, NULL, PAGE_SIZE); in krping_rdma_rkey()
729 BUG_ON(ret <= 0 || ret > cb->page_list_len); in krping_rdma_rkey()
734 cb->reg_mr_wr.key, in krping_rdma_rkey()
735 cb->reg_mr->page_size, in krping_rdma_rkey()
736 (unsigned)cb->reg_mr->length, in krping_rdma_rkey()
737 (unsigned long long)cb->reg_mr->iova); in krping_rdma_rkey()
740 ret = ib_post_send(cb->qp, &cb->invalidate_wr, &bad_wr); in krping_rdma_rkey()
742 ret = ib_post_send(cb->qp, &cb->reg_mr_wr.wr, &bad_wr); in krping_rdma_rkey()
745 cb->state = ERROR; in krping_rdma_rkey()
747 rkey = cb->reg_mr->rkey; in krping_rdma_rkey()
751 static void krping_format_send(struct krping_cb *cb, u64 buf) in krping_format_send() argument
753 struct krping_rdma_info *info = &cb->send_buf; in krping_format_send()
761 if (!cb->server || cb->wlat || cb->rlat || cb->bw) { in krping_format_send()
762 rkey = krping_rdma_rkey(cb, buf, !cb->server_invalidate); in krping_format_send()
765 info->size = htonl(cb->size); in krping_format_send()
767 (unsigned long long)buf, rkey, cb->size); in krping_format_send()
771 static void krping_test_server(struct krping_cb *cb) in krping_test_server() argument
779 wait_event_interruptible(cb->sem, cb->state >= RDMA_READ_ADV); in krping_test_server()
780 if (cb->state != RDMA_READ_ADV) { in krping_test_server()
782 cb->state); in krping_test_server()
788 cb->rdma_sq_wr.rkey = cb->remote_rkey; in krping_test_server()
789 cb->rdma_sq_wr.remote_addr = cb->remote_addr; in krping_test_server()
790 cb->rdma_sq_wr.wr.sg_list->length = cb->remote_len; in krping_test_server()
791 cb->rdma_sgl.lkey = krping_rdma_rkey(cb, cb->rdma_dma_addr, !cb->read_inv); in krping_test_server()
792 cb->rdma_sq_wr.wr.next = NULL; in krping_test_server()
795 if (cb->read_inv) in krping_test_server()
796 cb->rdma_sq_wr.wr.opcode = IB_WR_RDMA_READ_WITH_INV; in krping_test_server()
799 cb->rdma_sq_wr.wr.opcode = IB_WR_RDMA_READ; in krping_test_server()
804 cb->rdma_sq_wr.wr.next = &inv; in krping_test_server()
807 inv.ex.invalidate_rkey = cb->reg_mr->rkey; in krping_test_server()
811 ret = ib_post_send(cb->qp, &cb->rdma_sq_wr.wr, &bad_wr); in krping_test_server()
816 cb->rdma_sq_wr.wr.next = NULL; in krping_test_server()
821 wait_event_interruptible(cb->sem, in krping_test_server()
822 cb->state >= RDMA_READ_COMPLETE); in krping_test_server()
823 if (cb->state != RDMA_READ_COMPLETE) { in krping_test_server()
826 cb->state); in krping_test_server()
832 if (cb->verbose) in krping_test_server()
834 cb->rdma_buf); in krping_test_server()
837 if (cb->server && cb->server_invalidate) { in krping_test_server()
838 cb->sq_wr.ex.invalidate_rkey = cb->remote_rkey; in krping_test_server()
839 cb->sq_wr.opcode = IB_WR_SEND_WITH_INV; in krping_test_server()
840 DEBUG_LOG("send-w-inv rkey 0x%x\n", cb->remote_rkey); in krping_test_server()
842 ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); in krping_test_server()
850 wait_event_interruptible(cb->sem, cb->state >= RDMA_WRITE_ADV); in krping_test_server()
851 if (cb->state != RDMA_WRITE_ADV) { in krping_test_server()
854 cb->state); in krping_test_server()
860 cb->rdma_sq_wr.wr.opcode = IB_WR_RDMA_WRITE; in krping_test_server()
861 cb->rdma_sq_wr.rkey = cb->remote_rkey; in krping_test_server()
862 cb->rdma_sq_wr.remote_addr = cb->remote_addr; in krping_test_server()
863 cb->rdma_sq_wr.wr.sg_list->length = strlen(cb->rdma_buf) + 1; in krping_test_server()
864 if (cb->local_dma_lkey) in krping_test_server()
865 cb->rdma_sgl.lkey = cb->pd->local_dma_lkey; in krping_test_server()
867 cb->rdma_sgl.lkey = krping_rdma_rkey(cb, cb->rdma_dma_addr, 0); in krping_test_server()
870 cb->rdma_sq_wr.wr.sg_list->lkey, in krping_test_server()
871 (unsigned long long)cb->rdma_sq_wr.wr.sg_list->addr, in krping_test_server()
872 cb->rdma_sq_wr.wr.sg_list->length); in krping_test_server()
874 ret = ib_post_send(cb->qp, &cb->rdma_sq_wr.wr, &bad_wr); in krping_test_server()
881 ret = wait_event_interruptible(cb->sem, cb->state >= in krping_test_server()
883 if (cb->state != RDMA_WRITE_COMPLETE) { in krping_test_server()
886 cb->state); in krping_test_server()
891 cb->state = CONNECTED; in krping_test_server()
894 if (cb->server && cb->server_invalidate) { in krping_test_server()
895 cb->sq_wr.ex.invalidate_rkey = cb->remote_rkey; in krping_test_server()
896 cb->sq_wr.opcode = IB_WR_SEND_WITH_INV; in krping_test_server()
897 DEBUG_LOG("send-w-inv rkey 0x%x\n", cb->remote_rkey); in krping_test_server()
899 ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); in krping_test_server()
908 static void rlat_test(struct krping_cb *cb) in rlat_test() argument
911 int iters = cb->count; in rlat_test()
919 cb->rdma_sq_wr.wr.opcode = IB_WR_RDMA_READ; in rlat_test()
920 cb->rdma_sq_wr.rkey = cb->remote_rkey; in rlat_test()
921 cb->rdma_sq_wr.remote_addr = cb->remote_addr; in rlat_test()
922 cb->rdma_sq_wr.wr.sg_list->length = cb->size; in rlat_test()
925 if (!cb->poll) { in rlat_test()
926 cb->state = RDMA_READ_ADV; in rlat_test()
927 ib_req_notify_cq(cb->cq, IB_CQ_NEXT_COMP); in rlat_test()
931 cb->state = RDMA_READ_ADV; in rlat_test()
932 ret = ib_post_send(cb->qp, &cb->rdma_sq_wr.wr, &bad_wr); in rlat_test()
941 if (!cb->poll) { in rlat_test()
942 wait_event_interruptible(cb->sem, in rlat_test()
943 cb->state != RDMA_READ_ADV); in rlat_test()
944 if (cb->state == RDMA_READ_COMPLETE) { in rlat_test()
946 ib_req_notify_cq(cb->cq, in rlat_test()
952 ne = ib_poll_cq(cb->cq, 1, &wc); in rlat_test()
953 if (cb->state == ERROR) { in rlat_test()
965 if (cb->poll && wc.status != IB_WC_SUCCESS) { in rlat_test()
967 cb->server ? "server" : "client"); in rlat_test()
984 scnt, cb->size); in rlat_test()
987 static void wlat_test(struct krping_cb *cb) in wlat_test() argument
990 int iters=cb->count; in wlat_test()
991 volatile char *poll_buf = (char *) cb->start_buf; in wlat_test()
992 char *buf = (char *)cb->rdma_buf; in wlat_test()
1033 cb->rdma_sq_wr.wr.opcode = IB_WR_RDMA_WRITE; in wlat_test()
1034 cb->rdma_sq_wr.rkey = cb->remote_rkey; in wlat_test()
1035 cb->rdma_sq_wr.remote_addr = cb->remote_addr; in wlat_test()
1036 cb->rdma_sq_wr.wr.sg_list->length = cb->size; in wlat_test()
1044 if (rcnt < iters && !(scnt < 1 && !cb->server)) { in wlat_test()
1047 if (cb->state == ERROR) { in wlat_test()
1061 if (ib_post_send(cb->qp, &cb->rdma_sq_wr.wr, &bad_wr)) { in wlat_test()
1082 ne = ib_poll_cq(cb->cq, 1, &wc); in wlat_test()
1095 cb->server ? "server" : "client"); in wlat_test()
1123 scnt, cb->size, cycle_iters, in wlat_test()
1134 static void bw_test(struct krping_cb *cb) in bw_test() argument
1137 int iters=cb->count; in bw_test()
1177 cb->rdma_sq_wr.wr.opcode = IB_WR_RDMA_WRITE; in bw_test()
1178 cb->rdma_sq_wr.rkey = cb->remote_rkey; in bw_test()
1179 cb->rdma_sq_wr.remote_addr = cb->remote_addr; in bw_test()
1180 cb->rdma_sq_wr.wr.sg_list->length = cb->size; in bw_test()
1187 while (scnt < iters && scnt - ccnt < cb->txdepth) { in bw_test()
1192 if (ib_post_send(cb->qp, &cb->rdma_sq_wr.wr, &bad_wr)) { in bw_test()
1213 ne = ib_poll_cq(cb->cq, 1, &wc); in bw_test()
1226 cb->server ? "server" : "client"); in bw_test()
1251 scnt, cb->size, cycle_iters, in bw_test()
1262 static void krping_rlat_test_server(struct krping_cb *cb) in krping_rlat_test_server() argument
1269 while (cb->state < RDMA_READ_ADV) { in krping_rlat_test_server()
1270 krping_cq_event_handler(cb->cq, cb); in krping_rlat_test_server()
1274 krping_format_send(cb, cb->start_dma_addr); in krping_rlat_test_server()
1275 ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); in krping_rlat_test_server()
1282 while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); in krping_rlat_test_server()
1292 wait_event_interruptible(cb->sem, cb->state == ERROR); in krping_rlat_test_server()
1295 static void krping_wlat_test_server(struct krping_cb *cb) in krping_wlat_test_server() argument
1302 while (cb->state < RDMA_READ_ADV) { in krping_wlat_test_server()
1303 krping_cq_event_handler(cb->cq, cb); in krping_wlat_test_server()
1307 krping_format_send(cb, cb->start_dma_addr); in krping_wlat_test_server()
1308 ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); in krping_wlat_test_server()
1315 while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); in krping_wlat_test_server()
1325 wlat_test(cb); in krping_wlat_test_server()
1326 wait_event_interruptible(cb->sem, cb->state == ERROR); in krping_wlat_test_server()
1329 static void krping_bw_test_server(struct krping_cb *cb) in krping_bw_test_server() argument
1336 while (cb->state < RDMA_READ_ADV) { in krping_bw_test_server()
1337 krping_cq_event_handler(cb->cq, cb); in krping_bw_test_server()
1341 krping_format_send(cb, cb->start_dma_addr); in krping_bw_test_server()
1342 ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); in krping_bw_test_server()
1349 while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); in krping_bw_test_server()
1359 if (cb->duplex) in krping_bw_test_server()
1360 bw_test(cb); in krping_bw_test_server()
1361 wait_event_interruptible(cb->sem, cb->state == ERROR); in krping_bw_test_server()
1379 static void fill_sockaddr(struct sockaddr_storage *sin, struct krping_cb *cb) in fill_sockaddr() argument
1383 if (cb->addr_type == AF_INET) { in fill_sockaddr()
1387 memcpy((void *)&sin4->sin_addr.s_addr, cb->addr, 4); in fill_sockaddr()
1388 sin4->sin_port = cb->port; in fill_sockaddr()
1389 } else if (cb->addr_type == AF_INET6) { in fill_sockaddr()
1393 memcpy((void *)&sin6->sin6_addr, cb->addr, 16); in fill_sockaddr()
1394 sin6->sin6_port = cb->port; in fill_sockaddr()
1398 static int krping_bind_server(struct krping_cb *cb) in krping_bind_server() argument
1404 fill_sockaddr(&sin, cb); in krping_bind_server()
1406 ret = rdma_bind_addr(cb->cm_id, (struct sockaddr *)&sin); in krping_bind_server()
1414 ret = rdma_listen(cb->cm_id, 3); in krping_bind_server()
1420 wait_event_interruptible(cb->sem, cb->state >= CONNECT_REQUEST); in krping_bind_server()
1421 if (cb->state != CONNECT_REQUEST) { in krping_bind_server()
1423 cb->state); in krping_bind_server()
1427 if (!reg_supported(cb->child_cm_id->device)) in krping_bind_server()
1433 static void krping_run_server(struct krping_cb *cb) in krping_run_server() argument
1438 ret = krping_bind_server(cb); in krping_run_server()
1442 ret = krping_setup_qp(cb, cb->child_cm_id); in krping_run_server()
1448 ret = krping_setup_buffers(cb); in krping_run_server()
1454 ret = ib_post_recv(cb->qp, &cb->rq_wr, &bad_wr); in krping_run_server()
1460 ret = krping_accept(cb); in krping_run_server()
1466 if (cb->wlat) in krping_run_server()
1467 krping_wlat_test_server(cb); in krping_run_server()
1468 else if (cb->rlat) in krping_run_server()
1469 krping_rlat_test_server(cb); in krping_run_server()
1470 else if (cb->bw) in krping_run_server()
1471 krping_bw_test_server(cb); in krping_run_server()
1473 krping_test_server(cb); in krping_run_server()
1474 rdma_disconnect(cb->child_cm_id); in krping_run_server()
1476 krping_free_buffers(cb); in krping_run_server()
1478 krping_free_qp(cb); in krping_run_server()
1480 rdma_destroy_id(cb->child_cm_id); in krping_run_server()
1483 static void krping_test_client(struct krping_cb *cb) in krping_test_client() argument
1490 for (ping = 0; !cb->count || ping < cb->count; ping++) { in krping_test_client()
1491 cb->state = RDMA_READ_ADV; in krping_test_client()
1494 cc = sprintf(cb->start_buf, "rdma-ping-%d: ", ping); in krping_test_client()
1495 for (i = cc, c = start; i < cb->size; i++) { in krping_test_client()
1496 cb->start_buf[i] = c; in krping_test_client()
1504 cb->start_buf[cb->size - 1] = 0; in krping_test_client()
1506 krping_format_send(cb, cb->start_dma_addr); in krping_test_client()
1507 if (cb->state == ERROR) { in krping_test_client()
1511 ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); in krping_test_client()
1518 wait_event_interruptible(cb->sem, cb->state >= RDMA_WRITE_ADV); in krping_test_client()
1519 if (cb->state != RDMA_WRITE_ADV) { in krping_test_client()
1522 cb->state); in krping_test_client()
1526 krping_format_send(cb, cb->rdma_dma_addr); in krping_test_client()
1527 ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); in krping_test_client()
1534 wait_event_interruptible(cb->sem, in krping_test_client()
1535 cb->state >= RDMA_WRITE_COMPLETE); in krping_test_client()
1536 if (cb->state != RDMA_WRITE_COMPLETE) { in krping_test_client()
1539 cb->state); in krping_test_client()
1543 if (cb->validate) in krping_test_client()
1544 if (memcmp(cb->start_buf, cb->rdma_buf, cb->size)) { in krping_test_client()
1549 if (cb->verbose) in krping_test_client()
1550 printk(KERN_INFO PFX "ping data: %s\n", cb->rdma_buf); in krping_test_client()
1552 wait_event_interruptible_timeout(cb->sem, cb->state == ERROR, HZ); in krping_test_client()
1557 static void krping_rlat_test_client(struct krping_cb *cb) in krping_rlat_test_client() argument
1563 cb->state = RDMA_READ_ADV; in krping_rlat_test_client()
1566 krping_format_send(cb, cb->start_dma_addr); in krping_rlat_test_client()
1567 if (cb->state == ERROR) { in krping_rlat_test_client()
1571 ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); in krping_rlat_test_client()
1578 while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); in krping_rlat_test_client()
1589 while (cb->state < RDMA_WRITE_ADV) { in krping_rlat_test_client()
1590 krping_cq_event_handler(cb->cq, cb); in krping_rlat_test_client()
1604 cb->rdma_sq_wr.wr.opcode = IB_WR_RDMA_WRITE; in krping_rlat_test_client()
1605 cb->rdma_sq_wr.rkey = cb->remote_rkey; in krping_rlat_test_client()
1606 cb->rdma_sq_wr.remote_addr = cb->remote_addr; in krping_rlat_test_client()
1607 cb->rdma_sq_wr.wr.sg_list->length = 0; in krping_rlat_test_client()
1608 cb->rdma_sq_wr.wr.num_sge = 0; in krping_rlat_test_client()
1612 if (ib_post_send(cb->qp, &cb->rdma_sq_wr.wr, &bad_wr)) { in krping_rlat_test_client()
1617 ne = ib_poll_cq(cb->cq, 1, &wc); in krping_rlat_test_client()
1625 cb->server ? "server" : "client"); in krping_rlat_test_client()
1644 rlat_test(cb); in krping_rlat_test_client()
1647 static void krping_wlat_test_client(struct krping_cb *cb) in krping_wlat_test_client() argument
1653 cb->state = RDMA_READ_ADV; in krping_wlat_test_client()
1656 krping_format_send(cb, cb->start_dma_addr); in krping_wlat_test_client()
1657 if (cb->state == ERROR) { in krping_wlat_test_client()
1661 ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); in krping_wlat_test_client()
1668 while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); in krping_wlat_test_client()
1679 while (cb->state < RDMA_WRITE_ADV) { in krping_wlat_test_client()
1680 krping_cq_event_handler(cb->cq, cb); in krping_wlat_test_client()
1683 wlat_test(cb); in krping_wlat_test_client()
1686 static void krping_bw_test_client(struct krping_cb *cb) in krping_bw_test_client() argument
1692 cb->state = RDMA_READ_ADV; in krping_bw_test_client()
1695 krping_format_send(cb, cb->start_dma_addr); in krping_bw_test_client()
1696 if (cb->state == ERROR) { in krping_bw_test_client()
1700 ret = ib_post_send(cb->qp, &cb->sq_wr, &bad_wr); in krping_bw_test_client()
1707 while ((ret = ib_poll_cq(cb->cq, 1, &wc) == 0)); in krping_bw_test_client()
1718 while (cb->state < RDMA_WRITE_ADV) { in krping_bw_test_client()
1719 krping_cq_event_handler(cb->cq, cb); in krping_bw_test_client()
1722 bw_test(cb); in krping_bw_test_client()
1728 static void flush_qp(struct krping_cb *cb) in flush_qp() argument
1739 rdma_disconnect(cb->cm_id); in flush_qp()
1744 ret = ib_post_send(cb->qp, &wr, &bad); in flush_qp()
1751 ret = ib_post_recv(cb->qp, &recv_wr, &recv_bad); in flush_qp()
1759 ret = ib_poll_cq(cb->cq, 1, &wc); in flush_qp()
1774 static void krping_fr_test(struct krping_cb *cb) in krping_fr_test() argument
1783 int size = cb->size; in krping_fr_test()
1790 mr = ib_alloc_mr(cb->pd, IB_MR_TYPE_MEM_REG, plen); in krping_fr_test()
1814 …DEBUG_LOG("fr_test: stag index 0x%x plen %u size %u depth %u\n", mr->rkey >> 8, plen, cb->size, cb… in krping_fr_test()
1816 while (!cb->count || count <= cb->count) { in krping_fr_test()
1823 wait_event_interruptible_timeout(cb->sem, cb->state == ERROR, HZ); in krping_fr_test()
1824 if (cb->state == ERROR) in krping_fr_test()
1828 while (scnt < (cb->txdepth>>1)) { in krping_fr_test()
1833 size = arc4random() % cb->size; in krping_fr_test()
1835 size = cb->size; in krping_fr_test()
1842 ret = ib_post_send(cb->qp, &fr.wr, &bad); in krping_fr_test()
1850 ret = ib_poll_cq(cb->cq, 1, &wc); in krping_fr_test()
1865 flush_qp(cb); in krping_fr_test()
1870 static int krping_connect_client(struct krping_cb *cb) in krping_connect_client() argument
1880 ret = rdma_connect(cb->cm_id, &conn_param); in krping_connect_client()
1886 wait_event_interruptible(cb->sem, cb->state >= CONNECTED); in krping_connect_client()
1887 if (cb->state == ERROR) { in krping_connect_client()
1888 printk(KERN_ERR PFX "wait for CONNECTED state %d\n", cb->state); in krping_connect_client()
1896 static int krping_bind_client(struct krping_cb *cb) in krping_bind_client() argument
1901 fill_sockaddr(&sin, cb); in krping_bind_client()
1903 ret = rdma_resolve_addr(cb->cm_id, NULL, (struct sockaddr *)&sin, 2000); in krping_bind_client()
1909 wait_event_interruptible(cb->sem, cb->state >= ROUTE_RESOLVED); in krping_bind_client()
1910 if (cb->state != ROUTE_RESOLVED) { in krping_bind_client()
1913 cb->state); in krping_bind_client()
1917 if (!reg_supported(cb->cm_id->device)) in krping_bind_client()
1924 static void krping_run_client(struct krping_cb *cb) in krping_run_client() argument
1930 if (cb->tos != 0) in krping_run_client()
1931 rdma_set_service_type(cb->cm_id, cb->tos); in krping_run_client()
1933 ret = krping_bind_client(cb); in krping_run_client()
1937 ret = krping_setup_qp(cb, cb->cm_id); in krping_run_client()
1943 ret = krping_setup_buffers(cb); in krping_run_client()
1949 ret = ib_post_recv(cb->qp, &cb->rq_wr, &bad_wr); in krping_run_client()
1955 ret = krping_connect_client(cb); in krping_run_client()
1961 if (cb->wlat) in krping_run_client()
1962 krping_wlat_test_client(cb); in krping_run_client()
1963 else if (cb->rlat) in krping_run_client()
1964 krping_rlat_test_client(cb); in krping_run_client()
1965 else if (cb->bw) in krping_run_client()
1966 krping_bw_test_client(cb); in krping_run_client()
1967 else if (cb->frtest) in krping_run_client()
1968 krping_fr_test(cb); in krping_run_client()
1970 krping_test_client(cb); in krping_run_client()
1971 rdma_disconnect(cb->cm_id); in krping_run_client()
1973 krping_free_buffers(cb); in krping_run_client()
1975 krping_free_qp(cb); in krping_run_client()
1998 struct krping_cb *cb; in krping_doit() local
2005 cb = kzalloc(sizeof(*cb), GFP_KERNEL); in krping_doit()
2006 if (!cb) in krping_doit()
2010 list_add_tail(&cb->list, &krping_cbs); in krping_doit()
2013 cb->server = -1; in krping_doit()
2014 cb->state = IDLE; in krping_doit()
2015 cb->size = 64; in krping_doit()
2016 cb->txdepth = RPING_SQ_DEPTH; in krping_doit()
2017 init_waitqueue_head(&cb->sem); in krping_doit()
2023 cb->addr_str = optarg; in krping_doit()
2024 cb->addr_type = AF_INET; in krping_doit()
2026 if (inet_pton(AF_INET, optarg, cb->addr) != 1) { in krping_doit()
2033 cb->addr_str = optarg; in krping_doit()
2034 cb->addr_type = AF_INET6; in krping_doit()
2041 if (inet_pton(AF_INET6, optarg, cb->addr) != 1) { in krping_doit()
2045 } else if (IN6_IS_SCOPE_LINKLOCAL((struct in6_addr *)cb->addr) || in krping_doit()
2046 IN6_IS_ADDR_MC_INTFACELOCAL((struct in6_addr *)cb->addr)) { in krping_doit()
2049 cb->addr[2] = scope_id >> 8; in krping_doit()
2050 cb->addr[3] = scope_id & 0xFF; in krping_doit()
2054 cb->port = htons(optint); in krping_doit()
2058 cb->poll = 1; in krping_doit()
2062 cb->server = 1; in krping_doit()
2066 cb->server = 0; in krping_doit()
2070 cb->size = optint; in krping_doit()
2071 if ((cb->size < 1) || in krping_doit()
2072 (cb->size > RPING_BUFSIZE)) { in krping_doit()
2075 cb->size, RPING_BUFSIZE); in krping_doit()
2081 cb->count = optint; in krping_doit()
2082 if (cb->count < 0) { in krping_doit()
2084 cb->count); in krping_doit()
2087 DEBUG_LOG("count %d\n", (int) cb->count); in krping_doit()
2090 cb->verbose++; in krping_doit()
2094 cb->validate++; in krping_doit()
2098 cb->wlat++; in krping_doit()
2101 cb->rlat++; in krping_doit()
2104 cb->bw++; in krping_doit()
2107 cb->duplex++; in krping_doit()
2110 cb->server_invalidate = 1; in krping_doit()
2113 cb->tos = optint; in krping_doit()
2114 DEBUG_LOG("type of service, tos=%d\n", (int) cb->tos); in krping_doit()
2117 cb->txdepth = optint; in krping_doit()
2118 DEBUG_LOG("txdepth %d\n", (int) cb->txdepth); in krping_doit()
2121 cb->local_dma_lkey = 1; in krping_doit()
2125 cb->read_inv = 1; in krping_doit()
2129 cb->frtest = 1; in krping_doit()
2141 if (cb->server == -1) { in krping_doit()
2147 if (cb->server && cb->frtest) { in krping_doit()
2153 if ((cb->frtest + cb->bw + cb->rlat + cb->wlat) > 1) { in krping_doit()
2159 if (cb->wlat || cb->rlat || cb->bw) { in krping_doit()
2165 …cb->cm_id = rdma_create_id(TD_TO_VNET(curthread), krping_cma_event_handler, cb, RDMA_PS_TCP, IB_QP… in krping_doit()
2166 if (IS_ERR(cb->cm_id)) { in krping_doit()
2167 ret = PTR_ERR(cb->cm_id); in krping_doit()
2171 DEBUG_LOG("created cm_id %p\n", cb->cm_id); in krping_doit()
2173 if (cb->server) in krping_doit()
2174 krping_run_server(cb); in krping_doit()
2176 krping_run_client(cb); in krping_doit()
2178 DEBUG_LOG("destroy cm_id %p\n", cb->cm_id); in krping_doit()
2179 rdma_destroy_id(cb->cm_id); in krping_doit()
2182 list_del(&cb->list); in krping_doit()
2184 kfree(cb); in krping_doit()
2191 struct krping_cb *cb; in krping_walk_cb_list() local
2194 list_for_each_entry(cb, &krping_cbs, list) in krping_walk_cb_list()
2195 (*f)(cb->pd ? &cb->stats : NULL, arg); in krping_walk_cb_list()
2202 struct krping_cb *cb; in krping_cancel_all() local
2205 list_for_each_entry(cb, &krping_cbs, list) { in krping_cancel_all()
2206 cb->state = ERROR; in krping_cancel_all()
2207 wake_up_interruptible(&cb->sem); in krping_cancel_all()