1aa0a1e58SJeff Roberson /*
2aa0a1e58SJeff Roberson * Copyright (c) 2005-2007 Intel Corporation. All rights reserved.
3aa0a1e58SJeff Roberson *
4aa0a1e58SJeff Roberson * This software is available to you under a choice of one of two
5aa0a1e58SJeff Roberson * licenses. You may choose to be licensed under the terms of the GNU
6aa0a1e58SJeff Roberson * General Public License (GPL) Version 2, available from the file
7aa0a1e58SJeff Roberson * COPYING in the main directory of this source tree, or the
8aa0a1e58SJeff Roberson * OpenIB.org BSD license below:
9aa0a1e58SJeff Roberson *
10aa0a1e58SJeff Roberson * Redistribution and use in source and binary forms, with or
11aa0a1e58SJeff Roberson * without modification, are permitted provided that the following
12aa0a1e58SJeff Roberson * conditions are met:
13aa0a1e58SJeff Roberson *
14aa0a1e58SJeff Roberson * - Redistributions of source code must retain the above
15aa0a1e58SJeff Roberson * copyright notice, this list of conditions and the following
16aa0a1e58SJeff Roberson * disclaimer.
17aa0a1e58SJeff Roberson *
18aa0a1e58SJeff Roberson * - Redistributions in binary form must reproduce the above
19aa0a1e58SJeff Roberson * copyright notice, this list of conditions and the following
20aa0a1e58SJeff Roberson * disclaimer in the documentation and/or other materials
21aa0a1e58SJeff Roberson * provided with the distribution.
22aa0a1e58SJeff Roberson *
23aa0a1e58SJeff Roberson * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24aa0a1e58SJeff Roberson * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25aa0a1e58SJeff Roberson * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26aa0a1e58SJeff Roberson * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27aa0a1e58SJeff Roberson * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28aa0a1e58SJeff Roberson * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29aa0a1e58SJeff Roberson * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30aa0a1e58SJeff Roberson * SOFTWARE.
31aa0a1e58SJeff Roberson *
32aa0a1e58SJeff Roberson * $Id$
33aa0a1e58SJeff Roberson */
34aa0a1e58SJeff Roberson
35aa0a1e58SJeff Roberson #include <stdlib.h>
36aa0a1e58SJeff Roberson #include <string.h>
37aa0a1e58SJeff Roberson #include <stdio.h>
38aa0a1e58SJeff Roberson #include <errno.h>
39aa0a1e58SJeff Roberson #include <sys/types.h>
40aa0a1e58SJeff Roberson #include <arpa/inet.h>
41aa0a1e58SJeff Roberson #include <sys/socket.h>
42aa0a1e58SJeff Roberson #include <netdb.h>
43aa0a1e58SJeff Roberson #include <unistd.h>
44aa0a1e58SJeff Roberson #include <getopt.h>
45aa0a1e58SJeff Roberson
46aa0a1e58SJeff Roberson #include <rdma/rdma_cma.h>
47d6b92ffaSHans Petter Selasky #include <infiniband/ib.h>
48aa0a1e58SJeff Roberson
49aa0a1e58SJeff Roberson struct cmatest_node {
50aa0a1e58SJeff Roberson int id;
51aa0a1e58SJeff Roberson struct rdma_cm_id *cma_id;
52aa0a1e58SJeff Roberson int connected;
53aa0a1e58SJeff Roberson struct ibv_pd *pd;
54aa0a1e58SJeff Roberson struct ibv_cq *cq;
55aa0a1e58SJeff Roberson struct ibv_mr *mr;
56aa0a1e58SJeff Roberson struct ibv_ah *ah;
57aa0a1e58SJeff Roberson uint32_t remote_qpn;
58aa0a1e58SJeff Roberson uint32_t remote_qkey;
59aa0a1e58SJeff Roberson void *mem;
60aa0a1e58SJeff Roberson };
61aa0a1e58SJeff Roberson
62aa0a1e58SJeff Roberson struct cmatest {
63aa0a1e58SJeff Roberson struct rdma_event_channel *channel;
64aa0a1e58SJeff Roberson pthread_t cmathread;
65aa0a1e58SJeff Roberson struct cmatest_node *nodes;
66aa0a1e58SJeff Roberson int conn_index;
67aa0a1e58SJeff Roberson int connects_left;
68aa0a1e58SJeff Roberson
69d6b92ffaSHans Petter Selasky struct sockaddr_storage dst_in;
70aa0a1e58SJeff Roberson struct sockaddr *dst_addr;
71d6b92ffaSHans Petter Selasky struct sockaddr_storage src_in;
72aa0a1e58SJeff Roberson struct sockaddr *src_addr;
73aa0a1e58SJeff Roberson };
74aa0a1e58SJeff Roberson
75aa0a1e58SJeff Roberson static struct cmatest test;
76aa0a1e58SJeff Roberson static int connections = 1;
77aa0a1e58SJeff Roberson static int message_size = 100;
78aa0a1e58SJeff Roberson static int message_count = 10;
79aa0a1e58SJeff Roberson static int is_sender;
80aa0a1e58SJeff Roberson static int unmapped_addr;
81aa0a1e58SJeff Roberson static char *dst_addr;
82aa0a1e58SJeff Roberson static char *src_addr;
83aa0a1e58SJeff Roberson static enum rdma_port_space port_space = RDMA_PS_UDP;
84aa0a1e58SJeff Roberson
create_message(struct cmatest_node * node)85aa0a1e58SJeff Roberson static int create_message(struct cmatest_node *node)
86aa0a1e58SJeff Roberson {
87aa0a1e58SJeff Roberson if (!message_size)
88aa0a1e58SJeff Roberson message_count = 0;
89aa0a1e58SJeff Roberson
90aa0a1e58SJeff Roberson if (!message_count)
91aa0a1e58SJeff Roberson return 0;
92aa0a1e58SJeff Roberson
93aa0a1e58SJeff Roberson node->mem = malloc(message_size + sizeof(struct ibv_grh));
94aa0a1e58SJeff Roberson if (!node->mem) {
95aa0a1e58SJeff Roberson printf("failed message allocation\n");
96aa0a1e58SJeff Roberson return -1;
97aa0a1e58SJeff Roberson }
98aa0a1e58SJeff Roberson node->mr = ibv_reg_mr(node->pd, node->mem,
99aa0a1e58SJeff Roberson message_size + sizeof(struct ibv_grh),
100aa0a1e58SJeff Roberson IBV_ACCESS_LOCAL_WRITE);
101aa0a1e58SJeff Roberson if (!node->mr) {
102aa0a1e58SJeff Roberson printf("failed to reg MR\n");
103aa0a1e58SJeff Roberson goto err;
104aa0a1e58SJeff Roberson }
105aa0a1e58SJeff Roberson return 0;
106aa0a1e58SJeff Roberson err:
107aa0a1e58SJeff Roberson free(node->mem);
108aa0a1e58SJeff Roberson return -1;
109aa0a1e58SJeff Roberson }
110aa0a1e58SJeff Roberson
verify_test_params(struct cmatest_node * node)111aa0a1e58SJeff Roberson static int verify_test_params(struct cmatest_node *node)
112aa0a1e58SJeff Roberson {
113aa0a1e58SJeff Roberson struct ibv_port_attr port_attr;
114aa0a1e58SJeff Roberson int ret;
115aa0a1e58SJeff Roberson
116aa0a1e58SJeff Roberson ret = ibv_query_port(node->cma_id->verbs, node->cma_id->port_num,
117aa0a1e58SJeff Roberson &port_attr);
118aa0a1e58SJeff Roberson if (ret)
119aa0a1e58SJeff Roberson return ret;
120aa0a1e58SJeff Roberson
121aa0a1e58SJeff Roberson if (message_count && message_size > (1 << (port_attr.active_mtu + 7))) {
122aa0a1e58SJeff Roberson printf("mckey: message_size %d is larger than active mtu %d\n",
123aa0a1e58SJeff Roberson message_size, 1 << (port_attr.active_mtu + 7));
124aa0a1e58SJeff Roberson return -EINVAL;
125aa0a1e58SJeff Roberson }
126aa0a1e58SJeff Roberson
127aa0a1e58SJeff Roberson return 0;
128aa0a1e58SJeff Roberson }
129aa0a1e58SJeff Roberson
init_node(struct cmatest_node * node)130aa0a1e58SJeff Roberson static int init_node(struct cmatest_node *node)
131aa0a1e58SJeff Roberson {
132aa0a1e58SJeff Roberson struct ibv_qp_init_attr init_qp_attr;
133aa0a1e58SJeff Roberson int cqe, ret;
134aa0a1e58SJeff Roberson
135aa0a1e58SJeff Roberson node->pd = ibv_alloc_pd(node->cma_id->verbs);
136aa0a1e58SJeff Roberson if (!node->pd) {
137aa0a1e58SJeff Roberson ret = -ENOMEM;
138aa0a1e58SJeff Roberson printf("mckey: unable to allocate PD\n");
139aa0a1e58SJeff Roberson goto out;
140aa0a1e58SJeff Roberson }
141aa0a1e58SJeff Roberson
142aa0a1e58SJeff Roberson cqe = message_count ? message_count * 2 : 2;
143d6b92ffaSHans Petter Selasky node->cq = ibv_create_cq(node->cma_id->verbs, cqe, node, NULL, 0);
144aa0a1e58SJeff Roberson if (!node->cq) {
145aa0a1e58SJeff Roberson ret = -ENOMEM;
146aa0a1e58SJeff Roberson printf("mckey: unable to create CQ\n");
147aa0a1e58SJeff Roberson goto out;
148aa0a1e58SJeff Roberson }
149aa0a1e58SJeff Roberson
150aa0a1e58SJeff Roberson memset(&init_qp_attr, 0, sizeof init_qp_attr);
151aa0a1e58SJeff Roberson init_qp_attr.cap.max_send_wr = message_count ? message_count : 1;
152aa0a1e58SJeff Roberson init_qp_attr.cap.max_recv_wr = message_count ? message_count : 1;
153aa0a1e58SJeff Roberson init_qp_attr.cap.max_send_sge = 1;
154aa0a1e58SJeff Roberson init_qp_attr.cap.max_recv_sge = 1;
155aa0a1e58SJeff Roberson init_qp_attr.qp_context = node;
156aa0a1e58SJeff Roberson init_qp_attr.sq_sig_all = 0;
157aa0a1e58SJeff Roberson init_qp_attr.qp_type = IBV_QPT_UD;
158aa0a1e58SJeff Roberson init_qp_attr.send_cq = node->cq;
159aa0a1e58SJeff Roberson init_qp_attr.recv_cq = node->cq;
160aa0a1e58SJeff Roberson ret = rdma_create_qp(node->cma_id, node->pd, &init_qp_attr);
161aa0a1e58SJeff Roberson if (ret) {
162aa0a1e58SJeff Roberson perror("mckey: unable to create QP");
163aa0a1e58SJeff Roberson goto out;
164aa0a1e58SJeff Roberson }
165aa0a1e58SJeff Roberson
166aa0a1e58SJeff Roberson ret = create_message(node);
167aa0a1e58SJeff Roberson if (ret) {
168aa0a1e58SJeff Roberson printf("mckey: failed to create messages: %d\n", ret);
169aa0a1e58SJeff Roberson goto out;
170aa0a1e58SJeff Roberson }
171aa0a1e58SJeff Roberson out:
172aa0a1e58SJeff Roberson return ret;
173aa0a1e58SJeff Roberson }
174aa0a1e58SJeff Roberson
post_recvs(struct cmatest_node * node)175aa0a1e58SJeff Roberson static int post_recvs(struct cmatest_node *node)
176aa0a1e58SJeff Roberson {
177aa0a1e58SJeff Roberson struct ibv_recv_wr recv_wr, *recv_failure;
178aa0a1e58SJeff Roberson struct ibv_sge sge;
179aa0a1e58SJeff Roberson int i, ret = 0;
180aa0a1e58SJeff Roberson
181aa0a1e58SJeff Roberson if (!message_count)
182aa0a1e58SJeff Roberson return 0;
183aa0a1e58SJeff Roberson
184aa0a1e58SJeff Roberson recv_wr.next = NULL;
185aa0a1e58SJeff Roberson recv_wr.sg_list = &sge;
186aa0a1e58SJeff Roberson recv_wr.num_sge = 1;
187aa0a1e58SJeff Roberson recv_wr.wr_id = (uintptr_t) node;
188aa0a1e58SJeff Roberson
189aa0a1e58SJeff Roberson sge.length = message_size + sizeof(struct ibv_grh);
190aa0a1e58SJeff Roberson sge.lkey = node->mr->lkey;
191aa0a1e58SJeff Roberson sge.addr = (uintptr_t) node->mem;
192aa0a1e58SJeff Roberson
193aa0a1e58SJeff Roberson for (i = 0; i < message_count && !ret; i++ ) {
194aa0a1e58SJeff Roberson ret = ibv_post_recv(node->cma_id->qp, &recv_wr, &recv_failure);
195aa0a1e58SJeff Roberson if (ret) {
196aa0a1e58SJeff Roberson printf("failed to post receives: %d\n", ret);
197aa0a1e58SJeff Roberson break;
198aa0a1e58SJeff Roberson }
199aa0a1e58SJeff Roberson }
200aa0a1e58SJeff Roberson return ret;
201aa0a1e58SJeff Roberson }
202aa0a1e58SJeff Roberson
post_sends(struct cmatest_node * node,int signal_flag)203aa0a1e58SJeff Roberson static int post_sends(struct cmatest_node *node, int signal_flag)
204aa0a1e58SJeff Roberson {
205aa0a1e58SJeff Roberson struct ibv_send_wr send_wr, *bad_send_wr;
206aa0a1e58SJeff Roberson struct ibv_sge sge;
207aa0a1e58SJeff Roberson int i, ret = 0;
208aa0a1e58SJeff Roberson
209aa0a1e58SJeff Roberson if (!node->connected || !message_count)
210aa0a1e58SJeff Roberson return 0;
211aa0a1e58SJeff Roberson
212aa0a1e58SJeff Roberson send_wr.next = NULL;
213aa0a1e58SJeff Roberson send_wr.sg_list = &sge;
214aa0a1e58SJeff Roberson send_wr.num_sge = 1;
215aa0a1e58SJeff Roberson send_wr.opcode = IBV_WR_SEND_WITH_IMM;
216aa0a1e58SJeff Roberson send_wr.send_flags = signal_flag;
217aa0a1e58SJeff Roberson send_wr.wr_id = (unsigned long)node;
218d6b92ffaSHans Petter Selasky send_wr.imm_data = htobe32(node->cma_id->qp->qp_num);
219aa0a1e58SJeff Roberson
220aa0a1e58SJeff Roberson send_wr.wr.ud.ah = node->ah;
221aa0a1e58SJeff Roberson send_wr.wr.ud.remote_qpn = node->remote_qpn;
222aa0a1e58SJeff Roberson send_wr.wr.ud.remote_qkey = node->remote_qkey;
223aa0a1e58SJeff Roberson
224aa0a1e58SJeff Roberson sge.length = message_size;
225aa0a1e58SJeff Roberson sge.lkey = node->mr->lkey;
226aa0a1e58SJeff Roberson sge.addr = (uintptr_t) node->mem;
227aa0a1e58SJeff Roberson
228aa0a1e58SJeff Roberson for (i = 0; i < message_count && !ret; i++) {
229aa0a1e58SJeff Roberson ret = ibv_post_send(node->cma_id->qp, &send_wr, &bad_send_wr);
230aa0a1e58SJeff Roberson if (ret)
231aa0a1e58SJeff Roberson printf("failed to post sends: %d\n", ret);
232aa0a1e58SJeff Roberson }
233aa0a1e58SJeff Roberson return ret;
234aa0a1e58SJeff Roberson }
235aa0a1e58SJeff Roberson
connect_error(void)236aa0a1e58SJeff Roberson static void connect_error(void)
237aa0a1e58SJeff Roberson {
238aa0a1e58SJeff Roberson test.connects_left--;
239aa0a1e58SJeff Roberson }
240aa0a1e58SJeff Roberson
addr_handler(struct cmatest_node * node)241aa0a1e58SJeff Roberson static int addr_handler(struct cmatest_node *node)
242aa0a1e58SJeff Roberson {
243aa0a1e58SJeff Roberson int ret;
244aa0a1e58SJeff Roberson
245aa0a1e58SJeff Roberson ret = verify_test_params(node);
246aa0a1e58SJeff Roberson if (ret)
247aa0a1e58SJeff Roberson goto err;
248aa0a1e58SJeff Roberson
249aa0a1e58SJeff Roberson ret = init_node(node);
250aa0a1e58SJeff Roberson if (ret)
251aa0a1e58SJeff Roberson goto err;
252aa0a1e58SJeff Roberson
253aa0a1e58SJeff Roberson if (!is_sender) {
254aa0a1e58SJeff Roberson ret = post_recvs(node);
255aa0a1e58SJeff Roberson if (ret)
256aa0a1e58SJeff Roberson goto err;
257aa0a1e58SJeff Roberson }
258aa0a1e58SJeff Roberson
259aa0a1e58SJeff Roberson ret = rdma_join_multicast(node->cma_id, test.dst_addr, node);
260aa0a1e58SJeff Roberson if (ret) {
261aa0a1e58SJeff Roberson perror("mckey: failure joining");
262aa0a1e58SJeff Roberson goto err;
263aa0a1e58SJeff Roberson }
264aa0a1e58SJeff Roberson return 0;
265aa0a1e58SJeff Roberson err:
266aa0a1e58SJeff Roberson connect_error();
267aa0a1e58SJeff Roberson return ret;
268aa0a1e58SJeff Roberson }
269aa0a1e58SJeff Roberson
join_handler(struct cmatest_node * node,struct rdma_ud_param * param)270aa0a1e58SJeff Roberson static int join_handler(struct cmatest_node *node,
271aa0a1e58SJeff Roberson struct rdma_ud_param *param)
272aa0a1e58SJeff Roberson {
273aa0a1e58SJeff Roberson char buf[40];
274aa0a1e58SJeff Roberson
275aa0a1e58SJeff Roberson inet_ntop(AF_INET6, param->ah_attr.grh.dgid.raw, buf, 40);
276aa0a1e58SJeff Roberson printf("mckey: joined dgid: %s mlid 0x%x sl %d\n", buf,
277aa0a1e58SJeff Roberson param->ah_attr.dlid, param->ah_attr.sl);
278aa0a1e58SJeff Roberson
279aa0a1e58SJeff Roberson node->remote_qpn = param->qp_num;
280aa0a1e58SJeff Roberson node->remote_qkey = param->qkey;
281aa0a1e58SJeff Roberson node->ah = ibv_create_ah(node->pd, ¶m->ah_attr);
282aa0a1e58SJeff Roberson if (!node->ah) {
283aa0a1e58SJeff Roberson printf("mckey: failure creating address handle\n");
284aa0a1e58SJeff Roberson goto err;
285aa0a1e58SJeff Roberson }
286aa0a1e58SJeff Roberson
287aa0a1e58SJeff Roberson node->connected = 1;
288aa0a1e58SJeff Roberson test.connects_left--;
289aa0a1e58SJeff Roberson return 0;
290aa0a1e58SJeff Roberson err:
291aa0a1e58SJeff Roberson connect_error();
292aa0a1e58SJeff Roberson return -1;
293aa0a1e58SJeff Roberson }
294aa0a1e58SJeff Roberson
cma_handler(struct rdma_cm_id * cma_id,struct rdma_cm_event * event)295aa0a1e58SJeff Roberson static int cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
296aa0a1e58SJeff Roberson {
297aa0a1e58SJeff Roberson int ret = 0;
298aa0a1e58SJeff Roberson
299aa0a1e58SJeff Roberson switch (event->event) {
300aa0a1e58SJeff Roberson case RDMA_CM_EVENT_ADDR_RESOLVED:
301aa0a1e58SJeff Roberson ret = addr_handler(cma_id->context);
302aa0a1e58SJeff Roberson break;
303aa0a1e58SJeff Roberson case RDMA_CM_EVENT_MULTICAST_JOIN:
304aa0a1e58SJeff Roberson ret = join_handler(cma_id->context, &event->param.ud);
305aa0a1e58SJeff Roberson break;
306aa0a1e58SJeff Roberson case RDMA_CM_EVENT_ADDR_ERROR:
307aa0a1e58SJeff Roberson case RDMA_CM_EVENT_ROUTE_ERROR:
308aa0a1e58SJeff Roberson case RDMA_CM_EVENT_MULTICAST_ERROR:
309aa0a1e58SJeff Roberson printf("mckey: event: %s, error: %d\n",
310aa0a1e58SJeff Roberson rdma_event_str(event->event), event->status);
311aa0a1e58SJeff Roberson connect_error();
312aa0a1e58SJeff Roberson ret = event->status;
313aa0a1e58SJeff Roberson break;
314aa0a1e58SJeff Roberson case RDMA_CM_EVENT_DEVICE_REMOVAL:
315aa0a1e58SJeff Roberson /* Cleanup will occur after test completes. */
316aa0a1e58SJeff Roberson break;
317aa0a1e58SJeff Roberson default:
318aa0a1e58SJeff Roberson break;
319aa0a1e58SJeff Roberson }
320aa0a1e58SJeff Roberson return ret;
321aa0a1e58SJeff Roberson }
322aa0a1e58SJeff Roberson
cma_thread(void * arg)323aa0a1e58SJeff Roberson static void *cma_thread(void *arg)
324aa0a1e58SJeff Roberson {
325aa0a1e58SJeff Roberson struct rdma_cm_event *event;
326aa0a1e58SJeff Roberson int ret;
327aa0a1e58SJeff Roberson
328aa0a1e58SJeff Roberson while (1) {
329aa0a1e58SJeff Roberson ret = rdma_get_cm_event(test.channel, &event);
330aa0a1e58SJeff Roberson if (ret) {
331aa0a1e58SJeff Roberson perror("rdma_get_cm_event");
332aa0a1e58SJeff Roberson break;
333aa0a1e58SJeff Roberson }
334aa0a1e58SJeff Roberson
335aa0a1e58SJeff Roberson switch (event->event) {
336aa0a1e58SJeff Roberson case RDMA_CM_EVENT_MULTICAST_ERROR:
337aa0a1e58SJeff Roberson case RDMA_CM_EVENT_ADDR_CHANGE:
338aa0a1e58SJeff Roberson printf("mckey: event: %s, status: %d\n",
339aa0a1e58SJeff Roberson rdma_event_str(event->event), event->status);
340aa0a1e58SJeff Roberson break;
341aa0a1e58SJeff Roberson default:
342aa0a1e58SJeff Roberson break;
343aa0a1e58SJeff Roberson }
344aa0a1e58SJeff Roberson
345aa0a1e58SJeff Roberson rdma_ack_cm_event(event);
346aa0a1e58SJeff Roberson }
347aa0a1e58SJeff Roberson return NULL;
348aa0a1e58SJeff Roberson }
349aa0a1e58SJeff Roberson
destroy_node(struct cmatest_node * node)350aa0a1e58SJeff Roberson static void destroy_node(struct cmatest_node *node)
351aa0a1e58SJeff Roberson {
352aa0a1e58SJeff Roberson if (!node->cma_id)
353aa0a1e58SJeff Roberson return;
354aa0a1e58SJeff Roberson
355aa0a1e58SJeff Roberson if (node->ah)
356aa0a1e58SJeff Roberson ibv_destroy_ah(node->ah);
357aa0a1e58SJeff Roberson
358aa0a1e58SJeff Roberson if (node->cma_id->qp)
359aa0a1e58SJeff Roberson rdma_destroy_qp(node->cma_id);
360aa0a1e58SJeff Roberson
361aa0a1e58SJeff Roberson if (node->cq)
362aa0a1e58SJeff Roberson ibv_destroy_cq(node->cq);
363aa0a1e58SJeff Roberson
364aa0a1e58SJeff Roberson if (node->mem) {
365aa0a1e58SJeff Roberson ibv_dereg_mr(node->mr);
366aa0a1e58SJeff Roberson free(node->mem);
367aa0a1e58SJeff Roberson }
368aa0a1e58SJeff Roberson
369aa0a1e58SJeff Roberson if (node->pd)
370aa0a1e58SJeff Roberson ibv_dealloc_pd(node->pd);
371aa0a1e58SJeff Roberson
372aa0a1e58SJeff Roberson /* Destroy the RDMA ID after all device resources */
373aa0a1e58SJeff Roberson rdma_destroy_id(node->cma_id);
374aa0a1e58SJeff Roberson }
375aa0a1e58SJeff Roberson
alloc_nodes(void)376aa0a1e58SJeff Roberson static int alloc_nodes(void)
377aa0a1e58SJeff Roberson {
378aa0a1e58SJeff Roberson int ret, i;
379aa0a1e58SJeff Roberson
380aa0a1e58SJeff Roberson test.nodes = malloc(sizeof *test.nodes * connections);
381aa0a1e58SJeff Roberson if (!test.nodes) {
382aa0a1e58SJeff Roberson printf("mckey: unable to allocate memory for test nodes\n");
383aa0a1e58SJeff Roberson return -ENOMEM;
384aa0a1e58SJeff Roberson }
385aa0a1e58SJeff Roberson memset(test.nodes, 0, sizeof *test.nodes * connections);
386aa0a1e58SJeff Roberson
387aa0a1e58SJeff Roberson for (i = 0; i < connections; i++) {
388aa0a1e58SJeff Roberson test.nodes[i].id = i;
389aa0a1e58SJeff Roberson ret = rdma_create_id(test.channel, &test.nodes[i].cma_id,
390aa0a1e58SJeff Roberson &test.nodes[i], port_space);
391aa0a1e58SJeff Roberson if (ret)
392aa0a1e58SJeff Roberson goto err;
393aa0a1e58SJeff Roberson }
394aa0a1e58SJeff Roberson return 0;
395aa0a1e58SJeff Roberson err:
396aa0a1e58SJeff Roberson while (--i >= 0)
397aa0a1e58SJeff Roberson rdma_destroy_id(test.nodes[i].cma_id);
398aa0a1e58SJeff Roberson free(test.nodes);
399aa0a1e58SJeff Roberson return ret;
400aa0a1e58SJeff Roberson }
401aa0a1e58SJeff Roberson
destroy_nodes(void)402aa0a1e58SJeff Roberson static void destroy_nodes(void)
403aa0a1e58SJeff Roberson {
404aa0a1e58SJeff Roberson int i;
405aa0a1e58SJeff Roberson
406aa0a1e58SJeff Roberson for (i = 0; i < connections; i++)
407aa0a1e58SJeff Roberson destroy_node(&test.nodes[i]);
408aa0a1e58SJeff Roberson free(test.nodes);
409aa0a1e58SJeff Roberson }
410aa0a1e58SJeff Roberson
poll_cqs(void)411aa0a1e58SJeff Roberson static int poll_cqs(void)
412aa0a1e58SJeff Roberson {
413aa0a1e58SJeff Roberson struct ibv_wc wc[8];
414aa0a1e58SJeff Roberson int done, i, ret;
415aa0a1e58SJeff Roberson
416aa0a1e58SJeff Roberson for (i = 0; i < connections; i++) {
417aa0a1e58SJeff Roberson if (!test.nodes[i].connected)
418aa0a1e58SJeff Roberson continue;
419aa0a1e58SJeff Roberson
420aa0a1e58SJeff Roberson for (done = 0; done < message_count; done += ret) {
421aa0a1e58SJeff Roberson ret = ibv_poll_cq(test.nodes[i].cq, 8, wc);
422aa0a1e58SJeff Roberson if (ret < 0) {
423aa0a1e58SJeff Roberson printf("mckey: failed polling CQ: %d\n", ret);
424aa0a1e58SJeff Roberson return ret;
425aa0a1e58SJeff Roberson }
426aa0a1e58SJeff Roberson }
427aa0a1e58SJeff Roberson }
428aa0a1e58SJeff Roberson return 0;
429aa0a1e58SJeff Roberson }
430aa0a1e58SJeff Roberson
connect_events(void)431aa0a1e58SJeff Roberson static int connect_events(void)
432aa0a1e58SJeff Roberson {
433aa0a1e58SJeff Roberson struct rdma_cm_event *event;
434aa0a1e58SJeff Roberson int ret = 0;
435aa0a1e58SJeff Roberson
436aa0a1e58SJeff Roberson while (test.connects_left && !ret) {
437aa0a1e58SJeff Roberson ret = rdma_get_cm_event(test.channel, &event);
438aa0a1e58SJeff Roberson if (!ret) {
439aa0a1e58SJeff Roberson ret = cma_handler(event->id, event);
440aa0a1e58SJeff Roberson rdma_ack_cm_event(event);
441aa0a1e58SJeff Roberson }
442aa0a1e58SJeff Roberson }
443aa0a1e58SJeff Roberson return ret;
444aa0a1e58SJeff Roberson }
445aa0a1e58SJeff Roberson
get_addr(char * dst,struct sockaddr * addr)446aa0a1e58SJeff Roberson static int get_addr(char *dst, struct sockaddr *addr)
447aa0a1e58SJeff Roberson {
448aa0a1e58SJeff Roberson struct addrinfo *res;
449aa0a1e58SJeff Roberson int ret;
450aa0a1e58SJeff Roberson
451aa0a1e58SJeff Roberson ret = getaddrinfo(dst, NULL, NULL, &res);
452aa0a1e58SJeff Roberson if (ret) {
453d6b92ffaSHans Petter Selasky printf("getaddrinfo failed (%s) - invalid hostname or IP address\n", gai_strerror(ret));
454aa0a1e58SJeff Roberson return ret;
455aa0a1e58SJeff Roberson }
456aa0a1e58SJeff Roberson
457aa0a1e58SJeff Roberson memcpy(addr, res->ai_addr, res->ai_addrlen);
458aa0a1e58SJeff Roberson freeaddrinfo(res);
459aa0a1e58SJeff Roberson return ret;
460aa0a1e58SJeff Roberson }
461aa0a1e58SJeff Roberson
get_dst_addr(char * dst,struct sockaddr * addr)462d6b92ffaSHans Petter Selasky static int get_dst_addr(char *dst, struct sockaddr *addr)
463d6b92ffaSHans Petter Selasky {
464d6b92ffaSHans Petter Selasky struct sockaddr_ib *sib;
465d6b92ffaSHans Petter Selasky
466d6b92ffaSHans Petter Selasky if (!unmapped_addr)
467d6b92ffaSHans Petter Selasky return get_addr(dst, addr);
468d6b92ffaSHans Petter Selasky
469d6b92ffaSHans Petter Selasky sib = (struct sockaddr_ib *) addr;
470d6b92ffaSHans Petter Selasky memset(sib, 0, sizeof *sib);
471d6b92ffaSHans Petter Selasky sib->sib_family = AF_IB;
472*d13def78SEric van Gyzen return inet_pton(AF_INET6, dst, &sib->sib_addr) != 1;
473d6b92ffaSHans Petter Selasky }
474d6b92ffaSHans Petter Selasky
run(void)475aa0a1e58SJeff Roberson static int run(void)
476aa0a1e58SJeff Roberson {
477cd9cc48bSSlava Shwartsman int i, ret, err;
478aa0a1e58SJeff Roberson
479aa0a1e58SJeff Roberson printf("mckey: starting %s\n", is_sender ? "client" : "server");
480aa0a1e58SJeff Roberson if (src_addr) {
481aa0a1e58SJeff Roberson ret = get_addr(src_addr, (struct sockaddr *) &test.src_in);
482aa0a1e58SJeff Roberson if (ret)
483aa0a1e58SJeff Roberson return ret;
484aa0a1e58SJeff Roberson }
485aa0a1e58SJeff Roberson
486d6b92ffaSHans Petter Selasky ret = get_dst_addr(dst_addr, (struct sockaddr *) &test.dst_in);
487aa0a1e58SJeff Roberson if (ret)
488aa0a1e58SJeff Roberson return ret;
489aa0a1e58SJeff Roberson
490aa0a1e58SJeff Roberson printf("mckey: joining\n");
491aa0a1e58SJeff Roberson for (i = 0; i < connections; i++) {
492aa0a1e58SJeff Roberson if (src_addr) {
493aa0a1e58SJeff Roberson ret = rdma_bind_addr(test.nodes[i].cma_id,
494aa0a1e58SJeff Roberson test.src_addr);
495aa0a1e58SJeff Roberson if (ret) {
496aa0a1e58SJeff Roberson perror("mckey: addr bind failure");
497aa0a1e58SJeff Roberson connect_error();
498aa0a1e58SJeff Roberson return ret;
499aa0a1e58SJeff Roberson }
500aa0a1e58SJeff Roberson }
501aa0a1e58SJeff Roberson
502aa0a1e58SJeff Roberson if (unmapped_addr)
503aa0a1e58SJeff Roberson ret = addr_handler(&test.nodes[i]);
504aa0a1e58SJeff Roberson else
505aa0a1e58SJeff Roberson ret = rdma_resolve_addr(test.nodes[i].cma_id,
506aa0a1e58SJeff Roberson test.src_addr, test.dst_addr,
507aa0a1e58SJeff Roberson 2000);
508aa0a1e58SJeff Roberson if (ret) {
509aa0a1e58SJeff Roberson perror("mckey: resolve addr failure");
510aa0a1e58SJeff Roberson connect_error();
511aa0a1e58SJeff Roberson return ret;
512aa0a1e58SJeff Roberson }
513aa0a1e58SJeff Roberson }
514aa0a1e58SJeff Roberson
515aa0a1e58SJeff Roberson ret = connect_events();
516aa0a1e58SJeff Roberson if (ret)
517aa0a1e58SJeff Roberson goto out;
518aa0a1e58SJeff Roberson
519aa0a1e58SJeff Roberson pthread_create(&test.cmathread, NULL, cma_thread, NULL);
520aa0a1e58SJeff Roberson
521aa0a1e58SJeff Roberson /*
522aa0a1e58SJeff Roberson * Pause to give SM chance to configure switches. We don't want to
523aa0a1e58SJeff Roberson * handle reliability issue in this simple test program.
524aa0a1e58SJeff Roberson */
525aa0a1e58SJeff Roberson sleep(3);
526aa0a1e58SJeff Roberson
527aa0a1e58SJeff Roberson if (message_count) {
528aa0a1e58SJeff Roberson if (is_sender) {
529aa0a1e58SJeff Roberson printf("initiating data transfers\n");
530aa0a1e58SJeff Roberson for (i = 0; i < connections; i++) {
531aa0a1e58SJeff Roberson ret = post_sends(&test.nodes[i], 0);
532aa0a1e58SJeff Roberson if (ret)
533aa0a1e58SJeff Roberson goto out;
534aa0a1e58SJeff Roberson }
535aa0a1e58SJeff Roberson } else {
536aa0a1e58SJeff Roberson printf("receiving data transfers\n");
537aa0a1e58SJeff Roberson ret = poll_cqs();
538aa0a1e58SJeff Roberson if (ret)
539aa0a1e58SJeff Roberson goto out;
540aa0a1e58SJeff Roberson }
541aa0a1e58SJeff Roberson printf("data transfers complete\n");
542aa0a1e58SJeff Roberson }
543aa0a1e58SJeff Roberson out:
544aa0a1e58SJeff Roberson for (i = 0; i < connections; i++) {
545cd9cc48bSSlava Shwartsman err = rdma_leave_multicast(test.nodes[i].cma_id,
546aa0a1e58SJeff Roberson test.dst_addr);
547cd9cc48bSSlava Shwartsman if (err) {
548aa0a1e58SJeff Roberson perror("mckey: failure leaving");
549cd9cc48bSSlava Shwartsman ret = err;
550cd9cc48bSSlava Shwartsman }
551aa0a1e58SJeff Roberson }
552aa0a1e58SJeff Roberson return ret;
553aa0a1e58SJeff Roberson }
554aa0a1e58SJeff Roberson
main(int argc,char ** argv)555aa0a1e58SJeff Roberson int main(int argc, char **argv)
556aa0a1e58SJeff Roberson {
557aa0a1e58SJeff Roberson int op, ret;
558aa0a1e58SJeff Roberson
559aa0a1e58SJeff Roberson
560aa0a1e58SJeff Roberson while ((op = getopt(argc, argv, "m:M:sb:c:C:S:p:")) != -1) {
561aa0a1e58SJeff Roberson switch (op) {
562aa0a1e58SJeff Roberson case 'm':
563aa0a1e58SJeff Roberson dst_addr = optarg;
564aa0a1e58SJeff Roberson break;
565aa0a1e58SJeff Roberson case 'M':
566aa0a1e58SJeff Roberson unmapped_addr = 1;
567aa0a1e58SJeff Roberson dst_addr = optarg;
568aa0a1e58SJeff Roberson break;
569aa0a1e58SJeff Roberson case 's':
570aa0a1e58SJeff Roberson is_sender = 1;
571aa0a1e58SJeff Roberson break;
572aa0a1e58SJeff Roberson case 'b':
573aa0a1e58SJeff Roberson src_addr = optarg;
574aa0a1e58SJeff Roberson test.src_addr = (struct sockaddr *) &test.src_in;
575aa0a1e58SJeff Roberson break;
576aa0a1e58SJeff Roberson case 'c':
577aa0a1e58SJeff Roberson connections = atoi(optarg);
578aa0a1e58SJeff Roberson break;
579aa0a1e58SJeff Roberson case 'C':
580aa0a1e58SJeff Roberson message_count = atoi(optarg);
581aa0a1e58SJeff Roberson break;
582aa0a1e58SJeff Roberson case 'S':
583aa0a1e58SJeff Roberson message_size = atoi(optarg);
584aa0a1e58SJeff Roberson break;
585aa0a1e58SJeff Roberson case 'p':
586aa0a1e58SJeff Roberson port_space = strtol(optarg, NULL, 0);
587aa0a1e58SJeff Roberson break;
588aa0a1e58SJeff Roberson default:
589aa0a1e58SJeff Roberson printf("usage: %s\n", argv[0]);
590aa0a1e58SJeff Roberson printf("\t-m multicast_address\n");
591aa0a1e58SJeff Roberson printf("\t[-M unmapped_multicast_address]\n"
592aa0a1e58SJeff Roberson "\t replaces -m and requires -b\n");
593aa0a1e58SJeff Roberson printf("\t[-s(ender)]\n");
594aa0a1e58SJeff Roberson printf("\t[-b bind_address]\n");
595aa0a1e58SJeff Roberson printf("\t[-c connections]\n");
596aa0a1e58SJeff Roberson printf("\t[-C message_count]\n");
597aa0a1e58SJeff Roberson printf("\t[-S message_size]\n");
598aa0a1e58SJeff Roberson printf("\t[-p port_space - %#x for UDP (default), "
599aa0a1e58SJeff Roberson "%#x for IPOIB]\n", RDMA_PS_UDP, RDMA_PS_IPOIB);
600aa0a1e58SJeff Roberson exit(1);
601aa0a1e58SJeff Roberson }
602aa0a1e58SJeff Roberson }
603aa0a1e58SJeff Roberson
604aa0a1e58SJeff Roberson if (unmapped_addr && !src_addr) {
605aa0a1e58SJeff Roberson printf("unmapped multicast address requires binding "
606aa0a1e58SJeff Roberson "to source address\n");
607aa0a1e58SJeff Roberson exit(1);
608aa0a1e58SJeff Roberson }
609aa0a1e58SJeff Roberson
610aa0a1e58SJeff Roberson test.dst_addr = (struct sockaddr *) &test.dst_in;
611aa0a1e58SJeff Roberson test.connects_left = connections;
612aa0a1e58SJeff Roberson
613aa0a1e58SJeff Roberson test.channel = rdma_create_event_channel();
614aa0a1e58SJeff Roberson if (!test.channel) {
615aa0a1e58SJeff Roberson perror("failed to create event channel");
616aa0a1e58SJeff Roberson exit(1);
617aa0a1e58SJeff Roberson }
618aa0a1e58SJeff Roberson
619aa0a1e58SJeff Roberson if (alloc_nodes())
620aa0a1e58SJeff Roberson exit(1);
621aa0a1e58SJeff Roberson
622aa0a1e58SJeff Roberson ret = run();
623aa0a1e58SJeff Roberson
624aa0a1e58SJeff Roberson printf("test complete\n");
625aa0a1e58SJeff Roberson destroy_nodes();
626aa0a1e58SJeff Roberson rdma_destroy_event_channel(test.channel);
627aa0a1e58SJeff Roberson
628aa0a1e58SJeff Roberson printf("return status %d\n", ret);
629aa0a1e58SJeff Roberson return ret;
630aa0a1e58SJeff Roberson }
631