xref: /freebsd/contrib/ofed/librdmacm/examples/mckey.c (revision e2c0e292e8a7ca00ba99bcfccc9e637f45c3e8b1)
1aa0a1e58SJeff Roberson /*
2aa0a1e58SJeff Roberson  * Copyright (c) 2005-2007 Intel Corporation.  All rights reserved.
3aa0a1e58SJeff Roberson  *
4aa0a1e58SJeff Roberson  * This software is available to you under a choice of one of two
5aa0a1e58SJeff Roberson  * licenses.  You may choose to be licensed under the terms of the GNU
6aa0a1e58SJeff Roberson  * General Public License (GPL) Version 2, available from the file
7aa0a1e58SJeff Roberson  * COPYING in the main directory of this source tree, or the
8aa0a1e58SJeff Roberson  * OpenIB.org BSD license below:
9aa0a1e58SJeff Roberson  *
10aa0a1e58SJeff Roberson  *     Redistribution and use in source and binary forms, with or
11aa0a1e58SJeff Roberson  *     without modification, are permitted provided that the following
12aa0a1e58SJeff Roberson  *     conditions are met:
13aa0a1e58SJeff Roberson  *
14aa0a1e58SJeff Roberson  *      - Redistributions of source code must retain the above
15aa0a1e58SJeff Roberson  *        copyright notice, this list of conditions and the following
16aa0a1e58SJeff Roberson  *        disclaimer.
17aa0a1e58SJeff Roberson  *
18aa0a1e58SJeff Roberson  *      - Redistributions in binary form must reproduce the above
19aa0a1e58SJeff Roberson  *        copyright notice, this list of conditions and the following
20aa0a1e58SJeff Roberson  *        disclaimer in the documentation and/or other materials
21aa0a1e58SJeff Roberson  *        provided with the distribution.
22aa0a1e58SJeff Roberson  *
23aa0a1e58SJeff Roberson  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24aa0a1e58SJeff Roberson  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25aa0a1e58SJeff Roberson  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26aa0a1e58SJeff Roberson  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27aa0a1e58SJeff Roberson  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28aa0a1e58SJeff Roberson  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29aa0a1e58SJeff Roberson  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30aa0a1e58SJeff Roberson  * SOFTWARE.
31aa0a1e58SJeff Roberson  *
32aa0a1e58SJeff Roberson  * $Id$
33aa0a1e58SJeff Roberson  */
34aa0a1e58SJeff Roberson 
35aa0a1e58SJeff Roberson #include <stdlib.h>
36aa0a1e58SJeff Roberson #include <string.h>
37aa0a1e58SJeff Roberson #include <stdio.h>
38aa0a1e58SJeff Roberson #include <errno.h>
39aa0a1e58SJeff Roberson #include <sys/types.h>
40aa0a1e58SJeff Roberson #include <arpa/inet.h>
41aa0a1e58SJeff Roberson #include <sys/socket.h>
42aa0a1e58SJeff Roberson #include <netdb.h>
43aa0a1e58SJeff Roberson #include <unistd.h>
44aa0a1e58SJeff Roberson #include <getopt.h>
45aa0a1e58SJeff Roberson 
46aa0a1e58SJeff Roberson #include <rdma/rdma_cma.h>
47d6b92ffaSHans Petter Selasky #include <infiniband/ib.h>
48aa0a1e58SJeff Roberson 
49aa0a1e58SJeff Roberson struct cmatest_node {
50aa0a1e58SJeff Roberson 	int			id;
51aa0a1e58SJeff Roberson 	struct rdma_cm_id	*cma_id;
52aa0a1e58SJeff Roberson 	int			connected;
53aa0a1e58SJeff Roberson 	struct ibv_pd		*pd;
54aa0a1e58SJeff Roberson 	struct ibv_cq		*cq;
55aa0a1e58SJeff Roberson 	struct ibv_mr		*mr;
56aa0a1e58SJeff Roberson 	struct ibv_ah		*ah;
57aa0a1e58SJeff Roberson 	uint32_t		remote_qpn;
58aa0a1e58SJeff Roberson 	uint32_t		remote_qkey;
59aa0a1e58SJeff Roberson 	void			*mem;
60aa0a1e58SJeff Roberson };
61aa0a1e58SJeff Roberson 
62aa0a1e58SJeff Roberson struct cmatest {
63aa0a1e58SJeff Roberson 	struct rdma_event_channel *channel;
64aa0a1e58SJeff Roberson 	pthread_t 		cmathread;
65aa0a1e58SJeff Roberson 	struct cmatest_node	*nodes;
66aa0a1e58SJeff Roberson 	int			conn_index;
67aa0a1e58SJeff Roberson 	int			connects_left;
68aa0a1e58SJeff Roberson 
69d6b92ffaSHans Petter Selasky 	struct sockaddr_storage	dst_in;
70aa0a1e58SJeff Roberson 	struct sockaddr		*dst_addr;
71d6b92ffaSHans Petter Selasky 	struct sockaddr_storage	src_in;
72aa0a1e58SJeff Roberson 	struct sockaddr		*src_addr;
73aa0a1e58SJeff Roberson };
74aa0a1e58SJeff Roberson 
75aa0a1e58SJeff Roberson static struct cmatest test;
76aa0a1e58SJeff Roberson static int connections = 1;
77aa0a1e58SJeff Roberson static int message_size = 100;
78aa0a1e58SJeff Roberson static int message_count = 10;
79aa0a1e58SJeff Roberson static int is_sender;
80aa0a1e58SJeff Roberson static int unmapped_addr;
81aa0a1e58SJeff Roberson static char *dst_addr;
82aa0a1e58SJeff Roberson static char *src_addr;
83aa0a1e58SJeff Roberson static enum rdma_port_space port_space = RDMA_PS_UDP;
84aa0a1e58SJeff Roberson 
create_message(struct cmatest_node * node)85aa0a1e58SJeff Roberson static int create_message(struct cmatest_node *node)
86aa0a1e58SJeff Roberson {
87aa0a1e58SJeff Roberson 	if (!message_size)
88aa0a1e58SJeff Roberson 		message_count = 0;
89aa0a1e58SJeff Roberson 
90aa0a1e58SJeff Roberson 	if (!message_count)
91aa0a1e58SJeff Roberson 		return 0;
92aa0a1e58SJeff Roberson 
93aa0a1e58SJeff Roberson 	node->mem = malloc(message_size + sizeof(struct ibv_grh));
94aa0a1e58SJeff Roberson 	if (!node->mem) {
95aa0a1e58SJeff Roberson 		printf("failed message allocation\n");
96aa0a1e58SJeff Roberson 		return -1;
97aa0a1e58SJeff Roberson 	}
98aa0a1e58SJeff Roberson 	node->mr = ibv_reg_mr(node->pd, node->mem,
99aa0a1e58SJeff Roberson 			      message_size + sizeof(struct ibv_grh),
100aa0a1e58SJeff Roberson 			      IBV_ACCESS_LOCAL_WRITE);
101aa0a1e58SJeff Roberson 	if (!node->mr) {
102aa0a1e58SJeff Roberson 		printf("failed to reg MR\n");
103aa0a1e58SJeff Roberson 		goto err;
104aa0a1e58SJeff Roberson 	}
105aa0a1e58SJeff Roberson 	return 0;
106aa0a1e58SJeff Roberson err:
107aa0a1e58SJeff Roberson 	free(node->mem);
108aa0a1e58SJeff Roberson 	return -1;
109aa0a1e58SJeff Roberson }
110aa0a1e58SJeff Roberson 
verify_test_params(struct cmatest_node * node)111aa0a1e58SJeff Roberson static int verify_test_params(struct cmatest_node *node)
112aa0a1e58SJeff Roberson {
113aa0a1e58SJeff Roberson 	struct ibv_port_attr port_attr;
114aa0a1e58SJeff Roberson 	int ret;
115aa0a1e58SJeff Roberson 
116aa0a1e58SJeff Roberson 	ret = ibv_query_port(node->cma_id->verbs, node->cma_id->port_num,
117aa0a1e58SJeff Roberson 			     &port_attr);
118aa0a1e58SJeff Roberson 	if (ret)
119aa0a1e58SJeff Roberson 		return ret;
120aa0a1e58SJeff Roberson 
121aa0a1e58SJeff Roberson 	if (message_count && message_size > (1 << (port_attr.active_mtu + 7))) {
122aa0a1e58SJeff Roberson 		printf("mckey: message_size %d is larger than active mtu %d\n",
123aa0a1e58SJeff Roberson 		       message_size, 1 << (port_attr.active_mtu + 7));
124aa0a1e58SJeff Roberson 		return -EINVAL;
125aa0a1e58SJeff Roberson 	}
126aa0a1e58SJeff Roberson 
127aa0a1e58SJeff Roberson 	return 0;
128aa0a1e58SJeff Roberson }
129aa0a1e58SJeff Roberson 
init_node(struct cmatest_node * node)130aa0a1e58SJeff Roberson static int init_node(struct cmatest_node *node)
131aa0a1e58SJeff Roberson {
132aa0a1e58SJeff Roberson 	struct ibv_qp_init_attr init_qp_attr;
133aa0a1e58SJeff Roberson 	int cqe, ret;
134aa0a1e58SJeff Roberson 
135aa0a1e58SJeff Roberson 	node->pd = ibv_alloc_pd(node->cma_id->verbs);
136aa0a1e58SJeff Roberson 	if (!node->pd) {
137aa0a1e58SJeff Roberson 		ret = -ENOMEM;
138aa0a1e58SJeff Roberson 		printf("mckey: unable to allocate PD\n");
139aa0a1e58SJeff Roberson 		goto out;
140aa0a1e58SJeff Roberson 	}
141aa0a1e58SJeff Roberson 
142aa0a1e58SJeff Roberson 	cqe = message_count ? message_count * 2 : 2;
143d6b92ffaSHans Petter Selasky 	node->cq = ibv_create_cq(node->cma_id->verbs, cqe, node, NULL, 0);
144aa0a1e58SJeff Roberson 	if (!node->cq) {
145aa0a1e58SJeff Roberson 		ret = -ENOMEM;
146aa0a1e58SJeff Roberson 		printf("mckey: unable to create CQ\n");
147aa0a1e58SJeff Roberson 		goto out;
148aa0a1e58SJeff Roberson 	}
149aa0a1e58SJeff Roberson 
150aa0a1e58SJeff Roberson 	memset(&init_qp_attr, 0, sizeof init_qp_attr);
151aa0a1e58SJeff Roberson 	init_qp_attr.cap.max_send_wr = message_count ? message_count : 1;
152aa0a1e58SJeff Roberson 	init_qp_attr.cap.max_recv_wr = message_count ? message_count : 1;
153aa0a1e58SJeff Roberson 	init_qp_attr.cap.max_send_sge = 1;
154aa0a1e58SJeff Roberson 	init_qp_attr.cap.max_recv_sge = 1;
155aa0a1e58SJeff Roberson 	init_qp_attr.qp_context = node;
156aa0a1e58SJeff Roberson 	init_qp_attr.sq_sig_all = 0;
157aa0a1e58SJeff Roberson 	init_qp_attr.qp_type = IBV_QPT_UD;
158aa0a1e58SJeff Roberson 	init_qp_attr.send_cq = node->cq;
159aa0a1e58SJeff Roberson 	init_qp_attr.recv_cq = node->cq;
160aa0a1e58SJeff Roberson 	ret = rdma_create_qp(node->cma_id, node->pd, &init_qp_attr);
161aa0a1e58SJeff Roberson 	if (ret) {
162aa0a1e58SJeff Roberson 		perror("mckey: unable to create QP");
163aa0a1e58SJeff Roberson 		goto out;
164aa0a1e58SJeff Roberson 	}
165aa0a1e58SJeff Roberson 
166aa0a1e58SJeff Roberson 	ret = create_message(node);
167aa0a1e58SJeff Roberson 	if (ret) {
168aa0a1e58SJeff Roberson 		printf("mckey: failed to create messages: %d\n", ret);
169aa0a1e58SJeff Roberson 		goto out;
170aa0a1e58SJeff Roberson 	}
171aa0a1e58SJeff Roberson out:
172aa0a1e58SJeff Roberson 	return ret;
173aa0a1e58SJeff Roberson }
174aa0a1e58SJeff Roberson 
post_recvs(struct cmatest_node * node)175aa0a1e58SJeff Roberson static int post_recvs(struct cmatest_node *node)
176aa0a1e58SJeff Roberson {
177aa0a1e58SJeff Roberson 	struct ibv_recv_wr recv_wr, *recv_failure;
178aa0a1e58SJeff Roberson 	struct ibv_sge sge;
179aa0a1e58SJeff Roberson 	int i, ret = 0;
180aa0a1e58SJeff Roberson 
181aa0a1e58SJeff Roberson 	if (!message_count)
182aa0a1e58SJeff Roberson 		return 0;
183aa0a1e58SJeff Roberson 
184aa0a1e58SJeff Roberson 	recv_wr.next = NULL;
185aa0a1e58SJeff Roberson 	recv_wr.sg_list = &sge;
186aa0a1e58SJeff Roberson 	recv_wr.num_sge = 1;
187aa0a1e58SJeff Roberson 	recv_wr.wr_id = (uintptr_t) node;
188aa0a1e58SJeff Roberson 
189aa0a1e58SJeff Roberson 	sge.length = message_size + sizeof(struct ibv_grh);
190aa0a1e58SJeff Roberson 	sge.lkey = node->mr->lkey;
191aa0a1e58SJeff Roberson 	sge.addr = (uintptr_t) node->mem;
192aa0a1e58SJeff Roberson 
193aa0a1e58SJeff Roberson 	for (i = 0; i < message_count && !ret; i++ ) {
194aa0a1e58SJeff Roberson 		ret = ibv_post_recv(node->cma_id->qp, &recv_wr, &recv_failure);
195aa0a1e58SJeff Roberson 		if (ret) {
196aa0a1e58SJeff Roberson 			printf("failed to post receives: %d\n", ret);
197aa0a1e58SJeff Roberson 			break;
198aa0a1e58SJeff Roberson 		}
199aa0a1e58SJeff Roberson 	}
200aa0a1e58SJeff Roberson 	return ret;
201aa0a1e58SJeff Roberson }
202aa0a1e58SJeff Roberson 
post_sends(struct cmatest_node * node,int signal_flag)203aa0a1e58SJeff Roberson static int post_sends(struct cmatest_node *node, int signal_flag)
204aa0a1e58SJeff Roberson {
205aa0a1e58SJeff Roberson 	struct ibv_send_wr send_wr, *bad_send_wr;
206aa0a1e58SJeff Roberson 	struct ibv_sge sge;
207aa0a1e58SJeff Roberson 	int i, ret = 0;
208aa0a1e58SJeff Roberson 
209aa0a1e58SJeff Roberson 	if (!node->connected || !message_count)
210aa0a1e58SJeff Roberson 		return 0;
211aa0a1e58SJeff Roberson 
212aa0a1e58SJeff Roberson 	send_wr.next = NULL;
213aa0a1e58SJeff Roberson 	send_wr.sg_list = &sge;
214aa0a1e58SJeff Roberson 	send_wr.num_sge = 1;
215aa0a1e58SJeff Roberson 	send_wr.opcode = IBV_WR_SEND_WITH_IMM;
216aa0a1e58SJeff Roberson 	send_wr.send_flags = signal_flag;
217aa0a1e58SJeff Roberson 	send_wr.wr_id = (unsigned long)node;
218d6b92ffaSHans Petter Selasky 	send_wr.imm_data = htobe32(node->cma_id->qp->qp_num);
219aa0a1e58SJeff Roberson 
220aa0a1e58SJeff Roberson 	send_wr.wr.ud.ah = node->ah;
221aa0a1e58SJeff Roberson 	send_wr.wr.ud.remote_qpn = node->remote_qpn;
222aa0a1e58SJeff Roberson 	send_wr.wr.ud.remote_qkey = node->remote_qkey;
223aa0a1e58SJeff Roberson 
224aa0a1e58SJeff Roberson 	sge.length = message_size;
225aa0a1e58SJeff Roberson 	sge.lkey = node->mr->lkey;
226aa0a1e58SJeff Roberson 	sge.addr = (uintptr_t) node->mem;
227aa0a1e58SJeff Roberson 
228aa0a1e58SJeff Roberson 	for (i = 0; i < message_count && !ret; i++) {
229aa0a1e58SJeff Roberson 		ret = ibv_post_send(node->cma_id->qp, &send_wr, &bad_send_wr);
230aa0a1e58SJeff Roberson 		if (ret)
231aa0a1e58SJeff Roberson 			printf("failed to post sends: %d\n", ret);
232aa0a1e58SJeff Roberson 	}
233aa0a1e58SJeff Roberson 	return ret;
234aa0a1e58SJeff Roberson }
235aa0a1e58SJeff Roberson 
connect_error(void)236aa0a1e58SJeff Roberson static void connect_error(void)
237aa0a1e58SJeff Roberson {
238aa0a1e58SJeff Roberson 	test.connects_left--;
239aa0a1e58SJeff Roberson }
240aa0a1e58SJeff Roberson 
addr_handler(struct cmatest_node * node)241aa0a1e58SJeff Roberson static int addr_handler(struct cmatest_node *node)
242aa0a1e58SJeff Roberson {
243aa0a1e58SJeff Roberson 	int ret;
244aa0a1e58SJeff Roberson 
245aa0a1e58SJeff Roberson 	ret = verify_test_params(node);
246aa0a1e58SJeff Roberson 	if (ret)
247aa0a1e58SJeff Roberson 		goto err;
248aa0a1e58SJeff Roberson 
249aa0a1e58SJeff Roberson 	ret = init_node(node);
250aa0a1e58SJeff Roberson 	if (ret)
251aa0a1e58SJeff Roberson 		goto err;
252aa0a1e58SJeff Roberson 
253aa0a1e58SJeff Roberson 	if (!is_sender) {
254aa0a1e58SJeff Roberson 		ret = post_recvs(node);
255aa0a1e58SJeff Roberson 		if (ret)
256aa0a1e58SJeff Roberson 			goto err;
257aa0a1e58SJeff Roberson 	}
258aa0a1e58SJeff Roberson 
259aa0a1e58SJeff Roberson 	ret = rdma_join_multicast(node->cma_id, test.dst_addr, node);
260aa0a1e58SJeff Roberson 	if (ret) {
261aa0a1e58SJeff Roberson 		perror("mckey: failure joining");
262aa0a1e58SJeff Roberson 		goto err;
263aa0a1e58SJeff Roberson 	}
264aa0a1e58SJeff Roberson 	return 0;
265aa0a1e58SJeff Roberson err:
266aa0a1e58SJeff Roberson 	connect_error();
267aa0a1e58SJeff Roberson 	return ret;
268aa0a1e58SJeff Roberson }
269aa0a1e58SJeff Roberson 
join_handler(struct cmatest_node * node,struct rdma_ud_param * param)270aa0a1e58SJeff Roberson static int join_handler(struct cmatest_node *node,
271aa0a1e58SJeff Roberson 			struct rdma_ud_param *param)
272aa0a1e58SJeff Roberson {
273aa0a1e58SJeff Roberson 	char buf[40];
274aa0a1e58SJeff Roberson 
275aa0a1e58SJeff Roberson 	inet_ntop(AF_INET6, param->ah_attr.grh.dgid.raw, buf, 40);
276aa0a1e58SJeff Roberson 	printf("mckey: joined dgid: %s mlid 0x%x sl %d\n", buf,
277aa0a1e58SJeff Roberson 		param->ah_attr.dlid, param->ah_attr.sl);
278aa0a1e58SJeff Roberson 
279aa0a1e58SJeff Roberson 	node->remote_qpn = param->qp_num;
280aa0a1e58SJeff Roberson 	node->remote_qkey = param->qkey;
281aa0a1e58SJeff Roberson 	node->ah = ibv_create_ah(node->pd, &param->ah_attr);
282aa0a1e58SJeff Roberson 	if (!node->ah) {
283aa0a1e58SJeff Roberson 		printf("mckey: failure creating address handle\n");
284aa0a1e58SJeff Roberson 		goto err;
285aa0a1e58SJeff Roberson 	}
286aa0a1e58SJeff Roberson 
287aa0a1e58SJeff Roberson 	node->connected = 1;
288aa0a1e58SJeff Roberson 	test.connects_left--;
289aa0a1e58SJeff Roberson 	return 0;
290aa0a1e58SJeff Roberson err:
291aa0a1e58SJeff Roberson 	connect_error();
292aa0a1e58SJeff Roberson 	return -1;
293aa0a1e58SJeff Roberson }
294aa0a1e58SJeff Roberson 
cma_handler(struct rdma_cm_id * cma_id,struct rdma_cm_event * event)295aa0a1e58SJeff Roberson static int cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event)
296aa0a1e58SJeff Roberson {
297aa0a1e58SJeff Roberson 	int ret = 0;
298aa0a1e58SJeff Roberson 
299aa0a1e58SJeff Roberson 	switch (event->event) {
300aa0a1e58SJeff Roberson 	case RDMA_CM_EVENT_ADDR_RESOLVED:
301aa0a1e58SJeff Roberson 		ret = addr_handler(cma_id->context);
302aa0a1e58SJeff Roberson 		break;
303aa0a1e58SJeff Roberson 	case RDMA_CM_EVENT_MULTICAST_JOIN:
304aa0a1e58SJeff Roberson 		ret = join_handler(cma_id->context, &event->param.ud);
305aa0a1e58SJeff Roberson 		break;
306aa0a1e58SJeff Roberson 	case RDMA_CM_EVENT_ADDR_ERROR:
307aa0a1e58SJeff Roberson 	case RDMA_CM_EVENT_ROUTE_ERROR:
308aa0a1e58SJeff Roberson 	case RDMA_CM_EVENT_MULTICAST_ERROR:
309aa0a1e58SJeff Roberson 		printf("mckey: event: %s, error: %d\n",
310aa0a1e58SJeff Roberson 		       rdma_event_str(event->event), event->status);
311aa0a1e58SJeff Roberson 		connect_error();
312aa0a1e58SJeff Roberson 		ret = event->status;
313aa0a1e58SJeff Roberson 		break;
314aa0a1e58SJeff Roberson 	case RDMA_CM_EVENT_DEVICE_REMOVAL:
315aa0a1e58SJeff Roberson 		/* Cleanup will occur after test completes. */
316aa0a1e58SJeff Roberson 		break;
317aa0a1e58SJeff Roberson 	default:
318aa0a1e58SJeff Roberson 		break;
319aa0a1e58SJeff Roberson 	}
320aa0a1e58SJeff Roberson 	return ret;
321aa0a1e58SJeff Roberson }
322aa0a1e58SJeff Roberson 
cma_thread(void * arg)323aa0a1e58SJeff Roberson static void *cma_thread(void *arg)
324aa0a1e58SJeff Roberson {
325aa0a1e58SJeff Roberson 	struct rdma_cm_event *event;
326aa0a1e58SJeff Roberson 	int ret;
327aa0a1e58SJeff Roberson 
328aa0a1e58SJeff Roberson 	while (1) {
329aa0a1e58SJeff Roberson 		ret = rdma_get_cm_event(test.channel, &event);
330aa0a1e58SJeff Roberson 		if (ret) {
331aa0a1e58SJeff Roberson 			perror("rdma_get_cm_event");
332aa0a1e58SJeff Roberson 			break;
333aa0a1e58SJeff Roberson 		}
334aa0a1e58SJeff Roberson 
335aa0a1e58SJeff Roberson 		switch (event->event) {
336aa0a1e58SJeff Roberson 		case RDMA_CM_EVENT_MULTICAST_ERROR:
337aa0a1e58SJeff Roberson 		case RDMA_CM_EVENT_ADDR_CHANGE:
338aa0a1e58SJeff Roberson 			printf("mckey: event: %s, status: %d\n",
339aa0a1e58SJeff Roberson 			       rdma_event_str(event->event), event->status);
340aa0a1e58SJeff Roberson 			break;
341aa0a1e58SJeff Roberson 		default:
342aa0a1e58SJeff Roberson 			break;
343aa0a1e58SJeff Roberson 		}
344aa0a1e58SJeff Roberson 
345aa0a1e58SJeff Roberson 		rdma_ack_cm_event(event);
346aa0a1e58SJeff Roberson 	}
347aa0a1e58SJeff Roberson 	return NULL;
348aa0a1e58SJeff Roberson }
349aa0a1e58SJeff Roberson 
destroy_node(struct cmatest_node * node)350aa0a1e58SJeff Roberson static void destroy_node(struct cmatest_node *node)
351aa0a1e58SJeff Roberson {
352aa0a1e58SJeff Roberson 	if (!node->cma_id)
353aa0a1e58SJeff Roberson 		return;
354aa0a1e58SJeff Roberson 
355aa0a1e58SJeff Roberson 	if (node->ah)
356aa0a1e58SJeff Roberson 		ibv_destroy_ah(node->ah);
357aa0a1e58SJeff Roberson 
358aa0a1e58SJeff Roberson 	if (node->cma_id->qp)
359aa0a1e58SJeff Roberson 		rdma_destroy_qp(node->cma_id);
360aa0a1e58SJeff Roberson 
361aa0a1e58SJeff Roberson 	if (node->cq)
362aa0a1e58SJeff Roberson 		ibv_destroy_cq(node->cq);
363aa0a1e58SJeff Roberson 
364aa0a1e58SJeff Roberson 	if (node->mem) {
365aa0a1e58SJeff Roberson 		ibv_dereg_mr(node->mr);
366aa0a1e58SJeff Roberson 		free(node->mem);
367aa0a1e58SJeff Roberson 	}
368aa0a1e58SJeff Roberson 
369aa0a1e58SJeff Roberson 	if (node->pd)
370aa0a1e58SJeff Roberson 		ibv_dealloc_pd(node->pd);
371aa0a1e58SJeff Roberson 
372aa0a1e58SJeff Roberson 	/* Destroy the RDMA ID after all device resources */
373aa0a1e58SJeff Roberson 	rdma_destroy_id(node->cma_id);
374aa0a1e58SJeff Roberson }
375aa0a1e58SJeff Roberson 
alloc_nodes(void)376aa0a1e58SJeff Roberson static int alloc_nodes(void)
377aa0a1e58SJeff Roberson {
378aa0a1e58SJeff Roberson 	int ret, i;
379aa0a1e58SJeff Roberson 
380aa0a1e58SJeff Roberson 	test.nodes = malloc(sizeof *test.nodes * connections);
381aa0a1e58SJeff Roberson 	if (!test.nodes) {
382aa0a1e58SJeff Roberson 		printf("mckey: unable to allocate memory for test nodes\n");
383aa0a1e58SJeff Roberson 		return -ENOMEM;
384aa0a1e58SJeff Roberson 	}
385aa0a1e58SJeff Roberson 	memset(test.nodes, 0, sizeof *test.nodes * connections);
386aa0a1e58SJeff Roberson 
387aa0a1e58SJeff Roberson 	for (i = 0; i < connections; i++) {
388aa0a1e58SJeff Roberson 		test.nodes[i].id = i;
389aa0a1e58SJeff Roberson 		ret = rdma_create_id(test.channel, &test.nodes[i].cma_id,
390aa0a1e58SJeff Roberson 				     &test.nodes[i], port_space);
391aa0a1e58SJeff Roberson 		if (ret)
392aa0a1e58SJeff Roberson 			goto err;
393aa0a1e58SJeff Roberson 	}
394aa0a1e58SJeff Roberson 	return 0;
395aa0a1e58SJeff Roberson err:
396aa0a1e58SJeff Roberson 	while (--i >= 0)
397aa0a1e58SJeff Roberson 		rdma_destroy_id(test.nodes[i].cma_id);
398aa0a1e58SJeff Roberson 	free(test.nodes);
399aa0a1e58SJeff Roberson 	return ret;
400aa0a1e58SJeff Roberson }
401aa0a1e58SJeff Roberson 
destroy_nodes(void)402aa0a1e58SJeff Roberson static void destroy_nodes(void)
403aa0a1e58SJeff Roberson {
404aa0a1e58SJeff Roberson 	int i;
405aa0a1e58SJeff Roberson 
406aa0a1e58SJeff Roberson 	for (i = 0; i < connections; i++)
407aa0a1e58SJeff Roberson 		destroy_node(&test.nodes[i]);
408aa0a1e58SJeff Roberson 	free(test.nodes);
409aa0a1e58SJeff Roberson }
410aa0a1e58SJeff Roberson 
poll_cqs(void)411aa0a1e58SJeff Roberson static int poll_cqs(void)
412aa0a1e58SJeff Roberson {
413aa0a1e58SJeff Roberson 	struct ibv_wc wc[8];
414aa0a1e58SJeff Roberson 	int done, i, ret;
415aa0a1e58SJeff Roberson 
416aa0a1e58SJeff Roberson 	for (i = 0; i < connections; i++) {
417aa0a1e58SJeff Roberson 		if (!test.nodes[i].connected)
418aa0a1e58SJeff Roberson 			continue;
419aa0a1e58SJeff Roberson 
420aa0a1e58SJeff Roberson 		for (done = 0; done < message_count; done += ret) {
421aa0a1e58SJeff Roberson 			ret = ibv_poll_cq(test.nodes[i].cq, 8, wc);
422aa0a1e58SJeff Roberson 			if (ret < 0) {
423aa0a1e58SJeff Roberson 				printf("mckey: failed polling CQ: %d\n", ret);
424aa0a1e58SJeff Roberson 				return ret;
425aa0a1e58SJeff Roberson 			}
426aa0a1e58SJeff Roberson 		}
427aa0a1e58SJeff Roberson 	}
428aa0a1e58SJeff Roberson 	return 0;
429aa0a1e58SJeff Roberson }
430aa0a1e58SJeff Roberson 
connect_events(void)431aa0a1e58SJeff Roberson static int connect_events(void)
432aa0a1e58SJeff Roberson {
433aa0a1e58SJeff Roberson 	struct rdma_cm_event *event;
434aa0a1e58SJeff Roberson 	int ret = 0;
435aa0a1e58SJeff Roberson 
436aa0a1e58SJeff Roberson 	while (test.connects_left && !ret) {
437aa0a1e58SJeff Roberson 		ret = rdma_get_cm_event(test.channel, &event);
438aa0a1e58SJeff Roberson 		if (!ret) {
439aa0a1e58SJeff Roberson 			ret = cma_handler(event->id, event);
440aa0a1e58SJeff Roberson 			rdma_ack_cm_event(event);
441aa0a1e58SJeff Roberson 		}
442aa0a1e58SJeff Roberson 	}
443aa0a1e58SJeff Roberson 	return ret;
444aa0a1e58SJeff Roberson }
445aa0a1e58SJeff Roberson 
get_addr(char * dst,struct sockaddr * addr)446aa0a1e58SJeff Roberson static int get_addr(char *dst, struct sockaddr *addr)
447aa0a1e58SJeff Roberson {
448aa0a1e58SJeff Roberson 	struct addrinfo *res;
449aa0a1e58SJeff Roberson 	int ret;
450aa0a1e58SJeff Roberson 
451aa0a1e58SJeff Roberson 	ret = getaddrinfo(dst, NULL, NULL, &res);
452aa0a1e58SJeff Roberson 	if (ret) {
453d6b92ffaSHans Petter Selasky 		printf("getaddrinfo failed (%s) - invalid hostname or IP address\n", gai_strerror(ret));
454aa0a1e58SJeff Roberson 		return ret;
455aa0a1e58SJeff Roberson 	}
456aa0a1e58SJeff Roberson 
457aa0a1e58SJeff Roberson 	memcpy(addr, res->ai_addr, res->ai_addrlen);
458aa0a1e58SJeff Roberson 	freeaddrinfo(res);
459aa0a1e58SJeff Roberson 	return ret;
460aa0a1e58SJeff Roberson }
461aa0a1e58SJeff Roberson 
get_dst_addr(char * dst,struct sockaddr * addr)462d6b92ffaSHans Petter Selasky static int get_dst_addr(char *dst, struct sockaddr *addr)
463d6b92ffaSHans Petter Selasky {
464d6b92ffaSHans Petter Selasky 	struct sockaddr_ib *sib;
465d6b92ffaSHans Petter Selasky 
466d6b92ffaSHans Petter Selasky 	if (!unmapped_addr)
467d6b92ffaSHans Petter Selasky 		return get_addr(dst, addr);
468d6b92ffaSHans Petter Selasky 
469d6b92ffaSHans Petter Selasky 	sib = (struct sockaddr_ib *) addr;
470d6b92ffaSHans Petter Selasky 	memset(sib, 0, sizeof *sib);
471d6b92ffaSHans Petter Selasky 	sib->sib_family = AF_IB;
472*d13def78SEric van Gyzen 	return inet_pton(AF_INET6, dst, &sib->sib_addr) != 1;
473d6b92ffaSHans Petter Selasky }
474d6b92ffaSHans Petter Selasky 
run(void)475aa0a1e58SJeff Roberson static int run(void)
476aa0a1e58SJeff Roberson {
477cd9cc48bSSlava Shwartsman 	int i, ret, err;
478aa0a1e58SJeff Roberson 
479aa0a1e58SJeff Roberson 	printf("mckey: starting %s\n", is_sender ? "client" : "server");
480aa0a1e58SJeff Roberson 	if (src_addr) {
481aa0a1e58SJeff Roberson 		ret = get_addr(src_addr, (struct sockaddr *) &test.src_in);
482aa0a1e58SJeff Roberson 		if (ret)
483aa0a1e58SJeff Roberson 			return ret;
484aa0a1e58SJeff Roberson 	}
485aa0a1e58SJeff Roberson 
486d6b92ffaSHans Petter Selasky 	ret = get_dst_addr(dst_addr, (struct sockaddr *) &test.dst_in);
487aa0a1e58SJeff Roberson 	if (ret)
488aa0a1e58SJeff Roberson 		return ret;
489aa0a1e58SJeff Roberson 
490aa0a1e58SJeff Roberson 	printf("mckey: joining\n");
491aa0a1e58SJeff Roberson 	for (i = 0; i < connections; i++) {
492aa0a1e58SJeff Roberson 		if (src_addr) {
493aa0a1e58SJeff Roberson 			ret = rdma_bind_addr(test.nodes[i].cma_id,
494aa0a1e58SJeff Roberson 					     test.src_addr);
495aa0a1e58SJeff Roberson 			if (ret) {
496aa0a1e58SJeff Roberson 				perror("mckey: addr bind failure");
497aa0a1e58SJeff Roberson 				connect_error();
498aa0a1e58SJeff Roberson 				return ret;
499aa0a1e58SJeff Roberson 			}
500aa0a1e58SJeff Roberson 		}
501aa0a1e58SJeff Roberson 
502aa0a1e58SJeff Roberson 		if (unmapped_addr)
503aa0a1e58SJeff Roberson 			ret = addr_handler(&test.nodes[i]);
504aa0a1e58SJeff Roberson 		else
505aa0a1e58SJeff Roberson 			ret = rdma_resolve_addr(test.nodes[i].cma_id,
506aa0a1e58SJeff Roberson 						test.src_addr, test.dst_addr,
507aa0a1e58SJeff Roberson 						2000);
508aa0a1e58SJeff Roberson 		if (ret) {
509aa0a1e58SJeff Roberson 			perror("mckey: resolve addr failure");
510aa0a1e58SJeff Roberson 			connect_error();
511aa0a1e58SJeff Roberson 			return ret;
512aa0a1e58SJeff Roberson 		}
513aa0a1e58SJeff Roberson 	}
514aa0a1e58SJeff Roberson 
515aa0a1e58SJeff Roberson 	ret = connect_events();
516aa0a1e58SJeff Roberson 	if (ret)
517aa0a1e58SJeff Roberson 		goto out;
518aa0a1e58SJeff Roberson 
519aa0a1e58SJeff Roberson 	pthread_create(&test.cmathread, NULL, cma_thread, NULL);
520aa0a1e58SJeff Roberson 
521aa0a1e58SJeff Roberson 	/*
522aa0a1e58SJeff Roberson 	 * Pause to give SM chance to configure switches.  We don't want to
523aa0a1e58SJeff Roberson 	 * handle reliability issue in this simple test program.
524aa0a1e58SJeff Roberson 	 */
525aa0a1e58SJeff Roberson 	sleep(3);
526aa0a1e58SJeff Roberson 
527aa0a1e58SJeff Roberson 	if (message_count) {
528aa0a1e58SJeff Roberson 		if (is_sender) {
529aa0a1e58SJeff Roberson 			printf("initiating data transfers\n");
530aa0a1e58SJeff Roberson 			for (i = 0; i < connections; i++) {
531aa0a1e58SJeff Roberson 				ret = post_sends(&test.nodes[i], 0);
532aa0a1e58SJeff Roberson 				if (ret)
533aa0a1e58SJeff Roberson 					goto out;
534aa0a1e58SJeff Roberson 			}
535aa0a1e58SJeff Roberson 		} else {
536aa0a1e58SJeff Roberson 			printf("receiving data transfers\n");
537aa0a1e58SJeff Roberson 			ret = poll_cqs();
538aa0a1e58SJeff Roberson 			if (ret)
539aa0a1e58SJeff Roberson 				goto out;
540aa0a1e58SJeff Roberson 		}
541aa0a1e58SJeff Roberson 		printf("data transfers complete\n");
542aa0a1e58SJeff Roberson 	}
543aa0a1e58SJeff Roberson out:
544aa0a1e58SJeff Roberson 	for (i = 0; i < connections; i++) {
545cd9cc48bSSlava Shwartsman 		err = rdma_leave_multicast(test.nodes[i].cma_id,
546aa0a1e58SJeff Roberson 					   test.dst_addr);
547cd9cc48bSSlava Shwartsman 		if (err) {
548aa0a1e58SJeff Roberson 			perror("mckey: failure leaving");
549cd9cc48bSSlava Shwartsman 			ret = err;
550cd9cc48bSSlava Shwartsman 		}
551aa0a1e58SJeff Roberson 	}
552aa0a1e58SJeff Roberson 	return ret;
553aa0a1e58SJeff Roberson }
554aa0a1e58SJeff Roberson 
main(int argc,char ** argv)555aa0a1e58SJeff Roberson int main(int argc, char **argv)
556aa0a1e58SJeff Roberson {
557aa0a1e58SJeff Roberson 	int op, ret;
558aa0a1e58SJeff Roberson 
559aa0a1e58SJeff Roberson 
560aa0a1e58SJeff Roberson 	while ((op = getopt(argc, argv, "m:M:sb:c:C:S:p:")) != -1) {
561aa0a1e58SJeff Roberson 		switch (op) {
562aa0a1e58SJeff Roberson 		case 'm':
563aa0a1e58SJeff Roberson 			dst_addr = optarg;
564aa0a1e58SJeff Roberson 			break;
565aa0a1e58SJeff Roberson 		case 'M':
566aa0a1e58SJeff Roberson 			unmapped_addr = 1;
567aa0a1e58SJeff Roberson 			dst_addr = optarg;
568aa0a1e58SJeff Roberson 			break;
569aa0a1e58SJeff Roberson 		case 's':
570aa0a1e58SJeff Roberson 			is_sender = 1;
571aa0a1e58SJeff Roberson 			break;
572aa0a1e58SJeff Roberson 		case 'b':
573aa0a1e58SJeff Roberson 			src_addr = optarg;
574aa0a1e58SJeff Roberson 			test.src_addr = (struct sockaddr *) &test.src_in;
575aa0a1e58SJeff Roberson 			break;
576aa0a1e58SJeff Roberson 		case 'c':
577aa0a1e58SJeff Roberson 			connections = atoi(optarg);
578aa0a1e58SJeff Roberson 			break;
579aa0a1e58SJeff Roberson 		case 'C':
580aa0a1e58SJeff Roberson 			message_count = atoi(optarg);
581aa0a1e58SJeff Roberson 			break;
582aa0a1e58SJeff Roberson 		case 'S':
583aa0a1e58SJeff Roberson 			message_size = atoi(optarg);
584aa0a1e58SJeff Roberson 			break;
585aa0a1e58SJeff Roberson 		case 'p':
586aa0a1e58SJeff Roberson 			port_space = strtol(optarg, NULL, 0);
587aa0a1e58SJeff Roberson 			break;
588aa0a1e58SJeff Roberson 		default:
589aa0a1e58SJeff Roberson 			printf("usage: %s\n", argv[0]);
590aa0a1e58SJeff Roberson 			printf("\t-m multicast_address\n");
591aa0a1e58SJeff Roberson 			printf("\t[-M unmapped_multicast_address]\n"
592aa0a1e58SJeff Roberson 			       "\t replaces -m and requires -b\n");
593aa0a1e58SJeff Roberson 			printf("\t[-s(ender)]\n");
594aa0a1e58SJeff Roberson 			printf("\t[-b bind_address]\n");
595aa0a1e58SJeff Roberson 			printf("\t[-c connections]\n");
596aa0a1e58SJeff Roberson 			printf("\t[-C message_count]\n");
597aa0a1e58SJeff Roberson 			printf("\t[-S message_size]\n");
598aa0a1e58SJeff Roberson 			printf("\t[-p port_space - %#x for UDP (default), "
599aa0a1e58SJeff Roberson 			       "%#x for IPOIB]\n", RDMA_PS_UDP, RDMA_PS_IPOIB);
600aa0a1e58SJeff Roberson 			exit(1);
601aa0a1e58SJeff Roberson 		}
602aa0a1e58SJeff Roberson 	}
603aa0a1e58SJeff Roberson 
604aa0a1e58SJeff Roberson 	if (unmapped_addr && !src_addr) {
605aa0a1e58SJeff Roberson 		printf("unmapped multicast address requires binding "
606aa0a1e58SJeff Roberson 			"to source address\n");
607aa0a1e58SJeff Roberson 		exit(1);
608aa0a1e58SJeff Roberson 	}
609aa0a1e58SJeff Roberson 
610aa0a1e58SJeff Roberson 	test.dst_addr = (struct sockaddr *) &test.dst_in;
611aa0a1e58SJeff Roberson 	test.connects_left = connections;
612aa0a1e58SJeff Roberson 
613aa0a1e58SJeff Roberson 	test.channel = rdma_create_event_channel();
614aa0a1e58SJeff Roberson 	if (!test.channel) {
615aa0a1e58SJeff Roberson 		perror("failed to create event channel");
616aa0a1e58SJeff Roberson 		exit(1);
617aa0a1e58SJeff Roberson 	}
618aa0a1e58SJeff Roberson 
619aa0a1e58SJeff Roberson 	if (alloc_nodes())
620aa0a1e58SJeff Roberson 		exit(1);
621aa0a1e58SJeff Roberson 
622aa0a1e58SJeff Roberson 	ret = run();
623aa0a1e58SJeff Roberson 
624aa0a1e58SJeff Roberson 	printf("test complete\n");
625aa0a1e58SJeff Roberson 	destroy_nodes();
626aa0a1e58SJeff Roberson 	rdma_destroy_event_channel(test.channel);
627aa0a1e58SJeff Roberson 
628aa0a1e58SJeff Roberson 	printf("return status %d\n", ret);
629aa0a1e58SJeff Roberson 	return ret;
630aa0a1e58SJeff Roberson }
631