xref: /linux/drivers/infiniband/hw/irdma/cm.c (revision fd7d598270724cc787982ea48bbe17ad383a8b7f)
1 // SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
2 /* Copyright (c) 2015 - 2021 Intel Corporation */
3 #include "main.h"
4 #include "trace.h"
5 
6 static void irdma_cm_post_event(struct irdma_cm_event *event);
7 static void irdma_disconnect_worker(struct work_struct *work);
8 
9 /**
10  * irdma_free_sqbuf - put back puda buffer if refcount is 0
11  * @vsi: The VSI structure of the device
12  * @bufp: puda buffer to free
13  */
14 void irdma_free_sqbuf(struct irdma_sc_vsi *vsi, void *bufp)
15 {
16 	struct irdma_puda_buf *buf = bufp;
17 	struct irdma_puda_rsrc *ilq = vsi->ilq;
18 
19 	if (refcount_dec_and_test(&buf->refcount))
20 		irdma_puda_ret_bufpool(ilq, buf);
21 }
22 
23 /**
24  * irdma_record_ird_ord - Record IRD/ORD passed in
25  * @cm_node: connection's node
26  * @conn_ird: connection IRD
27  * @conn_ord: connection ORD
28  */
29 static void irdma_record_ird_ord(struct irdma_cm_node *cm_node, u32 conn_ird,
30 				 u32 conn_ord)
31 {
32 	if (conn_ird > cm_node->dev->hw_attrs.max_hw_ird)
33 		conn_ird = cm_node->dev->hw_attrs.max_hw_ird;
34 
35 	if (conn_ord > cm_node->dev->hw_attrs.max_hw_ord)
36 		conn_ord = cm_node->dev->hw_attrs.max_hw_ord;
37 	else if (!conn_ord && cm_node->send_rdma0_op == SEND_RDMA_READ_ZERO)
38 		conn_ord = 1;
39 	cm_node->ird_size = conn_ird;
40 	cm_node->ord_size = conn_ord;
41 }
42 
43 /**
44  * irdma_copy_ip_ntohl - copy IP address from  network to host
45  * @dst: IP address in host order
46  * @src: IP address in network order (big endian)
47  */
48 void irdma_copy_ip_ntohl(u32 *dst, __be32 *src)
49 {
50 	*dst++ = ntohl(*src++);
51 	*dst++ = ntohl(*src++);
52 	*dst++ = ntohl(*src++);
53 	*dst = ntohl(*src);
54 }
55 
56 /**
57  * irdma_copy_ip_htonl - copy IP address from host to network order
58  * @dst: IP address in network order (big endian)
59  * @src: IP address in host order
60  */
61 void irdma_copy_ip_htonl(__be32 *dst, u32 *src)
62 {
63 	*dst++ = htonl(*src++);
64 	*dst++ = htonl(*src++);
65 	*dst++ = htonl(*src++);
66 	*dst = htonl(*src);
67 }
68 
69 /**
70  * irdma_get_addr_info
71  * @cm_node: contains ip/tcp info
72  * @cm_info: to get a copy of the cm_node ip/tcp info
73  */
74 static void irdma_get_addr_info(struct irdma_cm_node *cm_node,
75 				struct irdma_cm_info *cm_info)
76 {
77 	memset(cm_info, 0, sizeof(*cm_info));
78 	cm_info->ipv4 = cm_node->ipv4;
79 	cm_info->vlan_id = cm_node->vlan_id;
80 	memcpy(cm_info->loc_addr, cm_node->loc_addr, sizeof(cm_info->loc_addr));
81 	memcpy(cm_info->rem_addr, cm_node->rem_addr, sizeof(cm_info->rem_addr));
82 	cm_info->loc_port = cm_node->loc_port;
83 	cm_info->rem_port = cm_node->rem_port;
84 }
85 
86 /**
87  * irdma_fill_sockaddr4 - fill in addr info for IPv4 connection
88  * @cm_node: connection's node
89  * @event: upper layer's cm event
90  */
91 static inline void irdma_fill_sockaddr4(struct irdma_cm_node *cm_node,
92 					struct iw_cm_event *event)
93 {
94 	struct sockaddr_in *laddr = (struct sockaddr_in *)&event->local_addr;
95 	struct sockaddr_in *raddr = (struct sockaddr_in *)&event->remote_addr;
96 
97 	laddr->sin_family = AF_INET;
98 	raddr->sin_family = AF_INET;
99 
100 	laddr->sin_port = htons(cm_node->loc_port);
101 	raddr->sin_port = htons(cm_node->rem_port);
102 
103 	laddr->sin_addr.s_addr = htonl(cm_node->loc_addr[0]);
104 	raddr->sin_addr.s_addr = htonl(cm_node->rem_addr[0]);
105 }
106 
107 /**
108  * irdma_fill_sockaddr6 - fill in addr info for IPv6 connection
109  * @cm_node: connection's node
110  * @event: upper layer's cm event
111  */
112 static inline void irdma_fill_sockaddr6(struct irdma_cm_node *cm_node,
113 					struct iw_cm_event *event)
114 {
115 	struct sockaddr_in6 *laddr6 = (struct sockaddr_in6 *)&event->local_addr;
116 	struct sockaddr_in6 *raddr6 = (struct sockaddr_in6 *)&event->remote_addr;
117 
118 	laddr6->sin6_family = AF_INET6;
119 	raddr6->sin6_family = AF_INET6;
120 
121 	laddr6->sin6_port = htons(cm_node->loc_port);
122 	raddr6->sin6_port = htons(cm_node->rem_port);
123 
124 	irdma_copy_ip_htonl(laddr6->sin6_addr.in6_u.u6_addr32,
125 			    cm_node->loc_addr);
126 	irdma_copy_ip_htonl(raddr6->sin6_addr.in6_u.u6_addr32,
127 			    cm_node->rem_addr);
128 }
129 
130 /**
131  * irdma_get_cmevent_info - for cm event upcall
132  * @cm_node: connection's node
133  * @cm_id: upper layers cm struct for the event
134  * @event: upper layer's cm event
135  */
136 static inline void irdma_get_cmevent_info(struct irdma_cm_node *cm_node,
137 					  struct iw_cm_id *cm_id,
138 					  struct iw_cm_event *event)
139 {
140 	memcpy(&event->local_addr, &cm_id->m_local_addr,
141 	       sizeof(event->local_addr));
142 	memcpy(&event->remote_addr, &cm_id->m_remote_addr,
143 	       sizeof(event->remote_addr));
144 	if (cm_node) {
145 		event->private_data = cm_node->pdata_buf;
146 		event->private_data_len = (u8)cm_node->pdata.size;
147 		event->ird = cm_node->ird_size;
148 		event->ord = cm_node->ord_size;
149 	}
150 }
151 
152 /**
153  * irdma_send_cm_event - upcall cm's event handler
154  * @cm_node: connection's node
155  * @cm_id: upper layer's cm info struct
156  * @type: Event type to indicate
157  * @status: status for the event type
158  */
159 static int irdma_send_cm_event(struct irdma_cm_node *cm_node,
160 			       struct iw_cm_id *cm_id,
161 			       enum iw_cm_event_type type, int status)
162 {
163 	struct iw_cm_event event = {};
164 
165 	event.event = type;
166 	event.status = status;
167 	trace_irdma_send_cm_event(cm_node, cm_id, type, status,
168 				  __builtin_return_address(0));
169 
170 	ibdev_dbg(&cm_node->iwdev->ibdev,
171 		  "CM: cm_node %p cm_id=%p state=%d accel=%d event_type=%d status=%d\n",
172 		  cm_node, cm_id, cm_node->accelerated, cm_node->state, type,
173 		  status);
174 
175 	switch (type) {
176 	case IW_CM_EVENT_CONNECT_REQUEST:
177 		if (cm_node->ipv4)
178 			irdma_fill_sockaddr4(cm_node, &event);
179 		else
180 			irdma_fill_sockaddr6(cm_node, &event);
181 		event.provider_data = cm_node;
182 		event.private_data = cm_node->pdata_buf;
183 		event.private_data_len = (u8)cm_node->pdata.size;
184 		event.ird = cm_node->ird_size;
185 		break;
186 	case IW_CM_EVENT_CONNECT_REPLY:
187 		irdma_get_cmevent_info(cm_node, cm_id, &event);
188 		break;
189 	case IW_CM_EVENT_ESTABLISHED:
190 		event.ird = cm_node->ird_size;
191 		event.ord = cm_node->ord_size;
192 		break;
193 	case IW_CM_EVENT_DISCONNECT:
194 	case IW_CM_EVENT_CLOSE:
195 		/* Wait if we are in RTS but havent issued the iwcm event upcall */
196 		if (!cm_node->accelerated)
197 			wait_for_completion(&cm_node->establish_comp);
198 		break;
199 	default:
200 		return -EINVAL;
201 	}
202 
203 	return cm_id->event_handler(cm_id, &event);
204 }
205 
206 /**
207  * irdma_timer_list_prep - add connection nodes to a list to perform timer tasks
208  * @cm_core: cm's core
209  * @timer_list: a timer list to which cm_node will be selected
210  */
211 static void irdma_timer_list_prep(struct irdma_cm_core *cm_core,
212 				  struct list_head *timer_list)
213 {
214 	struct irdma_cm_node *cm_node;
215 	int bkt;
216 
217 	hash_for_each_rcu(cm_core->cm_hash_tbl, bkt, cm_node, list) {
218 		if ((cm_node->close_entry || cm_node->send_entry) &&
219 		    refcount_inc_not_zero(&cm_node->refcnt))
220 			list_add(&cm_node->timer_entry, timer_list);
221 	}
222 }
223 
224 /**
225  * irdma_create_event - create cm event
226  * @cm_node: connection's node
227  * @type: Event type to generate
228  */
229 static struct irdma_cm_event *irdma_create_event(struct irdma_cm_node *cm_node,
230 						 enum irdma_cm_event_type type)
231 {
232 	struct irdma_cm_event *event;
233 
234 	if (!cm_node->cm_id)
235 		return NULL;
236 
237 	event = kzalloc(sizeof(*event), GFP_ATOMIC);
238 
239 	if (!event)
240 		return NULL;
241 
242 	event->type = type;
243 	event->cm_node = cm_node;
244 	memcpy(event->cm_info.rem_addr, cm_node->rem_addr,
245 	       sizeof(event->cm_info.rem_addr));
246 	memcpy(event->cm_info.loc_addr, cm_node->loc_addr,
247 	       sizeof(event->cm_info.loc_addr));
248 	event->cm_info.rem_port = cm_node->rem_port;
249 	event->cm_info.loc_port = cm_node->loc_port;
250 	event->cm_info.cm_id = cm_node->cm_id;
251 	ibdev_dbg(&cm_node->iwdev->ibdev,
252 		  "CM: node=%p event=%p type=%u dst=%pI4 src=%pI4\n", cm_node,
253 		  event, type, event->cm_info.loc_addr,
254 		  event->cm_info.rem_addr);
255 	trace_irdma_create_event(cm_node, type, __builtin_return_address(0));
256 	irdma_cm_post_event(event);
257 
258 	return event;
259 }
260 
261 /**
262  * irdma_free_retrans_entry - free send entry
263  * @cm_node: connection's node
264  */
265 static void irdma_free_retrans_entry(struct irdma_cm_node *cm_node)
266 {
267 	struct irdma_device *iwdev = cm_node->iwdev;
268 	struct irdma_timer_entry *send_entry;
269 
270 	send_entry = cm_node->send_entry;
271 	if (!send_entry)
272 		return;
273 
274 	cm_node->send_entry = NULL;
275 	irdma_free_sqbuf(&iwdev->vsi, send_entry->sqbuf);
276 	kfree(send_entry);
277 	refcount_dec(&cm_node->refcnt);
278 }
279 
280 /**
281  * irdma_cleanup_retrans_entry - free send entry with lock
282  * @cm_node: connection's node
283  */
284 static void irdma_cleanup_retrans_entry(struct irdma_cm_node *cm_node)
285 {
286 	unsigned long flags;
287 
288 	spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
289 	irdma_free_retrans_entry(cm_node);
290 	spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags);
291 }
292 
293 /**
294  * irdma_form_ah_cm_frame - get a free packet and build frame with address handle
295  * @cm_node: connection's node ionfo to use in frame
296  * @options: pointer to options info
297  * @hdr: pointer mpa header
298  * @pdata: pointer to private data
299  * @flags:  indicates FIN or ACK
300  */
301 static struct irdma_puda_buf *irdma_form_ah_cm_frame(struct irdma_cm_node *cm_node,
302 						     struct irdma_kmem_info *options,
303 						     struct irdma_kmem_info *hdr,
304 						     struct irdma_mpa_priv_info *pdata,
305 						     u8 flags)
306 {
307 	struct irdma_puda_buf *sqbuf;
308 	struct irdma_sc_vsi *vsi = &cm_node->iwdev->vsi;
309 	u8 *buf;
310 	struct tcphdr *tcph;
311 	u16 pktsize;
312 	u32 opts_len = 0;
313 	u32 pd_len = 0;
314 	u32 hdr_len = 0;
315 
316 	if (!cm_node->ah || !cm_node->ah->ah_info.ah_valid) {
317 		ibdev_dbg(&cm_node->iwdev->ibdev, "CM: AH invalid\n");
318 		return NULL;
319 	}
320 
321 	sqbuf = irdma_puda_get_bufpool(vsi->ilq);
322 	if (!sqbuf) {
323 		ibdev_dbg(&cm_node->iwdev->ibdev, "CM: SQ buf NULL\n");
324 		return NULL;
325 	}
326 
327 	sqbuf->ah_id = cm_node->ah->ah_info.ah_idx;
328 	buf = sqbuf->mem.va;
329 	if (options)
330 		opts_len = (u32)options->size;
331 
332 	if (hdr)
333 		hdr_len = hdr->size;
334 
335 	if (pdata)
336 		pd_len = pdata->size;
337 
338 	pktsize = sizeof(*tcph) + opts_len + hdr_len + pd_len;
339 
340 	memset(buf, 0, sizeof(*tcph));
341 
342 	sqbuf->totallen = pktsize;
343 	sqbuf->tcphlen = sizeof(*tcph) + opts_len;
344 	sqbuf->scratch = cm_node;
345 
346 	tcph = (struct tcphdr *)buf;
347 	buf += sizeof(*tcph);
348 
349 	tcph->source = htons(cm_node->loc_port);
350 	tcph->dest = htons(cm_node->rem_port);
351 	tcph->seq = htonl(cm_node->tcp_cntxt.loc_seq_num);
352 
353 	if (flags & SET_ACK) {
354 		cm_node->tcp_cntxt.loc_ack_num = cm_node->tcp_cntxt.rcv_nxt;
355 		tcph->ack_seq = htonl(cm_node->tcp_cntxt.loc_ack_num);
356 		tcph->ack = 1;
357 	} else {
358 		tcph->ack_seq = 0;
359 	}
360 
361 	if (flags & SET_SYN) {
362 		cm_node->tcp_cntxt.loc_seq_num++;
363 		tcph->syn = 1;
364 	} else {
365 		cm_node->tcp_cntxt.loc_seq_num += hdr_len + pd_len;
366 	}
367 
368 	if (flags & SET_FIN) {
369 		cm_node->tcp_cntxt.loc_seq_num++;
370 		tcph->fin = 1;
371 	}
372 
373 	if (flags & SET_RST)
374 		tcph->rst = 1;
375 
376 	tcph->doff = (u16)((sizeof(*tcph) + opts_len + 3) >> 2);
377 	sqbuf->tcphlen = tcph->doff << 2;
378 	tcph->window = htons(cm_node->tcp_cntxt.rcv_wnd);
379 	tcph->urg_ptr = 0;
380 
381 	if (opts_len) {
382 		memcpy(buf, options->addr, opts_len);
383 		buf += opts_len;
384 	}
385 
386 	if (hdr_len) {
387 		memcpy(buf, hdr->addr, hdr_len);
388 		buf += hdr_len;
389 	}
390 
391 	if (pdata && pdata->addr)
392 		memcpy(buf, pdata->addr, pdata->size);
393 
394 	refcount_set(&sqbuf->refcount, 1);
395 
396 	print_hex_dump_debug("ILQ: TRANSMIT ILQ BUFFER", DUMP_PREFIX_OFFSET,
397 			     16, 8, sqbuf->mem.va, sqbuf->totallen, false);
398 
399 	return sqbuf;
400 }
401 
402 /**
403  * irdma_form_uda_cm_frame - get a free packet and build frame full tcpip packet
404  * @cm_node: connection's node ionfo to use in frame
405  * @options: pointer to options info
406  * @hdr: pointer mpa header
407  * @pdata: pointer to private data
408  * @flags:  indicates FIN or ACK
409  */
410 static struct irdma_puda_buf *irdma_form_uda_cm_frame(struct irdma_cm_node *cm_node,
411 						      struct irdma_kmem_info *options,
412 						      struct irdma_kmem_info *hdr,
413 						      struct irdma_mpa_priv_info *pdata,
414 						      u8 flags)
415 {
416 	struct irdma_puda_buf *sqbuf;
417 	struct irdma_sc_vsi *vsi = &cm_node->iwdev->vsi;
418 	u8 *buf;
419 
420 	struct tcphdr *tcph;
421 	struct iphdr *iph;
422 	struct ipv6hdr *ip6h;
423 	struct ethhdr *ethh;
424 	u16 pktsize;
425 	u16 eth_hlen = ETH_HLEN;
426 	u32 opts_len = 0;
427 	u32 pd_len = 0;
428 	u32 hdr_len = 0;
429 
430 	u16 vtag;
431 
432 	sqbuf = irdma_puda_get_bufpool(vsi->ilq);
433 	if (!sqbuf)
434 		return NULL;
435 
436 	buf = sqbuf->mem.va;
437 
438 	if (options)
439 		opts_len = (u32)options->size;
440 
441 	if (hdr)
442 		hdr_len = hdr->size;
443 
444 	if (pdata)
445 		pd_len = pdata->size;
446 
447 	if (cm_node->vlan_id < VLAN_N_VID)
448 		eth_hlen += 4;
449 
450 	if (cm_node->ipv4)
451 		pktsize = sizeof(*iph) + sizeof(*tcph);
452 	else
453 		pktsize = sizeof(*ip6h) + sizeof(*tcph);
454 	pktsize += opts_len + hdr_len + pd_len;
455 
456 	memset(buf, 0, eth_hlen + pktsize);
457 
458 	sqbuf->totallen = pktsize + eth_hlen;
459 	sqbuf->maclen = eth_hlen;
460 	sqbuf->tcphlen = sizeof(*tcph) + opts_len;
461 	sqbuf->scratch = cm_node;
462 
463 	ethh = (struct ethhdr *)buf;
464 	buf += eth_hlen;
465 
466 	if (cm_node->do_lpb)
467 		sqbuf->do_lpb = true;
468 
469 	if (cm_node->ipv4) {
470 		sqbuf->ipv4 = true;
471 
472 		iph = (struct iphdr *)buf;
473 		buf += sizeof(*iph);
474 		tcph = (struct tcphdr *)buf;
475 		buf += sizeof(*tcph);
476 
477 		ether_addr_copy(ethh->h_dest, cm_node->rem_mac);
478 		ether_addr_copy(ethh->h_source, cm_node->loc_mac);
479 		if (cm_node->vlan_id < VLAN_N_VID) {
480 			((struct vlan_ethhdr *)ethh)->h_vlan_proto =
481 				htons(ETH_P_8021Q);
482 			vtag = (cm_node->user_pri << VLAN_PRIO_SHIFT) |
483 			       cm_node->vlan_id;
484 			((struct vlan_ethhdr *)ethh)->h_vlan_TCI = htons(vtag);
485 
486 			((struct vlan_ethhdr *)ethh)->h_vlan_encapsulated_proto =
487 				htons(ETH_P_IP);
488 		} else {
489 			ethh->h_proto = htons(ETH_P_IP);
490 		}
491 
492 		iph->version = IPVERSION;
493 		iph->ihl = 5; /* 5 * 4Byte words, IP headr len */
494 		iph->tos = cm_node->tos;
495 		iph->tot_len = htons(pktsize);
496 		iph->id = htons(++cm_node->tcp_cntxt.loc_id);
497 
498 		iph->frag_off = htons(0x4000);
499 		iph->ttl = 0x40;
500 		iph->protocol = IPPROTO_TCP;
501 		iph->saddr = htonl(cm_node->loc_addr[0]);
502 		iph->daddr = htonl(cm_node->rem_addr[0]);
503 	} else {
504 		sqbuf->ipv4 = false;
505 		ip6h = (struct ipv6hdr *)buf;
506 		buf += sizeof(*ip6h);
507 		tcph = (struct tcphdr *)buf;
508 		buf += sizeof(*tcph);
509 
510 		ether_addr_copy(ethh->h_dest, cm_node->rem_mac);
511 		ether_addr_copy(ethh->h_source, cm_node->loc_mac);
512 		if (cm_node->vlan_id < VLAN_N_VID) {
513 			((struct vlan_ethhdr *)ethh)->h_vlan_proto =
514 				htons(ETH_P_8021Q);
515 			vtag = (cm_node->user_pri << VLAN_PRIO_SHIFT) |
516 			       cm_node->vlan_id;
517 			((struct vlan_ethhdr *)ethh)->h_vlan_TCI = htons(vtag);
518 			((struct vlan_ethhdr *)ethh)->h_vlan_encapsulated_proto =
519 				htons(ETH_P_IPV6);
520 		} else {
521 			ethh->h_proto = htons(ETH_P_IPV6);
522 		}
523 		ip6h->version = 6;
524 		ip6h->priority = cm_node->tos >> 4;
525 		ip6h->flow_lbl[0] = cm_node->tos << 4;
526 		ip6h->flow_lbl[1] = 0;
527 		ip6h->flow_lbl[2] = 0;
528 		ip6h->payload_len = htons(pktsize - sizeof(*ip6h));
529 		ip6h->nexthdr = 6;
530 		ip6h->hop_limit = 128;
531 		irdma_copy_ip_htonl(ip6h->saddr.in6_u.u6_addr32,
532 				    cm_node->loc_addr);
533 		irdma_copy_ip_htonl(ip6h->daddr.in6_u.u6_addr32,
534 				    cm_node->rem_addr);
535 	}
536 
537 	tcph->source = htons(cm_node->loc_port);
538 	tcph->dest = htons(cm_node->rem_port);
539 	tcph->seq = htonl(cm_node->tcp_cntxt.loc_seq_num);
540 
541 	if (flags & SET_ACK) {
542 		cm_node->tcp_cntxt.loc_ack_num = cm_node->tcp_cntxt.rcv_nxt;
543 		tcph->ack_seq = htonl(cm_node->tcp_cntxt.loc_ack_num);
544 		tcph->ack = 1;
545 	} else {
546 		tcph->ack_seq = 0;
547 	}
548 
549 	if (flags & SET_SYN) {
550 		cm_node->tcp_cntxt.loc_seq_num++;
551 		tcph->syn = 1;
552 	} else {
553 		cm_node->tcp_cntxt.loc_seq_num += hdr_len + pd_len;
554 	}
555 
556 	if (flags & SET_FIN) {
557 		cm_node->tcp_cntxt.loc_seq_num++;
558 		tcph->fin = 1;
559 	}
560 
561 	if (flags & SET_RST)
562 		tcph->rst = 1;
563 
564 	tcph->doff = (u16)((sizeof(*tcph) + opts_len + 3) >> 2);
565 	sqbuf->tcphlen = tcph->doff << 2;
566 	tcph->window = htons(cm_node->tcp_cntxt.rcv_wnd);
567 	tcph->urg_ptr = 0;
568 
569 	if (opts_len) {
570 		memcpy(buf, options->addr, opts_len);
571 		buf += opts_len;
572 	}
573 
574 	if (hdr_len) {
575 		memcpy(buf, hdr->addr, hdr_len);
576 		buf += hdr_len;
577 	}
578 
579 	if (pdata && pdata->addr)
580 		memcpy(buf, pdata->addr, pdata->size);
581 
582 	refcount_set(&sqbuf->refcount, 1);
583 
584 	print_hex_dump_debug("ILQ: TRANSMIT ILQ BUFFER", DUMP_PREFIX_OFFSET,
585 			     16, 8, sqbuf->mem.va, sqbuf->totallen, false);
586 	return sqbuf;
587 }
588 
589 /**
590  * irdma_send_reset - Send RST packet
591  * @cm_node: connection's node
592  */
593 int irdma_send_reset(struct irdma_cm_node *cm_node)
594 {
595 	struct irdma_puda_buf *sqbuf;
596 	int flags = SET_RST | SET_ACK;
597 
598 	trace_irdma_send_reset(cm_node, 0, __builtin_return_address(0));
599 	sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL, NULL, NULL,
600 						flags);
601 	if (!sqbuf)
602 		return -ENOMEM;
603 
604 	ibdev_dbg(&cm_node->iwdev->ibdev,
605 		  "CM: caller: %pS cm_node %p cm_id=%p accel=%d state=%d rem_port=0x%04x, loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4\n",
606 		  __builtin_return_address(0), cm_node, cm_node->cm_id,
607 		  cm_node->accelerated, cm_node->state, cm_node->rem_port,
608 		  cm_node->loc_port, cm_node->rem_addr, cm_node->loc_addr);
609 
610 	return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 0,
611 				       1);
612 }
613 
614 /**
615  * irdma_active_open_err - send event for active side cm error
616  * @cm_node: connection's node
617  * @reset: Flag to send reset or not
618  */
619 static void irdma_active_open_err(struct irdma_cm_node *cm_node, bool reset)
620 {
621 	trace_irdma_active_open_err(cm_node, reset,
622 				    __builtin_return_address(0));
623 	irdma_cleanup_retrans_entry(cm_node);
624 	cm_node->cm_core->stats_connect_errs++;
625 	if (reset) {
626 		ibdev_dbg(&cm_node->iwdev->ibdev,
627 			  "CM: cm_node=%p state=%d\n", cm_node,
628 			  cm_node->state);
629 		refcount_inc(&cm_node->refcnt);
630 		irdma_send_reset(cm_node);
631 	}
632 
633 	cm_node->state = IRDMA_CM_STATE_CLOSED;
634 	irdma_create_event(cm_node, IRDMA_CM_EVENT_ABORTED);
635 }
636 
637 /**
638  * irdma_passive_open_err - handle passive side cm error
639  * @cm_node: connection's node
640  * @reset: send reset or just free cm_node
641  */
642 static void irdma_passive_open_err(struct irdma_cm_node *cm_node, bool reset)
643 {
644 	irdma_cleanup_retrans_entry(cm_node);
645 	cm_node->cm_core->stats_passive_errs++;
646 	cm_node->state = IRDMA_CM_STATE_CLOSED;
647 	ibdev_dbg(&cm_node->iwdev->ibdev, "CM: cm_node=%p state =%d\n",
648 		  cm_node, cm_node->state);
649 	trace_irdma_passive_open_err(cm_node, reset,
650 				     __builtin_return_address(0));
651 	if (reset)
652 		irdma_send_reset(cm_node);
653 	else
654 		irdma_rem_ref_cm_node(cm_node);
655 }
656 
657 /**
658  * irdma_event_connect_error - to create connect error event
659  * @event: cm information for connect event
660  */
661 static void irdma_event_connect_error(struct irdma_cm_event *event)
662 {
663 	struct irdma_qp *iwqp;
664 	struct iw_cm_id *cm_id;
665 
666 	cm_id = event->cm_node->cm_id;
667 	if (!cm_id)
668 		return;
669 
670 	iwqp = cm_id->provider_data;
671 
672 	if (!iwqp || !iwqp->iwdev)
673 		return;
674 
675 	iwqp->cm_id = NULL;
676 	cm_id->provider_data = NULL;
677 	irdma_send_cm_event(event->cm_node, cm_id, IW_CM_EVENT_CONNECT_REPLY,
678 			    -ECONNRESET);
679 	irdma_rem_ref_cm_node(event->cm_node);
680 }
681 
682 /**
683  * irdma_process_options - process options from TCP header
684  * @cm_node: connection's node
685  * @optionsloc: point to start of options
686  * @optionsize: size of all options
687  * @syn_pkt: flag if syn packet
688  */
689 static int irdma_process_options(struct irdma_cm_node *cm_node, u8 *optionsloc,
690 				 u32 optionsize, u32 syn_pkt)
691 {
692 	u32 tmp;
693 	u32 offset = 0;
694 	union all_known_options *all_options;
695 	char got_mss_option = 0;
696 
697 	while (offset < optionsize) {
698 		all_options = (union all_known_options *)(optionsloc + offset);
699 		switch (all_options->base.optionnum) {
700 		case OPTION_NUM_EOL:
701 			offset = optionsize;
702 			break;
703 		case OPTION_NUM_NONE:
704 			offset += 1;
705 			continue;
706 		case OPTION_NUM_MSS:
707 			ibdev_dbg(&cm_node->iwdev->ibdev,
708 				  "CM: MSS Length: %d Offset: %d Size: %d\n",
709 				  all_options->mss.len, offset, optionsize);
710 			got_mss_option = 1;
711 			if (all_options->mss.len != 4)
712 				return -EINVAL;
713 			tmp = ntohs(all_options->mss.mss);
714 			if ((cm_node->ipv4 &&
715 			     (tmp + IRDMA_MTU_TO_MSS_IPV4) < IRDMA_MIN_MTU_IPV4) ||
716 			    (!cm_node->ipv4 &&
717 			     (tmp + IRDMA_MTU_TO_MSS_IPV6) < IRDMA_MIN_MTU_IPV6))
718 				return -EINVAL;
719 			if (tmp < cm_node->tcp_cntxt.mss)
720 				cm_node->tcp_cntxt.mss = tmp;
721 			break;
722 		case OPTION_NUM_WINDOW_SCALE:
723 			cm_node->tcp_cntxt.snd_wscale =
724 				all_options->windowscale.shiftcount;
725 			break;
726 		default:
727 			ibdev_dbg(&cm_node->iwdev->ibdev,
728 				  "CM: Unsupported TCP Option: %x\n",
729 				  all_options->base.optionnum);
730 			break;
731 		}
732 		offset += all_options->base.len;
733 	}
734 	if (!got_mss_option && syn_pkt)
735 		cm_node->tcp_cntxt.mss = IRDMA_CM_DEFAULT_MSS;
736 
737 	return 0;
738 }
739 
740 /**
741  * irdma_handle_tcp_options - setup TCP context info after parsing TCP options
742  * @cm_node: connection's node
743  * @tcph: pointer tcp header
744  * @optionsize: size of options rcvd
745  * @passive: active or passive flag
746  */
747 static int irdma_handle_tcp_options(struct irdma_cm_node *cm_node,
748 				    struct tcphdr *tcph, int optionsize,
749 				    int passive)
750 {
751 	u8 *optionsloc = (u8 *)&tcph[1];
752 	int ret;
753 
754 	if (optionsize) {
755 		ret = irdma_process_options(cm_node, optionsloc, optionsize,
756 					    (u32)tcph->syn);
757 		if (ret) {
758 			ibdev_dbg(&cm_node->iwdev->ibdev,
759 				  "CM: Node %p, Sending Reset\n", cm_node);
760 			if (passive)
761 				irdma_passive_open_err(cm_node, true);
762 			else
763 				irdma_active_open_err(cm_node, true);
764 			return ret;
765 		}
766 	}
767 
768 	cm_node->tcp_cntxt.snd_wnd = ntohs(tcph->window)
769 				     << cm_node->tcp_cntxt.snd_wscale;
770 
771 	if (cm_node->tcp_cntxt.snd_wnd > cm_node->tcp_cntxt.max_snd_wnd)
772 		cm_node->tcp_cntxt.max_snd_wnd = cm_node->tcp_cntxt.snd_wnd;
773 
774 	return 0;
775 }
776 
777 /**
778  * irdma_build_mpa_v1 - build a MPA V1 frame
779  * @cm_node: connection's node
780  * @start_addr: address where to build frame
781  * @mpa_key: to do read0 or write0
782  */
783 static void irdma_build_mpa_v1(struct irdma_cm_node *cm_node, void *start_addr,
784 			       u8 mpa_key)
785 {
786 	struct ietf_mpa_v1 *mpa_frame = start_addr;
787 
788 	switch (mpa_key) {
789 	case MPA_KEY_REQUEST:
790 		memcpy(mpa_frame->key, IEFT_MPA_KEY_REQ, IETF_MPA_KEY_SIZE);
791 		break;
792 	case MPA_KEY_REPLY:
793 		memcpy(mpa_frame->key, IEFT_MPA_KEY_REP, IETF_MPA_KEY_SIZE);
794 		break;
795 	default:
796 		break;
797 	}
798 	mpa_frame->flags = IETF_MPA_FLAGS_CRC;
799 	mpa_frame->rev = cm_node->mpa_frame_rev;
800 	mpa_frame->priv_data_len = htons(cm_node->pdata.size);
801 }
802 
803 /**
804  * irdma_build_mpa_v2 - build a MPA V2 frame
805  * @cm_node: connection's node
806  * @start_addr: buffer start address
807  * @mpa_key: to do read0 or write0
808  */
809 static void irdma_build_mpa_v2(struct irdma_cm_node *cm_node, void *start_addr,
810 			       u8 mpa_key)
811 {
812 	struct ietf_mpa_v2 *mpa_frame = start_addr;
813 	struct ietf_rtr_msg *rtr_msg = &mpa_frame->rtr_msg;
814 	u16 ctrl_ird, ctrl_ord;
815 
816 	/* initialize the upper 5 bytes of the frame */
817 	irdma_build_mpa_v1(cm_node, start_addr, mpa_key);
818 	mpa_frame->flags |= IETF_MPA_V2_FLAG;
819 	if (cm_node->iwdev->iw_ooo) {
820 		mpa_frame->flags |= IETF_MPA_FLAGS_MARKERS;
821 		cm_node->rcv_mark_en = true;
822 	}
823 	mpa_frame->priv_data_len = cpu_to_be16(be16_to_cpu(mpa_frame->priv_data_len) +
824 					       IETF_RTR_MSG_SIZE);
825 
826 	/* initialize RTR msg */
827 	if (cm_node->mpav2_ird_ord == IETF_NO_IRD_ORD) {
828 		ctrl_ird = IETF_NO_IRD_ORD;
829 		ctrl_ord = IETF_NO_IRD_ORD;
830 	} else {
831 		ctrl_ird = (cm_node->ird_size > IETF_NO_IRD_ORD) ?
832 				   IETF_NO_IRD_ORD :
833 				   cm_node->ird_size;
834 		ctrl_ord = (cm_node->ord_size > IETF_NO_IRD_ORD) ?
835 				   IETF_NO_IRD_ORD :
836 				   cm_node->ord_size;
837 	}
838 	ctrl_ird |= IETF_PEER_TO_PEER;
839 
840 	switch (mpa_key) {
841 	case MPA_KEY_REQUEST:
842 		ctrl_ord |= IETF_RDMA0_WRITE;
843 		ctrl_ord |= IETF_RDMA0_READ;
844 		break;
845 	case MPA_KEY_REPLY:
846 		switch (cm_node->send_rdma0_op) {
847 		case SEND_RDMA_WRITE_ZERO:
848 			ctrl_ord |= IETF_RDMA0_WRITE;
849 			break;
850 		case SEND_RDMA_READ_ZERO:
851 			ctrl_ord |= IETF_RDMA0_READ;
852 			break;
853 		}
854 		break;
855 	default:
856 		break;
857 	}
858 	rtr_msg->ctrl_ird = htons(ctrl_ird);
859 	rtr_msg->ctrl_ord = htons(ctrl_ord);
860 }
861 
862 /**
863  * irdma_cm_build_mpa_frame - build mpa frame for mpa version 1 or version 2
864  * @cm_node: connection's node
865  * @mpa: mpa: data buffer
866  * @mpa_key: to do read0 or write0
867  */
868 static int irdma_cm_build_mpa_frame(struct irdma_cm_node *cm_node,
869 				    struct irdma_kmem_info *mpa, u8 mpa_key)
870 {
871 	int hdr_len = 0;
872 
873 	switch (cm_node->mpa_frame_rev) {
874 	case IETF_MPA_V1:
875 		hdr_len = sizeof(struct ietf_mpa_v1);
876 		irdma_build_mpa_v1(cm_node, mpa->addr, mpa_key);
877 		break;
878 	case IETF_MPA_V2:
879 		hdr_len = sizeof(struct ietf_mpa_v2);
880 		irdma_build_mpa_v2(cm_node, mpa->addr, mpa_key);
881 		break;
882 	default:
883 		break;
884 	}
885 
886 	return hdr_len;
887 }
888 
889 /**
890  * irdma_send_mpa_request - active node send mpa request to passive node
891  * @cm_node: connection's node
892  */
893 static int irdma_send_mpa_request(struct irdma_cm_node *cm_node)
894 {
895 	struct irdma_puda_buf *sqbuf;
896 
897 	cm_node->mpa_hdr.addr = &cm_node->mpa_v2_frame;
898 	cm_node->mpa_hdr.size = irdma_cm_build_mpa_frame(cm_node,
899 							 &cm_node->mpa_hdr,
900 							 MPA_KEY_REQUEST);
901 	if (!cm_node->mpa_hdr.size) {
902 		ibdev_dbg(&cm_node->iwdev->ibdev,
903 			  "CM: mpa size = %d\n", cm_node->mpa_hdr.size);
904 		return -EINVAL;
905 	}
906 
907 	sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL,
908 						&cm_node->mpa_hdr,
909 						&cm_node->pdata, SET_ACK);
910 	if (!sqbuf)
911 		return -ENOMEM;
912 
913 	return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 1,
914 				       0);
915 }
916 
917 /**
918  * irdma_send_mpa_reject -
919  * @cm_node: connection's node
920  * @pdata: reject data for connection
921  * @plen: length of reject data
922  */
923 static int irdma_send_mpa_reject(struct irdma_cm_node *cm_node,
924 				 const void *pdata, u8 plen)
925 {
926 	struct irdma_puda_buf *sqbuf;
927 	struct irdma_mpa_priv_info priv_info;
928 
929 	cm_node->mpa_hdr.addr = &cm_node->mpa_v2_frame;
930 	cm_node->mpa_hdr.size = irdma_cm_build_mpa_frame(cm_node,
931 							 &cm_node->mpa_hdr,
932 							 MPA_KEY_REPLY);
933 
934 	cm_node->mpa_frame.flags |= IETF_MPA_FLAGS_REJECT;
935 	priv_info.addr = pdata;
936 	priv_info.size = plen;
937 
938 	sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL,
939 						&cm_node->mpa_hdr, &priv_info,
940 						SET_ACK | SET_FIN);
941 	if (!sqbuf)
942 		return -ENOMEM;
943 
944 	cm_node->state = IRDMA_CM_STATE_FIN_WAIT1;
945 
946 	return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 1,
947 				       0);
948 }
949 
950 /**
951  * irdma_negotiate_mpa_v2_ird_ord - negotiate MPAv2 IRD/ORD
952  * @cm_node: connection's node
953  * @buf: Data pointer
954  */
955 static int irdma_negotiate_mpa_v2_ird_ord(struct irdma_cm_node *cm_node,
956 					  u8 *buf)
957 {
958 	struct ietf_mpa_v2 *mpa_v2_frame;
959 	struct ietf_rtr_msg *rtr_msg;
960 	u16 ird_size;
961 	u16 ord_size;
962 	u16 ctrl_ord;
963 	u16 ctrl_ird;
964 
965 	mpa_v2_frame = (struct ietf_mpa_v2 *)buf;
966 	rtr_msg = &mpa_v2_frame->rtr_msg;
967 
968 	/* parse rtr message */
969 	ctrl_ord = ntohs(rtr_msg->ctrl_ord);
970 	ctrl_ird = ntohs(rtr_msg->ctrl_ird);
971 	ird_size = ctrl_ird & IETF_NO_IRD_ORD;
972 	ord_size = ctrl_ord & IETF_NO_IRD_ORD;
973 
974 	if (!(ctrl_ird & IETF_PEER_TO_PEER))
975 		return -EOPNOTSUPP;
976 
977 	if (ird_size == IETF_NO_IRD_ORD || ord_size == IETF_NO_IRD_ORD) {
978 		cm_node->mpav2_ird_ord = IETF_NO_IRD_ORD;
979 		goto negotiate_done;
980 	}
981 
982 	if (cm_node->state != IRDMA_CM_STATE_MPAREQ_SENT) {
983 		/* responder */
984 		if (!ord_size && (ctrl_ord & IETF_RDMA0_READ))
985 			cm_node->ird_size = 1;
986 		if (cm_node->ord_size > ird_size)
987 			cm_node->ord_size = ird_size;
988 	} else {
989 		/* initiator */
990 		if (!ird_size && (ctrl_ord & IETF_RDMA0_READ))
991 			/* Remote peer doesn't support RDMA0_READ */
992 			return -EOPNOTSUPP;
993 
994 		if (cm_node->ord_size > ird_size)
995 			cm_node->ord_size = ird_size;
996 
997 		if (cm_node->ird_size < ord_size)
998 		/* no resources available */
999 			return -EINVAL;
1000 	}
1001 
1002 negotiate_done:
1003 	if (ctrl_ord & IETF_RDMA0_READ)
1004 		cm_node->send_rdma0_op = SEND_RDMA_READ_ZERO;
1005 	else if (ctrl_ord & IETF_RDMA0_WRITE)
1006 		cm_node->send_rdma0_op = SEND_RDMA_WRITE_ZERO;
1007 	else
1008 		/* Not supported RDMA0 operation */
1009 		return -EOPNOTSUPP;
1010 
1011 	ibdev_dbg(&cm_node->iwdev->ibdev,
1012 		  "CM: MPAV2 Negotiated ORD: %d, IRD: %d\n",
1013 		  cm_node->ord_size, cm_node->ird_size);
1014 	trace_irdma_negotiate_mpa_v2(cm_node);
1015 	return 0;
1016 }
1017 
1018 /**
1019  * irdma_parse_mpa - process an IETF MPA frame
1020  * @cm_node: connection's node
1021  * @buf: Data pointer
1022  * @type: to return accept or reject
1023  * @len: Len of mpa buffer
1024  */
1025 static int irdma_parse_mpa(struct irdma_cm_node *cm_node, u8 *buf, u32 *type,
1026 			   u32 len)
1027 {
1028 	struct ietf_mpa_v1 *mpa_frame;
1029 	int mpa_hdr_len, priv_data_len, ret;
1030 
1031 	*type = IRDMA_MPA_REQUEST_ACCEPT;
1032 
1033 	if (len < sizeof(struct ietf_mpa_v1)) {
1034 		ibdev_dbg(&cm_node->iwdev->ibdev,
1035 			  "CM: ietf buffer small (%x)\n", len);
1036 		return -EINVAL;
1037 	}
1038 
1039 	mpa_frame = (struct ietf_mpa_v1 *)buf;
1040 	mpa_hdr_len = sizeof(struct ietf_mpa_v1);
1041 	priv_data_len = ntohs(mpa_frame->priv_data_len);
1042 
1043 	if (priv_data_len > IETF_MAX_PRIV_DATA_LEN) {
1044 		ibdev_dbg(&cm_node->iwdev->ibdev,
1045 			  "CM: private_data too big %d\n", priv_data_len);
1046 		return -EOVERFLOW;
1047 	}
1048 
1049 	if (mpa_frame->rev != IETF_MPA_V1 && mpa_frame->rev != IETF_MPA_V2) {
1050 		ibdev_dbg(&cm_node->iwdev->ibdev,
1051 			  "CM: unsupported mpa rev = %d\n", mpa_frame->rev);
1052 		return -EINVAL;
1053 	}
1054 
1055 	if (mpa_frame->rev > cm_node->mpa_frame_rev) {
1056 		ibdev_dbg(&cm_node->iwdev->ibdev, "CM: rev %d\n",
1057 			  mpa_frame->rev);
1058 		return -EINVAL;
1059 	}
1060 
1061 	cm_node->mpa_frame_rev = mpa_frame->rev;
1062 	if (cm_node->state != IRDMA_CM_STATE_MPAREQ_SENT) {
1063 		if (memcmp(mpa_frame->key, IEFT_MPA_KEY_REQ,
1064 			   IETF_MPA_KEY_SIZE)) {
1065 			ibdev_dbg(&cm_node->iwdev->ibdev,
1066 				  "CM: Unexpected MPA Key received\n");
1067 			return -EINVAL;
1068 		}
1069 	} else {
1070 		if (memcmp(mpa_frame->key, IEFT_MPA_KEY_REP,
1071 			   IETF_MPA_KEY_SIZE)) {
1072 			ibdev_dbg(&cm_node->iwdev->ibdev,
1073 				  "CM: Unexpected MPA Key received\n");
1074 			return -EINVAL;
1075 		}
1076 	}
1077 
1078 	if (priv_data_len + mpa_hdr_len > len) {
1079 		ibdev_dbg(&cm_node->iwdev->ibdev,
1080 			  "CM: ietf buffer len(%x + %x != %x)\n",
1081 			  priv_data_len, mpa_hdr_len, len);
1082 		return -EOVERFLOW;
1083 	}
1084 
1085 	if (len > IRDMA_MAX_CM_BUF) {
1086 		ibdev_dbg(&cm_node->iwdev->ibdev,
1087 			  "CM: ietf buffer large len = %d\n", len);
1088 		return -EOVERFLOW;
1089 	}
1090 
1091 	switch (mpa_frame->rev) {
1092 	case IETF_MPA_V2:
1093 		mpa_hdr_len += IETF_RTR_MSG_SIZE;
1094 		ret = irdma_negotiate_mpa_v2_ird_ord(cm_node, buf);
1095 		if (ret)
1096 			return ret;
1097 		break;
1098 	case IETF_MPA_V1:
1099 	default:
1100 		break;
1101 	}
1102 
1103 	memcpy(cm_node->pdata_buf, buf + mpa_hdr_len, priv_data_len);
1104 	cm_node->pdata.size = priv_data_len;
1105 
1106 	if (mpa_frame->flags & IETF_MPA_FLAGS_REJECT)
1107 		*type = IRDMA_MPA_REQUEST_REJECT;
1108 
1109 	if (mpa_frame->flags & IETF_MPA_FLAGS_MARKERS)
1110 		cm_node->snd_mark_en = true;
1111 
1112 	return 0;
1113 }
1114 
1115 /**
1116  * irdma_schedule_cm_timer
1117  * @cm_node: connection's node
1118  * @sqbuf: buffer to send
1119  * @type: if it is send or close
1120  * @send_retrans: if rexmits to be done
1121  * @close_when_complete: is cm_node to be removed
1122  *
1123  * note - cm_node needs to be protected before calling this. Encase in:
1124  *		irdma_rem_ref_cm_node(cm_core, cm_node);
1125  *		irdma_schedule_cm_timer(...)
1126  *		refcount_inc(&cm_node->refcnt);
1127  */
1128 int irdma_schedule_cm_timer(struct irdma_cm_node *cm_node,
1129 			    struct irdma_puda_buf *sqbuf,
1130 			    enum irdma_timer_type type, int send_retrans,
1131 			    int close_when_complete)
1132 {
1133 	struct irdma_sc_vsi *vsi = &cm_node->iwdev->vsi;
1134 	struct irdma_cm_core *cm_core = cm_node->cm_core;
1135 	struct irdma_timer_entry *new_send;
1136 	u32 was_timer_set;
1137 	unsigned long flags;
1138 
1139 	new_send = kzalloc(sizeof(*new_send), GFP_ATOMIC);
1140 	if (!new_send) {
1141 		if (type != IRDMA_TIMER_TYPE_CLOSE)
1142 			irdma_free_sqbuf(vsi, sqbuf);
1143 		return -ENOMEM;
1144 	}
1145 
1146 	new_send->retrycount = IRDMA_DEFAULT_RETRYS;
1147 	new_send->retranscount = IRDMA_DEFAULT_RETRANS;
1148 	new_send->sqbuf = sqbuf;
1149 	new_send->timetosend = jiffies;
1150 	new_send->type = type;
1151 	new_send->send_retrans = send_retrans;
1152 	new_send->close_when_complete = close_when_complete;
1153 
1154 	if (type == IRDMA_TIMER_TYPE_CLOSE) {
1155 		new_send->timetosend += (HZ / 10);
1156 		if (cm_node->close_entry) {
1157 			kfree(new_send);
1158 			ibdev_dbg(&cm_node->iwdev->ibdev,
1159 				  "CM: already close entry\n");
1160 			return -EINVAL;
1161 		}
1162 
1163 		cm_node->close_entry = new_send;
1164 	} else { /* type == IRDMA_TIMER_TYPE_SEND */
1165 		spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
1166 		cm_node->send_entry = new_send;
1167 		refcount_inc(&cm_node->refcnt);
1168 		spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags);
1169 		new_send->timetosend = jiffies + IRDMA_RETRY_TIMEOUT;
1170 
1171 		refcount_inc(&sqbuf->refcount);
1172 		irdma_puda_send_buf(vsi->ilq, sqbuf);
1173 		if (!send_retrans) {
1174 			irdma_cleanup_retrans_entry(cm_node);
1175 			if (close_when_complete)
1176 				irdma_rem_ref_cm_node(cm_node);
1177 			return 0;
1178 		}
1179 	}
1180 
1181 	spin_lock_irqsave(&cm_core->ht_lock, flags);
1182 	was_timer_set = timer_pending(&cm_core->tcp_timer);
1183 
1184 	if (!was_timer_set) {
1185 		cm_core->tcp_timer.expires = new_send->timetosend;
1186 		add_timer(&cm_core->tcp_timer);
1187 	}
1188 	spin_unlock_irqrestore(&cm_core->ht_lock, flags);
1189 
1190 	return 0;
1191 }
1192 
1193 /**
1194  * irdma_retrans_expired - Could not rexmit the packet
1195  * @cm_node: connection's node
1196  */
1197 static void irdma_retrans_expired(struct irdma_cm_node *cm_node)
1198 {
1199 	enum irdma_cm_node_state state = cm_node->state;
1200 
1201 	cm_node->state = IRDMA_CM_STATE_CLOSED;
1202 	switch (state) {
1203 	case IRDMA_CM_STATE_SYN_RCVD:
1204 	case IRDMA_CM_STATE_CLOSING:
1205 		irdma_rem_ref_cm_node(cm_node);
1206 		break;
1207 	case IRDMA_CM_STATE_FIN_WAIT1:
1208 	case IRDMA_CM_STATE_LAST_ACK:
1209 		irdma_send_reset(cm_node);
1210 		break;
1211 	default:
1212 		refcount_inc(&cm_node->refcnt);
1213 		irdma_send_reset(cm_node);
1214 		irdma_create_event(cm_node, IRDMA_CM_EVENT_ABORTED);
1215 		break;
1216 	}
1217 }
1218 
1219 /**
1220  * irdma_handle_close_entry - for handling retry/timeouts
1221  * @cm_node: connection's node
1222  * @rem_node: flag for remove cm_node
1223  */
1224 static void irdma_handle_close_entry(struct irdma_cm_node *cm_node,
1225 				     u32 rem_node)
1226 {
1227 	struct irdma_timer_entry *close_entry = cm_node->close_entry;
1228 	struct irdma_qp *iwqp;
1229 	unsigned long flags;
1230 
1231 	if (!close_entry)
1232 		return;
1233 	iwqp = (struct irdma_qp *)close_entry->sqbuf;
1234 	if (iwqp) {
1235 		spin_lock_irqsave(&iwqp->lock, flags);
1236 		if (iwqp->cm_id) {
1237 			iwqp->hw_tcp_state = IRDMA_TCP_STATE_CLOSED;
1238 			iwqp->hw_iwarp_state = IRDMA_QP_STATE_ERROR;
1239 			iwqp->last_aeq = IRDMA_AE_RESET_SENT;
1240 			iwqp->ibqp_state = IB_QPS_ERR;
1241 			spin_unlock_irqrestore(&iwqp->lock, flags);
1242 			irdma_cm_disconn(iwqp);
1243 		} else {
1244 			spin_unlock_irqrestore(&iwqp->lock, flags);
1245 		}
1246 	} else if (rem_node) {
1247 		/* TIME_WAIT state */
1248 		irdma_rem_ref_cm_node(cm_node);
1249 	}
1250 
1251 	kfree(close_entry);
1252 	cm_node->close_entry = NULL;
1253 }
1254 
1255 /**
1256  * irdma_cm_timer_tick - system's timer expired callback
1257  * @t: Pointer to timer_list
1258  */
1259 static void irdma_cm_timer_tick(struct timer_list *t)
1260 {
1261 	unsigned long nexttimeout = jiffies + IRDMA_LONG_TIME;
1262 	struct irdma_cm_node *cm_node;
1263 	struct irdma_timer_entry *send_entry, *close_entry;
1264 	struct list_head *list_core_temp;
1265 	struct list_head *list_node;
1266 	struct irdma_cm_core *cm_core = from_timer(cm_core, t, tcp_timer);
1267 	struct irdma_sc_vsi *vsi;
1268 	u32 settimer = 0;
1269 	unsigned long timetosend;
1270 	unsigned long flags;
1271 	struct list_head timer_list;
1272 
1273 	INIT_LIST_HEAD(&timer_list);
1274 
1275 	rcu_read_lock();
1276 	irdma_timer_list_prep(cm_core, &timer_list);
1277 	rcu_read_unlock();
1278 
1279 	list_for_each_safe (list_node, list_core_temp, &timer_list) {
1280 		cm_node = container_of(list_node, struct irdma_cm_node,
1281 				       timer_entry);
1282 		close_entry = cm_node->close_entry;
1283 
1284 		if (close_entry) {
1285 			if (time_after(close_entry->timetosend, jiffies)) {
1286 				if (nexttimeout > close_entry->timetosend ||
1287 				    !settimer) {
1288 					nexttimeout = close_entry->timetosend;
1289 					settimer = 1;
1290 				}
1291 			} else {
1292 				irdma_handle_close_entry(cm_node, 1);
1293 			}
1294 		}
1295 
1296 		spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
1297 
1298 		send_entry = cm_node->send_entry;
1299 		if (!send_entry)
1300 			goto done;
1301 		if (time_after(send_entry->timetosend, jiffies)) {
1302 			if (cm_node->state != IRDMA_CM_STATE_OFFLOADED) {
1303 				if (nexttimeout > send_entry->timetosend ||
1304 				    !settimer) {
1305 					nexttimeout = send_entry->timetosend;
1306 					settimer = 1;
1307 				}
1308 			} else {
1309 				irdma_free_retrans_entry(cm_node);
1310 			}
1311 			goto done;
1312 		}
1313 
1314 		if (cm_node->state == IRDMA_CM_STATE_OFFLOADED ||
1315 		    cm_node->state == IRDMA_CM_STATE_CLOSED) {
1316 			irdma_free_retrans_entry(cm_node);
1317 			goto done;
1318 		}
1319 
1320 		if (!send_entry->retranscount || !send_entry->retrycount) {
1321 			irdma_free_retrans_entry(cm_node);
1322 
1323 			spin_unlock_irqrestore(&cm_node->retrans_list_lock,
1324 					       flags);
1325 			irdma_retrans_expired(cm_node);
1326 			cm_node->state = IRDMA_CM_STATE_CLOSED;
1327 			spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
1328 			goto done;
1329 		}
1330 		spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags);
1331 
1332 		vsi = &cm_node->iwdev->vsi;
1333 		if (!cm_node->ack_rcvd) {
1334 			refcount_inc(&send_entry->sqbuf->refcount);
1335 			irdma_puda_send_buf(vsi->ilq, send_entry->sqbuf);
1336 			cm_node->cm_core->stats_pkt_retrans++;
1337 		}
1338 
1339 		spin_lock_irqsave(&cm_node->retrans_list_lock, flags);
1340 		if (send_entry->send_retrans) {
1341 			send_entry->retranscount--;
1342 			timetosend = (IRDMA_RETRY_TIMEOUT <<
1343 				      (IRDMA_DEFAULT_RETRANS -
1344 				       send_entry->retranscount));
1345 
1346 			send_entry->timetosend = jiffies +
1347 			    min(timetosend, IRDMA_MAX_TIMEOUT);
1348 			if (nexttimeout > send_entry->timetosend || !settimer) {
1349 				nexttimeout = send_entry->timetosend;
1350 				settimer = 1;
1351 			}
1352 		} else {
1353 			int close_when_complete;
1354 
1355 			close_when_complete = send_entry->close_when_complete;
1356 			irdma_free_retrans_entry(cm_node);
1357 			if (close_when_complete)
1358 				irdma_rem_ref_cm_node(cm_node);
1359 		}
1360 done:
1361 		spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags);
1362 		irdma_rem_ref_cm_node(cm_node);
1363 	}
1364 
1365 	if (settimer) {
1366 		spin_lock_irqsave(&cm_core->ht_lock, flags);
1367 		if (!timer_pending(&cm_core->tcp_timer)) {
1368 			cm_core->tcp_timer.expires = nexttimeout;
1369 			add_timer(&cm_core->tcp_timer);
1370 		}
1371 		spin_unlock_irqrestore(&cm_core->ht_lock, flags);
1372 	}
1373 }
1374 
1375 /**
1376  * irdma_send_syn - send SYN packet
1377  * @cm_node: connection's node
1378  * @sendack: flag to set ACK bit or not
1379  */
1380 int irdma_send_syn(struct irdma_cm_node *cm_node, u32 sendack)
1381 {
1382 	struct irdma_puda_buf *sqbuf;
1383 	int flags = SET_SYN;
1384 	char optionsbuf[sizeof(struct option_mss) +
1385 			sizeof(struct option_windowscale) +
1386 			sizeof(struct option_base) + TCP_OPTIONS_PADDING];
1387 	struct irdma_kmem_info opts;
1388 	int optionssize = 0;
1389 	/* Sending MSS option */
1390 	union all_known_options *options;
1391 
1392 	opts.addr = optionsbuf;
1393 	if (!cm_node)
1394 		return -EINVAL;
1395 
1396 	options = (union all_known_options *)&optionsbuf[optionssize];
1397 	options->mss.optionnum = OPTION_NUM_MSS;
1398 	options->mss.len = sizeof(struct option_mss);
1399 	options->mss.mss = htons(cm_node->tcp_cntxt.mss);
1400 	optionssize += sizeof(struct option_mss);
1401 
1402 	options = (union all_known_options *)&optionsbuf[optionssize];
1403 	options->windowscale.optionnum = OPTION_NUM_WINDOW_SCALE;
1404 	options->windowscale.len = sizeof(struct option_windowscale);
1405 	options->windowscale.shiftcount = cm_node->tcp_cntxt.rcv_wscale;
1406 	optionssize += sizeof(struct option_windowscale);
1407 	options = (union all_known_options *)&optionsbuf[optionssize];
1408 	options->eol = OPTION_NUM_EOL;
1409 	optionssize += 1;
1410 
1411 	if (sendack)
1412 		flags |= SET_ACK;
1413 
1414 	opts.size = optionssize;
1415 
1416 	sqbuf = cm_node->cm_core->form_cm_frame(cm_node, &opts, NULL, NULL,
1417 						flags);
1418 	if (!sqbuf)
1419 		return -ENOMEM;
1420 
1421 	return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 1,
1422 				       0);
1423 }
1424 
1425 /**
1426  * irdma_send_ack - Send ACK packet
1427  * @cm_node: connection's node
1428  */
1429 void irdma_send_ack(struct irdma_cm_node *cm_node)
1430 {
1431 	struct irdma_puda_buf *sqbuf;
1432 	struct irdma_sc_vsi *vsi = &cm_node->iwdev->vsi;
1433 
1434 	sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL, NULL, NULL,
1435 						SET_ACK);
1436 	if (sqbuf)
1437 		irdma_puda_send_buf(vsi->ilq, sqbuf);
1438 }
1439 
1440 /**
1441  * irdma_send_fin - Send FIN pkt
1442  * @cm_node: connection's node
1443  */
1444 static int irdma_send_fin(struct irdma_cm_node *cm_node)
1445 {
1446 	struct irdma_puda_buf *sqbuf;
1447 
1448 	sqbuf = cm_node->cm_core->form_cm_frame(cm_node, NULL, NULL, NULL,
1449 						SET_ACK | SET_FIN);
1450 	if (!sqbuf)
1451 		return -ENOMEM;
1452 
1453 	return irdma_schedule_cm_timer(cm_node, sqbuf, IRDMA_TIMER_TYPE_SEND, 1,
1454 				       0);
1455 }
1456 
1457 /**
1458  * irdma_find_listener - find a cm node listening on this addr-port pair
1459  * @cm_core: cm's core
1460  * @dst_addr: listener ip addr
1461  * @ipv4: flag indicating IPv4 when true
1462  * @dst_port: listener tcp port num
1463  * @vlan_id: virtual LAN ID
1464  * @listener_state: state to match with listen node's
1465  */
1466 static struct irdma_cm_listener *
1467 irdma_find_listener(struct irdma_cm_core *cm_core, u32 *dst_addr, bool ipv4,
1468 		    u16 dst_port, u16 vlan_id,
1469 		    enum irdma_cm_listener_state listener_state)
1470 {
1471 	struct irdma_cm_listener *listen_node;
1472 	static const u32 ip_zero[4] = { 0, 0, 0, 0 };
1473 	u32 listen_addr[4];
1474 	u16 listen_port;
1475 	unsigned long flags;
1476 
1477 	/* walk list and find cm_node associated with this session ID */
1478 	spin_lock_irqsave(&cm_core->listen_list_lock, flags);
1479 	list_for_each_entry (listen_node, &cm_core->listen_list, list) {
1480 		memcpy(listen_addr, listen_node->loc_addr, sizeof(listen_addr));
1481 		listen_port = listen_node->loc_port;
1482 		if (listen_node->ipv4 != ipv4 || listen_port != dst_port ||
1483 		    !(listener_state & listen_node->listener_state))
1484 			continue;
1485 		/* compare node pair, return node handle if a match */
1486 		if (!memcmp(listen_addr, ip_zero, sizeof(listen_addr)) ||
1487 		    (!memcmp(listen_addr, dst_addr, sizeof(listen_addr)) &&
1488 		     vlan_id == listen_node->vlan_id)) {
1489 			refcount_inc(&listen_node->refcnt);
1490 			spin_unlock_irqrestore(&cm_core->listen_list_lock,
1491 					       flags);
1492 			trace_irdma_find_listener(listen_node);
1493 			return listen_node;
1494 		}
1495 	}
1496 	spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
1497 
1498 	return NULL;
1499 }
1500 
1501 /**
1502  * irdma_del_multiple_qhash - Remove qhash and child listens
1503  * @iwdev: iWarp device
1504  * @cm_info: CM info for parent listen node
1505  * @cm_parent_listen_node: The parent listen node
1506  */
1507 static int irdma_del_multiple_qhash(struct irdma_device *iwdev,
1508 				    struct irdma_cm_info *cm_info,
1509 				    struct irdma_cm_listener *cm_parent_listen_node)
1510 {
1511 	struct irdma_cm_listener *child_listen_node;
1512 	struct list_head *pos, *tpos;
1513 	unsigned long flags;
1514 	int ret = -EINVAL;
1515 
1516 	spin_lock_irqsave(&iwdev->cm_core.listen_list_lock, flags);
1517 	list_for_each_safe (pos, tpos,
1518 			    &cm_parent_listen_node->child_listen_list) {
1519 		child_listen_node = list_entry(pos, struct irdma_cm_listener,
1520 					       child_listen_list);
1521 		if (child_listen_node->ipv4)
1522 			ibdev_dbg(&iwdev->ibdev,
1523 				  "CM: removing child listen for IP=%pI4, port=%d, vlan=%d\n",
1524 				  child_listen_node->loc_addr,
1525 				  child_listen_node->loc_port,
1526 				  child_listen_node->vlan_id);
1527 		else
1528 			ibdev_dbg(&iwdev->ibdev,
1529 				  "CM: removing child listen for IP=%pI6, port=%d, vlan=%d\n",
1530 				  child_listen_node->loc_addr,
1531 				  child_listen_node->loc_port,
1532 				  child_listen_node->vlan_id);
1533 		trace_irdma_del_multiple_qhash(child_listen_node);
1534 		list_del(pos);
1535 		memcpy(cm_info->loc_addr, child_listen_node->loc_addr,
1536 		       sizeof(cm_info->loc_addr));
1537 		cm_info->vlan_id = child_listen_node->vlan_id;
1538 		if (child_listen_node->qhash_set) {
1539 			ret = irdma_manage_qhash(iwdev, cm_info,
1540 						 IRDMA_QHASH_TYPE_TCP_SYN,
1541 						 IRDMA_QHASH_MANAGE_TYPE_DELETE,
1542 						 NULL, false);
1543 			child_listen_node->qhash_set = false;
1544 		} else {
1545 			ret = 0;
1546 		}
1547 		ibdev_dbg(&iwdev->ibdev,
1548 			  "CM: Child listen node freed = %p\n",
1549 			  child_listen_node);
1550 		kfree(child_listen_node);
1551 		cm_parent_listen_node->cm_core->stats_listen_nodes_destroyed++;
1552 	}
1553 	spin_unlock_irqrestore(&iwdev->cm_core.listen_list_lock, flags);
1554 
1555 	return ret;
1556 }
1557 
1558 static u8 irdma_iw_get_vlan_prio(u32 *loc_addr, u8 prio, bool ipv4)
1559 {
1560 	struct net_device *ndev = NULL;
1561 
1562 	rcu_read_lock();
1563 	if (ipv4) {
1564 		ndev = ip_dev_find(&init_net, htonl(loc_addr[0]));
1565 	} else if (IS_ENABLED(CONFIG_IPV6)) {
1566 		struct net_device *ip_dev;
1567 		struct in6_addr laddr6;
1568 
1569 		irdma_copy_ip_htonl(laddr6.in6_u.u6_addr32, loc_addr);
1570 
1571 		for_each_netdev_rcu (&init_net, ip_dev) {
1572 			if (ipv6_chk_addr(&init_net, &laddr6, ip_dev, 1)) {
1573 				ndev = ip_dev;
1574 				break;
1575 			}
1576 		}
1577 	}
1578 
1579 	if (!ndev)
1580 		goto done;
1581 	if (is_vlan_dev(ndev))
1582 		prio = (vlan_dev_get_egress_qos_mask(ndev, prio) & VLAN_PRIO_MASK)
1583 			>> VLAN_PRIO_SHIFT;
1584 	if (ipv4)
1585 		dev_put(ndev);
1586 
1587 done:
1588 	rcu_read_unlock();
1589 
1590 	return prio;
1591 }
1592 
1593 /**
1594  * irdma_get_vlan_mac_ipv6 - Gets the vlan and mac
1595  * @addr: local IPv6 address
1596  * @vlan_id: vlan id for the given IPv6 address
1597  * @mac: mac address for the given IPv6 address
1598  *
1599  * Returns the vlan id and mac for an IPv6 address.
1600  */
1601 void irdma_get_vlan_mac_ipv6(u32 *addr, u16 *vlan_id, u8 *mac)
1602 {
1603 	struct net_device *ip_dev = NULL;
1604 	struct in6_addr laddr6;
1605 
1606 	if (!IS_ENABLED(CONFIG_IPV6))
1607 		return;
1608 
1609 	irdma_copy_ip_htonl(laddr6.in6_u.u6_addr32, addr);
1610 	if (vlan_id)
1611 		*vlan_id = 0xFFFF;	/* Match rdma_vlan_dev_vlan_id() */
1612 	if (mac)
1613 		eth_zero_addr(mac);
1614 
1615 	rcu_read_lock();
1616 	for_each_netdev_rcu (&init_net, ip_dev) {
1617 		if (ipv6_chk_addr(&init_net, &laddr6, ip_dev, 1)) {
1618 			if (vlan_id)
1619 				*vlan_id = rdma_vlan_dev_vlan_id(ip_dev);
1620 			if (ip_dev->dev_addr && mac)
1621 				ether_addr_copy(mac, ip_dev->dev_addr);
1622 			break;
1623 		}
1624 	}
1625 	rcu_read_unlock();
1626 }
1627 
1628 /**
1629  * irdma_get_vlan_ipv4 - Returns the vlan_id for IPv4 address
1630  * @addr: local IPv4 address
1631  */
1632 u16 irdma_get_vlan_ipv4(u32 *addr)
1633 {
1634 	struct net_device *netdev;
1635 	u16 vlan_id = 0xFFFF;
1636 
1637 	netdev = ip_dev_find(&init_net, htonl(addr[0]));
1638 	if (netdev) {
1639 		vlan_id = rdma_vlan_dev_vlan_id(netdev);
1640 		dev_put(netdev);
1641 	}
1642 
1643 	return vlan_id;
1644 }
1645 
1646 /**
1647  * irdma_add_mqh_6 - Adds multiple qhashes for IPv6
1648  * @iwdev: iWarp device
1649  * @cm_info: CM info for parent listen node
1650  * @cm_parent_listen_node: The parent listen node
1651  *
1652  * Adds a qhash and a child listen node for every IPv6 address
1653  * on the adapter and adds the associated qhash filter
1654  */
1655 static int irdma_add_mqh_6(struct irdma_device *iwdev,
1656 			   struct irdma_cm_info *cm_info,
1657 			   struct irdma_cm_listener *cm_parent_listen_node)
1658 {
1659 	struct net_device *ip_dev;
1660 	struct inet6_dev *idev;
1661 	struct inet6_ifaddr *ifp, *tmp;
1662 	struct irdma_cm_listener *child_listen_node;
1663 	unsigned long flags;
1664 	int ret = 0;
1665 
1666 	rtnl_lock();
1667 	for_each_netdev(&init_net, ip_dev) {
1668 		if (!(ip_dev->flags & IFF_UP))
1669 			continue;
1670 
1671 		if (((rdma_vlan_dev_vlan_id(ip_dev) >= VLAN_N_VID) ||
1672 		     (rdma_vlan_dev_real_dev(ip_dev) != iwdev->netdev)) &&
1673 		    ip_dev != iwdev->netdev)
1674 			continue;
1675 
1676 		idev = __in6_dev_get(ip_dev);
1677 		if (!idev) {
1678 			ibdev_dbg(&iwdev->ibdev, "CM: idev == NULL\n");
1679 			break;
1680 		}
1681 		list_for_each_entry_safe (ifp, tmp, &idev->addr_list, if_list) {
1682 			ibdev_dbg(&iwdev->ibdev, "CM: IP=%pI6, vlan_id=%d, MAC=%pM\n",
1683 				  &ifp->addr, rdma_vlan_dev_vlan_id(ip_dev),
1684 				  ip_dev->dev_addr);
1685 			child_listen_node = kzalloc(sizeof(*child_listen_node), GFP_KERNEL);
1686 			ibdev_dbg(&iwdev->ibdev, "CM: Allocating child listener %p\n",
1687 				  child_listen_node);
1688 			if (!child_listen_node) {
1689 				ibdev_dbg(&iwdev->ibdev, "CM: listener memory allocation\n");
1690 				ret = -ENOMEM;
1691 				goto exit;
1692 			}
1693 
1694 			cm_info->vlan_id = rdma_vlan_dev_vlan_id(ip_dev);
1695 			cm_parent_listen_node->vlan_id = cm_info->vlan_id;
1696 			memcpy(child_listen_node, cm_parent_listen_node,
1697 			       sizeof(*child_listen_node));
1698 			irdma_copy_ip_ntohl(child_listen_node->loc_addr,
1699 					    ifp->addr.in6_u.u6_addr32);
1700 			memcpy(cm_info->loc_addr, child_listen_node->loc_addr,
1701 			       sizeof(cm_info->loc_addr));
1702 			if (!iwdev->vsi.dscp_mode)
1703 				cm_info->user_pri =
1704 				irdma_iw_get_vlan_prio(child_listen_node->loc_addr,
1705 						       cm_info->user_pri,
1706 						       false);
1707 
1708 			ret = irdma_manage_qhash(iwdev, cm_info,
1709 						 IRDMA_QHASH_TYPE_TCP_SYN,
1710 						 IRDMA_QHASH_MANAGE_TYPE_ADD,
1711 						 NULL, true);
1712 			if (ret) {
1713 				kfree(child_listen_node);
1714 				continue;
1715 			}
1716 
1717 			trace_irdma_add_mqh_6(iwdev, child_listen_node,
1718 					      ip_dev->dev_addr);
1719 
1720 			child_listen_node->qhash_set = true;
1721 			spin_lock_irqsave(&iwdev->cm_core.listen_list_lock, flags);
1722 			list_add(&child_listen_node->child_listen_list,
1723 				 &cm_parent_listen_node->child_listen_list);
1724 			spin_unlock_irqrestore(&iwdev->cm_core.listen_list_lock, flags);
1725 			cm_parent_listen_node->cm_core->stats_listen_nodes_created++;
1726 		}
1727 	}
1728 exit:
1729 	rtnl_unlock();
1730 
1731 	return ret;
1732 }
1733 
1734 /**
1735  * irdma_add_mqh_4 - Adds multiple qhashes for IPv4
1736  * @iwdev: iWarp device
1737  * @cm_info: CM info for parent listen node
1738  * @cm_parent_listen_node: The parent listen node
1739  *
1740  * Adds a qhash and a child listen node for every IPv4 address
1741  * on the adapter and adds the associated qhash filter
1742  */
1743 static int irdma_add_mqh_4(struct irdma_device *iwdev,
1744 			   struct irdma_cm_info *cm_info,
1745 			   struct irdma_cm_listener *cm_parent_listen_node)
1746 {
1747 	struct net_device *ip_dev;
1748 	struct in_device *idev;
1749 	struct irdma_cm_listener *child_listen_node;
1750 	unsigned long flags;
1751 	const struct in_ifaddr *ifa;
1752 	int ret = 0;
1753 
1754 	rtnl_lock();
1755 	for_each_netdev(&init_net, ip_dev) {
1756 		if (!(ip_dev->flags & IFF_UP))
1757 			continue;
1758 
1759 		if (((rdma_vlan_dev_vlan_id(ip_dev) >= VLAN_N_VID) ||
1760 		     (rdma_vlan_dev_real_dev(ip_dev) != iwdev->netdev)) &&
1761 		    ip_dev != iwdev->netdev)
1762 			continue;
1763 
1764 		idev = in_dev_get(ip_dev);
1765 		if (!idev)
1766 			continue;
1767 
1768 		in_dev_for_each_ifa_rtnl(ifa, idev) {
1769 			ibdev_dbg(&iwdev->ibdev,
1770 				  "CM: Allocating child CM Listener forIP=%pI4, vlan_id=%d, MAC=%pM\n",
1771 				  &ifa->ifa_address, rdma_vlan_dev_vlan_id(ip_dev),
1772 				  ip_dev->dev_addr);
1773 			child_listen_node = kzalloc(sizeof(*child_listen_node), GFP_KERNEL);
1774 			cm_parent_listen_node->cm_core->stats_listen_nodes_created++;
1775 			ibdev_dbg(&iwdev->ibdev, "CM: Allocating child listener %p\n",
1776 				  child_listen_node);
1777 			if (!child_listen_node) {
1778 				ibdev_dbg(&iwdev->ibdev, "CM: listener memory allocation\n");
1779 				in_dev_put(idev);
1780 				ret = -ENOMEM;
1781 				goto exit;
1782 			}
1783 
1784 			cm_info->vlan_id = rdma_vlan_dev_vlan_id(ip_dev);
1785 			cm_parent_listen_node->vlan_id = cm_info->vlan_id;
1786 			memcpy(child_listen_node, cm_parent_listen_node,
1787 			       sizeof(*child_listen_node));
1788 			child_listen_node->loc_addr[0] =
1789 				ntohl(ifa->ifa_address);
1790 			memcpy(cm_info->loc_addr, child_listen_node->loc_addr,
1791 			       sizeof(cm_info->loc_addr));
1792 			if (!iwdev->vsi.dscp_mode)
1793 				cm_info->user_pri =
1794 				irdma_iw_get_vlan_prio(child_listen_node->loc_addr,
1795 						       cm_info->user_pri,
1796 						       true);
1797 			ret = irdma_manage_qhash(iwdev, cm_info,
1798 						 IRDMA_QHASH_TYPE_TCP_SYN,
1799 						 IRDMA_QHASH_MANAGE_TYPE_ADD,
1800 						 NULL, true);
1801 			if (ret) {
1802 				kfree(child_listen_node);
1803 				cm_parent_listen_node->cm_core
1804 					->stats_listen_nodes_created--;
1805 				continue;
1806 			}
1807 
1808 			trace_irdma_add_mqh_4(iwdev, child_listen_node,
1809 					      ip_dev->dev_addr);
1810 
1811 			child_listen_node->qhash_set = true;
1812 			spin_lock_irqsave(&iwdev->cm_core.listen_list_lock,
1813 					  flags);
1814 			list_add(&child_listen_node->child_listen_list,
1815 				 &cm_parent_listen_node->child_listen_list);
1816 			spin_unlock_irqrestore(&iwdev->cm_core.listen_list_lock, flags);
1817 		}
1818 		in_dev_put(idev);
1819 	}
1820 exit:
1821 	rtnl_unlock();
1822 
1823 	return ret;
1824 }
1825 
1826 /**
1827  * irdma_add_mqh - Adds multiple qhashes
1828  * @iwdev: iWarp device
1829  * @cm_info: CM info for parent listen node
1830  * @cm_listen_node: The parent listen node
1831  */
1832 static int irdma_add_mqh(struct irdma_device *iwdev,
1833 			 struct irdma_cm_info *cm_info,
1834 			 struct irdma_cm_listener *cm_listen_node)
1835 {
1836 	if (cm_info->ipv4)
1837 		return irdma_add_mqh_4(iwdev, cm_info, cm_listen_node);
1838 	else
1839 		return irdma_add_mqh_6(iwdev, cm_info, cm_listen_node);
1840 }
1841 
1842 /**
1843  * irdma_reset_list_prep - add connection nodes slated for reset to list
1844  * @cm_core: cm's core
1845  * @listener: pointer to listener node
1846  * @reset_list: a list to which cm_node will be selected
1847  */
1848 static void irdma_reset_list_prep(struct irdma_cm_core *cm_core,
1849 				  struct irdma_cm_listener *listener,
1850 				  struct list_head *reset_list)
1851 {
1852 	struct irdma_cm_node *cm_node;
1853 	int bkt;
1854 
1855 	hash_for_each_rcu(cm_core->cm_hash_tbl, bkt, cm_node, list) {
1856 		if (cm_node->listener == listener &&
1857 		    !cm_node->accelerated &&
1858 		    refcount_inc_not_zero(&cm_node->refcnt))
1859 			list_add(&cm_node->reset_entry, reset_list);
1860 	}
1861 }
1862 
1863 /**
1864  * irdma_dec_refcnt_listen - delete listener and associated cm nodes
1865  * @cm_core: cm's core
1866  * @listener: pointer to listener node
1867  * @free_hanging_nodes: to free associated cm_nodes
1868  * @apbvt_del: flag to delete the apbvt
1869  */
1870 static int irdma_dec_refcnt_listen(struct irdma_cm_core *cm_core,
1871 				   struct irdma_cm_listener *listener,
1872 				   int free_hanging_nodes, bool apbvt_del)
1873 {
1874 	int err;
1875 	struct list_head *list_pos;
1876 	struct list_head *list_temp;
1877 	struct irdma_cm_node *cm_node;
1878 	struct list_head reset_list;
1879 	struct irdma_cm_info nfo;
1880 	enum irdma_cm_node_state old_state;
1881 	unsigned long flags;
1882 
1883 	trace_irdma_dec_refcnt_listen(listener, __builtin_return_address(0));
1884 	/* free non-accelerated child nodes for this listener */
1885 	INIT_LIST_HEAD(&reset_list);
1886 	if (free_hanging_nodes) {
1887 		rcu_read_lock();
1888 		irdma_reset_list_prep(cm_core, listener, &reset_list);
1889 		rcu_read_unlock();
1890 	}
1891 
1892 	list_for_each_safe (list_pos, list_temp, &reset_list) {
1893 		cm_node = container_of(list_pos, struct irdma_cm_node,
1894 				       reset_entry);
1895 		if (cm_node->state >= IRDMA_CM_STATE_FIN_WAIT1) {
1896 			irdma_rem_ref_cm_node(cm_node);
1897 			continue;
1898 		}
1899 
1900 		irdma_cleanup_retrans_entry(cm_node);
1901 		err = irdma_send_reset(cm_node);
1902 		if (err) {
1903 			cm_node->state = IRDMA_CM_STATE_CLOSED;
1904 			ibdev_dbg(&cm_node->iwdev->ibdev,
1905 				  "CM: send reset failed\n");
1906 		} else {
1907 			old_state = cm_node->state;
1908 			cm_node->state = IRDMA_CM_STATE_LISTENER_DESTROYED;
1909 			if (old_state != IRDMA_CM_STATE_MPAREQ_RCVD)
1910 				irdma_rem_ref_cm_node(cm_node);
1911 		}
1912 	}
1913 
1914 	if (refcount_dec_and_test(&listener->refcnt)) {
1915 		spin_lock_irqsave(&cm_core->listen_list_lock, flags);
1916 		list_del(&listener->list);
1917 		spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
1918 
1919 		if (apbvt_del)
1920 			irdma_del_apbvt(listener->iwdev,
1921 					listener->apbvt_entry);
1922 		memcpy(nfo.loc_addr, listener->loc_addr, sizeof(nfo.loc_addr));
1923 		nfo.loc_port = listener->loc_port;
1924 		nfo.ipv4 = listener->ipv4;
1925 		nfo.vlan_id = listener->vlan_id;
1926 		nfo.user_pri = listener->user_pri;
1927 		nfo.qh_qpid = listener->iwdev->vsi.ilq->qp_id;
1928 
1929 		if (!list_empty(&listener->child_listen_list)) {
1930 			irdma_del_multiple_qhash(listener->iwdev, &nfo,
1931 						 listener);
1932 		} else {
1933 			if (listener->qhash_set)
1934 				irdma_manage_qhash(listener->iwdev,
1935 						   &nfo,
1936 						   IRDMA_QHASH_TYPE_TCP_SYN,
1937 						   IRDMA_QHASH_MANAGE_TYPE_DELETE,
1938 						   NULL, false);
1939 		}
1940 
1941 		cm_core->stats_listen_destroyed++;
1942 		cm_core->stats_listen_nodes_destroyed++;
1943 		ibdev_dbg(&listener->iwdev->ibdev,
1944 			  "CM: loc_port=0x%04x loc_addr=%pI4 cm_listen_node=%p cm_id=%p qhash_set=%d vlan_id=%d apbvt_del=%d\n",
1945 			  listener->loc_port, listener->loc_addr, listener,
1946 			  listener->cm_id, listener->qhash_set,
1947 			  listener->vlan_id, apbvt_del);
1948 		kfree(listener);
1949 		listener = NULL;
1950 		return 0;
1951 	}
1952 
1953 	return -EINVAL;
1954 }
1955 
1956 /**
1957  * irdma_cm_del_listen - delete a listener
1958  * @cm_core: cm's core
1959  * @listener: passive connection's listener
1960  * @apbvt_del: flag to delete apbvt
1961  */
1962 static int irdma_cm_del_listen(struct irdma_cm_core *cm_core,
1963 			       struct irdma_cm_listener *listener,
1964 			       bool apbvt_del)
1965 {
1966 	listener->listener_state = IRDMA_CM_LISTENER_PASSIVE_STATE;
1967 	listener->cm_id = NULL;
1968 
1969 	return irdma_dec_refcnt_listen(cm_core, listener, 1, apbvt_del);
1970 }
1971 
1972 /**
1973  * irdma_addr_resolve_neigh - resolve neighbor address
1974  * @iwdev: iwarp device structure
1975  * @src_ip: local ip address
1976  * @dst_ip: remote ip address
1977  * @arpindex: if there is an arp entry
1978  */
1979 static int irdma_addr_resolve_neigh(struct irdma_device *iwdev, u32 src_ip,
1980 				    u32 dst_ip, int arpindex)
1981 {
1982 	struct rtable *rt;
1983 	struct neighbour *neigh;
1984 	int rc = arpindex;
1985 	__be32 dst_ipaddr = htonl(dst_ip);
1986 	__be32 src_ipaddr = htonl(src_ip);
1987 
1988 	rt = ip_route_output(&init_net, dst_ipaddr, src_ipaddr, 0, 0);
1989 	if (IS_ERR(rt)) {
1990 		ibdev_dbg(&iwdev->ibdev, "CM: ip_route_output fail\n");
1991 		return -EINVAL;
1992 	}
1993 
1994 	neigh = dst_neigh_lookup(&rt->dst, &dst_ipaddr);
1995 	if (!neigh)
1996 		goto exit;
1997 
1998 	if (neigh->nud_state & NUD_VALID)
1999 		rc = irdma_add_arp(iwdev->rf, &dst_ip, true, neigh->ha);
2000 	else
2001 		neigh_event_send(neigh, NULL);
2002 	if (neigh)
2003 		neigh_release(neigh);
2004 exit:
2005 	ip_rt_put(rt);
2006 
2007 	return rc;
2008 }
2009 
2010 /**
2011  * irdma_get_dst_ipv6 - get destination cache entry via ipv6 lookup
2012  * @src_addr: local ipv6 sock address
2013  * @dst_addr: destination ipv6 sock address
2014  */
2015 static struct dst_entry *irdma_get_dst_ipv6(struct sockaddr_in6 *src_addr,
2016 					    struct sockaddr_in6 *dst_addr)
2017 {
2018 	struct dst_entry *dst = NULL;
2019 
2020 	if ((IS_ENABLED(CONFIG_IPV6))) {
2021 		struct flowi6 fl6 = {};
2022 
2023 		fl6.daddr = dst_addr->sin6_addr;
2024 		fl6.saddr = src_addr->sin6_addr;
2025 		if (ipv6_addr_type(&fl6.daddr) & IPV6_ADDR_LINKLOCAL)
2026 			fl6.flowi6_oif = dst_addr->sin6_scope_id;
2027 
2028 		dst = ip6_route_output(&init_net, NULL, &fl6);
2029 	}
2030 
2031 	return dst;
2032 }
2033 
2034 /**
2035  * irdma_addr_resolve_neigh_ipv6 - resolve neighbor ipv6 address
2036  * @iwdev: iwarp device structure
2037  * @src: local ip address
2038  * @dest: remote ip address
2039  * @arpindex: if there is an arp entry
2040  */
2041 static int irdma_addr_resolve_neigh_ipv6(struct irdma_device *iwdev, u32 *src,
2042 					 u32 *dest, int arpindex)
2043 {
2044 	struct neighbour *neigh;
2045 	int rc = arpindex;
2046 	struct dst_entry *dst;
2047 	struct sockaddr_in6 dst_addr = {};
2048 	struct sockaddr_in6 src_addr = {};
2049 
2050 	dst_addr.sin6_family = AF_INET6;
2051 	irdma_copy_ip_htonl(dst_addr.sin6_addr.in6_u.u6_addr32, dest);
2052 	src_addr.sin6_family = AF_INET6;
2053 	irdma_copy_ip_htonl(src_addr.sin6_addr.in6_u.u6_addr32, src);
2054 	dst = irdma_get_dst_ipv6(&src_addr, &dst_addr);
2055 	if (!dst || dst->error) {
2056 		if (dst) {
2057 			dst_release(dst);
2058 			ibdev_dbg(&iwdev->ibdev,
2059 				  "CM: ip6_route_output returned dst->error = %d\n",
2060 				  dst->error);
2061 		}
2062 		return -EINVAL;
2063 	}
2064 
2065 	neigh = dst_neigh_lookup(dst, dst_addr.sin6_addr.in6_u.u6_addr32);
2066 	if (!neigh)
2067 		goto exit;
2068 
2069 	ibdev_dbg(&iwdev->ibdev, "CM: dst_neigh_lookup MAC=%pM\n",
2070 		  neigh->ha);
2071 
2072 	trace_irdma_addr_resolve(iwdev, neigh->ha);
2073 
2074 	if (neigh->nud_state & NUD_VALID)
2075 		rc = irdma_add_arp(iwdev->rf, dest, false, neigh->ha);
2076 	else
2077 		neigh_event_send(neigh, NULL);
2078 	if (neigh)
2079 		neigh_release(neigh);
2080 exit:
2081 	dst_release(dst);
2082 
2083 	return rc;
2084 }
2085 
2086 /**
2087  * irdma_find_node - find a cm node that matches the reference cm node
2088  * @cm_core: cm's core
2089  * @rem_port: remote tcp port num
2090  * @rem_addr: remote ip addr
2091  * @loc_port: local tcp port num
2092  * @loc_addr: local ip addr
2093  * @vlan_id: local VLAN ID
2094  */
2095 struct irdma_cm_node *irdma_find_node(struct irdma_cm_core *cm_core,
2096 				      u16 rem_port, u32 *rem_addr, u16 loc_port,
2097 				      u32 *loc_addr, u16 vlan_id)
2098 {
2099 	struct irdma_cm_node *cm_node;
2100 	u32 key = (rem_port << 16) | loc_port;
2101 
2102 	rcu_read_lock();
2103 	hash_for_each_possible_rcu(cm_core->cm_hash_tbl, cm_node, list, key) {
2104 		if (cm_node->vlan_id == vlan_id &&
2105 		    cm_node->loc_port == loc_port && cm_node->rem_port == rem_port &&
2106 		    !memcmp(cm_node->loc_addr, loc_addr, sizeof(cm_node->loc_addr)) &&
2107 		    !memcmp(cm_node->rem_addr, rem_addr, sizeof(cm_node->rem_addr))) {
2108 			if (!refcount_inc_not_zero(&cm_node->refcnt))
2109 				goto exit;
2110 			rcu_read_unlock();
2111 			trace_irdma_find_node(cm_node, 0, NULL);
2112 			return cm_node;
2113 		}
2114 	}
2115 
2116 exit:
2117 	rcu_read_unlock();
2118 
2119 	/* no owner node */
2120 	return NULL;
2121 }
2122 
2123 /**
2124  * irdma_add_hte_node - add a cm node to the hash table
2125  * @cm_core: cm's core
2126  * @cm_node: connection's node
2127  */
2128 static void irdma_add_hte_node(struct irdma_cm_core *cm_core,
2129 			       struct irdma_cm_node *cm_node)
2130 {
2131 	unsigned long flags;
2132 	u32 key = (cm_node->rem_port << 16) | cm_node->loc_port;
2133 
2134 	spin_lock_irqsave(&cm_core->ht_lock, flags);
2135 	hash_add_rcu(cm_core->cm_hash_tbl, &cm_node->list, key);
2136 	spin_unlock_irqrestore(&cm_core->ht_lock, flags);
2137 }
2138 
2139 /**
2140  * irdma_ipv4_is_lpb - check if loopback
2141  * @loc_addr: local addr to compare
2142  * @rem_addr: remote address
2143  */
2144 bool irdma_ipv4_is_lpb(u32 loc_addr, u32 rem_addr)
2145 {
2146 	return ipv4_is_loopback(htonl(rem_addr)) || (loc_addr == rem_addr);
2147 }
2148 
2149 /**
2150  * irdma_ipv6_is_lpb - check if loopback
2151  * @loc_addr: local addr to compare
2152  * @rem_addr: remote address
2153  */
2154 bool irdma_ipv6_is_lpb(u32 *loc_addr, u32 *rem_addr)
2155 {
2156 	struct in6_addr raddr6;
2157 
2158 	irdma_copy_ip_htonl(raddr6.in6_u.u6_addr32, rem_addr);
2159 
2160 	return !memcmp(loc_addr, rem_addr, 16) || ipv6_addr_loopback(&raddr6);
2161 }
2162 
2163 /**
2164  * irdma_cm_create_ah - create a cm address handle
2165  * @cm_node: The connection manager node to create AH for
2166  * @wait: Provides option to wait for ah creation or not
2167  */
2168 static int irdma_cm_create_ah(struct irdma_cm_node *cm_node, bool wait)
2169 {
2170 	struct irdma_ah_info ah_info = {};
2171 	struct irdma_device *iwdev = cm_node->iwdev;
2172 
2173 	ether_addr_copy(ah_info.mac_addr, iwdev->netdev->dev_addr);
2174 
2175 	ah_info.hop_ttl = 0x40;
2176 	ah_info.tc_tos = cm_node->tos;
2177 	ah_info.vsi = &iwdev->vsi;
2178 
2179 	if (cm_node->ipv4) {
2180 		ah_info.ipv4_valid = true;
2181 		ah_info.dest_ip_addr[0] = cm_node->rem_addr[0];
2182 		ah_info.src_ip_addr[0] = cm_node->loc_addr[0];
2183 		ah_info.do_lpbk = irdma_ipv4_is_lpb(ah_info.src_ip_addr[0],
2184 						    ah_info.dest_ip_addr[0]);
2185 	} else {
2186 		memcpy(ah_info.dest_ip_addr, cm_node->rem_addr,
2187 		       sizeof(ah_info.dest_ip_addr));
2188 		memcpy(ah_info.src_ip_addr, cm_node->loc_addr,
2189 		       sizeof(ah_info.src_ip_addr));
2190 		ah_info.do_lpbk = irdma_ipv6_is_lpb(ah_info.src_ip_addr,
2191 						    ah_info.dest_ip_addr);
2192 	}
2193 
2194 	ah_info.vlan_tag = cm_node->vlan_id;
2195 	if (cm_node->vlan_id < VLAN_N_VID) {
2196 		ah_info.insert_vlan_tag = 1;
2197 		ah_info.vlan_tag |= cm_node->user_pri << VLAN_PRIO_SHIFT;
2198 	}
2199 
2200 	ah_info.dst_arpindex =
2201 		irdma_arp_table(iwdev->rf, ah_info.dest_ip_addr,
2202 				ah_info.ipv4_valid, NULL, IRDMA_ARP_RESOLVE);
2203 
2204 	if (irdma_puda_create_ah(&iwdev->rf->sc_dev, &ah_info, wait,
2205 				 IRDMA_PUDA_RSRC_TYPE_ILQ, cm_node,
2206 				 &cm_node->ah))
2207 		return -ENOMEM;
2208 
2209 	trace_irdma_create_ah(cm_node);
2210 	return 0;
2211 }
2212 
2213 /**
2214  * irdma_cm_free_ah - free a cm address handle
2215  * @cm_node: The connection manager node to create AH for
2216  */
2217 static void irdma_cm_free_ah(struct irdma_cm_node *cm_node)
2218 {
2219 	struct irdma_device *iwdev = cm_node->iwdev;
2220 
2221 	trace_irdma_cm_free_ah(cm_node);
2222 	irdma_puda_free_ah(&iwdev->rf->sc_dev, cm_node->ah);
2223 	cm_node->ah = NULL;
2224 }
2225 
2226 /**
2227  * irdma_make_cm_node - create a new instance of a cm node
2228  * @cm_core: cm's core
2229  * @iwdev: iwarp device structure
2230  * @cm_info: quad info for connection
2231  * @listener: passive connection's listener
2232  */
2233 static struct irdma_cm_node *
2234 irdma_make_cm_node(struct irdma_cm_core *cm_core, struct irdma_device *iwdev,
2235 		   struct irdma_cm_info *cm_info,
2236 		   struct irdma_cm_listener *listener)
2237 {
2238 	struct irdma_cm_node *cm_node;
2239 	int oldarpindex;
2240 	int arpindex;
2241 	struct net_device *netdev = iwdev->netdev;
2242 
2243 	/* create an hte and cm_node for this instance */
2244 	cm_node = kzalloc(sizeof(*cm_node), GFP_ATOMIC);
2245 	if (!cm_node)
2246 		return NULL;
2247 
2248 	/* set our node specific transport info */
2249 	cm_node->ipv4 = cm_info->ipv4;
2250 	cm_node->vlan_id = cm_info->vlan_id;
2251 	if (cm_node->vlan_id >= VLAN_N_VID && iwdev->dcb_vlan_mode)
2252 		cm_node->vlan_id = 0;
2253 	cm_node->tos = cm_info->tos;
2254 	cm_node->user_pri = cm_info->user_pri;
2255 	if (listener) {
2256 		if (listener->tos != cm_info->tos)
2257 			ibdev_warn(&iwdev->ibdev,
2258 				   "application TOS[%d] and remote client TOS[%d] mismatch\n",
2259 				   listener->tos, cm_info->tos);
2260 		if (iwdev->vsi.dscp_mode) {
2261 			cm_node->user_pri = listener->user_pri;
2262 		} else {
2263 			cm_node->tos = max(listener->tos, cm_info->tos);
2264 			cm_node->user_pri = rt_tos2priority(cm_node->tos);
2265 			cm_node->user_pri =
2266 				irdma_iw_get_vlan_prio(cm_info->loc_addr,
2267 						       cm_node->user_pri,
2268 						       cm_info->ipv4);
2269 		}
2270 		ibdev_dbg(&iwdev->ibdev,
2271 			  "DCB: listener: TOS:[%d] UP:[%d]\n", cm_node->tos,
2272 			  cm_node->user_pri);
2273 		trace_irdma_listener_tos(iwdev, cm_node->tos,
2274 					 cm_node->user_pri);
2275 	}
2276 	memcpy(cm_node->loc_addr, cm_info->loc_addr, sizeof(cm_node->loc_addr));
2277 	memcpy(cm_node->rem_addr, cm_info->rem_addr, sizeof(cm_node->rem_addr));
2278 	cm_node->loc_port = cm_info->loc_port;
2279 	cm_node->rem_port = cm_info->rem_port;
2280 
2281 	cm_node->mpa_frame_rev = IRDMA_CM_DEFAULT_MPA_VER;
2282 	cm_node->send_rdma0_op = SEND_RDMA_READ_ZERO;
2283 	cm_node->iwdev = iwdev;
2284 	cm_node->dev = &iwdev->rf->sc_dev;
2285 
2286 	cm_node->ird_size = cm_node->dev->hw_attrs.max_hw_ird;
2287 	cm_node->ord_size = cm_node->dev->hw_attrs.max_hw_ord;
2288 
2289 	cm_node->listener = listener;
2290 	cm_node->cm_id = cm_info->cm_id;
2291 	ether_addr_copy(cm_node->loc_mac, netdev->dev_addr);
2292 	spin_lock_init(&cm_node->retrans_list_lock);
2293 	cm_node->ack_rcvd = false;
2294 
2295 	init_completion(&cm_node->establish_comp);
2296 	refcount_set(&cm_node->refcnt, 1);
2297 	/* associate our parent CM core */
2298 	cm_node->cm_core = cm_core;
2299 	cm_node->tcp_cntxt.loc_id = IRDMA_CM_DEFAULT_LOCAL_ID;
2300 	cm_node->tcp_cntxt.rcv_wscale = iwdev->rcv_wscale;
2301 	cm_node->tcp_cntxt.rcv_wnd = iwdev->rcv_wnd >> cm_node->tcp_cntxt.rcv_wscale;
2302 	if (cm_node->ipv4) {
2303 		cm_node->tcp_cntxt.loc_seq_num = secure_tcp_seq(htonl(cm_node->loc_addr[0]),
2304 								htonl(cm_node->rem_addr[0]),
2305 								htons(cm_node->loc_port),
2306 								htons(cm_node->rem_port));
2307 		cm_node->tcp_cntxt.mss = iwdev->vsi.mtu - IRDMA_MTU_TO_MSS_IPV4;
2308 	} else if (IS_ENABLED(CONFIG_IPV6)) {
2309 		__be32 loc[4] = {
2310 			htonl(cm_node->loc_addr[0]), htonl(cm_node->loc_addr[1]),
2311 			htonl(cm_node->loc_addr[2]), htonl(cm_node->loc_addr[3])
2312 		};
2313 		__be32 rem[4] = {
2314 			htonl(cm_node->rem_addr[0]), htonl(cm_node->rem_addr[1]),
2315 			htonl(cm_node->rem_addr[2]), htonl(cm_node->rem_addr[3])
2316 		};
2317 		cm_node->tcp_cntxt.loc_seq_num = secure_tcpv6_seq(loc, rem,
2318 								  htons(cm_node->loc_port),
2319 								  htons(cm_node->rem_port));
2320 		cm_node->tcp_cntxt.mss = iwdev->vsi.mtu - IRDMA_MTU_TO_MSS_IPV6;
2321 	}
2322 
2323 	if ((cm_node->ipv4 &&
2324 	     irdma_ipv4_is_lpb(cm_node->loc_addr[0], cm_node->rem_addr[0])) ||
2325 	    (!cm_node->ipv4 &&
2326 	     irdma_ipv6_is_lpb(cm_node->loc_addr, cm_node->rem_addr))) {
2327 		cm_node->do_lpb = true;
2328 		arpindex = irdma_arp_table(iwdev->rf, cm_node->rem_addr,
2329 					   cm_node->ipv4, NULL,
2330 					   IRDMA_ARP_RESOLVE);
2331 	} else {
2332 		oldarpindex = irdma_arp_table(iwdev->rf, cm_node->rem_addr,
2333 					      cm_node->ipv4, NULL,
2334 					      IRDMA_ARP_RESOLVE);
2335 		if (cm_node->ipv4)
2336 			arpindex = irdma_addr_resolve_neigh(iwdev,
2337 							    cm_info->loc_addr[0],
2338 							    cm_info->rem_addr[0],
2339 							    oldarpindex);
2340 		else if (IS_ENABLED(CONFIG_IPV6))
2341 			arpindex = irdma_addr_resolve_neigh_ipv6(iwdev,
2342 								 cm_info->loc_addr,
2343 								 cm_info->rem_addr,
2344 								 oldarpindex);
2345 		else
2346 			arpindex = -EINVAL;
2347 	}
2348 
2349 	if (arpindex < 0)
2350 		goto err;
2351 
2352 	ether_addr_copy(cm_node->rem_mac,
2353 			iwdev->rf->arp_table[arpindex].mac_addr);
2354 	irdma_add_hte_node(cm_core, cm_node);
2355 	cm_core->stats_nodes_created++;
2356 	return cm_node;
2357 
2358 err:
2359 	kfree(cm_node);
2360 
2361 	return NULL;
2362 }
2363 
2364 static void irdma_destroy_connection(struct irdma_cm_node *cm_node)
2365 {
2366 	struct irdma_cm_core *cm_core = cm_node->cm_core;
2367 	struct irdma_qp *iwqp;
2368 	struct irdma_cm_info nfo;
2369 
2370 	/* if the node is destroyed before connection was accelerated */
2371 	if (!cm_node->accelerated && cm_node->accept_pend) {
2372 		ibdev_dbg(&cm_node->iwdev->ibdev,
2373 			  "CM: node destroyed before established\n");
2374 		atomic_dec(&cm_node->listener->pend_accepts_cnt);
2375 	}
2376 	if (cm_node->close_entry)
2377 		irdma_handle_close_entry(cm_node, 0);
2378 	if (cm_node->listener) {
2379 		irdma_dec_refcnt_listen(cm_core, cm_node->listener, 0, true);
2380 	} else {
2381 		if (cm_node->apbvt_set) {
2382 			irdma_del_apbvt(cm_node->iwdev, cm_node->apbvt_entry);
2383 			cm_node->apbvt_set = 0;
2384 		}
2385 		irdma_get_addr_info(cm_node, &nfo);
2386 		if (cm_node->qhash_set) {
2387 			nfo.qh_qpid = cm_node->iwdev->vsi.ilq->qp_id;
2388 			irdma_manage_qhash(cm_node->iwdev, &nfo,
2389 					   IRDMA_QHASH_TYPE_TCP_ESTABLISHED,
2390 					   IRDMA_QHASH_MANAGE_TYPE_DELETE, NULL,
2391 					   false);
2392 			cm_node->qhash_set = 0;
2393 		}
2394 	}
2395 
2396 	iwqp = cm_node->iwqp;
2397 	if (iwqp) {
2398 		cm_node->cm_id->rem_ref(cm_node->cm_id);
2399 		cm_node->cm_id = NULL;
2400 		iwqp->cm_id = NULL;
2401 		irdma_qp_rem_ref(&iwqp->ibqp);
2402 		cm_node->iwqp = NULL;
2403 	} else if (cm_node->qhash_set) {
2404 		irdma_get_addr_info(cm_node, &nfo);
2405 		nfo.qh_qpid = cm_node->iwdev->vsi.ilq->qp_id;
2406 		irdma_manage_qhash(cm_node->iwdev, &nfo,
2407 				   IRDMA_QHASH_TYPE_TCP_ESTABLISHED,
2408 				   IRDMA_QHASH_MANAGE_TYPE_DELETE, NULL, false);
2409 		cm_node->qhash_set = 0;
2410 	}
2411 
2412 	cm_core->cm_free_ah(cm_node);
2413 }
2414 
2415 /**
2416  * irdma_rem_ref_cm_node - destroy an instance of a cm node
2417  * @cm_node: connection's node
2418  */
2419 void irdma_rem_ref_cm_node(struct irdma_cm_node *cm_node)
2420 {
2421 	struct irdma_cm_core *cm_core = cm_node->cm_core;
2422 	unsigned long flags;
2423 
2424 	trace_irdma_rem_ref_cm_node(cm_node, 0, __builtin_return_address(0));
2425 	spin_lock_irqsave(&cm_core->ht_lock, flags);
2426 
2427 	if (!refcount_dec_and_test(&cm_node->refcnt)) {
2428 		spin_unlock_irqrestore(&cm_core->ht_lock, flags);
2429 		return;
2430 	}
2431 	if (cm_node->iwqp) {
2432 		cm_node->iwqp->cm_node = NULL;
2433 		cm_node->iwqp->cm_id = NULL;
2434 	}
2435 	hash_del_rcu(&cm_node->list);
2436 	cm_node->cm_core->stats_nodes_destroyed++;
2437 
2438 	spin_unlock_irqrestore(&cm_core->ht_lock, flags);
2439 
2440 	irdma_destroy_connection(cm_node);
2441 
2442 	kfree_rcu(cm_node, rcu_head);
2443 }
2444 
2445 /**
2446  * irdma_handle_fin_pkt - FIN packet received
2447  * @cm_node: connection's node
2448  */
2449 static void irdma_handle_fin_pkt(struct irdma_cm_node *cm_node)
2450 {
2451 	switch (cm_node->state) {
2452 	case IRDMA_CM_STATE_SYN_RCVD:
2453 	case IRDMA_CM_STATE_SYN_SENT:
2454 	case IRDMA_CM_STATE_ESTABLISHED:
2455 	case IRDMA_CM_STATE_MPAREJ_RCVD:
2456 		cm_node->tcp_cntxt.rcv_nxt++;
2457 		irdma_cleanup_retrans_entry(cm_node);
2458 		cm_node->state = IRDMA_CM_STATE_LAST_ACK;
2459 		irdma_send_fin(cm_node);
2460 		break;
2461 	case IRDMA_CM_STATE_MPAREQ_SENT:
2462 		irdma_create_event(cm_node, IRDMA_CM_EVENT_ABORTED);
2463 		cm_node->tcp_cntxt.rcv_nxt++;
2464 		irdma_cleanup_retrans_entry(cm_node);
2465 		cm_node->state = IRDMA_CM_STATE_CLOSED;
2466 		refcount_inc(&cm_node->refcnt);
2467 		irdma_send_reset(cm_node);
2468 		break;
2469 	case IRDMA_CM_STATE_FIN_WAIT1:
2470 		cm_node->tcp_cntxt.rcv_nxt++;
2471 		irdma_cleanup_retrans_entry(cm_node);
2472 		cm_node->state = IRDMA_CM_STATE_CLOSING;
2473 		irdma_send_ack(cm_node);
2474 		/*
2475 		 * Wait for ACK as this is simultaneous close.
2476 		 * After we receive ACK, do not send anything.
2477 		 * Just rm the node.
2478 		 */
2479 		break;
2480 	case IRDMA_CM_STATE_FIN_WAIT2:
2481 		cm_node->tcp_cntxt.rcv_nxt++;
2482 		irdma_cleanup_retrans_entry(cm_node);
2483 		cm_node->state = IRDMA_CM_STATE_TIME_WAIT;
2484 		irdma_send_ack(cm_node);
2485 		irdma_schedule_cm_timer(cm_node, NULL, IRDMA_TIMER_TYPE_CLOSE,
2486 					1, 0);
2487 		break;
2488 	case IRDMA_CM_STATE_TIME_WAIT:
2489 		cm_node->tcp_cntxt.rcv_nxt++;
2490 		irdma_cleanup_retrans_entry(cm_node);
2491 		cm_node->state = IRDMA_CM_STATE_CLOSED;
2492 		irdma_rem_ref_cm_node(cm_node);
2493 		break;
2494 	case IRDMA_CM_STATE_OFFLOADED:
2495 	default:
2496 		ibdev_dbg(&cm_node->iwdev->ibdev,
2497 			  "CM: bad state node state = %d\n", cm_node->state);
2498 		break;
2499 	}
2500 }
2501 
2502 /**
2503  * irdma_handle_rst_pkt - process received RST packet
2504  * @cm_node: connection's node
2505  * @rbuf: receive buffer
2506  */
2507 static void irdma_handle_rst_pkt(struct irdma_cm_node *cm_node,
2508 				 struct irdma_puda_buf *rbuf)
2509 {
2510 	ibdev_dbg(&cm_node->iwdev->ibdev,
2511 		  "CM: caller: %pS cm_node=%p state=%d rem_port=0x%04x loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4\n",
2512 		  __builtin_return_address(0), cm_node, cm_node->state,
2513 		  cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr,
2514 		  cm_node->loc_addr);
2515 
2516 	irdma_cleanup_retrans_entry(cm_node);
2517 	switch (cm_node->state) {
2518 	case IRDMA_CM_STATE_SYN_SENT:
2519 	case IRDMA_CM_STATE_MPAREQ_SENT:
2520 		switch (cm_node->mpa_frame_rev) {
2521 		case IETF_MPA_V2:
2522 			/* Drop down to MPA_V1*/
2523 			cm_node->mpa_frame_rev = IETF_MPA_V1;
2524 			/* send a syn and goto syn sent state */
2525 			cm_node->state = IRDMA_CM_STATE_SYN_SENT;
2526 			if (irdma_send_syn(cm_node, 0))
2527 				irdma_active_open_err(cm_node, false);
2528 			break;
2529 		case IETF_MPA_V1:
2530 		default:
2531 			irdma_active_open_err(cm_node, false);
2532 			break;
2533 		}
2534 		break;
2535 	case IRDMA_CM_STATE_MPAREQ_RCVD:
2536 		atomic_inc(&cm_node->passive_state);
2537 		break;
2538 	case IRDMA_CM_STATE_ESTABLISHED:
2539 	case IRDMA_CM_STATE_SYN_RCVD:
2540 	case IRDMA_CM_STATE_LISTENING:
2541 		irdma_passive_open_err(cm_node, false);
2542 		break;
2543 	case IRDMA_CM_STATE_OFFLOADED:
2544 		irdma_active_open_err(cm_node, false);
2545 		break;
2546 	case IRDMA_CM_STATE_CLOSED:
2547 		break;
2548 	case IRDMA_CM_STATE_FIN_WAIT2:
2549 	case IRDMA_CM_STATE_FIN_WAIT1:
2550 	case IRDMA_CM_STATE_LAST_ACK:
2551 	case IRDMA_CM_STATE_TIME_WAIT:
2552 		cm_node->state = IRDMA_CM_STATE_CLOSED;
2553 		irdma_rem_ref_cm_node(cm_node);
2554 		break;
2555 	default:
2556 		break;
2557 	}
2558 }
2559 
2560 /**
2561  * irdma_handle_rcv_mpa - Process a recv'd mpa buffer
2562  * @cm_node: connection's node
2563  * @rbuf: receive buffer
2564  */
2565 static void irdma_handle_rcv_mpa(struct irdma_cm_node *cm_node,
2566 				 struct irdma_puda_buf *rbuf)
2567 {
2568 	int err;
2569 	int datasize = rbuf->datalen;
2570 	u8 *dataloc = rbuf->data;
2571 
2572 	enum irdma_cm_event_type type = IRDMA_CM_EVENT_UNKNOWN;
2573 	u32 res_type;
2574 
2575 	err = irdma_parse_mpa(cm_node, dataloc, &res_type, datasize);
2576 	if (err) {
2577 		if (cm_node->state == IRDMA_CM_STATE_MPAREQ_SENT)
2578 			irdma_active_open_err(cm_node, true);
2579 		else
2580 			irdma_passive_open_err(cm_node, true);
2581 		return;
2582 	}
2583 
2584 	switch (cm_node->state) {
2585 	case IRDMA_CM_STATE_ESTABLISHED:
2586 		if (res_type == IRDMA_MPA_REQUEST_REJECT)
2587 			ibdev_dbg(&cm_node->iwdev->ibdev,
2588 				  "CM: state for reject\n");
2589 		cm_node->state = IRDMA_CM_STATE_MPAREQ_RCVD;
2590 		type = IRDMA_CM_EVENT_MPA_REQ;
2591 		irdma_send_ack(cm_node); /* ACK received MPA request */
2592 		atomic_set(&cm_node->passive_state,
2593 			   IRDMA_PASSIVE_STATE_INDICATED);
2594 		break;
2595 	case IRDMA_CM_STATE_MPAREQ_SENT:
2596 		irdma_cleanup_retrans_entry(cm_node);
2597 		if (res_type == IRDMA_MPA_REQUEST_REJECT) {
2598 			type = IRDMA_CM_EVENT_MPA_REJECT;
2599 			cm_node->state = IRDMA_CM_STATE_MPAREJ_RCVD;
2600 		} else {
2601 			type = IRDMA_CM_EVENT_CONNECTED;
2602 			cm_node->state = IRDMA_CM_STATE_OFFLOADED;
2603 		}
2604 		irdma_send_ack(cm_node);
2605 		break;
2606 	default:
2607 		ibdev_dbg(&cm_node->iwdev->ibdev,
2608 			  "CM: wrong cm_node state =%d\n", cm_node->state);
2609 		break;
2610 	}
2611 	irdma_create_event(cm_node, type);
2612 }
2613 
2614 /**
2615  * irdma_check_syn - Check for error on received syn ack
2616  * @cm_node: connection's node
2617  * @tcph: pointer tcp header
2618  */
2619 static int irdma_check_syn(struct irdma_cm_node *cm_node, struct tcphdr *tcph)
2620 {
2621 	if (ntohl(tcph->ack_seq) != cm_node->tcp_cntxt.loc_seq_num) {
2622 		irdma_active_open_err(cm_node, true);
2623 		return 1;
2624 	}
2625 
2626 	return 0;
2627 }
2628 
2629 /**
2630  * irdma_check_seq - check seq numbers if OK
2631  * @cm_node: connection's node
2632  * @tcph: pointer tcp header
2633  */
2634 static int irdma_check_seq(struct irdma_cm_node *cm_node, struct tcphdr *tcph)
2635 {
2636 	u32 seq;
2637 	u32 ack_seq;
2638 	u32 loc_seq_num = cm_node->tcp_cntxt.loc_seq_num;
2639 	u32 rcv_nxt = cm_node->tcp_cntxt.rcv_nxt;
2640 	u32 rcv_wnd;
2641 	int err = 0;
2642 
2643 	seq = ntohl(tcph->seq);
2644 	ack_seq = ntohl(tcph->ack_seq);
2645 	rcv_wnd = cm_node->tcp_cntxt.rcv_wnd;
2646 	if (ack_seq != loc_seq_num ||
2647 	    !between(seq, rcv_nxt, (rcv_nxt + rcv_wnd)))
2648 		err = -1;
2649 	if (err)
2650 		ibdev_dbg(&cm_node->iwdev->ibdev,
2651 			  "CM: seq number err\n");
2652 
2653 	return err;
2654 }
2655 
2656 void irdma_add_conn_est_qh(struct irdma_cm_node *cm_node)
2657 {
2658 	struct irdma_cm_info nfo;
2659 
2660 	irdma_get_addr_info(cm_node, &nfo);
2661 	nfo.qh_qpid = cm_node->iwdev->vsi.ilq->qp_id;
2662 	irdma_manage_qhash(cm_node->iwdev, &nfo,
2663 			   IRDMA_QHASH_TYPE_TCP_ESTABLISHED,
2664 			   IRDMA_QHASH_MANAGE_TYPE_ADD,
2665 			   cm_node, false);
2666 	cm_node->qhash_set = true;
2667 }
2668 
2669 /**
2670  * irdma_handle_syn_pkt - is for Passive node
2671  * @cm_node: connection's node
2672  * @rbuf: receive buffer
2673  */
2674 static void irdma_handle_syn_pkt(struct irdma_cm_node *cm_node,
2675 				 struct irdma_puda_buf *rbuf)
2676 {
2677 	struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph;
2678 	int err;
2679 	u32 inc_sequence;
2680 	int optionsize;
2681 
2682 	optionsize = (tcph->doff << 2) - sizeof(struct tcphdr);
2683 	inc_sequence = ntohl(tcph->seq);
2684 
2685 	switch (cm_node->state) {
2686 	case IRDMA_CM_STATE_SYN_SENT:
2687 	case IRDMA_CM_STATE_MPAREQ_SENT:
2688 		/* Rcvd syn on active open connection */
2689 		irdma_active_open_err(cm_node, 1);
2690 		break;
2691 	case IRDMA_CM_STATE_LISTENING:
2692 		/* Passive OPEN */
2693 		if (atomic_read(&cm_node->listener->pend_accepts_cnt) >
2694 		    cm_node->listener->backlog) {
2695 			cm_node->cm_core->stats_backlog_drops++;
2696 			irdma_passive_open_err(cm_node, false);
2697 			break;
2698 		}
2699 		err = irdma_handle_tcp_options(cm_node, tcph, optionsize, 1);
2700 		if (err) {
2701 			irdma_passive_open_err(cm_node, false);
2702 			/* drop pkt */
2703 			break;
2704 		}
2705 		err = cm_node->cm_core->cm_create_ah(cm_node, false);
2706 		if (err) {
2707 			irdma_passive_open_err(cm_node, false);
2708 			/* drop pkt */
2709 			break;
2710 		}
2711 		cm_node->tcp_cntxt.rcv_nxt = inc_sequence + 1;
2712 		cm_node->accept_pend = 1;
2713 		atomic_inc(&cm_node->listener->pend_accepts_cnt);
2714 
2715 		cm_node->state = IRDMA_CM_STATE_SYN_RCVD;
2716 		break;
2717 	case IRDMA_CM_STATE_CLOSED:
2718 		irdma_cleanup_retrans_entry(cm_node);
2719 		refcount_inc(&cm_node->refcnt);
2720 		irdma_send_reset(cm_node);
2721 		break;
2722 	case IRDMA_CM_STATE_OFFLOADED:
2723 	case IRDMA_CM_STATE_ESTABLISHED:
2724 	case IRDMA_CM_STATE_FIN_WAIT1:
2725 	case IRDMA_CM_STATE_FIN_WAIT2:
2726 	case IRDMA_CM_STATE_MPAREQ_RCVD:
2727 	case IRDMA_CM_STATE_LAST_ACK:
2728 	case IRDMA_CM_STATE_CLOSING:
2729 	case IRDMA_CM_STATE_UNKNOWN:
2730 	default:
2731 		break;
2732 	}
2733 }
2734 
2735 /**
2736  * irdma_handle_synack_pkt - Process SYN+ACK packet (active side)
2737  * @cm_node: connection's node
2738  * @rbuf: receive buffer
2739  */
2740 static void irdma_handle_synack_pkt(struct irdma_cm_node *cm_node,
2741 				    struct irdma_puda_buf *rbuf)
2742 {
2743 	struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph;
2744 	int err;
2745 	u32 inc_sequence;
2746 	int optionsize;
2747 
2748 	optionsize = (tcph->doff << 2) - sizeof(struct tcphdr);
2749 	inc_sequence = ntohl(tcph->seq);
2750 	switch (cm_node->state) {
2751 	case IRDMA_CM_STATE_SYN_SENT:
2752 		irdma_cleanup_retrans_entry(cm_node);
2753 		/* active open */
2754 		if (irdma_check_syn(cm_node, tcph)) {
2755 			ibdev_dbg(&cm_node->iwdev->ibdev,
2756 				  "CM: check syn fail\n");
2757 			return;
2758 		}
2759 		cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->ack_seq);
2760 		/* setup options */
2761 		err = irdma_handle_tcp_options(cm_node, tcph, optionsize, 0);
2762 		if (err) {
2763 			ibdev_dbg(&cm_node->iwdev->ibdev,
2764 				  "CM: cm_node=%p tcp_options failed\n",
2765 				  cm_node);
2766 			break;
2767 		}
2768 		irdma_cleanup_retrans_entry(cm_node);
2769 		cm_node->tcp_cntxt.rcv_nxt = inc_sequence + 1;
2770 		irdma_send_ack(cm_node); /* ACK  for the syn_ack */
2771 		err = irdma_send_mpa_request(cm_node);
2772 		if (err) {
2773 			ibdev_dbg(&cm_node->iwdev->ibdev,
2774 				  "CM: cm_node=%p irdma_send_mpa_request failed\n",
2775 				  cm_node);
2776 			break;
2777 		}
2778 		cm_node->state = IRDMA_CM_STATE_MPAREQ_SENT;
2779 		break;
2780 	case IRDMA_CM_STATE_MPAREQ_RCVD:
2781 		irdma_passive_open_err(cm_node, true);
2782 		break;
2783 	case IRDMA_CM_STATE_LISTENING:
2784 		cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq);
2785 		irdma_cleanup_retrans_entry(cm_node);
2786 		cm_node->state = IRDMA_CM_STATE_CLOSED;
2787 		irdma_send_reset(cm_node);
2788 		break;
2789 	case IRDMA_CM_STATE_CLOSED:
2790 		cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq);
2791 		irdma_cleanup_retrans_entry(cm_node);
2792 		refcount_inc(&cm_node->refcnt);
2793 		irdma_send_reset(cm_node);
2794 		break;
2795 	case IRDMA_CM_STATE_ESTABLISHED:
2796 	case IRDMA_CM_STATE_FIN_WAIT1:
2797 	case IRDMA_CM_STATE_FIN_WAIT2:
2798 	case IRDMA_CM_STATE_LAST_ACK:
2799 	case IRDMA_CM_STATE_OFFLOADED:
2800 	case IRDMA_CM_STATE_CLOSING:
2801 	case IRDMA_CM_STATE_UNKNOWN:
2802 	case IRDMA_CM_STATE_MPAREQ_SENT:
2803 	default:
2804 		break;
2805 	}
2806 }
2807 
2808 /**
2809  * irdma_handle_ack_pkt - process packet with ACK
2810  * @cm_node: connection's node
2811  * @rbuf: receive buffer
2812  */
2813 static int irdma_handle_ack_pkt(struct irdma_cm_node *cm_node,
2814 				struct irdma_puda_buf *rbuf)
2815 {
2816 	struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph;
2817 	u32 inc_sequence;
2818 	int ret;
2819 	int optionsize;
2820 	u32 datasize = rbuf->datalen;
2821 
2822 	optionsize = (tcph->doff << 2) - sizeof(struct tcphdr);
2823 
2824 	if (irdma_check_seq(cm_node, tcph))
2825 		return -EINVAL;
2826 
2827 	inc_sequence = ntohl(tcph->seq);
2828 	switch (cm_node->state) {
2829 	case IRDMA_CM_STATE_SYN_RCVD:
2830 		irdma_cleanup_retrans_entry(cm_node);
2831 		ret = irdma_handle_tcp_options(cm_node, tcph, optionsize, 1);
2832 		if (ret)
2833 			return ret;
2834 		cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->ack_seq);
2835 		cm_node->state = IRDMA_CM_STATE_ESTABLISHED;
2836 		if (datasize) {
2837 			cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize;
2838 			irdma_handle_rcv_mpa(cm_node, rbuf);
2839 		}
2840 		break;
2841 	case IRDMA_CM_STATE_ESTABLISHED:
2842 		irdma_cleanup_retrans_entry(cm_node);
2843 		if (datasize) {
2844 			cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize;
2845 			irdma_handle_rcv_mpa(cm_node, rbuf);
2846 		}
2847 		break;
2848 	case IRDMA_CM_STATE_MPAREQ_SENT:
2849 		cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->ack_seq);
2850 		if (datasize) {
2851 			cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize;
2852 			cm_node->ack_rcvd = false;
2853 			irdma_handle_rcv_mpa(cm_node, rbuf);
2854 		} else {
2855 			cm_node->ack_rcvd = true;
2856 		}
2857 		break;
2858 	case IRDMA_CM_STATE_LISTENING:
2859 		irdma_cleanup_retrans_entry(cm_node);
2860 		cm_node->state = IRDMA_CM_STATE_CLOSED;
2861 		irdma_send_reset(cm_node);
2862 		break;
2863 	case IRDMA_CM_STATE_CLOSED:
2864 		irdma_cleanup_retrans_entry(cm_node);
2865 		refcount_inc(&cm_node->refcnt);
2866 		irdma_send_reset(cm_node);
2867 		break;
2868 	case IRDMA_CM_STATE_LAST_ACK:
2869 	case IRDMA_CM_STATE_CLOSING:
2870 		irdma_cleanup_retrans_entry(cm_node);
2871 		cm_node->state = IRDMA_CM_STATE_CLOSED;
2872 		irdma_rem_ref_cm_node(cm_node);
2873 		break;
2874 	case IRDMA_CM_STATE_FIN_WAIT1:
2875 		irdma_cleanup_retrans_entry(cm_node);
2876 		cm_node->state = IRDMA_CM_STATE_FIN_WAIT2;
2877 		break;
2878 	case IRDMA_CM_STATE_SYN_SENT:
2879 	case IRDMA_CM_STATE_FIN_WAIT2:
2880 	case IRDMA_CM_STATE_OFFLOADED:
2881 	case IRDMA_CM_STATE_MPAREQ_RCVD:
2882 	case IRDMA_CM_STATE_UNKNOWN:
2883 	default:
2884 		irdma_cleanup_retrans_entry(cm_node);
2885 		break;
2886 	}
2887 
2888 	return 0;
2889 }
2890 
2891 /**
2892  * irdma_process_pkt - process cm packet
2893  * @cm_node: connection's node
2894  * @rbuf: receive buffer
2895  */
2896 static void irdma_process_pkt(struct irdma_cm_node *cm_node,
2897 			      struct irdma_puda_buf *rbuf)
2898 {
2899 	enum irdma_tcpip_pkt_type pkt_type = IRDMA_PKT_TYPE_UNKNOWN;
2900 	struct tcphdr *tcph = (struct tcphdr *)rbuf->tcph;
2901 	u32 fin_set = 0;
2902 	int err;
2903 
2904 	if (tcph->rst) {
2905 		pkt_type = IRDMA_PKT_TYPE_RST;
2906 	} else if (tcph->syn) {
2907 		pkt_type = IRDMA_PKT_TYPE_SYN;
2908 		if (tcph->ack)
2909 			pkt_type = IRDMA_PKT_TYPE_SYNACK;
2910 	} else if (tcph->ack) {
2911 		pkt_type = IRDMA_PKT_TYPE_ACK;
2912 	}
2913 	if (tcph->fin)
2914 		fin_set = 1;
2915 
2916 	switch (pkt_type) {
2917 	case IRDMA_PKT_TYPE_SYN:
2918 		irdma_handle_syn_pkt(cm_node, rbuf);
2919 		break;
2920 	case IRDMA_PKT_TYPE_SYNACK:
2921 		irdma_handle_synack_pkt(cm_node, rbuf);
2922 		break;
2923 	case IRDMA_PKT_TYPE_ACK:
2924 		err = irdma_handle_ack_pkt(cm_node, rbuf);
2925 		if (fin_set && !err)
2926 			irdma_handle_fin_pkt(cm_node);
2927 		break;
2928 	case IRDMA_PKT_TYPE_RST:
2929 		irdma_handle_rst_pkt(cm_node, rbuf);
2930 		break;
2931 	default:
2932 		if (fin_set &&
2933 		    (!irdma_check_seq(cm_node, (struct tcphdr *)rbuf->tcph)))
2934 			irdma_handle_fin_pkt(cm_node);
2935 		break;
2936 	}
2937 }
2938 
2939 /**
2940  * irdma_make_listen_node - create a listen node with params
2941  * @cm_core: cm's core
2942  * @iwdev: iwarp device structure
2943  * @cm_info: quad info for connection
2944  */
2945 static struct irdma_cm_listener *
2946 irdma_make_listen_node(struct irdma_cm_core *cm_core,
2947 		       struct irdma_device *iwdev,
2948 		       struct irdma_cm_info *cm_info)
2949 {
2950 	struct irdma_cm_listener *listener;
2951 	unsigned long flags;
2952 
2953 	/* cannot have multiple matching listeners */
2954 	listener =
2955 		irdma_find_listener(cm_core, cm_info->loc_addr, cm_info->ipv4,
2956 				    cm_info->loc_port, cm_info->vlan_id,
2957 				    IRDMA_CM_LISTENER_EITHER_STATE);
2958 	if (listener &&
2959 	    listener->listener_state == IRDMA_CM_LISTENER_ACTIVE_STATE) {
2960 		refcount_dec(&listener->refcnt);
2961 		return NULL;
2962 	}
2963 
2964 	if (!listener) {
2965 		/* create a CM listen node
2966 		 * 1/2 node to compare incoming traffic to
2967 		 */
2968 		listener = kzalloc(sizeof(*listener), GFP_KERNEL);
2969 		if (!listener)
2970 			return NULL;
2971 		cm_core->stats_listen_nodes_created++;
2972 		memcpy(listener->loc_addr, cm_info->loc_addr,
2973 		       sizeof(listener->loc_addr));
2974 		listener->loc_port = cm_info->loc_port;
2975 
2976 		INIT_LIST_HEAD(&listener->child_listen_list);
2977 
2978 		refcount_set(&listener->refcnt, 1);
2979 	} else {
2980 		listener->reused_node = 1;
2981 	}
2982 
2983 	listener->cm_id = cm_info->cm_id;
2984 	listener->ipv4 = cm_info->ipv4;
2985 	listener->vlan_id = cm_info->vlan_id;
2986 	atomic_set(&listener->pend_accepts_cnt, 0);
2987 	listener->cm_core = cm_core;
2988 	listener->iwdev = iwdev;
2989 
2990 	listener->backlog = cm_info->backlog;
2991 	listener->listener_state = IRDMA_CM_LISTENER_ACTIVE_STATE;
2992 
2993 	if (!listener->reused_node) {
2994 		spin_lock_irqsave(&cm_core->listen_list_lock, flags);
2995 		list_add(&listener->list, &cm_core->listen_list);
2996 		spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
2997 	}
2998 
2999 	return listener;
3000 }
3001 
3002 /**
3003  * irdma_create_cm_node - make a connection node with params
3004  * @cm_core: cm's core
3005  * @iwdev: iwarp device structure
3006  * @conn_param: connection parameters
3007  * @cm_info: quad info for connection
3008  * @caller_cm_node: pointer to cm_node structure to return
3009  */
3010 static int irdma_create_cm_node(struct irdma_cm_core *cm_core,
3011 				struct irdma_device *iwdev,
3012 				struct iw_cm_conn_param *conn_param,
3013 				struct irdma_cm_info *cm_info,
3014 				struct irdma_cm_node **caller_cm_node)
3015 {
3016 	struct irdma_cm_node *cm_node;
3017 	u16 private_data_len = conn_param->private_data_len;
3018 	const void *private_data = conn_param->private_data;
3019 
3020 	/* create a CM connection node */
3021 	cm_node = irdma_make_cm_node(cm_core, iwdev, cm_info, NULL);
3022 	if (!cm_node)
3023 		return -ENOMEM;
3024 
3025 	/* set our node side to client (active) side */
3026 	cm_node->tcp_cntxt.client = 1;
3027 	cm_node->tcp_cntxt.rcv_wscale = IRDMA_CM_DEFAULT_RCV_WND_SCALE;
3028 
3029 	irdma_record_ird_ord(cm_node, conn_param->ird, conn_param->ord);
3030 
3031 	cm_node->pdata.size = private_data_len;
3032 	cm_node->pdata.addr = cm_node->pdata_buf;
3033 
3034 	memcpy(cm_node->pdata_buf, private_data, private_data_len);
3035 	*caller_cm_node = cm_node;
3036 
3037 	return 0;
3038 }
3039 
3040 /**
3041  * irdma_cm_reject - reject and teardown a connection
3042  * @cm_node: connection's node
3043  * @pdata: ptr to private data for reject
3044  * @plen: size of private data
3045  */
3046 static int irdma_cm_reject(struct irdma_cm_node *cm_node, const void *pdata,
3047 			   u8 plen)
3048 {
3049 	int ret;
3050 	int passive_state;
3051 
3052 	if (cm_node->tcp_cntxt.client)
3053 		return 0;
3054 
3055 	irdma_cleanup_retrans_entry(cm_node);
3056 
3057 	passive_state = atomic_add_return(1, &cm_node->passive_state);
3058 	if (passive_state == IRDMA_SEND_RESET_EVENT) {
3059 		cm_node->state = IRDMA_CM_STATE_CLOSED;
3060 		irdma_rem_ref_cm_node(cm_node);
3061 		return 0;
3062 	}
3063 
3064 	if (cm_node->state == IRDMA_CM_STATE_LISTENER_DESTROYED) {
3065 		irdma_rem_ref_cm_node(cm_node);
3066 		return 0;
3067 	}
3068 
3069 	ret = irdma_send_mpa_reject(cm_node, pdata, plen);
3070 	if (!ret)
3071 		return 0;
3072 
3073 	cm_node->state = IRDMA_CM_STATE_CLOSED;
3074 	if (irdma_send_reset(cm_node))
3075 		ibdev_dbg(&cm_node->iwdev->ibdev,
3076 			  "CM: send reset failed\n");
3077 
3078 	return ret;
3079 }
3080 
3081 /**
3082  * irdma_cm_close - close of cm connection
3083  * @cm_node: connection's node
3084  */
3085 static int irdma_cm_close(struct irdma_cm_node *cm_node)
3086 {
3087 	switch (cm_node->state) {
3088 	case IRDMA_CM_STATE_SYN_RCVD:
3089 	case IRDMA_CM_STATE_SYN_SENT:
3090 	case IRDMA_CM_STATE_ONE_SIDE_ESTABLISHED:
3091 	case IRDMA_CM_STATE_ESTABLISHED:
3092 	case IRDMA_CM_STATE_ACCEPTING:
3093 	case IRDMA_CM_STATE_MPAREQ_SENT:
3094 	case IRDMA_CM_STATE_MPAREQ_RCVD:
3095 		irdma_cleanup_retrans_entry(cm_node);
3096 		irdma_send_reset(cm_node);
3097 		break;
3098 	case IRDMA_CM_STATE_CLOSE_WAIT:
3099 		cm_node->state = IRDMA_CM_STATE_LAST_ACK;
3100 		irdma_send_fin(cm_node);
3101 		break;
3102 	case IRDMA_CM_STATE_FIN_WAIT1:
3103 	case IRDMA_CM_STATE_FIN_WAIT2:
3104 	case IRDMA_CM_STATE_LAST_ACK:
3105 	case IRDMA_CM_STATE_TIME_WAIT:
3106 	case IRDMA_CM_STATE_CLOSING:
3107 		return -EINVAL;
3108 	case IRDMA_CM_STATE_LISTENING:
3109 		irdma_cleanup_retrans_entry(cm_node);
3110 		irdma_send_reset(cm_node);
3111 		break;
3112 	case IRDMA_CM_STATE_MPAREJ_RCVD:
3113 	case IRDMA_CM_STATE_UNKNOWN:
3114 	case IRDMA_CM_STATE_INITED:
3115 	case IRDMA_CM_STATE_CLOSED:
3116 	case IRDMA_CM_STATE_LISTENER_DESTROYED:
3117 		irdma_rem_ref_cm_node(cm_node);
3118 		break;
3119 	case IRDMA_CM_STATE_OFFLOADED:
3120 		if (cm_node->send_entry)
3121 			ibdev_dbg(&cm_node->iwdev->ibdev,
3122 				  "CM: CM send_entry in OFFLOADED state\n");
3123 		irdma_rem_ref_cm_node(cm_node);
3124 		break;
3125 	}
3126 
3127 	return 0;
3128 }
3129 
3130 /**
3131  * irdma_receive_ilq - recv an ETHERNET packet, and process it
3132  * through CM
3133  * @vsi: VSI structure of dev
3134  * @rbuf: receive buffer
3135  */
3136 void irdma_receive_ilq(struct irdma_sc_vsi *vsi, struct irdma_puda_buf *rbuf)
3137 {
3138 	struct irdma_cm_node *cm_node;
3139 	struct irdma_cm_listener *listener;
3140 	struct iphdr *iph;
3141 	struct ipv6hdr *ip6h;
3142 	struct tcphdr *tcph;
3143 	struct irdma_cm_info cm_info = {};
3144 	struct irdma_device *iwdev = vsi->back_vsi;
3145 	struct irdma_cm_core *cm_core = &iwdev->cm_core;
3146 	struct vlan_ethhdr *ethh;
3147 	u16 vtag;
3148 
3149 	/* if vlan, then maclen = 18 else 14 */
3150 	iph = (struct iphdr *)rbuf->iph;
3151 	print_hex_dump_debug("ILQ: RECEIVE ILQ BUFFER", DUMP_PREFIX_OFFSET,
3152 			     16, 8, rbuf->mem.va, rbuf->totallen, false);
3153 	if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
3154 		if (rbuf->vlan_valid) {
3155 			vtag = rbuf->vlan_id;
3156 			cm_info.user_pri = (vtag & VLAN_PRIO_MASK) >>
3157 					   VLAN_PRIO_SHIFT;
3158 			cm_info.vlan_id = vtag & VLAN_VID_MASK;
3159 		} else {
3160 			cm_info.vlan_id = 0xFFFF;
3161 		}
3162 	} else {
3163 		ethh = rbuf->mem.va;
3164 
3165 		if (ethh->h_vlan_proto == htons(ETH_P_8021Q)) {
3166 			vtag = ntohs(ethh->h_vlan_TCI);
3167 			cm_info.user_pri = (vtag & VLAN_PRIO_MASK) >>
3168 					   VLAN_PRIO_SHIFT;
3169 			cm_info.vlan_id = vtag & VLAN_VID_MASK;
3170 			ibdev_dbg(&cm_core->iwdev->ibdev,
3171 				  "CM: vlan_id=%d\n", cm_info.vlan_id);
3172 		} else {
3173 			cm_info.vlan_id = 0xFFFF;
3174 		}
3175 	}
3176 	tcph = (struct tcphdr *)rbuf->tcph;
3177 
3178 	if (rbuf->ipv4) {
3179 		cm_info.loc_addr[0] = ntohl(iph->daddr);
3180 		cm_info.rem_addr[0] = ntohl(iph->saddr);
3181 		cm_info.ipv4 = true;
3182 		cm_info.tos = iph->tos;
3183 	} else {
3184 		ip6h = (struct ipv6hdr *)rbuf->iph;
3185 		irdma_copy_ip_ntohl(cm_info.loc_addr,
3186 				    ip6h->daddr.in6_u.u6_addr32);
3187 		irdma_copy_ip_ntohl(cm_info.rem_addr,
3188 				    ip6h->saddr.in6_u.u6_addr32);
3189 		cm_info.ipv4 = false;
3190 		cm_info.tos = (ip6h->priority << 4) | (ip6h->flow_lbl[0] >> 4);
3191 	}
3192 	cm_info.loc_port = ntohs(tcph->dest);
3193 	cm_info.rem_port = ntohs(tcph->source);
3194 	cm_node = irdma_find_node(cm_core, cm_info.rem_port, cm_info.rem_addr,
3195 				  cm_info.loc_port, cm_info.loc_addr, cm_info.vlan_id);
3196 
3197 	if (!cm_node) {
3198 		/* Only type of packet accepted are for the
3199 		 * PASSIVE open (syn only)
3200 		 */
3201 		if (!tcph->syn || tcph->ack)
3202 			return;
3203 
3204 		listener = irdma_find_listener(cm_core,
3205 					       cm_info.loc_addr,
3206 					       cm_info.ipv4,
3207 					       cm_info.loc_port,
3208 					       cm_info.vlan_id,
3209 					       IRDMA_CM_LISTENER_ACTIVE_STATE);
3210 		if (!listener) {
3211 			cm_info.cm_id = NULL;
3212 			ibdev_dbg(&cm_core->iwdev->ibdev,
3213 				  "CM: no listener found\n");
3214 			return;
3215 		}
3216 
3217 		cm_info.cm_id = listener->cm_id;
3218 		cm_node = irdma_make_cm_node(cm_core, iwdev, &cm_info,
3219 					     listener);
3220 		if (!cm_node) {
3221 			ibdev_dbg(&cm_core->iwdev->ibdev,
3222 				  "CM: allocate node failed\n");
3223 			refcount_dec(&listener->refcnt);
3224 			return;
3225 		}
3226 
3227 		if (!tcph->rst && !tcph->fin) {
3228 			cm_node->state = IRDMA_CM_STATE_LISTENING;
3229 		} else {
3230 			irdma_rem_ref_cm_node(cm_node);
3231 			return;
3232 		}
3233 
3234 		refcount_inc(&cm_node->refcnt);
3235 	} else if (cm_node->state == IRDMA_CM_STATE_OFFLOADED) {
3236 		irdma_rem_ref_cm_node(cm_node);
3237 		return;
3238 	}
3239 
3240 	irdma_process_pkt(cm_node, rbuf);
3241 	irdma_rem_ref_cm_node(cm_node);
3242 }
3243 
3244 static int irdma_add_qh(struct irdma_cm_node *cm_node, bool active)
3245 {
3246 	if (!active)
3247 		irdma_add_conn_est_qh(cm_node);
3248 	return 0;
3249 }
3250 
3251 static void irdma_cm_free_ah_nop(struct irdma_cm_node *cm_node)
3252 {
3253 }
3254 
3255 /**
3256  * irdma_setup_cm_core - setup top level instance of a cm core
3257  * @iwdev: iwarp device structure
3258  * @rdma_ver: HW version
3259  */
3260 int irdma_setup_cm_core(struct irdma_device *iwdev, u8 rdma_ver)
3261 {
3262 	struct irdma_cm_core *cm_core = &iwdev->cm_core;
3263 
3264 	cm_core->iwdev = iwdev;
3265 	cm_core->dev = &iwdev->rf->sc_dev;
3266 
3267 	/* Handles CM event work items send to Iwarp core */
3268 	cm_core->event_wq = alloc_ordered_workqueue("iwarp-event-wq", 0);
3269 	if (!cm_core->event_wq)
3270 		return -ENOMEM;
3271 
3272 	INIT_LIST_HEAD(&cm_core->listen_list);
3273 
3274 	timer_setup(&cm_core->tcp_timer, irdma_cm_timer_tick, 0);
3275 
3276 	spin_lock_init(&cm_core->ht_lock);
3277 	spin_lock_init(&cm_core->listen_list_lock);
3278 	spin_lock_init(&cm_core->apbvt_lock);
3279 	switch (rdma_ver) {
3280 	case IRDMA_GEN_1:
3281 		cm_core->form_cm_frame = irdma_form_uda_cm_frame;
3282 		cm_core->cm_create_ah = irdma_add_qh;
3283 		cm_core->cm_free_ah = irdma_cm_free_ah_nop;
3284 		break;
3285 	case IRDMA_GEN_2:
3286 	default:
3287 		cm_core->form_cm_frame = irdma_form_ah_cm_frame;
3288 		cm_core->cm_create_ah = irdma_cm_create_ah;
3289 		cm_core->cm_free_ah = irdma_cm_free_ah;
3290 	}
3291 
3292 	return 0;
3293 }
3294 
3295 /**
3296  * irdma_cleanup_cm_core - deallocate a top level instance of a
3297  * cm core
3298  * @cm_core: cm's core
3299  */
3300 void irdma_cleanup_cm_core(struct irdma_cm_core *cm_core)
3301 {
3302 	if (!cm_core)
3303 		return;
3304 
3305 	del_timer_sync(&cm_core->tcp_timer);
3306 
3307 	destroy_workqueue(cm_core->event_wq);
3308 	cm_core->dev->ws_reset(&cm_core->iwdev->vsi);
3309 }
3310 
3311 /**
3312  * irdma_init_tcp_ctx - setup qp context
3313  * @cm_node: connection's node
3314  * @tcp_info: offload info for tcp
3315  * @iwqp: associate qp for the connection
3316  */
3317 static void irdma_init_tcp_ctx(struct irdma_cm_node *cm_node,
3318 			       struct irdma_tcp_offload_info *tcp_info,
3319 			       struct irdma_qp *iwqp)
3320 {
3321 	tcp_info->ipv4 = cm_node->ipv4;
3322 	tcp_info->drop_ooo_seg = !iwqp->iwdev->iw_ooo;
3323 	tcp_info->wscale = true;
3324 	tcp_info->ignore_tcp_opt = true;
3325 	tcp_info->ignore_tcp_uns_opt = true;
3326 	tcp_info->no_nagle = false;
3327 
3328 	tcp_info->ttl = IRDMA_DEFAULT_TTL;
3329 	tcp_info->rtt_var = IRDMA_DEFAULT_RTT_VAR;
3330 	tcp_info->ss_thresh = IRDMA_DEFAULT_SS_THRESH;
3331 	tcp_info->rexmit_thresh = IRDMA_DEFAULT_REXMIT_THRESH;
3332 
3333 	tcp_info->tcp_state = IRDMA_TCP_STATE_ESTABLISHED;
3334 	tcp_info->snd_wscale = cm_node->tcp_cntxt.snd_wscale;
3335 	tcp_info->rcv_wscale = cm_node->tcp_cntxt.rcv_wscale;
3336 
3337 	tcp_info->snd_nxt = cm_node->tcp_cntxt.loc_seq_num;
3338 	tcp_info->snd_wnd = cm_node->tcp_cntxt.snd_wnd;
3339 	tcp_info->rcv_nxt = cm_node->tcp_cntxt.rcv_nxt;
3340 	tcp_info->snd_max = cm_node->tcp_cntxt.loc_seq_num;
3341 
3342 	tcp_info->snd_una = cm_node->tcp_cntxt.loc_seq_num;
3343 	tcp_info->cwnd = 2 * cm_node->tcp_cntxt.mss;
3344 	tcp_info->snd_wl1 = cm_node->tcp_cntxt.rcv_nxt;
3345 	tcp_info->snd_wl2 = cm_node->tcp_cntxt.loc_seq_num;
3346 	tcp_info->max_snd_window = cm_node->tcp_cntxt.max_snd_wnd;
3347 	tcp_info->rcv_wnd = cm_node->tcp_cntxt.rcv_wnd
3348 			    << cm_node->tcp_cntxt.rcv_wscale;
3349 
3350 	tcp_info->flow_label = 0;
3351 	tcp_info->snd_mss = (u32)cm_node->tcp_cntxt.mss;
3352 	tcp_info->tos = cm_node->tos;
3353 	if (cm_node->vlan_id < VLAN_N_VID) {
3354 		tcp_info->insert_vlan_tag = true;
3355 		tcp_info->vlan_tag = cm_node->vlan_id;
3356 		tcp_info->vlan_tag |= cm_node->user_pri << VLAN_PRIO_SHIFT;
3357 	}
3358 	if (cm_node->ipv4) {
3359 		tcp_info->src_port = cm_node->loc_port;
3360 		tcp_info->dst_port = cm_node->rem_port;
3361 
3362 		tcp_info->dest_ip_addr[3] = cm_node->rem_addr[0];
3363 		tcp_info->local_ipaddr[3] = cm_node->loc_addr[0];
3364 		tcp_info->arp_idx = (u16)irdma_arp_table(iwqp->iwdev->rf,
3365 							 &tcp_info->dest_ip_addr[3],
3366 							 true, NULL,
3367 							 IRDMA_ARP_RESOLVE);
3368 	} else {
3369 		tcp_info->src_port = cm_node->loc_port;
3370 		tcp_info->dst_port = cm_node->rem_port;
3371 		memcpy(tcp_info->dest_ip_addr, cm_node->rem_addr,
3372 		       sizeof(tcp_info->dest_ip_addr));
3373 		memcpy(tcp_info->local_ipaddr, cm_node->loc_addr,
3374 		       sizeof(tcp_info->local_ipaddr));
3375 
3376 		tcp_info->arp_idx = (u16)irdma_arp_table(iwqp->iwdev->rf,
3377 							 &tcp_info->dest_ip_addr[0],
3378 							 false, NULL,
3379 							 IRDMA_ARP_RESOLVE);
3380 	}
3381 }
3382 
3383 /**
3384  * irdma_cm_init_tsa_conn - setup qp for RTS
3385  * @iwqp: associate qp for the connection
3386  * @cm_node: connection's node
3387  */
3388 static void irdma_cm_init_tsa_conn(struct irdma_qp *iwqp,
3389 				   struct irdma_cm_node *cm_node)
3390 {
3391 	struct irdma_iwarp_offload_info *iwarp_info;
3392 	struct irdma_qp_host_ctx_info *ctx_info;
3393 
3394 	iwarp_info = &iwqp->iwarp_info;
3395 	ctx_info = &iwqp->ctx_info;
3396 
3397 	ctx_info->tcp_info = &iwqp->tcp_info;
3398 	ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id;
3399 	ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id;
3400 
3401 	iwarp_info->ord_size = cm_node->ord_size;
3402 	iwarp_info->ird_size = cm_node->ird_size;
3403 	iwarp_info->rd_en = true;
3404 	iwarp_info->rdmap_ver = 1;
3405 	iwarp_info->ddp_ver = 1;
3406 	iwarp_info->pd_id = iwqp->iwpd->sc_pd.pd_id;
3407 
3408 	ctx_info->tcp_info_valid = true;
3409 	ctx_info->iwarp_info_valid = true;
3410 	ctx_info->user_pri = cm_node->user_pri;
3411 
3412 	irdma_init_tcp_ctx(cm_node, &iwqp->tcp_info, iwqp);
3413 	if (cm_node->snd_mark_en) {
3414 		iwarp_info->snd_mark_en = true;
3415 		iwarp_info->snd_mark_offset = (iwqp->tcp_info.snd_nxt & SNDMARKER_SEQNMASK) +
3416 					       cm_node->lsmm_size;
3417 	}
3418 
3419 	cm_node->state = IRDMA_CM_STATE_OFFLOADED;
3420 	iwqp->tcp_info.tcp_state = IRDMA_TCP_STATE_ESTABLISHED;
3421 	iwqp->tcp_info.src_mac_addr_idx = iwqp->iwdev->mac_ip_table_idx;
3422 
3423 	if (cm_node->rcv_mark_en) {
3424 		iwarp_info->rcv_mark_en = true;
3425 		iwarp_info->align_hdrs = true;
3426 	}
3427 
3428 	irdma_sc_qp_setctx(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info);
3429 
3430 	/* once tcp_info is set, no need to do it again */
3431 	ctx_info->tcp_info_valid = false;
3432 	ctx_info->iwarp_info_valid = false;
3433 }
3434 
3435 /**
3436  * irdma_cm_disconn - when a connection is being closed
3437  * @iwqp: associated qp for the connection
3438  */
3439 void irdma_cm_disconn(struct irdma_qp *iwqp)
3440 {
3441 	struct irdma_device *iwdev = iwqp->iwdev;
3442 	struct disconn_work *work;
3443 	unsigned long flags;
3444 
3445 	work = kzalloc(sizeof(*work), GFP_ATOMIC);
3446 	if (!work)
3447 		return;
3448 
3449 	spin_lock_irqsave(&iwdev->rf->qptable_lock, flags);
3450 	if (!iwdev->rf->qp_table[iwqp->ibqp.qp_num]) {
3451 		spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags);
3452 		ibdev_dbg(&iwdev->ibdev,
3453 			  "CM: qp_id %d is already freed\n",
3454 			  iwqp->ibqp.qp_num);
3455 		kfree(work);
3456 		return;
3457 	}
3458 	irdma_qp_add_ref(&iwqp->ibqp);
3459 	spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags);
3460 
3461 	work->iwqp = iwqp;
3462 	INIT_WORK(&work->work, irdma_disconnect_worker);
3463 	queue_work(iwdev->cleanup_wq, &work->work);
3464 }
3465 
3466 /**
3467  * irdma_qp_disconnect - free qp and close cm
3468  * @iwqp: associate qp for the connection
3469  */
3470 static void irdma_qp_disconnect(struct irdma_qp *iwqp)
3471 {
3472 	struct irdma_device *iwdev = iwqp->iwdev;
3473 
3474 	iwqp->active_conn = 0;
3475 	/* close the CM node down if it is still active */
3476 	ibdev_dbg(&iwdev->ibdev, "CM: Call close API\n");
3477 	irdma_cm_close(iwqp->cm_node);
3478 }
3479 
3480 /**
3481  * irdma_cm_disconn_true - called by worker thread to disconnect qp
3482  * @iwqp: associate qp for the connection
3483  */
3484 static void irdma_cm_disconn_true(struct irdma_qp *iwqp)
3485 {
3486 	struct iw_cm_id *cm_id;
3487 	struct irdma_device *iwdev;
3488 	struct irdma_sc_qp *qp = &iwqp->sc_qp;
3489 	u16 last_ae;
3490 	u8 original_hw_tcp_state;
3491 	u8 original_ibqp_state;
3492 	int disconn_status = 0;
3493 	int issue_disconn = 0;
3494 	int issue_close = 0;
3495 	int issue_flush = 0;
3496 	unsigned long flags;
3497 	int err;
3498 
3499 	iwdev = iwqp->iwdev;
3500 	spin_lock_irqsave(&iwqp->lock, flags);
3501 	if (rdma_protocol_roce(&iwdev->ibdev, 1)) {
3502 		struct ib_qp_attr attr;
3503 
3504 		if (iwqp->flush_issued || iwqp->sc_qp.qp_uk.destroy_pending) {
3505 			spin_unlock_irqrestore(&iwqp->lock, flags);
3506 			return;
3507 		}
3508 
3509 		spin_unlock_irqrestore(&iwqp->lock, flags);
3510 
3511 		attr.qp_state = IB_QPS_ERR;
3512 		irdma_modify_qp_roce(&iwqp->ibqp, &attr, IB_QP_STATE, NULL);
3513 		irdma_ib_qp_event(iwqp, qp->event_type);
3514 		return;
3515 	}
3516 
3517 	cm_id = iwqp->cm_id;
3518 	original_hw_tcp_state = iwqp->hw_tcp_state;
3519 	original_ibqp_state = iwqp->ibqp_state;
3520 	last_ae = iwqp->last_aeq;
3521 
3522 	if (qp->term_flags) {
3523 		issue_disconn = 1;
3524 		issue_close = 1;
3525 		iwqp->cm_id = NULL;
3526 		irdma_terminate_del_timer(qp);
3527 		if (!iwqp->flush_issued) {
3528 			iwqp->flush_issued = 1;
3529 			issue_flush = 1;
3530 		}
3531 	} else if ((original_hw_tcp_state == IRDMA_TCP_STATE_CLOSE_WAIT) ||
3532 		   ((original_ibqp_state == IB_QPS_RTS) &&
3533 		    (last_ae == IRDMA_AE_LLP_CONNECTION_RESET))) {
3534 		issue_disconn = 1;
3535 		if (last_ae == IRDMA_AE_LLP_CONNECTION_RESET)
3536 			disconn_status = -ECONNRESET;
3537 	}
3538 
3539 	if (original_hw_tcp_state == IRDMA_TCP_STATE_CLOSED ||
3540 	    original_hw_tcp_state == IRDMA_TCP_STATE_TIME_WAIT ||
3541 	    last_ae == IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE ||
3542 	    last_ae == IRDMA_AE_BAD_CLOSE ||
3543 	    last_ae == IRDMA_AE_LLP_CONNECTION_RESET || iwdev->rf->reset || !cm_id) {
3544 		issue_close = 1;
3545 		iwqp->cm_id = NULL;
3546 		qp->term_flags = 0;
3547 		if (!iwqp->flush_issued) {
3548 			iwqp->flush_issued = 1;
3549 			issue_flush = 1;
3550 		}
3551 	}
3552 
3553 	spin_unlock_irqrestore(&iwqp->lock, flags);
3554 	if (issue_flush && !iwqp->sc_qp.qp_uk.destroy_pending) {
3555 		irdma_flush_wqes(iwqp, IRDMA_FLUSH_SQ | IRDMA_FLUSH_RQ |
3556 				 IRDMA_FLUSH_WAIT);
3557 
3558 		if (qp->term_flags)
3559 			irdma_ib_qp_event(iwqp, qp->event_type);
3560 	}
3561 
3562 	if (!cm_id || !cm_id->event_handler)
3563 		return;
3564 
3565 	spin_lock_irqsave(&iwdev->cm_core.ht_lock, flags);
3566 	if (!iwqp->cm_node) {
3567 		spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags);
3568 		return;
3569 	}
3570 	refcount_inc(&iwqp->cm_node->refcnt);
3571 
3572 	spin_unlock_irqrestore(&iwdev->cm_core.ht_lock, flags);
3573 
3574 	if (issue_disconn) {
3575 		err = irdma_send_cm_event(iwqp->cm_node, cm_id,
3576 					  IW_CM_EVENT_DISCONNECT,
3577 					  disconn_status);
3578 		if (err)
3579 			ibdev_dbg(&iwdev->ibdev,
3580 				  "CM: disconnect event failed: - cm_id = %p\n",
3581 				  cm_id);
3582 	}
3583 	if (issue_close) {
3584 		cm_id->provider_data = iwqp;
3585 		err = irdma_send_cm_event(iwqp->cm_node, cm_id,
3586 					  IW_CM_EVENT_CLOSE, 0);
3587 		if (err)
3588 			ibdev_dbg(&iwdev->ibdev,
3589 				  "CM: close event failed: - cm_id = %p\n",
3590 				  cm_id);
3591 		irdma_qp_disconnect(iwqp);
3592 	}
3593 	irdma_rem_ref_cm_node(iwqp->cm_node);
3594 }
3595 
3596 /**
3597  * irdma_disconnect_worker - worker for connection close
3598  * @work: points or disconn structure
3599  */
3600 static void irdma_disconnect_worker(struct work_struct *work)
3601 {
3602 	struct disconn_work *dwork = container_of(work, struct disconn_work, work);
3603 	struct irdma_qp *iwqp = dwork->iwqp;
3604 
3605 	kfree(dwork);
3606 	irdma_cm_disconn_true(iwqp);
3607 	irdma_qp_rem_ref(&iwqp->ibqp);
3608 }
3609 
3610 /**
3611  * irdma_free_lsmm_rsrc - free lsmm memory and deregister
3612  * @iwqp: associate qp for the connection
3613  */
3614 void irdma_free_lsmm_rsrc(struct irdma_qp *iwqp)
3615 {
3616 	struct irdma_device *iwdev;
3617 
3618 	iwdev = iwqp->iwdev;
3619 
3620 	if (iwqp->ietf_mem.va) {
3621 		if (iwqp->lsmm_mr)
3622 			iwdev->ibdev.ops.dereg_mr(iwqp->lsmm_mr, NULL);
3623 		dma_free_coherent(iwdev->rf->sc_dev.hw->device,
3624 				  iwqp->ietf_mem.size, iwqp->ietf_mem.va,
3625 				  iwqp->ietf_mem.pa);
3626 		iwqp->ietf_mem.va = NULL;
3627 	}
3628 }
3629 
3630 /**
3631  * irdma_accept - registered call for connection to be accepted
3632  * @cm_id: cm information for passive connection
3633  * @conn_param: accpet parameters
3634  */
3635 int irdma_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
3636 {
3637 	struct ib_qp *ibqp;
3638 	struct irdma_qp *iwqp;
3639 	struct irdma_device *iwdev;
3640 	struct irdma_sc_dev *dev;
3641 	struct irdma_cm_node *cm_node;
3642 	struct ib_qp_attr attr = {};
3643 	int passive_state;
3644 	struct ib_mr *ibmr;
3645 	struct irdma_pd *iwpd;
3646 	u16 buf_len = 0;
3647 	struct irdma_kmem_info accept;
3648 	u64 tagged_offset;
3649 	int wait_ret;
3650 	int ret = 0;
3651 
3652 	ibqp = irdma_get_qp(cm_id->device, conn_param->qpn);
3653 	if (!ibqp)
3654 		return -EINVAL;
3655 
3656 	iwqp = to_iwqp(ibqp);
3657 	iwdev = iwqp->iwdev;
3658 	dev = &iwdev->rf->sc_dev;
3659 	cm_node = cm_id->provider_data;
3660 
3661 	if (((struct sockaddr_in *)&cm_id->local_addr)->sin_family == AF_INET) {
3662 		cm_node->ipv4 = true;
3663 		cm_node->vlan_id = irdma_get_vlan_ipv4(cm_node->loc_addr);
3664 	} else {
3665 		cm_node->ipv4 = false;
3666 		irdma_get_vlan_mac_ipv6(cm_node->loc_addr, &cm_node->vlan_id,
3667 					NULL);
3668 	}
3669 	ibdev_dbg(&iwdev->ibdev, "CM: Accept vlan_id=%d\n",
3670 		  cm_node->vlan_id);
3671 
3672 	trace_irdma_accept(cm_node, 0, NULL);
3673 
3674 	if (cm_node->state == IRDMA_CM_STATE_LISTENER_DESTROYED) {
3675 		ret = -EINVAL;
3676 		goto error;
3677 	}
3678 
3679 	passive_state = atomic_add_return(1, &cm_node->passive_state);
3680 	if (passive_state == IRDMA_SEND_RESET_EVENT) {
3681 		ret = -ECONNRESET;
3682 		goto error;
3683 	}
3684 
3685 	buf_len = conn_param->private_data_len + IRDMA_MAX_IETF_SIZE;
3686 	iwqp->ietf_mem.size = ALIGN(buf_len, 1);
3687 	iwqp->ietf_mem.va = dma_alloc_coherent(dev->hw->device,
3688 					       iwqp->ietf_mem.size,
3689 					       &iwqp->ietf_mem.pa, GFP_KERNEL);
3690 	if (!iwqp->ietf_mem.va) {
3691 		ret = -ENOMEM;
3692 		goto error;
3693 	}
3694 
3695 	cm_node->pdata.size = conn_param->private_data_len;
3696 	accept.addr = iwqp->ietf_mem.va;
3697 	accept.size = irdma_cm_build_mpa_frame(cm_node, &accept, MPA_KEY_REPLY);
3698 	memcpy((u8 *)accept.addr + accept.size, conn_param->private_data,
3699 	       conn_param->private_data_len);
3700 
3701 	if (cm_node->dev->ws_add(iwqp->sc_qp.vsi, cm_node->user_pri)) {
3702 		ret = -ENOMEM;
3703 		goto error;
3704 	}
3705 	iwqp->sc_qp.user_pri = cm_node->user_pri;
3706 	irdma_qp_add_qos(&iwqp->sc_qp);
3707 	/* setup our first outgoing iWarp send WQE (the IETF frame response) */
3708 	iwpd = iwqp->iwpd;
3709 	tagged_offset = (uintptr_t)iwqp->ietf_mem.va;
3710 	ibmr = irdma_reg_phys_mr(&iwpd->ibpd, iwqp->ietf_mem.pa, buf_len,
3711 				 IB_ACCESS_LOCAL_WRITE, &tagged_offset);
3712 	if (IS_ERR(ibmr)) {
3713 		ret = -ENOMEM;
3714 		goto error;
3715 	}
3716 
3717 	ibmr->pd = &iwpd->ibpd;
3718 	ibmr->device = iwpd->ibpd.device;
3719 	iwqp->lsmm_mr = ibmr;
3720 	if (iwqp->page)
3721 		iwqp->sc_qp.qp_uk.sq_base = kmap_local_page(iwqp->page);
3722 
3723 	cm_node->lsmm_size = accept.size + conn_param->private_data_len;
3724 	irdma_sc_send_lsmm(&iwqp->sc_qp, iwqp->ietf_mem.va, cm_node->lsmm_size,
3725 			   ibmr->lkey);
3726 
3727 	if (iwqp->page)
3728 		kunmap_local(iwqp->sc_qp.qp_uk.sq_base);
3729 
3730 	iwqp->cm_id = cm_id;
3731 	cm_node->cm_id = cm_id;
3732 
3733 	cm_id->provider_data = iwqp;
3734 	iwqp->active_conn = 0;
3735 	iwqp->cm_node = cm_node;
3736 	cm_node->iwqp = iwqp;
3737 	irdma_cm_init_tsa_conn(iwqp, cm_node);
3738 	irdma_qp_add_ref(&iwqp->ibqp);
3739 	cm_id->add_ref(cm_id);
3740 
3741 	attr.qp_state = IB_QPS_RTS;
3742 	cm_node->qhash_set = false;
3743 	cm_node->cm_core->cm_free_ah(cm_node);
3744 
3745 	irdma_modify_qp(&iwqp->ibqp, &attr, IB_QP_STATE, NULL);
3746 	if (dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE) {
3747 		wait_ret = wait_event_interruptible_timeout(iwqp->waitq,
3748 							    iwqp->rts_ae_rcvd,
3749 							    IRDMA_MAX_TIMEOUT);
3750 		if (!wait_ret) {
3751 			ibdev_dbg(&iwdev->ibdev,
3752 				  "CM: Slow Connection: cm_node=%p, loc_port=%d, rem_port=%d, cm_id=%p\n",
3753 				  cm_node, cm_node->loc_port,
3754 				  cm_node->rem_port, cm_node->cm_id);
3755 			ret = -ECONNRESET;
3756 			goto error;
3757 		}
3758 	}
3759 
3760 	irdma_send_cm_event(cm_node, cm_id, IW_CM_EVENT_ESTABLISHED, 0);
3761 	cm_node->accelerated = true;
3762 	complete(&cm_node->establish_comp);
3763 
3764 	if (cm_node->accept_pend) {
3765 		atomic_dec(&cm_node->listener->pend_accepts_cnt);
3766 		cm_node->accept_pend = 0;
3767 	}
3768 
3769 	ibdev_dbg(&iwdev->ibdev,
3770 		  "CM: rem_port=0x%04x, loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4 cm_node=%p cm_id=%p qp_id = %d\n\n",
3771 		  cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr,
3772 		  cm_node->loc_addr, cm_node, cm_id, ibqp->qp_num);
3773 	cm_node->cm_core->stats_accepts++;
3774 
3775 	return 0;
3776 error:
3777 	irdma_free_lsmm_rsrc(iwqp);
3778 	irdma_rem_ref_cm_node(cm_node);
3779 
3780 	return ret;
3781 }
3782 
3783 /**
3784  * irdma_reject - registered call for connection to be rejected
3785  * @cm_id: cm information for passive connection
3786  * @pdata: private data to be sent
3787  * @pdata_len: private data length
3788  */
3789 int irdma_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len)
3790 {
3791 	struct irdma_device *iwdev;
3792 	struct irdma_cm_node *cm_node;
3793 
3794 	cm_node = cm_id->provider_data;
3795 	cm_node->pdata.size = pdata_len;
3796 
3797 	trace_irdma_reject(cm_node, 0, NULL);
3798 
3799 	iwdev = to_iwdev(cm_id->device);
3800 	if (!iwdev)
3801 		return -EINVAL;
3802 
3803 	cm_node->cm_core->stats_rejects++;
3804 
3805 	if (pdata_len + sizeof(struct ietf_mpa_v2) > IRDMA_MAX_CM_BUF)
3806 		return -EINVAL;
3807 
3808 	return irdma_cm_reject(cm_node, pdata, pdata_len);
3809 }
3810 
3811 /**
3812  * irdma_connect - registered call for connection to be established
3813  * @cm_id: cm information for passive connection
3814  * @conn_param: Information about the connection
3815  */
3816 int irdma_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param)
3817 {
3818 	struct ib_qp *ibqp;
3819 	struct irdma_qp *iwqp;
3820 	struct irdma_device *iwdev;
3821 	struct irdma_cm_node *cm_node;
3822 	struct irdma_cm_info cm_info;
3823 	struct sockaddr_in *laddr;
3824 	struct sockaddr_in *raddr;
3825 	struct sockaddr_in6 *laddr6;
3826 	struct sockaddr_in6 *raddr6;
3827 	int ret = 0;
3828 
3829 	ibqp = irdma_get_qp(cm_id->device, conn_param->qpn);
3830 	if (!ibqp)
3831 		return -EINVAL;
3832 	iwqp = to_iwqp(ibqp);
3833 	if (!iwqp)
3834 		return -EINVAL;
3835 	iwdev = iwqp->iwdev;
3836 	if (!iwdev)
3837 		return -EINVAL;
3838 
3839 	laddr = (struct sockaddr_in *)&cm_id->m_local_addr;
3840 	raddr = (struct sockaddr_in *)&cm_id->m_remote_addr;
3841 	laddr6 = (struct sockaddr_in6 *)&cm_id->m_local_addr;
3842 	raddr6 = (struct sockaddr_in6 *)&cm_id->m_remote_addr;
3843 
3844 	if (!(laddr->sin_port) || !(raddr->sin_port))
3845 		return -EINVAL;
3846 
3847 	iwqp->active_conn = 1;
3848 	iwqp->cm_id = NULL;
3849 	cm_id->provider_data = iwqp;
3850 
3851 	/* set up the connection params for the node */
3852 	if (cm_id->remote_addr.ss_family == AF_INET) {
3853 		if (iwdev->vsi.mtu < IRDMA_MIN_MTU_IPV4)
3854 			return -EINVAL;
3855 
3856 		cm_info.ipv4 = true;
3857 		memset(cm_info.loc_addr, 0, sizeof(cm_info.loc_addr));
3858 		memset(cm_info.rem_addr, 0, sizeof(cm_info.rem_addr));
3859 		cm_info.loc_addr[0] = ntohl(laddr->sin_addr.s_addr);
3860 		cm_info.rem_addr[0] = ntohl(raddr->sin_addr.s_addr);
3861 		cm_info.loc_port = ntohs(laddr->sin_port);
3862 		cm_info.rem_port = ntohs(raddr->sin_port);
3863 		cm_info.vlan_id = irdma_get_vlan_ipv4(cm_info.loc_addr);
3864 	} else {
3865 		if (iwdev->vsi.mtu < IRDMA_MIN_MTU_IPV6)
3866 			return -EINVAL;
3867 
3868 		cm_info.ipv4 = false;
3869 		irdma_copy_ip_ntohl(cm_info.loc_addr,
3870 				    laddr6->sin6_addr.in6_u.u6_addr32);
3871 		irdma_copy_ip_ntohl(cm_info.rem_addr,
3872 				    raddr6->sin6_addr.in6_u.u6_addr32);
3873 		cm_info.loc_port = ntohs(laddr6->sin6_port);
3874 		cm_info.rem_port = ntohs(raddr6->sin6_port);
3875 		irdma_get_vlan_mac_ipv6(cm_info.loc_addr, &cm_info.vlan_id,
3876 					NULL);
3877 	}
3878 	cm_info.cm_id = cm_id;
3879 	cm_info.qh_qpid = iwdev->vsi.ilq->qp_id;
3880 	cm_info.tos = cm_id->tos;
3881 	if (iwdev->vsi.dscp_mode) {
3882 		cm_info.user_pri =
3883 			iwqp->sc_qp.vsi->dscp_map[irdma_tos2dscp(cm_info.tos)];
3884 	} else {
3885 		cm_info.user_pri = rt_tos2priority(cm_id->tos);
3886 		cm_info.user_pri = irdma_iw_get_vlan_prio(cm_info.loc_addr,
3887 							  cm_info.user_pri,
3888 							  cm_info.ipv4);
3889 	}
3890 
3891 	if (iwqp->sc_qp.dev->ws_add(iwqp->sc_qp.vsi, cm_info.user_pri))
3892 		return -ENOMEM;
3893 	iwqp->sc_qp.user_pri = cm_info.user_pri;
3894 	irdma_qp_add_qos(&iwqp->sc_qp);
3895 	ibdev_dbg(&iwdev->ibdev, "DCB: TOS:[%d] UP:[%d]\n", cm_id->tos,
3896 		  cm_info.user_pri);
3897 
3898 	trace_irdma_dcb_tos(iwdev, cm_id->tos, cm_info.user_pri);
3899 
3900 	ret = irdma_create_cm_node(&iwdev->cm_core, iwdev, conn_param, &cm_info,
3901 				   &cm_node);
3902 	if (ret)
3903 		return ret;
3904 	ret = cm_node->cm_core->cm_create_ah(cm_node, true);
3905 	if (ret)
3906 		goto err;
3907 	if (irdma_manage_qhash(iwdev, &cm_info,
3908 			       IRDMA_QHASH_TYPE_TCP_ESTABLISHED,
3909 			       IRDMA_QHASH_MANAGE_TYPE_ADD, NULL, true)) {
3910 		ret = -EINVAL;
3911 		goto err;
3912 	}
3913 	cm_node->qhash_set = true;
3914 
3915 	cm_node->apbvt_entry = irdma_add_apbvt(iwdev, cm_info.loc_port);
3916 	if (!cm_node->apbvt_entry) {
3917 		ret = -EINVAL;
3918 		goto err;
3919 	}
3920 
3921 	cm_node->apbvt_set = true;
3922 	iwqp->cm_node = cm_node;
3923 	cm_node->iwqp = iwqp;
3924 	iwqp->cm_id = cm_id;
3925 	irdma_qp_add_ref(&iwqp->ibqp);
3926 	cm_id->add_ref(cm_id);
3927 
3928 	if (cm_node->state != IRDMA_CM_STATE_OFFLOADED) {
3929 		cm_node->state = IRDMA_CM_STATE_SYN_SENT;
3930 		ret = irdma_send_syn(cm_node, 0);
3931 		if (ret)
3932 			goto err;
3933 	}
3934 
3935 	ibdev_dbg(&iwdev->ibdev,
3936 		  "CM: rem_port=0x%04x, loc_port=0x%04x rem_addr=%pI4 loc_addr=%pI4 cm_node=%p cm_id=%p qp_id = %d\n\n",
3937 		  cm_node->rem_port, cm_node->loc_port, cm_node->rem_addr,
3938 		  cm_node->loc_addr, cm_node, cm_id, ibqp->qp_num);
3939 
3940 	trace_irdma_connect(cm_node, 0, NULL);
3941 
3942 	return 0;
3943 
3944 err:
3945 	if (cm_info.ipv4)
3946 		ibdev_dbg(&iwdev->ibdev,
3947 			  "CM: connect() FAILED: dest addr=%pI4",
3948 			  cm_info.rem_addr);
3949 	else
3950 		ibdev_dbg(&iwdev->ibdev,
3951 			  "CM: connect() FAILED: dest addr=%pI6",
3952 			  cm_info.rem_addr);
3953 	irdma_rem_ref_cm_node(cm_node);
3954 	iwdev->cm_core.stats_connect_errs++;
3955 
3956 	return ret;
3957 }
3958 
3959 /**
3960  * irdma_create_listen - registered call creating listener
3961  * @cm_id: cm information for passive connection
3962  * @backlog: to max accept pending count
3963  */
3964 int irdma_create_listen(struct iw_cm_id *cm_id, int backlog)
3965 {
3966 	struct irdma_device *iwdev;
3967 	struct irdma_cm_listener *cm_listen_node;
3968 	struct irdma_cm_info cm_info = {};
3969 	struct sockaddr_in *laddr;
3970 	struct sockaddr_in6 *laddr6;
3971 	bool wildcard = false;
3972 	int err;
3973 
3974 	iwdev = to_iwdev(cm_id->device);
3975 	if (!iwdev)
3976 		return -EINVAL;
3977 
3978 	laddr = (struct sockaddr_in *)&cm_id->m_local_addr;
3979 	laddr6 = (struct sockaddr_in6 *)&cm_id->m_local_addr;
3980 	cm_info.qh_qpid = iwdev->vsi.ilq->qp_id;
3981 
3982 	if (laddr->sin_family == AF_INET) {
3983 		if (iwdev->vsi.mtu < IRDMA_MIN_MTU_IPV4)
3984 			return -EINVAL;
3985 
3986 		cm_info.ipv4 = true;
3987 		cm_info.loc_addr[0] = ntohl(laddr->sin_addr.s_addr);
3988 		cm_info.loc_port = ntohs(laddr->sin_port);
3989 
3990 		if (laddr->sin_addr.s_addr != htonl(INADDR_ANY)) {
3991 			cm_info.vlan_id = irdma_get_vlan_ipv4(cm_info.loc_addr);
3992 		} else {
3993 			cm_info.vlan_id = 0xFFFF;
3994 			wildcard = true;
3995 		}
3996 	} else {
3997 		if (iwdev->vsi.mtu < IRDMA_MIN_MTU_IPV6)
3998 			return -EINVAL;
3999 
4000 		cm_info.ipv4 = false;
4001 		irdma_copy_ip_ntohl(cm_info.loc_addr,
4002 				    laddr6->sin6_addr.in6_u.u6_addr32);
4003 		cm_info.loc_port = ntohs(laddr6->sin6_port);
4004 		if (ipv6_addr_type(&laddr6->sin6_addr) != IPV6_ADDR_ANY) {
4005 			irdma_get_vlan_mac_ipv6(cm_info.loc_addr,
4006 						&cm_info.vlan_id, NULL);
4007 		} else {
4008 			cm_info.vlan_id = 0xFFFF;
4009 			wildcard = true;
4010 		}
4011 	}
4012 
4013 	if (cm_info.vlan_id >= VLAN_N_VID && iwdev->dcb_vlan_mode)
4014 		cm_info.vlan_id = 0;
4015 	cm_info.backlog = backlog;
4016 	cm_info.cm_id = cm_id;
4017 
4018 	trace_irdma_create_listen(iwdev, &cm_info);
4019 
4020 	cm_listen_node = irdma_make_listen_node(&iwdev->cm_core, iwdev,
4021 						&cm_info);
4022 	if (!cm_listen_node) {
4023 		ibdev_dbg(&iwdev->ibdev,
4024 			  "CM: cm_listen_node == NULL\n");
4025 		return -ENOMEM;
4026 	}
4027 
4028 	cm_id->provider_data = cm_listen_node;
4029 
4030 	cm_listen_node->tos = cm_id->tos;
4031 	if (iwdev->vsi.dscp_mode)
4032 		cm_listen_node->user_pri =
4033 		iwdev->vsi.dscp_map[irdma_tos2dscp(cm_id->tos)];
4034 	else
4035 		cm_listen_node->user_pri = rt_tos2priority(cm_id->tos);
4036 	cm_info.user_pri = cm_listen_node->user_pri;
4037 	if (!cm_listen_node->reused_node) {
4038 		if (wildcard) {
4039 			err = irdma_add_mqh(iwdev, &cm_info, cm_listen_node);
4040 			if (err)
4041 				goto error;
4042 		} else {
4043 			if (!iwdev->vsi.dscp_mode)
4044 				cm_listen_node->user_pri =
4045 				irdma_iw_get_vlan_prio(cm_info.loc_addr,
4046 						       cm_info.user_pri,
4047 						       cm_info.ipv4);
4048 			cm_info.user_pri = cm_listen_node->user_pri;
4049 			err = irdma_manage_qhash(iwdev, &cm_info,
4050 						 IRDMA_QHASH_TYPE_TCP_SYN,
4051 						 IRDMA_QHASH_MANAGE_TYPE_ADD,
4052 						 NULL, true);
4053 			if (err)
4054 				goto error;
4055 
4056 			cm_listen_node->qhash_set = true;
4057 		}
4058 
4059 		cm_listen_node->apbvt_entry = irdma_add_apbvt(iwdev,
4060 							      cm_info.loc_port);
4061 		if (!cm_listen_node->apbvt_entry)
4062 			goto error;
4063 	}
4064 	cm_id->add_ref(cm_id);
4065 	cm_listen_node->cm_core->stats_listen_created++;
4066 	ibdev_dbg(&iwdev->ibdev,
4067 		  "CM: loc_port=0x%04x loc_addr=%pI4 cm_listen_node=%p cm_id=%p qhash_set=%d vlan_id=%d\n",
4068 		  cm_listen_node->loc_port, cm_listen_node->loc_addr,
4069 		  cm_listen_node, cm_listen_node->cm_id,
4070 		  cm_listen_node->qhash_set, cm_listen_node->vlan_id);
4071 
4072 	return 0;
4073 
4074 error:
4075 
4076 	irdma_cm_del_listen(&iwdev->cm_core, cm_listen_node, false);
4077 
4078 	return -EINVAL;
4079 }
4080 
4081 /**
4082  * irdma_destroy_listen - registered call to destroy listener
4083  * @cm_id: cm information for passive connection
4084  */
4085 int irdma_destroy_listen(struct iw_cm_id *cm_id)
4086 {
4087 	struct irdma_device *iwdev;
4088 
4089 	iwdev = to_iwdev(cm_id->device);
4090 	if (cm_id->provider_data)
4091 		irdma_cm_del_listen(&iwdev->cm_core, cm_id->provider_data,
4092 				    true);
4093 	else
4094 		ibdev_dbg(&iwdev->ibdev,
4095 			  "CM: cm_id->provider_data was NULL\n");
4096 
4097 	cm_id->rem_ref(cm_id);
4098 
4099 	return 0;
4100 }
4101 
4102 /**
4103  * irdma_teardown_list_prep - add conn nodes slated for tear down to list
4104  * @cm_core: cm's core
4105  * @teardown_list: a list to which cm_node will be selected
4106  * @ipaddr: pointer to ip address
4107  * @nfo: pointer to cm_info structure instance
4108  * @disconnect_all: flag indicating disconnect all QPs
4109  */
4110 static void irdma_teardown_list_prep(struct irdma_cm_core *cm_core,
4111 				     struct list_head *teardown_list,
4112 				     u32 *ipaddr,
4113 				     struct irdma_cm_info *nfo,
4114 				     bool disconnect_all)
4115 {
4116 	struct irdma_cm_node *cm_node;
4117 	int bkt;
4118 
4119 	hash_for_each_rcu(cm_core->cm_hash_tbl, bkt, cm_node, list) {
4120 		if ((disconnect_all ||
4121 		     (nfo->vlan_id == cm_node->vlan_id &&
4122 		      !memcmp(cm_node->loc_addr, ipaddr, nfo->ipv4 ? 4 : 16))) &&
4123 		    refcount_inc_not_zero(&cm_node->refcnt))
4124 			list_add(&cm_node->teardown_entry, teardown_list);
4125 	}
4126 }
4127 
4128 /**
4129  * irdma_cm_event_connected - handle connected active node
4130  * @event: the info for cm_node of connection
4131  */
4132 static void irdma_cm_event_connected(struct irdma_cm_event *event)
4133 {
4134 	struct irdma_qp *iwqp;
4135 	struct irdma_device *iwdev;
4136 	struct irdma_cm_node *cm_node;
4137 	struct irdma_sc_dev *dev;
4138 	struct ib_qp_attr attr = {};
4139 	struct iw_cm_id *cm_id;
4140 	int status;
4141 	bool read0;
4142 	int wait_ret = 0;
4143 
4144 	cm_node = event->cm_node;
4145 	cm_id = cm_node->cm_id;
4146 	iwqp = cm_id->provider_data;
4147 	iwdev = iwqp->iwdev;
4148 	dev = &iwdev->rf->sc_dev;
4149 	if (iwqp->sc_qp.qp_uk.destroy_pending) {
4150 		status = -ETIMEDOUT;
4151 		goto error;
4152 	}
4153 
4154 	irdma_cm_init_tsa_conn(iwqp, cm_node);
4155 	read0 = (cm_node->send_rdma0_op == SEND_RDMA_READ_ZERO);
4156 	if (iwqp->page)
4157 		iwqp->sc_qp.qp_uk.sq_base = kmap_local_page(iwqp->page);
4158 	irdma_sc_send_rtt(&iwqp->sc_qp, read0);
4159 	if (iwqp->page)
4160 		kunmap_local(iwqp->sc_qp.qp_uk.sq_base);
4161 
4162 	attr.qp_state = IB_QPS_RTS;
4163 	cm_node->qhash_set = false;
4164 	irdma_modify_qp(&iwqp->ibqp, &attr, IB_QP_STATE, NULL);
4165 	if (dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE) {
4166 		wait_ret = wait_event_interruptible_timeout(iwqp->waitq,
4167 							    iwqp->rts_ae_rcvd,
4168 							    IRDMA_MAX_TIMEOUT);
4169 		if (!wait_ret)
4170 			ibdev_dbg(&iwdev->ibdev,
4171 				  "CM: Slow Connection: cm_node=%p, loc_port=%d, rem_port=%d, cm_id=%p\n",
4172 				  cm_node, cm_node->loc_port,
4173 				  cm_node->rem_port, cm_node->cm_id);
4174 	}
4175 
4176 	irdma_send_cm_event(cm_node, cm_id, IW_CM_EVENT_CONNECT_REPLY, 0);
4177 	cm_node->accelerated = true;
4178 	complete(&cm_node->establish_comp);
4179 	cm_node->cm_core->cm_free_ah(cm_node);
4180 	return;
4181 
4182 error:
4183 	iwqp->cm_id = NULL;
4184 	cm_id->provider_data = NULL;
4185 	irdma_send_cm_event(event->cm_node, cm_id, IW_CM_EVENT_CONNECT_REPLY,
4186 			    status);
4187 	irdma_rem_ref_cm_node(event->cm_node);
4188 }
4189 
4190 /**
4191  * irdma_cm_event_reset - handle reset
4192  * @event: the info for cm_node of connection
4193  */
4194 static void irdma_cm_event_reset(struct irdma_cm_event *event)
4195 {
4196 	struct irdma_cm_node *cm_node = event->cm_node;
4197 	struct iw_cm_id *cm_id = cm_node->cm_id;
4198 	struct irdma_qp *iwqp;
4199 
4200 	if (!cm_id)
4201 		return;
4202 
4203 	iwqp = cm_id->provider_data;
4204 	if (!iwqp)
4205 		return;
4206 
4207 	ibdev_dbg(&cm_node->iwdev->ibdev,
4208 		  "CM: reset event %p - cm_id = %p\n", event->cm_node, cm_id);
4209 	iwqp->cm_id = NULL;
4210 
4211 	irdma_send_cm_event(cm_node, cm_node->cm_id, IW_CM_EVENT_DISCONNECT,
4212 			    -ECONNRESET);
4213 	irdma_send_cm_event(cm_node, cm_node->cm_id, IW_CM_EVENT_CLOSE, 0);
4214 }
4215 
4216 /**
4217  * irdma_cm_event_handler - send event to cm upper layer
4218  * @work: pointer of cm event info.
4219  */
4220 static void irdma_cm_event_handler(struct work_struct *work)
4221 {
4222 	struct irdma_cm_event *event = container_of(work, struct irdma_cm_event, event_work);
4223 	struct irdma_cm_node *cm_node;
4224 
4225 	if (!event || !event->cm_node || !event->cm_node->cm_core)
4226 		return;
4227 
4228 	cm_node = event->cm_node;
4229 	trace_irdma_cm_event_handler(cm_node, event->type, NULL);
4230 
4231 	switch (event->type) {
4232 	case IRDMA_CM_EVENT_MPA_REQ:
4233 		irdma_send_cm_event(cm_node, cm_node->cm_id,
4234 				    IW_CM_EVENT_CONNECT_REQUEST, 0);
4235 		break;
4236 	case IRDMA_CM_EVENT_RESET:
4237 		irdma_cm_event_reset(event);
4238 		break;
4239 	case IRDMA_CM_EVENT_CONNECTED:
4240 		if (!event->cm_node->cm_id ||
4241 		    event->cm_node->state != IRDMA_CM_STATE_OFFLOADED)
4242 			break;
4243 		irdma_cm_event_connected(event);
4244 		break;
4245 	case IRDMA_CM_EVENT_MPA_REJECT:
4246 		if (!event->cm_node->cm_id ||
4247 		    cm_node->state == IRDMA_CM_STATE_OFFLOADED)
4248 			break;
4249 		irdma_send_cm_event(cm_node, cm_node->cm_id,
4250 				    IW_CM_EVENT_CONNECT_REPLY, -ECONNREFUSED);
4251 		break;
4252 	case IRDMA_CM_EVENT_ABORTED:
4253 		if (!event->cm_node->cm_id ||
4254 		    event->cm_node->state == IRDMA_CM_STATE_OFFLOADED)
4255 			break;
4256 		irdma_event_connect_error(event);
4257 		break;
4258 	default:
4259 		ibdev_dbg(&cm_node->iwdev->ibdev,
4260 			  "CM: bad event type = %d\n", event->type);
4261 		break;
4262 	}
4263 
4264 	irdma_rem_ref_cm_node(event->cm_node);
4265 	kfree(event);
4266 }
4267 
4268 /**
4269  * irdma_cm_post_event - queue event request for worker thread
4270  * @event: cm node's info for up event call
4271  */
4272 static void irdma_cm_post_event(struct irdma_cm_event *event)
4273 {
4274 	refcount_inc(&event->cm_node->refcnt);
4275 	INIT_WORK(&event->event_work, irdma_cm_event_handler);
4276 	queue_work(event->cm_node->cm_core->event_wq, &event->event_work);
4277 }
4278 
4279 /**
4280  * irdma_cm_teardown_connections - teardown QPs
4281  * @iwdev: device pointer
4282  * @ipaddr: Pointer to IPv4 or IPv6 address
4283  * @nfo: Connection info
4284  * @disconnect_all: flag indicating disconnect all QPs
4285  *
4286  * teardown QPs where source or destination addr matches ip addr
4287  */
4288 void irdma_cm_teardown_connections(struct irdma_device *iwdev, u32 *ipaddr,
4289 				   struct irdma_cm_info *nfo,
4290 				   bool disconnect_all)
4291 {
4292 	struct irdma_cm_core *cm_core = &iwdev->cm_core;
4293 	struct list_head *list_core_temp;
4294 	struct list_head *list_node;
4295 	struct irdma_cm_node *cm_node;
4296 	struct list_head teardown_list;
4297 	struct ib_qp_attr attr;
4298 
4299 	INIT_LIST_HEAD(&teardown_list);
4300 
4301 	rcu_read_lock();
4302 	irdma_teardown_list_prep(cm_core, &teardown_list, ipaddr, nfo, disconnect_all);
4303 	rcu_read_unlock();
4304 
4305 	list_for_each_safe (list_node, list_core_temp, &teardown_list) {
4306 		cm_node = container_of(list_node, struct irdma_cm_node,
4307 				       teardown_entry);
4308 		attr.qp_state = IB_QPS_ERR;
4309 		irdma_modify_qp(&cm_node->iwqp->ibqp, &attr, IB_QP_STATE, NULL);
4310 		if (iwdev->rf->reset)
4311 			irdma_cm_disconn(cm_node->iwqp);
4312 		irdma_rem_ref_cm_node(cm_node);
4313 	}
4314 }
4315 
4316 /**
4317  * irdma_qhash_ctrl - enable/disable qhash for list
4318  * @iwdev: device pointer
4319  * @parent_listen_node: parent listen node
4320  * @nfo: cm info node
4321  * @ipaddr: Pointer to IPv4 or IPv6 address
4322  * @ipv4: flag indicating IPv4 when true
4323  * @ifup: flag indicating interface up when true
4324  *
4325  * Enables or disables the qhash for the node in the child
4326  * listen list that matches ipaddr. If no matching IP was found
4327  * it will allocate and add a new child listen node to the
4328  * parent listen node. The listen_list_lock is assumed to be
4329  * held when called.
4330  */
4331 static void irdma_qhash_ctrl(struct irdma_device *iwdev,
4332 			     struct irdma_cm_listener *parent_listen_node,
4333 			     struct irdma_cm_info *nfo, u32 *ipaddr, bool ipv4,
4334 			     bool ifup)
4335 {
4336 	struct list_head *child_listen_list = &parent_listen_node->child_listen_list;
4337 	struct irdma_cm_listener *child_listen_node;
4338 	struct list_head *pos, *tpos;
4339 	bool node_allocated = false;
4340 	enum irdma_quad_hash_manage_type op = ifup ?
4341 					      IRDMA_QHASH_MANAGE_TYPE_ADD :
4342 					      IRDMA_QHASH_MANAGE_TYPE_DELETE;
4343 	int err;
4344 
4345 	list_for_each_safe (pos, tpos, child_listen_list) {
4346 		child_listen_node = list_entry(pos, struct irdma_cm_listener,
4347 					       child_listen_list);
4348 		if (!memcmp(child_listen_node->loc_addr, ipaddr, ipv4 ? 4 : 16))
4349 			goto set_qhash;
4350 	}
4351 
4352 	/* if not found then add a child listener if interface is going up */
4353 	if (!ifup)
4354 		return;
4355 	child_listen_node = kmemdup(parent_listen_node,
4356 				    sizeof(*child_listen_node), GFP_ATOMIC);
4357 	if (!child_listen_node)
4358 		return;
4359 
4360 	node_allocated = true;
4361 	memcpy(child_listen_node->loc_addr, ipaddr, ipv4 ? 4 : 16);
4362 
4363 set_qhash:
4364 	memcpy(nfo->loc_addr, child_listen_node->loc_addr,
4365 	       sizeof(nfo->loc_addr));
4366 	nfo->vlan_id = child_listen_node->vlan_id;
4367 	err = irdma_manage_qhash(iwdev, nfo, IRDMA_QHASH_TYPE_TCP_SYN, op, NULL,
4368 				 false);
4369 	if (!err) {
4370 		child_listen_node->qhash_set = ifup;
4371 		if (node_allocated)
4372 			list_add(&child_listen_node->child_listen_list,
4373 				 &parent_listen_node->child_listen_list);
4374 	} else if (node_allocated) {
4375 		kfree(child_listen_node);
4376 	}
4377 }
4378 
4379 /**
4380  * irdma_if_notify - process an ifdown on an interface
4381  * @iwdev: device pointer
4382  * @netdev: network device structure
4383  * @ipaddr: Pointer to IPv4 or IPv6 address
4384  * @ipv4: flag indicating IPv4 when true
4385  * @ifup: flag indicating interface up when true
4386  */
4387 void irdma_if_notify(struct irdma_device *iwdev, struct net_device *netdev,
4388 		     u32 *ipaddr, bool ipv4, bool ifup)
4389 {
4390 	struct irdma_cm_core *cm_core = &iwdev->cm_core;
4391 	unsigned long flags;
4392 	struct irdma_cm_listener *listen_node;
4393 	static const u32 ip_zero[4] = { 0, 0, 0, 0 };
4394 	struct irdma_cm_info nfo = {};
4395 	u16 vlan_id = rdma_vlan_dev_vlan_id(netdev);
4396 	enum irdma_quad_hash_manage_type op = ifup ?
4397 					      IRDMA_QHASH_MANAGE_TYPE_ADD :
4398 					      IRDMA_QHASH_MANAGE_TYPE_DELETE;
4399 
4400 	nfo.vlan_id = vlan_id;
4401 	nfo.ipv4 = ipv4;
4402 	nfo.qh_qpid = 1;
4403 
4404 	/* Disable or enable qhash for listeners */
4405 	spin_lock_irqsave(&cm_core->listen_list_lock, flags);
4406 	list_for_each_entry (listen_node, &cm_core->listen_list, list) {
4407 		if (vlan_id != listen_node->vlan_id ||
4408 		    (memcmp(listen_node->loc_addr, ipaddr, ipv4 ? 4 : 16) &&
4409 		     memcmp(listen_node->loc_addr, ip_zero, ipv4 ? 4 : 16)))
4410 			continue;
4411 
4412 		memcpy(nfo.loc_addr, listen_node->loc_addr,
4413 		       sizeof(nfo.loc_addr));
4414 		nfo.loc_port = listen_node->loc_port;
4415 		nfo.user_pri = listen_node->user_pri;
4416 		if (!list_empty(&listen_node->child_listen_list)) {
4417 			irdma_qhash_ctrl(iwdev, listen_node, &nfo, ipaddr, ipv4,
4418 					 ifup);
4419 		} else if (memcmp(listen_node->loc_addr, ip_zero,
4420 				  ipv4 ? 4 : 16)) {
4421 			if (!irdma_manage_qhash(iwdev, &nfo,
4422 						IRDMA_QHASH_TYPE_TCP_SYN, op,
4423 						NULL, false))
4424 				listen_node->qhash_set = ifup;
4425 		}
4426 	}
4427 	spin_unlock_irqrestore(&cm_core->listen_list_lock, flags);
4428 
4429 	/* disconnect any connected qp's on ifdown */
4430 	if (!ifup)
4431 		irdma_cm_teardown_connections(iwdev, ipaddr, &nfo, false);
4432 }
4433