xref: /linux/drivers/infiniband/hw/irdma/utils.c (revision 06d07429858317ded2db7986113a9e0129cd599b)
1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2015 - 2021 Intel Corporation */
3 #include "main.h"
4 
5 /**
6  * irdma_arp_table -manage arp table
7  * @rf: RDMA PCI function
8  * @ip_addr: ip address for device
9  * @ipv4: IPv4 flag
10  * @mac_addr: mac address ptr
11  * @action: modify, delete or add
12  */
irdma_arp_table(struct irdma_pci_f * rf,u32 * ip_addr,bool ipv4,const u8 * mac_addr,u32 action)13 int irdma_arp_table(struct irdma_pci_f *rf, u32 *ip_addr, bool ipv4,
14 		    const u8 *mac_addr, u32 action)
15 {
16 	unsigned long flags;
17 	int arp_index;
18 	u32 ip[4] = {};
19 
20 	if (ipv4)
21 		ip[0] = *ip_addr;
22 	else
23 		memcpy(ip, ip_addr, sizeof(ip));
24 
25 	spin_lock_irqsave(&rf->arp_lock, flags);
26 	for (arp_index = 0; (u32)arp_index < rf->arp_table_size; arp_index++) {
27 		if (!memcmp(rf->arp_table[arp_index].ip_addr, ip, sizeof(ip)))
28 			break;
29 	}
30 
31 	switch (action) {
32 	case IRDMA_ARP_ADD:
33 		if (arp_index != rf->arp_table_size) {
34 			arp_index = -1;
35 			break;
36 		}
37 
38 		arp_index = 0;
39 		if (irdma_alloc_rsrc(rf, rf->allocated_arps, rf->arp_table_size,
40 				     (u32 *)&arp_index, &rf->next_arp_index)) {
41 			arp_index = -1;
42 			break;
43 		}
44 
45 		memcpy(rf->arp_table[arp_index].ip_addr, ip,
46 		       sizeof(rf->arp_table[arp_index].ip_addr));
47 		ether_addr_copy(rf->arp_table[arp_index].mac_addr, mac_addr);
48 		break;
49 	case IRDMA_ARP_RESOLVE:
50 		if (arp_index == rf->arp_table_size)
51 			arp_index = -1;
52 		break;
53 	case IRDMA_ARP_DELETE:
54 		if (arp_index == rf->arp_table_size) {
55 			arp_index = -1;
56 			break;
57 		}
58 
59 		memset(rf->arp_table[arp_index].ip_addr, 0,
60 		       sizeof(rf->arp_table[arp_index].ip_addr));
61 		eth_zero_addr(rf->arp_table[arp_index].mac_addr);
62 		irdma_free_rsrc(rf, rf->allocated_arps, arp_index);
63 		break;
64 	default:
65 		arp_index = -1;
66 		break;
67 	}
68 
69 	spin_unlock_irqrestore(&rf->arp_lock, flags);
70 	return arp_index;
71 }
72 
73 /**
74  * irdma_add_arp - add a new arp entry if needed
75  * @rf: RDMA function
76  * @ip: IP address
77  * @ipv4: IPv4 flag
78  * @mac: MAC address
79  */
irdma_add_arp(struct irdma_pci_f * rf,u32 * ip,bool ipv4,const u8 * mac)80 int irdma_add_arp(struct irdma_pci_f *rf, u32 *ip, bool ipv4, const u8 *mac)
81 {
82 	int arpidx;
83 
84 	arpidx = irdma_arp_table(rf, &ip[0], ipv4, NULL, IRDMA_ARP_RESOLVE);
85 	if (arpidx >= 0) {
86 		if (ether_addr_equal(rf->arp_table[arpidx].mac_addr, mac))
87 			return arpidx;
88 
89 		irdma_manage_arp_cache(rf, rf->arp_table[arpidx].mac_addr, ip,
90 				       ipv4, IRDMA_ARP_DELETE);
91 	}
92 
93 	irdma_manage_arp_cache(rf, mac, ip, ipv4, IRDMA_ARP_ADD);
94 
95 	return irdma_arp_table(rf, ip, ipv4, NULL, IRDMA_ARP_RESOLVE);
96 }
97 
98 /**
99  * wr32 - write 32 bits to hw register
100  * @hw: hardware information including registers
101  * @reg: register offset
102  * @val: value to write to register
103  */
wr32(struct irdma_hw * hw,u32 reg,u32 val)104 inline void wr32(struct irdma_hw *hw, u32 reg, u32 val)
105 {
106 	writel(val, hw->hw_addr + reg);
107 }
108 
109 /**
110  * rd32 - read a 32 bit hw register
111  * @hw: hardware information including registers
112  * @reg: register offset
113  *
114  * Return value of register content
115  */
rd32(struct irdma_hw * hw,u32 reg)116 inline u32 rd32(struct irdma_hw *hw, u32 reg)
117 {
118 	return readl(hw->hw_addr + reg);
119 }
120 
121 /**
122  * rd64 - read a 64 bit hw register
123  * @hw: hardware information including registers
124  * @reg: register offset
125  *
126  * Return value of register content
127  */
rd64(struct irdma_hw * hw,u32 reg)128 inline u64 rd64(struct irdma_hw *hw, u32 reg)
129 {
130 	return readq(hw->hw_addr + reg);
131 }
132 
irdma_gid_change_event(struct ib_device * ibdev)133 static void irdma_gid_change_event(struct ib_device *ibdev)
134 {
135 	struct ib_event ib_event;
136 
137 	ib_event.event = IB_EVENT_GID_CHANGE;
138 	ib_event.device = ibdev;
139 	ib_event.element.port_num = 1;
140 	ib_dispatch_event(&ib_event);
141 }
142 
143 /**
144  * irdma_inetaddr_event - system notifier for ipv4 addr events
145  * @notifier: not used
146  * @event: event for notifier
147  * @ptr: if address
148  */
irdma_inetaddr_event(struct notifier_block * notifier,unsigned long event,void * ptr)149 int irdma_inetaddr_event(struct notifier_block *notifier, unsigned long event,
150 			 void *ptr)
151 {
152 	struct in_ifaddr *ifa = ptr;
153 	struct net_device *real_dev, *netdev = ifa->ifa_dev->dev;
154 	struct irdma_device *iwdev;
155 	struct ib_device *ibdev;
156 	u32 local_ipaddr;
157 
158 	real_dev = rdma_vlan_dev_real_dev(netdev);
159 	if (!real_dev)
160 		real_dev = netdev;
161 
162 	ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA);
163 	if (!ibdev)
164 		return NOTIFY_DONE;
165 
166 	iwdev = to_iwdev(ibdev);
167 	local_ipaddr = ntohl(ifa->ifa_address);
168 	ibdev_dbg(&iwdev->ibdev,
169 		  "DEV: netdev %p event %lu local_ip=%pI4 MAC=%pM\n", real_dev,
170 		  event, &local_ipaddr, real_dev->dev_addr);
171 	switch (event) {
172 	case NETDEV_DOWN:
173 		irdma_manage_arp_cache(iwdev->rf, real_dev->dev_addr,
174 				       &local_ipaddr, true, IRDMA_ARP_DELETE);
175 		irdma_if_notify(iwdev, real_dev, &local_ipaddr, true, false);
176 		irdma_gid_change_event(&iwdev->ibdev);
177 		break;
178 	case NETDEV_UP:
179 	case NETDEV_CHANGEADDR:
180 		irdma_add_arp(iwdev->rf, &local_ipaddr, true, real_dev->dev_addr);
181 		irdma_if_notify(iwdev, real_dev, &local_ipaddr, true, true);
182 		irdma_gid_change_event(&iwdev->ibdev);
183 		break;
184 	default:
185 		break;
186 	}
187 
188 	ib_device_put(ibdev);
189 
190 	return NOTIFY_DONE;
191 }
192 
193 /**
194  * irdma_inet6addr_event - system notifier for ipv6 addr events
195  * @notifier: not used
196  * @event: event for notifier
197  * @ptr: if address
198  */
irdma_inet6addr_event(struct notifier_block * notifier,unsigned long event,void * ptr)199 int irdma_inet6addr_event(struct notifier_block *notifier, unsigned long event,
200 			  void *ptr)
201 {
202 	struct inet6_ifaddr *ifa = ptr;
203 	struct net_device *real_dev, *netdev = ifa->idev->dev;
204 	struct irdma_device *iwdev;
205 	struct ib_device *ibdev;
206 	u32 local_ipaddr6[4];
207 
208 	real_dev = rdma_vlan_dev_real_dev(netdev);
209 	if (!real_dev)
210 		real_dev = netdev;
211 
212 	ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA);
213 	if (!ibdev)
214 		return NOTIFY_DONE;
215 
216 	iwdev = to_iwdev(ibdev);
217 	irdma_copy_ip_ntohl(local_ipaddr6, ifa->addr.in6_u.u6_addr32);
218 	ibdev_dbg(&iwdev->ibdev,
219 		  "DEV: netdev %p event %lu local_ip=%pI6 MAC=%pM\n", real_dev,
220 		  event, local_ipaddr6, real_dev->dev_addr);
221 	switch (event) {
222 	case NETDEV_DOWN:
223 		irdma_manage_arp_cache(iwdev->rf, real_dev->dev_addr,
224 				       local_ipaddr6, false, IRDMA_ARP_DELETE);
225 		irdma_if_notify(iwdev, real_dev, local_ipaddr6, false, false);
226 		irdma_gid_change_event(&iwdev->ibdev);
227 		break;
228 	case NETDEV_UP:
229 	case NETDEV_CHANGEADDR:
230 		irdma_add_arp(iwdev->rf, local_ipaddr6, false,
231 			      real_dev->dev_addr);
232 		irdma_if_notify(iwdev, real_dev, local_ipaddr6, false, true);
233 		irdma_gid_change_event(&iwdev->ibdev);
234 		break;
235 	default:
236 		break;
237 	}
238 
239 	ib_device_put(ibdev);
240 
241 	return NOTIFY_DONE;
242 }
243 
244 /**
245  * irdma_net_event - system notifier for net events
246  * @notifier: not used
247  * @event: event for notifier
248  * @ptr: neighbor
249  */
irdma_net_event(struct notifier_block * notifier,unsigned long event,void * ptr)250 int irdma_net_event(struct notifier_block *notifier, unsigned long event,
251 		    void *ptr)
252 {
253 	struct neighbour *neigh = ptr;
254 	struct net_device *real_dev, *netdev = (struct net_device *)neigh->dev;
255 	struct irdma_device *iwdev;
256 	struct ib_device *ibdev;
257 	__be32 *p;
258 	u32 local_ipaddr[4] = {};
259 	bool ipv4 = true;
260 
261 	switch (event) {
262 	case NETEVENT_NEIGH_UPDATE:
263 		real_dev = rdma_vlan_dev_real_dev(netdev);
264 		if (!real_dev)
265 			real_dev = netdev;
266 		ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA);
267 		if (!ibdev)
268 			return NOTIFY_DONE;
269 
270 		iwdev = to_iwdev(ibdev);
271 		p = (__be32 *)neigh->primary_key;
272 		if (neigh->tbl->family == AF_INET6) {
273 			ipv4 = false;
274 			irdma_copy_ip_ntohl(local_ipaddr, p);
275 		} else {
276 			local_ipaddr[0] = ntohl(*p);
277 		}
278 
279 		ibdev_dbg(&iwdev->ibdev,
280 			  "DEV: netdev %p state %d local_ip=%pI4 MAC=%pM\n",
281 			  iwdev->netdev, neigh->nud_state, local_ipaddr,
282 			  neigh->ha);
283 
284 		if (neigh->nud_state & NUD_VALID)
285 			irdma_add_arp(iwdev->rf, local_ipaddr, ipv4, neigh->ha);
286 
287 		else
288 			irdma_manage_arp_cache(iwdev->rf, neigh->ha,
289 					       local_ipaddr, ipv4,
290 					       IRDMA_ARP_DELETE);
291 		ib_device_put(ibdev);
292 		break;
293 	default:
294 		break;
295 	}
296 
297 	return NOTIFY_DONE;
298 }
299 
300 /**
301  * irdma_netdevice_event - system notifier for netdev events
302  * @notifier: not used
303  * @event: event for notifier
304  * @ptr: netdev
305  */
irdma_netdevice_event(struct notifier_block * notifier,unsigned long event,void * ptr)306 int irdma_netdevice_event(struct notifier_block *notifier, unsigned long event,
307 			  void *ptr)
308 {
309 	struct irdma_device *iwdev;
310 	struct ib_device *ibdev;
311 	struct net_device *netdev = netdev_notifier_info_to_dev(ptr);
312 
313 	ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_IRDMA);
314 	if (!ibdev)
315 		return NOTIFY_DONE;
316 
317 	iwdev = to_iwdev(ibdev);
318 	iwdev->iw_status = 1;
319 	switch (event) {
320 	case NETDEV_DOWN:
321 		iwdev->iw_status = 0;
322 		fallthrough;
323 	case NETDEV_UP:
324 		irdma_port_ibevent(iwdev);
325 		break;
326 	default:
327 		break;
328 	}
329 	ib_device_put(ibdev);
330 
331 	return NOTIFY_DONE;
332 }
333 
334 /**
335  * irdma_add_ipv6_addr - add ipv6 address to the hw arp table
336  * @iwdev: irdma device
337  */
irdma_add_ipv6_addr(struct irdma_device * iwdev)338 static void irdma_add_ipv6_addr(struct irdma_device *iwdev)
339 {
340 	struct net_device *ip_dev;
341 	struct inet6_dev *idev;
342 	struct inet6_ifaddr *ifp, *tmp;
343 	u32 local_ipaddr6[4];
344 
345 	rcu_read_lock();
346 	for_each_netdev_rcu (&init_net, ip_dev) {
347 		if (((rdma_vlan_dev_vlan_id(ip_dev) < 0xFFFF &&
348 		      rdma_vlan_dev_real_dev(ip_dev) == iwdev->netdev) ||
349 		      ip_dev == iwdev->netdev) &&
350 		      (READ_ONCE(ip_dev->flags) & IFF_UP)) {
351 			idev = __in6_dev_get(ip_dev);
352 			if (!idev) {
353 				ibdev_err(&iwdev->ibdev, "ipv6 inet device not found\n");
354 				break;
355 			}
356 			list_for_each_entry_safe (ifp, tmp, &idev->addr_list,
357 						  if_list) {
358 				ibdev_dbg(&iwdev->ibdev,
359 					  "INIT: IP=%pI6, vlan_id=%d, MAC=%pM\n",
360 					  &ifp->addr,
361 					  rdma_vlan_dev_vlan_id(ip_dev),
362 					  ip_dev->dev_addr);
363 
364 				irdma_copy_ip_ntohl(local_ipaddr6,
365 						    ifp->addr.in6_u.u6_addr32);
366 				irdma_manage_arp_cache(iwdev->rf,
367 						       ip_dev->dev_addr,
368 						       local_ipaddr6, false,
369 						       IRDMA_ARP_ADD);
370 			}
371 		}
372 	}
373 	rcu_read_unlock();
374 }
375 
376 /**
377  * irdma_add_ipv4_addr - add ipv4 address to the hw arp table
378  * @iwdev: irdma device
379  */
irdma_add_ipv4_addr(struct irdma_device * iwdev)380 static void irdma_add_ipv4_addr(struct irdma_device *iwdev)
381 {
382 	struct net_device *dev;
383 	struct in_device *idev;
384 	u32 ip_addr;
385 
386 	rcu_read_lock();
387 	for_each_netdev_rcu (&init_net, dev) {
388 		if (((rdma_vlan_dev_vlan_id(dev) < 0xFFFF &&
389 		      rdma_vlan_dev_real_dev(dev) == iwdev->netdev) ||
390 		      dev == iwdev->netdev) && (READ_ONCE(dev->flags) & IFF_UP)) {
391 			const struct in_ifaddr *ifa;
392 
393 			idev = __in_dev_get_rcu(dev);
394 			if (!idev)
395 				continue;
396 
397 			in_dev_for_each_ifa_rcu(ifa, idev) {
398 				ibdev_dbg(&iwdev->ibdev, "CM: IP=%pI4, vlan_id=%d, MAC=%pM\n",
399 					  &ifa->ifa_address, rdma_vlan_dev_vlan_id(dev),
400 					  dev->dev_addr);
401 
402 				ip_addr = ntohl(ifa->ifa_address);
403 				irdma_manage_arp_cache(iwdev->rf, dev->dev_addr,
404 						       &ip_addr, true,
405 						       IRDMA_ARP_ADD);
406 			}
407 		}
408 	}
409 	rcu_read_unlock();
410 }
411 
412 /**
413  * irdma_add_ip - add ip addresses
414  * @iwdev: irdma device
415  *
416  * Add ipv4/ipv6 addresses to the arp cache
417  */
irdma_add_ip(struct irdma_device * iwdev)418 void irdma_add_ip(struct irdma_device *iwdev)
419 {
420 	irdma_add_ipv4_addr(iwdev);
421 	irdma_add_ipv6_addr(iwdev);
422 }
423 
424 /**
425  * irdma_alloc_and_get_cqp_request - get cqp struct
426  * @cqp: device cqp ptr
427  * @wait: cqp to be used in wait mode
428  */
irdma_alloc_and_get_cqp_request(struct irdma_cqp * cqp,bool wait)429 struct irdma_cqp_request *irdma_alloc_and_get_cqp_request(struct irdma_cqp *cqp,
430 							  bool wait)
431 {
432 	struct irdma_cqp_request *cqp_request = NULL;
433 	unsigned long flags;
434 
435 	spin_lock_irqsave(&cqp->req_lock, flags);
436 	if (!list_empty(&cqp->cqp_avail_reqs)) {
437 		cqp_request = list_first_entry(&cqp->cqp_avail_reqs,
438 					       struct irdma_cqp_request, list);
439 		list_del_init(&cqp_request->list);
440 	}
441 	spin_unlock_irqrestore(&cqp->req_lock, flags);
442 	if (!cqp_request) {
443 		cqp_request = kzalloc(sizeof(*cqp_request), GFP_ATOMIC);
444 		if (cqp_request) {
445 			cqp_request->dynamic = true;
446 			if (wait)
447 				init_waitqueue_head(&cqp_request->waitq);
448 		}
449 	}
450 	if (!cqp_request) {
451 		ibdev_dbg(to_ibdev(cqp->sc_cqp.dev), "ERR: CQP Request Fail: No Memory");
452 		return NULL;
453 	}
454 
455 	cqp_request->waiting = wait;
456 	refcount_set(&cqp_request->refcnt, 1);
457 	memset(&cqp_request->compl_info, 0, sizeof(cqp_request->compl_info));
458 
459 	return cqp_request;
460 }
461 
462 /**
463  * irdma_get_cqp_request - increase refcount for cqp_request
464  * @cqp_request: pointer to cqp_request instance
465  */
irdma_get_cqp_request(struct irdma_cqp_request * cqp_request)466 static inline void irdma_get_cqp_request(struct irdma_cqp_request *cqp_request)
467 {
468 	refcount_inc(&cqp_request->refcnt);
469 }
470 
471 /**
472  * irdma_free_cqp_request - free cqp request
473  * @cqp: cqp ptr
474  * @cqp_request: to be put back in cqp list
475  */
irdma_free_cqp_request(struct irdma_cqp * cqp,struct irdma_cqp_request * cqp_request)476 void irdma_free_cqp_request(struct irdma_cqp *cqp,
477 			    struct irdma_cqp_request *cqp_request)
478 {
479 	unsigned long flags;
480 
481 	if (cqp_request->dynamic) {
482 		kfree(cqp_request);
483 	} else {
484 		WRITE_ONCE(cqp_request->request_done, false);
485 		cqp_request->callback_fcn = NULL;
486 		cqp_request->waiting = false;
487 
488 		spin_lock_irqsave(&cqp->req_lock, flags);
489 		list_add_tail(&cqp_request->list, &cqp->cqp_avail_reqs);
490 		spin_unlock_irqrestore(&cqp->req_lock, flags);
491 	}
492 	wake_up(&cqp->remove_wq);
493 }
494 
495 /**
496  * irdma_put_cqp_request - dec ref count and free if 0
497  * @cqp: cqp ptr
498  * @cqp_request: to be put back in cqp list
499  */
irdma_put_cqp_request(struct irdma_cqp * cqp,struct irdma_cqp_request * cqp_request)500 void irdma_put_cqp_request(struct irdma_cqp *cqp,
501 			   struct irdma_cqp_request *cqp_request)
502 {
503 	if (refcount_dec_and_test(&cqp_request->refcnt))
504 		irdma_free_cqp_request(cqp, cqp_request);
505 }
506 
507 /**
508  * irdma_free_pending_cqp_request -free pending cqp request objs
509  * @cqp: cqp ptr
510  * @cqp_request: to be put back in cqp list
511  */
512 static void
irdma_free_pending_cqp_request(struct irdma_cqp * cqp,struct irdma_cqp_request * cqp_request)513 irdma_free_pending_cqp_request(struct irdma_cqp *cqp,
514 			       struct irdma_cqp_request *cqp_request)
515 {
516 	if (cqp_request->waiting) {
517 		cqp_request->compl_info.error = true;
518 		WRITE_ONCE(cqp_request->request_done, true);
519 		wake_up(&cqp_request->waitq);
520 	}
521 	wait_event_timeout(cqp->remove_wq,
522 			   refcount_read(&cqp_request->refcnt) == 1, 1000);
523 	irdma_put_cqp_request(cqp, cqp_request);
524 }
525 
526 /**
527  * irdma_cleanup_pending_cqp_op - clean-up cqp with no
528  * completions
529  * @rf: RDMA PCI function
530  */
irdma_cleanup_pending_cqp_op(struct irdma_pci_f * rf)531 void irdma_cleanup_pending_cqp_op(struct irdma_pci_f *rf)
532 {
533 	struct irdma_sc_dev *dev = &rf->sc_dev;
534 	struct irdma_cqp *cqp = &rf->cqp;
535 	struct irdma_cqp_request *cqp_request = NULL;
536 	struct cqp_cmds_info *pcmdinfo = NULL;
537 	u32 i, pending_work, wqe_idx;
538 
539 	pending_work = IRDMA_RING_USED_QUANTA(cqp->sc_cqp.sq_ring);
540 	wqe_idx = IRDMA_RING_CURRENT_TAIL(cqp->sc_cqp.sq_ring);
541 	for (i = 0; i < pending_work; i++) {
542 		cqp_request = (struct irdma_cqp_request *)(unsigned long)
543 				      cqp->scratch_array[wqe_idx];
544 		if (cqp_request)
545 			irdma_free_pending_cqp_request(cqp, cqp_request);
546 		wqe_idx = (wqe_idx + 1) % IRDMA_RING_SIZE(cqp->sc_cqp.sq_ring);
547 	}
548 
549 	while (!list_empty(&dev->cqp_cmd_head)) {
550 		pcmdinfo = irdma_remove_cqp_head(dev);
551 		cqp_request =
552 			container_of(pcmdinfo, struct irdma_cqp_request, info);
553 		if (cqp_request)
554 			irdma_free_pending_cqp_request(cqp, cqp_request);
555 	}
556 }
557 
558 /**
559  * irdma_wait_event - wait for completion
560  * @rf: RDMA PCI function
561  * @cqp_request: cqp request to wait
562  */
irdma_wait_event(struct irdma_pci_f * rf,struct irdma_cqp_request * cqp_request)563 static int irdma_wait_event(struct irdma_pci_f *rf,
564 			    struct irdma_cqp_request *cqp_request)
565 {
566 	struct irdma_cqp_timeout cqp_timeout = {};
567 	bool cqp_error = false;
568 	int err_code = 0;
569 
570 	cqp_timeout.compl_cqp_cmds = atomic64_read(&rf->sc_dev.cqp->completed_ops);
571 	do {
572 		irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq);
573 		if (wait_event_timeout(cqp_request->waitq,
574 				       READ_ONCE(cqp_request->request_done),
575 				       msecs_to_jiffies(CQP_COMPL_WAIT_TIME_MS)))
576 			break;
577 
578 		irdma_check_cqp_progress(&cqp_timeout, &rf->sc_dev);
579 
580 		if (cqp_timeout.count < CQP_TIMEOUT_THRESHOLD)
581 			continue;
582 
583 		if (!rf->reset) {
584 			rf->reset = true;
585 			rf->gen_ops.request_reset(rf);
586 		}
587 		return -ETIMEDOUT;
588 	} while (1);
589 
590 	cqp_error = cqp_request->compl_info.error;
591 	if (cqp_error) {
592 		err_code = -EIO;
593 		if (cqp_request->compl_info.maj_err_code == 0xFFFF) {
594 			if (cqp_request->compl_info.min_err_code == 0x8002)
595 				err_code = -EBUSY;
596 			else if (cqp_request->compl_info.min_err_code == 0x8029) {
597 				if (!rf->reset) {
598 					rf->reset = true;
599 					rf->gen_ops.request_reset(rf);
600 				}
601 			}
602 		}
603 	}
604 
605 	return err_code;
606 }
607 
608 static const char *const irdma_cqp_cmd_names[IRDMA_MAX_CQP_OPS] = {
609 	[IRDMA_OP_CEQ_DESTROY] = "Destroy CEQ Cmd",
610 	[IRDMA_OP_AEQ_DESTROY] = "Destroy AEQ Cmd",
611 	[IRDMA_OP_DELETE_ARP_CACHE_ENTRY] = "Delete ARP Cache Cmd",
612 	[IRDMA_OP_MANAGE_APBVT_ENTRY] = "Manage APBV Table Entry Cmd",
613 	[IRDMA_OP_CEQ_CREATE] = "CEQ Create Cmd",
614 	[IRDMA_OP_AEQ_CREATE] = "AEQ Destroy Cmd",
615 	[IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY] = "Manage Quad Hash Table Entry Cmd",
616 	[IRDMA_OP_QP_MODIFY] = "Modify QP Cmd",
617 	[IRDMA_OP_QP_UPLOAD_CONTEXT] = "Upload Context Cmd",
618 	[IRDMA_OP_CQ_CREATE] = "Create CQ Cmd",
619 	[IRDMA_OP_CQ_DESTROY] = "Destroy CQ Cmd",
620 	[IRDMA_OP_QP_CREATE] = "Create QP Cmd",
621 	[IRDMA_OP_QP_DESTROY] = "Destroy QP Cmd",
622 	[IRDMA_OP_ALLOC_STAG] = "Allocate STag Cmd",
623 	[IRDMA_OP_MR_REG_NON_SHARED] = "Register Non-Shared MR Cmd",
624 	[IRDMA_OP_DEALLOC_STAG] = "Deallocate STag Cmd",
625 	[IRDMA_OP_MW_ALLOC] = "Allocate Memory Window Cmd",
626 	[IRDMA_OP_QP_FLUSH_WQES] = "Flush QP Cmd",
627 	[IRDMA_OP_ADD_ARP_CACHE_ENTRY] = "Add ARP Cache Cmd",
628 	[IRDMA_OP_MANAGE_PUSH_PAGE] = "Manage Push Page Cmd",
629 	[IRDMA_OP_UPDATE_PE_SDS] = "Update PE SDs Cmd",
630 	[IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE] = "Manage HMC PM Function Table Cmd",
631 	[IRDMA_OP_SUSPEND] = "Suspend QP Cmd",
632 	[IRDMA_OP_RESUME] = "Resume QP Cmd",
633 	[IRDMA_OP_MANAGE_VF_PBLE_BP] = "Manage VF PBLE Backing Pages Cmd",
634 	[IRDMA_OP_QUERY_FPM_VAL] = "Query FPM Values Cmd",
635 	[IRDMA_OP_COMMIT_FPM_VAL] = "Commit FPM Values Cmd",
636 	[IRDMA_OP_AH_CREATE] = "Create Address Handle Cmd",
637 	[IRDMA_OP_AH_MODIFY] = "Modify Address Handle Cmd",
638 	[IRDMA_OP_AH_DESTROY] = "Destroy Address Handle Cmd",
639 	[IRDMA_OP_MC_CREATE] = "Create Multicast Group Cmd",
640 	[IRDMA_OP_MC_DESTROY] = "Destroy Multicast Group Cmd",
641 	[IRDMA_OP_MC_MODIFY] = "Modify Multicast Group Cmd",
642 	[IRDMA_OP_STATS_ALLOCATE] = "Add Statistics Instance Cmd",
643 	[IRDMA_OP_STATS_FREE] = "Free Statistics Instance Cmd",
644 	[IRDMA_OP_STATS_GATHER] = "Gather Statistics Cmd",
645 	[IRDMA_OP_WS_ADD_NODE] = "Add Work Scheduler Node Cmd",
646 	[IRDMA_OP_WS_MODIFY_NODE] = "Modify Work Scheduler Node Cmd",
647 	[IRDMA_OP_WS_DELETE_NODE] = "Delete Work Scheduler Node Cmd",
648 	[IRDMA_OP_SET_UP_MAP] = "Set UP-UP Mapping Cmd",
649 	[IRDMA_OP_GEN_AE] = "Generate AE Cmd",
650 	[IRDMA_OP_QUERY_RDMA_FEATURES] = "RDMA Get Features Cmd",
651 	[IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY] = "Allocate Local MAC Entry Cmd",
652 	[IRDMA_OP_ADD_LOCAL_MAC_ENTRY] = "Add Local MAC Entry Cmd",
653 	[IRDMA_OP_DELETE_LOCAL_MAC_ENTRY] = "Delete Local MAC Entry Cmd",
654 	[IRDMA_OP_CQ_MODIFY] = "CQ Modify Cmd",
655 };
656 
657 static const struct irdma_cqp_err_info irdma_noncrit_err_list[] = {
658 	{0xffff, 0x8002, "Invalid State"},
659 	{0xffff, 0x8006, "Flush No Wqe Pending"},
660 	{0xffff, 0x8007, "Modify QP Bad Close"},
661 	{0xffff, 0x8009, "LLP Closed"},
662 	{0xffff, 0x800a, "Reset Not Sent"}
663 };
664 
665 /**
666  * irdma_cqp_crit_err - check if CQP error is critical
667  * @dev: pointer to dev structure
668  * @cqp_cmd: code for last CQP operation
669  * @maj_err_code: major error code
670  * @min_err_code: minot error code
671  */
irdma_cqp_crit_err(struct irdma_sc_dev * dev,u8 cqp_cmd,u16 maj_err_code,u16 min_err_code)672 bool irdma_cqp_crit_err(struct irdma_sc_dev *dev, u8 cqp_cmd,
673 			u16 maj_err_code, u16 min_err_code)
674 {
675 	int i;
676 
677 	for (i = 0; i < ARRAY_SIZE(irdma_noncrit_err_list); ++i) {
678 		if (maj_err_code == irdma_noncrit_err_list[i].maj &&
679 		    min_err_code == irdma_noncrit_err_list[i].min) {
680 			ibdev_dbg(to_ibdev(dev),
681 				  "CQP: [%s Error][%s] maj=0x%x min=0x%x\n",
682 				  irdma_noncrit_err_list[i].desc,
683 				  irdma_cqp_cmd_names[cqp_cmd], maj_err_code,
684 				  min_err_code);
685 			return false;
686 		}
687 	}
688 	return true;
689 }
690 
691 /**
692  * irdma_handle_cqp_op - process cqp command
693  * @rf: RDMA PCI function
694  * @cqp_request: cqp request to process
695  */
irdma_handle_cqp_op(struct irdma_pci_f * rf,struct irdma_cqp_request * cqp_request)696 int irdma_handle_cqp_op(struct irdma_pci_f *rf,
697 			struct irdma_cqp_request *cqp_request)
698 {
699 	struct irdma_sc_dev *dev = &rf->sc_dev;
700 	struct cqp_cmds_info *info = &cqp_request->info;
701 	int status;
702 	bool put_cqp_request = true;
703 
704 	if (rf->reset)
705 		return -EBUSY;
706 
707 	irdma_get_cqp_request(cqp_request);
708 	status = irdma_process_cqp_cmd(dev, info);
709 	if (status)
710 		goto err;
711 
712 	if (cqp_request->waiting) {
713 		put_cqp_request = false;
714 		status = irdma_wait_event(rf, cqp_request);
715 		if (status)
716 			goto err;
717 	}
718 
719 	return 0;
720 
721 err:
722 	if (irdma_cqp_crit_err(dev, info->cqp_cmd,
723 			       cqp_request->compl_info.maj_err_code,
724 			       cqp_request->compl_info.min_err_code))
725 		ibdev_err(&rf->iwdev->ibdev,
726 			  "[%s Error][op_code=%d] status=%d waiting=%d completion_err=%d maj=0x%x min=0x%x\n",
727 			  irdma_cqp_cmd_names[info->cqp_cmd], info->cqp_cmd, status, cqp_request->waiting,
728 			  cqp_request->compl_info.error, cqp_request->compl_info.maj_err_code,
729 			  cqp_request->compl_info.min_err_code);
730 
731 	if (put_cqp_request)
732 		irdma_put_cqp_request(&rf->cqp, cqp_request);
733 
734 	return status;
735 }
736 
irdma_qp_add_ref(struct ib_qp * ibqp)737 void irdma_qp_add_ref(struct ib_qp *ibqp)
738 {
739 	struct irdma_qp *iwqp = (struct irdma_qp *)ibqp;
740 
741 	refcount_inc(&iwqp->refcnt);
742 }
743 
irdma_qp_rem_ref(struct ib_qp * ibqp)744 void irdma_qp_rem_ref(struct ib_qp *ibqp)
745 {
746 	struct irdma_qp *iwqp = to_iwqp(ibqp);
747 	struct irdma_device *iwdev = iwqp->iwdev;
748 	u32 qp_num;
749 	unsigned long flags;
750 
751 	spin_lock_irqsave(&iwdev->rf->qptable_lock, flags);
752 	if (!refcount_dec_and_test(&iwqp->refcnt)) {
753 		spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags);
754 		return;
755 	}
756 
757 	qp_num = iwqp->ibqp.qp_num;
758 	iwdev->rf->qp_table[qp_num] = NULL;
759 	spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags);
760 	complete(&iwqp->free_qp);
761 }
762 
irdma_cq_add_ref(struct ib_cq * ibcq)763 void irdma_cq_add_ref(struct ib_cq *ibcq)
764 {
765 	struct irdma_cq *iwcq = to_iwcq(ibcq);
766 
767 	refcount_inc(&iwcq->refcnt);
768 }
769 
irdma_cq_rem_ref(struct ib_cq * ibcq)770 void irdma_cq_rem_ref(struct ib_cq *ibcq)
771 {
772 	struct ib_device *ibdev = ibcq->device;
773 	struct irdma_device *iwdev = to_iwdev(ibdev);
774 	struct irdma_cq *iwcq = to_iwcq(ibcq);
775 	unsigned long flags;
776 
777 	spin_lock_irqsave(&iwdev->rf->cqtable_lock, flags);
778 	if (!refcount_dec_and_test(&iwcq->refcnt)) {
779 		spin_unlock_irqrestore(&iwdev->rf->cqtable_lock, flags);
780 		return;
781 	}
782 
783 	iwdev->rf->cq_table[iwcq->cq_num] = NULL;
784 	spin_unlock_irqrestore(&iwdev->rf->cqtable_lock, flags);
785 	complete(&iwcq->free_cq);
786 }
787 
to_ibdev(struct irdma_sc_dev * dev)788 struct ib_device *to_ibdev(struct irdma_sc_dev *dev)
789 {
790 	return &(container_of(dev, struct irdma_pci_f, sc_dev))->iwdev->ibdev;
791 }
792 
793 /**
794  * irdma_get_qp - get qp address
795  * @device: iwarp device
796  * @qpn: qp number
797  */
irdma_get_qp(struct ib_device * device,int qpn)798 struct ib_qp *irdma_get_qp(struct ib_device *device, int qpn)
799 {
800 	struct irdma_device *iwdev = to_iwdev(device);
801 
802 	if (qpn < IW_FIRST_QPN || qpn >= iwdev->rf->max_qp)
803 		return NULL;
804 
805 	return &iwdev->rf->qp_table[qpn]->ibqp;
806 }
807 
808 /**
809  * irdma_remove_cqp_head - return head entry and remove
810  * @dev: device
811  */
irdma_remove_cqp_head(struct irdma_sc_dev * dev)812 void *irdma_remove_cqp_head(struct irdma_sc_dev *dev)
813 {
814 	struct list_head *entry;
815 	struct list_head *list = &dev->cqp_cmd_head;
816 
817 	if (list_empty(list))
818 		return NULL;
819 
820 	entry = list->next;
821 	list_del(entry);
822 
823 	return entry;
824 }
825 
826 /**
827  * irdma_cqp_sds_cmd - create cqp command for sd
828  * @dev: hardware control device structure
829  * @sdinfo: information for sd cqp
830  *
831  */
irdma_cqp_sds_cmd(struct irdma_sc_dev * dev,struct irdma_update_sds_info * sdinfo)832 int irdma_cqp_sds_cmd(struct irdma_sc_dev *dev,
833 		      struct irdma_update_sds_info *sdinfo)
834 {
835 	struct irdma_cqp_request *cqp_request;
836 	struct cqp_cmds_info *cqp_info;
837 	struct irdma_pci_f *rf = dev_to_rf(dev);
838 	int status;
839 
840 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
841 	if (!cqp_request)
842 		return -ENOMEM;
843 
844 	cqp_info = &cqp_request->info;
845 	memcpy(&cqp_info->in.u.update_pe_sds.info, sdinfo,
846 	       sizeof(cqp_info->in.u.update_pe_sds.info));
847 	cqp_info->cqp_cmd = IRDMA_OP_UPDATE_PE_SDS;
848 	cqp_info->post_sq = 1;
849 	cqp_info->in.u.update_pe_sds.dev = dev;
850 	cqp_info->in.u.update_pe_sds.scratch = (uintptr_t)cqp_request;
851 
852 	status = irdma_handle_cqp_op(rf, cqp_request);
853 	irdma_put_cqp_request(&rf->cqp, cqp_request);
854 
855 	return status;
856 }
857 
858 /**
859  * irdma_cqp_qp_suspend_resume - cqp command for suspend/resume
860  * @qp: hardware control qp
861  * @op: suspend or resume
862  */
irdma_cqp_qp_suspend_resume(struct irdma_sc_qp * qp,u8 op)863 int irdma_cqp_qp_suspend_resume(struct irdma_sc_qp *qp, u8 op)
864 {
865 	struct irdma_sc_dev *dev = qp->dev;
866 	struct irdma_cqp_request *cqp_request;
867 	struct irdma_sc_cqp *cqp = dev->cqp;
868 	struct cqp_cmds_info *cqp_info;
869 	struct irdma_pci_f *rf = dev_to_rf(dev);
870 	int status;
871 
872 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false);
873 	if (!cqp_request)
874 		return -ENOMEM;
875 
876 	cqp_info = &cqp_request->info;
877 	cqp_info->cqp_cmd = op;
878 	cqp_info->in.u.suspend_resume.cqp = cqp;
879 	cqp_info->in.u.suspend_resume.qp = qp;
880 	cqp_info->in.u.suspend_resume.scratch = (uintptr_t)cqp_request;
881 
882 	status = irdma_handle_cqp_op(rf, cqp_request);
883 	irdma_put_cqp_request(&rf->cqp, cqp_request);
884 
885 	return status;
886 }
887 
888 /**
889  * irdma_term_modify_qp - modify qp for term message
890  * @qp: hardware control qp
891  * @next_state: qp's next state
892  * @term: terminate code
893  * @term_len: length
894  */
irdma_term_modify_qp(struct irdma_sc_qp * qp,u8 next_state,u8 term,u8 term_len)895 void irdma_term_modify_qp(struct irdma_sc_qp *qp, u8 next_state, u8 term,
896 			  u8 term_len)
897 {
898 	struct irdma_qp *iwqp;
899 
900 	iwqp = qp->qp_uk.back_qp;
901 	irdma_next_iw_state(iwqp, next_state, 0, term, term_len);
902 };
903 
904 /**
905  * irdma_terminate_done - after terminate is completed
906  * @qp: hardware control qp
907  * @timeout_occurred: indicates if terminate timer expired
908  */
irdma_terminate_done(struct irdma_sc_qp * qp,int timeout_occurred)909 void irdma_terminate_done(struct irdma_sc_qp *qp, int timeout_occurred)
910 {
911 	struct irdma_qp *iwqp;
912 	u8 hte = 0;
913 	bool first_time;
914 	unsigned long flags;
915 
916 	iwqp = qp->qp_uk.back_qp;
917 	spin_lock_irqsave(&iwqp->lock, flags);
918 	if (iwqp->hte_added) {
919 		iwqp->hte_added = 0;
920 		hte = 1;
921 	}
922 	first_time = !(qp->term_flags & IRDMA_TERM_DONE);
923 	qp->term_flags |= IRDMA_TERM_DONE;
924 	spin_unlock_irqrestore(&iwqp->lock, flags);
925 	if (first_time) {
926 		if (!timeout_occurred)
927 			irdma_terminate_del_timer(qp);
928 
929 		irdma_next_iw_state(iwqp, IRDMA_QP_STATE_ERROR, hte, 0, 0);
930 		irdma_cm_disconn(iwqp);
931 	}
932 }
933 
irdma_terminate_timeout(struct timer_list * t)934 static void irdma_terminate_timeout(struct timer_list *t)
935 {
936 	struct irdma_qp *iwqp = from_timer(iwqp, t, terminate_timer);
937 	struct irdma_sc_qp *qp = &iwqp->sc_qp;
938 
939 	irdma_terminate_done(qp, 1);
940 	irdma_qp_rem_ref(&iwqp->ibqp);
941 }
942 
943 /**
944  * irdma_terminate_start_timer - start terminate timeout
945  * @qp: hardware control qp
946  */
irdma_terminate_start_timer(struct irdma_sc_qp * qp)947 void irdma_terminate_start_timer(struct irdma_sc_qp *qp)
948 {
949 	struct irdma_qp *iwqp;
950 
951 	iwqp = qp->qp_uk.back_qp;
952 	irdma_qp_add_ref(&iwqp->ibqp);
953 	timer_setup(&iwqp->terminate_timer, irdma_terminate_timeout, 0);
954 	iwqp->terminate_timer.expires = jiffies + HZ;
955 
956 	add_timer(&iwqp->terminate_timer);
957 }
958 
959 /**
960  * irdma_terminate_del_timer - delete terminate timeout
961  * @qp: hardware control qp
962  */
irdma_terminate_del_timer(struct irdma_sc_qp * qp)963 void irdma_terminate_del_timer(struct irdma_sc_qp *qp)
964 {
965 	struct irdma_qp *iwqp;
966 	int ret;
967 
968 	iwqp = qp->qp_uk.back_qp;
969 	ret = del_timer(&iwqp->terminate_timer);
970 	if (ret)
971 		irdma_qp_rem_ref(&iwqp->ibqp);
972 }
973 
974 /**
975  * irdma_cqp_query_fpm_val_cmd - send cqp command for fpm
976  * @dev: function device struct
977  * @val_mem: buffer for fpm
978  * @hmc_fn_id: function id for fpm
979  */
irdma_cqp_query_fpm_val_cmd(struct irdma_sc_dev * dev,struct irdma_dma_mem * val_mem,u8 hmc_fn_id)980 int irdma_cqp_query_fpm_val_cmd(struct irdma_sc_dev *dev,
981 				struct irdma_dma_mem *val_mem, u8 hmc_fn_id)
982 {
983 	struct irdma_cqp_request *cqp_request;
984 	struct cqp_cmds_info *cqp_info;
985 	struct irdma_pci_f *rf = dev_to_rf(dev);
986 	int status;
987 
988 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
989 	if (!cqp_request)
990 		return -ENOMEM;
991 
992 	cqp_info = &cqp_request->info;
993 	cqp_request->param = NULL;
994 	cqp_info->in.u.query_fpm_val.cqp = dev->cqp;
995 	cqp_info->in.u.query_fpm_val.fpm_val_pa = val_mem->pa;
996 	cqp_info->in.u.query_fpm_val.fpm_val_va = val_mem->va;
997 	cqp_info->in.u.query_fpm_val.hmc_fn_id = hmc_fn_id;
998 	cqp_info->cqp_cmd = IRDMA_OP_QUERY_FPM_VAL;
999 	cqp_info->post_sq = 1;
1000 	cqp_info->in.u.query_fpm_val.scratch = (uintptr_t)cqp_request;
1001 
1002 	status = irdma_handle_cqp_op(rf, cqp_request);
1003 	irdma_put_cqp_request(&rf->cqp, cqp_request);
1004 
1005 	return status;
1006 }
1007 
1008 /**
1009  * irdma_cqp_commit_fpm_val_cmd - commit fpm values in hw
1010  * @dev: hardware control device structure
1011  * @val_mem: buffer with fpm values
1012  * @hmc_fn_id: function id for fpm
1013  */
irdma_cqp_commit_fpm_val_cmd(struct irdma_sc_dev * dev,struct irdma_dma_mem * val_mem,u8 hmc_fn_id)1014 int irdma_cqp_commit_fpm_val_cmd(struct irdma_sc_dev *dev,
1015 				 struct irdma_dma_mem *val_mem, u8 hmc_fn_id)
1016 {
1017 	struct irdma_cqp_request *cqp_request;
1018 	struct cqp_cmds_info *cqp_info;
1019 	struct irdma_pci_f *rf = dev_to_rf(dev);
1020 	int status;
1021 
1022 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
1023 	if (!cqp_request)
1024 		return -ENOMEM;
1025 
1026 	cqp_info = &cqp_request->info;
1027 	cqp_request->param = NULL;
1028 	cqp_info->in.u.commit_fpm_val.cqp = dev->cqp;
1029 	cqp_info->in.u.commit_fpm_val.fpm_val_pa = val_mem->pa;
1030 	cqp_info->in.u.commit_fpm_val.fpm_val_va = val_mem->va;
1031 	cqp_info->in.u.commit_fpm_val.hmc_fn_id = hmc_fn_id;
1032 	cqp_info->cqp_cmd = IRDMA_OP_COMMIT_FPM_VAL;
1033 	cqp_info->post_sq = 1;
1034 	cqp_info->in.u.commit_fpm_val.scratch = (uintptr_t)cqp_request;
1035 
1036 	status = irdma_handle_cqp_op(rf, cqp_request);
1037 	irdma_put_cqp_request(&rf->cqp, cqp_request);
1038 
1039 	return status;
1040 }
1041 
1042 /**
1043  * irdma_cqp_cq_create_cmd - create a cq for the cqp
1044  * @dev: device pointer
1045  * @cq: pointer to created cq
1046  */
irdma_cqp_cq_create_cmd(struct irdma_sc_dev * dev,struct irdma_sc_cq * cq)1047 int irdma_cqp_cq_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq)
1048 {
1049 	struct irdma_pci_f *rf = dev_to_rf(dev);
1050 	struct irdma_cqp *iwcqp = &rf->cqp;
1051 	struct irdma_cqp_request *cqp_request;
1052 	struct cqp_cmds_info *cqp_info;
1053 	int status;
1054 
1055 	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);
1056 	if (!cqp_request)
1057 		return -ENOMEM;
1058 
1059 	cqp_info = &cqp_request->info;
1060 	cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE;
1061 	cqp_info->post_sq = 1;
1062 	cqp_info->in.u.cq_create.cq = cq;
1063 	cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request;
1064 
1065 	status = irdma_handle_cqp_op(rf, cqp_request);
1066 	irdma_put_cqp_request(iwcqp, cqp_request);
1067 
1068 	return status;
1069 }
1070 
1071 /**
1072  * irdma_cqp_qp_create_cmd - create a qp for the cqp
1073  * @dev: device pointer
1074  * @qp: pointer to created qp
1075  */
irdma_cqp_qp_create_cmd(struct irdma_sc_dev * dev,struct irdma_sc_qp * qp)1076 int irdma_cqp_qp_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
1077 {
1078 	struct irdma_pci_f *rf = dev_to_rf(dev);
1079 	struct irdma_cqp *iwcqp = &rf->cqp;
1080 	struct irdma_cqp_request *cqp_request;
1081 	struct cqp_cmds_info *cqp_info;
1082 	struct irdma_create_qp_info *qp_info;
1083 	int status;
1084 
1085 	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);
1086 	if (!cqp_request)
1087 		return -ENOMEM;
1088 
1089 	cqp_info = &cqp_request->info;
1090 	qp_info = &cqp_request->info.in.u.qp_create.info;
1091 	memset(qp_info, 0, sizeof(*qp_info));
1092 	qp_info->cq_num_valid = true;
1093 	qp_info->next_iwarp_state = IRDMA_QP_STATE_RTS;
1094 	cqp_info->cqp_cmd = IRDMA_OP_QP_CREATE;
1095 	cqp_info->post_sq = 1;
1096 	cqp_info->in.u.qp_create.qp = qp;
1097 	cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request;
1098 
1099 	status = irdma_handle_cqp_op(rf, cqp_request);
1100 	irdma_put_cqp_request(iwcqp, cqp_request);
1101 
1102 	return status;
1103 }
1104 
1105 /**
1106  * irdma_dealloc_push_page - free a push page for qp
1107  * @rf: RDMA PCI function
1108  * @qp: hardware control qp
1109  */
irdma_dealloc_push_page(struct irdma_pci_f * rf,struct irdma_sc_qp * qp)1110 static void irdma_dealloc_push_page(struct irdma_pci_f *rf,
1111 				    struct irdma_sc_qp *qp)
1112 {
1113 	struct irdma_cqp_request *cqp_request;
1114 	struct cqp_cmds_info *cqp_info;
1115 	int status;
1116 
1117 	if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX)
1118 		return;
1119 
1120 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false);
1121 	if (!cqp_request)
1122 		return;
1123 
1124 	cqp_info = &cqp_request->info;
1125 	cqp_info->cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE;
1126 	cqp_info->post_sq = 1;
1127 	cqp_info->in.u.manage_push_page.info.push_idx = qp->push_idx;
1128 	cqp_info->in.u.manage_push_page.info.qs_handle = qp->qs_handle;
1129 	cqp_info->in.u.manage_push_page.info.free_page = 1;
1130 	cqp_info->in.u.manage_push_page.info.push_page_type = 0;
1131 	cqp_info->in.u.manage_push_page.cqp = &rf->cqp.sc_cqp;
1132 	cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request;
1133 	status = irdma_handle_cqp_op(rf, cqp_request);
1134 	if (!status)
1135 		qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX;
1136 	irdma_put_cqp_request(&rf->cqp, cqp_request);
1137 }
1138 
1139 /**
1140  * irdma_free_qp_rsrc - free up memory resources for qp
1141  * @iwqp: qp ptr (user or kernel)
1142  */
irdma_free_qp_rsrc(struct irdma_qp * iwqp)1143 void irdma_free_qp_rsrc(struct irdma_qp *iwqp)
1144 {
1145 	struct irdma_device *iwdev = iwqp->iwdev;
1146 	struct irdma_pci_f *rf = iwdev->rf;
1147 	u32 qp_num = iwqp->ibqp.qp_num;
1148 
1149 	irdma_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp);
1150 	irdma_dealloc_push_page(rf, &iwqp->sc_qp);
1151 	if (iwqp->sc_qp.vsi) {
1152 		irdma_qp_rem_qos(&iwqp->sc_qp);
1153 		iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi,
1154 					   iwqp->sc_qp.user_pri);
1155 	}
1156 
1157 	if (qp_num > 2)
1158 		irdma_free_rsrc(rf, rf->allocated_qps, qp_num);
1159 	dma_free_coherent(rf->sc_dev.hw->device, iwqp->q2_ctx_mem.size,
1160 			  iwqp->q2_ctx_mem.va, iwqp->q2_ctx_mem.pa);
1161 	iwqp->q2_ctx_mem.va = NULL;
1162 	dma_free_coherent(rf->sc_dev.hw->device, iwqp->kqp.dma_mem.size,
1163 			  iwqp->kqp.dma_mem.va, iwqp->kqp.dma_mem.pa);
1164 	iwqp->kqp.dma_mem.va = NULL;
1165 	kfree(iwqp->kqp.sq_wrid_mem);
1166 	kfree(iwqp->kqp.rq_wrid_mem);
1167 }
1168 
1169 /**
1170  * irdma_cq_wq_destroy - send cq destroy cqp
1171  * @rf: RDMA PCI function
1172  * @cq: hardware control cq
1173  */
irdma_cq_wq_destroy(struct irdma_pci_f * rf,struct irdma_sc_cq * cq)1174 void irdma_cq_wq_destroy(struct irdma_pci_f *rf, struct irdma_sc_cq *cq)
1175 {
1176 	struct irdma_cqp_request *cqp_request;
1177 	struct cqp_cmds_info *cqp_info;
1178 
1179 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
1180 	if (!cqp_request)
1181 		return;
1182 
1183 	cqp_info = &cqp_request->info;
1184 	cqp_info->cqp_cmd = IRDMA_OP_CQ_DESTROY;
1185 	cqp_info->post_sq = 1;
1186 	cqp_info->in.u.cq_destroy.cq = cq;
1187 	cqp_info->in.u.cq_destroy.scratch = (uintptr_t)cqp_request;
1188 
1189 	irdma_handle_cqp_op(rf, cqp_request);
1190 	irdma_put_cqp_request(&rf->cqp, cqp_request);
1191 }
1192 
1193 /**
1194  * irdma_hw_modify_qp_callback - handle state for modifyQPs that don't wait
1195  * @cqp_request: modify QP completion
1196  */
irdma_hw_modify_qp_callback(struct irdma_cqp_request * cqp_request)1197 static void irdma_hw_modify_qp_callback(struct irdma_cqp_request *cqp_request)
1198 {
1199 	struct cqp_cmds_info *cqp_info;
1200 	struct irdma_qp *iwqp;
1201 
1202 	cqp_info = &cqp_request->info;
1203 	iwqp = cqp_info->in.u.qp_modify.qp->qp_uk.back_qp;
1204 	atomic_dec(&iwqp->hw_mod_qp_pend);
1205 	wake_up(&iwqp->mod_qp_waitq);
1206 }
1207 
1208 /**
1209  * irdma_hw_modify_qp - setup cqp for modify qp
1210  * @iwdev: RDMA device
1211  * @iwqp: qp ptr (user or kernel)
1212  * @info: info for modify qp
1213  * @wait: flag to wait or not for modify qp completion
1214  */
irdma_hw_modify_qp(struct irdma_device * iwdev,struct irdma_qp * iwqp,struct irdma_modify_qp_info * info,bool wait)1215 int irdma_hw_modify_qp(struct irdma_device *iwdev, struct irdma_qp *iwqp,
1216 		       struct irdma_modify_qp_info *info, bool wait)
1217 {
1218 	int status;
1219 	struct irdma_pci_f *rf = iwdev->rf;
1220 	struct irdma_cqp_request *cqp_request;
1221 	struct cqp_cmds_info *cqp_info;
1222 	struct irdma_modify_qp_info *m_info;
1223 
1224 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait);
1225 	if (!cqp_request)
1226 		return -ENOMEM;
1227 
1228 	if (!wait) {
1229 		cqp_request->callback_fcn = irdma_hw_modify_qp_callback;
1230 		atomic_inc(&iwqp->hw_mod_qp_pend);
1231 	}
1232 	cqp_info = &cqp_request->info;
1233 	m_info = &cqp_info->in.u.qp_modify.info;
1234 	memcpy(m_info, info, sizeof(*m_info));
1235 	cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY;
1236 	cqp_info->post_sq = 1;
1237 	cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp;
1238 	cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request;
1239 	status = irdma_handle_cqp_op(rf, cqp_request);
1240 	irdma_put_cqp_request(&rf->cqp, cqp_request);
1241 	if (status) {
1242 		if (rdma_protocol_roce(&iwdev->ibdev, 1))
1243 			return status;
1244 
1245 		switch (m_info->next_iwarp_state) {
1246 			struct irdma_gen_ae_info ae_info;
1247 
1248 		case IRDMA_QP_STATE_RTS:
1249 		case IRDMA_QP_STATE_IDLE:
1250 		case IRDMA_QP_STATE_TERMINATE:
1251 		case IRDMA_QP_STATE_CLOSING:
1252 			if (info->curr_iwarp_state == IRDMA_QP_STATE_IDLE)
1253 				irdma_send_reset(iwqp->cm_node);
1254 			else
1255 				iwqp->sc_qp.term_flags = IRDMA_TERM_DONE;
1256 			if (!wait) {
1257 				ae_info.ae_code = IRDMA_AE_BAD_CLOSE;
1258 				ae_info.ae_src = 0;
1259 				irdma_gen_ae(rf, &iwqp->sc_qp, &ae_info, false);
1260 			} else {
1261 				cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp,
1262 									      wait);
1263 				if (!cqp_request)
1264 					return -ENOMEM;
1265 
1266 				cqp_info = &cqp_request->info;
1267 				m_info = &cqp_info->in.u.qp_modify.info;
1268 				memcpy(m_info, info, sizeof(*m_info));
1269 				cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY;
1270 				cqp_info->post_sq = 1;
1271 				cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp;
1272 				cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request;
1273 				m_info->next_iwarp_state = IRDMA_QP_STATE_ERROR;
1274 				m_info->reset_tcp_conn = true;
1275 				irdma_handle_cqp_op(rf, cqp_request);
1276 				irdma_put_cqp_request(&rf->cqp, cqp_request);
1277 			}
1278 			break;
1279 		case IRDMA_QP_STATE_ERROR:
1280 		default:
1281 			break;
1282 		}
1283 	}
1284 
1285 	return status;
1286 }
1287 
1288 /**
1289  * irdma_cqp_cq_destroy_cmd - destroy the cqp cq
1290  * @dev: device pointer
1291  * @cq: pointer to cq
1292  */
irdma_cqp_cq_destroy_cmd(struct irdma_sc_dev * dev,struct irdma_sc_cq * cq)1293 void irdma_cqp_cq_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq)
1294 {
1295 	struct irdma_pci_f *rf = dev_to_rf(dev);
1296 
1297 	irdma_cq_wq_destroy(rf, cq);
1298 }
1299 
1300 /**
1301  * irdma_cqp_qp_destroy_cmd - destroy the cqp
1302  * @dev: device pointer
1303  * @qp: pointer to qp
1304  */
irdma_cqp_qp_destroy_cmd(struct irdma_sc_dev * dev,struct irdma_sc_qp * qp)1305 int irdma_cqp_qp_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
1306 {
1307 	struct irdma_pci_f *rf = dev_to_rf(dev);
1308 	struct irdma_cqp *iwcqp = &rf->cqp;
1309 	struct irdma_cqp_request *cqp_request;
1310 	struct cqp_cmds_info *cqp_info;
1311 	int status;
1312 
1313 	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);
1314 	if (!cqp_request)
1315 		return -ENOMEM;
1316 
1317 	cqp_info = &cqp_request->info;
1318 	memset(cqp_info, 0, sizeof(*cqp_info));
1319 	cqp_info->cqp_cmd = IRDMA_OP_QP_DESTROY;
1320 	cqp_info->post_sq = 1;
1321 	cqp_info->in.u.qp_destroy.qp = qp;
1322 	cqp_info->in.u.qp_destroy.scratch = (uintptr_t)cqp_request;
1323 	cqp_info->in.u.qp_destroy.remove_hash_idx = true;
1324 
1325 	status = irdma_handle_cqp_op(rf, cqp_request);
1326 	irdma_put_cqp_request(&rf->cqp, cqp_request);
1327 
1328 	return status;
1329 }
1330 
1331 /**
1332  * irdma_ieq_mpa_crc_ae - generate AE for crc error
1333  * @dev: hardware control device structure
1334  * @qp: hardware control qp
1335  */
irdma_ieq_mpa_crc_ae(struct irdma_sc_dev * dev,struct irdma_sc_qp * qp)1336 void irdma_ieq_mpa_crc_ae(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
1337 {
1338 	struct irdma_gen_ae_info info = {};
1339 	struct irdma_pci_f *rf = dev_to_rf(dev);
1340 
1341 	ibdev_dbg(&rf->iwdev->ibdev, "AEQ: Generate MPA CRC AE\n");
1342 	info.ae_code = IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR;
1343 	info.ae_src = IRDMA_AE_SOURCE_RQ;
1344 	irdma_gen_ae(rf, qp, &info, false);
1345 }
1346 
1347 /**
1348  * irdma_init_hash_desc - initialize hash for crc calculation
1349  * @desc: cryption type
1350  */
irdma_init_hash_desc(struct shash_desc ** desc)1351 int irdma_init_hash_desc(struct shash_desc **desc)
1352 {
1353 	struct crypto_shash *tfm;
1354 	struct shash_desc *tdesc;
1355 
1356 	tfm = crypto_alloc_shash("crc32c", 0, 0);
1357 	if (IS_ERR(tfm))
1358 		return -EINVAL;
1359 
1360 	tdesc = kzalloc(sizeof(*tdesc) + crypto_shash_descsize(tfm),
1361 			GFP_KERNEL);
1362 	if (!tdesc) {
1363 		crypto_free_shash(tfm);
1364 		return -EINVAL;
1365 	}
1366 
1367 	tdesc->tfm = tfm;
1368 	*desc = tdesc;
1369 
1370 	return 0;
1371 }
1372 
1373 /**
1374  * irdma_free_hash_desc - free hash desc
1375  * @desc: to be freed
1376  */
irdma_free_hash_desc(struct shash_desc * desc)1377 void irdma_free_hash_desc(struct shash_desc *desc)
1378 {
1379 	if (desc) {
1380 		crypto_free_shash(desc->tfm);
1381 		kfree(desc);
1382 	}
1383 }
1384 
1385 /**
1386  * irdma_ieq_check_mpacrc - check if mpa crc is OK
1387  * @desc: desc for hash
1388  * @addr: address of buffer for crc
1389  * @len: length of buffer
1390  * @val: value to be compared
1391  */
irdma_ieq_check_mpacrc(struct shash_desc * desc,void * addr,u32 len,u32 val)1392 int irdma_ieq_check_mpacrc(struct shash_desc *desc, void *addr, u32 len,
1393 			   u32 val)
1394 {
1395 	u32 crc = 0;
1396 
1397 	crypto_shash_digest(desc, addr, len, (u8 *)&crc);
1398 	if (crc != val)
1399 		return -EINVAL;
1400 
1401 	return 0;
1402 }
1403 
1404 /**
1405  * irdma_ieq_get_qp - get qp based on quad in puda buffer
1406  * @dev: hardware control device structure
1407  * @buf: receive puda buffer on exception q
1408  */
irdma_ieq_get_qp(struct irdma_sc_dev * dev,struct irdma_puda_buf * buf)1409 struct irdma_sc_qp *irdma_ieq_get_qp(struct irdma_sc_dev *dev,
1410 				     struct irdma_puda_buf *buf)
1411 {
1412 	struct irdma_qp *iwqp;
1413 	struct irdma_cm_node *cm_node;
1414 	struct irdma_device *iwdev = buf->vsi->back_vsi;
1415 	u32 loc_addr[4] = {};
1416 	u32 rem_addr[4] = {};
1417 	u16 loc_port, rem_port;
1418 	struct ipv6hdr *ip6h;
1419 	struct iphdr *iph = (struct iphdr *)buf->iph;
1420 	struct tcphdr *tcph = (struct tcphdr *)buf->tcph;
1421 
1422 	if (iph->version == 4) {
1423 		loc_addr[0] = ntohl(iph->daddr);
1424 		rem_addr[0] = ntohl(iph->saddr);
1425 	} else {
1426 		ip6h = (struct ipv6hdr *)buf->iph;
1427 		irdma_copy_ip_ntohl(loc_addr, ip6h->daddr.in6_u.u6_addr32);
1428 		irdma_copy_ip_ntohl(rem_addr, ip6h->saddr.in6_u.u6_addr32);
1429 	}
1430 	loc_port = ntohs(tcph->dest);
1431 	rem_port = ntohs(tcph->source);
1432 	cm_node = irdma_find_node(&iwdev->cm_core, rem_port, rem_addr, loc_port,
1433 				  loc_addr, buf->vlan_valid ? buf->vlan_id : 0xFFFF);
1434 	if (!cm_node)
1435 		return NULL;
1436 
1437 	iwqp = cm_node->iwqp;
1438 	irdma_rem_ref_cm_node(cm_node);
1439 
1440 	return &iwqp->sc_qp;
1441 }
1442 
1443 /**
1444  * irdma_send_ieq_ack - ACKs for duplicate or OOO partials FPDUs
1445  * @qp: qp ptr
1446  */
irdma_send_ieq_ack(struct irdma_sc_qp * qp)1447 void irdma_send_ieq_ack(struct irdma_sc_qp *qp)
1448 {
1449 	struct irdma_cm_node *cm_node = ((struct irdma_qp *)qp->qp_uk.back_qp)->cm_node;
1450 	struct irdma_puda_buf *buf = qp->pfpdu.lastrcv_buf;
1451 	struct tcphdr *tcph = (struct tcphdr *)buf->tcph;
1452 
1453 	cm_node->tcp_cntxt.rcv_nxt = qp->pfpdu.nextseqnum;
1454 	cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq);
1455 
1456 	irdma_send_ack(cm_node);
1457 }
1458 
1459 /**
1460  * irdma_puda_ieq_get_ah_info - get AH info from IEQ buffer
1461  * @qp: qp pointer
1462  * @ah_info: AH info pointer
1463  */
irdma_puda_ieq_get_ah_info(struct irdma_sc_qp * qp,struct irdma_ah_info * ah_info)1464 void irdma_puda_ieq_get_ah_info(struct irdma_sc_qp *qp,
1465 				struct irdma_ah_info *ah_info)
1466 {
1467 	struct irdma_puda_buf *buf = qp->pfpdu.ah_buf;
1468 	struct iphdr *iph;
1469 	struct ipv6hdr *ip6h;
1470 
1471 	memset(ah_info, 0, sizeof(*ah_info));
1472 	ah_info->do_lpbk = true;
1473 	ah_info->vlan_tag = buf->vlan_id;
1474 	ah_info->insert_vlan_tag = buf->vlan_valid;
1475 	ah_info->ipv4_valid = buf->ipv4;
1476 	ah_info->vsi = qp->vsi;
1477 
1478 	if (buf->smac_valid)
1479 		ether_addr_copy(ah_info->mac_addr, buf->smac);
1480 
1481 	if (buf->ipv4) {
1482 		ah_info->ipv4_valid = true;
1483 		iph = (struct iphdr *)buf->iph;
1484 		ah_info->hop_ttl = iph->ttl;
1485 		ah_info->tc_tos = iph->tos;
1486 		ah_info->dest_ip_addr[0] = ntohl(iph->daddr);
1487 		ah_info->src_ip_addr[0] = ntohl(iph->saddr);
1488 	} else {
1489 		ip6h = (struct ipv6hdr *)buf->iph;
1490 		ah_info->hop_ttl = ip6h->hop_limit;
1491 		ah_info->tc_tos = ip6h->priority;
1492 		irdma_copy_ip_ntohl(ah_info->dest_ip_addr,
1493 				    ip6h->daddr.in6_u.u6_addr32);
1494 		irdma_copy_ip_ntohl(ah_info->src_ip_addr,
1495 				    ip6h->saddr.in6_u.u6_addr32);
1496 	}
1497 
1498 	ah_info->dst_arpindex = irdma_arp_table(dev_to_rf(qp->dev),
1499 						ah_info->dest_ip_addr,
1500 						ah_info->ipv4_valid,
1501 						NULL, IRDMA_ARP_RESOLVE);
1502 }
1503 
1504 /**
1505  * irdma_gen1_ieq_update_tcpip_info - update tcpip in the buffer
1506  * @buf: puda to update
1507  * @len: length of buffer
1508  * @seqnum: seq number for tcp
1509  */
irdma_gen1_ieq_update_tcpip_info(struct irdma_puda_buf * buf,u16 len,u32 seqnum)1510 static void irdma_gen1_ieq_update_tcpip_info(struct irdma_puda_buf *buf,
1511 					     u16 len, u32 seqnum)
1512 {
1513 	struct tcphdr *tcph;
1514 	struct iphdr *iph;
1515 	u16 iphlen;
1516 	u16 pktsize;
1517 	u8 *addr = buf->mem.va;
1518 
1519 	iphlen = (buf->ipv4) ? 20 : 40;
1520 	iph = (struct iphdr *)(addr + buf->maclen);
1521 	tcph = (struct tcphdr *)(addr + buf->maclen + iphlen);
1522 	pktsize = len + buf->tcphlen + iphlen;
1523 	iph->tot_len = htons(pktsize);
1524 	tcph->seq = htonl(seqnum);
1525 }
1526 
1527 /**
1528  * irdma_ieq_update_tcpip_info - update tcpip in the buffer
1529  * @buf: puda to update
1530  * @len: length of buffer
1531  * @seqnum: seq number for tcp
1532  */
irdma_ieq_update_tcpip_info(struct irdma_puda_buf * buf,u16 len,u32 seqnum)1533 void irdma_ieq_update_tcpip_info(struct irdma_puda_buf *buf, u16 len,
1534 				 u32 seqnum)
1535 {
1536 	struct tcphdr *tcph;
1537 	u8 *addr;
1538 
1539 	if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
1540 		return irdma_gen1_ieq_update_tcpip_info(buf, len, seqnum);
1541 
1542 	addr = buf->mem.va;
1543 	tcph = (struct tcphdr *)addr;
1544 	tcph->seq = htonl(seqnum);
1545 }
1546 
1547 /**
1548  * irdma_gen1_puda_get_tcpip_info - get tcpip info from puda
1549  * buffer
1550  * @info: to get information
1551  * @buf: puda buffer
1552  */
irdma_gen1_puda_get_tcpip_info(struct irdma_puda_cmpl_info * info,struct irdma_puda_buf * buf)1553 static int irdma_gen1_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,
1554 					  struct irdma_puda_buf *buf)
1555 {
1556 	struct iphdr *iph;
1557 	struct ipv6hdr *ip6h;
1558 	struct tcphdr *tcph;
1559 	u16 iphlen;
1560 	u16 pkt_len;
1561 	u8 *mem = buf->mem.va;
1562 	struct ethhdr *ethh = buf->mem.va;
1563 
1564 	if (ethh->h_proto == htons(0x8100)) {
1565 		info->vlan_valid = true;
1566 		buf->vlan_id = ntohs(((struct vlan_ethhdr *)ethh)->h_vlan_TCI) &
1567 			       VLAN_VID_MASK;
1568 	}
1569 
1570 	buf->maclen = (info->vlan_valid) ? 18 : 14;
1571 	iphlen = (info->l3proto) ? 40 : 20;
1572 	buf->ipv4 = (info->l3proto) ? false : true;
1573 	buf->iph = mem + buf->maclen;
1574 	iph = (struct iphdr *)buf->iph;
1575 	buf->tcph = buf->iph + iphlen;
1576 	tcph = (struct tcphdr *)buf->tcph;
1577 
1578 	if (buf->ipv4) {
1579 		pkt_len = ntohs(iph->tot_len);
1580 	} else {
1581 		ip6h = (struct ipv6hdr *)buf->iph;
1582 		pkt_len = ntohs(ip6h->payload_len) + iphlen;
1583 	}
1584 
1585 	buf->totallen = pkt_len + buf->maclen;
1586 
1587 	if (info->payload_len < buf->totallen) {
1588 		ibdev_dbg(to_ibdev(buf->vsi->dev),
1589 			  "ERR: payload_len = 0x%x totallen expected0x%x\n",
1590 			  info->payload_len, buf->totallen);
1591 		return -EINVAL;
1592 	}
1593 
1594 	buf->tcphlen = tcph->doff << 2;
1595 	buf->datalen = pkt_len - iphlen - buf->tcphlen;
1596 	buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL;
1597 	buf->hdrlen = buf->maclen + iphlen + buf->tcphlen;
1598 	buf->seqnum = ntohl(tcph->seq);
1599 
1600 	return 0;
1601 }
1602 
1603 /**
1604  * irdma_puda_get_tcpip_info - get tcpip info from puda buffer
1605  * @info: to get information
1606  * @buf: puda buffer
1607  */
irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info * info,struct irdma_puda_buf * buf)1608 int irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,
1609 			      struct irdma_puda_buf *buf)
1610 {
1611 	struct tcphdr *tcph;
1612 	u32 pkt_len;
1613 	u8 *mem;
1614 
1615 	if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
1616 		return irdma_gen1_puda_get_tcpip_info(info, buf);
1617 
1618 	mem = buf->mem.va;
1619 	buf->vlan_valid = info->vlan_valid;
1620 	if (info->vlan_valid)
1621 		buf->vlan_id = info->vlan;
1622 
1623 	buf->ipv4 = info->ipv4;
1624 	if (buf->ipv4)
1625 		buf->iph = mem + IRDMA_IPV4_PAD;
1626 	else
1627 		buf->iph = mem;
1628 
1629 	buf->tcph = mem + IRDMA_TCP_OFFSET;
1630 	tcph = (struct tcphdr *)buf->tcph;
1631 	pkt_len = info->payload_len;
1632 	buf->totallen = pkt_len;
1633 	buf->tcphlen = tcph->doff << 2;
1634 	buf->datalen = pkt_len - IRDMA_TCP_OFFSET - buf->tcphlen;
1635 	buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL;
1636 	buf->hdrlen = IRDMA_TCP_OFFSET + buf->tcphlen;
1637 	buf->seqnum = ntohl(tcph->seq);
1638 
1639 	if (info->smac_valid) {
1640 		ether_addr_copy(buf->smac, info->smac);
1641 		buf->smac_valid = true;
1642 	}
1643 
1644 	return 0;
1645 }
1646 
1647 /**
1648  * irdma_hw_stats_timeout - Stats timer-handler which updates all HW stats
1649  * @t: timer_list pointer
1650  */
irdma_hw_stats_timeout(struct timer_list * t)1651 static void irdma_hw_stats_timeout(struct timer_list *t)
1652 {
1653 	struct irdma_vsi_pestat *pf_devstat =
1654 		from_timer(pf_devstat, t, stats_timer);
1655 	struct irdma_sc_vsi *sc_vsi = pf_devstat->vsi;
1656 
1657 	if (sc_vsi->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
1658 		irdma_cqp_gather_stats_cmd(sc_vsi->dev, sc_vsi->pestat, false);
1659 	else
1660 		irdma_cqp_gather_stats_gen1(sc_vsi->dev, sc_vsi->pestat);
1661 
1662 	mod_timer(&pf_devstat->stats_timer,
1663 		  jiffies + msecs_to_jiffies(STATS_TIMER_DELAY));
1664 }
1665 
1666 /**
1667  * irdma_hw_stats_start_timer - Start periodic stats timer
1668  * @vsi: vsi structure pointer
1669  */
irdma_hw_stats_start_timer(struct irdma_sc_vsi * vsi)1670 void irdma_hw_stats_start_timer(struct irdma_sc_vsi *vsi)
1671 {
1672 	struct irdma_vsi_pestat *devstat = vsi->pestat;
1673 
1674 	timer_setup(&devstat->stats_timer, irdma_hw_stats_timeout, 0);
1675 	mod_timer(&devstat->stats_timer,
1676 		  jiffies + msecs_to_jiffies(STATS_TIMER_DELAY));
1677 }
1678 
1679 /**
1680  * irdma_hw_stats_stop_timer - Delete periodic stats timer
1681  * @vsi: pointer to vsi structure
1682  */
irdma_hw_stats_stop_timer(struct irdma_sc_vsi * vsi)1683 void irdma_hw_stats_stop_timer(struct irdma_sc_vsi *vsi)
1684 {
1685 	struct irdma_vsi_pestat *devstat = vsi->pestat;
1686 
1687 	del_timer_sync(&devstat->stats_timer);
1688 }
1689 
1690 /**
1691  * irdma_process_stats - Checking for wrap and update stats
1692  * @pestat: stats structure pointer
1693  */
irdma_process_stats(struct irdma_vsi_pestat * pestat)1694 static inline void irdma_process_stats(struct irdma_vsi_pestat *pestat)
1695 {
1696 	sc_vsi_update_stats(pestat->vsi);
1697 }
1698 
1699 /**
1700  * irdma_cqp_gather_stats_gen1 - Gather stats
1701  * @dev: pointer to device structure
1702  * @pestat: statistics structure
1703  */
irdma_cqp_gather_stats_gen1(struct irdma_sc_dev * dev,struct irdma_vsi_pestat * pestat)1704 void irdma_cqp_gather_stats_gen1(struct irdma_sc_dev *dev,
1705 				 struct irdma_vsi_pestat *pestat)
1706 {
1707 	struct irdma_gather_stats *gather_stats =
1708 		pestat->gather_info.gather_stats_va;
1709 	const struct irdma_hw_stat_map *map = dev->hw_stats_map;
1710 	u16 max_stats_idx = dev->hw_attrs.max_stat_idx;
1711 	u32 stats_inst_offset_32;
1712 	u32 stats_inst_offset_64;
1713 	u64 new_val;
1714 	u16 i;
1715 
1716 	stats_inst_offset_32 = (pestat->gather_info.use_stats_inst) ?
1717 				pestat->gather_info.stats_inst_index :
1718 				pestat->hw->hmc.hmc_fn_id;
1719 	stats_inst_offset_32 *= 4;
1720 	stats_inst_offset_64 = stats_inst_offset_32 * 2;
1721 
1722 	for (i = 0; i < max_stats_idx; i++) {
1723 		if (map[i].bitmask <= IRDMA_MAX_STATS_32)
1724 			new_val = rd32(dev->hw,
1725 				       dev->hw_stats_regs[i] + stats_inst_offset_32);
1726 		else
1727 			new_val = rd64(dev->hw,
1728 				       dev->hw_stats_regs[i] + stats_inst_offset_64);
1729 		gather_stats->val[map[i].byteoff / sizeof(u64)] = new_val;
1730 	}
1731 
1732 	irdma_process_stats(pestat);
1733 }
1734 
1735 /**
1736  * irdma_process_cqp_stats - Checking for wrap and update stats
1737  * @cqp_request: cqp_request structure pointer
1738  */
irdma_process_cqp_stats(struct irdma_cqp_request * cqp_request)1739 static void irdma_process_cqp_stats(struct irdma_cqp_request *cqp_request)
1740 {
1741 	struct irdma_vsi_pestat *pestat = cqp_request->param;
1742 
1743 	irdma_process_stats(pestat);
1744 }
1745 
1746 /**
1747  * irdma_cqp_gather_stats_cmd - Gather stats
1748  * @dev: pointer to device structure
1749  * @pestat: pointer to stats info
1750  * @wait: flag to wait or not wait for stats
1751  */
irdma_cqp_gather_stats_cmd(struct irdma_sc_dev * dev,struct irdma_vsi_pestat * pestat,bool wait)1752 int irdma_cqp_gather_stats_cmd(struct irdma_sc_dev *dev,
1753 			       struct irdma_vsi_pestat *pestat, bool wait)
1754 
1755 {
1756 	struct irdma_pci_f *rf = dev_to_rf(dev);
1757 	struct irdma_cqp *iwcqp = &rf->cqp;
1758 	struct irdma_cqp_request *cqp_request;
1759 	struct cqp_cmds_info *cqp_info;
1760 	int status;
1761 
1762 	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait);
1763 	if (!cqp_request)
1764 		return -ENOMEM;
1765 
1766 	cqp_info = &cqp_request->info;
1767 	memset(cqp_info, 0, sizeof(*cqp_info));
1768 	cqp_info->cqp_cmd = IRDMA_OP_STATS_GATHER;
1769 	cqp_info->post_sq = 1;
1770 	cqp_info->in.u.stats_gather.info = pestat->gather_info;
1771 	cqp_info->in.u.stats_gather.scratch = (uintptr_t)cqp_request;
1772 	cqp_info->in.u.stats_gather.cqp = &rf->cqp.sc_cqp;
1773 	cqp_request->param = pestat;
1774 	if (!wait)
1775 		cqp_request->callback_fcn = irdma_process_cqp_stats;
1776 	status = irdma_handle_cqp_op(rf, cqp_request);
1777 	if (wait)
1778 		irdma_process_stats(pestat);
1779 	irdma_put_cqp_request(&rf->cqp, cqp_request);
1780 
1781 	return status;
1782 }
1783 
1784 /**
1785  * irdma_cqp_stats_inst_cmd - Allocate/free stats instance
1786  * @vsi: pointer to vsi structure
1787  * @cmd: command to allocate or free
1788  * @stats_info: pointer to allocate stats info
1789  */
irdma_cqp_stats_inst_cmd(struct irdma_sc_vsi * vsi,u8 cmd,struct irdma_stats_inst_info * stats_info)1790 int irdma_cqp_stats_inst_cmd(struct irdma_sc_vsi *vsi, u8 cmd,
1791 			     struct irdma_stats_inst_info *stats_info)
1792 {
1793 	struct irdma_pci_f *rf = dev_to_rf(vsi->dev);
1794 	struct irdma_cqp *iwcqp = &rf->cqp;
1795 	struct irdma_cqp_request *cqp_request;
1796 	struct cqp_cmds_info *cqp_info;
1797 	int status;
1798 	bool wait = false;
1799 
1800 	if (cmd == IRDMA_OP_STATS_ALLOCATE)
1801 		wait = true;
1802 	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait);
1803 	if (!cqp_request)
1804 		return -ENOMEM;
1805 
1806 	cqp_info = &cqp_request->info;
1807 	memset(cqp_info, 0, sizeof(*cqp_info));
1808 	cqp_info->cqp_cmd = cmd;
1809 	cqp_info->post_sq = 1;
1810 	cqp_info->in.u.stats_manage.info = *stats_info;
1811 	cqp_info->in.u.stats_manage.scratch = (uintptr_t)cqp_request;
1812 	cqp_info->in.u.stats_manage.cqp = &rf->cqp.sc_cqp;
1813 	status = irdma_handle_cqp_op(rf, cqp_request);
1814 	if (wait)
1815 		stats_info->stats_idx = cqp_request->compl_info.op_ret_val;
1816 	irdma_put_cqp_request(iwcqp, cqp_request);
1817 
1818 	return status;
1819 }
1820 
1821 /**
1822  * irdma_cqp_ceq_cmd - Create/Destroy CEQ's after CEQ 0
1823  * @dev: pointer to device info
1824  * @sc_ceq: pointer to ceq structure
1825  * @op: Create or Destroy
1826  */
irdma_cqp_ceq_cmd(struct irdma_sc_dev * dev,struct irdma_sc_ceq * sc_ceq,u8 op)1827 int irdma_cqp_ceq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_ceq *sc_ceq,
1828 		      u8 op)
1829 {
1830 	struct irdma_cqp_request *cqp_request;
1831 	struct cqp_cmds_info *cqp_info;
1832 	struct irdma_pci_f *rf = dev_to_rf(dev);
1833 	int status;
1834 
1835 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
1836 	if (!cqp_request)
1837 		return -ENOMEM;
1838 
1839 	cqp_info = &cqp_request->info;
1840 	cqp_info->post_sq = 1;
1841 	cqp_info->cqp_cmd = op;
1842 	cqp_info->in.u.ceq_create.ceq = sc_ceq;
1843 	cqp_info->in.u.ceq_create.scratch = (uintptr_t)cqp_request;
1844 
1845 	status = irdma_handle_cqp_op(rf, cqp_request);
1846 	irdma_put_cqp_request(&rf->cqp, cqp_request);
1847 
1848 	return status;
1849 }
1850 
1851 /**
1852  * irdma_cqp_aeq_cmd - Create/Destroy AEQ
1853  * @dev: pointer to device info
1854  * @sc_aeq: pointer to aeq structure
1855  * @op: Create or Destroy
1856  */
irdma_cqp_aeq_cmd(struct irdma_sc_dev * dev,struct irdma_sc_aeq * sc_aeq,u8 op)1857 int irdma_cqp_aeq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_aeq *sc_aeq,
1858 		      u8 op)
1859 {
1860 	struct irdma_cqp_request *cqp_request;
1861 	struct cqp_cmds_info *cqp_info;
1862 	struct irdma_pci_f *rf = dev_to_rf(dev);
1863 	int status;
1864 
1865 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
1866 	if (!cqp_request)
1867 		return -ENOMEM;
1868 
1869 	cqp_info = &cqp_request->info;
1870 	cqp_info->post_sq = 1;
1871 	cqp_info->cqp_cmd = op;
1872 	cqp_info->in.u.aeq_create.aeq = sc_aeq;
1873 	cqp_info->in.u.aeq_create.scratch = (uintptr_t)cqp_request;
1874 
1875 	status = irdma_handle_cqp_op(rf, cqp_request);
1876 	irdma_put_cqp_request(&rf->cqp, cqp_request);
1877 
1878 	return status;
1879 }
1880 
1881 /**
1882  * irdma_cqp_ws_node_cmd - Add/modify/delete ws node
1883  * @dev: pointer to device structure
1884  * @cmd: Add, modify or delete
1885  * @node_info: pointer to ws node info
1886  */
irdma_cqp_ws_node_cmd(struct irdma_sc_dev * dev,u8 cmd,struct irdma_ws_node_info * node_info)1887 int irdma_cqp_ws_node_cmd(struct irdma_sc_dev *dev, u8 cmd,
1888 			  struct irdma_ws_node_info *node_info)
1889 {
1890 	struct irdma_pci_f *rf = dev_to_rf(dev);
1891 	struct irdma_cqp *iwcqp = &rf->cqp;
1892 	struct irdma_sc_cqp *cqp = &iwcqp->sc_cqp;
1893 	struct irdma_cqp_request *cqp_request;
1894 	struct cqp_cmds_info *cqp_info;
1895 	int status;
1896 	bool poll;
1897 
1898 	if (!rf->sc_dev.ceq_valid)
1899 		poll = true;
1900 	else
1901 		poll = false;
1902 
1903 	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, !poll);
1904 	if (!cqp_request)
1905 		return -ENOMEM;
1906 
1907 	cqp_info = &cqp_request->info;
1908 	memset(cqp_info, 0, sizeof(*cqp_info));
1909 	cqp_info->cqp_cmd = cmd;
1910 	cqp_info->post_sq = 1;
1911 	cqp_info->in.u.ws_node.info = *node_info;
1912 	cqp_info->in.u.ws_node.cqp = cqp;
1913 	cqp_info->in.u.ws_node.scratch = (uintptr_t)cqp_request;
1914 	status = irdma_handle_cqp_op(rf, cqp_request);
1915 	if (status)
1916 		goto exit;
1917 
1918 	if (poll) {
1919 		struct irdma_ccq_cqe_info compl_info;
1920 
1921 		status = irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_WORK_SCHED_NODE,
1922 						       &compl_info);
1923 		node_info->qs_handle = compl_info.op_ret_val;
1924 		ibdev_dbg(&rf->iwdev->ibdev, "DCB: opcode=%d, compl_info.retval=%d\n",
1925 			  compl_info.op_code, compl_info.op_ret_val);
1926 	} else {
1927 		node_info->qs_handle = cqp_request->compl_info.op_ret_val;
1928 	}
1929 
1930 exit:
1931 	irdma_put_cqp_request(&rf->cqp, cqp_request);
1932 
1933 	return status;
1934 }
1935 
1936 /**
1937  * irdma_ah_cqp_op - perform an AH cqp operation
1938  * @rf: RDMA PCI function
1939  * @sc_ah: address handle
1940  * @cmd: AH operation
1941  * @wait: wait if true
1942  * @callback_fcn: Callback function on CQP op completion
1943  * @cb_param: parameter for callback function
1944  *
1945  * returns errno
1946  */
irdma_ah_cqp_op(struct irdma_pci_f * rf,struct irdma_sc_ah * sc_ah,u8 cmd,bool wait,void (* callback_fcn)(struct irdma_cqp_request *),void * cb_param)1947 int irdma_ah_cqp_op(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, u8 cmd,
1948 		    bool wait,
1949 		    void (*callback_fcn)(struct irdma_cqp_request *),
1950 		    void *cb_param)
1951 {
1952 	struct irdma_cqp_request *cqp_request;
1953 	struct cqp_cmds_info *cqp_info;
1954 	int status;
1955 
1956 	if (cmd != IRDMA_OP_AH_CREATE && cmd != IRDMA_OP_AH_DESTROY)
1957 		return -EINVAL;
1958 
1959 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait);
1960 	if (!cqp_request)
1961 		return -ENOMEM;
1962 
1963 	cqp_info = &cqp_request->info;
1964 	cqp_info->cqp_cmd = cmd;
1965 	cqp_info->post_sq = 1;
1966 	if (cmd == IRDMA_OP_AH_CREATE) {
1967 		cqp_info->in.u.ah_create.info = sc_ah->ah_info;
1968 		cqp_info->in.u.ah_create.scratch = (uintptr_t)cqp_request;
1969 		cqp_info->in.u.ah_create.cqp = &rf->cqp.sc_cqp;
1970 	} else if (cmd == IRDMA_OP_AH_DESTROY) {
1971 		cqp_info->in.u.ah_destroy.info = sc_ah->ah_info;
1972 		cqp_info->in.u.ah_destroy.scratch = (uintptr_t)cqp_request;
1973 		cqp_info->in.u.ah_destroy.cqp = &rf->cqp.sc_cqp;
1974 	}
1975 
1976 	if (!wait) {
1977 		cqp_request->callback_fcn = callback_fcn;
1978 		cqp_request->param = cb_param;
1979 	}
1980 	status = irdma_handle_cqp_op(rf, cqp_request);
1981 	irdma_put_cqp_request(&rf->cqp, cqp_request);
1982 
1983 	if (status)
1984 		return -ENOMEM;
1985 
1986 	if (wait)
1987 		sc_ah->ah_info.ah_valid = (cmd == IRDMA_OP_AH_CREATE);
1988 
1989 	return 0;
1990 }
1991 
1992 /**
1993  * irdma_ieq_ah_cb - callback after creation of AH for IEQ
1994  * @cqp_request: pointer to cqp_request of create AH
1995  */
irdma_ieq_ah_cb(struct irdma_cqp_request * cqp_request)1996 static void irdma_ieq_ah_cb(struct irdma_cqp_request *cqp_request)
1997 {
1998 	struct irdma_sc_qp *qp = cqp_request->param;
1999 	struct irdma_sc_ah *sc_ah = qp->pfpdu.ah;
2000 	unsigned long flags;
2001 
2002 	spin_lock_irqsave(&qp->pfpdu.lock, flags);
2003 	if (!cqp_request->compl_info.op_ret_val) {
2004 		sc_ah->ah_info.ah_valid = true;
2005 		irdma_ieq_process_fpdus(qp, qp->vsi->ieq);
2006 	} else {
2007 		sc_ah->ah_info.ah_valid = false;
2008 		irdma_ieq_cleanup_qp(qp->vsi->ieq, qp);
2009 	}
2010 	spin_unlock_irqrestore(&qp->pfpdu.lock, flags);
2011 }
2012 
2013 /**
2014  * irdma_ilq_ah_cb - callback after creation of AH for ILQ
2015  * @cqp_request: pointer to cqp_request of create AH
2016  */
irdma_ilq_ah_cb(struct irdma_cqp_request * cqp_request)2017 static void irdma_ilq_ah_cb(struct irdma_cqp_request *cqp_request)
2018 {
2019 	struct irdma_cm_node *cm_node = cqp_request->param;
2020 	struct irdma_sc_ah *sc_ah = cm_node->ah;
2021 
2022 	sc_ah->ah_info.ah_valid = !cqp_request->compl_info.op_ret_val;
2023 	irdma_add_conn_est_qh(cm_node);
2024 }
2025 
2026 /**
2027  * irdma_puda_create_ah - create AH for ILQ/IEQ qp's
2028  * @dev: device pointer
2029  * @ah_info: Address handle info
2030  * @wait: When true will wait for operation to complete
2031  * @type: ILQ/IEQ
2032  * @cb_param: Callback param when not waiting
2033  * @ah_ret: Returned pointer to address handle if created
2034  *
2035  */
irdma_puda_create_ah(struct irdma_sc_dev * dev,struct irdma_ah_info * ah_info,bool wait,enum puda_rsrc_type type,void * cb_param,struct irdma_sc_ah ** ah_ret)2036 int irdma_puda_create_ah(struct irdma_sc_dev *dev,
2037 			 struct irdma_ah_info *ah_info, bool wait,
2038 			 enum puda_rsrc_type type, void *cb_param,
2039 			 struct irdma_sc_ah **ah_ret)
2040 {
2041 	struct irdma_sc_ah *ah;
2042 	struct irdma_pci_f *rf = dev_to_rf(dev);
2043 	int err;
2044 
2045 	ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
2046 	*ah_ret = ah;
2047 	if (!ah)
2048 		return -ENOMEM;
2049 
2050 	err = irdma_alloc_rsrc(rf, rf->allocated_ahs, rf->max_ah,
2051 			       &ah_info->ah_idx, &rf->next_ah);
2052 	if (err)
2053 		goto err_free;
2054 
2055 	ah->dev = dev;
2056 	ah->ah_info = *ah_info;
2057 
2058 	if (type == IRDMA_PUDA_RSRC_TYPE_ILQ)
2059 		err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait,
2060 				      irdma_ilq_ah_cb, cb_param);
2061 	else
2062 		err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait,
2063 				      irdma_ieq_ah_cb, cb_param);
2064 
2065 	if (err)
2066 		goto error;
2067 	return 0;
2068 
2069 error:
2070 	irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx);
2071 err_free:
2072 	kfree(ah);
2073 	*ah_ret = NULL;
2074 	return -ENOMEM;
2075 }
2076 
2077 /**
2078  * irdma_puda_free_ah - free a puda address handle
2079  * @dev: device pointer
2080  * @ah: The address handle to free
2081  */
irdma_puda_free_ah(struct irdma_sc_dev * dev,struct irdma_sc_ah * ah)2082 void irdma_puda_free_ah(struct irdma_sc_dev *dev, struct irdma_sc_ah *ah)
2083 {
2084 	struct irdma_pci_f *rf = dev_to_rf(dev);
2085 
2086 	if (!ah)
2087 		return;
2088 
2089 	if (ah->ah_info.ah_valid) {
2090 		irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_DESTROY, false, NULL, NULL);
2091 		irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx);
2092 	}
2093 
2094 	kfree(ah);
2095 }
2096 
2097 /**
2098  * irdma_gsi_ud_qp_ah_cb - callback after creation of AH for GSI/ID QP
2099  * @cqp_request: pointer to cqp_request of create AH
2100  */
irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request * cqp_request)2101 void irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request *cqp_request)
2102 {
2103 	struct irdma_sc_ah *sc_ah = cqp_request->param;
2104 
2105 	if (!cqp_request->compl_info.op_ret_val)
2106 		sc_ah->ah_info.ah_valid = true;
2107 	else
2108 		sc_ah->ah_info.ah_valid = false;
2109 }
2110 
2111 /**
2112  * irdma_prm_add_pble_mem - add moemory to pble resources
2113  * @pprm: pble resource manager
2114  * @pchunk: chunk of memory to add
2115  */
irdma_prm_add_pble_mem(struct irdma_pble_prm * pprm,struct irdma_chunk * pchunk)2116 int irdma_prm_add_pble_mem(struct irdma_pble_prm *pprm,
2117 			   struct irdma_chunk *pchunk)
2118 {
2119 	u64 sizeofbitmap;
2120 
2121 	if (pchunk->size & 0xfff)
2122 		return -EINVAL;
2123 
2124 	sizeofbitmap = (u64)pchunk->size >> pprm->pble_shift;
2125 
2126 	pchunk->bitmapbuf = bitmap_zalloc(sizeofbitmap, GFP_KERNEL);
2127 	if (!pchunk->bitmapbuf)
2128 		return -ENOMEM;
2129 
2130 	pchunk->sizeofbitmap = sizeofbitmap;
2131 	/* each pble is 8 bytes hence shift by 3 */
2132 	pprm->total_pble_alloc += pchunk->size >> 3;
2133 	pprm->free_pble_cnt += pchunk->size >> 3;
2134 
2135 	return 0;
2136 }
2137 
2138 /**
2139  * irdma_prm_get_pbles - get pble's from prm
2140  * @pprm: pble resource manager
2141  * @chunkinfo: nformation about chunk where pble's were acquired
2142  * @mem_size: size of pble memory needed
2143  * @vaddr: returns virtual address of pble memory
2144  * @fpm_addr: returns fpm address of pble memory
2145  */
irdma_prm_get_pbles(struct irdma_pble_prm * pprm,struct irdma_pble_chunkinfo * chunkinfo,u64 mem_size,u64 ** vaddr,u64 * fpm_addr)2146 int irdma_prm_get_pbles(struct irdma_pble_prm *pprm,
2147 			struct irdma_pble_chunkinfo *chunkinfo, u64 mem_size,
2148 			u64 **vaddr, u64 *fpm_addr)
2149 {
2150 	u64 bits_needed;
2151 	u64 bit_idx = PBLE_INVALID_IDX;
2152 	struct irdma_chunk *pchunk = NULL;
2153 	struct list_head *chunk_entry = pprm->clist.next;
2154 	u32 offset;
2155 	unsigned long flags;
2156 	*vaddr = NULL;
2157 	*fpm_addr = 0;
2158 
2159 	bits_needed = DIV_ROUND_UP_ULL(mem_size, BIT_ULL(pprm->pble_shift));
2160 
2161 	spin_lock_irqsave(&pprm->prm_lock, flags);
2162 	while (chunk_entry != &pprm->clist) {
2163 		pchunk = (struct irdma_chunk *)chunk_entry;
2164 		bit_idx = bitmap_find_next_zero_area(pchunk->bitmapbuf,
2165 						     pchunk->sizeofbitmap, 0,
2166 						     bits_needed, 0);
2167 		if (bit_idx < pchunk->sizeofbitmap)
2168 			break;
2169 
2170 		/* list.next used macro */
2171 		chunk_entry = pchunk->list.next;
2172 	}
2173 
2174 	if (!pchunk || bit_idx >= pchunk->sizeofbitmap) {
2175 		spin_unlock_irqrestore(&pprm->prm_lock, flags);
2176 		return -ENOMEM;
2177 	}
2178 
2179 	bitmap_set(pchunk->bitmapbuf, bit_idx, bits_needed);
2180 	offset = bit_idx << pprm->pble_shift;
2181 	*vaddr = pchunk->vaddr + offset;
2182 	*fpm_addr = pchunk->fpm_addr + offset;
2183 
2184 	chunkinfo->pchunk = pchunk;
2185 	chunkinfo->bit_idx = bit_idx;
2186 	chunkinfo->bits_used = bits_needed;
2187 	/* 3 is sizeof pble divide */
2188 	pprm->free_pble_cnt -= chunkinfo->bits_used << (pprm->pble_shift - 3);
2189 	spin_unlock_irqrestore(&pprm->prm_lock, flags);
2190 
2191 	return 0;
2192 }
2193 
2194 /**
2195  * irdma_prm_return_pbles - return pbles back to prm
2196  * @pprm: pble resource manager
2197  * @chunkinfo: chunk where pble's were acquired and to be freed
2198  */
irdma_prm_return_pbles(struct irdma_pble_prm * pprm,struct irdma_pble_chunkinfo * chunkinfo)2199 void irdma_prm_return_pbles(struct irdma_pble_prm *pprm,
2200 			    struct irdma_pble_chunkinfo *chunkinfo)
2201 {
2202 	unsigned long flags;
2203 
2204 	spin_lock_irqsave(&pprm->prm_lock, flags);
2205 	pprm->free_pble_cnt += chunkinfo->bits_used << (pprm->pble_shift - 3);
2206 	bitmap_clear(chunkinfo->pchunk->bitmapbuf, chunkinfo->bit_idx,
2207 		     chunkinfo->bits_used);
2208 	spin_unlock_irqrestore(&pprm->prm_lock, flags);
2209 }
2210 
irdma_map_vm_page_list(struct irdma_hw * hw,void * va,dma_addr_t * pg_dma,u32 pg_cnt)2211 int irdma_map_vm_page_list(struct irdma_hw *hw, void *va, dma_addr_t *pg_dma,
2212 			   u32 pg_cnt)
2213 {
2214 	struct page *vm_page;
2215 	int i;
2216 	u8 *addr;
2217 
2218 	addr = (u8 *)(uintptr_t)va;
2219 	for (i = 0; i < pg_cnt; i++) {
2220 		vm_page = vmalloc_to_page(addr);
2221 		if (!vm_page)
2222 			goto err;
2223 
2224 		pg_dma[i] = dma_map_page(hw->device, vm_page, 0, PAGE_SIZE,
2225 					 DMA_BIDIRECTIONAL);
2226 		if (dma_mapping_error(hw->device, pg_dma[i]))
2227 			goto err;
2228 
2229 		addr += PAGE_SIZE;
2230 	}
2231 
2232 	return 0;
2233 
2234 err:
2235 	irdma_unmap_vm_page_list(hw, pg_dma, i);
2236 	return -ENOMEM;
2237 }
2238 
irdma_unmap_vm_page_list(struct irdma_hw * hw,dma_addr_t * pg_dma,u32 pg_cnt)2239 void irdma_unmap_vm_page_list(struct irdma_hw *hw, dma_addr_t *pg_dma, u32 pg_cnt)
2240 {
2241 	int i;
2242 
2243 	for (i = 0; i < pg_cnt; i++)
2244 		dma_unmap_page(hw->device, pg_dma[i], PAGE_SIZE, DMA_BIDIRECTIONAL);
2245 }
2246 
2247 /**
2248  * irdma_pble_free_paged_mem - free virtual paged memory
2249  * @chunk: chunk to free with paged memory
2250  */
irdma_pble_free_paged_mem(struct irdma_chunk * chunk)2251 void irdma_pble_free_paged_mem(struct irdma_chunk *chunk)
2252 {
2253 	if (!chunk->pg_cnt)
2254 		goto done;
2255 
2256 	irdma_unmap_vm_page_list(chunk->dev->hw, chunk->dmainfo.dmaaddrs,
2257 				 chunk->pg_cnt);
2258 
2259 done:
2260 	kfree(chunk->dmainfo.dmaaddrs);
2261 	chunk->dmainfo.dmaaddrs = NULL;
2262 	vfree(chunk->vaddr);
2263 	chunk->vaddr = NULL;
2264 	chunk->type = 0;
2265 }
2266 
2267 /**
2268  * irdma_pble_get_paged_mem -allocate paged memory for pbles
2269  * @chunk: chunk to add for paged memory
2270  * @pg_cnt: number of pages needed
2271  */
irdma_pble_get_paged_mem(struct irdma_chunk * chunk,u32 pg_cnt)2272 int irdma_pble_get_paged_mem(struct irdma_chunk *chunk, u32 pg_cnt)
2273 {
2274 	u32 size;
2275 	void *va;
2276 
2277 	chunk->dmainfo.dmaaddrs = kzalloc(pg_cnt << 3, GFP_KERNEL);
2278 	if (!chunk->dmainfo.dmaaddrs)
2279 		return -ENOMEM;
2280 
2281 	size = PAGE_SIZE * pg_cnt;
2282 	va = vmalloc(size);
2283 	if (!va)
2284 		goto err;
2285 
2286 	if (irdma_map_vm_page_list(chunk->dev->hw, va, chunk->dmainfo.dmaaddrs,
2287 				   pg_cnt)) {
2288 		vfree(va);
2289 		goto err;
2290 	}
2291 	chunk->vaddr = va;
2292 	chunk->size = size;
2293 	chunk->pg_cnt = pg_cnt;
2294 	chunk->type = PBLE_SD_PAGED;
2295 
2296 	return 0;
2297 err:
2298 	kfree(chunk->dmainfo.dmaaddrs);
2299 	chunk->dmainfo.dmaaddrs = NULL;
2300 
2301 	return -ENOMEM;
2302 }
2303 
2304 /**
2305  * irdma_alloc_ws_node_id - Allocate a tx scheduler node ID
2306  * @dev: device pointer
2307  */
irdma_alloc_ws_node_id(struct irdma_sc_dev * dev)2308 u16 irdma_alloc_ws_node_id(struct irdma_sc_dev *dev)
2309 {
2310 	struct irdma_pci_f *rf = dev_to_rf(dev);
2311 	u32 next = 1;
2312 	u32 node_id;
2313 
2314 	if (irdma_alloc_rsrc(rf, rf->allocated_ws_nodes, rf->max_ws_node_id,
2315 			     &node_id, &next))
2316 		return IRDMA_WS_NODE_INVALID;
2317 
2318 	return (u16)node_id;
2319 }
2320 
2321 /**
2322  * irdma_free_ws_node_id - Free a tx scheduler node ID
2323  * @dev: device pointer
2324  * @node_id: Work scheduler node ID
2325  */
irdma_free_ws_node_id(struct irdma_sc_dev * dev,u16 node_id)2326 void irdma_free_ws_node_id(struct irdma_sc_dev *dev, u16 node_id)
2327 {
2328 	struct irdma_pci_f *rf = dev_to_rf(dev);
2329 
2330 	irdma_free_rsrc(rf, rf->allocated_ws_nodes, (u32)node_id);
2331 }
2332 
2333 /**
2334  * irdma_modify_qp_to_err - Modify a QP to error
2335  * @sc_qp: qp structure
2336  */
irdma_modify_qp_to_err(struct irdma_sc_qp * sc_qp)2337 void irdma_modify_qp_to_err(struct irdma_sc_qp *sc_qp)
2338 {
2339 	struct irdma_qp *qp = sc_qp->qp_uk.back_qp;
2340 	struct ib_qp_attr attr;
2341 
2342 	if (qp->iwdev->rf->reset)
2343 		return;
2344 	attr.qp_state = IB_QPS_ERR;
2345 
2346 	if (rdma_protocol_roce(qp->ibqp.device, 1))
2347 		irdma_modify_qp_roce(&qp->ibqp, &attr, IB_QP_STATE, NULL);
2348 	else
2349 		irdma_modify_qp(&qp->ibqp, &attr, IB_QP_STATE, NULL);
2350 }
2351 
irdma_ib_qp_event(struct irdma_qp * iwqp,enum irdma_qp_event_type event)2352 void irdma_ib_qp_event(struct irdma_qp *iwqp, enum irdma_qp_event_type event)
2353 {
2354 	struct ib_event ibevent;
2355 
2356 	if (!iwqp->ibqp.event_handler)
2357 		return;
2358 
2359 	switch (event) {
2360 	case IRDMA_QP_EVENT_CATASTROPHIC:
2361 		ibevent.event = IB_EVENT_QP_FATAL;
2362 		break;
2363 	case IRDMA_QP_EVENT_ACCESS_ERR:
2364 		ibevent.event = IB_EVENT_QP_ACCESS_ERR;
2365 		break;
2366 	case IRDMA_QP_EVENT_REQ_ERR:
2367 		ibevent.event = IB_EVENT_QP_REQ_ERR;
2368 		break;
2369 	}
2370 	ibevent.device = iwqp->ibqp.device;
2371 	ibevent.element.qp = &iwqp->ibqp;
2372 	iwqp->ibqp.event_handler(&ibevent, iwqp->ibqp.qp_context);
2373 }
2374 
irdma_cq_empty(struct irdma_cq * iwcq)2375 bool irdma_cq_empty(struct irdma_cq *iwcq)
2376 {
2377 	struct irdma_cq_uk *ukcq;
2378 	u64 qword3;
2379 	__le64 *cqe;
2380 	u8 polarity;
2381 
2382 	ukcq  = &iwcq->sc_cq.cq_uk;
2383 	cqe = IRDMA_GET_CURRENT_CQ_ELEM(ukcq);
2384 	get_64bit_val(cqe, 24, &qword3);
2385 	polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3);
2386 
2387 	return polarity != ukcq->polarity;
2388 }
2389 
irdma_remove_cmpls_list(struct irdma_cq * iwcq)2390 void irdma_remove_cmpls_list(struct irdma_cq *iwcq)
2391 {
2392 	struct irdma_cmpl_gen *cmpl_node;
2393 	struct list_head *tmp_node, *list_node;
2394 
2395 	list_for_each_safe (list_node, tmp_node, &iwcq->cmpl_generated) {
2396 		cmpl_node = list_entry(list_node, struct irdma_cmpl_gen, list);
2397 		list_del(&cmpl_node->list);
2398 		kfree(cmpl_node);
2399 	}
2400 }
2401 
irdma_generated_cmpls(struct irdma_cq * iwcq,struct irdma_cq_poll_info * cq_poll_info)2402 int irdma_generated_cmpls(struct irdma_cq *iwcq, struct irdma_cq_poll_info *cq_poll_info)
2403 {
2404 	struct irdma_cmpl_gen *cmpl;
2405 
2406 	if (list_empty(&iwcq->cmpl_generated))
2407 		return -ENOENT;
2408 	cmpl = list_first_entry_or_null(&iwcq->cmpl_generated, struct irdma_cmpl_gen, list);
2409 	list_del(&cmpl->list);
2410 	memcpy(cq_poll_info, &cmpl->cpi, sizeof(*cq_poll_info));
2411 	kfree(cmpl);
2412 
2413 	ibdev_dbg(iwcq->ibcq.device,
2414 		  "VERBS: %s: Poll artificially generated completion for QP 0x%X, op %u, wr_id=0x%llx\n",
2415 		  __func__, cq_poll_info->qp_id, cq_poll_info->op_type,
2416 		  cq_poll_info->wr_id);
2417 
2418 	return 0;
2419 }
2420 
2421 /**
2422  * irdma_set_cpi_common_values - fill in values for polling info struct
2423  * @cpi: resulting structure of cq_poll_info type
2424  * @qp: QPair
2425  * @qp_num: id of the QP
2426  */
irdma_set_cpi_common_values(struct irdma_cq_poll_info * cpi,struct irdma_qp_uk * qp,u32 qp_num)2427 static void irdma_set_cpi_common_values(struct irdma_cq_poll_info *cpi,
2428 					struct irdma_qp_uk *qp, u32 qp_num)
2429 {
2430 	cpi->comp_status = IRDMA_COMPL_STATUS_FLUSHED;
2431 	cpi->error = true;
2432 	cpi->major_err = IRDMA_FLUSH_MAJOR_ERR;
2433 	cpi->minor_err = FLUSH_GENERAL_ERR;
2434 	cpi->qp_handle = (irdma_qp_handle)(uintptr_t)qp;
2435 	cpi->qp_id = qp_num;
2436 }
2437 
irdma_comp_handler(struct irdma_cq * cq)2438 static inline void irdma_comp_handler(struct irdma_cq *cq)
2439 {
2440 	if (!cq->ibcq.comp_handler)
2441 		return;
2442 	if (atomic_cmpxchg(&cq->armed, 1, 0))
2443 		cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
2444 }
2445 
irdma_generate_flush_completions(struct irdma_qp * iwqp)2446 void irdma_generate_flush_completions(struct irdma_qp *iwqp)
2447 {
2448 	struct irdma_qp_uk *qp = &iwqp->sc_qp.qp_uk;
2449 	struct irdma_ring *sq_ring = &qp->sq_ring;
2450 	struct irdma_ring *rq_ring = &qp->rq_ring;
2451 	struct irdma_cmpl_gen *cmpl;
2452 	__le64 *sw_wqe;
2453 	u64 wqe_qword;
2454 	u32 wqe_idx;
2455 	bool compl_generated = false;
2456 	unsigned long flags1;
2457 
2458 	spin_lock_irqsave(&iwqp->iwscq->lock, flags1);
2459 	if (irdma_cq_empty(iwqp->iwscq)) {
2460 		unsigned long flags2;
2461 
2462 		spin_lock_irqsave(&iwqp->lock, flags2);
2463 		while (IRDMA_RING_MORE_WORK(*sq_ring)) {
2464 			cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC);
2465 			if (!cmpl) {
2466 				spin_unlock_irqrestore(&iwqp->lock, flags2);
2467 				spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1);
2468 				return;
2469 			}
2470 
2471 			wqe_idx = sq_ring->tail;
2472 			irdma_set_cpi_common_values(&cmpl->cpi, qp, qp->qp_id);
2473 
2474 			cmpl->cpi.wr_id = qp->sq_wrtrk_array[wqe_idx].wrid;
2475 			sw_wqe = qp->sq_base[wqe_idx].elem;
2476 			get_64bit_val(sw_wqe, 24, &wqe_qword);
2477 			cmpl->cpi.op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, IRDMAQPSQ_OPCODE);
2478 			cmpl->cpi.q_type = IRDMA_CQE_QTYPE_SQ;
2479 			/* remove the SQ WR by moving SQ tail*/
2480 			IRDMA_RING_SET_TAIL(*sq_ring,
2481 				sq_ring->tail + qp->sq_wrtrk_array[sq_ring->tail].quanta);
2482 			if (cmpl->cpi.op_type == IRDMAQP_OP_NOP) {
2483 				kfree(cmpl);
2484 				continue;
2485 			}
2486 			ibdev_dbg(iwqp->iwscq->ibcq.device,
2487 				  "DEV: %s: adding wr_id = 0x%llx SQ Completion to list qp_id=%d\n",
2488 				  __func__, cmpl->cpi.wr_id, qp->qp_id);
2489 			list_add_tail(&cmpl->list, &iwqp->iwscq->cmpl_generated);
2490 			compl_generated = true;
2491 		}
2492 		spin_unlock_irqrestore(&iwqp->lock, flags2);
2493 		spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1);
2494 		if (compl_generated)
2495 			irdma_comp_handler(iwqp->iwscq);
2496 	} else {
2497 		spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1);
2498 		mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush,
2499 				 msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS));
2500 	}
2501 
2502 	spin_lock_irqsave(&iwqp->iwrcq->lock, flags1);
2503 	if (irdma_cq_empty(iwqp->iwrcq)) {
2504 		unsigned long flags2;
2505 
2506 		spin_lock_irqsave(&iwqp->lock, flags2);
2507 		while (IRDMA_RING_MORE_WORK(*rq_ring)) {
2508 			cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC);
2509 			if (!cmpl) {
2510 				spin_unlock_irqrestore(&iwqp->lock, flags2);
2511 				spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1);
2512 				return;
2513 			}
2514 
2515 			wqe_idx = rq_ring->tail;
2516 			irdma_set_cpi_common_values(&cmpl->cpi, qp, qp->qp_id);
2517 
2518 			cmpl->cpi.wr_id = qp->rq_wrid_array[wqe_idx];
2519 			cmpl->cpi.op_type = IRDMA_OP_TYPE_REC;
2520 			cmpl->cpi.q_type = IRDMA_CQE_QTYPE_RQ;
2521 			/* remove the RQ WR by moving RQ tail */
2522 			IRDMA_RING_SET_TAIL(*rq_ring, rq_ring->tail + 1);
2523 			ibdev_dbg(iwqp->iwrcq->ibcq.device,
2524 				  "DEV: %s: adding wr_id = 0x%llx RQ Completion to list qp_id=%d, wqe_idx=%d\n",
2525 				  __func__, cmpl->cpi.wr_id, qp->qp_id,
2526 				  wqe_idx);
2527 			list_add_tail(&cmpl->list, &iwqp->iwrcq->cmpl_generated);
2528 
2529 			compl_generated = true;
2530 		}
2531 		spin_unlock_irqrestore(&iwqp->lock, flags2);
2532 		spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1);
2533 		if (compl_generated)
2534 			irdma_comp_handler(iwqp->iwrcq);
2535 	} else {
2536 		spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1);
2537 		mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush,
2538 				 msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS));
2539 	}
2540 }
2541