1 /*-
2 * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
3 *
4 * Copyright (c) 2015 - 2023 Intel Corporation
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenFabrics.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35 #include "irdma_main.h"
36
37 LIST_HEAD(irdma_handlers);
38 DEFINE_SPINLOCK(irdma_handler_lock);
39
40 static const struct ae_desc ae_desc_list[] = {
41 {IRDMA_AE_AMP_UNALLOCATED_STAG, "Unallocated memory key (L-Key/R-Key)"},
42 {IRDMA_AE_AMP_INVALID_STAG, "Invalid memory key (L-Key/R-Key)"},
43 {IRDMA_AE_AMP_BAD_QP,
44 "Memory protection error: Accessing Memory Window (MW) which belongs to a different QP"},
45 {IRDMA_AE_AMP_BAD_PD,
46 "Memory protection error: Accessing Memory Window (MW)/Memory Region (MR) which belongs to a different PD"},
47 {IRDMA_AE_AMP_BAD_STAG_KEY, "Bad memory key (L-Key/R-Key)"},
48 {IRDMA_AE_AMP_BAD_STAG_INDEX, "Bad memory key (L-Key/R-Key): Too large memory key index"},
49 {IRDMA_AE_AMP_BOUNDS_VIOLATION, "Memory Window (MW)/Memory Region (MR) bounds violation"},
50 {IRDMA_AE_AMP_RIGHTS_VIOLATION, "Memory Window (MW)/Memory Region (MR) rights violation"},
51 {IRDMA_AE_AMP_TO_WRAP,
52 "Memory protection error: The address within Memory Window (MW)/Memory Region (MR) wraps"},
53 {IRDMA_AE_AMP_FASTREG_VALID_STAG,
54 "Fastreg error: Registration to a valid MR"},
55 {IRDMA_AE_AMP_FASTREG_MW_STAG,
56 "Fastreg error: Registration to a valid Memory Window (MW)"},
57 {IRDMA_AE_AMP_FASTREG_INVALID_RIGHTS, "Fastreg error: Invalid rights"},
58 {IRDMA_AE_AMP_FASTREG_INVALID_LENGTH, "Fastreg error: Invalid length"},
59 {IRDMA_AE_AMP_INVALIDATE_SHARED, "Attempt to invalidate a shared MR"},
60 {IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS,
61 "Attempt to remotely invalidate Memory Window (MW)/Memory Region (MR) without rights"},
62 {IRDMA_AE_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS,
63 "Attempt to invalidate MR with a bound Memory Window (MW)"},
64 {IRDMA_AE_AMP_MWBIND_VALID_STAG,
65 "Attempt to bind an Memory Window (MW) with a valid MW memory key (L-Key/R-Key)"},
66 {IRDMA_AE_AMP_MWBIND_OF_MR_STAG,
67 "Attempt to bind an Memory Window (MW) with an MR memory key (L-Key/R-Key)"},
68 {IRDMA_AE_AMP_MWBIND_TO_ZERO_BASED_STAG,
69 "Attempt to bind an Memory Window (MW) to a zero based MR"},
70 {IRDMA_AE_AMP_MWBIND_TO_MW_STAG,
71 "Attempt to bind an Memory Window (MW) using MW memory key (L-Key/R-Key) instead of MR memory key (L-Key/R-Key)"},
72 {IRDMA_AE_AMP_MWBIND_INVALID_RIGHTS, "Memory Window (MW) bind error: Invalid rights"},
73 {IRDMA_AE_AMP_MWBIND_INVALID_BOUNDS, "Memory Window (MW) bind error: Invalid bounds"},
74 {IRDMA_AE_AMP_MWBIND_TO_INVALID_PARENT,
75 "Memory Window (MW) bind error: Invalid parent MR"},
76 {IRDMA_AE_AMP_MWBIND_BIND_DISABLED,
77 "Memory Window (MW) bind error: Disabled bind support"},
78 {IRDMA_AE_PRIV_OPERATION_DENIED,
79 "Denying a privileged operation on a non-privileged QP"},
80 {IRDMA_AE_AMP_INVALIDATE_TYPE1_MW, "Memory Window (MW) error: Invalidate type 1 MW"},
81 {IRDMA_AE_AMP_MWBIND_ZERO_BASED_TYPE1_MW,
82 "Memory Window (MW) bind error: Zero-based addressing for type 1 MW"},
83 {IRDMA_AE_AMP_FASTREG_INVALID_PBL_HPS_CFG,
84 "Fastreg error: Invalid host page size config"},
85 {IRDMA_AE_AMP_MWBIND_WRONG_TYPE, "MB bind error: Wrong Memory Window (MW) type"},
86 {IRDMA_AE_AMP_FASTREG_PBLE_MISMATCH,
87 "Fastreg error: Invalid request to change physical MR to virtual or vice versa"},
88 {IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG,
89 "Userspace Direct Access (UDA) QP xmit error: Packet length exceeds the QP MTU"},
90 {IRDMA_AE_UDA_XMIT_BAD_PD,
91 "Userspace Direct Access (UDA) QP xmit error: Attempt to access a different PD"},
92 {IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT,
93 "Userspace Direct Access (UDA) QP xmit error: Too short packet length"},
94 {IRDMA_AE_UDA_L4LEN_INVALID,
95 "Userspace Direct Access (UDA) error: Invalid packet length field"},
96 {IRDMA_AE_BAD_CLOSE,
97 "iWARP error: Data is received when QP state is closing"},
98 {IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE,
99 "iWARP error: FIN is received when xmit data is pending"},
100 {IRDMA_AE_CQ_OPERATION_ERROR, "CQ overflow"},
101 {IRDMA_AE_RDMA_READ_WHILE_ORD_ZERO,
102 "QP error: Attempted RDMA Read when the outbound RDMA Read queue depth is zero"},
103 {IRDMA_AE_STAG_ZERO_INVALID,
104 "Zero invalid memory key (L-Key/R-Key) on inbound RDMA R/W"},
105 {IRDMA_AE_IB_RREQ_AND_Q1_FULL,
106 "QP error: Received RDMA Read request when the inbound RDMA Read queue is full"},
107 {IRDMA_AE_IB_INVALID_REQUEST,
108 "QP error: Invalid operation detected by the remote peer"},
109 {IRDMA_AE_WQE_UNEXPECTED_OPCODE,
110 "QP error: Invalid opcode in SQ WQE"},
111 {IRDMA_AE_WQE_INVALID_PARAMETER,
112 "QP error: Invalid parameter in a WQE"},
113 {IRDMA_AE_WQE_INVALID_FRAG_DATA,
114 "QP error: Invalid fragment in a WQE"},
115 {IRDMA_AE_IB_REMOTE_ACCESS_ERROR,
116 "RoCEv2 error: Remote access error"},
117 {IRDMA_AE_IB_REMOTE_OP_ERROR,
118 "RoCEv2 error: Remote operation error"},
119 {IRDMA_AE_WQE_LSMM_TOO_LONG, "iWARP error: Connection error"},
120 {IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN,
121 "iWARP error: Invalid message sequence number"},
122 {IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER,
123 "iWARP error: Inbound message is too long for the available buffer"},
124 {IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION, "iWARP error: Invalid DDP protocol version"},
125 {IRDMA_AE_DDP_UBE_INVALID_MO, "Received message with too large offset"},
126 {IRDMA_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE,
127 "iWARP error: Inbound Send message when no receive buffer is available"},
128 {IRDMA_AE_DDP_UBE_INVALID_QN, "iWARP error: Invalid QP number in inbound packet"},
129 {IRDMA_AE_DDP_NO_L_BIT,
130 "iWARP error: Last bit not set in an inbound packet which completes RDMA Read"},
131 {IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION, "iWARP error: Invalid RDMAP protocol version"},
132 {IRDMA_AE_RDMAP_ROE_UNEXPECTED_OPCODE, "QP error: Invalid opcode"},
133 {IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST, "Inbound Read request when QP isn't enabled for RDMA Read"},
134 {IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP,
135 "Inbound RDMA Read response or RDMA Write when QP isn't enabled for RDMA R/W"},
136 {IRDMA_AE_ROCE_RSP_LENGTH_ERROR, "RoCEv2 error: Received packet with incorrect length field"},
137 {IRDMA_AE_ROCE_EMPTY_MCG, "RoCEv2 error: Multicast group has no valid members"},
138 {IRDMA_AE_ROCE_BAD_MC_IP_ADDR, "RoCEv2 error: Multicast IP address doesn't match"},
139 {IRDMA_AE_ROCE_BAD_MC_QPID, "RoCEv2 error: Multicast packet QP number isn't 0xffffff"},
140 {IRDMA_AE_MCG_QP_PROTOCOL_MISMATCH, "RoCEv2 error: Multicast packet protocol mismatch"},
141 {IRDMA_AE_INVALID_ARP_ENTRY, "Invalid ARP entry"},
142 {IRDMA_AE_INVALID_TCP_OPTION_RCVD, "iWARP error: Invalid TCP option"},
143 {IRDMA_AE_STALE_ARP_ENTRY, "Stale ARP entry"},
144 {IRDMA_AE_INVALID_AH_ENTRY, "Invalid AH entry"},
145 {IRDMA_AE_LLP_CLOSE_COMPLETE,
146 "iWARP event: Graceful close complete"},
147 {IRDMA_AE_LLP_CONNECTION_RESET,
148 "iWARP event: Received a TCP packet with a RST bit set"},
149 {IRDMA_AE_LLP_FIN_RECEIVED,
150 "iWARP event: Received a TCP packet with a FIN bit set"},
151 {IRDMA_AE_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH,
152 "iWARP error: Unable to close a gap in the TCP sequence"},
153 {IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR, "Received an ICRC error"},
154 {IRDMA_AE_LLP_SEGMENT_TOO_SMALL,
155 "iWARP error: Received a packet with insufficient space for protocol headers"},
156 {IRDMA_AE_LLP_SYN_RECEIVED,
157 "iWARP event: Received a TCP packet with a SYN bit set"},
158 {IRDMA_AE_LLP_TERMINATE_RECEIVED,
159 "iWARP error: Received a terminate message"},
160 {IRDMA_AE_LLP_TOO_MANY_RETRIES, "Connection error: The max number of retries has been reached"},
161 {IRDMA_AE_LLP_TOO_MANY_KEEPALIVE_RETRIES,
162 "Connection error: The max number of keepalive retries has been reached"},
163 {IRDMA_AE_LLP_DOUBT_REACHABILITY,
164 "Connection error: Doubt reachability (usually occurs after the max number of retries has been reached)"},
165 {IRDMA_AE_LLP_CONNECTION_ESTABLISHED,
166 "iWARP event: Connection established"},
167 {IRDMA_AE_LLP_TOO_MANY_RNRS, "RoCEv2: Too many RNR NACKs"},
168 {IRDMA_AE_RESOURCE_EXHAUSTION,
169 "QP error: Resource exhaustion"},
170 {IRDMA_AE_RESET_SENT,
171 "Reset sent (as requested via Modify QP)"},
172 {IRDMA_AE_TERMINATE_SENT,
173 "Terminate sent (as requested via Modify QP)"},
174 {IRDMA_AE_RESET_NOT_SENT,
175 "Reset not sent (but requested via Modify QP)"},
176 {IRDMA_AE_LCE_QP_CATASTROPHIC,
177 "QP error: HW transaction resulted in catastrophic error"},
178 {IRDMA_AE_LCE_FUNCTION_CATASTROPHIC,
179 "PCIe function error: HW transaction resulted in catastrophic error"},
180 {IRDMA_AE_LCE_CQ_CATASTROPHIC,
181 "CQ error: HW transaction resulted in catastrophic error"},
182 {IRDMA_AE_QP_SUSPEND_COMPLETE, "QP event: Suspend complete"},
183 };
184
185 /**
186 * irdma_get_ae_desc - returns AE description
187 * @ae_id: the AE number
188 */
189 const char *
irdma_get_ae_desc(u16 ae_id)190 irdma_get_ae_desc(u16 ae_id)
191 {
192 const char *desc = "";
193 int i;
194
195 for (i = 0; i < ARRAY_SIZE(ae_desc_list); i++) {
196 if (ae_desc_list[i].id == ae_id) {
197 desc = ae_desc_list[i].desc;
198 break;
199 }
200 }
201 return desc;
202 }
203
204 /**
205 * irdma_arp_table -manage arp table
206 * @rf: RDMA PCI function
207 * @ip_addr: ip address for device
208 * @mac_addr: mac address ptr
209 * @action: modify, delete or add
210 */
211 int
irdma_arp_table(struct irdma_pci_f * rf,u32 * ip_addr,const u8 * mac_addr,u32 action)212 irdma_arp_table(struct irdma_pci_f *rf, u32 *ip_addr, const u8 *mac_addr,
213 u32 action)
214 {
215 unsigned long flags;
216 int arp_index;
217 u32 ip[4] = {};
218
219 memcpy(ip, ip_addr, sizeof(ip));
220
221 spin_lock_irqsave(&rf->arp_lock, flags);
222 for (arp_index = 0; (u32)arp_index < rf->arp_table_size; arp_index++) {
223 if (!memcmp(rf->arp_table[arp_index].ip_addr, ip, sizeof(ip)))
224 break;
225 }
226
227 switch (action) {
228 case IRDMA_ARP_ADD:
229 if (arp_index != rf->arp_table_size) {
230 arp_index = -1;
231 break;
232 }
233
234 arp_index = 0;
235 if (irdma_alloc_rsrc(rf, rf->allocated_arps, rf->arp_table_size,
236 (u32 *)&arp_index, &rf->next_arp_index)) {
237 arp_index = -1;
238 break;
239 }
240
241 memcpy(rf->arp_table[arp_index].ip_addr, ip,
242 sizeof(rf->arp_table[arp_index].ip_addr));
243 ether_addr_copy(rf->arp_table[arp_index].mac_addr, mac_addr);
244 break;
245 case IRDMA_ARP_RESOLVE:
246 if (arp_index == rf->arp_table_size)
247 arp_index = -1;
248 break;
249 case IRDMA_ARP_DELETE:
250 if (arp_index == rf->arp_table_size) {
251 arp_index = -1;
252 break;
253 }
254
255 memset(rf->arp_table[arp_index].ip_addr, 0,
256 sizeof(rf->arp_table[arp_index].ip_addr));
257 eth_zero_addr(rf->arp_table[arp_index].mac_addr);
258 irdma_free_rsrc(rf, rf->allocated_arps, arp_index);
259 break;
260 default:
261 arp_index = -1;
262 break;
263 }
264
265 spin_unlock_irqrestore(&rf->arp_lock, flags);
266 return arp_index;
267 }
268
269 /**
270 * irdma_add_arp - add a new arp entry if needed
271 * @rf: RDMA function
272 * @ip: IP address
273 * @mac: MAC address
274 */
275 int
irdma_add_arp(struct irdma_pci_f * rf,u32 * ip,const u8 * mac)276 irdma_add_arp(struct irdma_pci_f *rf, u32 *ip, const u8 *mac)
277 {
278 int arpidx;
279
280 arpidx = irdma_arp_table(rf, &ip[0], NULL, IRDMA_ARP_RESOLVE);
281 if (arpidx >= 0) {
282 if (ether_addr_equal(rf->arp_table[arpidx].mac_addr, mac))
283 return arpidx;
284
285 irdma_manage_arp_cache(rf, rf->arp_table[arpidx].mac_addr, ip,
286 IRDMA_ARP_DELETE);
287 }
288
289 irdma_manage_arp_cache(rf, mac, ip, IRDMA_ARP_ADD);
290
291 return irdma_arp_table(rf, ip, NULL, IRDMA_ARP_RESOLVE);
292 }
293
294 /**
295 * irdma_netdevice_event - system notifier for netdev events
296 * @notifier: not used
297 * @event: event for notifier
298 * @ptr: netdev
299 */
300 int
irdma_netdevice_event(struct notifier_block * notifier,unsigned long event,void * ptr)301 irdma_netdevice_event(struct notifier_block *notifier, unsigned long event,
302 void *ptr)
303 {
304 struct irdma_device *iwdev;
305 struct ifnet *netdev = netdev_notifier_info_to_ifp(ptr);
306
307 iwdev = container_of(notifier, struct irdma_device, nb_netdevice_event);
308 if (iwdev->netdev != netdev)
309 return NOTIFY_DONE;
310
311 iwdev->iw_status = 1;
312 switch (event) {
313 case NETDEV_DOWN:
314 iwdev->iw_status = 0;
315 /* fallthrough */
316 case NETDEV_UP:
317 irdma_port_ibevent(iwdev);
318 break;
319 default:
320 break;
321 }
322
323 return NOTIFY_DONE;
324 }
325
326 void
irdma_unregister_notifiers(struct irdma_device * iwdev)327 irdma_unregister_notifiers(struct irdma_device *iwdev)
328 {
329 unregister_netdevice_notifier(&iwdev->nb_netdevice_event);
330 }
331
332 int
irdma_register_notifiers(struct irdma_device * iwdev)333 irdma_register_notifiers(struct irdma_device *iwdev)
334 {
335 int ret;
336
337 iwdev->nb_netdevice_event.notifier_call = irdma_netdevice_event;
338 ret = register_netdevice_notifier(&iwdev->nb_netdevice_event);
339 if (ret) {
340 irdma_dev_err(&iwdev->ibdev, "register_netdevice_notifier failed\n");
341 return ret;
342 }
343 return ret;
344 }
345 /**
346 * irdma_alloc_and_get_cqp_request - get cqp struct
347 * @cqp: device cqp ptr
348 * @wait: cqp to be used in wait mode
349 */
350 struct irdma_cqp_request *
irdma_alloc_and_get_cqp_request(struct irdma_cqp * cqp,bool wait)351 irdma_alloc_and_get_cqp_request(struct irdma_cqp *cqp,
352 bool wait)
353 {
354 struct irdma_cqp_request *cqp_request = NULL;
355 unsigned long flags;
356
357 spin_lock_irqsave(&cqp->req_lock, flags);
358 if (!list_empty(&cqp->cqp_avail_reqs)) {
359 cqp_request = list_entry(cqp->cqp_avail_reqs.next,
360 struct irdma_cqp_request, list);
361 list_del_init(&cqp_request->list);
362 }
363 spin_unlock_irqrestore(&cqp->req_lock, flags);
364 if (!cqp_request) {
365 cqp_request = kzalloc(sizeof(*cqp_request), GFP_ATOMIC);
366 if (cqp_request) {
367 cqp_request->dynamic = true;
368 if (wait)
369 init_waitqueue_head(&cqp_request->waitq);
370 }
371 }
372 if (!cqp_request) {
373 irdma_debug(cqp->sc_cqp.dev, IRDMA_DEBUG_ERR, "CQP Request Fail: No Memory");
374 return NULL;
375 }
376
377 cqp_request->waiting = wait;
378 atomic_set(&cqp_request->refcnt, 1);
379 memset(&cqp_request->compl_info, 0, sizeof(cqp_request->compl_info));
380
381 return cqp_request;
382 }
383
384 /**
385 * irdma_get_cqp_request - increase refcount for cqp_request
386 * @cqp_request: pointer to cqp_request instance
387 */
388 static inline void
irdma_get_cqp_request(struct irdma_cqp_request * cqp_request)389 irdma_get_cqp_request(struct irdma_cqp_request *cqp_request)
390 {
391 atomic_inc(&cqp_request->refcnt);
392 }
393
394 /**
395 * irdma_free_cqp_request - free cqp request
396 * @cqp: cqp ptr
397 * @cqp_request: to be put back in cqp list
398 */
399 void
irdma_free_cqp_request(struct irdma_cqp * cqp,struct irdma_cqp_request * cqp_request)400 irdma_free_cqp_request(struct irdma_cqp *cqp,
401 struct irdma_cqp_request *cqp_request)
402 {
403 unsigned long flags;
404
405 if (cqp_request->dynamic) {
406 kfree(cqp_request);
407 } else {
408 WRITE_ONCE(cqp_request->request_done, false);
409 cqp_request->callback_fcn = NULL;
410 cqp_request->waiting = false;
411
412 spin_lock_irqsave(&cqp->req_lock, flags);
413 list_add_tail(&cqp_request->list, &cqp->cqp_avail_reqs);
414 spin_unlock_irqrestore(&cqp->req_lock, flags);
415 }
416 wake_up(&cqp->remove_wq);
417 }
418
419 /**
420 * irdma_put_cqp_request - dec ref count and free if 0
421 * @cqp: cqp ptr
422 * @cqp_request: to be put back in cqp list
423 */
424 void
irdma_put_cqp_request(struct irdma_cqp * cqp,struct irdma_cqp_request * cqp_request)425 irdma_put_cqp_request(struct irdma_cqp *cqp,
426 struct irdma_cqp_request *cqp_request)
427 {
428 if (atomic_dec_and_test(&cqp_request->refcnt))
429 irdma_free_cqp_request(cqp, cqp_request);
430 }
431
432 /**
433 * irdma_free_pending_cqp_request -free pending cqp request objs
434 * @cqp: cqp ptr
435 * @cqp_request: to be put back in cqp list
436 */
437 static void
irdma_free_pending_cqp_request(struct irdma_cqp * cqp,struct irdma_cqp_request * cqp_request)438 irdma_free_pending_cqp_request(struct irdma_cqp *cqp,
439 struct irdma_cqp_request *cqp_request)
440 {
441 cqp_request->compl_info.error = true;
442 WRITE_ONCE(cqp_request->request_done, true);
443
444 if (cqp_request->waiting)
445 wake_up(&cqp_request->waitq);
446 wait_event_timeout(cqp->remove_wq,
447 atomic_read(&cqp_request->refcnt) == 1, 1000);
448 irdma_put_cqp_request(cqp, cqp_request);
449 }
450
451 /**
452 * irdma_cleanup_pending_cqp_op - clean-up cqp with no
453 * completions
454 * @rf: RDMA PCI function
455 */
456 void
irdma_cleanup_pending_cqp_op(struct irdma_pci_f * rf)457 irdma_cleanup_pending_cqp_op(struct irdma_pci_f *rf)
458 {
459 struct irdma_sc_dev *dev = &rf->sc_dev;
460 struct irdma_cqp *cqp = &rf->cqp;
461 struct irdma_cqp_request *cqp_request = NULL;
462 struct cqp_cmds_info *pcmdinfo = NULL;
463 u32 i, pending_work, wqe_idx;
464
465 pending_work = IRDMA_RING_USED_QUANTA(cqp->sc_cqp.sq_ring);
466 wqe_idx = IRDMA_RING_CURRENT_TAIL(cqp->sc_cqp.sq_ring);
467 for (i = 0; i < pending_work; i++) {
468 cqp_request = (struct irdma_cqp_request *)(uintptr_t)
469 cqp->scratch_array[wqe_idx];
470 if (cqp_request)
471 irdma_free_pending_cqp_request(cqp, cqp_request);
472 wqe_idx = (wqe_idx + 1) % IRDMA_RING_SIZE(cqp->sc_cqp.sq_ring);
473 }
474
475 while (!list_empty(&dev->cqp_cmd_head)) {
476 pcmdinfo = irdma_remove_cqp_head(dev);
477 cqp_request =
478 container_of(pcmdinfo, struct irdma_cqp_request, info);
479 if (cqp_request)
480 irdma_free_pending_cqp_request(cqp, cqp_request);
481 }
482 }
483
484 /**
485 * irdma_wait_event - wait for completion
486 * @rf: RDMA PCI function
487 * @cqp_request: cqp request to wait
488 */
489 static int
irdma_wait_event(struct irdma_pci_f * rf,struct irdma_cqp_request * cqp_request)490 irdma_wait_event(struct irdma_pci_f *rf,
491 struct irdma_cqp_request *cqp_request)
492 {
493 struct irdma_cqp_timeout cqp_timeout = {0};
494 bool cqp_error = false;
495 int err_code = 0;
496
497 cqp_timeout.compl_cqp_cmds = atomic64_read(&rf->sc_dev.cqp->completed_ops);
498 do {
499 int wait_time_ms = rf->sc_dev.hw_attrs.max_cqp_compl_wait_time_ms;
500
501 irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq);
502 if (wait_event_timeout(cqp_request->waitq,
503 READ_ONCE(cqp_request->request_done),
504 msecs_to_jiffies(wait_time_ms)))
505 break;
506
507 irdma_check_cqp_progress(&cqp_timeout, &rf->sc_dev);
508
509 if (cqp_timeout.count < CQP_TIMEOUT_THRESHOLD)
510 continue;
511
512 if (!rf->reset) {
513 rf->reset = true;
514 rf->gen_ops.request_reset(rf);
515 }
516 return -ETIMEDOUT;
517 } while (1);
518
519 cqp_error = cqp_request->compl_info.error;
520 if (cqp_error) {
521 err_code = -EIO;
522 if (cqp_request->compl_info.maj_err_code == 0xFFFF) {
523 if (cqp_request->compl_info.min_err_code == 0x8002) {
524 err_code = -EBUSY;
525 } else if (cqp_request->compl_info.min_err_code == 0x8029) {
526 if (!rf->reset) {
527 rf->reset = true;
528 rf->gen_ops.request_reset(rf);
529 }
530 }
531 }
532 }
533
534 return err_code;
535 }
536
537 static const char *const irdma_cqp_cmd_names[IRDMA_MAX_CQP_OPS] = {
538 [IRDMA_OP_CEQ_DESTROY] = "Destroy CEQ Cmd",
539 [IRDMA_OP_AEQ_DESTROY] = "Destroy AEQ Cmd",
540 [IRDMA_OP_DELETE_ARP_CACHE_ENTRY] = "Delete ARP Cache Cmd",
541 [IRDMA_OP_MANAGE_APBVT_ENTRY] = "Manage APBV Table Entry Cmd",
542 [IRDMA_OP_CEQ_CREATE] = "CEQ Create Cmd",
543 [IRDMA_OP_AEQ_CREATE] = "AEQ Destroy Cmd",
544 [IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY] = "Manage Quad Hash Table Entry Cmd",
545 [IRDMA_OP_QP_MODIFY] = "Modify QP Cmd",
546 [IRDMA_OP_QP_UPLOAD_CONTEXT] = "Upload Context Cmd",
547 [IRDMA_OP_CQ_CREATE] = "Create CQ Cmd",
548 [IRDMA_OP_CQ_DESTROY] = "Destroy CQ Cmd",
549 [IRDMA_OP_QP_CREATE] = "Create QP Cmd",
550 [IRDMA_OP_QP_DESTROY] = "Destroy QP Cmd",
551 [IRDMA_OP_ALLOC_STAG] = "Allocate STag Cmd",
552 [IRDMA_OP_MR_REG_NON_SHARED] = "Register Non-Shared MR Cmd",
553 [IRDMA_OP_DEALLOC_STAG] = "Deallocate STag Cmd",
554 [IRDMA_OP_MW_ALLOC] = "Allocate Memory Window Cmd",
555 [IRDMA_OP_QP_FLUSH_WQES] = "Flush QP Cmd",
556 [IRDMA_OP_ADD_ARP_CACHE_ENTRY] = "Add ARP Cache Cmd",
557 [IRDMA_OP_MANAGE_PUSH_PAGE] = "Manage Push Page Cmd",
558 [IRDMA_OP_UPDATE_PE_SDS] = "Update PE SDs Cmd",
559 [IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE] = "Manage HMC PM Function Table Cmd",
560 [IRDMA_OP_SUSPEND] = "Suspend QP Cmd",
561 [IRDMA_OP_RESUME] = "Resume QP Cmd",
562 [IRDMA_OP_QUERY_FPM_VAL] = "Query FPM Values Cmd",
563 [IRDMA_OP_COMMIT_FPM_VAL] = "Commit FPM Values Cmd",
564 [IRDMA_OP_AH_CREATE] = "Create Address Handle Cmd",
565 [IRDMA_OP_AH_MODIFY] = "Modify Address Handle Cmd",
566 [IRDMA_OP_AH_DESTROY] = "Destroy Address Handle Cmd",
567 [IRDMA_OP_MC_CREATE] = "Create Multicast Group Cmd",
568 [IRDMA_OP_MC_DESTROY] = "Destroy Multicast Group Cmd",
569 [IRDMA_OP_MC_MODIFY] = "Modify Multicast Group Cmd",
570 [IRDMA_OP_STATS_ALLOCATE] = "Add Statistics Instance Cmd",
571 [IRDMA_OP_STATS_FREE] = "Free Statistics Instance Cmd",
572 [IRDMA_OP_STATS_GATHER] = "Gather Statistics Cmd",
573 [IRDMA_OP_WS_ADD_NODE] = "Add Work Scheduler Node Cmd",
574 [IRDMA_OP_WS_MODIFY_NODE] = "Modify Work Scheduler Node Cmd",
575 [IRDMA_OP_WS_DELETE_NODE] = "Delete Work Scheduler Node Cmd",
576 [IRDMA_OP_WS_FAILOVER_START] = "Failover Start Cmd",
577 [IRDMA_OP_WS_FAILOVER_COMPLETE] = "Failover Complete Cmd",
578 [IRDMA_OP_SET_UP_MAP] = "Set UP-UP Mapping Cmd",
579 [IRDMA_OP_GEN_AE] = "Generate AE Cmd",
580 [IRDMA_OP_QUERY_RDMA_FEATURES] = "RDMA Get Features Cmd",
581 [IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY] = "Allocate Local MAC Entry Cmd",
582 [IRDMA_OP_ADD_LOCAL_MAC_ENTRY] = "Add Local MAC Entry Cmd",
583 [IRDMA_OP_DELETE_LOCAL_MAC_ENTRY] = "Delete Local MAC Entry Cmd",
584 [IRDMA_OP_CQ_MODIFY] = "CQ Modify Cmd",
585 };
586
587 static const struct irdma_cqp_err_info irdma_noncrit_err_list[] = {
588 {0xffff, 0x8002, "Invalid State"},
589 {0xffff, 0x8006, "Flush No Wqe Pending"},
590 {0xffff, 0x8007, "Modify QP Bad Close"},
591 {0xffff, 0x8009, "LLP Closed"},
592 {0xffff, 0x800a, "Reset Not Sent"},
593 {0xffff, 0x0200, "Failover Pending"},
594 };
595
596 /**
597 * irdma_cqp_crit_err - check if CQP error is critical
598 * @dev: pointer to dev structure
599 * @cqp_cmd: code for last CQP operation
600 * @maj_err_code: major error code
601 * @min_err_code: minot error code
602 */
603 bool
irdma_cqp_crit_err(struct irdma_sc_dev * dev,u8 cqp_cmd,u16 maj_err_code,u16 min_err_code)604 irdma_cqp_crit_err(struct irdma_sc_dev *dev, u8 cqp_cmd,
605 u16 maj_err_code, u16 min_err_code)
606 {
607 int i;
608
609 for (i = 0; i < ARRAY_SIZE(irdma_noncrit_err_list); ++i) {
610 if (maj_err_code == irdma_noncrit_err_list[i].maj &&
611 min_err_code == irdma_noncrit_err_list[i].min) {
612 irdma_debug(dev, IRDMA_DEBUG_CQP,
613 "[%s Error][%s] maj=0x%x min=0x%x\n",
614 irdma_noncrit_err_list[i].desc,
615 irdma_cqp_cmd_names[cqp_cmd], maj_err_code,
616 min_err_code);
617 return false;
618 }
619 }
620 return true;
621 }
622
623 /**
624 * irdma_handle_cqp_op - process cqp command
625 * @rf: RDMA PCI function
626 * @cqp_request: cqp request to process
627 */
628 int
irdma_handle_cqp_op(struct irdma_pci_f * rf,struct irdma_cqp_request * cqp_request)629 irdma_handle_cqp_op(struct irdma_pci_f *rf,
630 struct irdma_cqp_request *cqp_request)
631 {
632 struct irdma_sc_dev *dev = &rf->sc_dev;
633 struct cqp_cmds_info *info = &cqp_request->info;
634 int status;
635 bool put_cqp_request = true;
636
637 if (rf->reset)
638 return 0;
639
640 irdma_get_cqp_request(cqp_request);
641 status = irdma_process_cqp_cmd(dev, info);
642 if (status)
643 goto err;
644
645 if (cqp_request->waiting) {
646 put_cqp_request = false;
647 status = irdma_wait_event(rf, cqp_request);
648 if (status)
649 goto err;
650 }
651
652 return 0;
653
654 err:
655 if (irdma_cqp_crit_err(dev, info->cqp_cmd,
656 cqp_request->compl_info.maj_err_code,
657 cqp_request->compl_info.min_err_code))
658 irdma_dev_err(&rf->iwdev->ibdev,
659 "[%s Error][op_code=%d] status=%d waiting=%d completion_err=%d maj=0x%x min=0x%x\n",
660 irdma_cqp_cmd_names[info->cqp_cmd], info->cqp_cmd, status,
661 cqp_request->waiting, cqp_request->compl_info.error,
662 cqp_request->compl_info.maj_err_code,
663 cqp_request->compl_info.min_err_code);
664
665 if (put_cqp_request)
666 irdma_put_cqp_request(&rf->cqp, cqp_request);
667
668 return status;
669 }
670
671 void
irdma_qp_add_ref(struct ib_qp * ibqp)672 irdma_qp_add_ref(struct ib_qp *ibqp)
673 {
674 struct irdma_qp *iwqp = to_iwqp(ibqp);
675
676 atomic_inc(&iwqp->refcnt);
677 }
678
679 void
irdma_qp_rem_ref(struct ib_qp * ibqp)680 irdma_qp_rem_ref(struct ib_qp *ibqp)
681 {
682 struct irdma_qp *iwqp = to_iwqp(ibqp);
683 struct irdma_device *iwdev = iwqp->iwdev;
684 unsigned long flags;
685
686 spin_lock_irqsave(&iwdev->rf->qptable_lock, flags);
687 if (!atomic_dec_and_test(&iwqp->refcnt)) {
688 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags);
689 return;
690 }
691
692 iwdev->rf->qp_table[iwqp->ibqp.qp_num] = NULL;
693 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags);
694 complete(&iwqp->free_qp);
695 }
696
697 void
irdma_cq_add_ref(struct ib_cq * ibcq)698 irdma_cq_add_ref(struct ib_cq *ibcq)
699 {
700 struct irdma_cq *iwcq = to_iwcq(ibcq);
701
702 atomic_inc(&iwcq->refcnt);
703 }
704
705 void
irdma_cq_rem_ref(struct ib_cq * ibcq)706 irdma_cq_rem_ref(struct ib_cq *ibcq)
707 {
708 struct irdma_cq *iwcq = to_iwcq(ibcq);
709 struct irdma_pci_f *rf = container_of(iwcq->sc_cq.dev, struct irdma_pci_f, sc_dev);
710 unsigned long flags;
711
712 spin_lock_irqsave(&rf->cqtable_lock, flags);
713 if (!atomic_dec_and_test(&iwcq->refcnt)) {
714 spin_unlock_irqrestore(&rf->cqtable_lock, flags);
715 return;
716 }
717
718 rf->cq_table[iwcq->cq_num] = NULL;
719 spin_unlock_irqrestore(&rf->cqtable_lock, flags);
720 complete(&iwcq->free_cq);
721 }
722
723 struct ib_device *
to_ibdev(struct irdma_sc_dev * dev)724 to_ibdev(struct irdma_sc_dev *dev)
725 {
726 return &(container_of(dev, struct irdma_pci_f, sc_dev))->iwdev->ibdev;
727 }
728
729 /**
730 * irdma_get_qp - get qp address
731 * @device: iwarp device
732 * @qpn: qp number
733 */
734 struct ib_qp *
irdma_get_qp(struct ib_device * device,int qpn)735 irdma_get_qp(struct ib_device *device, int qpn)
736 {
737 struct irdma_device *iwdev = to_iwdev(device);
738
739 if (qpn < IW_FIRST_QPN || qpn >= iwdev->rf->max_qp)
740 return NULL;
741
742 return &iwdev->rf->qp_table[qpn]->ibqp;
743 }
744
745 /**
746 * irdma_remove_cqp_head - return head entry and remove
747 * @dev: device
748 */
749 void *
irdma_remove_cqp_head(struct irdma_sc_dev * dev)750 irdma_remove_cqp_head(struct irdma_sc_dev *dev)
751 {
752 struct list_head *entry;
753 struct list_head *list = &dev->cqp_cmd_head;
754
755 if (list_empty(list))
756 return NULL;
757
758 entry = list->next;
759 list_del(entry);
760
761 return entry;
762 }
763
764 /**
765 * irdma_cqp_sds_cmd - create cqp command for sd
766 * @dev: hardware control device structure
767 * @sdinfo: information for sd cqp
768 *
769 */
770 int
irdma_cqp_sds_cmd(struct irdma_sc_dev * dev,struct irdma_update_sds_info * sdinfo)771 irdma_cqp_sds_cmd(struct irdma_sc_dev *dev,
772 struct irdma_update_sds_info *sdinfo)
773 {
774 struct irdma_cqp_request *cqp_request;
775 struct cqp_cmds_info *cqp_info;
776 struct irdma_pci_f *rf = dev_to_rf(dev);
777 int status;
778
779 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
780 if (!cqp_request)
781 return -ENOMEM;
782
783 cqp_info = &cqp_request->info;
784 memcpy(&cqp_info->in.u.update_pe_sds.info, sdinfo,
785 sizeof(cqp_info->in.u.update_pe_sds.info));
786 cqp_info->cqp_cmd = IRDMA_OP_UPDATE_PE_SDS;
787 cqp_info->post_sq = 1;
788 cqp_info->in.u.update_pe_sds.dev = dev;
789 cqp_info->in.u.update_pe_sds.scratch = (uintptr_t)cqp_request;
790
791 status = irdma_handle_cqp_op(rf, cqp_request);
792 irdma_put_cqp_request(&rf->cqp, cqp_request);
793
794 return status;
795 }
796
797 /**
798 * irdma_cqp_qp_suspend_resume - cqp command for suspend/resume
799 * @qp: hardware control qp
800 * @op: suspend or resume
801 */
802 int
irdma_cqp_qp_suspend_resume(struct irdma_sc_qp * qp,u8 op)803 irdma_cqp_qp_suspend_resume(struct irdma_sc_qp *qp, u8 op)
804 {
805 struct irdma_sc_dev *dev = qp->dev;
806 struct irdma_cqp_request *cqp_request;
807 struct irdma_sc_cqp *cqp = dev->cqp;
808 struct cqp_cmds_info *cqp_info;
809 struct irdma_pci_f *rf = dev_to_rf(dev);
810 int status;
811
812 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false);
813 if (!cqp_request)
814 return -ENOMEM;
815
816 cqp_info = &cqp_request->info;
817 cqp_info->cqp_cmd = op;
818 cqp_info->in.u.suspend_resume.cqp = cqp;
819 cqp_info->in.u.suspend_resume.qp = qp;
820 cqp_info->in.u.suspend_resume.scratch = (uintptr_t)cqp_request;
821
822 status = irdma_handle_cqp_op(rf, cqp_request);
823 irdma_put_cqp_request(&rf->cqp, cqp_request);
824
825 return status;
826 }
827
828 /**
829 * irdma_term_modify_qp - modify qp for term message
830 * @qp: hardware control qp
831 * @next_state: qp's next state
832 * @term: terminate code
833 * @term_len: length
834 */
835 void
irdma_term_modify_qp(struct irdma_sc_qp * qp,u8 next_state,u8 term,u8 term_len)836 irdma_term_modify_qp(struct irdma_sc_qp *qp, u8 next_state, u8 term,
837 u8 term_len)
838 {
839 struct irdma_qp *iwqp;
840
841 iwqp = qp->qp_uk.back_qp;
842 irdma_next_iw_state(iwqp, next_state, 0, term, term_len);
843 };
844
845 /**
846 * irdma_terminate_done - after terminate is completed
847 * @qp: hardware control qp
848 * @timeout_occurred: indicates if terminate timer expired
849 */
850 void
irdma_terminate_done(struct irdma_sc_qp * qp,int timeout_occurred)851 irdma_terminate_done(struct irdma_sc_qp *qp, int timeout_occurred)
852 {
853 struct irdma_qp *iwqp;
854 u8 hte = 0;
855 bool first_time;
856 unsigned long flags;
857
858 iwqp = qp->qp_uk.back_qp;
859 spin_lock_irqsave(&iwqp->lock, flags);
860 if (iwqp->hte_added) {
861 iwqp->hte_added = 0;
862 hte = 1;
863 }
864 first_time = !(qp->term_flags & IRDMA_TERM_DONE);
865 qp->term_flags |= IRDMA_TERM_DONE;
866 spin_unlock_irqrestore(&iwqp->lock, flags);
867 if (first_time) {
868 if (!timeout_occurred)
869 irdma_terminate_del_timer(qp);
870
871 irdma_next_iw_state(iwqp, IRDMA_QP_STATE_ERROR, hte, 0, 0);
872 irdma_cm_disconn(iwqp);
873 }
874 }
875
876 static void
irdma_terminate_timeout(struct timer_list * t)877 irdma_terminate_timeout(struct timer_list *t)
878 {
879 struct irdma_qp *iwqp = from_timer(iwqp, t, terminate_timer);
880 struct irdma_sc_qp *qp = &iwqp->sc_qp;
881
882 irdma_terminate_done(qp, 1);
883 irdma_qp_rem_ref(&iwqp->ibqp);
884 }
885
886 /**
887 * irdma_terminate_start_timer - start terminate timeout
888 * @qp: hardware control qp
889 */
890 void
irdma_terminate_start_timer(struct irdma_sc_qp * qp)891 irdma_terminate_start_timer(struct irdma_sc_qp *qp)
892 {
893 struct irdma_qp *iwqp;
894
895 iwqp = qp->qp_uk.back_qp;
896 irdma_qp_add_ref(&iwqp->ibqp);
897 timer_setup(&iwqp->terminate_timer, irdma_terminate_timeout, 0);
898 iwqp->terminate_timer.expires = jiffies + HZ;
899
900 add_timer(&iwqp->terminate_timer);
901 }
902
903 /**
904 * irdma_terminate_del_timer - delete terminate timeout
905 * @qp: hardware control qp
906 */
907 void
irdma_terminate_del_timer(struct irdma_sc_qp * qp)908 irdma_terminate_del_timer(struct irdma_sc_qp *qp)
909 {
910 struct irdma_qp *iwqp;
911 int ret;
912
913 iwqp = qp->qp_uk.back_qp;
914 ret = irdma_del_timer_compat(&iwqp->terminate_timer);
915 if (ret)
916 irdma_qp_rem_ref(&iwqp->ibqp);
917 }
918
919 /**
920 * irdma_cqp_query_fpm_val_cmd - send cqp command for fpm
921 * @dev: function device struct
922 * @val_mem: buffer for fpm
923 * @hmc_fn_id: function id for fpm
924 */
925 int
irdma_cqp_query_fpm_val_cmd(struct irdma_sc_dev * dev,struct irdma_dma_mem * val_mem,u16 hmc_fn_id)926 irdma_cqp_query_fpm_val_cmd(struct irdma_sc_dev *dev,
927 struct irdma_dma_mem *val_mem, u16 hmc_fn_id)
928 {
929 struct irdma_cqp_request *cqp_request;
930 struct cqp_cmds_info *cqp_info;
931 struct irdma_pci_f *rf = dev_to_rf(dev);
932 int status;
933
934 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
935 if (!cqp_request)
936 return -ENOMEM;
937
938 cqp_info = &cqp_request->info;
939 cqp_request->param = NULL;
940 cqp_info->in.u.query_fpm_val.cqp = dev->cqp;
941 cqp_info->in.u.query_fpm_val.fpm_val_pa = val_mem->pa;
942 cqp_info->in.u.query_fpm_val.fpm_val_va = val_mem->va;
943 cqp_info->in.u.query_fpm_val.hmc_fn_id = hmc_fn_id;
944 cqp_info->cqp_cmd = IRDMA_OP_QUERY_FPM_VAL;
945 cqp_info->post_sq = 1;
946 cqp_info->in.u.query_fpm_val.scratch = (uintptr_t)cqp_request;
947
948 status = irdma_handle_cqp_op(rf, cqp_request);
949 irdma_put_cqp_request(&rf->cqp, cqp_request);
950
951 return status;
952 }
953
954 /**
955 * irdma_cqp_commit_fpm_val_cmd - commit fpm values in hw
956 * @dev: hardware control device structure
957 * @val_mem: buffer with fpm values
958 * @hmc_fn_id: function id for fpm
959 */
960 int
irdma_cqp_commit_fpm_val_cmd(struct irdma_sc_dev * dev,struct irdma_dma_mem * val_mem,u16 hmc_fn_id)961 irdma_cqp_commit_fpm_val_cmd(struct irdma_sc_dev *dev,
962 struct irdma_dma_mem *val_mem, u16 hmc_fn_id)
963 {
964 struct irdma_cqp_request *cqp_request;
965 struct cqp_cmds_info *cqp_info;
966 struct irdma_pci_f *rf = dev_to_rf(dev);
967 int status;
968
969 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
970 if (!cqp_request)
971 return -ENOMEM;
972
973 cqp_info = &cqp_request->info;
974 cqp_request->param = NULL;
975 cqp_info->in.u.commit_fpm_val.cqp = dev->cqp;
976 cqp_info->in.u.commit_fpm_val.fpm_val_pa = val_mem->pa;
977 cqp_info->in.u.commit_fpm_val.fpm_val_va = val_mem->va;
978 cqp_info->in.u.commit_fpm_val.hmc_fn_id = hmc_fn_id;
979 cqp_info->cqp_cmd = IRDMA_OP_COMMIT_FPM_VAL;
980 cqp_info->post_sq = 1;
981 cqp_info->in.u.commit_fpm_val.scratch = (uintptr_t)cqp_request;
982
983 status = irdma_handle_cqp_op(rf, cqp_request);
984 irdma_put_cqp_request(&rf->cqp, cqp_request);
985
986 return status;
987 }
988
989 /**
990 * irdma_cqp_cq_create_cmd - create a cq for the cqp
991 * @dev: device pointer
992 * @cq: pointer to created cq
993 */
994 int
irdma_cqp_cq_create_cmd(struct irdma_sc_dev * dev,struct irdma_sc_cq * cq)995 irdma_cqp_cq_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq)
996 {
997 struct irdma_pci_f *rf = dev_to_rf(dev);
998 struct irdma_cqp *iwcqp = &rf->cqp;
999 struct irdma_cqp_request *cqp_request;
1000 struct cqp_cmds_info *cqp_info;
1001 int status;
1002
1003 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);
1004 if (!cqp_request)
1005 return -ENOMEM;
1006
1007 cqp_info = &cqp_request->info;
1008 cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE;
1009 cqp_info->post_sq = 1;
1010 cqp_info->in.u.cq_create.cq = cq;
1011 cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request;
1012
1013 status = irdma_handle_cqp_op(rf, cqp_request);
1014 irdma_put_cqp_request(iwcqp, cqp_request);
1015
1016 return status;
1017 }
1018
1019 /**
1020 * irdma_cqp_qp_create_cmd - create a qp for the cqp
1021 * @dev: device pointer
1022 * @qp: pointer to created qp
1023 */
1024 int
irdma_cqp_qp_create_cmd(struct irdma_sc_dev * dev,struct irdma_sc_qp * qp)1025 irdma_cqp_qp_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
1026 {
1027 struct irdma_pci_f *rf = dev_to_rf(dev);
1028 struct irdma_cqp *iwcqp = &rf->cqp;
1029 struct irdma_cqp_request *cqp_request;
1030 struct cqp_cmds_info *cqp_info;
1031 struct irdma_create_qp_info *qp_info;
1032 int status;
1033
1034 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);
1035 if (!cqp_request)
1036 return -ENOMEM;
1037
1038 cqp_info = &cqp_request->info;
1039 qp_info = &cqp_request->info.in.u.qp_create.info;
1040 memset(qp_info, 0, sizeof(*qp_info));
1041 qp_info->cq_num_valid = true;
1042 qp_info->next_iwarp_state = IRDMA_QP_STATE_RTS;
1043 cqp_info->cqp_cmd = IRDMA_OP_QP_CREATE;
1044 cqp_info->post_sq = 1;
1045 cqp_info->in.u.qp_create.qp = qp;
1046 cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request;
1047
1048 status = irdma_handle_cqp_op(rf, cqp_request);
1049 irdma_put_cqp_request(iwcqp, cqp_request);
1050
1051 return status;
1052 }
1053
1054 /**
1055 * irdma_dealloc_push_page - free a push page for qp
1056 * @rf: RDMA PCI function
1057 * @iwqp: QP pointer
1058 */
1059 void
irdma_dealloc_push_page(struct irdma_pci_f * rf,struct irdma_qp * iwqp)1060 irdma_dealloc_push_page(struct irdma_pci_f *rf,
1061 struct irdma_qp *iwqp)
1062 {
1063 struct irdma_cqp_request *cqp_request;
1064 struct cqp_cmds_info *cqp_info;
1065 int status;
1066 struct irdma_sc_qp *qp = &iwqp->sc_qp;
1067
1068 if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX)
1069 return;
1070
1071 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false);
1072 if (!cqp_request)
1073 return;
1074
1075 cqp_info = &cqp_request->info;
1076 cqp_info->cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE;
1077 cqp_info->post_sq = 1;
1078 cqp_info->in.u.manage_push_page.info.push_idx = qp->push_idx;
1079 cqp_info->in.u.manage_push_page.info.qs_handle = qp->qs_handle;
1080 cqp_info->in.u.manage_push_page.info.free_page = 1;
1081 cqp_info->in.u.manage_push_page.info.push_page_type = 0;
1082 cqp_info->in.u.manage_push_page.cqp = &rf->cqp.sc_cqp;
1083 cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request;
1084 status = irdma_handle_cqp_op(rf, cqp_request);
1085 if (!status)
1086 qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX;
1087 irdma_put_cqp_request(&rf->cqp, cqp_request);
1088 }
1089
1090 /**
1091 * irdma_cq_wq_destroy - send cq destroy cqp
1092 * @rf: RDMA PCI function
1093 * @cq: hardware control cq
1094 */
1095 void
irdma_cq_wq_destroy(struct irdma_pci_f * rf,struct irdma_sc_cq * cq)1096 irdma_cq_wq_destroy(struct irdma_pci_f *rf, struct irdma_sc_cq *cq)
1097 {
1098 struct irdma_cqp_request *cqp_request;
1099 struct cqp_cmds_info *cqp_info;
1100
1101 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
1102 if (!cqp_request)
1103 return;
1104
1105 cqp_info = &cqp_request->info;
1106 cqp_info->cqp_cmd = IRDMA_OP_CQ_DESTROY;
1107 cqp_info->post_sq = 1;
1108 cqp_info->in.u.cq_destroy.cq = cq;
1109 cqp_info->in.u.cq_destroy.scratch = (uintptr_t)cqp_request;
1110
1111 irdma_handle_cqp_op(rf, cqp_request);
1112 irdma_put_cqp_request(&rf->cqp, cqp_request);
1113 }
1114
1115 /**
1116 * irdma_hw_modify_qp_callback - handle state for modifyQPs that don't wait
1117 * @cqp_request: modify QP completion
1118 */
1119 static void
irdma_hw_modify_qp_callback(struct irdma_cqp_request * cqp_request)1120 irdma_hw_modify_qp_callback(struct irdma_cqp_request *cqp_request)
1121 {
1122 struct cqp_cmds_info *cqp_info;
1123 struct irdma_qp *iwqp;
1124
1125 cqp_info = &cqp_request->info;
1126 iwqp = cqp_info->in.u.qp_modify.qp->qp_uk.back_qp;
1127 atomic_dec(&iwqp->hw_mod_qp_pend);
1128 wake_up(&iwqp->mod_qp_waitq);
1129 }
1130
1131 /**
1132 * irdma_hw_modify_qp - setup cqp for modify qp
1133 * @iwdev: RDMA device
1134 * @iwqp: qp ptr (user or kernel)
1135 * @info: info for modify qp
1136 * @wait: flag to wait or not for modify qp completion
1137 */
1138 int
irdma_hw_modify_qp(struct irdma_device * iwdev,struct irdma_qp * iwqp,struct irdma_modify_qp_info * info,bool wait)1139 irdma_hw_modify_qp(struct irdma_device *iwdev, struct irdma_qp *iwqp,
1140 struct irdma_modify_qp_info *info, bool wait)
1141 {
1142 int status;
1143 struct irdma_pci_f *rf = iwdev->rf;
1144 struct irdma_cqp_request *cqp_request;
1145 struct cqp_cmds_info *cqp_info;
1146 struct irdma_modify_qp_info *m_info;
1147
1148 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait);
1149 if (!cqp_request)
1150 return -ENOMEM;
1151
1152 if (!wait) {
1153 cqp_request->callback_fcn = irdma_hw_modify_qp_callback;
1154 atomic_inc(&iwqp->hw_mod_qp_pend);
1155 }
1156 cqp_info = &cqp_request->info;
1157 m_info = &cqp_info->in.u.qp_modify.info;
1158 memcpy(m_info, info, sizeof(*m_info));
1159 cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY;
1160 cqp_info->post_sq = 1;
1161 cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp;
1162 cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request;
1163 status = irdma_handle_cqp_op(rf, cqp_request);
1164 irdma_put_cqp_request(&rf->cqp, cqp_request);
1165 if (status) {
1166 if (rdma_protocol_roce(&iwdev->ibdev, 1))
1167 return status;
1168
1169 switch (m_info->next_iwarp_state) {
1170 struct irdma_gen_ae_info ae_info;
1171
1172 case IRDMA_QP_STATE_RTS:
1173 case IRDMA_QP_STATE_IDLE:
1174 case IRDMA_QP_STATE_TERMINATE:
1175 case IRDMA_QP_STATE_CLOSING:
1176 if (info->curr_iwarp_state == IRDMA_QP_STATE_IDLE)
1177 irdma_send_reset(iwqp->cm_node);
1178 else
1179 iwqp->sc_qp.term_flags = IRDMA_TERM_DONE;
1180 if (!wait) {
1181 ae_info.ae_code = IRDMA_AE_BAD_CLOSE;
1182 ae_info.ae_src = 0;
1183 irdma_gen_ae(rf, &iwqp->sc_qp, &ae_info, false);
1184 } else {
1185 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp,
1186 wait);
1187 if (!cqp_request)
1188 return -ENOMEM;
1189
1190 cqp_info = &cqp_request->info;
1191 m_info = &cqp_info->in.u.qp_modify.info;
1192 memcpy(m_info, info, sizeof(*m_info));
1193 cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY;
1194 cqp_info->post_sq = 1;
1195 cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp;
1196 cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request;
1197 m_info->next_iwarp_state = IRDMA_QP_STATE_ERROR;
1198 m_info->reset_tcp_conn = true;
1199 irdma_handle_cqp_op(rf, cqp_request);
1200 irdma_put_cqp_request(&rf->cqp, cqp_request);
1201 }
1202 break;
1203 case IRDMA_QP_STATE_ERROR:
1204 default:
1205 break;
1206 }
1207 }
1208
1209 return status;
1210 }
1211
1212 /**
1213 * irdma_cqp_cq_destroy_cmd - destroy the cqp cq
1214 * @dev: device pointer
1215 * @cq: pointer to cq
1216 */
1217 void
irdma_cqp_cq_destroy_cmd(struct irdma_sc_dev * dev,struct irdma_sc_cq * cq)1218 irdma_cqp_cq_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq)
1219 {
1220 struct irdma_pci_f *rf = dev_to_rf(dev);
1221
1222 irdma_cq_wq_destroy(rf, cq);
1223 }
1224
1225 /**
1226 * irdma_cqp_qp_destroy_cmd - destroy the cqp
1227 * @dev: device pointer
1228 * @qp: pointer to qp
1229 */
1230 int
irdma_cqp_qp_destroy_cmd(struct irdma_sc_dev * dev,struct irdma_sc_qp * qp)1231 irdma_cqp_qp_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
1232 {
1233 struct irdma_pci_f *rf = dev_to_rf(dev);
1234 struct irdma_cqp *iwcqp = &rf->cqp;
1235 struct irdma_cqp_request *cqp_request;
1236 struct cqp_cmds_info *cqp_info;
1237 int status;
1238
1239 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);
1240 if (!cqp_request)
1241 return -ENOMEM;
1242
1243 cqp_info = &cqp_request->info;
1244 memset(cqp_info, 0, sizeof(*cqp_info));
1245 cqp_info->cqp_cmd = IRDMA_OP_QP_DESTROY;
1246 cqp_info->post_sq = 1;
1247 cqp_info->in.u.qp_destroy.qp = qp;
1248 cqp_info->in.u.qp_destroy.scratch = (uintptr_t)cqp_request;
1249 cqp_info->in.u.qp_destroy.remove_hash_idx = true;
1250
1251 status = irdma_handle_cqp_op(rf, cqp_request);
1252 irdma_put_cqp_request(&rf->cqp, cqp_request);
1253
1254 return status;
1255 }
1256
1257 /**
1258 * irdma_ieq_mpa_crc_ae - generate AE for crc error
1259 * @dev: hardware control device structure
1260 * @qp: hardware control qp
1261 */
1262 void
irdma_ieq_mpa_crc_ae(struct irdma_sc_dev * dev,struct irdma_sc_qp * qp)1263 irdma_ieq_mpa_crc_ae(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
1264 {
1265 struct irdma_gen_ae_info info = {0};
1266 struct irdma_pci_f *rf = dev_to_rf(dev);
1267
1268 irdma_debug(&rf->sc_dev, IRDMA_DEBUG_AEQ, "Generate MPA CRC AE\n");
1269 info.ae_code = IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR;
1270 info.ae_src = IRDMA_AE_SOURCE_RQ;
1271 irdma_gen_ae(rf, qp, &info, false);
1272 }
1273
1274 /**
1275 * irdma_ieq_get_qp - get qp based on quad in puda buffer
1276 * @dev: hardware control device structure
1277 * @buf: receive puda buffer on exception q
1278 */
1279 struct irdma_sc_qp *
irdma_ieq_get_qp(struct irdma_sc_dev * dev,struct irdma_puda_buf * buf)1280 irdma_ieq_get_qp(struct irdma_sc_dev *dev,
1281 struct irdma_puda_buf *buf)
1282 {
1283 struct irdma_qp *iwqp;
1284 struct irdma_cm_node *cm_node;
1285 struct irdma_device *iwdev = buf->vsi->back_vsi;
1286 u32 loc_addr[4] = {0};
1287 u32 rem_addr[4] = {0};
1288 u16 loc_port, rem_port;
1289 struct ip6_hdr *ip6h;
1290 struct ip *iph = (struct ip *)buf->iph;
1291 struct tcphdr *tcph = (struct tcphdr *)buf->tcph;
1292
1293 if (iph->ip_v == 4) {
1294 loc_addr[0] = ntohl(iph->ip_dst.s_addr);
1295 rem_addr[0] = ntohl(iph->ip_src.s_addr);
1296 } else {
1297 ip6h = (struct ip6_hdr *)buf->iph;
1298 irdma_copy_ip_ntohl(loc_addr, ip6h->ip6_dst.__u6_addr.__u6_addr32);
1299 irdma_copy_ip_ntohl(rem_addr, ip6h->ip6_src.__u6_addr.__u6_addr32);
1300 }
1301 loc_port = ntohs(tcph->th_dport);
1302 rem_port = ntohs(tcph->th_sport);
1303 cm_node = irdma_find_node(&iwdev->cm_core, rem_port, rem_addr, loc_port,
1304 loc_addr, buf->vlan_valid ? buf->vlan_id : 0xFFFF);
1305 if (!cm_node)
1306 return NULL;
1307
1308 iwqp = cm_node->iwqp;
1309 irdma_rem_ref_cm_node(cm_node);
1310
1311 return &iwqp->sc_qp;
1312 }
1313
1314 /**
1315 * irdma_send_ieq_ack - ACKs for duplicate or OOO partials FPDUs
1316 * @qp: qp ptr
1317 */
1318 void
irdma_send_ieq_ack(struct irdma_sc_qp * qp)1319 irdma_send_ieq_ack(struct irdma_sc_qp *qp)
1320 {
1321 struct irdma_cm_node *cm_node = ((struct irdma_qp *)qp->qp_uk.back_qp)->cm_node;
1322 struct irdma_puda_buf *buf = qp->pfpdu.lastrcv_buf;
1323 struct tcphdr *tcph = (struct tcphdr *)buf->tcph;
1324
1325 cm_node->tcp_cntxt.rcv_nxt = qp->pfpdu.nextseqnum;
1326 cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->th_ack);
1327
1328 irdma_send_ack(cm_node);
1329 }
1330
1331 /**
1332 * irdma_puda_ieq_get_ah_info - get AH info from IEQ buffer
1333 * @qp: qp pointer
1334 * @ah_info: AH info pointer
1335 */
1336 void
irdma_puda_ieq_get_ah_info(struct irdma_sc_qp * qp,struct irdma_ah_info * ah_info)1337 irdma_puda_ieq_get_ah_info(struct irdma_sc_qp *qp,
1338 struct irdma_ah_info *ah_info)
1339 {
1340 struct irdma_puda_buf *buf = qp->pfpdu.ah_buf;
1341 struct ip *iph;
1342 struct ip6_hdr *ip6h;
1343
1344 memset(ah_info, 0, sizeof(*ah_info));
1345 ah_info->do_lpbk = true;
1346 ah_info->vlan_tag = buf->vlan_id;
1347 ah_info->insert_vlan_tag = buf->vlan_valid;
1348 ah_info->ipv4_valid = buf->ipv4;
1349 ah_info->vsi = qp->vsi;
1350
1351 if (buf->smac_valid)
1352 ether_addr_copy(ah_info->mac_addr, buf->smac);
1353
1354 if (buf->ipv4) {
1355 ah_info->ipv4_valid = true;
1356 iph = (struct ip *)buf->iph;
1357 ah_info->hop_ttl = iph->ip_ttl;
1358 ah_info->tc_tos = iph->ip_tos;
1359 ah_info->dest_ip_addr[0] = ntohl(iph->ip_dst.s_addr);
1360 ah_info->src_ip_addr[0] = ntohl(iph->ip_src.s_addr);
1361 } else {
1362 ip6h = (struct ip6_hdr *)buf->iph;
1363 ah_info->hop_ttl = ip6h->ip6_hops;
1364 ah_info->tc_tos = ip6h->ip6_vfc;
1365 irdma_copy_ip_ntohl(ah_info->dest_ip_addr,
1366 ip6h->ip6_dst.__u6_addr.__u6_addr32);
1367 irdma_copy_ip_ntohl(ah_info->src_ip_addr,
1368 ip6h->ip6_src.__u6_addr.__u6_addr32);
1369 }
1370
1371 ah_info->dst_arpindex = irdma_arp_table(dev_to_rf(qp->dev),
1372 ah_info->dest_ip_addr,
1373 NULL, IRDMA_ARP_RESOLVE);
1374 }
1375
1376 /**
1377 * irdma_gen1_ieq_update_tcpip_info - update tcpip in the buffer
1378 * @buf: puda to update
1379 * @len: length of buffer
1380 * @seqnum: seq number for tcp
1381 */
1382 static void
irdma_gen1_ieq_update_tcpip_info(struct irdma_puda_buf * buf,u16 len,u32 seqnum)1383 irdma_gen1_ieq_update_tcpip_info(struct irdma_puda_buf *buf,
1384 u16 len, u32 seqnum)
1385 {
1386 struct tcphdr *tcph;
1387 struct ip *iph;
1388 u16 iphlen;
1389 u16 pktsize;
1390 u8 *addr = buf->mem.va;
1391
1392 iphlen = (buf->ipv4) ? 20 : 40;
1393 iph = (struct ip *)(addr + buf->maclen);
1394 tcph = (struct tcphdr *)(addr + buf->maclen + iphlen);
1395 pktsize = len + buf->tcphlen + iphlen;
1396 iph->ip_len = htons(pktsize);
1397 tcph->th_seq = htonl(seqnum);
1398 }
1399
1400 /**
1401 * irdma_ieq_update_tcpip_info - update tcpip in the buffer
1402 * @buf: puda to update
1403 * @len: length of buffer
1404 * @seqnum: seq number for tcp
1405 */
1406 void
irdma_ieq_update_tcpip_info(struct irdma_puda_buf * buf,u16 len,u32 seqnum)1407 irdma_ieq_update_tcpip_info(struct irdma_puda_buf *buf, u16 len,
1408 u32 seqnum)
1409 {
1410 struct tcphdr *tcph;
1411 u8 *addr;
1412
1413 if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
1414 return irdma_gen1_ieq_update_tcpip_info(buf, len, seqnum);
1415
1416 addr = buf->mem.va;
1417 tcph = (struct tcphdr *)addr;
1418 tcph->th_seq = htonl(seqnum);
1419 }
1420
1421 /**
1422 * irdma_gen1_puda_get_tcpip_info - get tcpip info from puda
1423 * buffer
1424 * @info: to get information
1425 * @buf: puda buffer
1426 */
1427 static int
irdma_gen1_puda_get_tcpip_info(struct irdma_puda_cmpl_info * info,struct irdma_puda_buf * buf)1428 irdma_gen1_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,
1429 struct irdma_puda_buf *buf)
1430 {
1431 struct ip *iph;
1432 struct ip6_hdr *ip6h;
1433 struct tcphdr *tcph;
1434 u16 iphlen;
1435 u16 pkt_len;
1436 u8 *mem = buf->mem.va;
1437 struct ether_header *ethh = buf->mem.va;
1438
1439 if (ethh->ether_type == htons(0x8100)) {
1440 info->vlan_valid = true;
1441 buf->vlan_id = ntohs(((struct ether_vlan_header *)ethh)->evl_tag) &
1442 EVL_VLID_MASK;
1443 }
1444
1445 buf->maclen = (info->vlan_valid) ? 18 : 14;
1446 iphlen = (info->l3proto) ? 40 : 20;
1447 buf->ipv4 = (info->l3proto) ? false : true;
1448 buf->iph = mem + buf->maclen;
1449 iph = (struct ip *)buf->iph;
1450 buf->tcph = buf->iph + iphlen;
1451 tcph = (struct tcphdr *)buf->tcph;
1452
1453 if (buf->ipv4) {
1454 pkt_len = ntohs(iph->ip_len);
1455 } else {
1456 ip6h = (struct ip6_hdr *)buf->iph;
1457 pkt_len = ntohs(ip6h->ip6_plen) + iphlen;
1458 }
1459
1460 buf->totallen = pkt_len + buf->maclen;
1461
1462 if (info->payload_len < buf->totallen) {
1463 irdma_debug(buf->vsi->dev, IRDMA_DEBUG_ERR,
1464 "payload_len = 0x%x totallen expected0x%x\n",
1465 info->payload_len, buf->totallen);
1466 return -EINVAL;
1467 }
1468
1469 buf->tcphlen = tcph->th_off << 2;
1470 buf->datalen = pkt_len - iphlen - buf->tcphlen;
1471 buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL;
1472 buf->hdrlen = buf->maclen + iphlen + buf->tcphlen;
1473 buf->seqnum = ntohl(tcph->th_seq);
1474
1475 return 0;
1476 }
1477
1478 /**
1479 * irdma_puda_get_tcpip_info - get tcpip info from puda buffer
1480 * @info: to get information
1481 * @buf: puda buffer
1482 */
1483 int
irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info * info,struct irdma_puda_buf * buf)1484 irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,
1485 struct irdma_puda_buf *buf)
1486 {
1487 struct tcphdr *tcph;
1488 u32 pkt_len;
1489 u8 *mem;
1490
1491 if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
1492 return irdma_gen1_puda_get_tcpip_info(info, buf);
1493
1494 mem = buf->mem.va;
1495 buf->vlan_valid = info->vlan_valid;
1496 if (info->vlan_valid)
1497 buf->vlan_id = info->vlan;
1498
1499 buf->ipv4 = info->ipv4;
1500 if (buf->ipv4)
1501 buf->iph = mem + IRDMA_IPV4_PAD;
1502 else
1503 buf->iph = mem;
1504
1505 buf->tcph = mem + IRDMA_TCP_OFFSET;
1506 tcph = (struct tcphdr *)buf->tcph;
1507 pkt_len = info->payload_len;
1508 buf->totallen = pkt_len;
1509 buf->tcphlen = tcph->th_off << 2;
1510 buf->datalen = pkt_len - IRDMA_TCP_OFFSET - buf->tcphlen;
1511 buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL;
1512 buf->hdrlen = IRDMA_TCP_OFFSET + buf->tcphlen;
1513 buf->seqnum = ntohl(tcph->th_seq);
1514
1515 if (info->smac_valid) {
1516 ether_addr_copy(buf->smac, info->smac);
1517 buf->smac_valid = true;
1518 }
1519
1520 return 0;
1521 }
1522
1523 /**
1524 * irdma_hw_stats_timeout - Stats timer-handler which updates all HW stats
1525 * @t: timer_list pointer
1526 */
1527 static void
irdma_hw_stats_timeout(struct timer_list * t)1528 irdma_hw_stats_timeout(struct timer_list *t)
1529 {
1530 struct irdma_vsi_pestat *pf_devstat =
1531 from_timer(pf_devstat, t, stats_timer);
1532 struct irdma_sc_vsi *sc_vsi = pf_devstat->vsi;
1533
1534 if (sc_vsi->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
1535 irdma_cqp_gather_stats_cmd(sc_vsi->dev, sc_vsi->pestat, false);
1536
1537 mod_timer(&pf_devstat->stats_timer,
1538 jiffies + msecs_to_jiffies(STATS_TIMER_DELAY));
1539 }
1540
1541 /**
1542 * irdma_hw_stats_start_timer - Start periodic stats timer
1543 * @vsi: vsi structure pointer
1544 */
1545 void
irdma_hw_stats_start_timer(struct irdma_sc_vsi * vsi)1546 irdma_hw_stats_start_timer(struct irdma_sc_vsi *vsi)
1547 {
1548 struct irdma_vsi_pestat *devstat = vsi->pestat;
1549
1550 timer_setup(&devstat->stats_timer, irdma_hw_stats_timeout, 0);
1551 mod_timer(&devstat->stats_timer,
1552 jiffies + msecs_to_jiffies(STATS_TIMER_DELAY));
1553 }
1554
1555 /**
1556 * irdma_hw_stats_stop_timer - Delete periodic stats timer
1557 * @vsi: pointer to vsi structure
1558 */
1559 void
irdma_hw_stats_stop_timer(struct irdma_sc_vsi * vsi)1560 irdma_hw_stats_stop_timer(struct irdma_sc_vsi *vsi)
1561 {
1562 struct irdma_vsi_pestat *devstat = vsi->pestat;
1563
1564 del_timer_sync(&devstat->stats_timer);
1565 }
1566
1567 /**
1568 * irdma_process_cqp_stats - Checking for wrap and update stats
1569 * @cqp_request: cqp_request structure pointer
1570 */
1571 static void
irdma_process_cqp_stats(struct irdma_cqp_request * cqp_request)1572 irdma_process_cqp_stats(struct irdma_cqp_request *cqp_request)
1573 {
1574 struct irdma_vsi_pestat *pestat = cqp_request->param;
1575
1576 sc_vsi_update_stats(pestat->vsi);
1577 }
1578
1579 /**
1580 * irdma_cqp_gather_stats_cmd - Gather stats
1581 * @dev: pointer to device structure
1582 * @pestat: pointer to stats info
1583 * @wait: flag to wait or not wait for stats
1584 */
1585 int
irdma_cqp_gather_stats_cmd(struct irdma_sc_dev * dev,struct irdma_vsi_pestat * pestat,bool wait)1586 irdma_cqp_gather_stats_cmd(struct irdma_sc_dev *dev,
1587 struct irdma_vsi_pestat *pestat, bool wait)
1588 {
1589
1590 struct irdma_pci_f *rf = dev_to_rf(dev);
1591 struct irdma_cqp *iwcqp = &rf->cqp;
1592 struct irdma_cqp_request *cqp_request;
1593 struct cqp_cmds_info *cqp_info;
1594 int status;
1595
1596 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait);
1597 if (!cqp_request)
1598 return -ENOMEM;
1599
1600 cqp_info = &cqp_request->info;
1601 memset(cqp_info, 0, sizeof(*cqp_info));
1602 cqp_info->cqp_cmd = IRDMA_OP_STATS_GATHER;
1603 cqp_info->post_sq = 1;
1604 cqp_info->in.u.stats_gather.info = pestat->gather_info;
1605 cqp_info->in.u.stats_gather.scratch = (uintptr_t)cqp_request;
1606 cqp_info->in.u.stats_gather.cqp = &rf->cqp.sc_cqp;
1607 cqp_request->param = pestat;
1608 if (!wait)
1609 cqp_request->callback_fcn = irdma_process_cqp_stats;
1610 status = irdma_handle_cqp_op(rf, cqp_request);
1611 if (wait)
1612 sc_vsi_update_stats(pestat->vsi);
1613 irdma_put_cqp_request(&rf->cqp, cqp_request);
1614
1615 return status;
1616 }
1617
1618 /**
1619 * irdma_cqp_stats_inst_cmd - Allocate/free stats instance
1620 * @vsi: pointer to vsi structure
1621 * @cmd: command to allocate or free
1622 * @stats_info: pointer to allocate stats info
1623 */
1624 int
irdma_cqp_stats_inst_cmd(struct irdma_sc_vsi * vsi,u8 cmd,struct irdma_stats_inst_info * stats_info)1625 irdma_cqp_stats_inst_cmd(struct irdma_sc_vsi *vsi, u8 cmd,
1626 struct irdma_stats_inst_info *stats_info)
1627 {
1628 struct irdma_pci_f *rf = dev_to_rf(vsi->dev);
1629 struct irdma_cqp *iwcqp = &rf->cqp;
1630 struct irdma_cqp_request *cqp_request;
1631 struct cqp_cmds_info *cqp_info;
1632 int status;
1633 bool wait = false;
1634
1635 if (cmd == IRDMA_OP_STATS_ALLOCATE)
1636 wait = true;
1637 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait);
1638 if (!cqp_request)
1639 return -ENOMEM;
1640
1641 cqp_info = &cqp_request->info;
1642 memset(cqp_info, 0, sizeof(*cqp_info));
1643 cqp_info->cqp_cmd = cmd;
1644 cqp_info->post_sq = 1;
1645 cqp_info->in.u.stats_manage.info = *stats_info;
1646 cqp_info->in.u.stats_manage.scratch = (uintptr_t)cqp_request;
1647 cqp_info->in.u.stats_manage.cqp = &rf->cqp.sc_cqp;
1648 status = irdma_handle_cqp_op(rf, cqp_request);
1649 if (wait)
1650 stats_info->stats_idx = cqp_request->compl_info.op_ret_val;
1651 irdma_put_cqp_request(iwcqp, cqp_request);
1652
1653 return status;
1654 }
1655
1656 /**
1657 * irdma_cqp_ceq_cmd - Create/Destroy CEQ's after CEQ 0
1658 * @dev: pointer to device info
1659 * @sc_ceq: pointer to ceq structure
1660 * @op: Create or Destroy
1661 */
1662 int
irdma_cqp_ceq_cmd(struct irdma_sc_dev * dev,struct irdma_sc_ceq * sc_ceq,u8 op)1663 irdma_cqp_ceq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_ceq *sc_ceq,
1664 u8 op)
1665 {
1666 struct irdma_cqp_request *cqp_request;
1667 struct cqp_cmds_info *cqp_info;
1668 struct irdma_pci_f *rf = dev_to_rf(dev);
1669 int status;
1670
1671 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
1672 if (!cqp_request)
1673 return -ENOMEM;
1674
1675 cqp_info = &cqp_request->info;
1676 cqp_info->post_sq = 1;
1677 cqp_info->cqp_cmd = op;
1678 cqp_info->in.u.ceq_create.ceq = sc_ceq;
1679 cqp_info->in.u.ceq_create.scratch = (uintptr_t)cqp_request;
1680
1681 status = irdma_handle_cqp_op(rf, cqp_request);
1682 irdma_put_cqp_request(&rf->cqp, cqp_request);
1683
1684 return status;
1685 }
1686
1687 /**
1688 * irdma_cqp_aeq_cmd - Create/Destroy AEQ
1689 * @dev: pointer to device info
1690 * @sc_aeq: pointer to aeq structure
1691 * @op: Create or Destroy
1692 */
1693 int
irdma_cqp_aeq_cmd(struct irdma_sc_dev * dev,struct irdma_sc_aeq * sc_aeq,u8 op)1694 irdma_cqp_aeq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_aeq *sc_aeq,
1695 u8 op)
1696 {
1697 struct irdma_cqp_request *cqp_request;
1698 struct cqp_cmds_info *cqp_info;
1699 struct irdma_pci_f *rf = dev_to_rf(dev);
1700 int status;
1701
1702 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
1703 if (!cqp_request)
1704 return -ENOMEM;
1705
1706 cqp_info = &cqp_request->info;
1707 cqp_info->post_sq = 1;
1708 cqp_info->cqp_cmd = op;
1709 cqp_info->in.u.aeq_create.aeq = sc_aeq;
1710 cqp_info->in.u.aeq_create.scratch = (uintptr_t)cqp_request;
1711
1712 status = irdma_handle_cqp_op(rf, cqp_request);
1713 irdma_put_cqp_request(&rf->cqp, cqp_request);
1714
1715 return status;
1716 }
1717
1718 /**
1719 * irdma_cqp_ws_node_cmd - Add/modify/delete ws node
1720 * @dev: pointer to device structure
1721 * @cmd: Add, modify or delete
1722 * @node_info: pointer to ws node info
1723 */
1724 int
irdma_cqp_ws_node_cmd(struct irdma_sc_dev * dev,u8 cmd,struct irdma_ws_node_info * node_info)1725 irdma_cqp_ws_node_cmd(struct irdma_sc_dev *dev, u8 cmd,
1726 struct irdma_ws_node_info *node_info)
1727 {
1728 struct irdma_pci_f *rf = dev_to_rf(dev);
1729 struct irdma_cqp *iwcqp = &rf->cqp;
1730 struct irdma_sc_cqp *cqp = &iwcqp->sc_cqp;
1731 struct irdma_cqp_request *cqp_request;
1732 struct cqp_cmds_info *cqp_info;
1733 int status;
1734 bool poll;
1735
1736 if (!rf->sc_dev.ceq_valid)
1737 poll = true;
1738 else
1739 poll = false;
1740
1741 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, !poll);
1742 if (!cqp_request)
1743 return -ENOMEM;
1744
1745 cqp_info = &cqp_request->info;
1746 memset(cqp_info, 0, sizeof(*cqp_info));
1747 cqp_info->cqp_cmd = cmd;
1748 cqp_info->post_sq = 1;
1749 cqp_info->in.u.ws_node.info = *node_info;
1750 cqp_info->in.u.ws_node.cqp = cqp;
1751 cqp_info->in.u.ws_node.scratch = (uintptr_t)cqp_request;
1752 status = irdma_handle_cqp_op(rf, cqp_request);
1753 if (status)
1754 goto exit;
1755
1756 if (poll) {
1757 struct irdma_ccq_cqe_info compl_info;
1758
1759 status = irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_WORK_SCHED_NODE,
1760 &compl_info);
1761 node_info->qs_handle = compl_info.op_ret_val;
1762 irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB,
1763 "opcode=%d, compl_info.retval=%d\n",
1764 compl_info.op_code, compl_info.op_ret_val);
1765 } else {
1766 node_info->qs_handle = cqp_request->compl_info.op_ret_val;
1767 }
1768
1769 exit:
1770 irdma_put_cqp_request(&rf->cqp, cqp_request);
1771
1772 return status;
1773 }
1774
1775 /**
1776 * irdma_ah_cqp_op - perform an AH cqp operation
1777 * @rf: RDMA PCI function
1778 * @sc_ah: address handle
1779 * @cmd: AH operation
1780 * @wait: wait if true
1781 * @callback_fcn: Callback function on CQP op completion
1782 * @cb_param: parameter for callback function
1783 *
1784 * returns errno
1785 */
1786 int
irdma_ah_cqp_op(struct irdma_pci_f * rf,struct irdma_sc_ah * sc_ah,u8 cmd,bool wait,void (* callback_fcn)(struct irdma_cqp_request *),void * cb_param)1787 irdma_ah_cqp_op(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, u8 cmd,
1788 bool wait,
1789 void (*callback_fcn) (struct irdma_cqp_request *),
1790 void *cb_param)
1791 {
1792 struct irdma_cqp_request *cqp_request;
1793 struct cqp_cmds_info *cqp_info;
1794 int status;
1795
1796 if (cmd != IRDMA_OP_AH_CREATE && cmd != IRDMA_OP_AH_DESTROY)
1797 return -EINVAL;
1798
1799 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait);
1800 if (!cqp_request)
1801 return -ENOMEM;
1802
1803 cqp_info = &cqp_request->info;
1804 cqp_info->cqp_cmd = cmd;
1805 cqp_info->post_sq = 1;
1806 if (cmd == IRDMA_OP_AH_CREATE) {
1807 if (!wait)
1808 irdma_get_cqp_request(cqp_request);
1809 sc_ah->ah_info.cqp_request = cqp_request;
1810
1811 cqp_info->in.u.ah_create.info = sc_ah->ah_info;
1812 cqp_info->in.u.ah_create.scratch = (uintptr_t)cqp_request;
1813 cqp_info->in.u.ah_create.cqp = &rf->cqp.sc_cqp;
1814 } else if (cmd == IRDMA_OP_AH_DESTROY) {
1815 cqp_info->in.u.ah_destroy.info = sc_ah->ah_info;
1816 cqp_info->in.u.ah_destroy.scratch = (uintptr_t)cqp_request;
1817 cqp_info->in.u.ah_destroy.cqp = &rf->cqp.sc_cqp;
1818 }
1819
1820 if (!wait) {
1821 cqp_request->callback_fcn = callback_fcn;
1822 cqp_request->param = cb_param;
1823 }
1824 status = irdma_handle_cqp_op(rf, cqp_request);
1825 irdma_put_cqp_request(&rf->cqp, cqp_request);
1826
1827 if (status)
1828 return -ENOMEM;
1829
1830 if (wait)
1831 sc_ah->ah_info.ah_valid = (cmd != IRDMA_OP_AH_DESTROY);
1832
1833 return 0;
1834 }
1835
1836 /**
1837 * irdma_ieq_ah_cb - callback after creation of AH for IEQ
1838 * @cqp_request: pointer to cqp_request of create AH
1839 */
1840 static void
irdma_ieq_ah_cb(struct irdma_cqp_request * cqp_request)1841 irdma_ieq_ah_cb(struct irdma_cqp_request *cqp_request)
1842 {
1843 struct irdma_sc_qp *qp = cqp_request->param;
1844 struct irdma_sc_ah *sc_ah = qp->pfpdu.ah;
1845 unsigned long flags;
1846
1847 spin_lock_irqsave(&qp->pfpdu.lock, flags);
1848 if (!cqp_request->compl_info.op_ret_val) {
1849 sc_ah->ah_info.ah_valid = true;
1850 irdma_ieq_process_fpdus(qp, qp->vsi->ieq);
1851 } else {
1852 sc_ah->ah_info.ah_valid = false;
1853 irdma_ieq_cleanup_qp(qp->vsi->ieq, qp);
1854 }
1855 spin_unlock_irqrestore(&qp->pfpdu.lock, flags);
1856 }
1857
1858 /**
1859 * irdma_ilq_ah_cb - callback after creation of AH for ILQ
1860 * @cqp_request: pointer to cqp_request of create AH
1861 */
1862 static void
irdma_ilq_ah_cb(struct irdma_cqp_request * cqp_request)1863 irdma_ilq_ah_cb(struct irdma_cqp_request *cqp_request)
1864 {
1865 struct irdma_cm_node *cm_node = cqp_request->param;
1866 struct irdma_sc_ah *sc_ah = cm_node->ah;
1867
1868 sc_ah->ah_info.ah_valid = !cqp_request->compl_info.op_ret_val;
1869 irdma_add_conn_est_qh(cm_node);
1870 }
1871
1872 /**
1873 * irdma_puda_create_ah - create AH for ILQ/IEQ qp's
1874 * @dev: device pointer
1875 * @ah_info: Address handle info
1876 * @wait: When true will wait for operation to complete
1877 * @type: ILQ/IEQ
1878 * @cb_param: Callback param when not waiting
1879 * @ah_ret: Returned pointer to address handle if created
1880 *
1881 */
1882 int
irdma_puda_create_ah(struct irdma_sc_dev * dev,struct irdma_ah_info * ah_info,bool wait,enum puda_rsrc_type type,void * cb_param,struct irdma_sc_ah ** ah_ret)1883 irdma_puda_create_ah(struct irdma_sc_dev *dev,
1884 struct irdma_ah_info *ah_info, bool wait,
1885 enum puda_rsrc_type type, void *cb_param,
1886 struct irdma_sc_ah **ah_ret)
1887 {
1888 struct irdma_sc_ah *ah;
1889 struct irdma_pci_f *rf = dev_to_rf(dev);
1890 int err;
1891
1892 ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
1893 *ah_ret = ah;
1894 if (!ah)
1895 return -ENOMEM;
1896
1897 err = irdma_alloc_rsrc(rf, rf->allocated_ahs, rf->max_ah,
1898 &ah_info->ah_idx, &rf->next_ah);
1899 if (err)
1900 goto err_free;
1901
1902 ah->dev = dev;
1903 ah->ah_info = *ah_info;
1904
1905 if (type == IRDMA_PUDA_RSRC_TYPE_ILQ)
1906 err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait,
1907 irdma_ilq_ah_cb, cb_param);
1908 else
1909 err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait,
1910 irdma_ieq_ah_cb, cb_param);
1911
1912 if (err)
1913 goto error;
1914 return 0;
1915
1916 error:
1917 irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx);
1918 err_free:
1919 kfree(ah);
1920 *ah_ret = NULL;
1921 return -ENOMEM;
1922 }
1923
1924 /**
1925 * irdma_puda_free_ah - free a puda address handle
1926 * @dev: device pointer
1927 * @ah: The address handle to free
1928 */
1929 void
irdma_puda_free_ah(struct irdma_sc_dev * dev,struct irdma_sc_ah * ah)1930 irdma_puda_free_ah(struct irdma_sc_dev *dev, struct irdma_sc_ah *ah)
1931 {
1932 struct irdma_pci_f *rf = dev_to_rf(dev);
1933
1934 if (!ah)
1935 return;
1936
1937 if (ah->ah_info.ah_valid) {
1938 irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_DESTROY, false, NULL, NULL);
1939 irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx);
1940 }
1941
1942 kfree(ah);
1943 }
1944
1945 /**
1946 * irdma_prm_add_pble_mem - add moemory to pble resources
1947 * @pprm: pble resource manager
1948 * @pchunk: chunk of memory to add
1949 */
1950 int
irdma_prm_add_pble_mem(struct irdma_pble_prm * pprm,struct irdma_chunk * pchunk)1951 irdma_prm_add_pble_mem(struct irdma_pble_prm *pprm,
1952 struct irdma_chunk *pchunk)
1953 {
1954 u64 sizeofbitmap;
1955
1956 if (pchunk->size & 0xfff)
1957 return -EINVAL;
1958
1959 sizeofbitmap = (u64)pchunk->size >> pprm->pble_shift;
1960
1961 pchunk->bitmapbuf = bitmap_zalloc(sizeofbitmap, GFP_KERNEL);
1962 if (!pchunk->bitmapbuf)
1963 return -ENOMEM;
1964
1965 pchunk->sizeofbitmap = sizeofbitmap;
1966 /* each pble is 8 bytes hence shift by 3 */
1967 pprm->total_pble_alloc += pchunk->size >> 3;
1968 pprm->free_pble_cnt += pchunk->size >> 3;
1969
1970 return 0;
1971 }
1972
1973 /**
1974 * irdma_prm_get_pbles - get pble's from prm
1975 * @pprm: pble resource manager
1976 * @chunkinfo: nformation about chunk where pble's were acquired
1977 * @mem_size: size of pble memory needed
1978 * @vaddr: returns virtual address of pble memory
1979 * @fpm_addr: returns fpm address of pble memory
1980 */
1981 int
irdma_prm_get_pbles(struct irdma_pble_prm * pprm,struct irdma_pble_chunkinfo * chunkinfo,u64 mem_size,u64 ** vaddr,u64 * fpm_addr)1982 irdma_prm_get_pbles(struct irdma_pble_prm *pprm,
1983 struct irdma_pble_chunkinfo *chunkinfo, u64 mem_size,
1984 u64 **vaddr, u64 *fpm_addr)
1985 {
1986 u64 bits_needed;
1987 u64 bit_idx = PBLE_INVALID_IDX;
1988 struct irdma_chunk *pchunk = NULL;
1989 struct list_head *chunk_entry = (&pprm->clist)->next;
1990 u32 offset;
1991 unsigned long flags;
1992
1993 *vaddr = NULL;
1994 *fpm_addr = 0;
1995
1996 bits_needed = DIV_ROUND_UP_ULL(mem_size, BIT_ULL(pprm->pble_shift));
1997
1998 spin_lock_irqsave(&pprm->prm_lock, flags);
1999 while (chunk_entry != &pprm->clist) {
2000 pchunk = (struct irdma_chunk *)chunk_entry;
2001 bit_idx = bitmap_find_next_zero_area(pchunk->bitmapbuf,
2002 pchunk->sizeofbitmap, 0,
2003 bits_needed, 0);
2004 if (bit_idx < pchunk->sizeofbitmap)
2005 break;
2006
2007 /* list.next used macro */
2008 chunk_entry = (&pchunk->list)->next;
2009 }
2010
2011 if (!pchunk || bit_idx >= pchunk->sizeofbitmap) {
2012 spin_unlock_irqrestore(&pprm->prm_lock, flags);
2013 return -ENOMEM;
2014 }
2015
2016 bitmap_set(pchunk->bitmapbuf, bit_idx, bits_needed);
2017 offset = bit_idx << pprm->pble_shift;
2018 *vaddr = (u64 *)((u8 *)pchunk->vaddr + offset);
2019 *fpm_addr = pchunk->fpm_addr + offset;
2020
2021 chunkinfo->pchunk = pchunk;
2022 chunkinfo->bit_idx = bit_idx;
2023 chunkinfo->bits_used = bits_needed;
2024 /* 3 is sizeof pble divide */
2025 pprm->free_pble_cnt -= chunkinfo->bits_used << (pprm->pble_shift - 3);
2026 spin_unlock_irqrestore(&pprm->prm_lock, flags);
2027
2028 return 0;
2029 }
2030
2031 /**
2032 * irdma_prm_return_pbles - return pbles back to prm
2033 * @pprm: pble resource manager
2034 * @chunkinfo: chunk where pble's were acquired and to be freed
2035 */
2036 void
irdma_prm_return_pbles(struct irdma_pble_prm * pprm,struct irdma_pble_chunkinfo * chunkinfo)2037 irdma_prm_return_pbles(struct irdma_pble_prm *pprm,
2038 struct irdma_pble_chunkinfo *chunkinfo)
2039 {
2040 unsigned long flags;
2041
2042 spin_lock_irqsave(&pprm->prm_lock, flags);
2043 pprm->free_pble_cnt += chunkinfo->bits_used << (pprm->pble_shift - 3);
2044 bitmap_clear(chunkinfo->pchunk->bitmapbuf, chunkinfo->bit_idx,
2045 chunkinfo->bits_used);
2046 spin_unlock_irqrestore(&pprm->prm_lock, flags);
2047 }
2048
2049 int
irdma_map_vm_page_list(struct irdma_hw * hw,void * va,dma_addr_t * pg_dma,u32 pg_cnt)2050 irdma_map_vm_page_list(struct irdma_hw *hw, void *va, dma_addr_t * pg_dma,
2051 u32 pg_cnt)
2052 {
2053 struct page *vm_page;
2054 int i;
2055 u8 *addr;
2056
2057 addr = (u8 *)(uintptr_t)va;
2058 for (i = 0; i < pg_cnt; i++) {
2059 vm_page = vmalloc_to_page(addr);
2060 if (!vm_page)
2061 goto err;
2062
2063 pg_dma[i] = dma_map_page(hw_to_dev(hw), vm_page, 0, PAGE_SIZE, DMA_BIDIRECTIONAL);
2064 if (dma_mapping_error(hw_to_dev(hw), pg_dma[i]))
2065 goto err;
2066
2067 addr += PAGE_SIZE;
2068 }
2069
2070 return 0;
2071
2072 err:
2073 irdma_unmap_vm_page_list(hw, pg_dma, i);
2074 return -ENOMEM;
2075 }
2076
2077 void
irdma_unmap_vm_page_list(struct irdma_hw * hw,dma_addr_t * pg_dma,u32 pg_cnt)2078 irdma_unmap_vm_page_list(struct irdma_hw *hw, dma_addr_t * pg_dma, u32 pg_cnt)
2079 {
2080 int i;
2081
2082 for (i = 0; i < pg_cnt; i++)
2083 dma_unmap_page(hw_to_dev(hw), pg_dma[i], PAGE_SIZE, DMA_BIDIRECTIONAL);
2084 }
2085
2086 /**
2087 * irdma_pble_free_paged_mem - free virtual paged memory
2088 * @chunk: chunk to free with paged memory
2089 */
2090 void
irdma_pble_free_paged_mem(struct irdma_chunk * chunk)2091 irdma_pble_free_paged_mem(struct irdma_chunk *chunk)
2092 {
2093 if (!chunk->pg_cnt)
2094 goto done;
2095
2096 irdma_unmap_vm_page_list(chunk->dev->hw, chunk->dmainfo.dmaaddrs,
2097 chunk->pg_cnt);
2098
2099 done:
2100 kfree(chunk->dmainfo.dmaaddrs);
2101 chunk->dmainfo.dmaaddrs = NULL;
2102 vfree(chunk->vaddr);
2103 chunk->vaddr = NULL;
2104 chunk->type = 0;
2105 }
2106
2107 /**
2108 * irdma_pble_get_paged_mem -allocate paged memory for pbles
2109 * @chunk: chunk to add for paged memory
2110 * @pg_cnt: number of pages needed
2111 */
2112 int
irdma_pble_get_paged_mem(struct irdma_chunk * chunk,u32 pg_cnt)2113 irdma_pble_get_paged_mem(struct irdma_chunk *chunk, u32 pg_cnt)
2114 {
2115 u32 size;
2116 void *va;
2117
2118 chunk->dmainfo.dmaaddrs = kzalloc(pg_cnt << 3, GFP_KERNEL);
2119 if (!chunk->dmainfo.dmaaddrs)
2120 return -ENOMEM;
2121
2122 size = PAGE_SIZE * pg_cnt;
2123 va = vmalloc(size);
2124 if (!va)
2125 goto err;
2126
2127 if (irdma_map_vm_page_list(chunk->dev->hw, va, chunk->dmainfo.dmaaddrs,
2128 pg_cnt)) {
2129 vfree(va);
2130 goto err;
2131 }
2132 chunk->vaddr = va;
2133 chunk->size = size;
2134 chunk->pg_cnt = pg_cnt;
2135 chunk->type = PBLE_SD_PAGED;
2136
2137 return 0;
2138 err:
2139 kfree(chunk->dmainfo.dmaaddrs);
2140 chunk->dmainfo.dmaaddrs = NULL;
2141
2142 return -ENOMEM;
2143 }
2144
2145 /**
2146 * irdma_alloc_ws_node_id - Allocate a tx scheduler node ID
2147 * @dev: device pointer
2148 */
2149 u16
irdma_alloc_ws_node_id(struct irdma_sc_dev * dev)2150 irdma_alloc_ws_node_id(struct irdma_sc_dev *dev)
2151 {
2152 struct irdma_pci_f *rf = dev_to_rf(dev);
2153 u32 next = 1;
2154 u32 node_id;
2155
2156 if (irdma_alloc_rsrc(rf, rf->allocated_ws_nodes, rf->max_ws_node_id,
2157 &node_id, &next))
2158 return IRDMA_WS_NODE_INVALID;
2159
2160 return (u16)node_id;
2161 }
2162
2163 /**
2164 * irdma_free_ws_node_id - Free a tx scheduler node ID
2165 * @dev: device pointer
2166 * @node_id: Work scheduler node ID
2167 */
2168 void
irdma_free_ws_node_id(struct irdma_sc_dev * dev,u16 node_id)2169 irdma_free_ws_node_id(struct irdma_sc_dev *dev, u16 node_id)
2170 {
2171 struct irdma_pci_f *rf = dev_to_rf(dev);
2172
2173 irdma_free_rsrc(rf, rf->allocated_ws_nodes, (u32)node_id);
2174 }
2175
2176 /**
2177 * irdma_modify_qp_to_err - Modify a QP to error
2178 * @sc_qp: qp structure
2179 */
2180 void
irdma_modify_qp_to_err(struct irdma_sc_qp * sc_qp)2181 irdma_modify_qp_to_err(struct irdma_sc_qp *sc_qp)
2182 {
2183 struct irdma_qp *qp = sc_qp->qp_uk.back_qp;
2184 struct ib_qp_attr attr;
2185
2186 if (qp->iwdev->rf->reset)
2187 return;
2188 attr.qp_state = IB_QPS_ERR;
2189
2190 if (rdma_protocol_roce(qp->ibqp.device, 1))
2191 irdma_modify_qp_roce(&qp->ibqp, &attr, IB_QP_STATE, NULL);
2192 else
2193 irdma_modify_qp(&qp->ibqp, &attr, IB_QP_STATE, NULL);
2194 }
2195
2196 void
irdma_ib_qp_event(struct irdma_qp * iwqp,enum irdma_qp_event_type event)2197 irdma_ib_qp_event(struct irdma_qp *iwqp, enum irdma_qp_event_type event)
2198 {
2199 struct ib_event ibevent;
2200
2201 if (!iwqp->ibqp.event_handler)
2202 return;
2203
2204 switch (event) {
2205 case IRDMA_QP_EVENT_CATASTROPHIC:
2206 ibevent.event = IB_EVENT_QP_FATAL;
2207 break;
2208 case IRDMA_QP_EVENT_ACCESS_ERR:
2209 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
2210 break;
2211 case IRDMA_QP_EVENT_REQ_ERR:
2212 ibevent.event = IB_EVENT_QP_REQ_ERR;
2213 break;
2214 }
2215 ibevent.device = iwqp->ibqp.device;
2216 ibevent.element.qp = &iwqp->ibqp;
2217 iwqp->ibqp.event_handler(&ibevent, iwqp->ibqp.qp_context);
2218 }
2219
2220 static void
clear_qp_ctx_addr(__le64 * ctx)2221 clear_qp_ctx_addr(__le64 * ctx)
2222 {
2223 u64 tmp;
2224
2225 get_64bit_val(ctx, 272, &tmp);
2226 tmp &= GENMASK_ULL(63, 58);
2227 set_64bit_val(ctx, 272, tmp);
2228
2229 get_64bit_val(ctx, 296, &tmp);
2230 tmp &= GENMASK_ULL(7, 0);
2231 set_64bit_val(ctx, 296, tmp);
2232
2233 get_64bit_val(ctx, 312, &tmp);
2234 tmp &= GENMASK_ULL(7, 0);
2235 set_64bit_val(ctx, 312, tmp);
2236
2237 set_64bit_val(ctx, 368, 0);
2238 }
2239
2240 /**
2241 * irdma_upload_qp_context - upload raw QP context
2242 * @iwqp: QP pointer
2243 * @freeze: freeze QP
2244 * @raw: raw context flag
2245 */
2246 int
irdma_upload_qp_context(struct irdma_qp * iwqp,bool freeze,bool raw)2247 irdma_upload_qp_context(struct irdma_qp *iwqp, bool freeze, bool raw)
2248 {
2249 struct irdma_dma_mem dma_mem;
2250 struct irdma_sc_dev *dev;
2251 struct irdma_sc_qp *qp;
2252 struct irdma_cqp *iwcqp;
2253 struct irdma_cqp_request *cqp_request;
2254 struct cqp_cmds_info *cqp_info;
2255 struct irdma_upload_context_info *info;
2256 struct irdma_pci_f *rf;
2257 int ret;
2258 u32 *ctx;
2259
2260 rf = iwqp->iwdev->rf;
2261 if (!rf)
2262 return -EINVAL;
2263
2264 qp = &iwqp->sc_qp;
2265 dev = &rf->sc_dev;
2266 iwcqp = &rf->cqp;
2267
2268 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);
2269 if (!cqp_request)
2270 return -EINVAL;
2271
2272 cqp_info = &cqp_request->info;
2273 info = &cqp_info->in.u.qp_upload_context.info;
2274 memset(info, 0, sizeof(struct irdma_upload_context_info));
2275 cqp_info->cqp_cmd = IRDMA_OP_QP_UPLOAD_CONTEXT;
2276 cqp_info->post_sq = 1;
2277 cqp_info->in.u.qp_upload_context.dev = dev;
2278 cqp_info->in.u.qp_upload_context.scratch = (uintptr_t)cqp_request;
2279
2280 dma_mem.size = PAGE_SIZE;
2281 dma_mem.va = irdma_allocate_dma_mem(dev->hw, &dma_mem, dma_mem.size, PAGE_SIZE);
2282 if (!dma_mem.va) {
2283 irdma_put_cqp_request(&rf->cqp, cqp_request);
2284 return -ENOMEM;
2285 }
2286
2287 ctx = dma_mem.va;
2288 info->buf_pa = dma_mem.pa;
2289 info->raw_format = raw;
2290 info->freeze_qp = freeze;
2291 info->qp_type = qp->qp_uk.qp_type; /* 1 is iWARP and 2 UDA */
2292 info->qp_id = qp->qp_uk.qp_id;
2293 ret = irdma_handle_cqp_op(rf, cqp_request);
2294 if (ret)
2295 goto error;
2296 irdma_debug(dev, IRDMA_DEBUG_QP, "PRINT CONTXT QP [%d]\n", info->qp_id);
2297 {
2298 u32 i, j;
2299
2300 clear_qp_ctx_addr(dma_mem.va);
2301 for (i = 0, j = 0; i < 32; i++, j += 4)
2302 irdma_debug(dev, IRDMA_DEBUG_QP,
2303 "%d:\t [%08X %08x %08X %08X]\n", (j * 4),
2304 ctx[j], ctx[j + 1], ctx[j + 2], ctx[j + 3]);
2305 }
2306 error:
2307 irdma_put_cqp_request(iwcqp, cqp_request);
2308 irdma_free_dma_mem(dev->hw, &dma_mem);
2309
2310 return ret;
2311 }
2312
2313 bool
irdma_cq_empty(struct irdma_cq * iwcq)2314 irdma_cq_empty(struct irdma_cq *iwcq)
2315 {
2316 struct irdma_cq_uk *ukcq;
2317 u64 qword3;
2318 __le64 *cqe;
2319 u8 polarity;
2320
2321 ukcq = &iwcq->sc_cq.cq_uk;
2322 cqe = IRDMA_GET_CURRENT_CQ_ELEM(ukcq);
2323 get_64bit_val(cqe, 24, &qword3);
2324 polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3);
2325
2326 return polarity != ukcq->polarity;
2327 }
2328
2329 void
irdma_remove_cmpls_list(struct irdma_cq * iwcq)2330 irdma_remove_cmpls_list(struct irdma_cq *iwcq)
2331 {
2332 struct irdma_cmpl_gen *cmpl_node;
2333 struct list_head *tmp_node, *list_node;
2334
2335 list_for_each_safe(list_node, tmp_node, &iwcq->cmpl_generated) {
2336 cmpl_node = list_entry(list_node, struct irdma_cmpl_gen, list);
2337 list_del(&cmpl_node->list);
2338 kfree(cmpl_node);
2339 }
2340 }
2341
2342 int
irdma_generated_cmpls(struct irdma_cq * iwcq,struct irdma_cq_poll_info * cq_poll_info)2343 irdma_generated_cmpls(struct irdma_cq *iwcq, struct irdma_cq_poll_info *cq_poll_info)
2344 {
2345 struct irdma_cmpl_gen *cmpl;
2346
2347 if (list_empty(&iwcq->cmpl_generated))
2348 return -ENOENT;
2349 cmpl = list_first_entry_or_null(&iwcq->cmpl_generated, struct irdma_cmpl_gen, list);
2350 list_del(&cmpl->list);
2351 memcpy(cq_poll_info, &cmpl->cpi, sizeof(*cq_poll_info));
2352 kfree(cmpl);
2353
2354 irdma_debug(iwcq->sc_cq.dev, IRDMA_DEBUG_VERBS,
2355 "%s: Poll artificially generated completion for QP 0x%X, op %u, wr_id=0x%lx\n",
2356 __func__, cq_poll_info->qp_id, cq_poll_info->op_type,
2357 cq_poll_info->wr_id);
2358
2359 return 0;
2360 }
2361
2362 /**
2363 * irdma_set_cpi_common_values - fill in values for polling info struct
2364 * @cpi: resulting structure of cq_poll_info type
2365 * @qp: QPair
2366 * @qp_num: id of the QP
2367 */
2368 static void
irdma_set_cpi_common_values(struct irdma_cq_poll_info * cpi,struct irdma_qp_uk * qp,u32 qp_num)2369 irdma_set_cpi_common_values(struct irdma_cq_poll_info *cpi,
2370 struct irdma_qp_uk *qp, u32 qp_num)
2371 {
2372 cpi->comp_status = IRDMA_COMPL_STATUS_FLUSHED;
2373 cpi->error = 1;
2374 cpi->major_err = IRDMA_FLUSH_MAJOR_ERR;
2375 cpi->minor_err = FLUSH_GENERAL_ERR;
2376 cpi->qp_handle = (irdma_qp_handle) (uintptr_t)qp;
2377 cpi->qp_id = qp_num;
2378 }
2379
2380 static inline void
irdma_comp_handler(struct irdma_cq * cq)2381 irdma_comp_handler(struct irdma_cq *cq)
2382 {
2383 if (!cq->ibcq.comp_handler)
2384 return;
2385
2386 if (atomic_cmpxchg(&cq->armed, 1, 0))
2387 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
2388 }
2389
2390 /**
2391 * irdma_generate_flush_completions - generate completion from WRs
2392 * @iwqp: pointer to QP
2393 */
2394 void
irdma_generate_flush_completions(struct irdma_qp * iwqp)2395 irdma_generate_flush_completions(struct irdma_qp *iwqp)
2396 {
2397 struct irdma_qp_uk *qp = &iwqp->sc_qp.qp_uk;
2398 struct irdma_ring *sq_ring = &qp->sq_ring;
2399 struct irdma_ring *rq_ring = &qp->rq_ring;
2400 struct irdma_cmpl_gen *cmpl;
2401 __le64 *sw_wqe;
2402 u64 wqe_qword;
2403 u32 wqe_idx;
2404 bool compl_generated = false;
2405 unsigned long flags1;
2406
2407 spin_lock_irqsave(&iwqp->iwscq->lock, flags1);
2408 if (irdma_cq_empty(iwqp->iwscq)) {
2409 unsigned long flags2;
2410
2411 spin_lock_irqsave(&iwqp->lock, flags2);
2412 while (IRDMA_RING_MORE_WORK(*sq_ring)) {
2413 cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC);
2414 if (!cmpl) {
2415 spin_unlock_irqrestore(&iwqp->lock, flags2);
2416 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1);
2417 return;
2418 }
2419
2420 wqe_idx = sq_ring->tail;
2421 irdma_set_cpi_common_values(&cmpl->cpi, qp, qp->qp_id);
2422
2423 cmpl->cpi.wr_id = qp->sq_wrtrk_array[wqe_idx].wrid;
2424 cmpl->cpi.signaled = qp->sq_wrtrk_array[wqe_idx].signaled;
2425 sw_wqe = qp->sq_base[wqe_idx].elem;
2426 get_64bit_val(sw_wqe, IRDMA_BYTE_24, &wqe_qword);
2427 cmpl->cpi.op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, wqe_qword);
2428 cmpl->cpi.q_type = IRDMA_CQE_QTYPE_SQ;
2429 /* remove the SQ WR by moving SQ tail */
2430 IRDMA_RING_SET_TAIL(*sq_ring,
2431 sq_ring->tail + qp->sq_wrtrk_array[sq_ring->tail].quanta);
2432
2433 if (cmpl->cpi.op_type == IRDMAQP_OP_NOP) {
2434 kfree(cmpl);
2435 continue;
2436 }
2437 irdma_debug(iwqp->sc_qp.dev, IRDMA_DEBUG_DEV,
2438 "%s: adding wr_id = 0x%lx SQ Completion to list qp_id=%d\n",
2439 __func__, cmpl->cpi.wr_id, qp->qp_id);
2440 list_add_tail(&cmpl->list, &iwqp->iwscq->cmpl_generated);
2441 compl_generated = true;
2442 }
2443 spin_unlock_irqrestore(&iwqp->lock, flags2);
2444 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1);
2445 if (compl_generated) {
2446 irdma_comp_handler(iwqp->iwscq);
2447 compl_generated = false;
2448 }
2449 } else {
2450 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1);
2451 irdma_sched_qp_flush_work(iwqp);
2452 }
2453
2454 spin_lock_irqsave(&iwqp->iwrcq->lock, flags1);
2455 if (irdma_cq_empty(iwqp->iwrcq)) {
2456 unsigned long flags2;
2457
2458 spin_lock_irqsave(&iwqp->lock, flags2);
2459 while (IRDMA_RING_MORE_WORK(*rq_ring)) {
2460 cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC);
2461 if (!cmpl) {
2462 spin_unlock_irqrestore(&iwqp->lock, flags2);
2463 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1);
2464 return;
2465 }
2466
2467 wqe_idx = rq_ring->tail;
2468 irdma_set_cpi_common_values(&cmpl->cpi, qp, qp->qp_id);
2469
2470 cmpl->cpi.wr_id = qp->rq_wrid_array[wqe_idx];
2471 cmpl->cpi.signaled = 1;
2472 cmpl->cpi.op_type = IRDMA_OP_TYPE_REC;
2473 cmpl->cpi.q_type = IRDMA_CQE_QTYPE_RQ;
2474 /* remove the RQ WR by moving RQ tail */
2475 IRDMA_RING_SET_TAIL(*rq_ring, rq_ring->tail + 1);
2476 irdma_debug(iwqp->sc_qp.dev, IRDMA_DEBUG_DEV,
2477 "%s: adding wr_id = 0x%lx RQ Completion to list qp_id=%d, wqe_idx=%d\n",
2478 __func__, cmpl->cpi.wr_id, qp->qp_id,
2479 wqe_idx);
2480
2481 list_add_tail(&cmpl->list, &iwqp->iwrcq->cmpl_generated);
2482
2483 compl_generated = true;
2484 }
2485 spin_unlock_irqrestore(&iwqp->lock, flags2);
2486 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1);
2487 if (compl_generated)
2488 irdma_comp_handler(iwqp->iwrcq);
2489 } else {
2490 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1);
2491 irdma_sched_qp_flush_work(iwqp);
2492 }
2493 }
2494
2495 /**
2496 * irdma_udqp_qs_change - change qs for UD QP in a worker thread
2497 * @iwqp: QP pointer
2498 * @user_prio: new user priority value
2499 * @qs_change: when false, only user priority changes, QS handle do not need to change
2500 */
2501 static void
irdma_udqp_qs_change(struct irdma_qp * iwqp,u8 user_prio,bool qs_change)2502 irdma_udqp_qs_change(struct irdma_qp *iwqp, u8 user_prio, bool qs_change)
2503 {
2504 irdma_qp_rem_qos(&iwqp->sc_qp);
2505 if (qs_change)
2506 iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi, iwqp->ctx_info.user_pri);
2507
2508 iwqp->ctx_info.user_pri = user_prio;
2509 iwqp->sc_qp.user_pri = user_prio;
2510
2511 if (qs_change)
2512 if (iwqp->sc_qp.dev->ws_add(iwqp->sc_qp.vsi, user_prio))
2513 irdma_dev_warn(&iwqp->iwdev->ibdev,
2514 "WS add failed during %s, qp_id: %x user_pri: %x",
2515 __func__, iwqp->ibqp.qp_num, user_prio);
2516 irdma_qp_add_qos(&iwqp->sc_qp);
2517 }
2518
2519 void
irdma_udqp_qs_worker(struct work_struct * work)2520 irdma_udqp_qs_worker(struct work_struct *work)
2521 {
2522 struct irdma_udqs_work *udqs_work = container_of(work, struct irdma_udqs_work, work);
2523
2524 irdma_udqp_qs_change(udqs_work->iwqp, udqs_work->user_prio, udqs_work->qs_change);
2525 if (udqs_work->qs_change)
2526 irdma_cqp_qp_suspend_resume(&udqs_work->iwqp->sc_qp, IRDMA_OP_RESUME);
2527 irdma_qp_rem_ref(&udqs_work->iwqp->ibqp);
2528 kfree(udqs_work);
2529 }
2530