xref: /freebsd/sys/dev/irdma/irdma_utils.c (revision 5b5f7d0e77a9eee73eb5d596f43aef4e1a3674d8)
1cdcd52d4SBartosz Sobczak /*-
2cdcd52d4SBartosz Sobczak  * SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
3cdcd52d4SBartosz Sobczak  *
401fbb869SBartosz Sobczak  * Copyright (c) 2015 - 2023 Intel Corporation
5cdcd52d4SBartosz Sobczak  *
6cdcd52d4SBartosz Sobczak  * This software is available to you under a choice of one of two
7cdcd52d4SBartosz Sobczak  * licenses.  You may choose to be licensed under the terms of the GNU
8cdcd52d4SBartosz Sobczak  * General Public License (GPL) Version 2, available from the file
9cdcd52d4SBartosz Sobczak  * COPYING in the main directory of this source tree, or the
10cdcd52d4SBartosz Sobczak  * OpenFabrics.org BSD license below:
11cdcd52d4SBartosz Sobczak  *
12cdcd52d4SBartosz Sobczak  *   Redistribution and use in source and binary forms, with or
13cdcd52d4SBartosz Sobczak  *   without modification, are permitted provided that the following
14cdcd52d4SBartosz Sobczak  *   conditions are met:
15cdcd52d4SBartosz Sobczak  *
16cdcd52d4SBartosz Sobczak  *    - Redistributions of source code must retain the above
17cdcd52d4SBartosz Sobczak  *	copyright notice, this list of conditions and the following
18cdcd52d4SBartosz Sobczak  *	disclaimer.
19cdcd52d4SBartosz Sobczak  *
20cdcd52d4SBartosz Sobczak  *    - Redistributions in binary form must reproduce the above
21cdcd52d4SBartosz Sobczak  *	copyright notice, this list of conditions and the following
22cdcd52d4SBartosz Sobczak  *	disclaimer in the documentation and/or other materials
23cdcd52d4SBartosz Sobczak  *	provided with the distribution.
24cdcd52d4SBartosz Sobczak  *
25cdcd52d4SBartosz Sobczak  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26cdcd52d4SBartosz Sobczak  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27cdcd52d4SBartosz Sobczak  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28cdcd52d4SBartosz Sobczak  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29cdcd52d4SBartosz Sobczak  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30cdcd52d4SBartosz Sobczak  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31cdcd52d4SBartosz Sobczak  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32cdcd52d4SBartosz Sobczak  * SOFTWARE.
33cdcd52d4SBartosz Sobczak  */
34cdcd52d4SBartosz Sobczak 
35cdcd52d4SBartosz Sobczak #include "irdma_main.h"
36cdcd52d4SBartosz Sobczak 
37cdcd52d4SBartosz Sobczak LIST_HEAD(irdma_handlers);
38cdcd52d4SBartosz Sobczak DEFINE_SPINLOCK(irdma_handler_lock);
39cdcd52d4SBartosz Sobczak 
4001fbb869SBartosz Sobczak static const struct ae_desc ae_desc_list[] = {
4101fbb869SBartosz Sobczak 	{IRDMA_AE_AMP_UNALLOCATED_STAG, "Unallocated memory key (L-Key/R-Key)"},
4201fbb869SBartosz Sobczak 	{IRDMA_AE_AMP_INVALID_STAG, "Invalid memory key (L-Key/R-Key)"},
4301fbb869SBartosz Sobczak 	{IRDMA_AE_AMP_BAD_QP,
4401fbb869SBartosz Sobczak 	"Memory protection error: Accessing Memory Window (MW) which belongs to a different QP"},
4501fbb869SBartosz Sobczak 	{IRDMA_AE_AMP_BAD_PD,
4601fbb869SBartosz Sobczak 	"Memory protection error: Accessing Memory Window (MW)/Memory Region (MR) which belongs to a different PD"},
4701fbb869SBartosz Sobczak 	{IRDMA_AE_AMP_BAD_STAG_KEY, "Bad memory key (L-Key/R-Key)"},
4801fbb869SBartosz Sobczak 	{IRDMA_AE_AMP_BAD_STAG_INDEX, "Bad memory key (L-Key/R-Key): Too large memory key index"},
4901fbb869SBartosz Sobczak 	{IRDMA_AE_AMP_BOUNDS_VIOLATION, "Memory Window (MW)/Memory Region (MR) bounds violation"},
5001fbb869SBartosz Sobczak 	{IRDMA_AE_AMP_RIGHTS_VIOLATION, "Memory Window (MW)/Memory Region (MR) rights violation"},
5101fbb869SBartosz Sobczak 	{IRDMA_AE_AMP_TO_WRAP,
5201fbb869SBartosz Sobczak 	"Memory protection error: The address within Memory Window (MW)/Memory Region (MR) wraps"},
5301fbb869SBartosz Sobczak 	{IRDMA_AE_AMP_FASTREG_VALID_STAG,
5401fbb869SBartosz Sobczak 	"Fastreg error: Registration to a valid MR"},
5501fbb869SBartosz Sobczak 	{IRDMA_AE_AMP_FASTREG_MW_STAG,
5601fbb869SBartosz Sobczak 	"Fastreg error: Registration to a valid Memory Window (MW)"},
5701fbb869SBartosz Sobczak 	{IRDMA_AE_AMP_FASTREG_INVALID_RIGHTS, "Fastreg error: Invalid rights"},
5801fbb869SBartosz Sobczak 	{IRDMA_AE_AMP_FASTREG_INVALID_LENGTH, "Fastreg error: Invalid length"},
5901fbb869SBartosz Sobczak 	{IRDMA_AE_AMP_INVALIDATE_SHARED, "Attempt to invalidate a shared MR"},
6001fbb869SBartosz Sobczak 	{IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS,
6101fbb869SBartosz Sobczak 	"Attempt to remotely invalidate Memory Window (MW)/Memory Region (MR) without rights"},
6201fbb869SBartosz Sobczak 	{IRDMA_AE_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS,
6301fbb869SBartosz Sobczak 	"Attempt to invalidate MR with a bound Memory Window (MW)"},
6401fbb869SBartosz Sobczak 	{IRDMA_AE_AMP_MWBIND_VALID_STAG,
6501fbb869SBartosz Sobczak 	"Attempt to bind an Memory Window (MW) with a valid MW memory key (L-Key/R-Key)"},
6601fbb869SBartosz Sobczak 	{IRDMA_AE_AMP_MWBIND_OF_MR_STAG,
6701fbb869SBartosz Sobczak 	"Attempt to bind an Memory Window (MW) with an MR memory key (L-Key/R-Key)"},
6801fbb869SBartosz Sobczak 	{IRDMA_AE_AMP_MWBIND_TO_ZERO_BASED_STAG,
6901fbb869SBartosz Sobczak 	"Attempt to bind an Memory Window (MW) to a zero based MR"},
7001fbb869SBartosz Sobczak 	{IRDMA_AE_AMP_MWBIND_TO_MW_STAG,
7101fbb869SBartosz Sobczak 	"Attempt to bind an Memory Window (MW) using MW memory key (L-Key/R-Key) instead of MR memory key (L-Key/R-Key)"},
7201fbb869SBartosz Sobczak 	{IRDMA_AE_AMP_MWBIND_INVALID_RIGHTS, "Memory Window (MW) bind error: Invalid rights"},
7301fbb869SBartosz Sobczak 	{IRDMA_AE_AMP_MWBIND_INVALID_BOUNDS, "Memory Window (MW) bind error: Invalid bounds"},
7401fbb869SBartosz Sobczak 	{IRDMA_AE_AMP_MWBIND_TO_INVALID_PARENT,
7501fbb869SBartosz Sobczak 	"Memory Window (MW) bind error: Invalid parent MR"},
7601fbb869SBartosz Sobczak 	{IRDMA_AE_AMP_MWBIND_BIND_DISABLED,
7701fbb869SBartosz Sobczak 	"Memory Window (MW) bind error: Disabled bind support"},
7801fbb869SBartosz Sobczak 	{IRDMA_AE_PRIV_OPERATION_DENIED,
7901fbb869SBartosz Sobczak 	"Denying a privileged operation on a non-privileged QP"},
8001fbb869SBartosz Sobczak 	{IRDMA_AE_AMP_INVALIDATE_TYPE1_MW, "Memory Window (MW) error: Invalidate type 1 MW"},
8101fbb869SBartosz Sobczak 	{IRDMA_AE_AMP_MWBIND_ZERO_BASED_TYPE1_MW,
8201fbb869SBartosz Sobczak 	"Memory Window (MW) bind error: Zero-based addressing for type 1 MW"},
8301fbb869SBartosz Sobczak 	{IRDMA_AE_AMP_FASTREG_INVALID_PBL_HPS_CFG,
8401fbb869SBartosz Sobczak 	"Fastreg error: Invalid host page size config"},
8501fbb869SBartosz Sobczak 	{IRDMA_AE_AMP_MWBIND_WRONG_TYPE, "MB bind error: Wrong Memory Window (MW) type"},
8601fbb869SBartosz Sobczak 	{IRDMA_AE_AMP_FASTREG_PBLE_MISMATCH,
8701fbb869SBartosz Sobczak 	"Fastreg error: Invalid request to change physical MR to virtual or vice versa"},
8801fbb869SBartosz Sobczak 	{IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG,
8901fbb869SBartosz Sobczak 	"Userspace Direct Access (UDA) QP xmit error: Packet length exceeds the QP MTU"},
9001fbb869SBartosz Sobczak 	{IRDMA_AE_UDA_XMIT_BAD_PD,
9101fbb869SBartosz Sobczak 	"Userspace Direct Access (UDA) QP xmit error: Attempt to access a different PD"},
9201fbb869SBartosz Sobczak 	{IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT,
9301fbb869SBartosz Sobczak 	"Userspace Direct Access (UDA) QP xmit error: Too short packet length"},
9401fbb869SBartosz Sobczak 	{IRDMA_AE_UDA_L4LEN_INVALID,
9501fbb869SBartosz Sobczak 	"Userspace Direct Access (UDA) error: Invalid packet length field"},
9601fbb869SBartosz Sobczak 	{IRDMA_AE_BAD_CLOSE,
9701fbb869SBartosz Sobczak 	"iWARP error: Data is received when QP state is closing"},
9801fbb869SBartosz Sobczak 	{IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE,
9901fbb869SBartosz Sobczak 	"iWARP error: FIN is received when xmit data is pending"},
10001fbb869SBartosz Sobczak 	{IRDMA_AE_CQ_OPERATION_ERROR, "CQ overflow"},
10101fbb869SBartosz Sobczak 	{IRDMA_AE_RDMA_READ_WHILE_ORD_ZERO,
10201fbb869SBartosz Sobczak 	"QP error: Attempted RDMA Read when the outbound RDMA Read queue depth is zero"},
10301fbb869SBartosz Sobczak 	{IRDMA_AE_STAG_ZERO_INVALID,
10401fbb869SBartosz Sobczak 	"Zero invalid memory key (L-Key/R-Key) on inbound RDMA R/W"},
10501fbb869SBartosz Sobczak 	{IRDMA_AE_IB_RREQ_AND_Q1_FULL,
10601fbb869SBartosz Sobczak 	"QP error: Received RDMA Read request when the inbound RDMA Read queue is full"},
10701fbb869SBartosz Sobczak 	{IRDMA_AE_IB_INVALID_REQUEST,
10801fbb869SBartosz Sobczak 	"QP error: Invalid operation detected by the remote peer"},
10901fbb869SBartosz Sobczak 	{IRDMA_AE_WQE_UNEXPECTED_OPCODE,
11001fbb869SBartosz Sobczak 	"QP error: Invalid opcode in SQ WQE"},
11101fbb869SBartosz Sobczak 	{IRDMA_AE_WQE_INVALID_PARAMETER,
11201fbb869SBartosz Sobczak 	"QP error: Invalid parameter in a WQE"},
11301fbb869SBartosz Sobczak 	{IRDMA_AE_WQE_INVALID_FRAG_DATA,
11401fbb869SBartosz Sobczak 	"QP error: Invalid fragment in a WQE"},
11501fbb869SBartosz Sobczak 	{IRDMA_AE_IB_REMOTE_ACCESS_ERROR,
11601fbb869SBartosz Sobczak 	"RoCEv2 error: Remote access error"},
11701fbb869SBartosz Sobczak 	{IRDMA_AE_IB_REMOTE_OP_ERROR,
11801fbb869SBartosz Sobczak 	"RoCEv2 error: Remote operation error"},
11901fbb869SBartosz Sobczak 	{IRDMA_AE_WQE_LSMM_TOO_LONG, "iWARP error: Connection error"},
12001fbb869SBartosz Sobczak 	{IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN,
12101fbb869SBartosz Sobczak 	"iWARP error: Invalid message sequence number"},
12201fbb869SBartosz Sobczak 	{IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER,
12301fbb869SBartosz Sobczak 	"iWARP error: Inbound message is too long for the available buffer"},
12401fbb869SBartosz Sobczak 	{IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION, "iWARP error: Invalid DDP protocol version"},
12501fbb869SBartosz Sobczak 	{IRDMA_AE_DDP_UBE_INVALID_MO, "Received message with too large offset"},
12601fbb869SBartosz Sobczak 	{IRDMA_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE,
12701fbb869SBartosz Sobczak 	"iWARP error: Inbound Send message when no receive buffer is available"},
12801fbb869SBartosz Sobczak 	{IRDMA_AE_DDP_UBE_INVALID_QN, "iWARP error: Invalid QP number in inbound packet"},
12901fbb869SBartosz Sobczak 	{IRDMA_AE_DDP_NO_L_BIT,
13001fbb869SBartosz Sobczak 	"iWARP error: Last bit not set in an inbound packet which completes RDMA Read"},
13101fbb869SBartosz Sobczak 	{IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION, "iWARP error: Invalid RDMAP protocol version"},
13201fbb869SBartosz Sobczak 	{IRDMA_AE_RDMAP_ROE_UNEXPECTED_OPCODE, "QP error: Invalid opcode"},
13301fbb869SBartosz Sobczak 	{IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST, "Inbound Read request when QP isn't enabled for RDMA Read"},
13401fbb869SBartosz Sobczak 	{IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP,
13501fbb869SBartosz Sobczak 	"Inbound RDMA Read response or RDMA Write when QP isn't enabled for RDMA R/W"},
13601fbb869SBartosz Sobczak 	{IRDMA_AE_ROCE_RSP_LENGTH_ERROR, "RoCEv2 error: Received packet with incorrect length field"},
13701fbb869SBartosz Sobczak 	{IRDMA_AE_ROCE_EMPTY_MCG, "RoCEv2 error: Multicast group has no valid members"},
13801fbb869SBartosz Sobczak 	{IRDMA_AE_ROCE_BAD_MC_IP_ADDR, "RoCEv2 error: Multicast IP address doesn't match"},
13901fbb869SBartosz Sobczak 	{IRDMA_AE_ROCE_BAD_MC_QPID, "RoCEv2 error: Multicast packet QP number isn't 0xffffff"},
14001fbb869SBartosz Sobczak 	{IRDMA_AE_MCG_QP_PROTOCOL_MISMATCH, "RoCEv2 error: Multicast packet protocol mismatch"},
14101fbb869SBartosz Sobczak 	{IRDMA_AE_INVALID_ARP_ENTRY, "Invalid ARP entry"},
14201fbb869SBartosz Sobczak 	{IRDMA_AE_INVALID_TCP_OPTION_RCVD, "iWARP error: Invalid TCP option"},
14301fbb869SBartosz Sobczak 	{IRDMA_AE_STALE_ARP_ENTRY, "Stale ARP entry"},
14401fbb869SBartosz Sobczak 	{IRDMA_AE_INVALID_AH_ENTRY, "Invalid AH entry"},
14501fbb869SBartosz Sobczak 	{IRDMA_AE_LLP_CLOSE_COMPLETE,
14601fbb869SBartosz Sobczak 	"iWARP event: Graceful close complete"},
14701fbb869SBartosz Sobczak 	{IRDMA_AE_LLP_CONNECTION_RESET,
14801fbb869SBartosz Sobczak 	"iWARP event: Received a TCP packet with a RST bit set"},
14901fbb869SBartosz Sobczak 	{IRDMA_AE_LLP_FIN_RECEIVED,
15001fbb869SBartosz Sobczak 	"iWARP event: Received a TCP packet with a FIN bit set"},
15101fbb869SBartosz Sobczak 	{IRDMA_AE_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH,
15201fbb869SBartosz Sobczak 	"iWARP error: Unable to close a gap in the TCP sequence"},
15301fbb869SBartosz Sobczak 	{IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR, "Received an ICRC error"},
15401fbb869SBartosz Sobczak 	{IRDMA_AE_LLP_SEGMENT_TOO_SMALL,
15501fbb869SBartosz Sobczak 	"iWARP error: Received a packet with insufficient space for protocol headers"},
15601fbb869SBartosz Sobczak 	{IRDMA_AE_LLP_SYN_RECEIVED,
15701fbb869SBartosz Sobczak 	"iWARP event: Received a TCP packet with a SYN bit set"},
15801fbb869SBartosz Sobczak 	{IRDMA_AE_LLP_TERMINATE_RECEIVED,
15901fbb869SBartosz Sobczak 	"iWARP error: Received a terminate message"},
16001fbb869SBartosz Sobczak 	{IRDMA_AE_LLP_TOO_MANY_RETRIES, "Connection error: The max number of retries has been reached"},
16101fbb869SBartosz Sobczak 	{IRDMA_AE_LLP_TOO_MANY_KEEPALIVE_RETRIES,
16201fbb869SBartosz Sobczak 	"Connection error: The max number of keepalive retries has been reached"},
16301fbb869SBartosz Sobczak 	{IRDMA_AE_LLP_DOUBT_REACHABILITY,
16401fbb869SBartosz Sobczak 	"Connection error: Doubt reachability (usually occurs after the max number of retries has been reached)"},
16501fbb869SBartosz Sobczak 	{IRDMA_AE_LLP_CONNECTION_ESTABLISHED,
16601fbb869SBartosz Sobczak 	"iWARP event: Connection established"},
167*5b5f7d0eSBartosz Sobczak 	{IRDMA_AE_LLP_TOO_MANY_RNRS, "RoCEv2: Too many RNR NACKs"},
16801fbb869SBartosz Sobczak 	{IRDMA_AE_RESOURCE_EXHAUSTION,
16901fbb869SBartosz Sobczak 	"QP error: Resource exhaustion"},
17001fbb869SBartosz Sobczak 	{IRDMA_AE_RESET_SENT,
17101fbb869SBartosz Sobczak 	"Reset sent (as requested via Modify QP)"},
17201fbb869SBartosz Sobczak 	{IRDMA_AE_TERMINATE_SENT,
17301fbb869SBartosz Sobczak 	"Terminate sent (as requested via Modify QP)"},
17401fbb869SBartosz Sobczak 	{IRDMA_AE_RESET_NOT_SENT,
17501fbb869SBartosz Sobczak 	"Reset not sent (but requested via Modify QP)"},
17601fbb869SBartosz Sobczak 	{IRDMA_AE_LCE_QP_CATASTROPHIC,
17701fbb869SBartosz Sobczak 	"QP error: HW transaction resulted in catastrophic error"},
17801fbb869SBartosz Sobczak 	{IRDMA_AE_LCE_FUNCTION_CATASTROPHIC,
17901fbb869SBartosz Sobczak 	"PCIe function error: HW transaction resulted in catastrophic error"},
18001fbb869SBartosz Sobczak 	{IRDMA_AE_LCE_CQ_CATASTROPHIC,
18101fbb869SBartosz Sobczak 	"CQ error: HW transaction resulted in catastrophic error"},
18201fbb869SBartosz Sobczak 	{IRDMA_AE_QP_SUSPEND_COMPLETE, "QP event: Suspend complete"},
18301fbb869SBartosz Sobczak };
18401fbb869SBartosz Sobczak 
18501fbb869SBartosz Sobczak /**
18601fbb869SBartosz Sobczak  * irdma_get_ae_desc - returns AE description
18701fbb869SBartosz Sobczak  * @ae_id: the AE number
18801fbb869SBartosz Sobczak  */
18901fbb869SBartosz Sobczak const char *
irdma_get_ae_desc(u16 ae_id)19001fbb869SBartosz Sobczak irdma_get_ae_desc(u16 ae_id)
19101fbb869SBartosz Sobczak {
19201fbb869SBartosz Sobczak 	const char *desc = "";
19301fbb869SBartosz Sobczak 	int i;
19401fbb869SBartosz Sobczak 
19501fbb869SBartosz Sobczak 	for (i = 0; i < ARRAY_SIZE(ae_desc_list); i++) {
19601fbb869SBartosz Sobczak 		if (ae_desc_list[i].id == ae_id) {
19701fbb869SBartosz Sobczak 			desc = ae_desc_list[i].desc;
19801fbb869SBartosz Sobczak 			break;
19901fbb869SBartosz Sobczak 		}
20001fbb869SBartosz Sobczak 	}
20101fbb869SBartosz Sobczak 	return desc;
20201fbb869SBartosz Sobczak }
20301fbb869SBartosz Sobczak 
204cdcd52d4SBartosz Sobczak /**
205cdcd52d4SBartosz Sobczak  * irdma_arp_table -manage arp table
206cdcd52d4SBartosz Sobczak  * @rf: RDMA PCI function
207cdcd52d4SBartosz Sobczak  * @ip_addr: ip address for device
208cdcd52d4SBartosz Sobczak  * @mac_addr: mac address ptr
209cdcd52d4SBartosz Sobczak  * @action: modify, delete or add
210cdcd52d4SBartosz Sobczak  */
211cdcd52d4SBartosz Sobczak int
irdma_arp_table(struct irdma_pci_f * rf,u32 * ip_addr,const u8 * mac_addr,u32 action)212777e472cSBartosz Sobczak irdma_arp_table(struct irdma_pci_f *rf, u32 *ip_addr, const u8 *mac_addr,
213cdcd52d4SBartosz Sobczak 		u32 action)
214cdcd52d4SBartosz Sobczak {
215cdcd52d4SBartosz Sobczak 	unsigned long flags;
216cdcd52d4SBartosz Sobczak 	int arp_index;
217cdcd52d4SBartosz Sobczak 	u32 ip[4] = {};
218cdcd52d4SBartosz Sobczak 
219cdcd52d4SBartosz Sobczak 	memcpy(ip, ip_addr, sizeof(ip));
220cdcd52d4SBartosz Sobczak 
221cdcd52d4SBartosz Sobczak 	spin_lock_irqsave(&rf->arp_lock, flags);
222cdcd52d4SBartosz Sobczak 	for (arp_index = 0; (u32)arp_index < rf->arp_table_size; arp_index++) {
223cdcd52d4SBartosz Sobczak 		if (!memcmp(rf->arp_table[arp_index].ip_addr, ip, sizeof(ip)))
224cdcd52d4SBartosz Sobczak 			break;
225cdcd52d4SBartosz Sobczak 	}
226cdcd52d4SBartosz Sobczak 
227cdcd52d4SBartosz Sobczak 	switch (action) {
228cdcd52d4SBartosz Sobczak 	case IRDMA_ARP_ADD:
229cdcd52d4SBartosz Sobczak 		if (arp_index != rf->arp_table_size) {
230cdcd52d4SBartosz Sobczak 			arp_index = -1;
231cdcd52d4SBartosz Sobczak 			break;
232cdcd52d4SBartosz Sobczak 		}
233cdcd52d4SBartosz Sobczak 
234cdcd52d4SBartosz Sobczak 		arp_index = 0;
235cdcd52d4SBartosz Sobczak 		if (irdma_alloc_rsrc(rf, rf->allocated_arps, rf->arp_table_size,
236cdcd52d4SBartosz Sobczak 				     (u32 *)&arp_index, &rf->next_arp_index)) {
237cdcd52d4SBartosz Sobczak 			arp_index = -1;
238cdcd52d4SBartosz Sobczak 			break;
239cdcd52d4SBartosz Sobczak 		}
240cdcd52d4SBartosz Sobczak 
241cdcd52d4SBartosz Sobczak 		memcpy(rf->arp_table[arp_index].ip_addr, ip,
242cdcd52d4SBartosz Sobczak 		       sizeof(rf->arp_table[arp_index].ip_addr));
243cdcd52d4SBartosz Sobczak 		ether_addr_copy(rf->arp_table[arp_index].mac_addr, mac_addr);
244cdcd52d4SBartosz Sobczak 		break;
245cdcd52d4SBartosz Sobczak 	case IRDMA_ARP_RESOLVE:
246cdcd52d4SBartosz Sobczak 		if (arp_index == rf->arp_table_size)
247cdcd52d4SBartosz Sobczak 			arp_index = -1;
248cdcd52d4SBartosz Sobczak 		break;
249cdcd52d4SBartosz Sobczak 	case IRDMA_ARP_DELETE:
250cdcd52d4SBartosz Sobczak 		if (arp_index == rf->arp_table_size) {
251cdcd52d4SBartosz Sobczak 			arp_index = -1;
252cdcd52d4SBartosz Sobczak 			break;
253cdcd52d4SBartosz Sobczak 		}
254cdcd52d4SBartosz Sobczak 
255cdcd52d4SBartosz Sobczak 		memset(rf->arp_table[arp_index].ip_addr, 0,
256cdcd52d4SBartosz Sobczak 		       sizeof(rf->arp_table[arp_index].ip_addr));
257cdcd52d4SBartosz Sobczak 		eth_zero_addr(rf->arp_table[arp_index].mac_addr);
258cdcd52d4SBartosz Sobczak 		irdma_free_rsrc(rf, rf->allocated_arps, arp_index);
259cdcd52d4SBartosz Sobczak 		break;
260cdcd52d4SBartosz Sobczak 	default:
261cdcd52d4SBartosz Sobczak 		arp_index = -1;
262cdcd52d4SBartosz Sobczak 		break;
263cdcd52d4SBartosz Sobczak 	}
264cdcd52d4SBartosz Sobczak 
265cdcd52d4SBartosz Sobczak 	spin_unlock_irqrestore(&rf->arp_lock, flags);
266cdcd52d4SBartosz Sobczak 	return arp_index;
267cdcd52d4SBartosz Sobczak }
268cdcd52d4SBartosz Sobczak 
269cdcd52d4SBartosz Sobczak /**
270cdcd52d4SBartosz Sobczak  * irdma_add_arp - add a new arp entry if needed
271cdcd52d4SBartosz Sobczak  * @rf: RDMA function
272cdcd52d4SBartosz Sobczak  * @ip: IP address
273cdcd52d4SBartosz Sobczak  * @mac: MAC address
274cdcd52d4SBartosz Sobczak  */
275cdcd52d4SBartosz Sobczak int
irdma_add_arp(struct irdma_pci_f * rf,u32 * ip,const u8 * mac)276777e472cSBartosz Sobczak irdma_add_arp(struct irdma_pci_f *rf, u32 *ip, const u8 *mac)
277cdcd52d4SBartosz Sobczak {
278cdcd52d4SBartosz Sobczak 	int arpidx;
279cdcd52d4SBartosz Sobczak 
280cdcd52d4SBartosz Sobczak 	arpidx = irdma_arp_table(rf, &ip[0], NULL, IRDMA_ARP_RESOLVE);
281cdcd52d4SBartosz Sobczak 	if (arpidx >= 0) {
282cdcd52d4SBartosz Sobczak 		if (ether_addr_equal(rf->arp_table[arpidx].mac_addr, mac))
283cdcd52d4SBartosz Sobczak 			return arpidx;
284cdcd52d4SBartosz Sobczak 
285cdcd52d4SBartosz Sobczak 		irdma_manage_arp_cache(rf, rf->arp_table[arpidx].mac_addr, ip,
286cdcd52d4SBartosz Sobczak 				       IRDMA_ARP_DELETE);
287cdcd52d4SBartosz Sobczak 	}
288cdcd52d4SBartosz Sobczak 
289cdcd52d4SBartosz Sobczak 	irdma_manage_arp_cache(rf, mac, ip, IRDMA_ARP_ADD);
290cdcd52d4SBartosz Sobczak 
291cdcd52d4SBartosz Sobczak 	return irdma_arp_table(rf, ip, NULL, IRDMA_ARP_RESOLVE);
292cdcd52d4SBartosz Sobczak }
293cdcd52d4SBartosz Sobczak 
294cdcd52d4SBartosz Sobczak /**
295777e472cSBartosz Sobczak  * irdma_netdevice_event - system notifier for netdev events
296777e472cSBartosz Sobczak  * @notifier: not used
297777e472cSBartosz Sobczak  * @event: event for notifier
298777e472cSBartosz Sobczak  * @ptr: netdev
299777e472cSBartosz Sobczak  */
300777e472cSBartosz Sobczak int
irdma_netdevice_event(struct notifier_block * notifier,unsigned long event,void * ptr)301777e472cSBartosz Sobczak irdma_netdevice_event(struct notifier_block *notifier, unsigned long event,
302777e472cSBartosz Sobczak 		      void *ptr)
303777e472cSBartosz Sobczak {
304777e472cSBartosz Sobczak 	struct irdma_device *iwdev;
305777e472cSBartosz Sobczak 	struct ifnet *netdev = netdev_notifier_info_to_ifp(ptr);
306777e472cSBartosz Sobczak 
307777e472cSBartosz Sobczak 	iwdev = container_of(notifier, struct irdma_device, nb_netdevice_event);
308777e472cSBartosz Sobczak 	if (iwdev->netdev != netdev)
309777e472cSBartosz Sobczak 		return NOTIFY_DONE;
310777e472cSBartosz Sobczak 
311777e472cSBartosz Sobczak 	iwdev->iw_status = 1;
312777e472cSBartosz Sobczak 	switch (event) {
313777e472cSBartosz Sobczak 	case NETDEV_DOWN:
314777e472cSBartosz Sobczak 		iwdev->iw_status = 0;
315777e472cSBartosz Sobczak 		/* fallthrough */
316777e472cSBartosz Sobczak 	case NETDEV_UP:
317777e472cSBartosz Sobczak 		irdma_port_ibevent(iwdev);
318777e472cSBartosz Sobczak 		break;
319777e472cSBartosz Sobczak 	default:
320777e472cSBartosz Sobczak 		break;
321777e472cSBartosz Sobczak 	}
322777e472cSBartosz Sobczak 
323777e472cSBartosz Sobczak 	return NOTIFY_DONE;
324777e472cSBartosz Sobczak }
325777e472cSBartosz Sobczak 
326777e472cSBartosz Sobczak void
irdma_unregister_notifiers(struct irdma_device * iwdev)327777e472cSBartosz Sobczak irdma_unregister_notifiers(struct irdma_device *iwdev)
328777e472cSBartosz Sobczak {
329777e472cSBartosz Sobczak 	unregister_netdevice_notifier(&iwdev->nb_netdevice_event);
330777e472cSBartosz Sobczak }
331777e472cSBartosz Sobczak 
332777e472cSBartosz Sobczak int
irdma_register_notifiers(struct irdma_device * iwdev)333777e472cSBartosz Sobczak irdma_register_notifiers(struct irdma_device *iwdev)
334777e472cSBartosz Sobczak {
335777e472cSBartosz Sobczak 	int ret;
336777e472cSBartosz Sobczak 
337777e472cSBartosz Sobczak 	iwdev->nb_netdevice_event.notifier_call = irdma_netdevice_event;
338777e472cSBartosz Sobczak 	ret = register_netdevice_notifier(&iwdev->nb_netdevice_event);
339777e472cSBartosz Sobczak 	if (ret) {
34035105900SBartosz Sobczak 		irdma_dev_err(&iwdev->ibdev, "register_netdevice_notifier failed\n");
341777e472cSBartosz Sobczak 		return ret;
342777e472cSBartosz Sobczak 	}
343777e472cSBartosz Sobczak 	return ret;
344777e472cSBartosz Sobczak }
345777e472cSBartosz Sobczak /**
346cdcd52d4SBartosz Sobczak  * irdma_alloc_and_get_cqp_request - get cqp struct
347cdcd52d4SBartosz Sobczak  * @cqp: device cqp ptr
348cdcd52d4SBartosz Sobczak  * @wait: cqp to be used in wait mode
349cdcd52d4SBartosz Sobczak  */
350cdcd52d4SBartosz Sobczak struct irdma_cqp_request *
irdma_alloc_and_get_cqp_request(struct irdma_cqp * cqp,bool wait)351cdcd52d4SBartosz Sobczak irdma_alloc_and_get_cqp_request(struct irdma_cqp *cqp,
352cdcd52d4SBartosz Sobczak 				bool wait)
353cdcd52d4SBartosz Sobczak {
354cdcd52d4SBartosz Sobczak 	struct irdma_cqp_request *cqp_request = NULL;
355cdcd52d4SBartosz Sobczak 	unsigned long flags;
356cdcd52d4SBartosz Sobczak 
357cdcd52d4SBartosz Sobczak 	spin_lock_irqsave(&cqp->req_lock, flags);
358cdcd52d4SBartosz Sobczak 	if (!list_empty(&cqp->cqp_avail_reqs)) {
359cdcd52d4SBartosz Sobczak 		cqp_request = list_entry(cqp->cqp_avail_reqs.next,
360cdcd52d4SBartosz Sobczak 					 struct irdma_cqp_request, list);
361cdcd52d4SBartosz Sobczak 		list_del_init(&cqp_request->list);
362cdcd52d4SBartosz Sobczak 	}
363cdcd52d4SBartosz Sobczak 	spin_unlock_irqrestore(&cqp->req_lock, flags);
364cdcd52d4SBartosz Sobczak 	if (!cqp_request) {
365cdcd52d4SBartosz Sobczak 		cqp_request = kzalloc(sizeof(*cqp_request), GFP_ATOMIC);
366cdcd52d4SBartosz Sobczak 		if (cqp_request) {
367cdcd52d4SBartosz Sobczak 			cqp_request->dynamic = true;
368cdcd52d4SBartosz Sobczak 			if (wait)
369cdcd52d4SBartosz Sobczak 				init_waitqueue_head(&cqp_request->waitq);
370cdcd52d4SBartosz Sobczak 		}
371cdcd52d4SBartosz Sobczak 	}
372cdcd52d4SBartosz Sobczak 	if (!cqp_request) {
37335105900SBartosz Sobczak 		irdma_debug(cqp->sc_cqp.dev, IRDMA_DEBUG_ERR, "CQP Request Fail: No Memory");
374cdcd52d4SBartosz Sobczak 		return NULL;
375cdcd52d4SBartosz Sobczak 	}
376cdcd52d4SBartosz Sobczak 
377cdcd52d4SBartosz Sobczak 	cqp_request->waiting = wait;
378cdcd52d4SBartosz Sobczak 	atomic_set(&cqp_request->refcnt, 1);
379cdcd52d4SBartosz Sobczak 	memset(&cqp_request->compl_info, 0, sizeof(cqp_request->compl_info));
380cdcd52d4SBartosz Sobczak 
381cdcd52d4SBartosz Sobczak 	return cqp_request;
382cdcd52d4SBartosz Sobczak }
383cdcd52d4SBartosz Sobczak 
384cdcd52d4SBartosz Sobczak /**
385cdcd52d4SBartosz Sobczak  * irdma_get_cqp_request - increase refcount for cqp_request
386cdcd52d4SBartosz Sobczak  * @cqp_request: pointer to cqp_request instance
387cdcd52d4SBartosz Sobczak  */
388cdcd52d4SBartosz Sobczak static inline void
irdma_get_cqp_request(struct irdma_cqp_request * cqp_request)389cdcd52d4SBartosz Sobczak irdma_get_cqp_request(struct irdma_cqp_request *cqp_request)
390cdcd52d4SBartosz Sobczak {
391cdcd52d4SBartosz Sobczak 	atomic_inc(&cqp_request->refcnt);
392cdcd52d4SBartosz Sobczak }
393cdcd52d4SBartosz Sobczak 
394cdcd52d4SBartosz Sobczak /**
395cdcd52d4SBartosz Sobczak  * irdma_free_cqp_request - free cqp request
396cdcd52d4SBartosz Sobczak  * @cqp: cqp ptr
397cdcd52d4SBartosz Sobczak  * @cqp_request: to be put back in cqp list
398cdcd52d4SBartosz Sobczak  */
399cdcd52d4SBartosz Sobczak void
irdma_free_cqp_request(struct irdma_cqp * cqp,struct irdma_cqp_request * cqp_request)400cdcd52d4SBartosz Sobczak irdma_free_cqp_request(struct irdma_cqp *cqp,
401cdcd52d4SBartosz Sobczak 		       struct irdma_cqp_request *cqp_request)
402cdcd52d4SBartosz Sobczak {
403cdcd52d4SBartosz Sobczak 	unsigned long flags;
404cdcd52d4SBartosz Sobczak 
405cdcd52d4SBartosz Sobczak 	if (cqp_request->dynamic) {
406cdcd52d4SBartosz Sobczak 		kfree(cqp_request);
407cdcd52d4SBartosz Sobczak 	} else {
40801fbb869SBartosz Sobczak 		WRITE_ONCE(cqp_request->request_done, false);
409cdcd52d4SBartosz Sobczak 		cqp_request->callback_fcn = NULL;
410cdcd52d4SBartosz Sobczak 		cqp_request->waiting = false;
411cdcd52d4SBartosz Sobczak 
412cdcd52d4SBartosz Sobczak 		spin_lock_irqsave(&cqp->req_lock, flags);
413cdcd52d4SBartosz Sobczak 		list_add_tail(&cqp_request->list, &cqp->cqp_avail_reqs);
414cdcd52d4SBartosz Sobczak 		spin_unlock_irqrestore(&cqp->req_lock, flags);
415cdcd52d4SBartosz Sobczak 	}
416cdcd52d4SBartosz Sobczak 	wake_up(&cqp->remove_wq);
417cdcd52d4SBartosz Sobczak }
418cdcd52d4SBartosz Sobczak 
419cdcd52d4SBartosz Sobczak /**
420cdcd52d4SBartosz Sobczak  * irdma_put_cqp_request - dec ref count and free if 0
421cdcd52d4SBartosz Sobczak  * @cqp: cqp ptr
422cdcd52d4SBartosz Sobczak  * @cqp_request: to be put back in cqp list
423cdcd52d4SBartosz Sobczak  */
424cdcd52d4SBartosz Sobczak void
irdma_put_cqp_request(struct irdma_cqp * cqp,struct irdma_cqp_request * cqp_request)425cdcd52d4SBartosz Sobczak irdma_put_cqp_request(struct irdma_cqp *cqp,
426cdcd52d4SBartosz Sobczak 		      struct irdma_cqp_request *cqp_request)
427cdcd52d4SBartosz Sobczak {
428cdcd52d4SBartosz Sobczak 	if (atomic_dec_and_test(&cqp_request->refcnt))
429cdcd52d4SBartosz Sobczak 		irdma_free_cqp_request(cqp, cqp_request);
430cdcd52d4SBartosz Sobczak }
431cdcd52d4SBartosz Sobczak 
432cdcd52d4SBartosz Sobczak /**
433cdcd52d4SBartosz Sobczak  * irdma_free_pending_cqp_request -free pending cqp request objs
434cdcd52d4SBartosz Sobczak  * @cqp: cqp ptr
435cdcd52d4SBartosz Sobczak  * @cqp_request: to be put back in cqp list
436cdcd52d4SBartosz Sobczak  */
437cdcd52d4SBartosz Sobczak static void
irdma_free_pending_cqp_request(struct irdma_cqp * cqp,struct irdma_cqp_request * cqp_request)438cdcd52d4SBartosz Sobczak irdma_free_pending_cqp_request(struct irdma_cqp *cqp,
439cdcd52d4SBartosz Sobczak 			       struct irdma_cqp_request *cqp_request)
440cdcd52d4SBartosz Sobczak {
441cdcd52d4SBartosz Sobczak 	cqp_request->compl_info.error = true;
44201fbb869SBartosz Sobczak 	WRITE_ONCE(cqp_request->request_done, true);
443*5b5f7d0eSBartosz Sobczak 
444*5b5f7d0eSBartosz Sobczak 	if (cqp_request->waiting)
445cdcd52d4SBartosz Sobczak 		wake_up(&cqp_request->waitq);
446cdcd52d4SBartosz Sobczak 	wait_event_timeout(cqp->remove_wq,
447cdcd52d4SBartosz Sobczak 			   atomic_read(&cqp_request->refcnt) == 1, 1000);
448cdcd52d4SBartosz Sobczak 	irdma_put_cqp_request(cqp, cqp_request);
449cdcd52d4SBartosz Sobczak }
450cdcd52d4SBartosz Sobczak 
451cdcd52d4SBartosz Sobczak /**
452cdcd52d4SBartosz Sobczak  * irdma_cleanup_pending_cqp_op - clean-up cqp with no
453cdcd52d4SBartosz Sobczak  * completions
454cdcd52d4SBartosz Sobczak  * @rf: RDMA PCI function
455cdcd52d4SBartosz Sobczak  */
456cdcd52d4SBartosz Sobczak void
irdma_cleanup_pending_cqp_op(struct irdma_pci_f * rf)457cdcd52d4SBartosz Sobczak irdma_cleanup_pending_cqp_op(struct irdma_pci_f *rf)
458cdcd52d4SBartosz Sobczak {
459cdcd52d4SBartosz Sobczak 	struct irdma_sc_dev *dev = &rf->sc_dev;
460cdcd52d4SBartosz Sobczak 	struct irdma_cqp *cqp = &rf->cqp;
461cdcd52d4SBartosz Sobczak 	struct irdma_cqp_request *cqp_request = NULL;
462cdcd52d4SBartosz Sobczak 	struct cqp_cmds_info *pcmdinfo = NULL;
463cdcd52d4SBartosz Sobczak 	u32 i, pending_work, wqe_idx;
464cdcd52d4SBartosz Sobczak 
465cdcd52d4SBartosz Sobczak 	pending_work = IRDMA_RING_USED_QUANTA(cqp->sc_cqp.sq_ring);
466cdcd52d4SBartosz Sobczak 	wqe_idx = IRDMA_RING_CURRENT_TAIL(cqp->sc_cqp.sq_ring);
467cdcd52d4SBartosz Sobczak 	for (i = 0; i < pending_work; i++) {
468777e472cSBartosz Sobczak 		cqp_request = (struct irdma_cqp_request *)(uintptr_t)
469cdcd52d4SBartosz Sobczak 		    cqp->scratch_array[wqe_idx];
470cdcd52d4SBartosz Sobczak 		if (cqp_request)
471cdcd52d4SBartosz Sobczak 			irdma_free_pending_cqp_request(cqp, cqp_request);
472cdcd52d4SBartosz Sobczak 		wqe_idx = (wqe_idx + 1) % IRDMA_RING_SIZE(cqp->sc_cqp.sq_ring);
473cdcd52d4SBartosz Sobczak 	}
474cdcd52d4SBartosz Sobczak 
475cdcd52d4SBartosz Sobczak 	while (!list_empty(&dev->cqp_cmd_head)) {
476cdcd52d4SBartosz Sobczak 		pcmdinfo = irdma_remove_cqp_head(dev);
477cdcd52d4SBartosz Sobczak 		cqp_request =
478cdcd52d4SBartosz Sobczak 		    container_of(pcmdinfo, struct irdma_cqp_request, info);
479cdcd52d4SBartosz Sobczak 		if (cqp_request)
480cdcd52d4SBartosz Sobczak 			irdma_free_pending_cqp_request(cqp, cqp_request);
481cdcd52d4SBartosz Sobczak 	}
482cdcd52d4SBartosz Sobczak }
483cdcd52d4SBartosz Sobczak 
484cdcd52d4SBartosz Sobczak /**
485cdcd52d4SBartosz Sobczak  * irdma_wait_event - wait for completion
486cdcd52d4SBartosz Sobczak  * @rf: RDMA PCI function
487cdcd52d4SBartosz Sobczak  * @cqp_request: cqp request to wait
488cdcd52d4SBartosz Sobczak  */
489cdcd52d4SBartosz Sobczak static int
irdma_wait_event(struct irdma_pci_f * rf,struct irdma_cqp_request * cqp_request)490cdcd52d4SBartosz Sobczak irdma_wait_event(struct irdma_pci_f *rf,
491cdcd52d4SBartosz Sobczak 		 struct irdma_cqp_request *cqp_request)
492cdcd52d4SBartosz Sobczak {
493cdcd52d4SBartosz Sobczak 	struct irdma_cqp_timeout cqp_timeout = {0};
494cdcd52d4SBartosz Sobczak 	bool cqp_error = false;
495cdcd52d4SBartosz Sobczak 	int err_code = 0;
496cdcd52d4SBartosz Sobczak 
49701fbb869SBartosz Sobczak 	cqp_timeout.compl_cqp_cmds = atomic64_read(&rf->sc_dev.cqp->completed_ops);
498cdcd52d4SBartosz Sobczak 	do {
499777e472cSBartosz Sobczak 		int wait_time_ms = rf->sc_dev.hw_attrs.max_cqp_compl_wait_time_ms;
500777e472cSBartosz Sobczak 
501cdcd52d4SBartosz Sobczak 		irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq);
502cdcd52d4SBartosz Sobczak 		if (wait_event_timeout(cqp_request->waitq,
50301fbb869SBartosz Sobczak 				       READ_ONCE(cqp_request->request_done),
504777e472cSBartosz Sobczak 				       msecs_to_jiffies(wait_time_ms)))
505cdcd52d4SBartosz Sobczak 			break;
506cdcd52d4SBartosz Sobczak 
507cdcd52d4SBartosz Sobczak 		irdma_check_cqp_progress(&cqp_timeout, &rf->sc_dev);
508cdcd52d4SBartosz Sobczak 
50901fbb869SBartosz Sobczak 		if (cqp_timeout.count < CQP_TIMEOUT_THRESHOLD)
510cdcd52d4SBartosz Sobczak 			continue;
511cdcd52d4SBartosz Sobczak 
512cdcd52d4SBartosz Sobczak 		if (!rf->reset) {
513cdcd52d4SBartosz Sobczak 			rf->reset = true;
514cdcd52d4SBartosz Sobczak 			rf->gen_ops.request_reset(rf);
515cdcd52d4SBartosz Sobczak 		}
516cdcd52d4SBartosz Sobczak 		return -ETIMEDOUT;
517cdcd52d4SBartosz Sobczak 	} while (1);
518cdcd52d4SBartosz Sobczak 
519cdcd52d4SBartosz Sobczak 	cqp_error = cqp_request->compl_info.error;
520cdcd52d4SBartosz Sobczak 	if (cqp_error) {
521cdcd52d4SBartosz Sobczak 		err_code = -EIO;
522777e472cSBartosz Sobczak 		if (cqp_request->compl_info.maj_err_code == 0xFFFF) {
523777e472cSBartosz Sobczak 			if (cqp_request->compl_info.min_err_code == 0x8002) {
524777e472cSBartosz Sobczak 				err_code = -EBUSY;
525777e472cSBartosz Sobczak 			} else if (cqp_request->compl_info.min_err_code == 0x8029) {
526cdcd52d4SBartosz Sobczak 				if (!rf->reset) {
527cdcd52d4SBartosz Sobczak 					rf->reset = true;
528cdcd52d4SBartosz Sobczak 					rf->gen_ops.request_reset(rf);
529cdcd52d4SBartosz Sobczak 				}
530cdcd52d4SBartosz Sobczak 			}
531cdcd52d4SBartosz Sobczak 		}
532777e472cSBartosz Sobczak 	}
533cdcd52d4SBartosz Sobczak 
534cdcd52d4SBartosz Sobczak 	return err_code;
535cdcd52d4SBartosz Sobczak }
536cdcd52d4SBartosz Sobczak 
537cdcd52d4SBartosz Sobczak static const char *const irdma_cqp_cmd_names[IRDMA_MAX_CQP_OPS] = {
538cdcd52d4SBartosz Sobczak 	[IRDMA_OP_CEQ_DESTROY] = "Destroy CEQ Cmd",
539cdcd52d4SBartosz Sobczak 	[IRDMA_OP_AEQ_DESTROY] = "Destroy AEQ Cmd",
540cdcd52d4SBartosz Sobczak 	[IRDMA_OP_DELETE_ARP_CACHE_ENTRY] = "Delete ARP Cache Cmd",
541cdcd52d4SBartosz Sobczak 	[IRDMA_OP_MANAGE_APBVT_ENTRY] = "Manage APBV Table Entry Cmd",
542cdcd52d4SBartosz Sobczak 	[IRDMA_OP_CEQ_CREATE] = "CEQ Create Cmd",
543cdcd52d4SBartosz Sobczak 	[IRDMA_OP_AEQ_CREATE] = "AEQ Destroy Cmd",
544cdcd52d4SBartosz Sobczak 	[IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY] = "Manage Quad Hash Table Entry Cmd",
545cdcd52d4SBartosz Sobczak 	[IRDMA_OP_QP_MODIFY] = "Modify QP Cmd",
546cdcd52d4SBartosz Sobczak 	[IRDMA_OP_QP_UPLOAD_CONTEXT] = "Upload Context Cmd",
547cdcd52d4SBartosz Sobczak 	[IRDMA_OP_CQ_CREATE] = "Create CQ Cmd",
548cdcd52d4SBartosz Sobczak 	[IRDMA_OP_CQ_DESTROY] = "Destroy CQ Cmd",
549cdcd52d4SBartosz Sobczak 	[IRDMA_OP_QP_CREATE] = "Create QP Cmd",
550cdcd52d4SBartosz Sobczak 	[IRDMA_OP_QP_DESTROY] = "Destroy QP Cmd",
551cdcd52d4SBartosz Sobczak 	[IRDMA_OP_ALLOC_STAG] = "Allocate STag Cmd",
552cdcd52d4SBartosz Sobczak 	[IRDMA_OP_MR_REG_NON_SHARED] = "Register Non-Shared MR Cmd",
553cdcd52d4SBartosz Sobczak 	[IRDMA_OP_DEALLOC_STAG] = "Deallocate STag Cmd",
554cdcd52d4SBartosz Sobczak 	[IRDMA_OP_MW_ALLOC] = "Allocate Memory Window Cmd",
555cdcd52d4SBartosz Sobczak 	[IRDMA_OP_QP_FLUSH_WQES] = "Flush QP Cmd",
556cdcd52d4SBartosz Sobczak 	[IRDMA_OP_ADD_ARP_CACHE_ENTRY] = "Add ARP Cache Cmd",
557cdcd52d4SBartosz Sobczak 	[IRDMA_OP_MANAGE_PUSH_PAGE] = "Manage Push Page Cmd",
558cdcd52d4SBartosz Sobczak 	[IRDMA_OP_UPDATE_PE_SDS] = "Update PE SDs Cmd",
559cdcd52d4SBartosz Sobczak 	[IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE] = "Manage HMC PM Function Table Cmd",
560cdcd52d4SBartosz Sobczak 	[IRDMA_OP_SUSPEND] = "Suspend QP Cmd",
561cdcd52d4SBartosz Sobczak 	[IRDMA_OP_RESUME] = "Resume QP Cmd",
562cdcd52d4SBartosz Sobczak 	[IRDMA_OP_QUERY_FPM_VAL] = "Query FPM Values Cmd",
563cdcd52d4SBartosz Sobczak 	[IRDMA_OP_COMMIT_FPM_VAL] = "Commit FPM Values Cmd",
564cdcd52d4SBartosz Sobczak 	[IRDMA_OP_AH_CREATE] = "Create Address Handle Cmd",
565cdcd52d4SBartosz Sobczak 	[IRDMA_OP_AH_MODIFY] = "Modify Address Handle Cmd",
566cdcd52d4SBartosz Sobczak 	[IRDMA_OP_AH_DESTROY] = "Destroy Address Handle Cmd",
567cdcd52d4SBartosz Sobczak 	[IRDMA_OP_MC_CREATE] = "Create Multicast Group Cmd",
568cdcd52d4SBartosz Sobczak 	[IRDMA_OP_MC_DESTROY] = "Destroy Multicast Group Cmd",
569cdcd52d4SBartosz Sobczak 	[IRDMA_OP_MC_MODIFY] = "Modify Multicast Group Cmd",
570cdcd52d4SBartosz Sobczak 	[IRDMA_OP_STATS_ALLOCATE] = "Add Statistics Instance Cmd",
571cdcd52d4SBartosz Sobczak 	[IRDMA_OP_STATS_FREE] = "Free Statistics Instance Cmd",
572cdcd52d4SBartosz Sobczak 	[IRDMA_OP_STATS_GATHER] = "Gather Statistics Cmd",
573cdcd52d4SBartosz Sobczak 	[IRDMA_OP_WS_ADD_NODE] = "Add Work Scheduler Node Cmd",
574cdcd52d4SBartosz Sobczak 	[IRDMA_OP_WS_MODIFY_NODE] = "Modify Work Scheduler Node Cmd",
575cdcd52d4SBartosz Sobczak 	[IRDMA_OP_WS_DELETE_NODE] = "Delete Work Scheduler Node Cmd",
57635105900SBartosz Sobczak 	[IRDMA_OP_WS_FAILOVER_START] = "Failover Start Cmd",
57735105900SBartosz Sobczak 	[IRDMA_OP_WS_FAILOVER_COMPLETE] = "Failover Complete Cmd",
578cdcd52d4SBartosz Sobczak 	[IRDMA_OP_SET_UP_MAP] = "Set UP-UP Mapping Cmd",
579cdcd52d4SBartosz Sobczak 	[IRDMA_OP_GEN_AE] = "Generate AE Cmd",
580cdcd52d4SBartosz Sobczak 	[IRDMA_OP_QUERY_RDMA_FEATURES] = "RDMA Get Features Cmd",
581cdcd52d4SBartosz Sobczak 	[IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY] = "Allocate Local MAC Entry Cmd",
582cdcd52d4SBartosz Sobczak 	[IRDMA_OP_ADD_LOCAL_MAC_ENTRY] = "Add Local MAC Entry Cmd",
583cdcd52d4SBartosz Sobczak 	[IRDMA_OP_DELETE_LOCAL_MAC_ENTRY] = "Delete Local MAC Entry Cmd",
584cdcd52d4SBartosz Sobczak 	[IRDMA_OP_CQ_MODIFY] = "CQ Modify Cmd",
585cdcd52d4SBartosz Sobczak };
586cdcd52d4SBartosz Sobczak 
587cdcd52d4SBartosz Sobczak static const struct irdma_cqp_err_info irdma_noncrit_err_list[] = {
588777e472cSBartosz Sobczak 	{0xffff, 0x8002, "Invalid State"},
589cdcd52d4SBartosz Sobczak 	{0xffff, 0x8006, "Flush No Wqe Pending"},
590cdcd52d4SBartosz Sobczak 	{0xffff, 0x8007, "Modify QP Bad Close"},
591cdcd52d4SBartosz Sobczak 	{0xffff, 0x8009, "LLP Closed"},
592777e472cSBartosz Sobczak 	{0xffff, 0x800a, "Reset Not Sent"},
593*5b5f7d0eSBartosz Sobczak 	{0xffff, 0x0200, "Failover Pending"},
594cdcd52d4SBartosz Sobczak };
595cdcd52d4SBartosz Sobczak 
596cdcd52d4SBartosz Sobczak /**
597cdcd52d4SBartosz Sobczak  * irdma_cqp_crit_err - check if CQP error is critical
598cdcd52d4SBartosz Sobczak  * @dev: pointer to dev structure
599cdcd52d4SBartosz Sobczak  * @cqp_cmd: code for last CQP operation
600cdcd52d4SBartosz Sobczak  * @maj_err_code: major error code
601cdcd52d4SBartosz Sobczak  * @min_err_code: minot error code
602cdcd52d4SBartosz Sobczak  */
603cdcd52d4SBartosz Sobczak bool
irdma_cqp_crit_err(struct irdma_sc_dev * dev,u8 cqp_cmd,u16 maj_err_code,u16 min_err_code)604cdcd52d4SBartosz Sobczak irdma_cqp_crit_err(struct irdma_sc_dev *dev, u8 cqp_cmd,
605cdcd52d4SBartosz Sobczak 		   u16 maj_err_code, u16 min_err_code)
606cdcd52d4SBartosz Sobczak {
607cdcd52d4SBartosz Sobczak 	int i;
608cdcd52d4SBartosz Sobczak 
609cdcd52d4SBartosz Sobczak 	for (i = 0; i < ARRAY_SIZE(irdma_noncrit_err_list); ++i) {
610cdcd52d4SBartosz Sobczak 		if (maj_err_code == irdma_noncrit_err_list[i].maj &&
611cdcd52d4SBartosz Sobczak 		    min_err_code == irdma_noncrit_err_list[i].min) {
612cdcd52d4SBartosz Sobczak 			irdma_debug(dev, IRDMA_DEBUG_CQP,
613cdcd52d4SBartosz Sobczak 				    "[%s Error][%s] maj=0x%x min=0x%x\n",
614cdcd52d4SBartosz Sobczak 				    irdma_noncrit_err_list[i].desc,
61535105900SBartosz Sobczak 				    irdma_cqp_cmd_names[cqp_cmd], maj_err_code,
616cdcd52d4SBartosz Sobczak 				    min_err_code);
617cdcd52d4SBartosz Sobczak 			return false;
618cdcd52d4SBartosz Sobczak 		}
619cdcd52d4SBartosz Sobczak 	}
620cdcd52d4SBartosz Sobczak 	return true;
621cdcd52d4SBartosz Sobczak }
622cdcd52d4SBartosz Sobczak 
623cdcd52d4SBartosz Sobczak /**
624cdcd52d4SBartosz Sobczak  * irdma_handle_cqp_op - process cqp command
625cdcd52d4SBartosz Sobczak  * @rf: RDMA PCI function
626cdcd52d4SBartosz Sobczak  * @cqp_request: cqp request to process
627cdcd52d4SBartosz Sobczak  */
628cdcd52d4SBartosz Sobczak int
irdma_handle_cqp_op(struct irdma_pci_f * rf,struct irdma_cqp_request * cqp_request)629cdcd52d4SBartosz Sobczak irdma_handle_cqp_op(struct irdma_pci_f *rf,
630cdcd52d4SBartosz Sobczak 		    struct irdma_cqp_request *cqp_request)
631cdcd52d4SBartosz Sobczak {
632cdcd52d4SBartosz Sobczak 	struct irdma_sc_dev *dev = &rf->sc_dev;
633cdcd52d4SBartosz Sobczak 	struct cqp_cmds_info *info = &cqp_request->info;
634cdcd52d4SBartosz Sobczak 	int status;
635cdcd52d4SBartosz Sobczak 	bool put_cqp_request = true;
636cdcd52d4SBartosz Sobczak 
637cdcd52d4SBartosz Sobczak 	if (rf->reset)
63835105900SBartosz Sobczak 		return 0;
639cdcd52d4SBartosz Sobczak 
640cdcd52d4SBartosz Sobczak 	irdma_get_cqp_request(cqp_request);
641cdcd52d4SBartosz Sobczak 	status = irdma_process_cqp_cmd(dev, info);
642cdcd52d4SBartosz Sobczak 	if (status)
643cdcd52d4SBartosz Sobczak 		goto err;
644cdcd52d4SBartosz Sobczak 
645cdcd52d4SBartosz Sobczak 	if (cqp_request->waiting) {
646cdcd52d4SBartosz Sobczak 		put_cqp_request = false;
647cdcd52d4SBartosz Sobczak 		status = irdma_wait_event(rf, cqp_request);
648cdcd52d4SBartosz Sobczak 		if (status)
649cdcd52d4SBartosz Sobczak 			goto err;
650cdcd52d4SBartosz Sobczak 	}
651cdcd52d4SBartosz Sobczak 
652cdcd52d4SBartosz Sobczak 	return 0;
653cdcd52d4SBartosz Sobczak 
654cdcd52d4SBartosz Sobczak err:
655cdcd52d4SBartosz Sobczak 	if (irdma_cqp_crit_err(dev, info->cqp_cmd,
656cdcd52d4SBartosz Sobczak 			       cqp_request->compl_info.maj_err_code,
657cdcd52d4SBartosz Sobczak 			       cqp_request->compl_info.min_err_code))
65835105900SBartosz Sobczak 		irdma_dev_err(&rf->iwdev->ibdev,
659cdcd52d4SBartosz Sobczak 			      "[%s Error][op_code=%d] status=%d waiting=%d completion_err=%d maj=0x%x min=0x%x\n",
66035105900SBartosz Sobczak 			      irdma_cqp_cmd_names[info->cqp_cmd], info->cqp_cmd, status,
66135105900SBartosz Sobczak 			      cqp_request->waiting, cqp_request->compl_info.error,
66235105900SBartosz Sobczak 			      cqp_request->compl_info.maj_err_code,
663cdcd52d4SBartosz Sobczak 			      cqp_request->compl_info.min_err_code);
664cdcd52d4SBartosz Sobczak 
665cdcd52d4SBartosz Sobczak 	if (put_cqp_request)
666cdcd52d4SBartosz Sobczak 		irdma_put_cqp_request(&rf->cqp, cqp_request);
667cdcd52d4SBartosz Sobczak 
668cdcd52d4SBartosz Sobczak 	return status;
669cdcd52d4SBartosz Sobczak }
670cdcd52d4SBartosz Sobczak 
671cdcd52d4SBartosz Sobczak void
irdma_qp_add_ref(struct ib_qp * ibqp)672cdcd52d4SBartosz Sobczak irdma_qp_add_ref(struct ib_qp *ibqp)
673cdcd52d4SBartosz Sobczak {
674cdcd52d4SBartosz Sobczak 	struct irdma_qp *iwqp = to_iwqp(ibqp);
675cdcd52d4SBartosz Sobczak 
676cdcd52d4SBartosz Sobczak 	atomic_inc(&iwqp->refcnt);
677cdcd52d4SBartosz Sobczak }
678cdcd52d4SBartosz Sobczak 
679cdcd52d4SBartosz Sobczak void
irdma_qp_rem_ref(struct ib_qp * ibqp)680cdcd52d4SBartosz Sobczak irdma_qp_rem_ref(struct ib_qp *ibqp)
681cdcd52d4SBartosz Sobczak {
682cdcd52d4SBartosz Sobczak 	struct irdma_qp *iwqp = to_iwqp(ibqp);
683cdcd52d4SBartosz Sobczak 	struct irdma_device *iwdev = iwqp->iwdev;
684cdcd52d4SBartosz Sobczak 	unsigned long flags;
685cdcd52d4SBartosz Sobczak 
686cdcd52d4SBartosz Sobczak 	spin_lock_irqsave(&iwdev->rf->qptable_lock, flags);
687cdcd52d4SBartosz Sobczak 	if (!atomic_dec_and_test(&iwqp->refcnt)) {
688cdcd52d4SBartosz Sobczak 		spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags);
689cdcd52d4SBartosz Sobczak 		return;
690cdcd52d4SBartosz Sobczak 	}
691cdcd52d4SBartosz Sobczak 
692cdcd52d4SBartosz Sobczak 	iwdev->rf->qp_table[iwqp->ibqp.qp_num] = NULL;
693cdcd52d4SBartosz Sobczak 	spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags);
694cdcd52d4SBartosz Sobczak 	complete(&iwqp->free_qp);
695cdcd52d4SBartosz Sobczak }
696cdcd52d4SBartosz Sobczak 
697cdcd52d4SBartosz Sobczak void
irdma_cq_add_ref(struct ib_cq * ibcq)698cdcd52d4SBartosz Sobczak irdma_cq_add_ref(struct ib_cq *ibcq)
699cdcd52d4SBartosz Sobczak {
700cdcd52d4SBartosz Sobczak 	struct irdma_cq *iwcq = to_iwcq(ibcq);
701cdcd52d4SBartosz Sobczak 
702cdcd52d4SBartosz Sobczak 	atomic_inc(&iwcq->refcnt);
703cdcd52d4SBartosz Sobczak }
704cdcd52d4SBartosz Sobczak 
705cdcd52d4SBartosz Sobczak void
irdma_cq_rem_ref(struct ib_cq * ibcq)706cdcd52d4SBartosz Sobczak irdma_cq_rem_ref(struct ib_cq *ibcq)
707cdcd52d4SBartosz Sobczak {
708cdcd52d4SBartosz Sobczak 	struct irdma_cq *iwcq = to_iwcq(ibcq);
709cdcd52d4SBartosz Sobczak 	struct irdma_pci_f *rf = container_of(iwcq->sc_cq.dev, struct irdma_pci_f, sc_dev);
710cdcd52d4SBartosz Sobczak 	unsigned long flags;
711cdcd52d4SBartosz Sobczak 
712cdcd52d4SBartosz Sobczak 	spin_lock_irqsave(&rf->cqtable_lock, flags);
713cdcd52d4SBartosz Sobczak 	if (!atomic_dec_and_test(&iwcq->refcnt)) {
714cdcd52d4SBartosz Sobczak 		spin_unlock_irqrestore(&rf->cqtable_lock, flags);
715cdcd52d4SBartosz Sobczak 		return;
716cdcd52d4SBartosz Sobczak 	}
717cdcd52d4SBartosz Sobczak 
718cdcd52d4SBartosz Sobczak 	rf->cq_table[iwcq->cq_num] = NULL;
719cdcd52d4SBartosz Sobczak 	spin_unlock_irqrestore(&rf->cqtable_lock, flags);
720cdcd52d4SBartosz Sobczak 	complete(&iwcq->free_cq);
721cdcd52d4SBartosz Sobczak }
722cdcd52d4SBartosz Sobczak 
723cdcd52d4SBartosz Sobczak struct ib_device *
to_ibdev(struct irdma_sc_dev * dev)72435105900SBartosz Sobczak to_ibdev(struct irdma_sc_dev *dev)
725cdcd52d4SBartosz Sobczak {
726cdcd52d4SBartosz Sobczak 	return &(container_of(dev, struct irdma_pci_f, sc_dev))->iwdev->ibdev;
727cdcd52d4SBartosz Sobczak }
728cdcd52d4SBartosz Sobczak 
729cdcd52d4SBartosz Sobczak /**
730cdcd52d4SBartosz Sobczak  * irdma_get_qp - get qp address
731cdcd52d4SBartosz Sobczak  * @device: iwarp device
732cdcd52d4SBartosz Sobczak  * @qpn: qp number
733cdcd52d4SBartosz Sobczak  */
734cdcd52d4SBartosz Sobczak struct ib_qp *
irdma_get_qp(struct ib_device * device,int qpn)735cdcd52d4SBartosz Sobczak irdma_get_qp(struct ib_device *device, int qpn)
736cdcd52d4SBartosz Sobczak {
737cdcd52d4SBartosz Sobczak 	struct irdma_device *iwdev = to_iwdev(device);
738cdcd52d4SBartosz Sobczak 
739cdcd52d4SBartosz Sobczak 	if (qpn < IW_FIRST_QPN || qpn >= iwdev->rf->max_qp)
740cdcd52d4SBartosz Sobczak 		return NULL;
741cdcd52d4SBartosz Sobczak 
742cdcd52d4SBartosz Sobczak 	return &iwdev->rf->qp_table[qpn]->ibqp;
743cdcd52d4SBartosz Sobczak }
744cdcd52d4SBartosz Sobczak 
745cdcd52d4SBartosz Sobczak /**
746cdcd52d4SBartosz Sobczak  * irdma_remove_cqp_head - return head entry and remove
747cdcd52d4SBartosz Sobczak  * @dev: device
748cdcd52d4SBartosz Sobczak  */
749cdcd52d4SBartosz Sobczak void *
irdma_remove_cqp_head(struct irdma_sc_dev * dev)750cdcd52d4SBartosz Sobczak irdma_remove_cqp_head(struct irdma_sc_dev *dev)
751cdcd52d4SBartosz Sobczak {
752cdcd52d4SBartosz Sobczak 	struct list_head *entry;
753cdcd52d4SBartosz Sobczak 	struct list_head *list = &dev->cqp_cmd_head;
754cdcd52d4SBartosz Sobczak 
755cdcd52d4SBartosz Sobczak 	if (list_empty(list))
756cdcd52d4SBartosz Sobczak 		return NULL;
757cdcd52d4SBartosz Sobczak 
758cdcd52d4SBartosz Sobczak 	entry = list->next;
759cdcd52d4SBartosz Sobczak 	list_del(entry);
760cdcd52d4SBartosz Sobczak 
761cdcd52d4SBartosz Sobczak 	return entry;
762cdcd52d4SBartosz Sobczak }
763cdcd52d4SBartosz Sobczak 
764cdcd52d4SBartosz Sobczak /**
765cdcd52d4SBartosz Sobczak  * irdma_cqp_sds_cmd - create cqp command for sd
766cdcd52d4SBartosz Sobczak  * @dev: hardware control device structure
767cdcd52d4SBartosz Sobczak  * @sdinfo: information for sd cqp
768cdcd52d4SBartosz Sobczak  *
769cdcd52d4SBartosz Sobczak  */
770cdcd52d4SBartosz Sobczak int
irdma_cqp_sds_cmd(struct irdma_sc_dev * dev,struct irdma_update_sds_info * sdinfo)771cdcd52d4SBartosz Sobczak irdma_cqp_sds_cmd(struct irdma_sc_dev *dev,
772cdcd52d4SBartosz Sobczak 		  struct irdma_update_sds_info *sdinfo)
773cdcd52d4SBartosz Sobczak {
774cdcd52d4SBartosz Sobczak 	struct irdma_cqp_request *cqp_request;
775cdcd52d4SBartosz Sobczak 	struct cqp_cmds_info *cqp_info;
776cdcd52d4SBartosz Sobczak 	struct irdma_pci_f *rf = dev_to_rf(dev);
777cdcd52d4SBartosz Sobczak 	int status;
778cdcd52d4SBartosz Sobczak 
779cdcd52d4SBartosz Sobczak 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
780cdcd52d4SBartosz Sobczak 	if (!cqp_request)
781cdcd52d4SBartosz Sobczak 		return -ENOMEM;
782cdcd52d4SBartosz Sobczak 
783cdcd52d4SBartosz Sobczak 	cqp_info = &cqp_request->info;
784cdcd52d4SBartosz Sobczak 	memcpy(&cqp_info->in.u.update_pe_sds.info, sdinfo,
785cdcd52d4SBartosz Sobczak 	       sizeof(cqp_info->in.u.update_pe_sds.info));
786cdcd52d4SBartosz Sobczak 	cqp_info->cqp_cmd = IRDMA_OP_UPDATE_PE_SDS;
787cdcd52d4SBartosz Sobczak 	cqp_info->post_sq = 1;
788cdcd52d4SBartosz Sobczak 	cqp_info->in.u.update_pe_sds.dev = dev;
789cdcd52d4SBartosz Sobczak 	cqp_info->in.u.update_pe_sds.scratch = (uintptr_t)cqp_request;
790cdcd52d4SBartosz Sobczak 
791cdcd52d4SBartosz Sobczak 	status = irdma_handle_cqp_op(rf, cqp_request);
792cdcd52d4SBartosz Sobczak 	irdma_put_cqp_request(&rf->cqp, cqp_request);
793cdcd52d4SBartosz Sobczak 
794cdcd52d4SBartosz Sobczak 	return status;
795cdcd52d4SBartosz Sobczak }
796cdcd52d4SBartosz Sobczak 
797cdcd52d4SBartosz Sobczak /**
798cdcd52d4SBartosz Sobczak  * irdma_cqp_qp_suspend_resume - cqp command for suspend/resume
799cdcd52d4SBartosz Sobczak  * @qp: hardware control qp
800cdcd52d4SBartosz Sobczak  * @op: suspend or resume
801cdcd52d4SBartosz Sobczak  */
802cdcd52d4SBartosz Sobczak int
irdma_cqp_qp_suspend_resume(struct irdma_sc_qp * qp,u8 op)803cdcd52d4SBartosz Sobczak irdma_cqp_qp_suspend_resume(struct irdma_sc_qp *qp, u8 op)
804cdcd52d4SBartosz Sobczak {
805cdcd52d4SBartosz Sobczak 	struct irdma_sc_dev *dev = qp->dev;
806cdcd52d4SBartosz Sobczak 	struct irdma_cqp_request *cqp_request;
807cdcd52d4SBartosz Sobczak 	struct irdma_sc_cqp *cqp = dev->cqp;
808cdcd52d4SBartosz Sobczak 	struct cqp_cmds_info *cqp_info;
809cdcd52d4SBartosz Sobczak 	struct irdma_pci_f *rf = dev_to_rf(dev);
810cdcd52d4SBartosz Sobczak 	int status;
811cdcd52d4SBartosz Sobczak 
812cdcd52d4SBartosz Sobczak 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false);
813cdcd52d4SBartosz Sobczak 	if (!cqp_request)
814cdcd52d4SBartosz Sobczak 		return -ENOMEM;
815cdcd52d4SBartosz Sobczak 
816cdcd52d4SBartosz Sobczak 	cqp_info = &cqp_request->info;
817cdcd52d4SBartosz Sobczak 	cqp_info->cqp_cmd = op;
818cdcd52d4SBartosz Sobczak 	cqp_info->in.u.suspend_resume.cqp = cqp;
819cdcd52d4SBartosz Sobczak 	cqp_info->in.u.suspend_resume.qp = qp;
820cdcd52d4SBartosz Sobczak 	cqp_info->in.u.suspend_resume.scratch = (uintptr_t)cqp_request;
821cdcd52d4SBartosz Sobczak 
822cdcd52d4SBartosz Sobczak 	status = irdma_handle_cqp_op(rf, cqp_request);
823cdcd52d4SBartosz Sobczak 	irdma_put_cqp_request(&rf->cqp, cqp_request);
824cdcd52d4SBartosz Sobczak 
825cdcd52d4SBartosz Sobczak 	return status;
826cdcd52d4SBartosz Sobczak }
827cdcd52d4SBartosz Sobczak 
828cdcd52d4SBartosz Sobczak /**
829cdcd52d4SBartosz Sobczak  * irdma_term_modify_qp - modify qp for term message
830cdcd52d4SBartosz Sobczak  * @qp: hardware control qp
831cdcd52d4SBartosz Sobczak  * @next_state: qp's next state
832cdcd52d4SBartosz Sobczak  * @term: terminate code
833cdcd52d4SBartosz Sobczak  * @term_len: length
834cdcd52d4SBartosz Sobczak  */
835cdcd52d4SBartosz Sobczak void
irdma_term_modify_qp(struct irdma_sc_qp * qp,u8 next_state,u8 term,u8 term_len)836cdcd52d4SBartosz Sobczak irdma_term_modify_qp(struct irdma_sc_qp *qp, u8 next_state, u8 term,
837cdcd52d4SBartosz Sobczak 		     u8 term_len)
838cdcd52d4SBartosz Sobczak {
839cdcd52d4SBartosz Sobczak 	struct irdma_qp *iwqp;
840cdcd52d4SBartosz Sobczak 
841cdcd52d4SBartosz Sobczak 	iwqp = qp->qp_uk.back_qp;
842cdcd52d4SBartosz Sobczak 	irdma_next_iw_state(iwqp, next_state, 0, term, term_len);
843cdcd52d4SBartosz Sobczak };
844cdcd52d4SBartosz Sobczak 
845cdcd52d4SBartosz Sobczak /**
846cdcd52d4SBartosz Sobczak  * irdma_terminate_done - after terminate is completed
847cdcd52d4SBartosz Sobczak  * @qp: hardware control qp
848cdcd52d4SBartosz Sobczak  * @timeout_occurred: indicates if terminate timer expired
849cdcd52d4SBartosz Sobczak  */
850cdcd52d4SBartosz Sobczak void
irdma_terminate_done(struct irdma_sc_qp * qp,int timeout_occurred)851cdcd52d4SBartosz Sobczak irdma_terminate_done(struct irdma_sc_qp *qp, int timeout_occurred)
852cdcd52d4SBartosz Sobczak {
853cdcd52d4SBartosz Sobczak 	struct irdma_qp *iwqp;
854cdcd52d4SBartosz Sobczak 	u8 hte = 0;
855cdcd52d4SBartosz Sobczak 	bool first_time;
856cdcd52d4SBartosz Sobczak 	unsigned long flags;
857cdcd52d4SBartosz Sobczak 
858cdcd52d4SBartosz Sobczak 	iwqp = qp->qp_uk.back_qp;
859cdcd52d4SBartosz Sobczak 	spin_lock_irqsave(&iwqp->lock, flags);
860cdcd52d4SBartosz Sobczak 	if (iwqp->hte_added) {
861cdcd52d4SBartosz Sobczak 		iwqp->hte_added = 0;
862cdcd52d4SBartosz Sobczak 		hte = 1;
863cdcd52d4SBartosz Sobczak 	}
864cdcd52d4SBartosz Sobczak 	first_time = !(qp->term_flags & IRDMA_TERM_DONE);
865cdcd52d4SBartosz Sobczak 	qp->term_flags |= IRDMA_TERM_DONE;
866cdcd52d4SBartosz Sobczak 	spin_unlock_irqrestore(&iwqp->lock, flags);
867cdcd52d4SBartosz Sobczak 	if (first_time) {
868cdcd52d4SBartosz Sobczak 		if (!timeout_occurred)
869cdcd52d4SBartosz Sobczak 			irdma_terminate_del_timer(qp);
870cdcd52d4SBartosz Sobczak 
871cdcd52d4SBartosz Sobczak 		irdma_next_iw_state(iwqp, IRDMA_QP_STATE_ERROR, hte, 0, 0);
872cdcd52d4SBartosz Sobczak 		irdma_cm_disconn(iwqp);
873cdcd52d4SBartosz Sobczak 	}
874cdcd52d4SBartosz Sobczak }
875cdcd52d4SBartosz Sobczak 
876cdcd52d4SBartosz Sobczak static void
irdma_terminate_timeout(struct timer_list * t)877cdcd52d4SBartosz Sobczak irdma_terminate_timeout(struct timer_list *t)
878cdcd52d4SBartosz Sobczak {
879cdcd52d4SBartosz Sobczak 	struct irdma_qp *iwqp = from_timer(iwqp, t, terminate_timer);
880cdcd52d4SBartosz Sobczak 	struct irdma_sc_qp *qp = &iwqp->sc_qp;
881cdcd52d4SBartosz Sobczak 
882cdcd52d4SBartosz Sobczak 	irdma_terminate_done(qp, 1);
883cdcd52d4SBartosz Sobczak 	irdma_qp_rem_ref(&iwqp->ibqp);
884cdcd52d4SBartosz Sobczak }
885cdcd52d4SBartosz Sobczak 
886cdcd52d4SBartosz Sobczak /**
887cdcd52d4SBartosz Sobczak  * irdma_terminate_start_timer - start terminate timeout
888cdcd52d4SBartosz Sobczak  * @qp: hardware control qp
889cdcd52d4SBartosz Sobczak  */
890cdcd52d4SBartosz Sobczak void
irdma_terminate_start_timer(struct irdma_sc_qp * qp)891cdcd52d4SBartosz Sobczak irdma_terminate_start_timer(struct irdma_sc_qp *qp)
892cdcd52d4SBartosz Sobczak {
893cdcd52d4SBartosz Sobczak 	struct irdma_qp *iwqp;
894cdcd52d4SBartosz Sobczak 
895cdcd52d4SBartosz Sobczak 	iwqp = qp->qp_uk.back_qp;
896cdcd52d4SBartosz Sobczak 	irdma_qp_add_ref(&iwqp->ibqp);
897cdcd52d4SBartosz Sobczak 	timer_setup(&iwqp->terminate_timer, irdma_terminate_timeout, 0);
898cdcd52d4SBartosz Sobczak 	iwqp->terminate_timer.expires = jiffies + HZ;
899cdcd52d4SBartosz Sobczak 
900cdcd52d4SBartosz Sobczak 	add_timer(&iwqp->terminate_timer);
901cdcd52d4SBartosz Sobczak }
902cdcd52d4SBartosz Sobczak 
903cdcd52d4SBartosz Sobczak /**
904cdcd52d4SBartosz Sobczak  * irdma_terminate_del_timer - delete terminate timeout
905cdcd52d4SBartosz Sobczak  * @qp: hardware control qp
906cdcd52d4SBartosz Sobczak  */
907cdcd52d4SBartosz Sobczak void
irdma_terminate_del_timer(struct irdma_sc_qp * qp)908cdcd52d4SBartosz Sobczak irdma_terminate_del_timer(struct irdma_sc_qp *qp)
909cdcd52d4SBartosz Sobczak {
910cdcd52d4SBartosz Sobczak 	struct irdma_qp *iwqp;
911cdcd52d4SBartosz Sobczak 	int ret;
912cdcd52d4SBartosz Sobczak 
913cdcd52d4SBartosz Sobczak 	iwqp = qp->qp_uk.back_qp;
914cdcd52d4SBartosz Sobczak 	ret = irdma_del_timer_compat(&iwqp->terminate_timer);
915cdcd52d4SBartosz Sobczak 	if (ret)
916cdcd52d4SBartosz Sobczak 		irdma_qp_rem_ref(&iwqp->ibqp);
917cdcd52d4SBartosz Sobczak }
918cdcd52d4SBartosz Sobczak 
919cdcd52d4SBartosz Sobczak /**
920cdcd52d4SBartosz Sobczak  * irdma_cqp_query_fpm_val_cmd - send cqp command for fpm
921cdcd52d4SBartosz Sobczak  * @dev: function device struct
922cdcd52d4SBartosz Sobczak  * @val_mem: buffer for fpm
923cdcd52d4SBartosz Sobczak  * @hmc_fn_id: function id for fpm
924cdcd52d4SBartosz Sobczak  */
925cdcd52d4SBartosz Sobczak int
irdma_cqp_query_fpm_val_cmd(struct irdma_sc_dev * dev,struct irdma_dma_mem * val_mem,u16 hmc_fn_id)926cdcd52d4SBartosz Sobczak irdma_cqp_query_fpm_val_cmd(struct irdma_sc_dev *dev,
927777e472cSBartosz Sobczak 			    struct irdma_dma_mem *val_mem, u16 hmc_fn_id)
928cdcd52d4SBartosz Sobczak {
929cdcd52d4SBartosz Sobczak 	struct irdma_cqp_request *cqp_request;
930cdcd52d4SBartosz Sobczak 	struct cqp_cmds_info *cqp_info;
931cdcd52d4SBartosz Sobczak 	struct irdma_pci_f *rf = dev_to_rf(dev);
932cdcd52d4SBartosz Sobczak 	int status;
933cdcd52d4SBartosz Sobczak 
934cdcd52d4SBartosz Sobczak 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
935cdcd52d4SBartosz Sobczak 	if (!cqp_request)
936cdcd52d4SBartosz Sobczak 		return -ENOMEM;
937cdcd52d4SBartosz Sobczak 
938cdcd52d4SBartosz Sobczak 	cqp_info = &cqp_request->info;
939cdcd52d4SBartosz Sobczak 	cqp_request->param = NULL;
940cdcd52d4SBartosz Sobczak 	cqp_info->in.u.query_fpm_val.cqp = dev->cqp;
941cdcd52d4SBartosz Sobczak 	cqp_info->in.u.query_fpm_val.fpm_val_pa = val_mem->pa;
942cdcd52d4SBartosz Sobczak 	cqp_info->in.u.query_fpm_val.fpm_val_va = val_mem->va;
943cdcd52d4SBartosz Sobczak 	cqp_info->in.u.query_fpm_val.hmc_fn_id = hmc_fn_id;
944cdcd52d4SBartosz Sobczak 	cqp_info->cqp_cmd = IRDMA_OP_QUERY_FPM_VAL;
945cdcd52d4SBartosz Sobczak 	cqp_info->post_sq = 1;
946cdcd52d4SBartosz Sobczak 	cqp_info->in.u.query_fpm_val.scratch = (uintptr_t)cqp_request;
947cdcd52d4SBartosz Sobczak 
948cdcd52d4SBartosz Sobczak 	status = irdma_handle_cqp_op(rf, cqp_request);
949cdcd52d4SBartosz Sobczak 	irdma_put_cqp_request(&rf->cqp, cqp_request);
950cdcd52d4SBartosz Sobczak 
951cdcd52d4SBartosz Sobczak 	return status;
952cdcd52d4SBartosz Sobczak }
953cdcd52d4SBartosz Sobczak 
954cdcd52d4SBartosz Sobczak /**
955cdcd52d4SBartosz Sobczak  * irdma_cqp_commit_fpm_val_cmd - commit fpm values in hw
956cdcd52d4SBartosz Sobczak  * @dev: hardware control device structure
957cdcd52d4SBartosz Sobczak  * @val_mem: buffer with fpm values
958cdcd52d4SBartosz Sobczak  * @hmc_fn_id: function id for fpm
959cdcd52d4SBartosz Sobczak  */
960cdcd52d4SBartosz Sobczak int
irdma_cqp_commit_fpm_val_cmd(struct irdma_sc_dev * dev,struct irdma_dma_mem * val_mem,u16 hmc_fn_id)961cdcd52d4SBartosz Sobczak irdma_cqp_commit_fpm_val_cmd(struct irdma_sc_dev *dev,
962777e472cSBartosz Sobczak 			     struct irdma_dma_mem *val_mem, u16 hmc_fn_id)
963cdcd52d4SBartosz Sobczak {
964cdcd52d4SBartosz Sobczak 	struct irdma_cqp_request *cqp_request;
965cdcd52d4SBartosz Sobczak 	struct cqp_cmds_info *cqp_info;
966cdcd52d4SBartosz Sobczak 	struct irdma_pci_f *rf = dev_to_rf(dev);
967cdcd52d4SBartosz Sobczak 	int status;
968cdcd52d4SBartosz Sobczak 
969cdcd52d4SBartosz Sobczak 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
970cdcd52d4SBartosz Sobczak 	if (!cqp_request)
971cdcd52d4SBartosz Sobczak 		return -ENOMEM;
972cdcd52d4SBartosz Sobczak 
973cdcd52d4SBartosz Sobczak 	cqp_info = &cqp_request->info;
974cdcd52d4SBartosz Sobczak 	cqp_request->param = NULL;
975cdcd52d4SBartosz Sobczak 	cqp_info->in.u.commit_fpm_val.cqp = dev->cqp;
976cdcd52d4SBartosz Sobczak 	cqp_info->in.u.commit_fpm_val.fpm_val_pa = val_mem->pa;
977cdcd52d4SBartosz Sobczak 	cqp_info->in.u.commit_fpm_val.fpm_val_va = val_mem->va;
978cdcd52d4SBartosz Sobczak 	cqp_info->in.u.commit_fpm_val.hmc_fn_id = hmc_fn_id;
979cdcd52d4SBartosz Sobczak 	cqp_info->cqp_cmd = IRDMA_OP_COMMIT_FPM_VAL;
980cdcd52d4SBartosz Sobczak 	cqp_info->post_sq = 1;
981cdcd52d4SBartosz Sobczak 	cqp_info->in.u.commit_fpm_val.scratch = (uintptr_t)cqp_request;
982cdcd52d4SBartosz Sobczak 
983cdcd52d4SBartosz Sobczak 	status = irdma_handle_cqp_op(rf, cqp_request);
984cdcd52d4SBartosz Sobczak 	irdma_put_cqp_request(&rf->cqp, cqp_request);
985cdcd52d4SBartosz Sobczak 
986cdcd52d4SBartosz Sobczak 	return status;
987cdcd52d4SBartosz Sobczak }
988cdcd52d4SBartosz Sobczak 
989cdcd52d4SBartosz Sobczak /**
990cdcd52d4SBartosz Sobczak  * irdma_cqp_cq_create_cmd - create a cq for the cqp
991cdcd52d4SBartosz Sobczak  * @dev: device pointer
992cdcd52d4SBartosz Sobczak  * @cq: pointer to created cq
993cdcd52d4SBartosz Sobczak  */
994cdcd52d4SBartosz Sobczak int
irdma_cqp_cq_create_cmd(struct irdma_sc_dev * dev,struct irdma_sc_cq * cq)995cdcd52d4SBartosz Sobczak irdma_cqp_cq_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq)
996cdcd52d4SBartosz Sobczak {
997cdcd52d4SBartosz Sobczak 	struct irdma_pci_f *rf = dev_to_rf(dev);
998cdcd52d4SBartosz Sobczak 	struct irdma_cqp *iwcqp = &rf->cqp;
999cdcd52d4SBartosz Sobczak 	struct irdma_cqp_request *cqp_request;
1000cdcd52d4SBartosz Sobczak 	struct cqp_cmds_info *cqp_info;
1001cdcd52d4SBartosz Sobczak 	int status;
1002cdcd52d4SBartosz Sobczak 
1003cdcd52d4SBartosz Sobczak 	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);
1004cdcd52d4SBartosz Sobczak 	if (!cqp_request)
1005cdcd52d4SBartosz Sobczak 		return -ENOMEM;
1006cdcd52d4SBartosz Sobczak 
1007cdcd52d4SBartosz Sobczak 	cqp_info = &cqp_request->info;
1008cdcd52d4SBartosz Sobczak 	cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE;
1009cdcd52d4SBartosz Sobczak 	cqp_info->post_sq = 1;
1010cdcd52d4SBartosz Sobczak 	cqp_info->in.u.cq_create.cq = cq;
1011cdcd52d4SBartosz Sobczak 	cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request;
1012cdcd52d4SBartosz Sobczak 
1013cdcd52d4SBartosz Sobczak 	status = irdma_handle_cqp_op(rf, cqp_request);
1014cdcd52d4SBartosz Sobczak 	irdma_put_cqp_request(iwcqp, cqp_request);
1015cdcd52d4SBartosz Sobczak 
1016cdcd52d4SBartosz Sobczak 	return status;
1017cdcd52d4SBartosz Sobczak }
1018cdcd52d4SBartosz Sobczak 
1019cdcd52d4SBartosz Sobczak /**
1020cdcd52d4SBartosz Sobczak  * irdma_cqp_qp_create_cmd - create a qp for the cqp
1021cdcd52d4SBartosz Sobczak  * @dev: device pointer
1022cdcd52d4SBartosz Sobczak  * @qp: pointer to created qp
1023cdcd52d4SBartosz Sobczak  */
1024cdcd52d4SBartosz Sobczak int
irdma_cqp_qp_create_cmd(struct irdma_sc_dev * dev,struct irdma_sc_qp * qp)1025cdcd52d4SBartosz Sobczak irdma_cqp_qp_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
1026cdcd52d4SBartosz Sobczak {
1027cdcd52d4SBartosz Sobczak 	struct irdma_pci_f *rf = dev_to_rf(dev);
1028cdcd52d4SBartosz Sobczak 	struct irdma_cqp *iwcqp = &rf->cqp;
1029cdcd52d4SBartosz Sobczak 	struct irdma_cqp_request *cqp_request;
1030cdcd52d4SBartosz Sobczak 	struct cqp_cmds_info *cqp_info;
1031cdcd52d4SBartosz Sobczak 	struct irdma_create_qp_info *qp_info;
1032cdcd52d4SBartosz Sobczak 	int status;
1033cdcd52d4SBartosz Sobczak 
1034cdcd52d4SBartosz Sobczak 	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);
1035cdcd52d4SBartosz Sobczak 	if (!cqp_request)
1036cdcd52d4SBartosz Sobczak 		return -ENOMEM;
1037cdcd52d4SBartosz Sobczak 
1038cdcd52d4SBartosz Sobczak 	cqp_info = &cqp_request->info;
1039cdcd52d4SBartosz Sobczak 	qp_info = &cqp_request->info.in.u.qp_create.info;
1040cdcd52d4SBartosz Sobczak 	memset(qp_info, 0, sizeof(*qp_info));
1041cdcd52d4SBartosz Sobczak 	qp_info->cq_num_valid = true;
1042cdcd52d4SBartosz Sobczak 	qp_info->next_iwarp_state = IRDMA_QP_STATE_RTS;
1043cdcd52d4SBartosz Sobczak 	cqp_info->cqp_cmd = IRDMA_OP_QP_CREATE;
1044cdcd52d4SBartosz Sobczak 	cqp_info->post_sq = 1;
1045cdcd52d4SBartosz Sobczak 	cqp_info->in.u.qp_create.qp = qp;
1046cdcd52d4SBartosz Sobczak 	cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request;
1047cdcd52d4SBartosz Sobczak 
1048cdcd52d4SBartosz Sobczak 	status = irdma_handle_cqp_op(rf, cqp_request);
1049cdcd52d4SBartosz Sobczak 	irdma_put_cqp_request(iwcqp, cqp_request);
1050cdcd52d4SBartosz Sobczak 
1051cdcd52d4SBartosz Sobczak 	return status;
1052cdcd52d4SBartosz Sobczak }
1053cdcd52d4SBartosz Sobczak 
1054cdcd52d4SBartosz Sobczak /**
1055cdcd52d4SBartosz Sobczak  * irdma_dealloc_push_page - free a push page for qp
1056cdcd52d4SBartosz Sobczak  * @rf: RDMA PCI function
1057*5b5f7d0eSBartosz Sobczak  * @iwqp: QP pointer
1058cdcd52d4SBartosz Sobczak  */
1059cdcd52d4SBartosz Sobczak void
irdma_dealloc_push_page(struct irdma_pci_f * rf,struct irdma_qp * iwqp)1060cdcd52d4SBartosz Sobczak irdma_dealloc_push_page(struct irdma_pci_f *rf,
1061*5b5f7d0eSBartosz Sobczak 			struct irdma_qp *iwqp)
1062cdcd52d4SBartosz Sobczak {
1063cdcd52d4SBartosz Sobczak 	struct irdma_cqp_request *cqp_request;
1064cdcd52d4SBartosz Sobczak 	struct cqp_cmds_info *cqp_info;
1065cdcd52d4SBartosz Sobczak 	int status;
1066*5b5f7d0eSBartosz Sobczak 	struct irdma_sc_qp *qp = &iwqp->sc_qp;
1067cdcd52d4SBartosz Sobczak 
1068cdcd52d4SBartosz Sobczak 	if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX)
1069cdcd52d4SBartosz Sobczak 		return;
1070cdcd52d4SBartosz Sobczak 
1071cdcd52d4SBartosz Sobczak 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false);
1072cdcd52d4SBartosz Sobczak 	if (!cqp_request)
1073cdcd52d4SBartosz Sobczak 		return;
1074cdcd52d4SBartosz Sobczak 
1075cdcd52d4SBartosz Sobczak 	cqp_info = &cqp_request->info;
1076cdcd52d4SBartosz Sobczak 	cqp_info->cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE;
1077cdcd52d4SBartosz Sobczak 	cqp_info->post_sq = 1;
1078cdcd52d4SBartosz Sobczak 	cqp_info->in.u.manage_push_page.info.push_idx = qp->push_idx;
1079cdcd52d4SBartosz Sobczak 	cqp_info->in.u.manage_push_page.info.qs_handle = qp->qs_handle;
1080cdcd52d4SBartosz Sobczak 	cqp_info->in.u.manage_push_page.info.free_page = 1;
1081cdcd52d4SBartosz Sobczak 	cqp_info->in.u.manage_push_page.info.push_page_type = 0;
1082cdcd52d4SBartosz Sobczak 	cqp_info->in.u.manage_push_page.cqp = &rf->cqp.sc_cqp;
1083cdcd52d4SBartosz Sobczak 	cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request;
1084cdcd52d4SBartosz Sobczak 	status = irdma_handle_cqp_op(rf, cqp_request);
1085cdcd52d4SBartosz Sobczak 	if (!status)
1086cdcd52d4SBartosz Sobczak 		qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX;
1087cdcd52d4SBartosz Sobczak 	irdma_put_cqp_request(&rf->cqp, cqp_request);
1088cdcd52d4SBartosz Sobczak }
1089cdcd52d4SBartosz Sobczak 
1090cdcd52d4SBartosz Sobczak /**
1091cdcd52d4SBartosz Sobczak  * irdma_cq_wq_destroy - send cq destroy cqp
1092cdcd52d4SBartosz Sobczak  * @rf: RDMA PCI function
1093cdcd52d4SBartosz Sobczak  * @cq: hardware control cq
1094cdcd52d4SBartosz Sobczak  */
1095cdcd52d4SBartosz Sobczak void
irdma_cq_wq_destroy(struct irdma_pci_f * rf,struct irdma_sc_cq * cq)1096cdcd52d4SBartosz Sobczak irdma_cq_wq_destroy(struct irdma_pci_f *rf, struct irdma_sc_cq *cq)
1097cdcd52d4SBartosz Sobczak {
1098cdcd52d4SBartosz Sobczak 	struct irdma_cqp_request *cqp_request;
1099cdcd52d4SBartosz Sobczak 	struct cqp_cmds_info *cqp_info;
1100cdcd52d4SBartosz Sobczak 
1101cdcd52d4SBartosz Sobczak 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
1102cdcd52d4SBartosz Sobczak 	if (!cqp_request)
1103cdcd52d4SBartosz Sobczak 		return;
1104cdcd52d4SBartosz Sobczak 
1105cdcd52d4SBartosz Sobczak 	cqp_info = &cqp_request->info;
1106cdcd52d4SBartosz Sobczak 	cqp_info->cqp_cmd = IRDMA_OP_CQ_DESTROY;
1107cdcd52d4SBartosz Sobczak 	cqp_info->post_sq = 1;
1108cdcd52d4SBartosz Sobczak 	cqp_info->in.u.cq_destroy.cq = cq;
1109cdcd52d4SBartosz Sobczak 	cqp_info->in.u.cq_destroy.scratch = (uintptr_t)cqp_request;
1110cdcd52d4SBartosz Sobczak 
1111cdcd52d4SBartosz Sobczak 	irdma_handle_cqp_op(rf, cqp_request);
1112cdcd52d4SBartosz Sobczak 	irdma_put_cqp_request(&rf->cqp, cqp_request);
1113cdcd52d4SBartosz Sobczak }
1114cdcd52d4SBartosz Sobczak 
1115cdcd52d4SBartosz Sobczak /**
1116cdcd52d4SBartosz Sobczak  * irdma_hw_modify_qp_callback - handle state for modifyQPs that don't wait
1117cdcd52d4SBartosz Sobczak  * @cqp_request: modify QP completion
1118cdcd52d4SBartosz Sobczak  */
1119cdcd52d4SBartosz Sobczak static void
irdma_hw_modify_qp_callback(struct irdma_cqp_request * cqp_request)1120cdcd52d4SBartosz Sobczak irdma_hw_modify_qp_callback(struct irdma_cqp_request *cqp_request)
1121cdcd52d4SBartosz Sobczak {
1122cdcd52d4SBartosz Sobczak 	struct cqp_cmds_info *cqp_info;
1123cdcd52d4SBartosz Sobczak 	struct irdma_qp *iwqp;
1124cdcd52d4SBartosz Sobczak 
1125cdcd52d4SBartosz Sobczak 	cqp_info = &cqp_request->info;
1126cdcd52d4SBartosz Sobczak 	iwqp = cqp_info->in.u.qp_modify.qp->qp_uk.back_qp;
1127cdcd52d4SBartosz Sobczak 	atomic_dec(&iwqp->hw_mod_qp_pend);
1128cdcd52d4SBartosz Sobczak 	wake_up(&iwqp->mod_qp_waitq);
1129cdcd52d4SBartosz Sobczak }
1130cdcd52d4SBartosz Sobczak 
1131cdcd52d4SBartosz Sobczak /**
1132cdcd52d4SBartosz Sobczak  * irdma_hw_modify_qp - setup cqp for modify qp
1133cdcd52d4SBartosz Sobczak  * @iwdev: RDMA device
1134cdcd52d4SBartosz Sobczak  * @iwqp: qp ptr (user or kernel)
1135cdcd52d4SBartosz Sobczak  * @info: info for modify qp
1136cdcd52d4SBartosz Sobczak  * @wait: flag to wait or not for modify qp completion
1137cdcd52d4SBartosz Sobczak  */
1138cdcd52d4SBartosz Sobczak int
irdma_hw_modify_qp(struct irdma_device * iwdev,struct irdma_qp * iwqp,struct irdma_modify_qp_info * info,bool wait)1139cdcd52d4SBartosz Sobczak irdma_hw_modify_qp(struct irdma_device *iwdev, struct irdma_qp *iwqp,
1140cdcd52d4SBartosz Sobczak 		   struct irdma_modify_qp_info *info, bool wait)
1141cdcd52d4SBartosz Sobczak {
1142cdcd52d4SBartosz Sobczak 	int status;
1143cdcd52d4SBartosz Sobczak 	struct irdma_pci_f *rf = iwdev->rf;
1144cdcd52d4SBartosz Sobczak 	struct irdma_cqp_request *cqp_request;
1145cdcd52d4SBartosz Sobczak 	struct cqp_cmds_info *cqp_info;
1146cdcd52d4SBartosz Sobczak 	struct irdma_modify_qp_info *m_info;
1147cdcd52d4SBartosz Sobczak 
1148cdcd52d4SBartosz Sobczak 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait);
1149cdcd52d4SBartosz Sobczak 	if (!cqp_request)
1150cdcd52d4SBartosz Sobczak 		return -ENOMEM;
1151cdcd52d4SBartosz Sobczak 
1152cdcd52d4SBartosz Sobczak 	if (!wait) {
1153cdcd52d4SBartosz Sobczak 		cqp_request->callback_fcn = irdma_hw_modify_qp_callback;
1154cdcd52d4SBartosz Sobczak 		atomic_inc(&iwqp->hw_mod_qp_pend);
1155cdcd52d4SBartosz Sobczak 	}
1156cdcd52d4SBartosz Sobczak 	cqp_info = &cqp_request->info;
1157cdcd52d4SBartosz Sobczak 	m_info = &cqp_info->in.u.qp_modify.info;
1158cdcd52d4SBartosz Sobczak 	memcpy(m_info, info, sizeof(*m_info));
1159cdcd52d4SBartosz Sobczak 	cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY;
1160cdcd52d4SBartosz Sobczak 	cqp_info->post_sq = 1;
1161cdcd52d4SBartosz Sobczak 	cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp;
1162cdcd52d4SBartosz Sobczak 	cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request;
1163cdcd52d4SBartosz Sobczak 	status = irdma_handle_cqp_op(rf, cqp_request);
1164cdcd52d4SBartosz Sobczak 	irdma_put_cqp_request(&rf->cqp, cqp_request);
1165cdcd52d4SBartosz Sobczak 	if (status) {
1166cdcd52d4SBartosz Sobczak 		if (rdma_protocol_roce(&iwdev->ibdev, 1))
1167cdcd52d4SBartosz Sobczak 			return status;
1168cdcd52d4SBartosz Sobczak 
1169cdcd52d4SBartosz Sobczak 		switch (m_info->next_iwarp_state) {
1170cdcd52d4SBartosz Sobczak 			struct irdma_gen_ae_info ae_info;
1171cdcd52d4SBartosz Sobczak 
1172cdcd52d4SBartosz Sobczak 		case IRDMA_QP_STATE_RTS:
1173cdcd52d4SBartosz Sobczak 		case IRDMA_QP_STATE_IDLE:
1174cdcd52d4SBartosz Sobczak 		case IRDMA_QP_STATE_TERMINATE:
1175cdcd52d4SBartosz Sobczak 		case IRDMA_QP_STATE_CLOSING:
1176cdcd52d4SBartosz Sobczak 			if (info->curr_iwarp_state == IRDMA_QP_STATE_IDLE)
1177cdcd52d4SBartosz Sobczak 				irdma_send_reset(iwqp->cm_node);
1178cdcd52d4SBartosz Sobczak 			else
1179cdcd52d4SBartosz Sobczak 				iwqp->sc_qp.term_flags = IRDMA_TERM_DONE;
1180cdcd52d4SBartosz Sobczak 			if (!wait) {
1181cdcd52d4SBartosz Sobczak 				ae_info.ae_code = IRDMA_AE_BAD_CLOSE;
1182cdcd52d4SBartosz Sobczak 				ae_info.ae_src = 0;
1183cdcd52d4SBartosz Sobczak 				irdma_gen_ae(rf, &iwqp->sc_qp, &ae_info, false);
1184cdcd52d4SBartosz Sobczak 			} else {
1185cdcd52d4SBartosz Sobczak 				cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp,
1186cdcd52d4SBartosz Sobczak 									      wait);
1187cdcd52d4SBartosz Sobczak 				if (!cqp_request)
1188cdcd52d4SBartosz Sobczak 					return -ENOMEM;
1189cdcd52d4SBartosz Sobczak 
1190cdcd52d4SBartosz Sobczak 				cqp_info = &cqp_request->info;
1191cdcd52d4SBartosz Sobczak 				m_info = &cqp_info->in.u.qp_modify.info;
1192cdcd52d4SBartosz Sobczak 				memcpy(m_info, info, sizeof(*m_info));
1193cdcd52d4SBartosz Sobczak 				cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY;
1194cdcd52d4SBartosz Sobczak 				cqp_info->post_sq = 1;
1195cdcd52d4SBartosz Sobczak 				cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp;
1196cdcd52d4SBartosz Sobczak 				cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request;
1197cdcd52d4SBartosz Sobczak 				m_info->next_iwarp_state = IRDMA_QP_STATE_ERROR;
1198cdcd52d4SBartosz Sobczak 				m_info->reset_tcp_conn = true;
1199cdcd52d4SBartosz Sobczak 				irdma_handle_cqp_op(rf, cqp_request);
1200cdcd52d4SBartosz Sobczak 				irdma_put_cqp_request(&rf->cqp, cqp_request);
1201cdcd52d4SBartosz Sobczak 			}
1202cdcd52d4SBartosz Sobczak 			break;
1203cdcd52d4SBartosz Sobczak 		case IRDMA_QP_STATE_ERROR:
1204cdcd52d4SBartosz Sobczak 		default:
1205cdcd52d4SBartosz Sobczak 			break;
1206cdcd52d4SBartosz Sobczak 		}
1207cdcd52d4SBartosz Sobczak 	}
1208cdcd52d4SBartosz Sobczak 
1209cdcd52d4SBartosz Sobczak 	return status;
1210cdcd52d4SBartosz Sobczak }
1211cdcd52d4SBartosz Sobczak 
1212cdcd52d4SBartosz Sobczak /**
1213cdcd52d4SBartosz Sobczak  * irdma_cqp_cq_destroy_cmd - destroy the cqp cq
1214cdcd52d4SBartosz Sobczak  * @dev: device pointer
1215cdcd52d4SBartosz Sobczak  * @cq: pointer to cq
1216cdcd52d4SBartosz Sobczak  */
1217cdcd52d4SBartosz Sobczak void
irdma_cqp_cq_destroy_cmd(struct irdma_sc_dev * dev,struct irdma_sc_cq * cq)1218cdcd52d4SBartosz Sobczak irdma_cqp_cq_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq)
1219cdcd52d4SBartosz Sobczak {
1220cdcd52d4SBartosz Sobczak 	struct irdma_pci_f *rf = dev_to_rf(dev);
1221cdcd52d4SBartosz Sobczak 
1222cdcd52d4SBartosz Sobczak 	irdma_cq_wq_destroy(rf, cq);
1223cdcd52d4SBartosz Sobczak }
1224cdcd52d4SBartosz Sobczak 
1225cdcd52d4SBartosz Sobczak /**
1226cdcd52d4SBartosz Sobczak  * irdma_cqp_qp_destroy_cmd - destroy the cqp
1227cdcd52d4SBartosz Sobczak  * @dev: device pointer
1228cdcd52d4SBartosz Sobczak  * @qp: pointer to qp
1229cdcd52d4SBartosz Sobczak  */
1230cdcd52d4SBartosz Sobczak int
irdma_cqp_qp_destroy_cmd(struct irdma_sc_dev * dev,struct irdma_sc_qp * qp)1231cdcd52d4SBartosz Sobczak irdma_cqp_qp_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
1232cdcd52d4SBartosz Sobczak {
1233cdcd52d4SBartosz Sobczak 	struct irdma_pci_f *rf = dev_to_rf(dev);
1234cdcd52d4SBartosz Sobczak 	struct irdma_cqp *iwcqp = &rf->cqp;
1235cdcd52d4SBartosz Sobczak 	struct irdma_cqp_request *cqp_request;
1236cdcd52d4SBartosz Sobczak 	struct cqp_cmds_info *cqp_info;
1237cdcd52d4SBartosz Sobczak 	int status;
1238cdcd52d4SBartosz Sobczak 
1239cdcd52d4SBartosz Sobczak 	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);
1240cdcd52d4SBartosz Sobczak 	if (!cqp_request)
1241cdcd52d4SBartosz Sobczak 		return -ENOMEM;
1242cdcd52d4SBartosz Sobczak 
1243cdcd52d4SBartosz Sobczak 	cqp_info = &cqp_request->info;
1244cdcd52d4SBartosz Sobczak 	memset(cqp_info, 0, sizeof(*cqp_info));
1245cdcd52d4SBartosz Sobczak 	cqp_info->cqp_cmd = IRDMA_OP_QP_DESTROY;
1246cdcd52d4SBartosz Sobczak 	cqp_info->post_sq = 1;
1247cdcd52d4SBartosz Sobczak 	cqp_info->in.u.qp_destroy.qp = qp;
1248cdcd52d4SBartosz Sobczak 	cqp_info->in.u.qp_destroy.scratch = (uintptr_t)cqp_request;
1249cdcd52d4SBartosz Sobczak 	cqp_info->in.u.qp_destroy.remove_hash_idx = true;
1250cdcd52d4SBartosz Sobczak 
1251cdcd52d4SBartosz Sobczak 	status = irdma_handle_cqp_op(rf, cqp_request);
1252cdcd52d4SBartosz Sobczak 	irdma_put_cqp_request(&rf->cqp, cqp_request);
1253cdcd52d4SBartosz Sobczak 
1254cdcd52d4SBartosz Sobczak 	return status;
1255cdcd52d4SBartosz Sobczak }
1256cdcd52d4SBartosz Sobczak 
1257cdcd52d4SBartosz Sobczak /**
1258cdcd52d4SBartosz Sobczak  * irdma_ieq_mpa_crc_ae - generate AE for crc error
1259cdcd52d4SBartosz Sobczak  * @dev: hardware control device structure
1260cdcd52d4SBartosz Sobczak  * @qp: hardware control qp
1261cdcd52d4SBartosz Sobczak  */
1262cdcd52d4SBartosz Sobczak void
irdma_ieq_mpa_crc_ae(struct irdma_sc_dev * dev,struct irdma_sc_qp * qp)1263cdcd52d4SBartosz Sobczak irdma_ieq_mpa_crc_ae(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
1264cdcd52d4SBartosz Sobczak {
1265cdcd52d4SBartosz Sobczak 	struct irdma_gen_ae_info info = {0};
1266cdcd52d4SBartosz Sobczak 	struct irdma_pci_f *rf = dev_to_rf(dev);
1267cdcd52d4SBartosz Sobczak 
126835105900SBartosz Sobczak 	irdma_debug(&rf->sc_dev, IRDMA_DEBUG_AEQ, "Generate MPA CRC AE\n");
1269cdcd52d4SBartosz Sobczak 	info.ae_code = IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR;
1270cdcd52d4SBartosz Sobczak 	info.ae_src = IRDMA_AE_SOURCE_RQ;
1271cdcd52d4SBartosz Sobczak 	irdma_gen_ae(rf, qp, &info, false);
1272cdcd52d4SBartosz Sobczak }
1273cdcd52d4SBartosz Sobczak 
1274cdcd52d4SBartosz Sobczak /**
1275cdcd52d4SBartosz Sobczak  * irdma_ieq_get_qp - get qp based on quad in puda buffer
1276cdcd52d4SBartosz Sobczak  * @dev: hardware control device structure
1277cdcd52d4SBartosz Sobczak  * @buf: receive puda buffer on exception q
1278cdcd52d4SBartosz Sobczak  */
1279cdcd52d4SBartosz Sobczak struct irdma_sc_qp *
irdma_ieq_get_qp(struct irdma_sc_dev * dev,struct irdma_puda_buf * buf)1280cdcd52d4SBartosz Sobczak irdma_ieq_get_qp(struct irdma_sc_dev *dev,
1281cdcd52d4SBartosz Sobczak 		 struct irdma_puda_buf *buf)
1282cdcd52d4SBartosz Sobczak {
1283cdcd52d4SBartosz Sobczak 	struct irdma_qp *iwqp;
1284cdcd52d4SBartosz Sobczak 	struct irdma_cm_node *cm_node;
1285cdcd52d4SBartosz Sobczak 	struct irdma_device *iwdev = buf->vsi->back_vsi;
1286cdcd52d4SBartosz Sobczak 	u32 loc_addr[4] = {0};
1287cdcd52d4SBartosz Sobczak 	u32 rem_addr[4] = {0};
1288cdcd52d4SBartosz Sobczak 	u16 loc_port, rem_port;
1289cdcd52d4SBartosz Sobczak 	struct ip6_hdr *ip6h;
1290cdcd52d4SBartosz Sobczak 	struct ip *iph = (struct ip *)buf->iph;
1291cdcd52d4SBartosz Sobczak 	struct tcphdr *tcph = (struct tcphdr *)buf->tcph;
1292cdcd52d4SBartosz Sobczak 
1293cdcd52d4SBartosz Sobczak 	if (iph->ip_v == 4) {
1294cdcd52d4SBartosz Sobczak 		loc_addr[0] = ntohl(iph->ip_dst.s_addr);
1295cdcd52d4SBartosz Sobczak 		rem_addr[0] = ntohl(iph->ip_src.s_addr);
1296cdcd52d4SBartosz Sobczak 	} else {
1297cdcd52d4SBartosz Sobczak 		ip6h = (struct ip6_hdr *)buf->iph;
1298cdcd52d4SBartosz Sobczak 		irdma_copy_ip_ntohl(loc_addr, ip6h->ip6_dst.__u6_addr.__u6_addr32);
1299cdcd52d4SBartosz Sobczak 		irdma_copy_ip_ntohl(rem_addr, ip6h->ip6_src.__u6_addr.__u6_addr32);
1300cdcd52d4SBartosz Sobczak 	}
1301cdcd52d4SBartosz Sobczak 	loc_port = ntohs(tcph->th_dport);
1302cdcd52d4SBartosz Sobczak 	rem_port = ntohs(tcph->th_sport);
1303cdcd52d4SBartosz Sobczak 	cm_node = irdma_find_node(&iwdev->cm_core, rem_port, rem_addr, loc_port,
1304cdcd52d4SBartosz Sobczak 				  loc_addr, buf->vlan_valid ? buf->vlan_id : 0xFFFF);
1305cdcd52d4SBartosz Sobczak 	if (!cm_node)
1306cdcd52d4SBartosz Sobczak 		return NULL;
1307cdcd52d4SBartosz Sobczak 
1308cdcd52d4SBartosz Sobczak 	iwqp = cm_node->iwqp;
1309cdcd52d4SBartosz Sobczak 	irdma_rem_ref_cm_node(cm_node);
1310cdcd52d4SBartosz Sobczak 
1311cdcd52d4SBartosz Sobczak 	return &iwqp->sc_qp;
1312cdcd52d4SBartosz Sobczak }
1313cdcd52d4SBartosz Sobczak 
1314cdcd52d4SBartosz Sobczak /**
1315cdcd52d4SBartosz Sobczak  * irdma_send_ieq_ack - ACKs for duplicate or OOO partials FPDUs
1316cdcd52d4SBartosz Sobczak  * @qp: qp ptr
1317cdcd52d4SBartosz Sobczak  */
1318cdcd52d4SBartosz Sobczak void
irdma_send_ieq_ack(struct irdma_sc_qp * qp)1319cdcd52d4SBartosz Sobczak irdma_send_ieq_ack(struct irdma_sc_qp *qp)
1320cdcd52d4SBartosz Sobczak {
1321cdcd52d4SBartosz Sobczak 	struct irdma_cm_node *cm_node = ((struct irdma_qp *)qp->qp_uk.back_qp)->cm_node;
1322cdcd52d4SBartosz Sobczak 	struct irdma_puda_buf *buf = qp->pfpdu.lastrcv_buf;
1323cdcd52d4SBartosz Sobczak 	struct tcphdr *tcph = (struct tcphdr *)buf->tcph;
1324cdcd52d4SBartosz Sobczak 
1325cdcd52d4SBartosz Sobczak 	cm_node->tcp_cntxt.rcv_nxt = qp->pfpdu.nextseqnum;
1326cdcd52d4SBartosz Sobczak 	cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->th_ack);
1327cdcd52d4SBartosz Sobczak 
1328cdcd52d4SBartosz Sobczak 	irdma_send_ack(cm_node);
1329cdcd52d4SBartosz Sobczak }
1330cdcd52d4SBartosz Sobczak 
1331cdcd52d4SBartosz Sobczak /**
1332cdcd52d4SBartosz Sobczak  * irdma_puda_ieq_get_ah_info - get AH info from IEQ buffer
1333cdcd52d4SBartosz Sobczak  * @qp: qp pointer
1334cdcd52d4SBartosz Sobczak  * @ah_info: AH info pointer
1335cdcd52d4SBartosz Sobczak  */
1336cdcd52d4SBartosz Sobczak void
irdma_puda_ieq_get_ah_info(struct irdma_sc_qp * qp,struct irdma_ah_info * ah_info)1337cdcd52d4SBartosz Sobczak irdma_puda_ieq_get_ah_info(struct irdma_sc_qp *qp,
1338cdcd52d4SBartosz Sobczak 			   struct irdma_ah_info *ah_info)
1339cdcd52d4SBartosz Sobczak {
1340cdcd52d4SBartosz Sobczak 	struct irdma_puda_buf *buf = qp->pfpdu.ah_buf;
1341cdcd52d4SBartosz Sobczak 	struct ip *iph;
1342cdcd52d4SBartosz Sobczak 	struct ip6_hdr *ip6h;
1343cdcd52d4SBartosz Sobczak 
1344cdcd52d4SBartosz Sobczak 	memset(ah_info, 0, sizeof(*ah_info));
1345cdcd52d4SBartosz Sobczak 	ah_info->do_lpbk = true;
1346cdcd52d4SBartosz Sobczak 	ah_info->vlan_tag = buf->vlan_id;
1347cdcd52d4SBartosz Sobczak 	ah_info->insert_vlan_tag = buf->vlan_valid;
1348cdcd52d4SBartosz Sobczak 	ah_info->ipv4_valid = buf->ipv4;
1349cdcd52d4SBartosz Sobczak 	ah_info->vsi = qp->vsi;
1350cdcd52d4SBartosz Sobczak 
1351cdcd52d4SBartosz Sobczak 	if (buf->smac_valid)
1352cdcd52d4SBartosz Sobczak 		ether_addr_copy(ah_info->mac_addr, buf->smac);
1353cdcd52d4SBartosz Sobczak 
1354cdcd52d4SBartosz Sobczak 	if (buf->ipv4) {
1355cdcd52d4SBartosz Sobczak 		ah_info->ipv4_valid = true;
1356cdcd52d4SBartosz Sobczak 		iph = (struct ip *)buf->iph;
1357cdcd52d4SBartosz Sobczak 		ah_info->hop_ttl = iph->ip_ttl;
1358cdcd52d4SBartosz Sobczak 		ah_info->tc_tos = iph->ip_tos;
1359cdcd52d4SBartosz Sobczak 		ah_info->dest_ip_addr[0] = ntohl(iph->ip_dst.s_addr);
1360cdcd52d4SBartosz Sobczak 		ah_info->src_ip_addr[0] = ntohl(iph->ip_src.s_addr);
1361cdcd52d4SBartosz Sobczak 	} else {
1362cdcd52d4SBartosz Sobczak 		ip6h = (struct ip6_hdr *)buf->iph;
1363cdcd52d4SBartosz Sobczak 		ah_info->hop_ttl = ip6h->ip6_hops;
1364cdcd52d4SBartosz Sobczak 		ah_info->tc_tos = ip6h->ip6_vfc;
1365cdcd52d4SBartosz Sobczak 		irdma_copy_ip_ntohl(ah_info->dest_ip_addr,
1366cdcd52d4SBartosz Sobczak 				    ip6h->ip6_dst.__u6_addr.__u6_addr32);
1367cdcd52d4SBartosz Sobczak 		irdma_copy_ip_ntohl(ah_info->src_ip_addr,
1368cdcd52d4SBartosz Sobczak 				    ip6h->ip6_src.__u6_addr.__u6_addr32);
1369cdcd52d4SBartosz Sobczak 	}
1370cdcd52d4SBartosz Sobczak 
1371cdcd52d4SBartosz Sobczak 	ah_info->dst_arpindex = irdma_arp_table(dev_to_rf(qp->dev),
1372cdcd52d4SBartosz Sobczak 						ah_info->dest_ip_addr,
1373cdcd52d4SBartosz Sobczak 						NULL, IRDMA_ARP_RESOLVE);
1374cdcd52d4SBartosz Sobczak }
1375cdcd52d4SBartosz Sobczak 
1376cdcd52d4SBartosz Sobczak /**
1377cdcd52d4SBartosz Sobczak  * irdma_gen1_ieq_update_tcpip_info - update tcpip in the buffer
1378cdcd52d4SBartosz Sobczak  * @buf: puda to update
1379cdcd52d4SBartosz Sobczak  * @len: length of buffer
1380cdcd52d4SBartosz Sobczak  * @seqnum: seq number for tcp
1381cdcd52d4SBartosz Sobczak  */
1382cdcd52d4SBartosz Sobczak static void
irdma_gen1_ieq_update_tcpip_info(struct irdma_puda_buf * buf,u16 len,u32 seqnum)1383cdcd52d4SBartosz Sobczak irdma_gen1_ieq_update_tcpip_info(struct irdma_puda_buf *buf,
1384cdcd52d4SBartosz Sobczak 				 u16 len, u32 seqnum)
1385cdcd52d4SBartosz Sobczak {
1386cdcd52d4SBartosz Sobczak 	struct tcphdr *tcph;
1387cdcd52d4SBartosz Sobczak 	struct ip *iph;
1388cdcd52d4SBartosz Sobczak 	u16 iphlen;
1389cdcd52d4SBartosz Sobczak 	u16 pktsize;
1390cdcd52d4SBartosz Sobczak 	u8 *addr = buf->mem.va;
1391cdcd52d4SBartosz Sobczak 
1392cdcd52d4SBartosz Sobczak 	iphlen = (buf->ipv4) ? 20 : 40;
1393cdcd52d4SBartosz Sobczak 	iph = (struct ip *)(addr + buf->maclen);
1394cdcd52d4SBartosz Sobczak 	tcph = (struct tcphdr *)(addr + buf->maclen + iphlen);
1395cdcd52d4SBartosz Sobczak 	pktsize = len + buf->tcphlen + iphlen;
1396cdcd52d4SBartosz Sobczak 	iph->ip_len = htons(pktsize);
1397cdcd52d4SBartosz Sobczak 	tcph->th_seq = htonl(seqnum);
1398cdcd52d4SBartosz Sobczak }
1399cdcd52d4SBartosz Sobczak 
1400cdcd52d4SBartosz Sobczak /**
1401cdcd52d4SBartosz Sobczak  * irdma_ieq_update_tcpip_info - update tcpip in the buffer
1402cdcd52d4SBartosz Sobczak  * @buf: puda to update
1403cdcd52d4SBartosz Sobczak  * @len: length of buffer
1404cdcd52d4SBartosz Sobczak  * @seqnum: seq number for tcp
1405cdcd52d4SBartosz Sobczak  */
1406cdcd52d4SBartosz Sobczak void
irdma_ieq_update_tcpip_info(struct irdma_puda_buf * buf,u16 len,u32 seqnum)1407cdcd52d4SBartosz Sobczak irdma_ieq_update_tcpip_info(struct irdma_puda_buf *buf, u16 len,
1408cdcd52d4SBartosz Sobczak 			    u32 seqnum)
1409cdcd52d4SBartosz Sobczak {
1410cdcd52d4SBartosz Sobczak 	struct tcphdr *tcph;
1411cdcd52d4SBartosz Sobczak 	u8 *addr;
1412cdcd52d4SBartosz Sobczak 
1413cdcd52d4SBartosz Sobczak 	if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
1414cdcd52d4SBartosz Sobczak 		return irdma_gen1_ieq_update_tcpip_info(buf, len, seqnum);
1415cdcd52d4SBartosz Sobczak 
1416cdcd52d4SBartosz Sobczak 	addr = buf->mem.va;
1417cdcd52d4SBartosz Sobczak 	tcph = (struct tcphdr *)addr;
1418cdcd52d4SBartosz Sobczak 	tcph->th_seq = htonl(seqnum);
1419cdcd52d4SBartosz Sobczak }
1420cdcd52d4SBartosz Sobczak 
1421cdcd52d4SBartosz Sobczak /**
1422cdcd52d4SBartosz Sobczak  * irdma_gen1_puda_get_tcpip_info - get tcpip info from puda
1423cdcd52d4SBartosz Sobczak  * buffer
1424cdcd52d4SBartosz Sobczak  * @info: to get information
1425cdcd52d4SBartosz Sobczak  * @buf: puda buffer
1426cdcd52d4SBartosz Sobczak  */
1427cdcd52d4SBartosz Sobczak static int
irdma_gen1_puda_get_tcpip_info(struct irdma_puda_cmpl_info * info,struct irdma_puda_buf * buf)1428cdcd52d4SBartosz Sobczak irdma_gen1_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,
1429cdcd52d4SBartosz Sobczak 			       struct irdma_puda_buf *buf)
1430cdcd52d4SBartosz Sobczak {
1431cdcd52d4SBartosz Sobczak 	struct ip *iph;
1432cdcd52d4SBartosz Sobczak 	struct ip6_hdr *ip6h;
1433cdcd52d4SBartosz Sobczak 	struct tcphdr *tcph;
1434cdcd52d4SBartosz Sobczak 	u16 iphlen;
1435cdcd52d4SBartosz Sobczak 	u16 pkt_len;
1436cdcd52d4SBartosz Sobczak 	u8 *mem = buf->mem.va;
1437cdcd52d4SBartosz Sobczak 	struct ether_header *ethh = buf->mem.va;
1438cdcd52d4SBartosz Sobczak 
1439cdcd52d4SBartosz Sobczak 	if (ethh->ether_type == htons(0x8100)) {
1440cdcd52d4SBartosz Sobczak 		info->vlan_valid = true;
1441cdcd52d4SBartosz Sobczak 		buf->vlan_id = ntohs(((struct ether_vlan_header *)ethh)->evl_tag) &
1442cdcd52d4SBartosz Sobczak 		    EVL_VLID_MASK;
1443cdcd52d4SBartosz Sobczak 	}
1444cdcd52d4SBartosz Sobczak 
1445cdcd52d4SBartosz Sobczak 	buf->maclen = (info->vlan_valid) ? 18 : 14;
1446cdcd52d4SBartosz Sobczak 	iphlen = (info->l3proto) ? 40 : 20;
1447cdcd52d4SBartosz Sobczak 	buf->ipv4 = (info->l3proto) ? false : true;
1448cdcd52d4SBartosz Sobczak 	buf->iph = mem + buf->maclen;
1449cdcd52d4SBartosz Sobczak 	iph = (struct ip *)buf->iph;
1450cdcd52d4SBartosz Sobczak 	buf->tcph = buf->iph + iphlen;
1451cdcd52d4SBartosz Sobczak 	tcph = (struct tcphdr *)buf->tcph;
1452cdcd52d4SBartosz Sobczak 
1453cdcd52d4SBartosz Sobczak 	if (buf->ipv4) {
1454cdcd52d4SBartosz Sobczak 		pkt_len = ntohs(iph->ip_len);
1455cdcd52d4SBartosz Sobczak 	} else {
1456cdcd52d4SBartosz Sobczak 		ip6h = (struct ip6_hdr *)buf->iph;
1457cdcd52d4SBartosz Sobczak 		pkt_len = ntohs(ip6h->ip6_plen) + iphlen;
1458cdcd52d4SBartosz Sobczak 	}
1459cdcd52d4SBartosz Sobczak 
1460cdcd52d4SBartosz Sobczak 	buf->totallen = pkt_len + buf->maclen;
1461cdcd52d4SBartosz Sobczak 
1462cdcd52d4SBartosz Sobczak 	if (info->payload_len < buf->totallen) {
1463cdcd52d4SBartosz Sobczak 		irdma_debug(buf->vsi->dev, IRDMA_DEBUG_ERR,
1464cdcd52d4SBartosz Sobczak 			    "payload_len = 0x%x totallen expected0x%x\n",
1465cdcd52d4SBartosz Sobczak 			    info->payload_len, buf->totallen);
1466cdcd52d4SBartosz Sobczak 		return -EINVAL;
1467cdcd52d4SBartosz Sobczak 	}
1468cdcd52d4SBartosz Sobczak 
1469cdcd52d4SBartosz Sobczak 	buf->tcphlen = tcph->th_off << 2;
1470cdcd52d4SBartosz Sobczak 	buf->datalen = pkt_len - iphlen - buf->tcphlen;
1471cdcd52d4SBartosz Sobczak 	buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL;
1472cdcd52d4SBartosz Sobczak 	buf->hdrlen = buf->maclen + iphlen + buf->tcphlen;
1473cdcd52d4SBartosz Sobczak 	buf->seqnum = ntohl(tcph->th_seq);
1474cdcd52d4SBartosz Sobczak 
1475cdcd52d4SBartosz Sobczak 	return 0;
1476cdcd52d4SBartosz Sobczak }
1477cdcd52d4SBartosz Sobczak 
1478cdcd52d4SBartosz Sobczak /**
1479cdcd52d4SBartosz Sobczak  * irdma_puda_get_tcpip_info - get tcpip info from puda buffer
1480cdcd52d4SBartosz Sobczak  * @info: to get information
1481cdcd52d4SBartosz Sobczak  * @buf: puda buffer
1482cdcd52d4SBartosz Sobczak  */
1483cdcd52d4SBartosz Sobczak int
irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info * info,struct irdma_puda_buf * buf)1484cdcd52d4SBartosz Sobczak irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,
1485cdcd52d4SBartosz Sobczak 			  struct irdma_puda_buf *buf)
1486cdcd52d4SBartosz Sobczak {
1487cdcd52d4SBartosz Sobczak 	struct tcphdr *tcph;
1488cdcd52d4SBartosz Sobczak 	u32 pkt_len;
1489cdcd52d4SBartosz Sobczak 	u8 *mem;
1490cdcd52d4SBartosz Sobczak 
1491cdcd52d4SBartosz Sobczak 	if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
1492cdcd52d4SBartosz Sobczak 		return irdma_gen1_puda_get_tcpip_info(info, buf);
1493cdcd52d4SBartosz Sobczak 
1494cdcd52d4SBartosz Sobczak 	mem = buf->mem.va;
1495cdcd52d4SBartosz Sobczak 	buf->vlan_valid = info->vlan_valid;
1496cdcd52d4SBartosz Sobczak 	if (info->vlan_valid)
1497cdcd52d4SBartosz Sobczak 		buf->vlan_id = info->vlan;
1498cdcd52d4SBartosz Sobczak 
1499cdcd52d4SBartosz Sobczak 	buf->ipv4 = info->ipv4;
1500cdcd52d4SBartosz Sobczak 	if (buf->ipv4)
1501cdcd52d4SBartosz Sobczak 		buf->iph = mem + IRDMA_IPV4_PAD;
1502cdcd52d4SBartosz Sobczak 	else
1503cdcd52d4SBartosz Sobczak 		buf->iph = mem;
1504cdcd52d4SBartosz Sobczak 
1505cdcd52d4SBartosz Sobczak 	buf->tcph = mem + IRDMA_TCP_OFFSET;
1506cdcd52d4SBartosz Sobczak 	tcph = (struct tcphdr *)buf->tcph;
1507cdcd52d4SBartosz Sobczak 	pkt_len = info->payload_len;
1508cdcd52d4SBartosz Sobczak 	buf->totallen = pkt_len;
1509cdcd52d4SBartosz Sobczak 	buf->tcphlen = tcph->th_off << 2;
1510cdcd52d4SBartosz Sobczak 	buf->datalen = pkt_len - IRDMA_TCP_OFFSET - buf->tcphlen;
1511cdcd52d4SBartosz Sobczak 	buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL;
1512cdcd52d4SBartosz Sobczak 	buf->hdrlen = IRDMA_TCP_OFFSET + buf->tcphlen;
1513cdcd52d4SBartosz Sobczak 	buf->seqnum = ntohl(tcph->th_seq);
1514cdcd52d4SBartosz Sobczak 
1515cdcd52d4SBartosz Sobczak 	if (info->smac_valid) {
1516cdcd52d4SBartosz Sobczak 		ether_addr_copy(buf->smac, info->smac);
1517cdcd52d4SBartosz Sobczak 		buf->smac_valid = true;
1518cdcd52d4SBartosz Sobczak 	}
1519cdcd52d4SBartosz Sobczak 
1520cdcd52d4SBartosz Sobczak 	return 0;
1521cdcd52d4SBartosz Sobczak }
1522cdcd52d4SBartosz Sobczak 
1523cdcd52d4SBartosz Sobczak /**
1524cdcd52d4SBartosz Sobczak  * irdma_hw_stats_timeout - Stats timer-handler which updates all HW stats
1525cdcd52d4SBartosz Sobczak  * @t: timer_list pointer
1526cdcd52d4SBartosz Sobczak  */
1527cdcd52d4SBartosz Sobczak static void
irdma_hw_stats_timeout(struct timer_list * t)1528cdcd52d4SBartosz Sobczak irdma_hw_stats_timeout(struct timer_list *t)
1529cdcd52d4SBartosz Sobczak {
1530cdcd52d4SBartosz Sobczak 	struct irdma_vsi_pestat *pf_devstat =
1531cdcd52d4SBartosz Sobczak 	from_timer(pf_devstat, t, stats_timer);
1532cdcd52d4SBartosz Sobczak 	struct irdma_sc_vsi *sc_vsi = pf_devstat->vsi;
1533cdcd52d4SBartosz Sobczak 
1534cdcd52d4SBartosz Sobczak 	if (sc_vsi->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
1535cdcd52d4SBartosz Sobczak 		irdma_cqp_gather_stats_cmd(sc_vsi->dev, sc_vsi->pestat, false);
1536cdcd52d4SBartosz Sobczak 
1537cdcd52d4SBartosz Sobczak 	mod_timer(&pf_devstat->stats_timer,
1538cdcd52d4SBartosz Sobczak 		  jiffies + msecs_to_jiffies(STATS_TIMER_DELAY));
1539cdcd52d4SBartosz Sobczak }
1540cdcd52d4SBartosz Sobczak 
1541cdcd52d4SBartosz Sobczak /**
1542cdcd52d4SBartosz Sobczak  * irdma_hw_stats_start_timer - Start periodic stats timer
1543cdcd52d4SBartosz Sobczak  * @vsi: vsi structure pointer
1544cdcd52d4SBartosz Sobczak  */
1545cdcd52d4SBartosz Sobczak void
irdma_hw_stats_start_timer(struct irdma_sc_vsi * vsi)1546cdcd52d4SBartosz Sobczak irdma_hw_stats_start_timer(struct irdma_sc_vsi *vsi)
1547cdcd52d4SBartosz Sobczak {
1548cdcd52d4SBartosz Sobczak 	struct irdma_vsi_pestat *devstat = vsi->pestat;
1549cdcd52d4SBartosz Sobczak 
1550cdcd52d4SBartosz Sobczak 	timer_setup(&devstat->stats_timer, irdma_hw_stats_timeout, 0);
1551cdcd52d4SBartosz Sobczak 	mod_timer(&devstat->stats_timer,
1552cdcd52d4SBartosz Sobczak 		  jiffies + msecs_to_jiffies(STATS_TIMER_DELAY));
1553cdcd52d4SBartosz Sobczak }
1554cdcd52d4SBartosz Sobczak 
1555cdcd52d4SBartosz Sobczak /**
1556cdcd52d4SBartosz Sobczak  * irdma_hw_stats_stop_timer - Delete periodic stats timer
1557cdcd52d4SBartosz Sobczak  * @vsi: pointer to vsi structure
1558cdcd52d4SBartosz Sobczak  */
1559cdcd52d4SBartosz Sobczak void
irdma_hw_stats_stop_timer(struct irdma_sc_vsi * vsi)1560cdcd52d4SBartosz Sobczak irdma_hw_stats_stop_timer(struct irdma_sc_vsi *vsi)
1561cdcd52d4SBartosz Sobczak {
1562cdcd52d4SBartosz Sobczak 	struct irdma_vsi_pestat *devstat = vsi->pestat;
1563cdcd52d4SBartosz Sobczak 
1564cdcd52d4SBartosz Sobczak 	del_timer_sync(&devstat->stats_timer);
1565cdcd52d4SBartosz Sobczak }
1566cdcd52d4SBartosz Sobczak 
1567cdcd52d4SBartosz Sobczak /**
1568cdcd52d4SBartosz Sobczak  * irdma_process_cqp_stats - Checking for wrap and update stats
1569cdcd52d4SBartosz Sobczak  * @cqp_request: cqp_request structure pointer
1570cdcd52d4SBartosz Sobczak  */
1571cdcd52d4SBartosz Sobczak static void
irdma_process_cqp_stats(struct irdma_cqp_request * cqp_request)1572cdcd52d4SBartosz Sobczak irdma_process_cqp_stats(struct irdma_cqp_request *cqp_request)
1573cdcd52d4SBartosz Sobczak {
1574cdcd52d4SBartosz Sobczak 	struct irdma_vsi_pestat *pestat = cqp_request->param;
1575cdcd52d4SBartosz Sobczak 
1576*5b5f7d0eSBartosz Sobczak 	sc_vsi_update_stats(pestat->vsi);
1577cdcd52d4SBartosz Sobczak }
1578cdcd52d4SBartosz Sobczak 
1579cdcd52d4SBartosz Sobczak /**
1580cdcd52d4SBartosz Sobczak  * irdma_cqp_gather_stats_cmd - Gather stats
1581cdcd52d4SBartosz Sobczak  * @dev: pointer to device structure
1582cdcd52d4SBartosz Sobczak  * @pestat: pointer to stats info
1583cdcd52d4SBartosz Sobczak  * @wait: flag to wait or not wait for stats
1584cdcd52d4SBartosz Sobczak  */
1585cdcd52d4SBartosz Sobczak int
irdma_cqp_gather_stats_cmd(struct irdma_sc_dev * dev,struct irdma_vsi_pestat * pestat,bool wait)1586cdcd52d4SBartosz Sobczak irdma_cqp_gather_stats_cmd(struct irdma_sc_dev *dev,
1587cdcd52d4SBartosz Sobczak 			   struct irdma_vsi_pestat *pestat, bool wait)
1588cdcd52d4SBartosz Sobczak {
1589cdcd52d4SBartosz Sobczak 
1590cdcd52d4SBartosz Sobczak 	struct irdma_pci_f *rf = dev_to_rf(dev);
1591cdcd52d4SBartosz Sobczak 	struct irdma_cqp *iwcqp = &rf->cqp;
1592cdcd52d4SBartosz Sobczak 	struct irdma_cqp_request *cqp_request;
1593cdcd52d4SBartosz Sobczak 	struct cqp_cmds_info *cqp_info;
1594cdcd52d4SBartosz Sobczak 	int status;
1595cdcd52d4SBartosz Sobczak 
1596cdcd52d4SBartosz Sobczak 	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait);
1597cdcd52d4SBartosz Sobczak 	if (!cqp_request)
1598cdcd52d4SBartosz Sobczak 		return -ENOMEM;
1599cdcd52d4SBartosz Sobczak 
1600cdcd52d4SBartosz Sobczak 	cqp_info = &cqp_request->info;
1601cdcd52d4SBartosz Sobczak 	memset(cqp_info, 0, sizeof(*cqp_info));
1602cdcd52d4SBartosz Sobczak 	cqp_info->cqp_cmd = IRDMA_OP_STATS_GATHER;
1603cdcd52d4SBartosz Sobczak 	cqp_info->post_sq = 1;
1604cdcd52d4SBartosz Sobczak 	cqp_info->in.u.stats_gather.info = pestat->gather_info;
1605cdcd52d4SBartosz Sobczak 	cqp_info->in.u.stats_gather.scratch = (uintptr_t)cqp_request;
1606cdcd52d4SBartosz Sobczak 	cqp_info->in.u.stats_gather.cqp = &rf->cqp.sc_cqp;
1607cdcd52d4SBartosz Sobczak 	cqp_request->param = pestat;
1608cdcd52d4SBartosz Sobczak 	if (!wait)
1609cdcd52d4SBartosz Sobczak 		cqp_request->callback_fcn = irdma_process_cqp_stats;
1610cdcd52d4SBartosz Sobczak 	status = irdma_handle_cqp_op(rf, cqp_request);
1611cdcd52d4SBartosz Sobczak 	if (wait)
1612*5b5f7d0eSBartosz Sobczak 		sc_vsi_update_stats(pestat->vsi);
1613cdcd52d4SBartosz Sobczak 	irdma_put_cqp_request(&rf->cqp, cqp_request);
1614cdcd52d4SBartosz Sobczak 
1615cdcd52d4SBartosz Sobczak 	return status;
1616cdcd52d4SBartosz Sobczak }
1617cdcd52d4SBartosz Sobczak 
1618cdcd52d4SBartosz Sobczak /**
1619cdcd52d4SBartosz Sobczak  * irdma_cqp_stats_inst_cmd - Allocate/free stats instance
1620cdcd52d4SBartosz Sobczak  * @vsi: pointer to vsi structure
1621cdcd52d4SBartosz Sobczak  * @cmd: command to allocate or free
1622cdcd52d4SBartosz Sobczak  * @stats_info: pointer to allocate stats info
1623cdcd52d4SBartosz Sobczak  */
1624cdcd52d4SBartosz Sobczak int
irdma_cqp_stats_inst_cmd(struct irdma_sc_vsi * vsi,u8 cmd,struct irdma_stats_inst_info * stats_info)1625cdcd52d4SBartosz Sobczak irdma_cqp_stats_inst_cmd(struct irdma_sc_vsi *vsi, u8 cmd,
1626cdcd52d4SBartosz Sobczak 			 struct irdma_stats_inst_info *stats_info)
1627cdcd52d4SBartosz Sobczak {
1628cdcd52d4SBartosz Sobczak 	struct irdma_pci_f *rf = dev_to_rf(vsi->dev);
1629cdcd52d4SBartosz Sobczak 	struct irdma_cqp *iwcqp = &rf->cqp;
1630cdcd52d4SBartosz Sobczak 	struct irdma_cqp_request *cqp_request;
1631cdcd52d4SBartosz Sobczak 	struct cqp_cmds_info *cqp_info;
1632cdcd52d4SBartosz Sobczak 	int status;
1633cdcd52d4SBartosz Sobczak 	bool wait = false;
1634cdcd52d4SBartosz Sobczak 
1635cdcd52d4SBartosz Sobczak 	if (cmd == IRDMA_OP_STATS_ALLOCATE)
1636cdcd52d4SBartosz Sobczak 		wait = true;
1637cdcd52d4SBartosz Sobczak 	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait);
1638cdcd52d4SBartosz Sobczak 	if (!cqp_request)
1639cdcd52d4SBartosz Sobczak 		return -ENOMEM;
1640cdcd52d4SBartosz Sobczak 
1641cdcd52d4SBartosz Sobczak 	cqp_info = &cqp_request->info;
1642cdcd52d4SBartosz Sobczak 	memset(cqp_info, 0, sizeof(*cqp_info));
1643cdcd52d4SBartosz Sobczak 	cqp_info->cqp_cmd = cmd;
1644cdcd52d4SBartosz Sobczak 	cqp_info->post_sq = 1;
1645cdcd52d4SBartosz Sobczak 	cqp_info->in.u.stats_manage.info = *stats_info;
1646cdcd52d4SBartosz Sobczak 	cqp_info->in.u.stats_manage.scratch = (uintptr_t)cqp_request;
1647cdcd52d4SBartosz Sobczak 	cqp_info->in.u.stats_manage.cqp = &rf->cqp.sc_cqp;
1648cdcd52d4SBartosz Sobczak 	status = irdma_handle_cqp_op(rf, cqp_request);
1649cdcd52d4SBartosz Sobczak 	if (wait)
1650cdcd52d4SBartosz Sobczak 		stats_info->stats_idx = cqp_request->compl_info.op_ret_val;
1651cdcd52d4SBartosz Sobczak 	irdma_put_cqp_request(iwcqp, cqp_request);
1652cdcd52d4SBartosz Sobczak 
1653cdcd52d4SBartosz Sobczak 	return status;
1654cdcd52d4SBartosz Sobczak }
1655cdcd52d4SBartosz Sobczak 
1656cdcd52d4SBartosz Sobczak /**
1657cdcd52d4SBartosz Sobczak  * irdma_cqp_ceq_cmd - Create/Destroy CEQ's after CEQ 0
1658cdcd52d4SBartosz Sobczak  * @dev: pointer to device info
1659cdcd52d4SBartosz Sobczak  * @sc_ceq: pointer to ceq structure
1660cdcd52d4SBartosz Sobczak  * @op: Create or Destroy
1661cdcd52d4SBartosz Sobczak  */
1662cdcd52d4SBartosz Sobczak int
irdma_cqp_ceq_cmd(struct irdma_sc_dev * dev,struct irdma_sc_ceq * sc_ceq,u8 op)1663cdcd52d4SBartosz Sobczak irdma_cqp_ceq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_ceq *sc_ceq,
1664cdcd52d4SBartosz Sobczak 		  u8 op)
1665cdcd52d4SBartosz Sobczak {
1666cdcd52d4SBartosz Sobczak 	struct irdma_cqp_request *cqp_request;
1667cdcd52d4SBartosz Sobczak 	struct cqp_cmds_info *cqp_info;
1668cdcd52d4SBartosz Sobczak 	struct irdma_pci_f *rf = dev_to_rf(dev);
1669cdcd52d4SBartosz Sobczak 	int status;
1670cdcd52d4SBartosz Sobczak 
1671cdcd52d4SBartosz Sobczak 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
1672cdcd52d4SBartosz Sobczak 	if (!cqp_request)
1673cdcd52d4SBartosz Sobczak 		return -ENOMEM;
1674cdcd52d4SBartosz Sobczak 
1675cdcd52d4SBartosz Sobczak 	cqp_info = &cqp_request->info;
1676cdcd52d4SBartosz Sobczak 	cqp_info->post_sq = 1;
1677cdcd52d4SBartosz Sobczak 	cqp_info->cqp_cmd = op;
1678cdcd52d4SBartosz Sobczak 	cqp_info->in.u.ceq_create.ceq = sc_ceq;
1679cdcd52d4SBartosz Sobczak 	cqp_info->in.u.ceq_create.scratch = (uintptr_t)cqp_request;
1680cdcd52d4SBartosz Sobczak 
1681cdcd52d4SBartosz Sobczak 	status = irdma_handle_cqp_op(rf, cqp_request);
1682cdcd52d4SBartosz Sobczak 	irdma_put_cqp_request(&rf->cqp, cqp_request);
1683cdcd52d4SBartosz Sobczak 
1684cdcd52d4SBartosz Sobczak 	return status;
1685cdcd52d4SBartosz Sobczak }
1686cdcd52d4SBartosz Sobczak 
1687cdcd52d4SBartosz Sobczak /**
1688cdcd52d4SBartosz Sobczak  * irdma_cqp_aeq_cmd - Create/Destroy AEQ
1689cdcd52d4SBartosz Sobczak  * @dev: pointer to device info
1690cdcd52d4SBartosz Sobczak  * @sc_aeq: pointer to aeq structure
1691cdcd52d4SBartosz Sobczak  * @op: Create or Destroy
1692cdcd52d4SBartosz Sobczak  */
1693cdcd52d4SBartosz Sobczak int
irdma_cqp_aeq_cmd(struct irdma_sc_dev * dev,struct irdma_sc_aeq * sc_aeq,u8 op)1694cdcd52d4SBartosz Sobczak irdma_cqp_aeq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_aeq *sc_aeq,
1695cdcd52d4SBartosz Sobczak 		  u8 op)
1696cdcd52d4SBartosz Sobczak {
1697cdcd52d4SBartosz Sobczak 	struct irdma_cqp_request *cqp_request;
1698cdcd52d4SBartosz Sobczak 	struct cqp_cmds_info *cqp_info;
1699cdcd52d4SBartosz Sobczak 	struct irdma_pci_f *rf = dev_to_rf(dev);
1700cdcd52d4SBartosz Sobczak 	int status;
1701cdcd52d4SBartosz Sobczak 
1702cdcd52d4SBartosz Sobczak 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
1703cdcd52d4SBartosz Sobczak 	if (!cqp_request)
1704cdcd52d4SBartosz Sobczak 		return -ENOMEM;
1705cdcd52d4SBartosz Sobczak 
1706cdcd52d4SBartosz Sobczak 	cqp_info = &cqp_request->info;
1707cdcd52d4SBartosz Sobczak 	cqp_info->post_sq = 1;
1708cdcd52d4SBartosz Sobczak 	cqp_info->cqp_cmd = op;
1709cdcd52d4SBartosz Sobczak 	cqp_info->in.u.aeq_create.aeq = sc_aeq;
1710cdcd52d4SBartosz Sobczak 	cqp_info->in.u.aeq_create.scratch = (uintptr_t)cqp_request;
1711cdcd52d4SBartosz Sobczak 
1712cdcd52d4SBartosz Sobczak 	status = irdma_handle_cqp_op(rf, cqp_request);
1713cdcd52d4SBartosz Sobczak 	irdma_put_cqp_request(&rf->cqp, cqp_request);
1714cdcd52d4SBartosz Sobczak 
1715cdcd52d4SBartosz Sobczak 	return status;
1716cdcd52d4SBartosz Sobczak }
1717cdcd52d4SBartosz Sobczak 
1718cdcd52d4SBartosz Sobczak /**
1719cdcd52d4SBartosz Sobczak  * irdma_cqp_ws_node_cmd - Add/modify/delete ws node
1720cdcd52d4SBartosz Sobczak  * @dev: pointer to device structure
1721cdcd52d4SBartosz Sobczak  * @cmd: Add, modify or delete
1722cdcd52d4SBartosz Sobczak  * @node_info: pointer to ws node info
1723cdcd52d4SBartosz Sobczak  */
1724cdcd52d4SBartosz Sobczak int
irdma_cqp_ws_node_cmd(struct irdma_sc_dev * dev,u8 cmd,struct irdma_ws_node_info * node_info)1725cdcd52d4SBartosz Sobczak irdma_cqp_ws_node_cmd(struct irdma_sc_dev *dev, u8 cmd,
1726cdcd52d4SBartosz Sobczak 		      struct irdma_ws_node_info *node_info)
1727cdcd52d4SBartosz Sobczak {
1728cdcd52d4SBartosz Sobczak 	struct irdma_pci_f *rf = dev_to_rf(dev);
1729cdcd52d4SBartosz Sobczak 	struct irdma_cqp *iwcqp = &rf->cqp;
1730cdcd52d4SBartosz Sobczak 	struct irdma_sc_cqp *cqp = &iwcqp->sc_cqp;
1731cdcd52d4SBartosz Sobczak 	struct irdma_cqp_request *cqp_request;
1732cdcd52d4SBartosz Sobczak 	struct cqp_cmds_info *cqp_info;
1733cdcd52d4SBartosz Sobczak 	int status;
1734cdcd52d4SBartosz Sobczak 	bool poll;
1735cdcd52d4SBartosz Sobczak 
1736cdcd52d4SBartosz Sobczak 	if (!rf->sc_dev.ceq_valid)
1737cdcd52d4SBartosz Sobczak 		poll = true;
1738cdcd52d4SBartosz Sobczak 	else
1739cdcd52d4SBartosz Sobczak 		poll = false;
1740cdcd52d4SBartosz Sobczak 
1741cdcd52d4SBartosz Sobczak 	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, !poll);
1742cdcd52d4SBartosz Sobczak 	if (!cqp_request)
1743cdcd52d4SBartosz Sobczak 		return -ENOMEM;
1744cdcd52d4SBartosz Sobczak 
1745cdcd52d4SBartosz Sobczak 	cqp_info = &cqp_request->info;
1746cdcd52d4SBartosz Sobczak 	memset(cqp_info, 0, sizeof(*cqp_info));
1747cdcd52d4SBartosz Sobczak 	cqp_info->cqp_cmd = cmd;
1748cdcd52d4SBartosz Sobczak 	cqp_info->post_sq = 1;
1749cdcd52d4SBartosz Sobczak 	cqp_info->in.u.ws_node.info = *node_info;
1750cdcd52d4SBartosz Sobczak 	cqp_info->in.u.ws_node.cqp = cqp;
1751cdcd52d4SBartosz Sobczak 	cqp_info->in.u.ws_node.scratch = (uintptr_t)cqp_request;
1752cdcd52d4SBartosz Sobczak 	status = irdma_handle_cqp_op(rf, cqp_request);
1753cdcd52d4SBartosz Sobczak 	if (status)
1754cdcd52d4SBartosz Sobczak 		goto exit;
1755cdcd52d4SBartosz Sobczak 
1756cdcd52d4SBartosz Sobczak 	if (poll) {
1757cdcd52d4SBartosz Sobczak 		struct irdma_ccq_cqe_info compl_info;
1758cdcd52d4SBartosz Sobczak 
1759cdcd52d4SBartosz Sobczak 		status = irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_WORK_SCHED_NODE,
1760cdcd52d4SBartosz Sobczak 						       &compl_info);
1761cdcd52d4SBartosz Sobczak 		node_info->qs_handle = compl_info.op_ret_val;
176235105900SBartosz Sobczak 		irdma_debug(&rf->sc_dev, IRDMA_DEBUG_DCB,
1763cdcd52d4SBartosz Sobczak 			    "opcode=%d, compl_info.retval=%d\n",
1764cdcd52d4SBartosz Sobczak 			    compl_info.op_code, compl_info.op_ret_val);
1765cdcd52d4SBartosz Sobczak 	} else {
1766cdcd52d4SBartosz Sobczak 		node_info->qs_handle = cqp_request->compl_info.op_ret_val;
1767cdcd52d4SBartosz Sobczak 	}
1768cdcd52d4SBartosz Sobczak 
1769cdcd52d4SBartosz Sobczak exit:
1770cdcd52d4SBartosz Sobczak 	irdma_put_cqp_request(&rf->cqp, cqp_request);
1771cdcd52d4SBartosz Sobczak 
1772cdcd52d4SBartosz Sobczak 	return status;
1773cdcd52d4SBartosz Sobczak }
1774cdcd52d4SBartosz Sobczak 
1775cdcd52d4SBartosz Sobczak /**
1776cdcd52d4SBartosz Sobczak  * irdma_ah_cqp_op - perform an AH cqp operation
1777cdcd52d4SBartosz Sobczak  * @rf: RDMA PCI function
1778cdcd52d4SBartosz Sobczak  * @sc_ah: address handle
1779cdcd52d4SBartosz Sobczak  * @cmd: AH operation
1780cdcd52d4SBartosz Sobczak  * @wait: wait if true
1781cdcd52d4SBartosz Sobczak  * @callback_fcn: Callback function on CQP op completion
1782cdcd52d4SBartosz Sobczak  * @cb_param: parameter for callback function
1783cdcd52d4SBartosz Sobczak  *
1784cdcd52d4SBartosz Sobczak  * returns errno
1785cdcd52d4SBartosz Sobczak  */
1786cdcd52d4SBartosz Sobczak int
irdma_ah_cqp_op(struct irdma_pci_f * rf,struct irdma_sc_ah * sc_ah,u8 cmd,bool wait,void (* callback_fcn)(struct irdma_cqp_request *),void * cb_param)1787cdcd52d4SBartosz Sobczak irdma_ah_cqp_op(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, u8 cmd,
1788cdcd52d4SBartosz Sobczak 		bool wait,
1789cdcd52d4SBartosz Sobczak 		void (*callback_fcn) (struct irdma_cqp_request *),
1790cdcd52d4SBartosz Sobczak 		void *cb_param)
1791cdcd52d4SBartosz Sobczak {
1792cdcd52d4SBartosz Sobczak 	struct irdma_cqp_request *cqp_request;
1793cdcd52d4SBartosz Sobczak 	struct cqp_cmds_info *cqp_info;
1794cdcd52d4SBartosz Sobczak 	int status;
1795cdcd52d4SBartosz Sobczak 
1796cdcd52d4SBartosz Sobczak 	if (cmd != IRDMA_OP_AH_CREATE && cmd != IRDMA_OP_AH_DESTROY)
1797cdcd52d4SBartosz Sobczak 		return -EINVAL;
1798cdcd52d4SBartosz Sobczak 
1799cdcd52d4SBartosz Sobczak 	cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait);
1800cdcd52d4SBartosz Sobczak 	if (!cqp_request)
1801cdcd52d4SBartosz Sobczak 		return -ENOMEM;
1802cdcd52d4SBartosz Sobczak 
1803cdcd52d4SBartosz Sobczak 	cqp_info = &cqp_request->info;
1804cdcd52d4SBartosz Sobczak 	cqp_info->cqp_cmd = cmd;
1805cdcd52d4SBartosz Sobczak 	cqp_info->post_sq = 1;
1806cdcd52d4SBartosz Sobczak 	if (cmd == IRDMA_OP_AH_CREATE) {
1807*5b5f7d0eSBartosz Sobczak 		if (!wait)
1808*5b5f7d0eSBartosz Sobczak 			irdma_get_cqp_request(cqp_request);
1809*5b5f7d0eSBartosz Sobczak 		sc_ah->ah_info.cqp_request = cqp_request;
1810*5b5f7d0eSBartosz Sobczak 
1811cdcd52d4SBartosz Sobczak 		cqp_info->in.u.ah_create.info = sc_ah->ah_info;
1812cdcd52d4SBartosz Sobczak 		cqp_info->in.u.ah_create.scratch = (uintptr_t)cqp_request;
1813cdcd52d4SBartosz Sobczak 		cqp_info->in.u.ah_create.cqp = &rf->cqp.sc_cqp;
1814cdcd52d4SBartosz Sobczak 	} else if (cmd == IRDMA_OP_AH_DESTROY) {
1815cdcd52d4SBartosz Sobczak 		cqp_info->in.u.ah_destroy.info = sc_ah->ah_info;
1816cdcd52d4SBartosz Sobczak 		cqp_info->in.u.ah_destroy.scratch = (uintptr_t)cqp_request;
1817cdcd52d4SBartosz Sobczak 		cqp_info->in.u.ah_destroy.cqp = &rf->cqp.sc_cqp;
1818cdcd52d4SBartosz Sobczak 	}
1819cdcd52d4SBartosz Sobczak 
1820cdcd52d4SBartosz Sobczak 	if (!wait) {
1821cdcd52d4SBartosz Sobczak 		cqp_request->callback_fcn = callback_fcn;
1822cdcd52d4SBartosz Sobczak 		cqp_request->param = cb_param;
1823cdcd52d4SBartosz Sobczak 	}
1824cdcd52d4SBartosz Sobczak 	status = irdma_handle_cqp_op(rf, cqp_request);
1825cdcd52d4SBartosz Sobczak 	irdma_put_cqp_request(&rf->cqp, cqp_request);
1826cdcd52d4SBartosz Sobczak 
1827cdcd52d4SBartosz Sobczak 	if (status)
1828cdcd52d4SBartosz Sobczak 		return -ENOMEM;
1829cdcd52d4SBartosz Sobczak 
1830cdcd52d4SBartosz Sobczak 	if (wait)
183101fbb869SBartosz Sobczak 		sc_ah->ah_info.ah_valid = (cmd != IRDMA_OP_AH_DESTROY);
1832cdcd52d4SBartosz Sobczak 
1833cdcd52d4SBartosz Sobczak 	return 0;
1834cdcd52d4SBartosz Sobczak }
1835cdcd52d4SBartosz Sobczak 
1836cdcd52d4SBartosz Sobczak /**
1837cdcd52d4SBartosz Sobczak  * irdma_ieq_ah_cb - callback after creation of AH for IEQ
1838cdcd52d4SBartosz Sobczak  * @cqp_request: pointer to cqp_request of create AH
1839cdcd52d4SBartosz Sobczak  */
1840cdcd52d4SBartosz Sobczak static void
irdma_ieq_ah_cb(struct irdma_cqp_request * cqp_request)1841cdcd52d4SBartosz Sobczak irdma_ieq_ah_cb(struct irdma_cqp_request *cqp_request)
1842cdcd52d4SBartosz Sobczak {
1843cdcd52d4SBartosz Sobczak 	struct irdma_sc_qp *qp = cqp_request->param;
1844cdcd52d4SBartosz Sobczak 	struct irdma_sc_ah *sc_ah = qp->pfpdu.ah;
1845cdcd52d4SBartosz Sobczak 	unsigned long flags;
1846cdcd52d4SBartosz Sobczak 
1847cdcd52d4SBartosz Sobczak 	spin_lock_irqsave(&qp->pfpdu.lock, flags);
1848cdcd52d4SBartosz Sobczak 	if (!cqp_request->compl_info.op_ret_val) {
1849cdcd52d4SBartosz Sobczak 		sc_ah->ah_info.ah_valid = true;
1850cdcd52d4SBartosz Sobczak 		irdma_ieq_process_fpdus(qp, qp->vsi->ieq);
1851cdcd52d4SBartosz Sobczak 	} else {
1852cdcd52d4SBartosz Sobczak 		sc_ah->ah_info.ah_valid = false;
1853cdcd52d4SBartosz Sobczak 		irdma_ieq_cleanup_qp(qp->vsi->ieq, qp);
1854cdcd52d4SBartosz Sobczak 	}
1855cdcd52d4SBartosz Sobczak 	spin_unlock_irqrestore(&qp->pfpdu.lock, flags);
1856cdcd52d4SBartosz Sobczak }
1857cdcd52d4SBartosz Sobczak 
1858cdcd52d4SBartosz Sobczak /**
1859cdcd52d4SBartosz Sobczak  * irdma_ilq_ah_cb - callback after creation of AH for ILQ
1860cdcd52d4SBartosz Sobczak  * @cqp_request: pointer to cqp_request of create AH
1861cdcd52d4SBartosz Sobczak  */
1862cdcd52d4SBartosz Sobczak static void
irdma_ilq_ah_cb(struct irdma_cqp_request * cqp_request)1863cdcd52d4SBartosz Sobczak irdma_ilq_ah_cb(struct irdma_cqp_request *cqp_request)
1864cdcd52d4SBartosz Sobczak {
1865cdcd52d4SBartosz Sobczak 	struct irdma_cm_node *cm_node = cqp_request->param;
1866cdcd52d4SBartosz Sobczak 	struct irdma_sc_ah *sc_ah = cm_node->ah;
1867cdcd52d4SBartosz Sobczak 
1868cdcd52d4SBartosz Sobczak 	sc_ah->ah_info.ah_valid = !cqp_request->compl_info.op_ret_val;
1869cdcd52d4SBartosz Sobczak 	irdma_add_conn_est_qh(cm_node);
1870cdcd52d4SBartosz Sobczak }
1871cdcd52d4SBartosz Sobczak 
1872cdcd52d4SBartosz Sobczak /**
1873cdcd52d4SBartosz Sobczak  * irdma_puda_create_ah - create AH for ILQ/IEQ qp's
1874cdcd52d4SBartosz Sobczak  * @dev: device pointer
1875cdcd52d4SBartosz Sobczak  * @ah_info: Address handle info
1876cdcd52d4SBartosz Sobczak  * @wait: When true will wait for operation to complete
1877cdcd52d4SBartosz Sobczak  * @type: ILQ/IEQ
1878cdcd52d4SBartosz Sobczak  * @cb_param: Callback param when not waiting
1879cdcd52d4SBartosz Sobczak  * @ah_ret: Returned pointer to address handle if created
1880cdcd52d4SBartosz Sobczak  *
1881cdcd52d4SBartosz Sobczak  */
1882cdcd52d4SBartosz Sobczak int
irdma_puda_create_ah(struct irdma_sc_dev * dev,struct irdma_ah_info * ah_info,bool wait,enum puda_rsrc_type type,void * cb_param,struct irdma_sc_ah ** ah_ret)1883cdcd52d4SBartosz Sobczak irdma_puda_create_ah(struct irdma_sc_dev *dev,
1884cdcd52d4SBartosz Sobczak 		     struct irdma_ah_info *ah_info, bool wait,
1885cdcd52d4SBartosz Sobczak 		     enum puda_rsrc_type type, void *cb_param,
1886cdcd52d4SBartosz Sobczak 		     struct irdma_sc_ah **ah_ret)
1887cdcd52d4SBartosz Sobczak {
1888cdcd52d4SBartosz Sobczak 	struct irdma_sc_ah *ah;
1889cdcd52d4SBartosz Sobczak 	struct irdma_pci_f *rf = dev_to_rf(dev);
1890cdcd52d4SBartosz Sobczak 	int err;
1891cdcd52d4SBartosz Sobczak 
1892cdcd52d4SBartosz Sobczak 	ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
1893cdcd52d4SBartosz Sobczak 	*ah_ret = ah;
1894cdcd52d4SBartosz Sobczak 	if (!ah)
1895cdcd52d4SBartosz Sobczak 		return -ENOMEM;
1896cdcd52d4SBartosz Sobczak 
1897cdcd52d4SBartosz Sobczak 	err = irdma_alloc_rsrc(rf, rf->allocated_ahs, rf->max_ah,
1898cdcd52d4SBartosz Sobczak 			       &ah_info->ah_idx, &rf->next_ah);
1899cdcd52d4SBartosz Sobczak 	if (err)
1900cdcd52d4SBartosz Sobczak 		goto err_free;
1901cdcd52d4SBartosz Sobczak 
1902cdcd52d4SBartosz Sobczak 	ah->dev = dev;
1903cdcd52d4SBartosz Sobczak 	ah->ah_info = *ah_info;
1904cdcd52d4SBartosz Sobczak 
1905cdcd52d4SBartosz Sobczak 	if (type == IRDMA_PUDA_RSRC_TYPE_ILQ)
1906cdcd52d4SBartosz Sobczak 		err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait,
1907cdcd52d4SBartosz Sobczak 				      irdma_ilq_ah_cb, cb_param);
1908cdcd52d4SBartosz Sobczak 	else
1909cdcd52d4SBartosz Sobczak 		err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait,
1910cdcd52d4SBartosz Sobczak 				      irdma_ieq_ah_cb, cb_param);
1911cdcd52d4SBartosz Sobczak 
1912cdcd52d4SBartosz Sobczak 	if (err)
1913cdcd52d4SBartosz Sobczak 		goto error;
1914cdcd52d4SBartosz Sobczak 	return 0;
1915cdcd52d4SBartosz Sobczak 
1916cdcd52d4SBartosz Sobczak error:
1917cdcd52d4SBartosz Sobczak 	irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx);
1918cdcd52d4SBartosz Sobczak err_free:
1919cdcd52d4SBartosz Sobczak 	kfree(ah);
1920cdcd52d4SBartosz Sobczak 	*ah_ret = NULL;
1921cdcd52d4SBartosz Sobczak 	return -ENOMEM;
1922cdcd52d4SBartosz Sobczak }
1923cdcd52d4SBartosz Sobczak 
1924cdcd52d4SBartosz Sobczak /**
1925cdcd52d4SBartosz Sobczak  * irdma_puda_free_ah - free a puda address handle
1926cdcd52d4SBartosz Sobczak  * @dev: device pointer
1927cdcd52d4SBartosz Sobczak  * @ah: The address handle to free
1928cdcd52d4SBartosz Sobczak  */
1929cdcd52d4SBartosz Sobczak void
irdma_puda_free_ah(struct irdma_sc_dev * dev,struct irdma_sc_ah * ah)1930cdcd52d4SBartosz Sobczak irdma_puda_free_ah(struct irdma_sc_dev *dev, struct irdma_sc_ah *ah)
1931cdcd52d4SBartosz Sobczak {
1932cdcd52d4SBartosz Sobczak 	struct irdma_pci_f *rf = dev_to_rf(dev);
1933cdcd52d4SBartosz Sobczak 
1934cdcd52d4SBartosz Sobczak 	if (!ah)
1935cdcd52d4SBartosz Sobczak 		return;
1936cdcd52d4SBartosz Sobczak 
1937cdcd52d4SBartosz Sobczak 	if (ah->ah_info.ah_valid) {
1938cdcd52d4SBartosz Sobczak 		irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_DESTROY, false, NULL, NULL);
1939cdcd52d4SBartosz Sobczak 		irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx);
1940cdcd52d4SBartosz Sobczak 	}
1941cdcd52d4SBartosz Sobczak 
1942cdcd52d4SBartosz Sobczak 	kfree(ah);
1943cdcd52d4SBartosz Sobczak }
1944cdcd52d4SBartosz Sobczak 
1945cdcd52d4SBartosz Sobczak /**
1946cdcd52d4SBartosz Sobczak  * irdma_prm_add_pble_mem - add moemory to pble resources
1947cdcd52d4SBartosz Sobczak  * @pprm: pble resource manager
1948cdcd52d4SBartosz Sobczak  * @pchunk: chunk of memory to add
1949cdcd52d4SBartosz Sobczak  */
1950cdcd52d4SBartosz Sobczak int
irdma_prm_add_pble_mem(struct irdma_pble_prm * pprm,struct irdma_chunk * pchunk)1951cdcd52d4SBartosz Sobczak irdma_prm_add_pble_mem(struct irdma_pble_prm *pprm,
1952cdcd52d4SBartosz Sobczak 		       struct irdma_chunk *pchunk)
1953cdcd52d4SBartosz Sobczak {
1954cdcd52d4SBartosz Sobczak 	u64 sizeofbitmap;
1955cdcd52d4SBartosz Sobczak 
1956cdcd52d4SBartosz Sobczak 	if (pchunk->size & 0xfff)
1957cdcd52d4SBartosz Sobczak 		return -EINVAL;
1958cdcd52d4SBartosz Sobczak 
1959cdcd52d4SBartosz Sobczak 	sizeofbitmap = (u64)pchunk->size >> pprm->pble_shift;
1960cdcd52d4SBartosz Sobczak 
196101fbb869SBartosz Sobczak 	pchunk->bitmapbuf = bitmap_zalloc(sizeofbitmap, GFP_KERNEL);
196201fbb869SBartosz Sobczak 	if (!pchunk->bitmapbuf)
1963cdcd52d4SBartosz Sobczak 		return -ENOMEM;
1964cdcd52d4SBartosz Sobczak 
1965cdcd52d4SBartosz Sobczak 	pchunk->sizeofbitmap = sizeofbitmap;
1966cdcd52d4SBartosz Sobczak 	/* each pble is 8 bytes hence shift by 3 */
1967cdcd52d4SBartosz Sobczak 	pprm->total_pble_alloc += pchunk->size >> 3;
1968cdcd52d4SBartosz Sobczak 	pprm->free_pble_cnt += pchunk->size >> 3;
1969cdcd52d4SBartosz Sobczak 
1970cdcd52d4SBartosz Sobczak 	return 0;
1971cdcd52d4SBartosz Sobczak }
1972cdcd52d4SBartosz Sobczak 
1973cdcd52d4SBartosz Sobczak /**
1974cdcd52d4SBartosz Sobczak  * irdma_prm_get_pbles - get pble's from prm
1975cdcd52d4SBartosz Sobczak  * @pprm: pble resource manager
1976cdcd52d4SBartosz Sobczak  * @chunkinfo: nformation about chunk where pble's were acquired
1977cdcd52d4SBartosz Sobczak  * @mem_size: size of pble memory needed
1978cdcd52d4SBartosz Sobczak  * @vaddr: returns virtual address of pble memory
1979cdcd52d4SBartosz Sobczak  * @fpm_addr: returns fpm address of pble memory
1980cdcd52d4SBartosz Sobczak  */
1981cdcd52d4SBartosz Sobczak int
irdma_prm_get_pbles(struct irdma_pble_prm * pprm,struct irdma_pble_chunkinfo * chunkinfo,u64 mem_size,u64 ** vaddr,u64 * fpm_addr)1982cdcd52d4SBartosz Sobczak irdma_prm_get_pbles(struct irdma_pble_prm *pprm,
1983cdcd52d4SBartosz Sobczak 		    struct irdma_pble_chunkinfo *chunkinfo, u64 mem_size,
1984cdcd52d4SBartosz Sobczak 		    u64 **vaddr, u64 *fpm_addr)
1985cdcd52d4SBartosz Sobczak {
1986cdcd52d4SBartosz Sobczak 	u64 bits_needed;
1987cdcd52d4SBartosz Sobczak 	u64 bit_idx = PBLE_INVALID_IDX;
1988cdcd52d4SBartosz Sobczak 	struct irdma_chunk *pchunk = NULL;
1989cdcd52d4SBartosz Sobczak 	struct list_head *chunk_entry = (&pprm->clist)->next;
1990cdcd52d4SBartosz Sobczak 	u32 offset;
1991cdcd52d4SBartosz Sobczak 	unsigned long flags;
1992*5b5f7d0eSBartosz Sobczak 
1993cdcd52d4SBartosz Sobczak 	*vaddr = NULL;
1994cdcd52d4SBartosz Sobczak 	*fpm_addr = 0;
1995cdcd52d4SBartosz Sobczak 
1996cdcd52d4SBartosz Sobczak 	bits_needed = DIV_ROUND_UP_ULL(mem_size, BIT_ULL(pprm->pble_shift));
1997cdcd52d4SBartosz Sobczak 
1998cdcd52d4SBartosz Sobczak 	spin_lock_irqsave(&pprm->prm_lock, flags);
1999cdcd52d4SBartosz Sobczak 	while (chunk_entry != &pprm->clist) {
2000cdcd52d4SBartosz Sobczak 		pchunk = (struct irdma_chunk *)chunk_entry;
2001cdcd52d4SBartosz Sobczak 		bit_idx = bitmap_find_next_zero_area(pchunk->bitmapbuf,
2002cdcd52d4SBartosz Sobczak 						     pchunk->sizeofbitmap, 0,
2003cdcd52d4SBartosz Sobczak 						     bits_needed, 0);
2004cdcd52d4SBartosz Sobczak 		if (bit_idx < pchunk->sizeofbitmap)
2005cdcd52d4SBartosz Sobczak 			break;
2006cdcd52d4SBartosz Sobczak 
2007cdcd52d4SBartosz Sobczak 		/* list.next used macro */
2008cdcd52d4SBartosz Sobczak 		chunk_entry = (&pchunk->list)->next;
2009cdcd52d4SBartosz Sobczak 	}
2010cdcd52d4SBartosz Sobczak 
2011cdcd52d4SBartosz Sobczak 	if (!pchunk || bit_idx >= pchunk->sizeofbitmap) {
2012cdcd52d4SBartosz Sobczak 		spin_unlock_irqrestore(&pprm->prm_lock, flags);
2013cdcd52d4SBartosz Sobczak 		return -ENOMEM;
2014cdcd52d4SBartosz Sobczak 	}
2015cdcd52d4SBartosz Sobczak 
2016cdcd52d4SBartosz Sobczak 	bitmap_set(pchunk->bitmapbuf, bit_idx, bits_needed);
2017cdcd52d4SBartosz Sobczak 	offset = bit_idx << pprm->pble_shift;
2018cdcd52d4SBartosz Sobczak 	*vaddr = (u64 *)((u8 *)pchunk->vaddr + offset);
2019cdcd52d4SBartosz Sobczak 	*fpm_addr = pchunk->fpm_addr + offset;
2020cdcd52d4SBartosz Sobczak 
2021cdcd52d4SBartosz Sobczak 	chunkinfo->pchunk = pchunk;
2022cdcd52d4SBartosz Sobczak 	chunkinfo->bit_idx = bit_idx;
2023cdcd52d4SBartosz Sobczak 	chunkinfo->bits_used = bits_needed;
2024cdcd52d4SBartosz Sobczak 	/* 3 is sizeof pble divide */
2025cdcd52d4SBartosz Sobczak 	pprm->free_pble_cnt -= chunkinfo->bits_used << (pprm->pble_shift - 3);
2026cdcd52d4SBartosz Sobczak 	spin_unlock_irqrestore(&pprm->prm_lock, flags);
2027cdcd52d4SBartosz Sobczak 
2028cdcd52d4SBartosz Sobczak 	return 0;
2029cdcd52d4SBartosz Sobczak }
2030cdcd52d4SBartosz Sobczak 
2031cdcd52d4SBartosz Sobczak /**
2032cdcd52d4SBartosz Sobczak  * irdma_prm_return_pbles - return pbles back to prm
2033cdcd52d4SBartosz Sobczak  * @pprm: pble resource manager
2034cdcd52d4SBartosz Sobczak  * @chunkinfo: chunk where pble's were acquired and to be freed
2035cdcd52d4SBartosz Sobczak  */
2036cdcd52d4SBartosz Sobczak void
irdma_prm_return_pbles(struct irdma_pble_prm * pprm,struct irdma_pble_chunkinfo * chunkinfo)2037cdcd52d4SBartosz Sobczak irdma_prm_return_pbles(struct irdma_pble_prm *pprm,
2038cdcd52d4SBartosz Sobczak 		       struct irdma_pble_chunkinfo *chunkinfo)
2039cdcd52d4SBartosz Sobczak {
2040cdcd52d4SBartosz Sobczak 	unsigned long flags;
2041cdcd52d4SBartosz Sobczak 
2042cdcd52d4SBartosz Sobczak 	spin_lock_irqsave(&pprm->prm_lock, flags);
2043cdcd52d4SBartosz Sobczak 	pprm->free_pble_cnt += chunkinfo->bits_used << (pprm->pble_shift - 3);
2044cdcd52d4SBartosz Sobczak 	bitmap_clear(chunkinfo->pchunk->bitmapbuf, chunkinfo->bit_idx,
2045cdcd52d4SBartosz Sobczak 		     chunkinfo->bits_used);
2046cdcd52d4SBartosz Sobczak 	spin_unlock_irqrestore(&pprm->prm_lock, flags);
2047cdcd52d4SBartosz Sobczak }
2048cdcd52d4SBartosz Sobczak 
2049cdcd52d4SBartosz Sobczak int
irdma_map_vm_page_list(struct irdma_hw * hw,void * va,dma_addr_t * pg_dma,u32 pg_cnt)2050cdcd52d4SBartosz Sobczak irdma_map_vm_page_list(struct irdma_hw *hw, void *va, dma_addr_t * pg_dma,
2051cdcd52d4SBartosz Sobczak 		       u32 pg_cnt)
2052cdcd52d4SBartosz Sobczak {
2053cdcd52d4SBartosz Sobczak 	struct page *vm_page;
2054cdcd52d4SBartosz Sobczak 	int i;
2055cdcd52d4SBartosz Sobczak 	u8 *addr;
2056cdcd52d4SBartosz Sobczak 
2057cdcd52d4SBartosz Sobczak 	addr = (u8 *)(uintptr_t)va;
2058cdcd52d4SBartosz Sobczak 	for (i = 0; i < pg_cnt; i++) {
2059cdcd52d4SBartosz Sobczak 		vm_page = vmalloc_to_page(addr);
2060cdcd52d4SBartosz Sobczak 		if (!vm_page)
2061cdcd52d4SBartosz Sobczak 			goto err;
2062cdcd52d4SBartosz Sobczak 
2063cdcd52d4SBartosz Sobczak 		pg_dma[i] = dma_map_page(hw_to_dev(hw), vm_page, 0, PAGE_SIZE, DMA_BIDIRECTIONAL);
2064cdcd52d4SBartosz Sobczak 		if (dma_mapping_error(hw_to_dev(hw), pg_dma[i]))
2065cdcd52d4SBartosz Sobczak 			goto err;
2066cdcd52d4SBartosz Sobczak 
2067cdcd52d4SBartosz Sobczak 		addr += PAGE_SIZE;
2068cdcd52d4SBartosz Sobczak 	}
2069cdcd52d4SBartosz Sobczak 
2070cdcd52d4SBartosz Sobczak 	return 0;
2071cdcd52d4SBartosz Sobczak 
2072cdcd52d4SBartosz Sobczak err:
2073cdcd52d4SBartosz Sobczak 	irdma_unmap_vm_page_list(hw, pg_dma, i);
2074cdcd52d4SBartosz Sobczak 	return -ENOMEM;
2075cdcd52d4SBartosz Sobczak }
2076cdcd52d4SBartosz Sobczak 
2077cdcd52d4SBartosz Sobczak void
irdma_unmap_vm_page_list(struct irdma_hw * hw,dma_addr_t * pg_dma,u32 pg_cnt)2078cdcd52d4SBartosz Sobczak irdma_unmap_vm_page_list(struct irdma_hw *hw, dma_addr_t * pg_dma, u32 pg_cnt)
2079cdcd52d4SBartosz Sobczak {
2080cdcd52d4SBartosz Sobczak 	int i;
2081cdcd52d4SBartosz Sobczak 
2082cdcd52d4SBartosz Sobczak 	for (i = 0; i < pg_cnt; i++)
2083cdcd52d4SBartosz Sobczak 		dma_unmap_page(hw_to_dev(hw), pg_dma[i], PAGE_SIZE, DMA_BIDIRECTIONAL);
2084cdcd52d4SBartosz Sobczak }
2085cdcd52d4SBartosz Sobczak 
2086cdcd52d4SBartosz Sobczak /**
2087cdcd52d4SBartosz Sobczak  * irdma_pble_free_paged_mem - free virtual paged memory
2088cdcd52d4SBartosz Sobczak  * @chunk: chunk to free with paged memory
2089cdcd52d4SBartosz Sobczak  */
2090cdcd52d4SBartosz Sobczak void
irdma_pble_free_paged_mem(struct irdma_chunk * chunk)2091cdcd52d4SBartosz Sobczak irdma_pble_free_paged_mem(struct irdma_chunk *chunk)
2092cdcd52d4SBartosz Sobczak {
2093cdcd52d4SBartosz Sobczak 	if (!chunk->pg_cnt)
2094cdcd52d4SBartosz Sobczak 		goto done;
2095cdcd52d4SBartosz Sobczak 
2096cdcd52d4SBartosz Sobczak 	irdma_unmap_vm_page_list(chunk->dev->hw, chunk->dmainfo.dmaaddrs,
2097cdcd52d4SBartosz Sobczak 				 chunk->pg_cnt);
2098cdcd52d4SBartosz Sobczak 
2099cdcd52d4SBartosz Sobczak done:
2100cdcd52d4SBartosz Sobczak 	kfree(chunk->dmainfo.dmaaddrs);
2101cdcd52d4SBartosz Sobczak 	chunk->dmainfo.dmaaddrs = NULL;
2102cdcd52d4SBartosz Sobczak 	vfree(chunk->vaddr);
2103cdcd52d4SBartosz Sobczak 	chunk->vaddr = NULL;
2104cdcd52d4SBartosz Sobczak 	chunk->type = 0;
2105cdcd52d4SBartosz Sobczak }
2106cdcd52d4SBartosz Sobczak 
2107cdcd52d4SBartosz Sobczak /**
2108cdcd52d4SBartosz Sobczak  * irdma_pble_get_paged_mem -allocate paged memory for pbles
2109cdcd52d4SBartosz Sobczak  * @chunk: chunk to add for paged memory
2110cdcd52d4SBartosz Sobczak  * @pg_cnt: number of pages needed
2111cdcd52d4SBartosz Sobczak  */
2112cdcd52d4SBartosz Sobczak int
irdma_pble_get_paged_mem(struct irdma_chunk * chunk,u32 pg_cnt)2113cdcd52d4SBartosz Sobczak irdma_pble_get_paged_mem(struct irdma_chunk *chunk, u32 pg_cnt)
2114cdcd52d4SBartosz Sobczak {
2115cdcd52d4SBartosz Sobczak 	u32 size;
2116cdcd52d4SBartosz Sobczak 	void *va;
2117cdcd52d4SBartosz Sobczak 
2118cdcd52d4SBartosz Sobczak 	chunk->dmainfo.dmaaddrs = kzalloc(pg_cnt << 3, GFP_KERNEL);
2119cdcd52d4SBartosz Sobczak 	if (!chunk->dmainfo.dmaaddrs)
2120cdcd52d4SBartosz Sobczak 		return -ENOMEM;
2121cdcd52d4SBartosz Sobczak 
2122cdcd52d4SBartosz Sobczak 	size = PAGE_SIZE * pg_cnt;
2123cdcd52d4SBartosz Sobczak 	va = vmalloc(size);
2124cdcd52d4SBartosz Sobczak 	if (!va)
2125cdcd52d4SBartosz Sobczak 		goto err;
2126cdcd52d4SBartosz Sobczak 
2127cdcd52d4SBartosz Sobczak 	if (irdma_map_vm_page_list(chunk->dev->hw, va, chunk->dmainfo.dmaaddrs,
2128cdcd52d4SBartosz Sobczak 				   pg_cnt)) {
2129cdcd52d4SBartosz Sobczak 		vfree(va);
2130cdcd52d4SBartosz Sobczak 		goto err;
2131cdcd52d4SBartosz Sobczak 	}
2132cdcd52d4SBartosz Sobczak 	chunk->vaddr = va;
2133cdcd52d4SBartosz Sobczak 	chunk->size = size;
2134cdcd52d4SBartosz Sobczak 	chunk->pg_cnt = pg_cnt;
2135cdcd52d4SBartosz Sobczak 	chunk->type = PBLE_SD_PAGED;
2136cdcd52d4SBartosz Sobczak 
2137cdcd52d4SBartosz Sobczak 	return 0;
2138cdcd52d4SBartosz Sobczak err:
2139cdcd52d4SBartosz Sobczak 	kfree(chunk->dmainfo.dmaaddrs);
2140cdcd52d4SBartosz Sobczak 	chunk->dmainfo.dmaaddrs = NULL;
2141cdcd52d4SBartosz Sobczak 
2142cdcd52d4SBartosz Sobczak 	return -ENOMEM;
2143cdcd52d4SBartosz Sobczak }
2144cdcd52d4SBartosz Sobczak 
2145cdcd52d4SBartosz Sobczak /**
2146cdcd52d4SBartosz Sobczak  * irdma_alloc_ws_node_id - Allocate a tx scheduler node ID
2147cdcd52d4SBartosz Sobczak  * @dev: device pointer
2148cdcd52d4SBartosz Sobczak  */
2149cdcd52d4SBartosz Sobczak u16
irdma_alloc_ws_node_id(struct irdma_sc_dev * dev)2150cdcd52d4SBartosz Sobczak irdma_alloc_ws_node_id(struct irdma_sc_dev *dev)
2151cdcd52d4SBartosz Sobczak {
2152cdcd52d4SBartosz Sobczak 	struct irdma_pci_f *rf = dev_to_rf(dev);
2153cdcd52d4SBartosz Sobczak 	u32 next = 1;
2154cdcd52d4SBartosz Sobczak 	u32 node_id;
2155cdcd52d4SBartosz Sobczak 
2156cdcd52d4SBartosz Sobczak 	if (irdma_alloc_rsrc(rf, rf->allocated_ws_nodes, rf->max_ws_node_id,
2157cdcd52d4SBartosz Sobczak 			     &node_id, &next))
2158cdcd52d4SBartosz Sobczak 		return IRDMA_WS_NODE_INVALID;
2159cdcd52d4SBartosz Sobczak 
2160cdcd52d4SBartosz Sobczak 	return (u16)node_id;
2161cdcd52d4SBartosz Sobczak }
2162cdcd52d4SBartosz Sobczak 
2163cdcd52d4SBartosz Sobczak /**
2164cdcd52d4SBartosz Sobczak  * irdma_free_ws_node_id - Free a tx scheduler node ID
2165cdcd52d4SBartosz Sobczak  * @dev: device pointer
2166cdcd52d4SBartosz Sobczak  * @node_id: Work scheduler node ID
2167cdcd52d4SBartosz Sobczak  */
2168cdcd52d4SBartosz Sobczak void
irdma_free_ws_node_id(struct irdma_sc_dev * dev,u16 node_id)2169cdcd52d4SBartosz Sobczak irdma_free_ws_node_id(struct irdma_sc_dev *dev, u16 node_id)
2170cdcd52d4SBartosz Sobczak {
2171cdcd52d4SBartosz Sobczak 	struct irdma_pci_f *rf = dev_to_rf(dev);
2172cdcd52d4SBartosz Sobczak 
2173cdcd52d4SBartosz Sobczak 	irdma_free_rsrc(rf, rf->allocated_ws_nodes, (u32)node_id);
2174cdcd52d4SBartosz Sobczak }
2175cdcd52d4SBartosz Sobczak 
2176cdcd52d4SBartosz Sobczak /**
2177cdcd52d4SBartosz Sobczak  * irdma_modify_qp_to_err - Modify a QP to error
2178cdcd52d4SBartosz Sobczak  * @sc_qp: qp structure
2179cdcd52d4SBartosz Sobczak  */
2180cdcd52d4SBartosz Sobczak void
irdma_modify_qp_to_err(struct irdma_sc_qp * sc_qp)2181cdcd52d4SBartosz Sobczak irdma_modify_qp_to_err(struct irdma_sc_qp *sc_qp)
2182cdcd52d4SBartosz Sobczak {
2183cdcd52d4SBartosz Sobczak 	struct irdma_qp *qp = sc_qp->qp_uk.back_qp;
2184cdcd52d4SBartosz Sobczak 	struct ib_qp_attr attr;
2185cdcd52d4SBartosz Sobczak 
2186cdcd52d4SBartosz Sobczak 	if (qp->iwdev->rf->reset)
2187cdcd52d4SBartosz Sobczak 		return;
2188cdcd52d4SBartosz Sobczak 	attr.qp_state = IB_QPS_ERR;
2189cdcd52d4SBartosz Sobczak 
2190cdcd52d4SBartosz Sobczak 	if (rdma_protocol_roce(qp->ibqp.device, 1))
2191cdcd52d4SBartosz Sobczak 		irdma_modify_qp_roce(&qp->ibqp, &attr, IB_QP_STATE, NULL);
2192cdcd52d4SBartosz Sobczak 	else
2193cdcd52d4SBartosz Sobczak 		irdma_modify_qp(&qp->ibqp, &attr, IB_QP_STATE, NULL);
2194cdcd52d4SBartosz Sobczak }
2195cdcd52d4SBartosz Sobczak 
2196cdcd52d4SBartosz Sobczak void
irdma_ib_qp_event(struct irdma_qp * iwqp,enum irdma_qp_event_type event)2197cdcd52d4SBartosz Sobczak irdma_ib_qp_event(struct irdma_qp *iwqp, enum irdma_qp_event_type event)
2198cdcd52d4SBartosz Sobczak {
2199cdcd52d4SBartosz Sobczak 	struct ib_event ibevent;
2200cdcd52d4SBartosz Sobczak 
2201cdcd52d4SBartosz Sobczak 	if (!iwqp->ibqp.event_handler)
2202cdcd52d4SBartosz Sobczak 		return;
2203cdcd52d4SBartosz Sobczak 
2204cdcd52d4SBartosz Sobczak 	switch (event) {
2205cdcd52d4SBartosz Sobczak 	case IRDMA_QP_EVENT_CATASTROPHIC:
2206cdcd52d4SBartosz Sobczak 		ibevent.event = IB_EVENT_QP_FATAL;
2207cdcd52d4SBartosz Sobczak 		break;
2208cdcd52d4SBartosz Sobczak 	case IRDMA_QP_EVENT_ACCESS_ERR:
2209cdcd52d4SBartosz Sobczak 		ibevent.event = IB_EVENT_QP_ACCESS_ERR;
2210cdcd52d4SBartosz Sobczak 		break;
2211777e472cSBartosz Sobczak 	case IRDMA_QP_EVENT_REQ_ERR:
2212777e472cSBartosz Sobczak 		ibevent.event = IB_EVENT_QP_REQ_ERR;
2213777e472cSBartosz Sobczak 		break;
2214cdcd52d4SBartosz Sobczak 	}
2215cdcd52d4SBartosz Sobczak 	ibevent.device = iwqp->ibqp.device;
2216cdcd52d4SBartosz Sobczak 	ibevent.element.qp = &iwqp->ibqp;
2217cdcd52d4SBartosz Sobczak 	iwqp->ibqp.event_handler(&ibevent, iwqp->ibqp.qp_context);
2218cdcd52d4SBartosz Sobczak }
2219cdcd52d4SBartosz Sobczak 
2220cdcd52d4SBartosz Sobczak static void
clear_qp_ctx_addr(__le64 * ctx)2221cdcd52d4SBartosz Sobczak clear_qp_ctx_addr(__le64 * ctx)
2222cdcd52d4SBartosz Sobczak {
2223cdcd52d4SBartosz Sobczak 	u64 tmp;
2224cdcd52d4SBartosz Sobczak 
2225cdcd52d4SBartosz Sobczak 	get_64bit_val(ctx, 272, &tmp);
2226cdcd52d4SBartosz Sobczak 	tmp &= GENMASK_ULL(63, 58);
2227cdcd52d4SBartosz Sobczak 	set_64bit_val(ctx, 272, tmp);
2228cdcd52d4SBartosz Sobczak 
2229cdcd52d4SBartosz Sobczak 	get_64bit_val(ctx, 296, &tmp);
2230cdcd52d4SBartosz Sobczak 	tmp &= GENMASK_ULL(7, 0);
2231cdcd52d4SBartosz Sobczak 	set_64bit_val(ctx, 296, tmp);
2232cdcd52d4SBartosz Sobczak 
2233cdcd52d4SBartosz Sobczak 	get_64bit_val(ctx, 312, &tmp);
2234cdcd52d4SBartosz Sobczak 	tmp &= GENMASK_ULL(7, 0);
2235cdcd52d4SBartosz Sobczak 	set_64bit_val(ctx, 312, tmp);
2236cdcd52d4SBartosz Sobczak 
2237cdcd52d4SBartosz Sobczak 	set_64bit_val(ctx, 368, 0);
2238cdcd52d4SBartosz Sobczak }
2239cdcd52d4SBartosz Sobczak 
2240cdcd52d4SBartosz Sobczak /**
2241cdcd52d4SBartosz Sobczak  * irdma_upload_qp_context - upload raw QP context
2242cdcd52d4SBartosz Sobczak  * @iwqp: QP pointer
2243cdcd52d4SBartosz Sobczak  * @freeze: freeze QP
2244cdcd52d4SBartosz Sobczak  * @raw: raw context flag
2245cdcd52d4SBartosz Sobczak  */
2246cdcd52d4SBartosz Sobczak int
irdma_upload_qp_context(struct irdma_qp * iwqp,bool freeze,bool raw)2247cdcd52d4SBartosz Sobczak irdma_upload_qp_context(struct irdma_qp *iwqp, bool freeze, bool raw)
2248cdcd52d4SBartosz Sobczak {
2249cdcd52d4SBartosz Sobczak 	struct irdma_dma_mem dma_mem;
2250cdcd52d4SBartosz Sobczak 	struct irdma_sc_dev *dev;
2251cdcd52d4SBartosz Sobczak 	struct irdma_sc_qp *qp;
2252cdcd52d4SBartosz Sobczak 	struct irdma_cqp *iwcqp;
2253cdcd52d4SBartosz Sobczak 	struct irdma_cqp_request *cqp_request;
2254cdcd52d4SBartosz Sobczak 	struct cqp_cmds_info *cqp_info;
2255cdcd52d4SBartosz Sobczak 	struct irdma_upload_context_info *info;
2256cdcd52d4SBartosz Sobczak 	struct irdma_pci_f *rf;
2257cdcd52d4SBartosz Sobczak 	int ret;
2258cdcd52d4SBartosz Sobczak 	u32 *ctx;
2259cdcd52d4SBartosz Sobczak 
2260cdcd52d4SBartosz Sobczak 	rf = iwqp->iwdev->rf;
2261cdcd52d4SBartosz Sobczak 	if (!rf)
2262cdcd52d4SBartosz Sobczak 		return -EINVAL;
2263cdcd52d4SBartosz Sobczak 
2264cdcd52d4SBartosz Sobczak 	qp = &iwqp->sc_qp;
2265cdcd52d4SBartosz Sobczak 	dev = &rf->sc_dev;
2266cdcd52d4SBartosz Sobczak 	iwcqp = &rf->cqp;
2267cdcd52d4SBartosz Sobczak 
2268cdcd52d4SBartosz Sobczak 	cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);
2269cdcd52d4SBartosz Sobczak 	if (!cqp_request)
2270cdcd52d4SBartosz Sobczak 		return -EINVAL;
2271cdcd52d4SBartosz Sobczak 
2272cdcd52d4SBartosz Sobczak 	cqp_info = &cqp_request->info;
2273cdcd52d4SBartosz Sobczak 	info = &cqp_info->in.u.qp_upload_context.info;
2274cdcd52d4SBartosz Sobczak 	memset(info, 0, sizeof(struct irdma_upload_context_info));
2275cdcd52d4SBartosz Sobczak 	cqp_info->cqp_cmd = IRDMA_OP_QP_UPLOAD_CONTEXT;
2276cdcd52d4SBartosz Sobczak 	cqp_info->post_sq = 1;
2277cdcd52d4SBartosz Sobczak 	cqp_info->in.u.qp_upload_context.dev = dev;
2278cdcd52d4SBartosz Sobczak 	cqp_info->in.u.qp_upload_context.scratch = (uintptr_t)cqp_request;
2279cdcd52d4SBartosz Sobczak 
2280cdcd52d4SBartosz Sobczak 	dma_mem.size = PAGE_SIZE;
2281cdcd52d4SBartosz Sobczak 	dma_mem.va = irdma_allocate_dma_mem(dev->hw, &dma_mem, dma_mem.size, PAGE_SIZE);
2282cdcd52d4SBartosz Sobczak 	if (!dma_mem.va) {
2283cdcd52d4SBartosz Sobczak 		irdma_put_cqp_request(&rf->cqp, cqp_request);
2284cdcd52d4SBartosz Sobczak 		return -ENOMEM;
2285cdcd52d4SBartosz Sobczak 	}
2286cdcd52d4SBartosz Sobczak 
2287cdcd52d4SBartosz Sobczak 	ctx = dma_mem.va;
2288cdcd52d4SBartosz Sobczak 	info->buf_pa = dma_mem.pa;
2289cdcd52d4SBartosz Sobczak 	info->raw_format = raw;
2290cdcd52d4SBartosz Sobczak 	info->freeze_qp = freeze;
2291cdcd52d4SBartosz Sobczak 	info->qp_type = qp->qp_uk.qp_type;	/* 1 is iWARP and 2 UDA */
2292cdcd52d4SBartosz Sobczak 	info->qp_id = qp->qp_uk.qp_id;
2293cdcd52d4SBartosz Sobczak 	ret = irdma_handle_cqp_op(rf, cqp_request);
2294cdcd52d4SBartosz Sobczak 	if (ret)
2295cdcd52d4SBartosz Sobczak 		goto error;
2296cdcd52d4SBartosz Sobczak 	irdma_debug(dev, IRDMA_DEBUG_QP, "PRINT CONTXT QP [%d]\n", info->qp_id);
2297cdcd52d4SBartosz Sobczak 	{
2298cdcd52d4SBartosz Sobczak 		u32 i, j;
2299cdcd52d4SBartosz Sobczak 
2300cdcd52d4SBartosz Sobczak 		clear_qp_ctx_addr(dma_mem.va);
2301cdcd52d4SBartosz Sobczak 		for (i = 0, j = 0; i < 32; i++, j += 4)
2302cdcd52d4SBartosz Sobczak 			irdma_debug(dev, IRDMA_DEBUG_QP,
230301fbb869SBartosz Sobczak 				    "%d:\t [%08X %08x %08X %08X]\n", (j * 4),
230401fbb869SBartosz Sobczak 				    ctx[j], ctx[j + 1], ctx[j + 2], ctx[j + 3]);
2305cdcd52d4SBartosz Sobczak 	}
2306cdcd52d4SBartosz Sobczak error:
2307cdcd52d4SBartosz Sobczak 	irdma_put_cqp_request(iwcqp, cqp_request);
2308cdcd52d4SBartosz Sobczak 	irdma_free_dma_mem(dev->hw, &dma_mem);
2309cdcd52d4SBartosz Sobczak 
2310cdcd52d4SBartosz Sobczak 	return ret;
2311cdcd52d4SBartosz Sobczak }
2312cdcd52d4SBartosz Sobczak 
2313cdcd52d4SBartosz Sobczak bool
irdma_cq_empty(struct irdma_cq * iwcq)2314cdcd52d4SBartosz Sobczak irdma_cq_empty(struct irdma_cq *iwcq)
2315cdcd52d4SBartosz Sobczak {
2316cdcd52d4SBartosz Sobczak 	struct irdma_cq_uk *ukcq;
2317cdcd52d4SBartosz Sobczak 	u64 qword3;
2318cdcd52d4SBartosz Sobczak 	__le64 *cqe;
2319cdcd52d4SBartosz Sobczak 	u8 polarity;
2320cdcd52d4SBartosz Sobczak 
2321cdcd52d4SBartosz Sobczak 	ukcq = &iwcq->sc_cq.cq_uk;
2322cdcd52d4SBartosz Sobczak 	cqe = IRDMA_GET_CURRENT_CQ_ELEM(ukcq);
2323cdcd52d4SBartosz Sobczak 	get_64bit_val(cqe, 24, &qword3);
2324777e472cSBartosz Sobczak 	polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3);
2325cdcd52d4SBartosz Sobczak 
2326cdcd52d4SBartosz Sobczak 	return polarity != ukcq->polarity;
2327cdcd52d4SBartosz Sobczak }
2328cdcd52d4SBartosz Sobczak 
2329cdcd52d4SBartosz Sobczak void
irdma_remove_cmpls_list(struct irdma_cq * iwcq)2330cdcd52d4SBartosz Sobczak irdma_remove_cmpls_list(struct irdma_cq *iwcq)
2331cdcd52d4SBartosz Sobczak {
2332cdcd52d4SBartosz Sobczak 	struct irdma_cmpl_gen *cmpl_node;
2333cdcd52d4SBartosz Sobczak 	struct list_head *tmp_node, *list_node;
2334cdcd52d4SBartosz Sobczak 
2335cdcd52d4SBartosz Sobczak 	list_for_each_safe(list_node, tmp_node, &iwcq->cmpl_generated) {
2336cdcd52d4SBartosz Sobczak 		cmpl_node = list_entry(list_node, struct irdma_cmpl_gen, list);
2337cdcd52d4SBartosz Sobczak 		list_del(&cmpl_node->list);
2338cdcd52d4SBartosz Sobczak 		kfree(cmpl_node);
2339cdcd52d4SBartosz Sobczak 	}
2340cdcd52d4SBartosz Sobczak }
2341cdcd52d4SBartosz Sobczak 
2342cdcd52d4SBartosz Sobczak int
irdma_generated_cmpls(struct irdma_cq * iwcq,struct irdma_cq_poll_info * cq_poll_info)2343cdcd52d4SBartosz Sobczak irdma_generated_cmpls(struct irdma_cq *iwcq, struct irdma_cq_poll_info *cq_poll_info)
2344cdcd52d4SBartosz Sobczak {
2345cdcd52d4SBartosz Sobczak 	struct irdma_cmpl_gen *cmpl;
2346cdcd52d4SBartosz Sobczak 
2347777e472cSBartosz Sobczak 	if (list_empty(&iwcq->cmpl_generated))
2348cdcd52d4SBartosz Sobczak 		return -ENOENT;
2349cdcd52d4SBartosz Sobczak 	cmpl = list_first_entry_or_null(&iwcq->cmpl_generated, struct irdma_cmpl_gen, list);
2350cdcd52d4SBartosz Sobczak 	list_del(&cmpl->list);
2351cdcd52d4SBartosz Sobczak 	memcpy(cq_poll_info, &cmpl->cpi, sizeof(*cq_poll_info));
2352cdcd52d4SBartosz Sobczak 	kfree(cmpl);
2353cdcd52d4SBartosz Sobczak 
2354cdcd52d4SBartosz Sobczak 	irdma_debug(iwcq->sc_cq.dev, IRDMA_DEBUG_VERBS,
2355cdcd52d4SBartosz Sobczak 		    "%s: Poll artificially generated completion for QP 0x%X, op %u, wr_id=0x%lx\n",
235635105900SBartosz Sobczak 		    __func__, cq_poll_info->qp_id, cq_poll_info->op_type,
235735105900SBartosz Sobczak 		    cq_poll_info->wr_id);
2358cdcd52d4SBartosz Sobczak 
2359cdcd52d4SBartosz Sobczak 	return 0;
2360cdcd52d4SBartosz Sobczak }
2361cdcd52d4SBartosz Sobczak 
2362cdcd52d4SBartosz Sobczak /**
2363cdcd52d4SBartosz Sobczak  * irdma_set_cpi_common_values - fill in values for polling info struct
2364cdcd52d4SBartosz Sobczak  * @cpi: resulting structure of cq_poll_info type
2365cdcd52d4SBartosz Sobczak  * @qp: QPair
2366cdcd52d4SBartosz Sobczak  * @qp_num: id of the QP
2367cdcd52d4SBartosz Sobczak  */
2368cdcd52d4SBartosz Sobczak static void
irdma_set_cpi_common_values(struct irdma_cq_poll_info * cpi,struct irdma_qp_uk * qp,u32 qp_num)2369cdcd52d4SBartosz Sobczak irdma_set_cpi_common_values(struct irdma_cq_poll_info *cpi,
2370cdcd52d4SBartosz Sobczak 			    struct irdma_qp_uk *qp, u32 qp_num)
2371cdcd52d4SBartosz Sobczak {
2372cdcd52d4SBartosz Sobczak 	cpi->comp_status = IRDMA_COMPL_STATUS_FLUSHED;
2373cdcd52d4SBartosz Sobczak 	cpi->error = 1;
2374cdcd52d4SBartosz Sobczak 	cpi->major_err = IRDMA_FLUSH_MAJOR_ERR;
2375cdcd52d4SBartosz Sobczak 	cpi->minor_err = FLUSH_GENERAL_ERR;
2376cdcd52d4SBartosz Sobczak 	cpi->qp_handle = (irdma_qp_handle) (uintptr_t)qp;
2377cdcd52d4SBartosz Sobczak 	cpi->qp_id = qp_num;
2378cdcd52d4SBartosz Sobczak }
2379cdcd52d4SBartosz Sobczak 
2380cdcd52d4SBartosz Sobczak static inline void
irdma_comp_handler(struct irdma_cq * cq)2381cdcd52d4SBartosz Sobczak irdma_comp_handler(struct irdma_cq *cq)
2382cdcd52d4SBartosz Sobczak {
2383777e472cSBartosz Sobczak 	if (!cq->ibcq.comp_handler)
2384777e472cSBartosz Sobczak 		return;
2385777e472cSBartosz Sobczak 
2386777e472cSBartosz Sobczak 	if (atomic_cmpxchg(&cq->armed, 1, 0))
2387cdcd52d4SBartosz Sobczak 		cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
2388cdcd52d4SBartosz Sobczak }
2389cdcd52d4SBartosz Sobczak 
2390cdcd52d4SBartosz Sobczak /**
2391cdcd52d4SBartosz Sobczak  * irdma_generate_flush_completions - generate completion from WRs
2392cdcd52d4SBartosz Sobczak  * @iwqp: pointer to QP
2393cdcd52d4SBartosz Sobczak  */
2394cdcd52d4SBartosz Sobczak void
irdma_generate_flush_completions(struct irdma_qp * iwqp)2395cdcd52d4SBartosz Sobczak irdma_generate_flush_completions(struct irdma_qp *iwqp)
2396cdcd52d4SBartosz Sobczak {
2397cdcd52d4SBartosz Sobczak 	struct irdma_qp_uk *qp = &iwqp->sc_qp.qp_uk;
2398cdcd52d4SBartosz Sobczak 	struct irdma_ring *sq_ring = &qp->sq_ring;
2399cdcd52d4SBartosz Sobczak 	struct irdma_ring *rq_ring = &qp->rq_ring;
2400cdcd52d4SBartosz Sobczak 	struct irdma_cmpl_gen *cmpl;
2401cdcd52d4SBartosz Sobczak 	__le64 *sw_wqe;
2402cdcd52d4SBartosz Sobczak 	u64 wqe_qword;
2403cdcd52d4SBartosz Sobczak 	u32 wqe_idx;
240435105900SBartosz Sobczak 	bool compl_generated = false;
240535105900SBartosz Sobczak 	unsigned long flags1;
2406cdcd52d4SBartosz Sobczak 
240735105900SBartosz Sobczak 	spin_lock_irqsave(&iwqp->iwscq->lock, flags1);
2408cdcd52d4SBartosz Sobczak 	if (irdma_cq_empty(iwqp->iwscq)) {
2409777e472cSBartosz Sobczak 		unsigned long flags2;
2410777e472cSBartosz Sobczak 
2411777e472cSBartosz Sobczak 		spin_lock_irqsave(&iwqp->lock, flags2);
2412cdcd52d4SBartosz Sobczak 		while (IRDMA_RING_MORE_WORK(*sq_ring)) {
2413777e472cSBartosz Sobczak 			cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC);
2414cdcd52d4SBartosz Sobczak 			if (!cmpl) {
2415777e472cSBartosz Sobczak 				spin_unlock_irqrestore(&iwqp->lock, flags2);
241635105900SBartosz Sobczak 				spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1);
2417cdcd52d4SBartosz Sobczak 				return;
2418cdcd52d4SBartosz Sobczak 			}
2419cdcd52d4SBartosz Sobczak 
2420cdcd52d4SBartosz Sobczak 			wqe_idx = sq_ring->tail;
2421cdcd52d4SBartosz Sobczak 			irdma_set_cpi_common_values(&cmpl->cpi, qp, qp->qp_id);
2422cdcd52d4SBartosz Sobczak 
2423cdcd52d4SBartosz Sobczak 			cmpl->cpi.wr_id = qp->sq_wrtrk_array[wqe_idx].wrid;
2424cdcd52d4SBartosz Sobczak 			cmpl->cpi.signaled = qp->sq_wrtrk_array[wqe_idx].signaled;
2425cdcd52d4SBartosz Sobczak 			sw_wqe = qp->sq_base[wqe_idx].elem;
2426cdcd52d4SBartosz Sobczak 			get_64bit_val(sw_wqe, IRDMA_BYTE_24, &wqe_qword);
2427777e472cSBartosz Sobczak 			cmpl->cpi.op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, wqe_qword);
2428777e472cSBartosz Sobczak 			cmpl->cpi.q_type = IRDMA_CQE_QTYPE_SQ;
2429cdcd52d4SBartosz Sobczak 			/* remove the SQ WR by moving SQ tail */
243035105900SBartosz Sobczak 			IRDMA_RING_SET_TAIL(*sq_ring,
243135105900SBartosz Sobczak 					    sq_ring->tail + qp->sq_wrtrk_array[sq_ring->tail].quanta);
2432cdcd52d4SBartosz Sobczak 
243335105900SBartosz Sobczak 			if (cmpl->cpi.op_type == IRDMAQP_OP_NOP) {
243435105900SBartosz Sobczak 				kfree(cmpl);
243535105900SBartosz Sobczak 				continue;
243635105900SBartosz Sobczak 			}
2437cdcd52d4SBartosz Sobczak 			irdma_debug(iwqp->sc_qp.dev, IRDMA_DEBUG_DEV,
243835105900SBartosz Sobczak 				    "%s: adding wr_id = 0x%lx SQ Completion to list qp_id=%d\n",
243935105900SBartosz Sobczak 				    __func__, cmpl->cpi.wr_id, qp->qp_id);
2440cdcd52d4SBartosz Sobczak 			list_add_tail(&cmpl->list, &iwqp->iwscq->cmpl_generated);
244135105900SBartosz Sobczak 			compl_generated = true;
2442cdcd52d4SBartosz Sobczak 		}
2443777e472cSBartosz Sobczak 		spin_unlock_irqrestore(&iwqp->lock, flags2);
244435105900SBartosz Sobczak 		spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1);
244535105900SBartosz Sobczak 		if (compl_generated) {
244635105900SBartosz Sobczak 			irdma_comp_handler(iwqp->iwscq);
244735105900SBartosz Sobczak 			compl_generated = false;
244835105900SBartosz Sobczak 		}
2449777e472cSBartosz Sobczak 	} else {
245035105900SBartosz Sobczak 		spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1);
245135105900SBartosz Sobczak 		irdma_sched_qp_flush_work(iwqp);
2452777e472cSBartosz Sobczak 	}
2453cdcd52d4SBartosz Sobczak 
245435105900SBartosz Sobczak 	spin_lock_irqsave(&iwqp->iwrcq->lock, flags1);
2455cdcd52d4SBartosz Sobczak 	if (irdma_cq_empty(iwqp->iwrcq)) {
2456777e472cSBartosz Sobczak 		unsigned long flags2;
2457777e472cSBartosz Sobczak 
2458777e472cSBartosz Sobczak 		spin_lock_irqsave(&iwqp->lock, flags2);
2459cdcd52d4SBartosz Sobczak 		while (IRDMA_RING_MORE_WORK(*rq_ring)) {
2460777e472cSBartosz Sobczak 			cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC);
2461cdcd52d4SBartosz Sobczak 			if (!cmpl) {
2462777e472cSBartosz Sobczak 				spin_unlock_irqrestore(&iwqp->lock, flags2);
246335105900SBartosz Sobczak 				spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1);
2464cdcd52d4SBartosz Sobczak 				return;
2465cdcd52d4SBartosz Sobczak 			}
2466cdcd52d4SBartosz Sobczak 
2467cdcd52d4SBartosz Sobczak 			wqe_idx = rq_ring->tail;
2468cdcd52d4SBartosz Sobczak 			irdma_set_cpi_common_values(&cmpl->cpi, qp, qp->qp_id);
2469cdcd52d4SBartosz Sobczak 
2470cdcd52d4SBartosz Sobczak 			cmpl->cpi.wr_id = qp->rq_wrid_array[wqe_idx];
2471cdcd52d4SBartosz Sobczak 			cmpl->cpi.signaled = 1;
2472cdcd52d4SBartosz Sobczak 			cmpl->cpi.op_type = IRDMA_OP_TYPE_REC;
2473777e472cSBartosz Sobczak 			cmpl->cpi.q_type = IRDMA_CQE_QTYPE_RQ;
2474cdcd52d4SBartosz Sobczak 			/* remove the RQ WR by moving RQ tail */
2475cdcd52d4SBartosz Sobczak 			IRDMA_RING_SET_TAIL(*rq_ring, rq_ring->tail + 1);
2476cdcd52d4SBartosz Sobczak 			irdma_debug(iwqp->sc_qp.dev, IRDMA_DEBUG_DEV,
2477cdcd52d4SBartosz Sobczak 				    "%s: adding wr_id = 0x%lx RQ Completion to list qp_id=%d, wqe_idx=%d\n",
247835105900SBartosz Sobczak 				    __func__, cmpl->cpi.wr_id, qp->qp_id,
247935105900SBartosz Sobczak 				    wqe_idx);
248035105900SBartosz Sobczak 
2481cdcd52d4SBartosz Sobczak 			list_add_tail(&cmpl->list, &iwqp->iwrcq->cmpl_generated);
2482cdcd52d4SBartosz Sobczak 
248335105900SBartosz Sobczak 			compl_generated = true;
2484cdcd52d4SBartosz Sobczak 		}
2485777e472cSBartosz Sobczak 		spin_unlock_irqrestore(&iwqp->lock, flags2);
248635105900SBartosz Sobczak 		spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1);
248735105900SBartosz Sobczak 		if (compl_generated)
248835105900SBartosz Sobczak 			irdma_comp_handler(iwqp->iwrcq);
2489777e472cSBartosz Sobczak 	} else {
249035105900SBartosz Sobczak 		spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1);
249135105900SBartosz Sobczak 		irdma_sched_qp_flush_work(iwqp);
249235105900SBartosz Sobczak 	}
2493cdcd52d4SBartosz Sobczak }
2494777e472cSBartosz Sobczak 
249535105900SBartosz Sobczak /**
249635105900SBartosz Sobczak  * irdma_udqp_qs_change - change qs for UD QP in a worker thread
249735105900SBartosz Sobczak  * @iwqp: QP pointer
249835105900SBartosz Sobczak  * @user_prio: new user priority value
249935105900SBartosz Sobczak  * @qs_change: when false, only user priority changes, QS handle do not need to change
250035105900SBartosz Sobczak  */
250135105900SBartosz Sobczak static void
irdma_udqp_qs_change(struct irdma_qp * iwqp,u8 user_prio,bool qs_change)250235105900SBartosz Sobczak irdma_udqp_qs_change(struct irdma_qp *iwqp, u8 user_prio, bool qs_change)
250335105900SBartosz Sobczak {
250435105900SBartosz Sobczak 	irdma_qp_rem_qos(&iwqp->sc_qp);
250535105900SBartosz Sobczak 	if (qs_change)
250635105900SBartosz Sobczak 		iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi, iwqp->ctx_info.user_pri);
250735105900SBartosz Sobczak 
250835105900SBartosz Sobczak 	iwqp->ctx_info.user_pri = user_prio;
250935105900SBartosz Sobczak 	iwqp->sc_qp.user_pri = user_prio;
251035105900SBartosz Sobczak 
251135105900SBartosz Sobczak 	if (qs_change)
251235105900SBartosz Sobczak 		if (iwqp->sc_qp.dev->ws_add(iwqp->sc_qp.vsi, user_prio))
251335105900SBartosz Sobczak 			irdma_dev_warn(&iwqp->iwdev->ibdev,
251435105900SBartosz Sobczak 				       "WS add failed during %s, qp_id: %x user_pri: %x",
251535105900SBartosz Sobczak 				       __func__, iwqp->ibqp.qp_num, user_prio);
251635105900SBartosz Sobczak 	irdma_qp_add_qos(&iwqp->sc_qp);
2517777e472cSBartosz Sobczak }
251835105900SBartosz Sobczak 
251935105900SBartosz Sobczak void
irdma_udqp_qs_worker(struct work_struct * work)252035105900SBartosz Sobczak irdma_udqp_qs_worker(struct work_struct *work)
252135105900SBartosz Sobczak {
252235105900SBartosz Sobczak 	struct irdma_udqs_work *udqs_work = container_of(work, struct irdma_udqs_work, work);
252335105900SBartosz Sobczak 
252435105900SBartosz Sobczak 	irdma_udqp_qs_change(udqs_work->iwqp, udqs_work->user_prio, udqs_work->qs_change);
252535105900SBartosz Sobczak 	if (udqs_work->qs_change)
252635105900SBartosz Sobczak 		irdma_cqp_qp_suspend_resume(&udqs_work->iwqp->sc_qp, IRDMA_OP_RESUME);
252735105900SBartosz Sobczak 	irdma_qp_rem_ref(&udqs_work->iwqp->ibqp);
252835105900SBartosz Sobczak 	kfree(udqs_work);
2529777e472cSBartosz Sobczak }
2530