1 /*
2 * Copyright (c) 2017-2018 Cavium, Inc.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
16 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
19 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
20 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
21 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
22 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
23 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
24 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
25 * POSSIBILITY OF SUCH DAMAGE.
26 */
27 /*
28 * File : ecore_spq.c
29 */
30 #include <sys/cdefs.h>
31 #include "bcm_osal.h"
32 #include "reg_addr.h"
33 #include "ecore_gtt_reg_addr.h"
34 #include "ecore_hsi_common.h"
35 #include "ecore.h"
36 #include "ecore_sp_api.h"
37 #include "ecore_spq.h"
38 #include "ecore_iro.h"
39 #include "ecore_init_fw_funcs.h"
40 #include "ecore_cxt.h"
41 #include "ecore_int.h"
42 #include "ecore_dev_api.h"
43 #include "ecore_mcp.h"
44 #ifdef CONFIG_ECORE_RDMA
45 #include "ecore_rdma.h"
46 #endif
47 #include "ecore_hw.h"
48 #include "ecore_sriov.h"
49 #ifdef CONFIG_ECORE_ISCSI
50 #include "ecore_iscsi.h"
51 #include "ecore_ooo.h"
52 #endif
53
54 #ifdef _NTDDK_
55 #pragma warning(push)
56 #pragma warning(disable : 28167)
57 #pragma warning(disable : 28123)
58 #endif
59
60 /***************************************************************************
61 * Structures & Definitions
62 ***************************************************************************/
63
64 #define SPQ_HIGH_PRI_RESERVE_DEFAULT (1)
65
66 #define SPQ_BLOCK_DELAY_MAX_ITER (10)
67 #define SPQ_BLOCK_DELAY_US (10)
68 #define SPQ_BLOCK_SLEEP_MAX_ITER (200)
69 #define SPQ_BLOCK_SLEEP_MS (5)
70
71 #ifndef REMOVE_DBG
72 /***************************************************************************
73 * Debug [iSCSI] tool
74 ***************************************************************************/
ecore_iscsi_eq_dump(struct ecore_hwfn * p_hwfn,struct event_ring_entry * p_eqe)75 static void ecore_iscsi_eq_dump(struct ecore_hwfn *p_hwfn,
76 struct event_ring_entry *p_eqe)
77 {
78 if (p_eqe->opcode >= MAX_ISCSI_EQE_OPCODE) {
79 DP_NOTICE(p_hwfn, false, "Unknown iSCSI EQ: %x\n",
80 p_eqe->opcode);
81 }
82
83 switch (p_eqe->opcode) {
84 case ISCSI_EVENT_TYPE_INIT_FUNC:
85 case ISCSI_EVENT_TYPE_DESTROY_FUNC:
86 /* NOPE */
87 break;
88 case ISCSI_EVENT_TYPE_OFFLOAD_CONN:
89 case ISCSI_EVENT_TYPE_TERMINATE_CONN:
90 DP_VERBOSE(p_hwfn, ECORE_MSG_STORAGE,
91 "iSCSI EQE: Port %x, Op %x, echo %x, FWret %x, CID %x, ConnID %x, ERR %x\n",
92 p_hwfn->port_id, p_eqe->opcode,
93 OSAL_LE16_TO_CPU(p_eqe->echo),
94 p_eqe->fw_return_code,
95 OSAL_LE16_TO_CPU(p_eqe->data.iscsi_info.icid),
96 OSAL_LE16_TO_CPU(p_eqe->data.iscsi_info.conn_id),
97 p_eqe->data.iscsi_info.error_code);
98 break;
99 case ISCSI_EVENT_TYPE_UPDATE_CONN:
100 case ISCSI_EVENT_TYPE_CLEAR_SQ:
101 case ISCSI_EVENT_TYPE_ASYN_CONNECT_COMPLETE:
102 case ISCSI_EVENT_TYPE_ASYN_TERMINATE_DONE:
103 case ISCSI_EVENT_TYPE_ASYN_ABORT_RCVD:
104 case ISCSI_EVENT_TYPE_ASYN_CLOSE_RCVD:
105 case ISCSI_EVENT_TYPE_ASYN_SYN_RCVD:
106 case ISCSI_EVENT_TYPE_ASYN_MAX_RT_TIME:
107 case ISCSI_EVENT_TYPE_ASYN_MAX_RT_CNT:
108 case ISCSI_EVENT_TYPE_ASYN_MAX_KA_PROBES_CNT:
109 case ISCSI_EVENT_TYPE_ASYN_FIN_WAIT2:
110 case ISCSI_EVENT_TYPE_ISCSI_CONN_ERROR:
111 case ISCSI_EVENT_TYPE_TCP_CONN_ERROR:
112 default:
113 /* NOPE */
114 break;
115 }
116 }
117 #endif
118
119 /***************************************************************************
120 * Blocking Imp. (BLOCK/EBLOCK mode)
121 ***************************************************************************/
ecore_spq_blocking_cb(struct ecore_hwfn * p_hwfn,void * cookie,union event_ring_data OSAL_UNUSED * data,u8 fw_return_code)122 static void ecore_spq_blocking_cb(struct ecore_hwfn *p_hwfn, void *cookie,
123 union event_ring_data OSAL_UNUSED *data,
124 u8 fw_return_code)
125 {
126 struct ecore_spq_comp_done *comp_done;
127
128 comp_done = (struct ecore_spq_comp_done *)cookie;
129
130 comp_done->done = 0x1;
131 comp_done->fw_return_code = fw_return_code;
132
133 /* make update visible to waiting thread */
134 OSAL_SMP_WMB(p_hwfn->p_dev);
135 }
136
__ecore_spq_block(struct ecore_hwfn * p_hwfn,struct ecore_spq_entry * p_ent,u8 * p_fw_ret,bool sleep_between_iter)137 static enum _ecore_status_t __ecore_spq_block(struct ecore_hwfn *p_hwfn,
138 struct ecore_spq_entry *p_ent,
139 u8 *p_fw_ret,
140 bool sleep_between_iter)
141 {
142 struct ecore_spq_comp_done *comp_done;
143 u32 iter_cnt;
144
145 comp_done = (struct ecore_spq_comp_done *)p_ent->comp_cb.cookie;
146 iter_cnt = sleep_between_iter ? SPQ_BLOCK_SLEEP_MAX_ITER
147 : SPQ_BLOCK_DELAY_MAX_ITER;
148 #ifndef ASIC_ONLY
149 if (CHIP_REV_IS_EMUL(p_hwfn->p_dev) && sleep_between_iter)
150 iter_cnt *= 5;
151 #endif
152
153 while (iter_cnt--) {
154 OSAL_POLL_MODE_DPC(p_hwfn);
155 OSAL_SMP_RMB(p_hwfn->p_dev);
156 if (comp_done->done == 1) {
157 if (p_fw_ret)
158 *p_fw_ret = comp_done->fw_return_code;
159 return ECORE_SUCCESS;
160 }
161
162 if (sleep_between_iter) {
163 OSAL_MSLEEP(SPQ_BLOCK_SLEEP_MS);
164 } else {
165 OSAL_UDELAY(SPQ_BLOCK_DELAY_US);
166 }
167 }
168
169 return ECORE_TIMEOUT;
170 }
171
ecore_spq_block(struct ecore_hwfn * p_hwfn,struct ecore_spq_entry * p_ent,u8 * p_fw_ret,bool skip_quick_poll)172 static enum _ecore_status_t ecore_spq_block(struct ecore_hwfn *p_hwfn,
173 struct ecore_spq_entry *p_ent,
174 u8 *p_fw_ret, bool skip_quick_poll)
175 {
176 struct ecore_spq_comp_done *comp_done;
177 struct ecore_ptt *p_ptt;
178 enum _ecore_status_t rc;
179
180 /* A relatively short polling period w/o sleeping, to allow the FW to
181 * complete the ramrod and thus possibly to avoid the following sleeps.
182 */
183 if (!skip_quick_poll) {
184 rc = __ecore_spq_block(p_hwfn, p_ent, p_fw_ret, false);
185 if (rc == ECORE_SUCCESS)
186 return ECORE_SUCCESS;
187 }
188
189 /* Move to polling with a sleeping period between iterations */
190 rc = __ecore_spq_block(p_hwfn, p_ent, p_fw_ret, true);
191 if (rc == ECORE_SUCCESS)
192 return ECORE_SUCCESS;
193
194 p_ptt = ecore_ptt_acquire(p_hwfn);
195 if (!p_ptt)
196 return ECORE_AGAIN;
197
198 DP_INFO(p_hwfn, "Ramrod is stuck, requesting MCP drain\n");
199 rc = ecore_mcp_drain(p_hwfn, p_ptt);
200 ecore_ptt_release(p_hwfn, p_ptt);
201 if (rc != ECORE_SUCCESS) {
202 DP_NOTICE(p_hwfn, true, "MCP drain failed\n");
203 goto err;
204 }
205
206 /* Retry after drain */
207 rc = __ecore_spq_block(p_hwfn, p_ent, p_fw_ret, true);
208 if (rc == ECORE_SUCCESS)
209 return ECORE_SUCCESS;
210
211 comp_done = (struct ecore_spq_comp_done *)p_ent->comp_cb.cookie;
212 if (comp_done->done == 1) {
213 if (p_fw_ret)
214 *p_fw_ret = comp_done->fw_return_code;
215 return ECORE_SUCCESS;
216 }
217 err:
218 DP_NOTICE(p_hwfn, true,
219 "Ramrod is stuck [CID %08x cmd %02x protocol %02x echo %04x]\n",
220 OSAL_LE32_TO_CPU(p_ent->elem.hdr.cid),
221 p_ent->elem.hdr.cmd_id, p_ent->elem.hdr.protocol_id,
222 OSAL_LE16_TO_CPU(p_ent->elem.hdr.echo));
223
224 ecore_hw_err_notify(p_hwfn, ECORE_HW_ERR_RAMROD_FAIL);
225
226 return ECORE_BUSY;
227 }
228
229 /***************************************************************************
230 * SPQ entries inner API
231 ***************************************************************************/
ecore_spq_fill_entry(struct ecore_hwfn * p_hwfn,struct ecore_spq_entry * p_ent)232 static enum _ecore_status_t ecore_spq_fill_entry(struct ecore_hwfn *p_hwfn,
233 struct ecore_spq_entry *p_ent)
234 {
235 p_ent->flags = 0;
236
237 switch (p_ent->comp_mode) {
238 case ECORE_SPQ_MODE_EBLOCK:
239 case ECORE_SPQ_MODE_BLOCK:
240 p_ent->comp_cb.function = ecore_spq_blocking_cb;
241 break;
242 case ECORE_SPQ_MODE_CB:
243 break;
244 default:
245 DP_NOTICE(p_hwfn, true, "Unknown SPQE completion mode %d\n",
246 p_ent->comp_mode);
247 return ECORE_INVAL;
248 }
249
250 DP_VERBOSE(p_hwfn, ECORE_MSG_SPQ,
251 "Ramrod header: [CID 0x%08x CMD 0x%02x protocol 0x%02x] Data pointer: [%08x:%08x] Completion Mode: %s\n",
252 p_ent->elem.hdr.cid, p_ent->elem.hdr.cmd_id,
253 p_ent->elem.hdr.protocol_id,
254 p_ent->elem.data_ptr.hi, p_ent->elem.data_ptr.lo,
255 D_TRINE(p_ent->comp_mode, ECORE_SPQ_MODE_EBLOCK,
256 ECORE_SPQ_MODE_BLOCK, "MODE_EBLOCK", "MODE_BLOCK",
257 "MODE_CB"));
258
259 return ECORE_SUCCESS;
260 }
261
262 /***************************************************************************
263 * HSI access
264 ***************************************************************************/
ecore_spq_hw_initialize(struct ecore_hwfn * p_hwfn,struct ecore_spq * p_spq)265 static void ecore_spq_hw_initialize(struct ecore_hwfn *p_hwfn,
266 struct ecore_spq *p_spq)
267 {
268 struct e4_core_conn_context *p_cxt;
269 struct ecore_cxt_info cxt_info;
270 u16 physical_q;
271 enum _ecore_status_t rc;
272
273 cxt_info.iid = p_spq->cid;
274
275 rc = ecore_cxt_get_cid_info(p_hwfn, &cxt_info);
276
277 if (rc < 0) {
278 DP_NOTICE(p_hwfn, true, "Cannot find context info for cid=%d\n",
279 p_spq->cid);
280 return;
281 }
282
283 p_cxt = cxt_info.p_cxt;
284
285 /* @@@TBD we zero the context until we have ilt_reset implemented. */
286 OSAL_MEM_ZERO(p_cxt, sizeof(*p_cxt));
287
288 if (ECORE_IS_BB(p_hwfn->p_dev) || ECORE_IS_AH(p_hwfn->p_dev)) {
289 SET_FIELD(p_cxt->xstorm_ag_context.flags10,
290 E4_XSTORM_CORE_CONN_AG_CTX_DQ_CF_EN, 1);
291 SET_FIELD(p_cxt->xstorm_ag_context.flags1,
292 E4_XSTORM_CORE_CONN_AG_CTX_DQ_CF_ACTIVE, 1);
293 /*SET_FIELD(p_cxt->xstorm_ag_context.flags10,
294 E4_XSTORM_CORE_CONN_AG_CTX_SLOW_PATH_EN, 1);*/
295 SET_FIELD(p_cxt->xstorm_ag_context.flags9,
296 E4_XSTORM_CORE_CONN_AG_CTX_CONSOLID_PROD_CF_EN, 1);
297 } else { /* E5 */
298 ECORE_E5_MISSING_CODE;
299 }
300
301 /* CDU validation - FIXME currently disabled */
302
303 /* QM physical queue */
304 physical_q = ecore_get_cm_pq_idx(p_hwfn, PQ_FLAGS_LB);
305 p_cxt->xstorm_ag_context.physical_q0 = OSAL_CPU_TO_LE16(physical_q);
306
307 p_cxt->xstorm_st_context.spq_base_lo =
308 DMA_LO_LE(p_spq->chain.p_phys_addr);
309 p_cxt->xstorm_st_context.spq_base_hi =
310 DMA_HI_LE(p_spq->chain.p_phys_addr);
311
312 DMA_REGPAIR_LE(p_cxt->xstorm_st_context.consolid_base_addr,
313 p_hwfn->p_consq->chain.p_phys_addr);
314 }
315
ecore_spq_hw_post(struct ecore_hwfn * p_hwfn,struct ecore_spq * p_spq,struct ecore_spq_entry * p_ent)316 static enum _ecore_status_t ecore_spq_hw_post(struct ecore_hwfn *p_hwfn,
317 struct ecore_spq *p_spq,
318 struct ecore_spq_entry *p_ent)
319 {
320 struct ecore_chain *p_chain = &p_hwfn->p_spq->chain;
321 struct core_db_data *p_db_data = &p_spq->db_data;
322 u16 echo = ecore_chain_get_prod_idx(p_chain);
323 struct slow_path_element *elem;
324
325 p_ent->elem.hdr.echo = OSAL_CPU_TO_LE16(echo);
326 elem = ecore_chain_produce(p_chain);
327 if (!elem) {
328 DP_NOTICE(p_hwfn, true, "Failed to produce from SPQ chain\n");
329 return ECORE_INVAL;
330 }
331
332 *elem = p_ent->elem; /* Struct assignment */
333
334 p_db_data->spq_prod =
335 OSAL_CPU_TO_LE16(ecore_chain_get_prod_idx(p_chain));
336
337 /* Make sure the SPQE is updated before the doorbell */
338 OSAL_WMB(p_hwfn->p_dev);
339
340 DOORBELL(p_hwfn, p_spq->db_addr_offset, *(u32 *)p_db_data);
341
342 /* Make sure doorbell was rung */
343 OSAL_WMB(p_hwfn->p_dev);
344
345 DP_VERBOSE(p_hwfn, ECORE_MSG_SPQ,
346 "Doorbelled [0x%08x, CID 0x%08x] with Flags: %02x agg_params: %02x, prod: %04x\n",
347 p_spq->db_addr_offset, p_spq->cid, p_db_data->params,
348 p_db_data->agg_flags, ecore_chain_get_prod_idx(p_chain));
349
350 return ECORE_SUCCESS;
351 }
352
353 /***************************************************************************
354 * Asynchronous events
355 ***************************************************************************/
356
357 static enum _ecore_status_t
ecore_async_event_completion(struct ecore_hwfn * p_hwfn,struct event_ring_entry * p_eqe)358 ecore_async_event_completion(struct ecore_hwfn *p_hwfn,
359 struct event_ring_entry *p_eqe)
360 {
361 ecore_spq_async_comp_cb cb;
362
363 if (!p_hwfn->p_spq || (p_eqe->protocol_id >= MAX_PROTOCOL_TYPE)) {
364 return ECORE_INVAL;
365 }
366
367 cb = p_hwfn->p_spq->async_comp_cb[p_eqe->protocol_id];
368 if (cb) {
369 return cb(p_hwfn, p_eqe->opcode, p_eqe->echo,
370 &p_eqe->data, p_eqe->fw_return_code);
371 } else {
372 DP_NOTICE(p_hwfn,
373 true, "Unknown Async completion for protocol: %d\n",
374 p_eqe->protocol_id);
375 return ECORE_INVAL;
376 }
377 }
378
379 enum _ecore_status_t
ecore_spq_register_async_cb(struct ecore_hwfn * p_hwfn,enum protocol_type protocol_id,ecore_spq_async_comp_cb cb)380 ecore_spq_register_async_cb(struct ecore_hwfn *p_hwfn,
381 enum protocol_type protocol_id,
382 ecore_spq_async_comp_cb cb)
383 {
384 if (!p_hwfn->p_spq || (protocol_id >= MAX_PROTOCOL_TYPE)) {
385 return ECORE_INVAL;
386 }
387
388 p_hwfn->p_spq->async_comp_cb[protocol_id] = cb;
389 return ECORE_SUCCESS;
390 }
391
392 void
ecore_spq_unregister_async_cb(struct ecore_hwfn * p_hwfn,enum protocol_type protocol_id)393 ecore_spq_unregister_async_cb(struct ecore_hwfn *p_hwfn,
394 enum protocol_type protocol_id)
395 {
396 if (!p_hwfn->p_spq || (protocol_id >= MAX_PROTOCOL_TYPE)) {
397 return;
398 }
399
400 p_hwfn->p_spq->async_comp_cb[protocol_id] = OSAL_NULL;
401 }
402
403 /***************************************************************************
404 * EQ API
405 ***************************************************************************/
ecore_eq_prod_update(struct ecore_hwfn * p_hwfn,u16 prod)406 void ecore_eq_prod_update(struct ecore_hwfn *p_hwfn,
407 u16 prod)
408 {
409 u32 addr = GTT_BAR0_MAP_REG_USDM_RAM +
410 USTORM_EQE_CONS_OFFSET(p_hwfn->rel_pf_id);
411
412 REG_WR16(p_hwfn, addr, prod);
413
414 /* keep prod updates ordered */
415 OSAL_MMIOWB(p_hwfn->p_dev);
416 }
417
ecore_eq_completion(struct ecore_hwfn * p_hwfn,void * cookie)418 enum _ecore_status_t ecore_eq_completion(struct ecore_hwfn *p_hwfn,
419 void *cookie)
420
421 {
422 struct ecore_eq *p_eq = cookie;
423 struct ecore_chain *p_chain = &p_eq->chain;
424 enum _ecore_status_t rc = 0;
425
426 /* take a snapshot of the FW consumer */
427 u16 fw_cons_idx = OSAL_LE16_TO_CPU(*p_eq->p_fw_cons);
428
429 DP_VERBOSE(p_hwfn, ECORE_MSG_SPQ, "fw_cons_idx %x\n", fw_cons_idx);
430
431 /* Need to guarantee the fw_cons index we use points to a usuable
432 * element (to comply with our chain), so our macros would comply
433 */
434 if ((fw_cons_idx & ecore_chain_get_usable_per_page(p_chain)) ==
435 ecore_chain_get_usable_per_page(p_chain)) {
436 fw_cons_idx += ecore_chain_get_unusable_per_page(p_chain);
437 }
438
439 /* Complete current segment of eq entries */
440 while (fw_cons_idx != ecore_chain_get_cons_idx(p_chain)) {
441 struct event_ring_entry *p_eqe = ecore_chain_consume(p_chain);
442 if (!p_eqe) {
443 rc = ECORE_INVAL;
444 break;
445 }
446
447 DP_VERBOSE(p_hwfn,
448 ECORE_MSG_SPQ,
449 "op %x prot %x res0 %x echo %x fwret %x flags %x\n",
450 p_eqe->opcode, /* Event Opcode */
451 p_eqe->protocol_id, /* Event Protocol ID */
452 p_eqe->reserved0, /* Reserved */
453 OSAL_LE16_TO_CPU(p_eqe->echo),/* Echo value from
454 ramrod data on the host
455 */
456 p_eqe->fw_return_code, /* FW return code for SP
457 ramrods
458 */
459 p_eqe->flags);
460 #ifndef REMOVE_DBG
461 if (p_eqe->protocol_id == PROTOCOLID_ISCSI)
462 ecore_iscsi_eq_dump(p_hwfn, p_eqe);
463 #endif
464
465 if (GET_FIELD(p_eqe->flags, EVENT_RING_ENTRY_ASYNC)) {
466 if (ecore_async_event_completion(p_hwfn, p_eqe))
467 rc = ECORE_INVAL;
468 } else if (ecore_spq_completion(p_hwfn,
469 p_eqe->echo,
470 p_eqe->fw_return_code,
471 &p_eqe->data)) {
472 rc = ECORE_INVAL;
473 }
474
475 ecore_chain_recycle_consumed(p_chain);
476 }
477
478 ecore_eq_prod_update(p_hwfn, ecore_chain_get_prod_idx(p_chain));
479
480 /* Attempt to post pending requests */
481 OSAL_SPIN_LOCK(&p_hwfn->p_spq->lock);
482 rc = ecore_spq_pend_post(p_hwfn);
483 OSAL_SPIN_UNLOCK(&p_hwfn->p_spq->lock);
484
485 return rc;
486 }
487
ecore_eq_alloc(struct ecore_hwfn * p_hwfn,u16 num_elem)488 enum _ecore_status_t ecore_eq_alloc(struct ecore_hwfn *p_hwfn, u16 num_elem)
489 {
490 struct ecore_eq *p_eq;
491
492 /* Allocate EQ struct */
493 p_eq = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, sizeof(*p_eq));
494 if (!p_eq) {
495 DP_NOTICE(p_hwfn, false,
496 "Failed to allocate `struct ecore_eq'\n");
497 return ECORE_NOMEM;
498 }
499
500 /* Allocate and initialize EQ chain*/
501 if (ecore_chain_alloc(p_hwfn->p_dev,
502 ECORE_CHAIN_USE_TO_PRODUCE,
503 ECORE_CHAIN_MODE_PBL,
504 ECORE_CHAIN_CNT_TYPE_U16,
505 num_elem,
506 sizeof(union event_ring_element),
507 &p_eq->chain, OSAL_NULL) != ECORE_SUCCESS) {
508 DP_NOTICE(p_hwfn, false, "Failed to allocate eq chain\n");
509 goto eq_allocate_fail;
510 }
511
512 /* register EQ completion on the SP SB */
513 ecore_int_register_cb(p_hwfn, ecore_eq_completion,
514 p_eq, &p_eq->eq_sb_index, &p_eq->p_fw_cons);
515
516 p_hwfn->p_eq = p_eq;
517 return ECORE_SUCCESS;
518
519 eq_allocate_fail:
520 OSAL_FREE(p_hwfn->p_dev, p_eq);
521 return ECORE_NOMEM;
522 }
523
ecore_eq_setup(struct ecore_hwfn * p_hwfn)524 void ecore_eq_setup(struct ecore_hwfn *p_hwfn)
525 {
526 ecore_chain_reset(&p_hwfn->p_eq->chain);
527 }
528
ecore_eq_free(struct ecore_hwfn * p_hwfn)529 void ecore_eq_free(struct ecore_hwfn *p_hwfn)
530 {
531 if (!p_hwfn->p_eq)
532 return;
533
534 ecore_chain_free(p_hwfn->p_dev, &p_hwfn->p_eq->chain);
535
536 OSAL_FREE(p_hwfn->p_dev, p_hwfn->p_eq);
537 p_hwfn->p_eq = OSAL_NULL;
538 }
539
540 /***************************************************************************
541 * CQE API - manipulate EQ functionallity
542 ***************************************************************************/
ecore_cqe_completion(struct ecore_hwfn * p_hwfn,struct eth_slow_path_rx_cqe * cqe,enum protocol_type protocol)543 static enum _ecore_status_t ecore_cqe_completion(struct ecore_hwfn *p_hwfn,
544 struct eth_slow_path_rx_cqe *cqe,
545 enum protocol_type protocol)
546 {
547 if (IS_VF(p_hwfn->p_dev))
548 return OSAL_VF_CQE_COMPLETION(p_hwfn, cqe, protocol);
549
550 /* @@@tmp - it's possible we'll eventually want to handle some
551 * actual commands that can arrive here, but for now this is only
552 * used to complete the ramrod using the echo value on the cqe
553 */
554 return ecore_spq_completion(p_hwfn, cqe->echo, 0, OSAL_NULL);
555 }
556
ecore_eth_cqe_completion(struct ecore_hwfn * p_hwfn,struct eth_slow_path_rx_cqe * cqe)557 enum _ecore_status_t ecore_eth_cqe_completion(struct ecore_hwfn *p_hwfn,
558 struct eth_slow_path_rx_cqe *cqe)
559 {
560 enum _ecore_status_t rc;
561
562 rc = ecore_cqe_completion(p_hwfn, cqe, PROTOCOLID_ETH);
563 if (rc) {
564 DP_NOTICE(p_hwfn, true,
565 "Failed to handle RXQ CQE [cmd 0x%02x]\n",
566 cqe->ramrod_cmd_id);
567 }
568
569 return rc;
570 }
571
572 /***************************************************************************
573 * Slow hwfn Queue (spq)
574 ***************************************************************************/
ecore_spq_setup(struct ecore_hwfn * p_hwfn)575 void ecore_spq_setup(struct ecore_hwfn *p_hwfn)
576 {
577 struct ecore_spq *p_spq = p_hwfn->p_spq;
578 struct ecore_spq_entry *p_virt = OSAL_NULL;
579 struct core_db_data *p_db_data;
580 void OSAL_IOMEM *db_addr;
581 dma_addr_t p_phys = 0;
582 u32 i, capacity;
583 enum _ecore_status_t rc;
584
585 OSAL_LIST_INIT(&p_spq->pending);
586 OSAL_LIST_INIT(&p_spq->completion_pending);
587 OSAL_LIST_INIT(&p_spq->free_pool);
588 OSAL_LIST_INIT(&p_spq->unlimited_pending);
589 OSAL_SPIN_LOCK_INIT(&p_spq->lock);
590
591 /* SPQ empty pool */
592 p_phys = p_spq->p_phys + OFFSETOF(struct ecore_spq_entry, ramrod);
593 p_virt = p_spq->p_virt;
594
595 capacity = ecore_chain_get_capacity(&p_spq->chain);
596 for (i = 0; i < capacity; i++) {
597 DMA_REGPAIR_LE(p_virt->elem.data_ptr, p_phys);
598
599 OSAL_LIST_PUSH_TAIL(&p_virt->list, &p_spq->free_pool);
600
601 p_virt++;
602 p_phys += sizeof(struct ecore_spq_entry);
603 }
604
605 /* Statistics */
606 p_spq->normal_count = 0;
607 p_spq->comp_count = 0;
608 p_spq->comp_sent_count = 0;
609 p_spq->unlimited_pending_count = 0;
610
611 OSAL_MEM_ZERO(p_spq->p_comp_bitmap,
612 SPQ_COMP_BMAP_SIZE * sizeof(unsigned long));
613 p_spq->comp_bitmap_idx = 0;
614
615 /* SPQ cid, cannot fail */
616 ecore_cxt_acquire_cid(p_hwfn, PROTOCOLID_CORE, &p_spq->cid);
617 ecore_spq_hw_initialize(p_hwfn, p_spq);
618
619 /* reset the chain itself */
620 ecore_chain_reset(&p_spq->chain);
621
622 /* Initialize the address/data of the SPQ doorbell */
623 p_spq->db_addr_offset = DB_ADDR(p_spq->cid, DQ_DEMS_LEGACY);
624 p_db_data = &p_spq->db_data;
625 OSAL_MEM_ZERO(p_db_data, sizeof(*p_db_data));
626 SET_FIELD(p_db_data->params, CORE_DB_DATA_DEST, DB_DEST_XCM);
627 SET_FIELD(p_db_data->params, CORE_DB_DATA_AGG_CMD, DB_AGG_CMD_MAX);
628 SET_FIELD(p_db_data->params, CORE_DB_DATA_AGG_VAL_SEL,
629 DQ_XCM_CORE_SPQ_PROD_CMD);
630 p_db_data->agg_flags = DQ_XCM_CORE_DQ_CF_CMD;
631
632 /* Register the SPQ doorbell with the doorbell recovery mechanism */
633 db_addr = (void *)((u8 *)p_hwfn->doorbells + p_spq->db_addr_offset);
634 rc = ecore_db_recovery_add(p_hwfn->p_dev, db_addr, &p_spq->db_data,
635 DB_REC_WIDTH_32B, DB_REC_KERNEL);
636 if (rc != ECORE_SUCCESS)
637 DP_INFO(p_hwfn,
638 "Failed to register the SPQ doorbell with the doorbell recovery mechanism\n");
639 }
640
ecore_spq_alloc(struct ecore_hwfn * p_hwfn)641 enum _ecore_status_t ecore_spq_alloc(struct ecore_hwfn *p_hwfn)
642 {
643 struct ecore_spq_entry *p_virt = OSAL_NULL;
644 struct ecore_spq *p_spq = OSAL_NULL;
645 dma_addr_t p_phys = 0;
646 u32 capacity;
647
648 /* SPQ struct */
649 p_spq =
650 OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, sizeof(struct ecore_spq));
651 if (!p_spq) {
652 DP_NOTICE(p_hwfn, false, "Failed to allocate `struct ecore_spq'\n");
653 return ECORE_NOMEM;
654 }
655
656 /* SPQ ring */
657 if (ecore_chain_alloc(p_hwfn->p_dev,
658 ECORE_CHAIN_USE_TO_PRODUCE,
659 ECORE_CHAIN_MODE_SINGLE,
660 ECORE_CHAIN_CNT_TYPE_U16,
661 0, /* N/A when the mode is SINGLE */
662 sizeof(struct slow_path_element),
663 &p_spq->chain, OSAL_NULL)) {
664 DP_NOTICE(p_hwfn, false, "Failed to allocate spq chain\n");
665 goto spq_allocate_fail;
666 }
667
668 /* allocate and fill the SPQ elements (incl. ramrod data list) */
669 capacity = ecore_chain_get_capacity(&p_spq->chain);
670 p_virt = OSAL_DMA_ALLOC_COHERENT(p_hwfn->p_dev, &p_phys,
671 capacity *
672 sizeof(struct ecore_spq_entry));
673 if (!p_virt) {
674 goto spq_allocate_fail;
675 }
676
677 p_spq->p_virt = p_virt;
678 p_spq->p_phys = p_phys;
679
680 #ifdef CONFIG_ECORE_LOCK_ALLOC
681 if (OSAL_SPIN_LOCK_ALLOC(p_hwfn, &p_spq->lock))
682 goto spq_allocate_fail;
683 #endif
684
685 p_hwfn->p_spq = p_spq;
686 return ECORE_SUCCESS;
687
688 spq_allocate_fail:
689 ecore_chain_free(p_hwfn->p_dev, &p_spq->chain);
690 OSAL_FREE(p_hwfn->p_dev, p_spq);
691 return ECORE_NOMEM;
692 }
693
ecore_spq_free(struct ecore_hwfn * p_hwfn)694 void ecore_spq_free(struct ecore_hwfn *p_hwfn)
695 {
696 struct ecore_spq *p_spq = p_hwfn->p_spq;
697 void OSAL_IOMEM *db_addr;
698 u32 capacity;
699
700 if (!p_spq)
701 return;
702
703 /* Delete the SPQ doorbell from the doorbell recovery mechanism */
704 db_addr = (void *)((u8 *)p_hwfn->doorbells + p_spq->db_addr_offset);
705 ecore_db_recovery_del(p_hwfn->p_dev, db_addr, &p_spq->db_data);
706
707 if (p_spq->p_virt) {
708 capacity = ecore_chain_get_capacity(&p_spq->chain);
709 OSAL_DMA_FREE_COHERENT(p_hwfn->p_dev,
710 p_spq->p_virt,
711 p_spq->p_phys,
712 capacity *
713 sizeof(struct ecore_spq_entry));
714 }
715
716 ecore_chain_free(p_hwfn->p_dev, &p_spq->chain);
717 #ifdef CONFIG_ECORE_LOCK_ALLOC
718 OSAL_SPIN_LOCK_DEALLOC(&p_spq->lock);
719 #endif
720
721 OSAL_FREE(p_hwfn->p_dev, p_spq);
722 p_hwfn->p_spq = OSAL_NULL;
723 }
724
ecore_spq_get_entry(struct ecore_hwfn * p_hwfn,struct ecore_spq_entry ** pp_ent)725 enum _ecore_status_t ecore_spq_get_entry(struct ecore_hwfn *p_hwfn,
726 struct ecore_spq_entry **pp_ent)
727 {
728 struct ecore_spq *p_spq = p_hwfn->p_spq;
729 struct ecore_spq_entry *p_ent = OSAL_NULL;
730 enum _ecore_status_t rc = ECORE_SUCCESS;
731
732 OSAL_SPIN_LOCK(&p_spq->lock);
733
734 if (OSAL_LIST_IS_EMPTY(&p_spq->free_pool)) {
735 p_ent = OSAL_ZALLOC(p_hwfn->p_dev, GFP_ATOMIC, sizeof(*p_ent));
736 if (!p_ent) {
737 DP_NOTICE(p_hwfn, false, "Failed to allocate an SPQ entry for a pending ramrod\n");
738 rc = ECORE_NOMEM;
739 goto out_unlock;
740 }
741 p_ent->queue = &p_spq->unlimited_pending;
742 } else {
743 p_ent = OSAL_LIST_FIRST_ENTRY(&p_spq->free_pool,
744 struct ecore_spq_entry,
745 list);
746 OSAL_LIST_REMOVE_ENTRY(&p_ent->list, &p_spq->free_pool);
747 p_ent->queue = &p_spq->pending;
748 }
749
750 *pp_ent = p_ent;
751
752 out_unlock:
753 OSAL_SPIN_UNLOCK(&p_spq->lock);
754 return rc;
755 }
756
757 /* Locked variant; Should be called while the SPQ lock is taken */
__ecore_spq_return_entry(struct ecore_hwfn * p_hwfn,struct ecore_spq_entry * p_ent)758 static void __ecore_spq_return_entry(struct ecore_hwfn *p_hwfn,
759 struct ecore_spq_entry *p_ent)
760 {
761 OSAL_LIST_PUSH_TAIL(&p_ent->list, &p_hwfn->p_spq->free_pool);
762 }
763
ecore_spq_return_entry(struct ecore_hwfn * p_hwfn,struct ecore_spq_entry * p_ent)764 void ecore_spq_return_entry(struct ecore_hwfn *p_hwfn,
765 struct ecore_spq_entry *p_ent)
766 {
767 OSAL_SPIN_LOCK(&p_hwfn->p_spq->lock);
768 __ecore_spq_return_entry(p_hwfn, p_ent);
769 OSAL_SPIN_UNLOCK(&p_hwfn->p_spq->lock);
770 }
771
772 /**
773 * @brief ecore_spq_add_entry - adds a new entry to the pending
774 * list. Should be used while lock is being held.
775 *
776 * Addes an entry to the pending list is there is room (en empty
777 * element is avaliable in the free_pool), or else places the
778 * entry in the unlimited_pending pool.
779 *
780 * @param p_hwfn
781 * @param p_ent
782 * @param priority
783 *
784 * @return enum _ecore_status_t
785 */
ecore_spq_add_entry(struct ecore_hwfn * p_hwfn,struct ecore_spq_entry * p_ent,enum spq_priority priority)786 static enum _ecore_status_t ecore_spq_add_entry(struct ecore_hwfn *p_hwfn,
787 struct ecore_spq_entry *p_ent,
788 enum spq_priority priority)
789 {
790 struct ecore_spq *p_spq = p_hwfn->p_spq;
791
792 if (p_ent->queue == &p_spq->unlimited_pending) {
793 if (OSAL_LIST_IS_EMPTY(&p_spq->free_pool)) {
794 OSAL_LIST_PUSH_TAIL(&p_ent->list,
795 &p_spq->unlimited_pending);
796 p_spq->unlimited_pending_count++;
797
798 return ECORE_SUCCESS;
799
800 } else {
801 struct ecore_spq_entry *p_en2;
802
803 p_en2 = OSAL_LIST_FIRST_ENTRY(&p_spq->free_pool,
804 struct ecore_spq_entry,
805 list);
806 OSAL_LIST_REMOVE_ENTRY(&p_en2->list, &p_spq->free_pool);
807
808 /* Copy the ring element physical pointer to the new
809 * entry, since we are about to override the entire ring
810 * entry and don't want to lose the pointer.
811 */
812 p_ent->elem.data_ptr = p_en2->elem.data_ptr;
813
814 *p_en2 = *p_ent;
815
816 /* EBLOCK responsible to free the allocated p_ent */
817 if (p_ent->comp_mode != ECORE_SPQ_MODE_EBLOCK)
818 OSAL_FREE(p_hwfn->p_dev, p_ent);
819
820 p_ent = p_en2;
821 }
822 }
823
824 /* entry is to be placed in 'pending' queue */
825 switch (priority) {
826 case ECORE_SPQ_PRIORITY_NORMAL:
827 OSAL_LIST_PUSH_TAIL(&p_ent->list, &p_spq->pending);
828 p_spq->normal_count++;
829 break;
830 case ECORE_SPQ_PRIORITY_HIGH:
831 OSAL_LIST_PUSH_HEAD(&p_ent->list, &p_spq->pending);
832 p_spq->high_count++;
833 break;
834 default:
835 return ECORE_INVAL;
836 }
837
838 return ECORE_SUCCESS;
839 }
840
841 /***************************************************************************
842 * Accessor
843 ***************************************************************************/
844
ecore_spq_get_cid(struct ecore_hwfn * p_hwfn)845 u32 ecore_spq_get_cid(struct ecore_hwfn *p_hwfn)
846 {
847 if (!p_hwfn->p_spq) {
848 return 0xffffffff; /* illegal */
849 }
850 return p_hwfn->p_spq->cid;
851 }
852
853 /***************************************************************************
854 * Posting new Ramrods
855 ***************************************************************************/
856
ecore_spq_post_list(struct ecore_hwfn * p_hwfn,osal_list_t * head,u32 keep_reserve)857 static enum _ecore_status_t ecore_spq_post_list(struct ecore_hwfn *p_hwfn,
858 osal_list_t *head,
859 u32 keep_reserve)
860 {
861 struct ecore_spq *p_spq = p_hwfn->p_spq;
862 enum _ecore_status_t rc;
863
864 /* TODO - implementation might be wasteful; will always keep room
865 * for an additional high priority ramrod (even if one is already
866 * pending FW)
867 */
868 while (ecore_chain_get_elem_left(&p_spq->chain) > keep_reserve &&
869 !OSAL_LIST_IS_EMPTY(head)) {
870 struct ecore_spq_entry *p_ent =
871 OSAL_LIST_FIRST_ENTRY(head, struct ecore_spq_entry, list);
872 if (p_ent != OSAL_NULL) {
873 #if defined(_NTDDK_)
874 #pragma warning(suppress : 6011 28182)
875 #endif
876 OSAL_LIST_REMOVE_ENTRY(&p_ent->list, head);
877 OSAL_LIST_PUSH_TAIL(&p_ent->list, &p_spq->completion_pending);
878 p_spq->comp_sent_count++;
879
880 rc = ecore_spq_hw_post(p_hwfn, p_spq, p_ent);
881 if (rc) {
882 OSAL_LIST_REMOVE_ENTRY(&p_ent->list,
883 &p_spq->completion_pending);
884 __ecore_spq_return_entry(p_hwfn, p_ent);
885 return rc;
886 }
887 }
888 }
889
890 return ECORE_SUCCESS;
891 }
892
ecore_spq_pend_post(struct ecore_hwfn * p_hwfn)893 enum _ecore_status_t ecore_spq_pend_post(struct ecore_hwfn *p_hwfn)
894 {
895 struct ecore_spq *p_spq = p_hwfn->p_spq;
896 struct ecore_spq_entry *p_ent = OSAL_NULL;
897
898 while (!OSAL_LIST_IS_EMPTY(&p_spq->free_pool))
899 {
900 if (OSAL_LIST_IS_EMPTY(&p_spq->unlimited_pending))
901 break;
902
903 p_ent = OSAL_LIST_FIRST_ENTRY(&p_spq->unlimited_pending,
904 struct ecore_spq_entry,
905 list);
906 if (!p_ent)
907 return ECORE_INVAL;
908
909 #if defined(_NTDDK_)
910 #pragma warning(suppress : 6011)
911 #endif
912 OSAL_LIST_REMOVE_ENTRY(&p_ent->list, &p_spq->unlimited_pending);
913
914 ecore_spq_add_entry(p_hwfn, p_ent, p_ent->priority);
915 }
916
917 return ecore_spq_post_list(p_hwfn, &p_spq->pending,
918 SPQ_HIGH_PRI_RESERVE_DEFAULT);
919 }
920
ecore_spq_post(struct ecore_hwfn * p_hwfn,struct ecore_spq_entry * p_ent,u8 * fw_return_code)921 enum _ecore_status_t ecore_spq_post(struct ecore_hwfn *p_hwfn,
922 struct ecore_spq_entry *p_ent,
923 u8 *fw_return_code)
924 {
925 enum _ecore_status_t rc = ECORE_SUCCESS;
926 struct ecore_spq *p_spq = p_hwfn ? p_hwfn->p_spq : OSAL_NULL;
927 bool b_ret_ent = true;
928
929 if (!p_hwfn)
930 return ECORE_INVAL;
931
932 if (!p_ent) {
933 DP_NOTICE(p_hwfn, true, "Got a NULL pointer\n");
934 return ECORE_INVAL;
935 }
936
937 if (p_hwfn->p_dev->recov_in_prog) {
938 DP_VERBOSE(p_hwfn, ECORE_MSG_SPQ,
939 "Recovery is in progress -> skip spq post [cmd %02x protocol %02x]\n",
940 p_ent->elem.hdr.cmd_id, p_ent->elem.hdr.protocol_id);
941 /* Return success to let the flows to be completed successfully
942 * w/o any error handling.
943 */
944 return ECORE_SUCCESS;
945 }
946
947 OSAL_SPIN_LOCK(&p_spq->lock);
948
949 /* Complete the entry */
950 rc = ecore_spq_fill_entry(p_hwfn, p_ent);
951
952 /* Check return value after LOCK is taken for cleaner error flow */
953 if (rc)
954 goto spq_post_fail;
955
956 /* Add the request to the pending queue */
957 rc = ecore_spq_add_entry(p_hwfn, p_ent, p_ent->priority);
958 if (rc)
959 goto spq_post_fail;
960
961 rc = ecore_spq_pend_post(p_hwfn);
962 if (rc) {
963 /* Since it's possible that pending failed for a different
964 * entry [although unlikely], the failed entry was already
965 * dealt with; No need to return it here.
966 */
967 b_ret_ent = false;
968 goto spq_post_fail;
969 }
970
971 OSAL_SPIN_UNLOCK(&p_spq->lock);
972
973 if (p_ent->comp_mode == ECORE_SPQ_MODE_EBLOCK) {
974 /* For entries in ECORE BLOCK mode, the completion code cannot
975 * perform the necessary cleanup - if it did, we couldn't
976 * access p_ent here to see whether it's successful or not.
977 * Thus, after gaining the answer perform the cleanup here.
978 */
979 rc = ecore_spq_block(p_hwfn, p_ent, fw_return_code,
980 p_ent->queue == &p_spq->unlimited_pending);
981
982 if (p_ent->queue == &p_spq->unlimited_pending) {
983 /* This is an allocated p_ent which does not need to
984 * return to pool.
985 */
986 OSAL_FREE(p_hwfn->p_dev, p_ent);
987
988 /* TBD: handle error flow and remove p_ent from
989 * completion pending
990 */
991 return rc;
992 }
993
994 if (rc)
995 goto spq_post_fail2;
996
997 /* return to pool */
998 ecore_spq_return_entry(p_hwfn, p_ent);
999 }
1000 return rc;
1001
1002 spq_post_fail2:
1003 OSAL_SPIN_LOCK(&p_spq->lock);
1004 OSAL_LIST_REMOVE_ENTRY(&p_ent->list, &p_spq->completion_pending);
1005 ecore_chain_return_produced(&p_spq->chain);
1006
1007 spq_post_fail:
1008 /* return to the free pool */
1009 if (b_ret_ent)
1010 __ecore_spq_return_entry(p_hwfn, p_ent);
1011 OSAL_SPIN_UNLOCK(&p_spq->lock);
1012
1013 return rc;
1014 }
1015
ecore_spq_completion(struct ecore_hwfn * p_hwfn,__le16 echo,u8 fw_return_code,union event_ring_data * p_data)1016 enum _ecore_status_t ecore_spq_completion(struct ecore_hwfn *p_hwfn,
1017 __le16 echo,
1018 u8 fw_return_code,
1019 union event_ring_data *p_data)
1020 {
1021 struct ecore_spq *p_spq;
1022 struct ecore_spq_entry *p_ent = OSAL_NULL;
1023 struct ecore_spq_entry *tmp;
1024 struct ecore_spq_entry *found = OSAL_NULL;
1025
1026 if (!p_hwfn) {
1027 return ECORE_INVAL;
1028 }
1029
1030 p_spq = p_hwfn->p_spq;
1031 if (!p_spq) {
1032 return ECORE_INVAL;
1033 }
1034
1035 OSAL_SPIN_LOCK(&p_spq->lock);
1036 OSAL_LIST_FOR_EACH_ENTRY_SAFE(p_ent,
1037 tmp,
1038 &p_spq->completion_pending,
1039 list,
1040 struct ecore_spq_entry) {
1041 if (p_ent->elem.hdr.echo == echo) {
1042 OSAL_LIST_REMOVE_ENTRY(&p_ent->list,
1043 &p_spq->completion_pending);
1044
1045 /* Avoid overriding of SPQ entries when getting
1046 * out-of-order completions, by marking the completions
1047 * in a bitmap and increasing the chain consumer only
1048 * for the first successive completed entries.
1049 */
1050 SPQ_COMP_BMAP_SET_BIT(p_spq, echo);
1051 while (SPQ_COMP_BMAP_TEST_BIT(p_spq,
1052 p_spq->comp_bitmap_idx)) {
1053 SPQ_COMP_BMAP_CLEAR_BIT(p_spq,
1054 p_spq->comp_bitmap_idx);
1055 p_spq->comp_bitmap_idx++;
1056 ecore_chain_return_produced(&p_spq->chain);
1057 }
1058
1059 p_spq->comp_count++;
1060 found = p_ent;
1061 break;
1062 }
1063
1064 /* This is debug and should be relatively uncommon - depends
1065 * on scenarios which have mutliple per-PF sent ramrods.
1066 */
1067 DP_VERBOSE(p_hwfn, ECORE_MSG_SPQ,
1068 "Got completion for echo %04x - doesn't match echo %04x in completion pending list\n",
1069 OSAL_LE16_TO_CPU(echo),
1070 OSAL_LE16_TO_CPU(p_ent->elem.hdr.echo));
1071 }
1072
1073 /* Release lock before callback, as callback may post
1074 * an additional ramrod.
1075 */
1076 OSAL_SPIN_UNLOCK(&p_spq->lock);
1077
1078 if (!found) {
1079 DP_NOTICE(p_hwfn, true,
1080 "Failed to find an entry this EQE [echo %04x] completes\n",
1081 OSAL_LE16_TO_CPU(echo));
1082 return ECORE_EXISTS;
1083 }
1084
1085 DP_VERBOSE(p_hwfn, ECORE_MSG_SPQ,
1086 "Complete EQE [echo %04x]: func %p cookie %p)\n",
1087 OSAL_LE16_TO_CPU(echo),
1088 p_ent->comp_cb.function, p_ent->comp_cb.cookie);
1089 if (found->comp_cb.function)
1090 found->comp_cb.function(p_hwfn, found->comp_cb.cookie, p_data,
1091 fw_return_code);
1092 else
1093 DP_VERBOSE(p_hwfn, ECORE_MSG_SPQ, "Got a completion without a callback function\n");
1094
1095 if ((found->comp_mode != ECORE_SPQ_MODE_EBLOCK) ||
1096 (found->queue == &p_spq->unlimited_pending))
1097 /* EBLOCK is responsible for returning its own entry into the
1098 * free list, unless it originally added the entry into the
1099 * unlimited pending list.
1100 */
1101 ecore_spq_return_entry(p_hwfn, found);
1102
1103 return ECORE_SUCCESS;
1104 }
1105
ecore_consq_alloc(struct ecore_hwfn * p_hwfn)1106 enum _ecore_status_t ecore_consq_alloc(struct ecore_hwfn *p_hwfn)
1107 {
1108 struct ecore_consq *p_consq;
1109
1110 /* Allocate ConsQ struct */
1111 p_consq = OSAL_ZALLOC(p_hwfn->p_dev, GFP_KERNEL, sizeof(*p_consq));
1112 if (!p_consq) {
1113 DP_NOTICE(p_hwfn, false,
1114 "Failed to allocate `struct ecore_consq'\n");
1115 return ECORE_NOMEM;
1116 }
1117
1118 /* Allocate and initialize EQ chain*/
1119 if (ecore_chain_alloc(p_hwfn->p_dev,
1120 ECORE_CHAIN_USE_TO_PRODUCE,
1121 ECORE_CHAIN_MODE_PBL,
1122 ECORE_CHAIN_CNT_TYPE_U16,
1123 ECORE_CHAIN_PAGE_SIZE/0x80,
1124 0x80,
1125 &p_consq->chain, OSAL_NULL) != ECORE_SUCCESS) {
1126 DP_NOTICE(p_hwfn, false, "Failed to allocate consq chain");
1127 goto consq_allocate_fail;
1128 }
1129
1130 p_hwfn->p_consq = p_consq;
1131 return ECORE_SUCCESS;
1132
1133 consq_allocate_fail:
1134 OSAL_FREE(p_hwfn->p_dev, p_consq);
1135 return ECORE_NOMEM;
1136 }
1137
ecore_consq_setup(struct ecore_hwfn * p_hwfn)1138 void ecore_consq_setup(struct ecore_hwfn *p_hwfn)
1139 {
1140 ecore_chain_reset(&p_hwfn->p_consq->chain);
1141 }
1142
ecore_consq_free(struct ecore_hwfn * p_hwfn)1143 void ecore_consq_free(struct ecore_hwfn *p_hwfn)
1144 {
1145 if (!p_hwfn->p_consq)
1146 return;
1147
1148 ecore_chain_free(p_hwfn->p_dev, &p_hwfn->p_consq->chain);
1149
1150 OSAL_FREE(p_hwfn->p_dev, p_hwfn->p_consq);
1151 p_hwfn->p_consq = OSAL_NULL;
1152 }
1153
1154 #ifdef _NTDDK_
1155 #pragma warning(pop)
1156 #endif
1157