1 /* SPDX-License-Identifier: BSD-3-Clause */
2 /* Copyright (c) 2021, Intel Corporation
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 *
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * 3. Neither the name of the Intel Corporation nor the names of its
16 * contributors may be used to endorse or promote products derived from
17 * this software without specific prior written permission.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
23 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 #include "iavf_status.h"
33 #include "iavf_type.h"
34 #include "iavf_register.h"
35 #include "iavf_adminq.h"
36 #include "iavf_prototype.h"
37
38 /**
39 * iavf_adminq_init_regs - Initialize AdminQ registers
40 * @hw: pointer to the hardware structure
41 *
42 * This assumes the alloc_asq and alloc_arq functions have already been called
43 **/
iavf_adminq_init_regs(struct iavf_hw * hw)44 STATIC void iavf_adminq_init_regs(struct iavf_hw *hw)
45 {
46 /* set head and tail registers in our local struct */
47 hw->aq.asq.tail = IAVF_VF_ATQT1;
48 hw->aq.asq.head = IAVF_VF_ATQH1;
49 hw->aq.asq.len = IAVF_VF_ATQLEN1;
50 hw->aq.asq.bal = IAVF_VF_ATQBAL1;
51 hw->aq.asq.bah = IAVF_VF_ATQBAH1;
52 hw->aq.arq.tail = IAVF_VF_ARQT1;
53 hw->aq.arq.head = IAVF_VF_ARQH1;
54 hw->aq.arq.len = IAVF_VF_ARQLEN1;
55 hw->aq.arq.bal = IAVF_VF_ARQBAL1;
56 hw->aq.arq.bah = IAVF_VF_ARQBAH1;
57 }
58
59 /**
60 * iavf_alloc_adminq_asq_ring - Allocate Admin Queue send rings
61 * @hw: pointer to the hardware structure
62 **/
iavf_alloc_adminq_asq_ring(struct iavf_hw * hw)63 enum iavf_status iavf_alloc_adminq_asq_ring(struct iavf_hw *hw)
64 {
65 enum iavf_status ret_code;
66
67 ret_code = iavf_allocate_dma_mem(hw, &hw->aq.asq.desc_buf,
68 iavf_mem_atq_ring,
69 (hw->aq.num_asq_entries *
70 sizeof(struct iavf_aq_desc)),
71 IAVF_ADMINQ_DESC_ALIGNMENT);
72 if (ret_code)
73 return ret_code;
74
75 ret_code = iavf_allocate_virt_mem(hw, &hw->aq.asq.cmd_buf,
76 (hw->aq.num_asq_entries *
77 sizeof(struct iavf_asq_cmd_details)));
78 if (ret_code) {
79 iavf_free_dma_mem(hw, &hw->aq.asq.desc_buf);
80 return ret_code;
81 }
82
83 return ret_code;
84 }
85
86 /**
87 * iavf_alloc_adminq_arq_ring - Allocate Admin Queue receive rings
88 * @hw: pointer to the hardware structure
89 **/
iavf_alloc_adminq_arq_ring(struct iavf_hw * hw)90 enum iavf_status iavf_alloc_adminq_arq_ring(struct iavf_hw *hw)
91 {
92 enum iavf_status ret_code;
93
94 ret_code = iavf_allocate_dma_mem(hw, &hw->aq.arq.desc_buf,
95 iavf_mem_arq_ring,
96 (hw->aq.num_arq_entries *
97 sizeof(struct iavf_aq_desc)),
98 IAVF_ADMINQ_DESC_ALIGNMENT);
99
100 return ret_code;
101 }
102
103 /**
104 * iavf_free_adminq_asq - Free Admin Queue send rings
105 * @hw: pointer to the hardware structure
106 *
107 * This assumes the posted send buffers have already been cleaned
108 * and de-allocated
109 **/
iavf_free_adminq_asq(struct iavf_hw * hw)110 void iavf_free_adminq_asq(struct iavf_hw *hw)
111 {
112 iavf_free_virt_mem(hw, &hw->aq.asq.cmd_buf);
113 iavf_free_dma_mem(hw, &hw->aq.asq.desc_buf);
114 }
115
116 /**
117 * iavf_free_adminq_arq - Free Admin Queue receive rings
118 * @hw: pointer to the hardware structure
119 *
120 * This assumes the posted receive buffers have already been cleaned
121 * and de-allocated
122 **/
iavf_free_adminq_arq(struct iavf_hw * hw)123 void iavf_free_adminq_arq(struct iavf_hw *hw)
124 {
125 iavf_free_dma_mem(hw, &hw->aq.arq.desc_buf);
126 }
127
128 /**
129 * iavf_alloc_arq_bufs - Allocate pre-posted buffers for the receive queue
130 * @hw: pointer to the hardware structure
131 **/
iavf_alloc_arq_bufs(struct iavf_hw * hw)132 STATIC enum iavf_status iavf_alloc_arq_bufs(struct iavf_hw *hw)
133 {
134 enum iavf_status ret_code;
135 struct iavf_aq_desc *desc;
136 struct iavf_dma_mem *bi;
137 int i;
138
139 /* We'll be allocating the buffer info memory first, then we can
140 * allocate the mapped buffers for the event processing
141 */
142
143 /* buffer_info structures do not need alignment */
144 ret_code = iavf_allocate_virt_mem(hw, &hw->aq.arq.dma_head,
145 (hw->aq.num_arq_entries * sizeof(struct iavf_dma_mem)));
146 if (ret_code)
147 goto alloc_arq_bufs;
148 hw->aq.arq.r.arq_bi = (struct iavf_dma_mem *)hw->aq.arq.dma_head.va;
149
150 /* allocate the mapped buffers */
151 for (i = 0; i < hw->aq.num_arq_entries; i++) {
152 bi = &hw->aq.arq.r.arq_bi[i];
153 ret_code = iavf_allocate_dma_mem(hw, bi,
154 iavf_mem_arq_buf,
155 hw->aq.arq_buf_size,
156 IAVF_ADMINQ_DESC_ALIGNMENT);
157 if (ret_code)
158 goto unwind_alloc_arq_bufs;
159
160 /* now configure the descriptors for use */
161 desc = IAVF_ADMINQ_DESC(hw->aq.arq, i);
162
163 desc->flags = CPU_TO_LE16(IAVF_AQ_FLAG_BUF);
164 if (hw->aq.arq_buf_size > IAVF_AQ_LARGE_BUF)
165 desc->flags |= CPU_TO_LE16(IAVF_AQ_FLAG_LB);
166 desc->opcode = 0;
167 /* This is in accordance with Admin queue design, there is no
168 * register for buffer size configuration
169 */
170 desc->datalen = CPU_TO_LE16((u16)bi->size);
171 desc->retval = 0;
172 desc->cookie_high = 0;
173 desc->cookie_low = 0;
174 desc->params.external.addr_high =
175 CPU_TO_LE32(IAVF_HI_DWORD(bi->pa));
176 desc->params.external.addr_low =
177 CPU_TO_LE32(IAVF_LO_DWORD(bi->pa));
178 desc->params.external.param0 = 0;
179 desc->params.external.param1 = 0;
180 }
181
182 alloc_arq_bufs:
183 return ret_code;
184
185 unwind_alloc_arq_bufs:
186 /* don't try to free the one that failed... */
187 i--;
188 for (; i >= 0; i--)
189 iavf_free_dma_mem(hw, &hw->aq.arq.r.arq_bi[i]);
190 iavf_free_virt_mem(hw, &hw->aq.arq.dma_head);
191
192 return ret_code;
193 }
194
195 /**
196 * iavf_alloc_asq_bufs - Allocate empty buffer structs for the send queue
197 * @hw: pointer to the hardware structure
198 **/
iavf_alloc_asq_bufs(struct iavf_hw * hw)199 STATIC enum iavf_status iavf_alloc_asq_bufs(struct iavf_hw *hw)
200 {
201 enum iavf_status ret_code;
202 struct iavf_dma_mem *bi;
203 int i;
204
205 /* No mapped memory needed yet, just the buffer info structures */
206 ret_code = iavf_allocate_virt_mem(hw, &hw->aq.asq.dma_head,
207 (hw->aq.num_asq_entries * sizeof(struct iavf_dma_mem)));
208 if (ret_code)
209 goto alloc_asq_bufs;
210 hw->aq.asq.r.asq_bi = (struct iavf_dma_mem *)hw->aq.asq.dma_head.va;
211
212 /* allocate the mapped buffers */
213 for (i = 0; i < hw->aq.num_asq_entries; i++) {
214 bi = &hw->aq.asq.r.asq_bi[i];
215 ret_code = iavf_allocate_dma_mem(hw, bi,
216 iavf_mem_asq_buf,
217 hw->aq.asq_buf_size,
218 IAVF_ADMINQ_DESC_ALIGNMENT);
219 if (ret_code)
220 goto unwind_alloc_asq_bufs;
221 }
222 alloc_asq_bufs:
223 return ret_code;
224
225 unwind_alloc_asq_bufs:
226 /* don't try to free the one that failed... */
227 i--;
228 for (; i >= 0; i--)
229 iavf_free_dma_mem(hw, &hw->aq.asq.r.asq_bi[i]);
230 iavf_free_virt_mem(hw, &hw->aq.asq.dma_head);
231
232 return ret_code;
233 }
234
235 /**
236 * iavf_free_arq_bufs - Free receive queue buffer info elements
237 * @hw: pointer to the hardware structure
238 **/
iavf_free_arq_bufs(struct iavf_hw * hw)239 STATIC void iavf_free_arq_bufs(struct iavf_hw *hw)
240 {
241 int i;
242
243 /* free descriptors */
244 for (i = 0; i < hw->aq.num_arq_entries; i++)
245 iavf_free_dma_mem(hw, &hw->aq.arq.r.arq_bi[i]);
246
247 /* free the descriptor memory */
248 iavf_free_dma_mem(hw, &hw->aq.arq.desc_buf);
249
250 /* free the dma header */
251 iavf_free_virt_mem(hw, &hw->aq.arq.dma_head);
252 }
253
254 /**
255 * iavf_free_asq_bufs - Free send queue buffer info elements
256 * @hw: pointer to the hardware structure
257 **/
iavf_free_asq_bufs(struct iavf_hw * hw)258 STATIC void iavf_free_asq_bufs(struct iavf_hw *hw)
259 {
260 int i;
261
262 /* only unmap if the address is non-NULL */
263 for (i = 0; i < hw->aq.num_asq_entries; i++)
264 if (hw->aq.asq.r.asq_bi[i].pa)
265 iavf_free_dma_mem(hw, &hw->aq.asq.r.asq_bi[i]);
266
267 /* free the buffer info list */
268 iavf_free_virt_mem(hw, &hw->aq.asq.cmd_buf);
269
270 /* free the descriptor memory */
271 iavf_free_dma_mem(hw, &hw->aq.asq.desc_buf);
272
273 /* free the dma header */
274 iavf_free_virt_mem(hw, &hw->aq.asq.dma_head);
275 }
276
277 /**
278 * iavf_config_asq_regs - configure ASQ registers
279 * @hw: pointer to the hardware structure
280 *
281 * Configure base address and length registers for the transmit queue
282 **/
iavf_config_asq_regs(struct iavf_hw * hw)283 STATIC enum iavf_status iavf_config_asq_regs(struct iavf_hw *hw)
284 {
285 enum iavf_status ret_code = IAVF_SUCCESS;
286 u32 reg = 0;
287
288 /* Clear Head and Tail */
289 wr32(hw, hw->aq.asq.head, 0);
290 wr32(hw, hw->aq.asq.tail, 0);
291
292 /* set starting point */
293 wr32(hw, hw->aq.asq.len, (hw->aq.num_asq_entries |
294 IAVF_VF_ATQLEN1_ATQENABLE_MASK));
295 wr32(hw, hw->aq.asq.bal, IAVF_LO_DWORD(hw->aq.asq.desc_buf.pa));
296 wr32(hw, hw->aq.asq.bah, IAVF_HI_DWORD(hw->aq.asq.desc_buf.pa));
297
298 /* Check one register to verify that config was applied */
299 reg = rd32(hw, hw->aq.asq.bal);
300 if (reg != IAVF_LO_DWORD(hw->aq.asq.desc_buf.pa))
301 ret_code = IAVF_ERR_ADMIN_QUEUE_ERROR;
302
303 return ret_code;
304 }
305
306 /**
307 * iavf_config_arq_regs - ARQ register configuration
308 * @hw: pointer to the hardware structure
309 *
310 * Configure base address and length registers for the receive (event queue)
311 **/
iavf_config_arq_regs(struct iavf_hw * hw)312 STATIC enum iavf_status iavf_config_arq_regs(struct iavf_hw *hw)
313 {
314 enum iavf_status ret_code = IAVF_SUCCESS;
315 u32 reg = 0;
316
317 /* Clear Head and Tail */
318 wr32(hw, hw->aq.arq.head, 0);
319 wr32(hw, hw->aq.arq.tail, 0);
320
321 /* set starting point */
322 wr32(hw, hw->aq.arq.len, (hw->aq.num_arq_entries |
323 IAVF_VF_ARQLEN1_ARQENABLE_MASK));
324 wr32(hw, hw->aq.arq.bal, IAVF_LO_DWORD(hw->aq.arq.desc_buf.pa));
325 wr32(hw, hw->aq.arq.bah, IAVF_HI_DWORD(hw->aq.arq.desc_buf.pa));
326
327 /* Update tail in the HW to post pre-allocated buffers */
328 wr32(hw, hw->aq.arq.tail, hw->aq.num_arq_entries - 1);
329
330 /* Check one register to verify that config was applied */
331 reg = rd32(hw, hw->aq.arq.bal);
332 if (reg != IAVF_LO_DWORD(hw->aq.arq.desc_buf.pa))
333 ret_code = IAVF_ERR_ADMIN_QUEUE_ERROR;
334
335 return ret_code;
336 }
337
338 /**
339 * iavf_init_asq - main initialization routine for ASQ
340 * @hw: pointer to the hardware structure
341 *
342 * This is the main initialization routine for the Admin Send Queue
343 * Prior to calling this function, drivers *MUST* set the following fields
344 * in the hw->aq structure:
345 * - hw->aq.num_asq_entries
346 * - hw->aq.arq_buf_size
347 *
348 * Do *NOT* hold the lock when calling this as the memory allocation routines
349 * called are not going to be atomic context safe
350 **/
iavf_init_asq(struct iavf_hw * hw)351 enum iavf_status iavf_init_asq(struct iavf_hw *hw)
352 {
353 enum iavf_status ret_code = IAVF_SUCCESS;
354
355 if (hw->aq.asq.count > 0) {
356 /* queue already initialized */
357 ret_code = IAVF_ERR_NOT_READY;
358 goto init_adminq_exit;
359 }
360
361 /* verify input for valid configuration */
362 if ((hw->aq.num_asq_entries == 0) ||
363 (hw->aq.asq_buf_size == 0)) {
364 ret_code = IAVF_ERR_CONFIG;
365 goto init_adminq_exit;
366 }
367
368 hw->aq.asq.next_to_use = 0;
369 hw->aq.asq.next_to_clean = 0;
370
371 /* allocate the ring memory */
372 ret_code = iavf_alloc_adminq_asq_ring(hw);
373 if (ret_code != IAVF_SUCCESS)
374 goto init_adminq_exit;
375
376 /* allocate buffers in the rings */
377 ret_code = iavf_alloc_asq_bufs(hw);
378 if (ret_code != IAVF_SUCCESS)
379 goto init_adminq_free_rings;
380
381 /* initialize base registers */
382 ret_code = iavf_config_asq_regs(hw);
383 if (ret_code != IAVF_SUCCESS)
384 goto init_config_regs;
385
386 /* success! */
387 hw->aq.asq.count = hw->aq.num_asq_entries;
388 goto init_adminq_exit;
389
390 init_adminq_free_rings:
391 iavf_free_adminq_asq(hw);
392 return ret_code;
393
394 init_config_regs:
395 iavf_free_asq_bufs(hw);
396
397 init_adminq_exit:
398 return ret_code;
399 }
400
401 /**
402 * iavf_init_arq - initialize ARQ
403 * @hw: pointer to the hardware structure
404 *
405 * The main initialization routine for the Admin Receive (Event) Queue.
406 * Prior to calling this function, drivers *MUST* set the following fields
407 * in the hw->aq structure:
408 * - hw->aq.num_asq_entries
409 * - hw->aq.arq_buf_size
410 *
411 * Do *NOT* hold the lock when calling this as the memory allocation routines
412 * called are not going to be atomic context safe
413 **/
iavf_init_arq(struct iavf_hw * hw)414 enum iavf_status iavf_init_arq(struct iavf_hw *hw)
415 {
416 enum iavf_status ret_code = IAVF_SUCCESS;
417
418 if (hw->aq.arq.count > 0) {
419 /* queue already initialized */
420 ret_code = IAVF_ERR_NOT_READY;
421 goto init_adminq_exit;
422 }
423
424 /* verify input for valid configuration */
425 if ((hw->aq.num_arq_entries == 0) ||
426 (hw->aq.arq_buf_size == 0)) {
427 ret_code = IAVF_ERR_CONFIG;
428 goto init_adminq_exit;
429 }
430
431 hw->aq.arq.next_to_use = 0;
432 hw->aq.arq.next_to_clean = 0;
433
434 /* allocate the ring memory */
435 ret_code = iavf_alloc_adminq_arq_ring(hw);
436 if (ret_code != IAVF_SUCCESS)
437 goto init_adminq_exit;
438
439 /* allocate buffers in the rings */
440 ret_code = iavf_alloc_arq_bufs(hw);
441 if (ret_code != IAVF_SUCCESS)
442 goto init_adminq_free_rings;
443
444 /* initialize base registers */
445 ret_code = iavf_config_arq_regs(hw);
446 if (ret_code != IAVF_SUCCESS)
447 goto init_adminq_free_rings;
448
449 /* success! */
450 hw->aq.arq.count = hw->aq.num_arq_entries;
451 goto init_adminq_exit;
452
453 init_adminq_free_rings:
454 iavf_free_adminq_arq(hw);
455
456 init_adminq_exit:
457 return ret_code;
458 }
459
460 /**
461 * iavf_shutdown_asq - shutdown the ASQ
462 * @hw: pointer to the hardware structure
463 *
464 * The main shutdown routine for the Admin Send Queue
465 **/
iavf_shutdown_asq(struct iavf_hw * hw)466 enum iavf_status iavf_shutdown_asq(struct iavf_hw *hw)
467 {
468 enum iavf_status ret_code = IAVF_SUCCESS;
469
470 iavf_acquire_spinlock(&hw->aq.asq_spinlock);
471
472 if (hw->aq.asq.count == 0) {
473 ret_code = IAVF_ERR_NOT_READY;
474 goto shutdown_asq_out;
475 }
476
477 /* Stop firmware AdminQ processing */
478 wr32(hw, hw->aq.asq.head, 0);
479 wr32(hw, hw->aq.asq.tail, 0);
480 wr32(hw, hw->aq.asq.len, 0);
481 wr32(hw, hw->aq.asq.bal, 0);
482 wr32(hw, hw->aq.asq.bah, 0);
483
484 hw->aq.asq.count = 0; /* to indicate uninitialized queue */
485
486 /* free ring buffers */
487 iavf_free_asq_bufs(hw);
488
489 shutdown_asq_out:
490 iavf_release_spinlock(&hw->aq.asq_spinlock);
491 return ret_code;
492 }
493
494 /**
495 * iavf_shutdown_arq - shutdown ARQ
496 * @hw: pointer to the hardware structure
497 *
498 * The main shutdown routine for the Admin Receive Queue
499 **/
iavf_shutdown_arq(struct iavf_hw * hw)500 enum iavf_status iavf_shutdown_arq(struct iavf_hw *hw)
501 {
502 enum iavf_status ret_code = IAVF_SUCCESS;
503
504 iavf_acquire_spinlock(&hw->aq.arq_spinlock);
505
506 if (hw->aq.arq.count == 0) {
507 ret_code = IAVF_ERR_NOT_READY;
508 goto shutdown_arq_out;
509 }
510
511 /* Stop firmware AdminQ processing */
512 wr32(hw, hw->aq.arq.head, 0);
513 wr32(hw, hw->aq.arq.tail, 0);
514 wr32(hw, hw->aq.arq.len, 0);
515 wr32(hw, hw->aq.arq.bal, 0);
516 wr32(hw, hw->aq.arq.bah, 0);
517
518 hw->aq.arq.count = 0; /* to indicate uninitialized queue */
519
520 /* free ring buffers */
521 iavf_free_arq_bufs(hw);
522
523 shutdown_arq_out:
524 iavf_release_spinlock(&hw->aq.arq_spinlock);
525 return ret_code;
526 }
527
528 /**
529 * iavf_init_adminq - main initialization routine for Admin Queue
530 * @hw: pointer to the hardware structure
531 *
532 * Prior to calling this function, drivers *MUST* set the following fields
533 * in the hw->aq structure:
534 * - hw->aq.num_asq_entries
535 * - hw->aq.num_arq_entries
536 * - hw->aq.arq_buf_size
537 * - hw->aq.asq_buf_size
538 **/
iavf_init_adminq(struct iavf_hw * hw)539 enum iavf_status iavf_init_adminq(struct iavf_hw *hw)
540 {
541 enum iavf_status ret_code;
542
543 /* verify input for valid configuration */
544 if ((hw->aq.num_arq_entries == 0) ||
545 (hw->aq.num_asq_entries == 0) ||
546 (hw->aq.arq_buf_size == 0) ||
547 (hw->aq.asq_buf_size == 0)) {
548 ret_code = IAVF_ERR_CONFIG;
549 goto init_adminq_exit;
550 }
551 iavf_init_spinlock(&hw->aq.asq_spinlock);
552 iavf_init_spinlock(&hw->aq.arq_spinlock);
553
554 /* Set up register offsets */
555 iavf_adminq_init_regs(hw);
556
557 /* setup ASQ command write back timeout */
558 hw->aq.asq_cmd_timeout = IAVF_ASQ_CMD_TIMEOUT;
559
560 /* allocate the ASQ */
561 ret_code = iavf_init_asq(hw);
562 if (ret_code != IAVF_SUCCESS)
563 goto init_adminq_destroy_spinlocks;
564
565 /* allocate the ARQ */
566 ret_code = iavf_init_arq(hw);
567 if (ret_code != IAVF_SUCCESS)
568 goto init_adminq_free_asq;
569
570 /* success! */
571 goto init_adminq_exit;
572
573 init_adminq_free_asq:
574 iavf_shutdown_asq(hw);
575 init_adminq_destroy_spinlocks:
576 iavf_destroy_spinlock(&hw->aq.asq_spinlock);
577 iavf_destroy_spinlock(&hw->aq.arq_spinlock);
578
579 init_adminq_exit:
580 return ret_code;
581 }
582
583 /**
584 * iavf_shutdown_adminq - shutdown routine for the Admin Queue
585 * @hw: pointer to the hardware structure
586 **/
iavf_shutdown_adminq(struct iavf_hw * hw)587 enum iavf_status iavf_shutdown_adminq(struct iavf_hw *hw)
588 {
589 enum iavf_status ret_code = IAVF_SUCCESS;
590
591 if (iavf_check_asq_alive(hw))
592 iavf_aq_queue_shutdown(hw, true);
593
594 iavf_shutdown_asq(hw);
595 iavf_shutdown_arq(hw);
596 iavf_destroy_spinlock(&hw->aq.asq_spinlock);
597 iavf_destroy_spinlock(&hw->aq.arq_spinlock);
598
599 return ret_code;
600 }
601
602 /**
603 * iavf_clean_asq - cleans Admin send queue
604 * @hw: pointer to the hardware structure
605 *
606 * returns the number of free desc
607 **/
iavf_clean_asq(struct iavf_hw * hw)608 u16 iavf_clean_asq(struct iavf_hw *hw)
609 {
610 struct iavf_adminq_ring *asq = &(hw->aq.asq);
611 struct iavf_asq_cmd_details *details;
612 u16 ntc = asq->next_to_clean;
613 struct iavf_aq_desc desc_cb;
614 struct iavf_aq_desc *desc;
615
616 desc = IAVF_ADMINQ_DESC(*asq, ntc);
617 details = IAVF_ADMINQ_DETAILS(*asq, ntc);
618 while (rd32(hw, hw->aq.asq.head) != ntc) {
619 iavf_debug(hw, IAVF_DEBUG_AQ_MESSAGE,
620 "ntc %d head %d.\n", ntc, rd32(hw, hw->aq.asq.head));
621
622 if (details->callback) {
623 IAVF_ADMINQ_CALLBACK cb_func =
624 (IAVF_ADMINQ_CALLBACK)details->callback;
625 iavf_memcpy(&desc_cb, desc, sizeof(struct iavf_aq_desc),
626 IAVF_DMA_TO_DMA);
627 cb_func(hw, &desc_cb);
628 }
629 iavf_memset(desc, 0, sizeof(*desc), IAVF_DMA_MEM);
630 iavf_memset(details, 0, sizeof(*details), IAVF_NONDMA_MEM);
631 ntc++;
632 if (ntc == asq->count)
633 ntc = 0;
634 desc = IAVF_ADMINQ_DESC(*asq, ntc);
635 details = IAVF_ADMINQ_DETAILS(*asq, ntc);
636 }
637
638 asq->next_to_clean = ntc;
639
640 return IAVF_DESC_UNUSED(asq);
641 }
642
643 /**
644 * iavf_asq_done - check if FW has processed the Admin Send Queue
645 * @hw: pointer to the hw struct
646 *
647 * Returns true if the firmware has processed all descriptors on the
648 * admin send queue. Returns false if there are still requests pending.
649 **/
iavf_asq_done(struct iavf_hw * hw)650 bool iavf_asq_done(struct iavf_hw *hw)
651 {
652 /* AQ designers suggest use of head for better
653 * timing reliability than DD bit
654 */
655 return rd32(hw, hw->aq.asq.head) == hw->aq.asq.next_to_use;
656
657 }
658
659 /**
660 * iavf_asq_send_command - send command to Admin Queue
661 * @hw: pointer to the hw struct
662 * @desc: prefilled descriptor describing the command (non DMA mem)
663 * @buff: buffer to use for indirect commands
664 * @buff_size: size of buffer for indirect commands
665 * @cmd_details: pointer to command details structure
666 *
667 * This is the main send command driver routine for the Admin Queue send
668 * queue. It runs the queue, cleans the queue, etc
669 **/
iavf_asq_send_command(struct iavf_hw * hw,struct iavf_aq_desc * desc,void * buff,u16 buff_size,struct iavf_asq_cmd_details * cmd_details)670 enum iavf_status iavf_asq_send_command(struct iavf_hw *hw,
671 struct iavf_aq_desc *desc,
672 void *buff, /* can be NULL */
673 u16 buff_size,
674 struct iavf_asq_cmd_details *cmd_details)
675 {
676 enum iavf_status status = IAVF_SUCCESS;
677 struct iavf_dma_mem *dma_buff = NULL;
678 struct iavf_asq_cmd_details *details;
679 struct iavf_aq_desc *desc_on_ring;
680 bool cmd_completed = false;
681 u16 retval = 0;
682 u32 val = 0;
683
684 iavf_acquire_spinlock(&hw->aq.asq_spinlock);
685
686 hw->aq.asq_last_status = IAVF_AQ_RC_OK;
687
688 if (hw->aq.asq.count == 0) {
689 iavf_debug(hw, IAVF_DEBUG_AQ_MESSAGE,
690 "AQTX: Admin queue not initialized.\n");
691 status = IAVF_ERR_QUEUE_EMPTY;
692 goto asq_send_command_error;
693 }
694
695 val = rd32(hw, hw->aq.asq.head);
696 if (val >= hw->aq.num_asq_entries) {
697 iavf_debug(hw, IAVF_DEBUG_AQ_MESSAGE,
698 "AQTX: head overrun at %d\n", val);
699 status = IAVF_ERR_QUEUE_EMPTY;
700 goto asq_send_command_error;
701 }
702
703 details = IAVF_ADMINQ_DETAILS(hw->aq.asq, hw->aq.asq.next_to_use);
704 if (cmd_details) {
705 iavf_memcpy(details,
706 cmd_details,
707 sizeof(struct iavf_asq_cmd_details),
708 IAVF_NONDMA_TO_NONDMA);
709
710 /* If the cmd_details are defined copy the cookie. The
711 * CPU_TO_LE32 is not needed here because the data is ignored
712 * by the FW, only used by the driver
713 */
714 if (details->cookie) {
715 desc->cookie_high =
716 CPU_TO_LE32(IAVF_HI_DWORD(details->cookie));
717 desc->cookie_low =
718 CPU_TO_LE32(IAVF_LO_DWORD(details->cookie));
719 }
720 } else {
721 iavf_memset(details, 0,
722 sizeof(struct iavf_asq_cmd_details),
723 IAVF_NONDMA_MEM);
724 }
725
726 /* clear requested flags and then set additional flags if defined */
727 desc->flags &= ~CPU_TO_LE16(details->flags_dis);
728 desc->flags |= CPU_TO_LE16(details->flags_ena);
729
730 if (buff_size > hw->aq.asq_buf_size) {
731 iavf_debug(hw,
732 IAVF_DEBUG_AQ_MESSAGE,
733 "AQTX: Invalid buffer size: %d.\n",
734 buff_size);
735 status = IAVF_ERR_INVALID_SIZE;
736 goto asq_send_command_error;
737 }
738
739 if (details->postpone && !details->async) {
740 iavf_debug(hw,
741 IAVF_DEBUG_AQ_MESSAGE,
742 "AQTX: Async flag not set along with postpone flag");
743 status = IAVF_ERR_PARAM;
744 goto asq_send_command_error;
745 }
746
747 /* call clean and check queue available function to reclaim the
748 * descriptors that were processed by FW, the function returns the
749 * number of desc available
750 */
751 /* the clean function called here could be called in a separate thread
752 * in case of asynchronous completions
753 */
754 if (iavf_clean_asq(hw) == 0) {
755 iavf_debug(hw,
756 IAVF_DEBUG_AQ_MESSAGE,
757 "AQTX: Error queue is full.\n");
758 status = IAVF_ERR_ADMIN_QUEUE_FULL;
759 goto asq_send_command_error;
760 }
761
762 /* initialize the temp desc pointer with the right desc */
763 desc_on_ring = IAVF_ADMINQ_DESC(hw->aq.asq, hw->aq.asq.next_to_use);
764
765 /* if the desc is available copy the temp desc to the right place */
766 iavf_memcpy(desc_on_ring, desc, sizeof(struct iavf_aq_desc),
767 IAVF_NONDMA_TO_DMA);
768
769 /* if buff is not NULL assume indirect command */
770 if (buff != NULL) {
771 dma_buff = &(hw->aq.asq.r.asq_bi[hw->aq.asq.next_to_use]);
772 /* copy the user buff into the respective DMA buff */
773 iavf_memcpy(dma_buff->va, buff, buff_size,
774 IAVF_NONDMA_TO_DMA);
775 desc_on_ring->datalen = CPU_TO_LE16(buff_size);
776
777 /* Update the address values in the desc with the pa value
778 * for respective buffer
779 */
780 desc_on_ring->params.external.addr_high =
781 CPU_TO_LE32(IAVF_HI_DWORD(dma_buff->pa));
782 desc_on_ring->params.external.addr_low =
783 CPU_TO_LE32(IAVF_LO_DWORD(dma_buff->pa));
784 }
785
786 /* bump the tail */
787 iavf_debug(hw, IAVF_DEBUG_AQ_MESSAGE, "AQTX: desc and buffer:\n");
788 iavf_debug_aq(hw, IAVF_DEBUG_AQ_COMMAND, (void *)desc_on_ring,
789 buff, buff_size);
790 (hw->aq.asq.next_to_use)++;
791 if (hw->aq.asq.next_to_use == hw->aq.asq.count)
792 hw->aq.asq.next_to_use = 0;
793 if (!details->postpone)
794 wr32(hw, hw->aq.asq.tail, hw->aq.asq.next_to_use);
795
796 /* if cmd_details are not defined or async flag is not set,
797 * we need to wait for desc write back
798 */
799 if (!details->async && !details->postpone) {
800 u32 total_delay = 0;
801
802 do {
803 /* AQ designers suggest use of head for better
804 * timing reliability than DD bit
805 */
806 if (iavf_asq_done(hw))
807 break;
808 iavf_usec_delay(50);
809 total_delay += 50;
810 } while (total_delay < hw->aq.asq_cmd_timeout);
811 }
812
813 /* if ready, copy the desc back to temp */
814 if (iavf_asq_done(hw)) {
815 iavf_memcpy(desc, desc_on_ring, sizeof(struct iavf_aq_desc),
816 IAVF_DMA_TO_NONDMA);
817 if (buff != NULL)
818 iavf_memcpy(buff, dma_buff->va, buff_size,
819 IAVF_DMA_TO_NONDMA);
820 retval = LE16_TO_CPU(desc->retval);
821 if (retval != 0) {
822 iavf_debug(hw,
823 IAVF_DEBUG_AQ_MESSAGE,
824 "AQTX: Command completed with error 0x%X.\n",
825 retval);
826
827 /* strip off FW internal code */
828 retval &= 0xff;
829 }
830 cmd_completed = true;
831 if ((enum iavf_admin_queue_err)retval == IAVF_AQ_RC_OK)
832 status = IAVF_SUCCESS;
833 else if ((enum iavf_admin_queue_err)retval == IAVF_AQ_RC_EBUSY)
834 status = IAVF_ERR_NOT_READY;
835 else
836 status = IAVF_ERR_ADMIN_QUEUE_ERROR;
837 hw->aq.asq_last_status = (enum iavf_admin_queue_err)retval;
838 }
839
840 iavf_debug(hw, IAVF_DEBUG_AQ_MESSAGE,
841 "AQTX: desc and buffer writeback:\n");
842 iavf_debug_aq(hw, IAVF_DEBUG_AQ_COMMAND, (void *)desc, buff, buff_size);
843
844 /* save writeback aq if requested */
845 if (details->wb_desc)
846 iavf_memcpy(details->wb_desc, desc_on_ring,
847 sizeof(struct iavf_aq_desc), IAVF_DMA_TO_NONDMA);
848
849 /* update the error if time out occurred */
850 if ((!cmd_completed) &&
851 (!details->async && !details->postpone)) {
852 if (rd32(hw, hw->aq.asq.len) & IAVF_VF_ATQLEN1_ATQCRIT_MASK) {
853 iavf_debug(hw, IAVF_DEBUG_AQ_MESSAGE,
854 "AQTX: AQ Critical error.\n");
855 status = IAVF_ERR_ADMIN_QUEUE_CRITICAL_ERROR;
856 } else {
857 iavf_debug(hw, IAVF_DEBUG_AQ_MESSAGE,
858 "AQTX: Writeback timeout.\n");
859 status = IAVF_ERR_ADMIN_QUEUE_TIMEOUT;
860 }
861 }
862
863 asq_send_command_error:
864 iavf_release_spinlock(&hw->aq.asq_spinlock);
865 return status;
866 }
867
868 /**
869 * iavf_fill_default_direct_cmd_desc - AQ descriptor helper function
870 * @desc: pointer to the temp descriptor (non DMA mem)
871 * @opcode: the opcode can be used to decide which flags to turn off or on
872 *
873 * Fill the desc with default values
874 **/
iavf_fill_default_direct_cmd_desc(struct iavf_aq_desc * desc,u16 opcode)875 void iavf_fill_default_direct_cmd_desc(struct iavf_aq_desc *desc,
876 u16 opcode)
877 {
878 /* zero out the desc */
879 iavf_memset((void *)desc, 0, sizeof(struct iavf_aq_desc),
880 IAVF_NONDMA_MEM);
881 desc->opcode = CPU_TO_LE16(opcode);
882 desc->flags = CPU_TO_LE16(IAVF_AQ_FLAG_SI);
883 }
884
885 /**
886 * iavf_clean_arq_element
887 * @hw: pointer to the hw struct
888 * @e: event info from the receive descriptor, includes any buffers
889 * @pending: number of events that could be left to process
890 *
891 * This function cleans one Admin Receive Queue element and returns
892 * the contents through e. It can also return how many events are
893 * left to process through 'pending'
894 **/
iavf_clean_arq_element(struct iavf_hw * hw,struct iavf_arq_event_info * e,u16 * pending)895 enum iavf_status iavf_clean_arq_element(struct iavf_hw *hw,
896 struct iavf_arq_event_info *e,
897 u16 *pending)
898 {
899 enum iavf_status ret_code = IAVF_SUCCESS;
900 u16 ntc = hw->aq.arq.next_to_clean;
901 struct iavf_aq_desc *desc;
902 struct iavf_dma_mem *bi;
903 u16 desc_idx;
904 u16 datalen;
905 u16 flags;
906 u16 ntu;
907
908 /* pre-clean the event info */
909 iavf_memset(&e->desc, 0, sizeof(e->desc), IAVF_NONDMA_MEM);
910
911 /* take the lock before we start messing with the ring */
912 iavf_acquire_spinlock(&hw->aq.arq_spinlock);
913
914 if (hw->aq.arq.count == 0) {
915 iavf_debug(hw, IAVF_DEBUG_AQ_MESSAGE,
916 "AQRX: Admin queue not initialized.\n");
917 ret_code = IAVF_ERR_QUEUE_EMPTY;
918 goto clean_arq_element_err;
919 }
920
921 /* set next_to_use to head */
922 ntu = rd32(hw, hw->aq.arq.head) & IAVF_VF_ARQH1_ARQH_MASK;
923 if (ntu == ntc) {
924 /* nothing to do - shouldn't need to update ring's values */
925 ret_code = IAVF_ERR_ADMIN_QUEUE_NO_WORK;
926 goto clean_arq_element_out;
927 }
928
929 /* now clean the next descriptor */
930 desc = IAVF_ADMINQ_DESC(hw->aq.arq, ntc);
931 desc_idx = ntc;
932
933 hw->aq.arq_last_status =
934 (enum iavf_admin_queue_err)LE16_TO_CPU(desc->retval);
935 flags = LE16_TO_CPU(desc->flags);
936 if (flags & IAVF_AQ_FLAG_ERR) {
937 ret_code = IAVF_ERR_ADMIN_QUEUE_ERROR;
938 iavf_debug(hw,
939 IAVF_DEBUG_AQ_MESSAGE,
940 "AQRX: Event received with error 0x%X.\n",
941 hw->aq.arq_last_status);
942 }
943
944 iavf_memcpy(&e->desc, desc, sizeof(struct iavf_aq_desc),
945 IAVF_DMA_TO_NONDMA);
946 datalen = LE16_TO_CPU(desc->datalen);
947 e->msg_len = min(datalen, e->buf_len);
948 if (e->msg_buf != NULL && (e->msg_len != 0))
949 iavf_memcpy(e->msg_buf,
950 hw->aq.arq.r.arq_bi[desc_idx].va,
951 e->msg_len, IAVF_DMA_TO_NONDMA);
952
953 iavf_debug(hw, IAVF_DEBUG_AQ_MESSAGE, "AQRX: desc and buffer:\n");
954 iavf_debug_aq(hw, IAVF_DEBUG_AQ_COMMAND, (void *)desc, e->msg_buf,
955 hw->aq.arq_buf_size);
956
957 /* Restore the original datalen and buffer address in the desc,
958 * FW updates datalen to indicate the event message
959 * size
960 */
961 bi = &hw->aq.arq.r.arq_bi[ntc];
962 iavf_memset((void *)desc, 0, sizeof(struct iavf_aq_desc), IAVF_DMA_MEM);
963
964 desc->flags = CPU_TO_LE16(IAVF_AQ_FLAG_BUF);
965 if (hw->aq.arq_buf_size > IAVF_AQ_LARGE_BUF)
966 desc->flags |= CPU_TO_LE16(IAVF_AQ_FLAG_LB);
967 desc->datalen = CPU_TO_LE16((u16)bi->size);
968 desc->params.external.addr_high = CPU_TO_LE32(IAVF_HI_DWORD(bi->pa));
969 desc->params.external.addr_low = CPU_TO_LE32(IAVF_LO_DWORD(bi->pa));
970
971 /* set tail = the last cleaned desc index. */
972 wr32(hw, hw->aq.arq.tail, ntc);
973 /* ntc is updated to tail + 1 */
974 ntc++;
975 if (ntc == hw->aq.num_arq_entries)
976 ntc = 0;
977 hw->aq.arq.next_to_clean = ntc;
978 hw->aq.arq.next_to_use = ntu;
979
980 clean_arq_element_out:
981 /* Set pending if needed, unlock and return */
982 if (pending != NULL)
983 *pending = (ntc > ntu ? hw->aq.arq.count : 0) + (ntu - ntc);
984 clean_arq_element_err:
985 iavf_release_spinlock(&hw->aq.arq_spinlock);
986
987 return ret_code;
988 }
989
990