xref: /titanic_51/usr/src/uts/common/io/bnxe/577xx/drivers/common/lm/device/lm_er.c (revision d14abf155341d55053c76eeec58b787a456b753b)
1 /*******************************************************************************
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  *
21  * Copyright 2014 QLogic Corporation
22  * The contents of this file are subject to the terms of the
23  * QLogic End User License (the "License").
24  * You may not use this file except in compliance with the License.
25  *
26  * You can obtain a copy of the License at
27  * http://www.qlogic.com/Resources/Documents/DriverDownloadHelp/
28  * QLogic_End_User_Software_License.txt
29  * See the License for the specific language governing permissions
30  * and limitations under the License.
31  *
32  *
33  * Module Description:
34  *      This file encapsulates all the functions required to support error
35  *      recovery in E2. It is an isolated module that can be compiled in/out
36  *      according to error recovery support in upper layer
37  *
38  *
39  ******************************************************************************/
40 
41 #include "lm5710.h"
42 #include "aeu_inputs.h"
43 #include "lm_defs.h"
44 #include "general_atten_bits.h"
45 /**
46  * @Description
47  *      This function should be called to acquire the leader lock. the leader
48  *      lock should not be released until recovery process id done.
49  *      The leader lock is not waited for, its a non-blockinf function
50  *
51  * @param pdev
52  *
53  * @return lm_status_t SUCCESS or FAILURE
54  */
55 lm_status_t lm_er_acquire_leader_lock(lm_device_t * pdev)
56 {
57     return lm_hw_lock(pdev, HW_LOCK_RESOURCE_RECOVERY_LEADER_0, FALSE);
58 }
59 
60 /**
61  * @Description
62  *      release the lock acquired in the previous function
63  * @param pdev
64  *
65  * @return lm_status_t SUCCESS, INVALID_PARAM: if invalid input
66  *         is provided, LM_STATUS_OBJECT_NOT_FOUND if the lock
67  *         isn't taken.
68  */
69 lm_status_t lm_er_release_leader_lock(lm_device_t * pdev)
70 {
71     return lm_hw_unlock(pdev, HW_LOCK_RESOURCE_RECOVERY_LEADER_0);
72 }
73 
74 
75 /**
76  * @Description
77  *      This function disables close the gate functionality
78  *      should be called from the last driver that unloads
79  *      (unless recovery is in progress)
80  *
81  * @param pdev
82  */
83 void lm_er_disable_close_the_gate(lm_device_t *pdev)
84 {
85     u32_t val;
86 
87     DbgMessage(pdev, INFORMer, "Disabling \"close the gates\"\n");
88 
89     if (CHIP_IS_E2(pdev) || CHIP_IS_E3(pdev))
90     {
91         val = REG_RD(pdev, MISC_REG_AEU_GENERAL_MASK);
92         val &= ~(MISC_AEU_GENERAL_MASK_REG_AEU_PXP_CLOSE_MASK |
93              MISC_AEU_GENERAL_MASK_REG_AEU_NIG_CLOSE_MASK);
94         REG_WR(pdev, MISC_REG_AEU_GENERAL_MASK, val);
95     }
96 }
97 
98 /* Close gates #2, #3 and #4: */
99 static void lm_er_set_234_gates(lm_device_t *pdev, u8_t close_g8)
100 {
101     u32_t val, enable_bit;
102 
103     enable_bit = close_g8? 1 : 0;
104 
105     /* close gate #4 */
106     REG_WR(pdev, PXP_REG_HST_DISCARD_DOORBELLS, enable_bit);
107 
108     /* close gate #2 */
109     REG_WR(pdev, PXP_REG_HST_DISCARD_INTERNAL_WRITES, enable_bit);
110 
111     /* close gate #3 (this will disable new interrupts */
112     val = REG_RD(pdev, IGU_REG_BLOCK_CONFIGURATION);
113     close_g8? RESET_FLAGS(val, IGU_BLOCK_CONFIGURATION_REG_BLOCK_ENABLE) :
114         SET_FLAGS(val, IGU_BLOCK_CONFIGURATION_REG_BLOCK_ENABLE);
115 
116     REG_WR(pdev, IGU_REG_BLOCK_CONFIGURATION, val);
117 
118 
119     DbgMessage(pdev, FATAL, "%s gates #2, #3 and #4\n",
120         close_g8 ? "closing" : "opening");
121 
122 }
123 
124 
125 static void lm_er_pxp_prep(lm_device_t *pdev)
126 {
127     if (!CHIP_IS_E1(pdev))
128     {
129         REG_WR(pdev, PXP2_REG_RD_START_INIT, 0);
130         REG_WR(pdev, PXP2_REG_RQ_RBC_DONE, 0);
131     }
132 }
133 
134 /*
135  * Reset the whole chip except for:
136  *      - PCIE core
137  *      - PCI Glue, PSWHST, PXP/PXP2 RF (all controlled by
138  *              one reset bit)
139  *      - IGU
140  *      - MISC (including AEU)
141  *      - GRC
142  *      - RBCN, RBCP
143  * Reset MCP ONLY if reset_mcp is TRUE
144  */
145 static void lm_er_process_kill_chip_reset(lm_device_t *pdev, u8_t reset_mcp)
146 {
147     u32_t not_reset_mask1, reset_mask1, not_reset_mask2, reset_mask2;
148 
149     not_reset_mask1 =
150         MISC_REGISTERS_RESET_REG_1_RST_HC |
151         MISC_REGISTERS_RESET_REG_1_RST_PXPV |
152         MISC_REGISTERS_RESET_REG_1_RST_PXP;
153 
154     not_reset_mask2 =
155         MISC_REGISTERS_RESET_REG_2_RST_PCI_MDIO |
156         MISC_REGISTERS_RESET_REG_2_RST_EMAC0_HARD_CORE |
157         MISC_REGISTERS_RESET_REG_2_RST_EMAC1_HARD_CORE |
158         MISC_REGISTERS_RESET_REG_2_RST_MISC_CORE |
159         MISC_REGISTERS_RESET_REG_2_RST_RBCN |
160         MISC_REGISTERS_RESET_REG_2_RST_GRC  |
161         MISC_REGISTERS_RESET_REG_2_RST_MCP_N_RESET_REG_HARD_CORE |
162         MISC_REGISTERS_RESET_REG_2_RST_MCP_N_HARD_CORE_RST_B;
163 
164 
165     reset_mask1 = 0xffffffff;
166     reset_mask2 = 0x1ffff;
167 
168     if (!reset_mcp)
169     {
170         RESET_FLAGS(reset_mask2, MISC_REGISTERS_RESET_REG_2_RST_MCP_N_RESET_CMN_CPU);
171         RESET_FLAGS(reset_mask2, MISC_REGISTERS_RESET_REG_2_RST_MCP_N_RESET_CMN_CORE);
172     }
173 
174     if (CHIP_IS_E3(pdev)) /* Maybe some day... */
175     {
176         reset_mask2 |= MISC_REGISTERS_RESET_REG_2_MSTAT0;
177         reset_mask2 |= MISC_REGISTERS_RESET_REG_2_MSTAT1;
178     }
179 
180     /* CQ54250, CQ54294, CQ54298, CQ54396
181      * Error Recovery: break at evbda!lm_dmae_command+960 during error recovery,
182      * REG_1 must be called after reg_2 so that QM is reset AFTER PXP, this is because
183      * resetting QM cancels close the gates, initiates request to PXP
184      * <Ofer Zipin>: theory when QM is reset before PXP: 'close the gates' is de-activated shortly before resetting PXP.
185      * PSWRQ sends a write request to PGLUE. Then PXP is reset (PGLUE is not reset).
186      * PGLUE tries to read the payload data from PSWWR, but PSWWR does not respond. The write queue in PGLUE will be stuck.
187      */
188 
189     REG_WR(pdev, GRCBASE_MISC + MISC_REGISTERS_RESET_REG_2_CLEAR,
190            reset_mask2 & (~not_reset_mask2));
191 
192     REG_WR(pdev, GRCBASE_MISC + MISC_REGISTERS_RESET_REG_1_CLEAR,
193            reset_mask1 & (~not_reset_mask1));
194     /* Take blocks out of reset */
195     REG_WR(pdev, GRCBASE_MISC + MISC_REGISTERS_RESET_REG_2_SET, reset_mask2);
196 
197     REG_WR(pdev, GRCBASE_MISC + MISC_REGISTERS_RESET_REG_1_SET, reset_mask1);
198 
199 
200 
201 }
202 
203 /**
204  * @Description
205  *      13.2 Poll for Tetris buffer to empty. PSWWR FIFOs are
206  *      not guaraneteed to empty. wait for all Glue tags to
207  *      become unused (all read completions have returned).
208  *
209  * @param pdev
210  *
211  * @return lm_status_t
212  */
213 static lm_status_t lm_er_empty_tetris_buffer(lm_device_t * pdev)
214 {
215     u32_t cnt            = 1000;
216     u32_t sr_cnt         = 0;
217     u32_t blk_cnt        = 0;
218     u32_t port_is_idle_0 = 0;
219     u32_t port_is_idle_1 = 0;
220     u32_t pgl_exp_rom2   = 0;
221     u32_t pgl_b_reg_tags = 0;
222 
223     do {
224         sr_cnt  = REG_RD(pdev, PXP2_REG_RD_SR_CNT);
225         blk_cnt = REG_RD(pdev, PXP2_REG_RD_BLK_CNT);
226         port_is_idle_0 = REG_RD(pdev, PXP2_REG_RD_PORT_IS_IDLE_0);
227         port_is_idle_1 = REG_RD(pdev, PXP2_REG_RD_PORT_IS_IDLE_1);
228         pgl_exp_rom2 = REG_RD(pdev, PXP2_REG_PGL_EXP_ROM2);
229         pgl_b_reg_tags = REG_RD(pdev, PGLUE_B_REG_TAGS_63_32);
230 
231         if (TRUE
232             && (sr_cnt >= 0x7e)
233             && (blk_cnt == 0xa0)
234             && ((port_is_idle_0 & 0x1) == 0x1)
235             && ((port_is_idle_1 & 0x1) == 0x1)
236             && (pgl_exp_rom2 == 0xffffffff)
237             && (!CHIP_IS_E3(pdev) || (pgl_b_reg_tags == 0xffffffff)))
238         {
239             break;
240         }
241         mm_wait(pdev, 1000);
242     } while (cnt-- > 0);
243 
244     if (cnt == 0) {
245         DbgMessage(pdev, FATAL, "Tetris buffer didn't get empty or there are still outstanding read requests after 1s!\n");
246         DbgMessage(pdev, FATAL, "sr_cnt=0x%08x, blk_cnt=0x%08x, port_is_idle_0=0x%08x, port_is_idle_1=0x%08x, pgl_exp_rom2=0x%08x\n",
247               sr_cnt, blk_cnt, port_is_idle_0, port_is_idle_1, pgl_exp_rom2);
248         return LM_STATUS_BUSY;
249     }
250 
251     return LM_STATUS_SUCCESS;
252 }
253 
254 
255 /**
256  * @Description
257  *      13.5.   Poll for IGU VQ to become empty
258  *
259  * @param pdev
260  *
261  * @return lm_status_t
262  */
263 static lm_status_t lm_er_poll_igu_vq(lm_device_t * pdev)
264 {
265     u32_t cnt       = 1000;
266     u32_t pend_bits = 0;
267 
268     do {
269         pend_bits  = REG_RD(pdev, IGU_REG_PENDING_BITS_STATUS);
270 
271         if (pend_bits == 0)
272         {
273             break;
274         }
275         mm_wait(pdev, 1000);
276     } while (cnt-- > 0);
277 
278     if (cnt == 0) {
279         DbgMessage(pdev, FATAL, "Still pending IGU requests pend_bits=%x!\n", pend_bits);
280 
281         return LM_STATUS_BUSY;
282     }
283 
284     return LM_STATUS_SUCCESS;
285 }
286 
287 /**
288  * @Description
289  *      This section is based on E2 Recovery Flows Design
290  *      document by Yuval Eliyahu. Section 12.2 (process kill)
291  *      item #13. Number below refer to items inside item #13.
292  *      Some modifications were made to accomidate to E2.
293  *
294  * @param pdev
295  *
296  * @return lm_status_t
297  */
298 static lm_status_t lm_er_process_kill(lm_device_t *pdev, u8_t reset_mcp)
299 {
300     lm_status_t lm_status = LM_STATUS_FAILURE;
301     u32_t magic_val = 0;
302 
303     /* 13.2 Empty the Tetris buffer, wait for 1s TODO_ER: is this needed for E2? */
304     lm_status = lm_er_empty_tetris_buffer(pdev);
305     if (lm_status != LM_STATUS_SUCCESS)
306     {
307         return lm_status;
308     }
309 
310     /* 13.3, 13.4 Close gates #2, #3 and #4 */
311     lm_er_set_234_gates(pdev, TRUE);
312 
313     /* 13.5 Poll for IGU VQ to become empty */
314     lm_status = lm_er_poll_igu_vq(pdev);
315     if (lm_status != LM_STATUS_SUCCESS)
316     {
317         return lm_status;
318     }
319 
320     /* 13.6 Indicate that "process kill" is in progress to MCP TODO_ER: how/why?  */
321 
322     /* 13.7 Clear "unprepared" bit */
323     REG_WR(pdev, MISC_REG_UNPREPARED, 0);
324 
325     /* 13.8 Wait for 1ms to empty GLUE and PCI-E core queues,
326      * PSWHST, GRC and PSWRD Tetris buffer.
327      */
328     mm_wait(pdev, 1000);
329 
330     /* Prepare to chip reset: */
331     /* MCP */
332     if (reset_mcp)
333     {
334         lm_reset_mcp_prep(pdev, &magic_val);
335     }
336 
337     /* 13.11.1 PXP preparations TODO_ER: should this really be called before or only after
338      * spec says after, bnx2x implementation does this before as well.  */
339     lm_er_pxp_prep(pdev);
340 
341     /* 13.9 reset the chip */
342     /* 13.10 check that PSWRD, PSWRQ, PSWWR are reset : handled in function  */
343     lm_er_process_kill_chip_reset(pdev, reset_mcp);
344 
345 
346     /* 13.11 Recover after reset: */
347     /* MCP */
348     if (reset_mcp)
349     {
350         lm_status = lm_reset_mcp_comp(pdev, magic_val);
351         if (lm_status != LM_STATUS_SUCCESS)
352         {
353             return lm_status;
354         }
355     }
356 
357     /* Reset the loader for no-mcp mode, mcp has been reset!! */
358     lm_loader_reset(pdev);
359 
360     /* 13.11.1.2 PXP */
361 
362     // PXP2 initialization skipped here to address CQ61863:
363     //  - PXP2 must not be re-initialized.
364     //  - Starting MCP 7.0.45, PXP2 is initialized by MCP.
365 
366     //lm_er_pxp_prep(pdev);
367 
368     /* 13.11.2 Open the gates #2, #3 and #4 */
369     lm_er_set_234_gates(pdev, FALSE);
370 
371     /* 13.11.3 TODO_ER:  IGU/AEU preparation bring back the AEU/IGU to a
372      * reset state, re-enable attentions. This is done in "init" */
373 
374     /* Clear the general attention used to notify second engine */
375     REG_WR(pdev, MISC_REG_AEU_GENERAL_ATTN_20 , 0);
376     /* Some Notes:
377      * 1. parity bits will be cleard for blocks that are being reset, so no need to take care of it...
378      * 2. MCP notification isn't handled yet, when it is leader will need to nofity mcp reset is done.
379      */
380     return 0;
381 }
382 
383 
384 /**
385  * @Description
386  *     Perform the error recovery leader process kill flow.
387  *
388  * @param pdev
389  *
390  * @return lm_status_t SUCCESS or FAILURE
391  */
392 lm_status_t lm_er_leader_reset(lm_device_t *pdev)
393 {
394     lm_status_t lm_status = LM_STATUS_SUCCESS;
395     u32_t       cnt       = 1;
396     u8_t        reset_mcp = FALSE;
397     u8_t        function_of_opposite_path = 0;
398 
399     /* Try to recover after the failure */
400     /* need to recover on both paths using pretend */
401     function_of_opposite_path = !PATH_ID(pdev);
402     do
403     {
404         lm_status = lm_er_process_kill(pdev, reset_mcp);
405         if (lm_status != LM_STATUS_SUCCESS)
406         {
407             break;
408         }
409 
410         /* Pretend to the other path... */
411         if (!reset_mcp)
412         {
413             lm_pretend_func(pdev, function_of_opposite_path);
414             /* Only second go should reset MCP, so that the second engine doesn't get out of close-dg8 before the process is done */
415             reset_mcp = TRUE;
416         }
417     } while (cnt--);
418 
419     /* in anycase pretend back... */
420     lm_pretend_func(pdev, ABS_FUNC_ID(pdev));
421 
422     return lm_status;
423 }
424 
425 /**
426  * @Description
427  *      This function notifies the second engine that a
428  *      attention occured and error recovery will initiate on
429  *      second engine as well. Only the leader does this meaning
430  *      that the second engine either hasn't seen that there was
431  *      an error, or seen it and is waiting (won't initiate
432  *      leader reset) which means it won't see it anyway...
433  * @param pdev
434  *
435  * @return lm_status_t
436  */
437 lm_status_t lm_er_notify_other_path(lm_device_t *pdev)
438 {
439     u8_t function_of_opposite_path = 0;
440 
441     DbgMessage(pdev, FATAL, "lm_er_notify_other_path\n");
442     /* Pretend to the other path... */
443     function_of_opposite_path = lm_er_get_first_func_of_opp_path(pdev);
444     if (function_of_opposite_path != 0xFF)
445     {
446         lm_pretend_func(pdev, function_of_opposite_path);
447 
448         REG_WR(pdev, MISC_REG_AEU_GENERAL_ATTN_20 , 1);
449 
450         /* in anycase pretend back... */
451         lm_pretend_func(pdev, ABS_FUNC_ID(pdev));
452     }
453     else
454     {
455         DbgMessage(pdev, FATAL, "No ebnabled functions on path%d, the pah is not notfied about ER\n",!PATH_ID(pdev));
456     }
457 
458     return LM_STATUS_SUCCESS;
459 }
460 
461 /**
462  * @Description
463  *      This function attaches attentions to NIG / PXP
464  *      close-the-g8, any attention that is added here should
465  *      also be added to the lm_recoverable_error function.
466  * @param pdev
467  */
468 void lm_er_config_close_the_g8(lm_device_t *pdev)
469 {
470     u32_t val;
471 
472     if (!pdev->params.enable_error_recovery || CHIP_IS_E1x(pdev))
473     {
474         return;
475     }
476 
477     /* HW Attentions (Except Parity which is configured by init-tool / reset-values ) */
478 
479     /* QM Block */
480     val = REG_RD(pdev, MISC_REG_AEU_ENABLE2_NIG_0);
481     SET_FLAGS(val, AEU_INPUTS_ATTN_BITS_QM_HW_INTERRUPT); /* QM HW Interrupt */
482     REG_WR(pdev, MISC_REG_AEU_ENABLE2_NIG_0, val);
483 
484     val = REG_RD(pdev, MISC_REG_AEU_ENABLE2_PXP_0);
485     SET_FLAGS(val, AEU_INPUTS_ATTN_BITS_QM_HW_INTERRUPT); /* QM HW Interrupt */
486     REG_WR(pdev, MISC_REG_AEU_ENABLE2_PXP_0, val);
487 
488     /* General Attention 20 (error recovery attention) */
489     val = REG_RD(pdev, MISC_REG_AEU_ENABLE4_NIG_0);
490     SET_FLAGS(val, AEU_INPUTS_ATTN_BITS_GRC_MAPPED_GENERAL_ATTN20); /* general attention 20 */
491     REG_WR(pdev, MISC_REG_AEU_ENABLE4_NIG_0, val);
492 
493     val = REG_RD(pdev, MISC_REG_AEU_ENABLE4_PXP_0);
494     SET_FLAGS(val, AEU_INPUTS_ATTN_BITS_GRC_MAPPED_GENERAL_ATTN20); /* general attention 20 */
495     REG_WR(pdev, MISC_REG_AEU_ENABLE4_PXP_0, val);
496 
497 }
498 
499 u32_t lm_er_get_func_bit(struct _lm_device_t *pdev)
500 {
501     u32_t func_bit = 1 << ABS_FUNC_ID(pdev);
502     return func_bit;
503 }
504 
505 u32_t lm_er_get_number_of_functions(u32_t er_register)
506 {
507     u8_t i = 0;
508     u32_t func_num = 0;
509     for (i = 0; i < MAX_FUNC_NUM; i++)
510     {
511         if (er_register & (1 << i))
512         {
513             func_num++;
514         }
515     }
516     return func_num;
517 }
518 
519 u8_t lm_er_get_first_func_of_opp_path(struct _lm_device_t *pdev)
520 {
521     lm_status_t lm_status           = LM_STATUS_SUCCESS;
522     u32_t       por_aux_register    = 0;
523     u8_t        opposite_path       = 0;
524     u8_t        i                   = 0;
525     u8_t        func_of_opp_path    = 0xFF;
526 
527     if (IS_ASSIGNED_TO_VM_PFDEV(pdev))
528     {
529         lm_status = lm_hw_lock(pdev, HW_LOCK_RESOURCE_RECOVERY_REG, TRUE);
530         if (LM_STATUS_SUCCESS == lm_status)
531         {
532             por_aux_register = REG_RD(pdev, LM_ERROR_RECOVERY_COUNTER_HW_REGISTER);
533             lm_status = lm_hw_unlock(pdev, HW_LOCK_RESOURCE_RECOVERY_REG);
534             if (LM_STATUS_SUCCESS != lm_status)
535             {
536                 DbgBreakMsg("Failed to release HW Recovery Counter lock.\n");
537             }
538             opposite_path = !PATH_ID(pdev);
539             for (i = 0; i < MAX_FUNC_NUM/2; i++)
540             {
541                 if (por_aux_register & (1 << (i*2 + opposite_path)))
542                 {
543                     func_of_opp_path = i*2 + opposite_path;
544                     break;
545                 }
546             }
547         }
548         else
549         {
550             DbgBreakMsg("Failed to acquire HW Recovery Counter lock.\n");
551         }
552     }
553     else
554     {
555         func_of_opp_path = !PATH_ID(pdev);
556     }
557     return func_of_opp_path;
558 }
559 
560 u32_t lm_er_inc_load_cnt(lm_device_t *pdev, u8_t sync_it)
561 {
562     lm_status_t             lm_status = LM_STATUS_SUCCESS;
563     u32_t                   counter = 0;
564     u32_t                   por_aux_register = 0;
565     u32_t                   func_er_bit = 0;
566 
567     if (sync_it)
568     {
569         lm_status = lm_hw_lock(pdev, HW_LOCK_RESOURCE_RECOVERY_REG, TRUE);
570     }
571 
572     if (LM_STATUS_SUCCESS == lm_status)
573     {
574         por_aux_register = REG_RD(pdev, LM_ERROR_RECOVERY_COUNTER_HW_REGISTER);
575         func_er_bit = lm_er_get_func_bit(pdev);
576         if ((por_aux_register & func_er_bit))
577         {
578             if (sync_it)
579             {
580                 lm_status = lm_hw_unlock(pdev, HW_LOCK_RESOURCE_RECOVERY_REG);
581             }
582             DbgMessage(pdev, FATAL, "HW Recovery bit was not cleared in previous session.\n");
583             pdev->debug_info.er_bit_is_set_already = TRUE;
584             pdev->debug_info.er_bit_from_previous_sessions++;
585         }
586         else
587         {
588             por_aux_register |= func_er_bit;
589             REG_WR(pdev, LM_ERROR_RECOVERY_COUNTER_HW_REGISTER, por_aux_register);
590             pdev->debug_info.er_bit_is_set_already = FALSE;
591             if (sync_it)
592             {
593                 lm_status = lm_hw_unlock(pdev, HW_LOCK_RESOURCE_RECOVERY_REG);
594             }
595         }
596         if (LM_STATUS_SUCCESS != lm_status)
597         {
598             DbgBreakMsg("Failed to release HW Recovery Counter lock.\n");
599         }
600         counter = lm_er_get_number_of_functions(por_aux_register);
601     }
602     else
603     {
604         DbgBreakMsg("Failed to acquire HW Recovery Counter lock.\n");
605     }
606     return counter;
607 }
608 
609 u32_t lm_er_dec_load_cnt(lm_device_t *pdev, u8_t sync_it)
610 {
611     lm_status_t             lm_status = LM_STATUS_SUCCESS;
612     u32_t                   counter = 0;
613     u32_t                   por_aux_register = 0;
614     u32_t                   func_er_bit = 0;
615 
616     if (sync_it)
617     {
618         lm_status = lm_hw_lock(pdev, HW_LOCK_RESOURCE_RECOVERY_REG, TRUE);
619     }
620 
621     if (LM_STATUS_SUCCESS == lm_status)
622     {
623         por_aux_register = REG_RD(pdev, LM_ERROR_RECOVERY_COUNTER_HW_REGISTER);
624         func_er_bit = lm_er_get_func_bit(pdev);
625         if (!(por_aux_register & func_er_bit))
626         {
627             if (sync_it)
628             {
629                 lm_status = lm_hw_unlock(pdev, HW_LOCK_RESOURCE_RECOVERY_REG);
630             }
631             DbgMessage(pdev, FATAL, "HW Recovery bit is clear already.\n");
632         }
633         else
634         {
635             por_aux_register &= ~func_er_bit;
636             REG_WR(pdev, LM_ERROR_RECOVERY_COUNTER_HW_REGISTER, por_aux_register);
637             if (sync_it)
638             {
639                 lm_status = lm_hw_unlock(pdev, HW_LOCK_RESOURCE_RECOVERY_REG);
640             }
641         }
642         if (LM_STATUS_SUCCESS != lm_status)
643         {
644             DbgBreakMsg("Failed to release HW Recovery Counter lock.\n");
645         }
646         counter = lm_er_get_number_of_functions(por_aux_register);
647     }
648     else
649     {
650         DbgBreakMsg("Failed to acquire HW Recovery Counter lock.\n");
651     }
652     return counter;
653 }
654 
655 u32_t lm_er_get_load_cnt(lm_device_t *pdev, u8_t sync_it)
656 {
657     lm_status_t             lm_status = LM_STATUS_SUCCESS;
658     u32_t                   counter = 0;
659     u32_t                   por_aux_register = 0;
660 
661     if (sync_it)
662     {
663         lm_status = lm_hw_lock(pdev, HW_LOCK_RESOURCE_RECOVERY_REG, TRUE);
664     }
665 
666     if (LM_STATUS_SUCCESS == lm_status)
667     {
668         por_aux_register = REG_RD(pdev, LM_ERROR_RECOVERY_COUNTER_HW_REGISTER);
669         if (sync_it)
670         {
671             lm_status = lm_hw_unlock(pdev, HW_LOCK_RESOURCE_RECOVERY_REG);
672             if (LM_STATUS_SUCCESS != lm_status)
673             {
674                 DbgBreakMsg("Failed to release HW Recovery Counter lock.\n");
675             }
676         }
677         counter = lm_er_get_number_of_functions(por_aux_register);
678     }
679     else
680     {
681         DbgBreakMsg("Failed to acquire HW Recovery Counter lock.\n");
682     }
683     return counter;
684 }
685 
686 void lm_er_clear_load_cnt(lm_device_t *pdev, u8_t sync_it)
687 {
688     lm_status_t             lm_status = LM_STATUS_SUCCESS;
689 
690     if (sync_it)
691     {
692         lm_status = lm_hw_lock(pdev, HW_LOCK_RESOURCE_RECOVERY_REG, TRUE);
693     }
694     if (LM_STATUS_SUCCESS == lm_status)
695     {
696         REG_WR(pdev, LM_ERROR_RECOVERY_COUNTER_HW_REGISTER, 0);
697         if (sync_it)
698         {
699             lm_status = lm_hw_unlock(pdev, HW_LOCK_RESOURCE_RECOVERY_REG);
700             if (LM_STATUS_SUCCESS != lm_status)
701             {
702                 DbgBreakMsg("Failed to release HW Recovery Counter lock.\n");
703             }
704         }
705     }
706     else
707     {
708         DbgBreakMsg("Failed to acquire HW Recovery Counter lock.\n");
709     }
710 }
711 
712 void lm_er_set_recover_done(lm_device_t *pdev, u8_t sync_it)
713 {
714     lm_status_t             lm_status = LM_STATUS_SUCCESS;
715     u32_t                   por_aux_register = 0;
716 
717 
718     DbgMessage(pdev, FATAL, "Setting recovery in progress = 0\n");
719     if (sync_it)
720     {
721         lm_status = lm_hw_lock(pdev, HW_LOCK_RESOURCE_RECOVERY_REG, TRUE);
722     }
723     if (LM_STATUS_SUCCESS == lm_status)
724     {
725         por_aux_register = REG_RD(pdev, LM_ERROR_RECOVERY_COUNTER_HW_REGISTER);
726         por_aux_register &= ~LM_ERROR_RECOVERY_IN_PROGRESS_FLAG;
727         REG_WR(pdev, LM_ERROR_RECOVERY_COUNTER_HW_REGISTER, por_aux_register);
728         if (sync_it)
729         {
730             lm_status = lm_hw_unlock(pdev, HW_LOCK_RESOURCE_RECOVERY_REG);
731             if (LM_STATUS_SUCCESS != lm_status)
732             {
733                 DbgBreakMsg("Failed to release HW Recovery Counter lock.\n");
734             }
735         }
736     }
737     else
738     {
739         DbgBreakMsg("Failed to acquire HW Recovery Counter lock.\n");
740     }
741 }
742 
743 void lm_er_set_recover_in_progress(lm_device_t *pdev, u8_t sync_it)
744 {
745     lm_status_t             lm_status = LM_STATUS_SUCCESS;
746     u32_t                   por_aux_register = 0;
747 
748     DbgMessage(pdev, FATAL, "Setting recovery in progress = 1\n");
749     if (sync_it)
750     {
751         lm_status = lm_hw_lock(pdev, HW_LOCK_RESOURCE_RECOVERY_REG, TRUE);
752     }
753     if (LM_STATUS_SUCCESS == lm_status)
754     {
755         por_aux_register = REG_RD(pdev, LM_ERROR_RECOVERY_COUNTER_HW_REGISTER);
756         por_aux_register |= LM_ERROR_RECOVERY_IN_PROGRESS_FLAG;
757         REG_WR(pdev, LM_ERROR_RECOVERY_COUNTER_HW_REGISTER, por_aux_register);
758         if (sync_it)
759         {
760             lm_status = lm_hw_unlock(pdev, HW_LOCK_RESOURCE_RECOVERY_REG);
761             if (LM_STATUS_SUCCESS != lm_status)
762             {
763                 DbgBreakMsg("Failed to release HW Recovery Counter lock.\n");
764             }
765         }
766     }
767     else
768     {
769         DbgBreakMsg("Failed to acquire HW Recovery Counter lock.\n");
770     }
771 }
772 
773 
774 u8_t lm_er_recovery_in_progress(lm_device_t *pdev, u8_t sync_it)
775 {
776     lm_status_t             lm_status = LM_STATUS_SUCCESS;
777     u32_t                   por_aux_register = 0;
778     u8_t                    is_progress = FALSE;
779 
780     if (sync_it)
781     {
782         lm_status = lm_hw_lock(pdev, HW_LOCK_RESOURCE_RECOVERY_REG, TRUE);
783     }
784     if (LM_STATUS_SUCCESS == lm_status)
785     {
786         por_aux_register = REG_RD(pdev, LM_ERROR_RECOVERY_COUNTER_HW_REGISTER);
787         if (sync_it)
788         {
789             lm_status = lm_hw_unlock(pdev, HW_LOCK_RESOURCE_RECOVERY_REG);
790             if (LM_STATUS_SUCCESS != lm_status)
791             {
792                 DbgBreakMsg("Failed to release HW Recovery Counter lock.\n");
793             }
794         }
795         if (por_aux_register & LM_ERROR_RECOVERY_IN_PROGRESS_FLAG)
796         {
797             is_progress = TRUE;
798         }
799     }
800     else
801     {
802         DbgBreakMsg("Failed to acquire HW Recovery Counter lock.\n");
803     }
804     return is_progress;
805 }
806 
807