1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2025 Intel Corporation */ 3 4 #define pr_fmt(fmt) "QAT: " fmt 5 6 #include <linux/bits.h> 7 #include <linux/dev_printk.h> 8 #include <linux/printk.h> 9 #include "adf_accel_devices.h" 10 #include "adf_bank_state.h" 11 #include "adf_common_drv.h" 12 13 /* Ring interrupt masks */ 14 #define ADF_RP_INT_SRC_SEL_F_RISE_MASK GENMASK(1, 0) 15 #define ADF_RP_INT_SRC_SEL_F_FALL_MASK GENMASK(2, 0) 16 #define ADF_RP_INT_SRC_SEL_RANGE_WIDTH 4 17 18 static inline int check_stat(u32 (*op)(void __iomem *, u32), u32 expect_val, 19 const char *name, void __iomem *base, u32 bank) 20 { 21 u32 actual_val = op(base, bank); 22 23 if (expect_val == actual_val) 24 return 0; 25 26 pr_err("Fail to restore %s register. Expected %#x, actual %#x\n", 27 name, expect_val, actual_val); 28 29 return -EINVAL; 30 } 31 32 static void bank_state_save(struct adf_hw_csr_ops *ops, void __iomem *base, 33 u32 bank, struct adf_bank_state *state, u32 num_rings) 34 { 35 u32 i; 36 37 state->ringstat0 = ops->read_csr_stat(base, bank); 38 state->ringuostat = ops->read_csr_uo_stat(base, bank); 39 state->ringestat = ops->read_csr_e_stat(base, bank); 40 state->ringnestat = ops->read_csr_ne_stat(base, bank); 41 state->ringnfstat = ops->read_csr_nf_stat(base, bank); 42 state->ringfstat = ops->read_csr_f_stat(base, bank); 43 state->ringcstat0 = ops->read_csr_c_stat(base, bank); 44 state->iaintflagen = ops->read_csr_int_en(base, bank); 45 state->iaintflagreg = ops->read_csr_int_flag(base, bank); 46 state->iaintflagsrcsel0 = ops->read_csr_int_srcsel(base, bank); 47 state->iaintcolen = ops->read_csr_int_col_en(base, bank); 48 state->iaintcolctl = ops->read_csr_int_col_ctl(base, bank); 49 state->iaintflagandcolen = ops->read_csr_int_flag_and_col(base, bank); 50 state->ringexpstat = ops->read_csr_exp_stat(base, bank); 51 state->ringexpintenable = ops->read_csr_exp_int_en(base, bank); 52 state->ringsrvarben = ops->read_csr_ring_srv_arb_en(base, bank); 53 54 for (i = 0; i < num_rings; i++) { 55 state->rings[i].head = ops->read_csr_ring_head(base, bank, i); 56 state->rings[i].tail = ops->read_csr_ring_tail(base, bank, i); 57 state->rings[i].config = ops->read_csr_ring_config(base, bank, i); 58 state->rings[i].base = ops->read_csr_ring_base(base, bank, i); 59 } 60 } 61 62 static int bank_state_restore(struct adf_hw_csr_ops *ops, void __iomem *base, 63 u32 bank, struct adf_bank_state *state, u32 num_rings, 64 int tx_rx_gap) 65 { 66 u32 val, tmp_val, i; 67 int ret; 68 69 for (i = 0; i < num_rings; i++) 70 ops->write_csr_ring_base(base, bank, i, state->rings[i].base); 71 72 for (i = 0; i < num_rings; i++) 73 ops->write_csr_ring_config(base, bank, i, state->rings[i].config); 74 75 for (i = 0; i < num_rings / 2; i++) { 76 int tx = i * (tx_rx_gap + 1); 77 int rx = tx + tx_rx_gap; 78 79 ops->write_csr_ring_head(base, bank, tx, state->rings[tx].head); 80 ops->write_csr_ring_tail(base, bank, tx, state->rings[tx].tail); 81 82 /* 83 * The TX ring head needs to be updated again to make sure that 84 * the HW will not consider the ring as full when it is empty 85 * and the correct state flags are set to match the recovered state. 86 */ 87 if (state->ringestat & BIT(tx)) { 88 val = ops->read_csr_int_srcsel(base, bank); 89 val |= ADF_RP_INT_SRC_SEL_F_RISE_MASK; 90 ops->write_csr_int_srcsel_w_val(base, bank, val); 91 ops->write_csr_ring_head(base, bank, tx, state->rings[tx].head); 92 } 93 94 ops->write_csr_ring_tail(base, bank, rx, state->rings[rx].tail); 95 val = ops->read_csr_int_srcsel(base, bank); 96 val |= ADF_RP_INT_SRC_SEL_F_RISE_MASK << ADF_RP_INT_SRC_SEL_RANGE_WIDTH; 97 ops->write_csr_int_srcsel_w_val(base, bank, val); 98 99 ops->write_csr_ring_head(base, bank, rx, state->rings[rx].head); 100 val = ops->read_csr_int_srcsel(base, bank); 101 val |= ADF_RP_INT_SRC_SEL_F_FALL_MASK << ADF_RP_INT_SRC_SEL_RANGE_WIDTH; 102 ops->write_csr_int_srcsel_w_val(base, bank, val); 103 104 /* 105 * The RX ring tail needs to be updated again to make sure that 106 * the HW will not consider the ring as empty when it is full 107 * and the correct state flags are set to match the recovered state. 108 */ 109 if (state->ringfstat & BIT(rx)) 110 ops->write_csr_ring_tail(base, bank, rx, state->rings[rx].tail); 111 } 112 113 ops->write_csr_int_flag_and_col(base, bank, state->iaintflagandcolen); 114 ops->write_csr_int_en(base, bank, state->iaintflagen); 115 ops->write_csr_int_col_en(base, bank, state->iaintcolen); 116 ops->write_csr_int_srcsel_w_val(base, bank, state->iaintflagsrcsel0); 117 ops->write_csr_exp_int_en(base, bank, state->ringexpintenable); 118 ops->write_csr_int_col_ctl(base, bank, state->iaintcolctl); 119 120 /* 121 * Verify whether any exceptions were raised during the bank save process. 122 * If exceptions occurred, the status and exception registers cannot 123 * be directly restored. Consequently, further restoration is not 124 * feasible, and the current state of the ring should be maintained. 125 */ 126 val = state->ringexpstat; 127 if (val) { 128 pr_info("Bank %u state not fully restored due to exception in saved state (%#x)\n", 129 bank, val); 130 return 0; 131 } 132 133 /* Ensure that the restoration process completed without exceptions */ 134 tmp_val = ops->read_csr_exp_stat(base, bank); 135 if (tmp_val) { 136 pr_err("Bank %u restored with exception: %#x\n", bank, tmp_val); 137 return -EFAULT; 138 } 139 140 ops->write_csr_ring_srv_arb_en(base, bank, state->ringsrvarben); 141 142 /* Check that all ring statuses match the saved state. */ 143 ret = check_stat(ops->read_csr_stat, state->ringstat0, "ringstat", 144 base, bank); 145 if (ret) 146 return ret; 147 148 ret = check_stat(ops->read_csr_e_stat, state->ringestat, "ringestat", 149 base, bank); 150 if (ret) 151 return ret; 152 153 ret = check_stat(ops->read_csr_ne_stat, state->ringnestat, "ringnestat", 154 base, bank); 155 if (ret) 156 return ret; 157 158 ret = check_stat(ops->read_csr_nf_stat, state->ringnfstat, "ringnfstat", 159 base, bank); 160 if (ret) 161 return ret; 162 163 ret = check_stat(ops->read_csr_f_stat, state->ringfstat, "ringfstat", 164 base, bank); 165 if (ret) 166 return ret; 167 168 ret = check_stat(ops->read_csr_c_stat, state->ringcstat0, "ringcstat", 169 base, bank); 170 if (ret) 171 return ret; 172 173 return 0; 174 } 175 176 /** 177 * adf_bank_state_save() - save state of bank-related registers 178 * @accel_dev: Pointer to the device structure 179 * @bank_number: Bank number 180 * @state: Pointer to bank state structure 181 * 182 * This function saves the state of a bank by reading the bank CSRs and 183 * writing them in the @state structure. 184 * 185 * Returns 0 on success, error code otherwise 186 */ 187 int adf_bank_state_save(struct adf_accel_dev *accel_dev, u32 bank_number, 188 struct adf_bank_state *state) 189 { 190 struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); 191 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); 192 void __iomem *csr_base = adf_get_etr_base(accel_dev); 193 194 if (bank_number >= hw_data->num_banks || !state) 195 return -EINVAL; 196 197 dev_dbg(&GET_DEV(accel_dev), "Saving state of bank %d\n", bank_number); 198 199 bank_state_save(csr_ops, csr_base, bank_number, state, 200 hw_data->num_rings_per_bank); 201 202 return 0; 203 } 204 EXPORT_SYMBOL_GPL(adf_bank_state_save); 205 206 /** 207 * adf_bank_state_restore() - restore state of bank-related registers 208 * @accel_dev: Pointer to the device structure 209 * @bank_number: Bank number 210 * @state: Pointer to bank state structure 211 * 212 * This function attempts to restore the state of a bank by writing the 213 * bank CSRs to the values in the state structure. 214 * 215 * Returns 0 on success, error code otherwise 216 */ 217 int adf_bank_state_restore(struct adf_accel_dev *accel_dev, u32 bank_number, 218 struct adf_bank_state *state) 219 { 220 struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); 221 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); 222 void __iomem *csr_base = adf_get_etr_base(accel_dev); 223 int ret; 224 225 if (bank_number >= hw_data->num_banks || !state) 226 return -EINVAL; 227 228 dev_dbg(&GET_DEV(accel_dev), "Restoring state of bank %d\n", bank_number); 229 230 ret = bank_state_restore(csr_ops, csr_base, bank_number, state, 231 hw_data->num_rings_per_bank, hw_data->tx_rx_gap); 232 if (ret) 233 dev_err(&GET_DEV(accel_dev), 234 "Unable to restore state of bank %d\n", bank_number); 235 236 return ret; 237 } 238 EXPORT_SYMBOL_GPL(adf_bank_state_restore); 239