1e9eec291SMałgorzata Mielnik // SPDX-License-Identifier: GPL-2.0-only 2e9eec291SMałgorzata Mielnik /* Copyright(c) 2025 Intel Corporation */ 3e9eec291SMałgorzata Mielnik 4e9eec291SMałgorzata Mielnik #define pr_fmt(fmt) "QAT: " fmt 5e9eec291SMałgorzata Mielnik 6e9eec291SMałgorzata Mielnik #include <linux/bits.h> 7e9eec291SMałgorzata Mielnik #include <linux/dev_printk.h> 8e9eec291SMałgorzata Mielnik #include <linux/printk.h> 9e9eec291SMałgorzata Mielnik #include "adf_accel_devices.h" 10e9eec291SMałgorzata Mielnik #include "adf_bank_state.h" 11e9eec291SMałgorzata Mielnik #include "adf_common_drv.h" 12e9eec291SMałgorzata Mielnik 13e9eec291SMałgorzata Mielnik /* Ring interrupt masks */ 14e9eec291SMałgorzata Mielnik #define ADF_RP_INT_SRC_SEL_F_RISE_MASK GENMASK(1, 0) 15e9eec291SMałgorzata Mielnik #define ADF_RP_INT_SRC_SEL_F_FALL_MASK GENMASK(2, 0) 16e9eec291SMałgorzata Mielnik #define ADF_RP_INT_SRC_SEL_RANGE_WIDTH 4 17e9eec291SMałgorzata Mielnik 18e9eec291SMałgorzata Mielnik static inline int check_stat(u32 (*op)(void __iomem *, u32), u32 expect_val, 19e9eec291SMałgorzata Mielnik const char *name, void __iomem *base, u32 bank) 20e9eec291SMałgorzata Mielnik { 21e9eec291SMałgorzata Mielnik u32 actual_val = op(base, bank); 22e9eec291SMałgorzata Mielnik 23e9eec291SMałgorzata Mielnik if (expect_val == actual_val) 24e9eec291SMałgorzata Mielnik return 0; 25e9eec291SMałgorzata Mielnik 26e9eec291SMałgorzata Mielnik pr_err("Fail to restore %s register. Expected %#x, actual %#x\n", 27e9eec291SMałgorzata Mielnik name, expect_val, actual_val); 28e9eec291SMałgorzata Mielnik 29e9eec291SMałgorzata Mielnik return -EINVAL; 30e9eec291SMałgorzata Mielnik } 31e9eec291SMałgorzata Mielnik 32e9eec291SMałgorzata Mielnik static void bank_state_save(struct adf_hw_csr_ops *ops, void __iomem *base, 33*a47dc5d1SSuman Kumar Chakraborty u32 bank, struct adf_bank_state *state, u32 num_rings) 34e9eec291SMałgorzata Mielnik { 35e9eec291SMałgorzata Mielnik u32 i; 36e9eec291SMałgorzata Mielnik 37e9eec291SMałgorzata Mielnik state->ringstat0 = ops->read_csr_stat(base, bank); 38e9eec291SMałgorzata Mielnik state->ringuostat = ops->read_csr_uo_stat(base, bank); 39e9eec291SMałgorzata Mielnik state->ringestat = ops->read_csr_e_stat(base, bank); 40e9eec291SMałgorzata Mielnik state->ringnestat = ops->read_csr_ne_stat(base, bank); 41e9eec291SMałgorzata Mielnik state->ringnfstat = ops->read_csr_nf_stat(base, bank); 42e9eec291SMałgorzata Mielnik state->ringfstat = ops->read_csr_f_stat(base, bank); 43e9eec291SMałgorzata Mielnik state->ringcstat0 = ops->read_csr_c_stat(base, bank); 44e9eec291SMałgorzata Mielnik state->iaintflagen = ops->read_csr_int_en(base, bank); 45e9eec291SMałgorzata Mielnik state->iaintflagreg = ops->read_csr_int_flag(base, bank); 46e9eec291SMałgorzata Mielnik state->iaintflagsrcsel0 = ops->read_csr_int_srcsel(base, bank); 47e9eec291SMałgorzata Mielnik state->iaintcolen = ops->read_csr_int_col_en(base, bank); 48e9eec291SMałgorzata Mielnik state->iaintcolctl = ops->read_csr_int_col_ctl(base, bank); 49e9eec291SMałgorzata Mielnik state->iaintflagandcolen = ops->read_csr_int_flag_and_col(base, bank); 50e9eec291SMałgorzata Mielnik state->ringexpstat = ops->read_csr_exp_stat(base, bank); 51e9eec291SMałgorzata Mielnik state->ringexpintenable = ops->read_csr_exp_int_en(base, bank); 52e9eec291SMałgorzata Mielnik state->ringsrvarben = ops->read_csr_ring_srv_arb_en(base, bank); 53e9eec291SMałgorzata Mielnik 54e9eec291SMałgorzata Mielnik for (i = 0; i < num_rings; i++) { 55e9eec291SMałgorzata Mielnik state->rings[i].head = ops->read_csr_ring_head(base, bank, i); 56e9eec291SMałgorzata Mielnik state->rings[i].tail = ops->read_csr_ring_tail(base, bank, i); 57e9eec291SMałgorzata Mielnik state->rings[i].config = ops->read_csr_ring_config(base, bank, i); 58e9eec291SMałgorzata Mielnik state->rings[i].base = ops->read_csr_ring_base(base, bank, i); 59e9eec291SMałgorzata Mielnik } 60e9eec291SMałgorzata Mielnik } 61e9eec291SMałgorzata Mielnik 62e9eec291SMałgorzata Mielnik static int bank_state_restore(struct adf_hw_csr_ops *ops, void __iomem *base, 63*a47dc5d1SSuman Kumar Chakraborty u32 bank, struct adf_bank_state *state, u32 num_rings, 64e9eec291SMałgorzata Mielnik int tx_rx_gap) 65e9eec291SMałgorzata Mielnik { 66e9eec291SMałgorzata Mielnik u32 val, tmp_val, i; 67e9eec291SMałgorzata Mielnik int ret; 68e9eec291SMałgorzata Mielnik 69e9eec291SMałgorzata Mielnik for (i = 0; i < num_rings; i++) 70e9eec291SMałgorzata Mielnik ops->write_csr_ring_base(base, bank, i, state->rings[i].base); 71e9eec291SMałgorzata Mielnik 72e9eec291SMałgorzata Mielnik for (i = 0; i < num_rings; i++) 73e9eec291SMałgorzata Mielnik ops->write_csr_ring_config(base, bank, i, state->rings[i].config); 74e9eec291SMałgorzata Mielnik 75e9eec291SMałgorzata Mielnik for (i = 0; i < num_rings / 2; i++) { 76e9eec291SMałgorzata Mielnik int tx = i * (tx_rx_gap + 1); 77e9eec291SMałgorzata Mielnik int rx = tx + tx_rx_gap; 78e9eec291SMałgorzata Mielnik 79e9eec291SMałgorzata Mielnik ops->write_csr_ring_head(base, bank, tx, state->rings[tx].head); 80e9eec291SMałgorzata Mielnik ops->write_csr_ring_tail(base, bank, tx, state->rings[tx].tail); 81e9eec291SMałgorzata Mielnik 82e9eec291SMałgorzata Mielnik /* 83e9eec291SMałgorzata Mielnik * The TX ring head needs to be updated again to make sure that 84e9eec291SMałgorzata Mielnik * the HW will not consider the ring as full when it is empty 85e9eec291SMałgorzata Mielnik * and the correct state flags are set to match the recovered state. 86e9eec291SMałgorzata Mielnik */ 87e9eec291SMałgorzata Mielnik if (state->ringestat & BIT(tx)) { 88e9eec291SMałgorzata Mielnik val = ops->read_csr_int_srcsel(base, bank); 89e9eec291SMałgorzata Mielnik val |= ADF_RP_INT_SRC_SEL_F_RISE_MASK; 90e9eec291SMałgorzata Mielnik ops->write_csr_int_srcsel_w_val(base, bank, val); 91e9eec291SMałgorzata Mielnik ops->write_csr_ring_head(base, bank, tx, state->rings[tx].head); 92e9eec291SMałgorzata Mielnik } 93e9eec291SMałgorzata Mielnik 94e9eec291SMałgorzata Mielnik ops->write_csr_ring_tail(base, bank, rx, state->rings[rx].tail); 95e9eec291SMałgorzata Mielnik val = ops->read_csr_int_srcsel(base, bank); 96e9eec291SMałgorzata Mielnik val |= ADF_RP_INT_SRC_SEL_F_RISE_MASK << ADF_RP_INT_SRC_SEL_RANGE_WIDTH; 97e9eec291SMałgorzata Mielnik ops->write_csr_int_srcsel_w_val(base, bank, val); 98e9eec291SMałgorzata Mielnik 99e9eec291SMałgorzata Mielnik ops->write_csr_ring_head(base, bank, rx, state->rings[rx].head); 100e9eec291SMałgorzata Mielnik val = ops->read_csr_int_srcsel(base, bank); 101e9eec291SMałgorzata Mielnik val |= ADF_RP_INT_SRC_SEL_F_FALL_MASK << ADF_RP_INT_SRC_SEL_RANGE_WIDTH; 102e9eec291SMałgorzata Mielnik ops->write_csr_int_srcsel_w_val(base, bank, val); 103e9eec291SMałgorzata Mielnik 104e9eec291SMałgorzata Mielnik /* 105e9eec291SMałgorzata Mielnik * The RX ring tail needs to be updated again to make sure that 106e9eec291SMałgorzata Mielnik * the HW will not consider the ring as empty when it is full 107e9eec291SMałgorzata Mielnik * and the correct state flags are set to match the recovered state. 108e9eec291SMałgorzata Mielnik */ 109e9eec291SMałgorzata Mielnik if (state->ringfstat & BIT(rx)) 110e9eec291SMałgorzata Mielnik ops->write_csr_ring_tail(base, bank, rx, state->rings[rx].tail); 111e9eec291SMałgorzata Mielnik } 112e9eec291SMałgorzata Mielnik 113e9eec291SMałgorzata Mielnik ops->write_csr_int_flag_and_col(base, bank, state->iaintflagandcolen); 114e9eec291SMałgorzata Mielnik ops->write_csr_int_en(base, bank, state->iaintflagen); 115e9eec291SMałgorzata Mielnik ops->write_csr_int_col_en(base, bank, state->iaintcolen); 116e9eec291SMałgorzata Mielnik ops->write_csr_int_srcsel_w_val(base, bank, state->iaintflagsrcsel0); 117e9eec291SMałgorzata Mielnik ops->write_csr_exp_int_en(base, bank, state->ringexpintenable); 118e9eec291SMałgorzata Mielnik ops->write_csr_int_col_ctl(base, bank, state->iaintcolctl); 119e9eec291SMałgorzata Mielnik 120e9eec291SMałgorzata Mielnik /* 121e9eec291SMałgorzata Mielnik * Verify whether any exceptions were raised during the bank save process. 122e9eec291SMałgorzata Mielnik * If exceptions occurred, the status and exception registers cannot 123e9eec291SMałgorzata Mielnik * be directly restored. Consequently, further restoration is not 124e9eec291SMałgorzata Mielnik * feasible, and the current state of the ring should be maintained. 125e9eec291SMałgorzata Mielnik */ 126e9eec291SMałgorzata Mielnik val = state->ringexpstat; 127e9eec291SMałgorzata Mielnik if (val) { 128e9eec291SMałgorzata Mielnik pr_info("Bank %u state not fully restored due to exception in saved state (%#x)\n", 129e9eec291SMałgorzata Mielnik bank, val); 130e9eec291SMałgorzata Mielnik return 0; 131e9eec291SMałgorzata Mielnik } 132e9eec291SMałgorzata Mielnik 133e9eec291SMałgorzata Mielnik /* Ensure that the restoration process completed without exceptions */ 134e9eec291SMałgorzata Mielnik tmp_val = ops->read_csr_exp_stat(base, bank); 135e9eec291SMałgorzata Mielnik if (tmp_val) { 136e9eec291SMałgorzata Mielnik pr_err("Bank %u restored with exception: %#x\n", bank, tmp_val); 137e9eec291SMałgorzata Mielnik return -EFAULT; 138e9eec291SMałgorzata Mielnik } 139e9eec291SMałgorzata Mielnik 140e9eec291SMałgorzata Mielnik ops->write_csr_ring_srv_arb_en(base, bank, state->ringsrvarben); 141e9eec291SMałgorzata Mielnik 142e9eec291SMałgorzata Mielnik /* Check that all ring statuses match the saved state. */ 143e9eec291SMałgorzata Mielnik ret = check_stat(ops->read_csr_stat, state->ringstat0, "ringstat", 144e9eec291SMałgorzata Mielnik base, bank); 145e9eec291SMałgorzata Mielnik if (ret) 146e9eec291SMałgorzata Mielnik return ret; 147e9eec291SMałgorzata Mielnik 148e9eec291SMałgorzata Mielnik ret = check_stat(ops->read_csr_e_stat, state->ringestat, "ringestat", 149e9eec291SMałgorzata Mielnik base, bank); 150e9eec291SMałgorzata Mielnik if (ret) 151e9eec291SMałgorzata Mielnik return ret; 152e9eec291SMałgorzata Mielnik 153e9eec291SMałgorzata Mielnik ret = check_stat(ops->read_csr_ne_stat, state->ringnestat, "ringnestat", 154e9eec291SMałgorzata Mielnik base, bank); 155e9eec291SMałgorzata Mielnik if (ret) 156e9eec291SMałgorzata Mielnik return ret; 157e9eec291SMałgorzata Mielnik 158e9eec291SMałgorzata Mielnik ret = check_stat(ops->read_csr_nf_stat, state->ringnfstat, "ringnfstat", 159e9eec291SMałgorzata Mielnik base, bank); 160e9eec291SMałgorzata Mielnik if (ret) 161e9eec291SMałgorzata Mielnik return ret; 162e9eec291SMałgorzata Mielnik 163e9eec291SMałgorzata Mielnik ret = check_stat(ops->read_csr_f_stat, state->ringfstat, "ringfstat", 164e9eec291SMałgorzata Mielnik base, bank); 165e9eec291SMałgorzata Mielnik if (ret) 166e9eec291SMałgorzata Mielnik return ret; 167e9eec291SMałgorzata Mielnik 168e9eec291SMałgorzata Mielnik ret = check_stat(ops->read_csr_c_stat, state->ringcstat0, "ringcstat", 169e9eec291SMałgorzata Mielnik base, bank); 170e9eec291SMałgorzata Mielnik if (ret) 171e9eec291SMałgorzata Mielnik return ret; 172e9eec291SMałgorzata Mielnik 173e9eec291SMałgorzata Mielnik return 0; 174e9eec291SMałgorzata Mielnik } 175e9eec291SMałgorzata Mielnik 176e9eec291SMałgorzata Mielnik /** 177e9eec291SMałgorzata Mielnik * adf_bank_state_save() - save state of bank-related registers 178e9eec291SMałgorzata Mielnik * @accel_dev: Pointer to the device structure 179e9eec291SMałgorzata Mielnik * @bank_number: Bank number 180e9eec291SMałgorzata Mielnik * @state: Pointer to bank state structure 181e9eec291SMałgorzata Mielnik * 182e9eec291SMałgorzata Mielnik * This function saves the state of a bank by reading the bank CSRs and 183e9eec291SMałgorzata Mielnik * writing them in the @state structure. 184e9eec291SMałgorzata Mielnik * 185e9eec291SMałgorzata Mielnik * Returns 0 on success, error code otherwise 186e9eec291SMałgorzata Mielnik */ 187e9eec291SMałgorzata Mielnik int adf_bank_state_save(struct adf_accel_dev *accel_dev, u32 bank_number, 188*a47dc5d1SSuman Kumar Chakraborty struct adf_bank_state *state) 189e9eec291SMałgorzata Mielnik { 190e9eec291SMałgorzata Mielnik struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); 191e9eec291SMałgorzata Mielnik struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); 192e9eec291SMałgorzata Mielnik void __iomem *csr_base = adf_get_etr_base(accel_dev); 193e9eec291SMałgorzata Mielnik 194e9eec291SMałgorzata Mielnik if (bank_number >= hw_data->num_banks || !state) 195e9eec291SMałgorzata Mielnik return -EINVAL; 196e9eec291SMałgorzata Mielnik 197e9eec291SMałgorzata Mielnik dev_dbg(&GET_DEV(accel_dev), "Saving state of bank %d\n", bank_number); 198e9eec291SMałgorzata Mielnik 199e9eec291SMałgorzata Mielnik bank_state_save(csr_ops, csr_base, bank_number, state, 200e9eec291SMałgorzata Mielnik hw_data->num_rings_per_bank); 201e9eec291SMałgorzata Mielnik 202e9eec291SMałgorzata Mielnik return 0; 203e9eec291SMałgorzata Mielnik } 204e9eec291SMałgorzata Mielnik EXPORT_SYMBOL_GPL(adf_bank_state_save); 205e9eec291SMałgorzata Mielnik 206e9eec291SMałgorzata Mielnik /** 207e9eec291SMałgorzata Mielnik * adf_bank_state_restore() - restore state of bank-related registers 208e9eec291SMałgorzata Mielnik * @accel_dev: Pointer to the device structure 209e9eec291SMałgorzata Mielnik * @bank_number: Bank number 210e9eec291SMałgorzata Mielnik * @state: Pointer to bank state structure 211e9eec291SMałgorzata Mielnik * 212e9eec291SMałgorzata Mielnik * This function attempts to restore the state of a bank by writing the 213e9eec291SMałgorzata Mielnik * bank CSRs to the values in the state structure. 214e9eec291SMałgorzata Mielnik * 215e9eec291SMałgorzata Mielnik * Returns 0 on success, error code otherwise 216e9eec291SMałgorzata Mielnik */ 217e9eec291SMałgorzata Mielnik int adf_bank_state_restore(struct adf_accel_dev *accel_dev, u32 bank_number, 218*a47dc5d1SSuman Kumar Chakraborty struct adf_bank_state *state) 219e9eec291SMałgorzata Mielnik { 220e9eec291SMałgorzata Mielnik struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); 221e9eec291SMałgorzata Mielnik struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); 222e9eec291SMałgorzata Mielnik void __iomem *csr_base = adf_get_etr_base(accel_dev); 223e9eec291SMałgorzata Mielnik int ret; 224e9eec291SMałgorzata Mielnik 225e9eec291SMałgorzata Mielnik if (bank_number >= hw_data->num_banks || !state) 226e9eec291SMałgorzata Mielnik return -EINVAL; 227e9eec291SMałgorzata Mielnik 228e9eec291SMałgorzata Mielnik dev_dbg(&GET_DEV(accel_dev), "Restoring state of bank %d\n", bank_number); 229e9eec291SMałgorzata Mielnik 230e9eec291SMałgorzata Mielnik ret = bank_state_restore(csr_ops, csr_base, bank_number, state, 231e9eec291SMałgorzata Mielnik hw_data->num_rings_per_bank, hw_data->tx_rx_gap); 232e9eec291SMałgorzata Mielnik if (ret) 233e9eec291SMałgorzata Mielnik dev_err(&GET_DEV(accel_dev), 234e9eec291SMałgorzata Mielnik "Unable to restore state of bank %d\n", bank_number); 235e9eec291SMałgorzata Mielnik 236e9eec291SMałgorzata Mielnik return ret; 237e9eec291SMałgorzata Mielnik } 238e9eec291SMałgorzata Mielnik EXPORT_SYMBOL_GPL(adf_bank_state_restore); 239