xref: /linux/drivers/crypto/intel/qat/qat_common/adf_bank_state.c (revision 8d2b0853add1d7534dc0794e3c8e0b9e8c4ec640)
1e9eec291SMałgorzata Mielnik // SPDX-License-Identifier: GPL-2.0-only
2e9eec291SMałgorzata Mielnik /* Copyright(c) 2025 Intel Corporation */
3e9eec291SMałgorzata Mielnik 
4e9eec291SMałgorzata Mielnik #define pr_fmt(fmt)	"QAT: " fmt
5e9eec291SMałgorzata Mielnik 
6e9eec291SMałgorzata Mielnik #include <linux/bits.h>
7e9eec291SMałgorzata Mielnik #include <linux/dev_printk.h>
8e9eec291SMałgorzata Mielnik #include <linux/printk.h>
9e9eec291SMałgorzata Mielnik #include "adf_accel_devices.h"
10e9eec291SMałgorzata Mielnik #include "adf_bank_state.h"
11e9eec291SMałgorzata Mielnik #include "adf_common_drv.h"
12e9eec291SMałgorzata Mielnik 
13e9eec291SMałgorzata Mielnik /* Ring interrupt masks */
14e9eec291SMałgorzata Mielnik #define ADF_RP_INT_SRC_SEL_F_RISE_MASK	GENMASK(1, 0)
15e9eec291SMałgorzata Mielnik #define ADF_RP_INT_SRC_SEL_F_FALL_MASK	GENMASK(2, 0)
16e9eec291SMałgorzata Mielnik #define ADF_RP_INT_SRC_SEL_RANGE_WIDTH	4
17e9eec291SMałgorzata Mielnik 
18e9eec291SMałgorzata Mielnik static inline int check_stat(u32 (*op)(void __iomem *, u32), u32 expect_val,
19e9eec291SMałgorzata Mielnik 			     const char *name, void __iomem *base, u32 bank)
20e9eec291SMałgorzata Mielnik {
21e9eec291SMałgorzata Mielnik 	u32 actual_val = op(base, bank);
22e9eec291SMałgorzata Mielnik 
23e9eec291SMałgorzata Mielnik 	if (expect_val == actual_val)
24e9eec291SMałgorzata Mielnik 		return 0;
25e9eec291SMałgorzata Mielnik 
26e9eec291SMałgorzata Mielnik 	pr_err("Fail to restore %s register. Expected %#x, actual %#x\n",
27e9eec291SMałgorzata Mielnik 	       name, expect_val, actual_val);
28e9eec291SMałgorzata Mielnik 
29e9eec291SMałgorzata Mielnik 	return -EINVAL;
30e9eec291SMałgorzata Mielnik }
31e9eec291SMałgorzata Mielnik 
32e9eec291SMałgorzata Mielnik static void bank_state_save(struct adf_hw_csr_ops *ops, void __iomem *base,
33*a47dc5d1SSuman Kumar Chakraborty 			    u32 bank, struct adf_bank_state *state, u32 num_rings)
34e9eec291SMałgorzata Mielnik {
35e9eec291SMałgorzata Mielnik 	u32 i;
36e9eec291SMałgorzata Mielnik 
37e9eec291SMałgorzata Mielnik 	state->ringstat0 = ops->read_csr_stat(base, bank);
38e9eec291SMałgorzata Mielnik 	state->ringuostat = ops->read_csr_uo_stat(base, bank);
39e9eec291SMałgorzata Mielnik 	state->ringestat = ops->read_csr_e_stat(base, bank);
40e9eec291SMałgorzata Mielnik 	state->ringnestat = ops->read_csr_ne_stat(base, bank);
41e9eec291SMałgorzata Mielnik 	state->ringnfstat = ops->read_csr_nf_stat(base, bank);
42e9eec291SMałgorzata Mielnik 	state->ringfstat = ops->read_csr_f_stat(base, bank);
43e9eec291SMałgorzata Mielnik 	state->ringcstat0 = ops->read_csr_c_stat(base, bank);
44e9eec291SMałgorzata Mielnik 	state->iaintflagen = ops->read_csr_int_en(base, bank);
45e9eec291SMałgorzata Mielnik 	state->iaintflagreg = ops->read_csr_int_flag(base, bank);
46e9eec291SMałgorzata Mielnik 	state->iaintflagsrcsel0 = ops->read_csr_int_srcsel(base, bank);
47e9eec291SMałgorzata Mielnik 	state->iaintcolen = ops->read_csr_int_col_en(base, bank);
48e9eec291SMałgorzata Mielnik 	state->iaintcolctl = ops->read_csr_int_col_ctl(base, bank);
49e9eec291SMałgorzata Mielnik 	state->iaintflagandcolen = ops->read_csr_int_flag_and_col(base, bank);
50e9eec291SMałgorzata Mielnik 	state->ringexpstat = ops->read_csr_exp_stat(base, bank);
51e9eec291SMałgorzata Mielnik 	state->ringexpintenable = ops->read_csr_exp_int_en(base, bank);
52e9eec291SMałgorzata Mielnik 	state->ringsrvarben = ops->read_csr_ring_srv_arb_en(base, bank);
53e9eec291SMałgorzata Mielnik 
54e9eec291SMałgorzata Mielnik 	for (i = 0; i < num_rings; i++) {
55e9eec291SMałgorzata Mielnik 		state->rings[i].head = ops->read_csr_ring_head(base, bank, i);
56e9eec291SMałgorzata Mielnik 		state->rings[i].tail = ops->read_csr_ring_tail(base, bank, i);
57e9eec291SMałgorzata Mielnik 		state->rings[i].config = ops->read_csr_ring_config(base, bank, i);
58e9eec291SMałgorzata Mielnik 		state->rings[i].base = ops->read_csr_ring_base(base, bank, i);
59e9eec291SMałgorzata Mielnik 	}
60e9eec291SMałgorzata Mielnik }
61e9eec291SMałgorzata Mielnik 
62e9eec291SMałgorzata Mielnik static int bank_state_restore(struct adf_hw_csr_ops *ops, void __iomem *base,
63*a47dc5d1SSuman Kumar Chakraborty 			      u32 bank, struct adf_bank_state *state, u32 num_rings,
64e9eec291SMałgorzata Mielnik 			      int tx_rx_gap)
65e9eec291SMałgorzata Mielnik {
66e9eec291SMałgorzata Mielnik 	u32 val, tmp_val, i;
67e9eec291SMałgorzata Mielnik 	int ret;
68e9eec291SMałgorzata Mielnik 
69e9eec291SMałgorzata Mielnik 	for (i = 0; i < num_rings; i++)
70e9eec291SMałgorzata Mielnik 		ops->write_csr_ring_base(base, bank, i, state->rings[i].base);
71e9eec291SMałgorzata Mielnik 
72e9eec291SMałgorzata Mielnik 	for (i = 0; i < num_rings; i++)
73e9eec291SMałgorzata Mielnik 		ops->write_csr_ring_config(base, bank, i, state->rings[i].config);
74e9eec291SMałgorzata Mielnik 
75e9eec291SMałgorzata Mielnik 	for (i = 0; i < num_rings / 2; i++) {
76e9eec291SMałgorzata Mielnik 		int tx = i * (tx_rx_gap + 1);
77e9eec291SMałgorzata Mielnik 		int rx = tx + tx_rx_gap;
78e9eec291SMałgorzata Mielnik 
79e9eec291SMałgorzata Mielnik 		ops->write_csr_ring_head(base, bank, tx, state->rings[tx].head);
80e9eec291SMałgorzata Mielnik 		ops->write_csr_ring_tail(base, bank, tx, state->rings[tx].tail);
81e9eec291SMałgorzata Mielnik 
82e9eec291SMałgorzata Mielnik 		/*
83e9eec291SMałgorzata Mielnik 		 * The TX ring head needs to be updated again to make sure that
84e9eec291SMałgorzata Mielnik 		 * the HW will not consider the ring as full when it is empty
85e9eec291SMałgorzata Mielnik 		 * and the correct state flags are set to match the recovered state.
86e9eec291SMałgorzata Mielnik 		 */
87e9eec291SMałgorzata Mielnik 		if (state->ringestat & BIT(tx)) {
88e9eec291SMałgorzata Mielnik 			val = ops->read_csr_int_srcsel(base, bank);
89e9eec291SMałgorzata Mielnik 			val |= ADF_RP_INT_SRC_SEL_F_RISE_MASK;
90e9eec291SMałgorzata Mielnik 			ops->write_csr_int_srcsel_w_val(base, bank, val);
91e9eec291SMałgorzata Mielnik 			ops->write_csr_ring_head(base, bank, tx, state->rings[tx].head);
92e9eec291SMałgorzata Mielnik 		}
93e9eec291SMałgorzata Mielnik 
94e9eec291SMałgorzata Mielnik 		ops->write_csr_ring_tail(base, bank, rx, state->rings[rx].tail);
95e9eec291SMałgorzata Mielnik 		val = ops->read_csr_int_srcsel(base, bank);
96e9eec291SMałgorzata Mielnik 		val |= ADF_RP_INT_SRC_SEL_F_RISE_MASK << ADF_RP_INT_SRC_SEL_RANGE_WIDTH;
97e9eec291SMałgorzata Mielnik 		ops->write_csr_int_srcsel_w_val(base, bank, val);
98e9eec291SMałgorzata Mielnik 
99e9eec291SMałgorzata Mielnik 		ops->write_csr_ring_head(base, bank, rx, state->rings[rx].head);
100e9eec291SMałgorzata Mielnik 		val = ops->read_csr_int_srcsel(base, bank);
101e9eec291SMałgorzata Mielnik 		val |= ADF_RP_INT_SRC_SEL_F_FALL_MASK << ADF_RP_INT_SRC_SEL_RANGE_WIDTH;
102e9eec291SMałgorzata Mielnik 		ops->write_csr_int_srcsel_w_val(base, bank, val);
103e9eec291SMałgorzata Mielnik 
104e9eec291SMałgorzata Mielnik 		/*
105e9eec291SMałgorzata Mielnik 		 * The RX ring tail needs to be updated again to make sure that
106e9eec291SMałgorzata Mielnik 		 * the HW will not consider the ring as empty when it is full
107e9eec291SMałgorzata Mielnik 		 * and the correct state flags are set to match the recovered state.
108e9eec291SMałgorzata Mielnik 		 */
109e9eec291SMałgorzata Mielnik 		if (state->ringfstat & BIT(rx))
110e9eec291SMałgorzata Mielnik 			ops->write_csr_ring_tail(base, bank, rx, state->rings[rx].tail);
111e9eec291SMałgorzata Mielnik 	}
112e9eec291SMałgorzata Mielnik 
113e9eec291SMałgorzata Mielnik 	ops->write_csr_int_flag_and_col(base, bank, state->iaintflagandcolen);
114e9eec291SMałgorzata Mielnik 	ops->write_csr_int_en(base, bank, state->iaintflagen);
115e9eec291SMałgorzata Mielnik 	ops->write_csr_int_col_en(base, bank, state->iaintcolen);
116e9eec291SMałgorzata Mielnik 	ops->write_csr_int_srcsel_w_val(base, bank, state->iaintflagsrcsel0);
117e9eec291SMałgorzata Mielnik 	ops->write_csr_exp_int_en(base, bank, state->ringexpintenable);
118e9eec291SMałgorzata Mielnik 	ops->write_csr_int_col_ctl(base, bank, state->iaintcolctl);
119e9eec291SMałgorzata Mielnik 
120e9eec291SMałgorzata Mielnik 	/*
121e9eec291SMałgorzata Mielnik 	 * Verify whether any exceptions were raised during the bank save process.
122e9eec291SMałgorzata Mielnik 	 * If exceptions occurred, the status and exception registers cannot
123e9eec291SMałgorzata Mielnik 	 * be directly restored. Consequently, further restoration is not
124e9eec291SMałgorzata Mielnik 	 * feasible, and the current state of the ring should be maintained.
125e9eec291SMałgorzata Mielnik 	 */
126e9eec291SMałgorzata Mielnik 	val = state->ringexpstat;
127e9eec291SMałgorzata Mielnik 	if (val) {
128e9eec291SMałgorzata Mielnik 		pr_info("Bank %u state not fully restored due to exception in saved state (%#x)\n",
129e9eec291SMałgorzata Mielnik 			bank, val);
130e9eec291SMałgorzata Mielnik 		return 0;
131e9eec291SMałgorzata Mielnik 	}
132e9eec291SMałgorzata Mielnik 
133e9eec291SMałgorzata Mielnik 	/* Ensure that the restoration process completed without exceptions */
134e9eec291SMałgorzata Mielnik 	tmp_val = ops->read_csr_exp_stat(base, bank);
135e9eec291SMałgorzata Mielnik 	if (tmp_val) {
136e9eec291SMałgorzata Mielnik 		pr_err("Bank %u restored with exception: %#x\n", bank, tmp_val);
137e9eec291SMałgorzata Mielnik 		return -EFAULT;
138e9eec291SMałgorzata Mielnik 	}
139e9eec291SMałgorzata Mielnik 
140e9eec291SMałgorzata Mielnik 	ops->write_csr_ring_srv_arb_en(base, bank, state->ringsrvarben);
141e9eec291SMałgorzata Mielnik 
142e9eec291SMałgorzata Mielnik 	/* Check that all ring statuses match the saved state. */
143e9eec291SMałgorzata Mielnik 	ret = check_stat(ops->read_csr_stat, state->ringstat0, "ringstat",
144e9eec291SMałgorzata Mielnik 			 base, bank);
145e9eec291SMałgorzata Mielnik 	if (ret)
146e9eec291SMałgorzata Mielnik 		return ret;
147e9eec291SMałgorzata Mielnik 
148e9eec291SMałgorzata Mielnik 	ret = check_stat(ops->read_csr_e_stat, state->ringestat, "ringestat",
149e9eec291SMałgorzata Mielnik 			 base, bank);
150e9eec291SMałgorzata Mielnik 	if (ret)
151e9eec291SMałgorzata Mielnik 		return ret;
152e9eec291SMałgorzata Mielnik 
153e9eec291SMałgorzata Mielnik 	ret = check_stat(ops->read_csr_ne_stat, state->ringnestat, "ringnestat",
154e9eec291SMałgorzata Mielnik 			 base, bank);
155e9eec291SMałgorzata Mielnik 	if (ret)
156e9eec291SMałgorzata Mielnik 		return ret;
157e9eec291SMałgorzata Mielnik 
158e9eec291SMałgorzata Mielnik 	ret = check_stat(ops->read_csr_nf_stat, state->ringnfstat, "ringnfstat",
159e9eec291SMałgorzata Mielnik 			 base, bank);
160e9eec291SMałgorzata Mielnik 	if (ret)
161e9eec291SMałgorzata Mielnik 		return ret;
162e9eec291SMałgorzata Mielnik 
163e9eec291SMałgorzata Mielnik 	ret = check_stat(ops->read_csr_f_stat, state->ringfstat, "ringfstat",
164e9eec291SMałgorzata Mielnik 			 base, bank);
165e9eec291SMałgorzata Mielnik 	if (ret)
166e9eec291SMałgorzata Mielnik 		return ret;
167e9eec291SMałgorzata Mielnik 
168e9eec291SMałgorzata Mielnik 	ret = check_stat(ops->read_csr_c_stat, state->ringcstat0, "ringcstat",
169e9eec291SMałgorzata Mielnik 			 base, bank);
170e9eec291SMałgorzata Mielnik 	if (ret)
171e9eec291SMałgorzata Mielnik 		return ret;
172e9eec291SMałgorzata Mielnik 
173e9eec291SMałgorzata Mielnik 	return 0;
174e9eec291SMałgorzata Mielnik }
175e9eec291SMałgorzata Mielnik 
176e9eec291SMałgorzata Mielnik /**
177e9eec291SMałgorzata Mielnik  * adf_bank_state_save() - save state of bank-related registers
178e9eec291SMałgorzata Mielnik  * @accel_dev: Pointer to the device structure
179e9eec291SMałgorzata Mielnik  * @bank_number: Bank number
180e9eec291SMałgorzata Mielnik  * @state: Pointer to bank state structure
181e9eec291SMałgorzata Mielnik  *
182e9eec291SMałgorzata Mielnik  * This function saves the state of a bank by reading the bank CSRs and
183e9eec291SMałgorzata Mielnik  * writing them in the @state structure.
184e9eec291SMałgorzata Mielnik  *
185e9eec291SMałgorzata Mielnik  * Returns 0 on success, error code otherwise
186e9eec291SMałgorzata Mielnik  */
187e9eec291SMałgorzata Mielnik int adf_bank_state_save(struct adf_accel_dev *accel_dev, u32 bank_number,
188*a47dc5d1SSuman Kumar Chakraborty 			struct adf_bank_state *state)
189e9eec291SMałgorzata Mielnik {
190e9eec291SMałgorzata Mielnik 	struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);
191e9eec291SMałgorzata Mielnik 	struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev);
192e9eec291SMałgorzata Mielnik 	void __iomem *csr_base = adf_get_etr_base(accel_dev);
193e9eec291SMałgorzata Mielnik 
194e9eec291SMałgorzata Mielnik 	if (bank_number >= hw_data->num_banks || !state)
195e9eec291SMałgorzata Mielnik 		return -EINVAL;
196e9eec291SMałgorzata Mielnik 
197e9eec291SMałgorzata Mielnik 	dev_dbg(&GET_DEV(accel_dev), "Saving state of bank %d\n", bank_number);
198e9eec291SMałgorzata Mielnik 
199e9eec291SMałgorzata Mielnik 	bank_state_save(csr_ops, csr_base, bank_number, state,
200e9eec291SMałgorzata Mielnik 			hw_data->num_rings_per_bank);
201e9eec291SMałgorzata Mielnik 
202e9eec291SMałgorzata Mielnik 	return 0;
203e9eec291SMałgorzata Mielnik }
204e9eec291SMałgorzata Mielnik EXPORT_SYMBOL_GPL(adf_bank_state_save);
205e9eec291SMałgorzata Mielnik 
206e9eec291SMałgorzata Mielnik /**
207e9eec291SMałgorzata Mielnik  * adf_bank_state_restore() - restore state of bank-related registers
208e9eec291SMałgorzata Mielnik  * @accel_dev: Pointer to the device structure
209e9eec291SMałgorzata Mielnik  * @bank_number: Bank number
210e9eec291SMałgorzata Mielnik  * @state: Pointer to bank state structure
211e9eec291SMałgorzata Mielnik  *
212e9eec291SMałgorzata Mielnik  * This function attempts to restore the state of a bank by writing the
213e9eec291SMałgorzata Mielnik  * bank CSRs to the values in the state structure.
214e9eec291SMałgorzata Mielnik  *
215e9eec291SMałgorzata Mielnik  * Returns 0 on success, error code otherwise
216e9eec291SMałgorzata Mielnik  */
217e9eec291SMałgorzata Mielnik int adf_bank_state_restore(struct adf_accel_dev *accel_dev, u32 bank_number,
218*a47dc5d1SSuman Kumar Chakraborty 			   struct adf_bank_state *state)
219e9eec291SMałgorzata Mielnik {
220e9eec291SMałgorzata Mielnik 	struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev);
221e9eec291SMałgorzata Mielnik 	struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev);
222e9eec291SMałgorzata Mielnik 	void __iomem *csr_base = adf_get_etr_base(accel_dev);
223e9eec291SMałgorzata Mielnik 	int ret;
224e9eec291SMałgorzata Mielnik 
225e9eec291SMałgorzata Mielnik 	if (bank_number >= hw_data->num_banks  || !state)
226e9eec291SMałgorzata Mielnik 		return -EINVAL;
227e9eec291SMałgorzata Mielnik 
228e9eec291SMałgorzata Mielnik 	dev_dbg(&GET_DEV(accel_dev), "Restoring state of bank %d\n", bank_number);
229e9eec291SMałgorzata Mielnik 
230e9eec291SMałgorzata Mielnik 	ret = bank_state_restore(csr_ops, csr_base, bank_number, state,
231e9eec291SMałgorzata Mielnik 				 hw_data->num_rings_per_bank, hw_data->tx_rx_gap);
232e9eec291SMałgorzata Mielnik 	if (ret)
233e9eec291SMałgorzata Mielnik 		dev_err(&GET_DEV(accel_dev),
234e9eec291SMałgorzata Mielnik 			"Unable to restore state of bank %d\n", bank_number);
235e9eec291SMałgorzata Mielnik 
236e9eec291SMałgorzata Mielnik 	return ret;
237e9eec291SMałgorzata Mielnik }
238e9eec291SMałgorzata Mielnik EXPORT_SYMBOL_GPL(adf_bank_state_restore);
239