1 /* SPDX-License-Identifier: BSD-3-Clause */
2 /* Copyright(c) 2007-2022 Intel Corporation */
3 #include "qat_freebsd.h"
4 #include "adf_cfg.h"
5 #include "adf_common_drv.h"
6 #include "adf_accel_devices.h"
7 #include "icp_qat_uclo.h"
8 #include "icp_qat_fw.h"
9 #include "icp_qat_fw_init_admin.h"
10 #include "adf_cfg_strings.h"
11 #include "adf_transport_access_macros.h"
12 #include "adf_transport_internal.h"
13 #include "adf_accel_devices.h"
14 #include "adf_common_drv.h"
15 #include "adf_transport_internal.h"
16
17 #define ADF_ARB_NUM 4
18 #define ADF_ARB_REG_SIZE 0x4
19 #define ADF_ARB_WTR_SIZE 0x20
20 #define ADF_ARB_OFFSET 0x30000
21 #define ADF_ARB_REG_SLOT 0x1000
22 #define ADF_ARB_WTR_OFFSET 0x010
23 #define ADF_ARB_RO_EN_OFFSET 0x090
24 #define ADF_ARB_WQCFG_OFFSET 0x100
25 #define ADF_ARB_WRK_2_SER_MAP_OFFSET 0x180
26 #define ADF_ARB_RINGSRVARBEN_OFFSET 0x19C
27
28 #define WRITE_CSR_ARB_RINGSRVARBEN(csr_addr, index, value) \
29 ADF_CSR_WR(csr_addr, \
30 ADF_ARB_RINGSRVARBEN_OFFSET + (ADF_ARB_REG_SLOT * (index)), \
31 value)
32
33 #define WRITE_CSR_ARB_SARCONFIG(csr_addr, csr_offset, index, value) \
34 ADF_CSR_WR(csr_addr, (csr_offset) + (ADF_ARB_REG_SIZE * (index)), value)
35 #define READ_CSR_ARB_RINGSRVARBEN(csr_addr, index) \
36 ADF_CSR_RD(csr_addr, \
37 ADF_ARB_RINGSRVARBEN_OFFSET + (ADF_ARB_REG_SLOT * (index)))
38
39 static DEFINE_MUTEX(csr_arb_lock);
40
41 #define WRITE_CSR_ARB_WRK_2_SER_MAP( \
42 csr_addr, csr_offset, wrk_to_ser_map_offset, index, value) \
43 ADF_CSR_WR(csr_addr, \
44 ((csr_offset) + (wrk_to_ser_map_offset)) + \
45 (ADF_ARB_REG_SIZE * (index)), \
46 value)
47
48 int
adf_init_arb(struct adf_accel_dev * accel_dev)49 adf_init_arb(struct adf_accel_dev *accel_dev)
50 {
51 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
52 struct arb_info info;
53 struct resource *csr = accel_dev->transport->banks[0].csr_addr;
54 u32 arb_cfg = 0x1 << 31 | 0x4 << 4 | 0x1;
55 u32 arb;
56
57 hw_data->get_arb_info(&info);
58
59 /* Service arb configured for 32 bytes responses and
60 * ring flow control check enabled.
61 */
62 for (arb = 0; arb < ADF_ARB_NUM; arb++)
63 WRITE_CSR_ARB_SARCONFIG(csr, info.arbiter_offset, arb, arb_cfg);
64
65 return 0;
66 }
67
68 int
adf_init_gen2_arb(struct adf_accel_dev * accel_dev)69 adf_init_gen2_arb(struct adf_accel_dev *accel_dev)
70 {
71 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
72 struct arb_info info;
73 struct resource *csr = accel_dev->transport->banks[0].csr_addr;
74 u32 i;
75 const u32 *thd_2_arb_cfg;
76
77 /* invoke common adf_init_arb */
78 adf_init_arb(accel_dev);
79
80 hw_data->get_arb_info(&info);
81
82 /* Map worker threads to service arbiters */
83 hw_data->get_arb_mapping(accel_dev, &thd_2_arb_cfg);
84 if (!thd_2_arb_cfg)
85 return EFAULT;
86
87 for (i = 0; i < hw_data->num_engines; i++)
88 WRITE_CSR_ARB_WRK_2_SER_MAP(csr,
89 info.arbiter_offset,
90 info.wrk_thd_2_srv_arb_map,
91 i,
92 *(thd_2_arb_cfg + i));
93 return 0;
94 }
95
96 void
adf_update_ring_arb(struct adf_etr_ring_data * ring)97 adf_update_ring_arb(struct adf_etr_ring_data *ring)
98 {
99 int shift;
100 u32 arben, arben_tx, arben_rx, arb_mask;
101 struct adf_accel_dev *accel_dev = ring->bank->accel_dev;
102 struct adf_hw_csr_info *csr_info = &accel_dev->hw_device->csr_info;
103 struct adf_hw_csr_ops *csr_ops = &csr_info->csr_ops;
104
105 arb_mask = csr_info->arb_enable_mask;
106 shift = hweight32(arb_mask);
107
108 arben_tx = ring->bank->ring_mask & arb_mask;
109 arben_rx = (ring->bank->ring_mask >> shift) & arb_mask;
110 arben = arben_tx & arben_rx;
111 csr_ops->write_csr_ring_srv_arb_en(ring->bank->csr_addr,
112 ring->bank->bank_number,
113 arben);
114 }
115
116 void
adf_update_uio_ring_arb(struct adf_uio_control_bundle * bundle)117 adf_update_uio_ring_arb(struct adf_uio_control_bundle *bundle)
118 {
119 int shift;
120 u32 arben, arben_tx, arben_rx, arb_mask;
121 struct adf_accel_dev *accel_dev = bundle->uio_priv.accel->accel_dev;
122 struct adf_hw_csr_info *csr_info = &accel_dev->hw_device->csr_info;
123 struct adf_hw_csr_ops *csr_ops = &csr_info->csr_ops;
124
125 arb_mask = csr_info->arb_enable_mask;
126 shift = hweight32(arb_mask);
127
128 arben_tx = bundle->rings_enabled & arb_mask;
129 arben_rx = (bundle->rings_enabled >> shift) & arb_mask;
130 arben = arben_tx & arben_rx;
131 csr_ops->write_csr_ring_srv_arb_en(bundle->csr_addr,
132 bundle->hardware_bundle_number,
133 arben);
134 }
135 void
adf_enable_ring_arb(struct adf_accel_dev * accel_dev,void * csr_addr,unsigned int bank_nr,unsigned int mask)136 adf_enable_ring_arb(struct adf_accel_dev *accel_dev,
137 void *csr_addr,
138 unsigned int bank_nr,
139 unsigned int mask)
140 {
141 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev);
142 u32 arbenable;
143
144 if (!csr_addr)
145 return;
146
147 mutex_lock(&csr_arb_lock);
148 arbenable = csr_ops->read_csr_ring_srv_arb_en(csr_addr, bank_nr);
149 arbenable |= mask & 0xFF;
150 csr_ops->write_csr_ring_srv_arb_en(csr_addr, bank_nr, arbenable);
151
152 mutex_unlock(&csr_arb_lock);
153 }
154
155 void
adf_disable_ring_arb(struct adf_accel_dev * accel_dev,void * csr_addr,unsigned int bank_nr,unsigned int mask)156 adf_disable_ring_arb(struct adf_accel_dev *accel_dev,
157 void *csr_addr,
158 unsigned int bank_nr,
159 unsigned int mask)
160 {
161 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev);
162 struct resource *csr = csr_addr;
163 u32 arbenable;
164
165 if (!csr_addr)
166 return;
167
168 mutex_lock(&csr_arb_lock);
169 arbenable = csr_ops->read_csr_ring_srv_arb_en(csr, bank_nr);
170 arbenable &= ~mask & 0xFF;
171 csr_ops->write_csr_ring_srv_arb_en(csr, bank_nr, arbenable);
172 mutex_unlock(&csr_arb_lock);
173 }
174
175 void
adf_exit_arb(struct adf_accel_dev * accel_dev)176 adf_exit_arb(struct adf_accel_dev *accel_dev)
177 {
178 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev);
179 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
180 struct arb_info info;
181 struct resource *csr;
182 unsigned int i;
183
184 if (!accel_dev->transport)
185 return;
186
187 csr = accel_dev->transport->banks[0].csr_addr;
188
189 hw_data->get_arb_info(&info);
190
191 /* Reset arbiter configuration */
192 for (i = 0; i < ADF_ARB_NUM; i++)
193 WRITE_CSR_ARB_SARCONFIG(csr, info.arbiter_offset, i, 0);
194
195 /* Unmap worker threads to service arbiters */
196 if (hw_data->get_arb_mapping) {
197 for (i = 0; i < hw_data->num_engines; i++)
198 WRITE_CSR_ARB_WRK_2_SER_MAP(csr,
199 info.arbiter_offset,
200 info.wrk_thd_2_srv_arb_map,
201 i,
202 0);
203 }
204
205 /* Disable arbitration on all rings */
206 for (i = 0; i < GET_MAX_BANKS(accel_dev); i++)
207 csr_ops->write_csr_ring_srv_arb_en(csr, i, 0);
208 }
209
210 void
adf_disable_arb(struct adf_accel_dev * accel_dev)211 adf_disable_arb(struct adf_accel_dev *accel_dev)
212 {
213 struct adf_hw_csr_ops *csr_ops;
214 struct resource *csr;
215 unsigned int i;
216
217 if (!accel_dev || !accel_dev->transport)
218 return;
219
220 csr = accel_dev->transport->banks[0].csr_addr;
221 csr_ops = GET_CSR_OPS(accel_dev);
222
223 /* Disable arbitration on all rings */
224 for (i = 0; i < GET_MAX_BANKS(accel_dev); i++)
225 csr_ops->write_csr_ring_srv_arb_en(csr, i, 0);
226 }
227