1 /* SPDX-License-Identifier: BSD-3-Clause */
2 /* Copyright(c) 2007-2022 Intel Corporation */
3 #include "qat_freebsd.h"
4 #include "adf_cfg.h"
5 #include "adf_common_drv.h"
6 #include "adf_accel_devices.h"
7 #include "icp_qat_uclo.h"
8 #include "icp_qat_fw.h"
9 #include "icp_qat_fw_init_admin.h"
10 #include "adf_cfg_strings.h"
11 #include "adf_uio_control.h"
12 #include "adf_uio_cleanup.h"
13 #include "adf_uio.h"
14 #include "adf_transport_access_macros.h"
15 #include "adf_transport_internal.h"
16 #include <sys/param.h>
17 #include <sys/lock.h>
18 #include <sys/rwlock.h>
19 #include <sys/sglist.h>
20 #include <sys/systm.h>
21 #include <sys/proc.h>
22 #include <vm/vm.h>
23 #include <vm/pmap.h>
24 #include <vm/vm_kern.h>
25 #include <vm/vm_map.h>
26 #include <vm/vm_object.h>
27 #include <vm/vm_page.h>
28 #include <vm/vm_pager.h>
29 #include <vm/vm_param.h>
30
31 #define TX_RINGS_DISABLE 0
32 #define TX_RINGS_ENABLE 1
33 #define PKE_REQ_SIZE 64
34 #define BASE_ADDR_SHIFT 6
35 #define PKE_RX_RING_0 0
36 #define PKE_RX_RING_1 1
37
38 #define ADF_RING_EMPTY_RETRY_DELAY 2
39 #define ADF_RING_EMPTY_MAX_RETRY 15
40
41 struct bundle_orphan_ring {
42 unsigned long tx_mask;
43 unsigned long rx_mask;
44 unsigned long asym_mask;
45 int bank;
46 struct resource *csr_base;
47 struct adf_uio_control_bundle *bundle;
48 };
49
50 /*
51 * if orphan->tx_mask does not match with orphan->rx_mask
52 */
53 static void
check_orphan_ring(struct adf_accel_dev * accel_dev,struct bundle_orphan_ring * orphan,struct adf_hw_device_data * hw_data)54 check_orphan_ring(struct adf_accel_dev *accel_dev,
55 struct bundle_orphan_ring *orphan,
56 struct adf_hw_device_data *hw_data)
57 {
58 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev);
59 int i;
60 int tx_rx_gap = hw_data->tx_rx_gap;
61 u8 num_rings_per_bank = hw_data->num_rings_per_bank;
62 struct resource *csr_base = orphan->csr_base;
63 int bank = orphan->bank;
64
65 for (i = 0; i < num_rings_per_bank; i++) {
66 if (test_bit(i, &orphan->tx_mask)) {
67 int rx_ring = i + tx_rx_gap;
68
69 if (!test_bit(rx_ring, &orphan->rx_mask)) {
70 __clear_bit(i, &orphan->tx_mask);
71
72 /* clean up this tx ring */
73 csr_ops->write_csr_ring_config(csr_base,
74 bank,
75 i,
76 0);
77 csr_ops->write_csr_ring_base(csr_base,
78 bank,
79 i,
80 0);
81 }
82
83 } else if (test_bit(i, &orphan->rx_mask)) {
84 int tx_ring = i - tx_rx_gap;
85
86 if (!test_bit(tx_ring, &orphan->tx_mask)) {
87 __clear_bit(i, &orphan->rx_mask);
88
89 /* clean up this rx ring */
90 csr_ops->write_csr_ring_config(csr_base,
91 bank,
92 i,
93 0);
94 csr_ops->write_csr_ring_base(csr_base,
95 bank,
96 i,
97 0);
98 }
99 }
100 }
101 }
102
103 static int
get_orphan_bundle(int bank,struct adf_uio_control_accel * accel,struct bundle_orphan_ring ** orphan_bundle_out)104 get_orphan_bundle(int bank,
105 struct adf_uio_control_accel *accel,
106 struct bundle_orphan_ring **orphan_bundle_out)
107 {
108 int i;
109 int ret = 0;
110 struct resource *csr_base;
111 unsigned long tx_mask;
112 unsigned long asym_mask;
113 struct adf_accel_dev *accel_dev = accel->accel_dev;
114 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev);
115 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
116 u8 num_rings_per_bank = hw_data->num_rings_per_bank;
117 struct bundle_orphan_ring *orphan_bundle = NULL;
118 uint64_t base;
119 struct list_head *entry;
120 struct adf_uio_instance_rings *instance_rings;
121 struct adf_uio_control_bundle *bundle;
122 u16 ring_mask = 0;
123
124 orphan_bundle =
125 malloc(sizeof(*orphan_bundle), M_QAT, M_WAITOK | M_ZERO);
126 csr_base = accel->bar->virt_addr;
127 orphan_bundle->csr_base = csr_base;
128 orphan_bundle->bank = bank;
129
130 orphan_bundle->tx_mask = 0;
131 orphan_bundle->rx_mask = 0;
132 tx_mask = accel_dev->hw_device->tx_rings_mask;
133 asym_mask = accel_dev->hw_device->asym_rings_mask;
134
135 /* Get ring mask for this process. */
136 bundle = &accel->bundle[bank];
137 orphan_bundle->bundle = bundle;
138 mutex_lock(&bundle->list_lock);
139 list_for_each(entry, &bundle->list)
140 {
141 instance_rings =
142 list_entry(entry, struct adf_uio_instance_rings, list);
143 if (instance_rings->user_pid == curproc->p_pid) {
144 ring_mask = instance_rings->ring_mask;
145 break;
146 }
147 }
148 mutex_unlock(&bundle->list_lock);
149
150 for (i = 0; i < num_rings_per_bank; i++) {
151 base = csr_ops->read_csr_ring_base(csr_base, bank, i);
152
153 if (!base)
154 continue;
155 if (!(ring_mask & 1 << i))
156 continue; /* Not reserved for this process. */
157
158 if (test_bit(i, &tx_mask))
159 __set_bit(i, &orphan_bundle->tx_mask);
160 else
161 __set_bit(i, &orphan_bundle->rx_mask);
162
163 if (test_bit(i, &asym_mask))
164 __set_bit(i, &orphan_bundle->asym_mask);
165 }
166
167 if (orphan_bundle->tx_mask || orphan_bundle->rx_mask)
168 check_orphan_ring(accel_dev, orphan_bundle, hw_data);
169
170 *orphan_bundle_out = orphan_bundle;
171 return ret;
172 }
173
174 static void
put_orphan_bundle(struct bundle_orphan_ring * bundle)175 put_orphan_bundle(struct bundle_orphan_ring *bundle)
176 {
177 if (!bundle)
178 return;
179
180 free(bundle, M_QAT);
181 }
182
183 /* cleanup all ring */
184 static void
cleanup_all_ring(struct adf_uio_control_accel * accel,struct bundle_orphan_ring * orphan)185 cleanup_all_ring(struct adf_uio_control_accel *accel,
186 struct bundle_orphan_ring *orphan)
187 {
188 int i;
189 struct resource *csr_base = orphan->csr_base;
190 unsigned long mask = orphan->rx_mask | orphan->tx_mask;
191 struct adf_accel_dev *accel_dev = accel->accel_dev;
192 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev);
193 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
194 u8 num_rings_per_bank = hw_data->num_rings_per_bank;
195 int bank = orphan->bank;
196
197 mutex_lock(&orphan->bundle->lock);
198 orphan->bundle->rings_enabled &= ~mask;
199 adf_update_uio_ring_arb(orphan->bundle);
200 mutex_unlock(&orphan->bundle->lock);
201
202 for (i = 0; i < num_rings_per_bank; i++) {
203 if (!test_bit(i, &mask))
204 continue;
205
206 csr_ops->write_csr_ring_config(csr_base, bank, i, 0);
207 csr_ops->write_csr_ring_base(csr_base, bank, i, 0);
208 }
209 }
210
211 /*
212 * Return true, if number of messages in tx ring is equal to number
213 * of messages in corresponding rx ring, else false.
214 */
215 static bool
is_all_resp_recvd(struct adf_hw_csr_ops * csr_ops,struct bundle_orphan_ring * bundle,const u8 num_rings_per_bank)216 is_all_resp_recvd(struct adf_hw_csr_ops *csr_ops,
217 struct bundle_orphan_ring *bundle,
218 const u8 num_rings_per_bank)
219 {
220 u32 rx_tail = 0, tx_head = 0, rx_ring_msg_offset = 0,
221 tx_ring_msg_offset = 0, tx_rx_offset = num_rings_per_bank / 2,
222 idx = 0, retry = 0, delay = ADF_RING_EMPTY_RETRY_DELAY;
223
224 do {
225 for_each_set_bit(idx, &bundle->tx_mask, tx_rx_offset)
226 {
227 rx_tail =
228 csr_ops->read_csr_ring_tail(bundle->csr_base,
229 0,
230 (idx + tx_rx_offset));
231 tx_head = csr_ops->read_csr_ring_head(bundle->csr_base,
232 0,
233 idx);
234
235 /*
236 * Normalize messages in tx rings to match rx ring
237 * message size, i.e., size of response message(32).
238 * Asym messages are 64 bytes each, so right shift
239 * by 1 to normalize to 32. Sym and compression
240 * messages are 128 bytes each, so right shift by 2
241 * to normalize to 32.
242 */
243 if (bundle->asym_mask & (1 << idx))
244 tx_ring_msg_offset = (tx_head >> 1);
245 else
246 tx_ring_msg_offset = (tx_head >> 2);
247
248 rx_ring_msg_offset = rx_tail;
249
250 if (tx_ring_msg_offset != rx_ring_msg_offset)
251 break;
252 }
253 if (idx == tx_rx_offset)
254 /* All Tx and Rx ring message counts match */
255 return true;
256
257 DELAY(delay);
258 delay *= 2;
259 } while (++retry < ADF_RING_EMPTY_MAX_RETRY);
260
261 return false;
262 }
263
264 static int
bundle_need_cleanup(int bank,struct adf_uio_control_accel * accel)265 bundle_need_cleanup(int bank, struct adf_uio_control_accel *accel)
266 {
267 struct resource *csr_base = accel->bar->virt_addr;
268 struct adf_accel_dev *accel_dev = accel->accel_dev;
269 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev);
270 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
271 u8 num_rings_per_bank = hw_data->num_rings_per_bank;
272 int i;
273
274 if (!csr_base)
275 return 0;
276
277 for (i = 0; i < num_rings_per_bank; i++) {
278 if (csr_ops->read_csr_ring_base(csr_base, bank, i))
279 return 1;
280 }
281
282 return 0;
283 }
284
285 static void
cleanup_orphan_ring(struct bundle_orphan_ring * orphan,struct adf_uio_control_accel * accel)286 cleanup_orphan_ring(struct bundle_orphan_ring *orphan,
287 struct adf_uio_control_accel *accel)
288 {
289 struct adf_accel_dev *accel_dev = accel->accel_dev;
290 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev);
291 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
292 u8 number_rings_per_bank = hw_data->num_rings_per_bank;
293
294 /* disable the interrupt */
295 csr_ops->write_csr_int_col_en(orphan->csr_base, orphan->bank, 0);
296
297 /*
298 * wait firmware finish the in-process ring
299 * 1. disable all tx rings
300 * 2. check if all responses are received
301 * 3. reset all rings
302 */
303 adf_disable_ring_arb(accel_dev, orphan->csr_base, 0, orphan->tx_mask);
304
305 if (!is_all_resp_recvd(csr_ops, orphan, number_rings_per_bank)) {
306 device_printf(GET_DEV(accel_dev),
307 "Failed to clean up orphan rings");
308 return;
309 }
310
311 /*
312 * When the execution reaches here, it is assumed that
313 * there is no inflight request in the rings and that
314 * there is no in-process ring.
315 */
316
317 cleanup_all_ring(accel, orphan);
318 }
319
320 void
adf_uio_do_cleanup_orphan(int bank,struct adf_uio_control_accel * accel)321 adf_uio_do_cleanup_orphan(int bank, struct adf_uio_control_accel *accel)
322 {
323 int ret;
324 struct adf_uio_instance_rings *instance_rings, *tmp;
325 struct adf_uio_control_bundle *bundle;
326 /* orphan is local pointer allocated and deallocated in this function */
327 struct bundle_orphan_ring *orphan = NULL;
328 struct adf_accel_dev *accel_dev = accel->accel_dev;
329 struct adf_hw_device_data *hw_data = accel_dev->hw_device;
330
331 if (!bundle_need_cleanup(bank, accel))
332 goto release;
333
334 ret = get_orphan_bundle(bank, accel, &orphan);
335 if (ret != 0)
336 return;
337
338 /*
339 * If driver supports ring pair reset, no matter process
340 * exits normally or abnormally, just do ring pair reset.
341 * ring pair reset will reset all ring pair registers to
342 * default value. Driver only needs to reset ring mask
343 */
344 if (hw_data->ring_pair_reset) {
345 hw_data->ring_pair_reset(
346 accel_dev, orphan->bundle->hardware_bundle_number);
347 /*
348 * If processes exit normally, rx_mask, tx_mask
349 * and rings_enabled are all 0, below expression
350 * have no impact on rings_enabled.
351 * If processes exit abnormally, rings_enabled
352 * will be set as 0 by below expression.
353 */
354 orphan->bundle->rings_enabled &=
355 ~(orphan->rx_mask | orphan->tx_mask);
356 goto out;
357 }
358
359 if (!orphan->tx_mask && !orphan->rx_mask)
360 goto out;
361
362 device_printf(GET_DEV(accel_dev),
363 "Process %d %s exit with orphan rings %lx:%lx\n",
364 curproc->p_pid,
365 curproc->p_comm,
366 orphan->tx_mask,
367 orphan->rx_mask);
368
369 if (!test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) {
370 cleanup_orphan_ring(orphan, accel);
371 }
372 out:
373 put_orphan_bundle(orphan);
374
375 release:
376
377 bundle = &accel->bundle[bank];
378 /*
379 * If the user process died without releasing the rings
380 * then force a release here.
381 */
382 mutex_lock(&bundle->list_lock);
383 list_for_each_entry_safe(instance_rings, tmp, &bundle->list, list)
384 {
385 if (instance_rings->user_pid == curproc->p_pid) {
386 bundle->rings_used &= ~instance_rings->ring_mask;
387 break;
388 }
389 }
390 mutex_unlock(&bundle->list_lock);
391 }
392