1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright(c) 2007-2022 Intel Corporation */ 3 /* $FreeBSD$ */ 4 #include "qat_freebsd.h" 5 #include "adf_cfg.h" 6 #include "adf_common_drv.h" 7 #include "adf_accel_devices.h" 8 #include "icp_qat_uclo.h" 9 #include "icp_qat_fw.h" 10 #include "icp_qat_fw_init_admin.h" 11 #include "adf_cfg_strings.h" 12 #include "adf_uio_control.h" 13 #include "adf_uio_cleanup.h" 14 #include "adf_uio.h" 15 #include "adf_transport_access_macros.h" 16 #include "adf_transport_internal.h" 17 #include <sys/param.h> 18 #include <sys/lock.h> 19 #include <sys/rwlock.h> 20 #include <sys/sglist.h> 21 #include <sys/systm.h> 22 #include <sys/proc.h> 23 #include <vm/vm.h> 24 #include <vm/pmap.h> 25 #include <vm/vm_kern.h> 26 #include <vm/vm_map.h> 27 #include <vm/vm_object.h> 28 #include <vm/vm_page.h> 29 #include <vm/vm_pager.h> 30 #include <vm/vm_param.h> 31 32 #define TX_RINGS_DISABLE 0 33 #define TX_RINGS_ENABLE 1 34 #define PKE_REQ_SIZE 64 35 #define BASE_ADDR_SHIFT 6 36 #define PKE_RX_RING_0 0 37 #define PKE_RX_RING_1 1 38 39 #define ADF_RING_EMPTY_RETRY_DELAY 2 40 #define ADF_RING_EMPTY_MAX_RETRY 15 41 42 struct bundle_orphan_ring { 43 unsigned long tx_mask; 44 unsigned long rx_mask; 45 unsigned long asym_mask; 46 int bank; 47 struct resource *csr_base; 48 struct adf_uio_control_bundle *bundle; 49 }; 50 51 /* 52 * if orphan->tx_mask does not match with orphan->rx_mask 53 */ 54 static void 55 check_orphan_ring(struct adf_accel_dev *accel_dev, 56 struct bundle_orphan_ring *orphan, 57 struct adf_hw_device_data *hw_data) 58 { 59 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); 60 int i; 61 int tx_rx_gap = hw_data->tx_rx_gap; 62 u8 num_rings_per_bank = hw_data->num_rings_per_bank; 63 struct resource *csr_base = orphan->csr_base; 64 int bank = orphan->bank; 65 66 for (i = 0; i < num_rings_per_bank; i++) { 67 if (test_bit(i, &orphan->tx_mask)) { 68 int rx_ring = i + tx_rx_gap; 69 70 if (!test_bit(rx_ring, &orphan->rx_mask)) { 71 __clear_bit(i, &orphan->tx_mask); 72 73 /* clean up this tx ring */ 74 csr_ops->write_csr_ring_config(csr_base, 75 bank, 76 i, 77 0); 78 csr_ops->write_csr_ring_base(csr_base, 79 bank, 80 i, 81 0); 82 } 83 84 } else if (test_bit(i, &orphan->rx_mask)) { 85 int tx_ring = i - tx_rx_gap; 86 87 if (!test_bit(tx_ring, &orphan->tx_mask)) { 88 __clear_bit(i, &orphan->rx_mask); 89 90 /* clean up this rx ring */ 91 csr_ops->write_csr_ring_config(csr_base, 92 bank, 93 i, 94 0); 95 csr_ops->write_csr_ring_base(csr_base, 96 bank, 97 i, 98 0); 99 } 100 } 101 } 102 } 103 104 static int 105 get_orphan_bundle(int bank, 106 struct adf_uio_control_accel *accel, 107 struct bundle_orphan_ring **orphan_bundle_out) 108 { 109 int i; 110 int ret = 0; 111 struct resource *csr_base; 112 unsigned long tx_mask; 113 unsigned long asym_mask; 114 struct adf_accel_dev *accel_dev = accel->accel_dev; 115 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); 116 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 117 u8 num_rings_per_bank = hw_data->num_rings_per_bank; 118 struct bundle_orphan_ring *orphan_bundle = NULL; 119 uint64_t base; 120 struct list_head *entry; 121 struct adf_uio_instance_rings *instance_rings; 122 struct adf_uio_control_bundle *bundle; 123 u16 ring_mask = 0; 124 125 orphan_bundle = 126 malloc(sizeof(*orphan_bundle), M_QAT, M_WAITOK | M_ZERO); 127 if (!orphan_bundle) 128 return ENOMEM; 129 130 csr_base = accel->bar->virt_addr; 131 orphan_bundle->csr_base = csr_base; 132 orphan_bundle->bank = bank; 133 134 orphan_bundle->tx_mask = 0; 135 orphan_bundle->rx_mask = 0; 136 tx_mask = accel_dev->hw_device->tx_rings_mask; 137 asym_mask = accel_dev->hw_device->asym_rings_mask; 138 139 /* Get ring mask for this process. */ 140 bundle = &accel->bundle[bank]; 141 orphan_bundle->bundle = bundle; 142 mutex_lock(&bundle->list_lock); 143 list_for_each(entry, &bundle->list) 144 { 145 instance_rings = 146 list_entry(entry, struct adf_uio_instance_rings, list); 147 if (instance_rings->user_pid == curproc->p_pid) { 148 ring_mask = instance_rings->ring_mask; 149 break; 150 } 151 } 152 mutex_unlock(&bundle->list_lock); 153 154 for (i = 0; i < num_rings_per_bank; i++) { 155 base = csr_ops->read_csr_ring_base(csr_base, bank, i); 156 157 if (!base) 158 continue; 159 if (!(ring_mask & 1 << i)) 160 continue; /* Not reserved for this process. */ 161 162 if (test_bit(i, &tx_mask)) 163 __set_bit(i, &orphan_bundle->tx_mask); 164 else 165 __set_bit(i, &orphan_bundle->rx_mask); 166 167 if (test_bit(i, &asym_mask)) 168 __set_bit(i, &orphan_bundle->asym_mask); 169 } 170 171 if (orphan_bundle->tx_mask || orphan_bundle->rx_mask) 172 check_orphan_ring(accel_dev, orphan_bundle, hw_data); 173 174 *orphan_bundle_out = orphan_bundle; 175 return ret; 176 } 177 178 static void 179 put_orphan_bundle(struct bundle_orphan_ring *bundle) 180 { 181 if (!bundle) 182 return; 183 184 free(bundle, M_QAT); 185 } 186 187 /* cleanup all ring */ 188 static void 189 cleanup_all_ring(struct adf_uio_control_accel *accel, 190 struct bundle_orphan_ring *orphan) 191 { 192 int i; 193 struct resource *csr_base = orphan->csr_base; 194 unsigned long mask = orphan->rx_mask | orphan->tx_mask; 195 struct adf_accel_dev *accel_dev = accel->accel_dev; 196 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); 197 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 198 u8 num_rings_per_bank = hw_data->num_rings_per_bank; 199 int bank = orphan->bank; 200 201 mutex_lock(&orphan->bundle->lock); 202 orphan->bundle->rings_enabled &= ~mask; 203 adf_update_uio_ring_arb(orphan->bundle); 204 mutex_unlock(&orphan->bundle->lock); 205 206 for (i = 0; i < num_rings_per_bank; i++) { 207 if (!test_bit(i, &mask)) 208 continue; 209 210 csr_ops->write_csr_ring_config(csr_base, bank, i, 0); 211 csr_ops->write_csr_ring_base(csr_base, bank, i, 0); 212 } 213 } 214 215 /* 216 * Return true, if number of messages in tx ring is equal to number 217 * of messages in corresponding rx ring, else false. 218 */ 219 static bool 220 is_all_resp_recvd(struct adf_hw_csr_ops *csr_ops, 221 struct bundle_orphan_ring *bundle, 222 const u8 num_rings_per_bank) 223 { 224 u32 rx_tail = 0, tx_head = 0, rx_ring_msg_offset = 0, 225 tx_ring_msg_offset = 0, tx_rx_offset = num_rings_per_bank / 2, 226 idx = 0, retry = 0, delay = ADF_RING_EMPTY_RETRY_DELAY; 227 228 do { 229 for_each_set_bit(idx, &bundle->tx_mask, tx_rx_offset) 230 { 231 rx_tail = 232 csr_ops->read_csr_ring_tail(bundle->csr_base, 233 0, 234 (idx + tx_rx_offset)); 235 tx_head = csr_ops->read_csr_ring_head(bundle->csr_base, 236 0, 237 idx); 238 239 /* 240 * Normalize messages in tx rings to match rx ring 241 * message size, i.e., size of response message(32). 242 * Asym messages are 64 bytes each, so right shift 243 * by 1 to normalize to 32. Sym and compression 244 * messages are 128 bytes each, so right shift by 2 245 * to normalize to 32. 246 */ 247 if (bundle->asym_mask & (1 << idx)) 248 tx_ring_msg_offset = (tx_head >> 1); 249 else 250 tx_ring_msg_offset = (tx_head >> 2); 251 252 rx_ring_msg_offset = rx_tail; 253 254 if (tx_ring_msg_offset != rx_ring_msg_offset) 255 break; 256 } 257 if (idx == tx_rx_offset) 258 /* All Tx and Rx ring message counts match */ 259 return true; 260 261 DELAY(delay); 262 delay *= 2; 263 } while (++retry < ADF_RING_EMPTY_MAX_RETRY); 264 265 return false; 266 } 267 268 static int 269 bundle_need_cleanup(int bank, struct adf_uio_control_accel *accel) 270 { 271 struct resource *csr_base = accel->bar->virt_addr; 272 struct adf_accel_dev *accel_dev = accel->accel_dev; 273 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); 274 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 275 u8 num_rings_per_bank = hw_data->num_rings_per_bank; 276 int i; 277 278 if (!csr_base) 279 return 0; 280 281 for (i = 0; i < num_rings_per_bank; i++) { 282 if (csr_ops->read_csr_ring_base(csr_base, bank, i)) 283 return 1; 284 } 285 286 return 0; 287 } 288 289 static void 290 cleanup_orphan_ring(struct bundle_orphan_ring *orphan, 291 struct adf_uio_control_accel *accel) 292 { 293 struct adf_accel_dev *accel_dev = accel->accel_dev; 294 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); 295 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 296 u8 number_rings_per_bank = hw_data->num_rings_per_bank; 297 298 /* disable the interrupt */ 299 csr_ops->write_csr_int_col_en(orphan->csr_base, orphan->bank, 0); 300 301 /* 302 * wait firmware finish the in-process ring 303 * 1. disable all tx rings 304 * 2. check if all responses are received 305 * 3. reset all rings 306 */ 307 adf_disable_ring_arb(accel_dev, orphan->csr_base, 0, orphan->tx_mask); 308 309 if (!is_all_resp_recvd(csr_ops, orphan, number_rings_per_bank)) { 310 device_printf(GET_DEV(accel_dev), 311 "Failed to clean up orphan rings"); 312 return; 313 } 314 315 /* 316 * When the execution reaches here, it is assumed that 317 * there is no inflight request in the rings and that 318 * there is no in-process ring. 319 */ 320 321 cleanup_all_ring(accel, orphan); 322 } 323 324 void 325 adf_uio_do_cleanup_orphan(int bank, struct adf_uio_control_accel *accel) 326 { 327 int ret, pid_found; 328 struct adf_uio_instance_rings *instance_rings, *tmp; 329 struct adf_uio_control_bundle *bundle; 330 /* orphan is local pointer allocated and deallocated in this function */ 331 struct bundle_orphan_ring *orphan = NULL; 332 struct adf_accel_dev *accel_dev = accel->accel_dev; 333 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 334 335 if (!bundle_need_cleanup(bank, accel)) 336 goto release; 337 338 ret = get_orphan_bundle(bank, accel, &orphan); 339 if (ret != 0) 340 return; 341 342 /* 343 * If driver supports ring pair reset, no matter process 344 * exits normally or abnormally, just do ring pair reset. 345 * ring pair reset will reset all ring pair registers to 346 * default value. Driver only needs to reset ring mask 347 */ 348 if (hw_data->ring_pair_reset) { 349 hw_data->ring_pair_reset( 350 accel_dev, orphan->bundle->hardware_bundle_number); 351 mutex_lock(&orphan->bundle->lock); 352 /* 353 * If processes exit normally, rx_mask, tx_mask 354 * and rings_enabled are all 0, below expression 355 * have no impact on rings_enabled. 356 * If processes exit abnormally, rings_enabled 357 * will be set as 0 by below expression. 358 */ 359 orphan->bundle->rings_enabled &= 360 ~(orphan->rx_mask | orphan->tx_mask); 361 mutex_unlock(&orphan->bundle->lock); 362 goto out; 363 } 364 365 if (!orphan->tx_mask && !orphan->rx_mask) 366 goto out; 367 368 device_printf(GET_DEV(accel_dev), 369 "Process %d %s exit with orphan rings %lx:%lx\n", 370 curproc->p_pid, 371 curproc->p_comm, 372 orphan->tx_mask, 373 orphan->rx_mask); 374 375 if (!test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) { 376 cleanup_orphan_ring(orphan, accel); 377 } 378 out: 379 put_orphan_bundle(orphan); 380 381 release: 382 383 bundle = &accel->bundle[bank]; 384 /* 385 * If the user process died without releasing the rings 386 * then force a release here. 387 */ 388 mutex_lock(&bundle->list_lock); 389 pid_found = 0; 390 list_for_each_entry_safe(instance_rings, tmp, &bundle->list, list) 391 { 392 if (instance_rings->user_pid == curproc->p_pid) { 393 pid_found = 1; 394 break; 395 } 396 } 397 mutex_unlock(&bundle->list_lock); 398 399 if (pid_found) { 400 mutex_lock(&bundle->lock); 401 bundle->rings_used &= ~instance_rings->ring_mask; 402 mutex_unlock(&bundle->lock); 403 } 404 } 405