1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright(c) 2007-2022 Intel Corporation */ 3 #include "qat_freebsd.h" 4 #include "adf_cfg.h" 5 #include "adf_common_drv.h" 6 #include "adf_accel_devices.h" 7 #include "icp_qat_uclo.h" 8 #include "icp_qat_fw.h" 9 #include "icp_qat_fw_init_admin.h" 10 #include "adf_cfg_strings.h" 11 #include "adf_uio_control.h" 12 #include "adf_uio_cleanup.h" 13 #include "adf_uio.h" 14 #include "adf_transport_access_macros.h" 15 #include "adf_transport_internal.h" 16 #include <sys/param.h> 17 #include <sys/lock.h> 18 #include <sys/rwlock.h> 19 #include <sys/sglist.h> 20 #include <sys/systm.h> 21 #include <sys/proc.h> 22 #include <vm/vm.h> 23 #include <vm/pmap.h> 24 #include <vm/vm_kern.h> 25 #include <vm/vm_map.h> 26 #include <vm/vm_object.h> 27 #include <vm/vm_page.h> 28 #include <vm/vm_pager.h> 29 #include <vm/vm_param.h> 30 31 #define TX_RINGS_DISABLE 0 32 #define TX_RINGS_ENABLE 1 33 #define PKE_REQ_SIZE 64 34 #define BASE_ADDR_SHIFT 6 35 #define PKE_RX_RING_0 0 36 #define PKE_RX_RING_1 1 37 38 #define ADF_RING_EMPTY_RETRY_DELAY 2 39 #define ADF_RING_EMPTY_MAX_RETRY 15 40 41 struct bundle_orphan_ring { 42 unsigned long tx_mask; 43 unsigned long rx_mask; 44 unsigned long asym_mask; 45 int bank; 46 struct resource *csr_base; 47 struct adf_uio_control_bundle *bundle; 48 }; 49 50 /* 51 * if orphan->tx_mask does not match with orphan->rx_mask 52 */ 53 static void 54 check_orphan_ring(struct adf_accel_dev *accel_dev, 55 struct bundle_orphan_ring *orphan, 56 struct adf_hw_device_data *hw_data) 57 { 58 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); 59 int i; 60 int tx_rx_gap = hw_data->tx_rx_gap; 61 u8 num_rings_per_bank = hw_data->num_rings_per_bank; 62 struct resource *csr_base = orphan->csr_base; 63 int bank = orphan->bank; 64 65 for (i = 0; i < num_rings_per_bank; i++) { 66 if (test_bit(i, &orphan->tx_mask)) { 67 int rx_ring = i + tx_rx_gap; 68 69 if (!test_bit(rx_ring, &orphan->rx_mask)) { 70 __clear_bit(i, &orphan->tx_mask); 71 72 /* clean up this tx ring */ 73 csr_ops->write_csr_ring_config(csr_base, 74 bank, 75 i, 76 0); 77 csr_ops->write_csr_ring_base(csr_base, 78 bank, 79 i, 80 0); 81 } 82 83 } else if (test_bit(i, &orphan->rx_mask)) { 84 int tx_ring = i - tx_rx_gap; 85 86 if (!test_bit(tx_ring, &orphan->tx_mask)) { 87 __clear_bit(i, &orphan->rx_mask); 88 89 /* clean up this rx ring */ 90 csr_ops->write_csr_ring_config(csr_base, 91 bank, 92 i, 93 0); 94 csr_ops->write_csr_ring_base(csr_base, 95 bank, 96 i, 97 0); 98 } 99 } 100 } 101 } 102 103 static int 104 get_orphan_bundle(int bank, 105 struct adf_uio_control_accel *accel, 106 struct bundle_orphan_ring **orphan_bundle_out) 107 { 108 int i; 109 int ret = 0; 110 struct resource *csr_base; 111 unsigned long tx_mask; 112 unsigned long asym_mask; 113 struct adf_accel_dev *accel_dev = accel->accel_dev; 114 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); 115 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 116 u8 num_rings_per_bank = hw_data->num_rings_per_bank; 117 struct bundle_orphan_ring *orphan_bundle = NULL; 118 uint64_t base; 119 struct list_head *entry; 120 struct adf_uio_instance_rings *instance_rings; 121 struct adf_uio_control_bundle *bundle; 122 u16 ring_mask = 0; 123 124 orphan_bundle = 125 malloc(sizeof(*orphan_bundle), M_QAT, M_WAITOK | M_ZERO); 126 if (!orphan_bundle) 127 return ENOMEM; 128 129 csr_base = accel->bar->virt_addr; 130 orphan_bundle->csr_base = csr_base; 131 orphan_bundle->bank = bank; 132 133 orphan_bundle->tx_mask = 0; 134 orphan_bundle->rx_mask = 0; 135 tx_mask = accel_dev->hw_device->tx_rings_mask; 136 asym_mask = accel_dev->hw_device->asym_rings_mask; 137 138 /* Get ring mask for this process. */ 139 bundle = &accel->bundle[bank]; 140 orphan_bundle->bundle = bundle; 141 mutex_lock(&bundle->list_lock); 142 list_for_each(entry, &bundle->list) 143 { 144 instance_rings = 145 list_entry(entry, struct adf_uio_instance_rings, list); 146 if (instance_rings->user_pid == curproc->p_pid) { 147 ring_mask = instance_rings->ring_mask; 148 break; 149 } 150 } 151 mutex_unlock(&bundle->list_lock); 152 153 for (i = 0; i < num_rings_per_bank; i++) { 154 base = csr_ops->read_csr_ring_base(csr_base, bank, i); 155 156 if (!base) 157 continue; 158 if (!(ring_mask & 1 << i)) 159 continue; /* Not reserved for this process. */ 160 161 if (test_bit(i, &tx_mask)) 162 __set_bit(i, &orphan_bundle->tx_mask); 163 else 164 __set_bit(i, &orphan_bundle->rx_mask); 165 166 if (test_bit(i, &asym_mask)) 167 __set_bit(i, &orphan_bundle->asym_mask); 168 } 169 170 if (orphan_bundle->tx_mask || orphan_bundle->rx_mask) 171 check_orphan_ring(accel_dev, orphan_bundle, hw_data); 172 173 *orphan_bundle_out = orphan_bundle; 174 return ret; 175 } 176 177 static void 178 put_orphan_bundle(struct bundle_orphan_ring *bundle) 179 { 180 if (!bundle) 181 return; 182 183 free(bundle, M_QAT); 184 } 185 186 /* cleanup all ring */ 187 static void 188 cleanup_all_ring(struct adf_uio_control_accel *accel, 189 struct bundle_orphan_ring *orphan) 190 { 191 int i; 192 struct resource *csr_base = orphan->csr_base; 193 unsigned long mask = orphan->rx_mask | orphan->tx_mask; 194 struct adf_accel_dev *accel_dev = accel->accel_dev; 195 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); 196 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 197 u8 num_rings_per_bank = hw_data->num_rings_per_bank; 198 int bank = orphan->bank; 199 200 mutex_lock(&orphan->bundle->lock); 201 orphan->bundle->rings_enabled &= ~mask; 202 adf_update_uio_ring_arb(orphan->bundle); 203 mutex_unlock(&orphan->bundle->lock); 204 205 for (i = 0; i < num_rings_per_bank; i++) { 206 if (!test_bit(i, &mask)) 207 continue; 208 209 csr_ops->write_csr_ring_config(csr_base, bank, i, 0); 210 csr_ops->write_csr_ring_base(csr_base, bank, i, 0); 211 } 212 } 213 214 /* 215 * Return true, if number of messages in tx ring is equal to number 216 * of messages in corresponding rx ring, else false. 217 */ 218 static bool 219 is_all_resp_recvd(struct adf_hw_csr_ops *csr_ops, 220 struct bundle_orphan_ring *bundle, 221 const u8 num_rings_per_bank) 222 { 223 u32 rx_tail = 0, tx_head = 0, rx_ring_msg_offset = 0, 224 tx_ring_msg_offset = 0, tx_rx_offset = num_rings_per_bank / 2, 225 idx = 0, retry = 0, delay = ADF_RING_EMPTY_RETRY_DELAY; 226 227 do { 228 for_each_set_bit(idx, &bundle->tx_mask, tx_rx_offset) 229 { 230 rx_tail = 231 csr_ops->read_csr_ring_tail(bundle->csr_base, 232 0, 233 (idx + tx_rx_offset)); 234 tx_head = csr_ops->read_csr_ring_head(bundle->csr_base, 235 0, 236 idx); 237 238 /* 239 * Normalize messages in tx rings to match rx ring 240 * message size, i.e., size of response message(32). 241 * Asym messages are 64 bytes each, so right shift 242 * by 1 to normalize to 32. Sym and compression 243 * messages are 128 bytes each, so right shift by 2 244 * to normalize to 32. 245 */ 246 if (bundle->asym_mask & (1 << idx)) 247 tx_ring_msg_offset = (tx_head >> 1); 248 else 249 tx_ring_msg_offset = (tx_head >> 2); 250 251 rx_ring_msg_offset = rx_tail; 252 253 if (tx_ring_msg_offset != rx_ring_msg_offset) 254 break; 255 } 256 if (idx == tx_rx_offset) 257 /* All Tx and Rx ring message counts match */ 258 return true; 259 260 DELAY(delay); 261 delay *= 2; 262 } while (++retry < ADF_RING_EMPTY_MAX_RETRY); 263 264 return false; 265 } 266 267 static int 268 bundle_need_cleanup(int bank, struct adf_uio_control_accel *accel) 269 { 270 struct resource *csr_base = accel->bar->virt_addr; 271 struct adf_accel_dev *accel_dev = accel->accel_dev; 272 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); 273 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 274 u8 num_rings_per_bank = hw_data->num_rings_per_bank; 275 int i; 276 277 if (!csr_base) 278 return 0; 279 280 for (i = 0; i < num_rings_per_bank; i++) { 281 if (csr_ops->read_csr_ring_base(csr_base, bank, i)) 282 return 1; 283 } 284 285 return 0; 286 } 287 288 static void 289 cleanup_orphan_ring(struct bundle_orphan_ring *orphan, 290 struct adf_uio_control_accel *accel) 291 { 292 struct adf_accel_dev *accel_dev = accel->accel_dev; 293 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); 294 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 295 u8 number_rings_per_bank = hw_data->num_rings_per_bank; 296 297 /* disable the interrupt */ 298 csr_ops->write_csr_int_col_en(orphan->csr_base, orphan->bank, 0); 299 300 /* 301 * wait firmware finish the in-process ring 302 * 1. disable all tx rings 303 * 2. check if all responses are received 304 * 3. reset all rings 305 */ 306 adf_disable_ring_arb(accel_dev, orphan->csr_base, 0, orphan->tx_mask); 307 308 if (!is_all_resp_recvd(csr_ops, orphan, number_rings_per_bank)) { 309 device_printf(GET_DEV(accel_dev), 310 "Failed to clean up orphan rings"); 311 return; 312 } 313 314 /* 315 * When the execution reaches here, it is assumed that 316 * there is no inflight request in the rings and that 317 * there is no in-process ring. 318 */ 319 320 cleanup_all_ring(accel, orphan); 321 } 322 323 void 324 adf_uio_do_cleanup_orphan(int bank, struct adf_uio_control_accel *accel) 325 { 326 int ret; 327 struct adf_uio_instance_rings *instance_rings, *tmp; 328 struct adf_uio_control_bundle *bundle; 329 /* orphan is local pointer allocated and deallocated in this function */ 330 struct bundle_orphan_ring *orphan = NULL; 331 struct adf_accel_dev *accel_dev = accel->accel_dev; 332 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 333 334 if (!bundle_need_cleanup(bank, accel)) 335 goto release; 336 337 ret = get_orphan_bundle(bank, accel, &orphan); 338 if (ret != 0) 339 return; 340 341 /* 342 * If driver supports ring pair reset, no matter process 343 * exits normally or abnormally, just do ring pair reset. 344 * ring pair reset will reset all ring pair registers to 345 * default value. Driver only needs to reset ring mask 346 */ 347 if (hw_data->ring_pair_reset) { 348 hw_data->ring_pair_reset( 349 accel_dev, orphan->bundle->hardware_bundle_number); 350 /* 351 * If processes exit normally, rx_mask, tx_mask 352 * and rings_enabled are all 0, below expression 353 * have no impact on rings_enabled. 354 * If processes exit abnormally, rings_enabled 355 * will be set as 0 by below expression. 356 */ 357 orphan->bundle->rings_enabled &= 358 ~(orphan->rx_mask | orphan->tx_mask); 359 goto out; 360 } 361 362 if (!orphan->tx_mask && !orphan->rx_mask) 363 goto out; 364 365 device_printf(GET_DEV(accel_dev), 366 "Process %d %s exit with orphan rings %lx:%lx\n", 367 curproc->p_pid, 368 curproc->p_comm, 369 orphan->tx_mask, 370 orphan->rx_mask); 371 372 if (!test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) { 373 cleanup_orphan_ring(orphan, accel); 374 } 375 out: 376 put_orphan_bundle(orphan); 377 378 release: 379 380 bundle = &accel->bundle[bank]; 381 /* 382 * If the user process died without releasing the rings 383 * then force a release here. 384 */ 385 mutex_lock(&bundle->list_lock); 386 list_for_each_entry_safe(instance_rings, tmp, &bundle->list, list) 387 { 388 if (instance_rings->user_pid == curproc->p_pid) { 389 bundle->rings_used &= ~instance_rings->ring_mask; 390 break; 391 } 392 } 393 mutex_unlock(&bundle->list_lock); 394 } 395