1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* Copyright(c) 2007-2022 Intel Corporation */ 3 #include "qat_freebsd.h" 4 #include "adf_cfg.h" 5 #include "adf_common_drv.h" 6 #include "adf_accel_devices.h" 7 #include "icp_qat_uclo.h" 8 #include "icp_qat_fw.h" 9 #include "icp_qat_fw_init_admin.h" 10 #include "adf_cfg_strings.h" 11 #include "adf_uio_control.h" 12 #include "adf_uio_cleanup.h" 13 #include "adf_uio.h" 14 #include "adf_transport_access_macros.h" 15 #include "adf_transport_internal.h" 16 #include <sys/param.h> 17 #include <sys/lock.h> 18 #include <sys/rwlock.h> 19 #include <sys/sglist.h> 20 #include <sys/systm.h> 21 #include <sys/proc.h> 22 #include <vm/vm.h> 23 #include <vm/pmap.h> 24 #include <vm/vm_kern.h> 25 #include <vm/vm_map.h> 26 #include <vm/vm_object.h> 27 #include <vm/vm_page.h> 28 #include <vm/vm_pager.h> 29 #include <vm/vm_param.h> 30 31 #define TX_RINGS_DISABLE 0 32 #define TX_RINGS_ENABLE 1 33 #define PKE_REQ_SIZE 64 34 #define BASE_ADDR_SHIFT 6 35 #define PKE_RX_RING_0 0 36 #define PKE_RX_RING_1 1 37 38 #define ADF_RING_EMPTY_RETRY_DELAY 2 39 #define ADF_RING_EMPTY_MAX_RETRY 15 40 41 struct bundle_orphan_ring { 42 unsigned long tx_mask; 43 unsigned long rx_mask; 44 unsigned long asym_mask; 45 int bank; 46 struct resource *csr_base; 47 struct adf_uio_control_bundle *bundle; 48 }; 49 50 /* 51 * if orphan->tx_mask does not match with orphan->rx_mask 52 */ 53 static void 54 check_orphan_ring(struct adf_accel_dev *accel_dev, 55 struct bundle_orphan_ring *orphan, 56 struct adf_hw_device_data *hw_data) 57 { 58 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); 59 int i; 60 int tx_rx_gap = hw_data->tx_rx_gap; 61 u8 num_rings_per_bank = hw_data->num_rings_per_bank; 62 struct resource *csr_base = orphan->csr_base; 63 int bank = orphan->bank; 64 65 for (i = 0; i < num_rings_per_bank; i++) { 66 if (test_bit(i, &orphan->tx_mask)) { 67 int rx_ring = i + tx_rx_gap; 68 69 if (!test_bit(rx_ring, &orphan->rx_mask)) { 70 __clear_bit(i, &orphan->tx_mask); 71 72 /* clean up this tx ring */ 73 csr_ops->write_csr_ring_config(csr_base, 74 bank, 75 i, 76 0); 77 csr_ops->write_csr_ring_base(csr_base, 78 bank, 79 i, 80 0); 81 } 82 83 } else if (test_bit(i, &orphan->rx_mask)) { 84 int tx_ring = i - tx_rx_gap; 85 86 if (!test_bit(tx_ring, &orphan->tx_mask)) { 87 __clear_bit(i, &orphan->rx_mask); 88 89 /* clean up this rx ring */ 90 csr_ops->write_csr_ring_config(csr_base, 91 bank, 92 i, 93 0); 94 csr_ops->write_csr_ring_base(csr_base, 95 bank, 96 i, 97 0); 98 } 99 } 100 } 101 } 102 103 static int 104 get_orphan_bundle(int bank, 105 struct adf_uio_control_accel *accel, 106 struct bundle_orphan_ring **orphan_bundle_out) 107 { 108 int i; 109 int ret = 0; 110 struct resource *csr_base; 111 unsigned long tx_mask; 112 unsigned long asym_mask; 113 struct adf_accel_dev *accel_dev = accel->accel_dev; 114 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); 115 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 116 u8 num_rings_per_bank = hw_data->num_rings_per_bank; 117 struct bundle_orphan_ring *orphan_bundle = NULL; 118 uint64_t base; 119 struct list_head *entry; 120 struct adf_uio_instance_rings *instance_rings; 121 struct adf_uio_control_bundle *bundle; 122 u16 ring_mask = 0; 123 124 orphan_bundle = 125 malloc(sizeof(*orphan_bundle), M_QAT, M_WAITOK | M_ZERO); 126 csr_base = accel->bar->virt_addr; 127 orphan_bundle->csr_base = csr_base; 128 orphan_bundle->bank = bank; 129 130 orphan_bundle->tx_mask = 0; 131 orphan_bundle->rx_mask = 0; 132 tx_mask = accel_dev->hw_device->tx_rings_mask; 133 asym_mask = accel_dev->hw_device->asym_rings_mask; 134 135 /* Get ring mask for this process. */ 136 bundle = &accel->bundle[bank]; 137 orphan_bundle->bundle = bundle; 138 mutex_lock(&bundle->list_lock); 139 list_for_each(entry, &bundle->list) 140 { 141 instance_rings = 142 list_entry(entry, struct adf_uio_instance_rings, list); 143 if (instance_rings->user_pid == curproc->p_pid) { 144 ring_mask = instance_rings->ring_mask; 145 break; 146 } 147 } 148 mutex_unlock(&bundle->list_lock); 149 150 for (i = 0; i < num_rings_per_bank; i++) { 151 base = csr_ops->read_csr_ring_base(csr_base, bank, i); 152 153 if (!base) 154 continue; 155 if (!(ring_mask & 1 << i)) 156 continue; /* Not reserved for this process. */ 157 158 if (test_bit(i, &tx_mask)) 159 __set_bit(i, &orphan_bundle->tx_mask); 160 else 161 __set_bit(i, &orphan_bundle->rx_mask); 162 163 if (test_bit(i, &asym_mask)) 164 __set_bit(i, &orphan_bundle->asym_mask); 165 } 166 167 if (orphan_bundle->tx_mask || orphan_bundle->rx_mask) 168 check_orphan_ring(accel_dev, orphan_bundle, hw_data); 169 170 *orphan_bundle_out = orphan_bundle; 171 return ret; 172 } 173 174 static void 175 put_orphan_bundle(struct bundle_orphan_ring *bundle) 176 { 177 if (!bundle) 178 return; 179 180 free(bundle, M_QAT); 181 } 182 183 /* cleanup all ring */ 184 static void 185 cleanup_all_ring(struct adf_uio_control_accel *accel, 186 struct bundle_orphan_ring *orphan) 187 { 188 int i; 189 struct resource *csr_base = orphan->csr_base; 190 unsigned long mask = orphan->rx_mask | orphan->tx_mask; 191 struct adf_accel_dev *accel_dev = accel->accel_dev; 192 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); 193 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 194 u8 num_rings_per_bank = hw_data->num_rings_per_bank; 195 int bank = orphan->bank; 196 197 mutex_lock(&orphan->bundle->lock); 198 orphan->bundle->rings_enabled &= ~mask; 199 adf_update_uio_ring_arb(orphan->bundle); 200 mutex_unlock(&orphan->bundle->lock); 201 202 for (i = 0; i < num_rings_per_bank; i++) { 203 if (!test_bit(i, &mask)) 204 continue; 205 206 csr_ops->write_csr_ring_config(csr_base, bank, i, 0); 207 csr_ops->write_csr_ring_base(csr_base, bank, i, 0); 208 } 209 } 210 211 /* 212 * Return true, if number of messages in tx ring is equal to number 213 * of messages in corresponding rx ring, else false. 214 */ 215 static bool 216 is_all_resp_recvd(struct adf_hw_csr_ops *csr_ops, 217 struct bundle_orphan_ring *bundle, 218 const u8 num_rings_per_bank) 219 { 220 u32 rx_tail = 0, tx_head = 0, rx_ring_msg_offset = 0, 221 tx_ring_msg_offset = 0, tx_rx_offset = num_rings_per_bank / 2, 222 idx = 0, retry = 0, delay = ADF_RING_EMPTY_RETRY_DELAY; 223 224 do { 225 for_each_set_bit(idx, &bundle->tx_mask, tx_rx_offset) 226 { 227 rx_tail = 228 csr_ops->read_csr_ring_tail(bundle->csr_base, 229 0, 230 (idx + tx_rx_offset)); 231 tx_head = csr_ops->read_csr_ring_head(bundle->csr_base, 232 0, 233 idx); 234 235 /* 236 * Normalize messages in tx rings to match rx ring 237 * message size, i.e., size of response message(32). 238 * Asym messages are 64 bytes each, so right shift 239 * by 1 to normalize to 32. Sym and compression 240 * messages are 128 bytes each, so right shift by 2 241 * to normalize to 32. 242 */ 243 if (bundle->asym_mask & (1 << idx)) 244 tx_ring_msg_offset = (tx_head >> 1); 245 else 246 tx_ring_msg_offset = (tx_head >> 2); 247 248 rx_ring_msg_offset = rx_tail; 249 250 if (tx_ring_msg_offset != rx_ring_msg_offset) 251 break; 252 } 253 if (idx == tx_rx_offset) 254 /* All Tx and Rx ring message counts match */ 255 return true; 256 257 DELAY(delay); 258 delay *= 2; 259 } while (++retry < ADF_RING_EMPTY_MAX_RETRY); 260 261 return false; 262 } 263 264 static int 265 bundle_need_cleanup(int bank, struct adf_uio_control_accel *accel) 266 { 267 struct resource *csr_base = accel->bar->virt_addr; 268 struct adf_accel_dev *accel_dev = accel->accel_dev; 269 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); 270 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 271 u8 num_rings_per_bank = hw_data->num_rings_per_bank; 272 int i; 273 274 if (!csr_base) 275 return 0; 276 277 for (i = 0; i < num_rings_per_bank; i++) { 278 if (csr_ops->read_csr_ring_base(csr_base, bank, i)) 279 return 1; 280 } 281 282 return 0; 283 } 284 285 static void 286 cleanup_orphan_ring(struct bundle_orphan_ring *orphan, 287 struct adf_uio_control_accel *accel) 288 { 289 struct adf_accel_dev *accel_dev = accel->accel_dev; 290 struct adf_hw_csr_ops *csr_ops = GET_CSR_OPS(accel_dev); 291 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 292 u8 number_rings_per_bank = hw_data->num_rings_per_bank; 293 294 /* disable the interrupt */ 295 csr_ops->write_csr_int_col_en(orphan->csr_base, orphan->bank, 0); 296 297 /* 298 * wait firmware finish the in-process ring 299 * 1. disable all tx rings 300 * 2. check if all responses are received 301 * 3. reset all rings 302 */ 303 adf_disable_ring_arb(accel_dev, orphan->csr_base, 0, orphan->tx_mask); 304 305 if (!is_all_resp_recvd(csr_ops, orphan, number_rings_per_bank)) { 306 device_printf(GET_DEV(accel_dev), 307 "Failed to clean up orphan rings"); 308 return; 309 } 310 311 /* 312 * When the execution reaches here, it is assumed that 313 * there is no inflight request in the rings and that 314 * there is no in-process ring. 315 */ 316 317 cleanup_all_ring(accel, orphan); 318 } 319 320 void 321 adf_uio_do_cleanup_orphan(int bank, struct adf_uio_control_accel *accel) 322 { 323 int ret; 324 struct adf_uio_instance_rings *instance_rings, *tmp; 325 struct adf_uio_control_bundle *bundle; 326 /* orphan is local pointer allocated and deallocated in this function */ 327 struct bundle_orphan_ring *orphan = NULL; 328 struct adf_accel_dev *accel_dev = accel->accel_dev; 329 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 330 331 if (!bundle_need_cleanup(bank, accel)) 332 goto release; 333 334 ret = get_orphan_bundle(bank, accel, &orphan); 335 if (ret != 0) 336 return; 337 338 /* 339 * If driver supports ring pair reset, no matter process 340 * exits normally or abnormally, just do ring pair reset. 341 * ring pair reset will reset all ring pair registers to 342 * default value. Driver only needs to reset ring mask 343 */ 344 if (hw_data->ring_pair_reset) { 345 hw_data->ring_pair_reset( 346 accel_dev, orphan->bundle->hardware_bundle_number); 347 /* 348 * If processes exit normally, rx_mask, tx_mask 349 * and rings_enabled are all 0, below expression 350 * have no impact on rings_enabled. 351 * If processes exit abnormally, rings_enabled 352 * will be set as 0 by below expression. 353 */ 354 orphan->bundle->rings_enabled &= 355 ~(orphan->rx_mask | orphan->tx_mask); 356 goto out; 357 } 358 359 if (!orphan->tx_mask && !orphan->rx_mask) 360 goto out; 361 362 device_printf(GET_DEV(accel_dev), 363 "Process %d %s exit with orphan rings %lx:%lx\n", 364 curproc->p_pid, 365 curproc->p_comm, 366 orphan->tx_mask, 367 orphan->rx_mask); 368 369 if (!test_bit(ADF_STATUS_RESTARTING, &accel_dev->status)) { 370 cleanup_orphan_ring(orphan, accel); 371 } 372 out: 373 put_orphan_bundle(orphan); 374 375 release: 376 377 bundle = &accel->bundle[bank]; 378 /* 379 * If the user process died without releasing the rings 380 * then force a release here. 381 */ 382 mutex_lock(&bundle->list_lock); 383 list_for_each_entry_safe(instance_rings, tmp, &bundle->list, list) 384 { 385 if (instance_rings->user_pid == curproc->p_pid) { 386 bundle->rings_used &= ~instance_rings->ring_mask; 387 break; 388 } 389 } 390 mutex_unlock(&bundle->list_lock); 391 } 392