1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2022 Intel Corporation */ 3 #include <linux/module.h> 4 #include <linux/slab.h> 5 #include "adf_accel_devices.h" 6 #include "adf_common_drv.h" 7 #include "adf_transport.h" 8 #include "adf_transport_access_macros.h" 9 #include "adf_cfg.h" 10 #include "adf_cfg_strings.h" 11 #include "qat_compression.h" 12 #include "icp_qat_fw.h" 13 14 #define SEC ADF_KERNEL_SEC 15 16 static struct service_hndl qat_compression; 17 18 void qat_compression_put_instance(struct qat_compression_instance *inst) 19 { 20 atomic_dec(&inst->refctr); 21 adf_dev_put(inst->accel_dev); 22 } 23 24 static int qat_compression_free_instances(struct adf_accel_dev *accel_dev) 25 { 26 struct qat_compression_instance *inst; 27 struct list_head *list_ptr, *tmp; 28 int i; 29 30 list_for_each_safe(list_ptr, tmp, &accel_dev->compression_list) { 31 inst = list_entry(list_ptr, 32 struct qat_compression_instance, list); 33 34 for (i = 0; i < atomic_read(&inst->refctr); i++) 35 qat_compression_put_instance(inst); 36 37 if (inst->dc_tx) 38 adf_remove_ring(inst->dc_tx); 39 40 if (inst->dc_rx) 41 adf_remove_ring(inst->dc_rx); 42 43 list_del(list_ptr); 44 kfree(inst); 45 } 46 return 0; 47 } 48 49 struct qat_compression_instance *qat_compression_get_instance_node(int node, int alg) 50 { 51 struct qat_compression_instance *inst = NULL; 52 struct adf_hw_device_data *hw_data = NULL; 53 struct adf_accel_dev *accel_dev = NULL; 54 unsigned long best = ~0; 55 struct list_head *itr; 56 u32 caps, mask; 57 58 list_for_each(itr, adf_devmgr_get_head()) { 59 struct adf_accel_dev *tmp_dev; 60 unsigned long ctr; 61 int tmp_dev_node; 62 63 tmp_dev = list_entry(itr, struct adf_accel_dev, list); 64 tmp_dev_node = dev_to_node(&GET_DEV(tmp_dev)); 65 66 if (alg == QAT_ZSTD || alg == QAT_LZ4S) { 67 hw_data = tmp_dev->hw_device; 68 caps = hw_data->accel_capabilities_ext_mask; 69 mask = ADF_ACCEL_CAPABILITIES_EXT_ZSTD | 70 ADF_ACCEL_CAPABILITIES_EXT_ZSTD_LZ4S; 71 if (!(caps & mask)) 72 continue; 73 } 74 75 if ((node == tmp_dev_node || tmp_dev_node < 0) && 76 adf_dev_started(tmp_dev) && !list_empty(&tmp_dev->compression_list)) { 77 ctr = atomic_read(&tmp_dev->ref_count); 78 if (best > ctr) { 79 accel_dev = tmp_dev; 80 best = ctr; 81 } 82 } 83 } 84 85 if (!accel_dev) { 86 pr_debug_ratelimited("QAT: Could not find a device on node %d\n", node); 87 /* Get any started device */ 88 list_for_each(itr, adf_devmgr_get_head()) { 89 struct adf_accel_dev *tmp_dev; 90 91 tmp_dev = list_entry(itr, struct adf_accel_dev, list); 92 93 if (alg == QAT_ZSTD || alg == QAT_LZ4S) { 94 hw_data = tmp_dev->hw_device; 95 caps = hw_data->accel_capabilities_ext_mask; 96 mask = ADF_ACCEL_CAPABILITIES_EXT_ZSTD | 97 ADF_ACCEL_CAPABILITIES_EXT_ZSTD_LZ4S; 98 if (!(caps & mask)) 99 continue; 100 } 101 102 if (adf_dev_started(tmp_dev) && 103 !list_empty(&tmp_dev->compression_list)) { 104 accel_dev = tmp_dev; 105 break; 106 } 107 } 108 } 109 110 if (!accel_dev) 111 return NULL; 112 113 best = ~0; 114 list_for_each(itr, &accel_dev->compression_list) { 115 struct qat_compression_instance *tmp_inst; 116 unsigned long ctr; 117 118 tmp_inst = list_entry(itr, struct qat_compression_instance, list); 119 ctr = atomic_read(&tmp_inst->refctr); 120 if (best > ctr) { 121 inst = tmp_inst; 122 best = ctr; 123 } 124 } 125 if (inst) { 126 if (adf_dev_get(accel_dev)) { 127 dev_err(&GET_DEV(accel_dev), "Could not increment dev refctr\n"); 128 return NULL; 129 } 130 atomic_inc(&inst->refctr); 131 } 132 return inst; 133 } 134 135 static int qat_compression_create_instances(struct adf_accel_dev *accel_dev) 136 { 137 struct qat_compression_instance *inst; 138 char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; 139 char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; 140 unsigned long num_inst, num_msg_dc; 141 unsigned long bank; 142 int msg_size; 143 int ret; 144 int i; 145 146 INIT_LIST_HEAD(&accel_dev->compression_list); 147 strscpy(key, ADF_NUM_DC, sizeof(key)); 148 ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); 149 if (ret) 150 return ret; 151 152 ret = kstrtoul(val, 10, &num_inst); 153 if (ret) 154 return ret; 155 156 for (i = 0; i < num_inst; i++) { 157 inst = kzalloc_node(sizeof(*inst), GFP_KERNEL, 158 dev_to_node(&GET_DEV(accel_dev))); 159 if (!inst) { 160 ret = -ENOMEM; 161 goto err; 162 } 163 164 list_add_tail(&inst->list, &accel_dev->compression_list); 165 inst->id = i; 166 atomic_set(&inst->refctr, 0); 167 inst->accel_dev = accel_dev; 168 169 snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_BANK_NUM, i); 170 ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); 171 if (ret) 172 return ret; 173 174 ret = kstrtoul(val, 10, &bank); 175 if (ret) 176 return ret; 177 178 snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_SIZE, i); 179 ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); 180 if (ret) 181 return ret; 182 183 ret = kstrtoul(val, 10, &num_msg_dc); 184 if (ret) 185 return ret; 186 187 msg_size = ICP_QAT_FW_REQ_DEFAULT_SZ; 188 snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_TX, i); 189 ret = adf_create_ring(accel_dev, SEC, bank, num_msg_dc, 190 msg_size, key, NULL, 0, &inst->dc_tx); 191 if (ret) 192 return ret; 193 194 msg_size = ICP_QAT_FW_RESP_DEFAULT_SZ; 195 snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_RX, i); 196 ret = adf_create_ring(accel_dev, SEC, bank, num_msg_dc, 197 msg_size, key, qat_comp_alg_callback, 0, 198 &inst->dc_rx); 199 if (ret) 200 return ret; 201 202 inst->dc_data = accel_dev->dc_data; 203 INIT_LIST_HEAD(&inst->backlog.list); 204 spin_lock_init(&inst->backlog.lock); 205 } 206 return 0; 207 err: 208 qat_compression_free_instances(accel_dev); 209 return ret; 210 } 211 212 static int qat_compression_alloc_dc_data(struct adf_accel_dev *accel_dev) 213 { 214 struct device *dev = &GET_DEV(accel_dev); 215 dma_addr_t obuff_p = DMA_MAPPING_ERROR; 216 size_t ovf_buff_sz = QAT_COMP_MAX_SKID; 217 struct adf_dc_data *dc_data = NULL; 218 u8 *obuff = NULL; 219 220 dc_data = kzalloc_node(sizeof(*dc_data), GFP_KERNEL, dev_to_node(dev)); 221 if (!dc_data) 222 goto err; 223 224 obuff = kzalloc_node(ovf_buff_sz, GFP_KERNEL, dev_to_node(dev)); 225 if (!obuff) 226 goto err; 227 228 obuff_p = dma_map_single(dev, obuff, ovf_buff_sz, DMA_BIDIRECTIONAL); 229 if (unlikely(dma_mapping_error(dev, obuff_p))) 230 goto err; 231 232 dc_data->ovf_buff = obuff; 233 dc_data->ovf_buff_p = obuff_p; 234 dc_data->ovf_buff_sz = ovf_buff_sz; 235 236 accel_dev->dc_data = dc_data; 237 238 return 0; 239 240 err: 241 accel_dev->dc_data = NULL; 242 kfree(obuff); 243 devm_kfree(dev, dc_data); 244 return -ENOMEM; 245 } 246 247 static void qat_free_dc_data(struct adf_accel_dev *accel_dev) 248 { 249 struct adf_dc_data *dc_data = accel_dev->dc_data; 250 struct device *dev = &GET_DEV(accel_dev); 251 252 if (!dc_data) 253 return; 254 255 dma_unmap_single(dev, dc_data->ovf_buff_p, dc_data->ovf_buff_sz, 256 DMA_BIDIRECTIONAL); 257 kfree_sensitive(dc_data->ovf_buff); 258 kfree(dc_data); 259 accel_dev->dc_data = NULL; 260 } 261 262 static int qat_compression_init(struct adf_accel_dev *accel_dev) 263 { 264 int ret; 265 266 ret = qat_compression_alloc_dc_data(accel_dev); 267 if (ret) 268 return ret; 269 270 ret = qat_compression_create_instances(accel_dev); 271 if (ret) 272 qat_free_dc_data(accel_dev); 273 274 return ret; 275 } 276 277 static int qat_compression_shutdown(struct adf_accel_dev *accel_dev) 278 { 279 qat_free_dc_data(accel_dev); 280 return qat_compression_free_instances(accel_dev); 281 } 282 283 static int qat_compression_event_handler(struct adf_accel_dev *accel_dev, 284 enum adf_event event) 285 { 286 int ret; 287 288 switch (event) { 289 case ADF_EVENT_INIT: 290 ret = qat_compression_init(accel_dev); 291 break; 292 case ADF_EVENT_SHUTDOWN: 293 ret = qat_compression_shutdown(accel_dev); 294 break; 295 case ADF_EVENT_RESTARTING: 296 case ADF_EVENT_RESTARTED: 297 case ADF_EVENT_START: 298 case ADF_EVENT_STOP: 299 default: 300 ret = 0; 301 } 302 return ret; 303 } 304 305 int qat_compression_register(void) 306 { 307 memset(&qat_compression, 0, sizeof(qat_compression)); 308 qat_compression.event_hld = qat_compression_event_handler; 309 qat_compression.name = "qat_compression"; 310 return adf_service_register(&qat_compression); 311 } 312 313 int qat_compression_unregister(void) 314 { 315 return adf_service_unregister(&qat_compression); 316 } 317