1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2022 Intel Corporation */ 3 #include <linux/module.h> 4 #include <linux/slab.h> 5 #include <linux/string.h> 6 #include "adf_accel_devices.h" 7 #include "adf_common_drv.h" 8 #include "adf_transport.h" 9 #include "adf_transport_access_macros.h" 10 #include "adf_cfg.h" 11 #include "adf_cfg_strings.h" 12 #include "qat_compression.h" 13 #include "icp_qat_fw.h" 14 15 #define SEC ADF_KERNEL_SEC 16 17 static struct service_hndl qat_compression; 18 19 void qat_compression_put_instance(struct qat_compression_instance *inst) 20 { 21 atomic_dec(&inst->refctr); 22 adf_dev_put(inst->accel_dev); 23 } 24 25 static int qat_compression_free_instances(struct adf_accel_dev *accel_dev) 26 { 27 struct qat_compression_instance *inst; 28 struct list_head *list_ptr, *tmp; 29 int i; 30 31 list_for_each_safe(list_ptr, tmp, &accel_dev->compression_list) { 32 inst = list_entry(list_ptr, 33 struct qat_compression_instance, list); 34 35 for (i = 0; i < atomic_read(&inst->refctr); i++) 36 qat_compression_put_instance(inst); 37 38 if (inst->dc_tx) 39 adf_remove_ring(inst->dc_tx); 40 41 if (inst->dc_rx) 42 adf_remove_ring(inst->dc_rx); 43 44 list_del(list_ptr); 45 kfree(inst); 46 } 47 return 0; 48 } 49 50 struct qat_compression_instance *qat_compression_get_instance_node(int node, int alg) 51 { 52 struct qat_compression_instance *inst = NULL; 53 struct adf_hw_device_data *hw_data = NULL; 54 struct adf_accel_dev *accel_dev = NULL; 55 unsigned long best = ~0; 56 struct list_head *itr; 57 u32 caps, mask; 58 59 list_for_each(itr, adf_devmgr_get_head()) { 60 struct adf_accel_dev *tmp_dev; 61 unsigned long ctr; 62 int tmp_dev_node; 63 64 tmp_dev = list_entry(itr, struct adf_accel_dev, list); 65 tmp_dev_node = dev_to_node(&GET_DEV(tmp_dev)); 66 67 if (alg == QAT_ZSTD || alg == QAT_LZ4S) { 68 hw_data = tmp_dev->hw_device; 69 caps = hw_data->accel_capabilities_ext_mask; 70 mask = ADF_ACCEL_CAPABILITIES_EXT_ZSTD | 71 ADF_ACCEL_CAPABILITIES_EXT_ZSTD_LZ4S; 72 if (!(caps & mask)) 73 continue; 74 } 75 76 if ((node == tmp_dev_node || tmp_dev_node < 0) && 77 adf_dev_started(tmp_dev) && !list_empty(&tmp_dev->compression_list)) { 78 ctr = atomic_read(&tmp_dev->ref_count); 79 if (best > ctr) { 80 accel_dev = tmp_dev; 81 best = ctr; 82 } 83 } 84 } 85 86 if (!accel_dev) { 87 pr_debug_ratelimited("QAT: Could not find a device on node %d\n", node); 88 /* Get any started device */ 89 list_for_each(itr, adf_devmgr_get_head()) { 90 struct adf_accel_dev *tmp_dev; 91 92 tmp_dev = list_entry(itr, struct adf_accel_dev, list); 93 94 if (alg == QAT_ZSTD || alg == QAT_LZ4S) { 95 hw_data = tmp_dev->hw_device; 96 caps = hw_data->accel_capabilities_ext_mask; 97 mask = ADF_ACCEL_CAPABILITIES_EXT_ZSTD | 98 ADF_ACCEL_CAPABILITIES_EXT_ZSTD_LZ4S; 99 if (!(caps & mask)) 100 continue; 101 } 102 103 if (adf_dev_started(tmp_dev) && 104 !list_empty(&tmp_dev->compression_list)) { 105 accel_dev = tmp_dev; 106 break; 107 } 108 } 109 } 110 111 if (!accel_dev) 112 return NULL; 113 114 best = ~0; 115 list_for_each(itr, &accel_dev->compression_list) { 116 struct qat_compression_instance *tmp_inst; 117 unsigned long ctr; 118 119 tmp_inst = list_entry(itr, struct qat_compression_instance, list); 120 ctr = atomic_read(&tmp_inst->refctr); 121 if (best > ctr) { 122 inst = tmp_inst; 123 best = ctr; 124 } 125 } 126 if (inst) { 127 if (adf_dev_get(accel_dev)) { 128 dev_err(&GET_DEV(accel_dev), "Could not increment dev refctr\n"); 129 return NULL; 130 } 131 atomic_inc(&inst->refctr); 132 } 133 return inst; 134 } 135 136 static int qat_compression_create_instances(struct adf_accel_dev *accel_dev) 137 { 138 struct qat_compression_instance *inst; 139 char key[ADF_CFG_MAX_KEY_LEN_IN_BYTES]; 140 char val[ADF_CFG_MAX_VAL_LEN_IN_BYTES]; 141 unsigned long num_inst, num_msg_dc; 142 unsigned long bank; 143 int msg_size; 144 int ret; 145 int i; 146 147 INIT_LIST_HEAD(&accel_dev->compression_list); 148 strscpy(key, ADF_NUM_DC); 149 ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); 150 if (ret) 151 return ret; 152 153 ret = kstrtoul(val, 10, &num_inst); 154 if (ret) 155 return ret; 156 157 for (i = 0; i < num_inst; i++) { 158 inst = kzalloc_node(sizeof(*inst), GFP_KERNEL, 159 dev_to_node(&GET_DEV(accel_dev))); 160 if (!inst) { 161 ret = -ENOMEM; 162 goto err; 163 } 164 165 list_add_tail(&inst->list, &accel_dev->compression_list); 166 inst->id = i; 167 atomic_set(&inst->refctr, 0); 168 inst->accel_dev = accel_dev; 169 170 snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_BANK_NUM, i); 171 ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); 172 if (ret) 173 return ret; 174 175 ret = kstrtoul(val, 10, &bank); 176 if (ret) 177 return ret; 178 179 snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_SIZE, i); 180 ret = adf_cfg_get_param_value(accel_dev, SEC, key, val); 181 if (ret) 182 return ret; 183 184 ret = kstrtoul(val, 10, &num_msg_dc); 185 if (ret) 186 return ret; 187 188 msg_size = ICP_QAT_FW_REQ_DEFAULT_SZ; 189 snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_TX, i); 190 ret = adf_create_ring(accel_dev, SEC, bank, num_msg_dc, 191 msg_size, key, NULL, 0, &inst->dc_tx); 192 if (ret) 193 return ret; 194 195 msg_size = ICP_QAT_FW_RESP_DEFAULT_SZ; 196 snprintf(key, sizeof(key), ADF_DC "%d" ADF_RING_DC_RX, i); 197 ret = adf_create_ring(accel_dev, SEC, bank, num_msg_dc, 198 msg_size, key, qat_comp_alg_callback, 0, 199 &inst->dc_rx); 200 if (ret) 201 return ret; 202 203 inst->dc_data = accel_dev->dc_data; 204 INIT_LIST_HEAD(&inst->backlog.list); 205 spin_lock_init(&inst->backlog.lock); 206 } 207 return 0; 208 err: 209 qat_compression_free_instances(accel_dev); 210 return ret; 211 } 212 213 static int qat_compression_alloc_dc_data(struct adf_accel_dev *accel_dev) 214 { 215 struct device *dev = &GET_DEV(accel_dev); 216 dma_addr_t obuff_p = DMA_MAPPING_ERROR; 217 size_t ovf_buff_sz = QAT_COMP_MAX_SKID; 218 struct adf_dc_data *dc_data = NULL; 219 u8 *obuff = NULL; 220 221 dc_data = kzalloc_node(sizeof(*dc_data), GFP_KERNEL, dev_to_node(dev)); 222 if (!dc_data) 223 goto err; 224 225 obuff = kzalloc_node(ovf_buff_sz, GFP_KERNEL, dev_to_node(dev)); 226 if (!obuff) 227 goto err; 228 229 obuff_p = dma_map_single(dev, obuff, ovf_buff_sz, DMA_BIDIRECTIONAL); 230 if (unlikely(dma_mapping_error(dev, obuff_p))) 231 goto err; 232 233 dc_data->ovf_buff = obuff; 234 dc_data->ovf_buff_p = obuff_p; 235 dc_data->ovf_buff_sz = ovf_buff_sz; 236 237 accel_dev->dc_data = dc_data; 238 239 return 0; 240 241 err: 242 accel_dev->dc_data = NULL; 243 kfree(obuff); 244 devm_kfree(dev, dc_data); 245 return -ENOMEM; 246 } 247 248 static void qat_free_dc_data(struct adf_accel_dev *accel_dev) 249 { 250 struct adf_dc_data *dc_data = accel_dev->dc_data; 251 struct device *dev = &GET_DEV(accel_dev); 252 253 if (!dc_data) 254 return; 255 256 dma_unmap_single(dev, dc_data->ovf_buff_p, dc_data->ovf_buff_sz, 257 DMA_BIDIRECTIONAL); 258 kfree_sensitive(dc_data->ovf_buff); 259 kfree(dc_data); 260 accel_dev->dc_data = NULL; 261 } 262 263 static int qat_compression_init(struct adf_accel_dev *accel_dev) 264 { 265 int ret; 266 267 ret = qat_compression_alloc_dc_data(accel_dev); 268 if (ret) 269 return ret; 270 271 ret = qat_compression_create_instances(accel_dev); 272 if (ret) 273 qat_free_dc_data(accel_dev); 274 275 return ret; 276 } 277 278 static int qat_compression_shutdown(struct adf_accel_dev *accel_dev) 279 { 280 qat_free_dc_data(accel_dev); 281 return qat_compression_free_instances(accel_dev); 282 } 283 284 static int qat_compression_event_handler(struct adf_accel_dev *accel_dev, 285 enum adf_event event) 286 { 287 int ret; 288 289 switch (event) { 290 case ADF_EVENT_INIT: 291 ret = qat_compression_init(accel_dev); 292 break; 293 case ADF_EVENT_SHUTDOWN: 294 ret = qat_compression_shutdown(accel_dev); 295 break; 296 case ADF_EVENT_RESTARTING: 297 case ADF_EVENT_RESTARTED: 298 case ADF_EVENT_START: 299 case ADF_EVENT_STOP: 300 default: 301 ret = 0; 302 } 303 return ret; 304 } 305 306 int qat_compression_register(void) 307 { 308 memset(&qat_compression, 0, sizeof(qat_compression)); 309 qat_compression.event_hld = qat_compression_event_handler; 310 qat_compression.name = "qat_compression"; 311 return adf_service_register(&qat_compression); 312 } 313 314 int qat_compression_unregister(void) 315 { 316 return adf_service_unregister(&qat_compression); 317 } 318