1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) 2 /* Copyright(c) 2014 - 2020 Intel Corporation */ 3 #include <linux/types.h> 4 #include <linux/mutex.h> 5 #include <linux/slab.h> 6 #include <linux/iopoll.h> 7 #include <linux/pci.h> 8 #include <linux/dma-mapping.h> 9 #include <linux/delay.h> 10 #include "adf_accel_devices.h" 11 #include "adf_admin.h" 12 #include "adf_anti_rb.h" 13 #include "adf_common_drv.h" 14 #include "adf_cfg.h" 15 #include "adf_heartbeat.h" 16 #include "adf_kpt.h" 17 #include "icp_qat_fw_init_admin.h" 18 19 #define ADF_ADMIN_MAILBOX_STRIDE 0x1000 20 #define ADF_ADMINMSG_LEN 32 21 #define ADF_CONST_TABLE_SIZE 1024 22 #define ADF_ADMIN_POLL_DELAY_US 20 23 #define ADF_ADMIN_POLL_TIMEOUT_US (5 * USEC_PER_SEC) 24 #define ADF_ONE_AE 1 25 #define ADF_ADMIN_RETRY_MAX 60 26 27 static const u8 const_tab[1024] __aligned(1024) = { 28 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 29 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 30 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 31 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 32 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 33 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 34 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 35 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 36 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 37 0x00, 0x00, 0x00, 0x03, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x01, 38 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 39 0x00, 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 0x02, 0x00, 0x00, 40 0x00, 0x00, 0x00, 0x00, 0x13, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 41 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 42 0x00, 0x00, 0x33, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 43 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 44 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 45 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 46 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 47 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 48 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 49 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 50 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 51 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 52 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xfe, 0xdc, 0xba, 0x98, 0x76, 53 0x54, 0x32, 0x10, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 54 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x67, 0x45, 0x23, 0x01, 0xef, 0xcd, 0xab, 55 0x89, 0x98, 0xba, 0xdc, 0xfe, 0x10, 0x32, 0x54, 0x76, 0xc3, 0xd2, 0xe1, 0xf0, 56 0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 57 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 58 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc1, 0x05, 0x9e, 59 0xd8, 0x36, 0x7c, 0xd5, 0x07, 0x30, 0x70, 0xdd, 0x17, 0xf7, 0x0e, 0x59, 0x39, 60 0xff, 0xc0, 0x0b, 0x31, 0x68, 0x58, 0x15, 0x11, 0x64, 0xf9, 0x8f, 0xa7, 0xbe, 61 0xfa, 0x4f, 0xa4, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 62 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6a, 0x09, 0xe6, 0x67, 0xbb, 0x67, 0xae, 63 0x85, 0x3c, 0x6e, 0xf3, 0x72, 0xa5, 0x4f, 0xf5, 0x3a, 0x51, 0x0e, 0x52, 0x7f, 64 0x9b, 0x05, 0x68, 0x8c, 0x1f, 0x83, 0xd9, 0xab, 0x5b, 0xe0, 0xcd, 0x19, 0x05, 65 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 66 0x00, 0x00, 0xcb, 0xbb, 0x9d, 0x5d, 0xc1, 0x05, 0x9e, 0xd8, 0x62, 0x9a, 0x29, 67 0x2a, 0x36, 0x7c, 0xd5, 0x07, 0x91, 0x59, 0x01, 0x5a, 0x30, 0x70, 0xdd, 0x17, 68 0x15, 0x2f, 0xec, 0xd8, 0xf7, 0x0e, 0x59, 0x39, 0x67, 0x33, 0x26, 0x67, 0xff, 69 0xc0, 0x0b, 0x31, 0x8e, 0xb4, 0x4a, 0x87, 0x68, 0x58, 0x15, 0x11, 0xdb, 0x0c, 70 0x2e, 0x0d, 0x64, 0xf9, 0x8f, 0xa7, 0x47, 0xb5, 0x48, 0x1d, 0xbe, 0xfa, 0x4f, 71 0xa4, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 72 0x00, 0x00, 0x00, 0x00, 0x6a, 0x09, 0xe6, 0x67, 0xf3, 0xbc, 0xc9, 0x08, 0xbb, 73 0x67, 0xae, 0x85, 0x84, 0xca, 0xa7, 0x3b, 0x3c, 0x6e, 0xf3, 0x72, 0xfe, 0x94, 74 0xf8, 0x2b, 0xa5, 0x4f, 0xf5, 0x3a, 0x5f, 0x1d, 0x36, 0xf1, 0x51, 0x0e, 0x52, 75 0x7f, 0xad, 0xe6, 0x82, 0xd1, 0x9b, 0x05, 0x68, 0x8c, 0x2b, 0x3e, 0x6c, 0x1f, 76 0x1f, 0x83, 0xd9, 0xab, 0xfb, 0x41, 0xbd, 0x6b, 0x5b, 0xe0, 0xcd, 0x19, 0x13, 77 0x7e, 0x21, 0x79, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 78 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 79 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 80 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x18, 81 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 82 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x01, 0x00, 83 0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 84 0x15, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x02, 0x00, 0x00, 0x00, 85 0x00, 0x00, 0x00, 0x14, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x02, 86 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 87 0x00, 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, 0x00, 0x00, 0x00, 88 0x00, 0x00, 0x00, 0x00, 0x24, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, 89 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 90 0x00, 0x00, 0x12, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0x00, 0x00, 91 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 92 0x45, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x45, 0x01, 0x00, 0x00, 0x00, 93 0x00, 0x00, 0x00, 0x44, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x01, 94 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 95 0x00, 0x2B, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 96 0x00, 0x00, 0x00, 0x00, 0x15, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 97 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 98 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 99 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 100 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 101 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 102 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 103 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 104 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 105 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 106 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; 107 108 struct adf_admin_comms { 109 dma_addr_t phy_addr; 110 dma_addr_t const_tbl_addr; 111 void *virt_addr; 112 void *virt_tbl_addr; 113 void __iomem *mailbox_addr; 114 struct mutex lock; /* protects adf_admin_comms struct */ 115 }; 116 117 static int adf_put_admin_msg_sync(struct adf_accel_dev *accel_dev, u32 ae, 118 void *in, void *out) 119 { 120 int ret; 121 u32 status; 122 struct adf_admin_comms *admin = accel_dev->admin; 123 int offset = ae * ADF_ADMINMSG_LEN * 2; 124 void __iomem *mailbox = admin->mailbox_addr; 125 int mb_offset = ae * ADF_ADMIN_MAILBOX_STRIDE; 126 struct icp_qat_fw_init_admin_req *request = in; 127 128 mutex_lock(&admin->lock); 129 130 if (ADF_CSR_RD(mailbox, mb_offset) == 1) { 131 mutex_unlock(&admin->lock); 132 return -EAGAIN; 133 } 134 135 memcpy(admin->virt_addr + offset, in, ADF_ADMINMSG_LEN); 136 ADF_CSR_WR(mailbox, mb_offset, 1); 137 138 ret = read_poll_timeout(ADF_CSR_RD, status, status == 0, 139 ADF_ADMIN_POLL_DELAY_US, 140 ADF_ADMIN_POLL_TIMEOUT_US, true, 141 mailbox, mb_offset); 142 if (ret < 0) { 143 /* Response timeout */ 144 dev_err(&GET_DEV(accel_dev), 145 "Failed to send admin msg %d to accelerator %d\n", 146 request->cmd_id, ae); 147 } else { 148 /* Response received from admin message, we can now 149 * make response data available in "out" parameter. 150 */ 151 memcpy(out, admin->virt_addr + offset + 152 ADF_ADMINMSG_LEN, ADF_ADMINMSG_LEN); 153 } 154 155 mutex_unlock(&admin->lock); 156 return ret; 157 } 158 159 static int adf_send_admin(struct adf_accel_dev *accel_dev, 160 struct icp_qat_fw_init_admin_req *req, 161 struct icp_qat_fw_init_admin_resp *resp, 162 const unsigned long ae_mask) 163 { 164 u32 ae; 165 166 for_each_set_bit(ae, &ae_mask, ICP_QAT_HW_AE_DELIMITER) 167 if (adf_put_admin_msg_sync(accel_dev, ae, req, resp) || 168 resp->status) 169 return -EFAULT; 170 171 return 0; 172 } 173 174 static int adf_init_ae(struct adf_accel_dev *accel_dev) 175 { 176 struct icp_qat_fw_init_admin_req req; 177 struct icp_qat_fw_init_admin_resp resp; 178 struct adf_hw_device_data *hw_device = accel_dev->hw_device; 179 u32 ae_mask = hw_device->ae_mask; 180 181 memset(&req, 0, sizeof(req)); 182 memset(&resp, 0, sizeof(resp)); 183 req.cmd_id = ICP_QAT_FW_INIT_AE; 184 185 return adf_send_admin(accel_dev, &req, &resp, ae_mask); 186 } 187 188 static int adf_set_fw_constants(struct adf_accel_dev *accel_dev) 189 { 190 struct icp_qat_fw_init_admin_req req; 191 struct icp_qat_fw_init_admin_resp resp; 192 struct adf_hw_device_data *hw_device = accel_dev->hw_device; 193 u32 ae_mask = hw_device->admin_ae_mask ?: hw_device->ae_mask; 194 195 memset(&req, 0, sizeof(req)); 196 memset(&resp, 0, sizeof(resp)); 197 req.cmd_id = ICP_QAT_FW_CONSTANTS_CFG; 198 199 req.init_cfg_sz = ADF_CONST_TABLE_SIZE; 200 req.init_cfg_ptr = accel_dev->admin->const_tbl_addr; 201 202 return adf_send_admin(accel_dev, &req, &resp, ae_mask); 203 } 204 205 int adf_get_fw_timestamp(struct adf_accel_dev *accel_dev, u64 *timestamp) 206 { 207 struct icp_qat_fw_init_admin_req req = { }; 208 struct icp_qat_fw_init_admin_resp resp; 209 unsigned int ae_mask = ADF_ONE_AE; 210 int ret; 211 212 req.cmd_id = ICP_QAT_FW_TIMER_GET; 213 ret = adf_send_admin(accel_dev, &req, &resp, ae_mask); 214 if (ret) 215 return ret; 216 217 *timestamp = resp.timestamp; 218 return 0; 219 } 220 221 static int adf_set_chaining(struct adf_accel_dev *accel_dev) 222 { 223 u32 ae_mask = GET_HW_DATA(accel_dev)->ae_mask; 224 struct icp_qat_fw_init_admin_resp resp = { }; 225 struct icp_qat_fw_init_admin_req req = { }; 226 227 req.cmd_id = ICP_QAT_FW_DC_CHAIN_INIT; 228 229 return adf_send_admin(accel_dev, &req, &resp, ae_mask); 230 } 231 232 static int adf_get_dc_capabilities(struct adf_accel_dev *accel_dev, 233 u32 *capabilities) 234 { 235 struct adf_hw_device_data *hw_device = accel_dev->hw_device; 236 struct icp_qat_fw_init_admin_resp resp; 237 struct icp_qat_fw_init_admin_req req; 238 unsigned long ae_mask; 239 unsigned long ae; 240 int ret; 241 242 /* Target only service accelerator engines */ 243 ae_mask = hw_device->ae_mask & ~hw_device->admin_ae_mask; 244 245 memset(&req, 0, sizeof(req)); 246 memset(&resp, 0, sizeof(resp)); 247 req.cmd_id = ICP_QAT_FW_COMP_CAPABILITY_GET; 248 249 *capabilities = 0; 250 for_each_set_bit(ae, &ae_mask, GET_MAX_ACCELENGINES(accel_dev)) { 251 ret = adf_send_admin(accel_dev, &req, &resp, 1ULL << ae); 252 if (ret) 253 return ret; 254 255 *capabilities |= resp.extended_features; 256 } 257 258 return 0; 259 } 260 261 int adf_get_ae_fw_counters(struct adf_accel_dev *accel_dev, u16 ae, u64 *reqs, u64 *resps) 262 { 263 struct icp_qat_fw_init_admin_resp resp = { }; 264 struct icp_qat_fw_init_admin_req req = { }; 265 int ret; 266 267 req.cmd_id = ICP_QAT_FW_COUNTERS_GET; 268 269 ret = adf_put_admin_msg_sync(accel_dev, ae, &req, &resp); 270 if (ret || resp.status) 271 return -EFAULT; 272 273 *reqs = resp.req_rec_count; 274 *resps = resp.resp_sent_count; 275 276 return 0; 277 } 278 279 int adf_send_admin_tim_sync(struct adf_accel_dev *accel_dev, u32 cnt) 280 { 281 u32 ae_mask = accel_dev->hw_device->ae_mask; 282 struct icp_qat_fw_init_admin_req req = { }; 283 struct icp_qat_fw_init_admin_resp resp = { }; 284 285 req.cmd_id = ICP_QAT_FW_SYNC; 286 req.int_timer_ticks = cnt; 287 288 return adf_send_admin(accel_dev, &req, &resp, ae_mask); 289 } 290 291 int adf_send_admin_hb_timer(struct adf_accel_dev *accel_dev, uint32_t ticks) 292 { 293 u32 ae_mask = accel_dev->hw_device->ae_mask; 294 struct icp_qat_fw_init_admin_req req = { }; 295 struct icp_qat_fw_init_admin_resp resp; 296 297 req.cmd_id = ICP_QAT_FW_HEARTBEAT_TIMER_SET; 298 req.init_cfg_ptr = accel_dev->heartbeat->dma.phy_addr; 299 req.heartbeat_ticks = ticks; 300 301 return adf_send_admin(accel_dev, &req, &resp, ae_mask); 302 } 303 304 static bool is_dcc_enabled(struct adf_accel_dev *accel_dev) 305 { 306 char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = {0}; 307 int ret; 308 309 ret = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, 310 ADF_SERVICES_ENABLED, services); 311 if (ret) 312 return false; 313 314 return !strcmp(services, "dcc"); 315 } 316 317 static int adf_get_fw_capabilities(struct adf_accel_dev *accel_dev, u16 *caps) 318 { 319 u32 ae_mask = accel_dev->hw_device->admin_ae_mask; 320 struct icp_qat_fw_init_admin_resp resp = { }; 321 struct icp_qat_fw_init_admin_req req = { }; 322 int ret; 323 324 if (!ae_mask) 325 return 0; 326 327 req.cmd_id = ICP_QAT_FW_CAPABILITIES_GET; 328 ret = adf_send_admin(accel_dev, &req, &resp, ae_mask); 329 if (ret) 330 return ret; 331 332 *caps = resp.fw_capabilities; 333 334 return 0; 335 } 336 337 int adf_send_admin_rl_init(struct adf_accel_dev *accel_dev, 338 struct icp_qat_fw_init_admin_slice_cnt *slices) 339 { 340 u32 ae_mask = accel_dev->hw_device->admin_ae_mask; 341 struct icp_qat_fw_init_admin_resp resp = { }; 342 struct icp_qat_fw_init_admin_req req = { }; 343 int ret; 344 345 req.cmd_id = ICP_QAT_FW_RL_INIT; 346 347 ret = adf_send_admin(accel_dev, &req, &resp, ae_mask); 348 if (ret) 349 return ret; 350 351 memcpy(slices, &resp.slices, sizeof(*slices)); 352 353 return 0; 354 } 355 356 int adf_send_admin_rl_add_update(struct adf_accel_dev *accel_dev, 357 struct icp_qat_fw_init_admin_req *req) 358 { 359 u32 ae_mask = accel_dev->hw_device->admin_ae_mask; 360 struct icp_qat_fw_init_admin_resp resp = { }; 361 362 /* 363 * req struct filled in rl implementation. Used commands 364 * ICP_QAT_FW_RL_ADD for a new SLA 365 * ICP_QAT_FW_RL_UPDATE for update SLA 366 */ 367 return adf_send_admin(accel_dev, req, &resp, ae_mask); 368 } 369 370 int adf_send_admin_rl_delete(struct adf_accel_dev *accel_dev, u16 node_id, 371 u8 node_type) 372 { 373 u32 ae_mask = accel_dev->hw_device->admin_ae_mask; 374 struct icp_qat_fw_init_admin_resp resp = { }; 375 struct icp_qat_fw_init_admin_req req = { }; 376 377 req.cmd_id = ICP_QAT_FW_RL_REMOVE; 378 req.node_id = node_id; 379 req.node_type = node_type; 380 381 return adf_send_admin(accel_dev, &req, &resp, ae_mask); 382 } 383 384 /** 385 * adf_send_admin_init() - Function sends init message to FW 386 * @accel_dev: Pointer to acceleration device. 387 * 388 * Function sends admin init message to the FW 389 * 390 * Return: 0 on success, error code otherwise. 391 */ 392 int adf_send_admin_init(struct adf_accel_dev *accel_dev) 393 { 394 struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); 395 u32 dc_capabilities = 0; 396 int ret; 397 398 ret = adf_set_fw_constants(accel_dev); 399 if (ret) 400 return ret; 401 402 if (is_dcc_enabled(accel_dev)) { 403 ret = adf_set_chaining(accel_dev); 404 if (ret) 405 return ret; 406 } 407 408 ret = adf_get_dc_capabilities(accel_dev, &dc_capabilities); 409 if (ret) { 410 dev_err(&GET_DEV(accel_dev), "Cannot get dc capabilities\n"); 411 return ret; 412 } 413 accel_dev->hw_device->extended_dc_capabilities = dc_capabilities; 414 415 adf_get_fw_capabilities(accel_dev, &hw_data->fw_capabilities); 416 417 return adf_init_ae(accel_dev); 418 } 419 EXPORT_SYMBOL_GPL(adf_send_admin_init); 420 421 /** 422 * adf_init_admin_pm() - Function sends PM init message to FW 423 * @accel_dev: Pointer to acceleration device. 424 * @idle_delay: QAT HW idle time before power gating is initiated. 425 * 000 - 64us 426 * 001 - 128us 427 * 010 - 256us 428 * 011 - 512us 429 * 100 - 1ms 430 * 101 - 2ms 431 * 110 - 4ms 432 * 111 - 8ms 433 * 434 * Function sends to the FW the admin init message for the PM state 435 * configuration. 436 * 437 * Return: 0 on success, error code otherwise. 438 */ 439 int adf_init_admin_pm(struct adf_accel_dev *accel_dev, u32 idle_delay) 440 { 441 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 442 struct icp_qat_fw_init_admin_resp resp = {0}; 443 struct icp_qat_fw_init_admin_req req = {0}; 444 u32 ae_mask = hw_data->admin_ae_mask; 445 446 if (!accel_dev->admin) { 447 dev_err(&GET_DEV(accel_dev), "adf_admin is not available\n"); 448 return -EFAULT; 449 } 450 451 req.cmd_id = ICP_QAT_FW_PM_STATE_CONFIG; 452 req.idle_filter = idle_delay; 453 454 return adf_send_admin(accel_dev, &req, &resp, ae_mask); 455 } 456 EXPORT_SYMBOL_GPL(adf_init_admin_pm); 457 458 int adf_get_pm_info(struct adf_accel_dev *accel_dev, dma_addr_t p_state_addr, 459 size_t buff_size) 460 { 461 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 462 struct icp_qat_fw_init_admin_req req = { }; 463 struct icp_qat_fw_init_admin_resp resp; 464 u32 ae_mask = hw_data->admin_ae_mask; 465 int ret; 466 467 /* Query pm info via init/admin cmd */ 468 if (!accel_dev->admin) { 469 dev_err(&GET_DEV(accel_dev), "adf_admin is not available\n"); 470 return -EFAULT; 471 } 472 473 req.cmd_id = ICP_QAT_FW_PM_INFO; 474 req.init_cfg_sz = buff_size; 475 req.init_cfg_ptr = p_state_addr; 476 477 ret = adf_send_admin(accel_dev, &req, &resp, ae_mask); 478 if (ret) 479 dev_err(&GET_DEV(accel_dev), 480 "Failed to query power-management info\n"); 481 482 return ret; 483 } 484 485 int adf_get_cnv_stats(struct adf_accel_dev *accel_dev, u16 ae, u16 *err_cnt, 486 u16 *latest_err) 487 { 488 struct icp_qat_fw_init_admin_req req = { }; 489 struct icp_qat_fw_init_admin_resp resp; 490 int ret; 491 492 req.cmd_id = ICP_QAT_FW_CNV_STATS_GET; 493 494 ret = adf_put_admin_msg_sync(accel_dev, ae, &req, &resp); 495 if (ret) 496 return ret; 497 if (resp.status) 498 return -EPROTONOSUPPORT; 499 500 *err_cnt = resp.error_count; 501 *latest_err = resp.latest_error; 502 503 return ret; 504 } 505 506 int adf_send_admin_tl_start(struct adf_accel_dev *accel_dev, 507 dma_addr_t tl_dma_addr, size_t layout_sz, u8 *rp_indexes, 508 struct icp_qat_fw_init_admin_slice_cnt *slice_count) 509 { 510 u32 ae_mask = GET_HW_DATA(accel_dev)->admin_ae_mask; 511 struct icp_qat_fw_init_admin_resp resp = { }; 512 struct icp_qat_fw_init_admin_req req = { }; 513 int ret; 514 515 req.cmd_id = ICP_QAT_FW_TL_START; 516 req.init_cfg_ptr = tl_dma_addr; 517 req.init_cfg_sz = layout_sz; 518 519 if (rp_indexes) 520 memcpy(&req.rp_indexes, rp_indexes, sizeof(req.rp_indexes)); 521 522 ret = adf_send_admin(accel_dev, &req, &resp, ae_mask); 523 if (ret) 524 return ret; 525 526 memcpy(slice_count, &resp.slices, sizeof(*slice_count)); 527 528 return 0; 529 } 530 531 int adf_send_admin_tl_stop(struct adf_accel_dev *accel_dev) 532 { 533 struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); 534 struct icp_qat_fw_init_admin_resp resp = { }; 535 struct icp_qat_fw_init_admin_req req = { }; 536 u32 ae_mask = hw_data->admin_ae_mask; 537 538 req.cmd_id = ICP_QAT_FW_TL_STOP; 539 540 return adf_send_admin(accel_dev, &req, &resp, ae_mask); 541 } 542 543 static int adf_send_admin_retry(struct adf_accel_dev *accel_dev, u8 cmd_id, 544 struct icp_qat_fw_init_admin_resp *resp, 545 unsigned int sleep_ms) 546 { 547 u32 admin_ae_mask = GET_HW_DATA(accel_dev)->admin_ae_mask; 548 struct icp_qat_fw_init_admin_req req = { }; 549 unsigned int retries = ADF_ADMIN_RETRY_MAX; 550 int ret; 551 552 req.cmd_id = cmd_id; 553 554 do { 555 ret = adf_send_admin(accel_dev, &req, resp, admin_ae_mask); 556 if (!ret) 557 return 0; 558 559 if (resp->status != ICP_QAT_FW_INIT_RESP_STATUS_RETRY) 560 return ret; 561 562 msleep(sleep_ms); 563 } while (--retries); 564 565 return -ETIMEDOUT; 566 } 567 568 static int adf_send_admin_svn(struct adf_accel_dev *accel_dev, u8 cmd_id, 569 struct icp_qat_fw_init_admin_resp *resp) 570 { 571 return adf_send_admin_retry(accel_dev, cmd_id, resp, ADF_SVN_RETRY_MS); 572 } 573 574 int adf_send_admin_arb_query(struct adf_accel_dev *accel_dev, int cmd, u8 *svn) 575 { 576 struct icp_qat_fw_init_admin_resp resp = { }; 577 int ret; 578 579 ret = adf_send_admin_svn(accel_dev, ICP_QAT_FW_SVN_READ, &resp); 580 if (ret) 581 return ret; 582 583 switch (cmd) { 584 case ARB_ENFORCED_MIN_SVN: 585 *svn = resp.enforced_min_svn; 586 break; 587 case ARB_PERMANENT_MIN_SVN: 588 *svn = resp.permanent_min_svn; 589 break; 590 case ARB_ACTIVE_SVN: 591 *svn = resp.active_svn; 592 break; 593 default: 594 *svn = 0; 595 dev_err(&GET_DEV(accel_dev), 596 "Unknown secure version number request\n"); 597 ret = -EINVAL; 598 } 599 600 return ret; 601 } 602 603 int adf_send_admin_arb_commit(struct adf_accel_dev *accel_dev) 604 { 605 struct icp_qat_fw_init_admin_resp resp = { }; 606 607 return adf_send_admin_svn(accel_dev, ICP_QAT_FW_SVN_COMMIT, &resp); 608 } 609 610 static_assert(sizeof(struct icp_qat_fw_init_admin_kpt_cfg) < PAGE_SIZE); 611 612 static void adf_cfg_kpt_config(struct adf_accel_dev *accel_dev, 613 struct icp_qat_fw_init_admin_kpt_cfg *kpt_config) 614 { 615 struct adf_kpt_interface_data *user_data = GET_KPT_USER_DATA(accel_dev); 616 617 kpt_config->swk_cnt_per_fn = user_data->swk_cnt_per_fn; 618 kpt_config->swk_cnt_per_pasid = user_data->swk_cnt_per_pasid; 619 kpt_config->swk_ttl_in_secs = user_data->swk_max_ttl; 620 kpt_config->swk_shared_disable = !user_data->swk_shared; 621 } 622 623 int adf_send_admin_kpt_init(struct adf_accel_dev *accel_dev, void *init_cfg, 624 size_t init_cfg_sz, dma_addr_t init_ptr) 625 { 626 struct icp_qat_fw_init_admin_kpt_cfg *kpt_config = init_cfg; 627 u32 ae_mask = GET_HW_DATA(accel_dev)->admin_ae_mask; 628 struct icp_qat_fw_init_admin_resp resp = { }; 629 struct icp_qat_fw_init_admin_req req = { }; 630 int ret; 631 632 if (!kpt_config || init_cfg_sz < sizeof(*kpt_config)) 633 return -EINVAL; 634 635 adf_cfg_kpt_config(accel_dev, kpt_config); 636 637 req.cmd_id = ICP_QAT_FW_KPT_ENABLE; 638 req.init_cfg_ptr = init_ptr; 639 req.init_cfg_sz = sizeof(*kpt_config); 640 641 ret = adf_send_admin(accel_dev, &req, &resp, ae_mask); 642 if (ret) 643 dev_err(&GET_DEV(accel_dev), "Failed to send KPT init admin message\n"); 644 645 return ret; 646 } 647 648 int adf_init_admin_comms(struct adf_accel_dev *accel_dev) 649 { 650 struct adf_admin_comms *admin; 651 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 652 void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); 653 struct admin_info admin_csrs_info; 654 u32 mailbox_offset, adminmsg_u, adminmsg_l; 655 void __iomem *mailbox; 656 u64 reg_val; 657 658 admin = kzalloc_node(sizeof(*accel_dev->admin), GFP_KERNEL, 659 dev_to_node(&GET_DEV(accel_dev))); 660 if (!admin) 661 return -ENOMEM; 662 admin->virt_addr = dma_alloc_coherent(&GET_DEV(accel_dev), PAGE_SIZE, 663 &admin->phy_addr, GFP_KERNEL); 664 if (!admin->virt_addr) { 665 dev_err(&GET_DEV(accel_dev), "Failed to allocate dma buff\n"); 666 kfree(admin); 667 return -ENOMEM; 668 } 669 670 admin->virt_tbl_addr = dma_alloc_coherent(&GET_DEV(accel_dev), 671 PAGE_SIZE, 672 &admin->const_tbl_addr, 673 GFP_KERNEL); 674 if (!admin->virt_tbl_addr) { 675 dev_err(&GET_DEV(accel_dev), "Failed to allocate const_tbl\n"); 676 dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE, 677 admin->virt_addr, admin->phy_addr); 678 kfree(admin); 679 return -ENOMEM; 680 } 681 682 memcpy(admin->virt_tbl_addr, const_tab, sizeof(const_tab)); 683 hw_data->get_admin_info(&admin_csrs_info); 684 685 mailbox_offset = admin_csrs_info.mailbox_offset; 686 mailbox = pmisc_addr + mailbox_offset; 687 adminmsg_u = admin_csrs_info.admin_msg_ur; 688 adminmsg_l = admin_csrs_info.admin_msg_lr; 689 690 reg_val = (u64)admin->phy_addr; 691 ADF_CSR_WR(pmisc_addr, adminmsg_u, upper_32_bits(reg_val)); 692 ADF_CSR_WR(pmisc_addr, adminmsg_l, lower_32_bits(reg_val)); 693 694 mutex_init(&admin->lock); 695 admin->mailbox_addr = mailbox; 696 accel_dev->admin = admin; 697 return 0; 698 } 699 EXPORT_SYMBOL_GPL(adf_init_admin_comms); 700 701 void adf_exit_admin_comms(struct adf_accel_dev *accel_dev) 702 { 703 struct adf_admin_comms *admin = accel_dev->admin; 704 705 if (!admin) 706 return; 707 708 if (admin->virt_addr) 709 dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE, 710 admin->virt_addr, admin->phy_addr); 711 if (admin->virt_tbl_addr) 712 dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE, 713 admin->virt_tbl_addr, admin->const_tbl_addr); 714 715 mutex_destroy(&admin->lock); 716 kfree(admin); 717 accel_dev->admin = NULL; 718 } 719 EXPORT_SYMBOL_GPL(adf_exit_admin_comms); 720