1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) 2 /* Copyright(c) 2014 - 2020 Intel Corporation */ 3 #include <linux/types.h> 4 #include <linux/mutex.h> 5 #include <linux/slab.h> 6 #include <linux/iopoll.h> 7 #include <linux/pci.h> 8 #include <linux/dma-mapping.h> 9 #include <linux/delay.h> 10 #include "adf_accel_devices.h" 11 #include "adf_admin.h" 12 #include "adf_anti_rb.h" 13 #include "adf_common_drv.h" 14 #include "adf_cfg.h" 15 #include "adf_heartbeat.h" 16 #include "icp_qat_fw_init_admin.h" 17 18 #define ADF_ADMIN_MAILBOX_STRIDE 0x1000 19 #define ADF_ADMINMSG_LEN 32 20 #define ADF_CONST_TABLE_SIZE 1024 21 #define ADF_ADMIN_POLL_DELAY_US 20 22 #define ADF_ADMIN_POLL_TIMEOUT_US (5 * USEC_PER_SEC) 23 #define ADF_ONE_AE 1 24 #define ADF_ADMIN_RETRY_MAX 60 25 26 static const u8 const_tab[1024] __aligned(1024) = { 27 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 28 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 29 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 30 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 31 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 32 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 33 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 34 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 35 0x21, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 36 0x00, 0x00, 0x00, 0x03, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x01, 37 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 38 0x00, 0x13, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 0x02, 0x00, 0x00, 39 0x00, 0x00, 0x00, 0x00, 0x13, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 40 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x00, 0x00, 0x00, 0x00, 0x00, 41 0x00, 0x00, 0x33, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 42 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 43 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 44 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 45 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 46 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 47 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 48 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 49 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 50 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 51 0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef, 0xfe, 0xdc, 0xba, 0x98, 0x76, 52 0x54, 0x32, 0x10, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 53 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x67, 0x45, 0x23, 0x01, 0xef, 0xcd, 0xab, 54 0x89, 0x98, 0xba, 0xdc, 0xfe, 0x10, 0x32, 0x54, 0x76, 0xc3, 0xd2, 0xe1, 0xf0, 55 0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 56 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 57 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xc1, 0x05, 0x9e, 58 0xd8, 0x36, 0x7c, 0xd5, 0x07, 0x30, 0x70, 0xdd, 0x17, 0xf7, 0x0e, 0x59, 0x39, 59 0xff, 0xc0, 0x0b, 0x31, 0x68, 0x58, 0x15, 0x11, 0x64, 0xf9, 0x8f, 0xa7, 0xbe, 60 0xfa, 0x4f, 0xa4, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 61 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6a, 0x09, 0xe6, 0x67, 0xbb, 0x67, 0xae, 62 0x85, 0x3c, 0x6e, 0xf3, 0x72, 0xa5, 0x4f, 0xf5, 0x3a, 0x51, 0x0e, 0x52, 0x7f, 63 0x9b, 0x05, 0x68, 0x8c, 0x1f, 0x83, 0xd9, 0xab, 0x5b, 0xe0, 0xcd, 0x19, 0x05, 64 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 65 0x00, 0x00, 0xcb, 0xbb, 0x9d, 0x5d, 0xc1, 0x05, 0x9e, 0xd8, 0x62, 0x9a, 0x29, 66 0x2a, 0x36, 0x7c, 0xd5, 0x07, 0x91, 0x59, 0x01, 0x5a, 0x30, 0x70, 0xdd, 0x17, 67 0x15, 0x2f, 0xec, 0xd8, 0xf7, 0x0e, 0x59, 0x39, 0x67, 0x33, 0x26, 0x67, 0xff, 68 0xc0, 0x0b, 0x31, 0x8e, 0xb4, 0x4a, 0x87, 0x68, 0x58, 0x15, 0x11, 0xdb, 0x0c, 69 0x2e, 0x0d, 0x64, 0xf9, 0x8f, 0xa7, 0x47, 0xb5, 0x48, 0x1d, 0xbe, 0xfa, 0x4f, 70 0xa4, 0x06, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 71 0x00, 0x00, 0x00, 0x00, 0x6a, 0x09, 0xe6, 0x67, 0xf3, 0xbc, 0xc9, 0x08, 0xbb, 72 0x67, 0xae, 0x85, 0x84, 0xca, 0xa7, 0x3b, 0x3c, 0x6e, 0xf3, 0x72, 0xfe, 0x94, 73 0xf8, 0x2b, 0xa5, 0x4f, 0xf5, 0x3a, 0x5f, 0x1d, 0x36, 0xf1, 0x51, 0x0e, 0x52, 74 0x7f, 0xad, 0xe6, 0x82, 0xd1, 0x9b, 0x05, 0x68, 0x8c, 0x2b, 0x3e, 0x6c, 0x1f, 75 0x1f, 0x83, 0xd9, 0xab, 0xfb, 0x41, 0xbd, 0x6b, 0x5b, 0xe0, 0xcd, 0x19, 0x13, 76 0x7e, 0x21, 0x79, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 77 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 78 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 79 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x18, 80 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 81 0x00, 0x00, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x01, 0x00, 82 0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 83 0x15, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0x02, 0x00, 0x00, 0x00, 84 0x00, 0x00, 0x00, 0x14, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x02, 85 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x15, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 86 0x00, 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, 0x00, 0x00, 0x00, 87 0x00, 0x00, 0x00, 0x00, 0x24, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, 88 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0x00, 0x00, 89 0x00, 0x00, 0x12, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0x00, 0x00, 90 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 91 0x45, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x45, 0x01, 0x00, 0x00, 0x00, 92 0x00, 0x00, 0x00, 0x44, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x44, 0x01, 93 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2B, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 94 0x00, 0x2B, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 95 0x00, 0x00, 0x00, 0x00, 0x15, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 96 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x17, 0x00, 0x00, 0x00, 0x00, 0x00, 97 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 98 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 99 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 100 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 101 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 102 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 103 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 104 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 105 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; 106 107 struct adf_admin_comms { 108 dma_addr_t phy_addr; 109 dma_addr_t const_tbl_addr; 110 void *virt_addr; 111 void *virt_tbl_addr; 112 void __iomem *mailbox_addr; 113 struct mutex lock; /* protects adf_admin_comms struct */ 114 }; 115 116 static int adf_put_admin_msg_sync(struct adf_accel_dev *accel_dev, u32 ae, 117 void *in, void *out) 118 { 119 int ret; 120 u32 status; 121 struct adf_admin_comms *admin = accel_dev->admin; 122 int offset = ae * ADF_ADMINMSG_LEN * 2; 123 void __iomem *mailbox = admin->mailbox_addr; 124 int mb_offset = ae * ADF_ADMIN_MAILBOX_STRIDE; 125 struct icp_qat_fw_init_admin_req *request = in; 126 127 mutex_lock(&admin->lock); 128 129 if (ADF_CSR_RD(mailbox, mb_offset) == 1) { 130 mutex_unlock(&admin->lock); 131 return -EAGAIN; 132 } 133 134 memcpy(admin->virt_addr + offset, in, ADF_ADMINMSG_LEN); 135 ADF_CSR_WR(mailbox, mb_offset, 1); 136 137 ret = read_poll_timeout(ADF_CSR_RD, status, status == 0, 138 ADF_ADMIN_POLL_DELAY_US, 139 ADF_ADMIN_POLL_TIMEOUT_US, true, 140 mailbox, mb_offset); 141 if (ret < 0) { 142 /* Response timeout */ 143 dev_err(&GET_DEV(accel_dev), 144 "Failed to send admin msg %d to accelerator %d\n", 145 request->cmd_id, ae); 146 } else { 147 /* Response received from admin message, we can now 148 * make response data available in "out" parameter. 149 */ 150 memcpy(out, admin->virt_addr + offset + 151 ADF_ADMINMSG_LEN, ADF_ADMINMSG_LEN); 152 } 153 154 mutex_unlock(&admin->lock); 155 return ret; 156 } 157 158 static int adf_send_admin(struct adf_accel_dev *accel_dev, 159 struct icp_qat_fw_init_admin_req *req, 160 struct icp_qat_fw_init_admin_resp *resp, 161 const unsigned long ae_mask) 162 { 163 u32 ae; 164 165 for_each_set_bit(ae, &ae_mask, ICP_QAT_HW_AE_DELIMITER) 166 if (adf_put_admin_msg_sync(accel_dev, ae, req, resp) || 167 resp->status) 168 return -EFAULT; 169 170 return 0; 171 } 172 173 static int adf_init_ae(struct adf_accel_dev *accel_dev) 174 { 175 struct icp_qat_fw_init_admin_req req; 176 struct icp_qat_fw_init_admin_resp resp; 177 struct adf_hw_device_data *hw_device = accel_dev->hw_device; 178 u32 ae_mask = hw_device->ae_mask; 179 180 memset(&req, 0, sizeof(req)); 181 memset(&resp, 0, sizeof(resp)); 182 req.cmd_id = ICP_QAT_FW_INIT_AE; 183 184 return adf_send_admin(accel_dev, &req, &resp, ae_mask); 185 } 186 187 static int adf_set_fw_constants(struct adf_accel_dev *accel_dev) 188 { 189 struct icp_qat_fw_init_admin_req req; 190 struct icp_qat_fw_init_admin_resp resp; 191 struct adf_hw_device_data *hw_device = accel_dev->hw_device; 192 u32 ae_mask = hw_device->admin_ae_mask ?: hw_device->ae_mask; 193 194 memset(&req, 0, sizeof(req)); 195 memset(&resp, 0, sizeof(resp)); 196 req.cmd_id = ICP_QAT_FW_CONSTANTS_CFG; 197 198 req.init_cfg_sz = ADF_CONST_TABLE_SIZE; 199 req.init_cfg_ptr = accel_dev->admin->const_tbl_addr; 200 201 return adf_send_admin(accel_dev, &req, &resp, ae_mask); 202 } 203 204 int adf_get_fw_timestamp(struct adf_accel_dev *accel_dev, u64 *timestamp) 205 { 206 struct icp_qat_fw_init_admin_req req = { }; 207 struct icp_qat_fw_init_admin_resp resp; 208 unsigned int ae_mask = ADF_ONE_AE; 209 int ret; 210 211 req.cmd_id = ICP_QAT_FW_TIMER_GET; 212 ret = adf_send_admin(accel_dev, &req, &resp, ae_mask); 213 if (ret) 214 return ret; 215 216 *timestamp = resp.timestamp; 217 return 0; 218 } 219 220 static int adf_set_chaining(struct adf_accel_dev *accel_dev) 221 { 222 u32 ae_mask = GET_HW_DATA(accel_dev)->ae_mask; 223 struct icp_qat_fw_init_admin_resp resp = { }; 224 struct icp_qat_fw_init_admin_req req = { }; 225 226 req.cmd_id = ICP_QAT_FW_DC_CHAIN_INIT; 227 228 return adf_send_admin(accel_dev, &req, &resp, ae_mask); 229 } 230 231 static int adf_get_dc_capabilities(struct adf_accel_dev *accel_dev, 232 u32 *capabilities) 233 { 234 struct adf_hw_device_data *hw_device = accel_dev->hw_device; 235 struct icp_qat_fw_init_admin_resp resp; 236 struct icp_qat_fw_init_admin_req req; 237 unsigned long ae_mask; 238 unsigned long ae; 239 int ret; 240 241 /* Target only service accelerator engines */ 242 ae_mask = hw_device->ae_mask & ~hw_device->admin_ae_mask; 243 244 memset(&req, 0, sizeof(req)); 245 memset(&resp, 0, sizeof(resp)); 246 req.cmd_id = ICP_QAT_FW_COMP_CAPABILITY_GET; 247 248 *capabilities = 0; 249 for_each_set_bit(ae, &ae_mask, GET_MAX_ACCELENGINES(accel_dev)) { 250 ret = adf_send_admin(accel_dev, &req, &resp, 1ULL << ae); 251 if (ret) 252 return ret; 253 254 *capabilities |= resp.extended_features; 255 } 256 257 return 0; 258 } 259 260 int adf_get_ae_fw_counters(struct adf_accel_dev *accel_dev, u16 ae, u64 *reqs, u64 *resps) 261 { 262 struct icp_qat_fw_init_admin_resp resp = { }; 263 struct icp_qat_fw_init_admin_req req = { }; 264 int ret; 265 266 req.cmd_id = ICP_QAT_FW_COUNTERS_GET; 267 268 ret = adf_put_admin_msg_sync(accel_dev, ae, &req, &resp); 269 if (ret || resp.status) 270 return -EFAULT; 271 272 *reqs = resp.req_rec_count; 273 *resps = resp.resp_sent_count; 274 275 return 0; 276 } 277 278 int adf_send_admin_tim_sync(struct adf_accel_dev *accel_dev, u32 cnt) 279 { 280 u32 ae_mask = accel_dev->hw_device->ae_mask; 281 struct icp_qat_fw_init_admin_req req = { }; 282 struct icp_qat_fw_init_admin_resp resp = { }; 283 284 req.cmd_id = ICP_QAT_FW_SYNC; 285 req.int_timer_ticks = cnt; 286 287 return adf_send_admin(accel_dev, &req, &resp, ae_mask); 288 } 289 290 int adf_send_admin_hb_timer(struct adf_accel_dev *accel_dev, uint32_t ticks) 291 { 292 u32 ae_mask = accel_dev->hw_device->ae_mask; 293 struct icp_qat_fw_init_admin_req req = { }; 294 struct icp_qat_fw_init_admin_resp resp; 295 296 req.cmd_id = ICP_QAT_FW_HEARTBEAT_TIMER_SET; 297 req.init_cfg_ptr = accel_dev->heartbeat->dma.phy_addr; 298 req.heartbeat_ticks = ticks; 299 300 return adf_send_admin(accel_dev, &req, &resp, ae_mask); 301 } 302 303 static bool is_dcc_enabled(struct adf_accel_dev *accel_dev) 304 { 305 char services[ADF_CFG_MAX_VAL_LEN_IN_BYTES] = {0}; 306 int ret; 307 308 ret = adf_cfg_get_param_value(accel_dev, ADF_GENERAL_SEC, 309 ADF_SERVICES_ENABLED, services); 310 if (ret) 311 return false; 312 313 return !strcmp(services, "dcc"); 314 } 315 316 static int adf_get_fw_capabilities(struct adf_accel_dev *accel_dev, u16 *caps) 317 { 318 u32 ae_mask = accel_dev->hw_device->admin_ae_mask; 319 struct icp_qat_fw_init_admin_resp resp = { }; 320 struct icp_qat_fw_init_admin_req req = { }; 321 int ret; 322 323 if (!ae_mask) 324 return 0; 325 326 req.cmd_id = ICP_QAT_FW_CAPABILITIES_GET; 327 ret = adf_send_admin(accel_dev, &req, &resp, ae_mask); 328 if (ret) 329 return ret; 330 331 *caps = resp.fw_capabilities; 332 333 return 0; 334 } 335 336 int adf_send_admin_rl_init(struct adf_accel_dev *accel_dev, 337 struct icp_qat_fw_init_admin_slice_cnt *slices) 338 { 339 u32 ae_mask = accel_dev->hw_device->admin_ae_mask; 340 struct icp_qat_fw_init_admin_resp resp = { }; 341 struct icp_qat_fw_init_admin_req req = { }; 342 int ret; 343 344 req.cmd_id = ICP_QAT_FW_RL_INIT; 345 346 ret = adf_send_admin(accel_dev, &req, &resp, ae_mask); 347 if (ret) 348 return ret; 349 350 memcpy(slices, &resp.slices, sizeof(*slices)); 351 352 return 0; 353 } 354 355 int adf_send_admin_rl_add_update(struct adf_accel_dev *accel_dev, 356 struct icp_qat_fw_init_admin_req *req) 357 { 358 u32 ae_mask = accel_dev->hw_device->admin_ae_mask; 359 struct icp_qat_fw_init_admin_resp resp = { }; 360 361 /* 362 * req struct filled in rl implementation. Used commands 363 * ICP_QAT_FW_RL_ADD for a new SLA 364 * ICP_QAT_FW_RL_UPDATE for update SLA 365 */ 366 return adf_send_admin(accel_dev, req, &resp, ae_mask); 367 } 368 369 int adf_send_admin_rl_delete(struct adf_accel_dev *accel_dev, u16 node_id, 370 u8 node_type) 371 { 372 u32 ae_mask = accel_dev->hw_device->admin_ae_mask; 373 struct icp_qat_fw_init_admin_resp resp = { }; 374 struct icp_qat_fw_init_admin_req req = { }; 375 376 req.cmd_id = ICP_QAT_FW_RL_REMOVE; 377 req.node_id = node_id; 378 req.node_type = node_type; 379 380 return adf_send_admin(accel_dev, &req, &resp, ae_mask); 381 } 382 383 /** 384 * adf_send_admin_init() - Function sends init message to FW 385 * @accel_dev: Pointer to acceleration device. 386 * 387 * Function sends admin init message to the FW 388 * 389 * Return: 0 on success, error code otherwise. 390 */ 391 int adf_send_admin_init(struct adf_accel_dev *accel_dev) 392 { 393 struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); 394 u32 dc_capabilities = 0; 395 int ret; 396 397 ret = adf_set_fw_constants(accel_dev); 398 if (ret) 399 return ret; 400 401 if (is_dcc_enabled(accel_dev)) { 402 ret = adf_set_chaining(accel_dev); 403 if (ret) 404 return ret; 405 } 406 407 ret = adf_get_dc_capabilities(accel_dev, &dc_capabilities); 408 if (ret) { 409 dev_err(&GET_DEV(accel_dev), "Cannot get dc capabilities\n"); 410 return ret; 411 } 412 accel_dev->hw_device->extended_dc_capabilities = dc_capabilities; 413 414 adf_get_fw_capabilities(accel_dev, &hw_data->fw_capabilities); 415 416 return adf_init_ae(accel_dev); 417 } 418 EXPORT_SYMBOL_GPL(adf_send_admin_init); 419 420 /** 421 * adf_init_admin_pm() - Function sends PM init message to FW 422 * @accel_dev: Pointer to acceleration device. 423 * @idle_delay: QAT HW idle time before power gating is initiated. 424 * 000 - 64us 425 * 001 - 128us 426 * 010 - 256us 427 * 011 - 512us 428 * 100 - 1ms 429 * 101 - 2ms 430 * 110 - 4ms 431 * 111 - 8ms 432 * 433 * Function sends to the FW the admin init message for the PM state 434 * configuration. 435 * 436 * Return: 0 on success, error code otherwise. 437 */ 438 int adf_init_admin_pm(struct adf_accel_dev *accel_dev, u32 idle_delay) 439 { 440 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 441 struct icp_qat_fw_init_admin_resp resp = {0}; 442 struct icp_qat_fw_init_admin_req req = {0}; 443 u32 ae_mask = hw_data->admin_ae_mask; 444 445 if (!accel_dev->admin) { 446 dev_err(&GET_DEV(accel_dev), "adf_admin is not available\n"); 447 return -EFAULT; 448 } 449 450 req.cmd_id = ICP_QAT_FW_PM_STATE_CONFIG; 451 req.idle_filter = idle_delay; 452 453 return adf_send_admin(accel_dev, &req, &resp, ae_mask); 454 } 455 EXPORT_SYMBOL_GPL(adf_init_admin_pm); 456 457 int adf_get_pm_info(struct adf_accel_dev *accel_dev, dma_addr_t p_state_addr, 458 size_t buff_size) 459 { 460 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 461 struct icp_qat_fw_init_admin_req req = { }; 462 struct icp_qat_fw_init_admin_resp resp; 463 u32 ae_mask = hw_data->admin_ae_mask; 464 int ret; 465 466 /* Query pm info via init/admin cmd */ 467 if (!accel_dev->admin) { 468 dev_err(&GET_DEV(accel_dev), "adf_admin is not available\n"); 469 return -EFAULT; 470 } 471 472 req.cmd_id = ICP_QAT_FW_PM_INFO; 473 req.init_cfg_sz = buff_size; 474 req.init_cfg_ptr = p_state_addr; 475 476 ret = adf_send_admin(accel_dev, &req, &resp, ae_mask); 477 if (ret) 478 dev_err(&GET_DEV(accel_dev), 479 "Failed to query power-management info\n"); 480 481 return ret; 482 } 483 484 int adf_get_cnv_stats(struct adf_accel_dev *accel_dev, u16 ae, u16 *err_cnt, 485 u16 *latest_err) 486 { 487 struct icp_qat_fw_init_admin_req req = { }; 488 struct icp_qat_fw_init_admin_resp resp; 489 int ret; 490 491 req.cmd_id = ICP_QAT_FW_CNV_STATS_GET; 492 493 ret = adf_put_admin_msg_sync(accel_dev, ae, &req, &resp); 494 if (ret) 495 return ret; 496 if (resp.status) 497 return -EPROTONOSUPPORT; 498 499 *err_cnt = resp.error_count; 500 *latest_err = resp.latest_error; 501 502 return ret; 503 } 504 505 int adf_send_admin_tl_start(struct adf_accel_dev *accel_dev, 506 dma_addr_t tl_dma_addr, size_t layout_sz, u8 *rp_indexes, 507 struct icp_qat_fw_init_admin_slice_cnt *slice_count) 508 { 509 u32 ae_mask = GET_HW_DATA(accel_dev)->admin_ae_mask; 510 struct icp_qat_fw_init_admin_resp resp = { }; 511 struct icp_qat_fw_init_admin_req req = { }; 512 int ret; 513 514 req.cmd_id = ICP_QAT_FW_TL_START; 515 req.init_cfg_ptr = tl_dma_addr; 516 req.init_cfg_sz = layout_sz; 517 518 if (rp_indexes) 519 memcpy(&req.rp_indexes, rp_indexes, sizeof(req.rp_indexes)); 520 521 ret = adf_send_admin(accel_dev, &req, &resp, ae_mask); 522 if (ret) 523 return ret; 524 525 memcpy(slice_count, &resp.slices, sizeof(*slice_count)); 526 527 return 0; 528 } 529 530 int adf_send_admin_tl_stop(struct adf_accel_dev *accel_dev) 531 { 532 struct adf_hw_device_data *hw_data = GET_HW_DATA(accel_dev); 533 struct icp_qat_fw_init_admin_resp resp = { }; 534 struct icp_qat_fw_init_admin_req req = { }; 535 u32 ae_mask = hw_data->admin_ae_mask; 536 537 req.cmd_id = ICP_QAT_FW_TL_STOP; 538 539 return adf_send_admin(accel_dev, &req, &resp, ae_mask); 540 } 541 542 static int adf_send_admin_retry(struct adf_accel_dev *accel_dev, u8 cmd_id, 543 struct icp_qat_fw_init_admin_resp *resp, 544 unsigned int sleep_ms) 545 { 546 u32 admin_ae_mask = GET_HW_DATA(accel_dev)->admin_ae_mask; 547 struct icp_qat_fw_init_admin_req req = { }; 548 unsigned int retries = ADF_ADMIN_RETRY_MAX; 549 int ret; 550 551 req.cmd_id = cmd_id; 552 553 do { 554 ret = adf_send_admin(accel_dev, &req, resp, admin_ae_mask); 555 if (!ret) 556 return 0; 557 558 if (resp->status != ICP_QAT_FW_INIT_RESP_STATUS_RETRY) 559 return ret; 560 561 msleep(sleep_ms); 562 } while (--retries); 563 564 return -ETIMEDOUT; 565 } 566 567 static int adf_send_admin_svn(struct adf_accel_dev *accel_dev, u8 cmd_id, 568 struct icp_qat_fw_init_admin_resp *resp) 569 { 570 return adf_send_admin_retry(accel_dev, cmd_id, resp, ADF_SVN_RETRY_MS); 571 } 572 573 int adf_send_admin_arb_query(struct adf_accel_dev *accel_dev, int cmd, u8 *svn) 574 { 575 struct icp_qat_fw_init_admin_resp resp = { }; 576 int ret; 577 578 ret = adf_send_admin_svn(accel_dev, ICP_QAT_FW_SVN_READ, &resp); 579 if (ret) 580 return ret; 581 582 switch (cmd) { 583 case ARB_ENFORCED_MIN_SVN: 584 *svn = resp.enforced_min_svn; 585 break; 586 case ARB_PERMANENT_MIN_SVN: 587 *svn = resp.permanent_min_svn; 588 break; 589 case ARB_ACTIVE_SVN: 590 *svn = resp.active_svn; 591 break; 592 default: 593 *svn = 0; 594 dev_err(&GET_DEV(accel_dev), 595 "Unknown secure version number request\n"); 596 ret = -EINVAL; 597 } 598 599 return ret; 600 } 601 602 int adf_send_admin_arb_commit(struct adf_accel_dev *accel_dev) 603 { 604 struct icp_qat_fw_init_admin_resp resp = { }; 605 606 return adf_send_admin_svn(accel_dev, ICP_QAT_FW_SVN_COMMIT, &resp); 607 } 608 609 int adf_init_admin_comms(struct adf_accel_dev *accel_dev) 610 { 611 struct adf_admin_comms *admin; 612 struct adf_hw_device_data *hw_data = accel_dev->hw_device; 613 void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); 614 struct admin_info admin_csrs_info; 615 u32 mailbox_offset, adminmsg_u, adminmsg_l; 616 void __iomem *mailbox; 617 u64 reg_val; 618 619 admin = kzalloc_node(sizeof(*accel_dev->admin), GFP_KERNEL, 620 dev_to_node(&GET_DEV(accel_dev))); 621 if (!admin) 622 return -ENOMEM; 623 admin->virt_addr = dma_alloc_coherent(&GET_DEV(accel_dev), PAGE_SIZE, 624 &admin->phy_addr, GFP_KERNEL); 625 if (!admin->virt_addr) { 626 dev_err(&GET_DEV(accel_dev), "Failed to allocate dma buff\n"); 627 kfree(admin); 628 return -ENOMEM; 629 } 630 631 admin->virt_tbl_addr = dma_alloc_coherent(&GET_DEV(accel_dev), 632 PAGE_SIZE, 633 &admin->const_tbl_addr, 634 GFP_KERNEL); 635 if (!admin->virt_tbl_addr) { 636 dev_err(&GET_DEV(accel_dev), "Failed to allocate const_tbl\n"); 637 dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE, 638 admin->virt_addr, admin->phy_addr); 639 kfree(admin); 640 return -ENOMEM; 641 } 642 643 memcpy(admin->virt_tbl_addr, const_tab, sizeof(const_tab)); 644 hw_data->get_admin_info(&admin_csrs_info); 645 646 mailbox_offset = admin_csrs_info.mailbox_offset; 647 mailbox = pmisc_addr + mailbox_offset; 648 adminmsg_u = admin_csrs_info.admin_msg_ur; 649 adminmsg_l = admin_csrs_info.admin_msg_lr; 650 651 reg_val = (u64)admin->phy_addr; 652 ADF_CSR_WR(pmisc_addr, adminmsg_u, upper_32_bits(reg_val)); 653 ADF_CSR_WR(pmisc_addr, adminmsg_l, lower_32_bits(reg_val)); 654 655 mutex_init(&admin->lock); 656 admin->mailbox_addr = mailbox; 657 accel_dev->admin = admin; 658 return 0; 659 } 660 EXPORT_SYMBOL_GPL(adf_init_admin_comms); 661 662 void adf_exit_admin_comms(struct adf_accel_dev *accel_dev) 663 { 664 struct adf_admin_comms *admin = accel_dev->admin; 665 666 if (!admin) 667 return; 668 669 if (admin->virt_addr) 670 dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE, 671 admin->virt_addr, admin->phy_addr); 672 if (admin->virt_tbl_addr) 673 dma_free_coherent(&GET_DEV(accel_dev), PAGE_SIZE, 674 admin->virt_tbl_addr, admin->const_tbl_addr); 675 676 mutex_destroy(&admin->lock); 677 kfree(admin); 678 accel_dev->admin = NULL; 679 } 680 EXPORT_SYMBOL_GPL(adf_exit_admin_comms); 681