1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only) 2 /* Copyright(c) 2021 Intel Corporation */ 3 #include <linux/delay.h> 4 #include <linux/iopoll.h> 5 #include <linux/mutex.h> 6 #include <linux/types.h> 7 #include "adf_accel_devices.h" 8 #include "adf_common_drv.h" 9 #include "adf_gen2_pfvf.h" 10 #include "adf_pfvf_msg.h" 11 #include "adf_pfvf_pf_proto.h" 12 #include "adf_pfvf_vf_proto.h" 13 #include "adf_pfvf_utils.h" 14 15 /* VF2PF interrupts */ 16 #define ADF_GEN2_VF_MSK 0xFFFF 17 #define ADF_GEN2_ERR_REG_VF2PF(vf_src) (((vf_src) & 0x01FFFE00) >> 9) 18 #define ADF_GEN2_ERR_MSK_VF2PF(vf_mask) (((vf_mask) & ADF_GEN2_VF_MSK) << 9) 19 20 #define ADF_GEN2_PF_PF2VF_OFFSET(i) (0x3A000 + 0x280 + ((i) * 0x04)) 21 #define ADF_GEN2_VF_PF2VF_OFFSET 0x200 22 23 #define ADF_GEN2_CSR_IN_USE 0x6AC2 24 #define ADF_GEN2_CSR_IN_USE_MASK 0xFFFE 25 26 enum gen2_csr_pos { 27 ADF_GEN2_CSR_PF2VF_OFFSET = 0, 28 ADF_GEN2_CSR_VF2PF_OFFSET = 16, 29 }; 30 31 #define ADF_PFVF_GEN2_MSGTYPE_SHIFT 2 32 #define ADF_PFVF_GEN2_MSGTYPE_MASK 0x0F 33 #define ADF_PFVF_GEN2_MSGDATA_SHIFT 6 34 #define ADF_PFVF_GEN2_MSGDATA_MASK 0x3FF 35 36 static const struct pfvf_csr_format csr_gen2_fmt = { 37 { ADF_PFVF_GEN2_MSGTYPE_SHIFT, ADF_PFVF_GEN2_MSGTYPE_MASK }, 38 { ADF_PFVF_GEN2_MSGDATA_SHIFT, ADF_PFVF_GEN2_MSGDATA_MASK }, 39 }; 40 41 #define ADF_PFVF_MSG_RETRY_DELAY 5 42 #define ADF_PFVF_MSG_MAX_RETRIES 3 43 44 static u32 adf_gen2_pf_get_pfvf_offset(u32 i) 45 { 46 return ADF_GEN2_PF_PF2VF_OFFSET(i); 47 } 48 49 static u32 adf_gen2_vf_get_pfvf_offset(u32 i) 50 { 51 return ADF_GEN2_VF_PF2VF_OFFSET; 52 } 53 54 static void adf_gen2_enable_vf2pf_interrupts(void __iomem *pmisc_addr, u32 vf_mask) 55 { 56 /* Enable VF2PF Messaging Ints - VFs 0 through 15 per vf_mask[15:0] */ 57 if (vf_mask & ADF_GEN2_VF_MSK) { 58 u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3) 59 & ~ADF_GEN2_ERR_MSK_VF2PF(vf_mask); 60 ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val); 61 } 62 } 63 64 static void adf_gen2_disable_all_vf2pf_interrupts(void __iomem *pmisc_addr) 65 { 66 /* Disable VF2PF interrupts for VFs 0 through 15 per vf_mask[15:0] */ 67 u32 val = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3) 68 | ADF_GEN2_ERR_MSK_VF2PF(ADF_GEN2_VF_MSK); 69 ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, val); 70 } 71 72 static u32 adf_gen2_disable_pending_vf2pf_interrupts(void __iomem *pmisc_addr) 73 { 74 u32 sources, disabled, pending; 75 u32 errsou3, errmsk3; 76 77 /* Get the interrupt sources triggered by VFs */ 78 errsou3 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRSOU3); 79 sources = ADF_GEN2_ERR_REG_VF2PF(errsou3); 80 81 if (!sources) 82 return 0; 83 84 /* Get the already disabled interrupts */ 85 errmsk3 = ADF_CSR_RD(pmisc_addr, ADF_GEN2_ERRMSK3); 86 disabled = ADF_GEN2_ERR_REG_VF2PF(errmsk3); 87 88 pending = sources & ~disabled; 89 if (!pending) 90 return 0; 91 92 /* Due to HW limitations, when disabling the interrupts, we can't 93 * just disable the requested sources, as this would lead to missed 94 * interrupts if ERRSOU3 changes just before writing to ERRMSK3. 95 * To work around it, disable all and re-enable only the sources that 96 * are not in vf_mask and were not already disabled. Re-enabling will 97 * trigger a new interrupt for the sources that have changed in the 98 * meantime, if any. 99 */ 100 errmsk3 |= ADF_GEN2_ERR_MSK_VF2PF(ADF_GEN2_VF_MSK); 101 ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, errmsk3); 102 103 /* Update only section of errmsk3 related to VF2PF */ 104 errmsk3 &= ~ADF_GEN2_ERR_MSK_VF2PF(ADF_GEN2_VF_MSK); 105 errmsk3 |= ADF_GEN2_ERR_MSK_VF2PF(sources | disabled); 106 ADF_CSR_WR(pmisc_addr, ADF_GEN2_ERRMSK3, errmsk3); 107 108 /* Return the sources of the (new) interrupt(s) */ 109 return pending; 110 } 111 112 static u32 gen2_csr_get_int_bit(enum gen2_csr_pos offset) 113 { 114 return ADF_PFVF_INT << offset; 115 } 116 117 static u32 gen2_csr_msg_to_position(u32 csr_msg, enum gen2_csr_pos offset) 118 { 119 return (csr_msg & 0xFFFF) << offset; 120 } 121 122 static u32 gen2_csr_msg_from_position(u32 csr_val, enum gen2_csr_pos offset) 123 { 124 return (csr_val >> offset) & 0xFFFF; 125 } 126 127 static bool gen2_csr_is_in_use(u32 msg, enum gen2_csr_pos offset) 128 { 129 return ((msg >> offset) & ADF_GEN2_CSR_IN_USE_MASK) == ADF_GEN2_CSR_IN_USE; 130 } 131 132 static void gen2_csr_clear_in_use(u32 *msg, enum gen2_csr_pos offset) 133 { 134 *msg &= ~(ADF_GEN2_CSR_IN_USE_MASK << offset); 135 } 136 137 static void gen2_csr_set_in_use(u32 *msg, enum gen2_csr_pos offset) 138 { 139 *msg |= (ADF_GEN2_CSR_IN_USE << offset); 140 } 141 142 static bool is_legacy_user_pfvf_message(u32 msg) 143 { 144 return !(msg & ADF_PFVF_MSGORIGIN_SYSTEM); 145 } 146 147 static bool is_pf2vf_notification(u8 msg_type) 148 { 149 switch (msg_type) { 150 case ADF_PF2VF_MSGTYPE_RESTARTING: 151 return true; 152 default: 153 return false; 154 } 155 } 156 157 static bool is_vf2pf_notification(u8 msg_type) 158 { 159 switch (msg_type) { 160 case ADF_VF2PF_MSGTYPE_INIT: 161 case ADF_VF2PF_MSGTYPE_SHUTDOWN: 162 return true; 163 default: 164 return false; 165 } 166 } 167 168 struct pfvf_gen2_params { 169 u32 pfvf_offset; 170 struct mutex *csr_lock; /* lock preventing concurrent access of CSR */ 171 enum gen2_csr_pos local_offset; 172 enum gen2_csr_pos remote_offset; 173 bool (*is_notification_message)(u8 msg_type); 174 u8 compat_ver; 175 }; 176 177 static int adf_gen2_pfvf_send(struct adf_accel_dev *accel_dev, 178 struct pfvf_message msg, 179 struct pfvf_gen2_params *params) 180 { 181 void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); 182 enum gen2_csr_pos remote_offset = params->remote_offset; 183 enum gen2_csr_pos local_offset = params->local_offset; 184 unsigned int retries = ADF_PFVF_MSG_MAX_RETRIES; 185 struct mutex *lock = params->csr_lock; 186 u32 pfvf_offset = params->pfvf_offset; 187 u32 int_bit; 188 u32 csr_val; 189 u32 csr_msg; 190 int ret; 191 192 /* Gen2 messages, both PF->VF and VF->PF, are all 16 bits long. This 193 * allows us to build and read messages as if they where all 0 based. 194 * However, send and receive are in a single shared 32 bits register, 195 * so we need to shift and/or mask the message half before decoding 196 * it and after encoding it. Which one to shift depends on the 197 * direction. 198 */ 199 200 int_bit = gen2_csr_get_int_bit(local_offset); 201 202 csr_msg = adf_pfvf_csr_msg_of(accel_dev, msg, &csr_gen2_fmt); 203 if (unlikely(!csr_msg)) 204 return -EINVAL; 205 206 /* Prepare for CSR format, shifting the wire message in place and 207 * setting the in use pattern 208 */ 209 csr_msg = gen2_csr_msg_to_position(csr_msg, local_offset); 210 gen2_csr_set_in_use(&csr_msg, remote_offset); 211 212 mutex_lock(lock); 213 214 start: 215 /* Check if the PFVF CSR is in use by remote function */ 216 csr_val = ADF_CSR_RD(pmisc_addr, pfvf_offset); 217 if (gen2_csr_is_in_use(csr_val, local_offset)) { 218 dev_dbg(&GET_DEV(accel_dev), 219 "PFVF CSR in use by remote function\n"); 220 goto retry; 221 } 222 223 /* Attempt to get ownership of the PFVF CSR */ 224 ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_msg | int_bit); 225 226 /* Wait for confirmation from remote func it received the message */ 227 ret = read_poll_timeout(ADF_CSR_RD, csr_val, !(csr_val & int_bit), 228 ADF_PFVF_MSG_ACK_DELAY_US, 229 ADF_PFVF_MSG_ACK_MAX_DELAY_US, 230 true, pmisc_addr, pfvf_offset); 231 if (unlikely(ret < 0)) { 232 dev_dbg(&GET_DEV(accel_dev), "ACK not received from remote\n"); 233 csr_val &= ~int_bit; 234 } 235 236 /* For fire-and-forget notifications, the receiver does not clear 237 * the in-use pattern. This is used to detect collisions. 238 */ 239 if (params->is_notification_message(msg.type) && csr_val != csr_msg) { 240 /* Collision must have overwritten the message */ 241 dev_err(&GET_DEV(accel_dev), 242 "Collision on notification - PFVF CSR overwritten by remote function\n"); 243 goto retry; 244 } 245 246 /* If the far side did not clear the in-use pattern it is either 247 * 1) Notification - message left intact to detect collision 248 * 2) Older protocol (compatibility version < 3) on the far side 249 * where the sender is responsible for clearing the in-use 250 * pattern after the received has acknowledged receipt. 251 * In either case, clear the in-use pattern now. 252 */ 253 if (gen2_csr_is_in_use(csr_val, remote_offset)) { 254 gen2_csr_clear_in_use(&csr_val, remote_offset); 255 ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val); 256 } 257 258 out: 259 mutex_unlock(lock); 260 return ret; 261 262 retry: 263 if (--retries) { 264 msleep(ADF_PFVF_MSG_RETRY_DELAY); 265 goto start; 266 } else { 267 ret = -EBUSY; 268 goto out; 269 } 270 } 271 272 static struct pfvf_message adf_gen2_pfvf_recv(struct adf_accel_dev *accel_dev, 273 struct pfvf_gen2_params *params) 274 { 275 void __iomem *pmisc_addr = adf_get_pmisc_base(accel_dev); 276 enum gen2_csr_pos remote_offset = params->remote_offset; 277 enum gen2_csr_pos local_offset = params->local_offset; 278 u32 pfvf_offset = params->pfvf_offset; 279 struct pfvf_message msg = { 0 }; 280 u32 int_bit; 281 u32 csr_val; 282 u16 csr_msg; 283 284 int_bit = gen2_csr_get_int_bit(local_offset); 285 286 /* Read message */ 287 csr_val = ADF_CSR_RD(pmisc_addr, pfvf_offset); 288 if (!(csr_val & int_bit)) { 289 dev_info(&GET_DEV(accel_dev), 290 "Spurious PFVF interrupt, msg 0x%.8x. Ignored\n", csr_val); 291 return msg; 292 } 293 294 /* Extract the message from the CSR */ 295 csr_msg = gen2_csr_msg_from_position(csr_val, local_offset); 296 297 /* Ignore legacy non-system (non-kernel) messages */ 298 if (unlikely(is_legacy_user_pfvf_message(csr_msg))) { 299 dev_dbg(&GET_DEV(accel_dev), 300 "Ignored non-system message (0x%.8x);\n", csr_val); 301 /* Because this must be a legacy message, the far side 302 * must clear the in-use pattern, so don't do it. 303 */ 304 return msg; 305 } 306 307 /* Return the pfvf_message format */ 308 msg = adf_pfvf_message_of(accel_dev, csr_msg, &csr_gen2_fmt); 309 310 /* The in-use pattern is not cleared for notifications (so that 311 * it can be used for collision detection) or older implementations 312 */ 313 if (params->compat_ver >= ADF_PFVF_COMPAT_FAST_ACK && 314 !params->is_notification_message(msg.type)) 315 gen2_csr_clear_in_use(&csr_val, remote_offset); 316 317 /* To ACK, clear the INT bit */ 318 csr_val &= ~int_bit; 319 ADF_CSR_WR(pmisc_addr, pfvf_offset, csr_val); 320 321 return msg; 322 } 323 324 static int adf_gen2_pf2vf_send(struct adf_accel_dev *accel_dev, struct pfvf_message msg, 325 u32 pfvf_offset, struct mutex *csr_lock) 326 { 327 struct pfvf_gen2_params params = { 328 .csr_lock = csr_lock, 329 .pfvf_offset = pfvf_offset, 330 .local_offset = ADF_GEN2_CSR_PF2VF_OFFSET, 331 .remote_offset = ADF_GEN2_CSR_VF2PF_OFFSET, 332 .is_notification_message = is_pf2vf_notification, 333 }; 334 335 return adf_gen2_pfvf_send(accel_dev, msg, ¶ms); 336 } 337 338 static int adf_gen2_vf2pf_send(struct adf_accel_dev *accel_dev, struct pfvf_message msg, 339 u32 pfvf_offset, struct mutex *csr_lock) 340 { 341 struct pfvf_gen2_params params = { 342 .csr_lock = csr_lock, 343 .pfvf_offset = pfvf_offset, 344 .local_offset = ADF_GEN2_CSR_VF2PF_OFFSET, 345 .remote_offset = ADF_GEN2_CSR_PF2VF_OFFSET, 346 .is_notification_message = is_vf2pf_notification, 347 }; 348 349 return adf_gen2_pfvf_send(accel_dev, msg, ¶ms); 350 } 351 352 static struct pfvf_message adf_gen2_pf2vf_recv(struct adf_accel_dev *accel_dev, 353 u32 pfvf_offset, u8 compat_ver) 354 { 355 struct pfvf_gen2_params params = { 356 .pfvf_offset = pfvf_offset, 357 .local_offset = ADF_GEN2_CSR_PF2VF_OFFSET, 358 .remote_offset = ADF_GEN2_CSR_VF2PF_OFFSET, 359 .is_notification_message = is_pf2vf_notification, 360 .compat_ver = compat_ver, 361 }; 362 363 return adf_gen2_pfvf_recv(accel_dev, ¶ms); 364 } 365 366 static struct pfvf_message adf_gen2_vf2pf_recv(struct adf_accel_dev *accel_dev, 367 u32 pfvf_offset, u8 compat_ver) 368 { 369 struct pfvf_gen2_params params = { 370 .pfvf_offset = pfvf_offset, 371 .local_offset = ADF_GEN2_CSR_VF2PF_OFFSET, 372 .remote_offset = ADF_GEN2_CSR_PF2VF_OFFSET, 373 .is_notification_message = is_vf2pf_notification, 374 .compat_ver = compat_ver, 375 }; 376 377 return adf_gen2_pfvf_recv(accel_dev, ¶ms); 378 } 379 380 void adf_gen2_init_pf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) 381 { 382 pfvf_ops->enable_comms = adf_enable_pf2vf_comms; 383 pfvf_ops->get_pf2vf_offset = adf_gen2_pf_get_pfvf_offset; 384 pfvf_ops->get_vf2pf_offset = adf_gen2_pf_get_pfvf_offset; 385 pfvf_ops->enable_vf2pf_interrupts = adf_gen2_enable_vf2pf_interrupts; 386 pfvf_ops->disable_all_vf2pf_interrupts = adf_gen2_disable_all_vf2pf_interrupts; 387 pfvf_ops->disable_pending_vf2pf_interrupts = adf_gen2_disable_pending_vf2pf_interrupts; 388 pfvf_ops->send_msg = adf_gen2_pf2vf_send; 389 pfvf_ops->recv_msg = adf_gen2_vf2pf_recv; 390 } 391 EXPORT_SYMBOL_GPL(adf_gen2_init_pf_pfvf_ops); 392 393 void adf_gen2_init_vf_pfvf_ops(struct adf_pfvf_ops *pfvf_ops) 394 { 395 pfvf_ops->enable_comms = adf_enable_vf2pf_comms; 396 pfvf_ops->get_pf2vf_offset = adf_gen2_vf_get_pfvf_offset; 397 pfvf_ops->get_vf2pf_offset = adf_gen2_vf_get_pfvf_offset; 398 pfvf_ops->send_msg = adf_gen2_vf2pf_send; 399 pfvf_ops->recv_msg = adf_gen2_pf2vf_recv; 400 } 401 EXPORT_SYMBOL_GPL(adf_gen2_init_vf_pfvf_ops); 402