1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell RVU Admin Function driver 3 * 4 * Copyright (C) 2018 Marvell. 5 * 6 */ 7 8 #include <linux/module.h> 9 #include <linux/interrupt.h> 10 #include <linux/pci.h> 11 12 #include "rvu_reg.h" 13 #include "cn20k/reg.h" 14 #include "cn20k/api.h" 15 #include "mbox.h" 16 #include "rvu_trace.h" 17 #include "rvu.h" 18 19 static const u16 msgs_offset = ALIGN(sizeof(struct mbox_hdr), MBOX_MSG_ALIGN); 20 21 void __otx2_mbox_reset(struct otx2_mbox *mbox, int devid) 22 { 23 struct otx2_mbox_dev *mdev = &mbox->dev[devid]; 24 struct mbox_hdr *tx_hdr, *rx_hdr; 25 void *hw_mbase = mdev->hwbase; 26 27 tx_hdr = hw_mbase + mbox->tx_start; 28 rx_hdr = hw_mbase + mbox->rx_start; 29 30 mdev->msg_size = 0; 31 mdev->rsp_size = 0; 32 tx_hdr->num_msgs = 0; 33 tx_hdr->msg_size = 0; 34 tx_hdr->sig = 0; 35 rx_hdr->num_msgs = 0; 36 rx_hdr->msg_size = 0; 37 rx_hdr->sig = 0; 38 } 39 EXPORT_SYMBOL(__otx2_mbox_reset); 40 41 void otx2_mbox_reset(struct otx2_mbox *mbox, int devid) 42 { 43 struct otx2_mbox_dev *mdev = &mbox->dev[devid]; 44 45 spin_lock(&mdev->mbox_lock); 46 __otx2_mbox_reset(mbox, devid); 47 spin_unlock(&mdev->mbox_lock); 48 } 49 EXPORT_SYMBOL(otx2_mbox_reset); 50 51 void otx2_mbox_destroy(struct otx2_mbox *mbox) 52 { 53 mbox->reg_base = NULL; 54 mbox->hwbase = NULL; 55 56 kfree(mbox->dev); 57 mbox->dev = NULL; 58 } 59 EXPORT_SYMBOL(otx2_mbox_destroy); 60 61 int cn20k_mbox_setup(struct otx2_mbox *mbox, struct pci_dev *pdev, 62 void *reg_base, int direction, int ndevs) 63 { 64 switch (direction) { 65 case MBOX_DIR_AFPF: 66 case MBOX_DIR_PFVF: 67 mbox->tx_start = MBOX_DOWN_TX_START; 68 mbox->rx_start = MBOX_DOWN_RX_START; 69 mbox->tx_size = MBOX_DOWN_TX_SIZE; 70 mbox->rx_size = MBOX_DOWN_RX_SIZE; 71 break; 72 case MBOX_DIR_PFAF: 73 case MBOX_DIR_VFPF: 74 mbox->tx_start = MBOX_DOWN_RX_START; 75 mbox->rx_start = MBOX_DOWN_TX_START; 76 mbox->tx_size = MBOX_DOWN_RX_SIZE; 77 mbox->rx_size = MBOX_DOWN_TX_SIZE; 78 break; 79 case MBOX_DIR_AFPF_UP: 80 case MBOX_DIR_PFVF_UP: 81 mbox->tx_start = MBOX_UP_TX_START; 82 mbox->rx_start = MBOX_UP_RX_START; 83 mbox->tx_size = MBOX_UP_TX_SIZE; 84 mbox->rx_size = MBOX_UP_RX_SIZE; 85 break; 86 case MBOX_DIR_PFAF_UP: 87 case MBOX_DIR_VFPF_UP: 88 mbox->tx_start = MBOX_UP_RX_START; 89 mbox->rx_start = MBOX_UP_TX_START; 90 mbox->tx_size = MBOX_UP_RX_SIZE; 91 mbox->rx_size = MBOX_UP_TX_SIZE; 92 break; 93 default: 94 return -ENODEV; 95 } 96 97 switch (direction) { 98 case MBOX_DIR_AFPF: 99 mbox->trigger = RVU_MBOX_AF_AFPFX_TRIGX(1); 100 mbox->tr_shift = 4; 101 break; 102 case MBOX_DIR_AFPF_UP: 103 mbox->trigger = RVU_MBOX_AF_AFPFX_TRIGX(0); 104 mbox->tr_shift = 4; 105 break; 106 case MBOX_DIR_PFAF: 107 mbox->trigger = RVU_MBOX_PF_PFAF_TRIGX(0); 108 mbox->tr_shift = 0; 109 break; 110 case MBOX_DIR_PFAF_UP: 111 mbox->trigger = RVU_MBOX_PF_PFAF_TRIGX(1); 112 mbox->tr_shift = 0; 113 break; 114 case MBOX_DIR_PFVF: 115 mbox->trigger = RVU_MBOX_PF_VFX_PFVF_TRIGX(1); 116 mbox->tr_shift = 4; 117 break; 118 case MBOX_DIR_PFVF_UP: 119 mbox->trigger = RVU_MBOX_PF_VFX_PFVF_TRIGX(0); 120 mbox->tr_shift = 4; 121 break; 122 case MBOX_DIR_VFPF: 123 mbox->trigger = RVU_MBOX_VF_VFPF_TRIGX(0); 124 mbox->tr_shift = 0; 125 break; 126 case MBOX_DIR_VFPF_UP: 127 mbox->trigger = RVU_MBOX_VF_VFPF_TRIGX(1); 128 mbox->tr_shift = 0; 129 break; 130 default: 131 return -ENODEV; 132 } 133 mbox->reg_base = reg_base; 134 mbox->pdev = pdev; 135 136 mbox->dev = kcalloc(ndevs, sizeof(struct otx2_mbox_dev), GFP_KERNEL); 137 if (!mbox->dev) { 138 otx2_mbox_destroy(mbox); 139 return -ENOMEM; 140 } 141 mbox->ndevs = ndevs; 142 143 return 0; 144 } 145 146 static int otx2_mbox_setup(struct otx2_mbox *mbox, struct pci_dev *pdev, 147 void *reg_base, int direction, int ndevs) 148 { 149 if (is_cn20k(pdev)) 150 return cn20k_mbox_setup(mbox, pdev, reg_base, 151 direction, ndevs); 152 153 switch (direction) { 154 case MBOX_DIR_AFPF: 155 case MBOX_DIR_PFVF: 156 mbox->tx_start = MBOX_DOWN_TX_START; 157 mbox->rx_start = MBOX_DOWN_RX_START; 158 mbox->tx_size = MBOX_DOWN_TX_SIZE; 159 mbox->rx_size = MBOX_DOWN_RX_SIZE; 160 break; 161 case MBOX_DIR_PFAF: 162 case MBOX_DIR_VFPF: 163 mbox->tx_start = MBOX_DOWN_RX_START; 164 mbox->rx_start = MBOX_DOWN_TX_START; 165 mbox->tx_size = MBOX_DOWN_RX_SIZE; 166 mbox->rx_size = MBOX_DOWN_TX_SIZE; 167 break; 168 case MBOX_DIR_AFPF_UP: 169 case MBOX_DIR_PFVF_UP: 170 mbox->tx_start = MBOX_UP_TX_START; 171 mbox->rx_start = MBOX_UP_RX_START; 172 mbox->tx_size = MBOX_UP_TX_SIZE; 173 mbox->rx_size = MBOX_UP_RX_SIZE; 174 break; 175 case MBOX_DIR_PFAF_UP: 176 case MBOX_DIR_VFPF_UP: 177 mbox->tx_start = MBOX_UP_RX_START; 178 mbox->rx_start = MBOX_UP_TX_START; 179 mbox->tx_size = MBOX_UP_RX_SIZE; 180 mbox->rx_size = MBOX_UP_TX_SIZE; 181 break; 182 default: 183 return -ENODEV; 184 } 185 186 switch (direction) { 187 case MBOX_DIR_AFPF: 188 case MBOX_DIR_AFPF_UP: 189 mbox->trigger = RVU_AF_AFPF_MBOX0; 190 mbox->tr_shift = 4; 191 break; 192 case MBOX_DIR_PFAF: 193 case MBOX_DIR_PFAF_UP: 194 mbox->trigger = RVU_PF_PFAF_MBOX1; 195 mbox->tr_shift = 0; 196 break; 197 case MBOX_DIR_PFVF: 198 case MBOX_DIR_PFVF_UP: 199 mbox->trigger = RVU_PF_VFX_PFVF_MBOX0; 200 mbox->tr_shift = 12; 201 break; 202 case MBOX_DIR_VFPF: 203 case MBOX_DIR_VFPF_UP: 204 mbox->trigger = RVU_VF_VFPF_MBOX1; 205 mbox->tr_shift = 0; 206 break; 207 default: 208 return -ENODEV; 209 } 210 211 mbox->reg_base = reg_base; 212 mbox->pdev = pdev; 213 214 mbox->dev = kcalloc(ndevs, sizeof(struct otx2_mbox_dev), GFP_KERNEL); 215 if (!mbox->dev) { 216 otx2_mbox_destroy(mbox); 217 return -ENOMEM; 218 } 219 mbox->ndevs = ndevs; 220 221 return 0; 222 } 223 224 int otx2_mbox_init(struct otx2_mbox *mbox, void *hwbase, struct pci_dev *pdev, 225 void *reg_base, int direction, int ndevs) 226 { 227 struct otx2_mbox_dev *mdev; 228 int devid, err; 229 230 err = otx2_mbox_setup(mbox, pdev, reg_base, direction, ndevs); 231 if (err) 232 return err; 233 234 mbox->hwbase = hwbase; 235 236 for (devid = 0; devid < ndevs; devid++) { 237 mdev = &mbox->dev[devid]; 238 mdev->mbase = mbox->hwbase + (devid * MBOX_SIZE); 239 mdev->hwbase = mdev->mbase; 240 spin_lock_init(&mdev->mbox_lock); 241 /* Init header to reset value */ 242 otx2_mbox_reset(mbox, devid); 243 } 244 245 return 0; 246 } 247 EXPORT_SYMBOL(otx2_mbox_init); 248 249 /* Initialize mailbox with the set of mailbox region addresses 250 * in the array hwbase. 251 */ 252 int otx2_mbox_regions_init(struct otx2_mbox *mbox, void **hwbase, 253 struct pci_dev *pdev, void *reg_base, 254 int direction, int ndevs, unsigned long *pf_bmap) 255 { 256 struct otx2_mbox_dev *mdev; 257 int devid, err; 258 259 err = otx2_mbox_setup(mbox, pdev, reg_base, direction, ndevs); 260 if (err) 261 return err; 262 263 mbox->hwbase = hwbase[0]; 264 265 for (devid = 0; devid < ndevs; devid++) { 266 if (!test_bit(devid, pf_bmap)) 267 continue; 268 269 mdev = &mbox->dev[devid]; 270 mdev->mbase = hwbase[devid]; 271 mdev->hwbase = hwbase[devid]; 272 spin_lock_init(&mdev->mbox_lock); 273 /* Init header to reset value */ 274 otx2_mbox_reset(mbox, devid); 275 } 276 277 return 0; 278 } 279 EXPORT_SYMBOL(otx2_mbox_regions_init); 280 281 int otx2_mbox_wait_for_rsp(struct otx2_mbox *mbox, int devid) 282 { 283 unsigned long timeout = jiffies + msecs_to_jiffies(MBOX_RSP_TIMEOUT); 284 struct otx2_mbox_dev *mdev = &mbox->dev[devid]; 285 286 while (!time_after(jiffies, timeout)) { 287 if (mdev->num_msgs == mdev->msgs_acked) 288 return 0; 289 usleep_range(800, 1000); 290 } 291 trace_otx2_msg_wait_rsp(mbox->pdev); 292 return -EIO; 293 } 294 EXPORT_SYMBOL(otx2_mbox_wait_for_rsp); 295 296 int otx2_mbox_busy_poll_for_rsp(struct otx2_mbox *mbox, int devid) 297 { 298 struct otx2_mbox_dev *mdev = &mbox->dev[devid]; 299 unsigned long timeout = jiffies + 1 * HZ; 300 301 while (!time_after(jiffies, timeout)) { 302 if (mdev->num_msgs == mdev->msgs_acked) 303 return 0; 304 cpu_relax(); 305 } 306 return -EIO; 307 } 308 EXPORT_SYMBOL(otx2_mbox_busy_poll_for_rsp); 309 310 static void otx2_mbox_msg_send_data(struct otx2_mbox *mbox, int devid, u64 data) 311 { 312 struct otx2_mbox_dev *mdev = &mbox->dev[devid]; 313 struct mbox_hdr *tx_hdr, *rx_hdr; 314 void *hw_mbase = mdev->hwbase; 315 struct mbox_msghdr *msg; 316 u64 intr_val; 317 318 tx_hdr = hw_mbase + mbox->tx_start; 319 rx_hdr = hw_mbase + mbox->rx_start; 320 321 /* If bounce buffer is implemented copy mbox messages from 322 * bounce buffer to hw mbox memory. 323 */ 324 if (mdev->mbase != hw_mbase) 325 memcpy(hw_mbase + mbox->tx_start + msgs_offset, 326 mdev->mbase + mbox->tx_start + msgs_offset, 327 mdev->msg_size); 328 329 spin_lock(&mdev->mbox_lock); 330 331 if (!tx_hdr->sig) { 332 tx_hdr->msg_size = mdev->msg_size; 333 tx_hdr->num_msgs = mdev->num_msgs; 334 } 335 336 /* Reset header for next messages */ 337 mdev->msg_size = 0; 338 mdev->rsp_size = 0; 339 mdev->msgs_acked = 0; 340 341 /* Sync mbox data into memory */ 342 smp_wmb(); 343 344 /* num_msgs != 0 signals to the peer that the buffer has a number of 345 * messages. So this should be written after writing all the messages 346 * to the shared memory. 347 */ 348 rx_hdr->num_msgs = 0; 349 350 msg = (struct mbox_msghdr *)(hw_mbase + mbox->tx_start + msgs_offset); 351 352 trace_otx2_msg_send(mbox->pdev, tx_hdr->num_msgs, tx_hdr->msg_size, 353 msg->id, msg->pcifunc); 354 355 spin_unlock(&mdev->mbox_lock); 356 357 /* Check if interrupt pending */ 358 intr_val = readq((void __iomem *)mbox->reg_base + 359 (mbox->trigger | (devid << mbox->tr_shift))); 360 361 intr_val |= data; 362 /* The interrupt should be fired after num_msgs is written 363 * to the shared memory 364 */ 365 writeq(intr_val, (void __iomem *)mbox->reg_base + 366 (mbox->trigger | (devid << mbox->tr_shift))); 367 } 368 369 void otx2_mbox_msg_send(struct otx2_mbox *mbox, int devid) 370 { 371 otx2_mbox_msg_send_data(mbox, devid, MBOX_DOWN_MSG); 372 } 373 EXPORT_SYMBOL(otx2_mbox_msg_send); 374 375 void otx2_mbox_msg_send_up(struct otx2_mbox *mbox, int devid) 376 { 377 otx2_mbox_msg_send_data(mbox, devid, MBOX_UP_MSG); 378 } 379 EXPORT_SYMBOL(otx2_mbox_msg_send_up); 380 381 bool otx2_mbox_wait_for_zero(struct otx2_mbox *mbox, int devid) 382 { 383 u64 data; 384 385 data = readq((void __iomem *)mbox->reg_base + 386 (mbox->trigger | (devid << mbox->tr_shift))); 387 388 /* If data is non-zero wait for ~1ms and return to caller 389 * whether data has changed to zero or not after the wait. 390 */ 391 if (!data) 392 return true; 393 394 usleep_range(950, 1000); 395 396 data = readq((void __iomem *)mbox->reg_base + 397 (mbox->trigger | (devid << mbox->tr_shift))); 398 399 return data == 0; 400 } 401 EXPORT_SYMBOL(otx2_mbox_wait_for_zero); 402 403 struct mbox_msghdr *otx2_mbox_alloc_msg_rsp(struct otx2_mbox *mbox, int devid, 404 int size, int size_rsp) 405 { 406 struct otx2_mbox_dev *mdev = &mbox->dev[devid]; 407 struct mbox_msghdr *msghdr = NULL; 408 struct mbox_hdr *mboxhdr = NULL; 409 410 spin_lock(&mdev->mbox_lock); 411 size = ALIGN(size, MBOX_MSG_ALIGN); 412 size_rsp = ALIGN(size_rsp, MBOX_MSG_ALIGN); 413 /* Check if there is space in mailbox */ 414 if ((mdev->msg_size + size) > mbox->tx_size - msgs_offset) 415 goto exit; 416 if ((mdev->rsp_size + size_rsp) > mbox->rx_size - msgs_offset) 417 goto exit; 418 419 if (mdev->msg_size == 0) 420 mdev->num_msgs = 0; 421 mdev->num_msgs++; 422 423 msghdr = mdev->mbase + mbox->tx_start + msgs_offset + mdev->msg_size; 424 425 /* Clear the whole msg region */ 426 memset(msghdr, 0, size); 427 /* Init message header with reset values */ 428 msghdr->ver = OTX2_MBOX_VERSION; 429 mdev->msg_size += size; 430 mdev->rsp_size += size_rsp; 431 msghdr->next_msgoff = mdev->msg_size + msgs_offset; 432 433 mboxhdr = mdev->mbase + mbox->tx_start; 434 /* Clear the msg header region */ 435 memset(mboxhdr, 0, msgs_offset); 436 437 exit: 438 spin_unlock(&mdev->mbox_lock); 439 440 return msghdr; 441 } 442 EXPORT_SYMBOL(otx2_mbox_alloc_msg_rsp); 443 444 struct mbox_msghdr *otx2_mbox_get_rsp(struct otx2_mbox *mbox, int devid, 445 struct mbox_msghdr *msg) 446 { 447 unsigned long imsg = mbox->tx_start + msgs_offset; 448 unsigned long irsp = mbox->rx_start + msgs_offset; 449 struct otx2_mbox_dev *mdev = &mbox->dev[devid]; 450 u16 msgs; 451 452 spin_lock(&mdev->mbox_lock); 453 454 if (mdev->num_msgs != mdev->msgs_acked) 455 goto error; 456 457 for (msgs = 0; msgs < mdev->msgs_acked; msgs++) { 458 struct mbox_msghdr *pmsg = mdev->mbase + imsg; 459 struct mbox_msghdr *prsp = mdev->mbase + irsp; 460 461 if (msg == pmsg) { 462 if (pmsg->id != prsp->id) 463 goto error; 464 spin_unlock(&mdev->mbox_lock); 465 return prsp; 466 } 467 468 imsg = mbox->tx_start + pmsg->next_msgoff; 469 irsp = mbox->rx_start + prsp->next_msgoff; 470 } 471 472 error: 473 spin_unlock(&mdev->mbox_lock); 474 return ERR_PTR(-ENODEV); 475 } 476 EXPORT_SYMBOL(otx2_mbox_get_rsp); 477 478 int otx2_mbox_check_rsp_msgs(struct otx2_mbox *mbox, int devid) 479 { 480 unsigned long ireq = mbox->tx_start + msgs_offset; 481 unsigned long irsp = mbox->rx_start + msgs_offset; 482 struct otx2_mbox_dev *mdev = &mbox->dev[devid]; 483 int rc = -ENODEV; 484 u16 msgs; 485 486 spin_lock(&mdev->mbox_lock); 487 488 if (mdev->num_msgs != mdev->msgs_acked) 489 goto exit; 490 491 for (msgs = 0; msgs < mdev->msgs_acked; msgs++) { 492 struct mbox_msghdr *preq = mdev->mbase + ireq; 493 struct mbox_msghdr *prsp = mdev->mbase + irsp; 494 495 if (preq->id != prsp->id) { 496 trace_otx2_msg_check(mbox->pdev, preq->id, 497 prsp->id, prsp->rc); 498 goto exit; 499 } 500 if (prsp->rc) { 501 rc = prsp->rc; 502 trace_otx2_msg_check(mbox->pdev, preq->id, 503 prsp->id, prsp->rc); 504 goto exit; 505 } 506 507 ireq = mbox->tx_start + preq->next_msgoff; 508 irsp = mbox->rx_start + prsp->next_msgoff; 509 } 510 rc = 0; 511 exit: 512 spin_unlock(&mdev->mbox_lock); 513 return rc; 514 } 515 EXPORT_SYMBOL(otx2_mbox_check_rsp_msgs); 516 517 int 518 otx2_reply_invalid_msg(struct otx2_mbox *mbox, int devid, u16 pcifunc, u16 id) 519 { 520 struct msg_rsp *rsp; 521 522 rsp = (struct msg_rsp *) 523 otx2_mbox_alloc_msg(mbox, devid, sizeof(*rsp)); 524 if (!rsp) 525 return -ENOMEM; 526 rsp->hdr.id = id; 527 rsp->hdr.sig = OTX2_MBOX_RSP_SIG; 528 rsp->hdr.rc = MBOX_MSG_INVALID; 529 rsp->hdr.pcifunc = pcifunc; 530 return 0; 531 } 532 EXPORT_SYMBOL(otx2_reply_invalid_msg); 533 534 bool otx2_mbox_nonempty(struct otx2_mbox *mbox, int devid) 535 { 536 struct otx2_mbox_dev *mdev = &mbox->dev[devid]; 537 bool ret; 538 539 spin_lock(&mdev->mbox_lock); 540 ret = mdev->num_msgs != 0; 541 spin_unlock(&mdev->mbox_lock); 542 543 return ret; 544 } 545 EXPORT_SYMBOL(otx2_mbox_nonempty); 546 547 const char *otx2_mbox_id2name(u16 id) 548 { 549 switch (id) { 550 #define M(_name, _id, _1, _2, _3) case _id: return # _name; 551 MBOX_MESSAGES 552 #undef M 553 554 #define M(_name, _id, _1, _2, _3) case _id: return # _name; 555 MBOX_UP_CGX_MESSAGES 556 #undef M 557 558 #define M(_name, _id, _1, _2, _3) case _id: return # _name; 559 MBOX_UP_CPT_MESSAGES 560 #undef M 561 default: 562 return "INVALID ID"; 563 } 564 } 565 EXPORT_SYMBOL(otx2_mbox_id2name); 566 567 MODULE_AUTHOR("Marvell."); 568 MODULE_DESCRIPTION("Marvell RVU NIC Mbox helpers"); 569 MODULE_LICENSE("GPL v2"); 570