1 /* 2 * Microsemi Switchtec(tm) PCIe Management Driver 3 * Copyright (c) 2017, Microsemi Corporation 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 * 14 */ 15 16 #include <linux/switchtec.h> 17 #include <linux/module.h> 18 #include <linux/delay.h> 19 #include <linux/kthread.h> 20 #include <linux/interrupt.h> 21 22 MODULE_DESCRIPTION("Microsemi Switchtec(tm) NTB Driver"); 23 MODULE_VERSION("0.1"); 24 MODULE_LICENSE("GPL"); 25 MODULE_AUTHOR("Microsemi Corporation"); 26 27 static bool use_lut_mws; 28 module_param(use_lut_mws, bool, 0644); 29 MODULE_PARM_DESC(use_lut_mws, 30 "Enable the use of the LUT based memory windows"); 31 32 #ifndef ioread64 33 #ifdef readq 34 #define ioread64 readq 35 #else 36 #define ioread64 _ioread64 37 static inline u64 _ioread64(void __iomem *mmio) 38 { 39 u64 low, high; 40 41 low = ioread32(mmio); 42 high = ioread32(mmio + sizeof(u32)); 43 return low | (high << 32); 44 } 45 #endif 46 #endif 47 48 #ifndef iowrite64 49 #ifdef writeq 50 #define iowrite64 writeq 51 #else 52 #define iowrite64 _iowrite64 53 static inline void _iowrite64(u64 val, void __iomem *mmio) 54 { 55 iowrite32(val, mmio); 56 iowrite32(val >> 32, mmio + sizeof(u32)); 57 } 58 #endif 59 #endif 60 61 #define SWITCHTEC_NTB_MAGIC 0x45CC0001 62 #define MAX_MWS 128 63 64 struct shared_mw { 65 u32 magic; 66 u32 partition_id; 67 u64 mw_sizes[MAX_MWS]; 68 }; 69 70 #define MAX_DIRECT_MW ARRAY_SIZE(((struct ntb_ctrl_regs *)(0))->bar_entry) 71 #define LUT_SIZE SZ_64K 72 73 struct switchtec_ntb { 74 struct switchtec_dev *stdev; 75 76 int self_partition; 77 int peer_partition; 78 79 int doorbell_irq; 80 int message_irq; 81 82 struct ntb_info_regs __iomem *mmio_ntb; 83 struct ntb_ctrl_regs __iomem *mmio_ctrl; 84 struct ntb_dbmsg_regs __iomem *mmio_dbmsg; 85 struct ntb_ctrl_regs __iomem *mmio_self_ctrl; 86 struct ntb_ctrl_regs __iomem *mmio_peer_ctrl; 87 struct ntb_dbmsg_regs __iomem *mmio_self_dbmsg; 88 89 struct shared_mw *self_shared; 90 struct shared_mw __iomem *peer_shared; 91 dma_addr_t self_shared_dma; 92 93 u64 db_mask; 94 u64 db_valid_mask; 95 int db_shift; 96 int db_peer_shift; 97 98 int nr_direct_mw; 99 int nr_lut_mw; 100 int direct_mw_to_bar[MAX_DIRECT_MW]; 101 102 int peer_nr_direct_mw; 103 int peer_nr_lut_mw; 104 int peer_direct_mw_to_bar[MAX_DIRECT_MW]; 105 }; 106 107 static int switchtec_ntb_part_op(struct switchtec_ntb *sndev, 108 struct ntb_ctrl_regs __iomem *ctl, 109 u32 op, int wait_status) 110 { 111 static const char * const op_text[] = { 112 [NTB_CTRL_PART_OP_LOCK] = "lock", 113 [NTB_CTRL_PART_OP_CFG] = "configure", 114 [NTB_CTRL_PART_OP_RESET] = "reset", 115 }; 116 117 int i; 118 u32 ps; 119 int status; 120 121 switch (op) { 122 case NTB_CTRL_PART_OP_LOCK: 123 status = NTB_CTRL_PART_STATUS_LOCKING; 124 break; 125 case NTB_CTRL_PART_OP_CFG: 126 status = NTB_CTRL_PART_STATUS_CONFIGURING; 127 break; 128 case NTB_CTRL_PART_OP_RESET: 129 status = NTB_CTRL_PART_STATUS_RESETTING; 130 break; 131 default: 132 return -EINVAL; 133 } 134 135 iowrite32(op, &ctl->partition_op); 136 137 for (i = 0; i < 1000; i++) { 138 if (msleep_interruptible(50) != 0) { 139 iowrite32(NTB_CTRL_PART_OP_RESET, &ctl->partition_op); 140 return -EINTR; 141 } 142 143 ps = ioread32(&ctl->partition_status) & 0xFFFF; 144 145 if (ps != status) 146 break; 147 } 148 149 if (ps == wait_status) 150 return 0; 151 152 if (ps == status) { 153 dev_err(&sndev->stdev->dev, 154 "Timed out while peforming %s (%d). (%08x)", 155 op_text[op], op, 156 ioread32(&ctl->partition_status)); 157 158 return -ETIMEDOUT; 159 } 160 161 return -EIO; 162 } 163 164 static void switchtec_ntb_init_sndev(struct switchtec_ntb *sndev) 165 { 166 u64 part_map; 167 168 sndev->self_partition = sndev->stdev->partition; 169 170 sndev->mmio_ntb = sndev->stdev->mmio_ntb; 171 part_map = ioread64(&sndev->mmio_ntb->ep_map); 172 part_map &= ~(1 << sndev->self_partition); 173 sndev->peer_partition = ffs(part_map) - 1; 174 175 dev_dbg(&sndev->stdev->dev, "Partition ID %d of %d (%llx)", 176 sndev->self_partition, sndev->stdev->partition_count, 177 part_map); 178 179 sndev->mmio_ctrl = (void * __iomem)sndev->mmio_ntb + 180 SWITCHTEC_NTB_REG_CTRL_OFFSET; 181 sndev->mmio_dbmsg = (void * __iomem)sndev->mmio_ntb + 182 SWITCHTEC_NTB_REG_DBMSG_OFFSET; 183 184 sndev->mmio_self_ctrl = &sndev->mmio_ctrl[sndev->self_partition]; 185 sndev->mmio_peer_ctrl = &sndev->mmio_ctrl[sndev->peer_partition]; 186 sndev->mmio_self_dbmsg = &sndev->mmio_dbmsg[sndev->self_partition]; 187 } 188 189 static int map_bars(int *map, struct ntb_ctrl_regs __iomem *ctrl) 190 { 191 int i; 192 int cnt = 0; 193 194 for (i = 0; i < ARRAY_SIZE(ctrl->bar_entry); i++) { 195 u32 r = ioread32(&ctrl->bar_entry[i].ctl); 196 197 if (r & NTB_CTRL_BAR_VALID) 198 map[cnt++] = i; 199 } 200 201 return cnt; 202 } 203 204 static void switchtec_ntb_init_mw(struct switchtec_ntb *sndev) 205 { 206 sndev->nr_direct_mw = map_bars(sndev->direct_mw_to_bar, 207 sndev->mmio_self_ctrl); 208 209 sndev->nr_lut_mw = ioread16(&sndev->mmio_self_ctrl->lut_table_entries); 210 sndev->nr_lut_mw = rounddown_pow_of_two(sndev->nr_lut_mw); 211 212 dev_dbg(&sndev->stdev->dev, "MWs: %d direct, %d lut", 213 sndev->nr_direct_mw, sndev->nr_lut_mw); 214 215 sndev->peer_nr_direct_mw = map_bars(sndev->peer_direct_mw_to_bar, 216 sndev->mmio_peer_ctrl); 217 218 sndev->peer_nr_lut_mw = 219 ioread16(&sndev->mmio_peer_ctrl->lut_table_entries); 220 sndev->peer_nr_lut_mw = rounddown_pow_of_two(sndev->peer_nr_lut_mw); 221 222 dev_dbg(&sndev->stdev->dev, "Peer MWs: %d direct, %d lut", 223 sndev->peer_nr_direct_mw, sndev->peer_nr_lut_mw); 224 225 } 226 227 /* 228 * There are 64 doorbells in the switch hardware but this is 229 * shared among all partitions. So we must split them in half 230 * (32 for each partition). However, the message interrupts are 231 * also shared with the top 4 doorbells so we just limit this to 232 * 28 doorbells per partition 233 */ 234 static void switchtec_ntb_init_db(struct switchtec_ntb *sndev) 235 { 236 sndev->db_valid_mask = 0x0FFFFFFF; 237 238 if (sndev->self_partition < sndev->peer_partition) { 239 sndev->db_shift = 0; 240 sndev->db_peer_shift = 32; 241 } else { 242 sndev->db_shift = 32; 243 sndev->db_peer_shift = 0; 244 } 245 246 sndev->db_mask = 0x0FFFFFFFFFFFFFFFULL; 247 iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask); 248 iowrite64(sndev->db_valid_mask << sndev->db_peer_shift, 249 &sndev->mmio_self_dbmsg->odb_mask); 250 } 251 252 static void switchtec_ntb_init_msgs(struct switchtec_ntb *sndev) 253 { 254 int i; 255 u32 msg_map = 0; 256 257 for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) { 258 int m = i | sndev->peer_partition << 2; 259 260 msg_map |= m << i * 8; 261 } 262 263 iowrite32(msg_map, &sndev->mmio_self_dbmsg->msg_map); 264 265 for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) 266 iowrite64(NTB_DBMSG_IMSG_STATUS | NTB_DBMSG_IMSG_MASK, 267 &sndev->mmio_self_dbmsg->imsg[i]); 268 } 269 270 static int switchtec_ntb_init_req_id_table(struct switchtec_ntb *sndev) 271 { 272 int rc = 0; 273 u16 req_id; 274 u32 error; 275 276 req_id = ioread16(&sndev->mmio_ntb->requester_id); 277 278 if (ioread32(&sndev->mmio_self_ctrl->req_id_table_size) < 2) { 279 dev_err(&sndev->stdev->dev, 280 "Not enough requester IDs available."); 281 return -EFAULT; 282 } 283 284 rc = switchtec_ntb_part_op(sndev, sndev->mmio_self_ctrl, 285 NTB_CTRL_PART_OP_LOCK, 286 NTB_CTRL_PART_STATUS_LOCKED); 287 if (rc) 288 return rc; 289 290 iowrite32(NTB_PART_CTRL_ID_PROT_DIS, 291 &sndev->mmio_self_ctrl->partition_ctrl); 292 293 /* 294 * Root Complex Requester ID (which is 0:00.0) 295 */ 296 iowrite32(0 << 16 | NTB_CTRL_REQ_ID_EN, 297 &sndev->mmio_self_ctrl->req_id_table[0]); 298 299 /* 300 * Host Bridge Requester ID (as read from the mmap address) 301 */ 302 iowrite32(req_id << 16 | NTB_CTRL_REQ_ID_EN, 303 &sndev->mmio_self_ctrl->req_id_table[1]); 304 305 rc = switchtec_ntb_part_op(sndev, sndev->mmio_self_ctrl, 306 NTB_CTRL_PART_OP_CFG, 307 NTB_CTRL_PART_STATUS_NORMAL); 308 if (rc == -EIO) { 309 error = ioread32(&sndev->mmio_self_ctrl->req_id_error); 310 dev_err(&sndev->stdev->dev, 311 "Error setting up the requester ID table: %08x", 312 error); 313 } 314 315 return rc; 316 } 317 318 static void switchtec_ntb_init_shared(struct switchtec_ntb *sndev) 319 { 320 int i; 321 322 memset(sndev->self_shared, 0, LUT_SIZE); 323 sndev->self_shared->magic = SWITCHTEC_NTB_MAGIC; 324 sndev->self_shared->partition_id = sndev->stdev->partition; 325 326 for (i = 0; i < sndev->nr_direct_mw; i++) { 327 int bar = sndev->direct_mw_to_bar[i]; 328 resource_size_t sz = pci_resource_len(sndev->stdev->pdev, bar); 329 330 if (i == 0) 331 sz = min_t(resource_size_t, sz, 332 LUT_SIZE * sndev->nr_lut_mw); 333 334 sndev->self_shared->mw_sizes[i] = sz; 335 } 336 337 for (i = 0; i < sndev->nr_lut_mw; i++) { 338 int idx = sndev->nr_direct_mw + i; 339 340 sndev->self_shared->mw_sizes[idx] = LUT_SIZE; 341 } 342 } 343 344 static int switchtec_ntb_init_shared_mw(struct switchtec_ntb *sndev) 345 { 346 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl; 347 int bar = sndev->direct_mw_to_bar[0]; 348 u32 ctl_val; 349 int rc; 350 351 sndev->self_shared = dma_zalloc_coherent(&sndev->stdev->pdev->dev, 352 LUT_SIZE, 353 &sndev->self_shared_dma, 354 GFP_KERNEL); 355 if (!sndev->self_shared) { 356 dev_err(&sndev->stdev->dev, 357 "unable to allocate memory for shared mw"); 358 return -ENOMEM; 359 } 360 361 switchtec_ntb_init_shared(sndev); 362 363 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK, 364 NTB_CTRL_PART_STATUS_LOCKED); 365 if (rc) 366 goto unalloc_and_exit; 367 368 ctl_val = ioread32(&ctl->bar_entry[bar].ctl); 369 ctl_val &= 0xFF; 370 ctl_val |= NTB_CTRL_BAR_LUT_WIN_EN; 371 ctl_val |= ilog2(LUT_SIZE) << 8; 372 ctl_val |= (sndev->nr_lut_mw - 1) << 14; 373 iowrite32(ctl_val, &ctl->bar_entry[bar].ctl); 374 375 iowrite64((NTB_CTRL_LUT_EN | (sndev->self_partition << 1) | 376 sndev->self_shared_dma), 377 &ctl->lut_entry[0]); 378 379 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG, 380 NTB_CTRL_PART_STATUS_NORMAL); 381 if (rc) { 382 u32 bar_error, lut_error; 383 384 bar_error = ioread32(&ctl->bar_error); 385 lut_error = ioread32(&ctl->lut_error); 386 dev_err(&sndev->stdev->dev, 387 "Error setting up shared MW: %08x / %08x", 388 bar_error, lut_error); 389 goto unalloc_and_exit; 390 } 391 392 sndev->peer_shared = pci_iomap(sndev->stdev->pdev, bar, LUT_SIZE); 393 if (!sndev->peer_shared) { 394 rc = -ENOMEM; 395 goto unalloc_and_exit; 396 } 397 398 dev_dbg(&sndev->stdev->dev, "Shared MW Ready"); 399 return 0; 400 401 unalloc_and_exit: 402 dma_free_coherent(&sndev->stdev->pdev->dev, LUT_SIZE, 403 sndev->self_shared, sndev->self_shared_dma); 404 405 return rc; 406 } 407 408 static void switchtec_ntb_deinit_shared_mw(struct switchtec_ntb *sndev) 409 { 410 if (sndev->peer_shared) 411 pci_iounmap(sndev->stdev->pdev, sndev->peer_shared); 412 413 if (sndev->self_shared) 414 dma_free_coherent(&sndev->stdev->pdev->dev, LUT_SIZE, 415 sndev->self_shared, 416 sndev->self_shared_dma); 417 } 418 419 static irqreturn_t switchtec_ntb_doorbell_isr(int irq, void *dev) 420 { 421 struct switchtec_ntb *sndev = dev; 422 423 dev_dbg(&sndev->stdev->dev, "doorbell\n"); 424 425 return IRQ_HANDLED; 426 } 427 428 static irqreturn_t switchtec_ntb_message_isr(int irq, void *dev) 429 { 430 int i; 431 struct switchtec_ntb *sndev = dev; 432 433 for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) { 434 u64 msg = ioread64(&sndev->mmio_self_dbmsg->imsg[i]); 435 436 if (msg & NTB_DBMSG_IMSG_STATUS) { 437 dev_dbg(&sndev->stdev->dev, "message: %d %08x\n", i, 438 (u32)msg); 439 iowrite8(1, &sndev->mmio_self_dbmsg->imsg[i].status); 440 } 441 } 442 443 return IRQ_HANDLED; 444 } 445 446 static int switchtec_ntb_init_db_msg_irq(struct switchtec_ntb *sndev) 447 { 448 int i; 449 int rc; 450 int doorbell_irq = 0; 451 int message_irq = 0; 452 int event_irq; 453 int idb_vecs = sizeof(sndev->mmio_self_dbmsg->idb_vec_map); 454 455 event_irq = ioread32(&sndev->stdev->mmio_part_cfg->vep_vector_number); 456 457 while (doorbell_irq == event_irq) 458 doorbell_irq++; 459 while (message_irq == doorbell_irq || 460 message_irq == event_irq) 461 message_irq++; 462 463 dev_dbg(&sndev->stdev->dev, "irqs - event: %d, db: %d, msgs: %d", 464 event_irq, doorbell_irq, message_irq); 465 466 for (i = 0; i < idb_vecs - 4; i++) 467 iowrite8(doorbell_irq, 468 &sndev->mmio_self_dbmsg->idb_vec_map[i]); 469 470 for (; i < idb_vecs; i++) 471 iowrite8(message_irq, 472 &sndev->mmio_self_dbmsg->idb_vec_map[i]); 473 474 sndev->doorbell_irq = pci_irq_vector(sndev->stdev->pdev, doorbell_irq); 475 sndev->message_irq = pci_irq_vector(sndev->stdev->pdev, message_irq); 476 477 rc = request_irq(sndev->doorbell_irq, 478 switchtec_ntb_doorbell_isr, 0, 479 "switchtec_ntb_doorbell", sndev); 480 if (rc) 481 return rc; 482 483 rc = request_irq(sndev->message_irq, 484 switchtec_ntb_message_isr, 0, 485 "switchtec_ntb_message", sndev); 486 if (rc) { 487 free_irq(sndev->doorbell_irq, sndev); 488 return rc; 489 } 490 491 return 0; 492 } 493 494 static void switchtec_ntb_deinit_db_msg_irq(struct switchtec_ntb *sndev) 495 { 496 free_irq(sndev->doorbell_irq, sndev); 497 free_irq(sndev->message_irq, sndev); 498 } 499 500 static int switchtec_ntb_add(struct device *dev, 501 struct class_interface *class_intf) 502 { 503 struct switchtec_dev *stdev = to_stdev(dev); 504 struct switchtec_ntb *sndev; 505 int rc; 506 507 stdev->sndev = NULL; 508 509 if (stdev->pdev->class != MICROSEMI_NTB_CLASSCODE) 510 return -ENODEV; 511 512 if (stdev->partition_count != 2) 513 dev_warn(dev, "ntb driver only supports 2 partitions"); 514 515 sndev = kzalloc_node(sizeof(*sndev), GFP_KERNEL, dev_to_node(dev)); 516 if (!sndev) 517 return -ENOMEM; 518 519 sndev->stdev = stdev; 520 521 switchtec_ntb_init_sndev(sndev); 522 switchtec_ntb_init_mw(sndev); 523 switchtec_ntb_init_db(sndev); 524 switchtec_ntb_init_msgs(sndev); 525 526 rc = switchtec_ntb_init_req_id_table(sndev); 527 if (rc) 528 goto free_and_exit; 529 530 rc = switchtec_ntb_init_shared_mw(sndev); 531 if (rc) 532 goto free_and_exit; 533 534 rc = switchtec_ntb_init_db_msg_irq(sndev); 535 if (rc) 536 goto deinit_shared_and_exit; 537 538 stdev->sndev = sndev; 539 dev_info(dev, "NTB device registered"); 540 541 return 0; 542 543 deinit_shared_and_exit: 544 switchtec_ntb_deinit_shared_mw(sndev); 545 free_and_exit: 546 kfree(sndev); 547 dev_err(dev, "failed to register ntb device: %d", rc); 548 return rc; 549 } 550 551 void switchtec_ntb_remove(struct device *dev, 552 struct class_interface *class_intf) 553 { 554 struct switchtec_dev *stdev = to_stdev(dev); 555 struct switchtec_ntb *sndev = stdev->sndev; 556 557 if (!sndev) 558 return; 559 560 stdev->sndev = NULL; 561 switchtec_ntb_deinit_db_msg_irq(sndev); 562 switchtec_ntb_deinit_shared_mw(sndev); 563 kfree(sndev); 564 dev_info(dev, "ntb device unregistered"); 565 } 566 567 static struct class_interface switchtec_interface = { 568 .add_dev = switchtec_ntb_add, 569 .remove_dev = switchtec_ntb_remove, 570 }; 571 572 static int __init switchtec_ntb_init(void) 573 { 574 switchtec_interface.class = switchtec_class; 575 return class_interface_register(&switchtec_interface); 576 } 577 module_init(switchtec_ntb_init); 578 579 static void __exit switchtec_ntb_exit(void) 580 { 581 class_interface_unregister(&switchtec_interface); 582 } 583 module_exit(switchtec_ntb_exit); 584