1 /*- 2 * Copyright (c) 2017 Alexander Motin <mav@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 /* 28 * The Non-Transparent Bridge (NTB) is a device that allows you to connect 29 * two or more systems using a PCI-e links, providing remote memory access. 30 * 31 * This module contains a driver for NTBs in PLX/Avago/Broadcom PCIe bridges. 32 */ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include <sys/param.h> 38 #include <sys/kernel.h> 39 #include <sys/systm.h> 40 #include <sys/bus.h> 41 #include <sys/interrupt.h> 42 #include <sys/module.h> 43 #include <sys/rman.h> 44 #include <sys/sysctl.h> 45 #include <vm/vm.h> 46 #include <vm/pmap.h> 47 #include <machine/bus.h> 48 #include <machine/intr_machdep.h> 49 #include <machine/resource.h> 50 #include <dev/pci/pcireg.h> 51 #include <dev/pci/pcivar.h> 52 53 #include "../ntb.h" 54 55 #define PLX_MAX_BARS 4 /* There are at most 4 data BARs. */ 56 #define PLX_NUM_SPAD 8 /* There are 8 scratchpads. */ 57 #define PLX_NUM_SPAD_PATT 4 /* Use test pattern as 4 more. */ 58 #define PLX_NUM_DB 16 /* There are 16 doorbells. */ 59 60 struct ntb_plx_mw_info { 61 int mw_bar; 62 int mw_64bit; 63 int mw_rid; 64 struct resource *mw_res; 65 vm_paddr_t mw_pbase; 66 caddr_t mw_vbase; 67 vm_size_t mw_size; 68 vm_memattr_t mw_map_mode; 69 bus_addr_t mw_xlat_addr; 70 size_t mw_xlat_size; 71 }; 72 73 struct ntb_plx_softc { 74 /* ntb.c context. Do not move! Must go first! */ 75 void *ntb_store; 76 77 device_t dev; 78 struct resource *conf_res; 79 int conf_rid; 80 u_int ntx; /* NTx number within chip. */ 81 u_int link; /* Link v/s Virtual side. */ 82 u_int port; /* Port number within chip. */ 83 84 int int_rid; 85 struct resource *int_res; 86 void *int_tag; 87 88 struct ntb_plx_mw_info mw_info[PLX_MAX_BARS]; 89 int mw_count; /* Number of memory windows. */ 90 91 int spad_count1; /* Number of standard spads. */ 92 int spad_count2; /* Number of extra spads. */ 93 uint32_t spad_off1; /* Offset of our spads. */ 94 uint32_t spad_off2; /* Offset of our extra spads. */ 95 uint32_t spad_offp1; /* Offset of peer spads. */ 96 uint32_t spad_offp2; /* Offset of peer extra spads. */ 97 98 /* Parameters of window shared with peer config access in B2B mode. */ 99 int b2b_mw; /* Shared window number. */ 100 uint64_t b2b_off; /* Offset in shared window. */ 101 }; 102 103 #define PLX_NT0_BASE 0x3E000 104 #define PLX_NT1_BASE 0x3C000 105 #define PLX_NTX_BASE(sc) ((sc)->ntx ? PLX_NT1_BASE : PLX_NT0_BASE) 106 #define PLX_NTX_LINK_OFFSET 0x01000 107 108 /* Bases of NTx our/peer interface registers */ 109 #define PLX_NTX_OUR_BASE(sc) \ 110 (PLX_NTX_BASE(sc) + ((sc)->link ? PLX_NTX_LINK_OFFSET : 0)) 111 #define PLX_NTX_PEER_BASE(sc) \ 112 (PLX_NTX_BASE(sc) + ((sc)->link ? 0 : PLX_NTX_LINK_OFFSET)) 113 114 /* Read/write NTx our interface registers */ 115 #define NTX_READ(sc, reg) \ 116 bus_read_4((sc)->conf_res, PLX_NTX_OUR_BASE(sc) + (reg)) 117 #define NTX_WRITE(sc, reg, val) \ 118 bus_write_4((sc)->conf_res, PLX_NTX_OUR_BASE(sc) + (reg), (val)) 119 120 /* Read/write NTx peer interface registers */ 121 #define PNTX_READ(sc, reg) \ 122 bus_read_4((sc)->conf_res, PLX_NTX_PEER_BASE(sc) + (reg)) 123 #define PNTX_WRITE(sc, reg, val) \ 124 bus_write_4((sc)->conf_res, PLX_NTX_PEER_BASE(sc) + (reg), (val)) 125 126 /* Read/write B2B NTx registers */ 127 #define BNTX_READ(sc, reg) \ 128 bus_read_4((sc)->mw_info[(sc)->b2b_mw].mw_res, \ 129 PLX_NTX_BASE(sc) + (reg)) 130 #define BNTX_WRITE(sc, reg, val) \ 131 bus_write_4((sc)->mw_info[(sc)->b2b_mw].mw_res, \ 132 PLX_NTX_BASE(sc) + (reg), (val)) 133 134 #define PLX_PORT_BASE(p) ((p) << 12) 135 #define PLX_STATION_PORT_BASE(sc) PLX_PORT_BASE((sc)->port & ~7) 136 137 #define PLX_PORT_CONTROL(sc) (PLX_STATION_PORT_BASE(sc) + 0x208) 138 139 static int ntb_plx_init(device_t dev); 140 static int ntb_plx_detach(device_t dev); 141 static int ntb_plx_mw_set_trans_internal(device_t dev, unsigned mw_idx); 142 143 static int 144 ntb_plx_probe(device_t dev) 145 { 146 147 switch (pci_get_devid(dev)) { 148 case 0x87a010b5: 149 device_set_desc(dev, "PLX Non-Transparent Bridge NT0 Link"); 150 return (BUS_PROBE_DEFAULT); 151 case 0x87a110b5: 152 device_set_desc(dev, "PLX Non-Transparent Bridge NT1 Link"); 153 return (BUS_PROBE_DEFAULT); 154 case 0x87b010b5: 155 device_set_desc(dev, "PLX Non-Transparent Bridge NT0 Virtual"); 156 return (BUS_PROBE_DEFAULT); 157 case 0x87b110b5: 158 device_set_desc(dev, "PLX Non-Transparent Bridge NT1 Virtual"); 159 return (BUS_PROBE_DEFAULT); 160 } 161 return (ENXIO); 162 } 163 164 static int 165 ntb_plx_init(device_t dev) 166 { 167 struct ntb_plx_softc *sc = device_get_softc(dev); 168 struct ntb_plx_mw_info *mw; 169 uint64_t val64; 170 int i; 171 uint32_t val; 172 173 if (sc->b2b_mw >= 0) { 174 /* Set peer BAR0/1 size and address for B2B NTx access. */ 175 mw = &sc->mw_info[sc->b2b_mw]; 176 if (mw->mw_64bit) { 177 PNTX_WRITE(sc, 0xe4, 0x3); /* 64-bit */ 178 val64 = 0x2000000000000000 * mw->mw_bar | 0x4; 179 PNTX_WRITE(sc, PCIR_BAR(0), val64); 180 PNTX_WRITE(sc, PCIR_BAR(0) + 4, val64 >> 32); 181 } else { 182 PNTX_WRITE(sc, 0xe4, 0x2); /* 32-bit */ 183 val = 0x20000000 * mw->mw_bar; 184 PNTX_WRITE(sc, PCIR_BAR(0), val); 185 } 186 187 /* Set Virtual to Link address translation for B2B. */ 188 for (i = 0; i < sc->mw_count; i++) { 189 mw = &sc->mw_info[i]; 190 if (mw->mw_64bit) { 191 val64 = 0x2000000000000000 * mw->mw_bar; 192 NTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4, val64); 193 NTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4 + 4, val64 >> 32); 194 } else { 195 val = 0x20000000 * mw->mw_bar; 196 NTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4, val); 197 } 198 } 199 200 /* Enable Link Interface LUT entries 0/1 for peer 0/1. */ 201 PNTX_WRITE(sc, 0xdb4, 0x00090001); 202 } 203 204 /* 205 * Enable Virtual Interface LUT entry 0 for 0:0.0 and 206 * entry 1 for our Requester ID reported by chip. 207 */ 208 val = (NTX_READ(sc, 0xc90) << 16) | 0x00010001; 209 NTX_WRITE(sc, sc->link ? 0xdb4 : 0xd94, val); 210 211 /* Set Link to Virtual address translation. */ 212 for (i = 0; i < sc->mw_count; i++) { 213 mw = &sc->mw_info[i]; 214 if (mw->mw_xlat_size != 0) 215 ntb_plx_mw_set_trans_internal(dev, i); 216 } 217 218 pci_enable_busmaster(dev); 219 if (sc->b2b_mw >= 0) 220 PNTX_WRITE(sc, PCIR_COMMAND, PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 221 222 return (0); 223 } 224 225 static void 226 ntb_plx_isr(void *arg) 227 { 228 device_t dev = arg; 229 struct ntb_plx_softc *sc = device_get_softc(dev); 230 uint32_t val; 231 232 ntb_db_event((device_t)arg, 0); 233 234 if (sc->link) /* Link Interface has no Link Error registers. */ 235 return; 236 237 val = NTX_READ(sc, 0xfe0); 238 if (val == 0) 239 return; 240 NTX_WRITE(sc, 0xfe0, val); 241 if (val & 1) 242 device_printf(dev, "Correctable Error\n"); 243 if (val & 2) 244 device_printf(dev, "Uncorrectable Error\n"); 245 if (val & 4) { 246 /* DL_Down resets link side registers, have to reinit. */ 247 ntb_plx_init(dev); 248 ntb_link_event(dev); 249 } 250 if (val & 8) 251 device_printf(dev, "Uncorrectable Error Message Drop\n"); 252 } 253 254 static int 255 ntb_plx_setup_intr(device_t dev) 256 { 257 struct ntb_plx_softc *sc = device_get_softc(dev); 258 int error; 259 260 /* 261 * XXX: This hardware supports MSI, but I found it unusable. 262 * It generates new MSI only when doorbell register goes from 263 * zero, but does not generate it when another bit is set or on 264 * partial clear. It makes operation very racy and unreliable. 265 * The data book mentions some mask juggling magic to workaround 266 * that, but I failed to make it work. 267 */ 268 sc->int_rid = 0; 269 sc->int_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, 270 &sc->int_rid, RF_SHAREABLE|RF_ACTIVE); 271 if (sc->int_res == NULL) { 272 device_printf(dev, "bus_alloc_resource failed\n"); 273 return (ENOMEM); 274 } 275 error = bus_setup_intr(dev, sc->int_res, INTR_MPSAFE | INTR_TYPE_MISC, 276 NULL, ntb_plx_isr, dev, &sc->int_tag); 277 if (error != 0) { 278 device_printf(dev, "bus_setup_intr failed: %d\n", error); 279 return (error); 280 } 281 282 if (!sc->link) { /* Link Interface has no Link Error registers. */ 283 NTX_WRITE(sc, 0xfe0, 0xf); /* Clear link interrupts. */ 284 NTX_WRITE(sc, 0xfe4, 0x0); /* Unmask link interrupts. */ 285 } 286 return (0); 287 } 288 289 static void 290 ntb_plx_teardown_intr(device_t dev) 291 { 292 struct ntb_plx_softc *sc = device_get_softc(dev); 293 294 if (!sc->link) /* Link Interface has no Link Error registers. */ 295 NTX_WRITE(sc, 0xfe4, 0xf); /* Mask link interrupts. */ 296 297 if (sc->int_res) { 298 bus_teardown_intr(dev, sc->int_res, sc->int_tag); 299 bus_release_resource(dev, SYS_RES_IRQ, sc->int_rid, 300 sc->int_res); 301 } 302 } 303 304 static int 305 ntb_plx_attach(device_t dev) 306 { 307 struct ntb_plx_softc *sc = device_get_softc(dev); 308 struct ntb_plx_mw_info *mw; 309 int error = 0, i; 310 uint32_t val; 311 char buf[32]; 312 313 /* Identify what we are (what side of what NTx). */ 314 sc->dev = dev; 315 val = pci_read_config(dev, 0xc8c, 4); 316 sc->ntx = (val & 1) != 0; 317 sc->link = (val & 0x80000000) != 0; 318 319 /* Get access to whole 256KB of chip configuration space via BAR0/1. */ 320 sc->conf_rid = PCIR_BAR(0); 321 sc->conf_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 322 &sc->conf_rid, RF_ACTIVE); 323 if (sc->conf_res == NULL) { 324 device_printf(dev, "Can't allocate configuration BAR.\n"); 325 return (ENXIO); 326 } 327 328 /* Identify chip port we are connected to. */ 329 val = bus_read_4(sc->conf_res, 0x360); 330 sc->port = (val >> ((sc->ntx == 0) ? 8 : 16)) & 0x1f; 331 332 /* Find configured memory windows at BAR2-5. */ 333 sc->mw_count = 0; 334 for (i = 2; i <= 5; i++) { 335 mw = &sc->mw_info[sc->mw_count]; 336 mw->mw_bar = i; 337 mw->mw_rid = PCIR_BAR(mw->mw_bar); 338 mw->mw_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 339 &mw->mw_rid, RF_ACTIVE); 340 if (mw->mw_res == NULL) 341 continue; 342 mw->mw_pbase = rman_get_start(mw->mw_res); 343 mw->mw_size = rman_get_size(mw->mw_res); 344 mw->mw_vbase = rman_get_virtual(mw->mw_res); 345 mw->mw_map_mode = VM_MEMATTR_UNCACHEABLE; 346 sc->mw_count++; 347 348 /* Skip over adjacent BAR for 64-bit BARs. */ 349 val = pci_read_config(dev, PCIR_BAR(mw->mw_bar), 4); 350 if ((val & PCIM_BAR_MEM_TYPE) == PCIM_BAR_MEM_64) { 351 mw->mw_64bit = 1; 352 i++; 353 } 354 } 355 356 /* Try to identify B2B mode. */ 357 i = 1; 358 snprintf(buf, sizeof(buf), "hint.%s.%d.b2b", device_get_name(dev), 359 device_get_unit(dev)); 360 TUNABLE_INT_FETCH(buf, &i); 361 if (sc->link) { 362 device_printf(dev, "NTB-to-Root Port mode (Link Interface)\n"); 363 sc->b2b_mw = -1; 364 } else if (i == 0) { 365 device_printf(dev, "NTB-to-Root Port mode (Virtual Interface)\n"); 366 sc->b2b_mw = -1; 367 } else { 368 device_printf(dev, "NTB-to-NTB (back-to-back) mode\n"); 369 370 /* We need at least one memory window for B2B peer access. */ 371 if (sc->mw_count == 0) { 372 device_printf(dev, "No memory window BARs enabled.\n"); 373 error = ENXIO; 374 goto out; 375 } 376 sc->b2b_mw = sc->mw_count - 1; 377 378 /* Use half of the window for B2B, but no less then 1MB. */ 379 mw = &sc->mw_info[sc->b2b_mw]; 380 if (mw->mw_size >= 2 * 1024 * 1024) 381 sc->b2b_off = mw->mw_size / 2; 382 else 383 sc->b2b_off = 0; 384 } 385 386 /* 387 * Use Physical Layer User Test Pattern as additional scratchpad. 388 * Make sure they are present and enabled by writing to them. 389 * XXX: Its a hack, but standard 8 registers are not enough. 390 */ 391 sc->spad_offp1 = sc->spad_off1 = PLX_NTX_OUR_BASE(sc) + 0xc6c; 392 sc->spad_offp2 = sc->spad_off2 = PLX_PORT_BASE(sc->ntx * 8) + 0x20c; 393 if (sc->b2b_mw >= 0) { 394 /* In NTB-to-NTB mode each side has own scratchpads. */ 395 sc->spad_count1 = PLX_NUM_SPAD; 396 bus_write_4(sc->conf_res, sc->spad_off2, 0x12345678); 397 if (bus_read_4(sc->conf_res, sc->spad_off2) == 0x12345678) 398 sc->spad_count2 = PLX_NUM_SPAD_PATT; 399 } else { 400 /* Otherwise we have share scratchpads with the peer. */ 401 if (sc->link) { 402 sc->spad_off1 += PLX_NUM_SPAD / 2 * 4; 403 sc->spad_off2 += PLX_NUM_SPAD_PATT / 2 * 4; 404 } else { 405 sc->spad_offp1 += PLX_NUM_SPAD / 2 * 4; 406 sc->spad_offp2 += PLX_NUM_SPAD_PATT / 2 * 4; 407 } 408 sc->spad_count1 = PLX_NUM_SPAD / 2; 409 bus_write_4(sc->conf_res, sc->spad_off2, 0x12345678); 410 if (bus_read_4(sc->conf_res, sc->spad_off2) == 0x12345678) 411 sc->spad_count2 = PLX_NUM_SPAD_PATT / 2; 412 } 413 414 /* Apply static part of NTB configuration. */ 415 ntb_plx_init(dev); 416 417 /* Allocate and setup interrupts. */ 418 error = ntb_plx_setup_intr(dev); 419 if (error) 420 goto out; 421 422 /* Attach children to this controller */ 423 error = ntb_register_device(dev); 424 425 out: 426 if (error != 0) 427 ntb_plx_detach(dev); 428 return (error); 429 } 430 431 static int 432 ntb_plx_detach(device_t dev) 433 { 434 struct ntb_plx_softc *sc = device_get_softc(dev); 435 struct ntb_plx_mw_info *mw; 436 int i; 437 438 /* Detach & delete all children */ 439 ntb_unregister_device(dev); 440 441 /* Disable and free interrupts. */ 442 ntb_plx_teardown_intr(dev); 443 444 /* Free memory resources. */ 445 for (i = 0; i < sc->mw_count; i++) { 446 mw = &sc->mw_info[i]; 447 bus_release_resource(dev, SYS_RES_MEMORY, mw->mw_rid, 448 mw->mw_res); 449 } 450 bus_release_resource(dev, SYS_RES_MEMORY, sc->conf_rid, sc->conf_res); 451 return (0); 452 } 453 454 455 static bool 456 ntb_plx_link_is_up(device_t dev, enum ntb_speed *speed, enum ntb_width *width) 457 { 458 uint16_t link; 459 460 link = pcie_read_config(dev, PCIER_LINK_STA, 2); 461 if (speed != NULL) 462 *speed = (link & PCIEM_LINK_STA_SPEED); 463 if (width != NULL) 464 *width = (link & PCIEM_LINK_STA_WIDTH) >> 4; 465 return ((link & PCIEM_LINK_STA_WIDTH) != 0); 466 } 467 468 static int 469 ntb_plx_link_enable(device_t dev, enum ntb_speed speed __unused, 470 enum ntb_width width __unused) 471 { 472 struct ntb_plx_softc *sc = device_get_softc(dev); 473 uint32_t reg, val; 474 475 /* The fact that we see the Link Interface means link is enabled. */ 476 if (sc->link) { 477 ntb_link_event(dev); 478 return (0); 479 } 480 481 reg = PLX_PORT_CONTROL(sc); 482 val = bus_read_4(sc->conf_res, reg); 483 if ((val & (1 << (sc->port & 7))) == 0) { 484 /* If already enabled, generate fake link event and exit. */ 485 ntb_link_event(dev); 486 return (0); 487 } 488 val &= ~(1 << (sc->port & 7)); 489 bus_write_4(sc->conf_res, reg, val); 490 return (0); 491 } 492 493 static int 494 ntb_plx_link_disable(device_t dev) 495 { 496 struct ntb_plx_softc *sc = device_get_softc(dev); 497 uint32_t reg, val; 498 499 /* Link disable for Link Interface would be suicidal. */ 500 if (sc->link) 501 return (0); 502 503 reg = PLX_PORT_CONTROL(sc); 504 val = bus_read_4(sc->conf_res, reg); 505 val |= (1 << (sc->port & 7)); 506 bus_write_4(sc->conf_res, reg, val); 507 return (0); 508 } 509 510 static bool 511 ntb_plx_link_enabled(device_t dev) 512 { 513 struct ntb_plx_softc *sc = device_get_softc(dev); 514 uint32_t reg, val; 515 516 /* The fact that we see the Link Interface means link is enabled. */ 517 if (sc->link) 518 return (TRUE); 519 520 reg = PLX_PORT_CONTROL(sc); 521 val = bus_read_4(sc->conf_res, reg); 522 return ((val & (1 << (sc->port & 7))) == 0); 523 } 524 525 static uint8_t 526 ntb_plx_mw_count(device_t dev) 527 { 528 struct ntb_plx_softc *sc = device_get_softc(dev); 529 530 if (sc->b2b_mw >= 0 && sc->b2b_off == 0) 531 return (sc->mw_count - 1); /* B2B consumed whole window. */ 532 return (sc->mw_count); 533 } 534 535 static int 536 ntb_plx_mw_get_range(device_t dev, unsigned mw_idx, vm_paddr_t *base, 537 caddr_t *vbase, size_t *size, size_t *align, size_t *align_size, 538 bus_addr_t *plimit) 539 { 540 struct ntb_plx_softc *sc = device_get_softc(dev); 541 struct ntb_plx_mw_info *mw; 542 size_t off; 543 544 if (mw_idx >= sc->mw_count) 545 return (EINVAL); 546 off = 0; 547 if (mw_idx == sc->b2b_mw) { 548 KASSERT(sc->b2b_off != 0, 549 ("user shouldn't get non-shared b2b mw")); 550 off = sc->b2b_off; 551 } 552 mw = &sc->mw_info[mw_idx]; 553 554 /* Local to remote memory window parameters. */ 555 if (base != NULL) 556 *base = mw->mw_pbase + off; 557 if (vbase != NULL) 558 *vbase = mw->mw_vbase + off; 559 if (size != NULL) 560 *size = mw->mw_size - off; 561 562 /* 563 * Remote to local memory window translation address alignment. 564 * XXX: In B2B mode we can change window size (and so alignmet) 565 * live, but there is no way to report it, so report safe value. 566 */ 567 if (align != NULL) 568 *align = mw->mw_size - off; 569 570 /* 571 * Remote to local memory window size alignment. 572 * XXX: The chip has no limit registers. In B2B case size must be 573 * power of 2 (since we can reprogram BAR size), but there is no way 574 * to report it, so report 1MB -- minimal BAR size. In non-B2B case 575 * there is no control at all, so report the precofigured BAR size. 576 */ 577 if (align_size != NULL) { 578 if (sc->b2b_mw >= 0) 579 *align_size = 1024 * 1024; 580 else 581 *align_size = mw->mw_size - off; 582 } 583 584 /* Remote to local memory window translation address upper limit. */ 585 if (plimit != NULL) 586 *plimit = mw->mw_64bit ? BUS_SPACE_MAXADDR : 587 BUS_SPACE_MAXADDR_32BIT; 588 return (0); 589 } 590 591 static int 592 ntb_plx_mw_set_trans_internal(device_t dev, unsigned mw_idx) 593 { 594 struct ntb_plx_softc *sc = device_get_softc(dev); 595 struct ntb_plx_mw_info *mw; 596 uint64_t addr, off, size, val64; 597 uint32_t val; 598 599 mw = &sc->mw_info[mw_idx]; 600 addr = mw->mw_xlat_addr; 601 size = mw->mw_xlat_size; 602 off = 0; 603 if (mw_idx == sc->b2b_mw) { 604 off = sc->b2b_off; 605 KASSERT(off != 0, ("user shouldn't get non-shared b2b mw")); 606 607 /* 608 * While generally we can set any BAR size on link side, 609 * for B2B shared window we can't go above preconfigured 610 * size due to BAR address alignment requirements. 611 */ 612 if (size > mw->mw_size - off) 613 return (EINVAL); 614 } 615 616 if (size > 0) { 617 /* Round BAR size to next power of 2 or at least 1MB. */ 618 if (!powerof2(size)) 619 size = 1LL << flsll(size); 620 if (size < 1024 * 1024) 621 size = 1024 * 1024; 622 623 /* Hardware requires addr aligned to BAR size. */ 624 if ((addr & (size - 1)) != 0) 625 return (EINVAL); 626 } 627 628 if (mw->mw_64bit) { 629 if (sc->b2b_mw >= 0) { 630 /* Set Link Interface BAR size and enable/disable it. */ 631 val64 = 0; 632 if (size > 0) 633 val64 = (~(size - 1) & ~0xfffff); 634 val64 |= 0xc; 635 PNTX_WRITE(sc, 0xe8 + (mw->mw_bar - 2) * 4, val64); 636 PNTX_WRITE(sc, 0xe8 + (mw->mw_bar - 2) * 4 + 4, val64 >> 32); 637 638 /* Set Link Interface BAR address. */ 639 val64 = 0x2000000000000000 * mw->mw_bar + off; 640 PNTX_WRITE(sc, PCIR_BAR(mw->mw_bar), val64); 641 PNTX_WRITE(sc, PCIR_BAR(mw->mw_bar) + 4, val64 >> 32); 642 } 643 644 /* Set Virtual Interface BARs address translation */ 645 PNTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4, addr); 646 PNTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4 + 4, addr >> 32); 647 } else { 648 /* Make sure we fit into 32-bit address space. */ 649 if ((addr & UINT32_MAX) != addr) 650 return (ERANGE); 651 if (((addr + size) & UINT32_MAX) != (addr + size)) 652 return (ERANGE); 653 654 if (sc->b2b_mw >= 0) { 655 /* Set Link Interface BAR size and enable/disable it. */ 656 val = 0; 657 if (size > 0) 658 val = (~(size - 1) & ~0xfffff); 659 PNTX_WRITE(sc, 0xe8 + (mw->mw_bar - 2) * 4, val); 660 661 /* Set Link Interface BAR address. */ 662 val64 = 0x20000000 * mw->mw_bar + off; 663 PNTX_WRITE(sc, PCIR_BAR(mw->mw_bar), val64); 664 } 665 666 /* Set Virtual Interface BARs address translation */ 667 PNTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4, addr); 668 } 669 return (0); 670 } 671 672 static int 673 ntb_plx_mw_set_trans(device_t dev, unsigned mw_idx, bus_addr_t addr, size_t size) 674 { 675 struct ntb_plx_softc *sc = device_get_softc(dev); 676 struct ntb_plx_mw_info *mw; 677 678 if (mw_idx >= sc->mw_count) 679 return (EINVAL); 680 mw = &sc->mw_info[mw_idx]; 681 mw->mw_xlat_addr = addr; 682 mw->mw_xlat_size = size; 683 return (ntb_plx_mw_set_trans_internal(dev, mw_idx)); 684 } 685 686 static int 687 ntb_plx_mw_clear_trans(device_t dev, unsigned mw_idx) 688 { 689 690 return (ntb_plx_mw_set_trans(dev, mw_idx, 0, 0)); 691 } 692 693 static int 694 ntb_plx_mw_get_wc(device_t dev, unsigned idx, vm_memattr_t *mode) 695 { 696 struct ntb_plx_softc *sc = device_get_softc(dev); 697 struct ntb_plx_mw_info *mw; 698 699 if (idx >= sc->mw_count) 700 return (EINVAL); 701 mw = &sc->mw_info[idx]; 702 *mode = mw->mw_map_mode; 703 return (0); 704 } 705 706 static int 707 ntb_plx_mw_set_wc(device_t dev, unsigned idx, vm_memattr_t mode) 708 { 709 struct ntb_plx_softc *sc = device_get_softc(dev); 710 struct ntb_plx_mw_info *mw; 711 uint64_t off; 712 int rc; 713 714 if (idx >= sc->mw_count) 715 return (EINVAL); 716 mw = &sc->mw_info[idx]; 717 if (mw->mw_map_mode == mode) 718 return (0); 719 720 off = 0; 721 if (idx == sc->b2b_mw) { 722 KASSERT(sc->b2b_off != 0, 723 ("user shouldn't get non-shared b2b mw")); 724 off = sc->b2b_off; 725 } 726 727 rc = pmap_change_attr((vm_offset_t)mw->mw_vbase + off, 728 mw->mw_size - off, mode); 729 if (rc == 0) 730 mw->mw_map_mode = mode; 731 return (rc); 732 } 733 734 static uint8_t 735 ntb_plx_spad_count(device_t dev) 736 { 737 struct ntb_plx_softc *sc = device_get_softc(dev); 738 739 return (sc->spad_count1 + sc->spad_count2); 740 } 741 742 static int 743 ntb_plx_spad_write(device_t dev, unsigned int idx, uint32_t val) 744 { 745 struct ntb_plx_softc *sc = device_get_softc(dev); 746 u_int off; 747 748 if (idx >= sc->spad_count1 + sc->spad_count2) 749 return (EINVAL); 750 751 if (idx < sc->spad_count1) 752 off = sc->spad_off1 + idx * 4; 753 else 754 off = sc->spad_off2 + (idx - sc->spad_count1) * 4; 755 bus_write_4(sc->conf_res, off, val); 756 return (0); 757 } 758 759 static void 760 ntb_plx_spad_clear(device_t dev) 761 { 762 struct ntb_plx_softc *sc = device_get_softc(dev); 763 int i; 764 765 for (i = 0; i < sc->spad_count1 + sc->spad_count2; i++) 766 ntb_plx_spad_write(dev, i, 0); 767 } 768 769 static int 770 ntb_plx_spad_read(device_t dev, unsigned int idx, uint32_t *val) 771 { 772 struct ntb_plx_softc *sc = device_get_softc(dev); 773 u_int off; 774 775 if (idx >= sc->spad_count1 + sc->spad_count2) 776 return (EINVAL); 777 778 if (idx < sc->spad_count1) 779 off = sc->spad_off1 + idx * 4; 780 else 781 off = sc->spad_off2 + (idx - sc->spad_count1) * 4; 782 *val = bus_read_4(sc->conf_res, off); 783 return (0); 784 } 785 786 static int 787 ntb_plx_peer_spad_write(device_t dev, unsigned int idx, uint32_t val) 788 { 789 struct ntb_plx_softc *sc = device_get_softc(dev); 790 u_int off; 791 792 if (idx >= sc->spad_count1 + sc->spad_count2) 793 return (EINVAL); 794 795 if (idx < sc->spad_count1) 796 off = sc->spad_offp1 + idx * 4; 797 else 798 off = sc->spad_offp2 + (idx - sc->spad_count1) * 4; 799 if (sc->b2b_mw >= 0) 800 bus_write_4(sc->mw_info[sc->b2b_mw].mw_res, off, val); 801 else 802 bus_write_4(sc->conf_res, off, val); 803 return (0); 804 } 805 806 static int 807 ntb_plx_peer_spad_read(device_t dev, unsigned int idx, uint32_t *val) 808 { 809 struct ntb_plx_softc *sc = device_get_softc(dev); 810 u_int off; 811 812 if (idx >= sc->spad_count1 + sc->spad_count2) 813 return (EINVAL); 814 815 if (idx < sc->spad_count1) 816 off = sc->spad_offp1 + idx * 4; 817 else 818 off = sc->spad_offp2 + (idx - sc->spad_count1) * 4; 819 if (sc->b2b_mw >= 0) 820 *val = bus_read_4(sc->mw_info[sc->b2b_mw].mw_res, off); 821 else 822 *val = bus_read_4(sc->conf_res, off); 823 return (0); 824 } 825 826 static uint64_t 827 ntb_plx_db_valid_mask(device_t dev) 828 { 829 830 return ((1LL << PLX_NUM_DB) - 1); 831 } 832 833 static int 834 ntb_plx_db_vector_count(device_t dev) 835 { 836 837 return (1); 838 } 839 840 static uint64_t 841 ntb_plx_db_vector_mask(device_t dev, uint32_t vector) 842 { 843 844 if (vector > 0) 845 return (0); 846 return ((1LL << PLX_NUM_DB) - 1); 847 } 848 849 static void 850 ntb_plx_db_clear(device_t dev, uint64_t bits) 851 { 852 struct ntb_plx_softc *sc = device_get_softc(dev); 853 854 NTX_WRITE(sc, sc->link ? 0xc60 : 0xc50, bits); 855 } 856 857 static void 858 ntb_plx_db_clear_mask(device_t dev, uint64_t bits) 859 { 860 struct ntb_plx_softc *sc = device_get_softc(dev); 861 862 NTX_WRITE(sc, sc->link ? 0xc68 : 0xc58, bits); 863 } 864 865 static uint64_t 866 ntb_plx_db_read(device_t dev) 867 { 868 struct ntb_plx_softc *sc = device_get_softc(dev); 869 870 return (NTX_READ(sc, sc->link ? 0xc5c : 0xc4c)); 871 } 872 873 static void 874 ntb_plx_db_set_mask(device_t dev, uint64_t bits) 875 { 876 struct ntb_plx_softc *sc = device_get_softc(dev); 877 878 NTX_WRITE(sc, sc->link ? 0xc64 : 0xc54, bits); 879 } 880 881 static int 882 ntb_plx_peer_db_addr(device_t dev, bus_addr_t *db_addr, vm_size_t *db_size) 883 { 884 struct ntb_plx_softc *sc = device_get_softc(dev); 885 struct ntb_plx_mw_info *mw; 886 887 KASSERT((db_addr != NULL && db_size != NULL), ("must be non-NULL")); 888 889 if (sc->b2b_mw >= 0) { 890 mw = &sc->mw_info[sc->b2b_mw]; 891 *db_addr = (uint64_t)mw->mw_pbase + PLX_NTX_BASE(sc) + 0xc4c; 892 } else { 893 *db_addr = rman_get_start(sc->conf_res) + PLX_NTX_BASE(sc); 894 *db_addr += sc->link ? 0xc4c : 0xc5c; 895 } 896 *db_size = 4; 897 return (0); 898 } 899 900 static void 901 ntb_plx_peer_db_set(device_t dev, uint64_t bit) 902 { 903 struct ntb_plx_softc *sc = device_get_softc(dev); 904 905 if (sc->b2b_mw >= 0) 906 BNTX_WRITE(sc, 0xc4c, bit); 907 else 908 NTX_WRITE(sc, sc->link ? 0xc4c : 0xc5c, bit); 909 } 910 911 static device_method_t ntb_plx_methods[] = { 912 /* Device interface */ 913 DEVMETHOD(device_probe, ntb_plx_probe), 914 DEVMETHOD(device_attach, ntb_plx_attach), 915 DEVMETHOD(device_detach, ntb_plx_detach), 916 /* Bus interface */ 917 DEVMETHOD(bus_child_location_str, ntb_child_location_str), 918 DEVMETHOD(bus_print_child, ntb_print_child), 919 /* NTB interface */ 920 DEVMETHOD(ntb_link_is_up, ntb_plx_link_is_up), 921 DEVMETHOD(ntb_link_enable, ntb_plx_link_enable), 922 DEVMETHOD(ntb_link_disable, ntb_plx_link_disable), 923 DEVMETHOD(ntb_link_enabled, ntb_plx_link_enabled), 924 DEVMETHOD(ntb_mw_count, ntb_plx_mw_count), 925 DEVMETHOD(ntb_mw_get_range, ntb_plx_mw_get_range), 926 DEVMETHOD(ntb_mw_set_trans, ntb_plx_mw_set_trans), 927 DEVMETHOD(ntb_mw_clear_trans, ntb_plx_mw_clear_trans), 928 DEVMETHOD(ntb_mw_get_wc, ntb_plx_mw_get_wc), 929 DEVMETHOD(ntb_mw_set_wc, ntb_plx_mw_set_wc), 930 DEVMETHOD(ntb_spad_count, ntb_plx_spad_count), 931 DEVMETHOD(ntb_spad_clear, ntb_plx_spad_clear), 932 DEVMETHOD(ntb_spad_write, ntb_plx_spad_write), 933 DEVMETHOD(ntb_spad_read, ntb_plx_spad_read), 934 DEVMETHOD(ntb_peer_spad_write, ntb_plx_peer_spad_write), 935 DEVMETHOD(ntb_peer_spad_read, ntb_plx_peer_spad_read), 936 DEVMETHOD(ntb_db_valid_mask, ntb_plx_db_valid_mask), 937 DEVMETHOD(ntb_db_vector_count, ntb_plx_db_vector_count), 938 DEVMETHOD(ntb_db_vector_mask, ntb_plx_db_vector_mask), 939 DEVMETHOD(ntb_db_clear, ntb_plx_db_clear), 940 DEVMETHOD(ntb_db_clear_mask, ntb_plx_db_clear_mask), 941 DEVMETHOD(ntb_db_read, ntb_plx_db_read), 942 DEVMETHOD(ntb_db_set_mask, ntb_plx_db_set_mask), 943 DEVMETHOD(ntb_peer_db_addr, ntb_plx_peer_db_addr), 944 DEVMETHOD(ntb_peer_db_set, ntb_plx_peer_db_set), 945 DEVMETHOD_END 946 }; 947 948 static DEFINE_CLASS_0(ntb_hw, ntb_plx_driver, ntb_plx_methods, 949 sizeof(struct ntb_plx_softc)); 950 DRIVER_MODULE(ntb_hw_plx, pci, ntb_plx_driver, ntb_hw_devclass, NULL, NULL); 951 MODULE_DEPEND(ntb_hw_plx, ntb, 1, 1, 1); 952 MODULE_VERSION(ntb_hw_plx, 1); 953