1 /*- 2 * Copyright (c) 2017 Alexander Motin <mav@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 /* 28 * The Non-Transparent Bridge (NTB) is a device that allows you to connect 29 * two or more systems using a PCI-e links, providing remote memory access. 30 * 31 * This module contains a driver for NTBs in PLX/Avago/Broadcom PCIe bridges. 32 */ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include <sys/param.h> 38 #include <sys/kernel.h> 39 #include <sys/systm.h> 40 #include <sys/bus.h> 41 #include <sys/interrupt.h> 42 #include <sys/module.h> 43 #include <sys/rman.h> 44 #include <sys/sysctl.h> 45 #include <vm/vm.h> 46 #include <vm/pmap.h> 47 #include <machine/bus.h> 48 #include <machine/intr_machdep.h> 49 #include <machine/resource.h> 50 #include <dev/pci/pcireg.h> 51 #include <dev/pci/pcivar.h> 52 53 #include "../ntb.h" 54 55 #define PLX_MAX_BARS 4 /* There are at most 4 data BARs. */ 56 #define PLX_NUM_SPAD 8 /* There are 8 scratchpads. */ 57 #define PLX_NUM_SPAD_PATT 4 /* Use test pattern as 4 more. */ 58 #define PLX_NUM_DB 16 /* There are 16 doorbells. */ 59 60 struct ntb_plx_mw_info { 61 int mw_bar; 62 int mw_64bit; 63 int mw_rid; 64 struct resource *mw_res; 65 vm_paddr_t mw_pbase; 66 caddr_t mw_vbase; 67 vm_size_t mw_size; 68 vm_memattr_t mw_map_mode; 69 bus_addr_t mw_xlat_addr; 70 size_t mw_xlat_size; 71 }; 72 73 struct ntb_plx_softc { 74 /* ntb.c context. Do not move! Must go first! */ 75 void *ntb_store; 76 77 device_t dev; 78 struct resource *conf_res; 79 int conf_rid; 80 u_int ntx; /* NTx number within chip. */ 81 u_int link; /* Link v/s Virtual side. */ 82 u_int port; /* Port number within chip. */ 83 u_int alut; /* A-LUT is enabled for NTx */ 84 85 int int_rid; 86 struct resource *int_res; 87 void *int_tag; 88 89 struct ntb_plx_mw_info mw_info[PLX_MAX_BARS]; 90 int mw_count; /* Number of memory windows. */ 91 92 int spad_count1; /* Number of standard spads. */ 93 int spad_count2; /* Number of extra spads. */ 94 uint32_t spad_off1; /* Offset of our spads. */ 95 uint32_t spad_off2; /* Offset of our extra spads. */ 96 uint32_t spad_offp1; /* Offset of peer spads. */ 97 uint32_t spad_offp2; /* Offset of peer extra spads. */ 98 99 /* Parameters of window shared with peer config access in B2B mode. */ 100 int b2b_mw; /* Shared window number. */ 101 uint64_t b2b_off; /* Offset in shared window. */ 102 }; 103 104 #define PLX_NT0_BASE 0x3E000 105 #define PLX_NT1_BASE 0x3C000 106 #define PLX_NTX_BASE(sc) ((sc)->ntx ? PLX_NT1_BASE : PLX_NT0_BASE) 107 #define PLX_NTX_LINK_OFFSET 0x01000 108 109 /* Bases of NTx our/peer interface registers */ 110 #define PLX_NTX_OUR_BASE(sc) \ 111 (PLX_NTX_BASE(sc) + ((sc)->link ? PLX_NTX_LINK_OFFSET : 0)) 112 #define PLX_NTX_PEER_BASE(sc) \ 113 (PLX_NTX_BASE(sc) + ((sc)->link ? 0 : PLX_NTX_LINK_OFFSET)) 114 115 /* Read/write NTx our interface registers */ 116 #define NTX_READ(sc, reg) \ 117 bus_read_4((sc)->conf_res, PLX_NTX_OUR_BASE(sc) + (reg)) 118 #define NTX_WRITE(sc, reg, val) \ 119 bus_write_4((sc)->conf_res, PLX_NTX_OUR_BASE(sc) + (reg), (val)) 120 121 /* Read/write NTx peer interface registers */ 122 #define PNTX_READ(sc, reg) \ 123 bus_read_4((sc)->conf_res, PLX_NTX_PEER_BASE(sc) + (reg)) 124 #define PNTX_WRITE(sc, reg, val) \ 125 bus_write_4((sc)->conf_res, PLX_NTX_PEER_BASE(sc) + (reg), (val)) 126 127 /* Read/write B2B NTx registers */ 128 #define BNTX_READ(sc, reg) \ 129 bus_read_4((sc)->mw_info[(sc)->b2b_mw].mw_res, \ 130 PLX_NTX_BASE(sc) + (reg)) 131 #define BNTX_WRITE(sc, reg, val) \ 132 bus_write_4((sc)->mw_info[(sc)->b2b_mw].mw_res, \ 133 PLX_NTX_BASE(sc) + (reg), (val)) 134 135 #define PLX_PORT_BASE(p) ((p) << 12) 136 #define PLX_STATION_PORT_BASE(sc) PLX_PORT_BASE((sc)->port & ~7) 137 138 #define PLX_PORT_CONTROL(sc) (PLX_STATION_PORT_BASE(sc) + 0x208) 139 140 static int ntb_plx_init(device_t dev); 141 static int ntb_plx_detach(device_t dev); 142 static int ntb_plx_mw_set_trans_internal(device_t dev, unsigned mw_idx); 143 144 static int 145 ntb_plx_probe(device_t dev) 146 { 147 148 switch (pci_get_devid(dev)) { 149 case 0x87a010b5: 150 device_set_desc(dev, "PLX Non-Transparent Bridge NT0 Link"); 151 return (BUS_PROBE_DEFAULT); 152 case 0x87a110b5: 153 device_set_desc(dev, "PLX Non-Transparent Bridge NT1 Link"); 154 return (BUS_PROBE_DEFAULT); 155 case 0x87b010b5: 156 device_set_desc(dev, "PLX Non-Transparent Bridge NT0 Virtual"); 157 return (BUS_PROBE_DEFAULT); 158 case 0x87b110b5: 159 device_set_desc(dev, "PLX Non-Transparent Bridge NT1 Virtual"); 160 return (BUS_PROBE_DEFAULT); 161 } 162 return (ENXIO); 163 } 164 165 static int 166 ntb_plx_init(device_t dev) 167 { 168 struct ntb_plx_softc *sc = device_get_softc(dev); 169 struct ntb_plx_mw_info *mw; 170 uint64_t val64; 171 int i; 172 uint32_t val; 173 174 if (sc->b2b_mw >= 0) { 175 /* Set peer BAR0/1 size and address for B2B NTx access. */ 176 mw = &sc->mw_info[sc->b2b_mw]; 177 if (mw->mw_64bit) { 178 PNTX_WRITE(sc, 0xe4, 0x3); /* 64-bit */ 179 val64 = 0x2000000000000000 * mw->mw_bar | 0x4; 180 PNTX_WRITE(sc, PCIR_BAR(0), val64); 181 PNTX_WRITE(sc, PCIR_BAR(0) + 4, val64 >> 32); 182 } else { 183 PNTX_WRITE(sc, 0xe4, 0x2); /* 32-bit */ 184 val = 0x20000000 * mw->mw_bar; 185 PNTX_WRITE(sc, PCIR_BAR(0), val); 186 } 187 188 /* Set Virtual to Link address translation for B2B. */ 189 for (i = 0; i < sc->mw_count; i++) { 190 mw = &sc->mw_info[i]; 191 if (mw->mw_64bit) { 192 val64 = 0x2000000000000000 * mw->mw_bar; 193 NTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4, val64); 194 NTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4 + 4, val64 >> 32); 195 } else { 196 val = 0x20000000 * mw->mw_bar; 197 NTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4, val); 198 } 199 } 200 201 /* Make sure Virtual to Link A-LUT is disabled. */ 202 if (sc->alut) 203 PNTX_WRITE(sc, 0xc94, 0); 204 205 /* Enable all Link Interface LUT entries for peer. */ 206 for (i = 0; i < 32; i += 2) { 207 PNTX_WRITE(sc, 0xdb4 + i * 2, 208 0x00010001 | ((i + 1) << 19) | (i << 3)); 209 } 210 } 211 212 /* 213 * Enable Virtual Interface LUT entry 0 for 0:0.*. 214 * entry 1 for our Requester ID reported by the chip, 215 * entries 2-5 for 0/64/128/192:4.* of I/OAT DMA engines. 216 * XXX: Its a hack, we can't know all DMA engines, but this covers all 217 * I/OAT of Xeon E5/E7 at least from Sandy Bridge till Skylake I saw. 218 */ 219 val = (NTX_READ(sc, 0xc90) << 16) | 0x00010001; 220 NTX_WRITE(sc, sc->link ? 0xdb4 : 0xd94, val); 221 NTX_WRITE(sc, sc->link ? 0xdb8 : 0xd98, 0x40210021); 222 NTX_WRITE(sc, sc->link ? 0xdbc : 0xd9c, 0xc0218021); 223 224 /* Set Link to Virtual address translation. */ 225 for (i = 0; i < sc->mw_count; i++) { 226 mw = &sc->mw_info[i]; 227 if (mw->mw_xlat_size != 0) 228 ntb_plx_mw_set_trans_internal(dev, i); 229 } 230 231 pci_enable_busmaster(dev); 232 if (sc->b2b_mw >= 0) 233 PNTX_WRITE(sc, PCIR_COMMAND, PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 234 235 return (0); 236 } 237 238 static void 239 ntb_plx_isr(void *arg) 240 { 241 device_t dev = arg; 242 struct ntb_plx_softc *sc = device_get_softc(dev); 243 uint32_t val; 244 245 ntb_db_event((device_t)arg, 0); 246 247 if (sc->link) /* Link Interface has no Link Error registers. */ 248 return; 249 250 val = NTX_READ(sc, 0xfe0); 251 if (val == 0) 252 return; 253 NTX_WRITE(sc, 0xfe0, val); 254 if (val & 1) 255 device_printf(dev, "Correctable Error\n"); 256 if (val & 2) 257 device_printf(dev, "Uncorrectable Error\n"); 258 if (val & 4) { 259 /* DL_Down resets link side registers, have to reinit. */ 260 ntb_plx_init(dev); 261 ntb_link_event(dev); 262 } 263 if (val & 8) 264 device_printf(dev, "Uncorrectable Error Message Drop\n"); 265 } 266 267 static int 268 ntb_plx_setup_intr(device_t dev) 269 { 270 struct ntb_plx_softc *sc = device_get_softc(dev); 271 int error; 272 273 /* 274 * XXX: This hardware supports MSI, but I found it unusable. 275 * It generates new MSI only when doorbell register goes from 276 * zero, but does not generate it when another bit is set or on 277 * partial clear. It makes operation very racy and unreliable. 278 * The data book mentions some mask juggling magic to workaround 279 * that, but I failed to make it work. 280 */ 281 sc->int_rid = 0; 282 sc->int_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, 283 &sc->int_rid, RF_SHAREABLE|RF_ACTIVE); 284 if (sc->int_res == NULL) { 285 device_printf(dev, "bus_alloc_resource failed\n"); 286 return (ENOMEM); 287 } 288 error = bus_setup_intr(dev, sc->int_res, INTR_MPSAFE | INTR_TYPE_MISC, 289 NULL, ntb_plx_isr, dev, &sc->int_tag); 290 if (error != 0) { 291 device_printf(dev, "bus_setup_intr failed: %d\n", error); 292 return (error); 293 } 294 295 if (!sc->link) { /* Link Interface has no Link Error registers. */ 296 NTX_WRITE(sc, 0xfe0, 0xf); /* Clear link interrupts. */ 297 NTX_WRITE(sc, 0xfe4, 0x0); /* Unmask link interrupts. */ 298 } 299 return (0); 300 } 301 302 static void 303 ntb_plx_teardown_intr(device_t dev) 304 { 305 struct ntb_plx_softc *sc = device_get_softc(dev); 306 307 if (!sc->link) /* Link Interface has no Link Error registers. */ 308 NTX_WRITE(sc, 0xfe4, 0xf); /* Mask link interrupts. */ 309 310 if (sc->int_res) { 311 bus_teardown_intr(dev, sc->int_res, sc->int_tag); 312 bus_release_resource(dev, SYS_RES_IRQ, sc->int_rid, 313 sc->int_res); 314 } 315 } 316 317 static int 318 ntb_plx_attach(device_t dev) 319 { 320 struct ntb_plx_softc *sc = device_get_softc(dev); 321 struct ntb_plx_mw_info *mw; 322 int error = 0, i; 323 uint32_t val; 324 char buf[32]; 325 326 /* Identify what we are (what side of what NTx). */ 327 sc->dev = dev; 328 val = pci_read_config(dev, 0xc8c, 4); 329 sc->ntx = (val & 1) != 0; 330 sc->link = (val & 0x80000000) != 0; 331 332 /* Get access to whole 256KB of chip configuration space via BAR0/1. */ 333 sc->conf_rid = PCIR_BAR(0); 334 sc->conf_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 335 &sc->conf_rid, RF_ACTIVE); 336 if (sc->conf_res == NULL) { 337 device_printf(dev, "Can't allocate configuration BAR.\n"); 338 return (ENXIO); 339 } 340 341 /* Identify chip port we are connected to. */ 342 val = bus_read_4(sc->conf_res, 0x360); 343 sc->port = (val >> ((sc->ntx == 0) ? 8 : 16)) & 0x1f; 344 345 /* Detect A-LUT enable and size. */ 346 val >>= 30; 347 sc->alut = (val == 0x3) ? 1 : ((val & (1 << sc->ntx)) ? 2 : 0); 348 if (sc->alut) 349 device_printf(dev, "%u A-LUT entries\n", 128 * sc->alut); 350 351 /* Find configured memory windows at BAR2-5. */ 352 sc->mw_count = 0; 353 for (i = 2; i <= 5; i++) { 354 mw = &sc->mw_info[sc->mw_count]; 355 mw->mw_bar = i; 356 mw->mw_rid = PCIR_BAR(mw->mw_bar); 357 mw->mw_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 358 &mw->mw_rid, RF_ACTIVE); 359 if (mw->mw_res == NULL) 360 continue; 361 mw->mw_pbase = rman_get_start(mw->mw_res); 362 mw->mw_size = rman_get_size(mw->mw_res); 363 mw->mw_vbase = rman_get_virtual(mw->mw_res); 364 mw->mw_map_mode = VM_MEMATTR_UNCACHEABLE; 365 sc->mw_count++; 366 367 /* Skip over adjacent BAR for 64-bit BARs. */ 368 val = pci_read_config(dev, PCIR_BAR(mw->mw_bar), 4); 369 if ((val & PCIM_BAR_MEM_TYPE) == PCIM_BAR_MEM_64) { 370 mw->mw_64bit = 1; 371 i++; 372 } 373 } 374 375 /* Try to identify B2B mode. */ 376 i = 1; 377 snprintf(buf, sizeof(buf), "hint.%s.%d.b2b", device_get_name(dev), 378 device_get_unit(dev)); 379 TUNABLE_INT_FETCH(buf, &i); 380 if (sc->link) { 381 device_printf(dev, "NTB-to-Root Port mode (Link Interface)\n"); 382 sc->b2b_mw = -1; 383 } else if (i == 0) { 384 device_printf(dev, "NTB-to-Root Port mode (Virtual Interface)\n"); 385 sc->b2b_mw = -1; 386 } else { 387 device_printf(dev, "NTB-to-NTB (back-to-back) mode\n"); 388 389 /* We need at least one memory window for B2B peer access. */ 390 if (sc->mw_count == 0) { 391 device_printf(dev, "No memory window BARs enabled.\n"); 392 error = ENXIO; 393 goto out; 394 } 395 sc->b2b_mw = sc->mw_count - 1; 396 397 /* Use half of the window for B2B, but no less then 1MB. */ 398 mw = &sc->mw_info[sc->b2b_mw]; 399 if (mw->mw_size >= 2 * 1024 * 1024) 400 sc->b2b_off = mw->mw_size / 2; 401 else 402 sc->b2b_off = 0; 403 } 404 405 /* 406 * Use Physical Layer User Test Pattern as additional scratchpad. 407 * Make sure they are present and enabled by writing to them. 408 * XXX: Its a hack, but standard 8 registers are not enough. 409 */ 410 sc->spad_offp1 = sc->spad_off1 = PLX_NTX_OUR_BASE(sc) + 0xc6c; 411 sc->spad_offp2 = sc->spad_off2 = PLX_PORT_BASE(sc->ntx * 8) + 0x20c; 412 if (sc->b2b_mw >= 0) { 413 /* In NTB-to-NTB mode each side has own scratchpads. */ 414 sc->spad_count1 = PLX_NUM_SPAD; 415 bus_write_4(sc->conf_res, sc->spad_off2, 0x12345678); 416 if (bus_read_4(sc->conf_res, sc->spad_off2) == 0x12345678) 417 sc->spad_count2 = PLX_NUM_SPAD_PATT; 418 } else { 419 /* Otherwise we have share scratchpads with the peer. */ 420 if (sc->link) { 421 sc->spad_off1 += PLX_NUM_SPAD / 2 * 4; 422 sc->spad_off2 += PLX_NUM_SPAD_PATT / 2 * 4; 423 } else { 424 sc->spad_offp1 += PLX_NUM_SPAD / 2 * 4; 425 sc->spad_offp2 += PLX_NUM_SPAD_PATT / 2 * 4; 426 } 427 sc->spad_count1 = PLX_NUM_SPAD / 2; 428 bus_write_4(sc->conf_res, sc->spad_off2, 0x12345678); 429 if (bus_read_4(sc->conf_res, sc->spad_off2) == 0x12345678) 430 sc->spad_count2 = PLX_NUM_SPAD_PATT / 2; 431 } 432 433 /* Apply static part of NTB configuration. */ 434 ntb_plx_init(dev); 435 436 /* Allocate and setup interrupts. */ 437 error = ntb_plx_setup_intr(dev); 438 if (error) 439 goto out; 440 441 /* Attach children to this controller */ 442 error = ntb_register_device(dev); 443 444 out: 445 if (error != 0) 446 ntb_plx_detach(dev); 447 return (error); 448 } 449 450 static int 451 ntb_plx_detach(device_t dev) 452 { 453 struct ntb_plx_softc *sc = device_get_softc(dev); 454 struct ntb_plx_mw_info *mw; 455 int i; 456 457 /* Detach & delete all children */ 458 ntb_unregister_device(dev); 459 460 /* Disable and free interrupts. */ 461 ntb_plx_teardown_intr(dev); 462 463 /* Free memory resources. */ 464 for (i = 0; i < sc->mw_count; i++) { 465 mw = &sc->mw_info[i]; 466 bus_release_resource(dev, SYS_RES_MEMORY, mw->mw_rid, 467 mw->mw_res); 468 } 469 bus_release_resource(dev, SYS_RES_MEMORY, sc->conf_rid, sc->conf_res); 470 return (0); 471 } 472 473 static int 474 ntb_plx_port_number(device_t dev) 475 { 476 struct ntb_plx_softc *sc = device_get_softc(dev); 477 478 return (sc->link ? 1 : 0); 479 } 480 481 static int 482 ntb_plx_peer_port_count(device_t dev) 483 { 484 485 return (1); 486 } 487 488 static int 489 ntb_plx_peer_port_number(device_t dev, int pidx) 490 { 491 struct ntb_plx_softc *sc = device_get_softc(dev); 492 493 if (pidx != 0) 494 return (-EINVAL); 495 496 return (sc->link ? 0 : 1); 497 } 498 499 static int 500 ntb_plx_peer_port_idx(device_t dev, int port) 501 { 502 int peer_port; 503 504 peer_port = ntb_plx_peer_port_number(dev, 0); 505 if (peer_port == -EINVAL || port != peer_port) 506 return (-EINVAL); 507 508 return (0); 509 } 510 511 static bool 512 ntb_plx_link_is_up(device_t dev, enum ntb_speed *speed, enum ntb_width *width) 513 { 514 uint16_t link; 515 516 link = pcie_read_config(dev, PCIER_LINK_STA, 2); 517 if (speed != NULL) 518 *speed = (link & PCIEM_LINK_STA_SPEED); 519 if (width != NULL) 520 *width = (link & PCIEM_LINK_STA_WIDTH) >> 4; 521 return ((link & PCIEM_LINK_STA_WIDTH) != 0); 522 } 523 524 static int 525 ntb_plx_link_enable(device_t dev, enum ntb_speed speed __unused, 526 enum ntb_width width __unused) 527 { 528 struct ntb_plx_softc *sc = device_get_softc(dev); 529 uint32_t reg, val; 530 531 /* The fact that we see the Link Interface means link is enabled. */ 532 if (sc->link) { 533 ntb_link_event(dev); 534 return (0); 535 } 536 537 reg = PLX_PORT_CONTROL(sc); 538 val = bus_read_4(sc->conf_res, reg); 539 if ((val & (1 << (sc->port & 7))) == 0) { 540 /* If already enabled, generate fake link event and exit. */ 541 ntb_link_event(dev); 542 return (0); 543 } 544 val &= ~(1 << (sc->port & 7)); 545 bus_write_4(sc->conf_res, reg, val); 546 return (0); 547 } 548 549 static int 550 ntb_plx_link_disable(device_t dev) 551 { 552 struct ntb_plx_softc *sc = device_get_softc(dev); 553 uint32_t reg, val; 554 555 /* Link disable for Link Interface would be suicidal. */ 556 if (sc->link) 557 return (0); 558 559 reg = PLX_PORT_CONTROL(sc); 560 val = bus_read_4(sc->conf_res, reg); 561 val |= (1 << (sc->port & 7)); 562 bus_write_4(sc->conf_res, reg, val); 563 return (0); 564 } 565 566 static bool 567 ntb_plx_link_enabled(device_t dev) 568 { 569 struct ntb_plx_softc *sc = device_get_softc(dev); 570 uint32_t reg, val; 571 572 /* The fact that we see the Link Interface means link is enabled. */ 573 if (sc->link) 574 return (TRUE); 575 576 reg = PLX_PORT_CONTROL(sc); 577 val = bus_read_4(sc->conf_res, reg); 578 return ((val & (1 << (sc->port & 7))) == 0); 579 } 580 581 static uint8_t 582 ntb_plx_mw_count(device_t dev) 583 { 584 struct ntb_plx_softc *sc = device_get_softc(dev); 585 586 if (sc->b2b_mw >= 0 && sc->b2b_off == 0) 587 return (sc->mw_count - 1); /* B2B consumed whole window. */ 588 return (sc->mw_count); 589 } 590 591 static int 592 ntb_plx_mw_get_range(device_t dev, unsigned mw_idx, vm_paddr_t *base, 593 caddr_t *vbase, size_t *size, size_t *align, size_t *align_size, 594 bus_addr_t *plimit) 595 { 596 struct ntb_plx_softc *sc = device_get_softc(dev); 597 struct ntb_plx_mw_info *mw; 598 size_t off; 599 600 if (mw_idx >= sc->mw_count) 601 return (EINVAL); 602 off = 0; 603 if (mw_idx == sc->b2b_mw) { 604 KASSERT(sc->b2b_off != 0, 605 ("user shouldn't get non-shared b2b mw")); 606 off = sc->b2b_off; 607 } 608 mw = &sc->mw_info[mw_idx]; 609 610 /* Local to remote memory window parameters. */ 611 if (base != NULL) 612 *base = mw->mw_pbase + off; 613 if (vbase != NULL) 614 *vbase = mw->mw_vbase + off; 615 if (size != NULL) 616 *size = mw->mw_size - off; 617 618 /* 619 * Remote to local memory window translation address alignment. 620 * Translation address has to be aligned to the BAR size, but A-LUT 621 * entries re-map addresses can be aligned to 1/128 or 1/256 of it. 622 * XXX: In B2B mode we can change BAR size (and so alignmet) live, 623 * but there is no way to report it here, so report safe value. 624 */ 625 if (align != NULL) { 626 if (sc->alut && mw->mw_bar == 2) 627 *align = (mw->mw_size - off) / 128 / sc->alut; 628 else 629 *align = mw->mw_size - off; 630 } 631 632 /* 633 * Remote to local memory window size alignment. 634 * The chip has no limit registers, but A-LUT, when available, allows 635 * access control with granularity of 1/128 or 1/256 of the BAR size. 636 * XXX: In B2B case we can change BAR size live, but there is no way 637 * to report it, so report half of the BAR size, that should be safe. 638 * In non-B2B case there is no control at all, so report the BAR size. 639 */ 640 if (align_size != NULL) { 641 if (sc->alut && mw->mw_bar == 2) 642 *align_size = (mw->mw_size - off) / 128 / sc->alut; 643 else if (sc->b2b_mw >= 0) 644 *align_size = (mw->mw_size - off) / 2; 645 else 646 *align_size = mw->mw_size - off; 647 } 648 649 /* Remote to local memory window translation address upper limit. */ 650 if (plimit != NULL) 651 *plimit = mw->mw_64bit ? BUS_SPACE_MAXADDR : 652 BUS_SPACE_MAXADDR_32BIT; 653 return (0); 654 } 655 656 static int 657 ntb_plx_mw_set_trans_internal(device_t dev, unsigned mw_idx) 658 { 659 struct ntb_plx_softc *sc = device_get_softc(dev); 660 struct ntb_plx_mw_info *mw; 661 uint64_t addr, eaddr, off, size, bsize, esize, val64; 662 uint32_t val; 663 int i; 664 665 mw = &sc->mw_info[mw_idx]; 666 addr = mw->mw_xlat_addr; 667 size = mw->mw_xlat_size; 668 off = 0; 669 if (mw_idx == sc->b2b_mw) { 670 off = sc->b2b_off; 671 KASSERT(off != 0, ("user shouldn't get non-shared b2b mw")); 672 673 /* 674 * While generally we can set any BAR size on link side, 675 * for B2B shared window we can't go above preconfigured 676 * size due to BAR address alignment requirements. 677 */ 678 if (size > mw->mw_size - off) 679 return (EINVAL); 680 } 681 682 if (size > 0) { 683 /* Round BAR size to next power of 2 or at least 1MB. */ 684 bsize = size; 685 if (!powerof2(bsize)) 686 bsize = 1LL << flsll(bsize); 687 if (bsize < 1024 * 1024) 688 bsize = 1024 * 1024; 689 690 /* A-LUT has 128 or 256 times better granularity. */ 691 esize = bsize; 692 if (sc->alut && mw->mw_bar == 2) 693 esize /= 128 * sc->alut; 694 695 /* addr should be aligned to BAR or A-LUT element size. */ 696 if ((addr & (esize - 1)) != 0) 697 return (EINVAL); 698 } else 699 esize = bsize = 0; 700 701 if (mw->mw_64bit) { 702 if (sc->b2b_mw >= 0) { 703 /* Set Link Interface BAR size and enable/disable it. */ 704 val64 = 0; 705 if (bsize > 0) 706 val64 = (~(bsize - 1) & ~0xfffff); 707 val64 |= 0xc; 708 PNTX_WRITE(sc, 0xe8 + (mw->mw_bar - 2) * 4, val64); 709 PNTX_WRITE(sc, 0xe8 + (mw->mw_bar - 2) * 4 + 4, val64 >> 32); 710 711 /* Set Link Interface BAR address. */ 712 val64 = 0x2000000000000000 * mw->mw_bar + off; 713 PNTX_WRITE(sc, PCIR_BAR(mw->mw_bar), val64); 714 PNTX_WRITE(sc, PCIR_BAR(mw->mw_bar) + 4, val64 >> 32); 715 } 716 717 /* Set Virtual Interface BARs address translation */ 718 PNTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4, addr); 719 PNTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4 + 4, addr >> 32); 720 } else { 721 /* Make sure we fit into 32-bit address space. */ 722 if ((addr & UINT32_MAX) != addr) 723 return (ERANGE); 724 if (((addr + bsize) & UINT32_MAX) != (addr + bsize)) 725 return (ERANGE); 726 727 if (sc->b2b_mw >= 0) { 728 /* Set Link Interface BAR size and enable/disable it. */ 729 val = 0; 730 if (bsize > 0) 731 val = (~(bsize - 1) & ~0xfffff); 732 PNTX_WRITE(sc, 0xe8 + (mw->mw_bar - 2) * 4, val); 733 734 /* Set Link Interface BAR address. */ 735 val64 = 0x20000000 * mw->mw_bar + off; 736 PNTX_WRITE(sc, PCIR_BAR(mw->mw_bar), val64); 737 } 738 739 /* Set Virtual Interface BARs address translation */ 740 PNTX_WRITE(sc, 0xc3c + (mw->mw_bar - 2) * 4, addr); 741 } 742 743 /* Configure and enable Link to Virtual A-LUT if we need it. */ 744 if (sc->alut && mw->mw_bar == 2 && 745 ((addr & (bsize - 1)) != 0 || size != bsize)) { 746 eaddr = addr; 747 for (i = 0; i < 128 * sc->alut; i++) { 748 val = sc->link ? 0 : 1; 749 if (sc->alut == 1) 750 val += 2 * sc->ntx; 751 val *= 0x1000 * sc->alut; 752 val += 0x38000 + i * 4 + (i >= 128 ? 0x0e00 : 0); 753 bus_write_4(sc->conf_res, val, eaddr); 754 bus_write_4(sc->conf_res, val + 0x400, eaddr >> 32); 755 bus_write_4(sc->conf_res, val + 0x800, 756 (eaddr < addr + size) ? 0x3 : 0); 757 eaddr += esize; 758 } 759 NTX_WRITE(sc, 0xc94, 0x10000000); 760 } else if (sc->alut && mw->mw_bar == 2) 761 NTX_WRITE(sc, 0xc94, 0); 762 763 return (0); 764 } 765 766 static int 767 ntb_plx_mw_set_trans(device_t dev, unsigned mw_idx, bus_addr_t addr, size_t size) 768 { 769 struct ntb_plx_softc *sc = device_get_softc(dev); 770 struct ntb_plx_mw_info *mw; 771 772 if (mw_idx >= sc->mw_count) 773 return (EINVAL); 774 mw = &sc->mw_info[mw_idx]; 775 mw->mw_xlat_addr = addr; 776 mw->mw_xlat_size = size; 777 return (ntb_plx_mw_set_trans_internal(dev, mw_idx)); 778 } 779 780 static int 781 ntb_plx_mw_clear_trans(device_t dev, unsigned mw_idx) 782 { 783 784 return (ntb_plx_mw_set_trans(dev, mw_idx, 0, 0)); 785 } 786 787 static int 788 ntb_plx_mw_get_wc(device_t dev, unsigned idx, vm_memattr_t *mode) 789 { 790 struct ntb_plx_softc *sc = device_get_softc(dev); 791 struct ntb_plx_mw_info *mw; 792 793 if (idx >= sc->mw_count) 794 return (EINVAL); 795 mw = &sc->mw_info[idx]; 796 *mode = mw->mw_map_mode; 797 return (0); 798 } 799 800 static int 801 ntb_plx_mw_set_wc(device_t dev, unsigned idx, vm_memattr_t mode) 802 { 803 struct ntb_plx_softc *sc = device_get_softc(dev); 804 struct ntb_plx_mw_info *mw; 805 uint64_t off; 806 int rc; 807 808 if (idx >= sc->mw_count) 809 return (EINVAL); 810 mw = &sc->mw_info[idx]; 811 if (mw->mw_map_mode == mode) 812 return (0); 813 814 off = 0; 815 if (idx == sc->b2b_mw) { 816 KASSERT(sc->b2b_off != 0, 817 ("user shouldn't get non-shared b2b mw")); 818 off = sc->b2b_off; 819 } 820 821 rc = pmap_change_attr((vm_offset_t)mw->mw_vbase + off, 822 mw->mw_size - off, mode); 823 if (rc == 0) 824 mw->mw_map_mode = mode; 825 return (rc); 826 } 827 828 static uint8_t 829 ntb_plx_spad_count(device_t dev) 830 { 831 struct ntb_plx_softc *sc = device_get_softc(dev); 832 833 return (sc->spad_count1 + sc->spad_count2); 834 } 835 836 static int 837 ntb_plx_spad_write(device_t dev, unsigned int idx, uint32_t val) 838 { 839 struct ntb_plx_softc *sc = device_get_softc(dev); 840 u_int off; 841 842 if (idx >= sc->spad_count1 + sc->spad_count2) 843 return (EINVAL); 844 845 if (idx < sc->spad_count1) 846 off = sc->spad_off1 + idx * 4; 847 else 848 off = sc->spad_off2 + (idx - sc->spad_count1) * 4; 849 bus_write_4(sc->conf_res, off, val); 850 return (0); 851 } 852 853 static void 854 ntb_plx_spad_clear(device_t dev) 855 { 856 struct ntb_plx_softc *sc = device_get_softc(dev); 857 int i; 858 859 for (i = 0; i < sc->spad_count1 + sc->spad_count2; i++) 860 ntb_plx_spad_write(dev, i, 0); 861 } 862 863 static int 864 ntb_plx_spad_read(device_t dev, unsigned int idx, uint32_t *val) 865 { 866 struct ntb_plx_softc *sc = device_get_softc(dev); 867 u_int off; 868 869 if (idx >= sc->spad_count1 + sc->spad_count2) 870 return (EINVAL); 871 872 if (idx < sc->spad_count1) 873 off = sc->spad_off1 + idx * 4; 874 else 875 off = sc->spad_off2 + (idx - sc->spad_count1) * 4; 876 *val = bus_read_4(sc->conf_res, off); 877 return (0); 878 } 879 880 static int 881 ntb_plx_peer_spad_write(device_t dev, unsigned int idx, uint32_t val) 882 { 883 struct ntb_plx_softc *sc = device_get_softc(dev); 884 u_int off; 885 886 if (idx >= sc->spad_count1 + sc->spad_count2) 887 return (EINVAL); 888 889 if (idx < sc->spad_count1) 890 off = sc->spad_offp1 + idx * 4; 891 else 892 off = sc->spad_offp2 + (idx - sc->spad_count1) * 4; 893 if (sc->b2b_mw >= 0) 894 bus_write_4(sc->mw_info[sc->b2b_mw].mw_res, off, val); 895 else 896 bus_write_4(sc->conf_res, off, val); 897 return (0); 898 } 899 900 static int 901 ntb_plx_peer_spad_read(device_t dev, unsigned int idx, uint32_t *val) 902 { 903 struct ntb_plx_softc *sc = device_get_softc(dev); 904 u_int off; 905 906 if (idx >= sc->spad_count1 + sc->spad_count2) 907 return (EINVAL); 908 909 if (idx < sc->spad_count1) 910 off = sc->spad_offp1 + idx * 4; 911 else 912 off = sc->spad_offp2 + (idx - sc->spad_count1) * 4; 913 if (sc->b2b_mw >= 0) 914 *val = bus_read_4(sc->mw_info[sc->b2b_mw].mw_res, off); 915 else 916 *val = bus_read_4(sc->conf_res, off); 917 return (0); 918 } 919 920 static uint64_t 921 ntb_plx_db_valid_mask(device_t dev) 922 { 923 924 return ((1LL << PLX_NUM_DB) - 1); 925 } 926 927 static int 928 ntb_plx_db_vector_count(device_t dev) 929 { 930 931 return (1); 932 } 933 934 static uint64_t 935 ntb_plx_db_vector_mask(device_t dev, uint32_t vector) 936 { 937 938 if (vector > 0) 939 return (0); 940 return ((1LL << PLX_NUM_DB) - 1); 941 } 942 943 static void 944 ntb_plx_db_clear(device_t dev, uint64_t bits) 945 { 946 struct ntb_plx_softc *sc = device_get_softc(dev); 947 948 NTX_WRITE(sc, sc->link ? 0xc60 : 0xc50, bits); 949 } 950 951 static void 952 ntb_plx_db_clear_mask(device_t dev, uint64_t bits) 953 { 954 struct ntb_plx_softc *sc = device_get_softc(dev); 955 956 NTX_WRITE(sc, sc->link ? 0xc68 : 0xc58, bits); 957 } 958 959 static uint64_t 960 ntb_plx_db_read(device_t dev) 961 { 962 struct ntb_plx_softc *sc = device_get_softc(dev); 963 964 return (NTX_READ(sc, sc->link ? 0xc5c : 0xc4c)); 965 } 966 967 static void 968 ntb_plx_db_set_mask(device_t dev, uint64_t bits) 969 { 970 struct ntb_plx_softc *sc = device_get_softc(dev); 971 972 NTX_WRITE(sc, sc->link ? 0xc64 : 0xc54, bits); 973 } 974 975 static int 976 ntb_plx_peer_db_addr(device_t dev, bus_addr_t *db_addr, vm_size_t *db_size) 977 { 978 struct ntb_plx_softc *sc = device_get_softc(dev); 979 struct ntb_plx_mw_info *mw; 980 981 KASSERT((db_addr != NULL && db_size != NULL), ("must be non-NULL")); 982 983 if (sc->b2b_mw >= 0) { 984 mw = &sc->mw_info[sc->b2b_mw]; 985 *db_addr = (uint64_t)mw->mw_pbase + PLX_NTX_BASE(sc) + 0xc4c; 986 } else { 987 *db_addr = rman_get_start(sc->conf_res) + PLX_NTX_BASE(sc); 988 *db_addr += sc->link ? 0xc4c : 0xc5c; 989 } 990 *db_size = 4; 991 return (0); 992 } 993 994 static void 995 ntb_plx_peer_db_set(device_t dev, uint64_t bit) 996 { 997 struct ntb_plx_softc *sc = device_get_softc(dev); 998 999 if (sc->b2b_mw >= 0) 1000 BNTX_WRITE(sc, 0xc4c, bit); 1001 else 1002 NTX_WRITE(sc, sc->link ? 0xc4c : 0xc5c, bit); 1003 } 1004 1005 static device_method_t ntb_plx_methods[] = { 1006 /* Device interface */ 1007 DEVMETHOD(device_probe, ntb_plx_probe), 1008 DEVMETHOD(device_attach, ntb_plx_attach), 1009 DEVMETHOD(device_detach, ntb_plx_detach), 1010 /* Bus interface */ 1011 DEVMETHOD(bus_child_location_str, ntb_child_location_str), 1012 DEVMETHOD(bus_print_child, ntb_print_child), 1013 /* NTB interface */ 1014 DEVMETHOD(ntb_port_number, ntb_plx_port_number), 1015 DEVMETHOD(ntb_peer_port_count, ntb_plx_peer_port_count), 1016 DEVMETHOD(ntb_peer_port_number, ntb_plx_peer_port_number), 1017 DEVMETHOD(ntb_peer_port_idx, ntb_plx_peer_port_idx), 1018 DEVMETHOD(ntb_link_is_up, ntb_plx_link_is_up), 1019 DEVMETHOD(ntb_link_enable, ntb_plx_link_enable), 1020 DEVMETHOD(ntb_link_disable, ntb_plx_link_disable), 1021 DEVMETHOD(ntb_link_enabled, ntb_plx_link_enabled), 1022 DEVMETHOD(ntb_mw_count, ntb_plx_mw_count), 1023 DEVMETHOD(ntb_mw_get_range, ntb_plx_mw_get_range), 1024 DEVMETHOD(ntb_mw_set_trans, ntb_plx_mw_set_trans), 1025 DEVMETHOD(ntb_mw_clear_trans, ntb_plx_mw_clear_trans), 1026 DEVMETHOD(ntb_mw_get_wc, ntb_plx_mw_get_wc), 1027 DEVMETHOD(ntb_mw_set_wc, ntb_plx_mw_set_wc), 1028 DEVMETHOD(ntb_spad_count, ntb_plx_spad_count), 1029 DEVMETHOD(ntb_spad_clear, ntb_plx_spad_clear), 1030 DEVMETHOD(ntb_spad_write, ntb_plx_spad_write), 1031 DEVMETHOD(ntb_spad_read, ntb_plx_spad_read), 1032 DEVMETHOD(ntb_peer_spad_write, ntb_plx_peer_spad_write), 1033 DEVMETHOD(ntb_peer_spad_read, ntb_plx_peer_spad_read), 1034 DEVMETHOD(ntb_db_valid_mask, ntb_plx_db_valid_mask), 1035 DEVMETHOD(ntb_db_vector_count, ntb_plx_db_vector_count), 1036 DEVMETHOD(ntb_db_vector_mask, ntb_plx_db_vector_mask), 1037 DEVMETHOD(ntb_db_clear, ntb_plx_db_clear), 1038 DEVMETHOD(ntb_db_clear_mask, ntb_plx_db_clear_mask), 1039 DEVMETHOD(ntb_db_read, ntb_plx_db_read), 1040 DEVMETHOD(ntb_db_set_mask, ntb_plx_db_set_mask), 1041 DEVMETHOD(ntb_peer_db_addr, ntb_plx_peer_db_addr), 1042 DEVMETHOD(ntb_peer_db_set, ntb_plx_peer_db_set), 1043 DEVMETHOD_END 1044 }; 1045 1046 static DEFINE_CLASS_0(ntb_hw, ntb_plx_driver, ntb_plx_methods, 1047 sizeof(struct ntb_plx_softc)); 1048 DRIVER_MODULE(ntb_hw_plx, pci, ntb_plx_driver, ntb_hw_devclass, NULL, NULL); 1049 MODULE_DEPEND(ntb_hw_plx, ntb, 1, 1, 1); 1050 MODULE_VERSION(ntb_hw_plx, 1); 1051