1 /*- 2 * Copyright (c) 2016-2017 Alexander Motin <mav@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 #include <sys/param.h> 29 #include <sys/kernel.h> 30 #include <sys/systm.h> 31 #include <sys/bus.h> 32 #include <sys/rmlock.h> 33 #include <sys/malloc.h> 34 #include <sys/module.h> 35 #include <sys/sbuf.h> 36 #include <sys/sysctl.h> 37 38 #include "ntb.h" 39 40 SYSCTL_NODE(_hw, OID_AUTO, ntb, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 41 "NTB sysctls"); 42 43 struct ntb_child { 44 device_t dev; 45 int function; 46 int enabled; 47 int mwoff; 48 int mwcnt; 49 int spadoff; 50 int spadcnt; 51 int dboff; 52 int dbcnt; 53 uint64_t dbmask; 54 void *ctx; 55 const struct ntb_ctx_ops *ctx_ops; 56 struct rmlock ctx_lock; 57 struct ntb_child *next; 58 }; 59 60 int 61 ntb_register_device(device_t dev) 62 { 63 struct ntb_child **cpp = device_get_softc(dev); 64 struct ntb_child *nc; 65 int i, mw, mwu, mwt, spad, spadu, spadt, db, dbu, dbt; 66 char cfg[128] = ""; 67 char buf[32]; 68 char *n, *np, *c, *p, *name; 69 70 mwu = 0; 71 mwt = NTB_MW_COUNT(dev); 72 spadu = 0; 73 spadt = NTB_SPAD_COUNT(dev); 74 dbu = 0; 75 dbt = flsll(NTB_DB_VALID_MASK(dev)); 76 77 device_printf(dev, "%d memory windows, %d scratchpads, " 78 "%d doorbells\n", mwt, spadt, dbt); 79 80 snprintf(buf, sizeof(buf), "hint.%s.%d.config", device_get_name(dev), 81 device_get_unit(dev)); 82 TUNABLE_STR_FETCH(buf, cfg, sizeof(cfg)); 83 n = cfg; 84 i = 0; 85 while ((c = strsep(&n, ",")) != NULL) { 86 np = c; 87 name = strsep(&np, ":"); 88 if (name != NULL && name[0] == 0) 89 name = NULL; 90 p = strsep(&np, ":"); 91 mw = (p && p[0] != 0) ? strtol(p, NULL, 10) : mwt - mwu; 92 p = strsep(&np, ":"); 93 spad = (p && p[0] != 0) ? strtol(p, NULL, 10) : spadt - spadu; 94 db = (np && np[0] != 0) ? strtol(np, NULL, 10) : dbt - dbu; 95 96 if (mw > mwt - mwu || spad > spadt - spadu || db > dbt - dbu) { 97 device_printf(dev, "Not enough resources for config\n"); 98 break; 99 } 100 101 nc = malloc(sizeof(*nc), M_DEVBUF, M_WAITOK | M_ZERO); 102 nc->function = i; 103 nc->mwoff = mwu; 104 nc->mwcnt = mw; 105 nc->spadoff = spadu; 106 nc->spadcnt = spad; 107 nc->dboff = dbu; 108 nc->dbcnt = db; 109 nc->dbmask = (db == 0) ? 0 : (0xffffffffffffffff >> (64 - db)); 110 rm_init(&nc->ctx_lock, "ntb ctx"); 111 nc->dev = device_add_child(dev, name, -1); 112 if (nc->dev == NULL) { 113 ntb_unregister_device(dev); 114 return (ENOMEM); 115 } 116 device_set_ivars(nc->dev, nc); 117 *cpp = nc; 118 cpp = &nc->next; 119 120 if (bootverbose) { 121 device_printf(dev, "%d \"%s\":", i, name); 122 if (mw > 0) { 123 printf(" memory windows %d", mwu); 124 if (mw > 1) 125 printf("-%d", mwu + mw - 1); 126 } 127 if (spad > 0) { 128 printf(" scratchpads %d", spadu); 129 if (spad > 1) 130 printf("-%d", spadu + spad - 1); 131 } 132 if (db > 0) { 133 printf(" doorbells %d", dbu); 134 if (db > 1) 135 printf("-%d", dbu + db - 1); 136 } 137 printf("\n"); 138 } 139 140 mwu += mw; 141 spadu += spad; 142 dbu += db; 143 i++; 144 } 145 146 bus_generic_attach(dev); 147 return (0); 148 } 149 150 int 151 ntb_unregister_device(device_t dev) 152 { 153 struct ntb_child **cpp = device_get_softc(dev); 154 struct ntb_child *nc; 155 int error = 0; 156 157 while ((nc = *cpp) != NULL) { 158 *cpp = (*cpp)->next; 159 error = device_delete_child(dev, nc->dev); 160 if (error) 161 break; 162 rm_destroy(&nc->ctx_lock); 163 free(nc, M_DEVBUF); 164 } 165 return (error); 166 } 167 168 int 169 ntb_child_location(device_t dev, device_t child, struct sbuf *sb) 170 { 171 struct ntb_child *nc = device_get_ivars(child); 172 173 sbuf_printf(sb, "function=%d", nc->function); 174 return (0); 175 } 176 177 int 178 ntb_print_child(device_t dev, device_t child) 179 { 180 struct ntb_child *nc = device_get_ivars(child); 181 int retval; 182 183 retval = bus_print_child_header(dev, child); 184 if (nc->mwcnt > 0) { 185 printf(" mw %d", nc->mwoff); 186 if (nc->mwcnt > 1) 187 printf("-%d", nc->mwoff + nc->mwcnt - 1); 188 } 189 if (nc->spadcnt > 0) { 190 printf(" spad %d", nc->spadoff); 191 if (nc->spadcnt > 1) 192 printf("-%d", nc->spadoff + nc->spadcnt - 1); 193 } 194 if (nc->dbcnt > 0) { 195 printf(" db %d", nc->dboff); 196 if (nc->dbcnt > 1) 197 printf("-%d", nc->dboff + nc->dbcnt - 1); 198 } 199 retval += printf(" at function %d", nc->function); 200 retval += bus_print_child_domain(dev, child); 201 retval += bus_print_child_footer(dev, child); 202 203 return (retval); 204 } 205 206 bus_dma_tag_t 207 ntb_get_dma_tag(device_t bus, device_t child) 208 { 209 210 return (bus_get_dma_tag(bus)); 211 } 212 213 void 214 ntb_link_event(device_t dev) 215 { 216 struct ntb_child **cpp = device_get_softc(dev); 217 struct ntb_child *nc; 218 struct rm_priotracker ctx_tracker; 219 enum ntb_speed speed; 220 enum ntb_width width; 221 222 if (NTB_LINK_IS_UP(dev, &speed, &width)) { 223 device_printf(dev, "Link is up (PCIe %d.x / x%d)\n", 224 (int)speed, (int)width); 225 } else { 226 device_printf(dev, "Link is down\n"); 227 } 228 for (nc = *cpp; nc != NULL; nc = nc->next) { 229 rm_rlock(&nc->ctx_lock, &ctx_tracker); 230 if (nc->ctx_ops != NULL && nc->ctx_ops->link_event != NULL) 231 nc->ctx_ops->link_event(nc->ctx); 232 rm_runlock(&nc->ctx_lock, &ctx_tracker); 233 } 234 } 235 236 void 237 ntb_db_event(device_t dev, uint32_t vec) 238 { 239 struct ntb_child **cpp = device_get_softc(dev); 240 struct ntb_child *nc; 241 struct rm_priotracker ctx_tracker; 242 243 for (nc = *cpp; nc != NULL; nc = nc->next) { 244 rm_rlock(&nc->ctx_lock, &ctx_tracker); 245 if (nc->ctx_ops != NULL && nc->ctx_ops->db_event != NULL) 246 nc->ctx_ops->db_event(nc->ctx, vec); 247 rm_runlock(&nc->ctx_lock, &ctx_tracker); 248 } 249 } 250 251 int 252 ntb_port_number(device_t ntb) 253 { 254 return (NTB_PORT_NUMBER(device_get_parent(ntb))); 255 } 256 257 int 258 ntb_peer_port_count(device_t ntb) 259 { 260 return (NTB_PEER_PORT_COUNT(device_get_parent(ntb))); 261 } 262 263 int 264 ntb_peer_port_number(device_t ntb, int pidx) 265 { 266 return (NTB_PEER_PORT_NUMBER(device_get_parent(ntb), pidx)); 267 } 268 269 int 270 ntb_peer_port_idx(device_t ntb, int port) 271 { 272 return (NTB_PEER_PORT_IDX(device_get_parent(ntb), port)); 273 } 274 275 bool 276 ntb_link_is_up(device_t ntb, enum ntb_speed *speed, enum ntb_width *width) 277 { 278 279 return (NTB_LINK_IS_UP(device_get_parent(ntb), speed, width)); 280 } 281 282 int 283 ntb_link_enable(device_t ntb, enum ntb_speed speed, enum ntb_width width) 284 { 285 struct ntb_child *nc = device_get_ivars(ntb); 286 struct ntb_child **cpp = device_get_softc(device_get_parent(nc->dev)); 287 struct ntb_child *nc1; 288 289 for (nc1 = *cpp; nc1 != NULL; nc1 = nc1->next) { 290 if (nc1->enabled) { 291 nc->enabled = 1; 292 return (0); 293 } 294 } 295 nc->enabled = 1; 296 return (NTB_LINK_ENABLE(device_get_parent(ntb), speed, width)); 297 } 298 299 int 300 ntb_link_disable(device_t ntb) 301 { 302 struct ntb_child *nc = device_get_ivars(ntb); 303 struct ntb_child **cpp = device_get_softc(device_get_parent(nc->dev)); 304 struct ntb_child *nc1; 305 306 if (!nc->enabled) 307 return (0); 308 nc->enabled = 0; 309 for (nc1 = *cpp; nc1 != NULL; nc1 = nc1->next) { 310 if (nc1->enabled) 311 return (0); 312 } 313 return (NTB_LINK_DISABLE(device_get_parent(ntb))); 314 } 315 316 bool 317 ntb_link_enabled(device_t ntb) 318 { 319 struct ntb_child *nc = device_get_ivars(ntb); 320 321 return (nc->enabled && NTB_LINK_ENABLED(device_get_parent(ntb))); 322 } 323 324 int 325 ntb_set_ctx(device_t ntb, void *ctx, const struct ntb_ctx_ops *ctx_ops) 326 { 327 struct ntb_child *nc = device_get_ivars(ntb); 328 329 if (ctx == NULL || ctx_ops == NULL) 330 return (EINVAL); 331 332 rm_wlock(&nc->ctx_lock); 333 if (nc->ctx_ops != NULL) { 334 rm_wunlock(&nc->ctx_lock); 335 return (EINVAL); 336 } 337 nc->ctx = ctx; 338 nc->ctx_ops = ctx_ops; 339 340 /* 341 * If applicaiton driver asks for link events, generate fake one now 342 * to let it update link state without races while we hold the lock. 343 */ 344 if (ctx_ops->link_event != NULL) 345 ctx_ops->link_event(ctx); 346 rm_wunlock(&nc->ctx_lock); 347 348 return (0); 349 } 350 351 void * 352 ntb_get_ctx(device_t ntb, const struct ntb_ctx_ops **ctx_ops) 353 { 354 struct ntb_child *nc = device_get_ivars(ntb); 355 356 KASSERT(nc->ctx != NULL && nc->ctx_ops != NULL, ("bogus")); 357 if (ctx_ops != NULL) 358 *ctx_ops = nc->ctx_ops; 359 return (nc->ctx); 360 } 361 362 void 363 ntb_clear_ctx(device_t ntb) 364 { 365 struct ntb_child *nc = device_get_ivars(ntb); 366 367 rm_wlock(&nc->ctx_lock); 368 nc->ctx = NULL; 369 nc->ctx_ops = NULL; 370 rm_wunlock(&nc->ctx_lock); 371 } 372 373 uint8_t 374 ntb_mw_count(device_t ntb) 375 { 376 struct ntb_child *nc = device_get_ivars(ntb); 377 378 return (nc->mwcnt); 379 } 380 381 int 382 ntb_mw_get_range(device_t ntb, unsigned mw_idx, vm_paddr_t *base, 383 caddr_t *vbase, size_t *size, size_t *align, size_t *align_size, 384 bus_addr_t *plimit) 385 { 386 struct ntb_child *nc = device_get_ivars(ntb); 387 388 return (NTB_MW_GET_RANGE(device_get_parent(ntb), mw_idx + nc->mwoff, 389 base, vbase, size, align, align_size, plimit)); 390 } 391 392 int 393 ntb_mw_set_trans(device_t ntb, unsigned mw_idx, bus_addr_t addr, size_t size) 394 { 395 struct ntb_child *nc = device_get_ivars(ntb); 396 397 return (NTB_MW_SET_TRANS(device_get_parent(ntb), mw_idx + nc->mwoff, 398 addr, size)); 399 } 400 401 int 402 ntb_mw_clear_trans(device_t ntb, unsigned mw_idx) 403 { 404 struct ntb_child *nc = device_get_ivars(ntb); 405 406 return (NTB_MW_CLEAR_TRANS(device_get_parent(ntb), mw_idx + nc->mwoff)); 407 } 408 409 int 410 ntb_mw_get_wc(device_t ntb, unsigned mw_idx, vm_memattr_t *mode) 411 { 412 struct ntb_child *nc = device_get_ivars(ntb); 413 414 return (NTB_MW_GET_WC(device_get_parent(ntb), mw_idx + nc->mwoff, mode)); 415 } 416 417 int 418 ntb_mw_set_wc(device_t ntb, unsigned mw_idx, vm_memattr_t mode) 419 { 420 struct ntb_child *nc = device_get_ivars(ntb); 421 422 return (NTB_MW_SET_WC(device_get_parent(ntb), mw_idx + nc->mwoff, mode)); 423 } 424 425 uint8_t 426 ntb_spad_count(device_t ntb) 427 { 428 struct ntb_child *nc = device_get_ivars(ntb); 429 430 return (nc->spadcnt); 431 } 432 433 void 434 ntb_spad_clear(device_t ntb) 435 { 436 struct ntb_child *nc = device_get_ivars(ntb); 437 unsigned i; 438 439 for (i = 0; i < nc->spadcnt; i++) 440 NTB_SPAD_WRITE(device_get_parent(ntb), i + nc->spadoff, 0); 441 } 442 443 int 444 ntb_spad_write(device_t ntb, unsigned int idx, uint32_t val) 445 { 446 struct ntb_child *nc = device_get_ivars(ntb); 447 448 return (NTB_SPAD_WRITE(device_get_parent(ntb), idx + nc->spadoff, val)); 449 } 450 451 int 452 ntb_spad_read(device_t ntb, unsigned int idx, uint32_t *val) 453 { 454 struct ntb_child *nc = device_get_ivars(ntb); 455 456 return (NTB_SPAD_READ(device_get_parent(ntb), idx + nc->spadoff, val)); 457 } 458 459 int 460 ntb_peer_spad_write(device_t ntb, unsigned int idx, uint32_t val) 461 { 462 struct ntb_child *nc = device_get_ivars(ntb); 463 464 return (NTB_PEER_SPAD_WRITE(device_get_parent(ntb), idx + nc->spadoff, 465 val)); 466 } 467 468 int 469 ntb_peer_spad_read(device_t ntb, unsigned int idx, uint32_t *val) 470 { 471 struct ntb_child *nc = device_get_ivars(ntb); 472 473 return (NTB_PEER_SPAD_READ(device_get_parent(ntb), idx + nc->spadoff, 474 val)); 475 } 476 477 uint64_t 478 ntb_db_valid_mask(device_t ntb) 479 { 480 struct ntb_child *nc = device_get_ivars(ntb); 481 482 return (nc->dbmask); 483 } 484 485 int 486 ntb_db_vector_count(device_t ntb) 487 { 488 489 return (NTB_DB_VECTOR_COUNT(device_get_parent(ntb))); 490 } 491 492 uint64_t 493 ntb_db_vector_mask(device_t ntb, uint32_t vector) 494 { 495 struct ntb_child *nc = device_get_ivars(ntb); 496 497 return ((NTB_DB_VECTOR_MASK(device_get_parent(ntb), vector) 498 >> nc->dboff) & nc->dbmask); 499 } 500 501 int 502 ntb_peer_db_addr(device_t ntb, bus_addr_t *db_addr, vm_size_t *db_size) 503 { 504 505 return (NTB_PEER_DB_ADDR(device_get_parent(ntb), db_addr, db_size)); 506 } 507 508 void 509 ntb_db_clear(device_t ntb, uint64_t bits) 510 { 511 struct ntb_child *nc = device_get_ivars(ntb); 512 513 return (NTB_DB_CLEAR(device_get_parent(ntb), bits << nc->dboff)); 514 } 515 516 void 517 ntb_db_clear_mask(device_t ntb, uint64_t bits) 518 { 519 struct ntb_child *nc = device_get_ivars(ntb); 520 521 return (NTB_DB_CLEAR_MASK(device_get_parent(ntb), bits << nc->dboff)); 522 } 523 524 uint64_t 525 ntb_db_read(device_t ntb) 526 { 527 struct ntb_child *nc = device_get_ivars(ntb); 528 529 return ((NTB_DB_READ(device_get_parent(ntb)) >> nc->dboff) 530 & nc->dbmask); 531 } 532 533 void 534 ntb_db_set_mask(device_t ntb, uint64_t bits) 535 { 536 struct ntb_child *nc = device_get_ivars(ntb); 537 538 return (NTB_DB_SET_MASK(device_get_parent(ntb), bits << nc->dboff)); 539 } 540 541 void 542 ntb_peer_db_set(device_t ntb, uint64_t bits) 543 { 544 struct ntb_child *nc = device_get_ivars(ntb); 545 546 return (NTB_PEER_DB_SET(device_get_parent(ntb), bits << nc->dboff)); 547 } 548 549 MODULE_VERSION(ntb, 1); 550