1 /*- 2 * Copyright (c) 2016-2017 Alexander Motin <mav@FreeBSD.org> 3 * Copyright (C) 2013 Intel Corporation 4 * Copyright (C) 2015 EMC Corporation 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 /* 30 * The Non-Transparent Bridge (NTB) is a device that allows you to connect 31 * two or more systems using a PCI-e links, providing remote memory access. 32 * 33 * This module contains a driver for NTB hardware in Intel Xeon/Atom CPUs. 34 * 35 * NOTE: Much of the code in this module is shared with Linux. Any patches may 36 * be picked up and redistributed in Linux with a dual GPL/BSD license. 37 */ 38 39 #include <sys/cdefs.h> 40 __FBSDID("$FreeBSD$"); 41 42 #include <sys/param.h> 43 #include <sys/kernel.h> 44 #include <sys/systm.h> 45 #include <sys/bus.h> 46 #include <sys/endian.h> 47 #include <sys/interrupt.h> 48 #include <sys/lock.h> 49 #include <sys/malloc.h> 50 #include <sys/module.h> 51 #include <sys/mutex.h> 52 #include <sys/pciio.h> 53 #include <sys/queue.h> 54 #include <sys/rman.h> 55 #include <sys/sbuf.h> 56 #include <sys/sysctl.h> 57 #include <vm/vm.h> 58 #include <vm/pmap.h> 59 #include <machine/bus.h> 60 #include <machine/intr_machdep.h> 61 #include <machine/resource.h> 62 #include <dev/pci/pcireg.h> 63 #include <dev/pci/pcivar.h> 64 65 #include "ntb_hw_intel.h" 66 #include "../ntb.h" 67 68 #define MAX_MSIX_INTERRUPTS MAX(XEON_DB_COUNT, ATOM_DB_COUNT) 69 70 #define NTB_HB_TIMEOUT 1 /* second */ 71 #define ATOM_LINK_RECOVERY_TIME 500 /* ms */ 72 #define BAR_HIGH_MASK (~((1ull << 12) - 1)) 73 74 #define NTB_MSIX_VER_GUARD 0xaabbccdd 75 #define NTB_MSIX_RECEIVED 0xe0f0e0f0 76 77 /* 78 * PCI constants could be somewhere more generic, but aren't defined/used in 79 * pci.c. 80 */ 81 #define PCI_MSIX_ENTRY_SIZE 16 82 #define PCI_MSIX_ENTRY_LOWER_ADDR 0 83 #define PCI_MSIX_ENTRY_UPPER_ADDR 4 84 #define PCI_MSIX_ENTRY_DATA 8 85 86 enum ntb_device_type { 87 NTB_XEON, 88 NTB_ATOM 89 }; 90 91 /* ntb_conn_type are hardware numbers, cannot change. */ 92 enum ntb_conn_type { 93 NTB_CONN_TRANSPARENT = 0, 94 NTB_CONN_B2B = 1, 95 NTB_CONN_RP = 2, 96 }; 97 98 enum ntb_b2b_direction { 99 NTB_DEV_USD = 0, 100 NTB_DEV_DSD = 1, 101 }; 102 103 enum ntb_bar { 104 NTB_CONFIG_BAR = 0, 105 NTB_B2B_BAR_1, 106 NTB_B2B_BAR_2, 107 NTB_B2B_BAR_3, 108 NTB_MAX_BARS 109 }; 110 111 enum { 112 NTB_MSIX_GUARD = 0, 113 NTB_MSIX_DATA0, 114 NTB_MSIX_DATA1, 115 NTB_MSIX_DATA2, 116 NTB_MSIX_OFS0, 117 NTB_MSIX_OFS1, 118 NTB_MSIX_OFS2, 119 NTB_MSIX_DONE, 120 NTB_MAX_MSIX_SPAD 121 }; 122 123 /* Device features and workarounds */ 124 #define HAS_FEATURE(ntb, feature) \ 125 (((ntb)->features & (feature)) != 0) 126 127 struct ntb_hw_info { 128 uint32_t device_id; 129 const char *desc; 130 enum ntb_device_type type; 131 uint32_t features; 132 }; 133 134 struct ntb_pci_bar_info { 135 bus_space_tag_t pci_bus_tag; 136 bus_space_handle_t pci_bus_handle; 137 int pci_resource_id; 138 struct resource *pci_resource; 139 vm_paddr_t pbase; 140 caddr_t vbase; 141 vm_size_t size; 142 vm_memattr_t map_mode; 143 144 /* Configuration register offsets */ 145 uint32_t psz_off; 146 uint32_t ssz_off; 147 uint32_t pbarxlat_off; 148 }; 149 150 struct ntb_int_info { 151 struct resource *res; 152 int rid; 153 void *tag; 154 }; 155 156 struct ntb_vec { 157 struct ntb_softc *ntb; 158 uint32_t num; 159 unsigned masked; 160 }; 161 162 struct ntb_reg { 163 uint32_t ntb_ctl; 164 uint32_t lnk_sta; 165 uint8_t db_size; 166 unsigned mw_bar[NTB_MAX_BARS]; 167 }; 168 169 struct ntb_alt_reg { 170 uint32_t db_bell; 171 uint32_t db_mask; 172 uint32_t spad; 173 }; 174 175 struct ntb_xlat_reg { 176 uint32_t bar0_base; 177 uint32_t bar2_base; 178 uint32_t bar4_base; 179 uint32_t bar5_base; 180 181 uint32_t bar2_xlat; 182 uint32_t bar4_xlat; 183 uint32_t bar5_xlat; 184 185 uint32_t bar2_limit; 186 uint32_t bar4_limit; 187 uint32_t bar5_limit; 188 }; 189 190 struct ntb_b2b_addr { 191 uint64_t bar0_addr; 192 uint64_t bar2_addr64; 193 uint64_t bar4_addr64; 194 uint64_t bar4_addr32; 195 uint64_t bar5_addr32; 196 }; 197 198 struct ntb_msix_data { 199 uint32_t nmd_ofs; 200 uint32_t nmd_data; 201 }; 202 203 struct ntb_softc { 204 /* ntb.c context. Do not move! Must go first! */ 205 void *ntb_store; 206 207 device_t device; 208 enum ntb_device_type type; 209 uint32_t features; 210 211 struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 212 struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 213 uint32_t allocated_interrupts; 214 215 struct ntb_msix_data peer_msix_data[XEON_NONLINK_DB_MSIX_BITS]; 216 struct ntb_msix_data msix_data[XEON_NONLINK_DB_MSIX_BITS]; 217 bool peer_msix_good; 218 bool peer_msix_done; 219 struct ntb_pci_bar_info *peer_lapic_bar; 220 struct callout peer_msix_work; 221 222 struct callout heartbeat_timer; 223 struct callout lr_timer; 224 225 struct ntb_vec *msix_vec; 226 227 uint32_t ppd; 228 enum ntb_conn_type conn_type; 229 enum ntb_b2b_direction dev_type; 230 231 /* Offset of peer bar0 in B2B BAR */ 232 uint64_t b2b_off; 233 /* Memory window used to access peer bar0 */ 234 #define B2B_MW_DISABLED UINT8_MAX 235 uint8_t b2b_mw_idx; 236 uint32_t msix_xlat; 237 uint8_t msix_mw_idx; 238 239 uint8_t mw_count; 240 uint8_t spad_count; 241 uint8_t db_count; 242 uint8_t db_vec_count; 243 uint8_t db_vec_shift; 244 245 /* Protects local db_mask. */ 246 #define DB_MASK_LOCK(sc) mtx_lock_spin(&(sc)->db_mask_lock) 247 #define DB_MASK_UNLOCK(sc) mtx_unlock_spin(&(sc)->db_mask_lock) 248 #define DB_MASK_ASSERT(sc,f) mtx_assert(&(sc)->db_mask_lock, (f)) 249 struct mtx db_mask_lock; 250 251 volatile uint32_t ntb_ctl; 252 volatile uint32_t lnk_sta; 253 254 uint64_t db_valid_mask; 255 uint64_t db_link_mask; 256 uint64_t db_mask; 257 uint64_t fake_db; /* NTB_SB01BASE_LOCKUP*/ 258 uint64_t force_db; /* NTB_SB01BASE_LOCKUP*/ 259 260 int last_ts; /* ticks @ last irq */ 261 262 const struct ntb_reg *reg; 263 const struct ntb_alt_reg *self_reg; 264 const struct ntb_alt_reg *peer_reg; 265 const struct ntb_xlat_reg *xlat_reg; 266 }; 267 268 #ifdef __i386__ 269 static __inline uint64_t 270 bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 271 bus_size_t offset) 272 { 273 274 return (bus_space_read_4(tag, handle, offset) | 275 ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32); 276 } 277 278 static __inline void 279 bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle, 280 bus_size_t offset, uint64_t val) 281 { 282 283 bus_space_write_4(tag, handle, offset, val); 284 bus_space_write_4(tag, handle, offset + 4, val >> 32); 285 } 286 #endif 287 288 #define intel_ntb_bar_read(SIZE, bar, offset) \ 289 bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 290 ntb->bar_info[(bar)].pci_bus_handle, (offset)) 291 #define intel_ntb_bar_write(SIZE, bar, offset, val) \ 292 bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 293 ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) 294 #define intel_ntb_reg_read(SIZE, offset) \ 295 intel_ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) 296 #define intel_ntb_reg_write(SIZE, offset, val) \ 297 intel_ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) 298 #define intel_ntb_mw_read(SIZE, offset) \ 299 intel_ntb_bar_read(SIZE, intel_ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \ 300 offset) 301 #define intel_ntb_mw_write(SIZE, offset, val) \ 302 intel_ntb_bar_write(SIZE, intel_ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \ 303 offset, val) 304 305 static int intel_ntb_probe(device_t device); 306 static int intel_ntb_attach(device_t device); 307 static int intel_ntb_detach(device_t device); 308 static uint64_t intel_ntb_db_valid_mask(device_t dev); 309 static void intel_ntb_spad_clear(device_t dev); 310 static uint64_t intel_ntb_db_vector_mask(device_t dev, uint32_t vector); 311 static bool intel_ntb_link_is_up(device_t dev, enum ntb_speed *speed, 312 enum ntb_width *width); 313 static int intel_ntb_link_enable(device_t dev, enum ntb_speed speed, 314 enum ntb_width width); 315 static int intel_ntb_link_disable(device_t dev); 316 static int intel_ntb_spad_read(device_t dev, unsigned int idx, uint32_t *val); 317 static int intel_ntb_peer_spad_write(device_t dev, unsigned int idx, uint32_t val); 318 319 static unsigned intel_ntb_user_mw_to_idx(struct ntb_softc *, unsigned uidx); 320 static inline enum ntb_bar intel_ntb_mw_to_bar(struct ntb_softc *, unsigned mw); 321 static inline bool bar_is_64bit(struct ntb_softc *, enum ntb_bar); 322 static inline void bar_get_xlat_params(struct ntb_softc *, enum ntb_bar, 323 uint32_t *base, uint32_t *xlat, uint32_t *lmt); 324 static int intel_ntb_map_pci_bars(struct ntb_softc *ntb); 325 static int intel_ntb_mw_set_wc_internal(struct ntb_softc *, unsigned idx, 326 vm_memattr_t); 327 static void print_map_success(struct ntb_softc *, struct ntb_pci_bar_info *, 328 const char *); 329 static int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); 330 static int map_memory_window_bar(struct ntb_softc *ntb, 331 struct ntb_pci_bar_info *bar); 332 static void intel_ntb_unmap_pci_bar(struct ntb_softc *ntb); 333 static int intel_ntb_remap_msix(device_t, uint32_t desired, uint32_t avail); 334 static int intel_ntb_init_isr(struct ntb_softc *ntb); 335 static int intel_ntb_setup_legacy_interrupt(struct ntb_softc *ntb); 336 static int intel_ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors); 337 static void intel_ntb_teardown_interrupts(struct ntb_softc *ntb); 338 static inline uint64_t intel_ntb_vec_mask(struct ntb_softc *, uint64_t db_vector); 339 static void intel_ntb_interrupt(struct ntb_softc *, uint32_t vec); 340 static void ndev_vec_isr(void *arg); 341 static void ndev_irq_isr(void *arg); 342 static inline uint64_t db_ioread(struct ntb_softc *, uint64_t regoff); 343 static inline void db_iowrite(struct ntb_softc *, uint64_t regoff, uint64_t); 344 static inline void db_iowrite_raw(struct ntb_softc *, uint64_t regoff, uint64_t); 345 static int intel_ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors); 346 static void intel_ntb_free_msix_vec(struct ntb_softc *ntb); 347 static void intel_ntb_get_msix_info(struct ntb_softc *ntb); 348 static void intel_ntb_exchange_msix(void *); 349 static struct ntb_hw_info *intel_ntb_get_device_info(uint32_t device_id); 350 static void intel_ntb_detect_max_mw(struct ntb_softc *ntb); 351 static int intel_ntb_detect_xeon(struct ntb_softc *ntb); 352 static int intel_ntb_detect_atom(struct ntb_softc *ntb); 353 static int intel_ntb_xeon_init_dev(struct ntb_softc *ntb); 354 static int intel_ntb_atom_init_dev(struct ntb_softc *ntb); 355 static void intel_ntb_teardown_xeon(struct ntb_softc *ntb); 356 static void configure_atom_secondary_side_bars(struct ntb_softc *ntb); 357 static void xeon_reset_sbar_size(struct ntb_softc *, enum ntb_bar idx, 358 enum ntb_bar regbar); 359 static void xeon_set_sbar_base_and_limit(struct ntb_softc *, 360 uint64_t base_addr, enum ntb_bar idx, enum ntb_bar regbar); 361 static void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr, 362 enum ntb_bar idx); 363 static int xeon_setup_b2b_mw(struct ntb_softc *, 364 const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr); 365 static inline bool link_is_up(struct ntb_softc *ntb); 366 static inline bool _xeon_link_is_up(struct ntb_softc *ntb); 367 static inline bool atom_link_is_err(struct ntb_softc *ntb); 368 static inline enum ntb_speed intel_ntb_link_sta_speed(struct ntb_softc *); 369 static inline enum ntb_width intel_ntb_link_sta_width(struct ntb_softc *); 370 static void atom_link_hb(void *arg); 371 static void recover_atom_link(void *arg); 372 static bool intel_ntb_poll_link(struct ntb_softc *ntb); 373 static void save_bar_parameters(struct ntb_pci_bar_info *bar); 374 static void intel_ntb_sysctl_init(struct ntb_softc *); 375 static int sysctl_handle_features(SYSCTL_HANDLER_ARGS); 376 static int sysctl_handle_link_admin(SYSCTL_HANDLER_ARGS); 377 static int sysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS); 378 static int sysctl_handle_link_status(SYSCTL_HANDLER_ARGS); 379 static int sysctl_handle_register(SYSCTL_HANDLER_ARGS); 380 381 static unsigned g_ntb_hw_debug_level; 382 SYSCTL_UINT(_hw_ntb, OID_AUTO, debug_level, CTLFLAG_RWTUN, 383 &g_ntb_hw_debug_level, 0, "ntb_hw log level -- higher is more verbose"); 384 #define intel_ntb_printf(lvl, ...) do { \ 385 if ((lvl) <= g_ntb_hw_debug_level) { \ 386 device_printf(ntb->device, __VA_ARGS__); \ 387 } \ 388 } while (0) 389 390 #define _NTB_PAT_UC 0 391 #define _NTB_PAT_WC 1 392 #define _NTB_PAT_WT 4 393 #define _NTB_PAT_WP 5 394 #define _NTB_PAT_WB 6 395 #define _NTB_PAT_UCM 7 396 static unsigned g_ntb_mw_pat = _NTB_PAT_UC; 397 SYSCTL_UINT(_hw_ntb, OID_AUTO, default_mw_pat, CTLFLAG_RDTUN, 398 &g_ntb_mw_pat, 0, "Configure the default memory window cache flags (PAT): " 399 "UC: " __XSTRING(_NTB_PAT_UC) ", " 400 "WC: " __XSTRING(_NTB_PAT_WC) ", " 401 "WT: " __XSTRING(_NTB_PAT_WT) ", " 402 "WP: " __XSTRING(_NTB_PAT_WP) ", " 403 "WB: " __XSTRING(_NTB_PAT_WB) ", " 404 "UC-: " __XSTRING(_NTB_PAT_UCM)); 405 406 static inline vm_memattr_t 407 intel_ntb_pat_flags(void) 408 { 409 410 switch (g_ntb_mw_pat) { 411 case _NTB_PAT_WC: 412 return (VM_MEMATTR_WRITE_COMBINING); 413 case _NTB_PAT_WT: 414 return (VM_MEMATTR_WRITE_THROUGH); 415 case _NTB_PAT_WP: 416 return (VM_MEMATTR_WRITE_PROTECTED); 417 case _NTB_PAT_WB: 418 return (VM_MEMATTR_WRITE_BACK); 419 case _NTB_PAT_UCM: 420 return (VM_MEMATTR_WEAK_UNCACHEABLE); 421 case _NTB_PAT_UC: 422 /* FALLTHROUGH */ 423 default: 424 return (VM_MEMATTR_UNCACHEABLE); 425 } 426 } 427 428 /* 429 * Well, this obviously doesn't belong here, but it doesn't seem to exist 430 * anywhere better yet. 431 */ 432 static inline const char * 433 intel_ntb_vm_memattr_to_str(vm_memattr_t pat) 434 { 435 436 switch (pat) { 437 case VM_MEMATTR_WRITE_COMBINING: 438 return ("WRITE_COMBINING"); 439 case VM_MEMATTR_WRITE_THROUGH: 440 return ("WRITE_THROUGH"); 441 case VM_MEMATTR_WRITE_PROTECTED: 442 return ("WRITE_PROTECTED"); 443 case VM_MEMATTR_WRITE_BACK: 444 return ("WRITE_BACK"); 445 case VM_MEMATTR_WEAK_UNCACHEABLE: 446 return ("UNCACHED"); 447 case VM_MEMATTR_UNCACHEABLE: 448 return ("UNCACHEABLE"); 449 default: 450 return ("UNKNOWN"); 451 } 452 } 453 454 static int g_ntb_msix_idx = 1; 455 SYSCTL_INT(_hw_ntb, OID_AUTO, msix_mw_idx, CTLFLAG_RDTUN, &g_ntb_msix_idx, 456 0, "Use this memory window to access the peer MSIX message complex on " 457 "certain Xeon-based NTB systems, as a workaround for a hardware errata. " 458 "Like b2b_mw_idx, negative values index from the last available memory " 459 "window. (Applies on Xeon platforms with SB01BASE_LOCKUP errata.)"); 460 461 static int g_ntb_mw_idx = -1; 462 SYSCTL_INT(_hw_ntb, OID_AUTO, b2b_mw_idx, CTLFLAG_RDTUN, &g_ntb_mw_idx, 463 0, "Use this memory window to access the peer NTB registers. A " 464 "non-negative value starts from the first MW index; a negative value " 465 "starts from the last MW index. The default is -1, i.e., the last " 466 "available memory window. Both sides of the NTB MUST set the same " 467 "value here! (Applies on Xeon platforms with SDOORBELL_LOCKUP errata.)"); 468 469 /* Hardware owns the low 16 bits of features. */ 470 #define NTB_BAR_SIZE_4K (1 << 0) 471 #define NTB_SDOORBELL_LOCKUP (1 << 1) 472 #define NTB_SB01BASE_LOCKUP (1 << 2) 473 #define NTB_B2BDOORBELL_BIT14 (1 << 3) 474 /* Software/configuration owns the top 16 bits. */ 475 #define NTB_SPLIT_BAR (1ull << 16) 476 477 #define NTB_FEATURES_STR \ 478 "\20\21SPLIT_BAR4\04B2B_DOORBELL_BIT14\03SB01BASE_LOCKUP" \ 479 "\02SDOORBELL_LOCKUP\01BAR_SIZE_4K" 480 481 static struct ntb_hw_info pci_ids[] = { 482 /* XXX: PS/SS IDs left out until they are supported. */ 483 { 0x0C4E8086, "BWD Atom Processor S1200 Non-Transparent Bridge B2B", 484 NTB_ATOM, 0 }, 485 486 { 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B", 487 NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 488 { 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B", 489 NTB_XEON, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 490 { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", NTB_XEON, 491 NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 492 NTB_SB01BASE_LOCKUP | NTB_BAR_SIZE_4K }, 493 { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", NTB_XEON, 494 NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 495 NTB_SB01BASE_LOCKUP }, 496 { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", NTB_XEON, 497 NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 498 NTB_SB01BASE_LOCKUP }, 499 }; 500 501 static const struct ntb_reg atom_reg = { 502 .ntb_ctl = ATOM_NTBCNTL_OFFSET, 503 .lnk_sta = ATOM_LINK_STATUS_OFFSET, 504 .db_size = sizeof(uint64_t), 505 .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 }, 506 }; 507 508 static const struct ntb_alt_reg atom_pri_reg = { 509 .db_bell = ATOM_PDOORBELL_OFFSET, 510 .db_mask = ATOM_PDBMSK_OFFSET, 511 .spad = ATOM_SPAD_OFFSET, 512 }; 513 514 static const struct ntb_alt_reg atom_b2b_reg = { 515 .db_bell = ATOM_B2B_DOORBELL_OFFSET, 516 .spad = ATOM_B2B_SPAD_OFFSET, 517 }; 518 519 static const struct ntb_xlat_reg atom_sec_xlat = { 520 #if 0 521 /* "FIXME" says the Linux driver. */ 522 .bar0_base = ATOM_SBAR0BASE_OFFSET, 523 .bar2_base = ATOM_SBAR2BASE_OFFSET, 524 .bar4_base = ATOM_SBAR4BASE_OFFSET, 525 526 .bar2_limit = ATOM_SBAR2LMT_OFFSET, 527 .bar4_limit = ATOM_SBAR4LMT_OFFSET, 528 #endif 529 530 .bar2_xlat = ATOM_SBAR2XLAT_OFFSET, 531 .bar4_xlat = ATOM_SBAR4XLAT_OFFSET, 532 }; 533 534 static const struct ntb_reg xeon_reg = { 535 .ntb_ctl = XEON_NTBCNTL_OFFSET, 536 .lnk_sta = XEON_LINK_STATUS_OFFSET, 537 .db_size = sizeof(uint16_t), 538 .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2, NTB_B2B_BAR_3 }, 539 }; 540 541 static const struct ntb_alt_reg xeon_pri_reg = { 542 .db_bell = XEON_PDOORBELL_OFFSET, 543 .db_mask = XEON_PDBMSK_OFFSET, 544 .spad = XEON_SPAD_OFFSET, 545 }; 546 547 static const struct ntb_alt_reg xeon_b2b_reg = { 548 .db_bell = XEON_B2B_DOORBELL_OFFSET, 549 .spad = XEON_B2B_SPAD_OFFSET, 550 }; 551 552 static const struct ntb_xlat_reg xeon_sec_xlat = { 553 .bar0_base = XEON_SBAR0BASE_OFFSET, 554 .bar2_base = XEON_SBAR2BASE_OFFSET, 555 .bar4_base = XEON_SBAR4BASE_OFFSET, 556 .bar5_base = XEON_SBAR5BASE_OFFSET, 557 558 .bar2_limit = XEON_SBAR2LMT_OFFSET, 559 .bar4_limit = XEON_SBAR4LMT_OFFSET, 560 .bar5_limit = XEON_SBAR5LMT_OFFSET, 561 562 .bar2_xlat = XEON_SBAR2XLAT_OFFSET, 563 .bar4_xlat = XEON_SBAR4XLAT_OFFSET, 564 .bar5_xlat = XEON_SBAR5XLAT_OFFSET, 565 }; 566 567 static struct ntb_b2b_addr xeon_b2b_usd_addr = { 568 .bar0_addr = XEON_B2B_BAR0_ADDR, 569 .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 570 .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 571 .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 572 .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 573 }; 574 575 static struct ntb_b2b_addr xeon_b2b_dsd_addr = { 576 .bar0_addr = XEON_B2B_BAR0_ADDR, 577 .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 578 .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 579 .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 580 .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 581 }; 582 583 SYSCTL_NODE(_hw_ntb, OID_AUTO, xeon_b2b, CTLFLAG_RW, 0, 584 "B2B MW segment overrides -- MUST be the same on both sides"); 585 586 SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar2_addr64, CTLFLAG_RDTUN, 587 &xeon_b2b_usd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 588 "hardware, use this 64-bit address on the bus between the NTB devices for " 589 "the window at BAR2, on the upstream side of the link. MUST be the same " 590 "address on both sides."); 591 SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr64, CTLFLAG_RDTUN, 592 &xeon_b2b_usd_addr.bar4_addr64, 0, "See usd_bar2_addr64, but BAR4."); 593 SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr32, CTLFLAG_RDTUN, 594 &xeon_b2b_usd_addr.bar4_addr32, 0, "See usd_bar2_addr64, but BAR4 " 595 "(split-BAR mode)."); 596 SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar5_addr32, CTLFLAG_RDTUN, 597 &xeon_b2b_usd_addr.bar5_addr32, 0, "See usd_bar2_addr64, but BAR5 " 598 "(split-BAR mode)."); 599 600 SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar2_addr64, CTLFLAG_RDTUN, 601 &xeon_b2b_dsd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 602 "hardware, use this 64-bit address on the bus between the NTB devices for " 603 "the window at BAR2, on the downstream side of the link. MUST be the same" 604 " address on both sides."); 605 SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr64, CTLFLAG_RDTUN, 606 &xeon_b2b_dsd_addr.bar4_addr64, 0, "See dsd_bar2_addr64, but BAR4."); 607 SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr32, CTLFLAG_RDTUN, 608 &xeon_b2b_dsd_addr.bar4_addr32, 0, "See dsd_bar2_addr64, but BAR4 " 609 "(split-BAR mode)."); 610 SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar5_addr32, CTLFLAG_RDTUN, 611 &xeon_b2b_dsd_addr.bar5_addr32, 0, "See dsd_bar2_addr64, but BAR5 " 612 "(split-BAR mode)."); 613 614 /* 615 * OS <-> Driver interface structures 616 */ 617 MALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 618 619 /* 620 * OS <-> Driver linkage functions 621 */ 622 static int 623 intel_ntb_probe(device_t device) 624 { 625 struct ntb_hw_info *p; 626 627 p = intel_ntb_get_device_info(pci_get_devid(device)); 628 if (p == NULL) 629 return (ENXIO); 630 631 device_set_desc(device, p->desc); 632 return (0); 633 } 634 635 static int 636 intel_ntb_attach(device_t device) 637 { 638 struct ntb_softc *ntb; 639 struct ntb_hw_info *p; 640 int error; 641 642 ntb = device_get_softc(device); 643 p = intel_ntb_get_device_info(pci_get_devid(device)); 644 645 ntb->device = device; 646 ntb->type = p->type; 647 ntb->features = p->features; 648 ntb->b2b_mw_idx = B2B_MW_DISABLED; 649 ntb->msix_mw_idx = B2B_MW_DISABLED; 650 651 /* Heartbeat timer for NTB_ATOM since there is no link interrupt */ 652 callout_init(&ntb->heartbeat_timer, 1); 653 callout_init(&ntb->lr_timer, 1); 654 callout_init(&ntb->peer_msix_work, 1); 655 mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN); 656 657 if (ntb->type == NTB_ATOM) 658 error = intel_ntb_detect_atom(ntb); 659 else 660 error = intel_ntb_detect_xeon(ntb); 661 if (error != 0) 662 goto out; 663 664 intel_ntb_detect_max_mw(ntb); 665 666 pci_enable_busmaster(ntb->device); 667 668 error = intel_ntb_map_pci_bars(ntb); 669 if (error != 0) 670 goto out; 671 if (ntb->type == NTB_ATOM) 672 error = intel_ntb_atom_init_dev(ntb); 673 else 674 error = intel_ntb_xeon_init_dev(ntb); 675 if (error != 0) 676 goto out; 677 678 intel_ntb_spad_clear(device); 679 680 intel_ntb_poll_link(ntb); 681 682 intel_ntb_sysctl_init(ntb); 683 684 /* Attach children to this controller */ 685 error = ntb_register_device(device); 686 687 out: 688 if (error != 0) 689 intel_ntb_detach(device); 690 return (error); 691 } 692 693 static int 694 intel_ntb_detach(device_t device) 695 { 696 struct ntb_softc *ntb; 697 698 ntb = device_get_softc(device); 699 700 /* Detach & delete all children */ 701 ntb_unregister_device(device); 702 703 if (ntb->self_reg != NULL) { 704 DB_MASK_LOCK(ntb); 705 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_valid_mask); 706 DB_MASK_UNLOCK(ntb); 707 } 708 callout_drain(&ntb->heartbeat_timer); 709 callout_drain(&ntb->lr_timer); 710 callout_drain(&ntb->peer_msix_work); 711 pci_disable_busmaster(ntb->device); 712 if (ntb->type == NTB_XEON) 713 intel_ntb_teardown_xeon(ntb); 714 intel_ntb_teardown_interrupts(ntb); 715 716 mtx_destroy(&ntb->db_mask_lock); 717 718 intel_ntb_unmap_pci_bar(ntb); 719 720 return (0); 721 } 722 723 /* 724 * Driver internal routines 725 */ 726 static inline enum ntb_bar 727 intel_ntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw) 728 { 729 730 KASSERT(mw < ntb->mw_count, 731 ("%s: mw:%u > count:%u", __func__, mw, (unsigned)ntb->mw_count)); 732 KASSERT(ntb->reg->mw_bar[mw] != 0, ("invalid mw")); 733 734 return (ntb->reg->mw_bar[mw]); 735 } 736 737 static inline bool 738 bar_is_64bit(struct ntb_softc *ntb, enum ntb_bar bar) 739 { 740 /* XXX This assertion could be stronger. */ 741 KASSERT(bar < NTB_MAX_BARS, ("bogus bar")); 742 return (bar < NTB_B2B_BAR_2 || !HAS_FEATURE(ntb, NTB_SPLIT_BAR)); 743 } 744 745 static inline void 746 bar_get_xlat_params(struct ntb_softc *ntb, enum ntb_bar bar, uint32_t *base, 747 uint32_t *xlat, uint32_t *lmt) 748 { 749 uint32_t basev, lmtv, xlatv; 750 751 switch (bar) { 752 case NTB_B2B_BAR_1: 753 basev = ntb->xlat_reg->bar2_base; 754 lmtv = ntb->xlat_reg->bar2_limit; 755 xlatv = ntb->xlat_reg->bar2_xlat; 756 break; 757 case NTB_B2B_BAR_2: 758 basev = ntb->xlat_reg->bar4_base; 759 lmtv = ntb->xlat_reg->bar4_limit; 760 xlatv = ntb->xlat_reg->bar4_xlat; 761 break; 762 case NTB_B2B_BAR_3: 763 basev = ntb->xlat_reg->bar5_base; 764 lmtv = ntb->xlat_reg->bar5_limit; 765 xlatv = ntb->xlat_reg->bar5_xlat; 766 break; 767 default: 768 KASSERT(bar >= NTB_B2B_BAR_1 && bar < NTB_MAX_BARS, 769 ("bad bar")); 770 basev = lmtv = xlatv = 0; 771 break; 772 } 773 774 if (base != NULL) 775 *base = basev; 776 if (xlat != NULL) 777 *xlat = xlatv; 778 if (lmt != NULL) 779 *lmt = lmtv; 780 } 781 782 static int 783 intel_ntb_map_pci_bars(struct ntb_softc *ntb) 784 { 785 struct ntb_pci_bar_info *bar; 786 int rc; 787 788 bar = &ntb->bar_info[NTB_CONFIG_BAR]; 789 bar->pci_resource_id = PCIR_BAR(0); 790 rc = map_mmr_bar(ntb, bar); 791 if (rc != 0) 792 goto out; 793 794 bar = &ntb->bar_info[NTB_B2B_BAR_1]; 795 bar->pci_resource_id = PCIR_BAR(2); 796 rc = map_memory_window_bar(ntb, bar); 797 if (rc != 0) 798 goto out; 799 bar->psz_off = XEON_PBAR23SZ_OFFSET; 800 bar->ssz_off = XEON_SBAR23SZ_OFFSET; 801 bar->pbarxlat_off = XEON_PBAR2XLAT_OFFSET; 802 803 bar = &ntb->bar_info[NTB_B2B_BAR_2]; 804 bar->pci_resource_id = PCIR_BAR(4); 805 rc = map_memory_window_bar(ntb, bar); 806 if (rc != 0) 807 goto out; 808 bar->psz_off = XEON_PBAR4SZ_OFFSET; 809 bar->ssz_off = XEON_SBAR4SZ_OFFSET; 810 bar->pbarxlat_off = XEON_PBAR4XLAT_OFFSET; 811 812 if (!HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 813 goto out; 814 815 bar = &ntb->bar_info[NTB_B2B_BAR_3]; 816 bar->pci_resource_id = PCIR_BAR(5); 817 rc = map_memory_window_bar(ntb, bar); 818 bar->psz_off = XEON_PBAR5SZ_OFFSET; 819 bar->ssz_off = XEON_SBAR5SZ_OFFSET; 820 bar->pbarxlat_off = XEON_PBAR5XLAT_OFFSET; 821 822 out: 823 if (rc != 0) 824 device_printf(ntb->device, 825 "unable to allocate pci resource\n"); 826 return (rc); 827 } 828 829 static void 830 print_map_success(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar, 831 const char *kind) 832 { 833 834 device_printf(ntb->device, 835 "Mapped BAR%d v:[%p-%p] p:[%p-%p] (0x%jx bytes) (%s)\n", 836 PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 837 (char *)bar->vbase + bar->size - 1, 838 (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 839 (uintmax_t)bar->size, kind); 840 } 841 842 static int 843 map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 844 { 845 846 bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 847 &bar->pci_resource_id, RF_ACTIVE); 848 if (bar->pci_resource == NULL) 849 return (ENXIO); 850 851 save_bar_parameters(bar); 852 bar->map_mode = VM_MEMATTR_UNCACHEABLE; 853 print_map_success(ntb, bar, "mmr"); 854 return (0); 855 } 856 857 static int 858 map_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 859 { 860 int rc; 861 vm_memattr_t mapmode; 862 uint8_t bar_size_bits = 0; 863 864 bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 865 &bar->pci_resource_id, RF_ACTIVE); 866 867 if (bar->pci_resource == NULL) 868 return (ENXIO); 869 870 save_bar_parameters(bar); 871 /* 872 * Ivytown NTB BAR sizes are misreported by the hardware due to a 873 * hardware issue. To work around this, query the size it should be 874 * configured to by the device and modify the resource to correspond to 875 * this new size. The BIOS on systems with this problem is required to 876 * provide enough address space to allow the driver to make this change 877 * safely. 878 * 879 * Ideally I could have just specified the size when I allocated the 880 * resource like: 881 * bus_alloc_resource(ntb->device, 882 * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul, 883 * 1ul << bar_size_bits, RF_ACTIVE); 884 * but the PCI driver does not honor the size in this call, so we have 885 * to modify it after the fact. 886 */ 887 if (HAS_FEATURE(ntb, NTB_BAR_SIZE_4K)) { 888 if (bar->pci_resource_id == PCIR_BAR(2)) 889 bar_size_bits = pci_read_config(ntb->device, 890 XEON_PBAR23SZ_OFFSET, 1); 891 else 892 bar_size_bits = pci_read_config(ntb->device, 893 XEON_PBAR45SZ_OFFSET, 1); 894 895 rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 896 bar->pci_resource, bar->pbase, 897 bar->pbase + (1ul << bar_size_bits) - 1); 898 if (rc != 0) { 899 device_printf(ntb->device, 900 "unable to resize bar\n"); 901 return (rc); 902 } 903 904 save_bar_parameters(bar); 905 } 906 907 bar->map_mode = VM_MEMATTR_UNCACHEABLE; 908 print_map_success(ntb, bar, "mw"); 909 910 /* 911 * Optionally, mark MW BARs as anything other than UC to improve 912 * performance. 913 */ 914 mapmode = intel_ntb_pat_flags(); 915 if (mapmode == bar->map_mode) 916 return (0); 917 918 rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mapmode); 919 if (rc == 0) { 920 bar->map_mode = mapmode; 921 device_printf(ntb->device, 922 "Marked BAR%d v:[%p-%p] p:[%p-%p] as " 923 "%s.\n", 924 PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 925 (char *)bar->vbase + bar->size - 1, 926 (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 927 intel_ntb_vm_memattr_to_str(mapmode)); 928 } else 929 device_printf(ntb->device, 930 "Unable to mark BAR%d v:[%p-%p] p:[%p-%p] as " 931 "%s: %d\n", 932 PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 933 (char *)bar->vbase + bar->size - 1, 934 (void *)bar->pbase, (void *)(bar->pbase + bar->size - 1), 935 intel_ntb_vm_memattr_to_str(mapmode), rc); 936 /* Proceed anyway */ 937 return (0); 938 } 939 940 static void 941 intel_ntb_unmap_pci_bar(struct ntb_softc *ntb) 942 { 943 struct ntb_pci_bar_info *bar; 944 int i; 945 946 for (i = 0; i < NTB_MAX_BARS; i++) { 947 bar = &ntb->bar_info[i]; 948 if (bar->pci_resource != NULL) 949 bus_release_resource(ntb->device, SYS_RES_MEMORY, 950 bar->pci_resource_id, bar->pci_resource); 951 } 952 } 953 954 static int 955 intel_ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors) 956 { 957 uint32_t i; 958 int rc; 959 960 for (i = 0; i < num_vectors; i++) { 961 ntb->int_info[i].rid = i + 1; 962 ntb->int_info[i].res = bus_alloc_resource_any(ntb->device, 963 SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE); 964 if (ntb->int_info[i].res == NULL) { 965 device_printf(ntb->device, 966 "bus_alloc_resource failed\n"); 967 return (ENOMEM); 968 } 969 ntb->int_info[i].tag = NULL; 970 ntb->allocated_interrupts++; 971 rc = bus_setup_intr(ntb->device, ntb->int_info[i].res, 972 INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_vec_isr, 973 &ntb->msix_vec[i], &ntb->int_info[i].tag); 974 if (rc != 0) { 975 device_printf(ntb->device, "bus_setup_intr failed\n"); 976 return (ENXIO); 977 } 978 } 979 return (0); 980 } 981 982 /* 983 * The Linux NTB driver drops from MSI-X to legacy INTx if a unique vector 984 * cannot be allocated for each MSI-X message. JHB seems to think remapping 985 * should be okay. This tunable should enable us to test that hypothesis 986 * when someone gets their hands on some Xeon hardware. 987 */ 988 static int ntb_force_remap_mode; 989 SYSCTL_INT(_hw_ntb, OID_AUTO, force_remap_mode, CTLFLAG_RDTUN, 990 &ntb_force_remap_mode, 0, "If enabled, force MSI-X messages to be remapped" 991 " to a smaller number of ithreads, even if the desired number are " 992 "available"); 993 994 /* 995 * In case it is NOT ok, give consumers an abort button. 996 */ 997 static int ntb_prefer_intx; 998 SYSCTL_INT(_hw_ntb, OID_AUTO, prefer_intx_to_remap, CTLFLAG_RDTUN, 999 &ntb_prefer_intx, 0, "If enabled, prefer to use legacy INTx mode rather " 1000 "than remapping MSI-X messages over available slots (match Linux driver " 1001 "behavior)"); 1002 1003 /* 1004 * Remap the desired number of MSI-X messages to available ithreads in a simple 1005 * round-robin fashion. 1006 */ 1007 static int 1008 intel_ntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail) 1009 { 1010 u_int *vectors; 1011 uint32_t i; 1012 int rc; 1013 1014 if (ntb_prefer_intx != 0) 1015 return (ENXIO); 1016 1017 vectors = malloc(desired * sizeof(*vectors), M_NTB, M_ZERO | M_WAITOK); 1018 1019 for (i = 0; i < desired; i++) 1020 vectors[i] = (i % avail) + 1; 1021 1022 rc = pci_remap_msix(dev, desired, vectors); 1023 free(vectors, M_NTB); 1024 return (rc); 1025 } 1026 1027 static int 1028 intel_ntb_init_isr(struct ntb_softc *ntb) 1029 { 1030 uint32_t desired_vectors, num_vectors; 1031 int rc; 1032 1033 ntb->allocated_interrupts = 0; 1034 ntb->last_ts = ticks; 1035 1036 /* 1037 * Mask all doorbell interrupts. (Except link events!) 1038 */ 1039 DB_MASK_LOCK(ntb); 1040 ntb->db_mask = ntb->db_valid_mask; 1041 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1042 DB_MASK_UNLOCK(ntb); 1043 1044 num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device), 1045 ntb->db_count); 1046 if (desired_vectors >= 1) { 1047 rc = pci_alloc_msix(ntb->device, &num_vectors); 1048 1049 if (ntb_force_remap_mode != 0 && rc == 0 && 1050 num_vectors == desired_vectors) 1051 num_vectors--; 1052 1053 if (rc == 0 && num_vectors < desired_vectors) { 1054 rc = intel_ntb_remap_msix(ntb->device, desired_vectors, 1055 num_vectors); 1056 if (rc == 0) 1057 num_vectors = desired_vectors; 1058 else 1059 pci_release_msi(ntb->device); 1060 } 1061 if (rc != 0) 1062 num_vectors = 1; 1063 } else 1064 num_vectors = 1; 1065 1066 if (ntb->type == NTB_XEON && num_vectors < ntb->db_vec_count) { 1067 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1068 device_printf(ntb->device, 1069 "Errata workaround does not support MSI or INTX\n"); 1070 return (EINVAL); 1071 } 1072 1073 ntb->db_vec_count = 1; 1074 ntb->db_vec_shift = XEON_DB_TOTAL_SHIFT; 1075 rc = intel_ntb_setup_legacy_interrupt(ntb); 1076 } else { 1077 if (num_vectors - 1 != XEON_NONLINK_DB_MSIX_BITS && 1078 HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1079 device_printf(ntb->device, 1080 "Errata workaround expects %d doorbell bits\n", 1081 XEON_NONLINK_DB_MSIX_BITS); 1082 return (EINVAL); 1083 } 1084 1085 intel_ntb_create_msix_vec(ntb, num_vectors); 1086 rc = intel_ntb_setup_msix(ntb, num_vectors); 1087 } 1088 if (rc != 0) { 1089 device_printf(ntb->device, 1090 "Error allocating interrupts: %d\n", rc); 1091 intel_ntb_free_msix_vec(ntb); 1092 } 1093 1094 return (rc); 1095 } 1096 1097 static int 1098 intel_ntb_setup_legacy_interrupt(struct ntb_softc *ntb) 1099 { 1100 int rc; 1101 1102 ntb->int_info[0].rid = 0; 1103 ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ, 1104 &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 1105 if (ntb->int_info[0].res == NULL) { 1106 device_printf(ntb->device, "bus_alloc_resource failed\n"); 1107 return (ENOMEM); 1108 } 1109 1110 ntb->int_info[0].tag = NULL; 1111 ntb->allocated_interrupts = 1; 1112 1113 rc = bus_setup_intr(ntb->device, ntb->int_info[0].res, 1114 INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_irq_isr, 1115 ntb, &ntb->int_info[0].tag); 1116 if (rc != 0) { 1117 device_printf(ntb->device, "bus_setup_intr failed\n"); 1118 return (ENXIO); 1119 } 1120 1121 return (0); 1122 } 1123 1124 static void 1125 intel_ntb_teardown_interrupts(struct ntb_softc *ntb) 1126 { 1127 struct ntb_int_info *current_int; 1128 int i; 1129 1130 for (i = 0; i < ntb->allocated_interrupts; i++) { 1131 current_int = &ntb->int_info[i]; 1132 if (current_int->tag != NULL) 1133 bus_teardown_intr(ntb->device, current_int->res, 1134 current_int->tag); 1135 1136 if (current_int->res != NULL) 1137 bus_release_resource(ntb->device, SYS_RES_IRQ, 1138 rman_get_rid(current_int->res), current_int->res); 1139 } 1140 1141 intel_ntb_free_msix_vec(ntb); 1142 pci_release_msi(ntb->device); 1143 } 1144 1145 /* 1146 * Doorbell register and mask are 64-bit on Atom, 16-bit on Xeon. Abstract it 1147 * out to make code clearer. 1148 */ 1149 static inline uint64_t 1150 db_ioread(struct ntb_softc *ntb, uint64_t regoff) 1151 { 1152 1153 if (ntb->type == NTB_ATOM) 1154 return (intel_ntb_reg_read(8, regoff)); 1155 1156 KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 1157 1158 return (intel_ntb_reg_read(2, regoff)); 1159 } 1160 1161 static inline void 1162 db_iowrite(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1163 { 1164 1165 KASSERT((val & ~ntb->db_valid_mask) == 0, 1166 ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1167 (uintmax_t)(val & ~ntb->db_valid_mask), 1168 (uintmax_t)ntb->db_valid_mask)); 1169 1170 if (regoff == ntb->self_reg->db_mask) 1171 DB_MASK_ASSERT(ntb, MA_OWNED); 1172 db_iowrite_raw(ntb, regoff, val); 1173 } 1174 1175 static inline void 1176 db_iowrite_raw(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1177 { 1178 1179 if (ntb->type == NTB_ATOM) { 1180 intel_ntb_reg_write(8, regoff, val); 1181 return; 1182 } 1183 1184 KASSERT(ntb->type == NTB_XEON, ("bad ntb type")); 1185 intel_ntb_reg_write(2, regoff, (uint16_t)val); 1186 } 1187 1188 static void 1189 intel_ntb_db_set_mask(device_t dev, uint64_t bits) 1190 { 1191 struct ntb_softc *ntb = device_get_softc(dev); 1192 1193 DB_MASK_LOCK(ntb); 1194 ntb->db_mask |= bits; 1195 if (!HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) 1196 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1197 DB_MASK_UNLOCK(ntb); 1198 } 1199 1200 static void 1201 intel_ntb_db_clear_mask(device_t dev, uint64_t bits) 1202 { 1203 struct ntb_softc *ntb = device_get_softc(dev); 1204 uint64_t ibits; 1205 int i; 1206 1207 KASSERT((bits & ~ntb->db_valid_mask) == 0, 1208 ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1209 (uintmax_t)(bits & ~ntb->db_valid_mask), 1210 (uintmax_t)ntb->db_valid_mask)); 1211 1212 DB_MASK_LOCK(ntb); 1213 ibits = ntb->fake_db & ntb->db_mask & bits; 1214 ntb->db_mask &= ~bits; 1215 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1216 /* Simulate fake interrupts if unmasked DB bits are set. */ 1217 ntb->force_db |= ibits; 1218 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1219 if ((ibits & intel_ntb_db_vector_mask(dev, i)) != 0) 1220 swi_sched(ntb->int_info[i].tag, 0); 1221 } 1222 } else { 1223 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1224 } 1225 DB_MASK_UNLOCK(ntb); 1226 } 1227 1228 static uint64_t 1229 intel_ntb_db_read(device_t dev) 1230 { 1231 struct ntb_softc *ntb = device_get_softc(dev); 1232 1233 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) 1234 return (ntb->fake_db); 1235 1236 return (db_ioread(ntb, ntb->self_reg->db_bell)); 1237 } 1238 1239 static void 1240 intel_ntb_db_clear(device_t dev, uint64_t bits) 1241 { 1242 struct ntb_softc *ntb = device_get_softc(dev); 1243 1244 KASSERT((bits & ~ntb->db_valid_mask) == 0, 1245 ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1246 (uintmax_t)(bits & ~ntb->db_valid_mask), 1247 (uintmax_t)ntb->db_valid_mask)); 1248 1249 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1250 DB_MASK_LOCK(ntb); 1251 ntb->fake_db &= ~bits; 1252 DB_MASK_UNLOCK(ntb); 1253 return; 1254 } 1255 1256 db_iowrite(ntb, ntb->self_reg->db_bell, bits); 1257 } 1258 1259 static inline uint64_t 1260 intel_ntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector) 1261 { 1262 uint64_t shift, mask; 1263 1264 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1265 /* 1266 * Remap vectors in custom way to make at least first 1267 * three doorbells to not generate stray events. 1268 * This breaks Linux compatibility (if one existed) 1269 * when more then one DB is used (not by if_ntb). 1270 */ 1271 if (db_vector < XEON_NONLINK_DB_MSIX_BITS - 1) 1272 return (1 << db_vector); 1273 if (db_vector == XEON_NONLINK_DB_MSIX_BITS - 1) 1274 return (0x7ffc); 1275 } 1276 1277 shift = ntb->db_vec_shift; 1278 mask = (1ull << shift) - 1; 1279 return (mask << (shift * db_vector)); 1280 } 1281 1282 static void 1283 intel_ntb_interrupt(struct ntb_softc *ntb, uint32_t vec) 1284 { 1285 uint64_t vec_mask; 1286 1287 ntb->last_ts = ticks; 1288 vec_mask = intel_ntb_vec_mask(ntb, vec); 1289 1290 if ((vec_mask & ntb->db_link_mask) != 0) { 1291 if (intel_ntb_poll_link(ntb)) 1292 ntb_link_event(ntb->device); 1293 } 1294 1295 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP) && 1296 (vec_mask & ntb->db_link_mask) == 0) { 1297 DB_MASK_LOCK(ntb); 1298 1299 /* 1300 * Do not report same DB events again if not cleared yet, 1301 * unless the mask was just cleared for them and this 1302 * interrupt handler call can be the consequence of it. 1303 */ 1304 vec_mask &= ~ntb->fake_db | ntb->force_db; 1305 ntb->force_db &= ~vec_mask; 1306 1307 /* Update our internal doorbell register. */ 1308 ntb->fake_db |= vec_mask; 1309 1310 /* Do not report masked DB events. */ 1311 vec_mask &= ~ntb->db_mask; 1312 1313 DB_MASK_UNLOCK(ntb); 1314 } 1315 1316 if ((vec_mask & ntb->db_valid_mask) != 0) 1317 ntb_db_event(ntb->device, vec); 1318 } 1319 1320 static void 1321 ndev_vec_isr(void *arg) 1322 { 1323 struct ntb_vec *nvec = arg; 1324 1325 intel_ntb_interrupt(nvec->ntb, nvec->num); 1326 } 1327 1328 static void 1329 ndev_irq_isr(void *arg) 1330 { 1331 /* If we couldn't set up MSI-X, we only have the one vector. */ 1332 intel_ntb_interrupt(arg, 0); 1333 } 1334 1335 static int 1336 intel_ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors) 1337 { 1338 uint32_t i; 1339 1340 ntb->msix_vec = malloc(num_vectors * sizeof(*ntb->msix_vec), M_NTB, 1341 M_ZERO | M_WAITOK); 1342 for (i = 0; i < num_vectors; i++) { 1343 ntb->msix_vec[i].num = i; 1344 ntb->msix_vec[i].ntb = ntb; 1345 } 1346 1347 return (0); 1348 } 1349 1350 static void 1351 intel_ntb_free_msix_vec(struct ntb_softc *ntb) 1352 { 1353 1354 if (ntb->msix_vec == NULL) 1355 return; 1356 1357 free(ntb->msix_vec, M_NTB); 1358 ntb->msix_vec = NULL; 1359 } 1360 1361 static void 1362 intel_ntb_get_msix_info(struct ntb_softc *ntb) 1363 { 1364 struct pci_devinfo *dinfo; 1365 struct pcicfg_msix *msix; 1366 uint32_t laddr, data, i, offset; 1367 1368 dinfo = device_get_ivars(ntb->device); 1369 msix = &dinfo->cfg.msix; 1370 1371 CTASSERT(XEON_NONLINK_DB_MSIX_BITS == nitems(ntb->msix_data)); 1372 1373 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1374 offset = msix->msix_table_offset + i * PCI_MSIX_ENTRY_SIZE; 1375 1376 laddr = bus_read_4(msix->msix_table_res, offset + 1377 PCI_MSIX_ENTRY_LOWER_ADDR); 1378 intel_ntb_printf(2, "local MSIX addr(%u): 0x%x\n", i, laddr); 1379 1380 KASSERT((laddr & MSI_INTEL_ADDR_BASE) == MSI_INTEL_ADDR_BASE, 1381 ("local MSIX addr 0x%x not in MSI base 0x%x", laddr, 1382 MSI_INTEL_ADDR_BASE)); 1383 ntb->msix_data[i].nmd_ofs = laddr; 1384 1385 data = bus_read_4(msix->msix_table_res, offset + 1386 PCI_MSIX_ENTRY_DATA); 1387 intel_ntb_printf(2, "local MSIX data(%u): 0x%x\n", i, data); 1388 1389 ntb->msix_data[i].nmd_data = data; 1390 } 1391 } 1392 1393 static struct ntb_hw_info * 1394 intel_ntb_get_device_info(uint32_t device_id) 1395 { 1396 struct ntb_hw_info *ep; 1397 1398 for (ep = pci_ids; ep < &pci_ids[nitems(pci_ids)]; ep++) { 1399 if (ep->device_id == device_id) 1400 return (ep); 1401 } 1402 return (NULL); 1403 } 1404 1405 static void 1406 intel_ntb_teardown_xeon(struct ntb_softc *ntb) 1407 { 1408 1409 if (ntb->reg != NULL) 1410 intel_ntb_link_disable(ntb->device); 1411 } 1412 1413 static void 1414 intel_ntb_detect_max_mw(struct ntb_softc *ntb) 1415 { 1416 1417 if (ntb->type == NTB_ATOM) { 1418 ntb->mw_count = ATOM_MW_COUNT; 1419 return; 1420 } 1421 1422 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 1423 ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT; 1424 else 1425 ntb->mw_count = XEON_SNB_MW_COUNT; 1426 } 1427 1428 static int 1429 intel_ntb_detect_xeon(struct ntb_softc *ntb) 1430 { 1431 uint8_t ppd, conn_type; 1432 1433 ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 1434 ntb->ppd = ppd; 1435 1436 if ((ppd & XEON_PPD_DEV_TYPE) != 0) 1437 ntb->dev_type = NTB_DEV_DSD; 1438 else 1439 ntb->dev_type = NTB_DEV_USD; 1440 1441 if ((ppd & XEON_PPD_SPLIT_BAR) != 0) 1442 ntb->features |= NTB_SPLIT_BAR; 1443 1444 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP) && 1445 !HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 1446 device_printf(ntb->device, 1447 "Can not apply SB01BASE_LOCKUP workaround " 1448 "with split BARs disabled!\n"); 1449 device_printf(ntb->device, 1450 "Expect system hangs under heavy NTB traffic!\n"); 1451 ntb->features &= ~NTB_SB01BASE_LOCKUP; 1452 } 1453 1454 /* 1455 * SDOORBELL errata workaround gets in the way of SB01BASE_LOCKUP 1456 * errata workaround; only do one at a time. 1457 */ 1458 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) 1459 ntb->features &= ~NTB_SDOORBELL_LOCKUP; 1460 1461 conn_type = ppd & XEON_PPD_CONN_TYPE; 1462 switch (conn_type) { 1463 case NTB_CONN_B2B: 1464 ntb->conn_type = conn_type; 1465 break; 1466 case NTB_CONN_RP: 1467 case NTB_CONN_TRANSPARENT: 1468 default: 1469 device_printf(ntb->device, "Unsupported connection type: %u\n", 1470 (unsigned)conn_type); 1471 return (ENXIO); 1472 } 1473 return (0); 1474 } 1475 1476 static int 1477 intel_ntb_detect_atom(struct ntb_softc *ntb) 1478 { 1479 uint32_t ppd, conn_type; 1480 1481 ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 1482 ntb->ppd = ppd; 1483 1484 if ((ppd & ATOM_PPD_DEV_TYPE) != 0) 1485 ntb->dev_type = NTB_DEV_DSD; 1486 else 1487 ntb->dev_type = NTB_DEV_USD; 1488 1489 conn_type = (ppd & ATOM_PPD_CONN_TYPE) >> 8; 1490 switch (conn_type) { 1491 case NTB_CONN_B2B: 1492 ntb->conn_type = conn_type; 1493 break; 1494 default: 1495 device_printf(ntb->device, "Unsupported NTB configuration\n"); 1496 return (ENXIO); 1497 } 1498 return (0); 1499 } 1500 1501 static int 1502 intel_ntb_xeon_init_dev(struct ntb_softc *ntb) 1503 { 1504 int rc; 1505 1506 ntb->spad_count = XEON_SPAD_COUNT; 1507 ntb->db_count = XEON_DB_COUNT; 1508 ntb->db_link_mask = XEON_DB_LINK_BIT; 1509 ntb->db_vec_count = XEON_DB_MSIX_VECTOR_COUNT; 1510 ntb->db_vec_shift = XEON_DB_MSIX_VECTOR_SHIFT; 1511 1512 if (ntb->conn_type != NTB_CONN_B2B) { 1513 device_printf(ntb->device, "Connection type %d not supported\n", 1514 ntb->conn_type); 1515 return (ENXIO); 1516 } 1517 1518 ntb->reg = &xeon_reg; 1519 ntb->self_reg = &xeon_pri_reg; 1520 ntb->peer_reg = &xeon_b2b_reg; 1521 ntb->xlat_reg = &xeon_sec_xlat; 1522 1523 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1524 ntb->force_db = ntb->fake_db = 0; 1525 ntb->msix_mw_idx = (ntb->mw_count + g_ntb_msix_idx) % 1526 ntb->mw_count; 1527 intel_ntb_printf(2, "Setting up MSIX mw idx %d means %u\n", 1528 g_ntb_msix_idx, ntb->msix_mw_idx); 1529 rc = intel_ntb_mw_set_wc_internal(ntb, ntb->msix_mw_idx, 1530 VM_MEMATTR_UNCACHEABLE); 1531 KASSERT(rc == 0, ("shouldn't fail")); 1532 } else if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) { 1533 /* 1534 * There is a Xeon hardware errata related to writes to SDOORBELL or 1535 * B2BDOORBELL in conjunction with inbound access to NTB MMIO space, 1536 * which may hang the system. To workaround this, use a memory 1537 * window to access the interrupt and scratch pad registers on the 1538 * remote system. 1539 */ 1540 ntb->b2b_mw_idx = (ntb->mw_count + g_ntb_mw_idx) % 1541 ntb->mw_count; 1542 intel_ntb_printf(2, "Setting up b2b mw idx %d means %u\n", 1543 g_ntb_mw_idx, ntb->b2b_mw_idx); 1544 rc = intel_ntb_mw_set_wc_internal(ntb, ntb->b2b_mw_idx, 1545 VM_MEMATTR_UNCACHEABLE); 1546 KASSERT(rc == 0, ("shouldn't fail")); 1547 } else if (HAS_FEATURE(ntb, NTB_B2BDOORBELL_BIT14)) 1548 /* 1549 * HW Errata on bit 14 of b2bdoorbell register. Writes will not be 1550 * mirrored to the remote system. Shrink the number of bits by one, 1551 * since bit 14 is the last bit. 1552 * 1553 * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register 1554 * anyway. Nor for non-B2B connection types. 1555 */ 1556 ntb->db_count = XEON_DB_COUNT - 1; 1557 1558 ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1559 1560 if (ntb->dev_type == NTB_DEV_USD) 1561 rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_dsd_addr, 1562 &xeon_b2b_usd_addr); 1563 else 1564 rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_usd_addr, 1565 &xeon_b2b_dsd_addr); 1566 if (rc != 0) 1567 return (rc); 1568 1569 /* Enable Bus Master and Memory Space on the secondary side */ 1570 intel_ntb_reg_write(2, XEON_SPCICMD_OFFSET, 1571 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1572 1573 /* 1574 * Mask all doorbell interrupts. 1575 */ 1576 DB_MASK_LOCK(ntb); 1577 ntb->db_mask = ntb->db_valid_mask; 1578 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1579 DB_MASK_UNLOCK(ntb); 1580 1581 rc = intel_ntb_init_isr(ntb); 1582 return (rc); 1583 } 1584 1585 static int 1586 intel_ntb_atom_init_dev(struct ntb_softc *ntb) 1587 { 1588 int error; 1589 1590 KASSERT(ntb->conn_type == NTB_CONN_B2B, 1591 ("Unsupported NTB configuration (%d)\n", ntb->conn_type)); 1592 1593 ntb->spad_count = ATOM_SPAD_COUNT; 1594 ntb->db_count = ATOM_DB_COUNT; 1595 ntb->db_vec_count = ATOM_DB_MSIX_VECTOR_COUNT; 1596 ntb->db_vec_shift = ATOM_DB_MSIX_VECTOR_SHIFT; 1597 ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1598 1599 ntb->reg = &atom_reg; 1600 ntb->self_reg = &atom_pri_reg; 1601 ntb->peer_reg = &atom_b2b_reg; 1602 ntb->xlat_reg = &atom_sec_xlat; 1603 1604 /* 1605 * FIXME - MSI-X bug on early Atom HW, remove once internal issue is 1606 * resolved. Mask transaction layer internal parity errors. 1607 */ 1608 pci_write_config(ntb->device, 0xFC, 0x4, 4); 1609 1610 configure_atom_secondary_side_bars(ntb); 1611 1612 /* Enable Bus Master and Memory Space on the secondary side */ 1613 intel_ntb_reg_write(2, ATOM_SPCICMD_OFFSET, 1614 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1615 1616 error = intel_ntb_init_isr(ntb); 1617 if (error != 0) 1618 return (error); 1619 1620 /* Initiate PCI-E link training */ 1621 intel_ntb_link_enable(ntb->device, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1622 1623 callout_reset(&ntb->heartbeat_timer, 0, atom_link_hb, ntb); 1624 1625 return (0); 1626 } 1627 1628 /* XXX: Linux driver doesn't seem to do any of this for Atom. */ 1629 static void 1630 configure_atom_secondary_side_bars(struct ntb_softc *ntb) 1631 { 1632 1633 if (ntb->dev_type == NTB_DEV_USD) { 1634 intel_ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1635 XEON_B2B_BAR2_ADDR64); 1636 intel_ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1637 XEON_B2B_BAR4_ADDR64); 1638 intel_ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1639 intel_ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1640 } else { 1641 intel_ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1642 XEON_B2B_BAR2_ADDR64); 1643 intel_ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1644 XEON_B2B_BAR4_ADDR64); 1645 intel_ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1646 intel_ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1647 } 1648 } 1649 1650 1651 /* 1652 * When working around Xeon SDOORBELL errata by remapping remote registers in a 1653 * MW, limit the B2B MW to half a MW. By sharing a MW, half the shared MW 1654 * remains for use by a higher layer. 1655 * 1656 * Will only be used if working around SDOORBELL errata and the BIOS-configured 1657 * MW size is sufficiently large. 1658 */ 1659 static unsigned int ntb_b2b_mw_share; 1660 SYSCTL_UINT(_hw_ntb, OID_AUTO, b2b_mw_share, CTLFLAG_RDTUN, &ntb_b2b_mw_share, 1661 0, "If enabled (non-zero), prefer to share half of the B2B peer register " 1662 "MW with higher level consumers. Both sides of the NTB MUST set the same " 1663 "value here."); 1664 1665 static void 1666 xeon_reset_sbar_size(struct ntb_softc *ntb, enum ntb_bar idx, 1667 enum ntb_bar regbar) 1668 { 1669 struct ntb_pci_bar_info *bar; 1670 uint8_t bar_sz; 1671 1672 if (!HAS_FEATURE(ntb, NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_3) 1673 return; 1674 1675 bar = &ntb->bar_info[idx]; 1676 bar_sz = pci_read_config(ntb->device, bar->psz_off, 1); 1677 if (idx == regbar) { 1678 if (ntb->b2b_off != 0) 1679 bar_sz--; 1680 else 1681 bar_sz = 0; 1682 } 1683 pci_write_config(ntb->device, bar->ssz_off, bar_sz, 1); 1684 bar_sz = pci_read_config(ntb->device, bar->ssz_off, 1); 1685 (void)bar_sz; 1686 } 1687 1688 static void 1689 xeon_set_sbar_base_and_limit(struct ntb_softc *ntb, uint64_t bar_addr, 1690 enum ntb_bar idx, enum ntb_bar regbar) 1691 { 1692 uint64_t reg_val; 1693 uint32_t base_reg, lmt_reg; 1694 1695 bar_get_xlat_params(ntb, idx, &base_reg, NULL, &lmt_reg); 1696 if (idx == regbar) { 1697 if (ntb->b2b_off) 1698 bar_addr += ntb->b2b_off; 1699 else 1700 bar_addr = 0; 1701 } 1702 1703 if (!bar_is_64bit(ntb, idx)) { 1704 intel_ntb_reg_write(4, base_reg, bar_addr); 1705 reg_val = intel_ntb_reg_read(4, base_reg); 1706 (void)reg_val; 1707 1708 intel_ntb_reg_write(4, lmt_reg, bar_addr); 1709 reg_val = intel_ntb_reg_read(4, lmt_reg); 1710 (void)reg_val; 1711 } else { 1712 intel_ntb_reg_write(8, base_reg, bar_addr); 1713 reg_val = intel_ntb_reg_read(8, base_reg); 1714 (void)reg_val; 1715 1716 intel_ntb_reg_write(8, lmt_reg, bar_addr); 1717 reg_val = intel_ntb_reg_read(8, lmt_reg); 1718 (void)reg_val; 1719 } 1720 } 1721 1722 static void 1723 xeon_set_pbar_xlat(struct ntb_softc *ntb, uint64_t base_addr, enum ntb_bar idx) 1724 { 1725 struct ntb_pci_bar_info *bar; 1726 1727 bar = &ntb->bar_info[idx]; 1728 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_2) { 1729 intel_ntb_reg_write(4, bar->pbarxlat_off, base_addr); 1730 base_addr = intel_ntb_reg_read(4, bar->pbarxlat_off); 1731 } else { 1732 intel_ntb_reg_write(8, bar->pbarxlat_off, base_addr); 1733 base_addr = intel_ntb_reg_read(8, bar->pbarxlat_off); 1734 } 1735 (void)base_addr; 1736 } 1737 1738 static int 1739 xeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr, 1740 const struct ntb_b2b_addr *peer_addr) 1741 { 1742 struct ntb_pci_bar_info *b2b_bar; 1743 vm_size_t bar_size; 1744 uint64_t bar_addr; 1745 enum ntb_bar b2b_bar_num, i; 1746 1747 if (ntb->b2b_mw_idx == B2B_MW_DISABLED) { 1748 b2b_bar = NULL; 1749 b2b_bar_num = NTB_CONFIG_BAR; 1750 ntb->b2b_off = 0; 1751 } else { 1752 b2b_bar_num = intel_ntb_mw_to_bar(ntb, ntb->b2b_mw_idx); 1753 KASSERT(b2b_bar_num > 0 && b2b_bar_num < NTB_MAX_BARS, 1754 ("invalid b2b mw bar")); 1755 1756 b2b_bar = &ntb->bar_info[b2b_bar_num]; 1757 bar_size = b2b_bar->size; 1758 1759 if (ntb_b2b_mw_share != 0 && 1760 (bar_size >> 1) >= XEON_B2B_MIN_SIZE) 1761 ntb->b2b_off = bar_size >> 1; 1762 else if (bar_size >= XEON_B2B_MIN_SIZE) { 1763 ntb->b2b_off = 0; 1764 } else { 1765 device_printf(ntb->device, 1766 "B2B bar size is too small!\n"); 1767 return (EIO); 1768 } 1769 } 1770 1771 /* 1772 * Reset the secondary bar sizes to match the primary bar sizes. 1773 * (Except, disable or halve the size of the B2B secondary bar.) 1774 */ 1775 for (i = NTB_B2B_BAR_1; i < NTB_MAX_BARS; i++) 1776 xeon_reset_sbar_size(ntb, i, b2b_bar_num); 1777 1778 bar_addr = 0; 1779 if (b2b_bar_num == NTB_CONFIG_BAR) 1780 bar_addr = addr->bar0_addr; 1781 else if (b2b_bar_num == NTB_B2B_BAR_1) 1782 bar_addr = addr->bar2_addr64; 1783 else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 1784 bar_addr = addr->bar4_addr64; 1785 else if (b2b_bar_num == NTB_B2B_BAR_2) 1786 bar_addr = addr->bar4_addr32; 1787 else if (b2b_bar_num == NTB_B2B_BAR_3) 1788 bar_addr = addr->bar5_addr32; 1789 else 1790 KASSERT(false, ("invalid bar")); 1791 1792 intel_ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, bar_addr); 1793 1794 /* 1795 * Other SBARs are normally hit by the PBAR xlat, except for the b2b 1796 * register BAR. The B2B BAR is either disabled above or configured 1797 * half-size. It starts at PBAR xlat + offset. 1798 * 1799 * Also set up incoming BAR limits == base (zero length window). 1800 */ 1801 xeon_set_sbar_base_and_limit(ntb, addr->bar2_addr64, NTB_B2B_BAR_1, 1802 b2b_bar_num); 1803 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 1804 xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr32, 1805 NTB_B2B_BAR_2, b2b_bar_num); 1806 xeon_set_sbar_base_and_limit(ntb, addr->bar5_addr32, 1807 NTB_B2B_BAR_3, b2b_bar_num); 1808 } else 1809 xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr64, 1810 NTB_B2B_BAR_2, b2b_bar_num); 1811 1812 /* Zero incoming translation addrs */ 1813 intel_ntb_reg_write(8, XEON_SBAR2XLAT_OFFSET, 0); 1814 intel_ntb_reg_write(8, XEON_SBAR4XLAT_OFFSET, 0); 1815 1816 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1817 uint32_t xlat_reg, lmt_reg; 1818 enum ntb_bar bar_num; 1819 1820 /* 1821 * We point the chosen MSIX MW BAR xlat to remote LAPIC for 1822 * workaround 1823 */ 1824 bar_num = intel_ntb_mw_to_bar(ntb, ntb->msix_mw_idx); 1825 bar_get_xlat_params(ntb, bar_num, NULL, &xlat_reg, &lmt_reg); 1826 if (bar_is_64bit(ntb, bar_num)) { 1827 intel_ntb_reg_write(8, xlat_reg, MSI_INTEL_ADDR_BASE); 1828 ntb->msix_xlat = intel_ntb_reg_read(8, xlat_reg); 1829 intel_ntb_reg_write(8, lmt_reg, 0); 1830 } else { 1831 intel_ntb_reg_write(4, xlat_reg, MSI_INTEL_ADDR_BASE); 1832 ntb->msix_xlat = intel_ntb_reg_read(4, xlat_reg); 1833 intel_ntb_reg_write(4, lmt_reg, 0); 1834 } 1835 1836 ntb->peer_lapic_bar = &ntb->bar_info[bar_num]; 1837 } 1838 (void)intel_ntb_reg_read(8, XEON_SBAR2XLAT_OFFSET); 1839 (void)intel_ntb_reg_read(8, XEON_SBAR4XLAT_OFFSET); 1840 1841 /* Zero outgoing translation limits (whole bar size windows) */ 1842 intel_ntb_reg_write(8, XEON_PBAR2LMT_OFFSET, 0); 1843 intel_ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); 1844 1845 /* Set outgoing translation offsets */ 1846 xeon_set_pbar_xlat(ntb, peer_addr->bar2_addr64, NTB_B2B_BAR_1); 1847 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 1848 xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr32, NTB_B2B_BAR_2); 1849 xeon_set_pbar_xlat(ntb, peer_addr->bar5_addr32, NTB_B2B_BAR_3); 1850 } else 1851 xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr64, NTB_B2B_BAR_2); 1852 1853 /* Set the translation offset for B2B registers */ 1854 bar_addr = 0; 1855 if (b2b_bar_num == NTB_CONFIG_BAR) 1856 bar_addr = peer_addr->bar0_addr; 1857 else if (b2b_bar_num == NTB_B2B_BAR_1) 1858 bar_addr = peer_addr->bar2_addr64; 1859 else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 1860 bar_addr = peer_addr->bar4_addr64; 1861 else if (b2b_bar_num == NTB_B2B_BAR_2) 1862 bar_addr = peer_addr->bar4_addr32; 1863 else if (b2b_bar_num == NTB_B2B_BAR_3) 1864 bar_addr = peer_addr->bar5_addr32; 1865 else 1866 KASSERT(false, ("invalid bar")); 1867 1868 /* 1869 * B2B_XLAT_OFFSET is a 64-bit register but can only be written 32 bits 1870 * at a time. 1871 */ 1872 intel_ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, bar_addr & 0xffffffff); 1873 intel_ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, bar_addr >> 32); 1874 return (0); 1875 } 1876 1877 static inline bool 1878 _xeon_link_is_up(struct ntb_softc *ntb) 1879 { 1880 1881 if (ntb->conn_type == NTB_CONN_TRANSPARENT) 1882 return (true); 1883 return ((ntb->lnk_sta & NTB_LINK_STATUS_ACTIVE) != 0); 1884 } 1885 1886 static inline bool 1887 link_is_up(struct ntb_softc *ntb) 1888 { 1889 1890 if (ntb->type == NTB_XEON) 1891 return (_xeon_link_is_up(ntb) && (ntb->peer_msix_good || 1892 !HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP))); 1893 1894 KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1895 return ((ntb->ntb_ctl & ATOM_CNTL_LINK_DOWN) == 0); 1896 } 1897 1898 static inline bool 1899 atom_link_is_err(struct ntb_softc *ntb) 1900 { 1901 uint32_t status; 1902 1903 KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 1904 1905 status = intel_ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 1906 if ((status & ATOM_LTSSMSTATEJMP_FORCEDETECT) != 0) 1907 return (true); 1908 1909 status = intel_ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 1910 return ((status & ATOM_IBIST_ERR_OFLOW) != 0); 1911 } 1912 1913 /* Atom does not have link status interrupt, poll on that platform */ 1914 static void 1915 atom_link_hb(void *arg) 1916 { 1917 struct ntb_softc *ntb = arg; 1918 sbintime_t timo, poll_ts; 1919 1920 timo = NTB_HB_TIMEOUT * hz; 1921 poll_ts = ntb->last_ts + timo; 1922 1923 /* 1924 * Delay polling the link status if an interrupt was received, unless 1925 * the cached link status says the link is down. 1926 */ 1927 if ((sbintime_t)ticks - poll_ts < 0 && link_is_up(ntb)) { 1928 timo = poll_ts - ticks; 1929 goto out; 1930 } 1931 1932 if (intel_ntb_poll_link(ntb)) 1933 ntb_link_event(ntb->device); 1934 1935 if (!link_is_up(ntb) && atom_link_is_err(ntb)) { 1936 /* Link is down with error, proceed with recovery */ 1937 callout_reset(&ntb->lr_timer, 0, recover_atom_link, ntb); 1938 return; 1939 } 1940 1941 out: 1942 callout_reset(&ntb->heartbeat_timer, timo, atom_link_hb, ntb); 1943 } 1944 1945 static void 1946 atom_perform_link_restart(struct ntb_softc *ntb) 1947 { 1948 uint32_t status; 1949 1950 /* Driver resets the NTB ModPhy lanes - magic! */ 1951 intel_ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0xe0); 1952 intel_ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x40); 1953 intel_ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x60); 1954 intel_ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0x60); 1955 1956 /* Driver waits 100ms to allow the NTB ModPhy to settle */ 1957 pause("ModPhy", hz / 10); 1958 1959 /* Clear AER Errors, write to clear */ 1960 status = intel_ntb_reg_read(4, ATOM_ERRCORSTS_OFFSET); 1961 status &= PCIM_AER_COR_REPLAY_ROLLOVER; 1962 intel_ntb_reg_write(4, ATOM_ERRCORSTS_OFFSET, status); 1963 1964 /* Clear unexpected electrical idle event in LTSSM, write to clear */ 1965 status = intel_ntb_reg_read(4, ATOM_LTSSMERRSTS0_OFFSET); 1966 status |= ATOM_LTSSMERRSTS0_UNEXPECTEDEI; 1967 intel_ntb_reg_write(4, ATOM_LTSSMERRSTS0_OFFSET, status); 1968 1969 /* Clear DeSkew Buffer error, write to clear */ 1970 status = intel_ntb_reg_read(4, ATOM_DESKEWSTS_OFFSET); 1971 status |= ATOM_DESKEWSTS_DBERR; 1972 intel_ntb_reg_write(4, ATOM_DESKEWSTS_OFFSET, status); 1973 1974 status = intel_ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 1975 status &= ATOM_IBIST_ERR_OFLOW; 1976 intel_ntb_reg_write(4, ATOM_IBSTERRRCRVSTS0_OFFSET, status); 1977 1978 /* Releases the NTB state machine to allow the link to retrain */ 1979 status = intel_ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 1980 status &= ~ATOM_LTSSMSTATEJMP_FORCEDETECT; 1981 intel_ntb_reg_write(4, ATOM_LTSSMSTATEJMP_OFFSET, status); 1982 } 1983 1984 static int 1985 intel_ntb_port_number(device_t dev) 1986 { 1987 struct ntb_softc *ntb = device_get_softc(dev); 1988 1989 return (ntb->dev_type == NTB_DEV_USD ? 0 : 1); 1990 } 1991 1992 static int 1993 intel_ntb_peer_port_count(device_t dev) 1994 { 1995 1996 return (1); 1997 } 1998 1999 static int 2000 intel_ntb_peer_port_number(device_t dev, int pidx) 2001 { 2002 struct ntb_softc *ntb = device_get_softc(dev); 2003 2004 if (pidx != 0) 2005 return (-EINVAL); 2006 2007 return (ntb->dev_type == NTB_DEV_USD ? 1 : 0); 2008 } 2009 2010 static int 2011 intel_ntb_peer_port_idx(device_t dev, int port) 2012 { 2013 int peer_port; 2014 2015 peer_port = intel_ntb_peer_port_number(dev, 0); 2016 if (peer_port == -EINVAL || port != peer_port) 2017 return (-EINVAL); 2018 2019 return (0); 2020 } 2021 2022 static int 2023 intel_ntb_link_enable(device_t dev, enum ntb_speed speed __unused, 2024 enum ntb_width width __unused) 2025 { 2026 struct ntb_softc *ntb = device_get_softc(dev); 2027 uint32_t cntl; 2028 2029 intel_ntb_printf(2, "%s\n", __func__); 2030 2031 if (ntb->type == NTB_ATOM) { 2032 pci_write_config(ntb->device, NTB_PPD_OFFSET, 2033 ntb->ppd | ATOM_PPD_INIT_LINK, 4); 2034 return (0); 2035 } 2036 2037 if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 2038 ntb_link_event(dev); 2039 return (0); 2040 } 2041 2042 cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl); 2043 cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); 2044 cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; 2045 cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP; 2046 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 2047 cntl |= NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP; 2048 intel_ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 2049 return (0); 2050 } 2051 2052 static int 2053 intel_ntb_link_disable(device_t dev) 2054 { 2055 struct ntb_softc *ntb = device_get_softc(dev); 2056 uint32_t cntl; 2057 2058 intel_ntb_printf(2, "%s\n", __func__); 2059 2060 if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 2061 ntb_link_event(dev); 2062 return (0); 2063 } 2064 2065 cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl); 2066 cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); 2067 cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP); 2068 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 2069 cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP); 2070 cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; 2071 intel_ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 2072 return (0); 2073 } 2074 2075 static bool 2076 intel_ntb_link_enabled(device_t dev) 2077 { 2078 struct ntb_softc *ntb = device_get_softc(dev); 2079 uint32_t cntl; 2080 2081 if (ntb->type == NTB_ATOM) { 2082 cntl = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 2083 return ((cntl & ATOM_PPD_INIT_LINK) != 0); 2084 } 2085 2086 if (ntb->conn_type == NTB_CONN_TRANSPARENT) 2087 return (true); 2088 2089 cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl); 2090 return ((cntl & NTB_CNTL_LINK_DISABLE) == 0); 2091 } 2092 2093 static void 2094 recover_atom_link(void *arg) 2095 { 2096 struct ntb_softc *ntb = arg; 2097 unsigned speed, width, oldspeed, oldwidth; 2098 uint32_t status32; 2099 2100 atom_perform_link_restart(ntb); 2101 2102 /* 2103 * There is a potential race between the 2 NTB devices recovering at 2104 * the same time. If the times are the same, the link will not recover 2105 * and the driver will be stuck in this loop forever. Add a random 2106 * interval to the recovery time to prevent this race. 2107 */ 2108 status32 = arc4random() % ATOM_LINK_RECOVERY_TIME; 2109 pause("Link", (ATOM_LINK_RECOVERY_TIME + status32) * hz / 1000); 2110 2111 if (atom_link_is_err(ntb)) 2112 goto retry; 2113 2114 status32 = intel_ntb_reg_read(4, ntb->reg->ntb_ctl); 2115 if ((status32 & ATOM_CNTL_LINK_DOWN) != 0) 2116 goto out; 2117 2118 status32 = intel_ntb_reg_read(4, ntb->reg->lnk_sta); 2119 width = NTB_LNK_STA_WIDTH(status32); 2120 speed = status32 & NTB_LINK_SPEED_MASK; 2121 2122 oldwidth = NTB_LNK_STA_WIDTH(ntb->lnk_sta); 2123 oldspeed = ntb->lnk_sta & NTB_LINK_SPEED_MASK; 2124 if (oldwidth != width || oldspeed != speed) 2125 goto retry; 2126 2127 out: 2128 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, atom_link_hb, 2129 ntb); 2130 return; 2131 2132 retry: 2133 callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_atom_link, 2134 ntb); 2135 } 2136 2137 /* 2138 * Polls the HW link status register(s); returns true if something has changed. 2139 */ 2140 static bool 2141 intel_ntb_poll_link(struct ntb_softc *ntb) 2142 { 2143 uint32_t ntb_cntl; 2144 uint16_t reg_val; 2145 2146 if (ntb->type == NTB_ATOM) { 2147 ntb_cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl); 2148 if (ntb_cntl == ntb->ntb_ctl) 2149 return (false); 2150 2151 ntb->ntb_ctl = ntb_cntl; 2152 ntb->lnk_sta = intel_ntb_reg_read(4, ntb->reg->lnk_sta); 2153 } else { 2154 db_iowrite_raw(ntb, ntb->self_reg->db_bell, ntb->db_link_mask); 2155 2156 reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 2157 if (reg_val == ntb->lnk_sta) 2158 return (false); 2159 2160 ntb->lnk_sta = reg_val; 2161 2162 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 2163 if (_xeon_link_is_up(ntb)) { 2164 if (!ntb->peer_msix_good) { 2165 callout_reset(&ntb->peer_msix_work, 0, 2166 intel_ntb_exchange_msix, ntb); 2167 return (false); 2168 } 2169 } else { 2170 ntb->peer_msix_good = false; 2171 ntb->peer_msix_done = false; 2172 } 2173 } 2174 } 2175 return (true); 2176 } 2177 2178 static inline enum ntb_speed 2179 intel_ntb_link_sta_speed(struct ntb_softc *ntb) 2180 { 2181 2182 if (!link_is_up(ntb)) 2183 return (NTB_SPEED_NONE); 2184 return (ntb->lnk_sta & NTB_LINK_SPEED_MASK); 2185 } 2186 2187 static inline enum ntb_width 2188 intel_ntb_link_sta_width(struct ntb_softc *ntb) 2189 { 2190 2191 if (!link_is_up(ntb)) 2192 return (NTB_WIDTH_NONE); 2193 return (NTB_LNK_STA_WIDTH(ntb->lnk_sta)); 2194 } 2195 2196 SYSCTL_NODE(_hw_ntb, OID_AUTO, debug_info, CTLFLAG_RW, 0, 2197 "Driver state, statistics, and HW registers"); 2198 2199 #define NTB_REGSZ_MASK (3ul << 30) 2200 #define NTB_REG_64 (1ul << 30) 2201 #define NTB_REG_32 (2ul << 30) 2202 #define NTB_REG_16 (3ul << 30) 2203 #define NTB_REG_8 (0ul << 30) 2204 2205 #define NTB_DB_READ (1ul << 29) 2206 #define NTB_PCI_REG (1ul << 28) 2207 #define NTB_REGFLAGS_MASK (NTB_REGSZ_MASK | NTB_DB_READ | NTB_PCI_REG) 2208 2209 static void 2210 intel_ntb_sysctl_init(struct ntb_softc *ntb) 2211 { 2212 struct sysctl_oid_list *globals, *tree_par, *regpar, *statpar, *errpar; 2213 struct sysctl_ctx_list *ctx; 2214 struct sysctl_oid *tree, *tmptree; 2215 2216 ctx = device_get_sysctl_ctx(ntb->device); 2217 globals = SYSCTL_CHILDREN(device_get_sysctl_tree(ntb->device)); 2218 2219 SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "link_status", 2220 CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, 2221 sysctl_handle_link_status_human, "A", 2222 "Link status (human readable)"); 2223 SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "active", 2224 CTLFLAG_RD | CTLTYPE_UINT, ntb, 0, sysctl_handle_link_status, 2225 "IU", "Link status (1=active, 0=inactive)"); 2226 SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "admin_up", 2227 CTLFLAG_RW | CTLTYPE_UINT, ntb, 0, sysctl_handle_link_admin, 2228 "IU", "Set/get interface status (1=UP, 0=DOWN)"); 2229 2230 tree = SYSCTL_ADD_NODE(ctx, globals, OID_AUTO, "debug_info", 2231 CTLFLAG_RD, NULL, "Driver state, statistics, and HW registers"); 2232 tree_par = SYSCTL_CHILDREN(tree); 2233 2234 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "conn_type", CTLFLAG_RD, 2235 &ntb->conn_type, 0, "0 - Transparent; 1 - B2B; 2 - Root Port"); 2236 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "dev_type", CTLFLAG_RD, 2237 &ntb->dev_type, 0, "0 - USD; 1 - DSD"); 2238 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ppd", CTLFLAG_RD, 2239 &ntb->ppd, 0, "Raw PPD register (cached)"); 2240 2241 if (ntb->b2b_mw_idx != B2B_MW_DISABLED) { 2242 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "b2b_idx", CTLFLAG_RD, 2243 &ntb->b2b_mw_idx, 0, 2244 "Index of the MW used for B2B remote register access"); 2245 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "b2b_off", 2246 CTLFLAG_RD, &ntb->b2b_off, 2247 "If non-zero, offset of B2B register region in shared MW"); 2248 } 2249 2250 SYSCTL_ADD_PROC(ctx, tree_par, OID_AUTO, "features", 2251 CTLFLAG_RD | CTLTYPE_STRING, ntb, 0, sysctl_handle_features, "A", 2252 "Features/errata of this NTB device"); 2253 2254 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ntb_ctl", CTLFLAG_RD, 2255 __DEVOLATILE(uint32_t *, &ntb->ntb_ctl), 0, 2256 "NTB CTL register (cached)"); 2257 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "lnk_sta", CTLFLAG_RD, 2258 __DEVOLATILE(uint32_t *, &ntb->lnk_sta), 0, 2259 "LNK STA register (cached)"); 2260 2261 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "mw_count", CTLFLAG_RD, 2262 &ntb->mw_count, 0, "MW count"); 2263 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "spad_count", CTLFLAG_RD, 2264 &ntb->spad_count, 0, "Scratchpad count"); 2265 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_count", CTLFLAG_RD, 2266 &ntb->db_count, 0, "Doorbell count"); 2267 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_count", CTLFLAG_RD, 2268 &ntb->db_vec_count, 0, "Doorbell vector count"); 2269 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_shift", CTLFLAG_RD, 2270 &ntb->db_vec_shift, 0, "Doorbell vector shift"); 2271 2272 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_valid_mask", CTLFLAG_RD, 2273 &ntb->db_valid_mask, "Doorbell valid mask"); 2274 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_link_mask", CTLFLAG_RD, 2275 &ntb->db_link_mask, "Doorbell link mask"); 2276 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_mask", CTLFLAG_RD, 2277 &ntb->db_mask, "Doorbell mask (cached)"); 2278 2279 tmptree = SYSCTL_ADD_NODE(ctx, tree_par, OID_AUTO, "registers", 2280 CTLFLAG_RD, NULL, "Raw HW registers (big-endian)"); 2281 regpar = SYSCTL_CHILDREN(tmptree); 2282 2283 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ntbcntl", 2284 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2285 ntb->reg->ntb_ctl, sysctl_handle_register, "IU", 2286 "NTB Control register"); 2287 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcap", 2288 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2289 0x19c, sysctl_handle_register, "IU", 2290 "NTB Link Capabilities"); 2291 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcon", 2292 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, NTB_REG_32 | 2293 0x1a0, sysctl_handle_register, "IU", 2294 "NTB Link Control register"); 2295 2296 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_mask", 2297 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2298 NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_mask, 2299 sysctl_handle_register, "QU", "Doorbell mask register"); 2300 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_bell", 2301 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2302 NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_bell, 2303 sysctl_handle_register, "QU", "Doorbell register"); 2304 2305 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat23", 2306 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2307 NTB_REG_64 | ntb->xlat_reg->bar2_xlat, 2308 sysctl_handle_register, "QU", "Incoming XLAT23 register"); 2309 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2310 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat4", 2311 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2312 NTB_REG_32 | ntb->xlat_reg->bar4_xlat, 2313 sysctl_handle_register, "IU", "Incoming XLAT4 register"); 2314 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat5", 2315 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2316 NTB_REG_32 | ntb->xlat_reg->bar5_xlat, 2317 sysctl_handle_register, "IU", "Incoming XLAT5 register"); 2318 } else { 2319 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat45", 2320 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2321 NTB_REG_64 | ntb->xlat_reg->bar4_xlat, 2322 sysctl_handle_register, "QU", "Incoming XLAT45 register"); 2323 } 2324 2325 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt23", 2326 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2327 NTB_REG_64 | ntb->xlat_reg->bar2_limit, 2328 sysctl_handle_register, "QU", "Incoming LMT23 register"); 2329 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2330 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt4", 2331 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2332 NTB_REG_32 | ntb->xlat_reg->bar4_limit, 2333 sysctl_handle_register, "IU", "Incoming LMT4 register"); 2334 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt5", 2335 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2336 NTB_REG_32 | ntb->xlat_reg->bar5_limit, 2337 sysctl_handle_register, "IU", "Incoming LMT5 register"); 2338 } else { 2339 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt45", 2340 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2341 NTB_REG_64 | ntb->xlat_reg->bar4_limit, 2342 sysctl_handle_register, "QU", "Incoming LMT45 register"); 2343 } 2344 2345 if (ntb->type == NTB_ATOM) 2346 return; 2347 2348 tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_stats", 2349 CTLFLAG_RD, NULL, "Xeon HW statistics"); 2350 statpar = SYSCTL_CHILDREN(tmptree); 2351 SYSCTL_ADD_PROC(ctx, statpar, OID_AUTO, "upstream_mem_miss", 2352 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2353 NTB_REG_16 | XEON_USMEMMISS_OFFSET, 2354 sysctl_handle_register, "SU", "Upstream Memory Miss"); 2355 2356 tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_hw_err", 2357 CTLFLAG_RD, NULL, "Xeon HW errors"); 2358 errpar = SYSCTL_CHILDREN(tmptree); 2359 2360 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ppd", 2361 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2362 NTB_REG_8 | NTB_PCI_REG | NTB_PPD_OFFSET, 2363 sysctl_handle_register, "CU", "PPD"); 2364 2365 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar23_sz", 2366 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2367 NTB_REG_8 | NTB_PCI_REG | XEON_PBAR23SZ_OFFSET, 2368 sysctl_handle_register, "CU", "PBAR23 SZ (log2)"); 2369 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar4_sz", 2370 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2371 NTB_REG_8 | NTB_PCI_REG | XEON_PBAR4SZ_OFFSET, 2372 sysctl_handle_register, "CU", "PBAR4 SZ (log2)"); 2373 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar5_sz", 2374 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2375 NTB_REG_8 | NTB_PCI_REG | XEON_PBAR5SZ_OFFSET, 2376 sysctl_handle_register, "CU", "PBAR5 SZ (log2)"); 2377 2378 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_sz", 2379 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2380 NTB_REG_8 | NTB_PCI_REG | XEON_SBAR23SZ_OFFSET, 2381 sysctl_handle_register, "CU", "SBAR23 SZ (log2)"); 2382 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_sz", 2383 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2384 NTB_REG_8 | NTB_PCI_REG | XEON_SBAR4SZ_OFFSET, 2385 sysctl_handle_register, "CU", "SBAR4 SZ (log2)"); 2386 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_sz", 2387 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2388 NTB_REG_8 | NTB_PCI_REG | XEON_SBAR5SZ_OFFSET, 2389 sysctl_handle_register, "CU", "SBAR5 SZ (log2)"); 2390 2391 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "devsts", 2392 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2393 NTB_REG_16 | NTB_PCI_REG | XEON_DEVSTS_OFFSET, 2394 sysctl_handle_register, "SU", "DEVSTS"); 2395 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnksts", 2396 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2397 NTB_REG_16 | NTB_PCI_REG | XEON_LINK_STATUS_OFFSET, 2398 sysctl_handle_register, "SU", "LNKSTS"); 2399 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "slnksts", 2400 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2401 NTB_REG_16 | NTB_PCI_REG | XEON_SLINK_STATUS_OFFSET, 2402 sysctl_handle_register, "SU", "SLNKSTS"); 2403 2404 SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "uncerrsts", 2405 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2406 NTB_REG_32 | NTB_PCI_REG | XEON_UNCERRSTS_OFFSET, 2407 sysctl_handle_register, "IU", "UNCERRSTS"); 2408 SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "corerrsts", 2409 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2410 NTB_REG_32 | NTB_PCI_REG | XEON_CORERRSTS_OFFSET, 2411 sysctl_handle_register, "IU", "CORERRSTS"); 2412 2413 if (ntb->conn_type != NTB_CONN_B2B) 2414 return; 2415 2416 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat01l", 2417 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2418 NTB_REG_32 | XEON_B2B_XLAT_OFFSETL, 2419 sysctl_handle_register, "IU", "Outgoing XLAT0L register"); 2420 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat01u", 2421 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2422 NTB_REG_32 | XEON_B2B_XLAT_OFFSETU, 2423 sysctl_handle_register, "IU", "Outgoing XLAT0U register"); 2424 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat23", 2425 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2426 NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off, 2427 sysctl_handle_register, "QU", "Outgoing XLAT23 register"); 2428 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2429 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat4", 2430 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2431 NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2432 sysctl_handle_register, "IU", "Outgoing XLAT4 register"); 2433 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat5", 2434 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2435 NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off, 2436 sysctl_handle_register, "IU", "Outgoing XLAT5 register"); 2437 } else { 2438 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat45", 2439 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2440 NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2441 sysctl_handle_register, "QU", "Outgoing XLAT45 register"); 2442 } 2443 2444 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt23", 2445 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2446 NTB_REG_64 | XEON_PBAR2LMT_OFFSET, 2447 sysctl_handle_register, "QU", "Outgoing LMT23 register"); 2448 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2449 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt4", 2450 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2451 NTB_REG_32 | XEON_PBAR4LMT_OFFSET, 2452 sysctl_handle_register, "IU", "Outgoing LMT4 register"); 2453 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt5", 2454 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2455 NTB_REG_32 | XEON_PBAR5LMT_OFFSET, 2456 sysctl_handle_register, "IU", "Outgoing LMT5 register"); 2457 } else { 2458 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt45", 2459 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2460 NTB_REG_64 | XEON_PBAR4LMT_OFFSET, 2461 sysctl_handle_register, "QU", "Outgoing LMT45 register"); 2462 } 2463 2464 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar01_base", 2465 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2466 NTB_REG_64 | ntb->xlat_reg->bar0_base, 2467 sysctl_handle_register, "QU", "Secondary BAR01 base register"); 2468 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_base", 2469 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2470 NTB_REG_64 | ntb->xlat_reg->bar2_base, 2471 sysctl_handle_register, "QU", "Secondary BAR23 base register"); 2472 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2473 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_base", 2474 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2475 NTB_REG_32 | ntb->xlat_reg->bar4_base, 2476 sysctl_handle_register, "IU", 2477 "Secondary BAR4 base register"); 2478 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_base", 2479 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2480 NTB_REG_32 | ntb->xlat_reg->bar5_base, 2481 sysctl_handle_register, "IU", 2482 "Secondary BAR5 base register"); 2483 } else { 2484 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar45_base", 2485 CTLFLAG_RD | CTLTYPE_OPAQUE, ntb, 2486 NTB_REG_64 | ntb->xlat_reg->bar4_base, 2487 sysctl_handle_register, "QU", 2488 "Secondary BAR45 base register"); 2489 } 2490 } 2491 2492 static int 2493 sysctl_handle_features(SYSCTL_HANDLER_ARGS) 2494 { 2495 struct ntb_softc *ntb = arg1; 2496 struct sbuf sb; 2497 int error; 2498 2499 sbuf_new_for_sysctl(&sb, NULL, 256, req); 2500 2501 sbuf_printf(&sb, "%b", ntb->features, NTB_FEATURES_STR); 2502 error = sbuf_finish(&sb); 2503 sbuf_delete(&sb); 2504 2505 if (error || !req->newptr) 2506 return (error); 2507 return (EINVAL); 2508 } 2509 2510 static int 2511 sysctl_handle_link_admin(SYSCTL_HANDLER_ARGS) 2512 { 2513 struct ntb_softc *ntb = arg1; 2514 unsigned old, new; 2515 int error; 2516 2517 old = intel_ntb_link_enabled(ntb->device); 2518 2519 error = SYSCTL_OUT(req, &old, sizeof(old)); 2520 if (error != 0 || req->newptr == NULL) 2521 return (error); 2522 2523 error = SYSCTL_IN(req, &new, sizeof(new)); 2524 if (error != 0) 2525 return (error); 2526 2527 intel_ntb_printf(0, "Admin set interface state to '%sabled'\n", 2528 (new != 0)? "en" : "dis"); 2529 2530 if (new != 0) 2531 error = intel_ntb_link_enable(ntb->device, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 2532 else 2533 error = intel_ntb_link_disable(ntb->device); 2534 return (error); 2535 } 2536 2537 static int 2538 sysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS) 2539 { 2540 struct ntb_softc *ntb = arg1; 2541 struct sbuf sb; 2542 enum ntb_speed speed; 2543 enum ntb_width width; 2544 int error; 2545 2546 sbuf_new_for_sysctl(&sb, NULL, 32, req); 2547 2548 if (intel_ntb_link_is_up(ntb->device, &speed, &width)) 2549 sbuf_printf(&sb, "up / PCIe Gen %u / Width x%u", 2550 (unsigned)speed, (unsigned)width); 2551 else 2552 sbuf_printf(&sb, "down"); 2553 2554 error = sbuf_finish(&sb); 2555 sbuf_delete(&sb); 2556 2557 if (error || !req->newptr) 2558 return (error); 2559 return (EINVAL); 2560 } 2561 2562 static int 2563 sysctl_handle_link_status(SYSCTL_HANDLER_ARGS) 2564 { 2565 struct ntb_softc *ntb = arg1; 2566 unsigned res; 2567 int error; 2568 2569 res = intel_ntb_link_is_up(ntb->device, NULL, NULL); 2570 2571 error = SYSCTL_OUT(req, &res, sizeof(res)); 2572 if (error || !req->newptr) 2573 return (error); 2574 return (EINVAL); 2575 } 2576 2577 static int 2578 sysctl_handle_register(SYSCTL_HANDLER_ARGS) 2579 { 2580 struct ntb_softc *ntb; 2581 const void *outp; 2582 uintptr_t sz; 2583 uint64_t umv; 2584 char be[sizeof(umv)]; 2585 size_t outsz; 2586 uint32_t reg; 2587 bool db, pci; 2588 int error; 2589 2590 ntb = arg1; 2591 reg = arg2 & ~NTB_REGFLAGS_MASK; 2592 sz = arg2 & NTB_REGSZ_MASK; 2593 db = (arg2 & NTB_DB_READ) != 0; 2594 pci = (arg2 & NTB_PCI_REG) != 0; 2595 2596 KASSERT(!(db && pci), ("bogus")); 2597 2598 if (db) { 2599 KASSERT(sz == NTB_REG_64, ("bogus")); 2600 umv = db_ioread(ntb, reg); 2601 outsz = sizeof(uint64_t); 2602 } else { 2603 switch (sz) { 2604 case NTB_REG_64: 2605 if (pci) 2606 umv = pci_read_config(ntb->device, reg, 8); 2607 else 2608 umv = intel_ntb_reg_read(8, reg); 2609 outsz = sizeof(uint64_t); 2610 break; 2611 case NTB_REG_32: 2612 if (pci) 2613 umv = pci_read_config(ntb->device, reg, 4); 2614 else 2615 umv = intel_ntb_reg_read(4, reg); 2616 outsz = sizeof(uint32_t); 2617 break; 2618 case NTB_REG_16: 2619 if (pci) 2620 umv = pci_read_config(ntb->device, reg, 2); 2621 else 2622 umv = intel_ntb_reg_read(2, reg); 2623 outsz = sizeof(uint16_t); 2624 break; 2625 case NTB_REG_8: 2626 if (pci) 2627 umv = pci_read_config(ntb->device, reg, 1); 2628 else 2629 umv = intel_ntb_reg_read(1, reg); 2630 outsz = sizeof(uint8_t); 2631 break; 2632 default: 2633 panic("bogus"); 2634 break; 2635 } 2636 } 2637 2638 /* Encode bigendian so that sysctl -x is legible. */ 2639 be64enc(be, umv); 2640 outp = ((char *)be) + sizeof(umv) - outsz; 2641 2642 error = SYSCTL_OUT(req, outp, outsz); 2643 if (error || !req->newptr) 2644 return (error); 2645 return (EINVAL); 2646 } 2647 2648 static unsigned 2649 intel_ntb_user_mw_to_idx(struct ntb_softc *ntb, unsigned uidx) 2650 { 2651 2652 if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 && 2653 uidx >= ntb->b2b_mw_idx) || 2654 (ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx)) 2655 uidx++; 2656 if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 && 2657 uidx >= ntb->b2b_mw_idx) && 2658 (ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx)) 2659 uidx++; 2660 return (uidx); 2661 } 2662 2663 #ifndef EARLY_AP_STARTUP 2664 static int msix_ready; 2665 2666 static void 2667 intel_ntb_msix_ready(void *arg __unused) 2668 { 2669 2670 msix_ready = 1; 2671 } 2672 SYSINIT(intel_ntb_msix_ready, SI_SUB_SMP, SI_ORDER_ANY, 2673 intel_ntb_msix_ready, NULL); 2674 #endif 2675 2676 static void 2677 intel_ntb_exchange_msix(void *ctx) 2678 { 2679 struct ntb_softc *ntb; 2680 uint32_t val; 2681 unsigned i; 2682 2683 ntb = ctx; 2684 2685 if (ntb->peer_msix_good) 2686 goto msix_good; 2687 if (ntb->peer_msix_done) 2688 goto msix_done; 2689 2690 #ifndef EARLY_AP_STARTUP 2691 /* Block MSIX negotiation until SMP started and IRQ reshuffled. */ 2692 if (!msix_ready) 2693 goto reschedule; 2694 #endif 2695 2696 intel_ntb_get_msix_info(ntb); 2697 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 2698 intel_ntb_peer_spad_write(ntb->device, NTB_MSIX_DATA0 + i, 2699 ntb->msix_data[i].nmd_data); 2700 intel_ntb_peer_spad_write(ntb->device, NTB_MSIX_OFS0 + i, 2701 ntb->msix_data[i].nmd_ofs - ntb->msix_xlat); 2702 } 2703 intel_ntb_peer_spad_write(ntb->device, NTB_MSIX_GUARD, NTB_MSIX_VER_GUARD); 2704 2705 intel_ntb_spad_read(ntb->device, NTB_MSIX_GUARD, &val); 2706 if (val != NTB_MSIX_VER_GUARD) 2707 goto reschedule; 2708 2709 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 2710 intel_ntb_spad_read(ntb->device, NTB_MSIX_DATA0 + i, &val); 2711 intel_ntb_printf(2, "remote MSIX data(%u): 0x%x\n", i, val); 2712 ntb->peer_msix_data[i].nmd_data = val; 2713 intel_ntb_spad_read(ntb->device, NTB_MSIX_OFS0 + i, &val); 2714 intel_ntb_printf(2, "remote MSIX addr(%u): 0x%x\n", i, val); 2715 ntb->peer_msix_data[i].nmd_ofs = val; 2716 } 2717 2718 ntb->peer_msix_done = true; 2719 2720 msix_done: 2721 intel_ntb_peer_spad_write(ntb->device, NTB_MSIX_DONE, NTB_MSIX_RECEIVED); 2722 intel_ntb_spad_read(ntb->device, NTB_MSIX_DONE, &val); 2723 if (val != NTB_MSIX_RECEIVED) 2724 goto reschedule; 2725 2726 intel_ntb_spad_clear(ntb->device); 2727 ntb->peer_msix_good = true; 2728 /* Give peer time to see our NTB_MSIX_RECEIVED. */ 2729 goto reschedule; 2730 2731 msix_good: 2732 intel_ntb_poll_link(ntb); 2733 ntb_link_event(ntb->device); 2734 return; 2735 2736 reschedule: 2737 ntb->lnk_sta = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 2738 if (_xeon_link_is_up(ntb)) { 2739 callout_reset(&ntb->peer_msix_work, 2740 hz * (ntb->peer_msix_good ? 2 : 1) / 10, 2741 intel_ntb_exchange_msix, ntb); 2742 } else 2743 intel_ntb_spad_clear(ntb->device); 2744 } 2745 2746 /* 2747 * Public API to the rest of the OS 2748 */ 2749 2750 static uint8_t 2751 intel_ntb_spad_count(device_t dev) 2752 { 2753 struct ntb_softc *ntb = device_get_softc(dev); 2754 2755 return (ntb->spad_count); 2756 } 2757 2758 static uint8_t 2759 intel_ntb_mw_count(device_t dev) 2760 { 2761 struct ntb_softc *ntb = device_get_softc(dev); 2762 uint8_t res; 2763 2764 res = ntb->mw_count; 2765 if (ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0) 2766 res--; 2767 if (ntb->msix_mw_idx != B2B_MW_DISABLED) 2768 res--; 2769 return (res); 2770 } 2771 2772 static int 2773 intel_ntb_spad_write(device_t dev, unsigned int idx, uint32_t val) 2774 { 2775 struct ntb_softc *ntb = device_get_softc(dev); 2776 2777 if (idx >= ntb->spad_count) 2778 return (EINVAL); 2779 2780 intel_ntb_reg_write(4, ntb->self_reg->spad + idx * 4, val); 2781 2782 return (0); 2783 } 2784 2785 /* 2786 * Zeros the local scratchpad. 2787 */ 2788 static void 2789 intel_ntb_spad_clear(device_t dev) 2790 { 2791 struct ntb_softc *ntb = device_get_softc(dev); 2792 unsigned i; 2793 2794 for (i = 0; i < ntb->spad_count; i++) 2795 intel_ntb_spad_write(dev, i, 0); 2796 } 2797 2798 static int 2799 intel_ntb_spad_read(device_t dev, unsigned int idx, uint32_t *val) 2800 { 2801 struct ntb_softc *ntb = device_get_softc(dev); 2802 2803 if (idx >= ntb->spad_count) 2804 return (EINVAL); 2805 2806 *val = intel_ntb_reg_read(4, ntb->self_reg->spad + idx * 4); 2807 2808 return (0); 2809 } 2810 2811 static int 2812 intel_ntb_peer_spad_write(device_t dev, unsigned int idx, uint32_t val) 2813 { 2814 struct ntb_softc *ntb = device_get_softc(dev); 2815 2816 if (idx >= ntb->spad_count) 2817 return (EINVAL); 2818 2819 if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) 2820 intel_ntb_mw_write(4, XEON_SPAD_OFFSET + idx * 4, val); 2821 else 2822 intel_ntb_reg_write(4, ntb->peer_reg->spad + idx * 4, val); 2823 2824 return (0); 2825 } 2826 2827 static int 2828 intel_ntb_peer_spad_read(device_t dev, unsigned int idx, uint32_t *val) 2829 { 2830 struct ntb_softc *ntb = device_get_softc(dev); 2831 2832 if (idx >= ntb->spad_count) 2833 return (EINVAL); 2834 2835 if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) 2836 *val = intel_ntb_mw_read(4, XEON_SPAD_OFFSET + idx * 4); 2837 else 2838 *val = intel_ntb_reg_read(4, ntb->peer_reg->spad + idx * 4); 2839 2840 return (0); 2841 } 2842 2843 static int 2844 intel_ntb_mw_get_range(device_t dev, unsigned mw_idx, vm_paddr_t *base, 2845 caddr_t *vbase, size_t *size, size_t *align, size_t *align_size, 2846 bus_addr_t *plimit) 2847 { 2848 struct ntb_softc *ntb = device_get_softc(dev); 2849 struct ntb_pci_bar_info *bar; 2850 bus_addr_t limit; 2851 size_t bar_b2b_off; 2852 enum ntb_bar bar_num; 2853 2854 if (mw_idx >= intel_ntb_mw_count(dev)) 2855 return (EINVAL); 2856 mw_idx = intel_ntb_user_mw_to_idx(ntb, mw_idx); 2857 2858 bar_num = intel_ntb_mw_to_bar(ntb, mw_idx); 2859 bar = &ntb->bar_info[bar_num]; 2860 bar_b2b_off = 0; 2861 if (mw_idx == ntb->b2b_mw_idx) { 2862 KASSERT(ntb->b2b_off != 0, 2863 ("user shouldn't get non-shared b2b mw")); 2864 bar_b2b_off = ntb->b2b_off; 2865 } 2866 2867 if (bar_is_64bit(ntb, bar_num)) 2868 limit = BUS_SPACE_MAXADDR; 2869 else 2870 limit = BUS_SPACE_MAXADDR_32BIT; 2871 2872 if (base != NULL) 2873 *base = bar->pbase + bar_b2b_off; 2874 if (vbase != NULL) 2875 *vbase = bar->vbase + bar_b2b_off; 2876 if (size != NULL) 2877 *size = bar->size - bar_b2b_off; 2878 if (align != NULL) 2879 *align = bar->size; 2880 if (align_size != NULL) 2881 *align_size = 1; 2882 if (plimit != NULL) 2883 *plimit = limit; 2884 return (0); 2885 } 2886 2887 static int 2888 intel_ntb_mw_set_trans(device_t dev, unsigned idx, bus_addr_t addr, size_t size) 2889 { 2890 struct ntb_softc *ntb = device_get_softc(dev); 2891 struct ntb_pci_bar_info *bar; 2892 uint64_t base, limit, reg_val; 2893 size_t bar_size, mw_size; 2894 uint32_t base_reg, xlat_reg, limit_reg; 2895 enum ntb_bar bar_num; 2896 2897 if (idx >= intel_ntb_mw_count(dev)) 2898 return (EINVAL); 2899 idx = intel_ntb_user_mw_to_idx(ntb, idx); 2900 2901 bar_num = intel_ntb_mw_to_bar(ntb, idx); 2902 bar = &ntb->bar_info[bar_num]; 2903 2904 bar_size = bar->size; 2905 if (idx == ntb->b2b_mw_idx) 2906 mw_size = bar_size - ntb->b2b_off; 2907 else 2908 mw_size = bar_size; 2909 2910 /* Hardware requires that addr is aligned to bar size */ 2911 if ((addr & (bar_size - 1)) != 0) 2912 return (EINVAL); 2913 2914 if (size > mw_size) 2915 return (EINVAL); 2916 2917 bar_get_xlat_params(ntb, bar_num, &base_reg, &xlat_reg, &limit_reg); 2918 2919 limit = 0; 2920 if (bar_is_64bit(ntb, bar_num)) { 2921 base = intel_ntb_reg_read(8, base_reg) & BAR_HIGH_MASK; 2922 2923 if (limit_reg != 0 && size != mw_size) 2924 limit = base + size; 2925 2926 /* Set and verify translation address */ 2927 intel_ntb_reg_write(8, xlat_reg, addr); 2928 reg_val = intel_ntb_reg_read(8, xlat_reg) & BAR_HIGH_MASK; 2929 if (reg_val != addr) { 2930 intel_ntb_reg_write(8, xlat_reg, 0); 2931 return (EIO); 2932 } 2933 2934 /* Set and verify the limit */ 2935 intel_ntb_reg_write(8, limit_reg, limit); 2936 reg_val = intel_ntb_reg_read(8, limit_reg) & BAR_HIGH_MASK; 2937 if (reg_val != limit) { 2938 intel_ntb_reg_write(8, limit_reg, base); 2939 intel_ntb_reg_write(8, xlat_reg, 0); 2940 return (EIO); 2941 } 2942 } else { 2943 /* Configure 32-bit (split) BAR MW */ 2944 2945 if ((addr & UINT32_MAX) != addr) 2946 return (ERANGE); 2947 if (((addr + size) & UINT32_MAX) != (addr + size)) 2948 return (ERANGE); 2949 2950 base = intel_ntb_reg_read(4, base_reg) & BAR_HIGH_MASK; 2951 2952 if (limit_reg != 0 && size != mw_size) 2953 limit = base + size; 2954 2955 /* Set and verify translation address */ 2956 intel_ntb_reg_write(4, xlat_reg, addr); 2957 reg_val = intel_ntb_reg_read(4, xlat_reg) & BAR_HIGH_MASK; 2958 if (reg_val != addr) { 2959 intel_ntb_reg_write(4, xlat_reg, 0); 2960 return (EIO); 2961 } 2962 2963 /* Set and verify the limit */ 2964 intel_ntb_reg_write(4, limit_reg, limit); 2965 reg_val = intel_ntb_reg_read(4, limit_reg) & BAR_HIGH_MASK; 2966 if (reg_val != limit) { 2967 intel_ntb_reg_write(4, limit_reg, base); 2968 intel_ntb_reg_write(4, xlat_reg, 0); 2969 return (EIO); 2970 } 2971 } 2972 return (0); 2973 } 2974 2975 static int 2976 intel_ntb_mw_clear_trans(device_t dev, unsigned mw_idx) 2977 { 2978 2979 return (intel_ntb_mw_set_trans(dev, mw_idx, 0, 0)); 2980 } 2981 2982 static int 2983 intel_ntb_mw_get_wc(device_t dev, unsigned idx, vm_memattr_t *mode) 2984 { 2985 struct ntb_softc *ntb = device_get_softc(dev); 2986 struct ntb_pci_bar_info *bar; 2987 2988 if (idx >= intel_ntb_mw_count(dev)) 2989 return (EINVAL); 2990 idx = intel_ntb_user_mw_to_idx(ntb, idx); 2991 2992 bar = &ntb->bar_info[intel_ntb_mw_to_bar(ntb, idx)]; 2993 *mode = bar->map_mode; 2994 return (0); 2995 } 2996 2997 static int 2998 intel_ntb_mw_set_wc(device_t dev, unsigned idx, vm_memattr_t mode) 2999 { 3000 struct ntb_softc *ntb = device_get_softc(dev); 3001 3002 if (idx >= intel_ntb_mw_count(dev)) 3003 return (EINVAL); 3004 3005 idx = intel_ntb_user_mw_to_idx(ntb, idx); 3006 return (intel_ntb_mw_set_wc_internal(ntb, idx, mode)); 3007 } 3008 3009 static int 3010 intel_ntb_mw_set_wc_internal(struct ntb_softc *ntb, unsigned idx, vm_memattr_t mode) 3011 { 3012 struct ntb_pci_bar_info *bar; 3013 int rc; 3014 3015 bar = &ntb->bar_info[intel_ntb_mw_to_bar(ntb, idx)]; 3016 if (bar->map_mode == mode) 3017 return (0); 3018 3019 rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mode); 3020 if (rc == 0) 3021 bar->map_mode = mode; 3022 3023 return (rc); 3024 } 3025 3026 static void 3027 intel_ntb_peer_db_set(device_t dev, uint64_t bit) 3028 { 3029 struct ntb_softc *ntb = device_get_softc(dev); 3030 3031 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 3032 struct ntb_pci_bar_info *lapic; 3033 unsigned i; 3034 3035 lapic = ntb->peer_lapic_bar; 3036 3037 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 3038 if ((bit & intel_ntb_db_vector_mask(dev, i)) != 0) 3039 bus_space_write_4(lapic->pci_bus_tag, 3040 lapic->pci_bus_handle, 3041 ntb->peer_msix_data[i].nmd_ofs, 3042 ntb->peer_msix_data[i].nmd_data); 3043 } 3044 return; 3045 } 3046 3047 if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) { 3048 intel_ntb_mw_write(2, XEON_PDOORBELL_OFFSET, bit); 3049 return; 3050 } 3051 3052 db_iowrite(ntb, ntb->peer_reg->db_bell, bit); 3053 } 3054 3055 static int 3056 intel_ntb_peer_db_addr(device_t dev, bus_addr_t *db_addr, vm_size_t *db_size) 3057 { 3058 struct ntb_softc *ntb = device_get_softc(dev); 3059 struct ntb_pci_bar_info *bar; 3060 uint64_t regoff; 3061 3062 KASSERT((db_addr != NULL && db_size != NULL), ("must be non-NULL")); 3063 3064 if (!HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) { 3065 bar = &ntb->bar_info[NTB_CONFIG_BAR]; 3066 regoff = ntb->peer_reg->db_bell; 3067 } else { 3068 KASSERT(ntb->b2b_mw_idx != B2B_MW_DISABLED, 3069 ("invalid b2b idx")); 3070 3071 bar = &ntb->bar_info[intel_ntb_mw_to_bar(ntb, ntb->b2b_mw_idx)]; 3072 regoff = XEON_PDOORBELL_OFFSET; 3073 } 3074 KASSERT(bar->pci_bus_tag != X86_BUS_SPACE_IO, ("uh oh")); 3075 3076 /* HACK: Specific to current x86 bus implementation. */ 3077 *db_addr = ((uint64_t)bar->pci_bus_handle + regoff); 3078 *db_size = ntb->reg->db_size; 3079 return (0); 3080 } 3081 3082 static uint64_t 3083 intel_ntb_db_valid_mask(device_t dev) 3084 { 3085 struct ntb_softc *ntb = device_get_softc(dev); 3086 3087 return (ntb->db_valid_mask); 3088 } 3089 3090 static int 3091 intel_ntb_db_vector_count(device_t dev) 3092 { 3093 struct ntb_softc *ntb = device_get_softc(dev); 3094 3095 return (ntb->db_vec_count); 3096 } 3097 3098 static uint64_t 3099 intel_ntb_db_vector_mask(device_t dev, uint32_t vector) 3100 { 3101 struct ntb_softc *ntb = device_get_softc(dev); 3102 3103 if (vector > ntb->db_vec_count) 3104 return (0); 3105 return (ntb->db_valid_mask & intel_ntb_vec_mask(ntb, vector)); 3106 } 3107 3108 static bool 3109 intel_ntb_link_is_up(device_t dev, enum ntb_speed *speed, enum ntb_width *width) 3110 { 3111 struct ntb_softc *ntb = device_get_softc(dev); 3112 3113 if (speed != NULL) 3114 *speed = intel_ntb_link_sta_speed(ntb); 3115 if (width != NULL) 3116 *width = intel_ntb_link_sta_width(ntb); 3117 return (link_is_up(ntb)); 3118 } 3119 3120 static void 3121 save_bar_parameters(struct ntb_pci_bar_info *bar) 3122 { 3123 3124 bar->pci_bus_tag = rman_get_bustag(bar->pci_resource); 3125 bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource); 3126 bar->pbase = rman_get_start(bar->pci_resource); 3127 bar->size = rman_get_size(bar->pci_resource); 3128 bar->vbase = rman_get_virtual(bar->pci_resource); 3129 } 3130 3131 static device_method_t ntb_intel_methods[] = { 3132 /* Device interface */ 3133 DEVMETHOD(device_probe, intel_ntb_probe), 3134 DEVMETHOD(device_attach, intel_ntb_attach), 3135 DEVMETHOD(device_detach, intel_ntb_detach), 3136 /* Bus interface */ 3137 DEVMETHOD(bus_child_location_str, ntb_child_location_str), 3138 DEVMETHOD(bus_print_child, ntb_print_child), 3139 DEVMETHOD(bus_get_dma_tag, ntb_get_dma_tag), 3140 /* NTB interface */ 3141 DEVMETHOD(ntb_port_number, intel_ntb_port_number), 3142 DEVMETHOD(ntb_peer_port_count, intel_ntb_peer_port_count), 3143 DEVMETHOD(ntb_peer_port_number, intel_ntb_peer_port_number), 3144 DEVMETHOD(ntb_peer_port_idx, intel_ntb_peer_port_idx), 3145 DEVMETHOD(ntb_link_is_up, intel_ntb_link_is_up), 3146 DEVMETHOD(ntb_link_enable, intel_ntb_link_enable), 3147 DEVMETHOD(ntb_link_disable, intel_ntb_link_disable), 3148 DEVMETHOD(ntb_link_enabled, intel_ntb_link_enabled), 3149 DEVMETHOD(ntb_mw_count, intel_ntb_mw_count), 3150 DEVMETHOD(ntb_mw_get_range, intel_ntb_mw_get_range), 3151 DEVMETHOD(ntb_mw_set_trans, intel_ntb_mw_set_trans), 3152 DEVMETHOD(ntb_mw_clear_trans, intel_ntb_mw_clear_trans), 3153 DEVMETHOD(ntb_mw_get_wc, intel_ntb_mw_get_wc), 3154 DEVMETHOD(ntb_mw_set_wc, intel_ntb_mw_set_wc), 3155 DEVMETHOD(ntb_spad_count, intel_ntb_spad_count), 3156 DEVMETHOD(ntb_spad_clear, intel_ntb_spad_clear), 3157 DEVMETHOD(ntb_spad_write, intel_ntb_spad_write), 3158 DEVMETHOD(ntb_spad_read, intel_ntb_spad_read), 3159 DEVMETHOD(ntb_peer_spad_write, intel_ntb_peer_spad_write), 3160 DEVMETHOD(ntb_peer_spad_read, intel_ntb_peer_spad_read), 3161 DEVMETHOD(ntb_db_valid_mask, intel_ntb_db_valid_mask), 3162 DEVMETHOD(ntb_db_vector_count, intel_ntb_db_vector_count), 3163 DEVMETHOD(ntb_db_vector_mask, intel_ntb_db_vector_mask), 3164 DEVMETHOD(ntb_db_clear, intel_ntb_db_clear), 3165 DEVMETHOD(ntb_db_clear_mask, intel_ntb_db_clear_mask), 3166 DEVMETHOD(ntb_db_read, intel_ntb_db_read), 3167 DEVMETHOD(ntb_db_set_mask, intel_ntb_db_set_mask), 3168 DEVMETHOD(ntb_peer_db_addr, intel_ntb_peer_db_addr), 3169 DEVMETHOD(ntb_peer_db_set, intel_ntb_peer_db_set), 3170 DEVMETHOD_END 3171 }; 3172 3173 static DEFINE_CLASS_0(ntb_hw, ntb_intel_driver, ntb_intel_methods, 3174 sizeof(struct ntb_softc)); 3175 DRIVER_MODULE(ntb_hw_intel, pci, ntb_intel_driver, ntb_hw_devclass, NULL, NULL); 3176 MODULE_DEPEND(ntb_hw_intel, ntb, 1, 1, 1); 3177 MODULE_VERSION(ntb_hw_intel, 1); 3178 MODULE_PNP_INFO("W32:vendor/device;D:#", pci, ntb_hw_intel, pci_ids, 3179 nitems(pci_ids)); 3180