1 /*- 2 * Copyright (c) 2016-2017 Alexander Motin <mav@FreeBSD.org> 3 * Copyright (C) 2013 Intel Corporation 4 * Copyright (C) 2015 EMC Corporation 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 /* 30 * The Non-Transparent Bridge (NTB) is a device that allows you to connect 31 * two or more systems using a PCI-e links, providing remote memory access. 32 * 33 * This module contains a driver for NTB hardware in Intel Xeon/Atom CPUs. 34 * 35 * NOTE: Much of the code in this module is shared with Linux. Any patches may 36 * be picked up and redistributed in Linux with a dual GPL/BSD license. 37 */ 38 39 #include <sys/cdefs.h> 40 __FBSDID("$FreeBSD$"); 41 42 #include <sys/param.h> 43 #include <sys/kernel.h> 44 #include <sys/systm.h> 45 #include <sys/bus.h> 46 #include <sys/endian.h> 47 #include <sys/interrupt.h> 48 #include <sys/lock.h> 49 #include <sys/malloc.h> 50 #include <sys/module.h> 51 #include <sys/mutex.h> 52 #include <sys/pciio.h> 53 #include <sys/taskqueue.h> 54 #include <sys/tree.h> 55 #include <sys/queue.h> 56 #include <sys/rman.h> 57 #include <sys/sbuf.h> 58 #include <sys/sysctl.h> 59 #include <vm/vm.h> 60 #include <vm/pmap.h> 61 #include <machine/bus.h> 62 #include <machine/intr_machdep.h> 63 #include <machine/resource.h> 64 #include <dev/pci/pcireg.h> 65 #include <dev/pci/pcivar.h> 66 #include <dev/iommu/iommu.h> 67 68 #include "ntb_hw_intel.h" 69 #include "../ntb.h" 70 71 #define MAX_MSIX_INTERRUPTS \ 72 MAX(MAX(XEON_DB_COUNT, ATOM_DB_COUNT), XEON_GEN3_DB_COUNT) 73 74 #define NTB_HB_TIMEOUT 1 /* second */ 75 #define ATOM_LINK_RECOVERY_TIME 500 /* ms */ 76 #define BAR_HIGH_MASK (~((1ull << 12) - 1)) 77 78 #define NTB_MSIX_VER_GUARD 0xaabbccdd 79 #define NTB_MSIX_RECEIVED 0xe0f0e0f0 80 81 /* 82 * PCI constants could be somewhere more generic, but aren't defined/used in 83 * pci.c. 84 */ 85 #define PCI_MSIX_ENTRY_SIZE 16 86 #define PCI_MSIX_ENTRY_LOWER_ADDR 0 87 #define PCI_MSIX_ENTRY_UPPER_ADDR 4 88 #define PCI_MSIX_ENTRY_DATA 8 89 90 enum ntb_device_type { 91 NTB_XEON_GEN1, 92 NTB_XEON_GEN3, 93 NTB_ATOM 94 }; 95 96 /* ntb_conn_type are hardware numbers, cannot change. */ 97 enum ntb_conn_type { 98 NTB_CONN_TRANSPARENT = 0, 99 NTB_CONN_B2B = 1, 100 NTB_CONN_RP = 2, 101 }; 102 103 enum ntb_b2b_direction { 104 NTB_DEV_USD = 0, 105 NTB_DEV_DSD = 1, 106 }; 107 108 enum ntb_bar { 109 NTB_CONFIG_BAR = 0, 110 NTB_B2B_BAR_1, 111 NTB_B2B_BAR_2, 112 NTB_B2B_BAR_3, 113 NTB_MAX_BARS 114 }; 115 116 enum { 117 NTB_MSIX_GUARD = 0, 118 NTB_MSIX_DATA0, 119 NTB_MSIX_DATA1, 120 NTB_MSIX_DATA2, 121 NTB_MSIX_OFS0, 122 NTB_MSIX_OFS1, 123 NTB_MSIX_OFS2, 124 NTB_MSIX_DONE, 125 NTB_MAX_MSIX_SPAD 126 }; 127 128 /* Device features and workarounds */ 129 #define HAS_FEATURE(ntb, feature) \ 130 (((ntb)->features & (feature)) != 0) 131 132 struct ntb_hw_info { 133 uint32_t device_id; 134 const char *desc; 135 enum ntb_device_type type; 136 uint32_t features; 137 }; 138 139 struct ntb_pci_bar_info { 140 bus_space_tag_t pci_bus_tag; 141 bus_space_handle_t pci_bus_handle; 142 int pci_resource_id; 143 struct resource *pci_resource; 144 vm_paddr_t pbase; 145 caddr_t vbase; 146 vm_size_t size; 147 vm_memattr_t map_mode; 148 149 /* Configuration register offsets */ 150 uint32_t psz_off; 151 uint32_t ssz_off; 152 uint32_t pbarxlat_off; 153 }; 154 155 struct ntb_int_info { 156 struct resource *res; 157 int rid; 158 void *tag; 159 }; 160 161 struct ntb_vec { 162 struct ntb_softc *ntb; 163 uint32_t num; 164 unsigned masked; 165 }; 166 167 struct ntb_reg { 168 uint32_t ntb_ctl; 169 uint32_t lnk_sta; 170 uint8_t db_size; 171 unsigned mw_bar[NTB_MAX_BARS]; 172 }; 173 174 struct ntb_alt_reg { 175 uint32_t db_bell; 176 uint32_t db_mask; 177 uint32_t spad; 178 }; 179 180 struct ntb_xlat_reg { 181 uint32_t bar0_base; 182 uint32_t bar2_base; 183 uint32_t bar4_base; 184 uint32_t bar5_base; 185 186 uint32_t bar2_xlat; 187 uint32_t bar4_xlat; 188 uint32_t bar5_xlat; 189 190 uint32_t bar2_limit; 191 uint32_t bar4_limit; 192 uint32_t bar5_limit; 193 }; 194 195 struct ntb_b2b_addr { 196 uint64_t bar0_addr; 197 uint64_t bar2_addr64; 198 uint64_t bar4_addr64; 199 uint64_t bar4_addr32; 200 uint64_t bar5_addr32; 201 }; 202 203 struct ntb_msix_data { 204 uint32_t nmd_ofs; 205 uint32_t nmd_data; 206 }; 207 208 struct ntb_softc { 209 /* ntb.c context. Do not move! Must go first! */ 210 void *ntb_store; 211 212 device_t device; 213 enum ntb_device_type type; 214 uint32_t features; 215 216 struct ntb_pci_bar_info bar_info[NTB_MAX_BARS]; 217 struct ntb_int_info int_info[MAX_MSIX_INTERRUPTS]; 218 uint32_t allocated_interrupts; 219 220 struct ntb_msix_data peer_msix_data[XEON_NONLINK_DB_MSIX_BITS]; 221 struct ntb_msix_data msix_data[XEON_NONLINK_DB_MSIX_BITS]; 222 bool peer_msix_good; 223 bool peer_msix_done; 224 struct ntb_pci_bar_info *peer_lapic_bar; 225 struct callout peer_msix_work; 226 227 bus_dma_tag_t bar0_dma_tag; 228 bus_dmamap_t bar0_dma_map; 229 230 struct callout heartbeat_timer; 231 struct callout lr_timer; 232 233 struct ntb_vec *msix_vec; 234 235 uint32_t ppd; 236 enum ntb_conn_type conn_type; 237 enum ntb_b2b_direction dev_type; 238 239 /* Offset of peer bar0 in B2B BAR */ 240 uint64_t b2b_off; 241 /* Memory window used to access peer bar0 */ 242 #define B2B_MW_DISABLED UINT8_MAX 243 uint8_t b2b_mw_idx; 244 uint32_t msix_xlat; 245 uint8_t msix_mw_idx; 246 247 uint8_t mw_count; 248 uint8_t spad_count; 249 uint8_t db_count; 250 uint8_t db_vec_count; 251 uint8_t db_vec_shift; 252 253 /* Protects local db_mask. */ 254 #define DB_MASK_LOCK(sc) mtx_lock_spin(&(sc)->db_mask_lock) 255 #define DB_MASK_UNLOCK(sc) mtx_unlock_spin(&(sc)->db_mask_lock) 256 #define DB_MASK_ASSERT(sc,f) mtx_assert(&(sc)->db_mask_lock, (f)) 257 struct mtx db_mask_lock; 258 259 volatile uint32_t ntb_ctl; 260 volatile uint32_t lnk_sta; 261 262 uint64_t db_valid_mask; 263 uint64_t db_link_mask; 264 uint64_t db_mask; 265 uint64_t fake_db; /* NTB_SB01BASE_LOCKUP*/ 266 uint64_t force_db; /* NTB_SB01BASE_LOCKUP*/ 267 268 int last_ts; /* ticks @ last irq */ 269 270 const struct ntb_reg *reg; 271 const struct ntb_alt_reg *self_reg; 272 const struct ntb_alt_reg *peer_reg; 273 const struct ntb_xlat_reg *xlat_reg; 274 }; 275 276 #ifdef __i386__ 277 static __inline uint64_t 278 bus_space_read_8(bus_space_tag_t tag, bus_space_handle_t handle, 279 bus_size_t offset) 280 { 281 282 return (bus_space_read_4(tag, handle, offset) | 283 ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32); 284 } 285 286 static __inline void 287 bus_space_write_8(bus_space_tag_t tag, bus_space_handle_t handle, 288 bus_size_t offset, uint64_t val) 289 { 290 291 bus_space_write_4(tag, handle, offset, val); 292 bus_space_write_4(tag, handle, offset + 4, val >> 32); 293 } 294 #endif 295 296 #define intel_ntb_bar_read(SIZE, bar, offset) \ 297 bus_space_read_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 298 ntb->bar_info[(bar)].pci_bus_handle, (offset)) 299 #define intel_ntb_bar_write(SIZE, bar, offset, val) \ 300 bus_space_write_ ## SIZE (ntb->bar_info[(bar)].pci_bus_tag, \ 301 ntb->bar_info[(bar)].pci_bus_handle, (offset), (val)) 302 #define intel_ntb_reg_read(SIZE, offset) \ 303 intel_ntb_bar_read(SIZE, NTB_CONFIG_BAR, offset) 304 #define intel_ntb_reg_write(SIZE, offset, val) \ 305 intel_ntb_bar_write(SIZE, NTB_CONFIG_BAR, offset, val) 306 #define intel_ntb_mw_read(SIZE, offset) \ 307 intel_ntb_bar_read(SIZE, intel_ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \ 308 offset) 309 #define intel_ntb_mw_write(SIZE, offset, val) \ 310 intel_ntb_bar_write(SIZE, intel_ntb_mw_to_bar(ntb, ntb->b2b_mw_idx), \ 311 offset, val) 312 313 static int intel_ntb_probe(device_t device); 314 static int intel_ntb_attach(device_t device); 315 static int intel_ntb_detach(device_t device); 316 static uint64_t intel_ntb_db_valid_mask(device_t dev); 317 static void intel_ntb_spad_clear(device_t dev); 318 static uint64_t intel_ntb_db_vector_mask(device_t dev, uint32_t vector); 319 static bool intel_ntb_link_is_up(device_t dev, enum ntb_speed *speed, 320 enum ntb_width *width); 321 static int intel_ntb_link_enable(device_t dev, enum ntb_speed speed, 322 enum ntb_width width); 323 static int intel_ntb_link_disable(device_t dev); 324 static int intel_ntb_spad_read(device_t dev, unsigned int idx, uint32_t *val); 325 static int intel_ntb_peer_spad_write(device_t dev, unsigned int idx, uint32_t val); 326 327 static unsigned intel_ntb_user_mw_to_idx(struct ntb_softc *, unsigned uidx); 328 static inline enum ntb_bar intel_ntb_mw_to_bar(struct ntb_softc *, unsigned mw); 329 static inline bool bar_is_64bit(struct ntb_softc *, enum ntb_bar); 330 static inline void bar_get_xlat_params(struct ntb_softc *, enum ntb_bar, 331 uint32_t *base, uint32_t *xlat, uint32_t *lmt); 332 static int intel_ntb_map_pci_bars(struct ntb_softc *ntb); 333 static int intel_ntb_mw_set_wc_internal(struct ntb_softc *, unsigned idx, 334 vm_memattr_t); 335 static void print_map_success(struct ntb_softc *, struct ntb_pci_bar_info *, 336 const char *); 337 static int map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar); 338 static int map_memory_window_bar(struct ntb_softc *ntb, 339 struct ntb_pci_bar_info *bar); 340 static void intel_ntb_unmap_pci_bar(struct ntb_softc *ntb); 341 static int intel_ntb_remap_msix(device_t, uint32_t desired, uint32_t avail); 342 static int intel_ntb_init_isr(struct ntb_softc *ntb); 343 static int intel_ntb_xeon_gen3_init_isr(struct ntb_softc *ntb); 344 static int intel_ntb_setup_legacy_interrupt(struct ntb_softc *ntb); 345 static int intel_ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors); 346 static void intel_ntb_teardown_interrupts(struct ntb_softc *ntb); 347 static inline uint64_t intel_ntb_vec_mask(struct ntb_softc *, uint64_t db_vector); 348 static void intel_ntb_interrupt(struct ntb_softc *, uint32_t vec); 349 static void ndev_vec_isr(void *arg); 350 static void ndev_irq_isr(void *arg); 351 static inline uint64_t db_ioread(struct ntb_softc *, uint64_t regoff); 352 static inline void db_iowrite(struct ntb_softc *, uint64_t regoff, uint64_t); 353 static inline void db_iowrite_raw(struct ntb_softc *, uint64_t regoff, uint64_t); 354 static int intel_ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors); 355 static void intel_ntb_free_msix_vec(struct ntb_softc *ntb); 356 static void intel_ntb_get_msix_info(struct ntb_softc *ntb); 357 static void intel_ntb_exchange_msix(void *); 358 static struct ntb_hw_info *intel_ntb_get_device_info(uint32_t device_id); 359 static void intel_ntb_detect_max_mw(struct ntb_softc *ntb); 360 static int intel_ntb_detect_xeon(struct ntb_softc *ntb); 361 static int intel_ntb_detect_xeon_gen3(struct ntb_softc *ntb); 362 static int intel_ntb_detect_atom(struct ntb_softc *ntb); 363 static int intel_ntb_xeon_init_dev(struct ntb_softc *ntb); 364 static int intel_ntb_xeon_gen3_init_dev(struct ntb_softc *ntb); 365 static int intel_ntb_atom_init_dev(struct ntb_softc *ntb); 366 static void intel_ntb_teardown_xeon(struct ntb_softc *ntb); 367 static void configure_atom_secondary_side_bars(struct ntb_softc *ntb); 368 static void xeon_reset_sbar_size(struct ntb_softc *, enum ntb_bar idx, 369 enum ntb_bar regbar); 370 static void xeon_set_sbar_base_and_limit(struct ntb_softc *, 371 uint64_t base_addr, enum ntb_bar idx, enum ntb_bar regbar); 372 static void xeon_set_pbar_xlat(struct ntb_softc *, uint64_t base_addr, 373 enum ntb_bar idx); 374 static int xeon_setup_b2b_mw(struct ntb_softc *, 375 const struct ntb_b2b_addr *addr, const struct ntb_b2b_addr *peer_addr); 376 static int xeon_gen3_setup_b2b_mw(struct ntb_softc *); 377 static int intel_ntb_mw_set_trans(device_t dev, unsigned idx, bus_addr_t addr, 378 size_t size); 379 static inline bool link_is_up(struct ntb_softc *ntb); 380 static inline bool _xeon_link_is_up(struct ntb_softc *ntb); 381 static inline bool atom_link_is_err(struct ntb_softc *ntb); 382 static inline enum ntb_speed intel_ntb_link_sta_speed(struct ntb_softc *); 383 static inline enum ntb_width intel_ntb_link_sta_width(struct ntb_softc *); 384 static void atom_link_hb(void *arg); 385 static void recover_atom_link(void *arg); 386 static bool intel_ntb_poll_link(struct ntb_softc *ntb); 387 static void save_bar_parameters(struct ntb_pci_bar_info *bar); 388 static void intel_ntb_sysctl_init(struct ntb_softc *); 389 static int sysctl_handle_features(SYSCTL_HANDLER_ARGS); 390 static int sysctl_handle_link_admin(SYSCTL_HANDLER_ARGS); 391 static int sysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS); 392 static int sysctl_handle_link_status(SYSCTL_HANDLER_ARGS); 393 static int sysctl_handle_register(SYSCTL_HANDLER_ARGS); 394 395 static unsigned g_ntb_hw_debug_level; 396 SYSCTL_UINT(_hw_ntb, OID_AUTO, debug_level, CTLFLAG_RWTUN, 397 &g_ntb_hw_debug_level, 0, "ntb_hw log level -- higher is more verbose"); 398 #define intel_ntb_printf(lvl, ...) do { \ 399 if ((lvl) <= g_ntb_hw_debug_level) { \ 400 device_printf(ntb->device, __VA_ARGS__); \ 401 } \ 402 } while (0) 403 404 #define _NTB_PAT_UC 0 405 #define _NTB_PAT_WC 1 406 #define _NTB_PAT_WT 4 407 #define _NTB_PAT_WP 5 408 #define _NTB_PAT_WB 6 409 #define _NTB_PAT_UCM 7 410 static unsigned g_ntb_mw_pat = _NTB_PAT_UC; 411 SYSCTL_UINT(_hw_ntb, OID_AUTO, default_mw_pat, CTLFLAG_RDTUN, 412 &g_ntb_mw_pat, 0, "Configure the default memory window cache flags (PAT): " 413 "UC: " __XSTRING(_NTB_PAT_UC) ", " 414 "WC: " __XSTRING(_NTB_PAT_WC) ", " 415 "WT: " __XSTRING(_NTB_PAT_WT) ", " 416 "WP: " __XSTRING(_NTB_PAT_WP) ", " 417 "WB: " __XSTRING(_NTB_PAT_WB) ", " 418 "UC-: " __XSTRING(_NTB_PAT_UCM)); 419 420 static inline vm_memattr_t 421 intel_ntb_pat_flags(void) 422 { 423 424 switch (g_ntb_mw_pat) { 425 case _NTB_PAT_WC: 426 return (VM_MEMATTR_WRITE_COMBINING); 427 case _NTB_PAT_WT: 428 return (VM_MEMATTR_WRITE_THROUGH); 429 case _NTB_PAT_WP: 430 return (VM_MEMATTR_WRITE_PROTECTED); 431 case _NTB_PAT_WB: 432 return (VM_MEMATTR_WRITE_BACK); 433 case _NTB_PAT_UCM: 434 return (VM_MEMATTR_WEAK_UNCACHEABLE); 435 case _NTB_PAT_UC: 436 /* FALLTHROUGH */ 437 default: 438 return (VM_MEMATTR_UNCACHEABLE); 439 } 440 } 441 442 /* 443 * Well, this obviously doesn't belong here, but it doesn't seem to exist 444 * anywhere better yet. 445 */ 446 static inline const char * 447 intel_ntb_vm_memattr_to_str(vm_memattr_t pat) 448 { 449 450 switch (pat) { 451 case VM_MEMATTR_WRITE_COMBINING: 452 return ("WRITE_COMBINING"); 453 case VM_MEMATTR_WRITE_THROUGH: 454 return ("WRITE_THROUGH"); 455 case VM_MEMATTR_WRITE_PROTECTED: 456 return ("WRITE_PROTECTED"); 457 case VM_MEMATTR_WRITE_BACK: 458 return ("WRITE_BACK"); 459 case VM_MEMATTR_WEAK_UNCACHEABLE: 460 return ("UNCACHED"); 461 case VM_MEMATTR_UNCACHEABLE: 462 return ("UNCACHEABLE"); 463 default: 464 return ("UNKNOWN"); 465 } 466 } 467 468 static int g_ntb_msix_idx = 1; 469 SYSCTL_INT(_hw_ntb, OID_AUTO, msix_mw_idx, CTLFLAG_RDTUN, &g_ntb_msix_idx, 470 0, "Use this memory window to access the peer MSIX message complex on " 471 "certain Xeon-based NTB systems, as a workaround for a hardware errata. " 472 "Like b2b_mw_idx, negative values index from the last available memory " 473 "window. (Applies on Xeon platforms with SB01BASE_LOCKUP errata.)"); 474 475 static int g_ntb_mw_idx = -1; 476 SYSCTL_INT(_hw_ntb, OID_AUTO, b2b_mw_idx, CTLFLAG_RDTUN, &g_ntb_mw_idx, 477 0, "Use this memory window to access the peer NTB registers. A " 478 "non-negative value starts from the first MW index; a negative value " 479 "starts from the last MW index. The default is -1, i.e., the last " 480 "available memory window. Both sides of the NTB MUST set the same " 481 "value here! (Applies on Xeon platforms with SDOORBELL_LOCKUP errata.)"); 482 483 /* Hardware owns the low 16 bits of features. */ 484 #define NTB_BAR_SIZE_4K (1 << 0) 485 #define NTB_SDOORBELL_LOCKUP (1 << 1) 486 #define NTB_SB01BASE_LOCKUP (1 << 2) 487 #define NTB_B2BDOORBELL_BIT14 (1 << 3) 488 /* Software/configuration owns the top 16 bits. */ 489 #define NTB_SPLIT_BAR (1ull << 16) 490 #define NTB_ONE_MSIX (1ull << 17) 491 492 #define NTB_FEATURES_STR \ 493 "\20\21SPLIT_BAR4\04B2B_DOORBELL_BIT14\03SB01BASE_LOCKUP" \ 494 "\02SDOORBELL_LOCKUP\01BAR_SIZE_4K" 495 496 static struct ntb_hw_info pci_ids[] = { 497 /* XXX: PS/SS IDs left out until they are supported. */ 498 { 0x0C4E8086, "BWD Atom Processor S1200 Non-Transparent Bridge B2B", 499 NTB_ATOM, 0 }, 500 501 { 0x37258086, "JSF Xeon C35xx/C55xx Non-Transparent Bridge B2B", 502 NTB_XEON_GEN1, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 503 { 0x3C0D8086, "SNB Xeon E5/Core i7 Non-Transparent Bridge B2B", 504 NTB_XEON_GEN1, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 }, 505 { 0x0E0D8086, "IVT Xeon E5 V2 Non-Transparent Bridge B2B", 506 NTB_XEON_GEN1, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 507 NTB_SB01BASE_LOCKUP | NTB_BAR_SIZE_4K }, 508 { 0x2F0D8086, "HSX Xeon E5 V3 Non-Transparent Bridge B2B", 509 NTB_XEON_GEN1, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 510 NTB_SB01BASE_LOCKUP }, 511 { 0x6F0D8086, "BDX Xeon E5 V4 Non-Transparent Bridge B2B", 512 NTB_XEON_GEN1, NTB_SDOORBELL_LOCKUP | NTB_B2BDOORBELL_BIT14 | 513 NTB_SB01BASE_LOCKUP }, 514 515 { 0x201C8086, "SKL Xeon E5 V5 Non-Transparent Bridge B2B", 516 NTB_XEON_GEN3, 0 }, 517 }; 518 519 static const struct ntb_reg atom_reg = { 520 .ntb_ctl = ATOM_NTBCNTL_OFFSET, 521 .lnk_sta = ATOM_LINK_STATUS_OFFSET, 522 .db_size = sizeof(uint64_t), 523 .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 }, 524 }; 525 526 static const struct ntb_alt_reg atom_pri_reg = { 527 .db_bell = ATOM_PDOORBELL_OFFSET, 528 .db_mask = ATOM_PDBMSK_OFFSET, 529 .spad = ATOM_SPAD_OFFSET, 530 }; 531 532 static const struct ntb_alt_reg atom_b2b_reg = { 533 .db_bell = ATOM_B2B_DOORBELL_OFFSET, 534 .spad = ATOM_B2B_SPAD_OFFSET, 535 }; 536 537 static const struct ntb_xlat_reg atom_sec_xlat = { 538 #if 0 539 /* "FIXME" says the Linux driver. */ 540 .bar0_base = ATOM_SBAR0BASE_OFFSET, 541 .bar2_base = ATOM_SBAR2BASE_OFFSET, 542 .bar4_base = ATOM_SBAR4BASE_OFFSET, 543 544 .bar2_limit = ATOM_SBAR2LMT_OFFSET, 545 .bar4_limit = ATOM_SBAR4LMT_OFFSET, 546 #endif 547 548 .bar2_xlat = ATOM_SBAR2XLAT_OFFSET, 549 .bar4_xlat = ATOM_SBAR4XLAT_OFFSET, 550 }; 551 552 static const struct ntb_reg xeon_reg = { 553 .ntb_ctl = XEON_NTBCNTL_OFFSET, 554 .lnk_sta = XEON_LINK_STATUS_OFFSET, 555 .db_size = sizeof(uint16_t), 556 .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2, NTB_B2B_BAR_3 }, 557 }; 558 559 static const struct ntb_alt_reg xeon_pri_reg = { 560 .db_bell = XEON_PDOORBELL_OFFSET, 561 .db_mask = XEON_PDBMSK_OFFSET, 562 .spad = XEON_SPAD_OFFSET, 563 }; 564 565 static const struct ntb_alt_reg xeon_b2b_reg = { 566 .db_bell = XEON_B2B_DOORBELL_OFFSET, 567 .spad = XEON_B2B_SPAD_OFFSET, 568 }; 569 570 static const struct ntb_xlat_reg xeon_sec_xlat = { 571 .bar0_base = XEON_SBAR0BASE_OFFSET, 572 .bar2_base = XEON_SBAR2BASE_OFFSET, 573 .bar4_base = XEON_SBAR4BASE_OFFSET, 574 .bar5_base = XEON_SBAR5BASE_OFFSET, 575 576 .bar2_limit = XEON_SBAR2LMT_OFFSET, 577 .bar4_limit = XEON_SBAR4LMT_OFFSET, 578 .bar5_limit = XEON_SBAR5LMT_OFFSET, 579 580 .bar2_xlat = XEON_SBAR2XLAT_OFFSET, 581 .bar4_xlat = XEON_SBAR4XLAT_OFFSET, 582 .bar5_xlat = XEON_SBAR5XLAT_OFFSET, 583 }; 584 585 static struct ntb_b2b_addr xeon_b2b_usd_addr = { 586 .bar0_addr = XEON_B2B_BAR0_ADDR, 587 .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 588 .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 589 .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 590 .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 591 }; 592 593 static struct ntb_b2b_addr xeon_b2b_dsd_addr = { 594 .bar0_addr = XEON_B2B_BAR0_ADDR, 595 .bar2_addr64 = XEON_B2B_BAR2_ADDR64, 596 .bar4_addr64 = XEON_B2B_BAR4_ADDR64, 597 .bar4_addr32 = XEON_B2B_BAR4_ADDR32, 598 .bar5_addr32 = XEON_B2B_BAR5_ADDR32, 599 }; 600 601 static const struct ntb_reg xeon_gen3_reg = { 602 .ntb_ctl = XEON_GEN3_REG_IMNTB_CTRL, 603 .lnk_sta = XEON_GEN3_INT_LNK_STS_OFFSET, 604 .db_size = sizeof(uint32_t), 605 .mw_bar = { NTB_B2B_BAR_1, NTB_B2B_BAR_2 }, 606 }; 607 608 static const struct ntb_alt_reg xeon_gen3_pri_reg = { 609 .db_bell = XEON_GEN3_REG_EMDOORBELL, 610 .db_mask = XEON_GEN3_REG_IMINT_DISABLE, 611 .spad = XEON_GEN3_REG_IMSPAD, 612 }; 613 614 static const struct ntb_alt_reg xeon_gen3_b2b_reg = { 615 .db_bell = XEON_GEN3_REG_IMDOORBELL, 616 .db_mask = XEON_GEN3_REG_EMINT_DISABLE, 617 .spad = XEON_GEN3_REG_IMB2B_SSPAD, 618 }; 619 620 static const struct ntb_xlat_reg xeon_gen3_sec_xlat = { 621 .bar0_base = XEON_GEN3_EXT_REG_BAR0BASE, 622 .bar2_base = XEON_GEN3_EXT_REG_BAR1BASE, 623 .bar4_base = XEON_GEN3_EXT_REG_BAR2BASE, 624 625 .bar2_limit = XEON_GEN3_REG_IMBAR1XLIMIT, 626 .bar4_limit = XEON_GEN3_REG_IMBAR2XLIMIT, 627 628 .bar2_xlat = XEON_GEN3_REG_IMBAR1XBASE, 629 .bar4_xlat = XEON_GEN3_REG_IMBAR2XBASE, 630 }; 631 632 SYSCTL_NODE(_hw_ntb, OID_AUTO, xeon_b2b, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 633 "B2B MW segment overrides -- MUST be the same on both sides"); 634 635 SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar2_addr64, CTLFLAG_RDTUN, 636 &xeon_b2b_usd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 637 "hardware, use this 64-bit address on the bus between the NTB devices for " 638 "the window at BAR2, on the upstream side of the link. MUST be the same " 639 "address on both sides."); 640 SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr64, CTLFLAG_RDTUN, 641 &xeon_b2b_usd_addr.bar4_addr64, 0, "See usd_bar2_addr64, but BAR4."); 642 SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar4_addr32, CTLFLAG_RDTUN, 643 &xeon_b2b_usd_addr.bar4_addr32, 0, "See usd_bar2_addr64, but BAR4 " 644 "(split-BAR mode)."); 645 SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, usd_bar5_addr32, CTLFLAG_RDTUN, 646 &xeon_b2b_usd_addr.bar5_addr32, 0, "See usd_bar2_addr64, but BAR5 " 647 "(split-BAR mode)."); 648 649 SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar2_addr64, CTLFLAG_RDTUN, 650 &xeon_b2b_dsd_addr.bar2_addr64, 0, "If using B2B topology on Xeon " 651 "hardware, use this 64-bit address on the bus between the NTB devices for " 652 "the window at BAR2, on the downstream side of the link. MUST be the same" 653 " address on both sides."); 654 SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr64, CTLFLAG_RDTUN, 655 &xeon_b2b_dsd_addr.bar4_addr64, 0, "See dsd_bar2_addr64, but BAR4."); 656 SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar4_addr32, CTLFLAG_RDTUN, 657 &xeon_b2b_dsd_addr.bar4_addr32, 0, "See dsd_bar2_addr64, but BAR4 " 658 "(split-BAR mode)."); 659 SYSCTL_UQUAD(_hw_ntb_xeon_b2b, OID_AUTO, dsd_bar5_addr32, CTLFLAG_RDTUN, 660 &xeon_b2b_dsd_addr.bar5_addr32, 0, "See dsd_bar2_addr64, but BAR5 " 661 "(split-BAR mode)."); 662 663 /* 664 * OS <-> Driver interface structures 665 */ 666 MALLOC_DEFINE(M_NTB, "ntb_hw", "ntb_hw driver memory allocations"); 667 668 /* 669 * OS <-> Driver linkage functions 670 */ 671 static int 672 intel_ntb_probe(device_t device) 673 { 674 struct ntb_hw_info *p; 675 676 p = intel_ntb_get_device_info(pci_get_devid(device)); 677 if (p == NULL) 678 return (ENXIO); 679 680 device_set_desc(device, p->desc); 681 return (0); 682 } 683 684 static int 685 intel_ntb_attach(device_t device) 686 { 687 struct ntb_softc *ntb; 688 struct ntb_hw_info *p; 689 int error; 690 691 ntb = device_get_softc(device); 692 p = intel_ntb_get_device_info(pci_get_devid(device)); 693 694 ntb->device = device; 695 ntb->type = p->type; 696 ntb->features = p->features; 697 ntb->b2b_mw_idx = B2B_MW_DISABLED; 698 ntb->msix_mw_idx = B2B_MW_DISABLED; 699 700 /* Heartbeat timer for NTB_ATOM since there is no link interrupt */ 701 callout_init(&ntb->heartbeat_timer, 1); 702 callout_init(&ntb->lr_timer, 1); 703 callout_init(&ntb->peer_msix_work, 1); 704 mtx_init(&ntb->db_mask_lock, "ntb hw bits", NULL, MTX_SPIN); 705 706 if (ntb->type == NTB_ATOM) 707 error = intel_ntb_detect_atom(ntb); 708 else if (ntb->type == NTB_XEON_GEN3) 709 error = intel_ntb_detect_xeon_gen3(ntb); 710 else 711 error = intel_ntb_detect_xeon(ntb); 712 if (error != 0) 713 goto out; 714 715 intel_ntb_detect_max_mw(ntb); 716 717 pci_enable_busmaster(ntb->device); 718 719 error = intel_ntb_map_pci_bars(ntb); 720 if (error != 0) 721 goto out; 722 if (ntb->type == NTB_ATOM) 723 error = intel_ntb_atom_init_dev(ntb); 724 else if (ntb->type == NTB_XEON_GEN3) 725 error = intel_ntb_xeon_gen3_init_dev(ntb); 726 else 727 error = intel_ntb_xeon_init_dev(ntb); 728 if (error != 0) 729 goto out; 730 731 intel_ntb_spad_clear(device); 732 733 intel_ntb_poll_link(ntb); 734 735 intel_ntb_sysctl_init(ntb); 736 737 /* Attach children to this controller */ 738 error = ntb_register_device(device); 739 740 out: 741 if (error != 0) 742 intel_ntb_detach(device); 743 return (error); 744 } 745 746 static int 747 intel_ntb_detach(device_t device) 748 { 749 struct ntb_softc *ntb; 750 751 ntb = device_get_softc(device); 752 753 /* Detach & delete all children */ 754 ntb_unregister_device(device); 755 756 if (ntb->self_reg != NULL) { 757 DB_MASK_LOCK(ntb); 758 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_valid_mask); 759 DB_MASK_UNLOCK(ntb); 760 } 761 callout_drain(&ntb->heartbeat_timer); 762 callout_drain(&ntb->lr_timer); 763 callout_drain(&ntb->peer_msix_work); 764 pci_disable_busmaster(ntb->device); 765 if (ntb->type == NTB_XEON_GEN1) 766 intel_ntb_teardown_xeon(ntb); 767 intel_ntb_teardown_interrupts(ntb); 768 769 mtx_destroy(&ntb->db_mask_lock); 770 771 intel_ntb_unmap_pci_bar(ntb); 772 773 return (0); 774 } 775 776 /* 777 * Driver internal routines 778 */ 779 static inline enum ntb_bar 780 intel_ntb_mw_to_bar(struct ntb_softc *ntb, unsigned mw) 781 { 782 783 KASSERT(mw < ntb->mw_count, 784 ("%s: mw:%u > count:%u", __func__, mw, (unsigned)ntb->mw_count)); 785 KASSERT(ntb->reg->mw_bar[mw] != 0, ("invalid mw")); 786 787 return (ntb->reg->mw_bar[mw]); 788 } 789 790 static inline bool 791 bar_is_64bit(struct ntb_softc *ntb, enum ntb_bar bar) 792 { 793 /* XXX This assertion could be stronger. */ 794 KASSERT(bar < NTB_MAX_BARS, ("bogus bar")); 795 return (bar < NTB_B2B_BAR_2 || !HAS_FEATURE(ntb, NTB_SPLIT_BAR)); 796 } 797 798 static inline void 799 bar_get_xlat_params(struct ntb_softc *ntb, enum ntb_bar bar, uint32_t *base, 800 uint32_t *xlat, uint32_t *lmt) 801 { 802 uint32_t basev, lmtv, xlatv; 803 804 switch (bar) { 805 case NTB_B2B_BAR_1: 806 basev = ntb->xlat_reg->bar2_base; 807 lmtv = ntb->xlat_reg->bar2_limit; 808 xlatv = ntb->xlat_reg->bar2_xlat; 809 break; 810 case NTB_B2B_BAR_2: 811 basev = ntb->xlat_reg->bar4_base; 812 lmtv = ntb->xlat_reg->bar4_limit; 813 xlatv = ntb->xlat_reg->bar4_xlat; 814 break; 815 case NTB_B2B_BAR_3: 816 basev = ntb->xlat_reg->bar5_base; 817 lmtv = ntb->xlat_reg->bar5_limit; 818 xlatv = ntb->xlat_reg->bar5_xlat; 819 break; 820 default: 821 KASSERT(bar >= NTB_B2B_BAR_1 && bar < NTB_MAX_BARS, 822 ("bad bar")); 823 basev = lmtv = xlatv = 0; 824 break; 825 } 826 827 if (base != NULL) 828 *base = basev; 829 if (xlat != NULL) 830 *xlat = xlatv; 831 if (lmt != NULL) 832 *lmt = lmtv; 833 } 834 835 static int 836 intel_ntb_map_pci_bars(struct ntb_softc *ntb) 837 { 838 struct ntb_pci_bar_info *bar; 839 int rc; 840 841 bar = &ntb->bar_info[NTB_CONFIG_BAR]; 842 bar->pci_resource_id = PCIR_BAR(0); 843 rc = map_mmr_bar(ntb, bar); 844 if (rc != 0) 845 goto out; 846 847 /* 848 * At least on Xeon v4 NTB device leaks to host some remote side 849 * BAR0 writes supposed to update scratchpad registers. I am not 850 * sure why it happens, but it may be related to the fact that 851 * on a link side BAR0 is 32KB, while on a host side it is 64KB. 852 * Without this hack DMAR blocks those accesses as not allowed. 853 */ 854 if (bus_dma_tag_create(bus_get_dma_tag(ntb->device), 1, 0, 855 BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, 856 bar->size, 1, bar->size, 0, NULL, NULL, &ntb->bar0_dma_tag)) { 857 device_printf(ntb->device, "Unable to create BAR0 tag\n"); 858 return (ENOMEM); 859 } 860 if (bus_dmamap_create(ntb->bar0_dma_tag, 0, &ntb->bar0_dma_map)) { 861 device_printf(ntb->device, "Unable to create BAR0 map\n"); 862 return (ENOMEM); 863 } 864 if (bus_dma_iommu_load_ident(ntb->bar0_dma_tag, ntb->bar0_dma_map, 865 bar->pbase, bar->size, 0)) { 866 device_printf(ntb->device, "Unable to load BAR0 map\n"); 867 return (ENOMEM); 868 } 869 870 bar = &ntb->bar_info[NTB_B2B_BAR_1]; 871 bar->pci_resource_id = PCIR_BAR(2); 872 rc = map_memory_window_bar(ntb, bar); 873 if (rc != 0) 874 goto out; 875 if (ntb->type == NTB_XEON_GEN3) { 876 bar->psz_off = XEON_GEN3_INT_REG_IMBAR1SZ; 877 bar->ssz_off = XEON_GEN3_INT_REG_EMBAR1SZ; 878 bar->pbarxlat_off = XEON_GEN3_REG_EMBAR1XBASE; 879 } else { 880 bar->psz_off = XEON_PBAR23SZ_OFFSET; 881 bar->ssz_off = XEON_SBAR23SZ_OFFSET; 882 bar->pbarxlat_off = XEON_PBAR2XLAT_OFFSET; 883 } 884 885 bar = &ntb->bar_info[NTB_B2B_BAR_2]; 886 bar->pci_resource_id = PCIR_BAR(4); 887 rc = map_memory_window_bar(ntb, bar); 888 if (rc != 0) 889 goto out; 890 if (ntb->type == NTB_XEON_GEN3) { 891 bar->psz_off = XEON_GEN3_INT_REG_IMBAR2SZ; 892 bar->ssz_off = XEON_GEN3_INT_REG_EMBAR2SZ; 893 bar->pbarxlat_off = XEON_GEN3_REG_EMBAR2XBASE; 894 } else { 895 bar->psz_off = XEON_PBAR4SZ_OFFSET; 896 bar->ssz_off = XEON_SBAR4SZ_OFFSET; 897 bar->pbarxlat_off = XEON_PBAR4XLAT_OFFSET; 898 } 899 900 if (!HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 901 goto out; 902 903 if (ntb->type == NTB_XEON_GEN3) { 904 device_printf(ntb->device, "no split bar support\n"); 905 return (ENXIO); 906 } 907 908 bar = &ntb->bar_info[NTB_B2B_BAR_3]; 909 bar->pci_resource_id = PCIR_BAR(5); 910 rc = map_memory_window_bar(ntb, bar); 911 bar->psz_off = XEON_PBAR5SZ_OFFSET; 912 bar->ssz_off = XEON_SBAR5SZ_OFFSET; 913 bar->pbarxlat_off = XEON_PBAR5XLAT_OFFSET; 914 915 out: 916 if (rc != 0) 917 device_printf(ntb->device, 918 "unable to allocate pci resource\n"); 919 return (rc); 920 } 921 922 static void 923 print_map_success(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar, 924 const char *kind) 925 { 926 927 device_printf(ntb->device, 928 "Mapped BAR%d v:[%p-%p] p:[0x%jx-0x%jx] (0x%jx bytes) (%s)\n", 929 PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 930 (char *)bar->vbase + bar->size - 1, 931 (uintmax_t)bar->pbase, (uintmax_t)(bar->pbase + bar->size - 1), 932 (uintmax_t)bar->size, kind); 933 } 934 935 static int 936 map_mmr_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 937 { 938 939 bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 940 &bar->pci_resource_id, RF_ACTIVE); 941 if (bar->pci_resource == NULL) 942 return (ENXIO); 943 944 save_bar_parameters(bar); 945 bar->map_mode = VM_MEMATTR_UNCACHEABLE; 946 print_map_success(ntb, bar, "mmr"); 947 return (0); 948 } 949 950 static int 951 map_memory_window_bar(struct ntb_softc *ntb, struct ntb_pci_bar_info *bar) 952 { 953 int rc; 954 vm_memattr_t mapmode; 955 uint8_t bar_size_bits = 0; 956 957 bar->pci_resource = bus_alloc_resource_any(ntb->device, SYS_RES_MEMORY, 958 &bar->pci_resource_id, RF_ACTIVE); 959 960 if (bar->pci_resource == NULL) 961 return (ENXIO); 962 963 save_bar_parameters(bar); 964 /* 965 * Ivytown NTB BAR sizes are misreported by the hardware due to a 966 * hardware issue. To work around this, query the size it should be 967 * configured to by the device and modify the resource to correspond to 968 * this new size. The BIOS on systems with this problem is required to 969 * provide enough address space to allow the driver to make this change 970 * safely. 971 * 972 * Ideally I could have just specified the size when I allocated the 973 * resource like: 974 * bus_alloc_resource(ntb->device, 975 * SYS_RES_MEMORY, &bar->pci_resource_id, 0ul, ~0ul, 976 * 1ul << bar_size_bits, RF_ACTIVE); 977 * but the PCI driver does not honor the size in this call, so we have 978 * to modify it after the fact. 979 */ 980 if (HAS_FEATURE(ntb, NTB_BAR_SIZE_4K)) { 981 if (bar->pci_resource_id == PCIR_BAR(2)) 982 bar_size_bits = pci_read_config(ntb->device, 983 XEON_PBAR23SZ_OFFSET, 1); 984 else 985 bar_size_bits = pci_read_config(ntb->device, 986 XEON_PBAR45SZ_OFFSET, 1); 987 988 rc = bus_adjust_resource(ntb->device, SYS_RES_MEMORY, 989 bar->pci_resource, bar->pbase, 990 bar->pbase + (1ul << bar_size_bits) - 1); 991 if (rc != 0) { 992 device_printf(ntb->device, 993 "unable to resize bar\n"); 994 return (rc); 995 } 996 997 save_bar_parameters(bar); 998 } 999 1000 bar->map_mode = VM_MEMATTR_UNCACHEABLE; 1001 print_map_success(ntb, bar, "mw"); 1002 1003 /* 1004 * Optionally, mark MW BARs as anything other than UC to improve 1005 * performance. 1006 */ 1007 mapmode = intel_ntb_pat_flags(); 1008 if (mapmode == bar->map_mode) 1009 return (0); 1010 1011 rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mapmode); 1012 if (rc == 0) { 1013 bar->map_mode = mapmode; 1014 device_printf(ntb->device, 1015 "Marked BAR%d v:[%p-%p] p:[0x%jx-0x%jx] as " 1016 "%s.\n", 1017 PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 1018 (char *)bar->vbase + bar->size - 1, 1019 (uintmax_t)bar->pbase, 1020 (uintmax_t)(bar->pbase + bar->size - 1), 1021 intel_ntb_vm_memattr_to_str(mapmode)); 1022 } else 1023 device_printf(ntb->device, 1024 "Unable to mark BAR%d v:[%p-%p] p:[0x%jx-0x%jx] as " 1025 "%s: %d\n", 1026 PCI_RID2BAR(bar->pci_resource_id), bar->vbase, 1027 (char *)bar->vbase + bar->size - 1, 1028 (uintmax_t)bar->pbase, 1029 (uintmax_t)(bar->pbase + bar->size - 1), 1030 intel_ntb_vm_memattr_to_str(mapmode), rc); 1031 /* Proceed anyway */ 1032 return (0); 1033 } 1034 1035 static void 1036 intel_ntb_unmap_pci_bar(struct ntb_softc *ntb) 1037 { 1038 struct ntb_pci_bar_info *bar; 1039 int i; 1040 1041 if (ntb->bar0_dma_map != NULL) { 1042 bus_dmamap_unload(ntb->bar0_dma_tag, ntb->bar0_dma_map); 1043 bus_dmamap_destroy(ntb->bar0_dma_tag, ntb->bar0_dma_map); 1044 } 1045 if (ntb->bar0_dma_tag != NULL) 1046 bus_dma_tag_destroy(ntb->bar0_dma_tag); 1047 for (i = 0; i < NTB_MAX_BARS; i++) { 1048 bar = &ntb->bar_info[i]; 1049 if (bar->pci_resource != NULL) 1050 bus_release_resource(ntb->device, SYS_RES_MEMORY, 1051 bar->pci_resource_id, bar->pci_resource); 1052 } 1053 } 1054 1055 static int 1056 intel_ntb_setup_msix(struct ntb_softc *ntb, uint32_t num_vectors) 1057 { 1058 uint32_t i; 1059 int rc; 1060 1061 for (i = 0; i < num_vectors; i++) { 1062 ntb->int_info[i].rid = i + 1; 1063 ntb->int_info[i].res = bus_alloc_resource_any(ntb->device, 1064 SYS_RES_IRQ, &ntb->int_info[i].rid, RF_ACTIVE); 1065 if (ntb->int_info[i].res == NULL) { 1066 device_printf(ntb->device, 1067 "bus_alloc_resource failed\n"); 1068 return (ENOMEM); 1069 } 1070 ntb->int_info[i].tag = NULL; 1071 ntb->allocated_interrupts++; 1072 rc = bus_setup_intr(ntb->device, ntb->int_info[i].res, 1073 INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_vec_isr, 1074 &ntb->msix_vec[i], &ntb->int_info[i].tag); 1075 if (rc != 0) { 1076 device_printf(ntb->device, "bus_setup_intr failed\n"); 1077 return (ENXIO); 1078 } 1079 } 1080 return (0); 1081 } 1082 1083 /* 1084 * The Linux NTB driver drops from MSI-X to legacy INTx if a unique vector 1085 * cannot be allocated for each MSI-X message. JHB seems to think remapping 1086 * should be okay. This tunable should enable us to test that hypothesis 1087 * when someone gets their hands on some Xeon hardware. 1088 */ 1089 static int ntb_force_remap_mode; 1090 SYSCTL_INT(_hw_ntb, OID_AUTO, force_remap_mode, CTLFLAG_RDTUN, 1091 &ntb_force_remap_mode, 0, "If enabled, force MSI-X messages to be remapped" 1092 " to a smaller number of ithreads, even if the desired number are " 1093 "available"); 1094 1095 /* 1096 * In case it is NOT ok, give consumers an abort button. 1097 */ 1098 static int ntb_prefer_intx; 1099 SYSCTL_INT(_hw_ntb, OID_AUTO, prefer_intx_to_remap, CTLFLAG_RDTUN, 1100 &ntb_prefer_intx, 0, "If enabled, prefer to use legacy INTx mode rather " 1101 "than remapping MSI-X messages over available slots (match Linux driver " 1102 "behavior)"); 1103 1104 /* 1105 * Remap the desired number of MSI-X messages to available ithreads in a simple 1106 * round-robin fashion. 1107 */ 1108 static int 1109 intel_ntb_remap_msix(device_t dev, uint32_t desired, uint32_t avail) 1110 { 1111 u_int *vectors; 1112 uint32_t i; 1113 int rc; 1114 1115 if (ntb_prefer_intx != 0) 1116 return (ENXIO); 1117 1118 vectors = malloc(desired * sizeof(*vectors), M_NTB, M_ZERO | M_WAITOK); 1119 1120 for (i = 0; i < desired; i++) 1121 vectors[i] = (i % avail) + 1; 1122 1123 rc = pci_remap_msix(dev, desired, vectors); 1124 free(vectors, M_NTB); 1125 return (rc); 1126 } 1127 1128 static int 1129 intel_ntb_xeon_gen3_init_isr(struct ntb_softc *ntb) 1130 { 1131 uint64_t i, reg; 1132 uint32_t desired_vectors, num_vectors; 1133 int rc; 1134 1135 ntb->allocated_interrupts = 0; 1136 ntb->last_ts = ticks; 1137 1138 /* Mask all the interrupts, including hardware interrupt */ 1139 intel_ntb_reg_write(8, XEON_GEN3_REG_IMINT_DISABLE, ~0ULL); 1140 1141 /* Clear Interrupt Status */ 1142 reg = intel_ntb_reg_read(8, XEON_GEN3_REG_IMINT_STATUS); 1143 intel_ntb_reg_write(8, XEON_GEN3_REG_IMINT_STATUS, reg); 1144 1145 num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device), 1146 XEON_GEN3_DB_MSIX_VECTOR_COUNT); 1147 1148 rc = pci_alloc_msix(ntb->device, &num_vectors); 1149 if (rc != 0) { 1150 device_printf(ntb->device, 1151 "Interrupt allocation failed %d\n", rc); 1152 return (rc); 1153 } 1154 if (desired_vectors != num_vectors) { 1155 device_printf(ntb->device, "Couldn't get %d vectors\n", 1156 XEON_GEN3_DB_MSIX_VECTOR_COUNT); 1157 return (ENXIO); 1158 } 1159 /* 32 db + 1 hardware */ 1160 if (num_vectors == XEON_GEN3_DB_MSIX_VECTOR_COUNT) { 1161 /* Program INTVECXX source register */ 1162 for (i = 0; i < XEON_GEN3_DB_MSIX_VECTOR_COUNT; i++) { 1163 /* interrupt source i for vector i */ 1164 intel_ntb_reg_write(1, XEON_GEN3_REG_IMINTVEC00 + i, i); 1165 if (i == (XEON_GEN3_DB_MSIX_VECTOR_COUNT - 1)) { 1166 intel_ntb_reg_write(1, 1167 XEON_GEN3_REG_IMINTVEC00 + i, 1168 XEON_GEN3_LINK_VECTOR_INDEX); 1169 } 1170 } 1171 1172 intel_ntb_create_msix_vec(ntb, num_vectors); 1173 rc = intel_ntb_setup_msix(ntb, num_vectors); 1174 1175 /* enable all interrupts */ 1176 intel_ntb_reg_write(8, XEON_GEN3_REG_IMINT_DISABLE, 0ULL); 1177 } else { 1178 device_printf(ntb->device, "need to remap interrupts, giving up.\n"); 1179 return (ENXIO); 1180 } 1181 1182 return (0); 1183 } 1184 1185 static int 1186 intel_ntb_init_isr(struct ntb_softc *ntb) 1187 { 1188 uint32_t desired_vectors, num_vectors; 1189 int rc; 1190 1191 ntb->allocated_interrupts = 0; 1192 ntb->last_ts = ticks; 1193 1194 /* 1195 * Mask all doorbell interrupts. (Except link events!) 1196 */ 1197 DB_MASK_LOCK(ntb); 1198 ntb->db_mask = ntb->db_valid_mask; 1199 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1200 DB_MASK_UNLOCK(ntb); 1201 1202 num_vectors = desired_vectors = MIN(pci_msix_count(ntb->device), 1203 ntb->db_count); 1204 if (desired_vectors >= 1) { 1205 rc = pci_alloc_msix(ntb->device, &num_vectors); 1206 1207 if (ntb_force_remap_mode != 0 && rc == 0 && 1208 num_vectors == desired_vectors) 1209 num_vectors--; 1210 1211 if (rc == 0 && num_vectors < desired_vectors) { 1212 rc = intel_ntb_remap_msix(ntb->device, desired_vectors, 1213 num_vectors); 1214 if (rc == 0) 1215 num_vectors = desired_vectors; 1216 else 1217 pci_release_msi(ntb->device); 1218 } 1219 if (rc != 0) 1220 num_vectors = 1; 1221 } else 1222 num_vectors = 1; 1223 1224 if (ntb->type == NTB_XEON_GEN1 && num_vectors < ntb->db_vec_count) { 1225 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1226 device_printf(ntb->device, 1227 "Errata workaround does not support MSI or INTX\n"); 1228 return (EINVAL); 1229 } 1230 1231 ntb->db_vec_count = 1; 1232 ntb->db_vec_shift = XEON_DB_TOTAL_SHIFT; 1233 rc = intel_ntb_setup_legacy_interrupt(ntb); 1234 } else { 1235 if (num_vectors - 1 != XEON_NONLINK_DB_MSIX_BITS && 1236 HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1237 device_printf(ntb->device, 1238 "Errata workaround expects %d doorbell bits\n", 1239 XEON_NONLINK_DB_MSIX_BITS); 1240 return (EINVAL); 1241 } 1242 1243 intel_ntb_create_msix_vec(ntb, num_vectors); 1244 rc = intel_ntb_setup_msix(ntb, num_vectors); 1245 } 1246 if (rc != 0) { 1247 device_printf(ntb->device, 1248 "Error allocating interrupts: %d\n", rc); 1249 intel_ntb_free_msix_vec(ntb); 1250 } 1251 1252 return (rc); 1253 } 1254 1255 static int 1256 intel_ntb_setup_legacy_interrupt(struct ntb_softc *ntb) 1257 { 1258 int rc; 1259 1260 ntb->int_info[0].rid = 0; 1261 ntb->int_info[0].res = bus_alloc_resource_any(ntb->device, SYS_RES_IRQ, 1262 &ntb->int_info[0].rid, RF_SHAREABLE|RF_ACTIVE); 1263 if (ntb->int_info[0].res == NULL) { 1264 device_printf(ntb->device, "bus_alloc_resource failed\n"); 1265 return (ENOMEM); 1266 } 1267 1268 ntb->int_info[0].tag = NULL; 1269 ntb->allocated_interrupts = 1; 1270 1271 rc = bus_setup_intr(ntb->device, ntb->int_info[0].res, 1272 INTR_MPSAFE | INTR_TYPE_MISC, NULL, ndev_irq_isr, 1273 ntb, &ntb->int_info[0].tag); 1274 if (rc != 0) { 1275 device_printf(ntb->device, "bus_setup_intr failed\n"); 1276 return (ENXIO); 1277 } 1278 1279 return (0); 1280 } 1281 1282 static void 1283 intel_ntb_teardown_interrupts(struct ntb_softc *ntb) 1284 { 1285 struct ntb_int_info *current_int; 1286 int i; 1287 1288 for (i = 0; i < ntb->allocated_interrupts; i++) { 1289 current_int = &ntb->int_info[i]; 1290 if (current_int->tag != NULL) 1291 bus_teardown_intr(ntb->device, current_int->res, 1292 current_int->tag); 1293 1294 if (current_int->res != NULL) 1295 bus_release_resource(ntb->device, SYS_RES_IRQ, 1296 rman_get_rid(current_int->res), current_int->res); 1297 } 1298 1299 intel_ntb_free_msix_vec(ntb); 1300 pci_release_msi(ntb->device); 1301 } 1302 1303 static inline uint64_t 1304 db_ioread(struct ntb_softc *ntb, uint64_t regoff) 1305 { 1306 1307 switch (ntb->type) { 1308 case NTB_ATOM: 1309 case NTB_XEON_GEN3: 1310 return (intel_ntb_reg_read(8, regoff)); 1311 case NTB_XEON_GEN1: 1312 return (intel_ntb_reg_read(2, regoff)); 1313 } 1314 __assert_unreachable(); 1315 } 1316 1317 static inline void 1318 db_iowrite(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1319 { 1320 1321 KASSERT((val & ~ntb->db_valid_mask) == 0, 1322 ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1323 (uintmax_t)(val & ~ntb->db_valid_mask), 1324 (uintmax_t)ntb->db_valid_mask)); 1325 1326 if (regoff == ntb->self_reg->db_mask) 1327 DB_MASK_ASSERT(ntb, MA_OWNED); 1328 db_iowrite_raw(ntb, regoff, val); 1329 } 1330 1331 static inline void 1332 db_iowrite_raw(struct ntb_softc *ntb, uint64_t regoff, uint64_t val) 1333 { 1334 1335 switch (ntb->type) { 1336 case NTB_ATOM: 1337 case NTB_XEON_GEN3: 1338 intel_ntb_reg_write(8, regoff, val); 1339 break; 1340 case NTB_XEON_GEN1: 1341 intel_ntb_reg_write(2, regoff, (uint16_t)val); 1342 break; 1343 } 1344 } 1345 1346 static void 1347 intel_ntb_db_set_mask(device_t dev, uint64_t bits) 1348 { 1349 struct ntb_softc *ntb = device_get_softc(dev); 1350 1351 DB_MASK_LOCK(ntb); 1352 ntb->db_mask |= bits; 1353 if (!HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) 1354 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1355 DB_MASK_UNLOCK(ntb); 1356 } 1357 1358 static void 1359 intel_ntb_db_clear_mask(device_t dev, uint64_t bits) 1360 { 1361 struct ntb_softc *ntb = device_get_softc(dev); 1362 uint64_t ibits; 1363 int i; 1364 1365 KASSERT((bits & ~ntb->db_valid_mask) == 0, 1366 ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1367 (uintmax_t)(bits & ~ntb->db_valid_mask), 1368 (uintmax_t)ntb->db_valid_mask)); 1369 1370 DB_MASK_LOCK(ntb); 1371 ibits = ntb->fake_db & ntb->db_mask & bits; 1372 ntb->db_mask &= ~bits; 1373 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1374 /* Simulate fake interrupts if unmasked DB bits are set. */ 1375 ntb->force_db |= ibits; 1376 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1377 if ((ibits & intel_ntb_db_vector_mask(dev, i)) != 0) 1378 swi_sched(ntb->int_info[i].tag, 0); 1379 } 1380 } else { 1381 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1382 } 1383 DB_MASK_UNLOCK(ntb); 1384 } 1385 1386 static uint64_t 1387 intel_ntb_db_read(device_t dev) 1388 { 1389 struct ntb_softc *ntb = device_get_softc(dev); 1390 1391 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) 1392 return (ntb->fake_db); 1393 if (ntb->type == NTB_XEON_GEN3) 1394 return (intel_ntb_reg_read(8, XEON_GEN3_REG_IMINT_STATUS)); 1395 else 1396 return (db_ioread(ntb, ntb->self_reg->db_bell)); 1397 } 1398 1399 static void 1400 intel_ntb_db_clear(device_t dev, uint64_t bits) 1401 { 1402 struct ntb_softc *ntb = device_get_softc(dev); 1403 1404 KASSERT((bits & ~ntb->db_valid_mask) == 0, 1405 ("%s: Invalid bits 0x%jx (valid: 0x%jx)", __func__, 1406 (uintmax_t)(bits & ~ntb->db_valid_mask), 1407 (uintmax_t)ntb->db_valid_mask)); 1408 1409 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1410 DB_MASK_LOCK(ntb); 1411 ntb->fake_db &= ~bits; 1412 DB_MASK_UNLOCK(ntb); 1413 return; 1414 } 1415 1416 if (ntb->type == NTB_XEON_GEN3) 1417 intel_ntb_reg_write(4, XEON_GEN3_REG_IMINT_STATUS, 1418 (uint32_t)bits); 1419 else 1420 db_iowrite(ntb, ntb->self_reg->db_bell, bits); 1421 } 1422 1423 static inline uint64_t 1424 intel_ntb_vec_mask(struct ntb_softc *ntb, uint64_t db_vector) 1425 { 1426 uint64_t shift, mask; 1427 1428 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1429 /* 1430 * Remap vectors in custom way to make at least first 1431 * three doorbells to not generate stray events. 1432 * This breaks Linux compatibility (if one existed) 1433 * when more then one DB is used (not by if_ntb). 1434 */ 1435 if (db_vector < XEON_NONLINK_DB_MSIX_BITS - 1) 1436 return (1 << db_vector); 1437 if (db_vector == XEON_NONLINK_DB_MSIX_BITS - 1) 1438 return (0x7ffc); 1439 } 1440 1441 shift = ntb->db_vec_shift; 1442 mask = (1ull << shift) - 1; 1443 return (mask << (shift * db_vector)); 1444 } 1445 1446 static void 1447 intel_ntb_interrupt(struct ntb_softc *ntb, uint32_t vec) 1448 { 1449 uint64_t vec_mask; 1450 1451 ntb->last_ts = ticks; 1452 vec_mask = intel_ntb_vec_mask(ntb, vec); 1453 1454 if (ntb->type == NTB_XEON_GEN3 && vec == XEON_GEN3_LINK_VECTOR_INDEX) 1455 vec_mask |= ntb->db_link_mask; 1456 if ((vec_mask & ntb->db_link_mask) != 0) { 1457 if (intel_ntb_poll_link(ntb)) 1458 ntb_link_event(ntb->device); 1459 if (ntb->type == NTB_XEON_GEN3) 1460 intel_ntb_reg_write(8, XEON_GEN3_REG_IMINT_STATUS, 1461 intel_ntb_reg_read(8, XEON_GEN3_REG_IMINT_STATUS)); 1462 } 1463 1464 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP) && 1465 (vec_mask & ntb->db_link_mask) == 0) { 1466 DB_MASK_LOCK(ntb); 1467 1468 /* 1469 * Do not report same DB events again if not cleared yet, 1470 * unless the mask was just cleared for them and this 1471 * interrupt handler call can be the consequence of it. 1472 */ 1473 vec_mask &= ~ntb->fake_db | ntb->force_db; 1474 ntb->force_db &= ~vec_mask; 1475 1476 /* Update our internal doorbell register. */ 1477 ntb->fake_db |= vec_mask; 1478 1479 /* Do not report masked DB events. */ 1480 vec_mask &= ~ntb->db_mask; 1481 1482 DB_MASK_UNLOCK(ntb); 1483 } 1484 1485 if ((vec_mask & ntb->db_valid_mask) != 0) 1486 ntb_db_event(ntb->device, vec); 1487 } 1488 1489 static void 1490 ndev_vec_isr(void *arg) 1491 { 1492 struct ntb_vec *nvec = arg; 1493 1494 intel_ntb_interrupt(nvec->ntb, nvec->num); 1495 } 1496 1497 static void 1498 ndev_irq_isr(void *arg) 1499 { 1500 /* If we couldn't set up MSI-X, we only have the one vector. */ 1501 intel_ntb_interrupt(arg, 0); 1502 } 1503 1504 static int 1505 intel_ntb_create_msix_vec(struct ntb_softc *ntb, uint32_t num_vectors) 1506 { 1507 uint32_t i; 1508 1509 ntb->msix_vec = malloc(num_vectors * sizeof(*ntb->msix_vec), M_NTB, 1510 M_ZERO | M_WAITOK); 1511 for (i = 0; i < num_vectors; i++) { 1512 ntb->msix_vec[i].num = i; 1513 ntb->msix_vec[i].ntb = ntb; 1514 } 1515 1516 return (0); 1517 } 1518 1519 static void 1520 intel_ntb_free_msix_vec(struct ntb_softc *ntb) 1521 { 1522 1523 if (ntb->msix_vec == NULL) 1524 return; 1525 1526 free(ntb->msix_vec, M_NTB); 1527 ntb->msix_vec = NULL; 1528 } 1529 1530 static void 1531 intel_ntb_get_msix_info(struct ntb_softc *ntb) 1532 { 1533 struct pci_devinfo *dinfo; 1534 struct pcicfg_msix *msix; 1535 uint32_t laddr, data, i, offset; 1536 1537 dinfo = device_get_ivars(ntb->device); 1538 msix = &dinfo->cfg.msix; 1539 1540 CTASSERT(XEON_NONLINK_DB_MSIX_BITS == nitems(ntb->msix_data)); 1541 1542 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 1543 offset = msix->msix_table_offset + i * PCI_MSIX_ENTRY_SIZE; 1544 1545 laddr = bus_read_4(msix->msix_table_res, offset + 1546 PCI_MSIX_ENTRY_LOWER_ADDR); 1547 intel_ntb_printf(2, "local MSIX addr(%u): 0x%x\n", i, laddr); 1548 1549 KASSERT((laddr & MSI_INTEL_ADDR_BASE) == MSI_INTEL_ADDR_BASE, 1550 ("local MSIX addr 0x%x not in MSI base 0x%x", laddr, 1551 MSI_INTEL_ADDR_BASE)); 1552 ntb->msix_data[i].nmd_ofs = laddr; 1553 1554 data = bus_read_4(msix->msix_table_res, offset + 1555 PCI_MSIX_ENTRY_DATA); 1556 intel_ntb_printf(2, "local MSIX data(%u): 0x%x\n", i, data); 1557 1558 ntb->msix_data[i].nmd_data = data; 1559 } 1560 } 1561 1562 static struct ntb_hw_info * 1563 intel_ntb_get_device_info(uint32_t device_id) 1564 { 1565 struct ntb_hw_info *ep; 1566 1567 for (ep = pci_ids; ep < &pci_ids[nitems(pci_ids)]; ep++) { 1568 if (ep->device_id == device_id) 1569 return (ep); 1570 } 1571 return (NULL); 1572 } 1573 1574 static void 1575 intel_ntb_teardown_xeon(struct ntb_softc *ntb) 1576 { 1577 1578 if (ntb->reg != NULL) 1579 intel_ntb_link_disable(ntb->device); 1580 } 1581 1582 static void 1583 intel_ntb_detect_max_mw(struct ntb_softc *ntb) 1584 { 1585 1586 switch (ntb->type) { 1587 case NTB_ATOM: 1588 ntb->mw_count = ATOM_MW_COUNT; 1589 break; 1590 case NTB_XEON_GEN1: 1591 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 1592 ntb->mw_count = XEON_HSX_SPLIT_MW_COUNT; 1593 else 1594 ntb->mw_count = XEON_SNB_MW_COUNT; 1595 break; 1596 case NTB_XEON_GEN3: 1597 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 1598 ntb->mw_count = XEON_GEN3_SPLIT_MW_COUNT; 1599 else 1600 ntb->mw_count = XEON_GEN3_MW_COUNT; 1601 break; 1602 } 1603 } 1604 1605 static int 1606 intel_ntb_detect_xeon(struct ntb_softc *ntb) 1607 { 1608 uint8_t ppd, conn_type; 1609 1610 ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 1); 1611 ntb->ppd = ppd; 1612 1613 if ((ppd & XEON_PPD_DEV_TYPE) != 0) 1614 ntb->dev_type = NTB_DEV_DSD; 1615 else 1616 ntb->dev_type = NTB_DEV_USD; 1617 1618 if ((ppd & XEON_PPD_SPLIT_BAR) != 0) 1619 ntb->features |= NTB_SPLIT_BAR; 1620 1621 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP) && 1622 !HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 1623 device_printf(ntb->device, 1624 "Can not apply SB01BASE_LOCKUP workaround " 1625 "with split BARs disabled!\n"); 1626 device_printf(ntb->device, 1627 "Expect system hangs under heavy NTB traffic!\n"); 1628 ntb->features &= ~NTB_SB01BASE_LOCKUP; 1629 } 1630 1631 /* 1632 * SDOORBELL errata workaround gets in the way of SB01BASE_LOCKUP 1633 * errata workaround; only do one at a time. 1634 */ 1635 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) 1636 ntb->features &= ~NTB_SDOORBELL_LOCKUP; 1637 1638 conn_type = ppd & XEON_PPD_CONN_TYPE; 1639 switch (conn_type) { 1640 case NTB_CONN_B2B: 1641 ntb->conn_type = conn_type; 1642 break; 1643 case NTB_CONN_RP: 1644 case NTB_CONN_TRANSPARENT: 1645 default: 1646 device_printf(ntb->device, "Unsupported connection type: %u\n", 1647 (unsigned)conn_type); 1648 return (ENXIO); 1649 } 1650 return (0); 1651 } 1652 1653 static int 1654 intel_ntb_detect_atom(struct ntb_softc *ntb) 1655 { 1656 uint32_t ppd, conn_type; 1657 1658 ppd = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 1659 ntb->ppd = ppd; 1660 1661 if ((ppd & ATOM_PPD_DEV_TYPE) != 0) 1662 ntb->dev_type = NTB_DEV_DSD; 1663 else 1664 ntb->dev_type = NTB_DEV_USD; 1665 1666 conn_type = (ppd & ATOM_PPD_CONN_TYPE) >> 8; 1667 switch (conn_type) { 1668 case NTB_CONN_B2B: 1669 ntb->conn_type = conn_type; 1670 break; 1671 default: 1672 device_printf(ntb->device, "Unsupported NTB configuration\n"); 1673 return (ENXIO); 1674 } 1675 return (0); 1676 } 1677 1678 static int 1679 intel_ntb_detect_xeon_gen3(struct ntb_softc *ntb) 1680 { 1681 uint8_t ppd, conn_type; 1682 1683 ppd = pci_read_config(ntb->device, XEON_GEN3_INT_REG_PPD, 1); 1684 ntb->ppd = ppd; 1685 1686 /* check port definition */ 1687 conn_type = XEON_GEN3_REG_PPD_PORT_DEF_F(ppd); 1688 switch (conn_type) { 1689 case NTB_CONN_B2B: 1690 ntb->conn_type = conn_type; 1691 break; 1692 default: 1693 device_printf(ntb->device, "Unsupported connection type: %u\n", 1694 conn_type); 1695 return (ENXIO); 1696 } 1697 1698 /* check cross link configuration status */ 1699 if (XEON_GEN3_REG_PPD_CONF_STS_F(ppd)) { 1700 /* NTB Port is configured as DSD/USP */ 1701 ntb->dev_type = NTB_DEV_DSD; 1702 } else { 1703 /* NTB Port is configured as USD/DSP */ 1704 ntb->dev_type = NTB_DEV_USD; 1705 } 1706 1707 if (XEON_GEN3_REG_PPD_ONE_MSIX_F(ppd)) { 1708 /* 1709 * This bit when set, causes only a single MSI-X message to be 1710 * generated if MSI-X is enabled. 1711 */ 1712 ntb->features |= NTB_ONE_MSIX; 1713 } 1714 1715 if (XEON_GEN3_REG_PPD_BAR45_SPL_F(ppd)) { 1716 /* BARs 4 and 5 are presented as two 32b non-prefetchable BARs */ 1717 ntb->features |= NTB_SPLIT_BAR; 1718 } 1719 1720 device_printf(ntb->device, "conn type 0x%02x, dev type 0x%02x," 1721 "features 0x%02x\n", ntb->conn_type, ntb->dev_type, ntb->features); 1722 1723 return (0); 1724 } 1725 1726 static int 1727 intel_ntb_xeon_init_dev(struct ntb_softc *ntb) 1728 { 1729 int rc; 1730 1731 ntb->spad_count = XEON_SPAD_COUNT; 1732 ntb->db_count = XEON_DB_COUNT; 1733 ntb->db_link_mask = XEON_DB_LINK_BIT; 1734 ntb->db_vec_count = XEON_DB_MSIX_VECTOR_COUNT; 1735 ntb->db_vec_shift = XEON_DB_MSIX_VECTOR_SHIFT; 1736 1737 if (ntb->conn_type != NTB_CONN_B2B) { 1738 device_printf(ntb->device, "Connection type %d not supported\n", 1739 ntb->conn_type); 1740 return (ENXIO); 1741 } 1742 1743 ntb->reg = &xeon_reg; 1744 ntb->self_reg = &xeon_pri_reg; 1745 ntb->peer_reg = &xeon_b2b_reg; 1746 ntb->xlat_reg = &xeon_sec_xlat; 1747 1748 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 1749 ntb->force_db = ntb->fake_db = 0; 1750 ntb->msix_mw_idx = (ntb->mw_count + g_ntb_msix_idx) % 1751 ntb->mw_count; 1752 intel_ntb_printf(2, "Setting up MSIX mw idx %d means %u\n", 1753 g_ntb_msix_idx, ntb->msix_mw_idx); 1754 rc = intel_ntb_mw_set_wc_internal(ntb, ntb->msix_mw_idx, 1755 VM_MEMATTR_UNCACHEABLE); 1756 KASSERT(rc == 0, ("shouldn't fail")); 1757 } else if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) { 1758 /* 1759 * There is a Xeon hardware errata related to writes to SDOORBELL or 1760 * B2BDOORBELL in conjunction with inbound access to NTB MMIO space, 1761 * which may hang the system. To workaround this, use a memory 1762 * window to access the interrupt and scratch pad registers on the 1763 * remote system. 1764 */ 1765 ntb->b2b_mw_idx = (ntb->mw_count + g_ntb_mw_idx) % 1766 ntb->mw_count; 1767 intel_ntb_printf(2, "Setting up b2b mw idx %d means %u\n", 1768 g_ntb_mw_idx, ntb->b2b_mw_idx); 1769 rc = intel_ntb_mw_set_wc_internal(ntb, ntb->b2b_mw_idx, 1770 VM_MEMATTR_UNCACHEABLE); 1771 KASSERT(rc == 0, ("shouldn't fail")); 1772 } else if (HAS_FEATURE(ntb, NTB_B2BDOORBELL_BIT14)) 1773 /* 1774 * HW Errata on bit 14 of b2bdoorbell register. Writes will not be 1775 * mirrored to the remote system. Shrink the number of bits by one, 1776 * since bit 14 is the last bit. 1777 * 1778 * On REGS_THRU_MW errata mode, we don't use the b2bdoorbell register 1779 * anyway. Nor for non-B2B connection types. 1780 */ 1781 ntb->db_count = XEON_DB_COUNT - 1; 1782 1783 ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1784 1785 if (ntb->dev_type == NTB_DEV_USD) 1786 rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_dsd_addr, 1787 &xeon_b2b_usd_addr); 1788 else 1789 rc = xeon_setup_b2b_mw(ntb, &xeon_b2b_usd_addr, 1790 &xeon_b2b_dsd_addr); 1791 if (rc != 0) 1792 return (rc); 1793 1794 /* Enable Bus Master and Memory Space on the secondary side */ 1795 intel_ntb_reg_write(2, XEON_SPCICMD_OFFSET, 1796 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1797 1798 /* 1799 * Mask all doorbell interrupts. 1800 */ 1801 DB_MASK_LOCK(ntb); 1802 ntb->db_mask = ntb->db_valid_mask; 1803 db_iowrite(ntb, ntb->self_reg->db_mask, ntb->db_mask); 1804 DB_MASK_UNLOCK(ntb); 1805 1806 rc = intel_ntb_init_isr(ntb); 1807 return (rc); 1808 } 1809 1810 static int 1811 intel_ntb_xeon_gen3_init_dev(struct ntb_softc *ntb) 1812 { 1813 int rc; 1814 1815 ntb->spad_count = XEON_GEN3_SPAD_COUNT; 1816 ntb->db_count = XEON_GEN3_DB_COUNT; 1817 ntb->db_link_mask = XEON_GEN3_DB_LINK_BIT; 1818 ntb->db_vec_count = XEON_GEN3_DB_MSIX_VECTOR_COUNT; 1819 ntb->db_vec_shift = XEON_GEN3_DB_MSIX_VECTOR_SHIFT; 1820 1821 if (ntb->conn_type != NTB_CONN_B2B) { 1822 device_printf(ntb->device, "Connection type %d not supported\n", 1823 ntb->conn_type); 1824 return (ENXIO); 1825 } 1826 1827 ntb->reg = &xeon_gen3_reg; 1828 ntb->self_reg = &xeon_gen3_pri_reg; 1829 ntb->peer_reg = &xeon_gen3_b2b_reg; 1830 ntb->xlat_reg = &xeon_gen3_sec_xlat; 1831 1832 ntb->db_valid_mask = (1ULL << ntb->db_count) - 1; 1833 1834 xeon_gen3_setup_b2b_mw(ntb); 1835 1836 /* Enable Bus Master and Memory Space on the External Side */ 1837 intel_ntb_reg_write(2, XEON_GEN3_EXT_REG_PCI_CMD, 1838 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1839 1840 /* Setup Interrupt */ 1841 rc = intel_ntb_xeon_gen3_init_isr(ntb); 1842 1843 return (rc); 1844 } 1845 1846 static int 1847 intel_ntb_atom_init_dev(struct ntb_softc *ntb) 1848 { 1849 int error; 1850 1851 KASSERT(ntb->conn_type == NTB_CONN_B2B, 1852 ("Unsupported NTB configuration (%d)\n", ntb->conn_type)); 1853 1854 ntb->spad_count = ATOM_SPAD_COUNT; 1855 ntb->db_count = ATOM_DB_COUNT; 1856 ntb->db_vec_count = ATOM_DB_MSIX_VECTOR_COUNT; 1857 ntb->db_vec_shift = ATOM_DB_MSIX_VECTOR_SHIFT; 1858 ntb->db_valid_mask = (1ull << ntb->db_count) - 1; 1859 1860 ntb->reg = &atom_reg; 1861 ntb->self_reg = &atom_pri_reg; 1862 ntb->peer_reg = &atom_b2b_reg; 1863 ntb->xlat_reg = &atom_sec_xlat; 1864 1865 /* 1866 * FIXME - MSI-X bug on early Atom HW, remove once internal issue is 1867 * resolved. Mask transaction layer internal parity errors. 1868 */ 1869 pci_write_config(ntb->device, 0xFC, 0x4, 4); 1870 1871 configure_atom_secondary_side_bars(ntb); 1872 1873 /* Enable Bus Master and Memory Space on the secondary side */ 1874 intel_ntb_reg_write(2, ATOM_SPCICMD_OFFSET, 1875 PCIM_CMD_MEMEN | PCIM_CMD_BUSMASTEREN); 1876 1877 error = intel_ntb_init_isr(ntb); 1878 if (error != 0) 1879 return (error); 1880 1881 /* Initiate PCI-E link training */ 1882 intel_ntb_link_enable(ntb->device, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 1883 1884 callout_reset(&ntb->heartbeat_timer, 0, atom_link_hb, ntb); 1885 1886 return (0); 1887 } 1888 1889 /* XXX: Linux driver doesn't seem to do any of this for Atom. */ 1890 static void 1891 configure_atom_secondary_side_bars(struct ntb_softc *ntb) 1892 { 1893 1894 if (ntb->dev_type == NTB_DEV_USD) { 1895 intel_ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1896 XEON_B2B_BAR2_ADDR64); 1897 intel_ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1898 XEON_B2B_BAR4_ADDR64); 1899 intel_ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1900 intel_ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1901 } else { 1902 intel_ntb_reg_write(8, ATOM_PBAR2XLAT_OFFSET, 1903 XEON_B2B_BAR2_ADDR64); 1904 intel_ntb_reg_write(8, ATOM_PBAR4XLAT_OFFSET, 1905 XEON_B2B_BAR4_ADDR64); 1906 intel_ntb_reg_write(8, ATOM_MBAR23_OFFSET, XEON_B2B_BAR2_ADDR64); 1907 intel_ntb_reg_write(8, ATOM_MBAR45_OFFSET, XEON_B2B_BAR4_ADDR64); 1908 } 1909 } 1910 1911 /* 1912 * When working around Xeon SDOORBELL errata by remapping remote registers in a 1913 * MW, limit the B2B MW to half a MW. By sharing a MW, half the shared MW 1914 * remains for use by a higher layer. 1915 * 1916 * Will only be used if working around SDOORBELL errata and the BIOS-configured 1917 * MW size is sufficiently large. 1918 */ 1919 static unsigned int ntb_b2b_mw_share; 1920 SYSCTL_UINT(_hw_ntb, OID_AUTO, b2b_mw_share, CTLFLAG_RDTUN, &ntb_b2b_mw_share, 1921 0, "If enabled (non-zero), prefer to share half of the B2B peer register " 1922 "MW with higher level consumers. Both sides of the NTB MUST set the same " 1923 "value here."); 1924 1925 static void 1926 xeon_reset_sbar_size(struct ntb_softc *ntb, enum ntb_bar idx, 1927 enum ntb_bar regbar) 1928 { 1929 struct ntb_pci_bar_info *bar; 1930 uint8_t bar_sz; 1931 1932 if (!HAS_FEATURE(ntb, NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_3) 1933 return; 1934 1935 bar = &ntb->bar_info[idx]; 1936 bar_sz = pci_read_config(ntb->device, bar->psz_off, 1); 1937 if (idx == regbar) { 1938 if (ntb->b2b_off != 0) 1939 bar_sz--; 1940 else 1941 bar_sz = 0; 1942 } 1943 pci_write_config(ntb->device, bar->ssz_off, bar_sz, 1); 1944 bar_sz = pci_read_config(ntb->device, bar->ssz_off, 1); 1945 (void)bar_sz; 1946 } 1947 1948 static void 1949 xeon_set_sbar_base_and_limit(struct ntb_softc *ntb, uint64_t bar_addr, 1950 enum ntb_bar idx, enum ntb_bar regbar) 1951 { 1952 uint64_t reg_val; 1953 uint32_t base_reg, lmt_reg; 1954 1955 bar_get_xlat_params(ntb, idx, &base_reg, NULL, &lmt_reg); 1956 if (idx == regbar) { 1957 if (ntb->b2b_off) 1958 bar_addr += ntb->b2b_off; 1959 else 1960 bar_addr = 0; 1961 } 1962 1963 if (!bar_is_64bit(ntb, idx)) { 1964 intel_ntb_reg_write(4, base_reg, bar_addr); 1965 reg_val = intel_ntb_reg_read(4, base_reg); 1966 (void)reg_val; 1967 1968 intel_ntb_reg_write(4, lmt_reg, bar_addr); 1969 reg_val = intel_ntb_reg_read(4, lmt_reg); 1970 (void)reg_val; 1971 } else { 1972 intel_ntb_reg_write(8, base_reg, bar_addr); 1973 reg_val = intel_ntb_reg_read(8, base_reg); 1974 (void)reg_val; 1975 1976 intel_ntb_reg_write(8, lmt_reg, bar_addr); 1977 reg_val = intel_ntb_reg_read(8, lmt_reg); 1978 (void)reg_val; 1979 } 1980 } 1981 1982 static void 1983 xeon_set_pbar_xlat(struct ntb_softc *ntb, uint64_t base_addr, enum ntb_bar idx) 1984 { 1985 struct ntb_pci_bar_info *bar; 1986 1987 bar = &ntb->bar_info[idx]; 1988 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR) && idx >= NTB_B2B_BAR_2) { 1989 intel_ntb_reg_write(4, bar->pbarxlat_off, base_addr); 1990 base_addr = intel_ntb_reg_read(4, bar->pbarxlat_off); 1991 } else { 1992 intel_ntb_reg_write(8, bar->pbarxlat_off, base_addr); 1993 base_addr = intel_ntb_reg_read(8, bar->pbarxlat_off); 1994 } 1995 (void)base_addr; 1996 } 1997 1998 static int 1999 xeon_setup_b2b_mw(struct ntb_softc *ntb, const struct ntb_b2b_addr *addr, 2000 const struct ntb_b2b_addr *peer_addr) 2001 { 2002 struct ntb_pci_bar_info *b2b_bar; 2003 vm_size_t bar_size; 2004 uint64_t bar_addr; 2005 enum ntb_bar b2b_bar_num, i; 2006 2007 if (ntb->b2b_mw_idx == B2B_MW_DISABLED) { 2008 b2b_bar = NULL; 2009 b2b_bar_num = NTB_CONFIG_BAR; 2010 ntb->b2b_off = 0; 2011 } else { 2012 b2b_bar_num = intel_ntb_mw_to_bar(ntb, ntb->b2b_mw_idx); 2013 KASSERT(b2b_bar_num > 0 && b2b_bar_num < NTB_MAX_BARS, 2014 ("invalid b2b mw bar")); 2015 2016 b2b_bar = &ntb->bar_info[b2b_bar_num]; 2017 bar_size = b2b_bar->size; 2018 2019 if (ntb_b2b_mw_share != 0 && 2020 (bar_size >> 1) >= XEON_B2B_MIN_SIZE) 2021 ntb->b2b_off = bar_size >> 1; 2022 else if (bar_size >= XEON_B2B_MIN_SIZE) { 2023 ntb->b2b_off = 0; 2024 } else { 2025 device_printf(ntb->device, 2026 "B2B bar size is too small!\n"); 2027 return (EIO); 2028 } 2029 } 2030 2031 /* 2032 * Reset the secondary bar sizes to match the primary bar sizes. 2033 * (Except, disable or halve the size of the B2B secondary bar.) 2034 */ 2035 for (i = NTB_B2B_BAR_1; i < NTB_MAX_BARS; i++) 2036 xeon_reset_sbar_size(ntb, i, b2b_bar_num); 2037 2038 bar_addr = 0; 2039 if (b2b_bar_num == NTB_CONFIG_BAR) 2040 bar_addr = addr->bar0_addr; 2041 else if (b2b_bar_num == NTB_B2B_BAR_1) 2042 bar_addr = addr->bar2_addr64; 2043 else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 2044 bar_addr = addr->bar4_addr64; 2045 else if (b2b_bar_num == NTB_B2B_BAR_2) 2046 bar_addr = addr->bar4_addr32; 2047 else if (b2b_bar_num == NTB_B2B_BAR_3) 2048 bar_addr = addr->bar5_addr32; 2049 else 2050 KASSERT(false, ("invalid bar")); 2051 2052 intel_ntb_reg_write(8, XEON_SBAR0BASE_OFFSET, bar_addr); 2053 2054 /* 2055 * Other SBARs are normally hit by the PBAR xlat, except for the b2b 2056 * register BAR. The B2B BAR is either disabled above or configured 2057 * half-size. It starts at PBAR xlat + offset. 2058 * 2059 * Also set up incoming BAR limits == base (zero length window). 2060 */ 2061 xeon_set_sbar_base_and_limit(ntb, addr->bar2_addr64, NTB_B2B_BAR_1, 2062 b2b_bar_num); 2063 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2064 xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr32, 2065 NTB_B2B_BAR_2, b2b_bar_num); 2066 xeon_set_sbar_base_and_limit(ntb, addr->bar5_addr32, 2067 NTB_B2B_BAR_3, b2b_bar_num); 2068 } else 2069 xeon_set_sbar_base_and_limit(ntb, addr->bar4_addr64, 2070 NTB_B2B_BAR_2, b2b_bar_num); 2071 2072 /* Zero incoming translation addrs */ 2073 intel_ntb_reg_write(8, XEON_SBAR2XLAT_OFFSET, 0); 2074 intel_ntb_reg_write(8, XEON_SBAR4XLAT_OFFSET, 0); 2075 2076 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 2077 uint32_t xlat_reg, lmt_reg; 2078 enum ntb_bar bar_num; 2079 2080 /* 2081 * We point the chosen MSIX MW BAR xlat to remote LAPIC for 2082 * workaround 2083 */ 2084 bar_num = intel_ntb_mw_to_bar(ntb, ntb->msix_mw_idx); 2085 bar_get_xlat_params(ntb, bar_num, NULL, &xlat_reg, &lmt_reg); 2086 if (bar_is_64bit(ntb, bar_num)) { 2087 intel_ntb_reg_write(8, xlat_reg, MSI_INTEL_ADDR_BASE); 2088 ntb->msix_xlat = intel_ntb_reg_read(8, xlat_reg); 2089 intel_ntb_reg_write(8, lmt_reg, 0); 2090 } else { 2091 intel_ntb_reg_write(4, xlat_reg, MSI_INTEL_ADDR_BASE); 2092 ntb->msix_xlat = intel_ntb_reg_read(4, xlat_reg); 2093 intel_ntb_reg_write(4, lmt_reg, 0); 2094 } 2095 2096 ntb->peer_lapic_bar = &ntb->bar_info[bar_num]; 2097 } 2098 (void)intel_ntb_reg_read(8, XEON_SBAR2XLAT_OFFSET); 2099 (void)intel_ntb_reg_read(8, XEON_SBAR4XLAT_OFFSET); 2100 2101 /* Zero outgoing translation limits (whole bar size windows) */ 2102 intel_ntb_reg_write(8, XEON_PBAR2LMT_OFFSET, 0); 2103 intel_ntb_reg_write(8, XEON_PBAR4LMT_OFFSET, 0); 2104 2105 /* Set outgoing translation offsets */ 2106 xeon_set_pbar_xlat(ntb, peer_addr->bar2_addr64, NTB_B2B_BAR_1); 2107 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2108 xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr32, NTB_B2B_BAR_2); 2109 xeon_set_pbar_xlat(ntb, peer_addr->bar5_addr32, NTB_B2B_BAR_3); 2110 } else 2111 xeon_set_pbar_xlat(ntb, peer_addr->bar4_addr64, NTB_B2B_BAR_2); 2112 2113 /* Set the translation offset for B2B registers */ 2114 bar_addr = 0; 2115 if (b2b_bar_num == NTB_CONFIG_BAR) 2116 bar_addr = peer_addr->bar0_addr; 2117 else if (b2b_bar_num == NTB_B2B_BAR_1) 2118 bar_addr = peer_addr->bar2_addr64; 2119 else if (b2b_bar_num == NTB_B2B_BAR_2 && !HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 2120 bar_addr = peer_addr->bar4_addr64; 2121 else if (b2b_bar_num == NTB_B2B_BAR_2) 2122 bar_addr = peer_addr->bar4_addr32; 2123 else if (b2b_bar_num == NTB_B2B_BAR_3) 2124 bar_addr = peer_addr->bar5_addr32; 2125 else 2126 KASSERT(false, ("invalid bar")); 2127 2128 /* 2129 * B2B_XLAT_OFFSET is a 64-bit register but can only be written 32 bits 2130 * at a time. 2131 */ 2132 intel_ntb_reg_write(4, XEON_B2B_XLAT_OFFSETL, bar_addr & 0xffffffff); 2133 intel_ntb_reg_write(4, XEON_B2B_XLAT_OFFSETU, bar_addr >> 32); 2134 return (0); 2135 } 2136 2137 static int 2138 xeon_gen3_setup_b2b_mw(struct ntb_softc *ntb) 2139 { 2140 uint64_t reg; 2141 uint32_t embarsz, imbarsz; 2142 2143 /* IMBAR1SZ should be equal to EMBAR1SZ */ 2144 embarsz = pci_read_config(ntb->device, XEON_GEN3_INT_REG_EMBAR1SZ, 1); 2145 imbarsz = pci_read_config(ntb->device, XEON_GEN3_INT_REG_IMBAR1SZ, 1); 2146 if (embarsz != imbarsz) { 2147 device_printf(ntb->device, 2148 "IMBAR1SZ (%u) should be equal to EMBAR1SZ (%u)\n", 2149 imbarsz, embarsz); 2150 return (EIO); 2151 } 2152 2153 /* IMBAR2SZ should be equal to EMBAR2SZ */ 2154 embarsz = pci_read_config(ntb->device, XEON_GEN3_INT_REG_EMBAR2SZ, 1); 2155 imbarsz = pci_read_config(ntb->device, XEON_GEN3_INT_REG_IMBAR2SZ, 1); 2156 if (embarsz != imbarsz) { 2157 device_printf(ntb->device, 2158 "IMBAR2SZ (%u) should be equal to EMBAR2SZ (%u)\n", 2159 imbarsz, embarsz); 2160 return (EIO); 2161 } 2162 2163 /* Client will provide the incoming IMBAR1/2XBASE, zero it for now */ 2164 intel_ntb_reg_write(8, XEON_GEN3_REG_IMBAR1XBASE, 0); 2165 intel_ntb_reg_write(8, XEON_GEN3_REG_IMBAR2XBASE, 0); 2166 2167 /* 2168 * If the value in IMBAR1XLIMIT is set equal to the value in IMBAR1XBASE, 2169 * the local memory window exposure from EMBAR1 is disabled. 2170 * Note: It is needed to avoid malicious access. 2171 */ 2172 intel_ntb_reg_write(8, XEON_GEN3_REG_IMBAR1XLIMIT, 0); 2173 intel_ntb_reg_write(8, XEON_GEN3_REG_IMBAR2XLIMIT, 0); 2174 2175 /* Config outgoing translation limits (whole bar size windows) */ 2176 reg = intel_ntb_reg_read(8, XEON_GEN3_REG_EMBAR1XBASE); 2177 reg += ntb->bar_info[NTB_B2B_BAR_1].size; 2178 intel_ntb_reg_write(8, XEON_GEN3_REG_EMBAR1XLIMIT, reg); 2179 2180 reg = intel_ntb_reg_read(8, XEON_GEN3_REG_EMBAR2XBASE); 2181 reg += ntb->bar_info[NTB_B2B_BAR_2].size; 2182 intel_ntb_reg_write(8, XEON_GEN3_REG_EMBAR2XLIMIT, reg); 2183 2184 return (0); 2185 } 2186 2187 static inline bool 2188 _xeon_link_is_up(struct ntb_softc *ntb) 2189 { 2190 2191 if (ntb->conn_type == NTB_CONN_TRANSPARENT) 2192 return (true); 2193 return ((ntb->lnk_sta & NTB_LINK_STATUS_ACTIVE) != 0); 2194 } 2195 2196 static inline bool 2197 link_is_up(struct ntb_softc *ntb) 2198 { 2199 2200 if (ntb->type == NTB_XEON_GEN1 || ntb->type == NTB_XEON_GEN3) 2201 return (_xeon_link_is_up(ntb) && (ntb->peer_msix_good || 2202 !HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP))); 2203 2204 KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 2205 return ((ntb->ntb_ctl & ATOM_CNTL_LINK_DOWN) == 0); 2206 } 2207 2208 static inline bool 2209 atom_link_is_err(struct ntb_softc *ntb) 2210 { 2211 uint32_t status; 2212 2213 KASSERT(ntb->type == NTB_ATOM, ("ntb type")); 2214 2215 status = intel_ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 2216 if ((status & ATOM_LTSSMSTATEJMP_FORCEDETECT) != 0) 2217 return (true); 2218 2219 status = intel_ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 2220 return ((status & ATOM_IBIST_ERR_OFLOW) != 0); 2221 } 2222 2223 /* Atom does not have link status interrupt, poll on that platform */ 2224 static void 2225 atom_link_hb(void *arg) 2226 { 2227 struct ntb_softc *ntb = arg; 2228 sbintime_t timo, poll_ts; 2229 2230 timo = NTB_HB_TIMEOUT * hz; 2231 poll_ts = ntb->last_ts + timo; 2232 2233 /* 2234 * Delay polling the link status if an interrupt was received, unless 2235 * the cached link status says the link is down. 2236 */ 2237 if ((sbintime_t)ticks - poll_ts < 0 && link_is_up(ntb)) { 2238 timo = poll_ts - ticks; 2239 goto out; 2240 } 2241 2242 if (intel_ntb_poll_link(ntb)) 2243 ntb_link_event(ntb->device); 2244 2245 if (!link_is_up(ntb) && atom_link_is_err(ntb)) { 2246 /* Link is down with error, proceed with recovery */ 2247 callout_reset(&ntb->lr_timer, 0, recover_atom_link, ntb); 2248 return; 2249 } 2250 2251 out: 2252 callout_reset(&ntb->heartbeat_timer, timo, atom_link_hb, ntb); 2253 } 2254 2255 static void 2256 atom_perform_link_restart(struct ntb_softc *ntb) 2257 { 2258 uint32_t status; 2259 2260 /* Driver resets the NTB ModPhy lanes - magic! */ 2261 intel_ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0xe0); 2262 intel_ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x40); 2263 intel_ntb_reg_write(1, ATOM_MODPHY_PCSREG4, 0x60); 2264 intel_ntb_reg_write(1, ATOM_MODPHY_PCSREG6, 0x60); 2265 2266 /* Driver waits 100ms to allow the NTB ModPhy to settle */ 2267 pause("ModPhy", hz / 10); 2268 2269 /* Clear AER Errors, write to clear */ 2270 status = intel_ntb_reg_read(4, ATOM_ERRCORSTS_OFFSET); 2271 status &= PCIM_AER_COR_REPLAY_ROLLOVER; 2272 intel_ntb_reg_write(4, ATOM_ERRCORSTS_OFFSET, status); 2273 2274 /* Clear unexpected electrical idle event in LTSSM, write to clear */ 2275 status = intel_ntb_reg_read(4, ATOM_LTSSMERRSTS0_OFFSET); 2276 status |= ATOM_LTSSMERRSTS0_UNEXPECTEDEI; 2277 intel_ntb_reg_write(4, ATOM_LTSSMERRSTS0_OFFSET, status); 2278 2279 /* Clear DeSkew Buffer error, write to clear */ 2280 status = intel_ntb_reg_read(4, ATOM_DESKEWSTS_OFFSET); 2281 status |= ATOM_DESKEWSTS_DBERR; 2282 intel_ntb_reg_write(4, ATOM_DESKEWSTS_OFFSET, status); 2283 2284 status = intel_ntb_reg_read(4, ATOM_IBSTERRRCRVSTS0_OFFSET); 2285 status &= ATOM_IBIST_ERR_OFLOW; 2286 intel_ntb_reg_write(4, ATOM_IBSTERRRCRVSTS0_OFFSET, status); 2287 2288 /* Releases the NTB state machine to allow the link to retrain */ 2289 status = intel_ntb_reg_read(4, ATOM_LTSSMSTATEJMP_OFFSET); 2290 status &= ~ATOM_LTSSMSTATEJMP_FORCEDETECT; 2291 intel_ntb_reg_write(4, ATOM_LTSSMSTATEJMP_OFFSET, status); 2292 } 2293 2294 static int 2295 intel_ntb_port_number(device_t dev) 2296 { 2297 struct ntb_softc *ntb = device_get_softc(dev); 2298 2299 return (ntb->dev_type == NTB_DEV_USD ? 0 : 1); 2300 } 2301 2302 static int 2303 intel_ntb_peer_port_count(device_t dev) 2304 { 2305 2306 return (1); 2307 } 2308 2309 static int 2310 intel_ntb_peer_port_number(device_t dev, int pidx) 2311 { 2312 struct ntb_softc *ntb = device_get_softc(dev); 2313 2314 if (pidx != 0) 2315 return (-EINVAL); 2316 2317 return (ntb->dev_type == NTB_DEV_USD ? 1 : 0); 2318 } 2319 2320 static int 2321 intel_ntb_peer_port_idx(device_t dev, int port) 2322 { 2323 int peer_port; 2324 2325 peer_port = intel_ntb_peer_port_number(dev, 0); 2326 if (peer_port == -EINVAL || port != peer_port) 2327 return (-EINVAL); 2328 2329 return (0); 2330 } 2331 2332 static int 2333 intel_ntb_link_enable(device_t dev, enum ntb_speed speed __unused, 2334 enum ntb_width width __unused) 2335 { 2336 struct ntb_softc *ntb = device_get_softc(dev); 2337 uint32_t cntl; 2338 2339 intel_ntb_printf(2, "%s\n", __func__); 2340 2341 if (ntb->type == NTB_ATOM) { 2342 pci_write_config(ntb->device, NTB_PPD_OFFSET, 2343 ntb->ppd | ATOM_PPD_INIT_LINK, 4); 2344 return (0); 2345 } 2346 2347 if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 2348 ntb_link_event(dev); 2349 return (0); 2350 } 2351 2352 cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl); 2353 cntl &= ~(NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK); 2354 cntl |= NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP; 2355 cntl |= NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP; 2356 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 2357 cntl |= NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP; 2358 intel_ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 2359 return (0); 2360 } 2361 2362 static int 2363 intel_ntb_link_disable(device_t dev) 2364 { 2365 struct ntb_softc *ntb = device_get_softc(dev); 2366 uint32_t cntl; 2367 2368 intel_ntb_printf(2, "%s\n", __func__); 2369 2370 if (ntb->conn_type == NTB_CONN_TRANSPARENT) { 2371 ntb_link_event(dev); 2372 return (0); 2373 } 2374 2375 cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl); 2376 cntl &= ~(NTB_CNTL_P2S_BAR23_SNOOP | NTB_CNTL_S2P_BAR23_SNOOP); 2377 cntl &= ~(NTB_CNTL_P2S_BAR4_SNOOP | NTB_CNTL_S2P_BAR4_SNOOP); 2378 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) 2379 cntl &= ~(NTB_CNTL_P2S_BAR5_SNOOP | NTB_CNTL_S2P_BAR5_SNOOP); 2380 cntl |= NTB_CNTL_LINK_DISABLE | NTB_CNTL_CFG_LOCK; 2381 intel_ntb_reg_write(4, ntb->reg->ntb_ctl, cntl); 2382 return (0); 2383 } 2384 2385 static bool 2386 intel_ntb_link_enabled(device_t dev) 2387 { 2388 struct ntb_softc *ntb = device_get_softc(dev); 2389 uint32_t cntl; 2390 2391 if (ntb->type == NTB_ATOM) { 2392 cntl = pci_read_config(ntb->device, NTB_PPD_OFFSET, 4); 2393 return ((cntl & ATOM_PPD_INIT_LINK) != 0); 2394 } 2395 2396 if (ntb->conn_type == NTB_CONN_TRANSPARENT) 2397 return (true); 2398 2399 cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl); 2400 return ((cntl & NTB_CNTL_LINK_DISABLE) == 0); 2401 } 2402 2403 static void 2404 recover_atom_link(void *arg) 2405 { 2406 struct ntb_softc *ntb = arg; 2407 unsigned speed, width, oldspeed, oldwidth; 2408 uint32_t status32; 2409 2410 atom_perform_link_restart(ntb); 2411 2412 /* 2413 * There is a potential race between the 2 NTB devices recovering at 2414 * the same time. If the times are the same, the link will not recover 2415 * and the driver will be stuck in this loop forever. Add a random 2416 * interval to the recovery time to prevent this race. 2417 */ 2418 status32 = arc4random() % ATOM_LINK_RECOVERY_TIME; 2419 pause("Link", (ATOM_LINK_RECOVERY_TIME + status32) * hz / 1000); 2420 2421 if (atom_link_is_err(ntb)) 2422 goto retry; 2423 2424 status32 = intel_ntb_reg_read(4, ntb->reg->ntb_ctl); 2425 if ((status32 & ATOM_CNTL_LINK_DOWN) != 0) 2426 goto out; 2427 2428 status32 = intel_ntb_reg_read(4, ntb->reg->lnk_sta); 2429 width = NTB_LNK_STA_WIDTH(status32); 2430 speed = status32 & NTB_LINK_SPEED_MASK; 2431 2432 oldwidth = NTB_LNK_STA_WIDTH(ntb->lnk_sta); 2433 oldspeed = ntb->lnk_sta & NTB_LINK_SPEED_MASK; 2434 if (oldwidth != width || oldspeed != speed) 2435 goto retry; 2436 2437 out: 2438 callout_reset(&ntb->heartbeat_timer, NTB_HB_TIMEOUT * hz, atom_link_hb, 2439 ntb); 2440 return; 2441 2442 retry: 2443 callout_reset(&ntb->lr_timer, NTB_HB_TIMEOUT * hz, recover_atom_link, 2444 ntb); 2445 } 2446 2447 /* 2448 * Polls the HW link status register(s); returns true if something has changed. 2449 */ 2450 static bool 2451 intel_ntb_poll_link(struct ntb_softc *ntb) 2452 { 2453 uint32_t ntb_cntl; 2454 uint16_t reg_val; 2455 2456 if (ntb->type == NTB_ATOM) { 2457 ntb_cntl = intel_ntb_reg_read(4, ntb->reg->ntb_ctl); 2458 if (ntb_cntl == ntb->ntb_ctl) 2459 return (false); 2460 2461 ntb->ntb_ctl = ntb_cntl; 2462 ntb->lnk_sta = intel_ntb_reg_read(4, ntb->reg->lnk_sta); 2463 } else { 2464 if (ntb->type == NTB_XEON_GEN1) 2465 db_iowrite_raw(ntb, ntb->self_reg->db_bell, 2466 ntb->db_link_mask); 2467 2468 reg_val = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 2469 if (reg_val == ntb->lnk_sta) 2470 return (false); 2471 2472 ntb->lnk_sta = reg_val; 2473 2474 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 2475 if (_xeon_link_is_up(ntb)) { 2476 if (!ntb->peer_msix_good) { 2477 callout_reset(&ntb->peer_msix_work, 0, 2478 intel_ntb_exchange_msix, ntb); 2479 return (false); 2480 } 2481 } else { 2482 ntb->peer_msix_good = false; 2483 ntb->peer_msix_done = false; 2484 } 2485 } 2486 } 2487 return (true); 2488 } 2489 2490 static inline enum ntb_speed 2491 intel_ntb_link_sta_speed(struct ntb_softc *ntb) 2492 { 2493 2494 if (!link_is_up(ntb)) 2495 return (NTB_SPEED_NONE); 2496 return (ntb->lnk_sta & NTB_LINK_SPEED_MASK); 2497 } 2498 2499 static inline enum ntb_width 2500 intel_ntb_link_sta_width(struct ntb_softc *ntb) 2501 { 2502 2503 if (!link_is_up(ntb)) 2504 return (NTB_WIDTH_NONE); 2505 return (NTB_LNK_STA_WIDTH(ntb->lnk_sta)); 2506 } 2507 2508 SYSCTL_NODE(_hw_ntb, OID_AUTO, debug_info, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 2509 "Driver state, statistics, and HW registers"); 2510 2511 #define NTB_REGSZ_MASK (3ul << 30) 2512 #define NTB_REG_64 (1ul << 30) 2513 #define NTB_REG_32 (2ul << 30) 2514 #define NTB_REG_16 (3ul << 30) 2515 #define NTB_REG_8 (0ul << 30) 2516 2517 #define NTB_DB_READ (1ul << 29) 2518 #define NTB_PCI_REG (1ul << 28) 2519 #define NTB_REGFLAGS_MASK (NTB_REGSZ_MASK | NTB_DB_READ | NTB_PCI_REG) 2520 2521 static void 2522 intel_ntb_sysctl_init(struct ntb_softc *ntb) 2523 { 2524 struct sysctl_oid_list *globals, *tree_par, *regpar, *statpar, *errpar; 2525 struct sysctl_ctx_list *ctx; 2526 struct sysctl_oid *tree, *tmptree; 2527 2528 ctx = device_get_sysctl_ctx(ntb->device); 2529 globals = SYSCTL_CHILDREN(device_get_sysctl_tree(ntb->device)); 2530 2531 SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "link_status", 2532 CTLFLAG_RD | CTLTYPE_STRING | CTLFLAG_MPSAFE, ntb, 0, 2533 sysctl_handle_link_status_human, "A", 2534 "Link status (human readable)"); 2535 SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "active", 2536 CTLFLAG_RD | CTLTYPE_UINT | CTLFLAG_MPSAFE, ntb, 0, 2537 sysctl_handle_link_status, "IU", 2538 "Link status (1=active, 0=inactive)"); 2539 SYSCTL_ADD_PROC(ctx, globals, OID_AUTO, "admin_up", 2540 CTLFLAG_RW | CTLTYPE_UINT | CTLFLAG_MPSAFE, ntb, 0, 2541 sysctl_handle_link_admin, "IU", 2542 "Set/get interface status (1=UP, 0=DOWN)"); 2543 2544 tree = SYSCTL_ADD_NODE(ctx, globals, OID_AUTO, "debug_info", 2545 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 2546 "Driver state, statistics, and HW registers"); 2547 tree_par = SYSCTL_CHILDREN(tree); 2548 2549 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "conn_type", CTLFLAG_RD, 2550 &ntb->conn_type, 0, "0 - Transparent; 1 - B2B; 2 - Root Port"); 2551 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "dev_type", CTLFLAG_RD, 2552 &ntb->dev_type, 0, "0 - USD; 1 - DSD"); 2553 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ppd", CTLFLAG_RD, 2554 &ntb->ppd, 0, "Raw PPD register (cached)"); 2555 2556 if (ntb->b2b_mw_idx != B2B_MW_DISABLED) { 2557 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "b2b_idx", CTLFLAG_RD, 2558 &ntb->b2b_mw_idx, 0, 2559 "Index of the MW used for B2B remote register access"); 2560 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "b2b_off", 2561 CTLFLAG_RD, &ntb->b2b_off, 2562 "If non-zero, offset of B2B register region in shared MW"); 2563 } 2564 2565 SYSCTL_ADD_PROC(ctx, tree_par, OID_AUTO, "features", 2566 CTLFLAG_RD | CTLTYPE_STRING | CTLFLAG_MPSAFE, ntb, 0, 2567 sysctl_handle_features, "A", "Features/errata of this NTB device"); 2568 2569 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "ntb_ctl", CTLFLAG_RD, 2570 __DEVOLATILE(uint32_t *, &ntb->ntb_ctl), 0, 2571 "NTB CTL register (cached)"); 2572 SYSCTL_ADD_UINT(ctx, tree_par, OID_AUTO, "lnk_sta", CTLFLAG_RD, 2573 __DEVOLATILE(uint32_t *, &ntb->lnk_sta), 0, 2574 "LNK STA register (cached)"); 2575 2576 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "mw_count", CTLFLAG_RD, 2577 &ntb->mw_count, 0, "MW count"); 2578 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "spad_count", CTLFLAG_RD, 2579 &ntb->spad_count, 0, "Scratchpad count"); 2580 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_count", CTLFLAG_RD, 2581 &ntb->db_count, 0, "Doorbell count"); 2582 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_count", CTLFLAG_RD, 2583 &ntb->db_vec_count, 0, "Doorbell vector count"); 2584 SYSCTL_ADD_U8(ctx, tree_par, OID_AUTO, "db_vec_shift", CTLFLAG_RD, 2585 &ntb->db_vec_shift, 0, "Doorbell vector shift"); 2586 2587 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_valid_mask", CTLFLAG_RD, 2588 &ntb->db_valid_mask, "Doorbell valid mask"); 2589 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_link_mask", CTLFLAG_RD, 2590 &ntb->db_link_mask, "Doorbell link mask"); 2591 SYSCTL_ADD_UQUAD(ctx, tree_par, OID_AUTO, "db_mask", CTLFLAG_RD, 2592 &ntb->db_mask, "Doorbell mask (cached)"); 2593 2594 tmptree = SYSCTL_ADD_NODE(ctx, tree_par, OID_AUTO, "registers", 2595 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 2596 "Raw HW registers (big-endian)"); 2597 regpar = SYSCTL_CHILDREN(tmptree); 2598 2599 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ntbcntl", 2600 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2601 NTB_REG_32 | ntb->reg->ntb_ctl, sysctl_handle_register, "IU", 2602 "NTB Control register"); 2603 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcap", 2604 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2605 NTB_REG_32 | 0x19c, sysctl_handle_register, "IU", 2606 "NTB Link Capabilities"); 2607 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnkcon", 2608 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2609 NTB_REG_32 | 0x1a0, sysctl_handle_register, "IU", 2610 "NTB Link Control register"); 2611 2612 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_mask", 2613 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2614 NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_mask, 2615 sysctl_handle_register, "QU", "Doorbell mask register"); 2616 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "db_bell", 2617 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2618 NTB_REG_64 | NTB_DB_READ | ntb->self_reg->db_bell, 2619 sysctl_handle_register, "QU", "Doorbell register"); 2620 2621 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat23", 2622 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2623 NTB_REG_64 | ntb->xlat_reg->bar2_xlat, 2624 sysctl_handle_register, "QU", "Incoming XLAT23 register"); 2625 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2626 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat4", 2627 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2628 NTB_REG_32 | ntb->xlat_reg->bar4_xlat, 2629 sysctl_handle_register, "IU", "Incoming XLAT4 register"); 2630 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat5", 2631 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2632 NTB_REG_32 | ntb->xlat_reg->bar5_xlat, 2633 sysctl_handle_register, "IU", "Incoming XLAT5 register"); 2634 } else { 2635 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_xlat45", 2636 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2637 NTB_REG_64 | ntb->xlat_reg->bar4_xlat, 2638 sysctl_handle_register, "QU", "Incoming XLAT45 register"); 2639 } 2640 2641 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt23", 2642 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2643 NTB_REG_64 | ntb->xlat_reg->bar2_limit, 2644 sysctl_handle_register, "QU", "Incoming LMT23 register"); 2645 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2646 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt4", 2647 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2648 NTB_REG_32 | ntb->xlat_reg->bar4_limit, 2649 sysctl_handle_register, "IU", "Incoming LMT4 register"); 2650 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt5", 2651 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2652 NTB_REG_32 | ntb->xlat_reg->bar5_limit, 2653 sysctl_handle_register, "IU", "Incoming LMT5 register"); 2654 } else { 2655 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "incoming_lmt45", 2656 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2657 NTB_REG_64 | ntb->xlat_reg->bar4_limit, 2658 sysctl_handle_register, "QU", "Incoming LMT45 register"); 2659 } 2660 2661 if (ntb->type == NTB_ATOM) 2662 return; 2663 2664 tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_stats", 2665 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Xeon HW statistics"); 2666 statpar = SYSCTL_CHILDREN(tmptree); 2667 SYSCTL_ADD_PROC(ctx, statpar, OID_AUTO, "upstream_mem_miss", 2668 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2669 NTB_REG_16 | XEON_USMEMMISS_OFFSET, 2670 sysctl_handle_register, "SU", "Upstream Memory Miss"); 2671 2672 tmptree = SYSCTL_ADD_NODE(ctx, regpar, OID_AUTO, "xeon_hw_err", 2673 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, "Xeon HW errors"); 2674 errpar = SYSCTL_CHILDREN(tmptree); 2675 2676 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "ppd", 2677 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2678 NTB_REG_8 | NTB_PCI_REG | NTB_PPD_OFFSET, 2679 sysctl_handle_register, "CU", "PPD"); 2680 2681 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar23_sz", 2682 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2683 NTB_REG_8 | NTB_PCI_REG | XEON_PBAR23SZ_OFFSET, 2684 sysctl_handle_register, "CU", "PBAR23 SZ (log2)"); 2685 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar4_sz", 2686 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2687 NTB_REG_8 | NTB_PCI_REG | XEON_PBAR4SZ_OFFSET, 2688 sysctl_handle_register, "CU", "PBAR4 SZ (log2)"); 2689 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "pbar5_sz", 2690 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2691 NTB_REG_8 | NTB_PCI_REG | XEON_PBAR5SZ_OFFSET, 2692 sysctl_handle_register, "CU", "PBAR5 SZ (log2)"); 2693 2694 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_sz", 2695 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2696 NTB_REG_8 | NTB_PCI_REG | XEON_SBAR23SZ_OFFSET, 2697 sysctl_handle_register, "CU", "SBAR23 SZ (log2)"); 2698 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_sz", 2699 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2700 NTB_REG_8 | NTB_PCI_REG | XEON_SBAR4SZ_OFFSET, 2701 sysctl_handle_register, "CU", "SBAR4 SZ (log2)"); 2702 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_sz", 2703 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2704 NTB_REG_8 | NTB_PCI_REG | XEON_SBAR5SZ_OFFSET, 2705 sysctl_handle_register, "CU", "SBAR5 SZ (log2)"); 2706 2707 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "devsts", 2708 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2709 NTB_REG_16 | NTB_PCI_REG | XEON_DEVSTS_OFFSET, 2710 sysctl_handle_register, "SU", "DEVSTS"); 2711 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "lnksts", 2712 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2713 NTB_REG_16 | NTB_PCI_REG | XEON_LINK_STATUS_OFFSET, 2714 sysctl_handle_register, "SU", "LNKSTS"); 2715 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "slnksts", 2716 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2717 NTB_REG_16 | NTB_PCI_REG | XEON_SLINK_STATUS_OFFSET, 2718 sysctl_handle_register, "SU", "SLNKSTS"); 2719 2720 SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "uncerrsts", 2721 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2722 NTB_REG_32 | NTB_PCI_REG | XEON_UNCERRSTS_OFFSET, 2723 sysctl_handle_register, "IU", "UNCERRSTS"); 2724 SYSCTL_ADD_PROC(ctx, errpar, OID_AUTO, "corerrsts", 2725 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2726 NTB_REG_32 | NTB_PCI_REG | XEON_CORERRSTS_OFFSET, 2727 sysctl_handle_register, "IU", "CORERRSTS"); 2728 2729 if (ntb->conn_type != NTB_CONN_B2B) 2730 return; 2731 2732 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat01l", 2733 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2734 NTB_REG_32 | XEON_B2B_XLAT_OFFSETL, 2735 sysctl_handle_register, "IU", "Outgoing XLAT0L register"); 2736 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat01u", 2737 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2738 NTB_REG_32 | XEON_B2B_XLAT_OFFSETU, 2739 sysctl_handle_register, "IU", "Outgoing XLAT0U register"); 2740 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat23", 2741 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2742 NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_1].pbarxlat_off, 2743 sysctl_handle_register, "QU", "Outgoing XLAT23 register"); 2744 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2745 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat4", 2746 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2747 NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2748 sysctl_handle_register, "IU", "Outgoing XLAT4 register"); 2749 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat5", 2750 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2751 NTB_REG_32 | ntb->bar_info[NTB_B2B_BAR_3].pbarxlat_off, 2752 sysctl_handle_register, "IU", "Outgoing XLAT5 register"); 2753 } else { 2754 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_xlat45", 2755 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2756 NTB_REG_64 | ntb->bar_info[NTB_B2B_BAR_2].pbarxlat_off, 2757 sysctl_handle_register, "QU", "Outgoing XLAT45 register"); 2758 } 2759 2760 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt23", 2761 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2762 NTB_REG_64 | XEON_PBAR2LMT_OFFSET, 2763 sysctl_handle_register, "QU", "Outgoing LMT23 register"); 2764 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2765 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt4", 2766 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2767 NTB_REG_32 | XEON_PBAR4LMT_OFFSET, 2768 sysctl_handle_register, "IU", "Outgoing LMT4 register"); 2769 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt5", 2770 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2771 NTB_REG_32 | XEON_PBAR5LMT_OFFSET, 2772 sysctl_handle_register, "IU", "Outgoing LMT5 register"); 2773 } else { 2774 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "outgoing_lmt45", 2775 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2776 NTB_REG_64 | XEON_PBAR4LMT_OFFSET, 2777 sysctl_handle_register, "QU", "Outgoing LMT45 register"); 2778 } 2779 2780 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar01_base", 2781 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2782 NTB_REG_64 | ntb->xlat_reg->bar0_base, 2783 sysctl_handle_register, "QU", "Secondary BAR01 base register"); 2784 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar23_base", 2785 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2786 NTB_REG_64 | ntb->xlat_reg->bar2_base, 2787 sysctl_handle_register, "QU", "Secondary BAR23 base register"); 2788 if (HAS_FEATURE(ntb, NTB_SPLIT_BAR)) { 2789 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar4_base", 2790 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2791 NTB_REG_32 | ntb->xlat_reg->bar4_base, 2792 sysctl_handle_register, "IU", 2793 "Secondary BAR4 base register"); 2794 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar5_base", 2795 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2796 NTB_REG_32 | ntb->xlat_reg->bar5_base, 2797 sysctl_handle_register, "IU", 2798 "Secondary BAR5 base register"); 2799 } else { 2800 SYSCTL_ADD_PROC(ctx, regpar, OID_AUTO, "sbar45_base", 2801 CTLFLAG_RD | CTLTYPE_OPAQUE | CTLFLAG_MPSAFE, ntb, 2802 NTB_REG_64 | ntb->xlat_reg->bar4_base, 2803 sysctl_handle_register, "QU", 2804 "Secondary BAR45 base register"); 2805 } 2806 } 2807 2808 static int 2809 sysctl_handle_features(SYSCTL_HANDLER_ARGS) 2810 { 2811 struct ntb_softc *ntb = arg1; 2812 struct sbuf sb; 2813 int error; 2814 2815 sbuf_new_for_sysctl(&sb, NULL, 256, req); 2816 2817 sbuf_printf(&sb, "%b", ntb->features, NTB_FEATURES_STR); 2818 error = sbuf_finish(&sb); 2819 sbuf_delete(&sb); 2820 2821 if (error || !req->newptr) 2822 return (error); 2823 return (EINVAL); 2824 } 2825 2826 static int 2827 sysctl_handle_link_admin(SYSCTL_HANDLER_ARGS) 2828 { 2829 struct ntb_softc *ntb = arg1; 2830 unsigned old, new; 2831 int error; 2832 2833 old = intel_ntb_link_enabled(ntb->device); 2834 2835 error = SYSCTL_OUT(req, &old, sizeof(old)); 2836 if (error != 0 || req->newptr == NULL) 2837 return (error); 2838 2839 error = SYSCTL_IN(req, &new, sizeof(new)); 2840 if (error != 0) 2841 return (error); 2842 2843 intel_ntb_printf(0, "Admin set interface state to '%sabled'\n", 2844 (new != 0)? "en" : "dis"); 2845 2846 if (new != 0) 2847 error = intel_ntb_link_enable(ntb->device, NTB_SPEED_AUTO, NTB_WIDTH_AUTO); 2848 else 2849 error = intel_ntb_link_disable(ntb->device); 2850 return (error); 2851 } 2852 2853 static int 2854 sysctl_handle_link_status_human(SYSCTL_HANDLER_ARGS) 2855 { 2856 struct ntb_softc *ntb = arg1; 2857 struct sbuf sb; 2858 enum ntb_speed speed; 2859 enum ntb_width width; 2860 int error; 2861 2862 sbuf_new_for_sysctl(&sb, NULL, 32, req); 2863 2864 if (intel_ntb_link_is_up(ntb->device, &speed, &width)) 2865 sbuf_printf(&sb, "up / PCIe Gen %u / Width x%u", 2866 (unsigned)speed, (unsigned)width); 2867 else 2868 sbuf_printf(&sb, "down"); 2869 2870 error = sbuf_finish(&sb); 2871 sbuf_delete(&sb); 2872 2873 if (error || !req->newptr) 2874 return (error); 2875 return (EINVAL); 2876 } 2877 2878 static int 2879 sysctl_handle_link_status(SYSCTL_HANDLER_ARGS) 2880 { 2881 struct ntb_softc *ntb = arg1; 2882 unsigned res; 2883 int error; 2884 2885 res = intel_ntb_link_is_up(ntb->device, NULL, NULL); 2886 2887 error = SYSCTL_OUT(req, &res, sizeof(res)); 2888 if (error || !req->newptr) 2889 return (error); 2890 return (EINVAL); 2891 } 2892 2893 static int 2894 sysctl_handle_register(SYSCTL_HANDLER_ARGS) 2895 { 2896 struct ntb_softc *ntb; 2897 const void *outp; 2898 uintptr_t sz; 2899 uint64_t umv; 2900 char be[sizeof(umv)]; 2901 size_t outsz; 2902 uint32_t reg; 2903 bool db, pci; 2904 int error; 2905 2906 ntb = arg1; 2907 reg = arg2 & ~NTB_REGFLAGS_MASK; 2908 sz = arg2 & NTB_REGSZ_MASK; 2909 db = (arg2 & NTB_DB_READ) != 0; 2910 pci = (arg2 & NTB_PCI_REG) != 0; 2911 2912 KASSERT(!(db && pci), ("bogus")); 2913 2914 if (db) { 2915 KASSERT(sz == NTB_REG_64, ("bogus")); 2916 umv = db_ioread(ntb, reg); 2917 outsz = sizeof(uint64_t); 2918 } else { 2919 switch (sz) { 2920 case NTB_REG_64: 2921 if (pci) 2922 umv = pci_read_config(ntb->device, reg, 8); 2923 else 2924 umv = intel_ntb_reg_read(8, reg); 2925 outsz = sizeof(uint64_t); 2926 break; 2927 case NTB_REG_32: 2928 if (pci) 2929 umv = pci_read_config(ntb->device, reg, 4); 2930 else 2931 umv = intel_ntb_reg_read(4, reg); 2932 outsz = sizeof(uint32_t); 2933 break; 2934 case NTB_REG_16: 2935 if (pci) 2936 umv = pci_read_config(ntb->device, reg, 2); 2937 else 2938 umv = intel_ntb_reg_read(2, reg); 2939 outsz = sizeof(uint16_t); 2940 break; 2941 case NTB_REG_8: 2942 if (pci) 2943 umv = pci_read_config(ntb->device, reg, 1); 2944 else 2945 umv = intel_ntb_reg_read(1, reg); 2946 outsz = sizeof(uint8_t); 2947 break; 2948 default: 2949 panic("bogus"); 2950 break; 2951 } 2952 } 2953 2954 /* Encode bigendian so that sysctl -x is legible. */ 2955 be64enc(be, umv); 2956 outp = ((char *)be) + sizeof(umv) - outsz; 2957 2958 error = SYSCTL_OUT(req, outp, outsz); 2959 if (error || !req->newptr) 2960 return (error); 2961 return (EINVAL); 2962 } 2963 2964 static unsigned 2965 intel_ntb_user_mw_to_idx(struct ntb_softc *ntb, unsigned uidx) 2966 { 2967 2968 if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 && 2969 uidx >= ntb->b2b_mw_idx) || 2970 (ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx)) 2971 uidx++; 2972 if ((ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0 && 2973 uidx >= ntb->b2b_mw_idx) && 2974 (ntb->msix_mw_idx != B2B_MW_DISABLED && uidx >= ntb->msix_mw_idx)) 2975 uidx++; 2976 return (uidx); 2977 } 2978 2979 #ifndef EARLY_AP_STARTUP 2980 static int msix_ready; 2981 2982 static void 2983 intel_ntb_msix_ready(void *arg __unused) 2984 { 2985 2986 msix_ready = 1; 2987 } 2988 SYSINIT(intel_ntb_msix_ready, SI_SUB_SMP, SI_ORDER_ANY, 2989 intel_ntb_msix_ready, NULL); 2990 #endif 2991 2992 static void 2993 intel_ntb_exchange_msix(void *ctx) 2994 { 2995 struct ntb_softc *ntb; 2996 uint32_t val; 2997 unsigned i; 2998 2999 ntb = ctx; 3000 3001 if (ntb->peer_msix_good) 3002 goto msix_good; 3003 if (ntb->peer_msix_done) 3004 goto msix_done; 3005 3006 #ifndef EARLY_AP_STARTUP 3007 /* Block MSIX negotiation until SMP started and IRQ reshuffled. */ 3008 if (!msix_ready) 3009 goto reschedule; 3010 #endif 3011 3012 intel_ntb_get_msix_info(ntb); 3013 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 3014 intel_ntb_peer_spad_write(ntb->device, NTB_MSIX_DATA0 + i, 3015 ntb->msix_data[i].nmd_data); 3016 intel_ntb_peer_spad_write(ntb->device, NTB_MSIX_OFS0 + i, 3017 ntb->msix_data[i].nmd_ofs - ntb->msix_xlat); 3018 } 3019 intel_ntb_peer_spad_write(ntb->device, NTB_MSIX_GUARD, NTB_MSIX_VER_GUARD); 3020 3021 intel_ntb_spad_read(ntb->device, NTB_MSIX_GUARD, &val); 3022 if (val != NTB_MSIX_VER_GUARD) 3023 goto reschedule; 3024 3025 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 3026 intel_ntb_spad_read(ntb->device, NTB_MSIX_DATA0 + i, &val); 3027 intel_ntb_printf(2, "remote MSIX data(%u): 0x%x\n", i, val); 3028 ntb->peer_msix_data[i].nmd_data = val; 3029 intel_ntb_spad_read(ntb->device, NTB_MSIX_OFS0 + i, &val); 3030 intel_ntb_printf(2, "remote MSIX addr(%u): 0x%x\n", i, val); 3031 ntb->peer_msix_data[i].nmd_ofs = val; 3032 } 3033 3034 ntb->peer_msix_done = true; 3035 3036 msix_done: 3037 intel_ntb_peer_spad_write(ntb->device, NTB_MSIX_DONE, NTB_MSIX_RECEIVED); 3038 intel_ntb_spad_read(ntb->device, NTB_MSIX_DONE, &val); 3039 if (val != NTB_MSIX_RECEIVED) 3040 goto reschedule; 3041 3042 intel_ntb_spad_clear(ntb->device); 3043 ntb->peer_msix_good = true; 3044 /* Give peer time to see our NTB_MSIX_RECEIVED. */ 3045 goto reschedule; 3046 3047 msix_good: 3048 intel_ntb_poll_link(ntb); 3049 ntb_link_event(ntb->device); 3050 return; 3051 3052 reschedule: 3053 ntb->lnk_sta = pci_read_config(ntb->device, ntb->reg->lnk_sta, 2); 3054 if (_xeon_link_is_up(ntb)) { 3055 callout_reset(&ntb->peer_msix_work, 3056 hz * (ntb->peer_msix_good ? 2 : 1) / 10, 3057 intel_ntb_exchange_msix, ntb); 3058 } else 3059 intel_ntb_spad_clear(ntb->device); 3060 } 3061 3062 /* 3063 * Public API to the rest of the OS 3064 */ 3065 3066 static uint8_t 3067 intel_ntb_spad_count(device_t dev) 3068 { 3069 struct ntb_softc *ntb = device_get_softc(dev); 3070 3071 return (ntb->spad_count); 3072 } 3073 3074 static uint8_t 3075 intel_ntb_mw_count(device_t dev) 3076 { 3077 struct ntb_softc *ntb = device_get_softc(dev); 3078 uint8_t res; 3079 3080 res = ntb->mw_count; 3081 if (ntb->b2b_mw_idx != B2B_MW_DISABLED && ntb->b2b_off == 0) 3082 res--; 3083 if (ntb->msix_mw_idx != B2B_MW_DISABLED) 3084 res--; 3085 return (res); 3086 } 3087 3088 static int 3089 intel_ntb_spad_write(device_t dev, unsigned int idx, uint32_t val) 3090 { 3091 struct ntb_softc *ntb = device_get_softc(dev); 3092 3093 if (idx >= ntb->spad_count) 3094 return (EINVAL); 3095 3096 intel_ntb_reg_write(4, ntb->self_reg->spad + idx * 4, val); 3097 3098 return (0); 3099 } 3100 3101 /* 3102 * Zeros the local scratchpad. 3103 */ 3104 static void 3105 intel_ntb_spad_clear(device_t dev) 3106 { 3107 struct ntb_softc *ntb = device_get_softc(dev); 3108 unsigned i; 3109 3110 for (i = 0; i < ntb->spad_count; i++) 3111 intel_ntb_spad_write(dev, i, 0); 3112 } 3113 3114 static int 3115 intel_ntb_spad_read(device_t dev, unsigned int idx, uint32_t *val) 3116 { 3117 struct ntb_softc *ntb = device_get_softc(dev); 3118 3119 if (idx >= ntb->spad_count) 3120 return (EINVAL); 3121 3122 *val = intel_ntb_reg_read(4, ntb->self_reg->spad + idx * 4); 3123 3124 return (0); 3125 } 3126 3127 static int 3128 intel_ntb_peer_spad_write(device_t dev, unsigned int idx, uint32_t val) 3129 { 3130 struct ntb_softc *ntb = device_get_softc(dev); 3131 3132 if (idx >= ntb->spad_count) 3133 return (EINVAL); 3134 3135 if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) 3136 intel_ntb_mw_write(4, XEON_SPAD_OFFSET + idx * 4, val); 3137 else 3138 intel_ntb_reg_write(4, ntb->peer_reg->spad + idx * 4, val); 3139 3140 return (0); 3141 } 3142 3143 static int 3144 intel_ntb_peer_spad_read(device_t dev, unsigned int idx, uint32_t *val) 3145 { 3146 struct ntb_softc *ntb = device_get_softc(dev); 3147 3148 if (idx >= ntb->spad_count) 3149 return (EINVAL); 3150 3151 if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) 3152 *val = intel_ntb_mw_read(4, XEON_SPAD_OFFSET + idx * 4); 3153 else 3154 *val = intel_ntb_reg_read(4, ntb->peer_reg->spad + idx * 4); 3155 3156 return (0); 3157 } 3158 3159 static int 3160 intel_ntb_mw_get_range(device_t dev, unsigned mw_idx, vm_paddr_t *base, 3161 caddr_t *vbase, size_t *size, size_t *align, size_t *align_size, 3162 bus_addr_t *plimit) 3163 { 3164 struct ntb_softc *ntb = device_get_softc(dev); 3165 struct ntb_pci_bar_info *bar; 3166 bus_addr_t limit; 3167 size_t bar_b2b_off; 3168 enum ntb_bar bar_num; 3169 3170 if (mw_idx >= intel_ntb_mw_count(dev)) 3171 return (EINVAL); 3172 mw_idx = intel_ntb_user_mw_to_idx(ntb, mw_idx); 3173 3174 bar_num = intel_ntb_mw_to_bar(ntb, mw_idx); 3175 bar = &ntb->bar_info[bar_num]; 3176 bar_b2b_off = 0; 3177 if (mw_idx == ntb->b2b_mw_idx) { 3178 KASSERT(ntb->b2b_off != 0, 3179 ("user shouldn't get non-shared b2b mw")); 3180 bar_b2b_off = ntb->b2b_off; 3181 } 3182 3183 if (bar_is_64bit(ntb, bar_num)) 3184 limit = BUS_SPACE_MAXADDR; 3185 else 3186 limit = BUS_SPACE_MAXADDR_32BIT; 3187 3188 if (base != NULL) 3189 *base = bar->pbase + bar_b2b_off; 3190 if (vbase != NULL) 3191 *vbase = bar->vbase + bar_b2b_off; 3192 if (size != NULL) 3193 *size = bar->size - bar_b2b_off; 3194 if (align != NULL) 3195 *align = bar->size; 3196 if (align_size != NULL) 3197 *align_size = 1; 3198 if (plimit != NULL) 3199 *plimit = limit; 3200 return (0); 3201 } 3202 3203 static int 3204 intel_ntb_mw_set_trans(device_t dev, unsigned idx, bus_addr_t addr, size_t size) 3205 { 3206 struct ntb_softc *ntb = device_get_softc(dev); 3207 struct ntb_pci_bar_info *bar; 3208 uint64_t base, limit, reg_val; 3209 size_t bar_size, mw_size; 3210 uint32_t base_reg, xlat_reg, limit_reg; 3211 enum ntb_bar bar_num; 3212 3213 if (idx >= intel_ntb_mw_count(dev)) 3214 return (EINVAL); 3215 idx = intel_ntb_user_mw_to_idx(ntb, idx); 3216 3217 bar_num = intel_ntb_mw_to_bar(ntb, idx); 3218 bar = &ntb->bar_info[bar_num]; 3219 3220 bar_size = bar->size; 3221 if (idx == ntb->b2b_mw_idx) 3222 mw_size = bar_size - ntb->b2b_off; 3223 else 3224 mw_size = bar_size; 3225 3226 /* Hardware requires that addr is aligned to bar size */ 3227 if ((addr & (bar_size - 1)) != 0) 3228 return (EINVAL); 3229 3230 if (size > mw_size) 3231 return (EINVAL); 3232 3233 bar_get_xlat_params(ntb, bar_num, &base_reg, &xlat_reg, &limit_reg); 3234 3235 limit = 0; 3236 if (bar_is_64bit(ntb, bar_num)) { 3237 if (ntb->type == NTB_XEON_GEN3) 3238 base = addr; 3239 else 3240 base = intel_ntb_reg_read(8, base_reg) & BAR_HIGH_MASK; 3241 3242 if (limit_reg != 0 && size != mw_size) 3243 limit = base + size; 3244 else 3245 limit = base + mw_size; 3246 3247 /* Set and verify translation address */ 3248 intel_ntb_reg_write(8, xlat_reg, addr); 3249 reg_val = intel_ntb_reg_read(8, xlat_reg) & BAR_HIGH_MASK; 3250 if (reg_val != addr) { 3251 intel_ntb_reg_write(8, xlat_reg, 0); 3252 return (EIO); 3253 } 3254 3255 /* Set and verify the limit */ 3256 intel_ntb_reg_write(8, limit_reg, limit); 3257 reg_val = intel_ntb_reg_read(8, limit_reg) & BAR_HIGH_MASK; 3258 if (reg_val != limit) { 3259 intel_ntb_reg_write(8, limit_reg, base); 3260 intel_ntb_reg_write(8, xlat_reg, 0); 3261 return (EIO); 3262 } 3263 } else { 3264 /* Configure 32-bit (split) BAR MW */ 3265 if (ntb->type == NTB_XEON_GEN3) 3266 return (EIO); 3267 3268 if ((addr & UINT32_MAX) != addr) 3269 return (ERANGE); 3270 if (((addr + size) & UINT32_MAX) != (addr + size)) 3271 return (ERANGE); 3272 3273 base = intel_ntb_reg_read(4, base_reg) & BAR_HIGH_MASK; 3274 3275 if (limit_reg != 0 && size != mw_size) 3276 limit = base + size; 3277 3278 /* Set and verify translation address */ 3279 intel_ntb_reg_write(4, xlat_reg, addr); 3280 reg_val = intel_ntb_reg_read(4, xlat_reg) & BAR_HIGH_MASK; 3281 if (reg_val != addr) { 3282 intel_ntb_reg_write(4, xlat_reg, 0); 3283 return (EIO); 3284 } 3285 3286 /* Set and verify the limit */ 3287 intel_ntb_reg_write(4, limit_reg, limit); 3288 reg_val = intel_ntb_reg_read(4, limit_reg) & BAR_HIGH_MASK; 3289 if (reg_val != limit) { 3290 intel_ntb_reg_write(4, limit_reg, base); 3291 intel_ntb_reg_write(4, xlat_reg, 0); 3292 return (EIO); 3293 } 3294 } 3295 return (0); 3296 } 3297 3298 static int 3299 intel_ntb_mw_clear_trans(device_t dev, unsigned mw_idx) 3300 { 3301 3302 return (intel_ntb_mw_set_trans(dev, mw_idx, 0, 0)); 3303 } 3304 3305 static int 3306 intel_ntb_mw_get_wc(device_t dev, unsigned idx, vm_memattr_t *mode) 3307 { 3308 struct ntb_softc *ntb = device_get_softc(dev); 3309 struct ntb_pci_bar_info *bar; 3310 3311 if (idx >= intel_ntb_mw_count(dev)) 3312 return (EINVAL); 3313 idx = intel_ntb_user_mw_to_idx(ntb, idx); 3314 3315 bar = &ntb->bar_info[intel_ntb_mw_to_bar(ntb, idx)]; 3316 *mode = bar->map_mode; 3317 return (0); 3318 } 3319 3320 static int 3321 intel_ntb_mw_set_wc(device_t dev, unsigned idx, vm_memattr_t mode) 3322 { 3323 struct ntb_softc *ntb = device_get_softc(dev); 3324 3325 if (idx >= intel_ntb_mw_count(dev)) 3326 return (EINVAL); 3327 3328 idx = intel_ntb_user_mw_to_idx(ntb, idx); 3329 return (intel_ntb_mw_set_wc_internal(ntb, idx, mode)); 3330 } 3331 3332 static int 3333 intel_ntb_mw_set_wc_internal(struct ntb_softc *ntb, unsigned idx, vm_memattr_t mode) 3334 { 3335 struct ntb_pci_bar_info *bar; 3336 int rc; 3337 3338 bar = &ntb->bar_info[intel_ntb_mw_to_bar(ntb, idx)]; 3339 if (bar->map_mode == mode) 3340 return (0); 3341 3342 rc = pmap_change_attr((vm_offset_t)bar->vbase, bar->size, mode); 3343 if (rc == 0) 3344 bar->map_mode = mode; 3345 3346 return (rc); 3347 } 3348 3349 static void 3350 intel_ntb_peer_db_set(device_t dev, uint64_t bits) 3351 { 3352 struct ntb_softc *ntb = device_get_softc(dev); 3353 uint64_t db; 3354 3355 if ((bits & ~ntb->db_valid_mask) != 0) { 3356 device_printf(ntb->device, "Invalid doorbell bits %#jx\n", 3357 (uintmax_t)bits); 3358 return; 3359 } 3360 3361 if (HAS_FEATURE(ntb, NTB_SB01BASE_LOCKUP)) { 3362 struct ntb_pci_bar_info *lapic; 3363 unsigned i; 3364 3365 lapic = ntb->peer_lapic_bar; 3366 3367 for (i = 0; i < XEON_NONLINK_DB_MSIX_BITS; i++) { 3368 if ((bits & intel_ntb_db_vector_mask(dev, i)) != 0) 3369 bus_space_write_4(lapic->pci_bus_tag, 3370 lapic->pci_bus_handle, 3371 ntb->peer_msix_data[i].nmd_ofs, 3372 ntb->peer_msix_data[i].nmd_data); 3373 } 3374 return; 3375 } 3376 3377 if (HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) { 3378 intel_ntb_mw_write(2, XEON_PDOORBELL_OFFSET, bits); 3379 return; 3380 } 3381 3382 if (ntb->type == NTB_XEON_GEN3) { 3383 while (bits != 0) { 3384 db = ffsll(bits); 3385 3386 intel_ntb_reg_write(1, 3387 ntb->peer_reg->db_bell + (db - 1) * 4, 0x1); 3388 3389 bits = bits & (bits - 1); 3390 } 3391 } else { 3392 db_iowrite(ntb, ntb->peer_reg->db_bell, bits); 3393 } 3394 } 3395 3396 static int 3397 intel_ntb_peer_db_addr(device_t dev, bus_addr_t *db_addr, vm_size_t *db_size) 3398 { 3399 struct ntb_softc *ntb = device_get_softc(dev); 3400 struct ntb_pci_bar_info *bar; 3401 uint64_t regoff; 3402 3403 KASSERT((db_addr != NULL && db_size != NULL), ("must be non-NULL")); 3404 3405 if (!HAS_FEATURE(ntb, NTB_SDOORBELL_LOCKUP)) { 3406 bar = &ntb->bar_info[NTB_CONFIG_BAR]; 3407 regoff = ntb->peer_reg->db_bell; 3408 } else { 3409 KASSERT(ntb->b2b_mw_idx != B2B_MW_DISABLED, 3410 ("invalid b2b idx")); 3411 3412 bar = &ntb->bar_info[intel_ntb_mw_to_bar(ntb, ntb->b2b_mw_idx)]; 3413 regoff = XEON_PDOORBELL_OFFSET; 3414 } 3415 KASSERT(bar->pci_bus_tag != X86_BUS_SPACE_IO, ("uh oh")); 3416 3417 /* HACK: Specific to current x86 bus implementation. */ 3418 *db_addr = ((uint64_t)bar->pci_bus_handle + regoff); 3419 *db_size = ntb->reg->db_size; 3420 return (0); 3421 } 3422 3423 static uint64_t 3424 intel_ntb_db_valid_mask(device_t dev) 3425 { 3426 struct ntb_softc *ntb = device_get_softc(dev); 3427 3428 return (ntb->db_valid_mask); 3429 } 3430 3431 static int 3432 intel_ntb_db_vector_count(device_t dev) 3433 { 3434 struct ntb_softc *ntb = device_get_softc(dev); 3435 3436 return (ntb->db_vec_count); 3437 } 3438 3439 static uint64_t 3440 intel_ntb_db_vector_mask(device_t dev, uint32_t vector) 3441 { 3442 struct ntb_softc *ntb = device_get_softc(dev); 3443 3444 if (vector > ntb->db_vec_count) 3445 return (0); 3446 return (ntb->db_valid_mask & intel_ntb_vec_mask(ntb, vector)); 3447 } 3448 3449 static bool 3450 intel_ntb_link_is_up(device_t dev, enum ntb_speed *speed, enum ntb_width *width) 3451 { 3452 struct ntb_softc *ntb = device_get_softc(dev); 3453 3454 if (speed != NULL) 3455 *speed = intel_ntb_link_sta_speed(ntb); 3456 if (width != NULL) 3457 *width = intel_ntb_link_sta_width(ntb); 3458 return (link_is_up(ntb)); 3459 } 3460 3461 static void 3462 save_bar_parameters(struct ntb_pci_bar_info *bar) 3463 { 3464 3465 bar->pci_bus_tag = rman_get_bustag(bar->pci_resource); 3466 bar->pci_bus_handle = rman_get_bushandle(bar->pci_resource); 3467 bar->pbase = rman_get_start(bar->pci_resource); 3468 bar->size = rman_get_size(bar->pci_resource); 3469 bar->vbase = rman_get_virtual(bar->pci_resource); 3470 } 3471 3472 static device_method_t ntb_intel_methods[] = { 3473 /* Device interface */ 3474 DEVMETHOD(device_probe, intel_ntb_probe), 3475 DEVMETHOD(device_attach, intel_ntb_attach), 3476 DEVMETHOD(device_detach, intel_ntb_detach), 3477 /* Bus interface */ 3478 DEVMETHOD(bus_child_location, ntb_child_location), 3479 DEVMETHOD(bus_print_child, ntb_print_child), 3480 DEVMETHOD(bus_get_dma_tag, ntb_get_dma_tag), 3481 /* NTB interface */ 3482 DEVMETHOD(ntb_port_number, intel_ntb_port_number), 3483 DEVMETHOD(ntb_peer_port_count, intel_ntb_peer_port_count), 3484 DEVMETHOD(ntb_peer_port_number, intel_ntb_peer_port_number), 3485 DEVMETHOD(ntb_peer_port_idx, intel_ntb_peer_port_idx), 3486 DEVMETHOD(ntb_link_is_up, intel_ntb_link_is_up), 3487 DEVMETHOD(ntb_link_enable, intel_ntb_link_enable), 3488 DEVMETHOD(ntb_link_disable, intel_ntb_link_disable), 3489 DEVMETHOD(ntb_link_enabled, intel_ntb_link_enabled), 3490 DEVMETHOD(ntb_mw_count, intel_ntb_mw_count), 3491 DEVMETHOD(ntb_mw_get_range, intel_ntb_mw_get_range), 3492 DEVMETHOD(ntb_mw_set_trans, intel_ntb_mw_set_trans), 3493 DEVMETHOD(ntb_mw_clear_trans, intel_ntb_mw_clear_trans), 3494 DEVMETHOD(ntb_mw_get_wc, intel_ntb_mw_get_wc), 3495 DEVMETHOD(ntb_mw_set_wc, intel_ntb_mw_set_wc), 3496 DEVMETHOD(ntb_spad_count, intel_ntb_spad_count), 3497 DEVMETHOD(ntb_spad_clear, intel_ntb_spad_clear), 3498 DEVMETHOD(ntb_spad_write, intel_ntb_spad_write), 3499 DEVMETHOD(ntb_spad_read, intel_ntb_spad_read), 3500 DEVMETHOD(ntb_peer_spad_write, intel_ntb_peer_spad_write), 3501 DEVMETHOD(ntb_peer_spad_read, intel_ntb_peer_spad_read), 3502 DEVMETHOD(ntb_db_valid_mask, intel_ntb_db_valid_mask), 3503 DEVMETHOD(ntb_db_vector_count, intel_ntb_db_vector_count), 3504 DEVMETHOD(ntb_db_vector_mask, intel_ntb_db_vector_mask), 3505 DEVMETHOD(ntb_db_clear, intel_ntb_db_clear), 3506 DEVMETHOD(ntb_db_clear_mask, intel_ntb_db_clear_mask), 3507 DEVMETHOD(ntb_db_read, intel_ntb_db_read), 3508 DEVMETHOD(ntb_db_set_mask, intel_ntb_db_set_mask), 3509 DEVMETHOD(ntb_peer_db_addr, intel_ntb_peer_db_addr), 3510 DEVMETHOD(ntb_peer_db_set, intel_ntb_peer_db_set), 3511 DEVMETHOD_END 3512 }; 3513 3514 static DEFINE_CLASS_0(ntb_hw, ntb_intel_driver, ntb_intel_methods, 3515 sizeof(struct ntb_softc)); 3516 DRIVER_MODULE(ntb_hw_intel, pci, ntb_intel_driver, NULL, NULL); 3517 MODULE_DEPEND(ntb_hw_intel, ntb, 1, 1, 1); 3518 MODULE_VERSION(ntb_hw_intel, 1); 3519 MODULE_PNP_INFO("W32:vendor/device;D:#", pci, ntb_hw_intel, pci_ids, 3520 nitems(pci_ids)); 3521