1e974f91cSConrad Meyer /*- 2e974f91cSConrad Meyer * Copyright (C) 2012 Intel Corporation 3e974f91cSConrad Meyer * All rights reserved. 4e974f91cSConrad Meyer * 5e974f91cSConrad Meyer * Redistribution and use in source and binary forms, with or without 6e974f91cSConrad Meyer * modification, are permitted provided that the following conditions 7e974f91cSConrad Meyer * are met: 8e974f91cSConrad Meyer * 1. Redistributions of source code must retain the above copyright 9e974f91cSConrad Meyer * notice, this list of conditions and the following disclaimer. 10e974f91cSConrad Meyer * 2. Redistributions in binary form must reproduce the above copyright 11e974f91cSConrad Meyer * notice, this list of conditions and the following disclaimer in the 12e974f91cSConrad Meyer * documentation and/or other materials provided with the distribution. 13e974f91cSConrad Meyer * 14e974f91cSConrad Meyer * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15e974f91cSConrad Meyer * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16e974f91cSConrad Meyer * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17e974f91cSConrad Meyer * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18e974f91cSConrad Meyer * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19e974f91cSConrad Meyer * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20e974f91cSConrad Meyer * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21e974f91cSConrad Meyer * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22e974f91cSConrad Meyer * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23e974f91cSConrad Meyer * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24e974f91cSConrad Meyer * SUCH DAMAGE. 25e974f91cSConrad Meyer */ 26e974f91cSConrad Meyer 27e974f91cSConrad Meyer #include <sys/cdefs.h> 28e974f91cSConrad Meyer __FBSDID("$FreeBSD$"); 29e974f91cSConrad Meyer 30e974f91cSConrad Meyer #include <sys/param.h> 31e974f91cSConrad Meyer #include <sys/systm.h> 32e974f91cSConrad Meyer #include <sys/bus.h> 33e974f91cSConrad Meyer #include <sys/conf.h> 34e974f91cSConrad Meyer #include <sys/ioccom.h> 35e974f91cSConrad Meyer #include <sys/kernel.h> 36e974f91cSConrad Meyer #include <sys/lock.h> 37e974f91cSConrad Meyer #include <sys/malloc.h> 38e974f91cSConrad Meyer #include <sys/module.h> 39e974f91cSConrad Meyer #include <sys/mutex.h> 40e974f91cSConrad Meyer #include <sys/rman.h> 41e974f91cSConrad Meyer #include <sys/sysctl.h> 42e974f91cSConrad Meyer #include <sys/time.h> 43e974f91cSConrad Meyer #include <dev/pci/pcireg.h> 44e974f91cSConrad Meyer #include <dev/pci/pcivar.h> 45e974f91cSConrad Meyer #include <machine/bus.h> 46e974f91cSConrad Meyer #include <machine/resource.h> 47e974f91cSConrad Meyer #include <machine/stdarg.h> 48e974f91cSConrad Meyer 49e974f91cSConrad Meyer #include "ioat.h" 50e974f91cSConrad Meyer #include "ioat_hw.h" 51e974f91cSConrad Meyer #include "ioat_internal.h" 52e974f91cSConrad Meyer 53fe720f5aSConrad Meyer #define IOAT_INTR_TIMO (hz / 10) 54466b3540SConrad Meyer #define IOAT_REFLK (&ioat->submit_lock) 55fe720f5aSConrad Meyer 56e974f91cSConrad Meyer static int ioat_probe(device_t device); 57e974f91cSConrad Meyer static int ioat_attach(device_t device); 58e974f91cSConrad Meyer static int ioat_detach(device_t device); 594253ea50SConrad Meyer static int ioat_setup_intr(struct ioat_softc *ioat); 604253ea50SConrad Meyer static int ioat_teardown_intr(struct ioat_softc *ioat); 61e974f91cSConrad Meyer static int ioat3_attach(device_t device); 627afbb263SConrad Meyer static int ioat3_selftest(struct ioat_softc *ioat); 63e974f91cSConrad Meyer static int ioat_map_pci_bar(struct ioat_softc *ioat); 64e974f91cSConrad Meyer static void ioat_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, 65e974f91cSConrad Meyer int error); 66e974f91cSConrad Meyer static void ioat_interrupt_handler(void *arg); 670d1a05d9SConrad Meyer static boolean_t ioat_model_resets_msix(struct ioat_softc *ioat); 68e974f91cSConrad Meyer static void ioat_process_events(struct ioat_softc *ioat); 69e974f91cSConrad Meyer static inline uint32_t ioat_get_active(struct ioat_softc *ioat); 70e974f91cSConrad Meyer static inline uint32_t ioat_get_ring_space(struct ioat_softc *ioat); 71e974f91cSConrad Meyer static void ioat_free_ring_entry(struct ioat_softc *ioat, 72e974f91cSConrad Meyer struct ioat_descriptor *desc); 73e974f91cSConrad Meyer static struct ioat_descriptor *ioat_alloc_ring_entry(struct ioat_softc *ioat); 74e974f91cSConrad Meyer static int ioat_reserve_space_and_lock(struct ioat_softc *ioat, int num_descs); 75e974f91cSConrad Meyer static struct ioat_descriptor *ioat_get_ring_entry(struct ioat_softc *ioat, 76e974f91cSConrad Meyer uint32_t index); 77e974f91cSConrad Meyer static boolean_t resize_ring(struct ioat_softc *ioat, int order); 78e974f91cSConrad Meyer static void ioat_timer_callback(void *arg); 79e974f91cSConrad Meyer static void dump_descriptor(void *hw_desc); 80e974f91cSConrad Meyer static void ioat_submit_single(struct ioat_softc *ioat); 81e974f91cSConrad Meyer static void ioat_comp_update_map(void *arg, bus_dma_segment_t *seg, int nseg, 82e974f91cSConrad Meyer int error); 83e974f91cSConrad Meyer static int ioat_reset_hw(struct ioat_softc *ioat); 84e974f91cSConrad Meyer static void ioat_setup_sysctl(device_t device); 85f7157235SConrad Meyer static int sysctl_handle_reset(SYSCTL_HANDLER_ARGS); 86466b3540SConrad Meyer static inline struct ioat_softc *ioat_get(struct ioat_softc *, 87466b3540SConrad Meyer enum ioat_ref_kind); 88466b3540SConrad Meyer static inline void ioat_put(struct ioat_softc *, enum ioat_ref_kind); 89466b3540SConrad Meyer static inline void ioat_putn(struct ioat_softc *, uint32_t, 90466b3540SConrad Meyer enum ioat_ref_kind); 91466b3540SConrad Meyer static void ioat_drain(struct ioat_softc *); 92e974f91cSConrad Meyer 931c25420eSConrad Meyer #define ioat_log_message(v, ...) do { \ 941c25420eSConrad Meyer if ((v) <= g_ioat_debug_level) { \ 951c25420eSConrad Meyer device_printf(ioat->device, __VA_ARGS__); \ 961c25420eSConrad Meyer } \ 971c25420eSConrad Meyer } while (0) 981c25420eSConrad Meyer 99e974f91cSConrad Meyer MALLOC_DEFINE(M_IOAT, "ioat", "ioat driver memory allocations"); 100e974f91cSConrad Meyer SYSCTL_NODE(_hw, OID_AUTO, ioat, CTLFLAG_RD, 0, "ioat node"); 101e974f91cSConrad Meyer 102e974f91cSConrad Meyer static int g_force_legacy_interrupts; 103e974f91cSConrad Meyer SYSCTL_INT(_hw_ioat, OID_AUTO, force_legacy_interrupts, CTLFLAG_RDTUN, 104e974f91cSConrad Meyer &g_force_legacy_interrupts, 0, "Set to non-zero to force MSI-X disabled"); 105e974f91cSConrad Meyer 1061c25420eSConrad Meyer int g_ioat_debug_level = 0; 107e974f91cSConrad Meyer SYSCTL_INT(_hw_ioat, OID_AUTO, debug_level, CTLFLAG_RWTUN, &g_ioat_debug_level, 108e974f91cSConrad Meyer 0, "Set log level (0-3) for ioat(4). Higher is more verbose."); 109e974f91cSConrad Meyer 110e974f91cSConrad Meyer /* 111e974f91cSConrad Meyer * OS <-> Driver interface structures 112e974f91cSConrad Meyer */ 113e974f91cSConrad Meyer static device_method_t ioat_pci_methods[] = { 114e974f91cSConrad Meyer /* Device interface */ 115e974f91cSConrad Meyer DEVMETHOD(device_probe, ioat_probe), 116e974f91cSConrad Meyer DEVMETHOD(device_attach, ioat_attach), 117e974f91cSConrad Meyer DEVMETHOD(device_detach, ioat_detach), 118e974f91cSConrad Meyer { 0, 0 } 119e974f91cSConrad Meyer }; 120e974f91cSConrad Meyer 121e974f91cSConrad Meyer static driver_t ioat_pci_driver = { 122e974f91cSConrad Meyer "ioat", 123e974f91cSConrad Meyer ioat_pci_methods, 124e974f91cSConrad Meyer sizeof(struct ioat_softc), 125e974f91cSConrad Meyer }; 126e974f91cSConrad Meyer 127e974f91cSConrad Meyer static devclass_t ioat_devclass; 128e974f91cSConrad Meyer DRIVER_MODULE(ioat, pci, ioat_pci_driver, ioat_devclass, 0, 0); 129e974f91cSConrad Meyer 130e974f91cSConrad Meyer /* 131e974f91cSConrad Meyer * Private data structures 132e974f91cSConrad Meyer */ 133e974f91cSConrad Meyer static struct ioat_softc *ioat_channel[IOAT_MAX_CHANNELS]; 134e974f91cSConrad Meyer static int ioat_channel_index = 0; 135e974f91cSConrad Meyer SYSCTL_INT(_hw_ioat, OID_AUTO, channels, CTLFLAG_RD, &ioat_channel_index, 0, 136e974f91cSConrad Meyer "Number of IOAT channels attached"); 137e974f91cSConrad Meyer 138e974f91cSConrad Meyer static struct _pcsid 139e974f91cSConrad Meyer { 140e974f91cSConrad Meyer u_int32_t type; 141e974f91cSConrad Meyer const char *desc; 142e974f91cSConrad Meyer } pci_ids[] = { 143e974f91cSConrad Meyer { 0x34308086, "TBG IOAT Ch0" }, 144e974f91cSConrad Meyer { 0x34318086, "TBG IOAT Ch1" }, 145e974f91cSConrad Meyer { 0x34328086, "TBG IOAT Ch2" }, 146e974f91cSConrad Meyer { 0x34338086, "TBG IOAT Ch3" }, 147e974f91cSConrad Meyer { 0x34298086, "TBG IOAT Ch4" }, 148e974f91cSConrad Meyer { 0x342a8086, "TBG IOAT Ch5" }, 149e974f91cSConrad Meyer { 0x342b8086, "TBG IOAT Ch6" }, 150e974f91cSConrad Meyer { 0x342c8086, "TBG IOAT Ch7" }, 151e974f91cSConrad Meyer 152e974f91cSConrad Meyer { 0x37108086, "JSF IOAT Ch0" }, 153e974f91cSConrad Meyer { 0x37118086, "JSF IOAT Ch1" }, 154e974f91cSConrad Meyer { 0x37128086, "JSF IOAT Ch2" }, 155e974f91cSConrad Meyer { 0x37138086, "JSF IOAT Ch3" }, 156e974f91cSConrad Meyer { 0x37148086, "JSF IOAT Ch4" }, 157e974f91cSConrad Meyer { 0x37158086, "JSF IOAT Ch5" }, 158e974f91cSConrad Meyer { 0x37168086, "JSF IOAT Ch6" }, 159e974f91cSConrad Meyer { 0x37178086, "JSF IOAT Ch7" }, 160e974f91cSConrad Meyer { 0x37188086, "JSF IOAT Ch0 (RAID)" }, 161e974f91cSConrad Meyer { 0x37198086, "JSF IOAT Ch1 (RAID)" }, 162e974f91cSConrad Meyer 163e974f91cSConrad Meyer { 0x3c208086, "SNB IOAT Ch0" }, 164e974f91cSConrad Meyer { 0x3c218086, "SNB IOAT Ch1" }, 165e974f91cSConrad Meyer { 0x3c228086, "SNB IOAT Ch2" }, 166e974f91cSConrad Meyer { 0x3c238086, "SNB IOAT Ch3" }, 167e974f91cSConrad Meyer { 0x3c248086, "SNB IOAT Ch4" }, 168e974f91cSConrad Meyer { 0x3c258086, "SNB IOAT Ch5" }, 169e974f91cSConrad Meyer { 0x3c268086, "SNB IOAT Ch6" }, 170e974f91cSConrad Meyer { 0x3c278086, "SNB IOAT Ch7" }, 171e974f91cSConrad Meyer { 0x3c2e8086, "SNB IOAT Ch0 (RAID)" }, 172e974f91cSConrad Meyer { 0x3c2f8086, "SNB IOAT Ch1 (RAID)" }, 173e974f91cSConrad Meyer 174e974f91cSConrad Meyer { 0x0e208086, "IVB IOAT Ch0" }, 175e974f91cSConrad Meyer { 0x0e218086, "IVB IOAT Ch1" }, 176e974f91cSConrad Meyer { 0x0e228086, "IVB IOAT Ch2" }, 177e974f91cSConrad Meyer { 0x0e238086, "IVB IOAT Ch3" }, 178e974f91cSConrad Meyer { 0x0e248086, "IVB IOAT Ch4" }, 179e974f91cSConrad Meyer { 0x0e258086, "IVB IOAT Ch5" }, 180e974f91cSConrad Meyer { 0x0e268086, "IVB IOAT Ch6" }, 181e974f91cSConrad Meyer { 0x0e278086, "IVB IOAT Ch7" }, 182e974f91cSConrad Meyer { 0x0e2e8086, "IVB IOAT Ch0 (RAID)" }, 183e974f91cSConrad Meyer { 0x0e2f8086, "IVB IOAT Ch1 (RAID)" }, 184e974f91cSConrad Meyer 185e974f91cSConrad Meyer { 0x2f208086, "HSW IOAT Ch0" }, 186e974f91cSConrad Meyer { 0x2f218086, "HSW IOAT Ch1" }, 187e974f91cSConrad Meyer { 0x2f228086, "HSW IOAT Ch2" }, 188e974f91cSConrad Meyer { 0x2f238086, "HSW IOAT Ch3" }, 189e974f91cSConrad Meyer { 0x2f248086, "HSW IOAT Ch4" }, 190e974f91cSConrad Meyer { 0x2f258086, "HSW IOAT Ch5" }, 191e974f91cSConrad Meyer { 0x2f268086, "HSW IOAT Ch6" }, 192e974f91cSConrad Meyer { 0x2f278086, "HSW IOAT Ch7" }, 193e974f91cSConrad Meyer { 0x2f2e8086, "HSW IOAT Ch0 (RAID)" }, 194e974f91cSConrad Meyer { 0x2f2f8086, "HSW IOAT Ch1 (RAID)" }, 195e974f91cSConrad Meyer 196e974f91cSConrad Meyer { 0x0c508086, "BWD IOAT Ch0" }, 197e974f91cSConrad Meyer { 0x0c518086, "BWD IOAT Ch1" }, 198e974f91cSConrad Meyer { 0x0c528086, "BWD IOAT Ch2" }, 199e974f91cSConrad Meyer { 0x0c538086, "BWD IOAT Ch3" }, 200e974f91cSConrad Meyer 201e974f91cSConrad Meyer { 0x6f508086, "BDXDE IOAT Ch0" }, 202e974f91cSConrad Meyer { 0x6f518086, "BDXDE IOAT Ch1" }, 203e974f91cSConrad Meyer { 0x6f528086, "BDXDE IOAT Ch2" }, 204e974f91cSConrad Meyer { 0x6f538086, "BDXDE IOAT Ch3" }, 205e974f91cSConrad Meyer 206e974f91cSConrad Meyer { 0x00000000, NULL } 207e974f91cSConrad Meyer }; 208e974f91cSConrad Meyer 209e974f91cSConrad Meyer /* 210e974f91cSConrad Meyer * OS <-> Driver linkage functions 211e974f91cSConrad Meyer */ 212e974f91cSConrad Meyer static int 213e974f91cSConrad Meyer ioat_probe(device_t device) 214e974f91cSConrad Meyer { 215e974f91cSConrad Meyer struct _pcsid *ep; 216e974f91cSConrad Meyer u_int32_t type; 217e974f91cSConrad Meyer 218e974f91cSConrad Meyer type = pci_get_devid(device); 219e974f91cSConrad Meyer for (ep = pci_ids; ep->type; ep++) { 220e974f91cSConrad Meyer if (ep->type == type) { 221e974f91cSConrad Meyer device_set_desc(device, ep->desc); 222e974f91cSConrad Meyer return (0); 223e974f91cSConrad Meyer } 224e974f91cSConrad Meyer } 225e974f91cSConrad Meyer return (ENXIO); 226e974f91cSConrad Meyer } 227e974f91cSConrad Meyer 228e974f91cSConrad Meyer static int 229e974f91cSConrad Meyer ioat_attach(device_t device) 230e974f91cSConrad Meyer { 231e974f91cSConrad Meyer struct ioat_softc *ioat; 232e974f91cSConrad Meyer int error; 233e974f91cSConrad Meyer 234e974f91cSConrad Meyer ioat = DEVICE2SOFTC(device); 235e974f91cSConrad Meyer ioat->device = device; 236e974f91cSConrad Meyer 237e974f91cSConrad Meyer error = ioat_map_pci_bar(ioat); 238e974f91cSConrad Meyer if (error != 0) 239e974f91cSConrad Meyer goto err; 240e974f91cSConrad Meyer 241e974f91cSConrad Meyer ioat->version = ioat_read_cbver(ioat); 242e974f91cSConrad Meyer if (ioat->version < IOAT_VER_3_0) { 243e974f91cSConrad Meyer error = ENODEV; 244e974f91cSConrad Meyer goto err; 245e974f91cSConrad Meyer } 246e974f91cSConrad Meyer 247e974f91cSConrad Meyer error = ioat3_attach(device); 248e974f91cSConrad Meyer if (error != 0) 249e974f91cSConrad Meyer goto err; 250e974f91cSConrad Meyer 251e974f91cSConrad Meyer error = pci_enable_busmaster(device); 252e974f91cSConrad Meyer if (error != 0) 253e974f91cSConrad Meyer goto err; 254e974f91cSConrad Meyer 255466b3540SConrad Meyer error = ioat_setup_intr(ioat); 256466b3540SConrad Meyer if (error != 0) 257466b3540SConrad Meyer goto err; 258466b3540SConrad Meyer 2597afbb263SConrad Meyer error = ioat3_selftest(ioat); 2607afbb263SConrad Meyer if (error != 0) 261466b3540SConrad Meyer goto err; 2627afbb263SConrad Meyer 2637afbb263SConrad Meyer ioat_process_events(ioat); 2647afbb263SConrad Meyer ioat_setup_sysctl(device); 2657afbb263SConrad Meyer 266e974f91cSConrad Meyer ioat_channel[ioat_channel_index++] = ioat; 2677afbb263SConrad Meyer ioat_test_attach(); 268e974f91cSConrad Meyer 269e974f91cSConrad Meyer err: 270e974f91cSConrad Meyer if (error != 0) 271e974f91cSConrad Meyer ioat_detach(device); 272e974f91cSConrad Meyer return (error); 273e974f91cSConrad Meyer } 274e974f91cSConrad Meyer 275e974f91cSConrad Meyer static int 276e974f91cSConrad Meyer ioat_detach(device_t device) 277e974f91cSConrad Meyer { 278e974f91cSConrad Meyer struct ioat_softc *ioat; 279e974f91cSConrad Meyer uint32_t i; 280e974f91cSConrad Meyer 281e974f91cSConrad Meyer ioat = DEVICE2SOFTC(device); 2827afbb263SConrad Meyer 2837afbb263SConrad Meyer ioat_test_detach(); 284466b3540SConrad Meyer ioat_drain(ioat); 285fe720f5aSConrad Meyer 286fe720f5aSConrad Meyer ioat_teardown_intr(ioat); 287e974f91cSConrad Meyer callout_drain(&ioat->timer); 288e974f91cSConrad Meyer 289e974f91cSConrad Meyer pci_disable_busmaster(device); 290e974f91cSConrad Meyer 291e974f91cSConrad Meyer if (ioat->pci_resource != NULL) 292e974f91cSConrad Meyer bus_release_resource(device, SYS_RES_MEMORY, 293e974f91cSConrad Meyer ioat->pci_resource_id, ioat->pci_resource); 294e974f91cSConrad Meyer 295e974f91cSConrad Meyer if (ioat->ring != NULL) { 296e974f91cSConrad Meyer for (i = 0; i < (1 << ioat->ring_size_order); i++) 297e974f91cSConrad Meyer ioat_free_ring_entry(ioat, ioat->ring[i]); 298e974f91cSConrad Meyer free(ioat->ring, M_IOAT); 299e974f91cSConrad Meyer } 300e974f91cSConrad Meyer 301e974f91cSConrad Meyer if (ioat->comp_update != NULL) { 302e974f91cSConrad Meyer bus_dmamap_unload(ioat->comp_update_tag, ioat->comp_update_map); 303e974f91cSConrad Meyer bus_dmamem_free(ioat->comp_update_tag, ioat->comp_update, 304e974f91cSConrad Meyer ioat->comp_update_map); 305e974f91cSConrad Meyer bus_dma_tag_destroy(ioat->comp_update_tag); 306e974f91cSConrad Meyer } 307e974f91cSConrad Meyer 308e974f91cSConrad Meyer bus_dma_tag_destroy(ioat->hw_desc_tag); 309e974f91cSConrad Meyer 3104253ea50SConrad Meyer return (0); 3114253ea50SConrad Meyer } 3124253ea50SConrad Meyer 3134253ea50SConrad Meyer static int 3144253ea50SConrad Meyer ioat_teardown_intr(struct ioat_softc *ioat) 3154253ea50SConrad Meyer { 3164253ea50SConrad Meyer 317e974f91cSConrad Meyer if (ioat->tag != NULL) 3184253ea50SConrad Meyer bus_teardown_intr(ioat->device, ioat->res, ioat->tag); 319e974f91cSConrad Meyer 320e974f91cSConrad Meyer if (ioat->res != NULL) 3214253ea50SConrad Meyer bus_release_resource(ioat->device, SYS_RES_IRQ, 322e974f91cSConrad Meyer rman_get_rid(ioat->res), ioat->res); 323e974f91cSConrad Meyer 3244253ea50SConrad Meyer pci_release_msi(ioat->device); 325e974f91cSConrad Meyer return (0); 326e974f91cSConrad Meyer } 327e974f91cSConrad Meyer 328e974f91cSConrad Meyer static int 329e974f91cSConrad Meyer ioat3_selftest(struct ioat_softc *ioat) 330e974f91cSConrad Meyer { 331e974f91cSConrad Meyer uint64_t status; 332e974f91cSConrad Meyer uint32_t chanerr; 333e974f91cSConrad Meyer int i; 334e974f91cSConrad Meyer 335e974f91cSConrad Meyer ioat_acquire(&ioat->dmaengine); 336e974f91cSConrad Meyer ioat_null(&ioat->dmaengine, NULL, NULL, 0); 337e974f91cSConrad Meyer ioat_release(&ioat->dmaengine); 338e974f91cSConrad Meyer 339e974f91cSConrad Meyer for (i = 0; i < 100; i++) { 340e974f91cSConrad Meyer DELAY(1); 341e974f91cSConrad Meyer status = ioat_get_chansts(ioat); 342e974f91cSConrad Meyer if (is_ioat_idle(status)) 343e974f91cSConrad Meyer return (0); 344e974f91cSConrad Meyer } 345e974f91cSConrad Meyer 346e974f91cSConrad Meyer chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET); 347e974f91cSConrad Meyer ioat_log_message(0, "could not start channel: " 348e974f91cSConrad Meyer "status = %#jx error = %x\n", (uintmax_t)status, chanerr); 349e974f91cSConrad Meyer return (ENXIO); 350e974f91cSConrad Meyer } 351e974f91cSConrad Meyer 352e974f91cSConrad Meyer /* 353e974f91cSConrad Meyer * Initialize Hardware 354e974f91cSConrad Meyer */ 355e974f91cSConrad Meyer static int 356e974f91cSConrad Meyer ioat3_attach(device_t device) 357e974f91cSConrad Meyer { 358e974f91cSConrad Meyer struct ioat_softc *ioat; 359e974f91cSConrad Meyer struct ioat_descriptor **ring; 360e974f91cSConrad Meyer struct ioat_descriptor *next; 361e974f91cSConrad Meyer struct ioat_dma_hw_descriptor *dma_hw_desc; 362e974f91cSConrad Meyer uint32_t capabilities; 363e974f91cSConrad Meyer int i, num_descriptors; 364e974f91cSConrad Meyer int error; 365e974f91cSConrad Meyer uint8_t xfercap; 366e974f91cSConrad Meyer 367e974f91cSConrad Meyer error = 0; 368e974f91cSConrad Meyer ioat = DEVICE2SOFTC(device); 369e974f91cSConrad Meyer capabilities = ioat_read_dmacapability(ioat); 370e974f91cSConrad Meyer 371e974f91cSConrad Meyer xfercap = ioat_read_xfercap(ioat); 372e974f91cSConrad Meyer ioat->max_xfer_size = 1 << xfercap; 373e974f91cSConrad Meyer 374e974f91cSConrad Meyer /* TODO: need to check DCA here if we ever do XOR/PQ */ 375e974f91cSConrad Meyer 376e974f91cSConrad Meyer mtx_init(&ioat->submit_lock, "ioat_submit", NULL, MTX_DEF); 377e974f91cSConrad Meyer mtx_init(&ioat->cleanup_lock, "ioat_process_events", NULL, MTX_DEF); 3787afbb263SConrad Meyer callout_init(&ioat->timer, 1); 379e974f91cSConrad Meyer 380e974f91cSConrad Meyer ioat->is_resize_pending = FALSE; 381e974f91cSConrad Meyer ioat->is_completion_pending = FALSE; 382e974f91cSConrad Meyer ioat->is_reset_pending = FALSE; 383e974f91cSConrad Meyer ioat->is_channel_running = FALSE; 384e974f91cSConrad Meyer ioat->is_waiting_for_ack = FALSE; 385e974f91cSConrad Meyer 386e974f91cSConrad Meyer bus_dma_tag_create(bus_get_dma_tag(ioat->device), sizeof(uint64_t), 0x0, 387e974f91cSConrad Meyer BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, 388e974f91cSConrad Meyer sizeof(uint64_t), 1, sizeof(uint64_t), 0, NULL, NULL, 389e974f91cSConrad Meyer &ioat->comp_update_tag); 390e974f91cSConrad Meyer 391e974f91cSConrad Meyer error = bus_dmamem_alloc(ioat->comp_update_tag, 392e974f91cSConrad Meyer (void **)&ioat->comp_update, BUS_DMA_ZERO, &ioat->comp_update_map); 393e974f91cSConrad Meyer if (ioat->comp_update == NULL) 394e974f91cSConrad Meyer return (ENOMEM); 395e974f91cSConrad Meyer 396e974f91cSConrad Meyer error = bus_dmamap_load(ioat->comp_update_tag, ioat->comp_update_map, 397e974f91cSConrad Meyer ioat->comp_update, sizeof(uint64_t), ioat_comp_update_map, ioat, 398e974f91cSConrad Meyer 0); 399e974f91cSConrad Meyer if (error != 0) 400e974f91cSConrad Meyer return (error); 401e974f91cSConrad Meyer 402e974f91cSConrad Meyer ioat->ring_size_order = IOAT_MIN_ORDER; 403e974f91cSConrad Meyer 404e974f91cSConrad Meyer num_descriptors = 1 << ioat->ring_size_order; 405e974f91cSConrad Meyer 406e974f91cSConrad Meyer bus_dma_tag_create(bus_get_dma_tag(ioat->device), 0x40, 0x0, 407e974f91cSConrad Meyer BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL, 408e974f91cSConrad Meyer sizeof(struct ioat_dma_hw_descriptor), 1, 409e974f91cSConrad Meyer sizeof(struct ioat_dma_hw_descriptor), 0, NULL, NULL, 410e974f91cSConrad Meyer &ioat->hw_desc_tag); 411e974f91cSConrad Meyer 412e974f91cSConrad Meyer ioat->ring = malloc(num_descriptors * sizeof(*ring), M_IOAT, 413e974f91cSConrad Meyer M_ZERO | M_NOWAIT); 414e974f91cSConrad Meyer if (ioat->ring == NULL) 415e974f91cSConrad Meyer return (ENOMEM); 416e974f91cSConrad Meyer 417e974f91cSConrad Meyer ring = ioat->ring; 418e974f91cSConrad Meyer for (i = 0; i < num_descriptors; i++) { 419e974f91cSConrad Meyer ring[i] = ioat_alloc_ring_entry(ioat); 420e974f91cSConrad Meyer if (ring[i] == NULL) 421e974f91cSConrad Meyer return (ENOMEM); 422e974f91cSConrad Meyer 423e974f91cSConrad Meyer ring[i]->id = i; 424e974f91cSConrad Meyer } 425e974f91cSConrad Meyer 426e974f91cSConrad Meyer for (i = 0; i < num_descriptors - 1; i++) { 427e974f91cSConrad Meyer next = ring[i + 1]; 428e974f91cSConrad Meyer dma_hw_desc = ring[i]->u.dma; 429e974f91cSConrad Meyer 430e974f91cSConrad Meyer dma_hw_desc->next = next->hw_desc_bus_addr; 431e974f91cSConrad Meyer } 432e974f91cSConrad Meyer 433e974f91cSConrad Meyer ring[i]->u.dma->next = ring[0]->hw_desc_bus_addr; 434e974f91cSConrad Meyer 435e974f91cSConrad Meyer ioat->head = 0; 436e974f91cSConrad Meyer ioat->tail = 0; 437e974f91cSConrad Meyer ioat->last_seen = 0; 438e974f91cSConrad Meyer 439e974f91cSConrad Meyer error = ioat_reset_hw(ioat); 440e974f91cSConrad Meyer if (error != 0) 441e974f91cSConrad Meyer return (error); 442e974f91cSConrad Meyer 443e974f91cSConrad Meyer ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN); 444e974f91cSConrad Meyer ioat_write_chancmp(ioat, ioat->comp_update_bus_addr); 445e974f91cSConrad Meyer ioat_write_chainaddr(ioat, ring[0]->hw_desc_bus_addr); 446e974f91cSConrad Meyer return (0); 447e974f91cSConrad Meyer } 448e974f91cSConrad Meyer 449e974f91cSConrad Meyer static int 450e974f91cSConrad Meyer ioat_map_pci_bar(struct ioat_softc *ioat) 451e974f91cSConrad Meyer { 452e974f91cSConrad Meyer 453e974f91cSConrad Meyer ioat->pci_resource_id = PCIR_BAR(0); 454*e88e14b9SConrad Meyer ioat->pci_resource = bus_alloc_resource_any(ioat->device, 455*e88e14b9SConrad Meyer SYS_RES_MEMORY, &ioat->pci_resource_id, RF_ACTIVE); 456e974f91cSConrad Meyer 457e974f91cSConrad Meyer if (ioat->pci_resource == NULL) { 458e974f91cSConrad Meyer ioat_log_message(0, "unable to allocate pci resource\n"); 459e974f91cSConrad Meyer return (ENODEV); 460e974f91cSConrad Meyer } 461e974f91cSConrad Meyer 462e974f91cSConrad Meyer ioat->pci_bus_tag = rman_get_bustag(ioat->pci_resource); 463e974f91cSConrad Meyer ioat->pci_bus_handle = rman_get_bushandle(ioat->pci_resource); 464e974f91cSConrad Meyer return (0); 465e974f91cSConrad Meyer } 466e974f91cSConrad Meyer 467e974f91cSConrad Meyer static void 468e974f91cSConrad Meyer ioat_comp_update_map(void *arg, bus_dma_segment_t *seg, int nseg, int error) 469e974f91cSConrad Meyer { 470e974f91cSConrad Meyer struct ioat_softc *ioat = arg; 471e974f91cSConrad Meyer 472e974f91cSConrad Meyer ioat->comp_update_bus_addr = seg[0].ds_addr; 473e974f91cSConrad Meyer } 474e974f91cSConrad Meyer 475e974f91cSConrad Meyer static void 476e974f91cSConrad Meyer ioat_dmamap_cb(void *arg, bus_dma_segment_t *segs, int nseg, int error) 477e974f91cSConrad Meyer { 478e974f91cSConrad Meyer bus_addr_t *baddr; 479e974f91cSConrad Meyer 480e974f91cSConrad Meyer baddr = arg; 481e974f91cSConrad Meyer *baddr = segs->ds_addr; 482e974f91cSConrad Meyer } 483e974f91cSConrad Meyer 484e974f91cSConrad Meyer /* 485e974f91cSConrad Meyer * Interrupt setup and handlers 486e974f91cSConrad Meyer */ 487e974f91cSConrad Meyer static int 4884253ea50SConrad Meyer ioat_setup_intr(struct ioat_softc *ioat) 489e974f91cSConrad Meyer { 490e974f91cSConrad Meyer uint32_t num_vectors; 491e974f91cSConrad Meyer int error; 492e974f91cSConrad Meyer boolean_t use_msix; 493e974f91cSConrad Meyer boolean_t force_legacy_interrupts; 494e974f91cSConrad Meyer 495e974f91cSConrad Meyer use_msix = FALSE; 496e974f91cSConrad Meyer force_legacy_interrupts = FALSE; 497e974f91cSConrad Meyer 498e974f91cSConrad Meyer if (!g_force_legacy_interrupts && pci_msix_count(ioat->device) >= 1) { 499e974f91cSConrad Meyer num_vectors = 1; 500e974f91cSConrad Meyer pci_alloc_msix(ioat->device, &num_vectors); 501e974f91cSConrad Meyer if (num_vectors == 1) 502e974f91cSConrad Meyer use_msix = TRUE; 503e974f91cSConrad Meyer } 504e974f91cSConrad Meyer 505e974f91cSConrad Meyer if (use_msix) { 506e974f91cSConrad Meyer ioat->rid = 1; 507e974f91cSConrad Meyer ioat->res = bus_alloc_resource_any(ioat->device, SYS_RES_IRQ, 508e974f91cSConrad Meyer &ioat->rid, RF_ACTIVE); 509e974f91cSConrad Meyer } else { 510e974f91cSConrad Meyer ioat->rid = 0; 511e974f91cSConrad Meyer ioat->res = bus_alloc_resource_any(ioat->device, SYS_RES_IRQ, 512e974f91cSConrad Meyer &ioat->rid, RF_SHAREABLE | RF_ACTIVE); 513e974f91cSConrad Meyer } 514e974f91cSConrad Meyer if (ioat->res == NULL) { 515e974f91cSConrad Meyer ioat_log_message(0, "bus_alloc_resource failed\n"); 516e974f91cSConrad Meyer return (ENOMEM); 517e974f91cSConrad Meyer } 518e974f91cSConrad Meyer 519e974f91cSConrad Meyer ioat->tag = NULL; 520e974f91cSConrad Meyer error = bus_setup_intr(ioat->device, ioat->res, INTR_MPSAFE | 521e974f91cSConrad Meyer INTR_TYPE_MISC, NULL, ioat_interrupt_handler, ioat, &ioat->tag); 522e974f91cSConrad Meyer if (error != 0) { 523e974f91cSConrad Meyer ioat_log_message(0, "bus_setup_intr failed\n"); 524e974f91cSConrad Meyer return (error); 525e974f91cSConrad Meyer } 526e974f91cSConrad Meyer 527e974f91cSConrad Meyer ioat_write_intrctrl(ioat, IOAT_INTRCTRL_MASTER_INT_EN); 528e974f91cSConrad Meyer return (0); 529e974f91cSConrad Meyer } 530e974f91cSConrad Meyer 5314253ea50SConrad Meyer static boolean_t 5320d1a05d9SConrad Meyer ioat_model_resets_msix(struct ioat_softc *ioat) 5334253ea50SConrad Meyer { 5344253ea50SConrad Meyer u_int32_t pciid; 5354253ea50SConrad Meyer 5364253ea50SConrad Meyer pciid = pci_get_devid(ioat->device); 5374253ea50SConrad Meyer switch (pciid) { 5380d1a05d9SConrad Meyer /* BWD: */ 5390d1a05d9SConrad Meyer case 0x0c508086: 5400d1a05d9SConrad Meyer case 0x0c518086: 5410d1a05d9SConrad Meyer case 0x0c528086: 5420d1a05d9SConrad Meyer case 0x0c538086: 5430d1a05d9SConrad Meyer /* BDXDE: */ 5444253ea50SConrad Meyer case 0x6f508086: 5454253ea50SConrad Meyer case 0x6f518086: 5464253ea50SConrad Meyer case 0x6f528086: 5474253ea50SConrad Meyer case 0x6f538086: 5484253ea50SConrad Meyer return (TRUE); 5494253ea50SConrad Meyer } 5504253ea50SConrad Meyer 5514253ea50SConrad Meyer return (FALSE); 5524253ea50SConrad Meyer } 5534253ea50SConrad Meyer 554e974f91cSConrad Meyer static void 555e974f91cSConrad Meyer ioat_interrupt_handler(void *arg) 556e974f91cSConrad Meyer { 557e974f91cSConrad Meyer struct ioat_softc *ioat = arg; 558e974f91cSConrad Meyer 559e974f91cSConrad Meyer ioat_process_events(ioat); 560e974f91cSConrad Meyer } 561e974f91cSConrad Meyer 562e974f91cSConrad Meyer static void 563e974f91cSConrad Meyer ioat_process_events(struct ioat_softc *ioat) 564e974f91cSConrad Meyer { 565e974f91cSConrad Meyer struct ioat_descriptor *desc; 566e974f91cSConrad Meyer struct bus_dmadesc *dmadesc; 567e974f91cSConrad Meyer uint64_t comp_update, status; 568e974f91cSConrad Meyer uint32_t completed; 569e974f91cSConrad Meyer 570e974f91cSConrad Meyer mtx_lock(&ioat->cleanup_lock); 571e974f91cSConrad Meyer 572e974f91cSConrad Meyer completed = 0; 573e974f91cSConrad Meyer comp_update = *ioat->comp_update; 574e974f91cSConrad Meyer status = comp_update & IOAT_CHANSTS_COMPLETED_DESCRIPTOR_MASK; 575e974f91cSConrad Meyer 576e974f91cSConrad Meyer ioat_log_message(3, "%s\n", __func__); 577e974f91cSConrad Meyer 5784becebdfSConrad Meyer if (status == ioat->last_seen) 5794becebdfSConrad Meyer goto out; 580e974f91cSConrad Meyer 581e974f91cSConrad Meyer while (1) { 582e974f91cSConrad Meyer desc = ioat_get_ring_entry(ioat, ioat->tail); 583e974f91cSConrad Meyer dmadesc = &desc->bus_dmadesc; 584e974f91cSConrad Meyer ioat_log_message(3, "completing desc %d\n", ioat->tail); 585e974f91cSConrad Meyer 586e974f91cSConrad Meyer if (dmadesc->callback_fn) 587e974f91cSConrad Meyer (*dmadesc->callback_fn)(dmadesc->callback_arg); 588e974f91cSConrad Meyer 589466b3540SConrad Meyer completed++; 590e974f91cSConrad Meyer ioat->tail++; 591e974f91cSConrad Meyer if (desc->hw_desc_bus_addr == status) 592e974f91cSConrad Meyer break; 593e974f91cSConrad Meyer } 594e974f91cSConrad Meyer 595e974f91cSConrad Meyer ioat->last_seen = desc->hw_desc_bus_addr; 596e974f91cSConrad Meyer 597e974f91cSConrad Meyer if (ioat->head == ioat->tail) { 598e974f91cSConrad Meyer ioat->is_completion_pending = FALSE; 599fe720f5aSConrad Meyer callout_reset(&ioat->timer, IOAT_INTR_TIMO, 600fe720f5aSConrad Meyer ioat_timer_callback, ioat); 601e974f91cSConrad Meyer } 602e974f91cSConrad Meyer 6034becebdfSConrad Meyer out: 604e974f91cSConrad Meyer ioat_write_chanctrl(ioat, IOAT_CHANCTRL_RUN); 605e974f91cSConrad Meyer mtx_unlock(&ioat->cleanup_lock); 606466b3540SConrad Meyer 607466b3540SConrad Meyer ioat_putn(ioat, completed, IOAT_ACTIVE_DESCR_REF); 608e974f91cSConrad Meyer } 609e974f91cSConrad Meyer 610e974f91cSConrad Meyer /* 611e974f91cSConrad Meyer * User API functions 612e974f91cSConrad Meyer */ 613e974f91cSConrad Meyer bus_dmaengine_t 614e974f91cSConrad Meyer ioat_get_dmaengine(uint32_t index) 615e974f91cSConrad Meyer { 616e974f91cSConrad Meyer 617466b3540SConrad Meyer if (index >= ioat_channel_index) 618e974f91cSConrad Meyer return (NULL); 619466b3540SConrad Meyer return (&ioat_get(ioat_channel[index], IOAT_DMAENGINE_REF)->dmaengine); 620466b3540SConrad Meyer } 621466b3540SConrad Meyer 622466b3540SConrad Meyer void 623466b3540SConrad Meyer ioat_put_dmaengine(bus_dmaengine_t dmaengine) 624466b3540SConrad Meyer { 625466b3540SConrad Meyer struct ioat_softc *ioat; 626466b3540SConrad Meyer 627466b3540SConrad Meyer ioat = to_ioat_softc(dmaengine); 628466b3540SConrad Meyer ioat_put(ioat, IOAT_DMAENGINE_REF); 629e974f91cSConrad Meyer } 630e974f91cSConrad Meyer 631e974f91cSConrad Meyer void 632e974f91cSConrad Meyer ioat_acquire(bus_dmaengine_t dmaengine) 633e974f91cSConrad Meyer { 634e974f91cSConrad Meyer struct ioat_softc *ioat; 635e974f91cSConrad Meyer 636e974f91cSConrad Meyer ioat = to_ioat_softc(dmaengine); 637e974f91cSConrad Meyer mtx_lock(&ioat->submit_lock); 638e974f91cSConrad Meyer ioat_log_message(3, "%s\n", __func__); 639e974f91cSConrad Meyer } 640e974f91cSConrad Meyer 641e974f91cSConrad Meyer void 642e974f91cSConrad Meyer ioat_release(bus_dmaengine_t dmaengine) 643e974f91cSConrad Meyer { 644e974f91cSConrad Meyer struct ioat_softc *ioat; 645e974f91cSConrad Meyer 646e974f91cSConrad Meyer ioat = to_ioat_softc(dmaengine); 6471c25420eSConrad Meyer ioat_log_message(3, "%s\n", __func__); 648e974f91cSConrad Meyer ioat_write_2(ioat, IOAT_DMACOUNT_OFFSET, (uint16_t)ioat->head); 649e974f91cSConrad Meyer mtx_unlock(&ioat->submit_lock); 650e974f91cSConrad Meyer } 651e974f91cSConrad Meyer 652e974f91cSConrad Meyer struct bus_dmadesc * 653e974f91cSConrad Meyer ioat_null(bus_dmaengine_t dmaengine, bus_dmaengine_callback_t callback_fn, 654e974f91cSConrad Meyer void *callback_arg, uint32_t flags) 655e974f91cSConrad Meyer { 656e974f91cSConrad Meyer struct ioat_softc *ioat; 657e974f91cSConrad Meyer struct ioat_descriptor *desc; 658e974f91cSConrad Meyer struct ioat_dma_hw_descriptor *hw_desc; 659e974f91cSConrad Meyer 660e974f91cSConrad Meyer KASSERT((flags & ~DMA_ALL_FLAGS) == 0, ("Unrecognized flag(s): %#x", 661e974f91cSConrad Meyer flags & ~DMA_ALL_FLAGS)); 662e974f91cSConrad Meyer 663e974f91cSConrad Meyer ioat = to_ioat_softc(dmaengine); 66409f49f24SConrad Meyer mtx_assert(&ioat->submit_lock, MA_OWNED); 665e974f91cSConrad Meyer 666e974f91cSConrad Meyer if (ioat_reserve_space_and_lock(ioat, 1) != 0) 667e974f91cSConrad Meyer return (NULL); 668e974f91cSConrad Meyer 669e974f91cSConrad Meyer ioat_log_message(3, "%s\n", __func__); 670e974f91cSConrad Meyer 671e974f91cSConrad Meyer desc = ioat_get_ring_entry(ioat, ioat->head); 672e974f91cSConrad Meyer hw_desc = desc->u.dma; 673e974f91cSConrad Meyer 674e974f91cSConrad Meyer hw_desc->u.control_raw = 0; 675e974f91cSConrad Meyer hw_desc->u.control.null = 1; 676e974f91cSConrad Meyer hw_desc->u.control.completion_update = 1; 677e974f91cSConrad Meyer 678e974f91cSConrad Meyer if ((flags & DMA_INT_EN) != 0) 679e974f91cSConrad Meyer hw_desc->u.control.int_enable = 1; 680e974f91cSConrad Meyer 681e974f91cSConrad Meyer hw_desc->size = 8; 682e974f91cSConrad Meyer hw_desc->src_addr = 0; 683e974f91cSConrad Meyer hw_desc->dest_addr = 0; 684e974f91cSConrad Meyer 685e974f91cSConrad Meyer desc->bus_dmadesc.callback_fn = callback_fn; 686e974f91cSConrad Meyer desc->bus_dmadesc.callback_arg = callback_arg; 687e974f91cSConrad Meyer 688e974f91cSConrad Meyer ioat_submit_single(ioat); 689e974f91cSConrad Meyer return (&desc->bus_dmadesc); 690e974f91cSConrad Meyer } 691e974f91cSConrad Meyer 692e974f91cSConrad Meyer struct bus_dmadesc * 693e974f91cSConrad Meyer ioat_copy(bus_dmaengine_t dmaengine, bus_addr_t dst, 694e974f91cSConrad Meyer bus_addr_t src, bus_size_t len, bus_dmaengine_callback_t callback_fn, 695e974f91cSConrad Meyer void *callback_arg, uint32_t flags) 696e974f91cSConrad Meyer { 697e974f91cSConrad Meyer struct ioat_descriptor *desc; 698e974f91cSConrad Meyer struct ioat_dma_hw_descriptor *hw_desc; 699e974f91cSConrad Meyer struct ioat_softc *ioat; 700e974f91cSConrad Meyer 701e974f91cSConrad Meyer KASSERT((flags & ~DMA_ALL_FLAGS) == 0, ("Unrecognized flag(s): %#x", 702e974f91cSConrad Meyer flags & ~DMA_ALL_FLAGS)); 703e974f91cSConrad Meyer 704e974f91cSConrad Meyer ioat = to_ioat_softc(dmaengine); 70509f49f24SConrad Meyer mtx_assert(&ioat->submit_lock, MA_OWNED); 706e974f91cSConrad Meyer 707e974f91cSConrad Meyer if (len > ioat->max_xfer_size) { 708e974f91cSConrad Meyer ioat_log_message(0, "%s: max_xfer_size = %d, requested = %d\n", 709e974f91cSConrad Meyer __func__, ioat->max_xfer_size, (int)len); 710e974f91cSConrad Meyer return (NULL); 711e974f91cSConrad Meyer } 712e974f91cSConrad Meyer 713e974f91cSConrad Meyer if (ioat_reserve_space_and_lock(ioat, 1) != 0) 714e974f91cSConrad Meyer return (NULL); 715e974f91cSConrad Meyer 716e974f91cSConrad Meyer ioat_log_message(3, "%s\n", __func__); 717e974f91cSConrad Meyer 718e974f91cSConrad Meyer desc = ioat_get_ring_entry(ioat, ioat->head); 719e974f91cSConrad Meyer hw_desc = desc->u.dma; 720e974f91cSConrad Meyer 721e974f91cSConrad Meyer hw_desc->u.control_raw = 0; 722e974f91cSConrad Meyer hw_desc->u.control.completion_update = 1; 723e974f91cSConrad Meyer 724e974f91cSConrad Meyer if ((flags & DMA_INT_EN) != 0) 725e974f91cSConrad Meyer hw_desc->u.control.int_enable = 1; 726e974f91cSConrad Meyer 727e974f91cSConrad Meyer hw_desc->size = len; 728e974f91cSConrad Meyer hw_desc->src_addr = src; 729e974f91cSConrad Meyer hw_desc->dest_addr = dst; 730e974f91cSConrad Meyer 731e974f91cSConrad Meyer if (g_ioat_debug_level >= 3) 732e974f91cSConrad Meyer dump_descriptor(hw_desc); 733e974f91cSConrad Meyer 734e974f91cSConrad Meyer desc->bus_dmadesc.callback_fn = callback_fn; 735e974f91cSConrad Meyer desc->bus_dmadesc.callback_arg = callback_arg; 736e974f91cSConrad Meyer 737e974f91cSConrad Meyer ioat_submit_single(ioat); 738e974f91cSConrad Meyer return (&desc->bus_dmadesc); 739e974f91cSConrad Meyer } 740e974f91cSConrad Meyer 741e974f91cSConrad Meyer /* 742e974f91cSConrad Meyer * Ring Management 743e974f91cSConrad Meyer */ 744e974f91cSConrad Meyer static inline uint32_t 745e974f91cSConrad Meyer ioat_get_active(struct ioat_softc *ioat) 746e974f91cSConrad Meyer { 747e974f91cSConrad Meyer 748e974f91cSConrad Meyer return ((ioat->head - ioat->tail) & ((1 << ioat->ring_size_order) - 1)); 749e974f91cSConrad Meyer } 750e974f91cSConrad Meyer 751e974f91cSConrad Meyer static inline uint32_t 752e974f91cSConrad Meyer ioat_get_ring_space(struct ioat_softc *ioat) 753e974f91cSConrad Meyer { 754e974f91cSConrad Meyer 755e974f91cSConrad Meyer return ((1 << ioat->ring_size_order) - ioat_get_active(ioat) - 1); 756e974f91cSConrad Meyer } 757e974f91cSConrad Meyer 758e974f91cSConrad Meyer static struct ioat_descriptor * 759e974f91cSConrad Meyer ioat_alloc_ring_entry(struct ioat_softc *ioat) 760e974f91cSConrad Meyer { 761e974f91cSConrad Meyer struct ioat_dma_hw_descriptor *hw_desc; 762e974f91cSConrad Meyer struct ioat_descriptor *desc; 763f46011aeSConrad Meyer int error; 764e974f91cSConrad Meyer 765f46011aeSConrad Meyer error = ENOMEM; 766f46011aeSConrad Meyer hw_desc = NULL; 767f46011aeSConrad Meyer 768f46011aeSConrad Meyer desc = malloc(sizeof(*desc), M_IOAT, M_NOWAIT); 769e974f91cSConrad Meyer if (desc == NULL) 770f46011aeSConrad Meyer goto out; 771e974f91cSConrad Meyer 772f46011aeSConrad Meyer bus_dmamem_alloc(ioat->hw_desc_tag, (void **)&hw_desc, 773f46011aeSConrad Meyer BUS_DMA_ZERO | BUS_DMA_NOWAIT, &ioat->hw_desc_map); 774f46011aeSConrad Meyer if (hw_desc == NULL) 775f46011aeSConrad Meyer goto out; 776e974f91cSConrad Meyer 777e974f91cSConrad Meyer desc->u.dma = hw_desc; 778f46011aeSConrad Meyer 779f46011aeSConrad Meyer error = bus_dmamap_load(ioat->hw_desc_tag, ioat->hw_desc_map, hw_desc, 780f46011aeSConrad Meyer sizeof(*hw_desc), ioat_dmamap_cb, &desc->hw_desc_bus_addr, 781f46011aeSConrad Meyer BUS_DMA_NOWAIT); 782f46011aeSConrad Meyer if (error) 783f46011aeSConrad Meyer goto out; 784f46011aeSConrad Meyer 785f46011aeSConrad Meyer out: 786f46011aeSConrad Meyer if (error) { 787f46011aeSConrad Meyer ioat_free_ring_entry(ioat, desc); 788f46011aeSConrad Meyer return (NULL); 789f46011aeSConrad Meyer } 790e974f91cSConrad Meyer return (desc); 791e974f91cSConrad Meyer } 792e974f91cSConrad Meyer 793e974f91cSConrad Meyer static void 794e974f91cSConrad Meyer ioat_free_ring_entry(struct ioat_softc *ioat, struct ioat_descriptor *desc) 795e974f91cSConrad Meyer { 796e974f91cSConrad Meyer 797e974f91cSConrad Meyer if (desc == NULL) 798e974f91cSConrad Meyer return; 799e974f91cSConrad Meyer 800e974f91cSConrad Meyer if (desc->u.dma) 801e974f91cSConrad Meyer bus_dmamem_free(ioat->hw_desc_tag, desc->u.dma, 802e974f91cSConrad Meyer ioat->hw_desc_map); 803e974f91cSConrad Meyer free(desc, M_IOAT); 804e974f91cSConrad Meyer } 805e974f91cSConrad Meyer 806e974f91cSConrad Meyer static int 807e974f91cSConrad Meyer ioat_reserve_space_and_lock(struct ioat_softc *ioat, int num_descs) 808e974f91cSConrad Meyer { 809e974f91cSConrad Meyer boolean_t retry; 810e974f91cSConrad Meyer 811e974f91cSConrad Meyer while (1) { 812e974f91cSConrad Meyer if (ioat_get_ring_space(ioat) >= num_descs) 813e974f91cSConrad Meyer return (0); 814e974f91cSConrad Meyer 815e974f91cSConrad Meyer mtx_lock(&ioat->cleanup_lock); 816e974f91cSConrad Meyer retry = resize_ring(ioat, ioat->ring_size_order + 1); 817e974f91cSConrad Meyer mtx_unlock(&ioat->cleanup_lock); 818e974f91cSConrad Meyer 819e974f91cSConrad Meyer if (!retry) 820e974f91cSConrad Meyer return (ENOMEM); 821e974f91cSConrad Meyer } 822e974f91cSConrad Meyer } 823e974f91cSConrad Meyer 824e974f91cSConrad Meyer static struct ioat_descriptor * 825e974f91cSConrad Meyer ioat_get_ring_entry(struct ioat_softc *ioat, uint32_t index) 826e974f91cSConrad Meyer { 827e974f91cSConrad Meyer 828e974f91cSConrad Meyer return (ioat->ring[index % (1 << ioat->ring_size_order)]); 829e974f91cSConrad Meyer } 830e974f91cSConrad Meyer 831e974f91cSConrad Meyer static boolean_t 832e974f91cSConrad Meyer resize_ring(struct ioat_softc *ioat, int order) 833e974f91cSConrad Meyer { 834e974f91cSConrad Meyer struct ioat_descriptor **ring; 835e974f91cSConrad Meyer struct ioat_descriptor *next; 836e974f91cSConrad Meyer struct ioat_dma_hw_descriptor *hw; 837e974f91cSConrad Meyer struct ioat_descriptor *ent; 838e974f91cSConrad Meyer uint32_t current_size, active, new_size, i, new_idx, current_idx; 839e974f91cSConrad Meyer uint32_t new_idx2; 840e974f91cSConrad Meyer 841e974f91cSConrad Meyer current_size = 1 << ioat->ring_size_order; 842e974f91cSConrad Meyer active = (ioat->head - ioat->tail) & (current_size - 1); 843e974f91cSConrad Meyer new_size = 1 << order; 844e974f91cSConrad Meyer 845e974f91cSConrad Meyer if (order > IOAT_MAX_ORDER) 846e974f91cSConrad Meyer return (FALSE); 847e974f91cSConrad Meyer 848e974f91cSConrad Meyer /* 849e974f91cSConrad Meyer * when shrinking, verify that we can hold the current active 850e974f91cSConrad Meyer * set in the new ring 851e974f91cSConrad Meyer */ 852e974f91cSConrad Meyer if (active >= new_size) 853e974f91cSConrad Meyer return (FALSE); 854e974f91cSConrad Meyer 855e974f91cSConrad Meyer /* allocate the array to hold the software ring */ 856e974f91cSConrad Meyer ring = malloc(new_size * sizeof(*ring), M_IOAT, M_ZERO | M_NOWAIT); 857e974f91cSConrad Meyer if (ring == NULL) 858e974f91cSConrad Meyer return (FALSE); 859e974f91cSConrad Meyer 860e974f91cSConrad Meyer ioat_log_message(2, "ring resize: new: %d old: %d\n", 861e974f91cSConrad Meyer new_size, current_size); 862e974f91cSConrad Meyer 863e974f91cSConrad Meyer /* allocate/trim descriptors as needed */ 864e974f91cSConrad Meyer if (new_size > current_size) { 865e974f91cSConrad Meyer /* copy current descriptors to the new ring */ 866e974f91cSConrad Meyer for (i = 0; i < current_size; i++) { 867e974f91cSConrad Meyer current_idx = (ioat->tail + i) & (current_size - 1); 868e974f91cSConrad Meyer new_idx = (ioat->tail + i) & (new_size - 1); 869e974f91cSConrad Meyer 870e974f91cSConrad Meyer ring[new_idx] = ioat->ring[current_idx]; 871e974f91cSConrad Meyer ring[new_idx]->id = new_idx; 872e974f91cSConrad Meyer } 873e974f91cSConrad Meyer 874e974f91cSConrad Meyer /* add new descriptors to the ring */ 875e974f91cSConrad Meyer for (i = current_size; i < new_size; i++) { 876e974f91cSConrad Meyer new_idx = (ioat->tail + i) & (new_size - 1); 877e974f91cSConrad Meyer 878e974f91cSConrad Meyer ring[new_idx] = ioat_alloc_ring_entry(ioat); 8798c8e8487SConrad Meyer if (ring[new_idx] == NULL) { 880e974f91cSConrad Meyer while (i--) { 881e974f91cSConrad Meyer new_idx2 = (ioat->tail + i) & 882e974f91cSConrad Meyer (new_size - 1); 883e974f91cSConrad Meyer 884e974f91cSConrad Meyer ioat_free_ring_entry(ioat, 885e974f91cSConrad Meyer ring[new_idx2]); 886e974f91cSConrad Meyer } 887e974f91cSConrad Meyer free(ring, M_IOAT); 888e974f91cSConrad Meyer return (FALSE); 889e974f91cSConrad Meyer } 890e974f91cSConrad Meyer ring[new_idx]->id = new_idx; 891e974f91cSConrad Meyer } 892e974f91cSConrad Meyer 893e974f91cSConrad Meyer for (i = current_size - 1; i < new_size; i++) { 894e974f91cSConrad Meyer new_idx = (ioat->tail + i) & (new_size - 1); 895e974f91cSConrad Meyer next = ring[(new_idx + 1) & (new_size - 1)]; 896e974f91cSConrad Meyer hw = ring[new_idx]->u.dma; 897e974f91cSConrad Meyer 898e974f91cSConrad Meyer hw->next = next->hw_desc_bus_addr; 899e974f91cSConrad Meyer } 900e974f91cSConrad Meyer } else { 901e974f91cSConrad Meyer /* 902e974f91cSConrad Meyer * copy current descriptors to the new ring, dropping the 903e974f91cSConrad Meyer * removed descriptors 904e974f91cSConrad Meyer */ 905e974f91cSConrad Meyer for (i = 0; i < new_size; i++) { 906e974f91cSConrad Meyer current_idx = (ioat->tail + i) & (current_size - 1); 907e974f91cSConrad Meyer new_idx = (ioat->tail + i) & (new_size - 1); 908e974f91cSConrad Meyer 909e974f91cSConrad Meyer ring[new_idx] = ioat->ring[current_idx]; 910e974f91cSConrad Meyer ring[new_idx]->id = new_idx; 911e974f91cSConrad Meyer } 912e974f91cSConrad Meyer 913e974f91cSConrad Meyer /* free deleted descriptors */ 914e974f91cSConrad Meyer for (i = new_size; i < current_size; i++) { 915e974f91cSConrad Meyer ent = ioat_get_ring_entry(ioat, ioat->tail + i); 916e974f91cSConrad Meyer ioat_free_ring_entry(ioat, ent); 917e974f91cSConrad Meyer } 918e974f91cSConrad Meyer 919e974f91cSConrad Meyer /* fix up hardware ring */ 920e974f91cSConrad Meyer hw = ring[(ioat->tail + new_size - 1) & (new_size - 1)]->u.dma; 921e974f91cSConrad Meyer next = ring[(ioat->tail + new_size) & (new_size - 1)]; 922e974f91cSConrad Meyer hw->next = next->hw_desc_bus_addr; 923e974f91cSConrad Meyer } 924e974f91cSConrad Meyer 925e974f91cSConrad Meyer free(ioat->ring, M_IOAT); 926e974f91cSConrad Meyer ioat->ring = ring; 927e974f91cSConrad Meyer ioat->ring_size_order = order; 928e974f91cSConrad Meyer 929e974f91cSConrad Meyer return (TRUE); 930e974f91cSConrad Meyer } 931e974f91cSConrad Meyer 932e974f91cSConrad Meyer static void 9338f274637SConrad Meyer ioat_halted_debug(struct ioat_softc *ioat, uint32_t chanerr) 934e974f91cSConrad Meyer { 935e974f91cSConrad Meyer struct ioat_descriptor *desc; 9368f274637SConrad Meyer 9378f274637SConrad Meyer ioat_log_message(0, "Channel halted (%x)\n", chanerr); 9388f274637SConrad Meyer if (chanerr == 0) 9398f274637SConrad Meyer return; 9408f274637SConrad Meyer 9418f274637SConrad Meyer desc = ioat_get_ring_entry(ioat, ioat->tail + 0); 9428f274637SConrad Meyer dump_descriptor(desc->u.raw); 9438f274637SConrad Meyer 9448f274637SConrad Meyer desc = ioat_get_ring_entry(ioat, ioat->tail + 1); 9458f274637SConrad Meyer dump_descriptor(desc->u.raw); 9468f274637SConrad Meyer } 9478f274637SConrad Meyer 9488f274637SConrad Meyer static void 9498f274637SConrad Meyer ioat_timer_callback(void *arg) 9508f274637SConrad Meyer { 951e974f91cSConrad Meyer struct ioat_softc *ioat; 952e974f91cSConrad Meyer uint64_t status; 953e974f91cSConrad Meyer uint32_t chanerr; 954e974f91cSConrad Meyer 955e974f91cSConrad Meyer ioat = arg; 956fe720f5aSConrad Meyer ioat_log_message(1, "%s\n", __func__); 957e974f91cSConrad Meyer 958e974f91cSConrad Meyer if (ioat->is_completion_pending) { 959e974f91cSConrad Meyer status = ioat_get_chansts(ioat); 960e974f91cSConrad Meyer 961e974f91cSConrad Meyer /* 962e974f91cSConrad Meyer * When halted due to errors, check for channel programming 963e974f91cSConrad Meyer * errors before advancing the completion state. 964e974f91cSConrad Meyer */ 965e974f91cSConrad Meyer if (is_ioat_halted(status)) { 966e974f91cSConrad Meyer chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET); 9678f274637SConrad Meyer ioat_halted_debug(ioat, chanerr); 968e974f91cSConrad Meyer } 969e974f91cSConrad Meyer ioat_process_events(ioat); 970e974f91cSConrad Meyer } else { 971e974f91cSConrad Meyer mtx_lock(&ioat->submit_lock); 972e974f91cSConrad Meyer mtx_lock(&ioat->cleanup_lock); 973e974f91cSConrad Meyer 974e974f91cSConrad Meyer if (ioat_get_active(ioat) == 0 && 975e974f91cSConrad Meyer ioat->ring_size_order > IOAT_MIN_ORDER) 976e974f91cSConrad Meyer resize_ring(ioat, ioat->ring_size_order - 1); 977e974f91cSConrad Meyer 978e974f91cSConrad Meyer mtx_unlock(&ioat->cleanup_lock); 979e974f91cSConrad Meyer mtx_unlock(&ioat->submit_lock); 980e974f91cSConrad Meyer 981e974f91cSConrad Meyer if (ioat->ring_size_order > IOAT_MIN_ORDER) 982fe720f5aSConrad Meyer callout_reset(&ioat->timer, IOAT_INTR_TIMO, 983e974f91cSConrad Meyer ioat_timer_callback, ioat); 984e974f91cSConrad Meyer } 985e974f91cSConrad Meyer } 986e974f91cSConrad Meyer 987e974f91cSConrad Meyer /* 988e974f91cSConrad Meyer * Support Functions 989e974f91cSConrad Meyer */ 990e974f91cSConrad Meyer static void 991e974f91cSConrad Meyer ioat_submit_single(struct ioat_softc *ioat) 992e974f91cSConrad Meyer { 993e974f91cSConrad Meyer 994466b3540SConrad Meyer ioat_get(ioat, IOAT_ACTIVE_DESCR_REF); 995e974f91cSConrad Meyer atomic_add_rel_int(&ioat->head, 1); 996e974f91cSConrad Meyer 997e974f91cSConrad Meyer if (!ioat->is_completion_pending) { 998e974f91cSConrad Meyer ioat->is_completion_pending = TRUE; 999fe720f5aSConrad Meyer callout_reset(&ioat->timer, IOAT_INTR_TIMO, 1000fe720f5aSConrad Meyer ioat_timer_callback, ioat); 1001e974f91cSConrad Meyer } 1002e974f91cSConrad Meyer } 1003e974f91cSConrad Meyer 1004e974f91cSConrad Meyer static int 1005e974f91cSConrad Meyer ioat_reset_hw(struct ioat_softc *ioat) 1006e974f91cSConrad Meyer { 1007e974f91cSConrad Meyer uint64_t status; 1008e974f91cSConrad Meyer uint32_t chanerr; 10090d1a05d9SConrad Meyer int timeout; 1010e974f91cSConrad Meyer 1011e974f91cSConrad Meyer status = ioat_get_chansts(ioat); 1012e974f91cSConrad Meyer if (is_ioat_active(status) || is_ioat_idle(status)) 1013e974f91cSConrad Meyer ioat_suspend(ioat); 1014e974f91cSConrad Meyer 1015e974f91cSConrad Meyer /* Wait at most 20 ms */ 1016e974f91cSConrad Meyer for (timeout = 0; (is_ioat_active(status) || is_ioat_idle(status)) && 1017e974f91cSConrad Meyer timeout < 20; timeout++) { 1018e974f91cSConrad Meyer DELAY(1000); 1019e974f91cSConrad Meyer status = ioat_get_chansts(ioat); 1020e974f91cSConrad Meyer } 1021e974f91cSConrad Meyer if (timeout == 20) 1022e974f91cSConrad Meyer return (ETIMEDOUT); 1023e974f91cSConrad Meyer 1024e974f91cSConrad Meyer chanerr = ioat_read_4(ioat, IOAT_CHANERR_OFFSET); 1025e974f91cSConrad Meyer ioat_write_4(ioat, IOAT_CHANERR_OFFSET, chanerr); 1026e974f91cSConrad Meyer 1027e974f91cSConrad Meyer /* 1028e974f91cSConrad Meyer * IOAT v3 workaround - CHANERRMSK_INT with 3E07h to masks out errors 1029e974f91cSConrad Meyer * that can cause stability issues for IOAT v3. 1030e974f91cSConrad Meyer */ 1031e974f91cSConrad Meyer pci_write_config(ioat->device, IOAT_CFG_CHANERRMASK_INT_OFFSET, 0x3e07, 1032e974f91cSConrad Meyer 4); 1033e974f91cSConrad Meyer chanerr = pci_read_config(ioat->device, IOAT_CFG_CHANERR_INT_OFFSET, 4); 1034e974f91cSConrad Meyer pci_write_config(ioat->device, IOAT_CFG_CHANERR_INT_OFFSET, chanerr, 4); 1035e974f91cSConrad Meyer 10360d1a05d9SConrad Meyer /* 10370d1a05d9SConrad Meyer * BDXDE and BWD models reset MSI-X registers on device reset. 10380d1a05d9SConrad Meyer * Save/restore their contents manually. 10390d1a05d9SConrad Meyer */ 1040f7157235SConrad Meyer if (ioat_model_resets_msix(ioat)) { 1041f7157235SConrad Meyer ioat_log_message(1, "device resets MSI-X registers; saving\n"); 10420d1a05d9SConrad Meyer pci_save_state(ioat->device); 1043f7157235SConrad Meyer } 10440d1a05d9SConrad Meyer 1045e974f91cSConrad Meyer ioat_reset(ioat); 1046e974f91cSConrad Meyer 1047e974f91cSConrad Meyer /* Wait at most 20 ms */ 1048e974f91cSConrad Meyer for (timeout = 0; ioat_reset_pending(ioat) && timeout < 20; timeout++) 1049e974f91cSConrad Meyer DELAY(1000); 1050e974f91cSConrad Meyer if (timeout == 20) 1051e974f91cSConrad Meyer return (ETIMEDOUT); 1052e974f91cSConrad Meyer 1053f7157235SConrad Meyer if (ioat_model_resets_msix(ioat)) { 1054f7157235SConrad Meyer ioat_log_message(1, "device resets registers; restored\n"); 10550d1a05d9SConrad Meyer pci_restore_state(ioat->device); 1056f7157235SConrad Meyer } 10574253ea50SConrad Meyer 1058e974f91cSConrad Meyer return (0); 1059e974f91cSConrad Meyer } 1060e974f91cSConrad Meyer 1061f7157235SConrad Meyer static int 1062f7157235SConrad Meyer sysctl_handle_reset(SYSCTL_HANDLER_ARGS) 1063f7157235SConrad Meyer { 1064f7157235SConrad Meyer struct ioat_softc *ioat; 1065f7157235SConrad Meyer int error, arg; 1066f7157235SConrad Meyer 1067f7157235SConrad Meyer ioat = arg1; 1068f7157235SConrad Meyer 1069f7157235SConrad Meyer arg = 0; 1070f7157235SConrad Meyer error = SYSCTL_OUT(req, &arg, sizeof(arg)); 1071f7157235SConrad Meyer if (error != 0 || req->newptr == NULL) 1072f7157235SConrad Meyer return (error); 1073f7157235SConrad Meyer 1074f7157235SConrad Meyer error = SYSCTL_IN(req, &arg, sizeof(arg)); 1075f7157235SConrad Meyer if (error != 0) 1076f7157235SConrad Meyer return (error); 1077f7157235SConrad Meyer 1078f7157235SConrad Meyer if (arg != 0) 1079f7157235SConrad Meyer error = ioat_reset_hw(ioat); 1080f7157235SConrad Meyer 1081f7157235SConrad Meyer return (error); 1082f7157235SConrad Meyer } 1083f7157235SConrad Meyer 1084e974f91cSConrad Meyer static void 1085e974f91cSConrad Meyer dump_descriptor(void *hw_desc) 1086e974f91cSConrad Meyer { 1087e974f91cSConrad Meyer int i, j; 1088e974f91cSConrad Meyer 1089e974f91cSConrad Meyer for (i = 0; i < 2; i++) { 1090e974f91cSConrad Meyer for (j = 0; j < 8; j++) 1091e974f91cSConrad Meyer printf("%08x ", ((uint32_t *)hw_desc)[i * 8 + j]); 1092e974f91cSConrad Meyer printf("\n"); 1093e974f91cSConrad Meyer } 1094e974f91cSConrad Meyer } 1095e974f91cSConrad Meyer 1096e974f91cSConrad Meyer static void 1097e974f91cSConrad Meyer ioat_setup_sysctl(device_t device) 1098e974f91cSConrad Meyer { 1099f7157235SConrad Meyer struct sysctl_oid_list *par; 1100f7157235SConrad Meyer struct sysctl_ctx_list *ctx; 1101f7157235SConrad Meyer struct sysctl_oid *tree; 1102e974f91cSConrad Meyer struct ioat_softc *ioat; 1103e974f91cSConrad Meyer 1104e974f91cSConrad Meyer ioat = DEVICE2SOFTC(device); 1105f7157235SConrad Meyer ctx = device_get_sysctl_ctx(device); 1106f7157235SConrad Meyer tree = device_get_sysctl_tree(device); 1107f7157235SConrad Meyer par = SYSCTL_CHILDREN(tree); 1108e974f91cSConrad Meyer 1109f7157235SConrad Meyer SYSCTL_ADD_UINT(ctx, par, OID_AUTO, "ring_size_order", CTLFLAG_RD, 1110f7157235SConrad Meyer &ioat->ring_size_order, 0, "HW descriptor ring size order"); 1111f7157235SConrad Meyer SYSCTL_ADD_UINT(ctx, par, OID_AUTO, "head", CTLFLAG_RD, &ioat->head, 0, 1112f7157235SConrad Meyer "HW descriptor head pointer index"); 1113f7157235SConrad Meyer SYSCTL_ADD_UINT(ctx, par, OID_AUTO, "tail", CTLFLAG_RD, &ioat->tail, 0, 1114f7157235SConrad Meyer "HW descriptor tail pointer index"); 1115f7157235SConrad Meyer 1116f7157235SConrad Meyer SYSCTL_ADD_PROC(ctx, par, OID_AUTO, "force_hw_reset", 1117f7157235SConrad Meyer CTLTYPE_INT | CTLFLAG_RW, ioat, 0, sysctl_handle_reset, "I", 1118f7157235SConrad Meyer "Set to non-zero to reset the hardware"); 1119e974f91cSConrad Meyer } 1120466b3540SConrad Meyer 1121466b3540SConrad Meyer static inline struct ioat_softc * 1122466b3540SConrad Meyer ioat_get(struct ioat_softc *ioat, enum ioat_ref_kind kind) 1123466b3540SConrad Meyer { 1124466b3540SConrad Meyer uint32_t old; 1125466b3540SConrad Meyer 1126466b3540SConrad Meyer KASSERT(kind < IOAT_NUM_REF_KINDS, ("bogus")); 1127466b3540SConrad Meyer 1128466b3540SConrad Meyer old = atomic_fetchadd_32(&ioat->refcnt, 1); 1129466b3540SConrad Meyer KASSERT(old < UINT32_MAX, ("refcnt overflow")); 1130466b3540SConrad Meyer 1131466b3540SConrad Meyer #ifdef INVARIANTS 1132466b3540SConrad Meyer old = atomic_fetchadd_32(&ioat->refkinds[kind], 1); 1133466b3540SConrad Meyer KASSERT(old < UINT32_MAX, ("refcnt kind overflow")); 1134466b3540SConrad Meyer #endif 1135466b3540SConrad Meyer 1136466b3540SConrad Meyer return (ioat); 1137466b3540SConrad Meyer } 1138466b3540SConrad Meyer 1139466b3540SConrad Meyer static inline void 1140466b3540SConrad Meyer ioat_putn(struct ioat_softc *ioat, uint32_t n, enum ioat_ref_kind kind) 1141466b3540SConrad Meyer { 1142466b3540SConrad Meyer uint32_t old; 1143466b3540SConrad Meyer 1144466b3540SConrad Meyer KASSERT(kind < IOAT_NUM_REF_KINDS, ("bogus")); 1145466b3540SConrad Meyer 1146466b3540SConrad Meyer if (n == 0) 1147466b3540SConrad Meyer return; 1148466b3540SConrad Meyer 1149466b3540SConrad Meyer #ifdef INVARIANTS 1150466b3540SConrad Meyer old = atomic_fetchadd_32(&ioat->refkinds[kind], -n); 1151466b3540SConrad Meyer KASSERT(old >= n, ("refcnt kind underflow")); 1152466b3540SConrad Meyer #endif 1153466b3540SConrad Meyer 1154466b3540SConrad Meyer /* Skip acquiring the lock if resulting refcnt > 0. */ 1155466b3540SConrad Meyer for (;;) { 1156466b3540SConrad Meyer old = ioat->refcnt; 1157466b3540SConrad Meyer if (old <= n) 1158466b3540SConrad Meyer break; 1159466b3540SConrad Meyer if (atomic_cmpset_32(&ioat->refcnt, old, old - n)) 1160466b3540SConrad Meyer return; 1161466b3540SConrad Meyer } 1162466b3540SConrad Meyer 1163466b3540SConrad Meyer mtx_lock(IOAT_REFLK); 1164466b3540SConrad Meyer old = atomic_fetchadd_32(&ioat->refcnt, -n); 1165466b3540SConrad Meyer KASSERT(old >= n, ("refcnt error")); 1166466b3540SConrad Meyer 1167466b3540SConrad Meyer if (old == n) 1168466b3540SConrad Meyer wakeup(IOAT_REFLK); 1169466b3540SConrad Meyer mtx_unlock(IOAT_REFLK); 1170466b3540SConrad Meyer } 1171466b3540SConrad Meyer 1172466b3540SConrad Meyer static inline void 1173466b3540SConrad Meyer ioat_put(struct ioat_softc *ioat, enum ioat_ref_kind kind) 1174466b3540SConrad Meyer { 1175466b3540SConrad Meyer 1176466b3540SConrad Meyer ioat_putn(ioat, 1, kind); 1177466b3540SConrad Meyer } 1178466b3540SConrad Meyer 1179466b3540SConrad Meyer static void 1180466b3540SConrad Meyer ioat_drain(struct ioat_softc *ioat) 1181466b3540SConrad Meyer { 1182466b3540SConrad Meyer 1183466b3540SConrad Meyer mtx_lock(IOAT_REFLK); 1184466b3540SConrad Meyer while (ioat->refcnt > 0) 1185466b3540SConrad Meyer msleep(IOAT_REFLK, IOAT_REFLK, 0, "ioat_drain", 0); 1186466b3540SConrad Meyer mtx_unlock(IOAT_REFLK); 1187466b3540SConrad Meyer } 1188