1 /*- 2 * Copyright (C) 2012 Intel Corporation 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 __FBSDID("$FreeBSD$"); 28 29 #ifndef __IOAT_INTERNAL_H__ 30 #define __IOAT_INTERNAL_H__ 31 32 #define DEVICE2SOFTC(dev) ((struct ioat_softc *) device_get_softc(dev)) 33 #define KTR_IOAT KTR_SPARE3 34 35 #define ioat_read_chancnt(ioat) \ 36 ioat_read_1((ioat), IOAT_CHANCNT_OFFSET) 37 38 #define ioat_read_xfercap(ioat) \ 39 (ioat_read_1((ioat), IOAT_XFERCAP_OFFSET) & IOAT_XFERCAP_VALID_MASK) 40 41 #define ioat_write_intrctrl(ioat, value) \ 42 ioat_write_1((ioat), IOAT_INTRCTRL_OFFSET, (value)) 43 44 #define ioat_read_cbver(ioat) \ 45 (ioat_read_1((ioat), IOAT_CBVER_OFFSET) & 0xFF) 46 47 #define ioat_read_dmacapability(ioat) \ 48 ioat_read_4((ioat), IOAT_DMACAPABILITY_OFFSET) 49 50 #define ioat_write_chanctrl(ioat, value) \ 51 ioat_write_2((ioat), IOAT_CHANCTRL_OFFSET, (value)) 52 53 static __inline uint64_t 54 ioat_bus_space_read_8_lower_first(bus_space_tag_t tag, 55 bus_space_handle_t handle, bus_size_t offset) 56 { 57 return (bus_space_read_4(tag, handle, offset) | 58 ((uint64_t)bus_space_read_4(tag, handle, offset + 4)) << 32); 59 } 60 61 static __inline void 62 ioat_bus_space_write_8_lower_first(bus_space_tag_t tag, 63 bus_space_handle_t handle, bus_size_t offset, uint64_t val) 64 { 65 bus_space_write_4(tag, handle, offset, val); 66 bus_space_write_4(tag, handle, offset + 4, val >> 32); 67 } 68 69 #ifdef __i386__ 70 #define ioat_bus_space_read_8 ioat_bus_space_read_8_lower_first 71 #define ioat_bus_space_write_8 ioat_bus_space_write_8_lower_first 72 #else 73 #define ioat_bus_space_read_8(tag, handle, offset) \ 74 bus_space_read_8((tag), (handle), (offset)) 75 #define ioat_bus_space_write_8(tag, handle, offset, val) \ 76 bus_space_write_8((tag), (handle), (offset), (val)) 77 #endif 78 79 #define ioat_read_1(ioat, offset) \ 80 bus_space_read_1((ioat)->pci_bus_tag, (ioat)->pci_bus_handle, \ 81 (offset)) 82 83 #define ioat_read_2(ioat, offset) \ 84 bus_space_read_2((ioat)->pci_bus_tag, (ioat)->pci_bus_handle, \ 85 (offset)) 86 87 #define ioat_read_4(ioat, offset) \ 88 bus_space_read_4((ioat)->pci_bus_tag, (ioat)->pci_bus_handle, \ 89 (offset)) 90 91 #define ioat_read_8(ioat, offset) \ 92 ioat_bus_space_read_8((ioat)->pci_bus_tag, (ioat)->pci_bus_handle, \ 93 (offset)) 94 95 #define ioat_read_double_4(ioat, offset) \ 96 ioat_bus_space_read_8_lower_first((ioat)->pci_bus_tag, \ 97 (ioat)->pci_bus_handle, (offset)) 98 99 #define ioat_write_1(ioat, offset, value) \ 100 bus_space_write_1((ioat)->pci_bus_tag, (ioat)->pci_bus_handle, \ 101 (offset), (value)) 102 103 #define ioat_write_2(ioat, offset, value) \ 104 bus_space_write_2((ioat)->pci_bus_tag, (ioat)->pci_bus_handle, \ 105 (offset), (value)) 106 107 #define ioat_write_4(ioat, offset, value) \ 108 bus_space_write_4((ioat)->pci_bus_tag, (ioat)->pci_bus_handle, \ 109 (offset), (value)) 110 111 #define ioat_write_8(ioat, offset, value) \ 112 ioat_bus_space_write_8((ioat)->pci_bus_tag, (ioat)->pci_bus_handle, \ 113 (offset), (value)) 114 115 #define ioat_write_double_4(ioat, offset, value) \ 116 ioat_bus_space_write_8_lower_first((ioat)->pci_bus_tag, \ 117 (ioat)->pci_bus_handle, (offset), (value)) 118 119 MALLOC_DECLARE(M_IOAT); 120 121 SYSCTL_DECL(_hw_ioat); 122 123 extern int g_ioat_debug_level; 124 125 struct generic_dma_control { 126 uint32_t int_enable:1; 127 uint32_t src_snoop_disable:1; 128 uint32_t dest_snoop_disable:1; 129 uint32_t completion_update:1; 130 uint32_t fence:1; 131 uint32_t reserved1:1; 132 uint32_t src_page_break:1; 133 uint32_t dest_page_break:1; 134 uint32_t bundle:1; 135 uint32_t dest_dca:1; 136 uint32_t hint:1; 137 uint32_t reserved2:13; 138 uint32_t op:8; 139 }; 140 141 struct ioat_generic_hw_descriptor { 142 uint32_t size; 143 union { 144 uint32_t control_raw; 145 struct generic_dma_control control_generic; 146 } u; 147 uint64_t src_addr; 148 uint64_t dest_addr; 149 uint64_t next; 150 uint64_t reserved[4]; 151 }; 152 153 struct ioat_dma_hw_descriptor { 154 uint32_t size; 155 union { 156 uint32_t control_raw; 157 struct generic_dma_control control_generic; 158 struct { 159 uint32_t int_enable:1; 160 uint32_t src_snoop_disable:1; 161 uint32_t dest_snoop_disable:1; 162 uint32_t completion_update:1; 163 uint32_t fence:1; 164 uint32_t null:1; 165 uint32_t src_page_break:1; 166 uint32_t dest_page_break:1; 167 uint32_t bundle:1; 168 uint32_t dest_dca:1; 169 uint32_t hint:1; 170 uint32_t reserved:13; 171 #define IOAT_OP_COPY 0x00 172 uint32_t op:8; 173 } control; 174 } u; 175 uint64_t src_addr; 176 uint64_t dest_addr; 177 uint64_t next; 178 uint64_t next_src_addr; 179 uint64_t next_dest_addr; 180 uint64_t user1; 181 uint64_t user2; 182 }; 183 184 struct ioat_fill_hw_descriptor { 185 uint32_t size; 186 union { 187 uint32_t control_raw; 188 struct generic_dma_control control_generic; 189 struct { 190 uint32_t int_enable:1; 191 uint32_t reserved:1; 192 uint32_t dest_snoop_disable:1; 193 uint32_t completion_update:1; 194 uint32_t fence:1; 195 uint32_t reserved2:2; 196 uint32_t dest_page_break:1; 197 uint32_t bundle:1; 198 uint32_t reserved3:15; 199 #define IOAT_OP_FILL 0x01 200 uint32_t op:8; 201 } control; 202 } u; 203 uint64_t src_data; 204 uint64_t dest_addr; 205 uint64_t next; 206 uint64_t reserved; 207 uint64_t next_dest_addr; 208 uint64_t user1; 209 uint64_t user2; 210 }; 211 212 struct ioat_xor_hw_descriptor { 213 uint32_t size; 214 union { 215 uint32_t control_raw; 216 struct generic_dma_control control_generic; 217 struct { 218 uint32_t int_enable:1; 219 uint32_t src_snoop_disable:1; 220 uint32_t dest_snoop_disable:1; 221 uint32_t completion_update:1; 222 uint32_t fence:1; 223 uint32_t src_count:3; 224 uint32_t bundle:1; 225 uint32_t dest_dca:1; 226 uint32_t hint:1; 227 uint32_t reserved:13; 228 #define IOAT_OP_XOR 0x87 229 #define IOAT_OP_XOR_VAL 0x88 230 uint32_t op:8; 231 } control; 232 } u; 233 uint64_t src_addr; 234 uint64_t dest_addr; 235 uint64_t next; 236 uint64_t src_addr2; 237 uint64_t src_addr3; 238 uint64_t src_addr4; 239 uint64_t src_addr5; 240 }; 241 242 struct ioat_xor_ext_hw_descriptor { 243 uint64_t src_addr6; 244 uint64_t src_addr7; 245 uint64_t src_addr8; 246 uint64_t next; 247 uint64_t reserved[4]; 248 }; 249 250 struct ioat_pq_hw_descriptor { 251 uint32_t size; 252 union { 253 uint32_t control_raw; 254 struct generic_dma_control control_generic; 255 struct { 256 uint32_t int_enable:1; 257 uint32_t src_snoop_disable:1; 258 uint32_t dest_snoop_disable:1; 259 uint32_t completion_update:1; 260 uint32_t fence:1; 261 uint32_t src_count:3; 262 uint32_t bundle:1; 263 uint32_t dest_dca:1; 264 uint32_t hint:1; 265 uint32_t p_disable:1; 266 uint32_t q_disable:1; 267 uint32_t reserved:11; 268 #define IOAT_OP_PQ 0x89 269 #define IOAT_OP_PQ_VAL 0x8a 270 uint32_t op:8; 271 } control; 272 } u; 273 uint64_t src_addr; 274 uint64_t p_addr; 275 uint64_t next; 276 uint64_t src_addr2; 277 uint64_t src_addr3; 278 uint8_t coef[8]; 279 uint64_t q_addr; 280 }; 281 282 struct ioat_pq_ext_hw_descriptor { 283 uint64_t src_addr4; 284 uint64_t src_addr5; 285 uint64_t src_addr6; 286 uint64_t next; 287 uint64_t src_addr7; 288 uint64_t src_addr8; 289 uint64_t reserved[2]; 290 }; 291 292 struct ioat_pq_update_hw_descriptor { 293 uint32_t size; 294 union { 295 uint32_t control_raw; 296 struct generic_dma_control control_generic; 297 struct { 298 uint32_t int_enable:1; 299 uint32_t src_snoop_disable:1; 300 uint32_t dest_snoop_disable:1; 301 uint32_t completion_update:1; 302 uint32_t fence:1; 303 uint32_t src_cnt:3; 304 uint32_t bundle:1; 305 uint32_t dest_dca:1; 306 uint32_t hint:1; 307 uint32_t p_disable:1; 308 uint32_t q_disable:1; 309 uint32_t reserved:3; 310 uint32_t coef:8; 311 #define IOAT_OP_PQ_UP 0x8b 312 uint32_t op:8; 313 } control; 314 } u; 315 uint64_t src_addr; 316 uint64_t p_addr; 317 uint64_t next; 318 uint64_t src_addr2; 319 uint64_t p_src; 320 uint64_t q_src; 321 uint64_t q_addr; 322 }; 323 324 struct ioat_raw_hw_descriptor { 325 uint64_t field[8]; 326 }; 327 328 struct bus_dmadesc { 329 bus_dmaengine_callback_t callback_fn; 330 void *callback_arg; 331 }; 332 333 struct ioat_descriptor { 334 struct bus_dmadesc bus_dmadesc; 335 union { 336 struct ioat_generic_hw_descriptor *generic; 337 struct ioat_dma_hw_descriptor *dma; 338 struct ioat_fill_hw_descriptor *fill; 339 struct ioat_xor_hw_descriptor *xor; 340 struct ioat_xor_ext_hw_descriptor *xor_ext; 341 struct ioat_pq_hw_descriptor *pq; 342 struct ioat_pq_ext_hw_descriptor *pq_ext; 343 struct ioat_raw_hw_descriptor *raw; 344 } u; 345 uint32_t id; 346 bus_addr_t hw_desc_bus_addr; 347 }; 348 349 enum ioat_ref_kind { 350 IOAT_DMAENGINE_REF = 0, 351 IOAT_ACTIVE_DESCR_REF, 352 IOAT_NUM_REF_KINDS 353 }; 354 355 /* One of these per allocated PCI device. */ 356 struct ioat_softc { 357 bus_dmaengine_t dmaengine; 358 #define to_ioat_softc(_dmaeng) \ 359 ({ \ 360 bus_dmaengine_t *_p = (_dmaeng); \ 361 (struct ioat_softc *)((char *)_p - \ 362 offsetof(struct ioat_softc, dmaengine)); \ 363 }) 364 365 int version; 366 int chan_idx; 367 368 struct mtx submit_lock; 369 device_t device; 370 bus_space_tag_t pci_bus_tag; 371 bus_space_handle_t pci_bus_handle; 372 int pci_resource_id; 373 struct resource *pci_resource; 374 uint32_t max_xfer_size; 375 uint32_t capabilities; 376 uint16_t intrdelay_max; 377 uint16_t cached_intrdelay; 378 379 struct resource *res; 380 int rid; 381 void *tag; 382 383 bus_dma_tag_t hw_desc_tag; 384 bus_dmamap_t hw_desc_map; 385 386 bus_dma_tag_t comp_update_tag; 387 bus_dmamap_t comp_update_map; 388 uint64_t *comp_update; 389 bus_addr_t comp_update_bus_addr; 390 391 struct callout timer; 392 393 boolean_t quiescing; 394 boolean_t is_resize_pending; 395 boolean_t is_completion_pending; 396 boolean_t is_reset_pending; 397 boolean_t is_channel_running; 398 boolean_t intrdelay_supported; 399 400 uint32_t head; 401 uint32_t tail; 402 uint32_t hw_head; 403 uint32_t ring_size_order; 404 bus_addr_t last_seen; 405 406 struct ioat_descriptor **ring; 407 408 struct mtx cleanup_lock; 409 volatile uint32_t refcnt; 410 #ifdef INVARIANTS 411 volatile uint32_t refkinds[IOAT_NUM_REF_KINDS]; 412 #endif 413 414 struct { 415 uint64_t interrupts; 416 uint64_t descriptors_processed; 417 uint64_t descriptors_error; 418 uint64_t descriptors_submitted; 419 420 uint32_t channel_halts; 421 uint32_t last_halt_chanerr; 422 } stats; 423 }; 424 425 void ioat_test_attach(void); 426 void ioat_test_detach(void); 427 428 static inline uint64_t 429 ioat_get_chansts(struct ioat_softc *ioat) 430 { 431 uint64_t status; 432 433 if (ioat->version >= IOAT_VER_3_3) 434 status = ioat_read_8(ioat, IOAT_CHANSTS_OFFSET); 435 else 436 /* Must read lower 4 bytes before upper 4 bytes. */ 437 status = ioat_read_double_4(ioat, IOAT_CHANSTS_OFFSET); 438 return (status); 439 } 440 441 static inline void 442 ioat_write_chancmp(struct ioat_softc *ioat, uint64_t addr) 443 { 444 445 if (ioat->version >= IOAT_VER_3_3) 446 ioat_write_8(ioat, IOAT_CHANCMP_OFFSET_LOW, addr); 447 else 448 ioat_write_double_4(ioat, IOAT_CHANCMP_OFFSET_LOW, addr); 449 } 450 451 static inline void 452 ioat_write_chainaddr(struct ioat_softc *ioat, uint64_t addr) 453 { 454 455 if (ioat->version >= IOAT_VER_3_3) 456 ioat_write_8(ioat, IOAT_CHAINADDR_OFFSET_LOW, addr); 457 else 458 ioat_write_double_4(ioat, IOAT_CHAINADDR_OFFSET_LOW, addr); 459 } 460 461 static inline boolean_t 462 is_ioat_active(uint64_t status) 463 { 464 return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_ACTIVE); 465 } 466 467 static inline boolean_t 468 is_ioat_idle(uint64_t status) 469 { 470 return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_IDLE); 471 } 472 473 static inline boolean_t 474 is_ioat_halted(uint64_t status) 475 { 476 return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_HALTED); 477 } 478 479 static inline boolean_t 480 is_ioat_suspended(uint64_t status) 481 { 482 return ((status & IOAT_CHANSTS_STATUS) == IOAT_CHANSTS_SUSPENDED); 483 } 484 485 static inline void 486 ioat_suspend(struct ioat_softc *ioat) 487 { 488 ioat_write_1(ioat, IOAT_CHANCMD_OFFSET, IOAT_CHANCMD_SUSPEND); 489 } 490 491 static inline void 492 ioat_reset(struct ioat_softc *ioat) 493 { 494 ioat_write_1(ioat, IOAT_CHANCMD_OFFSET, IOAT_CHANCMD_RESET); 495 } 496 497 static inline boolean_t 498 ioat_reset_pending(struct ioat_softc *ioat) 499 { 500 uint8_t cmd; 501 502 cmd = ioat_read_1(ioat, IOAT_CHANCMD_OFFSET); 503 return ((cmd & IOAT_CHANCMD_RESET) != 0); 504 } 505 506 #endif /* __IOAT_INTERNAL_H__ */ 507