1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #ifndef _SYS_IOAT_H 28 #define _SYS_IOAT_H 29 30 #ifdef __cplusplus 31 extern "C" { 32 #endif 33 34 #include <sys/types.h> 35 #include <sys/dcopy.h> 36 #include <sys/dcopy_device.h> 37 38 39 /* ioat ioctls */ 40 #define IOATIOC ('T'<< 8) 41 typedef enum { 42 IOAT_IOCTL_WRITE_REG = (IOATIOC | 0x0), 43 IOAT_IOCTL_READ_REG = (IOATIOC | 0x1), 44 IOAT_IOCTL_TEST = (IOATIOC | 0x2) 45 } ioat_ioctl_enum_t; 46 47 typedef struct ioat_ioctl_reg_s { 48 uint_t size; 49 uint_t addr; 50 uint64_t data; 51 } ioat_ioctl_reg_t; 52 typedef ioat_ioctl_reg_t ioat_ioctl_wrreg_t; 53 typedef ioat_ioctl_reg_t ioat_ioctl_rdreg_t; 54 55 #ifdef _KERNEL 56 /* *** Driver Private Below *** */ 57 58 /* IOAT_DMACAPABILITY flags */ 59 #define IOAT_DMACAP_PAGEBREAK 0x1 60 #define IOAT_DMACAP_CRC 0x2 61 #define IOAT_DMACAP_MARKERSKIP 0x4 62 #define IOAT_DMACAP_XOR 0x8 63 #define IOAT_DMACAP_DCA 0x10 64 65 /* IOAT_INTRCTL bits */ 66 #define IOAT_INTRCTL_MASTER_EN 0x1 67 #define IOAT_INTRCTL_INTR_STAT 0x2 68 69 /* MMIO Registers */ 70 #define IOAT_CHANCNT 0x0 /* 8-bit */ 71 #define IOAT_XFERCAP 0x1 /* 8-bit */ 72 #define IOAT_GENCTRL 0x2 /* 8-bit */ 73 #define IOAT_INTRCTL 0x3 /* 8-bit */ 74 #define IOAT_ATTNSTATUS 0x4 /* 32-bit */ 75 #define IOAT_CBVER 0x8 /* 8-bit */ 76 #define IOAT_PERPORT_OFF 0xA /* 16-bit */ 77 #define IOAT_INTRDELAY 0xC /* 16-bit */ 78 #define IOAT_CSSTATUS 0xE /* 16-bit */ 79 #define IOAT_DMACAPABILITY 0x10 /* 32-bit */ 80 81 #define IOAT_CHANNELREG_OFFSET 0x80 82 83 /* Channel Registers */ 84 #define IOAT_CHAN_CTL 0x0 /* 16-bit */ 85 #define IOAT_CHAN_COMP 0x2 /* 16-bit */ 86 #define IOAT_CHAN_CMPL_LO 0x18 /* 32-bit */ 87 #define IOAT_CHAN_CMPL_HI 0x1C /* 32-bit */ 88 #define IOAT_CHAN_ERR 0x28 /* 32-bit */ 89 #define IOAT_CHAN_ERRMASK 0x2C /* 32-bit */ 90 #define IOAT_CHAN_DCACTRL 0x30 /* 32-bit */ 91 92 #define IOAT_V1_CHAN_STS_LO 0x4 /* 32-bit */ 93 #define IOAT_V1_CHAN_STS_HI 0x8 /* 32-bit */ 94 #define IOAT_V1_CHAN_ADDR_LO 0x0C /* 32-bit */ 95 #define IOAT_V1_CHAN_ADDR_HI 0x10 /* 32-bit */ 96 #define IOAT_V1_CHAN_CMD 0x14 /* 8-bit */ 97 98 #define IOAT_V2_CHAN_CMD 0x4 /* 8-bit */ 99 #define IOAT_V2_CHAN_CNT 0x6 /* 16-bit */ 100 #define IOAT_V2_CHAN_STS_LO 0x8 /* 32-bit */ 101 #define IOAT_V2_CHAN_STS_HI 0xC /* 32-bit */ 102 #define IOAT_V2_CHAN_ADDR_LO 0x10 /* 32-bit */ 103 #define IOAT_V2_CHAN_ADDR_HI 0x14 /* 32-bit */ 104 105 #define IOAT_CHAN_STS_ADDR_MASK 0xFFFFFFFFFFFFFFC0 106 #define IOAT_CHAN_STS_XFER_MASK 0x3F 107 #define IOAT_CHAN_STS_FAIL_MASK 0x6 108 #define IOAT_CMPL_INDEX(channel) \ 109 (((*channel->ic_cmpl & IOAT_CHAN_STS_ADDR_MASK) - \ 110 ring->cr_phys_desc) >> 6) 111 #define IOAT_CMPL_FAILED(channel) \ 112 (*channel->ic_cmpl & IOAT_CHAN_STS_FAIL_MASK) 113 114 115 typedef struct ioat_chan_desc_s { 116 uint32_t dd_res0; 117 uint32_t dd_ctrl; 118 uint64_t dd_res1; 119 uint64_t dd_res2; 120 uint64_t dd_next_desc; 121 uint64_t dd_res4; 122 uint64_t dd_res5; 123 uint64_t dd_res6; 124 uint64_t dd_res7; 125 } ioat_chan_desc_t; 126 127 /* dca dd_ctrl bits */ 128 #define IOAT_DESC_CTRL_OP_CNTX ((uint32_t)0xFF << 24) 129 #define IOAT_DESC_CTRL_CNTX_CHNG 0x1 130 typedef struct ioat_chan_dca_desc_s { 131 uint32_t dd_cntx; 132 uint32_t dd_ctrl; 133 uint64_t dd_res1; 134 uint64_t dd_res2; 135 uint64_t dd_next_desc; 136 uint64_t dd_res4; 137 uint64_t dd_res5; 138 uint64_t dd_res6; 139 uint64_t dd_res7; 140 } ioat_chan_dca_desc_t; 141 142 /* dma dd_ctrl bits */ 143 #define IOAT_DESC_CTRL_OP_DMA (0x0 << 24) 144 #define IOAT_DESC_DMACTRL_NULL 0x20 145 #define IOAT_DESC_CTRL_FENCE 0x10 146 #define IOAT_DESC_CTRL_CMPL 0x8 147 #define IOAT_DESC_CTRL_NODSTSNP 0x4 148 #define IOAT_DESC_CTRL_NOSRCSNP 0x2 149 #define IOAT_DESC_CTRL_INTR 0x1 150 typedef struct ioat_chan_dma_desc_s { 151 uint32_t dd_size; 152 uint32_t dd_ctrl; 153 uint64_t dd_src_paddr; 154 uint64_t dd_dest_paddr; 155 uint64_t dd_next_desc; 156 uint64_t dd_next_src_paddr; /* v2 only */ 157 uint64_t dd_next_dest_paddr; /* v2 only */ 158 uint64_t dd_res6; 159 uint64_t dd_res7; 160 } ioat_chan_dma_desc_t; 161 162 163 typedef enum { 164 IOAT_CBv1, 165 IOAT_CBv2 166 } ioat_version_t; 167 168 /* ioat private data per command */ 169 typedef struct ioat_cmd_private_s { 170 uint64_t ip_generation; 171 uint64_t ip_index; 172 uint64_t ip_start; 173 dcopy_cmd_t ip_next; 174 } ioat_cmd_private_t; 175 176 /* descriptor ring state */ 177 typedef struct ioat_channel_ring_s { 178 /* protects cr_cmpl_gen & cr_cmpl_last */ 179 kmutex_t cr_cmpl_mutex; 180 181 /* desc ring generation for the last completion we saw */ 182 uint64_t cr_cmpl_gen; 183 184 /* last descriptor index we saw complete */ 185 uint64_t cr_cmpl_last; 186 187 /* protects cr_desc_* */ 188 kmutex_t cr_desc_mutex; 189 190 /* 191 * last descriptor posted. used to update its next pointer when we 192 * add a new desc. Also used to tack the completion (See comment for 193 * cr_desc_gen_prev). 194 */ 195 uint64_t cr_desc_prev; 196 197 /* where to put the next descriptor */ 198 uint64_t cr_desc_next; 199 200 /* what the current desc ring generation is */ 201 uint64_t cr_desc_gen; 202 203 /* 204 * used during cmd_post to track the last desc posted. cr_desc_next 205 * and cr_desc_gen will be pointing to the next free desc after 206 * writing the descriptor to the ring. But we want to track the 207 * completion for the last descriptor posted. 208 */ 209 uint64_t cr_desc_gen_prev; 210 211 /* the last desc in the ring (for wrap) */ 212 uint64_t cr_desc_last; 213 214 /* pointer to the head of the ring */ 215 ioat_chan_desc_t *cr_desc; 216 217 /* physical address of the head of the ring */ 218 uint64_t cr_phys_desc; 219 220 /* back pointer to the channel state */ 221 struct ioat_channel_s *cr_chan; 222 223 /* for CB v2, number of desc posted (written to IOAT_V2_CHAN_CNT) */ 224 uint_t cr_post_cnt; 225 } ioat_channel_ring_t; 226 227 /* track channel state so we can handle a failure */ 228 typedef enum { 229 IOAT_CHANNEL_OK = 0, 230 IOAT_CHANNEL_IN_FAILURE = 1 231 } ic_channel_state_t; 232 233 typedef struct ioat_channel_s *ioat_channel_t; 234 struct ioat_channel_s { 235 /* channel's ring state */ 236 ioat_channel_ring_t *ic_ring; 237 238 /* IOAT_CBv1 || IOAT_CBv2 */ 239 ioat_version_t ic_ver; 240 241 /* 242 * state to determine if it's OK to post the the channel and if all 243 * future polls should return failure. 244 */ 245 ic_channel_state_t ic_channel_state; 246 247 /* channel command cache (*_cmd_alloc, *_cmd_free, etc) */ 248 kmem_cache_t *ic_cmd_cache; 249 250 /* dcopy state for dcopy_device_channel_notify() call */ 251 dcopy_handle_t ic_dcopy_handle; 252 253 /* location in memory where completions are DMA'ed into */ 254 volatile uint64_t *ic_cmpl; 255 256 /* channel specific registers */ 257 uint8_t *ic_regs; 258 259 /* if this channel is using DCA */ 260 boolean_t ic_dca_active; 261 262 /* DCA ID the channel is currently pointing to */ 263 uint32_t ic_dca_current; 264 265 /* devices channel number */ 266 uint_t ic_chan_num; 267 268 /* number of descriptors in ring */ 269 uint_t ic_chan_desc_cnt; 270 271 /* descriptor ring alloc state */ 272 ddi_dma_handle_t ic_desc_dma_handle; 273 size_t ic_desc_alloc_size; 274 ddi_acc_handle_t ic_desc_handle; 275 ddi_dma_cookie_t ic_desc_cookies; 276 277 /* completion buffer alloc state */ 278 ddi_dma_handle_t ic_cmpl_dma_handle; 279 size_t ic_cmpl_alloc_size; 280 ddi_acc_handle_t ic_cmpl_handle; 281 ddi_dma_cookie_t ic_cmpl_cookie; 282 uint64_t ic_phys_cmpl; 283 284 /* if inuse, we need to re-init the channel during resume */ 285 boolean_t ic_inuse; 286 287 /* backpointer to driver state */ 288 struct ioat_state_s *ic_state; 289 }; 290 291 typedef struct ioat_rs_s *ioat_rs_hdl_t; 292 293 /* driver state */ 294 typedef struct ioat_state_s { 295 dev_info_t *is_dip; 296 int is_instance; 297 298 kmutex_t is_mutex; 299 300 /* register handle and pointer to registers */ 301 ddi_acc_handle_t is_reg_handle; 302 uint8_t *is_genregs; 303 304 /* IOAT_CBv1 || IOAT_CBv2 */ 305 ioat_version_t is_ver; 306 307 /* channel state */ 308 ioat_channel_t is_channel; 309 size_t is_chansize; 310 ioat_rs_hdl_t is_channel_rs; 311 312 ddi_iblock_cookie_t is_iblock_cookie; 313 314 /* device info */ 315 uint_t is_chanoff; 316 uint_t is_num_channels; 317 uint_t is_maxxfer; 318 uint_t is_cbver; 319 uint_t is_intrdelay; 320 uint_t is_status; 321 uint_t is_capabilities; 322 323 /* dcopy_device_register()/dcopy_device_unregister() state */ 324 dcopy_device_handle_t is_device_handle; 325 dcopy_device_info_t is_deviceinfo; 326 } ioat_state_t; 327 328 329 int ioat_ioctl(dev_t dev, int cmd, intptr_t arg, int mode, cred_t *cred, 330 int *rval); 331 332 void ioat_rs_init(ioat_state_t *state, uint_t min_val, uint_t max_val, 333 ioat_rs_hdl_t *handle); 334 void ioat_rs_fini(ioat_rs_hdl_t *handle); 335 int ioat_rs_alloc(ioat_rs_hdl_t handle, uint_t *rs); 336 void ioat_rs_free(ioat_rs_hdl_t handle, uint_t rs); 337 338 int ioat_channel_init(ioat_state_t *state); 339 void ioat_channel_fini(ioat_state_t *state); 340 void ioat_channel_suspend(ioat_state_t *state); 341 int ioat_channel_resume(ioat_state_t *state); 342 void ioat_channel_quiesce(ioat_state_t *); 343 344 int ioat_channel_alloc(void *device_private, dcopy_handle_t handle, int flags, 345 uint_t size, dcopy_query_channel_t *info, void *channel_private); 346 void ioat_channel_free(void *channel_private); 347 void ioat_channel_intr(ioat_channel_t channel); 348 int ioat_cmd_alloc(void *channel, int flags, dcopy_cmd_t *cmd); 349 void ioat_cmd_free(void *channel, dcopy_cmd_t *cmd); 350 int ioat_cmd_post(void *channel, dcopy_cmd_t cmd); 351 int ioat_cmd_poll(void *channel, dcopy_cmd_t cmd); 352 void ioat_unregister_complete(void *device_private, int status); 353 354 355 #endif /* _KERNEL */ 356 357 #ifdef __cplusplus 358 } 359 #endif 360 361 #endif /* _SYS_IOAT_H */ 362