1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* Copyright (c) 2002-2005 Neterion, Inc. 23 * All right Reserved. 24 * 25 * FileName : xgehal-fifo.c 26 * 27 * Description: fifo object implementation 28 * 29 * Created: 10 May 2004 30 */ 31 32 #include "xgehal-fifo.h" 33 #include "xgehal-device.h" 34 35 static xge_hal_status_e 36 __hal_fifo_mempool_item_alloc(xge_hal_mempool_h mempoolh, 37 void *memblock, 38 int memblock_index, 39 xge_hal_mempool_dma_t *dma_object, 40 void *item, 41 int index, 42 int is_last, 43 void *userdata) 44 { 45 int memblock_item_idx; 46 xge_hal_fifo_txdl_priv_t *txdl_priv; 47 xge_hal_fifo_txd_t *txdp = (xge_hal_fifo_txd_t *)item; 48 xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)userdata; 49 50 xge_assert(item); 51 txdl_priv = __hal_mempool_item_priv(mempoolh, memblock_index, 52 item, &memblock_item_idx); 53 54 xge_assert(txdl_priv); 55 56 /* pre-format HAL's TxDL's private */ 57 txdl_priv->dma_offset = (char*)item - (char*)memblock; 58 txdl_priv->dma_addr = dma_object->addr + txdl_priv->dma_offset; 59 txdl_priv->dma_handle = dma_object->handle; 60 txdl_priv->memblock = memblock; 61 txdl_priv->first_txdp = (xge_hal_fifo_txd_t *)item; 62 txdl_priv->next_txdl_priv = NULL; 63 txdl_priv->dang_txdl = NULL; 64 txdl_priv->dang_frags = 0; 65 txdl_priv->alloc_frags = 0; 66 67 #ifdef XGE_DEBUG_ASSERT 68 txdl_priv->dma_object = dma_object; 69 #endif 70 txdp->host_control = (u64)(ulong_t)txdl_priv; 71 72 #ifdef XGE_HAL_ALIGN_XMIT 73 txdl_priv->align_vaddr = NULL; 74 txdl_priv->align_dma_addr = (dma_addr_t)0; 75 76 #ifndef XGE_HAL_ALIGN_XMIT_ALLOC_RT 77 { 78 xge_hal_status_e status; 79 if (fifo->config->alignment_size) { 80 status =__hal_fifo_dtr_align_alloc_map(fifo, txdp); 81 if (status != XGE_HAL_OK) { 82 xge_debug_mm(XGE_ERR, 83 "align buffer[%d] %d bytes, status %d", 84 index, 85 fifo->config->alignment_size * 86 fifo->config->max_aligned_frags, 87 status); 88 return status; 89 } 90 } 91 } 92 #endif 93 #endif 94 95 if (fifo->channel.dtr_init) { 96 fifo->channel.dtr_init(fifo, (xge_hal_dtr_h)txdp, index, 97 fifo->channel.userdata, XGE_HAL_CHANNEL_OC_NORMAL); 98 } 99 100 return XGE_HAL_OK; 101 } 102 103 104 static xge_hal_status_e 105 __hal_fifo_mempool_item_free(xge_hal_mempool_h mempoolh, 106 void *memblock, 107 int memblock_index, 108 xge_hal_mempool_dma_t *dma_object, 109 void *item, 110 int index, 111 int is_last, 112 void *userdata) 113 { 114 int memblock_item_idx; 115 xge_hal_fifo_txdl_priv_t *txdl_priv; 116 #ifdef XGE_HAL_ALIGN_XMIT 117 xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)userdata; 118 #endif 119 120 xge_assert(item); 121 122 txdl_priv = __hal_mempool_item_priv(mempoolh, memblock_index, 123 item, &memblock_item_idx); 124 xge_assert(txdl_priv); 125 126 #ifdef XGE_HAL_ALIGN_XMIT 127 if (fifo->config->alignment_size) { 128 if (txdl_priv->align_dma_addr != 0) { 129 xge_os_dma_unmap(fifo->channel.pdev, 130 txdl_priv->align_dma_handle, 131 txdl_priv->align_dma_addr, 132 fifo->config->alignment_size * 133 fifo->config->max_aligned_frags, 134 XGE_OS_DMA_DIR_TODEVICE); 135 136 txdl_priv->align_dma_addr = 0; 137 } 138 139 if (txdl_priv->align_vaddr != NULL) { 140 xge_os_dma_free(fifo->channel.pdev, 141 txdl_priv->align_vaddr, 142 fifo->config->alignment_size * 143 fifo->config->max_aligned_frags, 144 &txdl_priv->align_dma_acch, 145 &txdl_priv->align_dma_handle); 146 147 txdl_priv->align_vaddr = NULL; 148 } 149 } 150 #endif 151 152 return XGE_HAL_OK; 153 } 154 155 xge_hal_status_e 156 __hal_fifo_open(xge_hal_channel_h channelh, xge_hal_channel_attr_t *attr) 157 { 158 xge_hal_device_t *hldev; 159 xge_hal_status_e status; 160 xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh; 161 xge_hal_fifo_queue_t *queue; 162 int i, txdl_size, max_arr_index, mid_point; 163 xge_hal_dtr_h dtrh; 164 165 hldev = (xge_hal_device_t *)fifo->channel.devh; 166 fifo->config = &hldev->config.fifo; 167 queue = &fifo->config->queue[attr->post_qid]; 168 169 #if defined(XGE_HAL_TX_MULTI_RESERVE) 170 xge_os_spin_lock_init(&fifo->channel.reserve_lock, hldev->pdev); 171 #elif defined(XGE_HAL_TX_MULTI_RESERVE_IRQ) 172 xge_os_spin_lock_init_irq(&fifo->channel.reserve_lock, hldev->irqh); 173 #endif 174 #if defined(XGE_HAL_TX_MULTI_POST) 175 if (xge_hal_device_check_id(hldev) == XGE_HAL_CARD_XENA) { 176 fifo->post_lock_ptr = &hldev->xena_post_lock; 177 } else { 178 xge_os_spin_lock_init(&fifo->channel.post_lock, hldev->pdev); 179 fifo->post_lock_ptr = &fifo->channel.post_lock; 180 } 181 #elif defined(XGE_HAL_TX_MULTI_POST_IRQ) 182 if (xge_hal_device_check_id(hldev) == XGE_HAL_CARD_XENA) { 183 fifo->post_lock_ptr = &hldev->xena_post_lock; 184 } else { 185 xge_os_spin_lock_init_irq(&fifo->channel.post_lock, 186 hldev->irqh); 187 fifo->post_lock_ptr = &fifo->channel.post_lock; 188 } 189 #endif 190 191 /* Initializing the BAR1 address as the start of 192 * the FIFO queue pointer and as a location of FIFO control 193 * word. */ 194 fifo->hw_pair = 195 (xge_hal_fifo_hw_pair_t *) (void *)(hldev->bar1 + 196 (attr->post_qid * XGE_HAL_FIFO_HW_PAIR_OFFSET)); 197 198 /* apply "interrupts per txdl" attribute */ 199 fifo->interrupt_type = XGE_HAL_TXD_INT_TYPE_UTILZ; 200 if (queue->intr) { 201 fifo->interrupt_type = XGE_HAL_TXD_INT_TYPE_PER_LIST; 202 } 203 fifo->no_snoop_bits = 204 (int)(XGE_HAL_TX_FIFO_NO_SNOOP(queue->no_snoop_bits)); 205 206 /* 207 * FIFO memory management strategy: 208 * 209 * TxDL splitted into three independent parts: 210 * - set of TxD's 211 * - TxD HAL private part 212 * - upper layer private part 213 * 214 * Adaptative memory allocation used. i.e. Memory allocated on 215 * demand with the size which will fit into one memory block. 216 * One memory block may contain more than one TxDL. In simple case 217 * memory block size can be equal to CPU page size. On more 218 * sophisticated OS's memory block can be contigious across 219 * several pages. 220 * 221 * During "reserve" operations more memory can be allocated on demand 222 * for example due to FIFO full condition. 223 * 224 * Pool of memory memblocks never shrinks except __hal_fifo_close 225 * routine which will essentially stop channel and free the resources. 226 */ 227 228 /* TxDL common private size == TxDL private + ULD private */ 229 fifo->priv_size = sizeof(xge_hal_fifo_txdl_priv_t) + 230 attr->per_dtr_space; 231 fifo->priv_size = ((fifo->priv_size + __xge_os_cacheline_size -1) / 232 __xge_os_cacheline_size) * 233 __xge_os_cacheline_size; 234 235 /* recompute txdl size to be cacheline aligned */ 236 fifo->txdl_size = fifo->config->max_frags * sizeof(xge_hal_fifo_txd_t); 237 txdl_size = ((fifo->txdl_size + __xge_os_cacheline_size - 1) / 238 __xge_os_cacheline_size) * __xge_os_cacheline_size; 239 240 if (fifo->txdl_size != txdl_size) 241 xge_debug_fifo(XGE_ERR, "cacheline > 128 (??): %d, %d, %d, %d", 242 fifo->config->max_frags, fifo->txdl_size, txdl_size, 243 __xge_os_cacheline_size); 244 245 fifo->txdl_size = txdl_size; 246 247 /* since dtr_init() callback will be called from item_alloc(), 248 * the same way channels userdata might be used prior to 249 * channel_initialize() */ 250 fifo->channel.dtr_init = attr->dtr_init; 251 fifo->channel.userdata = attr->userdata; 252 fifo->txdl_per_memblock = fifo->config->memblock_size / 253 fifo->txdl_size; 254 255 fifo->mempool = __hal_mempool_create(hldev->pdev, 256 fifo->config->memblock_size, 257 fifo->txdl_size, 258 fifo->priv_size, 259 queue->initial, 260 queue->max, 261 __hal_fifo_mempool_item_alloc, 262 __hal_fifo_mempool_item_free, 263 fifo); 264 if (fifo->mempool == NULL) { 265 return XGE_HAL_ERR_OUT_OF_MEMORY; 266 } 267 268 status = __hal_channel_initialize(channelh, attr, 269 __hal_mempool_items_arr(fifo->mempool), 270 queue->initial, queue->max, 271 fifo->config->reserve_threshold); 272 if (status != XGE_HAL_OK) { 273 __hal_fifo_close(channelh); 274 return status; 275 } 276 xge_debug_fifo(XGE_TRACE, 277 "DTR reserve_length:%d reserve_top:%d\n" 278 "max_frags:%d reserve_threshold:%d\n" 279 "memblock_size:%d alignment_size:%d max_aligned_frags:%d\n", 280 fifo->channel.reserve_length, fifo->channel.reserve_top, 281 fifo->config->max_frags, fifo->config->reserve_threshold, 282 fifo->config->memblock_size, fifo->config->alignment_size, 283 fifo->config->max_aligned_frags); 284 285 #ifdef XGE_DEBUG_ASSERT 286 for ( i = 0; i < fifo->channel.reserve_length; i++) { 287 xge_debug_fifo(XGE_TRACE, "DTR before reversing index:%d" 288 " handle:%p\n", i, fifo->channel.reserve_arr[i]); 289 } 290 #endif 291 292 xge_assert(fifo->channel.reserve_length); 293 /* reverse the FIFO dtr array */ 294 max_arr_index = fifo->channel.reserve_length - 1; 295 max_arr_index -=fifo->channel.reserve_top; 296 xge_assert(max_arr_index); 297 mid_point = (fifo->channel.reserve_length - fifo->channel.reserve_top)/2; 298 for (i = 0; i < mid_point; i++) { 299 dtrh = fifo->channel.reserve_arr[i]; 300 fifo->channel.reserve_arr[i] = 301 fifo->channel.reserve_arr[max_arr_index - i]; 302 fifo->channel.reserve_arr[max_arr_index - i] = dtrh; 303 } 304 305 #ifdef XGE_DEBUG_ASSERT 306 for ( i = 0; i < fifo->channel.reserve_length; i++) { 307 xge_debug_fifo(XGE_TRACE, "DTR after reversing index:%d" 308 " handle:%p\n", i, fifo->channel.reserve_arr[i]); 309 } 310 #endif 311 312 return XGE_HAL_OK; 313 } 314 315 void 316 __hal_fifo_close(xge_hal_channel_h channelh) 317 { 318 xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh; 319 xge_hal_device_t *hldev = (xge_hal_device_t *)fifo->channel.devh; 320 321 if (fifo->mempool) { 322 __hal_mempool_destroy(fifo->mempool); 323 } 324 325 __hal_channel_terminate(channelh); 326 327 #if defined(XGE_HAL_TX_MULTI_RESERVE) 328 xge_os_spin_lock_destroy(&fifo->channel.reserve_lock, hldev->pdev); 329 #elif defined(XGE_HAL_TX_MULTI_RESERVE_IRQ) 330 xge_os_spin_lock_destroy_irq(&fifo->channel.reserve_lock, hldev->pdev); 331 #endif 332 if (xge_hal_device_check_id(hldev) == XGE_HAL_CARD_HERC) { 333 #if defined(XGE_HAL_TX_MULTI_POST) 334 xge_os_spin_lock_destroy(&fifo->channel.post_lock, hldev->pdev); 335 #elif defined(XGE_HAL_TX_MULTI_POST_IRQ) 336 xge_os_spin_lock_destroy_irq(&fifo->channel.post_lock, 337 hldev->pdev); 338 #endif 339 } 340 } 341 342 void 343 __hal_fifo_hw_initialize(xge_hal_device_h devh) 344 { 345 xge_hal_device_t *hldev = (xge_hal_device_t *)devh; 346 xge_hal_pci_bar0_t *bar0 = (xge_hal_pci_bar0_t *)(void 347 *)hldev->bar0; 348 u64* tx_fifo_partitions[4]; 349 u64* tx_fifo_wrr[5]; 350 u64 val64, part0; 351 int priority = 0; 352 int i; 353 354 /* Tx DMA Initialization */ 355 356 tx_fifo_partitions[0] = &bar0->tx_fifo_partition_0; 357 tx_fifo_partitions[1] = &bar0->tx_fifo_partition_1; 358 tx_fifo_partitions[2] = &bar0->tx_fifo_partition_2; 359 tx_fifo_partitions[3] = &bar0->tx_fifo_partition_3; 360 361 tx_fifo_wrr[0] = &bar0->tx_w_round_robin_0; 362 tx_fifo_wrr[1] = &bar0->tx_w_round_robin_1; 363 tx_fifo_wrr[2] = &bar0->tx_w_round_robin_2; 364 tx_fifo_wrr[3] = &bar0->tx_w_round_robin_3; 365 tx_fifo_wrr[4] = &bar0->tx_w_round_robin_4; 366 367 /* Note: WRR calendar must be configured before the transmit 368 FIFOs are enabled! page 6-77 user guide */ 369 370 /* all zeroes for Round-Robin */ 371 for (i = 0; i < XGE_HAL_FIFO_MAX_WRR; i++) { 372 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0, 0, 373 tx_fifo_wrr[i]); 374 } 375 376 /* reset all of them but '0' */ 377 for (i=1; i < XGE_HAL_FIFO_MAX_PARTITION; i++) { 378 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0, 0ULL, 379 tx_fifo_partitions[i]); 380 } 381 382 /* configure only configured FIFOs */ 383 val64 = 0; part0 = 0; 384 for (i = 0; i < XGE_HAL_MAX_FIFO_NUM; i++) { 385 int reg_half = i % 2; 386 int reg_num = i / 2; 387 388 priority = 0; 389 390 if (hldev->config.fifo.queue[i].configured) { 391 val64 |= 392 vBIT((hldev->config.fifo.queue[i].max-1), 393 (((reg_half) * 32) + 19), 394 13) | vBIT(priority, (((reg_half)*32) + 5), 3); 395 } 396 397 /* NOTE: do write operation for each second u64 half 398 or force for first one if configured number 399 is even */ 400 if (reg_half) { 401 if (reg_num == 0) { 402 /* skip partition '0', must write it once at 403 * the end */ 404 part0 = val64; 405 } else { 406 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0, 407 val64, tx_fifo_partitions[reg_num]); 408 xge_debug_fifo(XGE_TRACE, 409 "fifo partition_%d at: " 410 "0x%llx is: 0x%llx", reg_num, 411 (unsigned long long)(ulong_t) 412 tx_fifo_partitions[reg_num], 413 (unsigned long long)val64); 414 } 415 val64 = 0; 416 } 417 } 418 419 part0 |= BIT(0); /* to enable the FIFO partition. */ 420 __hal_pio_mem_write32_lower(hldev->pdev, hldev->regh0, (u32)part0, 421 tx_fifo_partitions[0]); 422 xge_os_wmb(); 423 __hal_pio_mem_write32_upper(hldev->pdev, hldev->regh0, (u32)(part0>>32), 424 tx_fifo_partitions[0]); 425 xge_debug_fifo(XGE_TRACE, "fifo partition_0 at: " 426 "0x%llx is: 0x%llx", 427 (unsigned long long)(ulong_t) 428 tx_fifo_partitions[0], 429 (unsigned long long) part0); 430 431 /* 432 * Initialization of Tx_PA_CONFIG register to ignore packet 433 * integrity checking. 434 */ 435 val64 = xge_os_pio_mem_read64(hldev->pdev, hldev->regh0, 436 &bar0->tx_pa_cfg); 437 val64 |= XGE_HAL_TX_PA_CFG_IGNORE_FRM_ERR | 438 XGE_HAL_TX_PA_CFG_IGNORE_SNAP_OUI | 439 XGE_HAL_TX_PA_CFG_IGNORE_LLC_CTRL | 440 XGE_HAL_TX_PA_CFG_IGNORE_L2_ERR; 441 xge_os_pio_mem_write64(hldev->pdev, hldev->regh0, val64, 442 &bar0->tx_pa_cfg); 443 xge_debug_fifo(XGE_TRACE, "%s", "fifo channels initialized"); 444 } 445 446 #ifdef XGE_HAL_ALIGN_XMIT 447 void 448 __hal_fifo_dtr_align_free_unmap(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh) 449 { 450 xge_hal_fifo_txdl_priv_t *txdl_priv; 451 xge_hal_fifo_txd_t *txdp = (xge_hal_fifo_txd_t *)dtrh; 452 xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh; 453 454 txdl_priv = __hal_fifo_txdl_priv(txdp); 455 456 if (txdl_priv->align_dma_addr != 0) { 457 xge_os_dma_unmap(fifo->channel.pdev, 458 txdl_priv->align_dma_handle, 459 txdl_priv->align_dma_addr, 460 fifo->config->alignment_size * 461 fifo->config->max_aligned_frags, 462 XGE_OS_DMA_DIR_TODEVICE); 463 464 txdl_priv->align_dma_addr = 0; 465 } 466 467 if (txdl_priv->align_vaddr != NULL) { 468 xge_os_dma_free(fifo->channel.pdev, 469 txdl_priv->align_vaddr, 470 fifo->config->alignment_size * 471 fifo->config->max_aligned_frags, 472 &txdl_priv->align_dma_acch, 473 &txdl_priv->align_dma_handle); 474 475 476 txdl_priv->align_vaddr = NULL; 477 } 478 } 479 480 xge_hal_status_e 481 __hal_fifo_dtr_align_alloc_map(xge_hal_channel_h channelh, xge_hal_dtr_h dtrh) 482 { 483 xge_hal_fifo_txdl_priv_t *txdl_priv; 484 xge_hal_fifo_txd_t *txdp = (xge_hal_fifo_txd_t *)dtrh; 485 xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh; 486 487 xge_assert(txdp); 488 489 txdl_priv = __hal_fifo_txdl_priv(txdp); 490 491 /* allocate alignment DMA-buffer */ 492 txdl_priv->align_vaddr = xge_os_dma_malloc(fifo->channel.pdev, 493 fifo->config->alignment_size * 494 fifo->config->max_aligned_frags, 495 XGE_OS_DMA_CACHELINE_ALIGNED | 496 XGE_OS_DMA_STREAMING, 497 &txdl_priv->align_dma_handle, 498 &txdl_priv->align_dma_acch); 499 if (txdl_priv->align_vaddr == NULL) { 500 return XGE_HAL_ERR_OUT_OF_MEMORY; 501 } 502 503 /* map it */ 504 txdl_priv->align_dma_addr = xge_os_dma_map(fifo->channel.pdev, 505 txdl_priv->align_dma_handle, txdl_priv->align_vaddr, 506 fifo->config->alignment_size * 507 fifo->config->max_aligned_frags, 508 XGE_OS_DMA_DIR_TODEVICE, XGE_OS_DMA_STREAMING); 509 510 if (txdl_priv->align_dma_addr == XGE_OS_INVALID_DMA_ADDR) { 511 __hal_fifo_dtr_align_free_unmap(channelh, dtrh); 512 return XGE_HAL_ERR_OUT_OF_MAPPING; 513 } 514 515 return XGE_HAL_OK; 516 } 517 #endif 518 519 520