40 	xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)userdata;  in __hal_fifo_mempool_item_alloc()  local
73 	if (fifo->config->alignment_size) {  in __hal_fifo_mempool_item_alloc()
74 	        status =__hal_fifo_dtr_align_alloc_map(fifo, txdp);  in __hal_fifo_mempool_item_alloc()
79 			      fifo->align_size,  in __hal_fifo_mempool_item_alloc()
88 	if (fifo->channel.dtr_init) {  in __hal_fifo_mempool_item_alloc()
89 		fifo->channel.dtr_init(fifo, (xge_hal_dtr_h)txdp, index,  in __hal_fifo_mempool_item_alloc()
90 			   fifo->channel.userdata, XGE_HAL_CHANNEL_OC_NORMAL);  in __hal_fifo_mempool_item_alloc()
110 	xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)userdata;  in __hal_fifo_mempool_item_free()  local
123 	if (fifo->config->alignment_size) {  in __hal_fifo_mempool_item_free()
125 			xge_os_dma_unmap(fifo->channel.pdev,  in __hal_fifo_mempool_item_free()
128 			       fifo->align_size,  in __hal_fifo_mempool_item_free()
135 			xge_os_dma_free(fifo->channel.pdev,  in __hal_fifo_mempool_item_free()
137 			      fifo->align_size,  in __hal_fifo_mempool_item_free()
154 	xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh;  in __hal_fifo_open()  local
159 	hldev = (xge_hal_device_t *)fifo->channel.devh;  in __hal_fifo_open()
160 	fifo->config = &hldev->config.fifo;  in __hal_fifo_open()
161 	queue = &fifo->config->queue[attr->post_qid];  in __hal_fifo_open()
164 	xge_os_spin_lock_init(&fifo->channel.reserve_lock, hldev->pdev);  in __hal_fifo_open()
166 	xge_os_spin_lock_init_irq(&fifo->channel.reserve_lock, hldev->irqh);  in __hal_fifo_open()
170                 fifo->post_lock_ptr = &hldev->xena_post_lock;  in __hal_fifo_open()
172 	        xge_os_spin_lock_init(&fifo->channel.post_lock, hldev->pdev);  in __hal_fifo_open()
173                 fifo->post_lock_ptr = &fifo->channel.post_lock;  in __hal_fifo_open()
177                 fifo->post_lock_ptr = &hldev->xena_post_lock;  in __hal_fifo_open()
179 	        xge_os_spin_lock_init_irq(&fifo->channel.post_lock,  in __hal_fifo_open()
181                 fifo->post_lock_ptr = &fifo->channel.post_lock;  in __hal_fifo_open()
185 	fifo->align_size =  in __hal_fifo_open()
186 		fifo->config->alignment_size * fifo->config->max_aligned_frags;  in __hal_fifo_open()
191 	fifo->hw_pair =  in __hal_fifo_open()
196 	fifo->interrupt_type = XGE_HAL_TXD_INT_TYPE_UTILZ;  in __hal_fifo_open()
198 		fifo->interrupt_type = XGE_HAL_TXD_INT_TYPE_PER_LIST;  in __hal_fifo_open()
200 	fifo->no_snoop_bits =  in __hal_fifo_open()
226 	fifo->priv_size = sizeof(xge_hal_fifo_txdl_priv_t) +  in __hal_fifo_open()
228 	fifo->priv_size = ((fifo->priv_size + __xge_os_cacheline_size -1) /  in __hal_fifo_open()
233 	fifo->txdl_size = fifo->config->max_frags * sizeof(xge_hal_fifo_txd_t);  in __hal_fifo_open()
234 	txdl_size = ((fifo->txdl_size + __xge_os_cacheline_size - 1) /  in __hal_fifo_open()
237 	if (fifo->txdl_size != txdl_size)  in __hal_fifo_open()
239 		fifo->config->max_frags, fifo->txdl_size, txdl_size,  in __hal_fifo_open()
242 	fifo->txdl_size = txdl_size;  in __hal_fifo_open()
247 	fifo->channel.dtr_init = attr->dtr_init;  in __hal_fifo_open()
248 	fifo->channel.userdata = attr->userdata;  in __hal_fifo_open()
249 	fifo->txdl_per_memblock = fifo->config->memblock_size /  in __hal_fifo_open()
250 		fifo->txdl_size;  in __hal_fifo_open()
252 	fifo->mempool = __hal_mempool_create(hldev->pdev,  in __hal_fifo_open()
253 					     fifo->config->memblock_size,  in __hal_fifo_open()
254 					     fifo->txdl_size,  in __hal_fifo_open()
255 					     fifo->priv_size,  in __hal_fifo_open()
260 					     fifo);  in __hal_fifo_open()
261 	if (fifo->mempool == NULL) {  in __hal_fifo_open()
266 					(void **) __hal_mempool_items_arr(fifo->mempool),  in __hal_fifo_open()
268 					fifo->config->reserve_threshold);  in __hal_fifo_open()
277 		fifo->channel.reserve_length, fifo->channel.reserve_top,  in __hal_fifo_open()
278 		fifo->config->max_frags, fifo->config->reserve_threshold,  in __hal_fifo_open()
279 		fifo->config->memblock_size, fifo->config->alignment_size,  in __hal_fifo_open()
280 		fifo->config->max_aligned_frags);  in __hal_fifo_open()
283 	for ( i = 0; i < fifo->channel.reserve_length; i++) {  in __hal_fifo_open()
285 		" handle:%p", i, fifo->channel.reserve_arr[i]);  in __hal_fifo_open()
289 	xge_assert(fifo->channel.reserve_length);  in __hal_fifo_open()
291 	max_arr_index	= fifo->channel.reserve_length - 1;  in __hal_fifo_open()
292 	max_arr_index	-=fifo->channel.reserve_top;  in __hal_fifo_open()
294 	mid_point = (fifo->channel.reserve_length - fifo->channel.reserve_top)/2;  in __hal_fifo_open()
296 		dtrh =	fifo->channel.reserve_arr[i];  in __hal_fifo_open()
297 		fifo->channel.reserve_arr[i] =  in __hal_fifo_open()
298 			fifo->channel.reserve_arr[max_arr_index - i];  in __hal_fifo_open()
299 		fifo->channel.reserve_arr[max_arr_index  - i] = dtrh;  in __hal_fifo_open()
303 	for ( i = 0; i < fifo->channel.reserve_length; i++) {  in __hal_fifo_open()
305 		" handle:%p", i, fifo->channel.reserve_arr[i]);  in __hal_fifo_open()
315 	xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh;  in __hal_fifo_close()  local
316 	xge_hal_device_t *hldev = (xge_hal_device_t *)fifo->channel.devh;  in __hal_fifo_close()
318 	if (fifo->mempool) {  in __hal_fifo_close()
319 		__hal_mempool_destroy(fifo->mempool);  in __hal_fifo_close()
325 	xge_os_spin_lock_destroy(&fifo->channel.reserve_lock, hldev->pdev);  in __hal_fifo_close()
327 	xge_os_spin_lock_destroy_irq(&fifo->channel.reserve_lock, hldev->pdev);  in __hal_fifo_close()
331 		xge_os_spin_lock_destroy(&fifo->channel.post_lock, hldev->pdev);  in __hal_fifo_close()
333 		xge_os_spin_lock_destroy_irq(&fifo->channel.post_lock,  in __hal_fifo_close()
398 		if (hldev->config.fifo.queue[i].configured) {  in __hal_fifo_hw_initialize()
399 			int priority = hldev->config.fifo.queue[i].priority;  in __hal_fifo_hw_initialize()
401 			    vBIT((hldev->config.fifo.queue[i].max-1),  in __hal_fifo_hw_initialize()
463 		if (!hldev->config.fifo.queue[i].configured ||  in __hal_fifo_hw_initialize()
464 		    !hldev->config.fifo.queue[i].intr_vector)  in __hal_fifo_hw_initialize()
481 				hldev->config.fifo.queue[i].intr_vector);  in __hal_fifo_hw_initialize()
494 	xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh;  in __hal_fifo_dtr_align_free_unmap()  local
499 		xge_os_dma_unmap(fifo->channel.pdev,  in __hal_fifo_dtr_align_free_unmap()
502 		       fifo->align_size,  in __hal_fifo_dtr_align_free_unmap()
509 	        xge_os_dma_free(fifo->channel.pdev,  in __hal_fifo_dtr_align_free_unmap()
511 	              fifo->align_size,  in __hal_fifo_dtr_align_free_unmap()
525 	xge_hal_fifo_t *fifo = (xge_hal_fifo_t *)channelh;  in __hal_fifo_dtr_align_alloc_map()  local
532 	txdl_priv->align_vaddr = (char *)xge_os_dma_malloc(fifo->channel.pdev,  in __hal_fifo_dtr_align_alloc_map()
533 				fifo->align_size,  in __hal_fifo_dtr_align_alloc_map()
543 	txdl_priv->align_dma_addr = xge_os_dma_map(fifo->channel.pdev,  in __hal_fifo_dtr_align_alloc_map()
545 		fifo->align_size,  in __hal_fifo_dtr_align_alloc_map()