Lines Matching +full:dma +full:- +full:poll +full:- +full:cnt
56 * @param octeon_dev - the octeon device pointer.
57 * @param opcode - the opcode for which the dispatch argument
59 * @param subcode - the subcode for which the dispatch argument
78 mtx_lock(&octeon_dev->dispatch.lock); in lio_get_dispatch_arg()
80 if (octeon_dev->dispatch.count == 0) { in lio_get_dispatch_arg()
81 mtx_unlock(&octeon_dev->dispatch.lock); in lio_get_dispatch_arg()
85 if (octeon_dev->dispatch.dlist[idx].opcode == combined_opcode) { in lio_get_dispatch_arg()
86 fn_arg = octeon_dev->dispatch.dlist[idx].arg; in lio_get_dispatch_arg()
89 &octeon_dev->dispatch.dlist[idx].head, entries) { in lio_get_dispatch_arg()
90 if (((struct lio_dispatch *)dispatch)->opcode == in lio_get_dispatch_arg()
92 fn_arg = ((struct lio_dispatch *)dispatch)->arg; in lio_get_dispatch_arg()
98 mtx_unlock(&octeon_dev->dispatch.lock); in lio_get_dispatch_arg()
104 * @param droq - Droq on which count is checked.
110 struct octeon_device *oct = droq->oct_dev; in lio_droq_check_hw_for_pkts()
114 pkt_count = lio_read_csr32(oct, droq->pkts_sent_reg); in lio_droq_check_hw_for_pkts()
116 last_count = pkt_count - droq->pkt_count; in lio_droq_check_hw_for_pkts()
117 droq->pkt_count = pkt_count; in lio_droq_check_hw_for_pkts()
121 atomic_add_int(&droq->pkts_pending, last_count); in lio_droq_check_hw_for_pkts()
137 droq->max_empty_descs = 0; in lio_droq_compute_max_packet_bufs()
140 droq->max_empty_descs++; in lio_droq_compute_max_packet_bufs()
141 count += droq->buffer_size; in lio_droq_compute_max_packet_bufs()
144 droq->max_empty_descs = droq->max_count - droq->max_empty_descs; in lio_droq_compute_max_packet_bufs()
151 droq->read_idx = 0; in lio_droq_reset_indices()
152 droq->refill_idx = 0; in lio_droq_reset_indices()
153 droq->refill_count = 0; in lio_droq_reset_indices()
154 atomic_store_rel_int(&droq->pkts_pending, 0); in lio_droq_reset_indices()
163 for (i = 0; i < droq->max_count; i++) { in lio_droq_destroy_ring_buffers()
164 if (droq->recv_buf_list[i].buffer != NULL) { in lio_droq_destroy_ring_buffers()
165 lio_recv_buffer_free(droq->recv_buf_list[i].buffer); in lio_droq_destroy_ring_buffers()
166 droq->recv_buf_list[i].buffer = NULL; in lio_droq_destroy_ring_buffers()
177 struct lio_droq_desc *desc_ring = droq->desc_ring; in lio_droq_setup_ring_buffers()
181 for (i = 0; i < droq->max_count; i++) { in lio_droq_setup_ring_buffers()
182 buf = lio_recv_buffer_alloc(droq->buffer_size); in lio_droq_setup_ring_buffers()
187 droq->stats.rx_alloc_failure++; in lio_droq_setup_ring_buffers()
188 return (-ENOMEM); in lio_droq_setup_ring_buffers()
191 droq->recv_buf_list[i].buffer = buf; in lio_droq_setup_ring_buffers()
192 droq->recv_buf_list[i].data = ((struct mbuf *)buf)->m_data; in lio_droq_setup_ring_buffers()
195 lio_map_ring(oct->device, droq->recv_buf_list[i].buffer, in lio_droq_setup_ring_buffers()
196 droq->buffer_size); in lio_droq_setup_ring_buffers()
209 struct lio_droq *droq = oct->droq[q_no]; in lio_delete_droq()
213 while (taskqueue_cancel(droq->droq_taskqueue, &droq->droq_task, NULL)) in lio_delete_droq()
214 taskqueue_drain(droq->droq_taskqueue, &droq->droq_task); in lio_delete_droq()
216 taskqueue_free(droq->droq_taskqueue); in lio_delete_droq()
217 droq->droq_taskqueue = NULL; in lio_delete_droq()
220 free(droq->recv_buf_list, M_DEVBUF); in lio_delete_droq()
222 if (droq->desc_ring != NULL) in lio_delete_droq()
223 lio_dma_free((droq->max_count * LIO_DROQ_DESC_SIZE), in lio_delete_droq()
224 droq->desc_ring); in lio_delete_droq()
226 oct->io_qmask.oq &= ~(1ULL << q_no); in lio_delete_droq()
227 bzero(oct->droq[q_no], sizeof(struct lio_droq)); in lio_delete_droq()
228 oct->num_oqs--; in lio_delete_droq()
237 struct octeon_device *oct = droq->oct_dev; in lio_droq_bh()
238 struct lio_instr_queue *iq = oct->instr_queue[droq->q_no]; in lio_droq_bh()
241 reschedule = lio_droq_process_packets(oct, droq, oct->rx_budget); in lio_droq_bh()
243 if (atomic_load_acq_int(&iq->instr_pending)) in lio_droq_bh()
244 tx_done = lio_flush_iq(oct, iq, oct->tx_budget); in lio_droq_bh()
247 taskqueue_enqueue(droq->droq_taskqueue, &droq->droq_task); in lio_droq_bh()
263 droq = oct->droq[q_no]; in lio_init_droq()
266 droq->oct_dev = oct; in lio_init_droq()
267 droq->q_no = q_no; in lio_init_droq()
269 droq->app_ctx = app_ctx; in lio_init_droq()
271 droq->app_ctx = (void *)(size_t)q_no; in lio_init_droq()
286 droq->max_count = c_num_descs; in lio_init_droq()
287 droq->buffer_size = c_buf_size; in lio_init_droq()
289 desc_ring_size = droq->max_count * LIO_DROQ_DESC_SIZE; in lio_init_droq()
290 droq->desc_ring = lio_dma_alloc(desc_ring_size, &droq->desc_ring_dma); in lio_init_droq()
291 if (droq->desc_ring == NULL) { in lio_init_droq()
296 lio_dev_dbg(oct, "droq[%d]: desc_ring: virt: 0x%p, dma: %llx\n", q_no, in lio_init_droq()
297 droq->desc_ring, LIO_CAST64(droq->desc_ring_dma)); in lio_init_droq()
298 lio_dev_dbg(oct, "droq[%d]: num_desc: %d\n", q_no, droq->max_count); in lio_init_droq()
300 size = droq->max_count * LIO_DROQ_RECVBUF_SIZE; in lio_init_droq()
301 droq->recv_buf_list = in lio_init_droq()
304 if (droq->recv_buf_list == NULL) { in lio_init_droq()
312 droq->pkts_per_intr = c_pkts_per_intr; in lio_init_droq()
313 droq->refill_threshold = c_refill_threshold; in lio_init_droq()
316 droq->max_empty_descs); in lio_init_droq()
318 mtx_init(&droq->lock, "droq_lock", NULL, MTX_DEF); in lio_init_droq()
320 STAILQ_INIT(&droq->dispatch_stq_head); in lio_init_droq()
322 oct->fn_list.setup_oq_regs(oct, q_no); in lio_init_droq()
324 oct->io_qmask.oq |= BIT_ULL(q_no); in lio_init_droq()
331 NET_TASK_INIT(&droq->droq_task, 0, lio_droq_bh, (void *)droq); in lio_init_droq()
333 droq->droq_taskqueue = taskqueue_create_fast("lio_droq_task", M_NOWAIT, in lio_init_droq()
335 &droq->droq_taskqueue); in lio_init_droq()
336 taskqueue_start_threads_cpuset(&droq->droq_taskqueue, 1, PI_NET, in lio_init_droq()
337 &oct->ioq_vector[q_no].affinity_mask, in lio_init_droq()
338 "lio%d_droq%d_task", oct->octeon_id, in lio_init_droq()
351 * octeon_dev - pointer to the octeon device structure
352 * droq - droq in which the packet arrived.
353 * buf_cnt - no. of buffers used by the packet.
354 * idx - index in the descriptor for the first buffer in the packet.
366 * The droq->lock is held when this routine is called.
377 info = (struct lio_droq_info *)droq->recv_buf_list[idx].data; in lio_create_recv_info()
383 recv_pkt = recv_info->recv_pkt; in lio_create_recv_info()
384 recv_pkt->rh = info->rh; in lio_create_recv_info()
385 recv_pkt->length = (uint32_t)info->length; in lio_create_recv_info()
386 recv_pkt->buffer_count = (uint16_t)buf_cnt; in lio_create_recv_info()
387 recv_pkt->octeon_id = (uint16_t)octeon_dev->octeon_id; in lio_create_recv_info()
390 bytes_left = (uint32_t)info->length; in lio_create_recv_info()
393 recv_pkt->buffer_size[i] = (bytes_left >= droq->buffer_size) ? in lio_create_recv_info()
394 droq->buffer_size : bytes_left; in lio_create_recv_info()
396 recv_pkt->buffer_ptr[i] = droq->recv_buf_list[idx].buffer; in lio_create_recv_info()
397 droq->recv_buf_list[idx].buffer = NULL; in lio_create_recv_info()
399 idx = lio_incr_index(idx, 1, droq->max_count); in lio_create_recv_info()
400 bytes_left -= droq->buffer_size; in lio_create_recv_info()
402 buf_cnt--; in lio_create_recv_info()
417 uint32_t refill_index = droq->refill_idx; in lio_droq_refill_pullup_descs()
419 while (refill_index != droq->read_idx) { in lio_droq_refill_pullup_descs()
420 if (droq->recv_buf_list[refill_index].buffer != NULL) { in lio_droq_refill_pullup_descs()
421 droq->recv_buf_list[droq->refill_idx].buffer = in lio_droq_refill_pullup_descs()
422 droq->recv_buf_list[refill_index].buffer; in lio_droq_refill_pullup_descs()
423 droq->recv_buf_list[droq->refill_idx].data = in lio_droq_refill_pullup_descs()
424 droq->recv_buf_list[refill_index].data; in lio_droq_refill_pullup_descs()
425 desc_ring[droq->refill_idx].buffer_ptr = in lio_droq_refill_pullup_descs()
427 droq->recv_buf_list[refill_index].buffer = NULL; in lio_droq_refill_pullup_descs()
430 droq->refill_idx = in lio_droq_refill_pullup_descs()
431 lio_incr_index(droq->refill_idx, 1, in lio_droq_refill_pullup_descs()
432 droq->max_count); in lio_droq_refill_pullup_descs()
434 droq->refill_count--; in lio_droq_refill_pullup_descs()
435 } while (droq->recv_buf_list[droq->refill_idx].buffer != in lio_droq_refill_pullup_descs()
438 refill_index = lio_incr_index(refill_index, 1, droq->max_count); in lio_droq_refill_pullup_descs()
446 * droq - droq in which descriptors require new buffers.
448 * Called during normal DROQ processing in interrupt mode or by the poll
455 * This routine is called with droq->lock held.
465 desc_ring = droq->desc_ring; in lio_droq_refill()
467 while (droq->refill_count && (desc_refilled < droq->max_count)) { in lio_droq_refill()
473 if (droq->recv_buf_list[droq->refill_idx].buffer == NULL) { in lio_droq_refill()
474 buf = lio_recv_buffer_alloc(droq->buffer_size); in lio_droq_refill()
480 droq->stats.rx_alloc_failure++; in lio_droq_refill()
484 droq->recv_buf_list[droq->refill_idx].buffer = buf; in lio_droq_refill()
485 data = ((struct mbuf *)buf)->m_data; in lio_droq_refill()
487 data = ((struct mbuf *)droq->recv_buf_list in lio_droq_refill()
488 [droq->refill_idx].buffer)->m_data; in lio_droq_refill()
491 droq->recv_buf_list[droq->refill_idx].data = data; in lio_droq_refill()
493 desc_ring[droq->refill_idx].buffer_ptr = in lio_droq_refill()
494 lio_map_ring(octeon_dev->device, in lio_droq_refill()
495 droq->recv_buf_list[droq->refill_idx].buffer, in lio_droq_refill()
496 droq->buffer_size); in lio_droq_refill()
498 droq->refill_idx = lio_incr_index(droq->refill_idx, 1, in lio_droq_refill()
499 droq->max_count); in lio_droq_refill()
501 droq->refill_count--; in lio_droq_refill()
504 if (droq->refill_count) in lio_droq_refill()
508 * if droq->refill_count in lio_droq_refill()
520 return ((total_len + buf_size - 1) / buf_size); in lio_droq_get_bufcount()
529 uint32_t cnt; in lio_droq_dispatch_pkt() local
531 cnt = lio_droq_get_bufcount(droq->buffer_size, (uint32_t)info->length); in lio_droq_dispatch_pkt()
533 disp_fn = lio_get_dispatch(oct, (uint16_t)rh->r.opcode, in lio_droq_dispatch_pkt()
534 (uint16_t)rh->r.subcode); in lio_droq_dispatch_pkt()
536 rinfo = lio_create_recv_info(oct, droq, cnt, droq->read_idx); in lio_droq_dispatch_pkt()
538 struct __dispatch *rdisp = rinfo->rsvd; in lio_droq_dispatch_pkt()
540 rdisp->rinfo = rinfo; in lio_droq_dispatch_pkt()
541 rdisp->disp_fn = disp_fn; in lio_droq_dispatch_pkt()
542 rinfo->recv_pkt->rh = *rh; in lio_droq_dispatch_pkt()
543 STAILQ_INSERT_TAIL(&droq->dispatch_stq_head, in lio_droq_dispatch_pkt()
544 &rdisp->node, entries); in lio_droq_dispatch_pkt()
546 droq->stats.dropped_nomem++; in lio_droq_dispatch_pkt()
550 (unsigned int)rh->r.opcode, in lio_droq_dispatch_pkt()
551 (unsigned int)rh->r.subcode); in lio_droq_dispatch_pkt()
552 droq->stats.dropped_nodispatch++; in lio_droq_dispatch_pkt()
555 return (cnt); in lio_droq_dispatch_pkt()
560 uint32_t cnt) in lio_droq_drop_packets() argument
565 for (i = 0; i < cnt; i++) { in lio_droq_drop_packets()
567 droq->recv_buf_list[droq->read_idx].data; in lio_droq_drop_packets()
571 if (info->length) { in lio_droq_drop_packets()
572 info->length += 8; in lio_droq_drop_packets()
573 droq->stats.bytes_received += info->length; in lio_droq_drop_packets()
574 buf_cnt = lio_droq_get_bufcount(droq->buffer_size, in lio_droq_drop_packets()
575 (uint32_t)info->length); in lio_droq_drop_packets()
581 droq->read_idx = lio_incr_index(droq->read_idx, buf_cnt, in lio_droq_drop_packets()
582 droq->max_count); in lio_droq_drop_packets()
583 droq->refill_count += buf_cnt; in lio_droq_drop_packets()
602 droq->recv_buf_list[droq->read_idx].data; in lio_droq_fast_process_packets()
606 if (!info->length) { in lio_droq_fast_process_packets()
609 droq->q_no, droq->read_idx, pkt_count); in lio_droq_fast_process_packets()
613 lio_incr_index(droq->read_idx, 1, droq->max_count); in lio_droq_fast_process_packets()
614 droq->refill_count++; in lio_droq_fast_process_packets()
618 rh = &info->rh; in lio_droq_fast_process_packets()
620 info->length += 8; in lio_droq_fast_process_packets()
621 rh->r_dh.len += (LIO_DROQ_INFO_SIZE + 7) / 8; in lio_droq_fast_process_packets()
623 total_len += (uint32_t)info->length; in lio_droq_fast_process_packets()
628 droq->read_idx = lio_incr_index(droq->read_idx, buf_cnt, in lio_droq_fast_process_packets()
629 droq->max_count); in lio_droq_fast_process_packets()
630 droq->refill_count += buf_cnt; in lio_droq_fast_process_packets()
632 if (info->length <= droq->buffer_size) { in lio_droq_fast_process_packets()
633 pkt_len = (uint32_t)info->length; in lio_droq_fast_process_packets()
634 nicbuf = droq->recv_buf_list[ in lio_droq_fast_process_packets()
635 droq->read_idx].buffer; in lio_droq_fast_process_packets()
636 nicbuf->m_len = pkt_len; in lio_droq_fast_process_packets()
637 droq->recv_buf_list[droq->read_idx].buffer = in lio_droq_fast_process_packets()
640 droq->read_idx = in lio_droq_fast_process_packets()
641 lio_incr_index(droq->read_idx, in lio_droq_fast_process_packets()
642 1, droq->max_count); in lio_droq_fast_process_packets()
643 droq->refill_count++; in lio_droq_fast_process_packets()
649 while (pkt_len < info->length) { in lio_droq_fast_process_packets()
650 int frag_len, idx = droq->read_idx; in lio_droq_fast_process_packets()
654 ((pkt_len + droq->buffer_size) > in lio_droq_fast_process_packets()
655 info->length) ? in lio_droq_fast_process_packets()
656 ((uint32_t)info->length - in lio_droq_fast_process_packets()
657 pkt_len) : droq->buffer_size; in lio_droq_fast_process_packets()
660 droq->recv_buf_list[idx]. in lio_droq_fast_process_packets()
662 buffer->m_len = frag_len; in lio_droq_fast_process_packets()
670 droq->recv_buf_list[droq->read_idx]. in lio_droq_fast_process_packets()
674 droq->read_idx = in lio_droq_fast_process_packets()
675 lio_incr_index(droq->read_idx, in lio_droq_fast_process_packets()
677 droq->max_count); in lio_droq_fast_process_packets()
678 droq->refill_count++; in lio_droq_fast_process_packets()
683 if (droq->ops.fptr != NULL) { in lio_droq_fast_process_packets()
684 droq->ops.fptr(nicbuf, pkt_len, rh, in lio_droq_fast_process_packets()
685 droq, droq->ops.farg); in lio_droq_fast_process_packets()
692 if (droq->refill_count >= droq->refill_threshold) { in lio_droq_fast_process_packets()
701 lio_write_csr32(oct, droq->pkts_credit_reg, in lio_droq_fast_process_packets()
709 droq->stats.pkts_received += pkt; in lio_droq_fast_process_packets()
710 droq->stats.bytes_received += total_len; in lio_droq_fast_process_packets()
712 tcp_lro_flush_all(&droq->lro); in lio_droq_fast_process_packets()
714 if ((droq->ops.drop_on_max) && (pkts_to_process - pkt)) { in lio_droq_fast_process_packets()
715 lio_droq_drop_packets(oct, droq, (pkts_to_process - pkt)); in lio_droq_fast_process_packets()
717 droq->stats.dropped_toomany += (pkts_to_process - pkt); in lio_droq_fast_process_packets()
732 mtx_lock(&droq->lock); in lio_droq_process_packets()
735 pkt_count = atomic_load_acq_int(&droq->pkts_pending); in lio_droq_process_packets()
738 mtx_unlock(&droq->lock); in lio_droq_process_packets()
746 atomic_subtract_int(&droq->pkts_pending, pkts_processed); in lio_droq_process_packets()
749 mtx_unlock(&droq->lock); in lio_droq_process_packets()
751 STAILQ_FOREACH_SAFE(tmp, &droq->dispatch_stq_head, entries, tmp2) { in lio_droq_process_packets()
754 STAILQ_REMOVE_HEAD(&droq->dispatch_stq_head, entries); in lio_droq_process_packets()
755 rdisp->disp_fn(rdisp->rinfo, lio_get_dispatch_arg(oct, in lio_droq_process_packets()
756 (uint16_t)rdisp->rinfo->recv_pkt->rh.r.opcode, in lio_droq_process_packets()
757 (uint16_t)rdisp->rinfo->recv_pkt->rh.r.subcode)); in lio_droq_process_packets()
761 if (atomic_load_acq_int(&droq->pkts_pending)) in lio_droq_process_packets()
777 return (-EINVAL); in lio_register_droq_ops()
781 return (-EINVAL); in lio_register_droq_ops()
786 __func__, q_no, (oct->num_oqs - 1)); in lio_register_droq_ops()
787 return (-EINVAL); in lio_register_droq_ops()
789 droq = oct->droq[q_no]; in lio_register_droq_ops()
791 mtx_lock(&droq->lock); in lio_register_droq_ops()
793 memcpy(&droq->ops, ops, sizeof(struct lio_droq_ops)); in lio_register_droq_ops()
795 mtx_unlock(&droq->lock); in lio_register_droq_ops()
809 return (-EINVAL); in lio_unregister_droq_ops()
813 __func__, q_no, oct->num_oqs - 1); in lio_unregister_droq_ops()
814 return (-EINVAL); in lio_unregister_droq_ops()
817 droq = oct->droq[q_no]; in lio_unregister_droq_ops()
824 mtx_lock(&droq->lock); in lio_unregister_droq_ops()
826 droq->ops.fptr = NULL; in lio_unregister_droq_ops()
827 droq->ops.farg = NULL; in lio_unregister_droq_ops()
828 droq->ops.drop_on_max = 0; in lio_unregister_droq_ops()
830 mtx_unlock(&droq->lock); in lio_unregister_droq_ops()
840 if (oct->droq[q_no]->oct_dev != NULL) { in lio_create_droq()
848 bzero(oct->droq[q_no], sizeof(struct lio_droq)); in lio_create_droq()
852 oct->num_oqs++; in lio_create_droq()
855 oct->num_oqs); in lio_create_droq()
866 return (-ENOMEM); in lio_create_droq()