Lines Matching +full:rx +full:- +full:status +full:- +full:len
1 // SPDX-License-Identifier: GPL-2.0+
24 #include "mlxbf-tmfifo-regs.h"
35 /* House-keeping timer interval. */
53 /* ACPI UID for BlueField-3. */
59 * struct mlxbf_tmfifo_vring - Structure of the TmFifo virtual ring
101 r->desc_head == &r->drop_desc; })
115 /* Ring types (Rx & Tx). */
123 * struct mlxbf_tmfifo_vdev - Structure of the TmFifo virtual device
126 * @status: status of the device
129 * @config: non-anonymous union for cons and net
130 * @config.cons: virtual console config -
132 * @config.net: virtual network config -
138 u8 status; member
149 * struct mlxbf_tmfifo_irq_info - Structure of the interrupt information
161 * struct mlxbf_tmfifo_io - Structure of the TmFifo IO resource (for both rx & tx)
163 * @sts: status register offset (TMFIFO_RX_STS / TMFIFO_TX_STS)
173 * struct mlxbf_tmfifo - Structure of the TmFifo
178 * @rx: rx io resource
180 * @rx_fifo_size: number of entries of the Rx FIFO
186 * @vring: Tx/Rx ring
187 * @spin_lock: Tx/Rx spin lock
195 struct mlxbf_tmfifo_io rx; member
209 * struct mlxbf_tmfifo_msg_hdr - Structure of the TmFifo message header
211 * @len: payload length in network byte order. Messages sent into the FIFO
218 __be16 len; member
238 /* Supported virtio-net features. */
252 for (i = 0; i < ARRAY_SIZE(tm_vdev->vrings); i++) { in mlxbf_tmfifo_free_vrings()
253 vring = &tm_vdev->vrings[i]; in mlxbf_tmfifo_free_vrings()
254 if (vring->va) { in mlxbf_tmfifo_free_vrings()
255 size = vring_size(vring->num, vring->align); in mlxbf_tmfifo_free_vrings()
256 dma_free_coherent(tm_vdev->vdev.dev.parent, size, in mlxbf_tmfifo_free_vrings()
257 vring->va, vring->dma); in mlxbf_tmfifo_free_vrings()
258 vring->va = NULL; in mlxbf_tmfifo_free_vrings()
259 if (vring->vq) { in mlxbf_tmfifo_free_vrings()
260 vring_del_virtqueue(vring->vq); in mlxbf_tmfifo_free_vrings()
261 vring->vq = NULL; in mlxbf_tmfifo_free_vrings()
277 for (i = 0; i < ARRAY_SIZE(tm_vdev->vrings); i++) { in mlxbf_tmfifo_alloc_vrings()
278 vring = &tm_vdev->vrings[i]; in mlxbf_tmfifo_alloc_vrings()
279 vring->fifo = fifo; in mlxbf_tmfifo_alloc_vrings()
280 vring->num = MLXBF_TMFIFO_VRING_SIZE; in mlxbf_tmfifo_alloc_vrings()
281 vring->align = SMP_CACHE_BYTES; in mlxbf_tmfifo_alloc_vrings()
282 vring->index = i; in mlxbf_tmfifo_alloc_vrings()
283 vring->vdev_id = tm_vdev->vdev.id.device; in mlxbf_tmfifo_alloc_vrings()
284 vring->drop_desc.len = cpu_to_virtio32(&tm_vdev->vdev, in mlxbf_tmfifo_alloc_vrings()
286 dev = &tm_vdev->vdev.dev; in mlxbf_tmfifo_alloc_vrings()
288 size = vring_size(vring->num, vring->align); in mlxbf_tmfifo_alloc_vrings()
289 va = dma_alloc_coherent(dev->parent, size, &dma, GFP_KERNEL); in mlxbf_tmfifo_alloc_vrings()
292 dev_err(dev->parent, "dma_alloc_coherent failed\n"); in mlxbf_tmfifo_alloc_vrings()
293 return -ENOMEM; in mlxbf_tmfifo_alloc_vrings()
296 vring->va = va; in mlxbf_tmfifo_alloc_vrings()
297 vring->dma = dma; in mlxbf_tmfifo_alloc_vrings()
309 irq = fifo->irq_info[i].irq; in mlxbf_tmfifo_disable_irqs()
310 fifo->irq_info[i].irq = 0; in mlxbf_tmfifo_disable_irqs()
320 if (!test_and_set_bit(irq_info->index, &irq_info->fifo->pend_events)) in mlxbf_tmfifo_irq_handler()
321 schedule_work(&irq_info->fifo->work); in mlxbf_tmfifo_irq_handler()
330 const struct vring *vr = virtqueue_get_vring(vring->vq); in mlxbf_tmfifo_get_next_desc()
331 struct virtio_device *vdev = vring->vq->vdev; in mlxbf_tmfifo_get_next_desc()
334 if (vring->next_avail == virtio16_to_cpu(vdev, vr->avail->idx)) in mlxbf_tmfifo_get_next_desc()
337 /* Make sure 'avail->idx' is visible already. */ in mlxbf_tmfifo_get_next_desc()
340 idx = vring->next_avail % vr->num; in mlxbf_tmfifo_get_next_desc()
341 head = virtio16_to_cpu(vdev, vr->avail->ring[idx]); in mlxbf_tmfifo_get_next_desc()
342 if (WARN_ON(head >= vr->num)) in mlxbf_tmfifo_get_next_desc()
345 vring->next_avail++; in mlxbf_tmfifo_get_next_desc()
347 return &vr->desc[head]; in mlxbf_tmfifo_get_next_desc()
352 struct vring_desc *desc, u32 len) in mlxbf_tmfifo_release_desc() argument
354 const struct vring *vr = virtqueue_get_vring(vring->vq); in mlxbf_tmfifo_release_desc()
355 struct virtio_device *vdev = vring->vq->vdev; in mlxbf_tmfifo_release_desc()
358 vr_idx = virtio16_to_cpu(vdev, vr->used->idx); in mlxbf_tmfifo_release_desc()
359 idx = vr_idx % vr->num; in mlxbf_tmfifo_release_desc()
360 vr->used->ring[idx].id = cpu_to_virtio32(vdev, desc - vr->desc); in mlxbf_tmfifo_release_desc()
361 vr->used->ring[idx].len = cpu_to_virtio32(vdev, len); in mlxbf_tmfifo_release_desc()
369 vr->used->idx = cpu_to_virtio16(vdev, vr_idx + 1); in mlxbf_tmfifo_release_desc()
376 const struct vring *vr = virtqueue_get_vring(vring->vq); in mlxbf_tmfifo_get_pkt_len()
377 struct virtio_device *vdev = vring->vq->vdev; in mlxbf_tmfifo_get_pkt_len()
378 u32 len = 0, idx; in mlxbf_tmfifo_get_pkt_len() local
381 len += virtio32_to_cpu(vdev, desc->len); in mlxbf_tmfifo_get_pkt_len()
382 if (!(virtio16_to_cpu(vdev, desc->flags) & VRING_DESC_F_NEXT)) in mlxbf_tmfifo_get_pkt_len()
384 idx = virtio16_to_cpu(vdev, desc->next); in mlxbf_tmfifo_get_pkt_len()
385 desc = &vr->desc[idx]; in mlxbf_tmfifo_get_pkt_len()
388 return len; in mlxbf_tmfifo_get_pkt_len()
394 u32 len = 0; in mlxbf_tmfifo_release_pkt() local
396 if (vring->desc_head) { in mlxbf_tmfifo_release_pkt()
397 desc_head = vring->desc_head; in mlxbf_tmfifo_release_pkt()
398 len = vring->pkt_len; in mlxbf_tmfifo_release_pkt()
401 len = mlxbf_tmfifo_get_pkt_len(vring, desc_head); in mlxbf_tmfifo_release_pkt()
405 mlxbf_tmfifo_release_desc(vring, desc_head, len); in mlxbf_tmfifo_release_pkt()
407 vring->pkt_len = 0; in mlxbf_tmfifo_release_pkt()
408 vring->desc = NULL; in mlxbf_tmfifo_release_pkt()
409 vring->desc_head = NULL; in mlxbf_tmfifo_release_pkt()
415 struct virtio_device *vdev = vring->vq->vdev; in mlxbf_tmfifo_init_net_desc()
418 net_hdr = phys_to_virt(virtio64_to_cpu(vdev, desc->addr)); in mlxbf_tmfifo_init_net_desc()
429 if (desc && is_rx && vring->vdev_id == VIRTIO_ID_NET) in mlxbf_tmfifo_get_next_pkt()
432 vring->desc_head = desc; in mlxbf_tmfifo_get_next_pkt()
433 vring->desc = desc; in mlxbf_tmfifo_get_next_pkt()
438 /* House-keeping timer. */
442 int rx, tx; in mlxbf_tmfifo_timer() local
444 rx = !test_and_set_bit(MLXBF_TM_RX_HWM_IRQ, &fifo->pend_events); in mlxbf_tmfifo_timer()
445 tx = !test_and_set_bit(MLXBF_TM_TX_LWM_IRQ, &fifo->pend_events); in mlxbf_tmfifo_timer()
447 if (rx || tx) in mlxbf_tmfifo_timer()
448 schedule_work(&fifo->work); in mlxbf_tmfifo_timer()
450 mod_timer(&fifo->timer, jiffies + MLXBF_TMFIFO_TIMER_INTERVAL); in mlxbf_tmfifo_timer()
458 const struct vring *vr = virtqueue_get_vring(vring->vq); in mlxbf_tmfifo_console_output_one()
459 struct virtio_device *vdev = &cons->vdev; in mlxbf_tmfifo_console_output_one()
460 u32 len, idx, seg; in mlxbf_tmfifo_console_output_one() local
464 addr = phys_to_virt(virtio64_to_cpu(vdev, desc->addr)); in mlxbf_tmfifo_console_output_one()
465 len = virtio32_to_cpu(vdev, desc->len); in mlxbf_tmfifo_console_output_one()
467 seg = CIRC_SPACE_TO_END(cons->tx_buf.head, cons->tx_buf.tail, in mlxbf_tmfifo_console_output_one()
469 if (len <= seg) { in mlxbf_tmfifo_console_output_one()
470 memcpy(cons->tx_buf.buf + cons->tx_buf.head, addr, len); in mlxbf_tmfifo_console_output_one()
472 memcpy(cons->tx_buf.buf + cons->tx_buf.head, addr, seg); in mlxbf_tmfifo_console_output_one()
474 memcpy(cons->tx_buf.buf, addr, len - seg); in mlxbf_tmfifo_console_output_one()
476 cons->tx_buf.head = (cons->tx_buf.head + len) % in mlxbf_tmfifo_console_output_one()
479 if (!(virtio16_to_cpu(vdev, desc->flags) & VRING_DESC_F_NEXT)) in mlxbf_tmfifo_console_output_one()
481 idx = virtio16_to_cpu(vdev, desc->next); in mlxbf_tmfifo_console_output_one()
482 desc = &vr->desc[idx]; in mlxbf_tmfifo_console_output_one()
491 u32 len, avail; in mlxbf_tmfifo_console_output() local
496 len = mlxbf_tmfifo_get_pkt_len(vring, desc); in mlxbf_tmfifo_console_output()
497 avail = CIRC_SPACE(cons->tx_buf.head, cons->tx_buf.tail, in mlxbf_tmfifo_console_output()
499 if (len + MLXBF_TMFIFO_CON_TX_BUF_RSV_SIZE > avail) { in mlxbf_tmfifo_console_output()
500 mlxbf_tmfifo_release_desc(vring, desc, len); in mlxbf_tmfifo_console_output()
505 mlxbf_tmfifo_release_desc(vring, desc, len); in mlxbf_tmfifo_console_output()
510 /* Get the number of available words in Rx FIFO for receiving. */
515 sts = readq(fifo->rx.sts); in mlxbf_tmfifo_get_rx_avail()
528 tx_reserve = fifo->tx_fifo_size / MLXBF_TMFIFO_RESERVE_RATIO; in mlxbf_tmfifo_get_tx_avail()
532 sts = readq(fifo->tx.sts); in mlxbf_tmfifo_get_tx_avail()
534 return fifo->tx_fifo_size - tx_reserve - count; in mlxbf_tmfifo_get_tx_avail()
551 cons = fifo->vdev[VIRTIO_ID_CONSOLE]; in mlxbf_tmfifo_console_tx()
552 if (!cons || !cons->tx_buf.buf) in mlxbf_tmfifo_console_tx()
556 size = CIRC_CNT(cons->tx_buf.head, cons->tx_buf.tail, in mlxbf_tmfifo_console_tx()
563 size = avail * sizeof(u64) - sizeof(hdr); in mlxbf_tmfifo_console_tx()
567 hdr.len = htons(size); in mlxbf_tmfifo_console_tx()
568 writeq(*(u64 *)&hdr, fifo->tx.data); in mlxbf_tmfifo_console_tx()
570 /* Use spin-lock to protect the 'cons->tx_buf'. */ in mlxbf_tmfifo_console_tx()
571 spin_lock_irqsave(&fifo->spin_lock[0], flags); in mlxbf_tmfifo_console_tx()
574 addr = cons->tx_buf.buf + cons->tx_buf.tail; in mlxbf_tmfifo_console_tx()
576 seg = CIRC_CNT_TO_END(cons->tx_buf.head, cons->tx_buf.tail, in mlxbf_tmfifo_console_tx()
582 memcpy((u8 *)&data + seg, cons->tx_buf.buf, in mlxbf_tmfifo_console_tx()
583 sizeof(u64) - seg); in mlxbf_tmfifo_console_tx()
585 writeq(data, fifo->tx.data); in mlxbf_tmfifo_console_tx()
588 cons->tx_buf.tail = (cons->tx_buf.tail + sizeof(u64)) % in mlxbf_tmfifo_console_tx()
590 size -= sizeof(u64); in mlxbf_tmfifo_console_tx()
592 cons->tx_buf.tail = (cons->tx_buf.tail + size) % in mlxbf_tmfifo_console_tx()
598 spin_unlock_irqrestore(&fifo->spin_lock[0], flags); in mlxbf_tmfifo_console_tx()
601 /* Rx/Tx one word in the descriptor buffer. */
604 bool is_rx, int len) in mlxbf_tmfifo_rxtx_word() argument
606 struct virtio_device *vdev = vring->vq->vdev; in mlxbf_tmfifo_rxtx_word()
607 struct mlxbf_tmfifo *fifo = vring->fifo; in mlxbf_tmfifo_rxtx_word()
612 addr = phys_to_virt(virtio64_to_cpu(vdev, desc->addr)); in mlxbf_tmfifo_rxtx_word()
614 /* Read a word from FIFO for Rx. */ in mlxbf_tmfifo_rxtx_word()
616 data = readq(fifo->rx.data); in mlxbf_tmfifo_rxtx_word()
618 if (vring->cur_len + sizeof(u64) <= len) { in mlxbf_tmfifo_rxtx_word()
622 memcpy(addr + vring->cur_len, &data, in mlxbf_tmfifo_rxtx_word()
625 memcpy(&data, addr + vring->cur_len, in mlxbf_tmfifo_rxtx_word()
628 vring->cur_len += sizeof(u64); in mlxbf_tmfifo_rxtx_word()
633 memcpy(addr + vring->cur_len, &data, in mlxbf_tmfifo_rxtx_word()
634 len - vring->cur_len); in mlxbf_tmfifo_rxtx_word()
637 memcpy(&data, addr + vring->cur_len, in mlxbf_tmfifo_rxtx_word()
638 len - vring->cur_len); in mlxbf_tmfifo_rxtx_word()
640 vring->cur_len = len; in mlxbf_tmfifo_rxtx_word()
645 writeq(data, fifo->tx.data); in mlxbf_tmfifo_rxtx_word()
649 * Rx/Tx packet header.
651 * In Rx case, the packet might be found to belong to a different vring since
659 struct mlxbf_tmfifo *fifo = vring->fifo; in mlxbf_tmfifo_rxtx_header()
668 *(u64 *)&hdr = readq(fifo->rx.data); in mlxbf_tmfifo_rxtx_header()
671 if (hdr.len == 0) in mlxbf_tmfifo_rxtx_header()
678 config = &fifo->vdev[vdev_id]->config.net; in mlxbf_tmfifo_rxtx_header()
679 /* A legacy-only interface for now. */ in mlxbf_tmfifo_rxtx_header()
680 if (ntohs(hdr.len) > in mlxbf_tmfifo_rxtx_header()
682 config->mtu) + in mlxbf_tmfifo_rxtx_header()
694 if (vdev_id != vring->vdev_id) { in mlxbf_tmfifo_rxtx_header()
695 struct mlxbf_tmfifo_vdev *tm_dev2 = fifo->vdev[vdev_id]; in mlxbf_tmfifo_rxtx_header()
699 vring->desc = *desc; in mlxbf_tmfifo_rxtx_header()
700 vring = &tm_dev2->vrings[MLXBF_TMFIFO_VRING_RX]; in mlxbf_tmfifo_rxtx_header()
705 if (vring->desc_head) in mlxbf_tmfifo_rxtx_header()
707 *desc = &vring->drop_desc; in mlxbf_tmfifo_rxtx_header()
708 vring->desc_head = *desc; in mlxbf_tmfifo_rxtx_header()
709 vring->desc = *desc; in mlxbf_tmfifo_rxtx_header()
712 vring->pkt_len = ntohs(hdr.len) + hdr_len; in mlxbf_tmfifo_rxtx_header()
715 hdr_len = (vring->vdev_id == VIRTIO_ID_NET) ? in mlxbf_tmfifo_rxtx_header()
717 vring->pkt_len = mlxbf_tmfifo_get_pkt_len(vring, *desc); in mlxbf_tmfifo_rxtx_header()
718 hdr.type = (vring->vdev_id == VIRTIO_ID_NET) ? in mlxbf_tmfifo_rxtx_header()
720 hdr.len = htons(vring->pkt_len - hdr_len); in mlxbf_tmfifo_rxtx_header()
721 writeq(*(u64 *)&hdr, fifo->tx.data); in mlxbf_tmfifo_rxtx_header()
724 vring->cur_len = hdr_len; in mlxbf_tmfifo_rxtx_header()
725 vring->rem_len = vring->pkt_len; in mlxbf_tmfifo_rxtx_header()
726 fifo->vring[is_rx] = vring; in mlxbf_tmfifo_rxtx_header()
730 * Rx/Tx one descriptor.
737 const struct vring *vr = virtqueue_get_vring(vring->vq); in mlxbf_tmfifo_rxtx_one_desc()
738 struct mlxbf_tmfifo *fifo = vring->fifo; in mlxbf_tmfifo_rxtx_one_desc()
743 u32 len, idx; in mlxbf_tmfifo_rxtx_one_desc() local
745 vdev = &fifo->vdev[vring->vdev_id]->vdev; in mlxbf_tmfifo_rxtx_one_desc()
748 if (!vring->desc) { in mlxbf_tmfifo_rxtx_one_desc()
751 /* Drop next Rx packet to avoid stuck. */ in mlxbf_tmfifo_rxtx_one_desc()
753 desc = &vring->drop_desc; in mlxbf_tmfifo_rxtx_one_desc()
754 vring->desc_head = desc; in mlxbf_tmfifo_rxtx_one_desc()
755 vring->desc = desc; in mlxbf_tmfifo_rxtx_one_desc()
761 desc = vring->desc; in mlxbf_tmfifo_rxtx_one_desc()
764 /* Beginning of a packet. Start to Rx/Tx packet header. */ in mlxbf_tmfifo_rxtx_one_desc()
765 if (vring->pkt_len == 0) { in mlxbf_tmfifo_rxtx_one_desc()
767 (*avail)--; in mlxbf_tmfifo_rxtx_one_desc()
776 len = virtio32_to_cpu(vdev, desc->len); in mlxbf_tmfifo_rxtx_one_desc()
777 if (len > vring->rem_len) in mlxbf_tmfifo_rxtx_one_desc()
778 len = vring->rem_len; in mlxbf_tmfifo_rxtx_one_desc()
780 /* Rx/Tx one word (8 bytes) if not done. */ in mlxbf_tmfifo_rxtx_one_desc()
781 if (vring->cur_len < len) { in mlxbf_tmfifo_rxtx_one_desc()
782 mlxbf_tmfifo_rxtx_word(vring, desc, is_rx, len); in mlxbf_tmfifo_rxtx_one_desc()
783 (*avail)--; in mlxbf_tmfifo_rxtx_one_desc()
787 if (vring->cur_len == len) { in mlxbf_tmfifo_rxtx_one_desc()
788 vring->cur_len = 0; in mlxbf_tmfifo_rxtx_one_desc()
789 vring->rem_len -= len; in mlxbf_tmfifo_rxtx_one_desc()
792 if (!IS_VRING_DROP(vring) && vring->rem_len > 0 && in mlxbf_tmfifo_rxtx_one_desc()
793 (virtio16_to_cpu(vdev, desc->flags) & VRING_DESC_F_NEXT)) { in mlxbf_tmfifo_rxtx_one_desc()
794 idx = virtio16_to_cpu(vdev, desc->next); in mlxbf_tmfifo_rxtx_one_desc()
795 desc = &vr->desc[idx]; in mlxbf_tmfifo_rxtx_one_desc()
801 fifo->vring[is_rx] = NULL; in mlxbf_tmfifo_rxtx_one_desc()
805 vring->pkt_len = 0; in mlxbf_tmfifo_rxtx_one_desc()
806 vring->desc_head = NULL; in mlxbf_tmfifo_rxtx_one_desc()
807 vring->desc = NULL; in mlxbf_tmfifo_rxtx_one_desc()
818 spin_lock_irqsave(&fifo->spin_lock[is_rx], flags); in mlxbf_tmfifo_rxtx_one_desc()
819 vring_interrupt(0, vring->vq); in mlxbf_tmfifo_rxtx_one_desc()
820 spin_unlock_irqrestore(&fifo->spin_lock[is_rx], flags); in mlxbf_tmfifo_rxtx_one_desc()
825 vring->desc = desc; in mlxbf_tmfifo_rxtx_one_desc()
835 if (vring->vdev_id != VIRTIO_ID_NET) in mlxbf_tmfifo_check_tx_timeout()
839 if (!vring->tx_timeout) { in mlxbf_tmfifo_check_tx_timeout()
841 vring->tx_timeout = jiffies + in mlxbf_tmfifo_check_tx_timeout()
844 } else if (time_before(jiffies, vring->tx_timeout)) { in mlxbf_tmfifo_check_tx_timeout()
856 vring->rem_padding = round_up(vring->rem_len, sizeof(u64)); in mlxbf_tmfifo_check_tx_timeout()
858 vring->cur_len = 0; in mlxbf_tmfifo_check_tx_timeout()
859 vring->rem_len = 0; in mlxbf_tmfifo_check_tx_timeout()
860 vring->fifo->vring[0] = NULL; in mlxbf_tmfifo_check_tx_timeout()
869 spin_lock_irqsave(&vring->fifo->spin_lock[0], flags); in mlxbf_tmfifo_check_tx_timeout()
870 vring_interrupt(0, vring->vq); in mlxbf_tmfifo_check_tx_timeout()
871 spin_unlock_irqrestore(&vring->fifo->spin_lock[0], flags); in mlxbf_tmfifo_check_tx_timeout()
874 /* Rx & Tx processing of a queue. */
877 int avail = 0, devid = vring->vdev_id; in mlxbf_tmfifo_rxtx()
881 fifo = vring->fifo; in mlxbf_tmfifo_rxtx()
884 if (!fifo || !fifo->vdev[devid]) in mlxbf_tmfifo_rxtx()
888 if (fifo->vring[is_rx] && fifo->vring[is_rx] != vring) in mlxbf_tmfifo_rxtx()
909 vring->tx_timeout = 0; in mlxbf_tmfifo_rxtx()
910 while (vring->rem_padding >= sizeof(u64)) { in mlxbf_tmfifo_rxtx()
911 writeq(0, vring->fifo->tx.data); in mlxbf_tmfifo_rxtx()
912 vring->rem_padding -= sizeof(u64); in mlxbf_tmfifo_rxtx()
913 if (--avail == 0) in mlxbf_tmfifo_rxtx()
933 /* Handle Rx or Tx queues. */
941 if (!test_and_clear_bit(irq_id, &fifo->pend_events) || in mlxbf_tmfifo_work_rxtx()
942 !fifo->irq_info[irq_id].irq) in mlxbf_tmfifo_work_rxtx()
946 tm_vdev = fifo->vdev[i]; in mlxbf_tmfifo_work_rxtx()
948 vring = &tm_vdev->vrings[queue_id]; in mlxbf_tmfifo_work_rxtx()
949 if (vring->vq) in mlxbf_tmfifo_work_rxtx()
955 /* Work handler for Rx and Tx case. */
961 if (!fifo->is_ready) in mlxbf_tmfifo_work_handler()
964 mutex_lock(&fifo->lock); in mlxbf_tmfifo_work_handler()
970 /* Rx (Receive data from the TmFifo). */ in mlxbf_tmfifo_work_handler()
974 mutex_unlock(&fifo->lock); in mlxbf_tmfifo_work_handler()
980 struct mlxbf_tmfifo_vring *vring = vq->priv; in mlxbf_tmfifo_virtio_notify()
985 fifo = vring->fifo; in mlxbf_tmfifo_virtio_notify()
988 * Virtio maintains vrings in pairs, even number ring for Rx in mlxbf_tmfifo_virtio_notify()
991 if (vring->index & BIT(0)) { in mlxbf_tmfifo_virtio_notify()
998 if (vring->vdev_id == VIRTIO_ID_CONSOLE) { in mlxbf_tmfifo_virtio_notify()
999 spin_lock_irqsave(&fifo->spin_lock[0], flags); in mlxbf_tmfifo_virtio_notify()
1000 tm_vdev = fifo->vdev[VIRTIO_ID_CONSOLE]; in mlxbf_tmfifo_virtio_notify()
1002 spin_unlock_irqrestore(&fifo->spin_lock[0], flags); in mlxbf_tmfifo_virtio_notify()
1003 set_bit(MLXBF_TM_TX_LWM_IRQ, &fifo->pend_events); in mlxbf_tmfifo_virtio_notify()
1005 &fifo->pend_events)) { in mlxbf_tmfifo_virtio_notify()
1009 if (test_and_set_bit(MLXBF_TM_RX_HWM_IRQ, &fifo->pend_events)) in mlxbf_tmfifo_virtio_notify()
1013 schedule_work(&fifo->work); in mlxbf_tmfifo_virtio_notify()
1023 return tm_vdev->features; in mlxbf_tmfifo_virtio_get_features()
1031 tm_vdev->features = vdev->features; in mlxbf_tmfifo_virtio_finalize_features()
1044 for (i = 0; i < ARRAY_SIZE(tm_vdev->vrings); i++) { in mlxbf_tmfifo_virtio_del_vqs()
1045 vring = &tm_vdev->vrings[i]; in mlxbf_tmfifo_virtio_del_vqs()
1048 if (vring->desc) in mlxbf_tmfifo_virtio_del_vqs()
1050 vq = vring->vq; in mlxbf_tmfifo_virtio_del_vqs()
1052 vring->vq = NULL; in mlxbf_tmfifo_virtio_del_vqs()
1070 if (nvqs > ARRAY_SIZE(tm_vdev->vrings)) in mlxbf_tmfifo_virtio_find_vqs()
1071 return -EINVAL; in mlxbf_tmfifo_virtio_find_vqs()
1076 if (!vqi->name) { in mlxbf_tmfifo_virtio_find_vqs()
1077 ret = -EINVAL; in mlxbf_tmfifo_virtio_find_vqs()
1080 vring = &tm_vdev->vrings[i]; in mlxbf_tmfifo_virtio_find_vqs()
1083 size = vring_size(vring->num, vring->align); in mlxbf_tmfifo_virtio_find_vqs()
1084 memset(vring->va, 0, size); in mlxbf_tmfifo_virtio_find_vqs()
1085 vq = vring_new_virtqueue(i, vring->num, vring->align, vdev, in mlxbf_tmfifo_virtio_find_vqs()
1086 false, false, vring->va, in mlxbf_tmfifo_virtio_find_vqs()
1088 vqi->callback, vqi->name); in mlxbf_tmfifo_virtio_find_vqs()
1090 dev_err(&vdev->dev, "vring_new_virtqueue failed\n"); in mlxbf_tmfifo_virtio_find_vqs()
1091 ret = -ENOMEM; in mlxbf_tmfifo_virtio_find_vqs()
1095 vq->num_max = vring->num; in mlxbf_tmfifo_virtio_find_vqs()
1097 vq->priv = vring; in mlxbf_tmfifo_virtio_find_vqs()
1103 vring->vq = vq; in mlxbf_tmfifo_virtio_find_vqs()
1113 /* Read the status byte. */
1118 return tm_vdev->status; in mlxbf_tmfifo_virtio_get_status()
1121 /* Write the status byte. */
1123 u8 status) in mlxbf_tmfifo_virtio_set_status() argument
1127 tm_vdev->status = status; in mlxbf_tmfifo_virtio_set_status()
1135 tm_vdev->status = 0; in mlxbf_tmfifo_virtio_reset()
1142 unsigned int len) in mlxbf_tmfifo_virtio_get() argument
1146 if ((u64)offset + len > sizeof(tm_vdev->config)) in mlxbf_tmfifo_virtio_get()
1149 memcpy(buf, (u8 *)&tm_vdev->config + offset, len); in mlxbf_tmfifo_virtio_get()
1156 unsigned int len) in mlxbf_tmfifo_virtio_set() argument
1160 if ((u64)offset + len > sizeof(tm_vdev->config)) in mlxbf_tmfifo_virtio_set()
1163 memcpy((u8 *)&tm_vdev->config + offset, buf, len); in mlxbf_tmfifo_virtio_set()
1197 mutex_lock(&fifo->lock); in mlxbf_tmfifo_create_vdev()
1199 tm_vdev = fifo->vdev[vdev_id]; in mlxbf_tmfifo_create_vdev()
1202 ret = -EEXIST; in mlxbf_tmfifo_create_vdev()
1208 ret = -ENOMEM; in mlxbf_tmfifo_create_vdev()
1212 tm_vdev->vdev.id.device = vdev_id; in mlxbf_tmfifo_create_vdev()
1213 tm_vdev->vdev.config = &mlxbf_tmfifo_virtio_config_ops; in mlxbf_tmfifo_create_vdev()
1214 tm_vdev->vdev.dev.parent = dev; in mlxbf_tmfifo_create_vdev()
1215 tm_vdev->vdev.dev.release = tmfifo_virtio_dev_release; in mlxbf_tmfifo_create_vdev()
1216 tm_vdev->features = features; in mlxbf_tmfifo_create_vdev()
1218 memcpy(&tm_vdev->config, config, size); in mlxbf_tmfifo_create_vdev()
1222 ret = -ENOMEM; in mlxbf_tmfifo_create_vdev()
1228 tm_vdev->tx_buf.buf = devm_kmalloc(dev, in mlxbf_tmfifo_create_vdev()
1231 fifo->vdev[vdev_id] = tm_vdev; in mlxbf_tmfifo_create_vdev()
1234 ret = register_virtio_device(&tm_vdev->vdev); in mlxbf_tmfifo_create_vdev()
1241 mutex_unlock(&fifo->lock); in mlxbf_tmfifo_create_vdev()
1246 fifo->vdev[vdev_id] = NULL; in mlxbf_tmfifo_create_vdev()
1248 put_device(&tm_vdev->vdev.dev); in mlxbf_tmfifo_create_vdev()
1252 mutex_unlock(&fifo->lock); in mlxbf_tmfifo_create_vdev()
1261 mutex_lock(&fifo->lock); in mlxbf_tmfifo_delete_vdev()
1264 tm_vdev = fifo->vdev[vdev_id]; in mlxbf_tmfifo_delete_vdev()
1266 unregister_virtio_device(&tm_vdev->vdev); in mlxbf_tmfifo_delete_vdev()
1268 fifo->vdev[vdev_id] = NULL; in mlxbf_tmfifo_delete_vdev()
1271 mutex_unlock(&fifo->lock); in mlxbf_tmfifo_delete_vdev()
1297 ctl = readq(fifo->tx.ctl); in mlxbf_tmfifo_set_threshold()
1298 fifo->tx_fifo_size = in mlxbf_tmfifo_set_threshold()
1302 fifo->tx_fifo_size / 2); in mlxbf_tmfifo_set_threshold()
1305 fifo->tx_fifo_size - 1); in mlxbf_tmfifo_set_threshold()
1306 writeq(ctl, fifo->tx.ctl); in mlxbf_tmfifo_set_threshold()
1308 /* Get Rx FIFO size and set the low/high watermark. */ in mlxbf_tmfifo_set_threshold()
1309 ctl = readq(fifo->rx.ctl); in mlxbf_tmfifo_set_threshold()
1310 fifo->rx_fifo_size = in mlxbf_tmfifo_set_threshold()
1316 writeq(ctl, fifo->rx.ctl); in mlxbf_tmfifo_set_threshold()
1323 fifo->is_ready = false; in mlxbf_tmfifo_cleanup()
1324 timer_delete_sync(&fifo->timer); in mlxbf_tmfifo_cleanup()
1326 cancel_work_sync(&fifo->work); in mlxbf_tmfifo_cleanup()
1335 struct device *dev = &pdev->dev; in mlxbf_tmfifo_probe()
1348 return -ENOMEM; in mlxbf_tmfifo_probe()
1350 spin_lock_init(&fifo->spin_lock[0]); in mlxbf_tmfifo_probe()
1351 spin_lock_init(&fifo->spin_lock[1]); in mlxbf_tmfifo_probe()
1352 INIT_WORK(&fifo->work, mlxbf_tmfifo_work_handler); in mlxbf_tmfifo_probe()
1353 mutex_init(&fifo->lock); in mlxbf_tmfifo_probe()
1355 /* Get the resource of the Rx FIFO. */ in mlxbf_tmfifo_probe()
1356 fifo->res0 = devm_platform_ioremap_resource(pdev, 0); in mlxbf_tmfifo_probe()
1357 if (IS_ERR(fifo->res0)) in mlxbf_tmfifo_probe()
1358 return PTR_ERR(fifo->res0); in mlxbf_tmfifo_probe()
1361 fifo->res1 = devm_platform_ioremap_resource(pdev, 1); in mlxbf_tmfifo_probe()
1362 if (IS_ERR(fifo->res1)) in mlxbf_tmfifo_probe()
1363 return PTR_ERR(fifo->res1); in mlxbf_tmfifo_probe()
1366 fifo->rx.ctl = fifo->res1 + MLXBF_TMFIFO_RX_CTL_BF3; in mlxbf_tmfifo_probe()
1367 fifo->rx.sts = fifo->res1 + MLXBF_TMFIFO_RX_STS_BF3; in mlxbf_tmfifo_probe()
1368 fifo->rx.data = fifo->res0 + MLXBF_TMFIFO_RX_DATA_BF3; in mlxbf_tmfifo_probe()
1369 fifo->tx.ctl = fifo->res1 + MLXBF_TMFIFO_TX_CTL_BF3; in mlxbf_tmfifo_probe()
1370 fifo->tx.sts = fifo->res1 + MLXBF_TMFIFO_TX_STS_BF3; in mlxbf_tmfifo_probe()
1371 fifo->tx.data = fifo->res0 + MLXBF_TMFIFO_TX_DATA_BF3; in mlxbf_tmfifo_probe()
1373 fifo->rx.ctl = fifo->res0 + MLXBF_TMFIFO_RX_CTL; in mlxbf_tmfifo_probe()
1374 fifo->rx.sts = fifo->res0 + MLXBF_TMFIFO_RX_STS; in mlxbf_tmfifo_probe()
1375 fifo->rx.data = fifo->res0 + MLXBF_TMFIFO_RX_DATA; in mlxbf_tmfifo_probe()
1376 fifo->tx.ctl = fifo->res1 + MLXBF_TMFIFO_TX_CTL; in mlxbf_tmfifo_probe()
1377 fifo->tx.sts = fifo->res1 + MLXBF_TMFIFO_TX_STS; in mlxbf_tmfifo_probe()
1378 fifo->tx.data = fifo->res1 + MLXBF_TMFIFO_TX_DATA; in mlxbf_tmfifo_probe()
1383 timer_setup(&fifo->timer, mlxbf_tmfifo_timer, 0); in mlxbf_tmfifo_probe()
1386 fifo->irq_info[i].index = i; in mlxbf_tmfifo_probe()
1387 fifo->irq_info[i].fifo = fifo; in mlxbf_tmfifo_probe()
1388 fifo->irq_info[i].irq = platform_get_irq(pdev, i); in mlxbf_tmfifo_probe()
1389 rc = devm_request_irq(dev, fifo->irq_info[i].irq, in mlxbf_tmfifo_probe()
1391 "tmfifo", &fifo->irq_info[i]); in mlxbf_tmfifo_probe()
1394 fifo->irq_info[i].irq = 0; in mlxbf_tmfifo_probe()
1409 /* A legacy-only interface for now. */ in mlxbf_tmfifo_probe()
1412 net_config.status = __cpu_to_virtio16(virtio_legacy_is_little_endian(), in mlxbf_tmfifo_probe()
1421 mod_timer(&fifo->timer, jiffies + MLXBF_TMFIFO_TIMER_INTERVAL); in mlxbf_tmfifo_probe()
1426 fifo->is_ready = true; in mlxbf_tmfifo_probe()
1452 .name = "bf-tmfifo",