xref: /freebsd/sys/dev/gve/gve_rx_dqo.c (revision d438b4ef0cfc6986b93d0754f49ebf3ead50f269)

/*-
 * SPDX-License-Identifier: BSD-3-Clause
 *
 * Copyright (c) 2024 Google LLC
 *
 * Redistribution and use in source and binary forms, with or without modification,
 * are permitted provided that the following conditions are met:
 *
 * 1. Redistributions of source code must retain the above copyright notice, this
 *    list of conditions and the following disclaimer.
 *
 * 2. Redistributions in binary form must reproduce the above copyright notice,
 *    this list of conditions and the following disclaimer in the documentation
 *    and/or other materials provided with the distribution.
 *
 * 3. Neither the name of the copyright holder nor the names of its contributors
 *    may be used to endorse or promote products derived from this software without
 *    specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */
#include "gve.h"
#include "gve_adminq.h"
#include "gve_dqo.h"

static void
gve_free_rx_mbufs_dqo(struct gve_rx_ring *rx)
{
	struct gve_rx_buf_dqo *buf;
	int i;

	for (i = 0; i < rx->dqo.buf_cnt; i++) {
		buf = &rx->dqo.bufs[i];
		if (!buf->mbuf)
			continue;

		bus_dmamap_sync(rx->dqo.buf_dmatag, buf->dmamap,
		    BUS_DMASYNC_POSTREAD);
		bus_dmamap_unload(rx->dqo.buf_dmatag, buf->dmamap);
		m_freem(buf->mbuf);
		buf->mbuf = NULL;
	}
}

void
gve_rx_free_ring_dqo(struct gve_priv *priv, int i)
{
	struct gve_rx_ring *rx = &priv->rx[i];
	int j;

	if (rx->dqo.compl_ring != NULL) {
		gve_dma_free_coherent(&rx->dqo.compl_ring_mem);
		rx->dqo.compl_ring = NULL;
	}

	if (rx->dqo.desc_ring != NULL) {
		gve_dma_free_coherent(&rx->desc_ring_mem);
		rx->dqo.desc_ring = NULL;
	}

	if (rx->dqo.bufs != NULL) {
		gve_free_rx_mbufs_dqo(rx);

		if (rx->dqo.buf_dmatag) {
			for (j = 0; j < rx->dqo.buf_cnt; j++)
				if (rx->dqo.bufs[j].mapped)
					bus_dmamap_destroy(rx->dqo.buf_dmatag,
					    rx->dqo.bufs[j].dmamap);
		}

		free(rx->dqo.bufs, M_GVE);
		rx->dqo.bufs = NULL;
	}

	if (rx->dqo.buf_dmatag)
		bus_dma_tag_destroy(rx->dqo.buf_dmatag);
}

int
gve_rx_alloc_ring_dqo(struct gve_priv *priv, int i)
{
	struct gve_rx_ring *rx = &priv->rx[i];
	int err;
	int j;

	err = gve_dma_alloc_coherent(priv,
	    sizeof(struct gve_rx_desc_dqo) * priv->rx_desc_cnt,
	    CACHE_LINE_SIZE, &rx->desc_ring_mem);
	if (err != 0) {
		device_printf(priv->dev,
		    "Failed to alloc desc ring for rx ring %d", i);
		goto abort;
	}
	rx->dqo.desc_ring = rx->desc_ring_mem.cpu_addr;
	rx->dqo.mask = priv->rx_desc_cnt - 1;

	err = bus_dma_tag_create(
	    bus_get_dma_tag(priv->dev),	/* parent */
	    1, 0,			/* alignment, bounds */
	    BUS_SPACE_MAXADDR,		/* lowaddr */
	    BUS_SPACE_MAXADDR,		/* highaddr */
	    NULL, NULL,			/* filter, filterarg */
	    MCLBYTES,			/* maxsize */
	    1,				/* nsegments */
	    MCLBYTES,			/* maxsegsize */
	    0,				/* flags */
	    NULL,			/* lockfunc */
	    NULL,			/* lockarg */
	    &rx->dqo.buf_dmatag);
	if (err != 0) {
		device_printf(priv->dev,
		    "%s: bus_dma_tag_create failed: %d\n",
		    __func__, err);
		goto abort;
	}

	rx->dqo.buf_cnt = priv->rx_desc_cnt;
	rx->dqo.bufs = malloc(rx->dqo.buf_cnt * sizeof(struct gve_rx_buf_dqo),
	    M_GVE, M_WAITOK | M_ZERO);
	for (j = 0; j < rx->dqo.buf_cnt; j++) {
		err = bus_dmamap_create(rx->dqo.buf_dmatag, 0,
		    &rx->dqo.bufs[j].dmamap);
		if (err != 0) {
			device_printf(priv->dev,
			    "err in creating rx buf dmamap %d: %d",
			    j, err);
			goto abort;
		}
		rx->dqo.bufs[j].mapped = true;
	}

	err = gve_dma_alloc_coherent(priv,
	    sizeof(struct gve_rx_compl_desc_dqo) * priv->rx_desc_cnt,
	    CACHE_LINE_SIZE, &rx->dqo.compl_ring_mem);
	if (err != 0) {
		device_printf(priv->dev,
		    "Failed to alloc compl ring for rx ring %d", i);
		goto abort;
	}
	rx->dqo.compl_ring = rx->dqo.compl_ring_mem.cpu_addr;
	rx->dqo.mask = priv->rx_desc_cnt - 1;

	return (0);

abort:
	gve_rx_free_ring_dqo(priv, i);
	return (err);
}

static void
gve_rx_clear_desc_ring_dqo(struct gve_rx_ring *rx)
{
	struct gve_ring_com *com = &rx->com;
	int entries;
	int i;

	entries = com->priv->rx_desc_cnt;
	for (i = 0; i < entries; i++)
		rx->dqo.desc_ring[i] = (struct gve_rx_desc_dqo){};

	bus_dmamap_sync(rx->desc_ring_mem.tag, rx->desc_ring_mem.map,
	    BUS_DMASYNC_PREWRITE);
}

static void
gve_rx_clear_compl_ring_dqo(struct gve_rx_ring *rx)
{
	struct gve_ring_com *com = &rx->com;
	int i;

	for (i = 0; i < com->priv->rx_desc_cnt; i++)
		rx->dqo.compl_ring[i] = (struct gve_rx_compl_desc_dqo){};

	bus_dmamap_sync(rx->dqo.compl_ring_mem.tag, rx->dqo.compl_ring_mem.map,
	    BUS_DMASYNC_PREWRITE);
}

void
gve_clear_rx_ring_dqo(struct gve_priv *priv, int i)
{
	struct gve_rx_ring *rx = &priv->rx[i];
	int j;

	rx->fill_cnt = 0;
	rx->cnt = 0;
	rx->dqo.mask = priv->rx_desc_cnt - 1;
	rx->dqo.head = 0;
	rx->dqo.tail = 0;
	rx->dqo.cur_gen_bit = 0;

	gve_rx_clear_desc_ring_dqo(rx);
	gve_rx_clear_compl_ring_dqo(rx);

	gve_free_rx_mbufs_dqo(rx);

	SLIST_INIT(&rx->dqo.free_bufs);
	for (j = 0; j < rx->dqo.buf_cnt; j++)
		SLIST_INSERT_HEAD(&rx->dqo.free_bufs,
		    &rx->dqo.bufs[j], slist_entry);
}

int
gve_rx_intr_dqo(void *arg)
{
	struct gve_rx_ring *rx = arg;
	struct gve_priv *priv = rx->com.priv;
	struct gve_ring_com *com = &rx->com;

	if (__predict_false((if_getdrvflags(priv->ifp) & IFF_DRV_RUNNING) == 0))
		return (FILTER_STRAY);

	/* Interrupts are automatically masked */
	taskqueue_enqueue(com->cleanup_tq, &com->cleanup_task);
	return (FILTER_HANDLED);
}

static void
gve_rx_post_buf_dqo(struct gve_rx_ring *rx, struct gve_rx_buf_dqo *buf)
{
	struct gve_rx_desc_dqo *desc;

	bus_dmamap_sync(rx->dqo.buf_dmatag, buf->dmamap,
	    BUS_DMASYNC_PREREAD);

	desc = &rx->dqo.desc_ring[rx->dqo.head];
	desc->buf_id = htole16(buf - rx->dqo.bufs);
	desc->buf_addr = htole64(buf->addr);

	rx->dqo.head = (rx->dqo.head + 1) & rx->dqo.mask;
	rx->fill_cnt++; /* rx->fill_cnt is just a sysctl counter */

	if ((rx->dqo.head & (GVE_RX_BUF_THRESH_DQO - 1)) == 0) {
		bus_dmamap_sync(rx->desc_ring_mem.tag, rx->desc_ring_mem.map,
		    BUS_DMASYNC_PREWRITE);
		gve_db_bar_dqo_write_4(rx->com.priv, rx->com.db_offset,
		    rx->dqo.head);
	}
}

static int
gve_rx_post_new_mbuf_dqo(struct gve_rx_ring *rx, int how)
{
	struct gve_rx_buf_dqo *buf;
	bus_dma_segment_t segs[1];
	int nsegs;
	int err;

	buf = SLIST_FIRST(&rx->dqo.free_bufs);
	if (__predict_false(!buf)) {
		device_printf(rx->com.priv->dev,
		    "Unexpected empty free bufs list\n");
		return (ENOBUFS);
	}
	SLIST_REMOVE_HEAD(&rx->dqo.free_bufs, slist_entry);

	buf->mbuf = m_getcl(how, MT_DATA, M_PKTHDR);
	if (__predict_false(!buf->mbuf)) {
		err = ENOMEM;
		counter_enter();
		counter_u64_add_protected(rx->stats.rx_mbuf_mclget_null, 1);
		counter_exit();
		goto abort_with_buf;
	}
	buf->mbuf->m_len = MCLBYTES;

	err = bus_dmamap_load_mbuf_sg(rx->dqo.buf_dmatag, buf->dmamap,
	    buf->mbuf, segs, &nsegs, BUS_DMA_NOWAIT);
	KASSERT(nsegs == 1, ("dma segs for a cluster mbuf is not 1"));
	if (__predict_false(err != 0)) {
		counter_enter();
		counter_u64_add_protected(rx->stats.rx_mbuf_dmamap_err, 1);
		counter_exit();
		goto abort_with_mbuf;
	}
	buf->addr = segs[0].ds_addr;

	gve_rx_post_buf_dqo(rx, buf);
	return (0);

abort_with_mbuf:
	m_freem(buf->mbuf);
	buf->mbuf = NULL;
abort_with_buf:
	SLIST_INSERT_HEAD(&rx->dqo.free_bufs, buf, slist_entry);
	return (err);
}

static void
gve_rx_post_buffers_dqo(struct gve_rx_ring *rx, int how)
{
	uint32_t num_pending_bufs;
	uint32_t num_to_post;
	uint32_t i;
	int err;

	num_pending_bufs = (rx->dqo.head - rx->dqo.tail) & rx->dqo.mask;
	num_to_post = rx->dqo.mask - num_pending_bufs;

	for (i = 0; i < num_to_post; i++) {
		err = gve_rx_post_new_mbuf_dqo(rx, how);
		if (err)
			break;
	}
}

void
gve_rx_prefill_buffers_dqo(struct gve_rx_ring *rx)
{
	gve_rx_post_buffers_dqo(rx, M_WAITOK);
}

static void
gve_rx_set_hashtype_dqo(struct mbuf *mbuf, struct gve_ptype *ptype, bool *is_tcp)
{
	switch (ptype->l3_type) {
	case GVE_L3_TYPE_IPV4:
		switch (ptype->l4_type) {
		case GVE_L4_TYPE_TCP:
			*is_tcp = true;
			M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV4);
			break;
		case GVE_L4_TYPE_UDP:
			M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_UDP_IPV4);
			break;
		default:
			M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV4);
		}
		break;
	case GVE_L3_TYPE_IPV6:
		switch (ptype->l4_type) {
		case GVE_L4_TYPE_TCP:
			*is_tcp = true;
			M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV6);
			break;
		case GVE_L4_TYPE_UDP:
			M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_UDP_IPV6);
			break;
		default:
			M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV6);
		}
		break;
	default:
		M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH);
	}
}

static void
gve_rx_set_csum_flags_dqo(struct mbuf *mbuf,
    struct gve_rx_compl_desc_dqo *desc,
    struct gve_ptype *ptype)
{
	/* HW did not identify and process L3 and L4 headers. */
	if (__predict_false(!desc->l3_l4_processed))
		return;

	if (ptype->l3_type == GVE_L3_TYPE_IPV4) {
		if (__predict_false(desc->csum_ip_err ||
		    desc->csum_external_ip_err))
			return;
	} else if (ptype->l3_type == GVE_L3_TYPE_IPV6) {
		/* Checksum should be skipped if this flag is set. */
		if (__predict_false(desc->ipv6_ex_add))
			return;
	}

	if (__predict_false(desc->csum_l4_err))
		return;

	switch (ptype->l4_type) {
	case GVE_L4_TYPE_TCP:
	case GVE_L4_TYPE_UDP:
	case GVE_L4_TYPE_ICMP:
	case GVE_L4_TYPE_SCTP:
		mbuf->m_pkthdr.csum_flags = CSUM_IP_CHECKED |
					    CSUM_IP_VALID |
					    CSUM_DATA_VALID |
					    CSUM_PSEUDO_HDR;
		mbuf->m_pkthdr.csum_data = 0xffff;
		break;
	default:
		break;
	}
}

static void
gve_rx_input_mbuf_dqo(struct gve_rx_ring *rx,
    struct gve_rx_compl_desc_dqo *compl_desc)
{
	struct mbuf *mbuf = rx->ctx.mbuf_head;
	if_t ifp = rx->com.priv->ifp;
	struct gve_ptype *ptype;
	bool do_if_input = true;
	bool is_tcp = false;

	ptype = &rx->com.priv->ptype_lut_dqo->ptypes[compl_desc->packet_type];
	gve_rx_set_hashtype_dqo(mbuf, ptype, &is_tcp);
	mbuf->m_pkthdr.flowid = le32toh(compl_desc->hash);
	gve_rx_set_csum_flags_dqo(mbuf, compl_desc, ptype);

	mbuf->m_pkthdr.rcvif = ifp;
	mbuf->m_pkthdr.len = rx->ctx.total_size;

	if (((if_getcapenable(rx->com.priv->ifp) & IFCAP_LRO) != 0) &&
	    is_tcp &&
	    (rx->lro.lro_cnt != 0) &&
	    (tcp_lro_rx(&rx->lro, mbuf, 0) == 0))
		do_if_input = false;

	if (do_if_input)
		if_input(ifp, mbuf);

	counter_enter();
	counter_u64_add_protected(rx->stats.rbytes, rx->ctx.total_size);
	counter_u64_add_protected(rx->stats.rpackets, 1);
	counter_exit();

	rx->ctx = (struct gve_rx_ctx){};
}

static int
gve_rx_copybreak_dqo(struct gve_rx_ring *rx, struct gve_rx_buf_dqo *buf,
    struct gve_rx_compl_desc_dqo *compl_desc, uint16_t frag_len)
{
	struct mbuf *mbuf;

	mbuf = m_get2(frag_len, M_NOWAIT, MT_DATA, M_PKTHDR);
	if (__predict_false(mbuf == NULL))
		return (ENOMEM);

	counter_enter();
	counter_u64_add_protected(rx->stats.rx_copybreak_cnt, 1);
	counter_exit();

	m_copyback(mbuf, 0, frag_len, mtod(buf->mbuf, char*));
	mbuf->m_len = frag_len;

	rx->ctx.mbuf_head = mbuf;
	rx->ctx.mbuf_tail = mbuf;
	rx->ctx.total_size += frag_len;

	gve_rx_post_buf_dqo(rx, buf);
	gve_rx_input_mbuf_dqo(rx, compl_desc);
	return (0);
}

static void
gve_rx_dqo(struct gve_priv *priv, struct gve_rx_ring *rx,
    struct gve_rx_compl_desc_dqo *compl_desc,
    int *work_done)
{
	bool is_last_frag = compl_desc->end_of_packet != 0;
	struct gve_rx_ctx *ctx = &rx->ctx;
	struct gve_rx_buf_dqo *buf;
	uint32_t num_pending_bufs;
	uint16_t frag_len;
	uint16_t buf_id;
	int err;

	buf_id = le16toh(compl_desc->buf_id);
	if (__predict_false(buf_id >= rx->dqo.buf_cnt)) {
		device_printf(priv->dev, "Invalid rx buf id %d on rxq %d, issuing reset\n",
		    buf_id, rx->com.id);
		gve_schedule_reset(priv);
		goto drop_frag_clear_ctx;
	}
	buf = &rx->dqo.bufs[buf_id];
	if (__predict_false(buf->mbuf == NULL)) {
		device_printf(priv->dev, "Spurious completion for buf id %d on rxq %d, issuing reset\n",
		    buf_id, rx->com.id);
		gve_schedule_reset(priv);
		goto drop_frag_clear_ctx;
	}

	if (__predict_false(ctx->drop_pkt))
		goto drop_frag;

	if (__predict_false(compl_desc->rx_error)) {
		counter_enter();
		counter_u64_add_protected(rx->stats.rx_dropped_pkt_desc_err, 1);
		counter_exit();
		goto drop_frag;
	}

	bus_dmamap_sync(rx->dqo.buf_dmatag, buf->dmamap,
	    BUS_DMASYNC_POSTREAD);

	frag_len = compl_desc->packet_len;
	if (frag_len <= priv->rx_copybreak && !ctx->mbuf_head && is_last_frag) {
		err = gve_rx_copybreak_dqo(rx, buf, compl_desc, frag_len);
		if (__predict_false(err != 0))
			goto drop_frag;
		(*work_done)++;
		return;
	}

	/*
	 * Although buffer completions may arrive out of order, buffer
	 * descriptors are consumed by the NIC in order. That is, the
	 * buffer at desc_ring[tail] might not be the buffer we got the
	 * completion compl_ring[tail] for: but we know that desc_ring[tail]
	 * has already been read by the NIC.
	 */
	num_pending_bufs = (rx->dqo.head - rx->dqo.tail) & rx->dqo.mask;

	/*
	 * For every fragment received, try to post a new buffer.
	 *
	 * Failures are okay but only so long as the number of outstanding
	 * buffers is above a threshold.
	 *
	 * Beyond that we drop new packets to reuse their buffers.
	 * Without ensuring a minimum number of buffers for the NIC to
	 * put packets in, we run the risk of getting the queue stuck
	 * for good.
	 */
	err = gve_rx_post_new_mbuf_dqo(rx, M_NOWAIT);
	if (__predict_false(err != 0 &&
	    num_pending_bufs <= GVE_RX_DQO_MIN_PENDING_BUFS)) {
		counter_enter();
		counter_u64_add_protected(
		    rx->stats.rx_dropped_pkt_mbuf_alloc_fail, 1);
		counter_exit();
		goto drop_frag;
	}

	buf->mbuf->m_len = frag_len;
	ctx->total_size += frag_len;
	if (ctx->mbuf_tail == NULL) {
		ctx->mbuf_head = buf->mbuf;
		ctx->mbuf_tail = buf->mbuf;
	} else {
		buf->mbuf->m_flags &= ~M_PKTHDR;
		ctx->mbuf_tail->m_next = buf->mbuf;
		ctx->mbuf_tail = buf->mbuf;
	}

	/*
	 * Disassociate the mbuf from buf and surrender buf to the free list to
	 * be used by a future mbuf.
	 */
	bus_dmamap_unload(rx->dqo.buf_dmatag, buf->dmamap);
	buf->mbuf = NULL;
	buf->addr = 0;
	SLIST_INSERT_HEAD(&rx->dqo.free_bufs, buf, slist_entry);

	if (is_last_frag) {
		gve_rx_input_mbuf_dqo(rx, compl_desc);
		(*work_done)++;
	}
	return;

drop_frag:
	/* Clear the earlier frags if there were any */
	m_freem(ctx->mbuf_head);
	rx->ctx = (struct gve_rx_ctx){};
	/* Drop the rest of the pkt if there are more frags */
	ctx->drop_pkt = true;
	/* Reuse the dropped frag's buffer */
	gve_rx_post_buf_dqo(rx, buf);

	if (is_last_frag)
		goto drop_frag_clear_ctx;
	return;

drop_frag_clear_ctx:
	counter_enter();
	counter_u64_add_protected(rx->stats.rx_dropped_pkt, 1);
	counter_exit();
	m_freem(ctx->mbuf_head);
	rx->ctx = (struct gve_rx_ctx){};
}

static bool
gve_rx_cleanup_dqo(struct gve_priv *priv, struct gve_rx_ring *rx, int budget)
{
	struct gve_rx_compl_desc_dqo *compl_desc;
	uint32_t work_done = 0;

	NET_EPOCH_ASSERT();

	while (work_done < budget) {
		bus_dmamap_sync(rx->dqo.compl_ring_mem.tag, rx->dqo.compl_ring_mem.map,
		    BUS_DMASYNC_POSTREAD);

		compl_desc = &rx->dqo.compl_ring[rx->dqo.tail];
		if (compl_desc->generation == rx->dqo.cur_gen_bit)
			break;
		/*
		 * Prevent generation bit from being read after the rest of the
		 * descriptor.
		 */
		rmb();

		rx->cnt++;
		rx->dqo.tail = (rx->dqo.tail + 1) & rx->dqo.mask;
		rx->dqo.cur_gen_bit ^= (rx->dqo.tail == 0);

		gve_rx_dqo(priv, rx, compl_desc, &work_done);
	}

	if (work_done != 0)
		tcp_lro_flush_all(&rx->lro);

	gve_rx_post_buffers_dqo(rx, M_NOWAIT);
	return (work_done == budget);
}

void
gve_rx_cleanup_tq_dqo(void *arg, int pending)
{
	struct gve_rx_ring *rx = arg;
	struct gve_priv *priv = rx->com.priv;

	if (__predict_false((if_getdrvflags(priv->ifp) & IFF_DRV_RUNNING) == 0))
		return;

	if (gve_rx_cleanup_dqo(priv, rx, /*budget=*/64)) {
		taskqueue_enqueue(rx->com.cleanup_tq, &rx->com.cleanup_task);
		return;
	}

	gve_db_bar_dqo_write_4(priv, rx->com.irq_db_offset,
	    GVE_ITR_NO_UPDATE_DQO | GVE_ITR_ENABLE_BIT_DQO);
}