151b35a45SEdward Cree // SPDX-License-Identifier: GPL-2.0-only
251b35a45SEdward Cree /****************************************************************************
351b35a45SEdward Cree * Driver for Solarflare network controllers and boards
451b35a45SEdward Cree * Copyright 2005-2019 Solarflare Communications Inc.
551b35a45SEdward Cree *
651b35a45SEdward Cree * This program is free software; you can redistribute it and/or modify it
751b35a45SEdward Cree * under the terms of the GNU General Public License version 2 as published
851b35a45SEdward Cree * by the Free Software Foundation, incorporated herein by reference.
951b35a45SEdward Cree */
1051b35a45SEdward Cree
1151b35a45SEdward Cree #include "net_driver.h"
1251b35a45SEdward Cree #include "ef100_rx.h"
1351b35a45SEdward Cree #include "rx_common.h"
1451b35a45SEdward Cree #include "efx.h"
158e57daf7SEdward Cree #include "nic_common.h"
168e57daf7SEdward Cree #include "mcdi_functions.h"
178e57daf7SEdward Cree #include "ef100_regs.h"
188e57daf7SEdward Cree #include "ef100_nic.h"
198e57daf7SEdward Cree #include "io.h"
2051b35a45SEdward Cree
218e57daf7SEdward Cree /* Get the value of a field in the RX prefix */
228e57daf7SEdward Cree #define PREFIX_OFFSET_W(_f) (ESF_GZ_RX_PREFIX_ ## _f ## _LBN / 32)
238e57daf7SEdward Cree #define PREFIX_OFFSET_B(_f) (ESF_GZ_RX_PREFIX_ ## _f ## _LBN % 32)
245ae0c226SEdward Cree #define PREFIX_WIDTH_MASK(_f) ((1ULL << ESF_GZ_RX_PREFIX_ ## _f ## _WIDTH) - 1)
258e57daf7SEdward Cree #define PREFIX_WORD(_p, _f) le32_to_cpu((__force __le32)(_p)[PREFIX_OFFSET_W(_f)])
268e57daf7SEdward Cree #define PREFIX_FIELD(_p, _f) ((PREFIX_WORD(_p, _f) >> PREFIX_OFFSET_B(_f)) & \
278e57daf7SEdward Cree PREFIX_WIDTH_MASK(_f))
28965b549fSEdward Cree
298e57daf7SEdward Cree #define ESF_GZ_RX_PREFIX_NT_OR_INNER_L3_CLASS_LBN \
308e57daf7SEdward Cree (ESF_GZ_RX_PREFIX_CLASS_LBN + ESF_GZ_RX_PREFIX_HCLASS_NT_OR_INNER_L3_CLASS_LBN)
318e57daf7SEdward Cree #define ESF_GZ_RX_PREFIX_NT_OR_INNER_L3_CLASS_WIDTH \
328e57daf7SEdward Cree ESF_GZ_RX_PREFIX_HCLASS_NT_OR_INNER_L3_CLASS_WIDTH
338e57daf7SEdward Cree
ef100_rx_buf_hash_valid(const u8 * prefix)3406888543SEdward Cree bool ef100_rx_buf_hash_valid(const u8 *prefix)
3506888543SEdward Cree {
3606888543SEdward Cree return PREFIX_FIELD(prefix, RSS_HASH_VALID);
3706888543SEdward Cree }
3806888543SEdward Cree
ef100_has_fcs_error(struct efx_channel * channel,u32 * prefix)39966b8266SEdward Cree static bool ef100_has_fcs_error(struct efx_channel *channel, u32 *prefix)
40965b549fSEdward Cree {
418e57daf7SEdward Cree u16 rxclass;
428e57daf7SEdward Cree u8 l2status;
438e57daf7SEdward Cree
448e57daf7SEdward Cree rxclass = le16_to_cpu((__force __le16)PREFIX_FIELD(prefix, CLASS));
458e57daf7SEdward Cree l2status = PREFIX_FIELD(&rxclass, HCLASS_L2_STATUS);
468e57daf7SEdward Cree
478e57daf7SEdward Cree if (likely(l2status == ESE_GZ_RH_HCLASS_L2_STATUS_OK))
488e57daf7SEdward Cree /* Everything is ok */
49966b8266SEdward Cree return false;
508e57daf7SEdward Cree
518e57daf7SEdward Cree if (l2status == ESE_GZ_RH_HCLASS_L2_STATUS_FCS_ERR)
528e57daf7SEdward Cree channel->n_rx_eth_crc_err++;
53966b8266SEdward Cree return true;
54965b549fSEdward Cree }
55965b549fSEdward Cree
__ef100_rx_packet(struct efx_channel * channel)5651b35a45SEdward Cree void __ef100_rx_packet(struct efx_channel *channel)
5751b35a45SEdward Cree {
5808d0b16eSEdward Cree struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel);
5908d0b16eSEdward Cree struct efx_rx_buffer *rx_buf = efx_rx_buffer(rx_queue,
6008d0b16eSEdward Cree channel->rx_pkt_index);
618e57daf7SEdward Cree struct efx_nic *efx = channel->efx;
6208d0b16eSEdward Cree struct ef100_nic_data *nic_data;
638e57daf7SEdward Cree u8 *eh = efx_rx_buf_va(rx_buf);
648e57daf7SEdward Cree __wsum csum = 0;
6508d0b16eSEdward Cree u16 ing_port;
668e57daf7SEdward Cree u32 *prefix;
6751b35a45SEdward Cree
688e57daf7SEdward Cree prefix = (u32 *)(eh - ESE_GZ_RX_PKT_PREFIX_LEN);
698e57daf7SEdward Cree
70*36df6136SEdward Cree if (channel->type->receive_raw) {
71*36df6136SEdward Cree u32 mark = PREFIX_FIELD(prefix, USER_MARK);
72*36df6136SEdward Cree
73*36df6136SEdward Cree if (channel->type->receive_raw(rx_queue, mark))
74*36df6136SEdward Cree return; /* packet was consumed */
75*36df6136SEdward Cree }
76*36df6136SEdward Cree
77966b8266SEdward Cree if (ef100_has_fcs_error(channel, prefix) &&
788e57daf7SEdward Cree unlikely(!(efx->net_dev->features & NETIF_F_RXALL)))
798e57daf7SEdward Cree goto out;
808e57daf7SEdward Cree
818e57daf7SEdward Cree rx_buf->len = le16_to_cpu((__force __le16)PREFIX_FIELD(prefix, LENGTH));
828e57daf7SEdward Cree if (rx_buf->len <= sizeof(struct ethhdr)) {
838e57daf7SEdward Cree if (net_ratelimit())
848e57daf7SEdward Cree netif_err(channel->efx, rx_err, channel->efx->net_dev,
858e57daf7SEdward Cree "RX packet too small (%d)\n", rx_buf->len);
868e57daf7SEdward Cree ++channel->n_rx_frm_trunc;
878e57daf7SEdward Cree goto out;
888e57daf7SEdward Cree }
898e57daf7SEdward Cree
9008d0b16eSEdward Cree ing_port = le16_to_cpu((__force __le16) PREFIX_FIELD(prefix, INGRESS_MPORT));
9108d0b16eSEdward Cree
9208d0b16eSEdward Cree nic_data = efx->nic_data;
9308d0b16eSEdward Cree
9408d0b16eSEdward Cree if (nic_data->have_mport && ing_port != nic_data->base_mport) {
95f50e8fcdSEdward Cree #ifdef CONFIG_SFC_SRIOV
96f50e8fcdSEdward Cree struct efx_rep *efv;
97f50e8fcdSEdward Cree
98f50e8fcdSEdward Cree rcu_read_lock();
99f50e8fcdSEdward Cree efv = efx_ef100_find_rep_by_mport(efx, ing_port);
100f50e8fcdSEdward Cree if (efv) {
101f50e8fcdSEdward Cree if (efv->net_dev->flags & IFF_UP)
102f50e8fcdSEdward Cree efx_ef100_rep_rx_packet(efv, rx_buf);
103f50e8fcdSEdward Cree rcu_read_unlock();
104f50e8fcdSEdward Cree /* Representor Rx doesn't care about PF Rx buffer
105f50e8fcdSEdward Cree * ownership, it just makes a copy. So, we are done
106f50e8fcdSEdward Cree * with the Rx buffer from PF point of view and should
107f50e8fcdSEdward Cree * free it.
108f50e8fcdSEdward Cree */
109f50e8fcdSEdward Cree goto free_rx_buffer;
110f50e8fcdSEdward Cree }
111f50e8fcdSEdward Cree rcu_read_unlock();
112f50e8fcdSEdward Cree #endif
11308d0b16eSEdward Cree if (net_ratelimit())
11408d0b16eSEdward Cree netif_warn(efx, drv, efx->net_dev,
11508d0b16eSEdward Cree "Unrecognised ing_port %04x (base %04x), dropping\n",
11608d0b16eSEdward Cree ing_port, nic_data->base_mport);
11708d0b16eSEdward Cree channel->n_rx_mport_bad++;
11808d0b16eSEdward Cree goto free_rx_buffer;
11908d0b16eSEdward Cree }
12008d0b16eSEdward Cree
1218e57daf7SEdward Cree if (likely(efx->net_dev->features & NETIF_F_RXCSUM)) {
1228e57daf7SEdward Cree if (PREFIX_FIELD(prefix, NT_OR_INNER_L3_CLASS) == 1) {
1238e57daf7SEdward Cree ++channel->n_rx_ip_hdr_chksum_err;
1248e57daf7SEdward Cree } else {
1258e57daf7SEdward Cree u16 sum = be16_to_cpu((__force __be16)PREFIX_FIELD(prefix, CSUM_FRAME));
1268e57daf7SEdward Cree
1278e57daf7SEdward Cree csum = (__force __wsum) sum;
1288e57daf7SEdward Cree }
1298e57daf7SEdward Cree }
1308e57daf7SEdward Cree
1318e57daf7SEdward Cree if (channel->type->receive_skb) {
1328e57daf7SEdward Cree /* no support for special channels yet, so just discard */
1338e57daf7SEdward Cree WARN_ON_ONCE(1);
13408d0b16eSEdward Cree goto free_rx_buffer;
1358e57daf7SEdward Cree }
1368e57daf7SEdward Cree
1378e57daf7SEdward Cree efx_rx_packet_gro(channel, rx_buf, channel->rx_pkt_n_frags, eh, csum);
13808d0b16eSEdward Cree goto out;
1398e57daf7SEdward Cree
14008d0b16eSEdward Cree free_rx_buffer:
14108d0b16eSEdward Cree efx_free_rx_buffers(rx_queue, rx_buf, 1);
1428e57daf7SEdward Cree out:
14351b35a45SEdward Cree channel->rx_pkt_n_frags = 0;
14451b35a45SEdward Cree }
1458e57daf7SEdward Cree
ef100_rx_packet(struct efx_rx_queue * rx_queue,unsigned int index)1468e57daf7SEdward Cree static void ef100_rx_packet(struct efx_rx_queue *rx_queue, unsigned int index)
1478e57daf7SEdward Cree {
1488e57daf7SEdward Cree struct efx_rx_buffer *rx_buf = efx_rx_buffer(rx_queue, index);
1498e57daf7SEdward Cree struct efx_channel *channel = efx_rx_queue_channel(rx_queue);
1508e57daf7SEdward Cree struct efx_nic *efx = rx_queue->efx;
1518e57daf7SEdward Cree
1528e57daf7SEdward Cree ++rx_queue->rx_packets;
1538e57daf7SEdward Cree
1548e57daf7SEdward Cree netif_vdbg(efx, rx_status, efx->net_dev,
1558e57daf7SEdward Cree "RX queue %d received id %x\n",
1568e57daf7SEdward Cree efx_rx_queue_index(rx_queue), index);
1578e57daf7SEdward Cree
1588e57daf7SEdward Cree efx_sync_rx_buffer(efx, rx_buf, efx->rx_dma_len);
1598e57daf7SEdward Cree
1608e57daf7SEdward Cree prefetch(efx_rx_buf_va(rx_buf));
1618e57daf7SEdward Cree
1628e57daf7SEdward Cree rx_buf->page_offset += efx->rx_prefix_size;
1638e57daf7SEdward Cree
1648e57daf7SEdward Cree efx_recycle_rx_pages(channel, rx_buf, 1);
1658e57daf7SEdward Cree
1668e57daf7SEdward Cree efx_rx_flush_packet(channel);
1678e57daf7SEdward Cree channel->rx_pkt_n_frags = 1;
1688e57daf7SEdward Cree channel->rx_pkt_index = index;
1698e57daf7SEdward Cree }
1708e57daf7SEdward Cree
efx_ef100_ev_rx(struct efx_channel * channel,const efx_qword_t * p_event)1718e57daf7SEdward Cree void efx_ef100_ev_rx(struct efx_channel *channel, const efx_qword_t *p_event)
1728e57daf7SEdward Cree {
1738e57daf7SEdward Cree struct efx_rx_queue *rx_queue = efx_channel_get_rx_queue(channel);
1748e57daf7SEdward Cree unsigned int n_packets =
1758e57daf7SEdward Cree EFX_QWORD_FIELD(*p_event, ESF_GZ_EV_RXPKTS_NUM_PKT);
1768e57daf7SEdward Cree int i;
1778e57daf7SEdward Cree
1788e57daf7SEdward Cree WARN_ON_ONCE(!n_packets);
1798e57daf7SEdward Cree if (n_packets > 1)
1808e57daf7SEdward Cree ++channel->n_rx_merge_events;
1818e57daf7SEdward Cree
1828e57daf7SEdward Cree channel->irq_mod_score += 2 * n_packets;
1838e57daf7SEdward Cree
1848e57daf7SEdward Cree for (i = 0; i < n_packets; ++i) {
1858e57daf7SEdward Cree ef100_rx_packet(rx_queue,
1868e57daf7SEdward Cree rx_queue->removed_count & rx_queue->ptr_mask);
1878e57daf7SEdward Cree ++rx_queue->removed_count;
1888e57daf7SEdward Cree }
1898e57daf7SEdward Cree }
1908e57daf7SEdward Cree
ef100_rx_write(struct efx_rx_queue * rx_queue)1918e57daf7SEdward Cree void ef100_rx_write(struct efx_rx_queue *rx_queue)
1928e57daf7SEdward Cree {
193e3951539SEdward Cree unsigned int notified_count = rx_queue->notified_count;
1948e57daf7SEdward Cree struct efx_rx_buffer *rx_buf;
1958e57daf7SEdward Cree unsigned int idx;
1968e57daf7SEdward Cree efx_qword_t *rxd;
1978e57daf7SEdward Cree efx_dword_t rxdb;
1988e57daf7SEdward Cree
199e3951539SEdward Cree while (notified_count != rx_queue->added_count) {
200e3951539SEdward Cree idx = notified_count & rx_queue->ptr_mask;
2018e57daf7SEdward Cree rx_buf = efx_rx_buffer(rx_queue, idx);
2028e57daf7SEdward Cree rxd = efx_rx_desc(rx_queue, idx);
2038e57daf7SEdward Cree
2048e57daf7SEdward Cree EFX_POPULATE_QWORD_1(*rxd, ESF_GZ_RX_BUF_ADDR, rx_buf->dma_addr);
2058e57daf7SEdward Cree
206e3951539SEdward Cree ++notified_count;
2078e57daf7SEdward Cree }
208e3951539SEdward Cree if (notified_count == rx_queue->notified_count)
209e3951539SEdward Cree return;
2108e57daf7SEdward Cree
2118e57daf7SEdward Cree wmb();
2128e57daf7SEdward Cree EFX_POPULATE_DWORD_1(rxdb, ERF_GZ_RX_RING_PIDX,
2138e57daf7SEdward Cree rx_queue->added_count & rx_queue->ptr_mask);
2148e57daf7SEdward Cree efx_writed_page(rx_queue->efx, &rxdb,
2158e57daf7SEdward Cree ER_GZ_RX_RING_DOORBELL, efx_rx_queue_index(rx_queue));
216e3951539SEdward Cree if (rx_queue->grant_credits)
217e3951539SEdward Cree wmb();
218e3951539SEdward Cree rx_queue->notified_count = notified_count;
219e3951539SEdward Cree if (rx_queue->grant_credits)
220e3951539SEdward Cree schedule_work(&rx_queue->grant_work);
2218e57daf7SEdward Cree }
222