1 // SPDX-License-Identifier: GPL-2.0-only OR BSD-3-Clause 2 3 /* Packet receive logic for Mellanox Gigabit Ethernet driver 4 * 5 * Copyright (C) 2020-2021 NVIDIA CORPORATION & AFFILIATES 6 */ 7 8 #include <linux/etherdevice.h> 9 #include <linux/skbuff.h> 10 11 #include "mlxbf_gige.h" 12 #include "mlxbf_gige_regs.h" 13 14 void mlxbf_gige_enable_multicast_rx(struct mlxbf_gige *priv) 15 { 16 void __iomem *base = priv->base; 17 u64 data; 18 19 data = readq(base + MLXBF_GIGE_RX_MAC_FILTER_GENERAL); 20 data |= MLXBF_GIGE_RX_MAC_FILTER_EN_MULTICAST; 21 writeq(data, base + MLXBF_GIGE_RX_MAC_FILTER_GENERAL); 22 } 23 24 void mlxbf_gige_disable_multicast_rx(struct mlxbf_gige *priv) 25 { 26 void __iomem *base = priv->base; 27 u64 data; 28 29 data = readq(base + MLXBF_GIGE_RX_MAC_FILTER_GENERAL); 30 data &= ~MLXBF_GIGE_RX_MAC_FILTER_EN_MULTICAST; 31 writeq(data, base + MLXBF_GIGE_RX_MAC_FILTER_GENERAL); 32 } 33 34 void mlxbf_gige_enable_mac_rx_filter(struct mlxbf_gige *priv, 35 unsigned int index) 36 { 37 void __iomem *base = priv->base; 38 u64 control; 39 40 /* Enable MAC receive filter mask for specified index */ 41 control = readq(base + MLXBF_GIGE_CONTROL); 42 control |= (MLXBF_GIGE_CONTROL_EN_SPECIFIC_MAC << index); 43 writeq(control, base + MLXBF_GIGE_CONTROL); 44 } 45 46 void mlxbf_gige_disable_mac_rx_filter(struct mlxbf_gige *priv, 47 unsigned int index) 48 { 49 void __iomem *base = priv->base; 50 u64 control; 51 52 /* Disable MAC receive filter mask for specified index */ 53 control = readq(base + MLXBF_GIGE_CONTROL); 54 control &= ~(MLXBF_GIGE_CONTROL_EN_SPECIFIC_MAC << index); 55 writeq(control, base + MLXBF_GIGE_CONTROL); 56 } 57 58 void mlxbf_gige_set_mac_rx_filter(struct mlxbf_gige *priv, 59 unsigned int index, u64 dmac) 60 { 61 void __iomem *base = priv->base; 62 63 /* Write destination MAC to specified MAC RX filter */ 64 writeq(dmac, base + MLXBF_GIGE_RX_MAC_FILTER + 65 (index * MLXBF_GIGE_RX_MAC_FILTER_STRIDE)); 66 } 67 68 void mlxbf_gige_get_mac_rx_filter(struct mlxbf_gige *priv, 69 unsigned int index, u64 *dmac) 70 { 71 void __iomem *base = priv->base; 72 73 /* Read destination MAC from specified MAC RX filter */ 74 *dmac = readq(base + MLXBF_GIGE_RX_MAC_FILTER + 75 (index * MLXBF_GIGE_RX_MAC_FILTER_STRIDE)); 76 } 77 78 void mlxbf_gige_enable_promisc(struct mlxbf_gige *priv) 79 { 80 void __iomem *base = priv->base; 81 u64 control; 82 u64 end_mac; 83 84 /* Enable MAC_ID_RANGE match functionality */ 85 control = readq(base + MLXBF_GIGE_CONTROL); 86 control |= MLXBF_GIGE_CONTROL_MAC_ID_RANGE_EN; 87 writeq(control, base + MLXBF_GIGE_CONTROL); 88 89 /* Set start of destination MAC range check to 0 */ 90 writeq(0, base + MLXBF_GIGE_RX_MAC_FILTER_DMAC_RANGE_START); 91 92 /* Set end of destination MAC range check to all FFs */ 93 end_mac = BCAST_MAC_ADDR; 94 writeq(end_mac, base + MLXBF_GIGE_RX_MAC_FILTER_DMAC_RANGE_END); 95 } 96 97 void mlxbf_gige_disable_promisc(struct mlxbf_gige *priv) 98 { 99 void __iomem *base = priv->base; 100 u64 control; 101 102 /* Disable MAC_ID_RANGE match functionality */ 103 control = readq(base + MLXBF_GIGE_CONTROL); 104 control &= ~MLXBF_GIGE_CONTROL_MAC_ID_RANGE_EN; 105 writeq(control, base + MLXBF_GIGE_CONTROL); 106 107 /* NOTE: no need to change DMAC_RANGE_START or END; 108 * those values are ignored since MAC_ID_RANGE_EN=0 109 */ 110 } 111 112 /* Receive Initialization 113 * 1) Configures RX MAC filters via MMIO registers 114 * 2) Allocates RX WQE array using coherent DMA mapping 115 * 3) Initializes each element of RX WQE array with a receive 116 * buffer pointer (also using coherent DMA mapping) 117 * 4) Allocates RX CQE array using coherent DMA mapping 118 * 5) Completes other misc receive initialization 119 */ 120 int mlxbf_gige_rx_init(struct mlxbf_gige *priv) 121 { 122 size_t wq_size, cq_size; 123 dma_addr_t *rx_wqe_ptr; 124 dma_addr_t rx_buf_dma; 125 u64 data; 126 int i, j; 127 128 /* Configure MAC RX filter #0 to allow RX of broadcast pkts */ 129 mlxbf_gige_set_mac_rx_filter(priv, MLXBF_GIGE_BCAST_MAC_FILTER_IDX, 130 BCAST_MAC_ADDR); 131 132 wq_size = MLXBF_GIGE_RX_WQE_SZ * priv->rx_q_entries; 133 priv->rx_wqe_base = dma_alloc_coherent(priv->dev, wq_size, 134 &priv->rx_wqe_base_dma, 135 GFP_KERNEL); 136 if (!priv->rx_wqe_base) 137 return -ENOMEM; 138 139 /* Initialize 'rx_wqe_ptr' to point to first RX WQE in array 140 * Each RX WQE is simply a receive buffer pointer, so walk 141 * the entire array, allocating a 2KB buffer for each element 142 */ 143 rx_wqe_ptr = priv->rx_wqe_base; 144 145 for (i = 0; i < priv->rx_q_entries; i++) { 146 priv->rx_skb[i] = mlxbf_gige_alloc_skb(priv, MLXBF_GIGE_DEFAULT_BUF_SZ, 147 &rx_buf_dma, DMA_FROM_DEVICE); 148 if (!priv->rx_skb[i]) 149 goto free_wqe_and_skb; 150 *rx_wqe_ptr++ = rx_buf_dma; 151 } 152 153 /* Write RX WQE base address into MMIO reg */ 154 writeq(priv->rx_wqe_base_dma, priv->base + MLXBF_GIGE_RX_WQ_BASE); 155 156 cq_size = MLXBF_GIGE_RX_CQE_SZ * priv->rx_q_entries; 157 priv->rx_cqe_base = dma_alloc_coherent(priv->dev, cq_size, 158 &priv->rx_cqe_base_dma, 159 GFP_KERNEL); 160 if (!priv->rx_cqe_base) 161 goto free_wqe_and_skb; 162 163 for (i = 0; i < priv->rx_q_entries; i++) 164 priv->rx_cqe_base[i] |= MLXBF_GIGE_RX_CQE_VALID_MASK; 165 166 /* Write RX CQE base address into MMIO reg */ 167 writeq(priv->rx_cqe_base_dma, priv->base + MLXBF_GIGE_RX_CQ_BASE); 168 169 /* Write RX_WQE_PI with current number of replenished buffers */ 170 writeq(priv->rx_q_entries, priv->base + MLXBF_GIGE_RX_WQE_PI); 171 172 /* Enable removal of CRC during RX */ 173 data = readq(priv->base + MLXBF_GIGE_RX); 174 data |= MLXBF_GIGE_RX_STRIP_CRC_EN; 175 writeq(data, priv->base + MLXBF_GIGE_RX); 176 177 /* Enable RX MAC filter pass and discard counters */ 178 writeq(MLXBF_GIGE_RX_MAC_FILTER_COUNT_DISC_EN, 179 priv->base + MLXBF_GIGE_RX_MAC_FILTER_COUNT_DISC); 180 writeq(MLXBF_GIGE_RX_MAC_FILTER_COUNT_PASS_EN, 181 priv->base + MLXBF_GIGE_RX_MAC_FILTER_COUNT_PASS); 182 183 writeq(ilog2(priv->rx_q_entries), 184 priv->base + MLXBF_GIGE_RX_WQE_SIZE_LOG2); 185 186 /* Clear MLXBF_GIGE_INT_MASK 'receive pkt' bit to 187 * indicate readiness to receive interrupts 188 */ 189 data = readq(priv->base + MLXBF_GIGE_INT_MASK); 190 data &= ~MLXBF_GIGE_INT_MASK_RX_RECEIVE_PACKET; 191 writeq(data, priv->base + MLXBF_GIGE_INT_MASK); 192 193 /* Enable RX DMA to write new packets to memory */ 194 data = readq(priv->base + MLXBF_GIGE_RX_DMA); 195 data |= MLXBF_GIGE_RX_DMA_EN; 196 writeq(data, priv->base + MLXBF_GIGE_RX_DMA); 197 198 return 0; 199 200 free_wqe_and_skb: 201 rx_wqe_ptr = priv->rx_wqe_base; 202 for (j = 0; j < i; j++) { 203 dma_unmap_single(priv->dev, *rx_wqe_ptr, 204 MLXBF_GIGE_DEFAULT_BUF_SZ, DMA_FROM_DEVICE); 205 dev_kfree_skb(priv->rx_skb[j]); 206 rx_wqe_ptr++; 207 } 208 dma_free_coherent(priv->dev, wq_size, 209 priv->rx_wqe_base, priv->rx_wqe_base_dma); 210 return -ENOMEM; 211 } 212 213 /* Receive Deinitialization 214 * This routine will free allocations done by mlxbf_gige_rx_init(), 215 * namely the RX WQE and RX CQE arrays, as well as all RX buffers 216 */ 217 void mlxbf_gige_rx_deinit(struct mlxbf_gige *priv) 218 { 219 dma_addr_t *rx_wqe_ptr; 220 size_t size; 221 u64 data; 222 int i; 223 224 /* Disable RX DMA to prevent packet transfers to memory */ 225 data = readq(priv->base + MLXBF_GIGE_RX_DMA); 226 data &= ~MLXBF_GIGE_RX_DMA_EN; 227 writeq(data, priv->base + MLXBF_GIGE_RX_DMA); 228 229 rx_wqe_ptr = priv->rx_wqe_base; 230 231 for (i = 0; i < priv->rx_q_entries; i++) { 232 dma_unmap_single(priv->dev, *rx_wqe_ptr, MLXBF_GIGE_DEFAULT_BUF_SZ, 233 DMA_FROM_DEVICE); 234 dev_kfree_skb(priv->rx_skb[i]); 235 rx_wqe_ptr++; 236 } 237 238 size = MLXBF_GIGE_RX_WQE_SZ * priv->rx_q_entries; 239 dma_free_coherent(priv->dev, size, 240 priv->rx_wqe_base, priv->rx_wqe_base_dma); 241 242 size = MLXBF_GIGE_RX_CQE_SZ * priv->rx_q_entries; 243 dma_free_coherent(priv->dev, size, 244 priv->rx_cqe_base, priv->rx_cqe_base_dma); 245 246 priv->rx_wqe_base = NULL; 247 priv->rx_wqe_base_dma = 0; 248 priv->rx_cqe_base = NULL; 249 priv->rx_cqe_base_dma = 0; 250 writeq(0, priv->base + MLXBF_GIGE_RX_WQ_BASE); 251 writeq(0, priv->base + MLXBF_GIGE_RX_CQ_BASE); 252 } 253 254 static bool mlxbf_gige_rx_packet(struct mlxbf_gige *priv, int *rx_pkts) 255 { 256 struct net_device *netdev = priv->netdev; 257 struct sk_buff *skb = NULL, *rx_skb; 258 u16 rx_pi_rem, rx_ci_rem; 259 dma_addr_t *rx_wqe_addr; 260 dma_addr_t rx_buf_dma; 261 u64 *rx_cqe_addr; 262 u64 datalen; 263 u64 rx_cqe; 264 u16 rx_ci; 265 u16 rx_pi; 266 267 /* Index into RX buffer array is rx_pi w/wrap based on RX_CQE_SIZE */ 268 rx_pi = readq(priv->base + MLXBF_GIGE_RX_WQE_PI); 269 rx_pi_rem = rx_pi % priv->rx_q_entries; 270 271 rx_wqe_addr = priv->rx_wqe_base + rx_pi_rem; 272 rx_cqe_addr = priv->rx_cqe_base + rx_pi_rem; 273 rx_cqe = *rx_cqe_addr; 274 275 if ((!!(rx_cqe & MLXBF_GIGE_RX_CQE_VALID_MASK)) != priv->valid_polarity) 276 return false; 277 278 if ((rx_cqe & MLXBF_GIGE_RX_CQE_PKT_STATUS_MASK) == 0) { 279 /* Packet is OK, increment stats */ 280 datalen = rx_cqe & MLXBF_GIGE_RX_CQE_PKT_LEN_MASK; 281 netdev->stats.rx_packets++; 282 netdev->stats.rx_bytes += datalen; 283 284 skb = priv->rx_skb[rx_pi_rem]; 285 286 /* Alloc another RX SKB for this same index */ 287 rx_skb = mlxbf_gige_alloc_skb(priv, MLXBF_GIGE_DEFAULT_BUF_SZ, 288 &rx_buf_dma, DMA_FROM_DEVICE); 289 if (!rx_skb) 290 return false; 291 priv->rx_skb[rx_pi_rem] = rx_skb; 292 dma_unmap_single(priv->dev, *rx_wqe_addr, 293 MLXBF_GIGE_DEFAULT_BUF_SZ, DMA_FROM_DEVICE); 294 295 skb_put(skb, datalen); 296 297 skb->ip_summed = CHECKSUM_NONE; /* device did not checksum packet */ 298 299 skb->protocol = eth_type_trans(skb, netdev); 300 301 *rx_wqe_addr = rx_buf_dma; 302 } else if (rx_cqe & MLXBF_GIGE_RX_CQE_PKT_STATUS_MAC_ERR) { 303 priv->stats.rx_mac_errors++; 304 } else if (rx_cqe & MLXBF_GIGE_RX_CQE_PKT_STATUS_TRUNCATED) { 305 priv->stats.rx_truncate_errors++; 306 } 307 308 /* Read receive consumer index before replenish so that this routine 309 * returns accurate return value even if packet is received into 310 * just-replenished buffer prior to exiting this routine. 311 */ 312 rx_ci = readq(priv->base + MLXBF_GIGE_RX_CQE_PACKET_CI); 313 rx_ci_rem = rx_ci % priv->rx_q_entries; 314 315 /* Let hardware know we've replenished one buffer */ 316 rx_pi++; 317 318 /* Ensure completion of all writes before notifying HW of replenish */ 319 wmb(); 320 writeq(rx_pi, priv->base + MLXBF_GIGE_RX_WQE_PI); 321 322 (*rx_pkts)++; 323 324 rx_pi_rem = rx_pi % priv->rx_q_entries; 325 if (rx_pi_rem == 0) 326 priv->valid_polarity ^= 1; 327 328 if (skb) 329 netif_receive_skb(skb); 330 331 return rx_pi_rem != rx_ci_rem; 332 } 333 334 /* Driver poll() function called by NAPI infrastructure */ 335 int mlxbf_gige_poll(struct napi_struct *napi, int budget) 336 { 337 struct mlxbf_gige *priv; 338 bool remaining_pkts; 339 int work_done = 0; 340 u64 data; 341 342 priv = container_of(napi, struct mlxbf_gige, napi); 343 344 mlxbf_gige_handle_tx_complete(priv); 345 346 do { 347 remaining_pkts = mlxbf_gige_rx_packet(priv, &work_done); 348 } while (remaining_pkts && work_done < budget); 349 350 /* If amount of work done < budget, turn off NAPI polling 351 * via napi_complete_done(napi, work_done) and then 352 * re-enable interrupts. 353 */ 354 if (work_done < budget && napi_complete_done(napi, work_done)) { 355 /* Clear MLXBF_GIGE_INT_MASK 'receive pkt' bit to 356 * indicate receive readiness 357 */ 358 data = readq(priv->base + MLXBF_GIGE_INT_MASK); 359 data &= ~MLXBF_GIGE_INT_MASK_RX_RECEIVE_PACKET; 360 writeq(data, priv->base + MLXBF_GIGE_INT_MASK); 361 } 362 363 return work_done; 364 } 365