1 /*- 2 * Copyright (c) 2015 Mellanox Technologies. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * 25 * $FreeBSD$ 26 */ 27 28 #include "en.h" 29 #include <machine/atomic.h> 30 31 void 32 mlx5e_send_nop(struct mlx5e_sq *sq, u32 ds_cnt, bool notify_hw) 33 { 34 u16 pi = sq->pc & sq->wq.sz_m1; 35 struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi); 36 37 memset(&wqe->ctrl, 0, sizeof(wqe->ctrl)); 38 39 wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_NOP); 40 wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt); 41 wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE; 42 43 sq->mbuf[pi].mbuf = NULL; 44 sq->mbuf[pi].num_bytes = 0; 45 sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS); 46 sq->pc += sq->mbuf[pi].num_wqebbs; 47 if (notify_hw) 48 mlx5e_tx_notify_hw(sq, wqe, 0); 49 } 50 51 #if (__FreeBSD_version >= 1100000) 52 static uint32_t mlx5e_hash_value; 53 54 static void 55 mlx5e_hash_init(void *arg) 56 { 57 mlx5e_hash_value = m_ether_tcpip_hash_init(); 58 } 59 60 /* Make kernel call mlx5e_hash_init after the random stack finished initializing */ 61 SYSINIT(mlx5e_hash_init, SI_SUB_RANDOM, SI_ORDER_ANY, &mlx5e_hash_init, NULL); 62 #endif 63 64 static struct mlx5e_sq * 65 mlx5e_select_queue(struct ifnet *ifp, struct mbuf *mb) 66 { 67 struct mlx5e_priv *priv = ifp->if_softc; 68 u32 ch; 69 u32 tc; 70 71 /* check if channels are successfully opened */ 72 if (unlikely(priv->channel == NULL)) 73 return (NULL); 74 75 /* obtain VLAN information if present */ 76 if (mb->m_flags & M_VLANTAG) { 77 tc = (mb->m_pkthdr.ether_vtag >> 13); 78 if (tc >= priv->num_tc) 79 tc = priv->default_vlan_prio; 80 } else { 81 tc = priv->default_vlan_prio; 82 } 83 84 ch = priv->params.num_channels; 85 86 /* check if flowid is set */ 87 if (M_HASHTYPE_GET(mb) != M_HASHTYPE_NONE) { 88 #ifdef RSS 89 u32 temp; 90 91 if (rss_hash2bucket(mb->m_pkthdr.flowid, 92 M_HASHTYPE_GET(mb), &temp) == 0) 93 ch = temp % ch; 94 else 95 #endif 96 ch = (mb->m_pkthdr.flowid % 128) % ch; 97 } else { 98 #if (__FreeBSD_version >= 1100000) 99 ch = m_ether_tcpip_hash(MBUF_HASHFLAG_L3 | 100 MBUF_HASHFLAG_L4, mb, mlx5e_hash_value) % ch; 101 #else 102 /* 103 * m_ether_tcpip_hash not present in stable, so just 104 * throw unhashed mbufs on queue 0 105 */ 106 ch = 0; 107 #endif 108 } 109 110 /* check if channel is allocated */ 111 if (unlikely(priv->channel[ch] == NULL)) 112 return (NULL); 113 114 return (&priv->channel[ch]->sq[tc]); 115 } 116 117 static inline u16 118 mlx5e_get_inline_hdr_size(struct mlx5e_sq *sq, struct mbuf *mb) 119 { 120 return (MIN(MLX5E_MAX_TX_INLINE, mb->m_len)); 121 } 122 123 static int 124 mlx5e_get_header_size(struct mbuf *mb) 125 { 126 struct ether_vlan_header *eh; 127 struct tcphdr *th; 128 struct ip *ip; 129 int ip_hlen, tcp_hlen; 130 struct ip6_hdr *ip6; 131 uint16_t eth_type; 132 int eth_hdr_len; 133 134 eh = mtod(mb, struct ether_vlan_header *); 135 if (mb->m_len < ETHER_HDR_LEN) 136 return (0); 137 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) { 138 eth_type = ntohs(eh->evl_proto); 139 eth_hdr_len = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN; 140 } else { 141 eth_type = ntohs(eh->evl_encap_proto); 142 eth_hdr_len = ETHER_HDR_LEN; 143 } 144 if (mb->m_len < eth_hdr_len) 145 return (0); 146 switch (eth_type) { 147 case ETHERTYPE_IP: 148 ip = (struct ip *)(mb->m_data + eth_hdr_len); 149 if (mb->m_len < eth_hdr_len + sizeof(*ip)) 150 return (0); 151 if (ip->ip_p != IPPROTO_TCP) 152 return (0); 153 ip_hlen = ip->ip_hl << 2; 154 eth_hdr_len += ip_hlen; 155 break; 156 case ETHERTYPE_IPV6: 157 ip6 = (struct ip6_hdr *)(mb->m_data + eth_hdr_len); 158 if (mb->m_len < eth_hdr_len + sizeof(*ip6)) 159 return (0); 160 if (ip6->ip6_nxt != IPPROTO_TCP) 161 return (0); 162 eth_hdr_len += sizeof(*ip6); 163 break; 164 default: 165 return (0); 166 } 167 if (mb->m_len < eth_hdr_len + sizeof(*th)) 168 return (0); 169 th = (struct tcphdr *)(mb->m_data + eth_hdr_len); 170 tcp_hlen = th->th_off << 2; 171 eth_hdr_len += tcp_hlen; 172 if (mb->m_len < eth_hdr_len) 173 return (0); 174 return (eth_hdr_len); 175 } 176 177 /* 178 * The return value is not going back to the stack because of 179 * the drbr 180 */ 181 static int 182 mlx5e_sq_xmit(struct mlx5e_sq *sq, struct mbuf **mbp) 183 { 184 bus_dma_segment_t segs[MLX5E_MAX_TX_MBUF_FRAGS]; 185 struct mlx5_wqe_data_seg *dseg; 186 struct mlx5e_tx_wqe *wqe; 187 struct ifnet *ifp; 188 int nsegs; 189 int err; 190 int x; 191 struct mbuf *mb = *mbp; 192 u16 ds_cnt; 193 u16 ihs; 194 u16 pi; 195 u8 opcode; 196 197 /* 198 * Return ENOBUFS if the queue is full, this may trigger reinsertion 199 * of the mbuf into the drbr (see mlx5e_xmit_locked) 200 */ 201 if (unlikely(!mlx5e_sq_has_room_for(sq, 2 * MLX5_SEND_WQE_MAX_WQEBBS))) { 202 return (ENOBUFS); 203 } 204 205 /* Align SQ edge with NOPs to avoid WQE wrap around */ 206 pi = ((~sq->pc) & sq->wq.sz_m1); 207 if (pi < (MLX5_SEND_WQE_MAX_WQEBBS - 1)) { 208 /* Send one multi NOP message instead of many */ 209 mlx5e_send_nop(sq, (pi + 1) * MLX5_SEND_WQEBB_NUM_DS, false); 210 pi = ((~sq->pc) & sq->wq.sz_m1); 211 if (pi < (MLX5_SEND_WQE_MAX_WQEBBS - 1)) { 212 m_freem(mb); 213 return (ENOMEM); 214 } 215 } 216 217 /* Setup local variables */ 218 pi = sq->pc & sq->wq.sz_m1; 219 wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi); 220 ifp = sq->channel->ifp; 221 222 memset(wqe, 0, sizeof(*wqe)); 223 224 /* Send a copy of the frame to the BPF listener, if any */ 225 if (ifp != NULL && ifp->if_bpf != NULL) 226 ETHER_BPF_MTAP(ifp, mb); 227 228 if (mb->m_pkthdr.csum_flags & (CSUM_IP | CSUM_TSO)) { 229 wqe->eth.cs_flags |= MLX5_ETH_WQE_L3_CSUM; 230 } 231 if (mb->m_pkthdr.csum_flags & (CSUM_TCP | CSUM_UDP | CSUM_UDP_IPV6 | CSUM_TCP_IPV6 | CSUM_TSO)) { 232 wqe->eth.cs_flags |= MLX5_ETH_WQE_L4_CSUM; 233 } 234 if (wqe->eth.cs_flags == 0) { 235 sq->stats.csum_offload_none++; 236 } 237 if (mb->m_pkthdr.csum_flags & CSUM_TSO) { 238 u32 payload_len; 239 u32 mss = mb->m_pkthdr.tso_segsz; 240 u32 num_pkts; 241 242 wqe->eth.mss = cpu_to_be16(mss); 243 opcode = MLX5_OPCODE_LSO; 244 ihs = mlx5e_get_header_size(mb); 245 payload_len = mb->m_pkthdr.len - ihs; 246 if (payload_len == 0) 247 num_pkts = 1; 248 else 249 num_pkts = DIV_ROUND_UP(payload_len, mss); 250 sq->mbuf[pi].num_bytes = payload_len + (num_pkts * ihs); 251 252 sq->stats.tso_packets++; 253 sq->stats.tso_bytes += payload_len; 254 } else { 255 opcode = MLX5_OPCODE_SEND; 256 ihs = mlx5e_get_inline_hdr_size(sq, mb); 257 sq->mbuf[pi].num_bytes = max_t (unsigned int, 258 mb->m_pkthdr.len, ETHER_MIN_LEN - ETHER_CRC_LEN); 259 } 260 if (mb->m_flags & M_VLANTAG) { 261 struct ether_vlan_header *eh = 262 (struct ether_vlan_header *)wqe->eth.inline_hdr_start; 263 264 /* Range checks */ 265 if (ihs > (MLX5E_MAX_TX_INLINE - ETHER_VLAN_ENCAP_LEN)) 266 ihs = (MLX5E_MAX_TX_INLINE - ETHER_VLAN_ENCAP_LEN); 267 else if (ihs < ETHER_HDR_LEN) { 268 err = EINVAL; 269 goto tx_drop; 270 } 271 m_copydata(mb, 0, ETHER_HDR_LEN, (caddr_t)eh); 272 m_adj(mb, ETHER_HDR_LEN); 273 /* Insert 4 bytes VLAN tag into data stream */ 274 eh->evl_proto = eh->evl_encap_proto; 275 eh->evl_encap_proto = htons(ETHERTYPE_VLAN); 276 eh->evl_tag = htons(mb->m_pkthdr.ether_vtag); 277 /* Copy rest of header data, if any */ 278 m_copydata(mb, 0, ihs - ETHER_HDR_LEN, (caddr_t)(eh + 1)); 279 m_adj(mb, ihs - ETHER_HDR_LEN); 280 /* Extend header by 4 bytes */ 281 ihs += ETHER_VLAN_ENCAP_LEN; 282 } else { 283 m_copydata(mb, 0, ihs, wqe->eth.inline_hdr_start); 284 m_adj(mb, ihs); 285 } 286 287 wqe->eth.inline_hdr_sz = cpu_to_be16(ihs); 288 289 ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS; 290 if (likely(ihs > sizeof(wqe->eth.inline_hdr_start))) { 291 ds_cnt += DIV_ROUND_UP(ihs - sizeof(wqe->eth.inline_hdr_start), 292 MLX5_SEND_WQE_DS); 293 } 294 dseg = ((struct mlx5_wqe_data_seg *)&wqe->ctrl) + ds_cnt; 295 296 /* Trim off empty mbufs */ 297 while (mb->m_len == 0) { 298 mb = m_free(mb); 299 /* Check if all data has been inlined */ 300 if (mb == NULL) 301 goto skip_dma; 302 } 303 304 err = bus_dmamap_load_mbuf_sg(sq->dma_tag, sq->mbuf[pi].dma_map, 305 mb, segs, &nsegs, BUS_DMA_NOWAIT); 306 if (err == EFBIG) { 307 /* 308 * Update *mbp before defrag in case it was trimmed in the 309 * loop above 310 */ 311 *mbp = mb; 312 /* Update statistics */ 313 sq->stats.defragged++; 314 /* Too many mbuf fragments */ 315 mb = m_defrag(*mbp, M_NOWAIT); 316 if (mb == NULL) { 317 mb = *mbp; 318 goto tx_drop; 319 } 320 /* Try again */ 321 err = bus_dmamap_load_mbuf_sg(sq->dma_tag, sq->mbuf[pi].dma_map, 322 mb, segs, &nsegs, BUS_DMA_NOWAIT); 323 } 324 /* Catch errors */ 325 if (err != 0) { 326 goto tx_drop; 327 } 328 *mbp = mb; 329 330 for (x = 0; x != nsegs; x++) { 331 if (segs[x].ds_len == 0) 332 continue; 333 dseg->addr = cpu_to_be64((uint64_t)segs[x].ds_addr); 334 dseg->lkey = sq->mkey_be; 335 dseg->byte_count = cpu_to_be32((uint32_t)segs[x].ds_len); 336 dseg++; 337 } 338 skip_dma: 339 ds_cnt = (dseg - ((struct mlx5_wqe_data_seg *)&wqe->ctrl)); 340 341 wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | opcode); 342 wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt); 343 wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE; 344 345 /* Store pointer to mbuf */ 346 sq->mbuf[pi].mbuf = mb; 347 sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS); 348 sq->pc += sq->mbuf[pi].num_wqebbs; 349 350 /* Make sure all mbuf data is written to RAM */ 351 if (mb != NULL) 352 bus_dmamap_sync(sq->dma_tag, sq->mbuf[pi].dma_map, BUS_DMASYNC_PREWRITE); 353 354 mlx5e_tx_notify_hw(sq, wqe, 0); 355 356 sq->stats.packets++; 357 return (0); 358 359 tx_drop: 360 sq->stats.dropped++; 361 *mbp = NULL; 362 m_freem(mb); 363 return err; 364 } 365 366 static void 367 mlx5e_poll_tx_cq(struct mlx5e_sq *sq, int budget) 368 { 369 u16 sqcc; 370 371 /* 372 * sq->cc must be updated only after mlx5_cqwq_update_db_record(), 373 * otherwise a cq overrun may occur 374 */ 375 sqcc = sq->cc; 376 377 while (budget--) { 378 struct mlx5_cqe64 *cqe; 379 struct mbuf *mb; 380 u16 ci; 381 382 cqe = mlx5e_get_cqe(&sq->cq); 383 if (!cqe) 384 break; 385 386 ci = sqcc & sq->wq.sz_m1; 387 mb = sq->mbuf[ci].mbuf; 388 sq->mbuf[ci].mbuf = NULL; /* Safety clear */ 389 390 if (mb == NULL) { 391 if (sq->mbuf[ci].num_bytes == 0) { 392 /* NOP */ 393 sq->stats.nop++; 394 } 395 } else { 396 bus_dmamap_sync(sq->dma_tag, sq->mbuf[ci].dma_map, 397 BUS_DMASYNC_POSTWRITE); 398 bus_dmamap_unload(sq->dma_tag, sq->mbuf[ci].dma_map); 399 400 /* Free transmitted mbuf */ 401 m_freem(mb); 402 } 403 sqcc += sq->mbuf[ci].num_wqebbs; 404 } 405 406 mlx5_cqwq_update_db_record(&sq->cq.wq); 407 408 /* Ensure cq space is freed before enabling more cqes */ 409 wmb(); 410 411 sq->cc = sqcc; 412 413 if (atomic_cmpset_int(&sq->queue_state, MLX5E_SQ_FULL, MLX5E_SQ_READY)) 414 taskqueue_enqueue(sq->sq_tq, &sq->sq_task); 415 } 416 417 static int 418 mlx5e_xmit_locked(struct ifnet *ifp, struct mlx5e_sq *sq, struct mbuf *mb) 419 { 420 struct mbuf *next; 421 int err = 0; 422 423 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) { 424 if (mb) 425 err = drbr_enqueue(ifp, sq->br, mb); 426 return (err); 427 } 428 429 if (mb != NULL) 430 /* 431 * If we can't insert mbuf into drbr, try to xmit anyway. 432 * We keep the error we got so we could return that after xmit. 433 */ 434 err = drbr_enqueue(ifp, sq->br, mb); 435 436 /* Process the queue */ 437 while ((next = drbr_peek(ifp, sq->br)) != NULL) { 438 if (mlx5e_sq_xmit(sq, &next) != 0) { 439 if (next == NULL) { 440 drbr_advance(ifp, sq->br); 441 } else { 442 drbr_putback(ifp, sq->br, next); 443 atomic_store_rel_int(&sq->queue_state, MLX5E_SQ_FULL); 444 } 445 break; 446 } 447 drbr_advance(ifp, sq->br); 448 if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) 449 break; 450 } 451 return (err); 452 } 453 454 int 455 mlx5e_xmit(struct ifnet *ifp, struct mbuf *mb) 456 { 457 struct mlx5e_sq *sq; 458 int ret; 459 460 sq = mlx5e_select_queue(ifp, mb); 461 if (unlikely(sq == NULL)) { 462 /* Invalid send queue */ 463 m_freem(mb); 464 return (ENXIO); 465 } 466 if (mtx_trylock(&sq->lock)) { 467 ret = mlx5e_xmit_locked(ifp, sq, mb); 468 mtx_unlock(&sq->lock); 469 } else { 470 ret = drbr_enqueue(ifp, sq->br, mb); 471 taskqueue_enqueue(sq->sq_tq, &sq->sq_task); 472 } 473 474 return (ret); 475 } 476 477 void 478 mlx5e_tx_cq_comp(struct mlx5_core_cq *mcq) 479 { 480 struct mlx5e_sq *sq = container_of(mcq, struct mlx5e_sq, cq.mcq); 481 482 mtx_lock(&sq->comp_lock); 483 mlx5e_poll_tx_cq(sq, MLX5E_BUDGET_MAX); 484 mlx5e_cq_arm(&sq->cq); 485 mtx_unlock(&sq->comp_lock); 486 } 487 488 void 489 mlx5e_tx_que(void *context, int pending) 490 { 491 struct mlx5e_sq *sq = context; 492 struct ifnet *ifp = sq->channel->ifp; 493 494 if (ifp->if_drv_flags & IFF_DRV_RUNNING) { 495 mtx_lock(&sq->lock); 496 if (!drbr_empty(ifp, sq->br)) 497 mlx5e_xmit_locked(ifp, sq, NULL); 498 mtx_unlock(&sq->lock); 499 } 500 } 501