1 /*- 2 * Copyright (c) 2015 Mellanox Technologies. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * 25 * $FreeBSD$ 26 */ 27 28 #include "en.h" 29 #include <machine/in_cksum.h> 30 31 static inline int 32 mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, 33 struct mlx5e_rx_wqe *wqe, u16 ix) 34 { 35 bus_dma_segment_t segs[1]; 36 struct mbuf *mb; 37 int nsegs; 38 int err; 39 40 if (rq->mbuf[ix].mbuf != NULL) 41 return (0); 42 43 mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, rq->wqe_sz); 44 if (unlikely(!mb)) 45 return (-ENOMEM); 46 47 /* set initial mbuf length */ 48 mb->m_pkthdr.len = mb->m_len = rq->wqe_sz; 49 50 /* get IP header aligned */ 51 m_adj(mb, MLX5E_NET_IP_ALIGN); 52 53 err = -bus_dmamap_load_mbuf_sg(rq->dma_tag, rq->mbuf[ix].dma_map, 54 mb, segs, &nsegs, BUS_DMA_NOWAIT); 55 if (err != 0) 56 goto err_free_mbuf; 57 if (unlikely(nsegs != 1)) { 58 bus_dmamap_unload(rq->dma_tag, rq->mbuf[ix].dma_map); 59 err = -ENOMEM; 60 goto err_free_mbuf; 61 } 62 wqe->data.addr = cpu_to_be64(segs[0].ds_addr); 63 64 rq->mbuf[ix].mbuf = mb; 65 rq->mbuf[ix].data = mb->m_data; 66 67 bus_dmamap_sync(rq->dma_tag, rq->mbuf[ix].dma_map, 68 BUS_DMASYNC_PREREAD); 69 return (0); 70 71 err_free_mbuf: 72 m_freem(mb); 73 return (err); 74 } 75 76 static void 77 mlx5e_post_rx_wqes(struct mlx5e_rq *rq) 78 { 79 if (unlikely(rq->enabled == 0)) 80 return; 81 82 while (!mlx5_wq_ll_is_full(&rq->wq)) { 83 struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, rq->wq.head); 84 85 if (unlikely(mlx5e_alloc_rx_wqe(rq, wqe, rq->wq.head))) 86 break; 87 88 mlx5_wq_ll_push(&rq->wq, be16_to_cpu(wqe->next.next_wqe_index)); 89 } 90 91 /* ensure wqes are visible to device before updating doorbell record */ 92 wmb(); 93 94 mlx5_wq_ll_update_db_record(&rq->wq); 95 } 96 97 static void 98 mlx5e_lro_update_hdr(struct mbuf *mb, struct mlx5_cqe64 *cqe) 99 { 100 /* TODO: consider vlans, ip options, ... */ 101 struct ether_header *eh; 102 uint16_t eh_type; 103 struct ip6_hdr *ip6 = NULL; 104 struct ip *ip4 = NULL; 105 struct tcphdr *th; 106 uint32_t *ts_ptr; 107 108 eh = mtod(mb, struct ether_header *); 109 eh_type = ntohs(eh->ether_type); 110 111 u8 l4_hdr_type = get_cqe_l4_hdr_type(cqe); 112 int tcp_ack = ((CQE_L4_HDR_TYPE_TCP_ACK_NO_DATA == l4_hdr_type) || 113 (CQE_L4_HDR_TYPE_TCP_ACK_AND_DATA == l4_hdr_type)); 114 115 /* TODO: consider vlan */ 116 u16 tot_len = be32_to_cpu(cqe->byte_cnt) - ETHER_HDR_LEN; 117 118 switch (eh_type) { 119 case ETHERTYPE_IP: 120 ip4 = (struct ip *)(eh + 1); 121 th = (struct tcphdr *)(ip4 + 1); 122 break; 123 case ETHERTYPE_IPV6: 124 ip6 = (struct ip6_hdr *)(eh + 1); 125 th = (struct tcphdr *)(ip6 + 1); 126 break; 127 default: 128 return; 129 } 130 131 ts_ptr = (uint32_t *)(th + 1); 132 133 if (get_cqe_lro_tcppsh(cqe)) 134 th->th_flags |= TH_PUSH; 135 136 if (tcp_ack) { 137 th->th_flags |= TH_ACK; 138 th->th_ack = cqe->lro_ack_seq_num; 139 th->th_win = cqe->lro_tcp_win; 140 141 /* 142 * FreeBSD handles only 32bit aligned timestamp right after 143 * the TCP hdr 144 * +--------+--------+--------+--------+ 145 * | NOP | NOP | TSopt | 10 | 146 * +--------+--------+--------+--------+ 147 * | TSval timestamp | 148 * +--------+--------+--------+--------+ 149 * | TSecr timestamp | 150 * +--------+--------+--------+--------+ 151 */ 152 if (get_cqe_lro_timestamp_valid(cqe) && 153 (__predict_true(*ts_ptr) == ntohl(TCPOPT_NOP << 24 | 154 TCPOPT_NOP << 16 | TCPOPT_TIMESTAMP << 8 | 155 TCPOLEN_TIMESTAMP))) { 156 /* 157 * cqe->timestamp is 64bit long. 158 * [0-31] - timestamp. 159 * [32-64] - timestamp echo replay. 160 */ 161 ts_ptr[1] = *(uint32_t *)&cqe->timestamp; 162 ts_ptr[2] = *((uint32_t *)&cqe->timestamp + 1); 163 } 164 } 165 if (ip4) { 166 ip4->ip_ttl = cqe->lro_min_ttl; 167 ip4->ip_len = cpu_to_be16(tot_len); 168 ip4->ip_sum = 0; 169 ip4->ip_sum = in_cksum(mb, ip4->ip_hl << 2); 170 } else { 171 ip6->ip6_hlim = cqe->lro_min_ttl; 172 ip6->ip6_plen = cpu_to_be16(tot_len - 173 sizeof(struct ip6_hdr)); 174 } 175 /* TODO: handle tcp checksum */ 176 } 177 178 static inline void 179 mlx5e_build_rx_mbuf(struct mlx5_cqe64 *cqe, 180 struct mlx5e_rq *rq, struct mbuf *mb, 181 u32 cqe_bcnt) 182 { 183 struct ifnet *ifp = rq->ifp; 184 int lro_num_seg; /* HW LRO session aggregated packets counter */ 185 186 lro_num_seg = be32_to_cpu(cqe->srqn) >> 24; 187 if (lro_num_seg > 1) { 188 mlx5e_lro_update_hdr(mb, cqe); 189 rq->stats.lro_packets++; 190 rq->stats.lro_bytes += cqe_bcnt; 191 } 192 193 mb->m_pkthdr.len = mb->m_len = cqe_bcnt; 194 /* check if a Toeplitz hash was computed */ 195 if (cqe->rss_hash_type != 0) 196 mb->m_pkthdr.flowid = be32_to_cpu(cqe->rss_hash_result); 197 else 198 mb->m_pkthdr.flowid = rq->ix; 199 200 M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE); 201 mb->m_pkthdr.rcvif = ifp; 202 203 if (likely(ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) && 204 ((cqe->hds_ip_ext & (CQE_L2_OK | CQE_L3_OK | CQE_L4_OK)) == 205 (CQE_L2_OK | CQE_L3_OK | CQE_L4_OK))) { 206 mb->m_pkthdr.csum_flags = 207 CSUM_IP_CHECKED | CSUM_IP_VALID | 208 CSUM_DATA_VALID | CSUM_PSEUDO_HDR; 209 mb->m_pkthdr.csum_data = htons(0xffff); 210 } else { 211 rq->stats.csum_none++; 212 } 213 214 if (cqe_has_vlan(cqe)) { 215 mb->m_pkthdr.ether_vtag = be16_to_cpu(cqe->vlan_info); 216 mb->m_flags |= M_VLANTAG; 217 } 218 } 219 220 static int 221 mlx5e_poll_rx_cq(struct mlx5e_rq *rq, int budget) 222 { 223 #ifndef HAVE_TURBO_LRO 224 struct lro_entry *queued; 225 #endif 226 int i; 227 228 for (i = 0; i < budget; i++) { 229 struct mlx5e_rx_wqe *wqe; 230 struct mlx5_cqe64 *cqe; 231 struct mbuf *mb; 232 __be16 wqe_counter_be; 233 u16 wqe_counter; 234 u32 byte_cnt; 235 236 cqe = mlx5e_get_cqe(&rq->cq); 237 if (!cqe) 238 break; 239 240 wqe_counter_be = cqe->wqe_counter; 241 wqe_counter = be16_to_cpu(wqe_counter_be); 242 wqe = mlx5_wq_ll_get_wqe(&rq->wq, wqe_counter); 243 byte_cnt = be32_to_cpu(cqe->byte_cnt); 244 245 bus_dmamap_sync(rq->dma_tag, 246 rq->mbuf[wqe_counter].dma_map, 247 BUS_DMASYNC_POSTREAD); 248 249 if (unlikely((cqe->op_own >> 4) != MLX5_CQE_RESP_SEND)) { 250 rq->stats.wqe_err++; 251 goto wq_ll_pop; 252 } 253 254 if (MHLEN >= byte_cnt && 255 (mb = m_gethdr(M_NOWAIT, MT_DATA)) != NULL) { 256 bcopy(rq->mbuf[wqe_counter].data, mtod(mb, caddr_t), 257 byte_cnt); 258 } else { 259 mb = rq->mbuf[wqe_counter].mbuf; 260 rq->mbuf[wqe_counter].mbuf = NULL; /* safety clear */ 261 262 bus_dmamap_unload(rq->dma_tag, 263 rq->mbuf[wqe_counter].dma_map); 264 } 265 266 mlx5e_build_rx_mbuf(cqe, rq, mb, byte_cnt); 267 rq->stats.packets++; 268 #ifdef HAVE_TURBO_LRO 269 if (mb->m_pkthdr.csum_flags == 0 || 270 (rq->ifp->if_capenable & IFCAP_LRO) == 0 || 271 rq->lro.mbuf == NULL) { 272 /* normal input */ 273 rq->ifp->if_input(rq->ifp, mb); 274 } else { 275 tcp_tlro_rx(&rq->lro, mb); 276 } 277 #else 278 if (mb->m_pkthdr.csum_flags == 0 || 279 (rq->ifp->if_capenable & IFCAP_LRO) == 0 || 280 rq->lro.lro_cnt == 0 || 281 tcp_lro_rx(&rq->lro, mb, 0) != 0) { 282 rq->ifp->if_input(rq->ifp, mb); 283 } 284 #endif 285 wq_ll_pop: 286 mlx5_wq_ll_pop(&rq->wq, wqe_counter_be, 287 &wqe->next.next_wqe_index); 288 } 289 290 mlx5_cqwq_update_db_record(&rq->cq.wq); 291 292 /* ensure cq space is freed before enabling more cqes */ 293 wmb(); 294 #ifndef HAVE_TURBO_LRO 295 while ((queued = SLIST_FIRST(&rq->lro.lro_active)) != NULL) { 296 SLIST_REMOVE_HEAD(&rq->lro.lro_active, next); 297 tcp_lro_flush(&rq->lro, queued); 298 } 299 #endif 300 return (i); 301 } 302 303 void 304 mlx5e_rx_cq_comp(struct mlx5_core_cq *mcq) 305 { 306 struct mlx5e_rq *rq = container_of(mcq, struct mlx5e_rq, cq.mcq); 307 int i = 0; 308 309 #ifdef HAVE_PER_CQ_EVENT_PACKET 310 struct mbuf *mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, rq->wqe_sz); 311 312 if (mb != NULL) { 313 /* this code is used for debugging purpose only */ 314 mb->m_pkthdr.len = mb->m_len = 15; 315 memset(mb->m_data, 255, 14); 316 mb->m_data[14] = rq->ix; 317 mb->m_pkthdr.rcvif = rq->ifp; 318 rq->ifp->if_input(rq->ifp, mb); 319 } 320 #endif 321 322 mtx_lock(&rq->mtx); 323 324 /* 325 * Polling the entire CQ without posting new WQEs results in 326 * lack of receive WQEs during heavy traffic scenarios. 327 */ 328 while (1) { 329 if (mlx5e_poll_rx_cq(rq, MLX5E_RX_BUDGET_MAX) != 330 MLX5E_RX_BUDGET_MAX) 331 break; 332 i += MLX5E_RX_BUDGET_MAX; 333 if (i >= MLX5E_BUDGET_MAX) 334 break; 335 mlx5e_post_rx_wqes(rq); 336 } 337 mlx5e_post_rx_wqes(rq); 338 mlx5e_cq_arm(&rq->cq); 339 #ifdef HAVE_TURBO_LRO 340 tcp_tlro_flush(&rq->lro, 1); 341 #endif 342 mtx_unlock(&rq->mtx); 343 } 344