1 /*- 2 * Copyright (c) 2015 Mellanox Technologies. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * 25 * $FreeBSD$ 26 */ 27 28 #include "en.h" 29 #include <machine/in_cksum.h> 30 31 static inline int 32 mlx5e_alloc_rx_wqe(struct mlx5e_rq *rq, 33 struct mlx5e_rx_wqe *wqe, u16 ix) 34 { 35 bus_dma_segment_t segs[1]; 36 struct mbuf *mb; 37 int nsegs; 38 int err; 39 40 if (rq->mbuf[ix].mbuf != NULL) 41 return (0); 42 43 mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, rq->wqe_sz); 44 if (unlikely(!mb)) 45 return (-ENOMEM); 46 47 /* set initial mbuf length */ 48 mb->m_pkthdr.len = mb->m_len = rq->wqe_sz; 49 50 /* get IP header aligned */ 51 m_adj(mb, MLX5E_NET_IP_ALIGN); 52 53 err = -bus_dmamap_load_mbuf_sg(rq->dma_tag, rq->mbuf[ix].dma_map, 54 mb, segs, &nsegs, BUS_DMA_NOWAIT); 55 if (err != 0) 56 goto err_free_mbuf; 57 if (unlikely(nsegs != 1)) { 58 bus_dmamap_unload(rq->dma_tag, rq->mbuf[ix].dma_map); 59 err = -ENOMEM; 60 goto err_free_mbuf; 61 } 62 wqe->data.addr = cpu_to_be64(segs[0].ds_addr); 63 64 rq->mbuf[ix].mbuf = mb; 65 rq->mbuf[ix].data = mb->m_data; 66 67 bus_dmamap_sync(rq->dma_tag, rq->mbuf[ix].dma_map, 68 BUS_DMASYNC_PREREAD); 69 return (0); 70 71 err_free_mbuf: 72 m_freem(mb); 73 return (err); 74 } 75 76 static void 77 mlx5e_post_rx_wqes(struct mlx5e_rq *rq) 78 { 79 if (unlikely(rq->enabled == 0)) 80 return; 81 82 while (!mlx5_wq_ll_is_full(&rq->wq)) { 83 struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, rq->wq.head); 84 85 if (unlikely(mlx5e_alloc_rx_wqe(rq, wqe, rq->wq.head))) { 86 callout_reset_curcpu(&rq->watchdog, 1, (void *)&mlx5e_post_rx_wqes, rq); 87 break; 88 } 89 mlx5_wq_ll_push(&rq->wq, be16_to_cpu(wqe->next.next_wqe_index)); 90 } 91 92 /* ensure wqes are visible to device before updating doorbell record */ 93 atomic_thread_fence_rel(); 94 95 mlx5_wq_ll_update_db_record(&rq->wq); 96 } 97 98 static void 99 mlx5e_lro_update_hdr(struct mbuf *mb, struct mlx5_cqe64 *cqe) 100 { 101 /* TODO: consider vlans, ip options, ... */ 102 struct ether_header *eh; 103 uint16_t eh_type; 104 uint16_t tot_len; 105 struct ip6_hdr *ip6 = NULL; 106 struct ip *ip4 = NULL; 107 struct tcphdr *th; 108 uint32_t *ts_ptr; 109 uint8_t l4_hdr_type; 110 int tcp_ack; 111 112 eh = mtod(mb, struct ether_header *); 113 eh_type = ntohs(eh->ether_type); 114 115 l4_hdr_type = get_cqe_l4_hdr_type(cqe); 116 tcp_ack = ((CQE_L4_HDR_TYPE_TCP_ACK_NO_DATA == l4_hdr_type) || 117 (CQE_L4_HDR_TYPE_TCP_ACK_AND_DATA == l4_hdr_type)); 118 119 /* TODO: consider vlan */ 120 tot_len = be32_to_cpu(cqe->byte_cnt) - ETHER_HDR_LEN; 121 122 switch (eh_type) { 123 case ETHERTYPE_IP: 124 ip4 = (struct ip *)(eh + 1); 125 th = (struct tcphdr *)(ip4 + 1); 126 break; 127 case ETHERTYPE_IPV6: 128 ip6 = (struct ip6_hdr *)(eh + 1); 129 th = (struct tcphdr *)(ip6 + 1); 130 break; 131 default: 132 return; 133 } 134 135 ts_ptr = (uint32_t *)(th + 1); 136 137 if (get_cqe_lro_tcppsh(cqe)) 138 th->th_flags |= TH_PUSH; 139 140 if (tcp_ack) { 141 th->th_flags |= TH_ACK; 142 th->th_ack = cqe->lro_ack_seq_num; 143 th->th_win = cqe->lro_tcp_win; 144 145 /* 146 * FreeBSD handles only 32bit aligned timestamp right after 147 * the TCP hdr 148 * +--------+--------+--------+--------+ 149 * | NOP | NOP | TSopt | 10 | 150 * +--------+--------+--------+--------+ 151 * | TSval timestamp | 152 * +--------+--------+--------+--------+ 153 * | TSecr timestamp | 154 * +--------+--------+--------+--------+ 155 */ 156 if (get_cqe_lro_timestamp_valid(cqe) && 157 (__predict_true(*ts_ptr) == ntohl(TCPOPT_NOP << 24 | 158 TCPOPT_NOP << 16 | TCPOPT_TIMESTAMP << 8 | 159 TCPOLEN_TIMESTAMP))) { 160 /* 161 * cqe->timestamp is 64bit long. 162 * [0-31] - timestamp. 163 * [32-64] - timestamp echo replay. 164 */ 165 ts_ptr[1] = *(uint32_t *)&cqe->timestamp; 166 ts_ptr[2] = *((uint32_t *)&cqe->timestamp + 1); 167 } 168 } 169 if (ip4) { 170 ip4->ip_ttl = cqe->lro_min_ttl; 171 ip4->ip_len = cpu_to_be16(tot_len); 172 ip4->ip_sum = 0; 173 ip4->ip_sum = in_cksum(mb, ip4->ip_hl << 2); 174 } else { 175 ip6->ip6_hlim = cqe->lro_min_ttl; 176 ip6->ip6_plen = cpu_to_be16(tot_len - 177 sizeof(struct ip6_hdr)); 178 } 179 /* TODO: handle tcp checksum */ 180 } 181 182 static uint64_t 183 mlx5e_mbuf_tstmp(struct mlx5e_priv *priv, uint64_t hw_tstmp) 184 { 185 struct mlx5e_clbr_point *cp; 186 uint64_t a1, a2, res; 187 u_int gen; 188 189 do { 190 cp = &priv->clbr_points[priv->clbr_curr]; 191 gen = atomic_load_acq_int(&cp->clbr_gen); 192 a1 = (hw_tstmp - cp->clbr_hw_prev) >> MLX5E_TSTMP_PREC; 193 a2 = (cp->base_curr - cp->base_prev) >> MLX5E_TSTMP_PREC; 194 res = (a1 * a2) << MLX5E_TSTMP_PREC; 195 196 /* 197 * Divisor cannot be zero because calibration callback 198 * checks for the condition and disables timestamping 199 * if clock halted. 200 */ 201 res /= (cp->clbr_hw_curr - cp->clbr_hw_prev) >> 202 MLX5E_TSTMP_PREC; 203 204 res += cp->base_prev; 205 atomic_thread_fence_acq(); 206 } while (gen == 0 || gen != cp->clbr_gen); 207 return (res); 208 } 209 210 static inline void 211 mlx5e_build_rx_mbuf(struct mlx5_cqe64 *cqe, 212 struct mlx5e_rq *rq, struct mbuf *mb, 213 u32 cqe_bcnt) 214 { 215 struct ifnet *ifp = rq->ifp; 216 struct mlx5e_channel *c; 217 int lro_num_seg; /* HW LRO session aggregated packets counter */ 218 uint64_t tstmp; 219 220 lro_num_seg = be32_to_cpu(cqe->srqn) >> 24; 221 if (lro_num_seg > 1) { 222 mlx5e_lro_update_hdr(mb, cqe); 223 rq->stats.lro_packets++; 224 rq->stats.lro_bytes += cqe_bcnt; 225 } 226 227 mb->m_pkthdr.len = mb->m_len = cqe_bcnt; 228 /* check if a Toeplitz hash was computed */ 229 if (cqe->rss_hash_type != 0) { 230 mb->m_pkthdr.flowid = be32_to_cpu(cqe->rss_hash_result); 231 #ifdef RSS 232 /* decode the RSS hash type */ 233 switch (cqe->rss_hash_type & 234 (CQE_RSS_DST_HTYPE_L4 | CQE_RSS_DST_HTYPE_IP)) { 235 /* IPv4 */ 236 case (CQE_RSS_DST_HTYPE_TCP | CQE_RSS_DST_HTYPE_IPV4): 237 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_TCP_IPV4); 238 break; 239 case (CQE_RSS_DST_HTYPE_UDP | CQE_RSS_DST_HTYPE_IPV4): 240 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_UDP_IPV4); 241 break; 242 case CQE_RSS_DST_HTYPE_IPV4: 243 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_IPV4); 244 break; 245 /* IPv6 */ 246 case (CQE_RSS_DST_HTYPE_TCP | CQE_RSS_DST_HTYPE_IPV6): 247 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_TCP_IPV6); 248 break; 249 case (CQE_RSS_DST_HTYPE_UDP | CQE_RSS_DST_HTYPE_IPV6): 250 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_UDP_IPV6); 251 break; 252 case CQE_RSS_DST_HTYPE_IPV6: 253 M_HASHTYPE_SET(mb, M_HASHTYPE_RSS_IPV6); 254 break; 255 default: /* Other */ 256 M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE_HASH); 257 break; 258 } 259 #else 260 M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE_HASH); 261 #endif 262 } else { 263 mb->m_pkthdr.flowid = rq->ix; 264 M_HASHTYPE_SET(mb, M_HASHTYPE_OPAQUE); 265 } 266 mb->m_pkthdr.rcvif = ifp; 267 268 if (likely(ifp->if_capenable & (IFCAP_RXCSUM | IFCAP_RXCSUM_IPV6)) && 269 ((cqe->hds_ip_ext & (CQE_L2_OK | CQE_L3_OK | CQE_L4_OK)) == 270 (CQE_L2_OK | CQE_L3_OK | CQE_L4_OK))) { 271 mb->m_pkthdr.csum_flags = 272 CSUM_IP_CHECKED | CSUM_IP_VALID | 273 CSUM_DATA_VALID | CSUM_PSEUDO_HDR; 274 mb->m_pkthdr.csum_data = htons(0xffff); 275 } else { 276 rq->stats.csum_none++; 277 } 278 279 if (cqe_has_vlan(cqe)) { 280 mb->m_pkthdr.ether_vtag = be16_to_cpu(cqe->vlan_info); 281 mb->m_flags |= M_VLANTAG; 282 } 283 284 c = container_of(rq, struct mlx5e_channel, rq); 285 if (c->priv->clbr_done >= 2) { 286 tstmp = mlx5e_mbuf_tstmp(c->priv, be64_to_cpu(cqe->timestamp)); 287 if ((tstmp & MLX5_CQE_TSTMP_PTP) != 0) { 288 /* 289 * Timestamp was taken on the packet entrance, 290 * instead of the cqe generation. 291 */ 292 tstmp &= ~MLX5_CQE_TSTMP_PTP; 293 mb->m_flags |= M_TSTMP_HPREC; 294 } 295 mb->m_pkthdr.rcv_tstmp = tstmp; 296 mb->m_flags |= M_TSTMP; 297 } 298 } 299 300 static inline void 301 mlx5e_read_cqe_slot(struct mlx5e_cq *cq, u32 cc, void *data) 302 { 303 memcpy(data, mlx5_cqwq_get_wqe(&cq->wq, (cc & cq->wq.sz_m1)), 304 sizeof(struct mlx5_cqe64)); 305 } 306 307 static inline void 308 mlx5e_write_cqe_slot(struct mlx5e_cq *cq, u32 cc, void *data) 309 { 310 memcpy(mlx5_cqwq_get_wqe(&cq->wq, cc & cq->wq.sz_m1), 311 data, sizeof(struct mlx5_cqe64)); 312 } 313 314 static inline void 315 mlx5e_decompress_cqe(struct mlx5e_cq *cq, struct mlx5_cqe64 *title, 316 struct mlx5_mini_cqe8 *mini, 317 u16 wqe_counter, int i) 318 { 319 /* 320 * NOTE: The fields which are not set here are copied from the 321 * initial and common title. See memcpy() in 322 * mlx5e_write_cqe_slot(). 323 */ 324 title->byte_cnt = mini->byte_cnt; 325 title->wqe_counter = cpu_to_be16((wqe_counter + i) & cq->wq.sz_m1); 326 title->check_sum = mini->checksum; 327 title->op_own = (title->op_own & 0xf0) | 328 (((cq->wq.cc + i) >> cq->wq.log_sz) & 1); 329 } 330 331 #define MLX5E_MINI_ARRAY_SZ 8 332 /* Make sure structs are not packet differently */ 333 CTASSERT(sizeof(struct mlx5_cqe64) == 334 sizeof(struct mlx5_mini_cqe8) * MLX5E_MINI_ARRAY_SZ); 335 static void 336 mlx5e_decompress_cqes(struct mlx5e_cq *cq) 337 { 338 struct mlx5_mini_cqe8 mini_array[MLX5E_MINI_ARRAY_SZ]; 339 struct mlx5_cqe64 title; 340 u32 cqe_count; 341 u32 i = 0; 342 u16 title_wqe_counter; 343 344 mlx5e_read_cqe_slot(cq, cq->wq.cc, &title); 345 title_wqe_counter = be16_to_cpu(title.wqe_counter); 346 cqe_count = be32_to_cpu(title.byte_cnt); 347 348 /* Make sure we won't overflow */ 349 KASSERT(cqe_count <= cq->wq.sz_m1, 350 ("%s: cqe_count %u > cq->wq.sz_m1 %u", __func__, 351 cqe_count, cq->wq.sz_m1)); 352 353 mlx5e_read_cqe_slot(cq, cq->wq.cc + 1, mini_array); 354 while (true) { 355 mlx5e_decompress_cqe(cq, &title, 356 &mini_array[i % MLX5E_MINI_ARRAY_SZ], 357 title_wqe_counter, i); 358 mlx5e_write_cqe_slot(cq, cq->wq.cc + i, &title); 359 i++; 360 361 if (i == cqe_count) 362 break; 363 if (i % MLX5E_MINI_ARRAY_SZ == 0) 364 mlx5e_read_cqe_slot(cq, cq->wq.cc + i, mini_array); 365 } 366 } 367 368 static int 369 mlx5e_poll_rx_cq(struct mlx5e_rq *rq, int budget) 370 { 371 int i; 372 373 for (i = 0; i < budget; i++) { 374 struct mlx5e_rx_wqe *wqe; 375 struct mlx5_cqe64 *cqe; 376 struct mbuf *mb; 377 __be16 wqe_counter_be; 378 u16 wqe_counter; 379 u32 byte_cnt; 380 381 cqe = mlx5e_get_cqe(&rq->cq); 382 if (!cqe) 383 break; 384 385 if (mlx5_get_cqe_format(cqe) == MLX5_COMPRESSED) 386 mlx5e_decompress_cqes(&rq->cq); 387 388 mlx5_cqwq_pop(&rq->cq.wq); 389 390 wqe_counter_be = cqe->wqe_counter; 391 wqe_counter = be16_to_cpu(wqe_counter_be); 392 wqe = mlx5_wq_ll_get_wqe(&rq->wq, wqe_counter); 393 byte_cnt = be32_to_cpu(cqe->byte_cnt); 394 395 bus_dmamap_sync(rq->dma_tag, 396 rq->mbuf[wqe_counter].dma_map, 397 BUS_DMASYNC_POSTREAD); 398 399 if (unlikely((cqe->op_own >> 4) != MLX5_CQE_RESP_SEND)) { 400 rq->stats.wqe_err++; 401 goto wq_ll_pop; 402 } 403 if ((MHLEN - MLX5E_NET_IP_ALIGN) >= byte_cnt && 404 (mb = m_gethdr(M_NOWAIT, MT_DATA)) != NULL) { 405 /* get IP header aligned */ 406 mb->m_data += MLX5E_NET_IP_ALIGN; 407 408 bcopy(rq->mbuf[wqe_counter].data, mtod(mb, caddr_t), 409 byte_cnt); 410 } else { 411 mb = rq->mbuf[wqe_counter].mbuf; 412 rq->mbuf[wqe_counter].mbuf = NULL; /* safety clear */ 413 414 bus_dmamap_unload(rq->dma_tag, 415 rq->mbuf[wqe_counter].dma_map); 416 } 417 418 mlx5e_build_rx_mbuf(cqe, rq, mb, byte_cnt); 419 rq->stats.packets++; 420 421 #if !defined(HAVE_TCP_LRO_RX) 422 tcp_lro_queue_mbuf(&rq->lro, mb); 423 #else 424 if (mb->m_pkthdr.csum_flags == 0 || 425 (rq->ifp->if_capenable & IFCAP_LRO) == 0 || 426 rq->lro.lro_cnt == 0 || 427 tcp_lro_rx(&rq->lro, mb, 0) != 0) { 428 rq->ifp->if_input(rq->ifp, mb); 429 } 430 #endif 431 wq_ll_pop: 432 mlx5_wq_ll_pop(&rq->wq, wqe_counter_be, 433 &wqe->next.next_wqe_index); 434 } 435 436 mlx5_cqwq_update_db_record(&rq->cq.wq); 437 438 /* ensure cq space is freed before enabling more cqes */ 439 atomic_thread_fence_rel(); 440 return (i); 441 } 442 443 void 444 mlx5e_rx_cq_comp(struct mlx5_core_cq *mcq) 445 { 446 struct mlx5e_rq *rq = container_of(mcq, struct mlx5e_rq, cq.mcq); 447 int i = 0; 448 449 #ifdef HAVE_PER_CQ_EVENT_PACKET 450 struct mbuf *mb = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, rq->wqe_sz); 451 452 if (mb != NULL) { 453 /* this code is used for debugging purpose only */ 454 mb->m_pkthdr.len = mb->m_len = 15; 455 memset(mb->m_data, 255, 14); 456 mb->m_data[14] = rq->ix; 457 mb->m_pkthdr.rcvif = rq->ifp; 458 rq->ifp->if_input(rq->ifp, mb); 459 } 460 #endif 461 462 mtx_lock(&rq->mtx); 463 464 /* 465 * Polling the entire CQ without posting new WQEs results in 466 * lack of receive WQEs during heavy traffic scenarios. 467 */ 468 while (1) { 469 if (mlx5e_poll_rx_cq(rq, MLX5E_RX_BUDGET_MAX) != 470 MLX5E_RX_BUDGET_MAX) 471 break; 472 i += MLX5E_RX_BUDGET_MAX; 473 if (i >= MLX5E_BUDGET_MAX) 474 break; 475 mlx5e_post_rx_wqes(rq); 476 } 477 mlx5e_post_rx_wqes(rq); 478 mlx5e_cq_arm(&rq->cq, MLX5_GET_DOORBELL_LOCK(&rq->channel->priv->doorbell_lock)); 479 tcp_lro_flush_all(&rq->lro); 480 mtx_unlock(&rq->mtx); 481 } 482