1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright(c) 2013 - 2018 Intel Corporation. */ 3 4 #include <linux/prefetch.h> 5 #include <linux/bpf_trace.h> 6 #include <net/xdp.h> 7 #include "i40e.h" 8 #include "i40e_trace.h" 9 #include "i40e_prototype.h" 10 #include "i40e_txrx_common.h" 11 #include "i40e_xsk.h" 12 13 #define I40E_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS) 14 /** 15 * i40e_fdir - Generate a Flow Director descriptor based on fdata 16 * @tx_ring: Tx ring to send buffer on 17 * @fdata: Flow director filter data 18 * @add: Indicate if we are adding a rule or deleting one 19 * 20 **/ 21 static void i40e_fdir(struct i40e_ring *tx_ring, 22 struct i40e_fdir_filter *fdata, bool add) 23 { 24 struct i40e_filter_program_desc *fdir_desc; 25 struct i40e_pf *pf = tx_ring->vsi->back; 26 u32 flex_ptype, dtype_cmd; 27 u16 i; 28 29 /* grab the next descriptor */ 30 i = tx_ring->next_to_use; 31 fdir_desc = I40E_TX_FDIRDESC(tx_ring, i); 32 33 i++; 34 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 35 36 flex_ptype = I40E_TXD_FLTR_QW0_QINDEX_MASK & 37 (fdata->q_index << I40E_TXD_FLTR_QW0_QINDEX_SHIFT); 38 39 flex_ptype |= I40E_TXD_FLTR_QW0_FLEXOFF_MASK & 40 (fdata->flex_off << I40E_TXD_FLTR_QW0_FLEXOFF_SHIFT); 41 42 flex_ptype |= I40E_TXD_FLTR_QW0_PCTYPE_MASK & 43 (fdata->pctype << I40E_TXD_FLTR_QW0_PCTYPE_SHIFT); 44 45 flex_ptype |= I40E_TXD_FLTR_QW0_PCTYPE_MASK & 46 (fdata->flex_offset << I40E_TXD_FLTR_QW0_FLEXOFF_SHIFT); 47 48 /* Use LAN VSI Id if not programmed by user */ 49 flex_ptype |= I40E_TXD_FLTR_QW0_DEST_VSI_MASK & 50 ((u32)(fdata->dest_vsi ? : pf->vsi[pf->lan_vsi]->id) << 51 I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT); 52 53 dtype_cmd = I40E_TX_DESC_DTYPE_FILTER_PROG; 54 55 dtype_cmd |= add ? 56 I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE << 57 I40E_TXD_FLTR_QW1_PCMD_SHIFT : 58 I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE << 59 I40E_TXD_FLTR_QW1_PCMD_SHIFT; 60 61 dtype_cmd |= I40E_TXD_FLTR_QW1_DEST_MASK & 62 (fdata->dest_ctl << I40E_TXD_FLTR_QW1_DEST_SHIFT); 63 64 dtype_cmd |= I40E_TXD_FLTR_QW1_FD_STATUS_MASK & 65 (fdata->fd_status << I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT); 66 67 if (fdata->cnt_index) { 68 dtype_cmd |= I40E_TXD_FLTR_QW1_CNT_ENA_MASK; 69 dtype_cmd |= I40E_TXD_FLTR_QW1_CNTINDEX_MASK & 70 ((u32)fdata->cnt_index << 71 I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT); 72 } 73 74 fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32(flex_ptype); 75 fdir_desc->rsvd = cpu_to_le32(0); 76 fdir_desc->dtype_cmd_cntindex = cpu_to_le32(dtype_cmd); 77 fdir_desc->fd_id = cpu_to_le32(fdata->fd_id); 78 } 79 80 #define I40E_FD_CLEAN_DELAY 10 81 /** 82 * i40e_program_fdir_filter - Program a Flow Director filter 83 * @fdir_data: Packet data that will be filter parameters 84 * @raw_packet: the pre-allocated packet buffer for FDir 85 * @pf: The PF pointer 86 * @add: True for add/update, False for remove 87 **/ 88 static int i40e_program_fdir_filter(struct i40e_fdir_filter *fdir_data, 89 u8 *raw_packet, struct i40e_pf *pf, 90 bool add) 91 { 92 struct i40e_tx_buffer *tx_buf, *first; 93 struct i40e_tx_desc *tx_desc; 94 struct i40e_ring *tx_ring; 95 struct i40e_vsi *vsi; 96 struct device *dev; 97 dma_addr_t dma; 98 u32 td_cmd = 0; 99 u16 i; 100 101 /* find existing FDIR VSI */ 102 vsi = i40e_find_vsi_by_type(pf, I40E_VSI_FDIR); 103 if (!vsi) 104 return -ENOENT; 105 106 tx_ring = vsi->tx_rings[0]; 107 dev = tx_ring->dev; 108 109 /* we need two descriptors to add/del a filter and we can wait */ 110 for (i = I40E_FD_CLEAN_DELAY; I40E_DESC_UNUSED(tx_ring) < 2; i--) { 111 if (!i) 112 return -EAGAIN; 113 msleep_interruptible(1); 114 } 115 116 dma = dma_map_single(dev, raw_packet, 117 I40E_FDIR_MAX_RAW_PACKET_SIZE, DMA_TO_DEVICE); 118 if (dma_mapping_error(dev, dma)) 119 goto dma_fail; 120 121 /* grab the next descriptor */ 122 i = tx_ring->next_to_use; 123 first = &tx_ring->tx_bi[i]; 124 i40e_fdir(tx_ring, fdir_data, add); 125 126 /* Now program a dummy descriptor */ 127 i = tx_ring->next_to_use; 128 tx_desc = I40E_TX_DESC(tx_ring, i); 129 tx_buf = &tx_ring->tx_bi[i]; 130 131 tx_ring->next_to_use = ((i + 1) < tx_ring->count) ? i + 1 : 0; 132 133 memset(tx_buf, 0, sizeof(struct i40e_tx_buffer)); 134 135 /* record length, and DMA address */ 136 dma_unmap_len_set(tx_buf, len, I40E_FDIR_MAX_RAW_PACKET_SIZE); 137 dma_unmap_addr_set(tx_buf, dma, dma); 138 139 tx_desc->buffer_addr = cpu_to_le64(dma); 140 td_cmd = I40E_TXD_CMD | I40E_TX_DESC_CMD_DUMMY; 141 142 tx_buf->tx_flags = I40E_TX_FLAGS_FD_SB; 143 tx_buf->raw_buf = (void *)raw_packet; 144 145 tx_desc->cmd_type_offset_bsz = 146 build_ctob(td_cmd, 0, I40E_FDIR_MAX_RAW_PACKET_SIZE, 0); 147 148 /* Force memory writes to complete before letting h/w 149 * know there are new descriptors to fetch. 150 */ 151 wmb(); 152 153 /* Mark the data descriptor to be watched */ 154 first->next_to_watch = tx_desc; 155 156 writel(tx_ring->next_to_use, tx_ring->tail); 157 return 0; 158 159 dma_fail: 160 return -1; 161 } 162 163 #define IP_HEADER_OFFSET 14 164 #define I40E_UDPIP_DUMMY_PACKET_LEN 42 165 /** 166 * i40e_add_del_fdir_udpv4 - Add/Remove UDPv4 filters 167 * @vsi: pointer to the targeted VSI 168 * @fd_data: the flow director data required for the FDir descriptor 169 * @add: true adds a filter, false removes it 170 * 171 * Returns 0 if the filters were successfully added or removed 172 **/ 173 static int i40e_add_del_fdir_udpv4(struct i40e_vsi *vsi, 174 struct i40e_fdir_filter *fd_data, 175 bool add) 176 { 177 struct i40e_pf *pf = vsi->back; 178 struct udphdr *udp; 179 struct iphdr *ip; 180 u8 *raw_packet; 181 int ret; 182 static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0, 183 0x45, 0, 0, 0x1c, 0, 0, 0x40, 0, 0x40, 0x11, 0, 0, 0, 0, 0, 0, 184 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; 185 186 raw_packet = kzalloc(I40E_FDIR_MAX_RAW_PACKET_SIZE, GFP_KERNEL); 187 if (!raw_packet) 188 return -ENOMEM; 189 memcpy(raw_packet, packet, I40E_UDPIP_DUMMY_PACKET_LEN); 190 191 ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET); 192 udp = (struct udphdr *)(raw_packet + IP_HEADER_OFFSET 193 + sizeof(struct iphdr)); 194 195 ip->daddr = fd_data->dst_ip; 196 udp->dest = fd_data->dst_port; 197 ip->saddr = fd_data->src_ip; 198 udp->source = fd_data->src_port; 199 200 if (fd_data->flex_filter) { 201 u8 *payload = raw_packet + I40E_UDPIP_DUMMY_PACKET_LEN; 202 __be16 pattern = fd_data->flex_word; 203 u16 off = fd_data->flex_offset; 204 205 *((__force __be16 *)(payload + off)) = pattern; 206 } 207 208 fd_data->pctype = I40E_FILTER_PCTYPE_NONF_IPV4_UDP; 209 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add); 210 if (ret) { 211 dev_info(&pf->pdev->dev, 212 "PCTYPE:%d, Filter command send failed for fd_id:%d (ret = %d)\n", 213 fd_data->pctype, fd_data->fd_id, ret); 214 /* Free the packet buffer since it wasn't added to the ring */ 215 kfree(raw_packet); 216 return -EOPNOTSUPP; 217 } else if (I40E_DEBUG_FD & pf->hw.debug_mask) { 218 if (add) 219 dev_info(&pf->pdev->dev, 220 "Filter OK for PCTYPE %d loc = %d\n", 221 fd_data->pctype, fd_data->fd_id); 222 else 223 dev_info(&pf->pdev->dev, 224 "Filter deleted for PCTYPE %d loc = %d\n", 225 fd_data->pctype, fd_data->fd_id); 226 } 227 228 if (add) 229 pf->fd_udp4_filter_cnt++; 230 else 231 pf->fd_udp4_filter_cnt--; 232 233 return 0; 234 } 235 236 #define I40E_TCPIP_DUMMY_PACKET_LEN 54 237 /** 238 * i40e_add_del_fdir_tcpv4 - Add/Remove TCPv4 filters 239 * @vsi: pointer to the targeted VSI 240 * @fd_data: the flow director data required for the FDir descriptor 241 * @add: true adds a filter, false removes it 242 * 243 * Returns 0 if the filters were successfully added or removed 244 **/ 245 static int i40e_add_del_fdir_tcpv4(struct i40e_vsi *vsi, 246 struct i40e_fdir_filter *fd_data, 247 bool add) 248 { 249 struct i40e_pf *pf = vsi->back; 250 struct tcphdr *tcp; 251 struct iphdr *ip; 252 u8 *raw_packet; 253 int ret; 254 /* Dummy packet */ 255 static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0, 256 0x45, 0, 0, 0x28, 0, 0, 0x40, 0, 0x40, 0x6, 0, 0, 0, 0, 0, 0, 257 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x80, 0x11, 258 0x0, 0x72, 0, 0, 0, 0}; 259 260 raw_packet = kzalloc(I40E_FDIR_MAX_RAW_PACKET_SIZE, GFP_KERNEL); 261 if (!raw_packet) 262 return -ENOMEM; 263 memcpy(raw_packet, packet, I40E_TCPIP_DUMMY_PACKET_LEN); 264 265 ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET); 266 tcp = (struct tcphdr *)(raw_packet + IP_HEADER_OFFSET 267 + sizeof(struct iphdr)); 268 269 ip->daddr = fd_data->dst_ip; 270 tcp->dest = fd_data->dst_port; 271 ip->saddr = fd_data->src_ip; 272 tcp->source = fd_data->src_port; 273 274 if (fd_data->flex_filter) { 275 u8 *payload = raw_packet + I40E_TCPIP_DUMMY_PACKET_LEN; 276 __be16 pattern = fd_data->flex_word; 277 u16 off = fd_data->flex_offset; 278 279 *((__force __be16 *)(payload + off)) = pattern; 280 } 281 282 fd_data->pctype = I40E_FILTER_PCTYPE_NONF_IPV4_TCP; 283 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add); 284 if (ret) { 285 dev_info(&pf->pdev->dev, 286 "PCTYPE:%d, Filter command send failed for fd_id:%d (ret = %d)\n", 287 fd_data->pctype, fd_data->fd_id, ret); 288 /* Free the packet buffer since it wasn't added to the ring */ 289 kfree(raw_packet); 290 return -EOPNOTSUPP; 291 } else if (I40E_DEBUG_FD & pf->hw.debug_mask) { 292 if (add) 293 dev_info(&pf->pdev->dev, "Filter OK for PCTYPE %d loc = %d)\n", 294 fd_data->pctype, fd_data->fd_id); 295 else 296 dev_info(&pf->pdev->dev, 297 "Filter deleted for PCTYPE %d loc = %d\n", 298 fd_data->pctype, fd_data->fd_id); 299 } 300 301 if (add) { 302 pf->fd_tcp4_filter_cnt++; 303 if ((pf->flags & I40E_FLAG_FD_ATR_ENABLED) && 304 I40E_DEBUG_FD & pf->hw.debug_mask) 305 dev_info(&pf->pdev->dev, "Forcing ATR off, sideband rules for TCP/IPv4 flow being applied\n"); 306 set_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state); 307 } else { 308 pf->fd_tcp4_filter_cnt--; 309 } 310 311 return 0; 312 } 313 314 #define I40E_SCTPIP_DUMMY_PACKET_LEN 46 315 /** 316 * i40e_add_del_fdir_sctpv4 - Add/Remove SCTPv4 Flow Director filters for 317 * a specific flow spec 318 * @vsi: pointer to the targeted VSI 319 * @fd_data: the flow director data required for the FDir descriptor 320 * @add: true adds a filter, false removes it 321 * 322 * Returns 0 if the filters were successfully added or removed 323 **/ 324 static int i40e_add_del_fdir_sctpv4(struct i40e_vsi *vsi, 325 struct i40e_fdir_filter *fd_data, 326 bool add) 327 { 328 struct i40e_pf *pf = vsi->back; 329 struct sctphdr *sctp; 330 struct iphdr *ip; 331 u8 *raw_packet; 332 int ret; 333 /* Dummy packet */ 334 static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0, 335 0x45, 0, 0, 0x20, 0, 0, 0x40, 0, 0x40, 0x84, 0, 0, 0, 0, 0, 0, 336 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}; 337 338 raw_packet = kzalloc(I40E_FDIR_MAX_RAW_PACKET_SIZE, GFP_KERNEL); 339 if (!raw_packet) 340 return -ENOMEM; 341 memcpy(raw_packet, packet, I40E_SCTPIP_DUMMY_PACKET_LEN); 342 343 ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET); 344 sctp = (struct sctphdr *)(raw_packet + IP_HEADER_OFFSET 345 + sizeof(struct iphdr)); 346 347 ip->daddr = fd_data->dst_ip; 348 sctp->dest = fd_data->dst_port; 349 ip->saddr = fd_data->src_ip; 350 sctp->source = fd_data->src_port; 351 352 if (fd_data->flex_filter) { 353 u8 *payload = raw_packet + I40E_SCTPIP_DUMMY_PACKET_LEN; 354 __be16 pattern = fd_data->flex_word; 355 u16 off = fd_data->flex_offset; 356 357 *((__force __be16 *)(payload + off)) = pattern; 358 } 359 360 fd_data->pctype = I40E_FILTER_PCTYPE_NONF_IPV4_SCTP; 361 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add); 362 if (ret) { 363 dev_info(&pf->pdev->dev, 364 "PCTYPE:%d, Filter command send failed for fd_id:%d (ret = %d)\n", 365 fd_data->pctype, fd_data->fd_id, ret); 366 /* Free the packet buffer since it wasn't added to the ring */ 367 kfree(raw_packet); 368 return -EOPNOTSUPP; 369 } else if (I40E_DEBUG_FD & pf->hw.debug_mask) { 370 if (add) 371 dev_info(&pf->pdev->dev, 372 "Filter OK for PCTYPE %d loc = %d\n", 373 fd_data->pctype, fd_data->fd_id); 374 else 375 dev_info(&pf->pdev->dev, 376 "Filter deleted for PCTYPE %d loc = %d\n", 377 fd_data->pctype, fd_data->fd_id); 378 } 379 380 if (add) 381 pf->fd_sctp4_filter_cnt++; 382 else 383 pf->fd_sctp4_filter_cnt--; 384 385 return 0; 386 } 387 388 #define I40E_IP_DUMMY_PACKET_LEN 34 389 /** 390 * i40e_add_del_fdir_ipv4 - Add/Remove IPv4 Flow Director filters for 391 * a specific flow spec 392 * @vsi: pointer to the targeted VSI 393 * @fd_data: the flow director data required for the FDir descriptor 394 * @add: true adds a filter, false removes it 395 * 396 * Returns 0 if the filters were successfully added or removed 397 **/ 398 static int i40e_add_del_fdir_ipv4(struct i40e_vsi *vsi, 399 struct i40e_fdir_filter *fd_data, 400 bool add) 401 { 402 struct i40e_pf *pf = vsi->back; 403 struct iphdr *ip; 404 u8 *raw_packet; 405 int ret; 406 int i; 407 static char packet[] = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x08, 0, 408 0x45, 0, 0, 0x14, 0, 0, 0x40, 0, 0x40, 0x10, 0, 0, 0, 0, 0, 0, 409 0, 0, 0, 0}; 410 411 for (i = I40E_FILTER_PCTYPE_NONF_IPV4_OTHER; 412 i <= I40E_FILTER_PCTYPE_FRAG_IPV4; i++) { 413 raw_packet = kzalloc(I40E_FDIR_MAX_RAW_PACKET_SIZE, GFP_KERNEL); 414 if (!raw_packet) 415 return -ENOMEM; 416 memcpy(raw_packet, packet, I40E_IP_DUMMY_PACKET_LEN); 417 ip = (struct iphdr *)(raw_packet + IP_HEADER_OFFSET); 418 419 ip->saddr = fd_data->src_ip; 420 ip->daddr = fd_data->dst_ip; 421 ip->protocol = 0; 422 423 if (fd_data->flex_filter) { 424 u8 *payload = raw_packet + I40E_IP_DUMMY_PACKET_LEN; 425 __be16 pattern = fd_data->flex_word; 426 u16 off = fd_data->flex_offset; 427 428 *((__force __be16 *)(payload + off)) = pattern; 429 } 430 431 fd_data->pctype = i; 432 ret = i40e_program_fdir_filter(fd_data, raw_packet, pf, add); 433 if (ret) { 434 dev_info(&pf->pdev->dev, 435 "PCTYPE:%d, Filter command send failed for fd_id:%d (ret = %d)\n", 436 fd_data->pctype, fd_data->fd_id, ret); 437 /* The packet buffer wasn't added to the ring so we 438 * need to free it now. 439 */ 440 kfree(raw_packet); 441 return -EOPNOTSUPP; 442 } else if (I40E_DEBUG_FD & pf->hw.debug_mask) { 443 if (add) 444 dev_info(&pf->pdev->dev, 445 "Filter OK for PCTYPE %d loc = %d\n", 446 fd_data->pctype, fd_data->fd_id); 447 else 448 dev_info(&pf->pdev->dev, 449 "Filter deleted for PCTYPE %d loc = %d\n", 450 fd_data->pctype, fd_data->fd_id); 451 } 452 } 453 454 if (add) 455 pf->fd_ip4_filter_cnt++; 456 else 457 pf->fd_ip4_filter_cnt--; 458 459 return 0; 460 } 461 462 /** 463 * i40e_add_del_fdir - Build raw packets to add/del fdir filter 464 * @vsi: pointer to the targeted VSI 465 * @input: filter to add or delete 466 * @add: true adds a filter, false removes it 467 * 468 **/ 469 int i40e_add_del_fdir(struct i40e_vsi *vsi, 470 struct i40e_fdir_filter *input, bool add) 471 { 472 struct i40e_pf *pf = vsi->back; 473 int ret; 474 475 switch (input->flow_type & ~FLOW_EXT) { 476 case TCP_V4_FLOW: 477 ret = i40e_add_del_fdir_tcpv4(vsi, input, add); 478 break; 479 case UDP_V4_FLOW: 480 ret = i40e_add_del_fdir_udpv4(vsi, input, add); 481 break; 482 case SCTP_V4_FLOW: 483 ret = i40e_add_del_fdir_sctpv4(vsi, input, add); 484 break; 485 case IP_USER_FLOW: 486 switch (input->ip4_proto) { 487 case IPPROTO_TCP: 488 ret = i40e_add_del_fdir_tcpv4(vsi, input, add); 489 break; 490 case IPPROTO_UDP: 491 ret = i40e_add_del_fdir_udpv4(vsi, input, add); 492 break; 493 case IPPROTO_SCTP: 494 ret = i40e_add_del_fdir_sctpv4(vsi, input, add); 495 break; 496 case IPPROTO_IP: 497 ret = i40e_add_del_fdir_ipv4(vsi, input, add); 498 break; 499 default: 500 /* We cannot support masking based on protocol */ 501 dev_info(&pf->pdev->dev, "Unsupported IPv4 protocol 0x%02x\n", 502 input->ip4_proto); 503 return -EINVAL; 504 } 505 break; 506 default: 507 dev_info(&pf->pdev->dev, "Unsupported flow type 0x%02x\n", 508 input->flow_type); 509 return -EINVAL; 510 } 511 512 /* The buffer allocated here will be normally be freed by 513 * i40e_clean_fdir_tx_irq() as it reclaims resources after transmit 514 * completion. In the event of an error adding the buffer to the FDIR 515 * ring, it will immediately be freed. It may also be freed by 516 * i40e_clean_tx_ring() when closing the VSI. 517 */ 518 return ret; 519 } 520 521 /** 522 * i40e_fd_handle_status - check the Programming Status for FD 523 * @rx_ring: the Rx ring for this descriptor 524 * @rx_desc: the Rx descriptor for programming Status, not a packet descriptor. 525 * @prog_id: the id originally used for programming 526 * 527 * This is used to verify if the FD programming or invalidation 528 * requested by SW to the HW is successful or not and take actions accordingly. 529 **/ 530 void i40e_fd_handle_status(struct i40e_ring *rx_ring, 531 union i40e_rx_desc *rx_desc, u8 prog_id) 532 { 533 struct i40e_pf *pf = rx_ring->vsi->back; 534 struct pci_dev *pdev = pf->pdev; 535 u32 fcnt_prog, fcnt_avail; 536 u32 error; 537 u64 qw; 538 539 qw = le64_to_cpu(rx_desc->wb.qword1.status_error_len); 540 error = (qw & I40E_RX_PROG_STATUS_DESC_QW1_ERROR_MASK) >> 541 I40E_RX_PROG_STATUS_DESC_QW1_ERROR_SHIFT; 542 543 if (error == BIT(I40E_RX_PROG_STATUS_DESC_FD_TBL_FULL_SHIFT)) { 544 pf->fd_inv = le32_to_cpu(rx_desc->wb.qword0.hi_dword.fd_id); 545 if ((rx_desc->wb.qword0.hi_dword.fd_id != 0) || 546 (I40E_DEBUG_FD & pf->hw.debug_mask)) 547 dev_warn(&pdev->dev, "ntuple filter loc = %d, could not be added\n", 548 pf->fd_inv); 549 550 /* Check if the programming error is for ATR. 551 * If so, auto disable ATR and set a state for 552 * flush in progress. Next time we come here if flush is in 553 * progress do nothing, once flush is complete the state will 554 * be cleared. 555 */ 556 if (test_bit(__I40E_FD_FLUSH_REQUESTED, pf->state)) 557 return; 558 559 pf->fd_add_err++; 560 /* store the current atr filter count */ 561 pf->fd_atr_cnt = i40e_get_current_atr_cnt(pf); 562 563 if ((rx_desc->wb.qword0.hi_dword.fd_id == 0) && 564 test_bit(__I40E_FD_SB_AUTO_DISABLED, pf->state)) { 565 /* These set_bit() calls aren't atomic with the 566 * test_bit() here, but worse case we potentially 567 * disable ATR and queue a flush right after SB 568 * support is re-enabled. That shouldn't cause an 569 * issue in practice 570 */ 571 set_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state); 572 set_bit(__I40E_FD_FLUSH_REQUESTED, pf->state); 573 } 574 575 /* filter programming failed most likely due to table full */ 576 fcnt_prog = i40e_get_global_fd_count(pf); 577 fcnt_avail = pf->fdir_pf_filter_count; 578 /* If ATR is running fcnt_prog can quickly change, 579 * if we are very close to full, it makes sense to disable 580 * FD ATR/SB and then re-enable it when there is room. 581 */ 582 if (fcnt_prog >= (fcnt_avail - I40E_FDIR_BUFFER_FULL_MARGIN)) { 583 if ((pf->flags & I40E_FLAG_FD_SB_ENABLED) && 584 !test_and_set_bit(__I40E_FD_SB_AUTO_DISABLED, 585 pf->state)) 586 if (I40E_DEBUG_FD & pf->hw.debug_mask) 587 dev_warn(&pdev->dev, "FD filter space full, new ntuple rules will not be added\n"); 588 } 589 } else if (error == BIT(I40E_RX_PROG_STATUS_DESC_NO_FD_ENTRY_SHIFT)) { 590 if (I40E_DEBUG_FD & pf->hw.debug_mask) 591 dev_info(&pdev->dev, "ntuple filter fd_id = %d, could not be removed\n", 592 rx_desc->wb.qword0.hi_dword.fd_id); 593 } 594 } 595 596 /** 597 * i40e_unmap_and_free_tx_resource - Release a Tx buffer 598 * @ring: the ring that owns the buffer 599 * @tx_buffer: the buffer to free 600 **/ 601 static void i40e_unmap_and_free_tx_resource(struct i40e_ring *ring, 602 struct i40e_tx_buffer *tx_buffer) 603 { 604 if (tx_buffer->skb) { 605 if (tx_buffer->tx_flags & I40E_TX_FLAGS_FD_SB) 606 kfree(tx_buffer->raw_buf); 607 else if (ring_is_xdp(ring)) 608 xdp_return_frame(tx_buffer->xdpf); 609 else 610 dev_kfree_skb_any(tx_buffer->skb); 611 if (dma_unmap_len(tx_buffer, len)) 612 dma_unmap_single(ring->dev, 613 dma_unmap_addr(tx_buffer, dma), 614 dma_unmap_len(tx_buffer, len), 615 DMA_TO_DEVICE); 616 } else if (dma_unmap_len(tx_buffer, len)) { 617 dma_unmap_page(ring->dev, 618 dma_unmap_addr(tx_buffer, dma), 619 dma_unmap_len(tx_buffer, len), 620 DMA_TO_DEVICE); 621 } 622 623 tx_buffer->next_to_watch = NULL; 624 tx_buffer->skb = NULL; 625 dma_unmap_len_set(tx_buffer, len, 0); 626 /* tx_buffer must be completely set up in the transmit path */ 627 } 628 629 /** 630 * i40e_clean_tx_ring - Free any empty Tx buffers 631 * @tx_ring: ring to be cleaned 632 **/ 633 void i40e_clean_tx_ring(struct i40e_ring *tx_ring) 634 { 635 unsigned long bi_size; 636 u16 i; 637 638 if (ring_is_xdp(tx_ring) && tx_ring->xsk_umem) { 639 i40e_xsk_clean_tx_ring(tx_ring); 640 } else { 641 /* ring already cleared, nothing to do */ 642 if (!tx_ring->tx_bi) 643 return; 644 645 /* Free all the Tx ring sk_buffs */ 646 for (i = 0; i < tx_ring->count; i++) 647 i40e_unmap_and_free_tx_resource(tx_ring, 648 &tx_ring->tx_bi[i]); 649 } 650 651 bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count; 652 memset(tx_ring->tx_bi, 0, bi_size); 653 654 /* Zero out the descriptor ring */ 655 memset(tx_ring->desc, 0, tx_ring->size); 656 657 tx_ring->next_to_use = 0; 658 tx_ring->next_to_clean = 0; 659 660 if (!tx_ring->netdev) 661 return; 662 663 /* cleanup Tx queue statistics */ 664 netdev_tx_reset_queue(txring_txq(tx_ring)); 665 } 666 667 /** 668 * i40e_free_tx_resources - Free Tx resources per queue 669 * @tx_ring: Tx descriptor ring for a specific queue 670 * 671 * Free all transmit software resources 672 **/ 673 void i40e_free_tx_resources(struct i40e_ring *tx_ring) 674 { 675 i40e_clean_tx_ring(tx_ring); 676 kfree(tx_ring->tx_bi); 677 tx_ring->tx_bi = NULL; 678 679 if (tx_ring->desc) { 680 dma_free_coherent(tx_ring->dev, tx_ring->size, 681 tx_ring->desc, tx_ring->dma); 682 tx_ring->desc = NULL; 683 } 684 } 685 686 /** 687 * i40e_get_tx_pending - how many tx descriptors not processed 688 * @ring: the ring of descriptors 689 * @in_sw: use SW variables 690 * 691 * Since there is no access to the ring head register 692 * in XL710, we need to use our local copies 693 **/ 694 u32 i40e_get_tx_pending(struct i40e_ring *ring, bool in_sw) 695 { 696 u32 head, tail; 697 698 if (!in_sw) { 699 head = i40e_get_head(ring); 700 tail = readl(ring->tail); 701 } else { 702 head = ring->next_to_clean; 703 tail = ring->next_to_use; 704 } 705 706 if (head != tail) 707 return (head < tail) ? 708 tail - head : (tail + ring->count - head); 709 710 return 0; 711 } 712 713 /** 714 * i40e_detect_recover_hung - Function to detect and recover hung_queues 715 * @vsi: pointer to vsi struct with tx queues 716 * 717 * VSI has netdev and netdev has TX queues. This function is to check each of 718 * those TX queues if they are hung, trigger recovery by issuing SW interrupt. 719 **/ 720 void i40e_detect_recover_hung(struct i40e_vsi *vsi) 721 { 722 struct i40e_ring *tx_ring = NULL; 723 struct net_device *netdev; 724 unsigned int i; 725 int packets; 726 727 if (!vsi) 728 return; 729 730 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 731 return; 732 733 netdev = vsi->netdev; 734 if (!netdev) 735 return; 736 737 if (!netif_carrier_ok(netdev)) 738 return; 739 740 for (i = 0; i < vsi->num_queue_pairs; i++) { 741 tx_ring = vsi->tx_rings[i]; 742 if (tx_ring && tx_ring->desc) { 743 /* If packet counter has not changed the queue is 744 * likely stalled, so force an interrupt for this 745 * queue. 746 * 747 * prev_pkt_ctr would be negative if there was no 748 * pending work. 749 */ 750 packets = tx_ring->stats.packets & INT_MAX; 751 if (tx_ring->tx_stats.prev_pkt_ctr == packets) { 752 i40e_force_wb(vsi, tx_ring->q_vector); 753 continue; 754 } 755 756 /* Memory barrier between read of packet count and call 757 * to i40e_get_tx_pending() 758 */ 759 smp_rmb(); 760 tx_ring->tx_stats.prev_pkt_ctr = 761 i40e_get_tx_pending(tx_ring, true) ? packets : -1; 762 } 763 } 764 } 765 766 /** 767 * i40e_clean_tx_irq - Reclaim resources after transmit completes 768 * @vsi: the VSI we care about 769 * @tx_ring: Tx ring to clean 770 * @napi_budget: Used to determine if we are in netpoll 771 * 772 * Returns true if there's any budget left (e.g. the clean is finished) 773 **/ 774 static bool i40e_clean_tx_irq(struct i40e_vsi *vsi, 775 struct i40e_ring *tx_ring, int napi_budget) 776 { 777 int i = tx_ring->next_to_clean; 778 struct i40e_tx_buffer *tx_buf; 779 struct i40e_tx_desc *tx_head; 780 struct i40e_tx_desc *tx_desc; 781 unsigned int total_bytes = 0, total_packets = 0; 782 unsigned int budget = vsi->work_limit; 783 784 tx_buf = &tx_ring->tx_bi[i]; 785 tx_desc = I40E_TX_DESC(tx_ring, i); 786 i -= tx_ring->count; 787 788 tx_head = I40E_TX_DESC(tx_ring, i40e_get_head(tx_ring)); 789 790 do { 791 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch; 792 793 /* if next_to_watch is not set then there is no work pending */ 794 if (!eop_desc) 795 break; 796 797 /* prevent any other reads prior to eop_desc */ 798 smp_rmb(); 799 800 i40e_trace(clean_tx_irq, tx_ring, tx_desc, tx_buf); 801 /* we have caught up to head, no work left to do */ 802 if (tx_head == tx_desc) 803 break; 804 805 /* clear next_to_watch to prevent false hangs */ 806 tx_buf->next_to_watch = NULL; 807 808 /* update the statistics for this packet */ 809 total_bytes += tx_buf->bytecount; 810 total_packets += tx_buf->gso_segs; 811 812 /* free the skb/XDP data */ 813 if (ring_is_xdp(tx_ring)) 814 xdp_return_frame(tx_buf->xdpf); 815 else 816 napi_consume_skb(tx_buf->skb, napi_budget); 817 818 /* unmap skb header data */ 819 dma_unmap_single(tx_ring->dev, 820 dma_unmap_addr(tx_buf, dma), 821 dma_unmap_len(tx_buf, len), 822 DMA_TO_DEVICE); 823 824 /* clear tx_buffer data */ 825 tx_buf->skb = NULL; 826 dma_unmap_len_set(tx_buf, len, 0); 827 828 /* unmap remaining buffers */ 829 while (tx_desc != eop_desc) { 830 i40e_trace(clean_tx_irq_unmap, 831 tx_ring, tx_desc, tx_buf); 832 833 tx_buf++; 834 tx_desc++; 835 i++; 836 if (unlikely(!i)) { 837 i -= tx_ring->count; 838 tx_buf = tx_ring->tx_bi; 839 tx_desc = I40E_TX_DESC(tx_ring, 0); 840 } 841 842 /* unmap any remaining paged data */ 843 if (dma_unmap_len(tx_buf, len)) { 844 dma_unmap_page(tx_ring->dev, 845 dma_unmap_addr(tx_buf, dma), 846 dma_unmap_len(tx_buf, len), 847 DMA_TO_DEVICE); 848 dma_unmap_len_set(tx_buf, len, 0); 849 } 850 } 851 852 /* move us one more past the eop_desc for start of next pkt */ 853 tx_buf++; 854 tx_desc++; 855 i++; 856 if (unlikely(!i)) { 857 i -= tx_ring->count; 858 tx_buf = tx_ring->tx_bi; 859 tx_desc = I40E_TX_DESC(tx_ring, 0); 860 } 861 862 prefetch(tx_desc); 863 864 /* update budget accounting */ 865 budget--; 866 } while (likely(budget)); 867 868 i += tx_ring->count; 869 tx_ring->next_to_clean = i; 870 i40e_update_tx_stats(tx_ring, total_packets, total_bytes); 871 i40e_arm_wb(tx_ring, vsi, budget); 872 873 if (ring_is_xdp(tx_ring)) 874 return !!budget; 875 876 /* notify netdev of completed buffers */ 877 netdev_tx_completed_queue(txring_txq(tx_ring), 878 total_packets, total_bytes); 879 880 #define TX_WAKE_THRESHOLD ((s16)(DESC_NEEDED * 2)) 881 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) && 882 (I40E_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) { 883 /* Make sure that anybody stopping the queue after this 884 * sees the new next_to_clean. 885 */ 886 smp_mb(); 887 if (__netif_subqueue_stopped(tx_ring->netdev, 888 tx_ring->queue_index) && 889 !test_bit(__I40E_VSI_DOWN, vsi->state)) { 890 netif_wake_subqueue(tx_ring->netdev, 891 tx_ring->queue_index); 892 ++tx_ring->tx_stats.restart_queue; 893 } 894 } 895 896 return !!budget; 897 } 898 899 /** 900 * i40e_enable_wb_on_itr - Arm hardware to do a wb, interrupts are not enabled 901 * @vsi: the VSI we care about 902 * @q_vector: the vector on which to enable writeback 903 * 904 **/ 905 static void i40e_enable_wb_on_itr(struct i40e_vsi *vsi, 906 struct i40e_q_vector *q_vector) 907 { 908 u16 flags = q_vector->tx.ring[0].flags; 909 u32 val; 910 911 if (!(flags & I40E_TXR_FLAGS_WB_ON_ITR)) 912 return; 913 914 if (q_vector->arm_wb_state) 915 return; 916 917 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) { 918 val = I40E_PFINT_DYN_CTLN_WB_ON_ITR_MASK | 919 I40E_PFINT_DYN_CTLN_ITR_INDX_MASK; /* set noitr */ 920 921 wr32(&vsi->back->hw, 922 I40E_PFINT_DYN_CTLN(q_vector->reg_idx), 923 val); 924 } else { 925 val = I40E_PFINT_DYN_CTL0_WB_ON_ITR_MASK | 926 I40E_PFINT_DYN_CTL0_ITR_INDX_MASK; /* set noitr */ 927 928 wr32(&vsi->back->hw, I40E_PFINT_DYN_CTL0, val); 929 } 930 q_vector->arm_wb_state = true; 931 } 932 933 /** 934 * i40e_force_wb - Issue SW Interrupt so HW does a wb 935 * @vsi: the VSI we care about 936 * @q_vector: the vector on which to force writeback 937 * 938 **/ 939 void i40e_force_wb(struct i40e_vsi *vsi, struct i40e_q_vector *q_vector) 940 { 941 if (vsi->back->flags & I40E_FLAG_MSIX_ENABLED) { 942 u32 val = I40E_PFINT_DYN_CTLN_INTENA_MASK | 943 I40E_PFINT_DYN_CTLN_ITR_INDX_MASK | /* set noitr */ 944 I40E_PFINT_DYN_CTLN_SWINT_TRIG_MASK | 945 I40E_PFINT_DYN_CTLN_SW_ITR_INDX_ENA_MASK; 946 /* allow 00 to be written to the index */ 947 948 wr32(&vsi->back->hw, 949 I40E_PFINT_DYN_CTLN(q_vector->reg_idx), val); 950 } else { 951 u32 val = I40E_PFINT_DYN_CTL0_INTENA_MASK | 952 I40E_PFINT_DYN_CTL0_ITR_INDX_MASK | /* set noitr */ 953 I40E_PFINT_DYN_CTL0_SWINT_TRIG_MASK | 954 I40E_PFINT_DYN_CTL0_SW_ITR_INDX_ENA_MASK; 955 /* allow 00 to be written to the index */ 956 957 wr32(&vsi->back->hw, I40E_PFINT_DYN_CTL0, val); 958 } 959 } 960 961 static inline bool i40e_container_is_rx(struct i40e_q_vector *q_vector, 962 struct i40e_ring_container *rc) 963 { 964 return &q_vector->rx == rc; 965 } 966 967 static inline unsigned int i40e_itr_divisor(struct i40e_q_vector *q_vector) 968 { 969 unsigned int divisor; 970 971 switch (q_vector->vsi->back->hw.phy.link_info.link_speed) { 972 case I40E_LINK_SPEED_40GB: 973 divisor = I40E_ITR_ADAPTIVE_MIN_INC * 1024; 974 break; 975 case I40E_LINK_SPEED_25GB: 976 case I40E_LINK_SPEED_20GB: 977 divisor = I40E_ITR_ADAPTIVE_MIN_INC * 512; 978 break; 979 default: 980 case I40E_LINK_SPEED_10GB: 981 divisor = I40E_ITR_ADAPTIVE_MIN_INC * 256; 982 break; 983 case I40E_LINK_SPEED_1GB: 984 case I40E_LINK_SPEED_100MB: 985 divisor = I40E_ITR_ADAPTIVE_MIN_INC * 32; 986 break; 987 } 988 989 return divisor; 990 } 991 992 /** 993 * i40e_update_itr - update the dynamic ITR value based on statistics 994 * @q_vector: structure containing interrupt and ring information 995 * @rc: structure containing ring performance data 996 * 997 * Stores a new ITR value based on packets and byte 998 * counts during the last interrupt. The advantage of per interrupt 999 * computation is faster updates and more accurate ITR for the current 1000 * traffic pattern. Constants in this function were computed 1001 * based on theoretical maximum wire speed and thresholds were set based 1002 * on testing data as well as attempting to minimize response time 1003 * while increasing bulk throughput. 1004 **/ 1005 static void i40e_update_itr(struct i40e_q_vector *q_vector, 1006 struct i40e_ring_container *rc) 1007 { 1008 unsigned int avg_wire_size, packets, bytes, itr; 1009 unsigned long next_update = jiffies; 1010 1011 /* If we don't have any rings just leave ourselves set for maximum 1012 * possible latency so we take ourselves out of the equation. 1013 */ 1014 if (!rc->ring || !ITR_IS_DYNAMIC(rc->ring->itr_setting)) 1015 return; 1016 1017 /* For Rx we want to push the delay up and default to low latency. 1018 * for Tx we want to pull the delay down and default to high latency. 1019 */ 1020 itr = i40e_container_is_rx(q_vector, rc) ? 1021 I40E_ITR_ADAPTIVE_MIN_USECS | I40E_ITR_ADAPTIVE_LATENCY : 1022 I40E_ITR_ADAPTIVE_MAX_USECS | I40E_ITR_ADAPTIVE_LATENCY; 1023 1024 /* If we didn't update within up to 1 - 2 jiffies we can assume 1025 * that either packets are coming in so slow there hasn't been 1026 * any work, or that there is so much work that NAPI is dealing 1027 * with interrupt moderation and we don't need to do anything. 1028 */ 1029 if (time_after(next_update, rc->next_update)) 1030 goto clear_counts; 1031 1032 /* If itr_countdown is set it means we programmed an ITR within 1033 * the last 4 interrupt cycles. This has a side effect of us 1034 * potentially firing an early interrupt. In order to work around 1035 * this we need to throw out any data received for a few 1036 * interrupts following the update. 1037 */ 1038 if (q_vector->itr_countdown) { 1039 itr = rc->target_itr; 1040 goto clear_counts; 1041 } 1042 1043 packets = rc->total_packets; 1044 bytes = rc->total_bytes; 1045 1046 if (i40e_container_is_rx(q_vector, rc)) { 1047 /* If Rx there are 1 to 4 packets and bytes are less than 1048 * 9000 assume insufficient data to use bulk rate limiting 1049 * approach unless Tx is already in bulk rate limiting. We 1050 * are likely latency driven. 1051 */ 1052 if (packets && packets < 4 && bytes < 9000 && 1053 (q_vector->tx.target_itr & I40E_ITR_ADAPTIVE_LATENCY)) { 1054 itr = I40E_ITR_ADAPTIVE_LATENCY; 1055 goto adjust_by_size; 1056 } 1057 } else if (packets < 4) { 1058 /* If we have Tx and Rx ITR maxed and Tx ITR is running in 1059 * bulk mode and we are receiving 4 or fewer packets just 1060 * reset the ITR_ADAPTIVE_LATENCY bit for latency mode so 1061 * that the Rx can relax. 1062 */ 1063 if (rc->target_itr == I40E_ITR_ADAPTIVE_MAX_USECS && 1064 (q_vector->rx.target_itr & I40E_ITR_MASK) == 1065 I40E_ITR_ADAPTIVE_MAX_USECS) 1066 goto clear_counts; 1067 } else if (packets > 32) { 1068 /* If we have processed over 32 packets in a single interrupt 1069 * for Tx assume we need to switch over to "bulk" mode. 1070 */ 1071 rc->target_itr &= ~I40E_ITR_ADAPTIVE_LATENCY; 1072 } 1073 1074 /* We have no packets to actually measure against. This means 1075 * either one of the other queues on this vector is active or 1076 * we are a Tx queue doing TSO with too high of an interrupt rate. 1077 * 1078 * Between 4 and 56 we can assume that our current interrupt delay 1079 * is only slightly too low. As such we should increase it by a small 1080 * fixed amount. 1081 */ 1082 if (packets < 56) { 1083 itr = rc->target_itr + I40E_ITR_ADAPTIVE_MIN_INC; 1084 if ((itr & I40E_ITR_MASK) > I40E_ITR_ADAPTIVE_MAX_USECS) { 1085 itr &= I40E_ITR_ADAPTIVE_LATENCY; 1086 itr += I40E_ITR_ADAPTIVE_MAX_USECS; 1087 } 1088 goto clear_counts; 1089 } 1090 1091 if (packets <= 256) { 1092 itr = min(q_vector->tx.current_itr, q_vector->rx.current_itr); 1093 itr &= I40E_ITR_MASK; 1094 1095 /* Between 56 and 112 is our "goldilocks" zone where we are 1096 * working out "just right". Just report that our current 1097 * ITR is good for us. 1098 */ 1099 if (packets <= 112) 1100 goto clear_counts; 1101 1102 /* If packet count is 128 or greater we are likely looking 1103 * at a slight overrun of the delay we want. Try halving 1104 * our delay to see if that will cut the number of packets 1105 * in half per interrupt. 1106 */ 1107 itr /= 2; 1108 itr &= I40E_ITR_MASK; 1109 if (itr < I40E_ITR_ADAPTIVE_MIN_USECS) 1110 itr = I40E_ITR_ADAPTIVE_MIN_USECS; 1111 1112 goto clear_counts; 1113 } 1114 1115 /* The paths below assume we are dealing with a bulk ITR since 1116 * number of packets is greater than 256. We are just going to have 1117 * to compute a value and try to bring the count under control, 1118 * though for smaller packet sizes there isn't much we can do as 1119 * NAPI polling will likely be kicking in sooner rather than later. 1120 */ 1121 itr = I40E_ITR_ADAPTIVE_BULK; 1122 1123 adjust_by_size: 1124 /* If packet counts are 256 or greater we can assume we have a gross 1125 * overestimation of what the rate should be. Instead of trying to fine 1126 * tune it just use the formula below to try and dial in an exact value 1127 * give the current packet size of the frame. 1128 */ 1129 avg_wire_size = bytes / packets; 1130 1131 /* The following is a crude approximation of: 1132 * wmem_default / (size + overhead) = desired_pkts_per_int 1133 * rate / bits_per_byte / (size + ethernet overhead) = pkt_rate 1134 * (desired_pkt_rate / pkt_rate) * usecs_per_sec = ITR value 1135 * 1136 * Assuming wmem_default is 212992 and overhead is 640 bytes per 1137 * packet, (256 skb, 64 headroom, 320 shared info), we can reduce the 1138 * formula down to 1139 * 1140 * (170 * (size + 24)) / (size + 640) = ITR 1141 * 1142 * We first do some math on the packet size and then finally bitshift 1143 * by 8 after rounding up. We also have to account for PCIe link speed 1144 * difference as ITR scales based on this. 1145 */ 1146 if (avg_wire_size <= 60) { 1147 /* Start at 250k ints/sec */ 1148 avg_wire_size = 4096; 1149 } else if (avg_wire_size <= 380) { 1150 /* 250K ints/sec to 60K ints/sec */ 1151 avg_wire_size *= 40; 1152 avg_wire_size += 1696; 1153 } else if (avg_wire_size <= 1084) { 1154 /* 60K ints/sec to 36K ints/sec */ 1155 avg_wire_size *= 15; 1156 avg_wire_size += 11452; 1157 } else if (avg_wire_size <= 1980) { 1158 /* 36K ints/sec to 30K ints/sec */ 1159 avg_wire_size *= 5; 1160 avg_wire_size += 22420; 1161 } else { 1162 /* plateau at a limit of 30K ints/sec */ 1163 avg_wire_size = 32256; 1164 } 1165 1166 /* If we are in low latency mode halve our delay which doubles the 1167 * rate to somewhere between 100K to 16K ints/sec 1168 */ 1169 if (itr & I40E_ITR_ADAPTIVE_LATENCY) 1170 avg_wire_size /= 2; 1171 1172 /* Resultant value is 256 times larger than it needs to be. This 1173 * gives us room to adjust the value as needed to either increase 1174 * or decrease the value based on link speeds of 10G, 2.5G, 1G, etc. 1175 * 1176 * Use addition as we have already recorded the new latency flag 1177 * for the ITR value. 1178 */ 1179 itr += DIV_ROUND_UP(avg_wire_size, i40e_itr_divisor(q_vector)) * 1180 I40E_ITR_ADAPTIVE_MIN_INC; 1181 1182 if ((itr & I40E_ITR_MASK) > I40E_ITR_ADAPTIVE_MAX_USECS) { 1183 itr &= I40E_ITR_ADAPTIVE_LATENCY; 1184 itr += I40E_ITR_ADAPTIVE_MAX_USECS; 1185 } 1186 1187 clear_counts: 1188 /* write back value */ 1189 rc->target_itr = itr; 1190 1191 /* next update should occur within next jiffy */ 1192 rc->next_update = next_update + 1; 1193 1194 rc->total_bytes = 0; 1195 rc->total_packets = 0; 1196 } 1197 1198 /** 1199 * i40e_reuse_rx_page - page flip buffer and store it back on the ring 1200 * @rx_ring: rx descriptor ring to store buffers on 1201 * @old_buff: donor buffer to have page reused 1202 * 1203 * Synchronizes page for reuse by the adapter 1204 **/ 1205 static void i40e_reuse_rx_page(struct i40e_ring *rx_ring, 1206 struct i40e_rx_buffer *old_buff) 1207 { 1208 struct i40e_rx_buffer *new_buff; 1209 u16 nta = rx_ring->next_to_alloc; 1210 1211 new_buff = &rx_ring->rx_bi[nta]; 1212 1213 /* update, and store next to alloc */ 1214 nta++; 1215 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; 1216 1217 /* transfer page from old buffer to new buffer */ 1218 new_buff->dma = old_buff->dma; 1219 new_buff->page = old_buff->page; 1220 new_buff->page_offset = old_buff->page_offset; 1221 new_buff->pagecnt_bias = old_buff->pagecnt_bias; 1222 1223 rx_ring->rx_stats.page_reuse_count++; 1224 1225 /* clear contents of buffer_info */ 1226 old_buff->page = NULL; 1227 } 1228 1229 /** 1230 * i40e_rx_is_programming_status - check for programming status descriptor 1231 * @qw: qword representing status_error_len in CPU ordering 1232 * 1233 * The value of in the descriptor length field indicate if this 1234 * is a programming status descriptor for flow director or FCoE 1235 * by the value of I40E_RX_PROG_STATUS_DESC_LENGTH, otherwise 1236 * it is a packet descriptor. 1237 **/ 1238 static inline bool i40e_rx_is_programming_status(u64 qw) 1239 { 1240 /* The Rx filter programming status and SPH bit occupy the same 1241 * spot in the descriptor. Since we don't support packet split we 1242 * can just reuse the bit as an indication that this is a 1243 * programming status descriptor. 1244 */ 1245 return qw & I40E_RXD_QW1_LENGTH_SPH_MASK; 1246 } 1247 1248 /** 1249 * i40e_clean_programming_status - try clean the programming status descriptor 1250 * @rx_ring: the rx ring that has this descriptor 1251 * @rx_desc: the rx descriptor written back by HW 1252 * @qw: qword representing status_error_len in CPU ordering 1253 * 1254 * Flow director should handle FD_FILTER_STATUS to check its filter programming 1255 * status being successful or not and take actions accordingly. FCoE should 1256 * handle its context/filter programming/invalidation status and take actions. 1257 * 1258 * Returns an i40e_rx_buffer to reuse if the cleanup occurred, otherwise NULL. 1259 **/ 1260 struct i40e_rx_buffer *i40e_clean_programming_status( 1261 struct i40e_ring *rx_ring, 1262 union i40e_rx_desc *rx_desc, 1263 u64 qw) 1264 { 1265 struct i40e_rx_buffer *rx_buffer; 1266 u32 ntc; 1267 u8 id; 1268 1269 if (!i40e_rx_is_programming_status(qw)) 1270 return NULL; 1271 1272 ntc = rx_ring->next_to_clean; 1273 1274 /* fetch, update, and store next to clean */ 1275 rx_buffer = &rx_ring->rx_bi[ntc++]; 1276 ntc = (ntc < rx_ring->count) ? ntc : 0; 1277 rx_ring->next_to_clean = ntc; 1278 1279 prefetch(I40E_RX_DESC(rx_ring, ntc)); 1280 1281 id = (qw & I40E_RX_PROG_STATUS_DESC_QW1_PROGID_MASK) >> 1282 I40E_RX_PROG_STATUS_DESC_QW1_PROGID_SHIFT; 1283 1284 if (id == I40E_RX_PROG_STATUS_DESC_FD_FILTER_STATUS) 1285 i40e_fd_handle_status(rx_ring, rx_desc, id); 1286 1287 return rx_buffer; 1288 } 1289 1290 /** 1291 * i40e_setup_tx_descriptors - Allocate the Tx descriptors 1292 * @tx_ring: the tx ring to set up 1293 * 1294 * Return 0 on success, negative on error 1295 **/ 1296 int i40e_setup_tx_descriptors(struct i40e_ring *tx_ring) 1297 { 1298 struct device *dev = tx_ring->dev; 1299 int bi_size; 1300 1301 if (!dev) 1302 return -ENOMEM; 1303 1304 /* warn if we are about to overwrite the pointer */ 1305 WARN_ON(tx_ring->tx_bi); 1306 bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count; 1307 tx_ring->tx_bi = kzalloc(bi_size, GFP_KERNEL); 1308 if (!tx_ring->tx_bi) 1309 goto err; 1310 1311 u64_stats_init(&tx_ring->syncp); 1312 1313 /* round up to nearest 4K */ 1314 tx_ring->size = tx_ring->count * sizeof(struct i40e_tx_desc); 1315 /* add u32 for head writeback, align after this takes care of 1316 * guaranteeing this is at least one cache line in size 1317 */ 1318 tx_ring->size += sizeof(u32); 1319 tx_ring->size = ALIGN(tx_ring->size, 4096); 1320 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size, 1321 &tx_ring->dma, GFP_KERNEL); 1322 if (!tx_ring->desc) { 1323 dev_info(dev, "Unable to allocate memory for the Tx descriptor ring, size=%d\n", 1324 tx_ring->size); 1325 goto err; 1326 } 1327 1328 tx_ring->next_to_use = 0; 1329 tx_ring->next_to_clean = 0; 1330 tx_ring->tx_stats.prev_pkt_ctr = -1; 1331 return 0; 1332 1333 err: 1334 kfree(tx_ring->tx_bi); 1335 tx_ring->tx_bi = NULL; 1336 return -ENOMEM; 1337 } 1338 1339 /** 1340 * i40e_clean_rx_ring - Free Rx buffers 1341 * @rx_ring: ring to be cleaned 1342 **/ 1343 void i40e_clean_rx_ring(struct i40e_ring *rx_ring) 1344 { 1345 unsigned long bi_size; 1346 u16 i; 1347 1348 /* ring already cleared, nothing to do */ 1349 if (!rx_ring->rx_bi) 1350 return; 1351 1352 if (rx_ring->skb) { 1353 dev_kfree_skb(rx_ring->skb); 1354 rx_ring->skb = NULL; 1355 } 1356 1357 if (rx_ring->xsk_umem) { 1358 i40e_xsk_clean_rx_ring(rx_ring); 1359 goto skip_free; 1360 } 1361 1362 /* Free all the Rx ring sk_buffs */ 1363 for (i = 0; i < rx_ring->count; i++) { 1364 struct i40e_rx_buffer *rx_bi = &rx_ring->rx_bi[i]; 1365 1366 if (!rx_bi->page) 1367 continue; 1368 1369 /* Invalidate cache lines that may have been written to by 1370 * device so that we avoid corrupting memory. 1371 */ 1372 dma_sync_single_range_for_cpu(rx_ring->dev, 1373 rx_bi->dma, 1374 rx_bi->page_offset, 1375 rx_ring->rx_buf_len, 1376 DMA_FROM_DEVICE); 1377 1378 /* free resources associated with mapping */ 1379 dma_unmap_page_attrs(rx_ring->dev, rx_bi->dma, 1380 i40e_rx_pg_size(rx_ring), 1381 DMA_FROM_DEVICE, 1382 I40E_RX_DMA_ATTR); 1383 1384 __page_frag_cache_drain(rx_bi->page, rx_bi->pagecnt_bias); 1385 1386 rx_bi->page = NULL; 1387 rx_bi->page_offset = 0; 1388 } 1389 1390 skip_free: 1391 bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count; 1392 memset(rx_ring->rx_bi, 0, bi_size); 1393 1394 /* Zero out the descriptor ring */ 1395 memset(rx_ring->desc, 0, rx_ring->size); 1396 1397 rx_ring->next_to_alloc = 0; 1398 rx_ring->next_to_clean = 0; 1399 rx_ring->next_to_use = 0; 1400 } 1401 1402 /** 1403 * i40e_free_rx_resources - Free Rx resources 1404 * @rx_ring: ring to clean the resources from 1405 * 1406 * Free all receive software resources 1407 **/ 1408 void i40e_free_rx_resources(struct i40e_ring *rx_ring) 1409 { 1410 i40e_clean_rx_ring(rx_ring); 1411 if (rx_ring->vsi->type == I40E_VSI_MAIN) 1412 xdp_rxq_info_unreg(&rx_ring->xdp_rxq); 1413 rx_ring->xdp_prog = NULL; 1414 kfree(rx_ring->rx_bi); 1415 rx_ring->rx_bi = NULL; 1416 1417 if (rx_ring->desc) { 1418 dma_free_coherent(rx_ring->dev, rx_ring->size, 1419 rx_ring->desc, rx_ring->dma); 1420 rx_ring->desc = NULL; 1421 } 1422 } 1423 1424 /** 1425 * i40e_setup_rx_descriptors - Allocate Rx descriptors 1426 * @rx_ring: Rx descriptor ring (for a specific queue) to setup 1427 * 1428 * Returns 0 on success, negative on failure 1429 **/ 1430 int i40e_setup_rx_descriptors(struct i40e_ring *rx_ring) 1431 { 1432 struct device *dev = rx_ring->dev; 1433 int err = -ENOMEM; 1434 int bi_size; 1435 1436 /* warn if we are about to overwrite the pointer */ 1437 WARN_ON(rx_ring->rx_bi); 1438 bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count; 1439 rx_ring->rx_bi = kzalloc(bi_size, GFP_KERNEL); 1440 if (!rx_ring->rx_bi) 1441 goto err; 1442 1443 u64_stats_init(&rx_ring->syncp); 1444 1445 /* Round up to nearest 4K */ 1446 rx_ring->size = rx_ring->count * sizeof(union i40e_32byte_rx_desc); 1447 rx_ring->size = ALIGN(rx_ring->size, 4096); 1448 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, 1449 &rx_ring->dma, GFP_KERNEL); 1450 1451 if (!rx_ring->desc) { 1452 dev_info(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n", 1453 rx_ring->size); 1454 goto err; 1455 } 1456 1457 rx_ring->next_to_alloc = 0; 1458 rx_ring->next_to_clean = 0; 1459 rx_ring->next_to_use = 0; 1460 1461 /* XDP RX-queue info only needed for RX rings exposed to XDP */ 1462 if (rx_ring->vsi->type == I40E_VSI_MAIN) { 1463 err = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, 1464 rx_ring->queue_index); 1465 if (err < 0) 1466 goto err; 1467 } 1468 1469 rx_ring->xdp_prog = rx_ring->vsi->xdp_prog; 1470 1471 return 0; 1472 err: 1473 kfree(rx_ring->rx_bi); 1474 rx_ring->rx_bi = NULL; 1475 return err; 1476 } 1477 1478 /** 1479 * i40e_release_rx_desc - Store the new tail and head values 1480 * @rx_ring: ring to bump 1481 * @val: new head index 1482 **/ 1483 void i40e_release_rx_desc(struct i40e_ring *rx_ring, u32 val) 1484 { 1485 rx_ring->next_to_use = val; 1486 1487 /* update next to alloc since we have filled the ring */ 1488 rx_ring->next_to_alloc = val; 1489 1490 /* Force memory writes to complete before letting h/w 1491 * know there are new descriptors to fetch. (Only 1492 * applicable for weak-ordered memory model archs, 1493 * such as IA-64). 1494 */ 1495 wmb(); 1496 writel(val, rx_ring->tail); 1497 } 1498 1499 /** 1500 * i40e_rx_offset - Return expected offset into page to access data 1501 * @rx_ring: Ring we are requesting offset of 1502 * 1503 * Returns the offset value for ring into the data buffer. 1504 */ 1505 static inline unsigned int i40e_rx_offset(struct i40e_ring *rx_ring) 1506 { 1507 return ring_uses_build_skb(rx_ring) ? I40E_SKB_PAD : 0; 1508 } 1509 1510 static unsigned int i40e_rx_frame_truesize(struct i40e_ring *rx_ring, 1511 unsigned int size) 1512 { 1513 unsigned int truesize; 1514 1515 #if (PAGE_SIZE < 8192) 1516 truesize = i40e_rx_pg_size(rx_ring) / 2; /* Must be power-of-2 */ 1517 #else 1518 truesize = i40e_rx_offset(rx_ring) ? 1519 SKB_DATA_ALIGN(size + i40e_rx_offset(rx_ring)) + 1520 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) : 1521 SKB_DATA_ALIGN(size); 1522 #endif 1523 return truesize; 1524 } 1525 1526 /** 1527 * i40e_alloc_mapped_page - recycle or make a new page 1528 * @rx_ring: ring to use 1529 * @bi: rx_buffer struct to modify 1530 * 1531 * Returns true if the page was successfully allocated or 1532 * reused. 1533 **/ 1534 static bool i40e_alloc_mapped_page(struct i40e_ring *rx_ring, 1535 struct i40e_rx_buffer *bi) 1536 { 1537 struct page *page = bi->page; 1538 dma_addr_t dma; 1539 1540 /* since we are recycling buffers we should seldom need to alloc */ 1541 if (likely(page)) { 1542 rx_ring->rx_stats.page_reuse_count++; 1543 return true; 1544 } 1545 1546 /* alloc new page for storage */ 1547 page = dev_alloc_pages(i40e_rx_pg_order(rx_ring)); 1548 if (unlikely(!page)) { 1549 rx_ring->rx_stats.alloc_page_failed++; 1550 return false; 1551 } 1552 1553 /* map page for use */ 1554 dma = dma_map_page_attrs(rx_ring->dev, page, 0, 1555 i40e_rx_pg_size(rx_ring), 1556 DMA_FROM_DEVICE, 1557 I40E_RX_DMA_ATTR); 1558 1559 /* if mapping failed free memory back to system since 1560 * there isn't much point in holding memory we can't use 1561 */ 1562 if (dma_mapping_error(rx_ring->dev, dma)) { 1563 __free_pages(page, i40e_rx_pg_order(rx_ring)); 1564 rx_ring->rx_stats.alloc_page_failed++; 1565 return false; 1566 } 1567 1568 bi->dma = dma; 1569 bi->page = page; 1570 bi->page_offset = i40e_rx_offset(rx_ring); 1571 page_ref_add(page, USHRT_MAX - 1); 1572 bi->pagecnt_bias = USHRT_MAX; 1573 1574 return true; 1575 } 1576 1577 /** 1578 * i40e_alloc_rx_buffers - Replace used receive buffers 1579 * @rx_ring: ring to place buffers on 1580 * @cleaned_count: number of buffers to replace 1581 * 1582 * Returns false if all allocations were successful, true if any fail 1583 **/ 1584 bool i40e_alloc_rx_buffers(struct i40e_ring *rx_ring, u16 cleaned_count) 1585 { 1586 u16 ntu = rx_ring->next_to_use; 1587 union i40e_rx_desc *rx_desc; 1588 struct i40e_rx_buffer *bi; 1589 1590 /* do nothing if no valid netdev defined */ 1591 if (!rx_ring->netdev || !cleaned_count) 1592 return false; 1593 1594 rx_desc = I40E_RX_DESC(rx_ring, ntu); 1595 bi = &rx_ring->rx_bi[ntu]; 1596 1597 do { 1598 if (!i40e_alloc_mapped_page(rx_ring, bi)) 1599 goto no_buffers; 1600 1601 /* sync the buffer for use by the device */ 1602 dma_sync_single_range_for_device(rx_ring->dev, bi->dma, 1603 bi->page_offset, 1604 rx_ring->rx_buf_len, 1605 DMA_FROM_DEVICE); 1606 1607 /* Refresh the desc even if buffer_addrs didn't change 1608 * because each write-back erases this info. 1609 */ 1610 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); 1611 1612 rx_desc++; 1613 bi++; 1614 ntu++; 1615 if (unlikely(ntu == rx_ring->count)) { 1616 rx_desc = I40E_RX_DESC(rx_ring, 0); 1617 bi = rx_ring->rx_bi; 1618 ntu = 0; 1619 } 1620 1621 /* clear the status bits for the next_to_use descriptor */ 1622 rx_desc->wb.qword1.status_error_len = 0; 1623 1624 cleaned_count--; 1625 } while (cleaned_count); 1626 1627 if (rx_ring->next_to_use != ntu) 1628 i40e_release_rx_desc(rx_ring, ntu); 1629 1630 return false; 1631 1632 no_buffers: 1633 if (rx_ring->next_to_use != ntu) 1634 i40e_release_rx_desc(rx_ring, ntu); 1635 1636 /* make sure to come back via polling to try again after 1637 * allocation failure 1638 */ 1639 return true; 1640 } 1641 1642 /** 1643 * i40e_rx_checksum - Indicate in skb if hw indicated a good cksum 1644 * @vsi: the VSI we care about 1645 * @skb: skb currently being received and modified 1646 * @rx_desc: the receive descriptor 1647 **/ 1648 static inline void i40e_rx_checksum(struct i40e_vsi *vsi, 1649 struct sk_buff *skb, 1650 union i40e_rx_desc *rx_desc) 1651 { 1652 struct i40e_rx_ptype_decoded decoded; 1653 u32 rx_error, rx_status; 1654 bool ipv4, ipv6; 1655 u8 ptype; 1656 u64 qword; 1657 1658 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len); 1659 ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) >> I40E_RXD_QW1_PTYPE_SHIFT; 1660 rx_error = (qword & I40E_RXD_QW1_ERROR_MASK) >> 1661 I40E_RXD_QW1_ERROR_SHIFT; 1662 rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >> 1663 I40E_RXD_QW1_STATUS_SHIFT; 1664 decoded = decode_rx_desc_ptype(ptype); 1665 1666 skb->ip_summed = CHECKSUM_NONE; 1667 1668 skb_checksum_none_assert(skb); 1669 1670 /* Rx csum enabled and ip headers found? */ 1671 if (!(vsi->netdev->features & NETIF_F_RXCSUM)) 1672 return; 1673 1674 /* did the hardware decode the packet and checksum? */ 1675 if (!(rx_status & BIT(I40E_RX_DESC_STATUS_L3L4P_SHIFT))) 1676 return; 1677 1678 /* both known and outer_ip must be set for the below code to work */ 1679 if (!(decoded.known && decoded.outer_ip)) 1680 return; 1681 1682 ipv4 = (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP) && 1683 (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV4); 1684 ipv6 = (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP) && 1685 (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6); 1686 1687 if (ipv4 && 1688 (rx_error & (BIT(I40E_RX_DESC_ERROR_IPE_SHIFT) | 1689 BIT(I40E_RX_DESC_ERROR_EIPE_SHIFT)))) 1690 goto checksum_fail; 1691 1692 /* likely incorrect csum if alternate IP extension headers found */ 1693 if (ipv6 && 1694 rx_status & BIT(I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT)) 1695 /* don't increment checksum err here, non-fatal err */ 1696 return; 1697 1698 /* there was some L4 error, count error and punt packet to the stack */ 1699 if (rx_error & BIT(I40E_RX_DESC_ERROR_L4E_SHIFT)) 1700 goto checksum_fail; 1701 1702 /* handle packets that were not able to be checksummed due 1703 * to arrival speed, in this case the stack can compute 1704 * the csum. 1705 */ 1706 if (rx_error & BIT(I40E_RX_DESC_ERROR_PPRS_SHIFT)) 1707 return; 1708 1709 /* If there is an outer header present that might contain a checksum 1710 * we need to bump the checksum level by 1 to reflect the fact that 1711 * we are indicating we validated the inner checksum. 1712 */ 1713 if (decoded.tunnel_type >= I40E_RX_PTYPE_TUNNEL_IP_GRENAT) 1714 skb->csum_level = 1; 1715 1716 /* Only report checksum unnecessary for TCP, UDP, or SCTP */ 1717 switch (decoded.inner_prot) { 1718 case I40E_RX_PTYPE_INNER_PROT_TCP: 1719 case I40E_RX_PTYPE_INNER_PROT_UDP: 1720 case I40E_RX_PTYPE_INNER_PROT_SCTP: 1721 skb->ip_summed = CHECKSUM_UNNECESSARY; 1722 /* fall though */ 1723 default: 1724 break; 1725 } 1726 1727 return; 1728 1729 checksum_fail: 1730 vsi->back->hw_csum_rx_error++; 1731 } 1732 1733 /** 1734 * i40e_ptype_to_htype - get a hash type 1735 * @ptype: the ptype value from the descriptor 1736 * 1737 * Returns a hash type to be used by skb_set_hash 1738 **/ 1739 static inline int i40e_ptype_to_htype(u8 ptype) 1740 { 1741 struct i40e_rx_ptype_decoded decoded = decode_rx_desc_ptype(ptype); 1742 1743 if (!decoded.known) 1744 return PKT_HASH_TYPE_NONE; 1745 1746 if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP && 1747 decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY4) 1748 return PKT_HASH_TYPE_L4; 1749 else if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP && 1750 decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY3) 1751 return PKT_HASH_TYPE_L3; 1752 else 1753 return PKT_HASH_TYPE_L2; 1754 } 1755 1756 /** 1757 * i40e_rx_hash - set the hash value in the skb 1758 * @ring: descriptor ring 1759 * @rx_desc: specific descriptor 1760 * @skb: skb currently being received and modified 1761 * @rx_ptype: Rx packet type 1762 **/ 1763 static inline void i40e_rx_hash(struct i40e_ring *ring, 1764 union i40e_rx_desc *rx_desc, 1765 struct sk_buff *skb, 1766 u8 rx_ptype) 1767 { 1768 u32 hash; 1769 const __le64 rss_mask = 1770 cpu_to_le64((u64)I40E_RX_DESC_FLTSTAT_RSS_HASH << 1771 I40E_RX_DESC_STATUS_FLTSTAT_SHIFT); 1772 1773 if (!(ring->netdev->features & NETIF_F_RXHASH)) 1774 return; 1775 1776 if ((rx_desc->wb.qword1.status_error_len & rss_mask) == rss_mask) { 1777 hash = le32_to_cpu(rx_desc->wb.qword0.hi_dword.rss); 1778 skb_set_hash(skb, hash, i40e_ptype_to_htype(rx_ptype)); 1779 } 1780 } 1781 1782 /** 1783 * i40e_process_skb_fields - Populate skb header fields from Rx descriptor 1784 * @rx_ring: rx descriptor ring packet is being transacted on 1785 * @rx_desc: pointer to the EOP Rx descriptor 1786 * @skb: pointer to current skb being populated 1787 * @rx_ptype: the packet type decoded by hardware 1788 * 1789 * This function checks the ring, descriptor, and packet information in 1790 * order to populate the hash, checksum, VLAN, protocol, and 1791 * other fields within the skb. 1792 **/ 1793 void i40e_process_skb_fields(struct i40e_ring *rx_ring, 1794 union i40e_rx_desc *rx_desc, struct sk_buff *skb) 1795 { 1796 u64 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len); 1797 u32 rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >> 1798 I40E_RXD_QW1_STATUS_SHIFT; 1799 u32 tsynvalid = rx_status & I40E_RXD_QW1_STATUS_TSYNVALID_MASK; 1800 u32 tsyn = (rx_status & I40E_RXD_QW1_STATUS_TSYNINDX_MASK) >> 1801 I40E_RXD_QW1_STATUS_TSYNINDX_SHIFT; 1802 u8 rx_ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) >> 1803 I40E_RXD_QW1_PTYPE_SHIFT; 1804 1805 if (unlikely(tsynvalid)) 1806 i40e_ptp_rx_hwtstamp(rx_ring->vsi->back, skb, tsyn); 1807 1808 i40e_rx_hash(rx_ring, rx_desc, skb, rx_ptype); 1809 1810 i40e_rx_checksum(rx_ring->vsi, skb, rx_desc); 1811 1812 skb_record_rx_queue(skb, rx_ring->queue_index); 1813 1814 if (qword & BIT(I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)) { 1815 u16 vlan_tag = rx_desc->wb.qword0.lo_dword.l2tag1; 1816 1817 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), 1818 le16_to_cpu(vlan_tag)); 1819 } 1820 1821 /* modifies the skb - consumes the enet header */ 1822 skb->protocol = eth_type_trans(skb, rx_ring->netdev); 1823 } 1824 1825 /** 1826 * i40e_cleanup_headers - Correct empty headers 1827 * @rx_ring: rx descriptor ring packet is being transacted on 1828 * @skb: pointer to current skb being fixed 1829 * @rx_desc: pointer to the EOP Rx descriptor 1830 * 1831 * Also address the case where we are pulling data in on pages only 1832 * and as such no data is present in the skb header. 1833 * 1834 * In addition if skb is not at least 60 bytes we need to pad it so that 1835 * it is large enough to qualify as a valid Ethernet frame. 1836 * 1837 * Returns true if an error was encountered and skb was freed. 1838 **/ 1839 static bool i40e_cleanup_headers(struct i40e_ring *rx_ring, struct sk_buff *skb, 1840 union i40e_rx_desc *rx_desc) 1841 1842 { 1843 /* XDP packets use error pointer so abort at this point */ 1844 if (IS_ERR(skb)) 1845 return true; 1846 1847 /* ERR_MASK will only have valid bits if EOP set, and 1848 * what we are doing here is actually checking 1849 * I40E_RX_DESC_ERROR_RXE_SHIFT, since it is the zeroth bit in 1850 * the error field 1851 */ 1852 if (unlikely(i40e_test_staterr(rx_desc, 1853 BIT(I40E_RXD_QW1_ERROR_SHIFT)))) { 1854 dev_kfree_skb_any(skb); 1855 return true; 1856 } 1857 1858 /* if eth_skb_pad returns an error the skb was freed */ 1859 if (eth_skb_pad(skb)) 1860 return true; 1861 1862 return false; 1863 } 1864 1865 /** 1866 * i40e_page_is_reusable - check if any reuse is possible 1867 * @page: page struct to check 1868 * 1869 * A page is not reusable if it was allocated under low memory 1870 * conditions, or it's not in the same NUMA node as this CPU. 1871 */ 1872 static inline bool i40e_page_is_reusable(struct page *page) 1873 { 1874 return (page_to_nid(page) == numa_mem_id()) && 1875 !page_is_pfmemalloc(page); 1876 } 1877 1878 /** 1879 * i40e_can_reuse_rx_page - Determine if this page can be reused by 1880 * the adapter for another receive 1881 * 1882 * @rx_buffer: buffer containing the page 1883 * 1884 * If page is reusable, rx_buffer->page_offset is adjusted to point to 1885 * an unused region in the page. 1886 * 1887 * For small pages, @truesize will be a constant value, half the size 1888 * of the memory at page. We'll attempt to alternate between high and 1889 * low halves of the page, with one half ready for use by the hardware 1890 * and the other half being consumed by the stack. We use the page 1891 * ref count to determine whether the stack has finished consuming the 1892 * portion of this page that was passed up with a previous packet. If 1893 * the page ref count is >1, we'll assume the "other" half page is 1894 * still busy, and this page cannot be reused. 1895 * 1896 * For larger pages, @truesize will be the actual space used by the 1897 * received packet (adjusted upward to an even multiple of the cache 1898 * line size). This will advance through the page by the amount 1899 * actually consumed by the received packets while there is still 1900 * space for a buffer. Each region of larger pages will be used at 1901 * most once, after which the page will not be reused. 1902 * 1903 * In either case, if the page is reusable its refcount is increased. 1904 **/ 1905 static bool i40e_can_reuse_rx_page(struct i40e_rx_buffer *rx_buffer) 1906 { 1907 unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; 1908 struct page *page = rx_buffer->page; 1909 1910 /* Is any reuse possible? */ 1911 if (unlikely(!i40e_page_is_reusable(page))) 1912 return false; 1913 1914 #if (PAGE_SIZE < 8192) 1915 /* if we are only owner of page we can reuse it */ 1916 if (unlikely((page_count(page) - pagecnt_bias) > 1)) 1917 return false; 1918 #else 1919 #define I40E_LAST_OFFSET \ 1920 (SKB_WITH_OVERHEAD(PAGE_SIZE) - I40E_RXBUFFER_2048) 1921 if (rx_buffer->page_offset > I40E_LAST_OFFSET) 1922 return false; 1923 #endif 1924 1925 /* If we have drained the page fragment pool we need to update 1926 * the pagecnt_bias and page count so that we fully restock the 1927 * number of references the driver holds. 1928 */ 1929 if (unlikely(pagecnt_bias == 1)) { 1930 page_ref_add(page, USHRT_MAX - 1); 1931 rx_buffer->pagecnt_bias = USHRT_MAX; 1932 } 1933 1934 return true; 1935 } 1936 1937 /** 1938 * i40e_add_rx_frag - Add contents of Rx buffer to sk_buff 1939 * @rx_ring: rx descriptor ring to transact packets on 1940 * @rx_buffer: buffer containing page to add 1941 * @skb: sk_buff to place the data into 1942 * @size: packet length from rx_desc 1943 * 1944 * This function will add the data contained in rx_buffer->page to the skb. 1945 * It will just attach the page as a frag to the skb. 1946 * 1947 * The function will then update the page offset. 1948 **/ 1949 static void i40e_add_rx_frag(struct i40e_ring *rx_ring, 1950 struct i40e_rx_buffer *rx_buffer, 1951 struct sk_buff *skb, 1952 unsigned int size) 1953 { 1954 #if (PAGE_SIZE < 8192) 1955 unsigned int truesize = i40e_rx_pg_size(rx_ring) / 2; 1956 #else 1957 unsigned int truesize = SKB_DATA_ALIGN(size + i40e_rx_offset(rx_ring)); 1958 #endif 1959 1960 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, 1961 rx_buffer->page_offset, size, truesize); 1962 1963 /* page is being used so we must update the page offset */ 1964 #if (PAGE_SIZE < 8192) 1965 rx_buffer->page_offset ^= truesize; 1966 #else 1967 rx_buffer->page_offset += truesize; 1968 #endif 1969 } 1970 1971 /** 1972 * i40e_get_rx_buffer - Fetch Rx buffer and synchronize data for use 1973 * @rx_ring: rx descriptor ring to transact packets on 1974 * @size: size of buffer to add to skb 1975 * 1976 * This function will pull an Rx buffer from the ring and synchronize it 1977 * for use by the CPU. 1978 */ 1979 static struct i40e_rx_buffer *i40e_get_rx_buffer(struct i40e_ring *rx_ring, 1980 const unsigned int size) 1981 { 1982 struct i40e_rx_buffer *rx_buffer; 1983 1984 rx_buffer = &rx_ring->rx_bi[rx_ring->next_to_clean]; 1985 prefetchw(rx_buffer->page); 1986 1987 /* we are reusing so sync this buffer for CPU use */ 1988 dma_sync_single_range_for_cpu(rx_ring->dev, 1989 rx_buffer->dma, 1990 rx_buffer->page_offset, 1991 size, 1992 DMA_FROM_DEVICE); 1993 1994 /* We have pulled a buffer for use, so decrement pagecnt_bias */ 1995 rx_buffer->pagecnt_bias--; 1996 1997 return rx_buffer; 1998 } 1999 2000 /** 2001 * i40e_construct_skb - Allocate skb and populate it 2002 * @rx_ring: rx descriptor ring to transact packets on 2003 * @rx_buffer: rx buffer to pull data from 2004 * @xdp: xdp_buff pointing to the data 2005 * 2006 * This function allocates an skb. It then populates it with the page 2007 * data from the current receive descriptor, taking care to set up the 2008 * skb correctly. 2009 */ 2010 static struct sk_buff *i40e_construct_skb(struct i40e_ring *rx_ring, 2011 struct i40e_rx_buffer *rx_buffer, 2012 struct xdp_buff *xdp) 2013 { 2014 unsigned int size = xdp->data_end - xdp->data; 2015 #if (PAGE_SIZE < 8192) 2016 unsigned int truesize = i40e_rx_pg_size(rx_ring) / 2; 2017 #else 2018 unsigned int truesize = SKB_DATA_ALIGN(size); 2019 #endif 2020 unsigned int headlen; 2021 struct sk_buff *skb; 2022 2023 /* prefetch first cache line of first page */ 2024 prefetch(xdp->data); 2025 #if L1_CACHE_BYTES < 128 2026 prefetch(xdp->data + L1_CACHE_BYTES); 2027 #endif 2028 /* Note, we get here by enabling legacy-rx via: 2029 * 2030 * ethtool --set-priv-flags <dev> legacy-rx on 2031 * 2032 * In this mode, we currently get 0 extra XDP headroom as 2033 * opposed to having legacy-rx off, where we process XDP 2034 * packets going to stack via i40e_build_skb(). The latter 2035 * provides us currently with 192 bytes of headroom. 2036 * 2037 * For i40e_construct_skb() mode it means that the 2038 * xdp->data_meta will always point to xdp->data, since 2039 * the helper cannot expand the head. Should this ever 2040 * change in future for legacy-rx mode on, then lets also 2041 * add xdp->data_meta handling here. 2042 */ 2043 2044 /* allocate a skb to store the frags */ 2045 skb = __napi_alloc_skb(&rx_ring->q_vector->napi, 2046 I40E_RX_HDR_SIZE, 2047 GFP_ATOMIC | __GFP_NOWARN); 2048 if (unlikely(!skb)) 2049 return NULL; 2050 2051 /* Determine available headroom for copy */ 2052 headlen = size; 2053 if (headlen > I40E_RX_HDR_SIZE) 2054 headlen = eth_get_headlen(skb->dev, xdp->data, 2055 I40E_RX_HDR_SIZE); 2056 2057 /* align pull length to size of long to optimize memcpy performance */ 2058 memcpy(__skb_put(skb, headlen), xdp->data, 2059 ALIGN(headlen, sizeof(long))); 2060 2061 /* update all of the pointers */ 2062 size -= headlen; 2063 if (size) { 2064 skb_add_rx_frag(skb, 0, rx_buffer->page, 2065 rx_buffer->page_offset + headlen, 2066 size, truesize); 2067 2068 /* buffer is used by skb, update page_offset */ 2069 #if (PAGE_SIZE < 8192) 2070 rx_buffer->page_offset ^= truesize; 2071 #else 2072 rx_buffer->page_offset += truesize; 2073 #endif 2074 } else { 2075 /* buffer is unused, reset bias back to rx_buffer */ 2076 rx_buffer->pagecnt_bias++; 2077 } 2078 2079 return skb; 2080 } 2081 2082 /** 2083 * i40e_build_skb - Build skb around an existing buffer 2084 * @rx_ring: Rx descriptor ring to transact packets on 2085 * @rx_buffer: Rx buffer to pull data from 2086 * @xdp: xdp_buff pointing to the data 2087 * 2088 * This function builds an skb around an existing Rx buffer, taking care 2089 * to set up the skb correctly and avoid any memcpy overhead. 2090 */ 2091 static struct sk_buff *i40e_build_skb(struct i40e_ring *rx_ring, 2092 struct i40e_rx_buffer *rx_buffer, 2093 struct xdp_buff *xdp) 2094 { 2095 unsigned int metasize = xdp->data - xdp->data_meta; 2096 #if (PAGE_SIZE < 8192) 2097 unsigned int truesize = i40e_rx_pg_size(rx_ring) / 2; 2098 #else 2099 unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + 2100 SKB_DATA_ALIGN(xdp->data_end - 2101 xdp->data_hard_start); 2102 #endif 2103 struct sk_buff *skb; 2104 2105 /* Prefetch first cache line of first page. If xdp->data_meta 2106 * is unused, this points exactly as xdp->data, otherwise we 2107 * likely have a consumer accessing first few bytes of meta 2108 * data, and then actual data. 2109 */ 2110 prefetch(xdp->data_meta); 2111 #if L1_CACHE_BYTES < 128 2112 prefetch(xdp->data_meta + L1_CACHE_BYTES); 2113 #endif 2114 /* build an skb around the page buffer */ 2115 skb = build_skb(xdp->data_hard_start, truesize); 2116 if (unlikely(!skb)) 2117 return NULL; 2118 2119 /* update pointers within the skb to store the data */ 2120 skb_reserve(skb, xdp->data - xdp->data_hard_start); 2121 __skb_put(skb, xdp->data_end - xdp->data); 2122 if (metasize) 2123 skb_metadata_set(skb, metasize); 2124 2125 /* buffer is used by skb, update page_offset */ 2126 #if (PAGE_SIZE < 8192) 2127 rx_buffer->page_offset ^= truesize; 2128 #else 2129 rx_buffer->page_offset += truesize; 2130 #endif 2131 2132 return skb; 2133 } 2134 2135 /** 2136 * i40e_put_rx_buffer - Clean up used buffer and either recycle or free 2137 * @rx_ring: rx descriptor ring to transact packets on 2138 * @rx_buffer: rx buffer to pull data from 2139 * 2140 * This function will clean up the contents of the rx_buffer. It will 2141 * either recycle the buffer or unmap it and free the associated resources. 2142 */ 2143 static void i40e_put_rx_buffer(struct i40e_ring *rx_ring, 2144 struct i40e_rx_buffer *rx_buffer) 2145 { 2146 if (i40e_can_reuse_rx_page(rx_buffer)) { 2147 /* hand second half of page back to the ring */ 2148 i40e_reuse_rx_page(rx_ring, rx_buffer); 2149 } else { 2150 /* we are not reusing the buffer so unmap it */ 2151 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, 2152 i40e_rx_pg_size(rx_ring), 2153 DMA_FROM_DEVICE, I40E_RX_DMA_ATTR); 2154 __page_frag_cache_drain(rx_buffer->page, 2155 rx_buffer->pagecnt_bias); 2156 /* clear contents of buffer_info */ 2157 rx_buffer->page = NULL; 2158 } 2159 } 2160 2161 /** 2162 * i40e_is_non_eop - process handling of non-EOP buffers 2163 * @rx_ring: Rx ring being processed 2164 * @rx_desc: Rx descriptor for current buffer 2165 * @skb: Current socket buffer containing buffer in progress 2166 * 2167 * This function updates next to clean. If the buffer is an EOP buffer 2168 * this function exits returning false, otherwise it will place the 2169 * sk_buff in the next buffer to be chained and return true indicating 2170 * that this is in fact a non-EOP buffer. 2171 **/ 2172 static bool i40e_is_non_eop(struct i40e_ring *rx_ring, 2173 union i40e_rx_desc *rx_desc, 2174 struct sk_buff *skb) 2175 { 2176 u32 ntc = rx_ring->next_to_clean + 1; 2177 2178 /* fetch, update, and store next to clean */ 2179 ntc = (ntc < rx_ring->count) ? ntc : 0; 2180 rx_ring->next_to_clean = ntc; 2181 2182 prefetch(I40E_RX_DESC(rx_ring, ntc)); 2183 2184 /* if we are the last buffer then there is nothing else to do */ 2185 #define I40E_RXD_EOF BIT(I40E_RX_DESC_STATUS_EOF_SHIFT) 2186 if (likely(i40e_test_staterr(rx_desc, I40E_RXD_EOF))) 2187 return false; 2188 2189 rx_ring->rx_stats.non_eop_descs++; 2190 2191 return true; 2192 } 2193 2194 static int i40e_xmit_xdp_ring(struct xdp_frame *xdpf, 2195 struct i40e_ring *xdp_ring); 2196 2197 int i40e_xmit_xdp_tx_ring(struct xdp_buff *xdp, struct i40e_ring *xdp_ring) 2198 { 2199 struct xdp_frame *xdpf = convert_to_xdp_frame(xdp); 2200 2201 if (unlikely(!xdpf)) 2202 return I40E_XDP_CONSUMED; 2203 2204 return i40e_xmit_xdp_ring(xdpf, xdp_ring); 2205 } 2206 2207 /** 2208 * i40e_run_xdp - run an XDP program 2209 * @rx_ring: Rx ring being processed 2210 * @xdp: XDP buffer containing the frame 2211 **/ 2212 static struct sk_buff *i40e_run_xdp(struct i40e_ring *rx_ring, 2213 struct xdp_buff *xdp) 2214 { 2215 int err, result = I40E_XDP_PASS; 2216 struct i40e_ring *xdp_ring; 2217 struct bpf_prog *xdp_prog; 2218 u32 act; 2219 2220 rcu_read_lock(); 2221 xdp_prog = READ_ONCE(rx_ring->xdp_prog); 2222 2223 if (!xdp_prog) 2224 goto xdp_out; 2225 2226 prefetchw(xdp->data_hard_start); /* xdp_frame write */ 2227 2228 act = bpf_prog_run_xdp(xdp_prog, xdp); 2229 switch (act) { 2230 case XDP_PASS: 2231 break; 2232 case XDP_TX: 2233 xdp_ring = rx_ring->vsi->xdp_rings[rx_ring->queue_index]; 2234 result = i40e_xmit_xdp_tx_ring(xdp, xdp_ring); 2235 break; 2236 case XDP_REDIRECT: 2237 err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog); 2238 result = !err ? I40E_XDP_REDIR : I40E_XDP_CONSUMED; 2239 break; 2240 default: 2241 bpf_warn_invalid_xdp_action(act); 2242 /* fall through */ 2243 case XDP_ABORTED: 2244 trace_xdp_exception(rx_ring->netdev, xdp_prog, act); 2245 /* fall through -- handle aborts by dropping packet */ 2246 case XDP_DROP: 2247 result = I40E_XDP_CONSUMED; 2248 break; 2249 } 2250 xdp_out: 2251 rcu_read_unlock(); 2252 return ERR_PTR(-result); 2253 } 2254 2255 /** 2256 * i40e_rx_buffer_flip - adjusted rx_buffer to point to an unused region 2257 * @rx_ring: Rx ring 2258 * @rx_buffer: Rx buffer to adjust 2259 * @size: Size of adjustment 2260 **/ 2261 static void i40e_rx_buffer_flip(struct i40e_ring *rx_ring, 2262 struct i40e_rx_buffer *rx_buffer, 2263 unsigned int size) 2264 { 2265 unsigned int truesize = i40e_rx_frame_truesize(rx_ring, size); 2266 2267 #if (PAGE_SIZE < 8192) 2268 rx_buffer->page_offset ^= truesize; 2269 #else 2270 rx_buffer->page_offset += truesize; 2271 #endif 2272 } 2273 2274 /** 2275 * i40e_xdp_ring_update_tail - Updates the XDP Tx ring tail register 2276 * @xdp_ring: XDP Tx ring 2277 * 2278 * This function updates the XDP Tx ring tail register. 2279 **/ 2280 void i40e_xdp_ring_update_tail(struct i40e_ring *xdp_ring) 2281 { 2282 /* Force memory writes to complete before letting h/w 2283 * know there are new descriptors to fetch. 2284 */ 2285 wmb(); 2286 writel_relaxed(xdp_ring->next_to_use, xdp_ring->tail); 2287 } 2288 2289 /** 2290 * i40e_update_rx_stats - Update Rx ring statistics 2291 * @rx_ring: rx descriptor ring 2292 * @total_rx_bytes: number of bytes received 2293 * @total_rx_packets: number of packets received 2294 * 2295 * This function updates the Rx ring statistics. 2296 **/ 2297 void i40e_update_rx_stats(struct i40e_ring *rx_ring, 2298 unsigned int total_rx_bytes, 2299 unsigned int total_rx_packets) 2300 { 2301 u64_stats_update_begin(&rx_ring->syncp); 2302 rx_ring->stats.packets += total_rx_packets; 2303 rx_ring->stats.bytes += total_rx_bytes; 2304 u64_stats_update_end(&rx_ring->syncp); 2305 rx_ring->q_vector->rx.total_packets += total_rx_packets; 2306 rx_ring->q_vector->rx.total_bytes += total_rx_bytes; 2307 } 2308 2309 /** 2310 * i40e_finalize_xdp_rx - Bump XDP Tx tail and/or flush redirect map 2311 * @rx_ring: Rx ring 2312 * @xdp_res: Result of the receive batch 2313 * 2314 * This function bumps XDP Tx tail and/or flush redirect map, and 2315 * should be called when a batch of packets has been processed in the 2316 * napi loop. 2317 **/ 2318 void i40e_finalize_xdp_rx(struct i40e_ring *rx_ring, unsigned int xdp_res) 2319 { 2320 if (xdp_res & I40E_XDP_REDIR) 2321 xdp_do_flush_map(); 2322 2323 if (xdp_res & I40E_XDP_TX) { 2324 struct i40e_ring *xdp_ring = 2325 rx_ring->vsi->xdp_rings[rx_ring->queue_index]; 2326 2327 i40e_xdp_ring_update_tail(xdp_ring); 2328 } 2329 } 2330 2331 /** 2332 * i40e_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf 2333 * @rx_ring: rx descriptor ring to transact packets on 2334 * @budget: Total limit on number of packets to process 2335 * 2336 * This function provides a "bounce buffer" approach to Rx interrupt 2337 * processing. The advantage to this is that on systems that have 2338 * expensive overhead for IOMMU access this provides a means of avoiding 2339 * it by maintaining the mapping of the page to the system. 2340 * 2341 * Returns amount of work completed 2342 **/ 2343 static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget) 2344 { 2345 unsigned int total_rx_bytes = 0, total_rx_packets = 0; 2346 struct sk_buff *skb = rx_ring->skb; 2347 u16 cleaned_count = I40E_DESC_UNUSED(rx_ring); 2348 unsigned int xdp_xmit = 0; 2349 bool failure = false; 2350 struct xdp_buff xdp; 2351 2352 #if (PAGE_SIZE < 8192) 2353 xdp.frame_sz = i40e_rx_frame_truesize(rx_ring, 0); 2354 #endif 2355 xdp.rxq = &rx_ring->xdp_rxq; 2356 2357 while (likely(total_rx_packets < (unsigned int)budget)) { 2358 struct i40e_rx_buffer *rx_buffer; 2359 union i40e_rx_desc *rx_desc; 2360 unsigned int size; 2361 u64 qword; 2362 2363 /* return some buffers to hardware, one at a time is too slow */ 2364 if (cleaned_count >= I40E_RX_BUFFER_WRITE) { 2365 failure = failure || 2366 i40e_alloc_rx_buffers(rx_ring, cleaned_count); 2367 cleaned_count = 0; 2368 } 2369 2370 rx_desc = I40E_RX_DESC(rx_ring, rx_ring->next_to_clean); 2371 2372 /* status_error_len will always be zero for unused descriptors 2373 * because it's cleared in cleanup, and overlaps with hdr_addr 2374 * which is always zero because packet split isn't used, if the 2375 * hardware wrote DD then the length will be non-zero 2376 */ 2377 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len); 2378 2379 /* This memory barrier is needed to keep us from reading 2380 * any other fields out of the rx_desc until we have 2381 * verified the descriptor has been written back. 2382 */ 2383 dma_rmb(); 2384 2385 rx_buffer = i40e_clean_programming_status(rx_ring, rx_desc, 2386 qword); 2387 if (unlikely(rx_buffer)) { 2388 i40e_reuse_rx_page(rx_ring, rx_buffer); 2389 cleaned_count++; 2390 continue; 2391 } 2392 2393 size = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) >> 2394 I40E_RXD_QW1_LENGTH_PBUF_SHIFT; 2395 if (!size) 2396 break; 2397 2398 i40e_trace(clean_rx_irq, rx_ring, rx_desc, skb); 2399 rx_buffer = i40e_get_rx_buffer(rx_ring, size); 2400 2401 /* retrieve a buffer from the ring */ 2402 if (!skb) { 2403 xdp.data = page_address(rx_buffer->page) + 2404 rx_buffer->page_offset; 2405 xdp.data_meta = xdp.data; 2406 xdp.data_hard_start = xdp.data - 2407 i40e_rx_offset(rx_ring); 2408 xdp.data_end = xdp.data + size; 2409 #if (PAGE_SIZE > 4096) 2410 /* At larger PAGE_SIZE, frame_sz depend on len size */ 2411 xdp.frame_sz = i40e_rx_frame_truesize(rx_ring, size); 2412 #endif 2413 skb = i40e_run_xdp(rx_ring, &xdp); 2414 } 2415 2416 if (IS_ERR(skb)) { 2417 unsigned int xdp_res = -PTR_ERR(skb); 2418 2419 if (xdp_res & (I40E_XDP_TX | I40E_XDP_REDIR)) { 2420 xdp_xmit |= xdp_res; 2421 i40e_rx_buffer_flip(rx_ring, rx_buffer, size); 2422 } else { 2423 rx_buffer->pagecnt_bias++; 2424 } 2425 total_rx_bytes += size; 2426 total_rx_packets++; 2427 } else if (skb) { 2428 i40e_add_rx_frag(rx_ring, rx_buffer, skb, size); 2429 } else if (ring_uses_build_skb(rx_ring)) { 2430 skb = i40e_build_skb(rx_ring, rx_buffer, &xdp); 2431 } else { 2432 skb = i40e_construct_skb(rx_ring, rx_buffer, &xdp); 2433 } 2434 2435 /* exit if we failed to retrieve a buffer */ 2436 if (!skb) { 2437 rx_ring->rx_stats.alloc_buff_failed++; 2438 rx_buffer->pagecnt_bias++; 2439 break; 2440 } 2441 2442 i40e_put_rx_buffer(rx_ring, rx_buffer); 2443 cleaned_count++; 2444 2445 if (i40e_is_non_eop(rx_ring, rx_desc, skb)) 2446 continue; 2447 2448 if (i40e_cleanup_headers(rx_ring, skb, rx_desc)) { 2449 skb = NULL; 2450 continue; 2451 } 2452 2453 /* probably a little skewed due to removing CRC */ 2454 total_rx_bytes += skb->len; 2455 2456 /* populate checksum, VLAN, and protocol */ 2457 i40e_process_skb_fields(rx_ring, rx_desc, skb); 2458 2459 i40e_trace(clean_rx_irq_rx, rx_ring, rx_desc, skb); 2460 napi_gro_receive(&rx_ring->q_vector->napi, skb); 2461 skb = NULL; 2462 2463 /* update budget accounting */ 2464 total_rx_packets++; 2465 } 2466 2467 i40e_finalize_xdp_rx(rx_ring, xdp_xmit); 2468 rx_ring->skb = skb; 2469 2470 i40e_update_rx_stats(rx_ring, total_rx_bytes, total_rx_packets); 2471 2472 /* guarantee a trip back through this routine if there was a failure */ 2473 return failure ? budget : (int)total_rx_packets; 2474 } 2475 2476 static inline u32 i40e_buildreg_itr(const int type, u16 itr) 2477 { 2478 u32 val; 2479 2480 /* We don't bother with setting the CLEARPBA bit as the data sheet 2481 * points out doing so is "meaningless since it was already 2482 * auto-cleared". The auto-clearing happens when the interrupt is 2483 * asserted. 2484 * 2485 * Hardware errata 28 for also indicates that writing to a 2486 * xxINT_DYN_CTLx CSR with INTENA_MSK (bit 31) set to 0 will clear 2487 * an event in the PBA anyway so we need to rely on the automask 2488 * to hold pending events for us until the interrupt is re-enabled 2489 * 2490 * The itr value is reported in microseconds, and the register 2491 * value is recorded in 2 microsecond units. For this reason we 2492 * only need to shift by the interval shift - 1 instead of the 2493 * full value. 2494 */ 2495 itr &= I40E_ITR_MASK; 2496 2497 val = I40E_PFINT_DYN_CTLN_INTENA_MASK | 2498 (type << I40E_PFINT_DYN_CTLN_ITR_INDX_SHIFT) | 2499 (itr << (I40E_PFINT_DYN_CTLN_INTERVAL_SHIFT - 1)); 2500 2501 return val; 2502 } 2503 2504 /* a small macro to shorten up some long lines */ 2505 #define INTREG I40E_PFINT_DYN_CTLN 2506 2507 /* The act of updating the ITR will cause it to immediately trigger. In order 2508 * to prevent this from throwing off adaptive update statistics we defer the 2509 * update so that it can only happen so often. So after either Tx or Rx are 2510 * updated we make the adaptive scheme wait until either the ITR completely 2511 * expires via the next_update expiration or we have been through at least 2512 * 3 interrupts. 2513 */ 2514 #define ITR_COUNTDOWN_START 3 2515 2516 /** 2517 * i40e_update_enable_itr - Update itr and re-enable MSIX interrupt 2518 * @vsi: the VSI we care about 2519 * @q_vector: q_vector for which itr is being updated and interrupt enabled 2520 * 2521 **/ 2522 static inline void i40e_update_enable_itr(struct i40e_vsi *vsi, 2523 struct i40e_q_vector *q_vector) 2524 { 2525 struct i40e_hw *hw = &vsi->back->hw; 2526 u32 intval; 2527 2528 /* If we don't have MSIX, then we only need to re-enable icr0 */ 2529 if (!(vsi->back->flags & I40E_FLAG_MSIX_ENABLED)) { 2530 i40e_irq_dynamic_enable_icr0(vsi->back); 2531 return; 2532 } 2533 2534 /* These will do nothing if dynamic updates are not enabled */ 2535 i40e_update_itr(q_vector, &q_vector->tx); 2536 i40e_update_itr(q_vector, &q_vector->rx); 2537 2538 /* This block of logic allows us to get away with only updating 2539 * one ITR value with each interrupt. The idea is to perform a 2540 * pseudo-lazy update with the following criteria. 2541 * 2542 * 1. Rx is given higher priority than Tx if both are in same state 2543 * 2. If we must reduce an ITR that is given highest priority. 2544 * 3. We then give priority to increasing ITR based on amount. 2545 */ 2546 if (q_vector->rx.target_itr < q_vector->rx.current_itr) { 2547 /* Rx ITR needs to be reduced, this is highest priority */ 2548 intval = i40e_buildreg_itr(I40E_RX_ITR, 2549 q_vector->rx.target_itr); 2550 q_vector->rx.current_itr = q_vector->rx.target_itr; 2551 q_vector->itr_countdown = ITR_COUNTDOWN_START; 2552 } else if ((q_vector->tx.target_itr < q_vector->tx.current_itr) || 2553 ((q_vector->rx.target_itr - q_vector->rx.current_itr) < 2554 (q_vector->tx.target_itr - q_vector->tx.current_itr))) { 2555 /* Tx ITR needs to be reduced, this is second priority 2556 * Tx ITR needs to be increased more than Rx, fourth priority 2557 */ 2558 intval = i40e_buildreg_itr(I40E_TX_ITR, 2559 q_vector->tx.target_itr); 2560 q_vector->tx.current_itr = q_vector->tx.target_itr; 2561 q_vector->itr_countdown = ITR_COUNTDOWN_START; 2562 } else if (q_vector->rx.current_itr != q_vector->rx.target_itr) { 2563 /* Rx ITR needs to be increased, third priority */ 2564 intval = i40e_buildreg_itr(I40E_RX_ITR, 2565 q_vector->rx.target_itr); 2566 q_vector->rx.current_itr = q_vector->rx.target_itr; 2567 q_vector->itr_countdown = ITR_COUNTDOWN_START; 2568 } else { 2569 /* No ITR update, lowest priority */ 2570 intval = i40e_buildreg_itr(I40E_ITR_NONE, 0); 2571 if (q_vector->itr_countdown) 2572 q_vector->itr_countdown--; 2573 } 2574 2575 if (!test_bit(__I40E_VSI_DOWN, vsi->state)) 2576 wr32(hw, INTREG(q_vector->reg_idx), intval); 2577 } 2578 2579 /** 2580 * i40e_napi_poll - NAPI polling Rx/Tx cleanup routine 2581 * @napi: napi struct with our devices info in it 2582 * @budget: amount of work driver is allowed to do this pass, in packets 2583 * 2584 * This function will clean all queues associated with a q_vector. 2585 * 2586 * Returns the amount of work done 2587 **/ 2588 int i40e_napi_poll(struct napi_struct *napi, int budget) 2589 { 2590 struct i40e_q_vector *q_vector = 2591 container_of(napi, struct i40e_q_vector, napi); 2592 struct i40e_vsi *vsi = q_vector->vsi; 2593 struct i40e_ring *ring; 2594 bool clean_complete = true; 2595 bool arm_wb = false; 2596 int budget_per_ring; 2597 int work_done = 0; 2598 2599 if (test_bit(__I40E_VSI_DOWN, vsi->state)) { 2600 napi_complete(napi); 2601 return 0; 2602 } 2603 2604 /* Since the actual Tx work is minimal, we can give the Tx a larger 2605 * budget and be more aggressive about cleaning up the Tx descriptors. 2606 */ 2607 i40e_for_each_ring(ring, q_vector->tx) { 2608 bool wd = ring->xsk_umem ? 2609 i40e_clean_xdp_tx_irq(vsi, ring, budget) : 2610 i40e_clean_tx_irq(vsi, ring, budget); 2611 2612 if (!wd) { 2613 clean_complete = false; 2614 continue; 2615 } 2616 arm_wb |= ring->arm_wb; 2617 ring->arm_wb = false; 2618 } 2619 2620 /* Handle case where we are called by netpoll with a budget of 0 */ 2621 if (budget <= 0) 2622 goto tx_only; 2623 2624 /* We attempt to distribute budget to each Rx queue fairly, but don't 2625 * allow the budget to go below 1 because that would exit polling early. 2626 */ 2627 budget_per_ring = max(budget/q_vector->num_ringpairs, 1); 2628 2629 i40e_for_each_ring(ring, q_vector->rx) { 2630 int cleaned = ring->xsk_umem ? 2631 i40e_clean_rx_irq_zc(ring, budget_per_ring) : 2632 i40e_clean_rx_irq(ring, budget_per_ring); 2633 2634 work_done += cleaned; 2635 /* if we clean as many as budgeted, we must not be done */ 2636 if (cleaned >= budget_per_ring) 2637 clean_complete = false; 2638 } 2639 2640 /* If work not completed, return budget and polling will return */ 2641 if (!clean_complete) { 2642 int cpu_id = smp_processor_id(); 2643 2644 /* It is possible that the interrupt affinity has changed but, 2645 * if the cpu is pegged at 100%, polling will never exit while 2646 * traffic continues and the interrupt will be stuck on this 2647 * cpu. We check to make sure affinity is correct before we 2648 * continue to poll, otherwise we must stop polling so the 2649 * interrupt can move to the correct cpu. 2650 */ 2651 if (!cpumask_test_cpu(cpu_id, &q_vector->affinity_mask)) { 2652 /* Tell napi that we are done polling */ 2653 napi_complete_done(napi, work_done); 2654 2655 /* Force an interrupt */ 2656 i40e_force_wb(vsi, q_vector); 2657 2658 /* Return budget-1 so that polling stops */ 2659 return budget - 1; 2660 } 2661 tx_only: 2662 if (arm_wb) { 2663 q_vector->tx.ring[0].tx_stats.tx_force_wb++; 2664 i40e_enable_wb_on_itr(vsi, q_vector); 2665 } 2666 return budget; 2667 } 2668 2669 if (vsi->back->flags & I40E_TXR_FLAGS_WB_ON_ITR) 2670 q_vector->arm_wb_state = false; 2671 2672 /* Exit the polling mode, but don't re-enable interrupts if stack might 2673 * poll us due to busy-polling 2674 */ 2675 if (likely(napi_complete_done(napi, work_done))) 2676 i40e_update_enable_itr(vsi, q_vector); 2677 2678 return min(work_done, budget - 1); 2679 } 2680 2681 /** 2682 * i40e_atr - Add a Flow Director ATR filter 2683 * @tx_ring: ring to add programming descriptor to 2684 * @skb: send buffer 2685 * @tx_flags: send tx flags 2686 **/ 2687 static void i40e_atr(struct i40e_ring *tx_ring, struct sk_buff *skb, 2688 u32 tx_flags) 2689 { 2690 struct i40e_filter_program_desc *fdir_desc; 2691 struct i40e_pf *pf = tx_ring->vsi->back; 2692 union { 2693 unsigned char *network; 2694 struct iphdr *ipv4; 2695 struct ipv6hdr *ipv6; 2696 } hdr; 2697 struct tcphdr *th; 2698 unsigned int hlen; 2699 u32 flex_ptype, dtype_cmd; 2700 int l4_proto; 2701 u16 i; 2702 2703 /* make sure ATR is enabled */ 2704 if (!(pf->flags & I40E_FLAG_FD_ATR_ENABLED)) 2705 return; 2706 2707 if (test_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) 2708 return; 2709 2710 /* if sampling is disabled do nothing */ 2711 if (!tx_ring->atr_sample_rate) 2712 return; 2713 2714 /* Currently only IPv4/IPv6 with TCP is supported */ 2715 if (!(tx_flags & (I40E_TX_FLAGS_IPV4 | I40E_TX_FLAGS_IPV6))) 2716 return; 2717 2718 /* snag network header to get L4 type and address */ 2719 hdr.network = (tx_flags & I40E_TX_FLAGS_UDP_TUNNEL) ? 2720 skb_inner_network_header(skb) : skb_network_header(skb); 2721 2722 /* Note: tx_flags gets modified to reflect inner protocols in 2723 * tx_enable_csum function if encap is enabled. 2724 */ 2725 if (tx_flags & I40E_TX_FLAGS_IPV4) { 2726 /* access ihl as u8 to avoid unaligned access on ia64 */ 2727 hlen = (hdr.network[0] & 0x0F) << 2; 2728 l4_proto = hdr.ipv4->protocol; 2729 } else { 2730 /* find the start of the innermost ipv6 header */ 2731 unsigned int inner_hlen = hdr.network - skb->data; 2732 unsigned int h_offset = inner_hlen; 2733 2734 /* this function updates h_offset to the end of the header */ 2735 l4_proto = 2736 ipv6_find_hdr(skb, &h_offset, IPPROTO_TCP, NULL, NULL); 2737 /* hlen will contain our best estimate of the tcp header */ 2738 hlen = h_offset - inner_hlen; 2739 } 2740 2741 if (l4_proto != IPPROTO_TCP) 2742 return; 2743 2744 th = (struct tcphdr *)(hdr.network + hlen); 2745 2746 /* Due to lack of space, no more new filters can be programmed */ 2747 if (th->syn && test_bit(__I40E_FD_ATR_AUTO_DISABLED, pf->state)) 2748 return; 2749 if (pf->flags & I40E_FLAG_HW_ATR_EVICT_ENABLED) { 2750 /* HW ATR eviction will take care of removing filters on FIN 2751 * and RST packets. 2752 */ 2753 if (th->fin || th->rst) 2754 return; 2755 } 2756 2757 tx_ring->atr_count++; 2758 2759 /* sample on all syn/fin/rst packets or once every atr sample rate */ 2760 if (!th->fin && 2761 !th->syn && 2762 !th->rst && 2763 (tx_ring->atr_count < tx_ring->atr_sample_rate)) 2764 return; 2765 2766 tx_ring->atr_count = 0; 2767 2768 /* grab the next descriptor */ 2769 i = tx_ring->next_to_use; 2770 fdir_desc = I40E_TX_FDIRDESC(tx_ring, i); 2771 2772 i++; 2773 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 2774 2775 flex_ptype = (tx_ring->queue_index << I40E_TXD_FLTR_QW0_QINDEX_SHIFT) & 2776 I40E_TXD_FLTR_QW0_QINDEX_MASK; 2777 flex_ptype |= (tx_flags & I40E_TX_FLAGS_IPV4) ? 2778 (I40E_FILTER_PCTYPE_NONF_IPV4_TCP << 2779 I40E_TXD_FLTR_QW0_PCTYPE_SHIFT) : 2780 (I40E_FILTER_PCTYPE_NONF_IPV6_TCP << 2781 I40E_TXD_FLTR_QW0_PCTYPE_SHIFT); 2782 2783 flex_ptype |= tx_ring->vsi->id << I40E_TXD_FLTR_QW0_DEST_VSI_SHIFT; 2784 2785 dtype_cmd = I40E_TX_DESC_DTYPE_FILTER_PROG; 2786 2787 dtype_cmd |= (th->fin || th->rst) ? 2788 (I40E_FILTER_PROGRAM_DESC_PCMD_REMOVE << 2789 I40E_TXD_FLTR_QW1_PCMD_SHIFT) : 2790 (I40E_FILTER_PROGRAM_DESC_PCMD_ADD_UPDATE << 2791 I40E_TXD_FLTR_QW1_PCMD_SHIFT); 2792 2793 dtype_cmd |= I40E_FILTER_PROGRAM_DESC_DEST_DIRECT_PACKET_QINDEX << 2794 I40E_TXD_FLTR_QW1_DEST_SHIFT; 2795 2796 dtype_cmd |= I40E_FILTER_PROGRAM_DESC_FD_STATUS_FD_ID << 2797 I40E_TXD_FLTR_QW1_FD_STATUS_SHIFT; 2798 2799 dtype_cmd |= I40E_TXD_FLTR_QW1_CNT_ENA_MASK; 2800 if (!(tx_flags & I40E_TX_FLAGS_UDP_TUNNEL)) 2801 dtype_cmd |= 2802 ((u32)I40E_FD_ATR_STAT_IDX(pf->hw.pf_id) << 2803 I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) & 2804 I40E_TXD_FLTR_QW1_CNTINDEX_MASK; 2805 else 2806 dtype_cmd |= 2807 ((u32)I40E_FD_ATR_TUNNEL_STAT_IDX(pf->hw.pf_id) << 2808 I40E_TXD_FLTR_QW1_CNTINDEX_SHIFT) & 2809 I40E_TXD_FLTR_QW1_CNTINDEX_MASK; 2810 2811 if (pf->flags & I40E_FLAG_HW_ATR_EVICT_ENABLED) 2812 dtype_cmd |= I40E_TXD_FLTR_QW1_ATR_MASK; 2813 2814 fdir_desc->qindex_flex_ptype_vsi = cpu_to_le32(flex_ptype); 2815 fdir_desc->rsvd = cpu_to_le32(0); 2816 fdir_desc->dtype_cmd_cntindex = cpu_to_le32(dtype_cmd); 2817 fdir_desc->fd_id = cpu_to_le32(0); 2818 } 2819 2820 /** 2821 * i40e_tx_prepare_vlan_flags - prepare generic TX VLAN tagging flags for HW 2822 * @skb: send buffer 2823 * @tx_ring: ring to send buffer on 2824 * @flags: the tx flags to be set 2825 * 2826 * Checks the skb and set up correspondingly several generic transmit flags 2827 * related to VLAN tagging for the HW, such as VLAN, DCB, etc. 2828 * 2829 * Returns error code indicate the frame should be dropped upon error and the 2830 * otherwise returns 0 to indicate the flags has been set properly. 2831 **/ 2832 static inline int i40e_tx_prepare_vlan_flags(struct sk_buff *skb, 2833 struct i40e_ring *tx_ring, 2834 u32 *flags) 2835 { 2836 __be16 protocol = skb->protocol; 2837 u32 tx_flags = 0; 2838 2839 if (protocol == htons(ETH_P_8021Q) && 2840 !(tx_ring->netdev->features & NETIF_F_HW_VLAN_CTAG_TX)) { 2841 /* When HW VLAN acceleration is turned off by the user the 2842 * stack sets the protocol to 8021q so that the driver 2843 * can take any steps required to support the SW only 2844 * VLAN handling. In our case the driver doesn't need 2845 * to take any further steps so just set the protocol 2846 * to the encapsulated ethertype. 2847 */ 2848 skb->protocol = vlan_get_protocol(skb); 2849 goto out; 2850 } 2851 2852 /* if we have a HW VLAN tag being added, default to the HW one */ 2853 if (skb_vlan_tag_present(skb)) { 2854 tx_flags |= skb_vlan_tag_get(skb) << I40E_TX_FLAGS_VLAN_SHIFT; 2855 tx_flags |= I40E_TX_FLAGS_HW_VLAN; 2856 /* else if it is a SW VLAN, check the next protocol and store the tag */ 2857 } else if (protocol == htons(ETH_P_8021Q)) { 2858 struct vlan_hdr *vhdr, _vhdr; 2859 2860 vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr); 2861 if (!vhdr) 2862 return -EINVAL; 2863 2864 protocol = vhdr->h_vlan_encapsulated_proto; 2865 tx_flags |= ntohs(vhdr->h_vlan_TCI) << I40E_TX_FLAGS_VLAN_SHIFT; 2866 tx_flags |= I40E_TX_FLAGS_SW_VLAN; 2867 } 2868 2869 if (!(tx_ring->vsi->back->flags & I40E_FLAG_DCB_ENABLED)) 2870 goto out; 2871 2872 /* Insert 802.1p priority into VLAN header */ 2873 if ((tx_flags & (I40E_TX_FLAGS_HW_VLAN | I40E_TX_FLAGS_SW_VLAN)) || 2874 (skb->priority != TC_PRIO_CONTROL)) { 2875 tx_flags &= ~I40E_TX_FLAGS_VLAN_PRIO_MASK; 2876 tx_flags |= (skb->priority & 0x7) << 2877 I40E_TX_FLAGS_VLAN_PRIO_SHIFT; 2878 if (tx_flags & I40E_TX_FLAGS_SW_VLAN) { 2879 struct vlan_ethhdr *vhdr; 2880 int rc; 2881 2882 rc = skb_cow_head(skb, 0); 2883 if (rc < 0) 2884 return rc; 2885 vhdr = (struct vlan_ethhdr *)skb->data; 2886 vhdr->h_vlan_TCI = htons(tx_flags >> 2887 I40E_TX_FLAGS_VLAN_SHIFT); 2888 } else { 2889 tx_flags |= I40E_TX_FLAGS_HW_VLAN; 2890 } 2891 } 2892 2893 out: 2894 *flags = tx_flags; 2895 return 0; 2896 } 2897 2898 /** 2899 * i40e_tso - set up the tso context descriptor 2900 * @first: pointer to first Tx buffer for xmit 2901 * @hdr_len: ptr to the size of the packet header 2902 * @cd_type_cmd_tso_mss: Quad Word 1 2903 * 2904 * Returns 0 if no TSO can happen, 1 if tso is going, or error 2905 **/ 2906 static int i40e_tso(struct i40e_tx_buffer *first, u8 *hdr_len, 2907 u64 *cd_type_cmd_tso_mss) 2908 { 2909 struct sk_buff *skb = first->skb; 2910 u64 cd_cmd, cd_tso_len, cd_mss; 2911 union { 2912 struct iphdr *v4; 2913 struct ipv6hdr *v6; 2914 unsigned char *hdr; 2915 } ip; 2916 union { 2917 struct tcphdr *tcp; 2918 struct udphdr *udp; 2919 unsigned char *hdr; 2920 } l4; 2921 u32 paylen, l4_offset; 2922 u16 gso_segs, gso_size; 2923 int err; 2924 2925 if (skb->ip_summed != CHECKSUM_PARTIAL) 2926 return 0; 2927 2928 if (!skb_is_gso(skb)) 2929 return 0; 2930 2931 err = skb_cow_head(skb, 0); 2932 if (err < 0) 2933 return err; 2934 2935 ip.hdr = skb_network_header(skb); 2936 l4.hdr = skb_transport_header(skb); 2937 2938 /* initialize outer IP header fields */ 2939 if (ip.v4->version == 4) { 2940 ip.v4->tot_len = 0; 2941 ip.v4->check = 0; 2942 } else { 2943 ip.v6->payload_len = 0; 2944 } 2945 2946 if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE | 2947 SKB_GSO_GRE_CSUM | 2948 SKB_GSO_IPXIP4 | 2949 SKB_GSO_IPXIP6 | 2950 SKB_GSO_UDP_TUNNEL | 2951 SKB_GSO_UDP_TUNNEL_CSUM)) { 2952 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) && 2953 (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM)) { 2954 l4.udp->len = 0; 2955 2956 /* determine offset of outer transport header */ 2957 l4_offset = l4.hdr - skb->data; 2958 2959 /* remove payload length from outer checksum */ 2960 paylen = skb->len - l4_offset; 2961 csum_replace_by_diff(&l4.udp->check, 2962 (__force __wsum)htonl(paylen)); 2963 } 2964 2965 /* reset pointers to inner headers */ 2966 ip.hdr = skb_inner_network_header(skb); 2967 l4.hdr = skb_inner_transport_header(skb); 2968 2969 /* initialize inner IP header fields */ 2970 if (ip.v4->version == 4) { 2971 ip.v4->tot_len = 0; 2972 ip.v4->check = 0; 2973 } else { 2974 ip.v6->payload_len = 0; 2975 } 2976 } 2977 2978 /* determine offset of inner transport header */ 2979 l4_offset = l4.hdr - skb->data; 2980 2981 /* remove payload length from inner checksum */ 2982 paylen = skb->len - l4_offset; 2983 2984 if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) { 2985 csum_replace_by_diff(&l4.udp->check, (__force __wsum)htonl(paylen)); 2986 /* compute length of segmentation header */ 2987 *hdr_len = sizeof(*l4.udp) + l4_offset; 2988 } else { 2989 csum_replace_by_diff(&l4.tcp->check, (__force __wsum)htonl(paylen)); 2990 /* compute length of segmentation header */ 2991 *hdr_len = (l4.tcp->doff * 4) + l4_offset; 2992 } 2993 2994 /* pull values out of skb_shinfo */ 2995 gso_size = skb_shinfo(skb)->gso_size; 2996 gso_segs = skb_shinfo(skb)->gso_segs; 2997 2998 /* update GSO size and bytecount with header size */ 2999 first->gso_segs = gso_segs; 3000 first->bytecount += (first->gso_segs - 1) * *hdr_len; 3001 3002 /* find the field values */ 3003 cd_cmd = I40E_TX_CTX_DESC_TSO; 3004 cd_tso_len = skb->len - *hdr_len; 3005 cd_mss = gso_size; 3006 *cd_type_cmd_tso_mss |= (cd_cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) | 3007 (cd_tso_len << I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) | 3008 (cd_mss << I40E_TXD_CTX_QW1_MSS_SHIFT); 3009 return 1; 3010 } 3011 3012 /** 3013 * i40e_tsyn - set up the tsyn context descriptor 3014 * @tx_ring: ptr to the ring to send 3015 * @skb: ptr to the skb we're sending 3016 * @tx_flags: the collected send information 3017 * @cd_type_cmd_tso_mss: Quad Word 1 3018 * 3019 * Returns 0 if no Tx timestamp can happen and 1 if the timestamp will happen 3020 **/ 3021 static int i40e_tsyn(struct i40e_ring *tx_ring, struct sk_buff *skb, 3022 u32 tx_flags, u64 *cd_type_cmd_tso_mss) 3023 { 3024 struct i40e_pf *pf; 3025 3026 if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))) 3027 return 0; 3028 3029 /* Tx timestamps cannot be sampled when doing TSO */ 3030 if (tx_flags & I40E_TX_FLAGS_TSO) 3031 return 0; 3032 3033 /* only timestamp the outbound packet if the user has requested it and 3034 * we are not already transmitting a packet to be timestamped 3035 */ 3036 pf = i40e_netdev_to_pf(tx_ring->netdev); 3037 if (!(pf->flags & I40E_FLAG_PTP)) 3038 return 0; 3039 3040 if (pf->ptp_tx && 3041 !test_and_set_bit_lock(__I40E_PTP_TX_IN_PROGRESS, pf->state)) { 3042 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; 3043 pf->ptp_tx_start = jiffies; 3044 pf->ptp_tx_skb = skb_get(skb); 3045 } else { 3046 pf->tx_hwtstamp_skipped++; 3047 return 0; 3048 } 3049 3050 *cd_type_cmd_tso_mss |= (u64)I40E_TX_CTX_DESC_TSYN << 3051 I40E_TXD_CTX_QW1_CMD_SHIFT; 3052 3053 return 1; 3054 } 3055 3056 /** 3057 * i40e_tx_enable_csum - Enable Tx checksum offloads 3058 * @skb: send buffer 3059 * @tx_flags: pointer to Tx flags currently set 3060 * @td_cmd: Tx descriptor command bits to set 3061 * @td_offset: Tx descriptor header offsets to set 3062 * @tx_ring: Tx descriptor ring 3063 * @cd_tunneling: ptr to context desc bits 3064 **/ 3065 static int i40e_tx_enable_csum(struct sk_buff *skb, u32 *tx_flags, 3066 u32 *td_cmd, u32 *td_offset, 3067 struct i40e_ring *tx_ring, 3068 u32 *cd_tunneling) 3069 { 3070 union { 3071 struct iphdr *v4; 3072 struct ipv6hdr *v6; 3073 unsigned char *hdr; 3074 } ip; 3075 union { 3076 struct tcphdr *tcp; 3077 struct udphdr *udp; 3078 unsigned char *hdr; 3079 } l4; 3080 unsigned char *exthdr; 3081 u32 offset, cmd = 0; 3082 __be16 frag_off; 3083 u8 l4_proto = 0; 3084 3085 if (skb->ip_summed != CHECKSUM_PARTIAL) 3086 return 0; 3087 3088 ip.hdr = skb_network_header(skb); 3089 l4.hdr = skb_transport_header(skb); 3090 3091 /* compute outer L2 header size */ 3092 offset = ((ip.hdr - skb->data) / 2) << I40E_TX_DESC_LENGTH_MACLEN_SHIFT; 3093 3094 if (skb->encapsulation) { 3095 u32 tunnel = 0; 3096 /* define outer network header type */ 3097 if (*tx_flags & I40E_TX_FLAGS_IPV4) { 3098 tunnel |= (*tx_flags & I40E_TX_FLAGS_TSO) ? 3099 I40E_TX_CTX_EXT_IP_IPV4 : 3100 I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM; 3101 3102 l4_proto = ip.v4->protocol; 3103 } else if (*tx_flags & I40E_TX_FLAGS_IPV6) { 3104 tunnel |= I40E_TX_CTX_EXT_IP_IPV6; 3105 3106 exthdr = ip.hdr + sizeof(*ip.v6); 3107 l4_proto = ip.v6->nexthdr; 3108 if (l4.hdr != exthdr) 3109 ipv6_skip_exthdr(skb, exthdr - skb->data, 3110 &l4_proto, &frag_off); 3111 } 3112 3113 /* define outer transport */ 3114 switch (l4_proto) { 3115 case IPPROTO_UDP: 3116 tunnel |= I40E_TXD_CTX_UDP_TUNNELING; 3117 *tx_flags |= I40E_TX_FLAGS_UDP_TUNNEL; 3118 break; 3119 case IPPROTO_GRE: 3120 tunnel |= I40E_TXD_CTX_GRE_TUNNELING; 3121 *tx_flags |= I40E_TX_FLAGS_UDP_TUNNEL; 3122 break; 3123 case IPPROTO_IPIP: 3124 case IPPROTO_IPV6: 3125 *tx_flags |= I40E_TX_FLAGS_UDP_TUNNEL; 3126 l4.hdr = skb_inner_network_header(skb); 3127 break; 3128 default: 3129 if (*tx_flags & I40E_TX_FLAGS_TSO) 3130 return -1; 3131 3132 skb_checksum_help(skb); 3133 return 0; 3134 } 3135 3136 /* compute outer L3 header size */ 3137 tunnel |= ((l4.hdr - ip.hdr) / 4) << 3138 I40E_TXD_CTX_QW0_EXT_IPLEN_SHIFT; 3139 3140 /* switch IP header pointer from outer to inner header */ 3141 ip.hdr = skb_inner_network_header(skb); 3142 3143 /* compute tunnel header size */ 3144 tunnel |= ((ip.hdr - l4.hdr) / 2) << 3145 I40E_TXD_CTX_QW0_NATLEN_SHIFT; 3146 3147 /* indicate if we need to offload outer UDP header */ 3148 if ((*tx_flags & I40E_TX_FLAGS_TSO) && 3149 !(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) && 3150 (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM)) 3151 tunnel |= I40E_TXD_CTX_QW0_L4T_CS_MASK; 3152 3153 /* record tunnel offload values */ 3154 *cd_tunneling |= tunnel; 3155 3156 /* switch L4 header pointer from outer to inner */ 3157 l4.hdr = skb_inner_transport_header(skb); 3158 l4_proto = 0; 3159 3160 /* reset type as we transition from outer to inner headers */ 3161 *tx_flags &= ~(I40E_TX_FLAGS_IPV4 | I40E_TX_FLAGS_IPV6); 3162 if (ip.v4->version == 4) 3163 *tx_flags |= I40E_TX_FLAGS_IPV4; 3164 if (ip.v6->version == 6) 3165 *tx_flags |= I40E_TX_FLAGS_IPV6; 3166 } 3167 3168 /* Enable IP checksum offloads */ 3169 if (*tx_flags & I40E_TX_FLAGS_IPV4) { 3170 l4_proto = ip.v4->protocol; 3171 /* the stack computes the IP header already, the only time we 3172 * need the hardware to recompute it is in the case of TSO. 3173 */ 3174 cmd |= (*tx_flags & I40E_TX_FLAGS_TSO) ? 3175 I40E_TX_DESC_CMD_IIPT_IPV4_CSUM : 3176 I40E_TX_DESC_CMD_IIPT_IPV4; 3177 } else if (*tx_flags & I40E_TX_FLAGS_IPV6) { 3178 cmd |= I40E_TX_DESC_CMD_IIPT_IPV6; 3179 3180 exthdr = ip.hdr + sizeof(*ip.v6); 3181 l4_proto = ip.v6->nexthdr; 3182 if (l4.hdr != exthdr) 3183 ipv6_skip_exthdr(skb, exthdr - skb->data, 3184 &l4_proto, &frag_off); 3185 } 3186 3187 /* compute inner L3 header size */ 3188 offset |= ((l4.hdr - ip.hdr) / 4) << I40E_TX_DESC_LENGTH_IPLEN_SHIFT; 3189 3190 /* Enable L4 checksum offloads */ 3191 switch (l4_proto) { 3192 case IPPROTO_TCP: 3193 /* enable checksum offloads */ 3194 cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP; 3195 offset |= l4.tcp->doff << I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; 3196 break; 3197 case IPPROTO_SCTP: 3198 /* enable SCTP checksum offload */ 3199 cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP; 3200 offset |= (sizeof(struct sctphdr) >> 2) << 3201 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; 3202 break; 3203 case IPPROTO_UDP: 3204 /* enable UDP checksum offload */ 3205 cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP; 3206 offset |= (sizeof(struct udphdr) >> 2) << 3207 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT; 3208 break; 3209 default: 3210 if (*tx_flags & I40E_TX_FLAGS_TSO) 3211 return -1; 3212 skb_checksum_help(skb); 3213 return 0; 3214 } 3215 3216 *td_cmd |= cmd; 3217 *td_offset |= offset; 3218 3219 return 1; 3220 } 3221 3222 /** 3223 * i40e_create_tx_ctx Build the Tx context descriptor 3224 * @tx_ring: ring to create the descriptor on 3225 * @cd_type_cmd_tso_mss: Quad Word 1 3226 * @cd_tunneling: Quad Word 0 - bits 0-31 3227 * @cd_l2tag2: Quad Word 0 - bits 32-63 3228 **/ 3229 static void i40e_create_tx_ctx(struct i40e_ring *tx_ring, 3230 const u64 cd_type_cmd_tso_mss, 3231 const u32 cd_tunneling, const u32 cd_l2tag2) 3232 { 3233 struct i40e_tx_context_desc *context_desc; 3234 int i = tx_ring->next_to_use; 3235 3236 if ((cd_type_cmd_tso_mss == I40E_TX_DESC_DTYPE_CONTEXT) && 3237 !cd_tunneling && !cd_l2tag2) 3238 return; 3239 3240 /* grab the next descriptor */ 3241 context_desc = I40E_TX_CTXTDESC(tx_ring, i); 3242 3243 i++; 3244 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0; 3245 3246 /* cpu_to_le32 and assign to struct fields */ 3247 context_desc->tunneling_params = cpu_to_le32(cd_tunneling); 3248 context_desc->l2tag2 = cpu_to_le16(cd_l2tag2); 3249 context_desc->rsvd = cpu_to_le16(0); 3250 context_desc->type_cmd_tso_mss = cpu_to_le64(cd_type_cmd_tso_mss); 3251 } 3252 3253 /** 3254 * __i40e_maybe_stop_tx - 2nd level check for tx stop conditions 3255 * @tx_ring: the ring to be checked 3256 * @size: the size buffer we want to assure is available 3257 * 3258 * Returns -EBUSY if a stop is needed, else 0 3259 **/ 3260 int __i40e_maybe_stop_tx(struct i40e_ring *tx_ring, int size) 3261 { 3262 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index); 3263 /* Memory barrier before checking head and tail */ 3264 smp_mb(); 3265 3266 /* Check again in a case another CPU has just made room available. */ 3267 if (likely(I40E_DESC_UNUSED(tx_ring) < size)) 3268 return -EBUSY; 3269 3270 /* A reprieve! - use start_queue because it doesn't call schedule */ 3271 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index); 3272 ++tx_ring->tx_stats.restart_queue; 3273 return 0; 3274 } 3275 3276 /** 3277 * __i40e_chk_linearize - Check if there are more than 8 buffers per packet 3278 * @skb: send buffer 3279 * 3280 * Note: Our HW can't DMA more than 8 buffers to build a packet on the wire 3281 * and so we need to figure out the cases where we need to linearize the skb. 3282 * 3283 * For TSO we need to count the TSO header and segment payload separately. 3284 * As such we need to check cases where we have 7 fragments or more as we 3285 * can potentially require 9 DMA transactions, 1 for the TSO header, 1 for 3286 * the segment payload in the first descriptor, and another 7 for the 3287 * fragments. 3288 **/ 3289 bool __i40e_chk_linearize(struct sk_buff *skb) 3290 { 3291 const skb_frag_t *frag, *stale; 3292 int nr_frags, sum; 3293 3294 /* no need to check if number of frags is less than 7 */ 3295 nr_frags = skb_shinfo(skb)->nr_frags; 3296 if (nr_frags < (I40E_MAX_BUFFER_TXD - 1)) 3297 return false; 3298 3299 /* We need to walk through the list and validate that each group 3300 * of 6 fragments totals at least gso_size. 3301 */ 3302 nr_frags -= I40E_MAX_BUFFER_TXD - 2; 3303 frag = &skb_shinfo(skb)->frags[0]; 3304 3305 /* Initialize size to the negative value of gso_size minus 1. We 3306 * use this as the worst case scenerio in which the frag ahead 3307 * of us only provides one byte which is why we are limited to 6 3308 * descriptors for a single transmit as the header and previous 3309 * fragment are already consuming 2 descriptors. 3310 */ 3311 sum = 1 - skb_shinfo(skb)->gso_size; 3312 3313 /* Add size of frags 0 through 4 to create our initial sum */ 3314 sum += skb_frag_size(frag++); 3315 sum += skb_frag_size(frag++); 3316 sum += skb_frag_size(frag++); 3317 sum += skb_frag_size(frag++); 3318 sum += skb_frag_size(frag++); 3319 3320 /* Walk through fragments adding latest fragment, testing it, and 3321 * then removing stale fragments from the sum. 3322 */ 3323 for (stale = &skb_shinfo(skb)->frags[0];; stale++) { 3324 int stale_size = skb_frag_size(stale); 3325 3326 sum += skb_frag_size(frag++); 3327 3328 /* The stale fragment may present us with a smaller 3329 * descriptor than the actual fragment size. To account 3330 * for that we need to remove all the data on the front and 3331 * figure out what the remainder would be in the last 3332 * descriptor associated with the fragment. 3333 */ 3334 if (stale_size > I40E_MAX_DATA_PER_TXD) { 3335 int align_pad = -(skb_frag_off(stale)) & 3336 (I40E_MAX_READ_REQ_SIZE - 1); 3337 3338 sum -= align_pad; 3339 stale_size -= align_pad; 3340 3341 do { 3342 sum -= I40E_MAX_DATA_PER_TXD_ALIGNED; 3343 stale_size -= I40E_MAX_DATA_PER_TXD_ALIGNED; 3344 } while (stale_size > I40E_MAX_DATA_PER_TXD); 3345 } 3346 3347 /* if sum is negative we failed to make sufficient progress */ 3348 if (sum < 0) 3349 return true; 3350 3351 if (!nr_frags--) 3352 break; 3353 3354 sum -= stale_size; 3355 } 3356 3357 return false; 3358 } 3359 3360 /** 3361 * i40e_tx_map - Build the Tx descriptor 3362 * @tx_ring: ring to send buffer on 3363 * @skb: send buffer 3364 * @first: first buffer info buffer to use 3365 * @tx_flags: collected send information 3366 * @hdr_len: size of the packet header 3367 * @td_cmd: the command field in the descriptor 3368 * @td_offset: offset for checksum or crc 3369 * 3370 * Returns 0 on success, -1 on failure to DMA 3371 **/ 3372 static inline int i40e_tx_map(struct i40e_ring *tx_ring, struct sk_buff *skb, 3373 struct i40e_tx_buffer *first, u32 tx_flags, 3374 const u8 hdr_len, u32 td_cmd, u32 td_offset) 3375 { 3376 unsigned int data_len = skb->data_len; 3377 unsigned int size = skb_headlen(skb); 3378 skb_frag_t *frag; 3379 struct i40e_tx_buffer *tx_bi; 3380 struct i40e_tx_desc *tx_desc; 3381 u16 i = tx_ring->next_to_use; 3382 u32 td_tag = 0; 3383 dma_addr_t dma; 3384 u16 desc_count = 1; 3385 3386 if (tx_flags & I40E_TX_FLAGS_HW_VLAN) { 3387 td_cmd |= I40E_TX_DESC_CMD_IL2TAG1; 3388 td_tag = (tx_flags & I40E_TX_FLAGS_VLAN_MASK) >> 3389 I40E_TX_FLAGS_VLAN_SHIFT; 3390 } 3391 3392 first->tx_flags = tx_flags; 3393 3394 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE); 3395 3396 tx_desc = I40E_TX_DESC(tx_ring, i); 3397 tx_bi = first; 3398 3399 for (frag = &skb_shinfo(skb)->frags[0];; frag++) { 3400 unsigned int max_data = I40E_MAX_DATA_PER_TXD_ALIGNED; 3401 3402 if (dma_mapping_error(tx_ring->dev, dma)) 3403 goto dma_error; 3404 3405 /* record length, and DMA address */ 3406 dma_unmap_len_set(tx_bi, len, size); 3407 dma_unmap_addr_set(tx_bi, dma, dma); 3408 3409 /* align size to end of page */ 3410 max_data += -dma & (I40E_MAX_READ_REQ_SIZE - 1); 3411 tx_desc->buffer_addr = cpu_to_le64(dma); 3412 3413 while (unlikely(size > I40E_MAX_DATA_PER_TXD)) { 3414 tx_desc->cmd_type_offset_bsz = 3415 build_ctob(td_cmd, td_offset, 3416 max_data, td_tag); 3417 3418 tx_desc++; 3419 i++; 3420 desc_count++; 3421 3422 if (i == tx_ring->count) { 3423 tx_desc = I40E_TX_DESC(tx_ring, 0); 3424 i = 0; 3425 } 3426 3427 dma += max_data; 3428 size -= max_data; 3429 3430 max_data = I40E_MAX_DATA_PER_TXD_ALIGNED; 3431 tx_desc->buffer_addr = cpu_to_le64(dma); 3432 } 3433 3434 if (likely(!data_len)) 3435 break; 3436 3437 tx_desc->cmd_type_offset_bsz = build_ctob(td_cmd, td_offset, 3438 size, td_tag); 3439 3440 tx_desc++; 3441 i++; 3442 desc_count++; 3443 3444 if (i == tx_ring->count) { 3445 tx_desc = I40E_TX_DESC(tx_ring, 0); 3446 i = 0; 3447 } 3448 3449 size = skb_frag_size(frag); 3450 data_len -= size; 3451 3452 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, 3453 DMA_TO_DEVICE); 3454 3455 tx_bi = &tx_ring->tx_bi[i]; 3456 } 3457 3458 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount); 3459 3460 i++; 3461 if (i == tx_ring->count) 3462 i = 0; 3463 3464 tx_ring->next_to_use = i; 3465 3466 i40e_maybe_stop_tx(tx_ring, DESC_NEEDED); 3467 3468 /* write last descriptor with EOP bit */ 3469 td_cmd |= I40E_TX_DESC_CMD_EOP; 3470 3471 /* We OR these values together to check both against 4 (WB_STRIDE) 3472 * below. This is safe since we don't re-use desc_count afterwards. 3473 */ 3474 desc_count |= ++tx_ring->packet_stride; 3475 3476 if (desc_count >= WB_STRIDE) { 3477 /* write last descriptor with RS bit set */ 3478 td_cmd |= I40E_TX_DESC_CMD_RS; 3479 tx_ring->packet_stride = 0; 3480 } 3481 3482 tx_desc->cmd_type_offset_bsz = 3483 build_ctob(td_cmd, td_offset, size, td_tag); 3484 3485 skb_tx_timestamp(skb); 3486 3487 /* Force memory writes to complete before letting h/w know there 3488 * are new descriptors to fetch. 3489 * 3490 * We also use this memory barrier to make certain all of the 3491 * status bits have been updated before next_to_watch is written. 3492 */ 3493 wmb(); 3494 3495 /* set next_to_watch value indicating a packet is present */ 3496 first->next_to_watch = tx_desc; 3497 3498 /* notify HW of packet */ 3499 if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) { 3500 writel(i, tx_ring->tail); 3501 } 3502 3503 return 0; 3504 3505 dma_error: 3506 dev_info(tx_ring->dev, "TX DMA map failed\n"); 3507 3508 /* clear dma mappings for failed tx_bi map */ 3509 for (;;) { 3510 tx_bi = &tx_ring->tx_bi[i]; 3511 i40e_unmap_and_free_tx_resource(tx_ring, tx_bi); 3512 if (tx_bi == first) 3513 break; 3514 if (i == 0) 3515 i = tx_ring->count; 3516 i--; 3517 } 3518 3519 tx_ring->next_to_use = i; 3520 3521 return -1; 3522 } 3523 3524 /** 3525 * i40e_xmit_xdp_ring - transmits an XDP buffer to an XDP Tx ring 3526 * @xdp: data to transmit 3527 * @xdp_ring: XDP Tx ring 3528 **/ 3529 static int i40e_xmit_xdp_ring(struct xdp_frame *xdpf, 3530 struct i40e_ring *xdp_ring) 3531 { 3532 u16 i = xdp_ring->next_to_use; 3533 struct i40e_tx_buffer *tx_bi; 3534 struct i40e_tx_desc *tx_desc; 3535 void *data = xdpf->data; 3536 u32 size = xdpf->len; 3537 dma_addr_t dma; 3538 3539 if (!unlikely(I40E_DESC_UNUSED(xdp_ring))) { 3540 xdp_ring->tx_stats.tx_busy++; 3541 return I40E_XDP_CONSUMED; 3542 } 3543 dma = dma_map_single(xdp_ring->dev, data, size, DMA_TO_DEVICE); 3544 if (dma_mapping_error(xdp_ring->dev, dma)) 3545 return I40E_XDP_CONSUMED; 3546 3547 tx_bi = &xdp_ring->tx_bi[i]; 3548 tx_bi->bytecount = size; 3549 tx_bi->gso_segs = 1; 3550 tx_bi->xdpf = xdpf; 3551 3552 /* record length, and DMA address */ 3553 dma_unmap_len_set(tx_bi, len, size); 3554 dma_unmap_addr_set(tx_bi, dma, dma); 3555 3556 tx_desc = I40E_TX_DESC(xdp_ring, i); 3557 tx_desc->buffer_addr = cpu_to_le64(dma); 3558 tx_desc->cmd_type_offset_bsz = build_ctob(I40E_TX_DESC_CMD_ICRC 3559 | I40E_TXD_CMD, 3560 0, size, 0); 3561 3562 /* Make certain all of the status bits have been updated 3563 * before next_to_watch is written. 3564 */ 3565 smp_wmb(); 3566 3567 i++; 3568 if (i == xdp_ring->count) 3569 i = 0; 3570 3571 tx_bi->next_to_watch = tx_desc; 3572 xdp_ring->next_to_use = i; 3573 3574 return I40E_XDP_TX; 3575 } 3576 3577 /** 3578 * i40e_xmit_frame_ring - Sends buffer on Tx ring 3579 * @skb: send buffer 3580 * @tx_ring: ring to send buffer on 3581 * 3582 * Returns NETDEV_TX_OK if sent, else an error code 3583 **/ 3584 static netdev_tx_t i40e_xmit_frame_ring(struct sk_buff *skb, 3585 struct i40e_ring *tx_ring) 3586 { 3587 u64 cd_type_cmd_tso_mss = I40E_TX_DESC_DTYPE_CONTEXT; 3588 u32 cd_tunneling = 0, cd_l2tag2 = 0; 3589 struct i40e_tx_buffer *first; 3590 u32 td_offset = 0; 3591 u32 tx_flags = 0; 3592 __be16 protocol; 3593 u32 td_cmd = 0; 3594 u8 hdr_len = 0; 3595 int tso, count; 3596 int tsyn; 3597 3598 /* prefetch the data, we'll need it later */ 3599 prefetch(skb->data); 3600 3601 i40e_trace(xmit_frame_ring, skb, tx_ring); 3602 3603 count = i40e_xmit_descriptor_count(skb); 3604 if (i40e_chk_linearize(skb, count)) { 3605 if (__skb_linearize(skb)) { 3606 dev_kfree_skb_any(skb); 3607 return NETDEV_TX_OK; 3608 } 3609 count = i40e_txd_use_count(skb->len); 3610 tx_ring->tx_stats.tx_linearize++; 3611 } 3612 3613 /* need: 1 descriptor per page * PAGE_SIZE/I40E_MAX_DATA_PER_TXD, 3614 * + 1 desc for skb_head_len/I40E_MAX_DATA_PER_TXD, 3615 * + 4 desc gap to avoid the cache line where head is, 3616 * + 1 desc for context descriptor, 3617 * otherwise try next time 3618 */ 3619 if (i40e_maybe_stop_tx(tx_ring, count + 4 + 1)) { 3620 tx_ring->tx_stats.tx_busy++; 3621 return NETDEV_TX_BUSY; 3622 } 3623 3624 /* record the location of the first descriptor for this packet */ 3625 first = &tx_ring->tx_bi[tx_ring->next_to_use]; 3626 first->skb = skb; 3627 first->bytecount = skb->len; 3628 first->gso_segs = 1; 3629 3630 /* prepare the xmit flags */ 3631 if (i40e_tx_prepare_vlan_flags(skb, tx_ring, &tx_flags)) 3632 goto out_drop; 3633 3634 /* obtain protocol of skb */ 3635 protocol = vlan_get_protocol(skb); 3636 3637 /* setup IPv4/IPv6 offloads */ 3638 if (protocol == htons(ETH_P_IP)) 3639 tx_flags |= I40E_TX_FLAGS_IPV4; 3640 else if (protocol == htons(ETH_P_IPV6)) 3641 tx_flags |= I40E_TX_FLAGS_IPV6; 3642 3643 tso = i40e_tso(first, &hdr_len, &cd_type_cmd_tso_mss); 3644 3645 if (tso < 0) 3646 goto out_drop; 3647 else if (tso) 3648 tx_flags |= I40E_TX_FLAGS_TSO; 3649 3650 /* Always offload the checksum, since it's in the data descriptor */ 3651 tso = i40e_tx_enable_csum(skb, &tx_flags, &td_cmd, &td_offset, 3652 tx_ring, &cd_tunneling); 3653 if (tso < 0) 3654 goto out_drop; 3655 3656 tsyn = i40e_tsyn(tx_ring, skb, tx_flags, &cd_type_cmd_tso_mss); 3657 3658 if (tsyn) 3659 tx_flags |= I40E_TX_FLAGS_TSYN; 3660 3661 /* always enable CRC insertion offload */ 3662 td_cmd |= I40E_TX_DESC_CMD_ICRC; 3663 3664 i40e_create_tx_ctx(tx_ring, cd_type_cmd_tso_mss, 3665 cd_tunneling, cd_l2tag2); 3666 3667 /* Add Flow Director ATR if it's enabled. 3668 * 3669 * NOTE: this must always be directly before the data descriptor. 3670 */ 3671 i40e_atr(tx_ring, skb, tx_flags); 3672 3673 if (i40e_tx_map(tx_ring, skb, first, tx_flags, hdr_len, 3674 td_cmd, td_offset)) 3675 goto cleanup_tx_tstamp; 3676 3677 return NETDEV_TX_OK; 3678 3679 out_drop: 3680 i40e_trace(xmit_frame_ring_drop, first->skb, tx_ring); 3681 dev_kfree_skb_any(first->skb); 3682 first->skb = NULL; 3683 cleanup_tx_tstamp: 3684 if (unlikely(tx_flags & I40E_TX_FLAGS_TSYN)) { 3685 struct i40e_pf *pf = i40e_netdev_to_pf(tx_ring->netdev); 3686 3687 dev_kfree_skb_any(pf->ptp_tx_skb); 3688 pf->ptp_tx_skb = NULL; 3689 clear_bit_unlock(__I40E_PTP_TX_IN_PROGRESS, pf->state); 3690 } 3691 3692 return NETDEV_TX_OK; 3693 } 3694 3695 /** 3696 * i40e_lan_xmit_frame - Selects the correct VSI and Tx queue to send buffer 3697 * @skb: send buffer 3698 * @netdev: network interface device structure 3699 * 3700 * Returns NETDEV_TX_OK if sent, else an error code 3701 **/ 3702 netdev_tx_t i40e_lan_xmit_frame(struct sk_buff *skb, struct net_device *netdev) 3703 { 3704 struct i40e_netdev_priv *np = netdev_priv(netdev); 3705 struct i40e_vsi *vsi = np->vsi; 3706 struct i40e_ring *tx_ring = vsi->tx_rings[skb->queue_mapping]; 3707 3708 /* hardware can't handle really short frames, hardware padding works 3709 * beyond this point 3710 */ 3711 if (skb_put_padto(skb, I40E_MIN_TX_LEN)) 3712 return NETDEV_TX_OK; 3713 3714 return i40e_xmit_frame_ring(skb, tx_ring); 3715 } 3716 3717 /** 3718 * i40e_xdp_xmit - Implements ndo_xdp_xmit 3719 * @dev: netdev 3720 * @xdp: XDP buffer 3721 * 3722 * Returns number of frames successfully sent. Frames that fail are 3723 * free'ed via XDP return API. 3724 * 3725 * For error cases, a negative errno code is returned and no-frames 3726 * are transmitted (caller must handle freeing frames). 3727 **/ 3728 int i40e_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames, 3729 u32 flags) 3730 { 3731 struct i40e_netdev_priv *np = netdev_priv(dev); 3732 unsigned int queue_index = smp_processor_id(); 3733 struct i40e_vsi *vsi = np->vsi; 3734 struct i40e_pf *pf = vsi->back; 3735 struct i40e_ring *xdp_ring; 3736 int drops = 0; 3737 int i; 3738 3739 if (test_bit(__I40E_VSI_DOWN, vsi->state)) 3740 return -ENETDOWN; 3741 3742 if (!i40e_enabled_xdp_vsi(vsi) || queue_index >= vsi->num_queue_pairs || 3743 test_bit(__I40E_CONFIG_BUSY, pf->state)) 3744 return -ENXIO; 3745 3746 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 3747 return -EINVAL; 3748 3749 xdp_ring = vsi->xdp_rings[queue_index]; 3750 3751 for (i = 0; i < n; i++) { 3752 struct xdp_frame *xdpf = frames[i]; 3753 int err; 3754 3755 err = i40e_xmit_xdp_ring(xdpf, xdp_ring); 3756 if (err != I40E_XDP_TX) { 3757 xdp_return_frame_rx_napi(xdpf); 3758 drops++; 3759 } 3760 } 3761 3762 if (unlikely(flags & XDP_XMIT_FLUSH)) 3763 i40e_xdp_ring_update_tail(xdp_ring); 3764 3765 return n - drops; 3766 } 3767