Lines Matching full:tso
121 "Bitmask of FW-assisted TSO allowed to use if supported by NIC firmware");
366 * large number of descriptors for TSO. in sfxge_tx_qlist_post()
415 * If software TSO is used, we still need to copy packet header, in sfxge_tx_queue_mbuf()
828 ("TSO required on non-TCP packet")); in sfxge_parse_tx_packet()
833 ("TSO required on non-IP packet")); in sfxge_parse_tx_packet()
836 ("TSO required on non-TCP packet")); in sfxge_parse_tx_packet()
859 * generates TSO packets with RST flag. So, do not assert in sfxge_parse_tx_packet()
863 ("incompatible TCP flag 0x%x on TSO packet", in sfxge_parse_tx_packet()
938 * Software "TSO". Not quite as good as doing it in hardware, but
960 int fw_assisted; /* Use FW-assisted TSO */
964 * FW-assisted TSO */
968 static const struct ip *tso_iph(const struct sfxge_tso_state *tso) in tso_iph() argument
970 KASSERT(tso->protocol == htons(ETHERTYPE_IP), in tso_iph()
972 return (const struct ip *)(tso->mbuf->m_data + tso->nh_off); in tso_iph()
975 static __unused const struct ip6_hdr *tso_ip6h(const struct sfxge_tso_state *tso) in tso_ip6h() argument
977 KASSERT(tso->protocol == htons(ETHERTYPE_IPV6), in tso_ip6h()
979 return (const struct ip6_hdr *)(tso->mbuf->m_data + tso->nh_off); in tso_ip6h()
982 static const struct tcphdr *tso_tcph(const struct sfxge_tso_state *tso) in tso_tcph() argument
984 return (const struct tcphdr *)(tso->mbuf->m_data + tso->tcph_off); in tso_tcph()
988 /* Size of preallocated TSO header buffers. Larger blocks must be
994 * a TSO header buffer, since they must always be followed by a
1008 /* Allocate TSO header buffers */ in tso_init()
1039 static void tso_start(struct sfxge_txq *txq, struct sfxge_tso_state *tso, in tso_start() argument
1050 tso->fw_assisted = txq->tso_fw_assisted; in tso_start()
1051 tso->mbuf = mbuf; in tso_start()
1055 tso->protocol = eh->ether_type; in tso_start()
1056 if (tso->protocol == htons(ETHERTYPE_VLAN)) { in tso_start()
1059 tso->protocol = veh->evl_proto; in tso_start()
1060 tso->nh_off = sizeof(*veh); in tso_start()
1062 tso->nh_off = sizeof(*eh); in tso_start()
1065 tso->protocol = TSO_MBUF_PROTO(mbuf); in tso_start()
1066 tso->nh_off = mbuf->m_pkthdr.l2hlen; in tso_start()
1067 tso->tcph_off = mbuf->m_pkthdr.l3hlen; in tso_start()
1068 tso->packet_id = ntohs(TSO_MBUF_PACKETID(mbuf)); in tso_start()
1073 if (tso->protocol == htons(ETHERTYPE_IP)) { in tso_start()
1074 KASSERT(tso_iph(tso)->ip_p == IPPROTO_TCP, in tso_start()
1075 ("TSO required on non-TCP packet")); in tso_start()
1076 tso->tcph_off = tso->nh_off + 4 * tso_iph(tso)->ip_hl; in tso_start()
1077 tso->packet_id = ntohs(tso_iph(tso)->ip_id); in tso_start()
1079 KASSERT(tso->protocol == htons(ETHERTYPE_IPV6), in tso_start()
1080 ("TSO required on non-IP packet")); in tso_start()
1081 KASSERT(tso_ip6h(tso)->ip6_nxt == IPPROTO_TCP, in tso_start()
1082 ("TSO required on non-TCP packet")); in tso_start()
1083 tso->tcph_off = tso->nh_off + sizeof(struct ip6_hdr); in tso_start()
1084 tso->packet_id = 0; in tso_start()
1088 if (tso->fw_assisted && in tso_start()
1089 __predict_false(tso->tcph_off > in tso_start()
1091 tso->fw_assisted = 0; in tso_start()
1095 KASSERT(mbuf->m_len >= tso->tcph_off, in tso_start()
1098 if (mbuf->m_len < tso->tcph_off + offsetof(struct tcphdr, th_win)) { in tso_start()
1099 m_copydata(tso->mbuf, tso->tcph_off, sizeof(th_copy), in tso_start()
1103 th = tso_tcph(tso); in tso_start()
1105 tso->header_len = tso->tcph_off + 4 * th->th_off; in tso_start()
1107 tso->header_len = mbuf->m_pkthdr.l4hlen; in tso_start()
1109 tso->seg_size = mbuf->m_pkthdr.tso_segsz; in tso_start()
1112 tso->seqnum = ntohl(th->th_seq); in tso_start()
1117 * generates TSO packets with RST flag. So, do not assert in tso_start()
1121 ("incompatible TCP flag 0x%x on TSO packet", in tso_start()
1123 tso->tcp_flags = tcp_get_flags(th); in tso_start()
1125 tso->seqnum = TSO_MBUF_SEQNUM(mbuf); in tso_start()
1126 tso->tcp_flags = TSO_MBUF_FLAGS(mbuf); in tso_start()
1129 tso->out_len = mbuf->m_pkthdr.len - tso->header_len; in tso_start()
1131 if (tso->fw_assisted) { in tso_start()
1132 if (hdr_dma_seg->ds_len >= tso->header_len) in tso_start()
1135 tso->header_len, in tso_start()
1137 &tso->header_desc); in tso_start()
1139 tso->fw_assisted = 0; in tso_start()
1151 struct sfxge_tso_state *tso) in tso_fill_packet_with_fragment() argument
1155 uint64_t dma_addr = tso->dma_addr; in tso_fill_packet_with_fragment()
1158 if (tso->in_len == 0 || tso->packet_space == 0) in tso_fill_packet_with_fragment()
1161 KASSERT(tso->in_len > 0, ("TSO input length went negative")); in tso_fill_packet_with_fragment()
1162 KASSERT(tso->packet_space > 0, ("TSO packet space went negative")); in tso_fill_packet_with_fragment()
1164 if (tso->fw_assisted & SFXGE_FATSOV2) { in tso_fill_packet_with_fragment()
1165 n = tso->in_len; in tso_fill_packet_with_fragment()
1166 tso->out_len -= n; in tso_fill_packet_with_fragment()
1167 tso->seqnum += n; in tso_fill_packet_with_fragment()
1168 tso->in_len = 0; in tso_fill_packet_with_fragment()
1169 if (n < tso->packet_space) { in tso_fill_packet_with_fragment()
1170 tso->packet_space -= n; in tso_fill_packet_with_fragment()
1171 tso->segs_space--; in tso_fill_packet_with_fragment()
1173 tso->packet_space = tso->seg_size - in tso_fill_packet_with_fragment()
1174 (n - tso->packet_space) % tso->seg_size; in tso_fill_packet_with_fragment()
1175 tso->segs_space = in tso_fill_packet_with_fragment()
1177 (tso->packet_space != tso->seg_size); in tso_fill_packet_with_fragment()
1180 n = min(tso->in_len, tso->packet_space); in tso_fill_packet_with_fragment()
1181 tso->packet_space -= n; in tso_fill_packet_with_fragment()
1182 tso->out_len -= n; in tso_fill_packet_with_fragment()
1183 tso->dma_addr += n; in tso_fill_packet_with_fragment()
1184 tso->in_len -= n; in tso_fill_packet_with_fragment()
1191 eop = (tso->out_len == 0) | (tso->packet_space == 0) | in tso_fill_packet_with_fragment()
1192 (tso->segs_space == 0); in tso_fill_packet_with_fragment()
1198 /* Callback from bus_dmamap_load() for long TSO headers. */
1215 struct sfxge_tso_state *tso, in tso_start_new_packet() argument
1227 if (tso->fw_assisted) { in tso_start_new_packet()
1228 if (tso->fw_assisted & SFXGE_FATSOV2) { in tso_start_new_packet()
1232 tso->packet_id, in tso_start_new_packet()
1234 tso->seqnum, in tso_start_new_packet()
1235 tso->seg_size, in tso_start_new_packet()
1243 tso->segs_space = in tso_start_new_packet()
1246 uint8_t tcp_flags = tso->tcp_flags; in tso_start_new_packet()
1248 if (tso->out_len > tso->seg_size) in tso_start_new_packet()
1254 tso->packet_id, in tso_start_new_packet()
1255 tso->seqnum, in tso_start_new_packet()
1261 tso->seqnum += tso->seg_size; in tso_start_new_packet()
1262 tso->segs_space = UINT_MAX; in tso_start_new_packet()
1266 *desc = tso->header_desc; in tso_start_new_packet()
1272 if (__predict_true(tso->header_len <= TSOH_STD_SIZE)) { in tso_start_new_packet()
1288 header = malloc(tso->header_len, M_SFXGE, M_NOWAIT); in tso_start_new_packet()
1292 header, tso->header_len, in tso_start_new_packet()
1312 tsoh_th = (struct tcphdr *)(header + tso->tcph_off); in tso_start_new_packet()
1315 m_copydata(tso->mbuf, 0, tso->header_len, header); in tso_start_new_packet()
1317 tsoh_th->th_seq = htonl(tso->seqnum); in tso_start_new_packet()
1318 tso->seqnum += tso->seg_size; in tso_start_new_packet()
1319 if (tso->out_len > tso->seg_size) { in tso_start_new_packet()
1320 /* This packet will not finish the TSO burst. */ in tso_start_new_packet()
1321 ip_length = tso->header_len - tso->nh_off + tso->seg_size; in tso_start_new_packet()
1324 /* This packet will be the last in the TSO burst. */ in tso_start_new_packet()
1325 ip_length = tso->header_len - tso->nh_off + tso->out_len; in tso_start_new_packet()
1328 if (tso->protocol == htons(ETHERTYPE_IP)) { in tso_start_new_packet()
1329 struct ip *tsoh_iph = (struct ip *)(header + tso->nh_off); in tso_start_new_packet()
1336 (struct ip6_hdr *)(header + tso->nh_off); in tso_start_new_packet()
1347 tso->header_len, in tso_start_new_packet()
1352 tso->segs_space = UINT_MAX; in tso_start_new_packet()
1354 tso->packet_space = tso->seg_size; in tso_start_new_packet()
1366 struct sfxge_tso_state tso; in sfxge_tx_queue_tso() local
1370 tso_start(txq, &tso, dma_seg, mbuf); in sfxge_tx_queue_tso()
1372 while (dma_seg->ds_len + skipped <= tso.header_len) { in sfxge_tx_queue_tso()
1375 KASSERT(n_dma_seg, ("no payload found in TSO packet")); in sfxge_tx_queue_tso()
1378 tso.in_len = dma_seg->ds_len - (tso.header_len - skipped); in sfxge_tx_queue_tso()
1379 tso.dma_addr = dma_seg->ds_addr + (tso.header_len - skipped); in sfxge_tx_queue_tso()
1382 if (__predict_false(tso_start_new_packet(txq, &tso, &id))) in sfxge_tx_queue_tso()
1386 tso_fill_packet_with_fragment(txq, &tso); in sfxge_tx_queue_tso()
1392 if (tso.in_len == 0) { in sfxge_tx_queue_tso()
1397 tso.in_len = dma_seg->ds_len; in sfxge_tx_queue_tso()
1398 tso.dma_addr = dma_seg->ds_addr; in sfxge_tx_queue_tso()
1402 if ((tso.packet_space == 0) | (tso.segs_space == 0)) { in sfxge_tx_queue_tso()
1404 (tso.fw_assisted & SFXGE_FATSOV2) ? in sfxge_tx_queue_tso()
1406 (tso.fw_assisted & SFXGE_FATSOV1) ? 1 : 0; in sfxge_tx_queue_tso()
1418 if (__predict_false(tso_start_new_packet(txq, &tso, in sfxge_tx_queue_tso()
1552 * Estimate maximum number of Tx descriptors required for TSO packet.
1586 * in comparison to SW TSO */ in sfxge_tx_max_pkt_desc()