1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/export.h> 3 #include <linux/if_vlan.h> 4 #include <net/ip.h> 5 #include <net/tso.h> 6 #include <linux/dma-mapping.h> 7 #include <linux/unaligned.h> 8 9 void tso_build_hdr(const struct sk_buff *skb, char *hdr, struct tso_t *tso, 10 int size, bool is_last) 11 { 12 int hdr_len = skb_transport_offset(skb) + tso->tlen; 13 int mac_hdr_len = skb_network_offset(skb); 14 15 memcpy(hdr, skb->data, hdr_len); 16 if (!tso->ipv6) { 17 struct iphdr *iph = (void *)(hdr + mac_hdr_len); 18 19 iph->id = htons(tso->ip_id); 20 iph->tot_len = htons(size + hdr_len - mac_hdr_len); 21 tso->ip_id++; 22 } else { 23 struct ipv6hdr *iph = (void *)(hdr + mac_hdr_len); 24 25 iph->payload_len = htons(size + tso->tlen); 26 } 27 hdr += skb_transport_offset(skb); 28 if (tso->tlen != sizeof(struct udphdr)) { 29 struct tcphdr *tcph = (struct tcphdr *)hdr; 30 31 put_unaligned_be32(tso->tcp_seq, &tcph->seq); 32 33 if (!is_last) { 34 /* Clear all special flags for not last packet */ 35 tcph->psh = 0; 36 tcph->fin = 0; 37 tcph->rst = 0; 38 } 39 } else { 40 struct udphdr *uh = (struct udphdr *)hdr; 41 42 /* size is after segmentation. */ 43 udp_set_len_short(uh, sizeof(*uh) + size); 44 } 45 } 46 EXPORT_SYMBOL(tso_build_hdr); 47 48 void tso_build_data(const struct sk_buff *skb, struct tso_t *tso, int size) 49 { 50 tso->tcp_seq += size; /* not worth avoiding this operation for UDP */ 51 tso->size -= size; 52 tso->data += size; 53 54 if ((tso->size == 0) && 55 (tso->next_frag_idx < skb_shinfo(skb)->nr_frags)) { 56 skb_frag_t *frag = &skb_shinfo(skb)->frags[tso->next_frag_idx]; 57 58 /* Move to next segment */ 59 tso->size = skb_frag_size(frag); 60 tso->data = skb_frag_address(frag); 61 tso->next_frag_idx++; 62 } 63 } 64 EXPORT_SYMBOL(tso_build_data); 65 66 int tso_start(struct sk_buff *skb, struct tso_t *tso) 67 { 68 int tlen = skb_is_gso_tcp(skb) ? tcp_hdrlen(skb) : sizeof(struct udphdr); 69 int hdr_len = skb_transport_offset(skb) + tlen; 70 71 tso->tlen = tlen; 72 tso->ip_id = ntohs(ip_hdr(skb)->id); 73 tso->tcp_seq = (tlen != sizeof(struct udphdr)) ? ntohl(tcp_hdr(skb)->seq) : 0; 74 tso->next_frag_idx = 0; 75 tso->ipv6 = vlan_get_protocol(skb) == htons(ETH_P_IPV6); 76 77 /* Build first data */ 78 tso->size = skb_headlen(skb) - hdr_len; 79 tso->data = skb->data + hdr_len; 80 if ((tso->size == 0) && 81 (tso->next_frag_idx < skb_shinfo(skb)->nr_frags)) { 82 skb_frag_t *frag = &skb_shinfo(skb)->frags[tso->next_frag_idx]; 83 84 /* Move to next segment */ 85 tso->size = skb_frag_size(frag); 86 tso->data = skb_frag_address(frag); 87 tso->next_frag_idx++; 88 } 89 return hdr_len; 90 } 91 EXPORT_SYMBOL(tso_start); 92 93 static int tso_dma_iova_try(struct device *dev, struct tso_dma_map *map, 94 phys_addr_t phys, size_t linear_len, 95 size_t total_len, size_t *offset) 96 { 97 const struct sk_buff *skb; 98 unsigned int nr_frags; 99 int i; 100 101 if (!dma_iova_try_alloc(dev, &map->iova_state, phys, total_len)) 102 return 1; 103 104 skb = map->skb; 105 nr_frags = skb_shinfo(skb)->nr_frags; 106 107 if (linear_len) { 108 if (dma_iova_link(dev, &map->iova_state, 109 phys, *offset, linear_len, 110 DMA_TO_DEVICE, 0)) 111 goto iova_fail; 112 map->linear_len = linear_len; 113 *offset += linear_len; 114 } 115 116 for (i = 0; i < nr_frags; i++) { 117 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 118 unsigned int frag_len = skb_frag_size(frag); 119 120 if (dma_iova_link(dev, &map->iova_state, 121 skb_frag_phys(frag), *offset, 122 frag_len, DMA_TO_DEVICE, 0)) { 123 map->nr_frags = i; 124 goto iova_fail; 125 } 126 map->frags[i].len = frag_len; 127 *offset += frag_len; 128 map->nr_frags = i + 1; 129 } 130 131 if (dma_iova_sync(dev, &map->iova_state, 0, total_len)) 132 goto iova_fail; 133 134 return 0; 135 136 iova_fail: 137 dma_iova_destroy(dev, &map->iova_state, *offset, 138 DMA_TO_DEVICE, 0); 139 memset(&map->iova_state, 0, sizeof(map->iova_state)); 140 141 /* reset map state */ 142 map->frag_idx = -1; 143 map->offset = 0; 144 map->linear_len = 0; 145 map->nr_frags = 0; 146 147 return 1; 148 } 149 150 /** 151 * tso_dma_map_init - DMA-map GSO payload regions 152 * @map: map struct to initialize 153 * @dev: device for DMA mapping 154 * @skb: the GSO skb 155 * @hdr_len: per-segment header length in bytes 156 * 157 * DMA-maps the linear payload (after headers) and all frags. 158 * Prefers the DMA IOVA API (one contiguous mapping, one IOTLB sync); 159 * falls back to per-region dma_map_phys() when IOVA is not available. 160 * Positions the iterator at byte 0 of the payload. 161 * 162 * Return: 0 on success, -ENOMEM on DMA mapping failure (partial mappings 163 * are cleaned up internally). 164 */ 165 int tso_dma_map_init(struct tso_dma_map *map, struct device *dev, 166 const struct sk_buff *skb, unsigned int hdr_len) 167 { 168 unsigned int linear_len = skb_headlen(skb) - hdr_len; 169 unsigned int nr_frags = skb_shinfo(skb)->nr_frags; 170 size_t total_len = skb->len - hdr_len; 171 size_t offset = 0; 172 phys_addr_t phys; 173 int i; 174 175 map->dev = dev; 176 map->skb = skb; 177 map->hdr_len = hdr_len; 178 map->frag_idx = -1; 179 map->offset = 0; 180 map->iova_offset = 0; 181 map->total_len = total_len; 182 map->linear_len = 0; 183 map->nr_frags = 0; 184 memset(&map->iova_state, 0, sizeof(map->iova_state)); 185 186 if (!total_len) 187 return 0; 188 189 if (linear_len) 190 phys = virt_to_phys(skb->data + hdr_len); 191 else 192 phys = skb_frag_phys(&skb_shinfo(skb)->frags[0]); 193 194 if (tso_dma_iova_try(dev, map, phys, linear_len, total_len, &offset)) { 195 /* IOVA path failed, map state was reset. Fallback to 196 * per-region dma_map_phys() 197 */ 198 if (linear_len) { 199 map->linear_dma = dma_map_phys(dev, phys, linear_len, 200 DMA_TO_DEVICE, 0); 201 if (dma_mapping_error(dev, map->linear_dma)) 202 return -ENOMEM; 203 map->linear_len = linear_len; 204 } 205 206 for (i = 0; i < nr_frags; i++) { 207 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 208 unsigned int frag_len = skb_frag_size(frag); 209 210 map->frags[i].len = frag_len; 211 map->frags[i].dma = dma_map_phys(dev, skb_frag_phys(frag), 212 frag_len, DMA_TO_DEVICE, 0); 213 if (dma_mapping_error(dev, map->frags[i].dma)) { 214 tso_dma_map_cleanup(map); 215 return -ENOMEM; 216 } 217 map->nr_frags = i + 1; 218 } 219 } 220 221 if (linear_len == 0 && nr_frags > 0) 222 map->frag_idx = 0; 223 224 return 0; 225 } 226 EXPORT_SYMBOL(tso_dma_map_init); 227 228 /** 229 * tso_dma_map_cleanup - unmap all DMA regions in a tso_dma_map 230 * @map: the map to clean up 231 * 232 * Handles both IOVA and fallback paths. For IOVA, calls 233 * dma_iova_destroy(). For fallback, unmaps each region individually. 234 */ 235 void tso_dma_map_cleanup(struct tso_dma_map *map) 236 { 237 int i; 238 239 if (dma_use_iova(&map->iova_state)) { 240 dma_iova_destroy(map->dev, &map->iova_state, map->total_len, 241 DMA_TO_DEVICE, 0); 242 memset(&map->iova_state, 0, sizeof(map->iova_state)); 243 } else { 244 if (map->linear_len) 245 dma_unmap_phys(map->dev, map->linear_dma, 246 map->linear_len, DMA_TO_DEVICE, 0); 247 248 for (i = 0; i < map->nr_frags; i++) 249 dma_unmap_phys(map->dev, map->frags[i].dma, 250 map->frags[i].len, DMA_TO_DEVICE, 0); 251 } 252 253 map->linear_len = 0; 254 map->nr_frags = 0; 255 } 256 EXPORT_SYMBOL(tso_dma_map_cleanup); 257 258 /** 259 * tso_dma_map_count - count descriptors for a payload range 260 * @map: the payload map 261 * @len: number of payload bytes in this segment 262 * 263 * Counts how many contiguous DMA region chunks the next @len bytes 264 * will span, without advancing the iterator. On the IOVA path this 265 * is always 1 (contiguous). On the fallback path, uses region sizes 266 * from the current position. 267 * 268 * Return: the number of descriptors needed for @len bytes of payload. 269 */ 270 unsigned int tso_dma_map_count(struct tso_dma_map *map, unsigned int len) 271 { 272 unsigned int offset = map->offset; 273 int idx = map->frag_idx; 274 unsigned int count = 0; 275 276 if (!len) 277 return 0; 278 279 if (dma_use_iova(&map->iova_state)) 280 return 1; 281 282 while (len > 0) { 283 unsigned int region_len, chunk; 284 285 if (idx == -1) 286 region_len = map->linear_len; 287 else 288 region_len = map->frags[idx].len; 289 290 chunk = min(len, region_len - offset); 291 len -= chunk; 292 count++; 293 offset = 0; 294 idx++; 295 } 296 297 return count; 298 } 299 EXPORT_SYMBOL(tso_dma_map_count); 300 301 /** 302 * tso_dma_map_next - yield the next DMA address range 303 * @map: the payload map 304 * @addr: output DMA address 305 * @chunk_len: output chunk length 306 * @mapping_len: full DMA mapping length when this chunk starts a new 307 * mapping region, or 0 when continuing a previous one. 308 * On the IOVA path this is always 0 (driver must not 309 * do per-region unmaps; use tso_dma_map_cleanup instead). 310 * @seg_remaining: bytes left in current segment 311 * 312 * Yields the next (dma_addr, chunk_len) pair and advances the iterator. 313 * On the IOVA path, the entire payload is contiguous so each segment 314 * is always a single chunk. 315 * 316 * Return: true if a chunk was yielded, false when @seg_remaining is 0. 317 */ 318 bool tso_dma_map_next(struct tso_dma_map *map, dma_addr_t *addr, 319 unsigned int *chunk_len, unsigned int *mapping_len, 320 unsigned int seg_remaining) 321 { 322 unsigned int region_len, chunk; 323 324 if (!seg_remaining) 325 return false; 326 327 /* IOVA path: contiguous DMA range, no region boundaries */ 328 if (dma_use_iova(&map->iova_state)) { 329 *addr = map->iova_state.addr + map->iova_offset; 330 *chunk_len = seg_remaining; 331 *mapping_len = 0; 332 map->iova_offset += seg_remaining; 333 return true; 334 } 335 336 /* Fallback path: per-region iteration */ 337 338 if (map->frag_idx == -1) { 339 region_len = map->linear_len; 340 chunk = min(seg_remaining, region_len - map->offset); 341 *addr = map->linear_dma + map->offset; 342 } else { 343 region_len = map->frags[map->frag_idx].len; 344 chunk = min(seg_remaining, region_len - map->offset); 345 *addr = map->frags[map->frag_idx].dma + map->offset; 346 } 347 348 *mapping_len = (map->offset == 0) ? region_len : 0; 349 *chunk_len = chunk; 350 map->offset += chunk; 351 352 if (map->offset >= region_len) { 353 map->frag_idx++; 354 map->offset = 0; 355 } 356 357 return true; 358 } 359 EXPORT_SYMBOL(tso_dma_map_next); 360