1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/export.h> 3 #include <linux/if_vlan.h> 4 #include <net/ip.h> 5 #include <net/tso.h> 6 #include <linux/dma-mapping.h> 7 #include <linux/unaligned.h> 8 9 void tso_build_hdr(const struct sk_buff *skb, char *hdr, struct tso_t *tso, 10 int size, bool is_last) 11 { 12 int hdr_len = skb_transport_offset(skb) + tso->tlen; 13 int mac_hdr_len = skb_network_offset(skb); 14 15 memcpy(hdr, skb->data, hdr_len); 16 if (!tso->ipv6) { 17 struct iphdr *iph = (void *)(hdr + mac_hdr_len); 18 19 iph->id = htons(tso->ip_id); 20 iph->tot_len = htons(size + hdr_len - mac_hdr_len); 21 tso->ip_id++; 22 } else { 23 struct ipv6hdr *iph = (void *)(hdr + mac_hdr_len); 24 25 iph->payload_len = htons(size + tso->tlen); 26 } 27 hdr += skb_transport_offset(skb); 28 if (tso->tlen != sizeof(struct udphdr)) { 29 struct tcphdr *tcph = (struct tcphdr *)hdr; 30 31 put_unaligned_be32(tso->tcp_seq, &tcph->seq); 32 33 if (!is_last) { 34 /* Clear all special flags for not last packet */ 35 tcph->psh = 0; 36 tcph->fin = 0; 37 tcph->rst = 0; 38 } 39 } else { 40 struct udphdr *uh = (struct udphdr *)hdr; 41 42 uh->len = htons(sizeof(*uh) + size); 43 } 44 } 45 EXPORT_SYMBOL(tso_build_hdr); 46 47 void tso_build_data(const struct sk_buff *skb, struct tso_t *tso, int size) 48 { 49 tso->tcp_seq += size; /* not worth avoiding this operation for UDP */ 50 tso->size -= size; 51 tso->data += size; 52 53 if ((tso->size == 0) && 54 (tso->next_frag_idx < skb_shinfo(skb)->nr_frags)) { 55 skb_frag_t *frag = &skb_shinfo(skb)->frags[tso->next_frag_idx]; 56 57 /* Move to next segment */ 58 tso->size = skb_frag_size(frag); 59 tso->data = skb_frag_address(frag); 60 tso->next_frag_idx++; 61 } 62 } 63 EXPORT_SYMBOL(tso_build_data); 64 65 int tso_start(struct sk_buff *skb, struct tso_t *tso) 66 { 67 int tlen = skb_is_gso_tcp(skb) ? tcp_hdrlen(skb) : sizeof(struct udphdr); 68 int hdr_len = skb_transport_offset(skb) + tlen; 69 70 tso->tlen = tlen; 71 tso->ip_id = ntohs(ip_hdr(skb)->id); 72 tso->tcp_seq = (tlen != sizeof(struct udphdr)) ? ntohl(tcp_hdr(skb)->seq) : 0; 73 tso->next_frag_idx = 0; 74 tso->ipv6 = vlan_get_protocol(skb) == htons(ETH_P_IPV6); 75 76 /* Build first data */ 77 tso->size = skb_headlen(skb) - hdr_len; 78 tso->data = skb->data + hdr_len; 79 if ((tso->size == 0) && 80 (tso->next_frag_idx < skb_shinfo(skb)->nr_frags)) { 81 skb_frag_t *frag = &skb_shinfo(skb)->frags[tso->next_frag_idx]; 82 83 /* Move to next segment */ 84 tso->size = skb_frag_size(frag); 85 tso->data = skb_frag_address(frag); 86 tso->next_frag_idx++; 87 } 88 return hdr_len; 89 } 90 EXPORT_SYMBOL(tso_start); 91 92 static int tso_dma_iova_try(struct device *dev, struct tso_dma_map *map, 93 phys_addr_t phys, size_t linear_len, 94 size_t total_len, size_t *offset) 95 { 96 const struct sk_buff *skb; 97 unsigned int nr_frags; 98 int i; 99 100 if (!dma_iova_try_alloc(dev, &map->iova_state, phys, total_len)) 101 return 1; 102 103 skb = map->skb; 104 nr_frags = skb_shinfo(skb)->nr_frags; 105 106 if (linear_len) { 107 if (dma_iova_link(dev, &map->iova_state, 108 phys, *offset, linear_len, 109 DMA_TO_DEVICE, 0)) 110 goto iova_fail; 111 map->linear_len = linear_len; 112 *offset += linear_len; 113 } 114 115 for (i = 0; i < nr_frags; i++) { 116 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 117 unsigned int frag_len = skb_frag_size(frag); 118 119 if (dma_iova_link(dev, &map->iova_state, 120 skb_frag_phys(frag), *offset, 121 frag_len, DMA_TO_DEVICE, 0)) { 122 map->nr_frags = i; 123 goto iova_fail; 124 } 125 map->frags[i].len = frag_len; 126 *offset += frag_len; 127 map->nr_frags = i + 1; 128 } 129 130 if (dma_iova_sync(dev, &map->iova_state, 0, total_len)) 131 goto iova_fail; 132 133 return 0; 134 135 iova_fail: 136 dma_iova_destroy(dev, &map->iova_state, *offset, 137 DMA_TO_DEVICE, 0); 138 memset(&map->iova_state, 0, sizeof(map->iova_state)); 139 140 /* reset map state */ 141 map->frag_idx = -1; 142 map->offset = 0; 143 map->linear_len = 0; 144 map->nr_frags = 0; 145 146 return 1; 147 } 148 149 /** 150 * tso_dma_map_init - DMA-map GSO payload regions 151 * @map: map struct to initialize 152 * @dev: device for DMA mapping 153 * @skb: the GSO skb 154 * @hdr_len: per-segment header length in bytes 155 * 156 * DMA-maps the linear payload (after headers) and all frags. 157 * Prefers the DMA IOVA API (one contiguous mapping, one IOTLB sync); 158 * falls back to per-region dma_map_phys() when IOVA is not available. 159 * Positions the iterator at byte 0 of the payload. 160 * 161 * Return: 0 on success, -ENOMEM on DMA mapping failure (partial mappings 162 * are cleaned up internally). 163 */ 164 int tso_dma_map_init(struct tso_dma_map *map, struct device *dev, 165 const struct sk_buff *skb, unsigned int hdr_len) 166 { 167 unsigned int linear_len = skb_headlen(skb) - hdr_len; 168 unsigned int nr_frags = skb_shinfo(skb)->nr_frags; 169 size_t total_len = skb->len - hdr_len; 170 size_t offset = 0; 171 phys_addr_t phys; 172 int i; 173 174 map->dev = dev; 175 map->skb = skb; 176 map->hdr_len = hdr_len; 177 map->frag_idx = -1; 178 map->offset = 0; 179 map->iova_offset = 0; 180 map->total_len = total_len; 181 map->linear_len = 0; 182 map->nr_frags = 0; 183 memset(&map->iova_state, 0, sizeof(map->iova_state)); 184 185 if (!total_len) 186 return 0; 187 188 if (linear_len) 189 phys = virt_to_phys(skb->data + hdr_len); 190 else 191 phys = skb_frag_phys(&skb_shinfo(skb)->frags[0]); 192 193 if (tso_dma_iova_try(dev, map, phys, linear_len, total_len, &offset)) { 194 /* IOVA path failed, map state was reset. Fallback to 195 * per-region dma_map_phys() 196 */ 197 if (linear_len) { 198 map->linear_dma = dma_map_phys(dev, phys, linear_len, 199 DMA_TO_DEVICE, 0); 200 if (dma_mapping_error(dev, map->linear_dma)) 201 return -ENOMEM; 202 map->linear_len = linear_len; 203 } 204 205 for (i = 0; i < nr_frags; i++) { 206 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 207 unsigned int frag_len = skb_frag_size(frag); 208 209 map->frags[i].len = frag_len; 210 map->frags[i].dma = dma_map_phys(dev, skb_frag_phys(frag), 211 frag_len, DMA_TO_DEVICE, 0); 212 if (dma_mapping_error(dev, map->frags[i].dma)) { 213 tso_dma_map_cleanup(map); 214 return -ENOMEM; 215 } 216 map->nr_frags = i + 1; 217 } 218 } 219 220 if (linear_len == 0 && nr_frags > 0) 221 map->frag_idx = 0; 222 223 return 0; 224 } 225 EXPORT_SYMBOL(tso_dma_map_init); 226 227 /** 228 * tso_dma_map_cleanup - unmap all DMA regions in a tso_dma_map 229 * @map: the map to clean up 230 * 231 * Handles both IOVA and fallback paths. For IOVA, calls 232 * dma_iova_destroy(). For fallback, unmaps each region individually. 233 */ 234 void tso_dma_map_cleanup(struct tso_dma_map *map) 235 { 236 int i; 237 238 if (dma_use_iova(&map->iova_state)) { 239 dma_iova_destroy(map->dev, &map->iova_state, map->total_len, 240 DMA_TO_DEVICE, 0); 241 memset(&map->iova_state, 0, sizeof(map->iova_state)); 242 } else { 243 if (map->linear_len) 244 dma_unmap_phys(map->dev, map->linear_dma, 245 map->linear_len, DMA_TO_DEVICE, 0); 246 247 for (i = 0; i < map->nr_frags; i++) 248 dma_unmap_phys(map->dev, map->frags[i].dma, 249 map->frags[i].len, DMA_TO_DEVICE, 0); 250 } 251 252 map->linear_len = 0; 253 map->nr_frags = 0; 254 } 255 EXPORT_SYMBOL(tso_dma_map_cleanup); 256 257 /** 258 * tso_dma_map_count - count descriptors for a payload range 259 * @map: the payload map 260 * @len: number of payload bytes in this segment 261 * 262 * Counts how many contiguous DMA region chunks the next @len bytes 263 * will span, without advancing the iterator. On the IOVA path this 264 * is always 1 (contiguous). On the fallback path, uses region sizes 265 * from the current position. 266 * 267 * Return: the number of descriptors needed for @len bytes of payload. 268 */ 269 unsigned int tso_dma_map_count(struct tso_dma_map *map, unsigned int len) 270 { 271 unsigned int offset = map->offset; 272 int idx = map->frag_idx; 273 unsigned int count = 0; 274 275 if (!len) 276 return 0; 277 278 if (dma_use_iova(&map->iova_state)) 279 return 1; 280 281 while (len > 0) { 282 unsigned int region_len, chunk; 283 284 if (idx == -1) 285 region_len = map->linear_len; 286 else 287 region_len = map->frags[idx].len; 288 289 chunk = min(len, region_len - offset); 290 len -= chunk; 291 count++; 292 offset = 0; 293 idx++; 294 } 295 296 return count; 297 } 298 EXPORT_SYMBOL(tso_dma_map_count); 299 300 /** 301 * tso_dma_map_next - yield the next DMA address range 302 * @map: the payload map 303 * @addr: output DMA address 304 * @chunk_len: output chunk length 305 * @mapping_len: full DMA mapping length when this chunk starts a new 306 * mapping region, or 0 when continuing a previous one. 307 * On the IOVA path this is always 0 (driver must not 308 * do per-region unmaps; use tso_dma_map_cleanup instead). 309 * @seg_remaining: bytes left in current segment 310 * 311 * Yields the next (dma_addr, chunk_len) pair and advances the iterator. 312 * On the IOVA path, the entire payload is contiguous so each segment 313 * is always a single chunk. 314 * 315 * Return: true if a chunk was yielded, false when @seg_remaining is 0. 316 */ 317 bool tso_dma_map_next(struct tso_dma_map *map, dma_addr_t *addr, 318 unsigned int *chunk_len, unsigned int *mapping_len, 319 unsigned int seg_remaining) 320 { 321 unsigned int region_len, chunk; 322 323 if (!seg_remaining) 324 return false; 325 326 /* IOVA path: contiguous DMA range, no region boundaries */ 327 if (dma_use_iova(&map->iova_state)) { 328 *addr = map->iova_state.addr + map->iova_offset; 329 *chunk_len = seg_remaining; 330 *mapping_len = 0; 331 map->iova_offset += seg_remaining; 332 return true; 333 } 334 335 /* Fallback path: per-region iteration */ 336 337 if (map->frag_idx == -1) { 338 region_len = map->linear_len; 339 chunk = min(seg_remaining, region_len - map->offset); 340 *addr = map->linear_dma + map->offset; 341 } else { 342 region_len = map->frags[map->frag_idx].len; 343 chunk = min(seg_remaining, region_len - map->offset); 344 *addr = map->frags[map->frag_idx].dma + map->offset; 345 } 346 347 *mapping_len = (map->offset == 0) ? region_len : 0; 348 *chunk_len = chunk; 349 map->offset += chunk; 350 351 if (map->offset >= region_len) { 352 map->frag_idx++; 353 map->offset = 0; 354 } 355 356 return true; 357 } 358 EXPORT_SYMBOL(tso_dma_map_next); 359