1 /* 2 * Copyright (c) 2016 Citrix Systems Inc. 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License version 2 6 * as published by the Free Softare Foundation; or, when distributed 7 * separately from the Linux kernel or incorporated into other 8 * software packages, subject to the following license: 9 * 10 * Permission is hereby granted, free of charge, to any person obtaining a copy 11 * of this source file (the "Software"), to deal in the Software without 12 * restriction, including without limitation the rights to use, copy, modify, 13 * merge, publish, distribute, sublicense, and/or sell copies of the Software, 14 * and to permit persons to whom the Software is furnished to do so, subject to 15 * the following conditions: 16 * 17 * The above copyright notice and this permission notice shall be included in 18 * all copies or substantial portions of the Software. 19 * 20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 22 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 23 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 24 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 25 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 26 * IN THE SOFTWARE. 27 */ 28 29 #define XEN_NETIF_DEFINE_TOEPLITZ 30 31 #include "common.h" 32 #include <linux/vmalloc.h> 33 #include <linux/rculist.h> 34 35 static void xenvif_add_hash(struct xenvif *vif, const u8 *tag, 36 unsigned int len, u32 val) 37 { 38 struct xenvif_hash_cache_entry *new, *entry, *oldest; 39 unsigned long flags; 40 bool found; 41 42 new = kmalloc(sizeof(*entry), GFP_ATOMIC); 43 if (!new) 44 return; 45 46 memcpy(new->tag, tag, len); 47 new->len = len; 48 new->val = val; 49 50 spin_lock_irqsave(&vif->hash.cache.lock, flags); 51 52 found = false; 53 oldest = NULL; 54 list_for_each_entry_rcu(entry, &vif->hash.cache.list, link, 55 lockdep_is_held(&vif->hash.cache.lock)) { 56 /* Make sure we don't add duplicate entries */ 57 if (entry->len == len && 58 memcmp(entry->tag, tag, len) == 0) 59 found = true; 60 if (!oldest || entry->seq < oldest->seq) 61 oldest = entry; 62 } 63 64 if (!found) { 65 new->seq = atomic_inc_return(&vif->hash.cache.seq); 66 list_add_rcu(&new->link, &vif->hash.cache.list); 67 68 if (++vif->hash.cache.count > xenvif_hash_cache_size) { 69 list_del_rcu(&oldest->link); 70 vif->hash.cache.count--; 71 kfree_rcu(oldest, rcu); 72 } 73 } 74 75 spin_unlock_irqrestore(&vif->hash.cache.lock, flags); 76 77 if (found) 78 kfree(new); 79 } 80 81 static u32 xenvif_new_hash(struct xenvif *vif, const u8 *data, 82 unsigned int len) 83 { 84 u32 val; 85 86 val = xen_netif_toeplitz_hash(vif->hash.key, 87 sizeof(vif->hash.key), 88 data, len); 89 90 if (xenvif_hash_cache_size != 0) 91 xenvif_add_hash(vif, data, len, val); 92 93 return val; 94 } 95 96 static void xenvif_flush_hash(struct xenvif *vif) 97 { 98 struct xenvif_hash_cache_entry *entry, *n; 99 unsigned long flags; 100 101 if (xenvif_hash_cache_size == 0) 102 return; 103 104 spin_lock_irqsave(&vif->hash.cache.lock, flags); 105 106 list_for_each_entry_safe(entry, n, &vif->hash.cache.list, link) { 107 list_del_rcu(&entry->link); 108 vif->hash.cache.count--; 109 kfree_rcu(entry, rcu); 110 } 111 112 spin_unlock_irqrestore(&vif->hash.cache.lock, flags); 113 } 114 115 static u32 xenvif_find_hash(struct xenvif *vif, const u8 *data, 116 unsigned int len) 117 { 118 struct xenvif_hash_cache_entry *entry; 119 u32 val; 120 bool found; 121 122 if (len >= XEN_NETBK_HASH_TAG_SIZE) 123 return 0; 124 125 if (xenvif_hash_cache_size == 0) 126 return xenvif_new_hash(vif, data, len); 127 128 rcu_read_lock(); 129 130 found = false; 131 132 list_for_each_entry_rcu(entry, &vif->hash.cache.list, link) { 133 if (entry->len == len && 134 memcmp(entry->tag, data, len) == 0) { 135 val = entry->val; 136 entry->seq = atomic_inc_return(&vif->hash.cache.seq); 137 found = true; 138 break; 139 } 140 } 141 142 rcu_read_unlock(); 143 144 if (!found) 145 val = xenvif_new_hash(vif, data, len); 146 147 return val; 148 } 149 150 void xenvif_set_skb_hash(struct xenvif *vif, struct sk_buff *skb) 151 { 152 struct flow_keys flow; 153 u32 hash = 0; 154 enum pkt_hash_types type = PKT_HASH_TYPE_NONE; 155 u32 flags = vif->hash.flags; 156 bool has_tcp_hdr; 157 158 /* Quick rejection test: If the network protocol doesn't 159 * correspond to any enabled hash type then there's no point 160 * in parsing the packet header. 161 */ 162 switch (skb->protocol) { 163 case htons(ETH_P_IP): 164 if (flags & (XEN_NETIF_CTRL_HASH_TYPE_IPV4_TCP | 165 XEN_NETIF_CTRL_HASH_TYPE_IPV4)) 166 break; 167 168 goto done; 169 170 case htons(ETH_P_IPV6): 171 if (flags & (XEN_NETIF_CTRL_HASH_TYPE_IPV6_TCP | 172 XEN_NETIF_CTRL_HASH_TYPE_IPV6)) 173 break; 174 175 goto done; 176 177 default: 178 goto done; 179 } 180 181 memset(&flow, 0, sizeof(flow)); 182 if (!skb_flow_dissect_flow_keys(skb, &flow, 0)) 183 goto done; 184 185 has_tcp_hdr = (flow.basic.ip_proto == IPPROTO_TCP) && 186 !(flow.control.flags & FLOW_DIS_IS_FRAGMENT); 187 188 switch (skb->protocol) { 189 case htons(ETH_P_IP): 190 if (has_tcp_hdr && 191 (flags & XEN_NETIF_CTRL_HASH_TYPE_IPV4_TCP)) { 192 u8 data[12]; 193 194 memcpy(&data[0], &flow.addrs.v4addrs.src, 4); 195 memcpy(&data[4], &flow.addrs.v4addrs.dst, 4); 196 memcpy(&data[8], &flow.ports.src, 2); 197 memcpy(&data[10], &flow.ports.dst, 2); 198 199 hash = xenvif_find_hash(vif, data, sizeof(data)); 200 type = PKT_HASH_TYPE_L4; 201 } else if (flags & XEN_NETIF_CTRL_HASH_TYPE_IPV4) { 202 u8 data[8]; 203 204 memcpy(&data[0], &flow.addrs.v4addrs.src, 4); 205 memcpy(&data[4], &flow.addrs.v4addrs.dst, 4); 206 207 hash = xenvif_find_hash(vif, data, sizeof(data)); 208 type = PKT_HASH_TYPE_L3; 209 } 210 211 break; 212 213 case htons(ETH_P_IPV6): 214 if (has_tcp_hdr && 215 (flags & XEN_NETIF_CTRL_HASH_TYPE_IPV6_TCP)) { 216 u8 data[36]; 217 218 memcpy(&data[0], &flow.addrs.v6addrs.src, 16); 219 memcpy(&data[16], &flow.addrs.v6addrs.dst, 16); 220 memcpy(&data[32], &flow.ports.src, 2); 221 memcpy(&data[34], &flow.ports.dst, 2); 222 223 hash = xenvif_find_hash(vif, data, sizeof(data)); 224 type = PKT_HASH_TYPE_L4; 225 } else if (flags & XEN_NETIF_CTRL_HASH_TYPE_IPV6) { 226 u8 data[32]; 227 228 memcpy(&data[0], &flow.addrs.v6addrs.src, 16); 229 memcpy(&data[16], &flow.addrs.v6addrs.dst, 16); 230 231 hash = xenvif_find_hash(vif, data, sizeof(data)); 232 type = PKT_HASH_TYPE_L3; 233 } 234 235 break; 236 } 237 238 done: 239 if (type == PKT_HASH_TYPE_NONE) 240 skb_clear_hash(skb); 241 else 242 __skb_set_sw_hash(skb, hash, type == PKT_HASH_TYPE_L4); 243 } 244 245 u32 xenvif_set_hash_alg(struct xenvif *vif, u32 alg) 246 { 247 switch (alg) { 248 case XEN_NETIF_CTRL_HASH_ALGORITHM_NONE: 249 case XEN_NETIF_CTRL_HASH_ALGORITHM_TOEPLITZ: 250 break; 251 252 default: 253 return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER; 254 } 255 256 vif->hash.alg = alg; 257 258 return XEN_NETIF_CTRL_STATUS_SUCCESS; 259 } 260 261 u32 xenvif_get_hash_flags(struct xenvif *vif, u32 *flags) 262 { 263 if (vif->hash.alg == XEN_NETIF_CTRL_HASH_ALGORITHM_NONE) 264 return XEN_NETIF_CTRL_STATUS_NOT_SUPPORTED; 265 266 *flags = XEN_NETIF_CTRL_HASH_TYPE_IPV4 | 267 XEN_NETIF_CTRL_HASH_TYPE_IPV4_TCP | 268 XEN_NETIF_CTRL_HASH_TYPE_IPV6 | 269 XEN_NETIF_CTRL_HASH_TYPE_IPV6_TCP; 270 271 return XEN_NETIF_CTRL_STATUS_SUCCESS; 272 } 273 274 u32 xenvif_set_hash_flags(struct xenvif *vif, u32 flags) 275 { 276 if (flags & ~(XEN_NETIF_CTRL_HASH_TYPE_IPV4 | 277 XEN_NETIF_CTRL_HASH_TYPE_IPV4_TCP | 278 XEN_NETIF_CTRL_HASH_TYPE_IPV6 | 279 XEN_NETIF_CTRL_HASH_TYPE_IPV6_TCP)) 280 return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER; 281 282 if (vif->hash.alg == XEN_NETIF_CTRL_HASH_ALGORITHM_NONE) 283 return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER; 284 285 vif->hash.flags = flags; 286 287 return XEN_NETIF_CTRL_STATUS_SUCCESS; 288 } 289 290 u32 xenvif_set_hash_key(struct xenvif *vif, u32 gref, u32 len) 291 { 292 u8 *key = vif->hash.key; 293 struct gnttab_copy copy_op = { 294 .source.u.ref = gref, 295 .source.domid = vif->domid, 296 .dest.u.gmfn = virt_to_gfn(key), 297 .dest.domid = DOMID_SELF, 298 .dest.offset = xen_offset_in_page(key), 299 .len = len, 300 .flags = GNTCOPY_source_gref 301 }; 302 303 if (len > XEN_NETBK_MAX_HASH_KEY_SIZE) 304 return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER; 305 306 if (copy_op.len != 0) { 307 gnttab_batch_copy(©_op, 1); 308 309 if (copy_op.status != GNTST_okay) 310 return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER; 311 } 312 313 /* Clear any remaining key octets */ 314 if (len < XEN_NETBK_MAX_HASH_KEY_SIZE) 315 memset(key + len, 0, XEN_NETBK_MAX_HASH_KEY_SIZE - len); 316 317 xenvif_flush_hash(vif); 318 319 return XEN_NETIF_CTRL_STATUS_SUCCESS; 320 } 321 322 u32 xenvif_set_hash_mapping_size(struct xenvif *vif, u32 size) 323 { 324 if (size > XEN_NETBK_MAX_HASH_MAPPING_SIZE) 325 return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER; 326 327 vif->hash.size = size; 328 memset(vif->hash.mapping[vif->hash.mapping_sel], 0, 329 sizeof(u32) * size); 330 331 return XEN_NETIF_CTRL_STATUS_SUCCESS; 332 } 333 334 u32 xenvif_set_hash_mapping(struct xenvif *vif, u32 gref, u32 len, 335 u32 off) 336 { 337 u32 *mapping = vif->hash.mapping[!vif->hash.mapping_sel]; 338 unsigned int nr = 1; 339 struct gnttab_copy copy_op[2] = {{ 340 .source.u.ref = gref, 341 .source.domid = vif->domid, 342 .dest.domid = DOMID_SELF, 343 .len = len * sizeof(*mapping), 344 .flags = GNTCOPY_source_gref 345 }}; 346 347 if ((off + len < off) || (off + len > vif->hash.size) || 348 len > XEN_PAGE_SIZE / sizeof(*mapping)) 349 return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER; 350 351 copy_op[0].dest.u.gmfn = virt_to_gfn(mapping + off); 352 copy_op[0].dest.offset = xen_offset_in_page(mapping + off); 353 if (copy_op[0].dest.offset + copy_op[0].len > XEN_PAGE_SIZE) { 354 copy_op[1] = copy_op[0]; 355 copy_op[1].source.offset = XEN_PAGE_SIZE - copy_op[0].dest.offset; 356 copy_op[1].dest.u.gmfn = virt_to_gfn(mapping + off + len); 357 copy_op[1].dest.offset = 0; 358 copy_op[1].len = copy_op[0].len - copy_op[1].source.offset; 359 copy_op[0].len = copy_op[1].source.offset; 360 nr = 2; 361 } 362 363 memcpy(mapping, vif->hash.mapping[vif->hash.mapping_sel], 364 vif->hash.size * sizeof(*mapping)); 365 366 if (copy_op[0].len != 0) { 367 gnttab_batch_copy(copy_op, nr); 368 369 if (copy_op[0].status != GNTST_okay || 370 copy_op[nr - 1].status != GNTST_okay) 371 return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER; 372 } 373 374 while (len-- != 0) 375 if (mapping[off++] >= vif->num_queues) 376 return XEN_NETIF_CTRL_STATUS_INVALID_PARAMETER; 377 378 vif->hash.mapping_sel = !vif->hash.mapping_sel; 379 380 return XEN_NETIF_CTRL_STATUS_SUCCESS; 381 } 382 383 #ifdef CONFIG_DEBUG_FS 384 void xenvif_dump_hash_info(struct xenvif *vif, struct seq_file *m) 385 { 386 unsigned int i; 387 388 switch (vif->hash.alg) { 389 case XEN_NETIF_CTRL_HASH_ALGORITHM_TOEPLITZ: 390 seq_puts(m, "Hash Algorithm: TOEPLITZ\n"); 391 break; 392 393 case XEN_NETIF_CTRL_HASH_ALGORITHM_NONE: 394 seq_puts(m, "Hash Algorithm: NONE\n"); 395 fallthrough; 396 default: 397 return; 398 } 399 400 if (vif->hash.flags) { 401 seq_puts(m, "\nHash Flags:\n"); 402 403 if (vif->hash.flags & XEN_NETIF_CTRL_HASH_TYPE_IPV4) 404 seq_puts(m, "- IPv4\n"); 405 if (vif->hash.flags & XEN_NETIF_CTRL_HASH_TYPE_IPV4_TCP) 406 seq_puts(m, "- IPv4 + TCP\n"); 407 if (vif->hash.flags & XEN_NETIF_CTRL_HASH_TYPE_IPV6) 408 seq_puts(m, "- IPv6\n"); 409 if (vif->hash.flags & XEN_NETIF_CTRL_HASH_TYPE_IPV6_TCP) 410 seq_puts(m, "- IPv6 + TCP\n"); 411 } 412 413 seq_puts(m, "\nHash Key:\n"); 414 415 for (i = 0; i < XEN_NETBK_MAX_HASH_KEY_SIZE; ) { 416 unsigned int j, n; 417 418 n = 8; 419 if (i + n >= XEN_NETBK_MAX_HASH_KEY_SIZE) 420 n = XEN_NETBK_MAX_HASH_KEY_SIZE - i; 421 422 seq_printf(m, "[%2u - %2u]: ", i, i + n - 1); 423 424 for (j = 0; j < n; j++, i++) 425 seq_printf(m, "%02x ", vif->hash.key[i]); 426 427 seq_puts(m, "\n"); 428 } 429 430 if (vif->hash.size != 0) { 431 const u32 *mapping = vif->hash.mapping[vif->hash.mapping_sel]; 432 433 seq_puts(m, "\nHash Mapping:\n"); 434 435 for (i = 0; i < vif->hash.size; ) { 436 unsigned int j, n; 437 438 n = 8; 439 if (i + n >= vif->hash.size) 440 n = vif->hash.size - i; 441 442 seq_printf(m, "[%4u - %4u]: ", i, i + n - 1); 443 444 for (j = 0; j < n; j++, i++) 445 seq_printf(m, "%4u ", mapping[i]); 446 447 seq_puts(m, "\n"); 448 } 449 } 450 } 451 #endif /* CONFIG_DEBUG_FS */ 452 453 void xenvif_init_hash(struct xenvif *vif) 454 { 455 if (xenvif_hash_cache_size == 0) 456 return; 457 458 BUG_ON(vif->hash.cache.count); 459 460 spin_lock_init(&vif->hash.cache.lock); 461 INIT_LIST_HEAD(&vif->hash.cache.list); 462 } 463 464 void xenvif_deinit_hash(struct xenvif *vif) 465 { 466 xenvif_flush_hash(vif); 467 } 468