xfrm_policy.c (2518c7c2b3d7f0a6b302b4efe17c911f8dd4049f) xfrm_policy.c (44e36b42a8378be1dcf7e6f8a1cb2710a8903387)
1/*
2 * xfrm_policy.c
3 *
4 * Changes:
5 * Mitsuru KANDA @USAGI
6 * Kazunori MIYAZAWA @USAGI
7 * Kunihiro Ishiguro <kunihiro@ipinfusion.com>
8 * IPv6 support

--- 8 unchanged lines hidden (view full) ---

17#include <linux/kmod.h>
18#include <linux/list.h>
19#include <linux/spinlock.h>
20#include <linux/workqueue.h>
21#include <linux/notifier.h>
22#include <linux/netdevice.h>
23#include <linux/netfilter.h>
24#include <linux/module.h>
1/*
2 * xfrm_policy.c
3 *
4 * Changes:
5 * Mitsuru KANDA @USAGI
6 * Kazunori MIYAZAWA @USAGI
7 * Kunihiro Ishiguro <kunihiro@ipinfusion.com>
8 * IPv6 support

--- 8 unchanged lines hidden (view full) ---

17#include <linux/kmod.h>
18#include <linux/list.h>
19#include <linux/spinlock.h>
20#include <linux/workqueue.h>
21#include <linux/notifier.h>
22#include <linux/netdevice.h>
23#include <linux/netfilter.h>
24#include <linux/module.h>
25#include <linux/bootmem.h>
26#include <linux/vmalloc.h>
27#include <linux/cache.h>
28#include <net/xfrm.h>
29#include <net/ip.h>
30
25#include <linux/cache.h>
26#include <net/xfrm.h>
27#include <net/ip.h>
28
29#include "xfrm_hash.h"
30
31DEFINE_MUTEX(xfrm_cfg_mutex);
32EXPORT_SYMBOL(xfrm_cfg_mutex);
33
34static DEFINE_RWLOCK(xfrm_policy_lock);
35
36unsigned int xfrm_policy_count[XFRM_POLICY_MAX*2];
37EXPORT_SYMBOL(xfrm_policy_count);
38

--- 365 unchanged lines hidden (view full) ---

404};
405
406static struct hlist_head xfrm_policy_inexact[XFRM_POLICY_MAX*2];
407static struct xfrm_policy_hash xfrm_policy_bydst[XFRM_POLICY_MAX*2] __read_mostly;
408static struct hlist_head *xfrm_policy_byidx __read_mostly;
409static unsigned int xfrm_idx_hmask __read_mostly;
410static unsigned int xfrm_policy_hashmax __read_mostly = 1 * 1024 * 1024;
411
31DEFINE_MUTEX(xfrm_cfg_mutex);
32EXPORT_SYMBOL(xfrm_cfg_mutex);
33
34static DEFINE_RWLOCK(xfrm_policy_lock);
35
36unsigned int xfrm_policy_count[XFRM_POLICY_MAX*2];
37EXPORT_SYMBOL(xfrm_policy_count);
38

--- 365 unchanged lines hidden (view full) ---

404};
405
406static struct hlist_head xfrm_policy_inexact[XFRM_POLICY_MAX*2];
407static struct xfrm_policy_hash xfrm_policy_bydst[XFRM_POLICY_MAX*2] __read_mostly;
408static struct hlist_head *xfrm_policy_byidx __read_mostly;
409static unsigned int xfrm_idx_hmask __read_mostly;
410static unsigned int xfrm_policy_hashmax __read_mostly = 1 * 1024 * 1024;
411
412static inline unsigned int __idx_hash(u32 index, unsigned int hmask)
413{
414 return (index ^ (index >> 8)) & hmask;
415}
416
417static inline unsigned int idx_hash(u32 index)
418{
419 return __idx_hash(index, xfrm_idx_hmask);
420}
421
412static inline unsigned int idx_hash(u32 index)
413{
414 return __idx_hash(index, xfrm_idx_hmask);
415}
416
422static inline unsigned int __sel_hash(struct xfrm_selector *sel, unsigned short family, unsigned int hmask)
423{
424 xfrm_address_t *daddr = &sel->daddr;
425 xfrm_address_t *saddr = &sel->saddr;
426 unsigned int h = 0;
427
428 switch (family) {
429 case AF_INET:
430 if (sel->prefixlen_d != 32 ||
431 sel->prefixlen_s != 32)
432 return hmask + 1;
433
434 h = ntohl(daddr->a4 ^ saddr->a4);
435 break;
436
437 case AF_INET6:
438 if (sel->prefixlen_d != 128 ||
439 sel->prefixlen_s != 128)
440 return hmask + 1;
441
442 h = ntohl(daddr->a6[2] ^ daddr->a6[3] ^
443 saddr->a6[2] ^ saddr->a6[3]);
444 break;
445 };
446 h ^= (h >> 16);
447 return h & hmask;
448}
449
450static inline unsigned int __addr_hash(xfrm_address_t *daddr, xfrm_address_t *saddr, unsigned short family, unsigned int hmask)
451{
452 unsigned int h = 0;
453
454 switch (family) {
455 case AF_INET:
456 h = ntohl(daddr->a4 ^ saddr->a4);
457 break;
458
459 case AF_INET6:
460 h = ntohl(daddr->a6[2] ^ daddr->a6[3] ^
461 saddr->a6[2] ^ saddr->a6[3]);
462 break;
463 };
464 h ^= (h >> 16);
465 return h & hmask;
466}
467
468static struct hlist_head *policy_hash_bysel(struct xfrm_selector *sel, unsigned short family, int dir)
469{
470 unsigned int hmask = xfrm_policy_bydst[dir].hmask;
471 unsigned int hash = __sel_hash(sel, family, hmask);
472
473 return (hash == hmask + 1 ?
474 &xfrm_policy_inexact[dir] :
475 xfrm_policy_bydst[dir].table + hash);
476}
477
478static struct hlist_head *policy_hash_direct(xfrm_address_t *daddr, xfrm_address_t *saddr, unsigned short family, int dir)
479{
480 unsigned int hmask = xfrm_policy_bydst[dir].hmask;
481 unsigned int hash = __addr_hash(daddr, saddr, family, hmask);
482
483 return xfrm_policy_bydst[dir].table + hash;
484}
485
417static struct hlist_head *policy_hash_bysel(struct xfrm_selector *sel, unsigned short family, int dir)
418{
419 unsigned int hmask = xfrm_policy_bydst[dir].hmask;
420 unsigned int hash = __sel_hash(sel, family, hmask);
421
422 return (hash == hmask + 1 ?
423 &xfrm_policy_inexact[dir] :
424 xfrm_policy_bydst[dir].table + hash);
425}
426
427static struct hlist_head *policy_hash_direct(xfrm_address_t *daddr, xfrm_address_t *saddr, unsigned short family, int dir)
428{
429 unsigned int hmask = xfrm_policy_bydst[dir].hmask;
430 unsigned int hash = __addr_hash(daddr, saddr, family, hmask);
431
432 return xfrm_policy_bydst[dir].table + hash;
433}
434
486static struct hlist_head *xfrm_policy_hash_alloc(unsigned int sz)
487{
488 struct hlist_head *n;
489
490 if (sz <= PAGE_SIZE)
491 n = kmalloc(sz, GFP_KERNEL);
492 else if (hashdist)
493 n = __vmalloc(sz, GFP_KERNEL, PAGE_KERNEL);
494 else
495 n = (struct hlist_head *)
496 __get_free_pages(GFP_KERNEL, get_order(sz));
497
498 if (n)
499 memset(n, 0, sz);
500
501 return n;
502}
503
504static void xfrm_policy_hash_free(struct hlist_head *n, unsigned int sz)
505{
506 if (sz <= PAGE_SIZE)
507 kfree(n);
508 else if (hashdist)
509 vfree(n);
510 else
511 free_pages((unsigned long)n, get_order(sz));
512}
513
514static void xfrm_dst_hash_transfer(struct hlist_head *list,
515 struct hlist_head *ndsttable,
516 unsigned int nhashmask)
517{
518 struct hlist_node *entry, *tmp;
519 struct xfrm_policy *pol;
520
521 hlist_for_each_entry_safe(pol, entry, tmp, list, bydst) {

--- 26 unchanged lines hidden (view full) ---

548}
549
550static void xfrm_bydst_resize(int dir)
551{
552 unsigned int hmask = xfrm_policy_bydst[dir].hmask;
553 unsigned int nhashmask = xfrm_new_hash_mask(hmask);
554 unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
555 struct hlist_head *odst = xfrm_policy_bydst[dir].table;
435static void xfrm_dst_hash_transfer(struct hlist_head *list,
436 struct hlist_head *ndsttable,
437 unsigned int nhashmask)
438{
439 struct hlist_node *entry, *tmp;
440 struct xfrm_policy *pol;
441
442 hlist_for_each_entry_safe(pol, entry, tmp, list, bydst) {

--- 26 unchanged lines hidden (view full) ---

469}
470
471static void xfrm_bydst_resize(int dir)
472{
473 unsigned int hmask = xfrm_policy_bydst[dir].hmask;
474 unsigned int nhashmask = xfrm_new_hash_mask(hmask);
475 unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
476 struct hlist_head *odst = xfrm_policy_bydst[dir].table;
556 struct hlist_head *ndst = xfrm_policy_hash_alloc(nsize);
477 struct hlist_head *ndst = xfrm_hash_alloc(nsize);
557 int i;
558
559 if (!ndst)
560 return;
561
562 write_lock_bh(&xfrm_policy_lock);
563
564 for (i = hmask; i >= 0; i--)
565 xfrm_dst_hash_transfer(odst + i, ndst, nhashmask);
566
567 xfrm_policy_bydst[dir].table = ndst;
568 xfrm_policy_bydst[dir].hmask = nhashmask;
569
570 write_unlock_bh(&xfrm_policy_lock);
571
478 int i;
479
480 if (!ndst)
481 return;
482
483 write_lock_bh(&xfrm_policy_lock);
484
485 for (i = hmask; i >= 0; i--)
486 xfrm_dst_hash_transfer(odst + i, ndst, nhashmask);
487
488 xfrm_policy_bydst[dir].table = ndst;
489 xfrm_policy_bydst[dir].hmask = nhashmask;
490
491 write_unlock_bh(&xfrm_policy_lock);
492
572 xfrm_policy_hash_free(odst, (hmask + 1) * sizeof(struct hlist_head));
493 xfrm_hash_free(odst, (hmask + 1) * sizeof(struct hlist_head));
573}
574
575static void xfrm_byidx_resize(int total)
576{
577 unsigned int hmask = xfrm_idx_hmask;
578 unsigned int nhashmask = xfrm_new_hash_mask(hmask);
579 unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
580 struct hlist_head *oidx = xfrm_policy_byidx;
494}
495
496static void xfrm_byidx_resize(int total)
497{
498 unsigned int hmask = xfrm_idx_hmask;
499 unsigned int nhashmask = xfrm_new_hash_mask(hmask);
500 unsigned int nsize = (nhashmask + 1) * sizeof(struct hlist_head);
501 struct hlist_head *oidx = xfrm_policy_byidx;
581 struct hlist_head *nidx = xfrm_policy_hash_alloc(nsize);
502 struct hlist_head *nidx = xfrm_hash_alloc(nsize);
582 int i;
583
584 if (!nidx)
585 return;
586
587 write_lock_bh(&xfrm_policy_lock);
588
589 for (i = hmask; i >= 0; i--)
590 xfrm_idx_hash_transfer(oidx + i, nidx, nhashmask);
591
592 xfrm_policy_byidx = nidx;
593 xfrm_idx_hmask = nhashmask;
594
595 write_unlock_bh(&xfrm_policy_lock);
596
503 int i;
504
505 if (!nidx)
506 return;
507
508 write_lock_bh(&xfrm_policy_lock);
509
510 for (i = hmask; i >= 0; i--)
511 xfrm_idx_hash_transfer(oidx + i, nidx, nhashmask);
512
513 xfrm_policy_byidx = nidx;
514 xfrm_idx_hmask = nhashmask;
515
516 write_unlock_bh(&xfrm_policy_lock);
517
597 xfrm_policy_hash_free(oidx, (hmask + 1) * sizeof(struct hlist_head));
518 xfrm_hash_free(oidx, (hmask + 1) * sizeof(struct hlist_head));
598}
599
600static inline int xfrm_bydst_should_resize(int dir, int *total)
601{
602 unsigned int cnt = xfrm_policy_count[dir];
603 unsigned int hmask = xfrm_policy_bydst[dir].hmask;
604
605 if (total)

--- 1460 unchanged lines hidden (view full) ---

2066 0, SLAB_HWCACHE_ALIGN,
2067 NULL, NULL);
2068 if (!xfrm_dst_cache)
2069 panic("XFRM: failed to allocate xfrm_dst_cache\n");
2070
2071 hmask = 8 - 1;
2072 sz = (hmask+1) * sizeof(struct hlist_head);
2073
519}
520
521static inline int xfrm_bydst_should_resize(int dir, int *total)
522{
523 unsigned int cnt = xfrm_policy_count[dir];
524 unsigned int hmask = xfrm_policy_bydst[dir].hmask;
525
526 if (total)

--- 1460 unchanged lines hidden (view full) ---

1987 0, SLAB_HWCACHE_ALIGN,
1988 NULL, NULL);
1989 if (!xfrm_dst_cache)
1990 panic("XFRM: failed to allocate xfrm_dst_cache\n");
1991
1992 hmask = 8 - 1;
1993 sz = (hmask+1) * sizeof(struct hlist_head);
1994
2074 xfrm_policy_byidx = xfrm_policy_hash_alloc(sz);
1995 xfrm_policy_byidx = xfrm_hash_alloc(sz);
2075 xfrm_idx_hmask = hmask;
2076 if (!xfrm_policy_byidx)
2077 panic("XFRM: failed to allocate byidx hash\n");
2078
2079 for (dir = 0; dir < XFRM_POLICY_MAX * 2; dir++) {
2080 struct xfrm_policy_hash *htab;
2081
2082 INIT_HLIST_HEAD(&xfrm_policy_inexact[dir]);
2083
2084 htab = &xfrm_policy_bydst[dir];
1996 xfrm_idx_hmask = hmask;
1997 if (!xfrm_policy_byidx)
1998 panic("XFRM: failed to allocate byidx hash\n");
1999
2000 for (dir = 0; dir < XFRM_POLICY_MAX * 2; dir++) {
2001 struct xfrm_policy_hash *htab;
2002
2003 INIT_HLIST_HEAD(&xfrm_policy_inexact[dir]);
2004
2005 htab = &xfrm_policy_bydst[dir];
2085 htab->table = xfrm_policy_hash_alloc(sz);
2006 htab->table = xfrm_hash_alloc(sz);
2086 htab->hmask = hmask;
2087 if (!htab->table)
2088 panic("XFRM: failed to allocate bydst hash\n");
2089 }
2090
2091 INIT_WORK(&xfrm_policy_gc_work, xfrm_policy_gc_task, NULL);
2092 register_netdevice_notifier(&xfrm_dev_notifier);
2093}
2094
2095void __init xfrm_init(void)
2096{
2097 xfrm_state_init();
2098 xfrm_policy_init();
2099 xfrm_input_init();
2100}
2101
2007 htab->hmask = hmask;
2008 if (!htab->table)
2009 panic("XFRM: failed to allocate bydst hash\n");
2010 }
2011
2012 INIT_WORK(&xfrm_policy_gc_work, xfrm_policy_gc_task, NULL);
2013 register_netdevice_notifier(&xfrm_dev_notifier);
2014}
2015
2016void __init xfrm_init(void)
2017{
2018 xfrm_state_init();
2019 xfrm_policy_init();
2020 xfrm_input_init();
2021}
2022