1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /* Copyright (C) 2013 Jozsef Kadlecsik <kadlec@netfilter.org> */
3
4 #ifndef _IP_SET_HASH_GEN_H
5 #define _IP_SET_HASH_GEN_H
6
7 #include <linux/rcupdate.h>
8 #include <linux/rcupdate_wait.h>
9 #include <linux/jhash.h>
10 #include <linux/types.h>
11 #include <linux/seqlock.h>
12 #include <linux/netfilter/nfnetlink.h>
13 #include <linux/netfilter/ipset/ip_set.h>
14
15 #define __ipset_dereference(p) \
16 rcu_dereference_protected(p, 1)
17 #define ipset_dereference_nfnl(p) \
18 rcu_dereference_protected(p, \
19 lockdep_nfnl_is_held(NFNL_SUBSYS_IPSET))
20 #define ipset_dereference_set(p, set) \
21 rcu_dereference_protected(p, \
22 lockdep_nfnl_is_held(NFNL_SUBSYS_IPSET) || \
23 lockdep_is_held(&(set)->lock))
24 #define ipset_dereference_bh_nfnl(p) \
25 rcu_dereference_bh_check(p, \
26 lockdep_nfnl_is_held(NFNL_SUBSYS_IPSET))
27
28 /* Hashing which uses arrays to resolve clashing. The hash table is resized
29 * (doubled) when searching becomes too long.
30 * Internally jhash is used with the assumption that the size of the
31 * stored data is a multiple of sizeof(u32).
32 *
33 * Readers and resizing
34 *
35 * Resizing can be triggered by userspace command only, and those
36 * are serialized by the nfnl mutex. During resizing the set is
37 * read-locked, so the only possible concurrent operations are
38 * the kernel side readers. Those must be protected by proper RCU locking.
39 */
40
41 /* Number of elements to store in an initial array block */
42 #define AHASH_INIT_SIZE 2
43 /* Max number of elements to store in an array block */
44 #define AHASH_MAX_SIZE (6 * AHASH_INIT_SIZE)
45 /* Max muber of elements in the array block when tuned */
46 #define AHASH_MAX_TUNED 64
47 #define AHASH_MAX(h) ((h)->bucketsize)
48
49 /* A hash bucket */
50 struct hbucket {
51 struct rcu_head rcu; /* for call_rcu */
52 /* Which positions are used in the array */
53 DECLARE_BITMAP(used, AHASH_MAX_TUNED);
54 u8 size; /* size of the array */
55 u8 pos; /* position of the first free entry */
56 unsigned char value[] /* the array of the values */
57 __aligned(__alignof__(u64));
58 };
59
60 /* Region size for locking == 2^HTABLE_REGION_BITS */
61 #define HTABLE_REGION_BITS 10
62 #define ahash_numof_locks(htable_bits) \
63 ((htable_bits) < HTABLE_REGION_BITS ? 1 \
64 : jhash_size((htable_bits) - HTABLE_REGION_BITS))
65 #define ahash_sizeof_regions(htable_bits) \
66 (ahash_numof_locks(htable_bits) * sizeof(struct ip_set_region))
67 #define ahash_region(n) \
68 ((n) / jhash_size(HTABLE_REGION_BITS))
69 #define ahash_bucket_start(h, htable_bits) \
70 ((htable_bits) < HTABLE_REGION_BITS ? 0 \
71 : (h) * jhash_size(HTABLE_REGION_BITS))
72 #define ahash_bucket_end(h, htable_bits) \
73 ((htable_bits) < HTABLE_REGION_BITS ? jhash_size(htable_bits) \
74 : ((h) + 1) * jhash_size(HTABLE_REGION_BITS))
75
76 struct htable_gc {
77 struct delayed_work dwork;
78 struct ip_set *set; /* Set the gc belongs to */
79 spinlock_t lock; /* Lock to exclude gc and resize */
80 u32 region; /* Last gc run position */
81 };
82
83 /* The hash table: the table size stored here in order to make resizing easy */
84 struct htable {
85 bool resizing; /* Mark ongoing resize */
86 atomic_t uref; /* References for dumping and gc */
87 u8 htable_bits; /* size of hash table == 2^htable_bits */
88 u32 maxelem; /* Maxelem per region */
89 struct list_head ad; /* Resize add|del backlist */
90 struct ip_set_region *hregion; /* Region locks and ext sizes */
91 struct hbucket __rcu *bucket[]; /* hashtable buckets */
92 };
93
94 #define hbucket(h, i) ((h)->bucket[i])
95 #define ext_size(n, dsize) \
96 (sizeof(struct hbucket) + (n) * (dsize))
97
98 #ifndef IPSET_NET_COUNT
99 #define IPSET_NET_COUNT 1
100 #endif
101
102 /**
103 * struct net_prefix - Representation of a network prefix.
104 * @cidr: The CIDR prefix length.
105 * @count: Number of occurrences.
106 */
107 struct net_prefix {
108 u32 cidr:8;
109 u32 count:24;
110 };
111
112 #define CIDR_MAX_COUNT ((1 << 24) - 1)
113
114 /**
115 * struct net_prefixes - A collection of network prefixes.
116 * @rcu: RCU head
117 * @seq: Sequence counter guarding in-place reordering of @nets
118 * @len: Number of entries in the array.
119 * @nets: Array of net_prefix structures (sorted by CIDR descending).
120 *
121 * @nets entries are updated in place under @set's lock. A single entry's
122 * cidr/count pair is always updated atomically via READ_ONCE()/WRITE_ONCE(),
123 * but removing an entry also shifts every following entry down by one slot.
124 * Lockless readers that scan the whole array (i.e. more than a single
125 * indexed slot) must use @seq to detect and retry across such a shift.
126 */
127 struct net_prefixes {
128 struct rcu_head rcu;
129 seqcount_spinlock_t seq;
130 u8 len;
131 struct net_prefix nets[] __counted_by(len);
132 };
133
134 /* Compute the hash table size */
135 static size_t
htable_size(u8 hbits)136 htable_size(u8 hbits)
137 {
138 size_t hsize;
139
140 /* We must fit both into u32 in jhash and INT_MAX in kvmalloc_node() */
141 if (hbits > 31)
142 return 0;
143 hsize = jhash_size(hbits);
144 if ((INT_MAX - sizeof(struct htable)) / sizeof(struct hbucket *)
145 < hsize)
146 return 0;
147
148 return hsize * sizeof(struct hbucket *) + sizeof(struct htable);
149 }
150
151 #ifdef IP_SET_HASH_WITH_NETS
152 #if IPSET_NET_COUNT > 1
153 #define __CIDR(cidr, i) (cidr[i])
154 #else
155 #define __CIDR(cidr, i) (cidr)
156 #endif
157 #ifdef IP_SET_HASH_WITH_NETS_PACKED
158 /* When cidr is packed with nomatch, cidr - 1 is stored in the data entry */
159 #define DCIDR_PUT(cidr) ((cidr) - 1)
160 #define DCIDR_GET(cidr, i) (__CIDR(cidr, i) + 1)
161 #else
162 #define DCIDR_PUT(cidr) (cidr)
163 #define DCIDR_GET(cidr, i) __CIDR(cidr, i)
164 #endif
165
166 #define INIT_CIDR(n, host_mask) ({ \
167 const struct net_prefixes *__n = rcu_dereference(n); \
168 struct net_prefix __p = \
169 __n->len ? READ_ONCE(__n->nets[0]) \
170 : (struct net_prefix){}; \
171 DCIDR_PUT(__p.count ? __p.cidr : host_mask); \
172 })
173
174 #endif /* IP_SET_HASH_WITH_NETS */
175
176 #define SET_ELEM_EXPIRED(set, d) \
177 (SET_WITH_TIMEOUT(set) && \
178 ip_set_timeout_expired(ext_timeout(d, set)))
179
180 #if defined(IP_SET_HASH_WITH_NETMASK) || defined(IP_SET_HASH_WITH_BITMASK)
181 static const union nf_inet_addr onesmask = {
182 .all[0] = 0xffffffff,
183 .all[1] = 0xffffffff,
184 .all[2] = 0xffffffff,
185 .all[3] = 0xffffffff
186 };
187
188 static const union nf_inet_addr zeromask = {};
189 #endif
190
191 #endif /* _IP_SET_HASH_GEN_H */
192
193 #ifndef MTYPE
194 #error "MTYPE is not defined!"
195 #endif
196
197 #ifndef HTYPE
198 #error "HTYPE is not defined!"
199 #endif
200
201 #ifndef HOST_MASK
202 #error "HOST_MASK is not defined!"
203 #endif
204
205 /* Family dependent templates */
206
207 #undef ahash_data
208 #undef mtype_data_equal
209 #undef mtype_do_data_match
210 #undef mtype_data_set_flags
211 #undef mtype_data_reset_elem
212 #undef mtype_data_reset_flags
213 #undef mtype_data_netmask
214 #undef mtype_data_list
215 #undef mtype_data_next
216 #undef mtype_elem
217
218 #undef mtype_ahash_destroy
219 #undef mtype_ext_cleanup
220 #undef mtype_add_cidr
221 #undef mtype_del_cidr
222 #undef mtype_del_cidr_all
223 #undef mtype_ahash_memsize
224 #undef mtype_flush
225 #undef mtype_destroy
226 #undef mtype_same_set
227 #undef mtype_kadt
228 #undef mtype_uadt
229 #undef mtype_bucket_size
230 #undef mtype_hash_size
231
232 #undef mtype_add
233 #undef mtype_del
234 #undef mtype_test_cidrs
235 #undef mtype_test
236 #undef mtype_uref
237 #undef mtype_resize
238 #undef mtype_ext_size
239 #undef mtype_resize_ad
240 #undef mtype_head
241 #undef mtype_list
242 #undef mtype_gc_do
243 #undef mtype_gc
244 #undef mtype_gc_init
245 #undef mtype_cancel_gc
246 #undef mtype_variant
247 #undef mtype_data_match
248
249 #undef htype
250 #undef HKEY
251
252 #define mtype_data_equal IPSET_TOKEN(MTYPE, _data_equal)
253 #ifdef IP_SET_HASH_WITH_NETS
254 #define mtype_do_data_match IPSET_TOKEN(MTYPE, _do_data_match)
255 #else
256 #define mtype_do_data_match(d) 1
257 #endif
258 #define mtype_data_set_flags IPSET_TOKEN(MTYPE, _data_set_flags)
259 #define mtype_data_reset_elem IPSET_TOKEN(MTYPE, _data_reset_elem)
260 #define mtype_data_reset_flags IPSET_TOKEN(MTYPE, _data_reset_flags)
261 #define mtype_data_netmask IPSET_TOKEN(MTYPE, _data_netmask)
262 #define mtype_data_list IPSET_TOKEN(MTYPE, _data_list)
263 #define mtype_data_next IPSET_TOKEN(MTYPE, _data_next)
264 #define mtype_elem IPSET_TOKEN(MTYPE, _elem)
265
266 #define mtype_ahash_destroy IPSET_TOKEN(MTYPE, _ahash_destroy)
267 #define mtype_ext_cleanup IPSET_TOKEN(MTYPE, _ext_cleanup)
268 #define mtype_add_cidr IPSET_TOKEN(MTYPE, _add_cidr)
269 #define mtype_del_cidr IPSET_TOKEN(MTYPE, _del_cidr)
270 #define mtype_del_cidr_all IPSET_TOKEN(MTYPE, _del_cidr_all)
271 #define mtype_ahash_memsize IPSET_TOKEN(MTYPE, _ahash_memsize)
272 #define mtype_flush IPSET_TOKEN(MTYPE, _flush)
273 #define mtype_destroy IPSET_TOKEN(MTYPE, _destroy)
274 #define mtype_same_set IPSET_TOKEN(MTYPE, _same_set)
275 #define mtype_kadt IPSET_TOKEN(MTYPE, _kadt)
276 #define mtype_uadt IPSET_TOKEN(MTYPE, _uadt)
277 #define mtype_bucket_size IPSET_TOKEN(MTYPE, _bucket_size)
278 #define mtype_hash_size IPSET_TOKEN(MTYPE, _hash_size)
279
280 #define mtype_add IPSET_TOKEN(MTYPE, _add)
281 #define mtype_del IPSET_TOKEN(MTYPE, _del)
282 #define mtype_test_cidrs IPSET_TOKEN(MTYPE, _test_cidrs)
283 #define mtype_test IPSET_TOKEN(MTYPE, _test)
284 #define mtype_uref IPSET_TOKEN(MTYPE, _uref)
285 #define mtype_resize IPSET_TOKEN(MTYPE, _resize)
286 #define mtype_ext_size IPSET_TOKEN(MTYPE, _ext_size)
287 #define mtype_resize_ad IPSET_TOKEN(MTYPE, _resize_ad)
288 #define mtype_head IPSET_TOKEN(MTYPE, _head)
289 #define mtype_list IPSET_TOKEN(MTYPE, _list)
290 #define mtype_gc_do IPSET_TOKEN(MTYPE, _gc_do)
291 #define mtype_gc IPSET_TOKEN(MTYPE, _gc)
292 #define mtype_gc_init IPSET_TOKEN(MTYPE, _gc_init)
293 #define mtype_cancel_gc IPSET_TOKEN(MTYPE, _cancel_gc)
294 #define mtype_variant IPSET_TOKEN(MTYPE, _variant)
295 #define mtype_data_match IPSET_TOKEN(MTYPE, _data_match)
296
297 #ifndef HKEY_DATALEN
298 #define HKEY_DATALEN sizeof(struct mtype_elem)
299 #endif
300
301 #define htype MTYPE
302
303 #define HKEY(data, initval, htable_bits) \
304 ({ \
305 const u32 *__k = (const u32 *)data; \
306 u32 __l = HKEY_DATALEN / sizeof(u32); \
307 \
308 BUILD_BUG_ON(HKEY_DATALEN % sizeof(u32) != 0); \
309 \
310 jhash2(__k, __l, initval) & jhash_mask(htable_bits); \
311 })
312
313 /* The generic hash structure */
314 struct htype {
315 struct htable __rcu *table; /* the hash table */
316 struct net_prefixes __rcu *rnets[IPSET_NET_COUNT]; /* cidr prefixes */
317 struct htable_gc gc; /* gc workqueue */
318 u32 maxelem; /* max elements in the hash */
319 u32 initval; /* random jhash init value */
320 #ifdef IP_SET_HASH_WITH_MARKMASK
321 u32 markmask; /* markmask value for mark mask to store */
322 #endif
323 u8 bucketsize; /* max elements in an array block */
324 #if defined(IP_SET_HASH_WITH_NETMASK) || defined(IP_SET_HASH_WITH_BITMASK)
325 u8 netmask; /* netmask value for subnets to store */
326 union nf_inet_addr bitmask; /* stores bitmask */
327 #endif
328 /* Because 'next' is IPv4/IPv6 dependent, no elements of this
329 * structure and referred in create() may come after 'next'.
330 */
331 struct mtype_elem next; /* temporary storage for uadd */
332 };
333
334 /* ADD|DEL entries saved during resize */
335 struct mtype_resize_ad {
336 struct list_head list;
337 enum ipset_adt ad; /* ADD|DEL element */
338 struct mtype_elem d; /* Element value */
339 struct ip_set_ext ext; /* Extensions for ADD */
340 struct ip_set_ext mext; /* Target extensions for ADD */
341 u32 flags; /* Flags for ADD */
342 };
343
344 #ifdef IP_SET_HASH_WITH_NETS
345 /**
346 * mtype_add_cidr - Add a CIDR entry to hash table bookkeeping
347 * @set: Pointer to the ip_set
348 * @h: Pointer to the htype
349 * @cidr: The CIDR prefix length
350 * @n: The index of the net_prefix array to add @cidr to
351 *
352 * Performs an update if @cidr is found, otherwise performs COW-style
353 * allocation and replacement via RCU.
354 *
355 * Return: 0 on success, negative error code on failure.
356 */
357 static int
mtype_add_cidr(struct ip_set * set,struct htype * h,u8 cidr,u8 n)358 mtype_add_cidr(struct ip_set *set, struct htype *h, u8 cidr, u8 n)
359 {
360 int i, j, found, len = 0, ret = 0;
361 struct net_prefixes *nets, *tmp;
362 struct net_prefix np;
363
364 spin_lock_bh(&set->lock);
365 nets = __ipset_dereference(h->rnets[n]);
366 /* Add in increasing prefix order, so larger cidr first */
367 for (i = 0, found = -1; i < nets->len; i++) {
368 np = READ_ONCE(nets->nets[i]);
369 if (np.count)
370 len++;
371 if (found != -1) {
372 continue;
373 } else if (np.cidr < cidr) {
374 found = i;
375 } else if (np.cidr == cidr) {
376 if (np.count < CIDR_MAX_COUNT) {
377 np.count++;
378 WRITE_ONCE(nets->nets[i], np);
379 } else {
380 ret = -EOVERFLOW;
381 }
382 goto unlock;
383 }
384 }
385 len++;
386 tmp = kzalloc_flex(*tmp, nets, len, GFP_ATOMIC);
387 if (!tmp) {
388 ret = -ENOMEM;
389 goto unlock;
390 }
391
392 tmp->len = len;
393 seqcount_spinlock_init(&tmp->seq, &set->lock);
394 for (i = 0, j = 0; i < nets->len; i++) {
395 if (i == found) {
396 tmp->nets[j].cidr = cidr;
397 tmp->nets[j++].count = 1;
398 }
399 if (!nets->nets[i].count)
400 continue;
401 tmp->nets[j].cidr = nets->nets[i].cidr;
402 tmp->nets[j++].count = nets->nets[i].count;
403 }
404 if (found == -1) {
405 tmp->nets[j].cidr = cidr;
406 tmp->nets[j].count = 1;
407 }
408 rcu_assign_pointer(h->rnets[n], tmp);
409 kfree_rcu(nets, rcu);
410 unlock:
411 spin_unlock_bh(&set->lock);
412 return ret;
413 }
414
415 /**
416 * mtype_del_cidr - Remove CIDR entry and maintain array integrity.
417 * @set: Pointer to the ip_set.
418 * @h: Pointer to the htype.
419 * @cidr: The CIDR prefix length.
420 * @n: The index of the net_prefix array to remove @cidr from
421 *
422 * If CIDR entry count falls to 0, this function performs a "shift-left"
423 * operation on all following elements. This ensures that the array remains
424 * contiguous and maintains its descending order by CIDR. The vacated slot
425 * at the end of the array is zeroed out (cidr=0, count=0).
426 */
427 static void
mtype_del_cidr(struct ip_set * set,struct htype * h,u8 cidr,u8 n)428 mtype_del_cidr(struct ip_set *set, struct htype *h, u8 cidr, u8 n)
429 {
430 struct net_prefixes *nets;
431 struct net_prefix np;
432 int found;
433 u8 i, j;
434
435 BUILD_BUG_ON(sizeof(struct net_prefix) != sizeof(u32));
436
437 spin_lock_bh(&set->lock);
438 nets = __ipset_dereference(h->rnets[n]);
439 for (i = 0, found = -1; i < nets->len; i++) {
440 np = READ_ONCE(nets->nets[i]);
441 if (np.count && np.cidr == cidr) {
442 np.count--;
443 found = i;
444 break;
445 }
446 }
447 if (unlikely(found == -1))
448 goto unlock;
449
450 if (np.count) {
451 WRITE_ONCE(nets->nets[found], np);
452 goto unlock;
453 }
454
455 write_seqcount_begin(&nets->seq);
456 for (i = 0, j = 0; i < nets->len; i++) {
457 if (i == found)
458 continue;
459
460 np = READ_ONCE(nets->nets[i]);
461 if (i != j)
462 WRITE_ONCE(nets->nets[j], np);
463 j++;
464 }
465
466 while (j < nets->len)
467 WRITE_ONCE(nets->nets[j++], (struct net_prefix){});
468 write_seqcount_end(&nets->seq);
469 unlock:
470 spin_unlock_bh(&set->lock);
471 }
472 #endif
473
474 static void
mtype_del_cidr_all(struct ip_set * set,struct htype * h,const struct mtype_elem * data)475 mtype_del_cidr_all(struct ip_set *set, struct htype *h, const struct mtype_elem *data)
476 {
477 #ifdef IP_SET_HASH_WITH_NETS
478 int k;
479
480 for (k = 0; k < IPSET_NET_COUNT; k++)
481 mtype_del_cidr(set, h, DCIDR_GET(data->cidr, k), k);
482 #endif
483 }
484
485 /* Calculate the actual memory size of the set data */
486 static size_t
mtype_ahash_memsize(const struct htype * h,const struct htable * t)487 mtype_ahash_memsize(const struct htype *h, const struct htable *t)
488 {
489 return sizeof(*h) + sizeof(*t) + ahash_sizeof_regions(t->htable_bits);
490 }
491
492 /* Get the ith element from the array block n */
493 #define ahash_data(n, i, dsize) \
494 ((struct mtype_elem *)((n)->value + ((i) * (dsize))))
495
496 static void
mtype_ext_cleanup(struct ip_set * set,struct hbucket * n)497 mtype_ext_cleanup(struct ip_set *set, struct hbucket *n)
498 {
499 int i;
500 u8 pos = smp_load_acquire(&n->pos);
501
502 for (i = 0; i < pos; i++)
503 if (test_bit(i, n->used))
504 ip_set_ext_destroy(set, ahash_data(n, i, set->dsize));
505 }
506
507 /* Flush a hash type of set: destroy all elements */
508 static void
mtype_flush(struct ip_set * set)509 mtype_flush(struct ip_set *set)
510 {
511 struct htype *h = set->data;
512 #ifdef IP_SET_HASH_WITH_NETS
513 struct net_prefixes *nets;
514 #endif
515 struct htable *t;
516 struct hbucket *n;
517 u32 r, i;
518
519 t = ipset_dereference_nfnl(h->table);
520 for (r = 0; r < ahash_numof_locks(t->htable_bits); r++) {
521 spin_lock_bh(&t->hregion[r].lock);
522 for (i = ahash_bucket_start(r, t->htable_bits);
523 i < ahash_bucket_end(r, t->htable_bits); i++) {
524 n = __ipset_dereference(hbucket(t, i));
525 if (!n)
526 continue;
527 if (set->extensions & IPSET_EXT_DESTROY)
528 mtype_ext_cleanup(set, n);
529 /* FIXME: use slab cache */
530 rcu_assign_pointer(hbucket(t, i), NULL);
531 kfree_rcu(n, rcu);
532 }
533 t->hregion[r].ext_size = 0;
534 t->hregion[r].elements = 0;
535 spin_unlock_bh(&t->hregion[r].lock);
536 }
537 #ifdef IP_SET_HASH_WITH_NETS
538 for (i = 0; i < IPSET_NET_COUNT; i++) {
539 u8 j;
540
541 spin_lock_bh(&set->lock);
542 nets = ipset_dereference_nfnl(h->rnets[i]);
543 write_seqcount_begin(&nets->seq);
544 for (j = 0; j < nets->len; j++)
545 WRITE_ONCE(nets->nets[j], (struct net_prefix){});
546 write_seqcount_end(&nets->seq);
547 spin_unlock_bh(&set->lock);
548 }
549 #endif
550 }
551
552 /* Destroy the hashtable part of the set */
553 static void
mtype_ahash_destroy(struct ip_set * set,struct htable * t,bool ext_destroy)554 mtype_ahash_destroy(struct ip_set *set, struct htable *t, bool ext_destroy)
555 {
556 #ifdef IP_SET_HASH_WITH_NETS
557 struct htype *h = set->data;
558 #endif
559 struct hbucket *n;
560 u32 i;
561
562 for (i = 0; i < jhash_size(t->htable_bits); i++) {
563 n = (__force struct hbucket *)hbucket(t, i);
564 if (!n)
565 continue;
566 if (set->extensions & IPSET_EXT_DESTROY && ext_destroy)
567 mtype_ext_cleanup(set, n);
568 /* FIXME: use slab cache */
569 kfree(n);
570 }
571
572 #ifdef IP_SET_HASH_WITH_NETS
573 if (ext_destroy)
574 for (i = 0; i < IPSET_NET_COUNT; i++)
575 kfree(rcu_dereference_raw(h->rnets[i]));
576 #endif
577 ip_set_free(t->hregion);
578 ip_set_free(t);
579 }
580
581 /* Destroy a hash type of set */
582 static void
mtype_destroy(struct ip_set * set)583 mtype_destroy(struct ip_set *set)
584 {
585 struct htype *h = set->data;
586 struct htable *t = (__force struct htable *)h->table;
587 struct list_head *l, *lt;
588
589 list_for_each_safe(l, lt, &t->ad) {
590 list_del(l);
591 kfree(l);
592 }
593 mtype_ahash_destroy(set, t, true);
594 kfree(h);
595
596 set->data = NULL;
597 }
598
599 static bool
mtype_same_set(const struct ip_set * a,const struct ip_set * b)600 mtype_same_set(const struct ip_set *a, const struct ip_set *b)
601 {
602 const struct htype *x = a->data;
603 const struct htype *y = b->data;
604
605 /* Resizing changes htable_bits, so we ignore it */
606 return x->maxelem == y->maxelem &&
607 a->timeout == b->timeout &&
608 #if defined(IP_SET_HASH_WITH_NETMASK) || defined(IP_SET_HASH_WITH_BITMASK)
609 nf_inet_addr_cmp(&x->bitmask, &y->bitmask) &&
610 #endif
611 #ifdef IP_SET_HASH_WITH_MARKMASK
612 x->markmask == y->markmask &&
613 #endif
614 a->extensions == b->extensions;
615 }
616
617 static void
mtype_gc_do(struct ip_set * set,struct htype * h,struct htable * t,u32 r)618 mtype_gc_do(struct ip_set *set, struct htype *h, struct htable *t, u32 r)
619 {
620 struct hbucket *n, *tmp;
621 struct mtype_elem *data;
622 u32 i, j, d;
623 size_t dsize = set->dsize;
624 u8 pos, htable_bits = t->htable_bits;
625
626 spin_lock_bh(&t->hregion[r].lock);
627 for (i = ahash_bucket_start(r, htable_bits);
628 i < ahash_bucket_end(r, htable_bits); i++) {
629 n = __ipset_dereference(hbucket(t, i));
630 if (!n)
631 continue;
632 pos = smp_load_acquire(&n->pos);
633 for (j = 0, d = 0; j < pos; j++) {
634 if (!test_bit(j, n->used)) {
635 d++;
636 continue;
637 }
638 data = ahash_data(n, j, dsize);
639 if (!ip_set_timeout_expired(ext_timeout(data, set)))
640 continue;
641 pr_debug("expired %u/%u\n", i, j);
642 clear_bit(j, n->used);
643 smp_mb__after_atomic();
644 mtype_del_cidr_all(set, h, data);
645 t->hregion[r].elements--;
646 ip_set_ext_destroy(set, data);
647 d++;
648 }
649 if (d >= AHASH_INIT_SIZE) {
650 if (d >= n->size) {
651 t->hregion[r].ext_size -=
652 ext_size(n->size, dsize);
653 rcu_assign_pointer(hbucket(t, i), NULL);
654 kfree_rcu(n, rcu);
655 continue;
656 }
657 tmp = kzalloc(sizeof(*tmp) +
658 (n->size - AHASH_INIT_SIZE) * dsize,
659 GFP_ATOMIC);
660 if (!tmp)
661 /* Still try to delete expired elements. */
662 continue;
663 tmp->size = n->size - AHASH_INIT_SIZE;
664 for (j = 0, d = 0; j < pos; j++) {
665 if (!test_bit(j, n->used))
666 continue;
667 data = ahash_data(n, j, dsize);
668 memcpy(tmp->value + d * dsize,
669 data, dsize);
670 set_bit(d, tmp->used);
671 d++;
672 }
673 tmp->pos = d;
674 t->hregion[r].ext_size -=
675 ext_size(AHASH_INIT_SIZE, dsize);
676 rcu_assign_pointer(hbucket(t, i), tmp);
677 kfree_rcu(n, rcu);
678 }
679 }
680 spin_unlock_bh(&t->hregion[r].lock);
681 }
682
683 static void
mtype_gc(struct work_struct * work)684 mtype_gc(struct work_struct *work)
685 {
686 struct htable_gc *gc;
687 struct ip_set *set;
688 struct htype *h;
689 struct htable *t;
690 u32 r, numof_locks;
691 unsigned int next_run;
692
693 gc = container_of(work, struct htable_gc, dwork.work);
694 set = gc->set;
695 h = set->data;
696
697 rcu_read_lock_bh();
698 t = rcu_dereference_bh(h->table);
699 atomic_inc(&t->uref);
700 rcu_read_unlock_bh();
701 numof_locks = ahash_numof_locks(t->htable_bits);
702 r = gc->region++;
703 if (r >= numof_locks) {
704 r = gc->region = 0;
705 }
706 next_run = (IPSET_GC_PERIOD(set->timeout) * HZ) / numof_locks;
707 if (next_run < HZ/10)
708 next_run = HZ/10;
709
710 spin_lock_bh(&gc->lock);
711 if (!t->resizing)
712 mtype_gc_do(set, h, t, r);
713 spin_unlock_bh(&gc->lock);
714
715 if (atomic_dec_and_test(&t->uref) && t->resizing) {
716 pr_debug("Table destroy after resize by expire: %p\n", t);
717 mtype_ahash_destroy(set, t, false);
718 }
719
720 queue_delayed_work(system_power_efficient_wq, &gc->dwork, next_run);
721
722 }
723
724 static void
mtype_gc_init(struct htable_gc * gc)725 mtype_gc_init(struct htable_gc *gc)
726 {
727 INIT_DEFERRABLE_WORK(&gc->dwork, mtype_gc);
728 queue_delayed_work(system_power_efficient_wq, &gc->dwork, HZ);
729 }
730
731 static void
mtype_cancel_gc(struct ip_set * set)732 mtype_cancel_gc(struct ip_set *set)
733 {
734 struct htype *h = set->data;
735
736 if (SET_WITH_TIMEOUT(set))
737 disable_delayed_work_sync(&h->gc.dwork);
738 }
739
740 static int
741 mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext,
742 struct ip_set_ext *mext, u32 flags);
743 static int
744 mtype_del(struct ip_set *set, void *value, const struct ip_set_ext *ext,
745 struct ip_set_ext *mext, u32 flags);
746
747 /* Resize a hash: create a new hash table with doubling the hashsize
748 * and inserting the elements to it. Repeat until we succeed or
749 * fail due to memory pressures.
750 */
751 static int
mtype_resize(struct ip_set * set,bool retried)752 mtype_resize(struct ip_set *set, bool retried)
753 {
754 struct htype *h = set->data;
755 struct htable *t, *orig;
756 u8 pos, htable_bits;
757 size_t hsize, dsize = set->dsize;
758 #ifdef IP_SET_HASH_WITH_NETS
759 u8 flags;
760 struct mtype_elem *tmp;
761 #endif
762 struct mtype_elem *data;
763 struct mtype_elem *d;
764 struct hbucket *n, *m;
765 struct list_head *l, *lt;
766 struct mtype_resize_ad *x;
767 u32 i, j, r, nr, key;
768 int ret;
769
770 #ifdef IP_SET_HASH_WITH_NETS
771 tmp = kmalloc(dsize, GFP_KERNEL);
772 if (!tmp)
773 return -ENOMEM;
774 #endif
775 orig = ipset_dereference_bh_nfnl(h->table);
776 htable_bits = orig->htable_bits;
777
778 retry:
779 ret = 0;
780 htable_bits++;
781 if (!htable_bits)
782 goto hbwarn;
783 hsize = htable_size(htable_bits);
784 if (!hsize)
785 goto hbwarn;
786 t = ip_set_alloc(hsize);
787 if (!t) {
788 ret = -ENOMEM;
789 goto out;
790 }
791 t->hregion = ip_set_alloc(ahash_sizeof_regions(htable_bits));
792 if (!t->hregion) {
793 ip_set_free(t);
794 ret = -ENOMEM;
795 goto out;
796 }
797 t->htable_bits = htable_bits;
798 t->maxelem = h->maxelem / ahash_numof_locks(htable_bits);
799 INIT_LIST_HEAD(&t->ad);
800 for (i = 0; i < ahash_numof_locks(htable_bits); i++)
801 spin_lock_init(&t->hregion[i].lock);
802
803 /* There can't be another parallel resizing,
804 * but dumping and kernel side add/del are possible
805 */
806 orig = ipset_dereference_bh_nfnl(h->table);
807 atomic_inc(&orig->uref);
808 spin_lock_bh(&h->gc.lock);
809 orig->resizing = true;
810 spin_unlock_bh(&h->gc.lock);
811 pr_debug("attempt to resize set %s from %u to %u, t %p\n",
812 set->name, orig->htable_bits, htable_bits, orig);
813 for (r = 0; r < ahash_numof_locks(orig->htable_bits); r++) {
814 /* Expire may replace a hbucket with another one */
815 rcu_read_lock_bh();
816 for (i = ahash_bucket_start(r, orig->htable_bits);
817 i < ahash_bucket_end(r, orig->htable_bits); i++) {
818 n = __ipset_dereference(hbucket(orig, i));
819 if (!n)
820 continue;
821 pos = smp_load_acquire(&n->pos);
822 for (j = 0; j < pos; j++) {
823 if (!test_bit_acquire(j, n->used))
824 continue;
825 data = ahash_data(n, j, dsize);
826 if (SET_ELEM_EXPIRED(set, data))
827 continue;
828 #ifdef IP_SET_HASH_WITH_NETS
829 /* We have readers running parallel with us,
830 * so the live data cannot be modified.
831 */
832 flags = 0;
833 memcpy(tmp, data, dsize);
834 data = tmp;
835 mtype_data_reset_flags(data, &flags);
836 #endif
837 key = HKEY(data, h->initval, htable_bits);
838 m = __ipset_dereference(hbucket(t, key));
839 nr = ahash_region(key);
840 if (!m) {
841 m = kzalloc(sizeof(*m) +
842 AHASH_INIT_SIZE * dsize,
843 GFP_ATOMIC);
844 if (!m) {
845 ret = -ENOMEM;
846 goto cleanup;
847 }
848 m->size = AHASH_INIT_SIZE;
849 t->hregion[nr].ext_size +=
850 ext_size(AHASH_INIT_SIZE,
851 dsize);
852 RCU_INIT_POINTER(hbucket(t, key), m);
853 } else if (m->pos >= m->size) {
854 struct hbucket *ht;
855
856 if (m->size >= AHASH_MAX(h)) {
857 ret = -EAGAIN;
858 } else {
859 ht = kzalloc(sizeof(*ht) +
860 (m->size + AHASH_INIT_SIZE)
861 * dsize,
862 GFP_ATOMIC);
863 if (!ht)
864 ret = -ENOMEM;
865 }
866 if (ret < 0)
867 goto cleanup;
868 memcpy(ht, m, sizeof(struct hbucket) +
869 m->size * dsize);
870 ht->size = m->size + AHASH_INIT_SIZE;
871 t->hregion[nr].ext_size +=
872 ext_size(AHASH_INIT_SIZE,
873 dsize);
874 kfree(m);
875 m = ht;
876 RCU_INIT_POINTER(hbucket(t, key), ht);
877 }
878 d = ahash_data(m, m->pos, dsize);
879 memcpy(d, data, dsize);
880 set_bit(m->pos++, m->used);
881 t->hregion[nr].elements++;
882 #ifdef IP_SET_HASH_WITH_NETS
883 mtype_data_reset_flags(d, &flags);
884 #endif
885 }
886 }
887 rcu_read_unlock_bh();
888 }
889
890 /* There can't be any other writer. */
891 rcu_assign_pointer(h->table, t);
892
893 /* Give time to other readers of the set */
894 synchronize_rcu();
895
896 pr_debug("set %s resized from %u (%p) to %u (%p)\n", set->name,
897 orig->htable_bits, orig, t->htable_bits, t);
898 /* Add/delete elements processed by the SET target during resize.
899 * Kernel-side add cannot trigger a resize and userspace actions
900 * are serialized by the mutex.
901 */
902 list_for_each_safe(l, lt, &orig->ad) {
903 x = list_entry(l, struct mtype_resize_ad, list);
904 if (x->ad == IPSET_ADD) {
905 mtype_add(set, &x->d, &x->ext, &x->mext, x->flags);
906 } else {
907 mtype_del(set, &x->d, NULL, NULL, 0);
908 }
909 list_del(l);
910 kfree(l);
911 }
912 /* If there's nobody else using the table, destroy it */
913 if (atomic_dec_and_test(&orig->uref)) {
914 pr_debug("Table destroy by resize %p\n", orig);
915 mtype_ahash_destroy(set, orig, false);
916 }
917
918 out:
919 #ifdef IP_SET_HASH_WITH_NETS
920 kfree(tmp);
921 #endif
922 return ret;
923
924 cleanup:
925 rcu_read_unlock_bh();
926 spin_lock_bh(&h->gc.lock);
927 orig->resizing = false;
928 spin_unlock_bh(&h->gc.lock);
929 /* Make sure parallel readers see that orig->resizing is false
930 * before we decrement uref */
931 synchronize_rcu();
932 atomic_dec(&orig->uref);
933 mtype_ahash_destroy(set, t, false);
934 if (ret == -EAGAIN)
935 goto retry;
936
937 /* Cleanup the backlog of ADD/DEL elements */
938 spin_lock_bh(&set->lock);
939 list_for_each_safe(l, lt, &orig->ad) {
940 list_del(l);
941 kfree(l);
942 }
943 spin_unlock_bh(&set->lock);
944 goto out;
945
946 hbwarn:
947 /* In case we have plenty of memory :-) */
948 pr_warn("Cannot increase the hashsize of set %s further\n", set->name);
949 ret = -IPSET_ERR_HASH_FULL;
950 goto out;
951 }
952
953 /* Get the current number of elements and ext_size in the set */
954 static void
mtype_ext_size(struct ip_set * set,u32 * elements,size_t * ext_size)955 mtype_ext_size(struct ip_set *set, u32 *elements, size_t *ext_size)
956 {
957 struct htype *h = set->data;
958 const struct htable *t;
959 struct hbucket *n;
960 struct mtype_elem *data;
961 u32 i, j, r;
962 u8 pos;
963
964 t = rcu_dereference_bh(h->table);
965 for (r = 0; r < ahash_numof_locks(t->htable_bits); r++) {
966 for (i = ahash_bucket_start(r, t->htable_bits);
967 i < ahash_bucket_end(r, t->htable_bits); i++) {
968 n = rcu_dereference_bh(hbucket(t, i));
969 if (!n)
970 continue;
971 pos = smp_load_acquire(&n->pos);
972 for (j = 0; j < pos; j++) {
973 if (!test_bit_acquire(j, n->used))
974 continue;
975 data = ahash_data(n, j, set->dsize);
976 if (!SET_ELEM_EXPIRED(set, data))
977 (*elements)++;
978 }
979 }
980 *ext_size += t->hregion[r].ext_size;
981 }
982 }
983
984 /* Add an element to a hash and update the internal counters when succeeded,
985 * otherwise report the proper error code.
986 */
987 static int
mtype_add(struct ip_set * set,void * value,const struct ip_set_ext * ext,struct ip_set_ext * mext,u32 flags)988 mtype_add(struct ip_set *set, void *value, const struct ip_set_ext *ext,
989 struct ip_set_ext *mext, u32 flags)
990 {
991 struct htype *h = set->data;
992 struct htable *t;
993 const struct mtype_elem *d = value;
994 struct mtype_elem *data;
995 struct hbucket *n, *old = ERR_PTR(-ENOENT);
996 int i, j = -1, ret;
997 bool flag_exist = flags & IPSET_FLAG_EXIST;
998 bool deleted = false, forceadd = false, reuse = false;
999 u32 r, key, multi = 0, elements, maxelem;
1000 u8 npos = 0;
1001
1002 rcu_read_lock_bh();
1003 t = rcu_dereference_bh(h->table);
1004 key = HKEY(value, h->initval, t->htable_bits);
1005 r = ahash_region(key);
1006 atomic_inc(&t->uref);
1007 rcu_read_unlock_bh();
1008 elements = t->hregion[r].elements;
1009 maxelem = t->maxelem;
1010 if (elements >= maxelem) {
1011 u32 e;
1012 if (SET_WITH_TIMEOUT(set))
1013 mtype_gc_do(set, h, t, r);
1014 maxelem = h->maxelem;
1015 elements = 0;
1016 for (e = 0; e < ahash_numof_locks(t->htable_bits); e++)
1017 elements += t->hregion[e].elements;
1018 if (elements >= maxelem && SET_WITH_FORCEADD(set))
1019 forceadd = true;
1020 }
1021
1022 spin_lock_bh(&t->hregion[r].lock);
1023 n = rcu_dereference_bh(hbucket(t, key));
1024 if (!n) {
1025 if (forceadd || elements >= maxelem)
1026 goto set_full;
1027 old = NULL;
1028 n = kzalloc(sizeof(*n) + AHASH_INIT_SIZE * set->dsize,
1029 GFP_ATOMIC);
1030 if (!n) {
1031 ret = -ENOMEM;
1032 goto unlock;
1033 }
1034 n->size = AHASH_INIT_SIZE;
1035 t->hregion[r].ext_size +=
1036 ext_size(AHASH_INIT_SIZE, set->dsize);
1037 goto copy_elem;
1038 }
1039 npos = smp_load_acquire(&n->pos);
1040 for (i = 0; i < npos; i++) {
1041 if (!test_bit(i, n->used)) {
1042 /* Reuse first deleted entry */
1043 if (j == -1) {
1044 deleted = reuse = true;
1045 j = i;
1046 }
1047 continue;
1048 }
1049 data = ahash_data(n, i, set->dsize);
1050 if (mtype_data_equal(data, d, &multi)) {
1051 if (flag_exist || SET_ELEM_EXPIRED(set, data)) {
1052 /* Just the extensions could be overwritten */
1053 j = i;
1054 goto overwrite_extensions;
1055 }
1056 ret = -IPSET_ERR_EXIST;
1057 goto unlock;
1058 }
1059 /* Reuse first timed out entry */
1060 if (SET_ELEM_EXPIRED(set, data) && j == -1) {
1061 j = i;
1062 reuse = true;
1063 }
1064 }
1065 if (reuse || forceadd) {
1066 if (j == -1)
1067 j = 0;
1068 data = ahash_data(n, j, set->dsize);
1069 if (!deleted) {
1070 mtype_del_cidr_all(set, h, data);
1071 ip_set_ext_destroy(set, data);
1072 t->hregion[r].elements--;
1073 }
1074 goto copy_data;
1075 }
1076 if (elements >= maxelem)
1077 goto set_full;
1078 /* Create a new slot */
1079 if (npos >= n->size) {
1080 #ifdef IP_SET_HASH_WITH_MULTI
1081 if (h->bucketsize >= AHASH_MAX_TUNED)
1082 goto set_full;
1083 else if (h->bucketsize <= multi)
1084 h->bucketsize += AHASH_INIT_SIZE;
1085 #endif
1086 if (n->size >= AHASH_MAX(h)) {
1087 /* Trigger rehashing */
1088 mtype_data_next(&h->next, d);
1089 ret = -EAGAIN;
1090 goto resize;
1091 }
1092 old = n;
1093 n = kzalloc(sizeof(*n) +
1094 (old->size + AHASH_INIT_SIZE) * set->dsize,
1095 GFP_ATOMIC);
1096 if (!n) {
1097 ret = -ENOMEM;
1098 goto unlock;
1099 }
1100 memcpy(n, old, sizeof(struct hbucket) +
1101 old->size * set->dsize);
1102 n->size = old->size + AHASH_INIT_SIZE;
1103 t->hregion[r].ext_size +=
1104 ext_size(AHASH_INIT_SIZE, set->dsize);
1105 }
1106
1107 copy_elem:
1108 j = npos++;
1109 data = ahash_data(n, j, set->dsize);
1110 copy_data:
1111 t->hregion[r].elements++;
1112 #ifdef IP_SET_HASH_WITH_NETS
1113 for (i = 0; i < IPSET_NET_COUNT; i++)
1114 mtype_add_cidr(set, h, DCIDR_GET(d->cidr, i), i);
1115 #endif
1116 memcpy(data, d, sizeof(struct mtype_elem));
1117 overwrite_extensions:
1118 #ifdef IP_SET_HASH_WITH_NETS
1119 mtype_data_set_flags(data, flags);
1120 #endif
1121 if (SET_WITH_COUNTER(set))
1122 ip_set_init_counter(ext_counter(data, set), ext);
1123 if (SET_WITH_COMMENT(set) && !ext->target)
1124 ip_set_init_comment(set, ext_comment(data, set), ext);
1125 if (SET_WITH_SKBINFO(set))
1126 ip_set_init_skbinfo(ext_skbinfo(data, set), ext);
1127 /* Must come last for the case when timed out entry is reused */
1128 if (SET_WITH_TIMEOUT(set))
1129 ip_set_timeout_set(ext_timeout(data, set), ext->timeout);
1130 smp_mb__before_atomic();
1131 /* Ensure all data writes are visible before updating position */
1132 smp_store_release(&n->pos, npos);
1133 set_bit(j, n->used);
1134 if (old != ERR_PTR(-ENOENT)) {
1135 rcu_assign_pointer(hbucket(t, key), n);
1136 if (old)
1137 kfree_rcu(old, rcu);
1138 }
1139 ret = 0;
1140 resize:
1141 spin_unlock_bh(&t->hregion[r].lock);
1142 if (t->resizing && ext && ext->target) {
1143 /* Resize is in process and kernel side add, save values */
1144 struct mtype_resize_ad *x;
1145
1146 x = kzalloc_obj(struct mtype_resize_ad, GFP_ATOMIC);
1147 if (!x)
1148 /* Don't bother */
1149 goto out;
1150 x->ad = IPSET_ADD;
1151 memcpy(&x->d, value, sizeof(struct mtype_elem));
1152 memcpy(&x->ext, ext, sizeof(struct ip_set_ext));
1153 memcpy(&x->mext, mext, sizeof(struct ip_set_ext));
1154 x->flags = flags;
1155 spin_lock_bh(&set->lock);
1156 list_add_tail(&x->list, &t->ad);
1157 spin_unlock_bh(&set->lock);
1158 }
1159 goto out;
1160
1161 set_full:
1162 if (net_ratelimit())
1163 pr_warn("Set %s is full, maxelem %u reached\n",
1164 set->name, maxelem);
1165 ret = -IPSET_ERR_HASH_FULL;
1166 unlock:
1167 spin_unlock_bh(&t->hregion[r].lock);
1168 out:
1169 if (atomic_dec_and_test(&t->uref) && t->resizing) {
1170 pr_debug("Table destroy after resize by add: %p\n", t);
1171 mtype_ahash_destroy(set, t, false);
1172 }
1173 return ret;
1174 }
1175
1176 /* Delete an element from the hash and free up space if possible.
1177 */
1178 static int
mtype_del(struct ip_set * set,void * value,const struct ip_set_ext * ext,struct ip_set_ext * mext,u32 flags)1179 mtype_del(struct ip_set *set, void *value, const struct ip_set_ext *ext,
1180 struct ip_set_ext *mext, u32 flags)
1181 {
1182 struct htype *h = set->data;
1183 struct htable *t;
1184 const struct mtype_elem *d = value;
1185 struct mtype_elem *data;
1186 struct hbucket *n;
1187 struct mtype_resize_ad *x = NULL;
1188 int i, j, k, r, ret = -IPSET_ERR_EXIST;
1189 u32 key, multi = 0;
1190 size_t dsize = set->dsize;
1191 u8 pos;
1192
1193 /* Userspace add and resize is excluded by the mutex.
1194 * Kernespace add does not trigger resize.
1195 */
1196 rcu_read_lock_bh();
1197 t = rcu_dereference_bh(h->table);
1198 key = HKEY(value, h->initval, t->htable_bits);
1199 r = ahash_region(key);
1200 atomic_inc(&t->uref);
1201 rcu_read_unlock_bh();
1202
1203 spin_lock_bh(&t->hregion[r].lock);
1204 n = rcu_dereference_bh(hbucket(t, key));
1205 if (!n)
1206 goto out;
1207 pos = smp_load_acquire(&n->pos);
1208 for (i = 0, k = 0; i < pos; i++) {
1209 if (!test_bit(i, n->used)) {
1210 k++;
1211 continue;
1212 }
1213 data = ahash_data(n, i, dsize);
1214 if (!mtype_data_equal(data, d, &multi))
1215 continue;
1216 if (SET_ELEM_EXPIRED(set, data))
1217 goto out;
1218
1219 ret = 0;
1220 clear_bit(i, n->used);
1221 smp_mb__after_atomic();
1222 if (i + 1 == pos)
1223 smp_store_release(&n->pos, --pos);
1224 t->hregion[r].elements--;
1225 mtype_del_cidr_all(set, h, d);
1226 ip_set_ext_destroy(set, data);
1227
1228 if (t->resizing && ext && ext->target) {
1229 /* Resize is in process and kernel side del,
1230 * save values
1231 */
1232 x = kzalloc_obj(struct mtype_resize_ad, GFP_ATOMIC);
1233 if (x) {
1234 x->ad = IPSET_DEL;
1235 memcpy(&x->d, value,
1236 sizeof(struct mtype_elem));
1237 x->flags = flags;
1238 }
1239 }
1240 for (; i < pos; i++) {
1241 if (!test_bit(i, n->used))
1242 k++;
1243 }
1244 if (k == pos) {
1245 t->hregion[r].ext_size -= ext_size(n->size, dsize);
1246 rcu_assign_pointer(hbucket(t, key), NULL);
1247 kfree_rcu(n, rcu);
1248 } else if (k >= AHASH_INIT_SIZE) {
1249 struct hbucket *tmp = kzalloc(sizeof(*tmp) +
1250 (n->size - AHASH_INIT_SIZE) * dsize,
1251 GFP_ATOMIC);
1252 if (!tmp)
1253 goto out;
1254 tmp->size = n->size - AHASH_INIT_SIZE;
1255 for (j = 0, k = 0; j < pos; j++) {
1256 if (!test_bit(j, n->used))
1257 continue;
1258 data = ahash_data(n, j, dsize);
1259 memcpy(tmp->value + k * dsize, data, dsize);
1260 set_bit(k, tmp->used);
1261 k++;
1262 }
1263 tmp->pos = k;
1264 t->hregion[r].ext_size -=
1265 ext_size(AHASH_INIT_SIZE, dsize);
1266 rcu_assign_pointer(hbucket(t, key), tmp);
1267 kfree_rcu(n, rcu);
1268 }
1269 goto out;
1270 }
1271
1272 out:
1273 spin_unlock_bh(&t->hregion[r].lock);
1274 if (x) {
1275 spin_lock_bh(&set->lock);
1276 list_add(&x->list, &t->ad);
1277 spin_unlock_bh(&set->lock);
1278 }
1279 if (atomic_dec_and_test(&t->uref) && t->resizing) {
1280 pr_debug("Table destroy after resize by del: %p\n", t);
1281 mtype_ahash_destroy(set, t, false);
1282 }
1283 return ret;
1284 }
1285
1286 static int
mtype_data_match(struct mtype_elem * data,const struct ip_set_ext * ext,struct ip_set_ext * mext,struct ip_set * set,u32 flags)1287 mtype_data_match(struct mtype_elem *data, const struct ip_set_ext *ext,
1288 struct ip_set_ext *mext, struct ip_set *set, u32 flags)
1289 {
1290 if (!ip_set_match_extensions(set, ext, mext, flags, data))
1291 return 0;
1292 /* nomatch entries return -ENOTEMPTY */
1293 return mtype_do_data_match(data);
1294 }
1295
1296 #ifdef IP_SET_HASH_WITH_NETS
1297 /* Special test function which takes into account the different network
1298 * sizes added to the set
1299 */
1300 static int
mtype_test_cidrs(struct ip_set * set,struct mtype_elem * d,const struct ip_set_ext * ext,struct ip_set_ext * mext,u32 flags)1301 mtype_test_cidrs(struct ip_set *set, struct mtype_elem *d,
1302 const struct ip_set_ext *ext,
1303 struct ip_set_ext *mext, u32 flags)
1304 {
1305 struct htype *h = set->data;
1306 struct htable *t = rcu_dereference_bh(h->table);
1307 struct net_prefixes *nets0;
1308 struct hbucket *n;
1309 struct mtype_elem *data;
1310 #if IPSET_NET_COUNT == 2
1311 struct net_prefixes *nets1;
1312 struct mtype_elem orig = *d;
1313 unsigned int seq1;
1314 int ret, i, j, k;
1315 #else
1316 int ret, i, j;
1317 #endif
1318 unsigned int seq0;
1319 u32 key, multi;
1320 u8 pos;
1321
1322 pr_debug("test by nets\n");
1323 rcu_read_lock_bh();
1324 retry:
1325 multi = 0;
1326 nets0 = rcu_dereference_bh(h->rnets[0]);
1327 seq0 = read_seqcount_begin(&nets0->seq);
1328 #if IPSET_NET_COUNT == 2
1329 nets1 = rcu_dereference_bh(h->rnets[1]);
1330 seq1 = read_seqcount_begin(&nets1->seq);
1331 #endif
1332 for (j = 0; j < nets0->len && !multi; j++) {
1333 struct net_prefix p0 = READ_ONCE(nets0->nets[j]);
1334
1335 if (!p0.count)
1336 continue;
1337 #if IPSET_NET_COUNT == 2
1338 mtype_data_reset_elem(d, &orig);
1339 mtype_data_netmask(d, p0.cidr, false);
1340 for (k = 0; k < nets1->len && !multi; k++) {
1341 struct net_prefix p1 = READ_ONCE(nets1->nets[k]);
1342
1343 if (!p1.count)
1344 continue;
1345 mtype_data_netmask(d, p1.cidr, true);
1346 #else
1347 mtype_data_netmask(d, p0.cidr);
1348 #endif
1349 key = HKEY(d, h->initval, t->htable_bits);
1350 n = rcu_dereference_bh(hbucket(t, key));
1351 if (!n)
1352 continue;
1353 pos = smp_load_acquire(&n->pos);
1354 for (i = 0; i < pos; i++) {
1355 if (!test_bit_acquire(i, n->used))
1356 continue;
1357 data = ahash_data(n, i, set->dsize);
1358 if (!mtype_data_equal(data, d, &multi))
1359 continue;
1360 ret = mtype_data_match(data, ext, mext, set, flags);
1361 if (ret != 0)
1362 goto unlock;
1363 #ifdef IP_SET_HASH_WITH_MULTI
1364 /* No match, reset multiple match flag */
1365 multi = 0;
1366 #endif
1367 }
1368 #if IPSET_NET_COUNT == 2
1369 }
1370 #endif
1371 }
1372 ret = 0;
1373 unlock:
1374 if (read_seqcount_retry(&nets0->seq, seq0))
1375 goto retry;
1376 #if IPSET_NET_COUNT == 2
1377 if (read_seqcount_retry(&nets1->seq, seq1))
1378 goto retry;
1379 #endif
1380 rcu_read_unlock_bh();
1381 return ret;
1382 }
1383 #endif
1384
1385 /* Test whether the element is added to the set */
1386 static int
mtype_test(struct ip_set * set,void * value,const struct ip_set_ext * ext,struct ip_set_ext * mext,u32 flags)1387 mtype_test(struct ip_set *set, void *value, const struct ip_set_ext *ext,
1388 struct ip_set_ext *mext, u32 flags)
1389 {
1390 struct htype *h = set->data;
1391 struct htable *t;
1392 struct mtype_elem *d = value;
1393 struct hbucket *n;
1394 struct mtype_elem *data;
1395 int i, ret = 0;
1396 u32 key, multi = 0;
1397 u8 pos;
1398
1399 rcu_read_lock_bh();
1400 t = rcu_dereference_bh(h->table);
1401 #ifdef IP_SET_HASH_WITH_NETS
1402 /* If we test an IP address and not a network address,
1403 * try all possible network sizes
1404 */
1405 for (i = 0; i < IPSET_NET_COUNT; i++)
1406 if (DCIDR_GET(d->cidr, i) != HOST_MASK)
1407 break;
1408 if (i == IPSET_NET_COUNT) {
1409 ret = mtype_test_cidrs(set, d, ext, mext, flags);
1410 goto out;
1411 }
1412 #endif
1413
1414 key = HKEY(d, h->initval, t->htable_bits);
1415 n = rcu_dereference_bh(hbucket(t, key));
1416 if (!n) {
1417 ret = 0;
1418 goto out;
1419 }
1420 pos = smp_load_acquire(&n->pos);
1421 for (i = 0; i < pos; i++) {
1422 if (!test_bit_acquire(i, n->used))
1423 continue;
1424 data = ahash_data(n, i, set->dsize);
1425 if (!mtype_data_equal(data, d, &multi))
1426 continue;
1427 ret = mtype_data_match(data, ext, mext, set, flags);
1428 if (ret != 0)
1429 goto out;
1430 }
1431 out:
1432 rcu_read_unlock_bh();
1433 return ret;
1434 }
1435
mtype_hash_size(const struct htype * h)1436 static u32 mtype_hash_size(const struct htype *h)
1437 {
1438 const struct htable *t;
1439 u8 htable_bits;
1440
1441 rcu_read_lock();
1442 t = rcu_dereference(h->table);
1443 htable_bits = t->htable_bits;
1444 rcu_read_unlock();
1445
1446 return jhash_size(htable_bits);
1447 }
1448
mtype_bucket_size(const struct htype * h)1449 static u32 mtype_bucket_size(const struct htype *h)
1450 {
1451 return h->bucketsize;
1452 }
1453
1454 /* Reply a HEADER request: fill out the header part of the set */
1455 static int
mtype_head(struct ip_set * set,struct sk_buff * skb)1456 mtype_head(struct ip_set *set, struct sk_buff *skb)
1457 {
1458 struct htype *h = set->data;
1459 const struct htable *t;
1460 struct nlattr *nested;
1461 size_t memsize;
1462 u32 elements = 0;
1463 size_t ext_size = 0;
1464
1465 rcu_read_lock_bh();
1466 t = rcu_dereference_bh(h->table);
1467 mtype_ext_size(set, &elements, &ext_size);
1468 memsize = mtype_ahash_memsize(h, t) + ext_size + atomic64_read(&set->ext_size);
1469 rcu_read_unlock_bh();
1470
1471 nested = nla_nest_start(skb, IPSET_ATTR_DATA);
1472 if (!nested)
1473 goto nla_put_failure;
1474
1475 if (nla_put_net32(skb, IPSET_ATTR_HASHSIZE, htonl(mtype_hash_size(h))))
1476 goto nla_put_failure;
1477 if (nla_put_net32(skb, IPSET_ATTR_MAXELEM, htonl(h->maxelem)))
1478 goto nla_put_failure;
1479 #ifdef IP_SET_HASH_WITH_BITMASK
1480 /* if netmask is set to anything other than HOST_MASK we know that the user supplied netmask
1481 * and not bitmask. These two are mutually exclusive. */
1482 if (h->netmask == HOST_MASK && !nf_inet_addr_cmp(&onesmask, &h->bitmask)) {
1483 if (set->family == NFPROTO_IPV4) {
1484 if (nla_put_ipaddr4(skb, IPSET_ATTR_BITMASK, h->bitmask.ip))
1485 goto nla_put_failure;
1486 } else if (set->family == NFPROTO_IPV6) {
1487 if (nla_put_ipaddr6(skb, IPSET_ATTR_BITMASK, &h->bitmask.in6))
1488 goto nla_put_failure;
1489 }
1490 }
1491 #endif
1492 #ifdef IP_SET_HASH_WITH_NETMASK
1493 if (h->netmask != HOST_MASK && nla_put_u8(skb, IPSET_ATTR_NETMASK, h->netmask))
1494 goto nla_put_failure;
1495 #endif
1496 #ifdef IP_SET_HASH_WITH_MARKMASK
1497 if (nla_put_u32(skb, IPSET_ATTR_MARKMASK, h->markmask))
1498 goto nla_put_failure;
1499 #endif
1500 if (set->flags & IPSET_CREATE_FLAG_BUCKETSIZE) {
1501 if (nla_put_u8(skb, IPSET_ATTR_BUCKETSIZE, mtype_bucket_size(h)))
1502 goto nla_put_failure;
1503 if (nla_put_net32(skb, IPSET_ATTR_INITVAL, htonl(h->initval)))
1504 goto nla_put_failure;
1505 }
1506 if (nla_put_net32(skb, IPSET_ATTR_REFERENCES, htonl(set->ref)) ||
1507 nla_put_net32(skb, IPSET_ATTR_MEMSIZE, htonl(memsize)) ||
1508 nla_put_net32(skb, IPSET_ATTR_ELEMENTS, htonl(elements)))
1509 goto nla_put_failure;
1510 if (unlikely(ip_set_put_flags(skb, set)))
1511 goto nla_put_failure;
1512 nla_nest_end(skb, nested);
1513
1514 return 0;
1515 nla_put_failure:
1516 return -EMSGSIZE;
1517 }
1518
1519 /* Make possible to run dumping parallel with resizing */
1520 static void
mtype_uref(struct ip_set * set,struct netlink_callback * cb,bool start)1521 mtype_uref(struct ip_set *set, struct netlink_callback *cb, bool start)
1522 {
1523 struct htype *h = set->data;
1524 struct htable *t;
1525
1526 if (start) {
1527 rcu_read_lock_bh();
1528 t = ipset_dereference_bh_nfnl(h->table);
1529 atomic_inc(&t->uref);
1530 cb->args[IPSET_CB_PRIVATE] = (unsigned long)t;
1531 rcu_read_unlock_bh();
1532 } else if (cb->args[IPSET_CB_PRIVATE]) {
1533 t = (struct htable *)cb->args[IPSET_CB_PRIVATE];
1534 if (atomic_dec_and_test(&t->uref) && t->resizing) {
1535 pr_debug("Table destroy after resize "
1536 " by dump: %p\n", t);
1537 mtype_ahash_destroy(set, t, false);
1538 }
1539 cb->args[IPSET_CB_PRIVATE] = 0;
1540 }
1541 }
1542
1543 /* Reply a LIST/SAVE request: dump the elements of the specified set */
1544 static int
mtype_list(const struct ip_set * set,struct sk_buff * skb,struct netlink_callback * cb)1545 mtype_list(const struct ip_set *set,
1546 struct sk_buff *skb, struct netlink_callback *cb)
1547 {
1548 const struct htable *t;
1549 struct nlattr *atd, *nested;
1550 const struct hbucket *n;
1551 const struct mtype_elem *e;
1552 u32 first = cb->args[IPSET_CB_ARG0];
1553 /* We assume that one hash bucket fills into one page */
1554 void *incomplete;
1555 int i, ret = 0;
1556 u8 pos;
1557
1558 atd = nla_nest_start(skb, IPSET_ATTR_ADT);
1559 if (!atd)
1560 return -EMSGSIZE;
1561
1562 pr_debug("list hash set %s\n", set->name);
1563 t = (const struct htable *)cb->args[IPSET_CB_PRIVATE];
1564 /* Expire may replace a hbucket with another one */
1565 rcu_read_lock();
1566 for (; cb->args[IPSET_CB_ARG0] < jhash_size(t->htable_bits);
1567 cb->args[IPSET_CB_ARG0]++) {
1568 cond_resched_rcu();
1569 incomplete = skb_tail_pointer(skb);
1570 n = rcu_dereference(hbucket(t, cb->args[IPSET_CB_ARG0]));
1571 pr_debug("cb->arg bucket: %lu, t %p n %p\n",
1572 cb->args[IPSET_CB_ARG0], t, n);
1573 if (!n)
1574 continue;
1575 pos = smp_load_acquire(&n->pos);
1576 for (i = 0; i < pos; i++) {
1577 if (!test_bit_acquire(i, n->used))
1578 continue;
1579 e = ahash_data(n, i, set->dsize);
1580 if (SET_ELEM_EXPIRED(set, e))
1581 continue;
1582 pr_debug("list hash %lu hbucket %p i %u, data %p\n",
1583 cb->args[IPSET_CB_ARG0], n, i, e);
1584 nested = nla_nest_start(skb, IPSET_ATTR_DATA);
1585 if (!nested) {
1586 if (cb->args[IPSET_CB_ARG0] == first) {
1587 nla_nest_cancel(skb, atd);
1588 ret = -EMSGSIZE;
1589 goto out;
1590 }
1591 goto nla_put_failure;
1592 }
1593 if (mtype_data_list(skb, e))
1594 goto nla_put_failure;
1595 if (ip_set_put_extensions(skb, set, e, true))
1596 goto nla_put_failure;
1597 nla_nest_end(skb, nested);
1598 }
1599 }
1600 nla_nest_end(skb, atd);
1601 /* Set listing finished */
1602 cb->args[IPSET_CB_ARG0] = 0;
1603
1604 goto out;
1605
1606 nla_put_failure:
1607 nlmsg_trim(skb, incomplete);
1608 if (unlikely(first == cb->args[IPSET_CB_ARG0])) {
1609 pr_warn("Can't list set %s: one bucket does not fit into a message. Please report it!\n",
1610 set->name);
1611 cb->args[IPSET_CB_ARG0] = 0;
1612 ret = -EMSGSIZE;
1613 } else {
1614 nla_nest_end(skb, atd);
1615 }
1616 out:
1617 rcu_read_unlock();
1618 return ret;
1619 }
1620
1621 static int
1622 IPSET_TOKEN(MTYPE, _kadt)(struct ip_set *set, const struct sk_buff *skb,
1623 const struct xt_action_param *par,
1624 enum ipset_adt adt, struct ip_set_adt_opt *opt);
1625
1626 static int
1627 IPSET_TOKEN(MTYPE, _uadt)(struct ip_set *set, struct nlattr *tb[],
1628 enum ipset_adt adt, u32 *lineno, u32 flags,
1629 bool retried);
1630
1631 static const struct ip_set_type_variant mtype_variant = {
1632 .kadt = mtype_kadt,
1633 .uadt = mtype_uadt,
1634 .adt = {
1635 [IPSET_ADD] = mtype_add,
1636 [IPSET_DEL] = mtype_del,
1637 [IPSET_TEST] = mtype_test,
1638 },
1639 .destroy = mtype_destroy,
1640 .flush = mtype_flush,
1641 .head = mtype_head,
1642 .list = mtype_list,
1643 .uref = mtype_uref,
1644 .resize = mtype_resize,
1645 .same_set = mtype_same_set,
1646 .cancel_gc = mtype_cancel_gc,
1647 .region_lock = true,
1648 };
1649
1650 #ifdef IP_SET_EMIT_CREATE
1651 static int
IPSET_TOKEN(HTYPE,_create)1652 IPSET_TOKEN(HTYPE, _create)(struct net *net, struct ip_set *set,
1653 struct nlattr *tb[], u32 flags)
1654 {
1655 u32 hashsize = IPSET_DEFAULT_HASHSIZE, maxelem = IPSET_DEFAULT_MAXELEM;
1656 #ifdef IP_SET_HASH_WITH_MARKMASK
1657 u32 markmask;
1658 #endif
1659 u8 hbits;
1660 #if defined(IP_SET_HASH_WITH_NETMASK) || defined(IP_SET_HASH_WITH_BITMASK)
1661 int ret __attribute__((unused)) = 0;
1662 u8 netmask = set->family == NFPROTO_IPV4 ? 32 : 128;
1663 union nf_inet_addr bitmask = onesmask;
1664 #endif
1665 #ifdef IP_SET_HASH_WITH_NETS
1666 struct net_prefixes *nets;
1667 #endif
1668 size_t hsize;
1669 struct htype *h;
1670 struct htable *t;
1671 u32 i;
1672
1673 pr_debug("Create set %s with family %s\n",
1674 set->name, set->family == NFPROTO_IPV4 ? "inet" : "inet6");
1675
1676 #ifdef IP_SET_PROTO_UNDEF
1677 if (set->family != NFPROTO_UNSPEC)
1678 return -IPSET_ERR_INVALID_FAMILY;
1679 #else
1680 if (!(set->family == NFPROTO_IPV4 || set->family == NFPROTO_IPV6))
1681 return -IPSET_ERR_INVALID_FAMILY;
1682 #endif
1683
1684 if (unlikely(!ip_set_optattr_netorder(tb, IPSET_ATTR_HASHSIZE) ||
1685 !ip_set_optattr_netorder(tb, IPSET_ATTR_MAXELEM) ||
1686 !ip_set_optattr_netorder(tb, IPSET_ATTR_TIMEOUT) ||
1687 !ip_set_optattr_netorder(tb, IPSET_ATTR_CADT_FLAGS)))
1688 return -IPSET_ERR_PROTOCOL;
1689
1690 #ifdef IP_SET_HASH_WITH_MARKMASK
1691 /* Separated condition in order to avoid directive in argument list */
1692 if (unlikely(!ip_set_optattr_netorder(tb, IPSET_ATTR_MARKMASK)))
1693 return -IPSET_ERR_PROTOCOL;
1694
1695 markmask = 0xffffffff;
1696 if (tb[IPSET_ATTR_MARKMASK]) {
1697 markmask = ntohl(nla_get_be32(tb[IPSET_ATTR_MARKMASK]));
1698 if (markmask == 0)
1699 return -IPSET_ERR_INVALID_MARKMASK;
1700 }
1701 #endif
1702
1703 #ifdef IP_SET_HASH_WITH_NETMASK
1704 if (tb[IPSET_ATTR_NETMASK]) {
1705 netmask = nla_get_u8(tb[IPSET_ATTR_NETMASK]);
1706
1707 if ((set->family == NFPROTO_IPV4 && netmask > 32) ||
1708 (set->family == NFPROTO_IPV6 && netmask > 128) ||
1709 netmask == 0)
1710 return -IPSET_ERR_INVALID_NETMASK;
1711
1712 /* we convert netmask to bitmask and store it */
1713 if (set->family == NFPROTO_IPV4)
1714 bitmask.ip = ip_set_netmask(netmask);
1715 else
1716 ip6_netmask(&bitmask, netmask);
1717 }
1718 #endif
1719
1720 #ifdef IP_SET_HASH_WITH_BITMASK
1721 if (tb[IPSET_ATTR_BITMASK]) {
1722 /* bitmask and netmask do the same thing, allow only one of these options */
1723 if (tb[IPSET_ATTR_NETMASK])
1724 return -IPSET_ERR_BITMASK_NETMASK_EXCL;
1725
1726 if (set->family == NFPROTO_IPV4) {
1727 ret = ip_set_get_ipaddr4(tb[IPSET_ATTR_BITMASK], &bitmask.ip);
1728 if (ret || !bitmask.ip)
1729 return -IPSET_ERR_INVALID_NETMASK;
1730 } else if (set->family == NFPROTO_IPV6) {
1731 ret = ip_set_get_ipaddr6(tb[IPSET_ATTR_BITMASK], &bitmask);
1732 if (ret || ipv6_addr_any(&bitmask.in6))
1733 return -IPSET_ERR_INVALID_NETMASK;
1734 }
1735
1736 if (nf_inet_addr_cmp(&bitmask, &zeromask))
1737 return -IPSET_ERR_INVALID_NETMASK;
1738 }
1739 #endif
1740
1741 if (tb[IPSET_ATTR_HASHSIZE]) {
1742 hashsize = ip_set_get_h32(tb[IPSET_ATTR_HASHSIZE]);
1743 if (hashsize < IPSET_MIMINAL_HASHSIZE)
1744 hashsize = IPSET_MIMINAL_HASHSIZE;
1745 }
1746
1747 if (tb[IPSET_ATTR_MAXELEM])
1748 maxelem = ip_set_get_h32(tb[IPSET_ATTR_MAXELEM]);
1749
1750 #ifdef IP_SET_PROTO_UNDEF
1751 hsize = sizeof(struct htype);
1752 #else
1753 hsize = set->family == NFPROTO_IPV6 ?
1754 sizeof(struct IPSET_TOKEN(HTYPE, 6)) :
1755 sizeof(struct IPSET_TOKEN(HTYPE, 4));
1756 #endif
1757 h = kzalloc(hsize, GFP_KERNEL);
1758 if (!h)
1759 return -ENOMEM;
1760
1761 /* Compute htable_bits from the user input parameter hashsize.
1762 * Assume that hashsize == 2^htable_bits,
1763 * otherwise round up to the first 2^n value.
1764 */
1765 hbits = fls(hashsize - 1);
1766 hsize = htable_size(hbits);
1767 if (hsize == 0)
1768 goto free_h;
1769 t = ip_set_alloc(hsize);
1770 if (!t)
1771 goto free_h;
1772 t->hregion = ip_set_alloc(ahash_sizeof_regions(hbits));
1773 if (!t->hregion)
1774 goto free_t;
1775 #ifdef IP_SET_HASH_WITH_NETS
1776 for (i = 0; i < IPSET_NET_COUNT; i++) {
1777 nets = kzalloc_obj(*nets);
1778 if (!nets) {
1779 while (i > 0)
1780 kfree(rcu_dereference_raw(h->rnets[--i]));
1781 goto free_hregion;
1782 }
1783 seqcount_spinlock_init(&nets->seq, &set->lock);
1784 RCU_INIT_POINTER(h->rnets[i], nets);
1785 }
1786 #endif
1787 h->gc.set = set;
1788 spin_lock_init(&h->gc.lock);
1789 for (i = 0; i < ahash_numof_locks(hbits); i++)
1790 spin_lock_init(&t->hregion[i].lock);
1791 h->maxelem = maxelem;
1792 #if defined(IP_SET_HASH_WITH_NETMASK) || defined(IP_SET_HASH_WITH_BITMASK)
1793 h->bitmask = bitmask;
1794 h->netmask = netmask;
1795 #endif
1796 #ifdef IP_SET_HASH_WITH_MARKMASK
1797 h->markmask = markmask;
1798 #endif
1799 if (tb[IPSET_ATTR_INITVAL])
1800 h->initval = ntohl(nla_get_be32(tb[IPSET_ATTR_INITVAL]));
1801 else
1802 get_random_bytes(&h->initval, sizeof(h->initval));
1803 h->bucketsize = AHASH_MAX_SIZE;
1804 if (tb[IPSET_ATTR_BUCKETSIZE]) {
1805 h->bucketsize = nla_get_u8(tb[IPSET_ATTR_BUCKETSIZE]);
1806 if (h->bucketsize < AHASH_INIT_SIZE)
1807 h->bucketsize = AHASH_INIT_SIZE;
1808 else if (h->bucketsize > AHASH_MAX_SIZE)
1809 h->bucketsize = AHASH_MAX_SIZE;
1810 else if (h->bucketsize % 2)
1811 h->bucketsize += 1;
1812 }
1813 t->htable_bits = hbits;
1814 t->maxelem = h->maxelem / ahash_numof_locks(hbits);
1815 INIT_LIST_HEAD(&t->ad);
1816 RCU_INIT_POINTER(h->table, t);
1817 set->data = h;
1818
1819 #ifndef IP_SET_PROTO_UNDEF
1820 if (set->family == NFPROTO_IPV4) {
1821 #endif
1822 set->variant = &IPSET_TOKEN(HTYPE, 4_variant);
1823 set->dsize = ip_set_elem_len(set, tb,
1824 sizeof(struct IPSET_TOKEN(HTYPE, 4_elem)),
1825 __alignof__(struct IPSET_TOKEN(HTYPE, 4_elem)));
1826 #ifndef IP_SET_PROTO_UNDEF
1827 } else {
1828 set->variant = &IPSET_TOKEN(HTYPE, 6_variant);
1829 set->dsize = ip_set_elem_len(set, tb,
1830 sizeof(struct IPSET_TOKEN(HTYPE, 6_elem)),
1831 __alignof__(struct IPSET_TOKEN(HTYPE, 6_elem)));
1832 }
1833 #endif
1834 set->timeout = IPSET_NO_TIMEOUT;
1835 if (tb[IPSET_ATTR_TIMEOUT]) {
1836 set->timeout = ip_set_timeout_uget(tb[IPSET_ATTR_TIMEOUT]);
1837 #ifndef IP_SET_PROTO_UNDEF
1838 if (set->family == NFPROTO_IPV4)
1839 #endif
1840 IPSET_TOKEN(HTYPE, 4_gc_init)(&h->gc);
1841 #ifndef IP_SET_PROTO_UNDEF
1842 else
1843 IPSET_TOKEN(HTYPE, 6_gc_init)(&h->gc);
1844 #endif
1845 }
1846 pr_debug("create %s hashsize %u (%u) maxelem %u: %p(%p)\n",
1847 set->name, mtype_hash_size(h),
1848 t->htable_bits, h->maxelem, set->data, t);
1849
1850 return 0;
1851
1852 #ifdef IP_SET_HASH_WITH_NETS
1853 free_hregion:
1854 ip_set_free(t->hregion);
1855 #endif
1856 free_t:
1857 ip_set_free(t);
1858 free_h:
1859 kfree(h);
1860 return -ENOMEM;
1861 }
1862 #endif /* IP_SET_EMIT_CREATE */
1863
1864 #undef HKEY_DATALEN
1865