1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* IP Virtual Server 3 * data structure and functionality definitions 4 */ 5 6 #ifndef _NET_IP_VS_H 7 #define _NET_IP_VS_H 8 9 #include <linux/ip_vs.h> /* definitions shared with userland */ 10 11 #include <asm/types.h> /* for __uXX types */ 12 13 #include <linux/list.h> /* for struct list_head */ 14 #include <linux/rculist_bl.h> /* for struct hlist_bl_head */ 15 #include <linux/spinlock.h> /* for struct rwlock_t */ 16 #include <linux/atomic.h> /* for struct atomic_t */ 17 #include <linux/refcount.h> /* for struct refcount_t */ 18 #include <linux/workqueue.h> 19 20 #include <linux/compiler.h> 21 #include <linux/timer.h> 22 #include <linux/bug.h> 23 24 #include <net/checksum.h> 25 #include <linux/netfilter.h> /* for union nf_inet_addr */ 26 #include <linux/ip.h> 27 #include <linux/ipv6.h> /* for struct ipv6hdr */ 28 #include <net/ipv6.h> 29 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 30 #include <net/netfilter/nf_conntrack.h> 31 #endif 32 #include <net/net_namespace.h> /* Netw namespace */ 33 #include <linux/sched/isolation.h> 34 #include <linux/siphash.h> 35 36 #define IP_VS_HDR_INVERSE 1 37 #define IP_VS_HDR_ICMP 2 38 39 /* Destination Server Config Flags */ 40 #define IP_VS_DEST_CF_AVAILABLE 0x0001 /* server is available */ 41 42 /* conn_tab limits (as per Kconfig) */ 43 #define IP_VS_CONN_TAB_MIN_BITS 8 44 #if BITS_PER_LONG > 32 45 #define IP_VS_CONN_TAB_MAX_BITS 27 46 #else 47 #define IP_VS_CONN_TAB_MAX_BITS 20 48 #endif 49 50 /* conn_max limits */ 51 #if BITS_PER_LONG > 32 52 /* Limit of atomic_t but restricted by roundup_pow_of_two() in ip_vs_core.c */ 53 #define IP_VS_CONN_MAX (1 << 30) 54 #else 55 #define IP_VS_CONN_MAX (1 << 24) 56 #endif 57 58 /* svc_table limits */ 59 #define IP_VS_SVC_TAB_MIN_BITS 4 60 #define IP_VS_SVC_TAB_MAX_BITS 20 61 62 /* Generic access of ipvs struct */ 63 static inline struct netns_ipvs *net_ipvs(struct net* net) 64 { 65 return net->ipvs; 66 } 67 68 /* Connections' size value needed by ip_vs_ctl.c */ 69 extern int ip_vs_conn_tab_size; 70 71 struct ip_vs_iphdr { 72 int hdr_flags; /* ipvs flags */ 73 __u32 off; /* Where IP or IPv4 header starts */ 74 __u32 len; /* IPv4 simply where L4 starts 75 * IPv6 where L4 Transport Header starts */ 76 __u16 fragoffs; /* IPv6 fragment offset, 0 if first frag (or not frag)*/ 77 __s16 protocol; 78 __s32 flags; 79 union nf_inet_addr saddr; 80 union nf_inet_addr daddr; 81 }; 82 83 static inline void *frag_safe_skb_hp(const struct sk_buff *skb, int offset, 84 int len, void *buffer) 85 { 86 return skb_header_pointer(skb, offset, len, buffer); 87 } 88 89 /* This function handles filling *ip_vs_iphdr, both for IPv4 and IPv6. 90 * IPv6 requires some extra work, as finding proper header position, 91 * depend on the IPv6 extension headers. 92 */ 93 static inline int 94 ip_vs_fill_iph_skb_off(int af, const struct sk_buff *skb, int offset, 95 int hdr_flags, struct ip_vs_iphdr *iphdr) 96 { 97 iphdr->hdr_flags = hdr_flags; 98 iphdr->off = offset; 99 100 #ifdef CONFIG_IP_VS_IPV6 101 if (af == AF_INET6) { 102 struct ipv6hdr _iph; 103 const struct ipv6hdr *iph = skb_header_pointer( 104 skb, offset, sizeof(_iph), &_iph); 105 if (!iph) 106 return 0; 107 108 iphdr->saddr.in6 = iph->saddr; 109 iphdr->daddr.in6 = iph->daddr; 110 /* ipv6_find_hdr() updates len, flags */ 111 iphdr->len = offset; 112 iphdr->flags = 0; 113 iphdr->protocol = ipv6_find_hdr(skb, &iphdr->len, -1, 114 &iphdr->fragoffs, 115 &iphdr->flags); 116 if (iphdr->protocol < 0) 117 return 0; 118 } else 119 #endif 120 { 121 struct iphdr _iph; 122 const struct iphdr *iph = skb_header_pointer( 123 skb, offset, sizeof(_iph), &_iph); 124 if (!iph) 125 return 0; 126 127 iphdr->len = offset + iph->ihl * 4; 128 iphdr->fragoffs = 0; 129 iphdr->protocol = iph->protocol; 130 iphdr->saddr.ip = iph->saddr; 131 iphdr->daddr.ip = iph->daddr; 132 } 133 134 return 1; 135 } 136 137 static inline int 138 ip_vs_fill_iph_skb_icmp(int af, const struct sk_buff *skb, int offset, 139 bool inverse, struct ip_vs_iphdr *iphdr) 140 { 141 int hdr_flags = IP_VS_HDR_ICMP; 142 143 if (inverse) 144 hdr_flags |= IP_VS_HDR_INVERSE; 145 146 return ip_vs_fill_iph_skb_off(af, skb, offset, hdr_flags, iphdr); 147 } 148 149 static inline int 150 ip_vs_fill_iph_skb(int af, const struct sk_buff *skb, bool inverse, 151 struct ip_vs_iphdr *iphdr) 152 { 153 int hdr_flags = 0; 154 155 if (inverse) 156 hdr_flags |= IP_VS_HDR_INVERSE; 157 158 return ip_vs_fill_iph_skb_off(af, skb, skb_network_offset(skb), 159 hdr_flags, iphdr); 160 } 161 162 static inline bool 163 ip_vs_iph_inverse(const struct ip_vs_iphdr *iph) 164 { 165 return !!(iph->hdr_flags & IP_VS_HDR_INVERSE); 166 } 167 168 static inline bool 169 ip_vs_iph_icmp(const struct ip_vs_iphdr *iph) 170 { 171 return !!(iph->hdr_flags & IP_VS_HDR_ICMP); 172 } 173 174 static inline void ip_vs_addr_copy(int af, union nf_inet_addr *dst, 175 const union nf_inet_addr *src) 176 { 177 #ifdef CONFIG_IP_VS_IPV6 178 if (af == AF_INET6) 179 dst->in6 = src->in6; 180 else 181 #endif 182 dst->ip = src->ip; 183 } 184 185 static inline void ip_vs_addr_set(int af, union nf_inet_addr *dst, 186 const union nf_inet_addr *src) 187 { 188 #ifdef CONFIG_IP_VS_IPV6 189 if (af == AF_INET6) { 190 dst->in6 = src->in6; 191 return; 192 } 193 #endif 194 dst->ip = src->ip; 195 dst->all[1] = 0; 196 dst->all[2] = 0; 197 dst->all[3] = 0; 198 } 199 200 static inline int ip_vs_addr_equal(int af, const union nf_inet_addr *a, 201 const union nf_inet_addr *b) 202 { 203 #ifdef CONFIG_IP_VS_IPV6 204 if (af == AF_INET6) 205 return ipv6_addr_equal(&a->in6, &b->in6); 206 #endif 207 return a->ip == b->ip; 208 } 209 210 #ifdef CONFIG_IP_VS_DEBUG 211 #include <linux/net.h> 212 213 int ip_vs_get_debug_level(void); 214 215 static inline const char *ip_vs_dbg_addr(int af, char *buf, size_t buf_len, 216 const union nf_inet_addr *addr, 217 int *idx) 218 { 219 int len; 220 #ifdef CONFIG_IP_VS_IPV6 221 if (af == AF_INET6) 222 len = snprintf(&buf[*idx], buf_len - *idx, "[%pI6c]", 223 &addr->in6) + 1; 224 else 225 #endif 226 len = snprintf(&buf[*idx], buf_len - *idx, "%pI4", 227 &addr->ip) + 1; 228 229 *idx += len; 230 BUG_ON(*idx > buf_len + 1); 231 return &buf[*idx - len]; 232 } 233 234 #define IP_VS_DBG_BUF(level, msg, ...) \ 235 do { \ 236 char ip_vs_dbg_buf[160]; \ 237 int ip_vs_dbg_idx = 0; \ 238 if (level <= ip_vs_get_debug_level()) \ 239 printk(KERN_DEBUG pr_fmt(msg), ##__VA_ARGS__); \ 240 } while (0) 241 #define IP_VS_ERR_BUF(msg...) \ 242 do { \ 243 char ip_vs_dbg_buf[160]; \ 244 int ip_vs_dbg_idx = 0; \ 245 pr_err(msg); \ 246 } while (0) 247 248 /* Only use from within IP_VS_DBG_BUF() or IP_VS_ERR_BUF macros */ 249 #define IP_VS_DBG_ADDR(af, addr) \ 250 ip_vs_dbg_addr(af, ip_vs_dbg_buf, \ 251 sizeof(ip_vs_dbg_buf), addr, \ 252 &ip_vs_dbg_idx) 253 254 #define IP_VS_DBG(level, msg, ...) \ 255 do { \ 256 if (level <= ip_vs_get_debug_level()) \ 257 printk(KERN_DEBUG pr_fmt(msg), ##__VA_ARGS__); \ 258 } while (0) 259 #define IP_VS_DBG_RL(msg, ...) \ 260 do { \ 261 if (net_ratelimit()) \ 262 printk(KERN_DEBUG pr_fmt(msg), ##__VA_ARGS__); \ 263 } while (0) 264 #define IP_VS_DBG_PKT(level, af, pp, skb, ofs, msg) \ 265 do { \ 266 if (level <= ip_vs_get_debug_level()) \ 267 pp->debug_packet(af, pp, skb, ofs, msg); \ 268 } while (0) 269 #define IP_VS_DBG_RL_PKT(level, af, pp, skb, ofs, msg) \ 270 do { \ 271 if (level <= ip_vs_get_debug_level() && \ 272 net_ratelimit()) \ 273 pp->debug_packet(af, pp, skb, ofs, msg); \ 274 } while (0) 275 #else /* NO DEBUGGING at ALL */ 276 #define IP_VS_DBG_BUF(level, msg...) do {} while (0) 277 #define IP_VS_ERR_BUF(msg...) do {} while (0) 278 #define IP_VS_DBG(level, msg...) do {} while (0) 279 #define IP_VS_DBG_RL(msg...) do {} while (0) 280 #define IP_VS_DBG_PKT(level, af, pp, skb, ofs, msg) do {} while (0) 281 #define IP_VS_DBG_RL_PKT(level, af, pp, skb, ofs, msg) do {} while (0) 282 #endif 283 284 #define IP_VS_BUG() BUG() 285 #define IP_VS_ERR_RL(msg, ...) \ 286 do { \ 287 if (net_ratelimit()) \ 288 pr_err(msg, ##__VA_ARGS__); \ 289 } while (0) 290 291 struct ip_vs_aligned_lock { 292 spinlock_t l; /* Protect buckets */ 293 } ____cacheline_aligned_in_smp; 294 295 /* For arrays per family */ 296 enum { 297 IP_VS_AF_INET, 298 IP_VS_AF_INET6, 299 IP_VS_AF_MAX 300 }; 301 302 static inline int ip_vs_af_index(int af) 303 { 304 return af == AF_INET6 ? IP_VS_AF_INET6 : IP_VS_AF_INET; 305 } 306 307 /* work_flags */ 308 enum { 309 IP_VS_WORK_SVC_RESIZE, /* Schedule svc_resize_work */ 310 IP_VS_WORK_SVC_NORESIZE, /* Stopping svc_resize_work */ 311 IP_VS_WORK_CONN_RESIZE, /* Schedule conn_resize_work */ 312 }; 313 314 /* The port number of FTP service (in network order). */ 315 #define FTPPORT cpu_to_be16(21) 316 #define FTPDATA cpu_to_be16(20) 317 318 /* TCP State Values */ 319 enum { 320 IP_VS_TCP_S_NONE = 0, 321 IP_VS_TCP_S_ESTABLISHED, 322 IP_VS_TCP_S_SYN_SENT, 323 IP_VS_TCP_S_SYN_RECV, 324 IP_VS_TCP_S_FIN_WAIT, 325 IP_VS_TCP_S_TIME_WAIT, 326 IP_VS_TCP_S_CLOSE, 327 IP_VS_TCP_S_CLOSE_WAIT, 328 IP_VS_TCP_S_LAST_ACK, 329 IP_VS_TCP_S_LISTEN, 330 IP_VS_TCP_S_SYNACK, 331 IP_VS_TCP_S_LAST 332 }; 333 334 /* UDP State Values */ 335 enum { 336 IP_VS_UDP_S_NORMAL, 337 IP_VS_UDP_S_LAST, 338 }; 339 340 /* ICMP State Values */ 341 enum { 342 IP_VS_ICMP_S_NORMAL, 343 IP_VS_ICMP_S_LAST, 344 }; 345 346 /* SCTP State Values */ 347 enum ip_vs_sctp_states { 348 IP_VS_SCTP_S_NONE, 349 IP_VS_SCTP_S_INIT1, 350 IP_VS_SCTP_S_INIT, 351 IP_VS_SCTP_S_COOKIE_SENT, 352 IP_VS_SCTP_S_COOKIE_REPLIED, 353 IP_VS_SCTP_S_COOKIE_WAIT, 354 IP_VS_SCTP_S_COOKIE, 355 IP_VS_SCTP_S_COOKIE_ECHOED, 356 IP_VS_SCTP_S_ESTABLISHED, 357 IP_VS_SCTP_S_SHUTDOWN_SENT, 358 IP_VS_SCTP_S_SHUTDOWN_RECEIVED, 359 IP_VS_SCTP_S_SHUTDOWN_ACK_SENT, 360 IP_VS_SCTP_S_REJECTED, 361 IP_VS_SCTP_S_CLOSED, 362 IP_VS_SCTP_S_LAST 363 }; 364 365 /* Connection templates use bits from state */ 366 #define IP_VS_CTPL_S_NONE 0x0000 367 #define IP_VS_CTPL_S_ASSURED 0x0001 368 #define IP_VS_CTPL_S_LAST 0x0002 369 370 /* Delta sequence info structure 371 * Each ip_vs_conn has 2 (output AND input seq. changes). 372 * Only used in the VS/NAT. 373 */ 374 struct ip_vs_seq { 375 __u32 init_seq; /* Add delta from this seq */ 376 __u32 delta; /* Delta in sequence numbers */ 377 __u32 previous_delta; /* Delta in sequence numbers 378 * before last resized pkt */ 379 }; 380 381 /* counters per cpu */ 382 struct ip_vs_counters { 383 u64_stats_t conns; /* connections scheduled */ 384 u64_stats_t inpkts; /* incoming packets */ 385 u64_stats_t outpkts; /* outgoing packets */ 386 u64_stats_t inbytes; /* incoming bytes */ 387 u64_stats_t outbytes; /* outgoing bytes */ 388 }; 389 /* Stats per cpu */ 390 struct ip_vs_cpu_stats { 391 struct ip_vs_counters cnt; 392 struct u64_stats_sync syncp; 393 }; 394 395 /* Default nice for estimator kthreads */ 396 #define IPVS_EST_NICE 0 397 398 /* IPVS statistics objects */ 399 struct ip_vs_estimator { 400 struct hlist_node list; 401 402 u64 last_inbytes; 403 u64 last_outbytes; 404 u64 last_conns; 405 u64 last_inpkts; 406 u64 last_outpkts; 407 408 u64 cps; 409 u64 inpps; 410 u64 outpps; 411 u64 inbps; 412 u64 outbps; 413 414 s32 ktid:16, /* kthread ID, -1=temp list */ 415 ktrow:8, /* row/tick ID for kthread */ 416 ktcid:8; /* chain ID for kthread tick */ 417 }; 418 419 /* 420 * IPVS statistics object, 64-bit kernel version of struct ip_vs_stats_user 421 */ 422 struct ip_vs_kstats { 423 u64 conns; /* connections scheduled */ 424 u64 inpkts; /* incoming packets */ 425 u64 outpkts; /* outgoing packets */ 426 u64 inbytes; /* incoming bytes */ 427 u64 outbytes; /* outgoing bytes */ 428 429 u64 cps; /* current connection rate */ 430 u64 inpps; /* current in packet rate */ 431 u64 outpps; /* current out packet rate */ 432 u64 inbps; /* current in byte rate */ 433 u64 outbps; /* current out byte rate */ 434 }; 435 436 struct ip_vs_stats { 437 struct ip_vs_kstats kstats; /* kernel statistics */ 438 struct ip_vs_estimator est; /* estimator */ 439 struct ip_vs_cpu_stats __percpu *cpustats; /* per cpu counters */ 440 spinlock_t lock; /* spin lock */ 441 struct ip_vs_kstats kstats0; /* reset values */ 442 }; 443 444 struct ip_vs_stats_rcu { 445 struct ip_vs_stats s; 446 struct rcu_head rcu_head; 447 }; 448 449 int ip_vs_stats_init_alloc(struct ip_vs_stats *s); 450 struct ip_vs_stats *ip_vs_stats_alloc(void); 451 void ip_vs_stats_release(struct ip_vs_stats *stats); 452 void ip_vs_stats_free(struct ip_vs_stats *stats); 453 454 /* Process estimators in multiple timer ticks (20/50/100, see ktrow) */ 455 #define IPVS_EST_NTICKS 50 456 /* Estimation uses a 2-second period containing ticks (in jiffies) */ 457 #define IPVS_EST_TICK ((2 * HZ) / IPVS_EST_NTICKS) 458 459 /* Limit of CPU load per kthread (8 for 12.5%), ratio of CPU capacity (1/C). 460 * Value of 4 and above ensures kthreads will take work without exceeding 461 * the CPU capacity under different circumstances. 462 */ 463 #define IPVS_EST_LOAD_DIVISOR 8 464 465 /* Kthreads should not have work that exceeds the CPU load above 50% */ 466 #define IPVS_EST_CPU_KTHREADS (IPVS_EST_LOAD_DIVISOR / 2) 467 468 /* Desired number of chains per timer tick (chain load factor in 100us units), 469 * 48=4.8ms of 40ms tick (12% CPU usage): 470 * 2 sec * 1000 ms in sec * 10 (100us in ms) / 8 (12.5%) / 50 471 */ 472 #define IPVS_EST_CHAIN_FACTOR \ 473 ALIGN_DOWN(2 * 1000 * 10 / IPVS_EST_LOAD_DIVISOR / IPVS_EST_NTICKS, 8) 474 475 /* Compiled number of chains per tick 476 * The defines should match cond_resched_rcu 477 */ 478 #if defined(CONFIG_DEBUG_ATOMIC_SLEEP) || !defined(CONFIG_PREEMPT_RCU) 479 #define IPVS_EST_TICK_CHAINS IPVS_EST_CHAIN_FACTOR 480 #else 481 #define IPVS_EST_TICK_CHAINS 1 482 #endif 483 484 #if IPVS_EST_NTICKS > 127 485 #error Too many timer ticks for ktrow 486 #endif 487 488 /* Multiple chains processed in same tick */ 489 struct ip_vs_est_tick_data { 490 struct rcu_head rcu_head; 491 struct hlist_head chains[IPVS_EST_TICK_CHAINS]; 492 DECLARE_BITMAP(present, IPVS_EST_TICK_CHAINS); 493 DECLARE_BITMAP(full, IPVS_EST_TICK_CHAINS); 494 int chain_len[IPVS_EST_TICK_CHAINS]; 495 }; 496 497 /* Context for estimation kthread */ 498 struct ip_vs_est_kt_data { 499 struct netns_ipvs *ipvs; 500 struct task_struct *task; /* task if running */ 501 struct ip_vs_est_tick_data __rcu *ticks[IPVS_EST_NTICKS]; 502 DECLARE_BITMAP(avail, IPVS_EST_NTICKS); /* tick has space for ests */ 503 unsigned long est_timer; /* estimation timer (jiffies) */ 504 struct ip_vs_stats *calc_stats; /* Used for calculation */ 505 int needed; /* task is needed */ 506 int tick_len[IPVS_EST_NTICKS]; /* est count */ 507 int id; /* ktid per netns */ 508 int chain_max; /* max ests per tick chain */ 509 int tick_max; /* max ests per tick */ 510 int est_count; /* attached ests to kthread */ 511 int est_max_count; /* max ests per kthread */ 512 int add_row; /* row for new ests */ 513 int est_row; /* estimated row */ 514 }; 515 516 /* IPVS resizable hash tables */ 517 struct ip_vs_rht { 518 struct hlist_bl_head *buckets; 519 struct ip_vs_rht __rcu *new_tbl; /* New/Same table */ 520 seqcount_t *seqc; /* Protects moves */ 521 struct ip_vs_aligned_lock *lock; /* Protect seqc */ 522 int mask; /* Buckets mask */ 523 int size; /* Buckets */ 524 int seqc_mask; /* seqc mask */ 525 int lock_mask; /* lock mask */ 526 u32 table_id; 527 int u_thresh; /* upper threshold */ 528 int l_thresh; /* lower threshold */ 529 int lfactor; /* Load Factor (shift)*/ 530 int bits; /* size = 1 << bits */ 531 siphash_key_t hash_key; 532 struct rcu_head rcu_head; 533 }; 534 535 /** 536 * ip_vs_rht_for_each_table() - Walk the hash tables 537 * @table: struct ip_vs_rht __rcu *table 538 * @t: current table, used as cursor, struct ip_vs_rht *var 539 * @p: previous table, temp struct ip_vs_rht *var 540 * 541 * Walk tables assuming others can not change the installed tables 542 */ 543 #define ip_vs_rht_for_each_table(table, t, p) \ 544 for (p = NULL, t = rcu_dereference_protected(table, 1); \ 545 t != p; \ 546 p = t, t = rcu_dereference_protected(t->new_tbl, 1)) 547 548 /** 549 * ip_vs_rht_for_each_table_rcu() - Walk the hash tables under RCU reader lock 550 * @table: struct ip_vs_rht __rcu *table 551 * @t: current table, used as cursor, struct ip_vs_rht *var 552 * @p: previous table, temp struct ip_vs_rht *var 553 * 554 * We usually search in one table and also in second table on resizing 555 */ 556 #define ip_vs_rht_for_each_table_rcu(table, t, p) \ 557 for (p = NULL, t = rcu_dereference(table); \ 558 t != p; \ 559 p = t, t = rcu_dereference(t->new_tbl)) 560 561 /** 562 * ip_vs_rht_for_each_bucket() - Walk all table buckets 563 * @t: current table, used as cursor, struct ip_vs_rht *var 564 * @bucket: bucket index, used as cursor, u32 var 565 * @head: bucket address, used as cursor, struct hlist_bl_head *var 566 */ 567 #define ip_vs_rht_for_each_bucket(t, bucket, head) \ 568 for (bucket = 0, head = (t)->buckets; \ 569 bucket < t->size; bucket++, head++) 570 571 /** 572 * ip_vs_rht_for_bucket_retry() - Retry bucket if entries are moved 573 * @t: current table, used as cursor, struct ip_vs_rht *var 574 * @bucket: index of current bucket or hash key 575 * @sc: temp seqcount_t *var 576 * @seq: temp unsigned int var for sequence count 577 * @retry: temp int var 578 */ 579 #define ip_vs_rht_for_bucket_retry(t, bucket, sc, seq, retry) \ 580 for (retry = 1, sc = &(t)->seqc[(bucket) & (t)->seqc_mask]; \ 581 retry && ({ seq = read_seqcount_begin(sc); 1; }); \ 582 retry = read_seqcount_retry(sc, seq)) 583 584 /** 585 * DECLARE_IP_VS_RHT_WALK_BUCKETS_RCU() - Declare variables 586 * 587 * Variables for ip_vs_rht_walk_buckets_rcu 588 */ 589 #define DECLARE_IP_VS_RHT_WALK_BUCKETS_RCU() \ 590 struct ip_vs_rht *_t, *_p; \ 591 unsigned int _seq; \ 592 seqcount_t *_sc; \ 593 u32 _bucket; \ 594 int _retry 595 /** 596 * ip_vs_rht_walk_buckets_rcu() - Walk all buckets under RCU read lock 597 * @table: struct ip_vs_rht __rcu *table 598 * @head: bucket address, used as cursor, struct hlist_bl_head *var 599 * 600 * Can be used while others add/delete/move entries 601 * Not suitable if duplicates are not desired 602 * Possible cases for reader that uses cond_resched_rcu() in the loop: 603 * - new table can not be installed, no need to repeat 604 * - new table can be installed => check and repeat if new table is 605 * installed, needed for !PREEMPT_RCU 606 */ 607 #define ip_vs_rht_walk_buckets_rcu(table, head) \ 608 ip_vs_rht_for_each_table_rcu(table, _t, _p) \ 609 ip_vs_rht_for_each_bucket(_t, _bucket, head) \ 610 ip_vs_rht_for_bucket_retry(_t, _bucket, _sc, \ 611 _seq, _retry) 612 613 /** 614 * DECLARE_IP_VS_RHT_WALK_BUCKET_RCU() - Declare variables 615 * 616 * Variables for ip_vs_rht_walk_bucket_rcu 617 */ 618 #define DECLARE_IP_VS_RHT_WALK_BUCKET_RCU() \ 619 unsigned int _seq; \ 620 seqcount_t *_sc; \ 621 int _retry 622 /** 623 * ip_vs_rht_walk_bucket_rcu() - Walk bucket under RCU read lock 624 * @t: current table, struct ip_vs_rht *var 625 * @bucket: index of current bucket or hash key 626 * @head: bucket address, used as cursor, struct hlist_bl_head *var 627 * 628 * Can be used while others add/delete/move entries 629 * Not suitable if duplicates are not desired 630 * Possible cases for reader that uses cond_resched_rcu() in the loop: 631 * - new table can not be installed, no need to repeat 632 * - new table can be installed => check and repeat if new table is 633 * installed, needed for !PREEMPT_RCU 634 */ 635 #define ip_vs_rht_walk_bucket_rcu(t, bucket, head) \ 636 if (({ head = (t)->buckets + ((bucket) & (t)->mask); 0; })) \ 637 {} \ 638 else \ 639 ip_vs_rht_for_bucket_retry(t, (bucket), _sc, _seq, _retry) 640 641 /** 642 * DECLARE_IP_VS_RHT_WALK_BUCKETS_SAFE_RCU() - Declare variables 643 * 644 * Variables for ip_vs_rht_walk_buckets_safe_rcu 645 */ 646 #define DECLARE_IP_VS_RHT_WALK_BUCKETS_SAFE_RCU() \ 647 struct ip_vs_rht *_t, *_p; \ 648 u32 _bucket 649 /** 650 * ip_vs_rht_walk_buckets_safe_rcu() - Walk all buckets under RCU read lock 651 * @table: struct ip_vs_rht __rcu *table 652 * @head: bucket address, used as cursor, struct hlist_bl_head *var 653 * 654 * Can be used while others add/delete entries but moving is disabled 655 * Using cond_resched_rcu() should be safe if tables do not change 656 */ 657 #define ip_vs_rht_walk_buckets_safe_rcu(table, head) \ 658 ip_vs_rht_for_each_table_rcu(table, _t, _p) \ 659 ip_vs_rht_for_each_bucket(_t, _bucket, head) 660 661 /** 662 * DECLARE_IP_VS_RHT_WALK_BUCKETS() - Declare variables 663 * 664 * Variables for ip_vs_rht_walk_buckets 665 */ 666 #define DECLARE_IP_VS_RHT_WALK_BUCKETS() \ 667 struct ip_vs_rht *_t, *_p; \ 668 u32 _bucket 669 670 /** 671 * ip_vs_rht_walk_buckets() - Walk all buckets 672 * @table: struct ip_vs_rht __rcu *table 673 * @head: bucket address, used as cursor, struct hlist_bl_head *var 674 * 675 * Use if others can not add/delete/move entries 676 */ 677 #define ip_vs_rht_walk_buckets(table, head) \ 678 ip_vs_rht_for_each_table(table, _t, _p) \ 679 ip_vs_rht_for_each_bucket(_t, _bucket, head) 680 681 /* Entries can be in one of two tables, so we flip bit when new table is 682 * created and store it as highest bit in hash keys 683 */ 684 #define IP_VS_RHT_TABLE_ID_MASK BIT(31) 685 686 /* Check if hash key is from this table */ 687 static inline bool ip_vs_rht_same_table(struct ip_vs_rht *t, u32 hash_key) 688 { 689 return !((t->table_id ^ hash_key) & IP_VS_RHT_TABLE_ID_MASK); 690 } 691 692 /* Build per-table hash key from hash value */ 693 static inline u32 ip_vs_rht_build_hash_key(struct ip_vs_rht *t, u32 hash) 694 { 695 return t->table_id | (hash & ~IP_VS_RHT_TABLE_ID_MASK); 696 } 697 698 void ip_vs_rht_free(struct ip_vs_rht *t); 699 void ip_vs_rht_rcu_free(struct rcu_head *head); 700 struct ip_vs_rht *ip_vs_rht_alloc(int buckets, int scounts, int locks); 701 int ip_vs_rht_desired_size(struct netns_ipvs *ipvs, struct ip_vs_rht *t, int n, 702 int lfactor, int min_bits, int max_bits); 703 void ip_vs_rht_set_thresholds(struct ip_vs_rht *t, int size, int lfactor, 704 int min_bits, int max_bits); 705 u32 ip_vs_rht_hash_linfo(struct ip_vs_rht *t, int af, 706 const union nf_inet_addr *addr, u32 v1, u32 v2); 707 708 struct dst_entry; 709 struct iphdr; 710 struct ip_vs_conn; 711 struct ip_vs_app; 712 struct sk_buff; 713 struct ip_vs_proto_data; 714 715 struct ip_vs_protocol { 716 struct ip_vs_protocol *next; 717 char *name; 718 u16 protocol; 719 u16 num_states; 720 int dont_defrag; 721 722 void (*init)(struct ip_vs_protocol *pp); 723 724 void (*exit)(struct ip_vs_protocol *pp); 725 726 int (*init_netns)(struct netns_ipvs *ipvs, struct ip_vs_proto_data *pd); 727 728 void (*exit_netns)(struct netns_ipvs *ipvs, struct ip_vs_proto_data *pd); 729 730 int (*conn_schedule)(struct netns_ipvs *ipvs, 731 int af, struct sk_buff *skb, 732 struct ip_vs_proto_data *pd, 733 int *verdict, struct ip_vs_conn **cpp, 734 struct ip_vs_iphdr *iph); 735 736 struct ip_vs_conn * 737 (*conn_in_get)(struct netns_ipvs *ipvs, 738 int af, 739 const struct sk_buff *skb, 740 const struct ip_vs_iphdr *iph); 741 742 struct ip_vs_conn * 743 (*conn_out_get)(struct netns_ipvs *ipvs, 744 int af, 745 const struct sk_buff *skb, 746 const struct ip_vs_iphdr *iph); 747 748 int (*snat_handler)(struct sk_buff *skb, struct ip_vs_protocol *pp, 749 struct ip_vs_conn *cp, struct ip_vs_iphdr *iph); 750 751 int (*dnat_handler)(struct sk_buff *skb, struct ip_vs_protocol *pp, 752 struct ip_vs_conn *cp, struct ip_vs_iphdr *iph); 753 754 const char *(*state_name)(int state); 755 756 void (*state_transition)(struct ip_vs_conn *cp, int direction, 757 const struct sk_buff *skb, 758 struct ip_vs_proto_data *pd, 759 unsigned int iph_len); 760 761 int (*register_app)(struct netns_ipvs *ipvs, struct ip_vs_app *inc); 762 763 void (*unregister_app)(struct netns_ipvs *ipvs, struct ip_vs_app *inc); 764 765 int (*app_conn_bind)(struct ip_vs_conn *cp); 766 767 void (*debug_packet)(int af, struct ip_vs_protocol *pp, 768 const struct sk_buff *skb, 769 int offset, 770 const char *msg); 771 772 void (*timeout_change)(struct ip_vs_proto_data *pd, int flags); 773 }; 774 775 /* protocol data per netns */ 776 struct ip_vs_proto_data { 777 struct ip_vs_proto_data *next; 778 struct ip_vs_protocol *pp; 779 int *timeout_table; /* protocol timeout table */ 780 atomic_t appcnt; /* counter of proto app incs. */ 781 struct tcp_states_t *tcp_state_table; 782 }; 783 784 struct ip_vs_protocol *ip_vs_proto_get(unsigned short proto); 785 struct ip_vs_proto_data *ip_vs_proto_data_get(struct netns_ipvs *ipvs, 786 unsigned short proto); 787 788 struct ip_vs_conn_param { 789 struct netns_ipvs *ipvs; 790 const union nf_inet_addr *caddr; 791 const union nf_inet_addr *vaddr; 792 __be16 cport; 793 __be16 vport; 794 __u16 protocol; 795 u16 af; 796 797 const struct ip_vs_pe *pe; 798 char *pe_data; 799 __u8 pe_data_len; 800 }; 801 802 /* Hash node in conn_tab */ 803 struct ip_vs_conn_hnode { 804 struct hlist_bl_node node; /* node in conn_tab */ 805 u32 hash_key; /* Key for the hash table */ 806 u8 dir; /* 0=out->in, 1=in->out */ 807 } __packed; 808 809 /* IP_VS structure allocated for each dynamically scheduled connection */ 810 struct ip_vs_conn { 811 /* Cacheline for hash table nodes - rarely modified */ 812 813 struct ip_vs_conn_hnode hn0; /* Original direction */ 814 u8 af; /* address family */ 815 __be16 cport; 816 struct ip_vs_conn_hnode hn1; /* Reply direction */ 817 u8 daf; /* Address family of the dest */ 818 __be16 dport; 819 struct ip_vs_dest *dest; /* real server */ 820 atomic_t n_control; /* Number of controlled ones */ 821 volatile __u32 flags; /* status flags */ 822 /* 44/64 */ 823 824 struct ip_vs_conn *control; /* Master control connection */ 825 const struct ip_vs_pe *pe; 826 char *pe_data; 827 __u8 pe_data_len; 828 volatile __u16 state; /* state info */ 829 volatile __u16 old_state; /* old state, to be used for 830 * state transition triggered 831 * synchronization 832 */ 833 /* 2-byte hole */ 834 /* 64/96 */ 835 836 union nf_inet_addr caddr; /* client address */ 837 union nf_inet_addr vaddr; /* virtual address */ 838 /* 96/128 */ 839 840 union nf_inet_addr daddr; /* destination address */ 841 __u32 fwmark; /* Fire wall mark from skb */ 842 __be16 vport; 843 __u16 protocol; /* Which protocol (TCP/UDP) */ 844 845 /* Note: we can group the following members into a structure, 846 * in order to save more space, and the following members are 847 * only used in VS/NAT anyway 848 */ 849 struct ip_vs_app *app; /* bound ip_vs_app object */ 850 void *app_data; /* Application private data */ 851 /* 128/168 */ 852 struct_group(sync_conn_opt, 853 struct ip_vs_seq in_seq; /* incoming seq. struct */ 854 struct ip_vs_seq out_seq; /* outgoing seq. struct */ 855 ); 856 /* 152/192 */ 857 858 struct timer_list timer; /* Expiration timer */ 859 volatile unsigned long timeout; /* timeout */ 860 spinlock_t lock; /* lock for state transition */ 861 refcount_t refcnt; /* reference count */ 862 atomic_t in_pkts; /* incoming packet counter */ 863 /* 64-bit: 4-byte gap */ 864 865 /* 188/256 */ 866 unsigned long sync_endtime; /* jiffies + sent_retries */ 867 struct netns_ipvs *ipvs; 868 869 /* Packet transmitter for different forwarding methods. If it 870 * mangles the packet, it must return NF_DROP or better NF_STOLEN, 871 * otherwise this must be changed to a sk_buff **. 872 * NF_ACCEPT can be returned when destination is local. 873 */ 874 int (*packet_xmit)(struct sk_buff *skb, struct ip_vs_conn *cp, 875 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 876 877 struct rcu_head rcu_head; 878 }; 879 880 /* Extended internal versions of struct ip_vs_service_user and ip_vs_dest_user 881 * for IPv6 support. 882 * 883 * We need these to conveniently pass around service and destination 884 * options, but unfortunately, we also need to keep the old definitions to 885 * maintain userspace backwards compatibility for the setsockopt interface. 886 */ 887 struct ip_vs_service_user_kern { 888 /* virtual service addresses */ 889 u16 af; 890 u16 protocol; 891 union nf_inet_addr addr; /* virtual ip address */ 892 __be16 port; 893 u32 fwmark; /* firewall mark of service */ 894 895 /* virtual service options */ 896 char *sched_name; 897 char *pe_name; 898 unsigned int flags; /* virtual service flags */ 899 unsigned int timeout; /* persistent timeout in sec */ 900 __be32 netmask; /* persistent netmask or plen */ 901 }; 902 903 904 struct ip_vs_dest_user_kern { 905 /* destination server address */ 906 union nf_inet_addr addr; 907 __be16 port; 908 909 /* real server options */ 910 unsigned int conn_flags; /* connection flags */ 911 int weight; /* destination weight */ 912 913 /* thresholds for active connections */ 914 u32 u_threshold; /* upper threshold */ 915 u32 l_threshold; /* lower threshold */ 916 917 /* Address family of addr */ 918 u16 af; 919 920 u16 tun_type; /* tunnel type */ 921 __be16 tun_port; /* tunnel port */ 922 u16 tun_flags; /* tunnel flags */ 923 }; 924 925 926 /* 927 * The information about the virtual service offered to the net and the 928 * forwarding entries. 929 */ 930 struct ip_vs_service { 931 struct hlist_bl_node s_list; /* node in service table */ 932 u32 hash_key; /* Key for the hash table */ 933 u16 af; /* address family */ 934 __u16 protocol; /* which protocol (TCP/UDP) */ 935 936 union nf_inet_addr addr; /* IP address for virtual service */ 937 __u32 fwmark; /* firewall mark of the service */ 938 atomic_t refcnt; /* reference counter */ 939 __be16 port; /* port number for the service */ 940 unsigned int flags; /* service status flags */ 941 unsigned int timeout; /* persistent timeout in ticks */ 942 __be32 netmask; /* grouping granularity, mask/plen */ 943 struct netns_ipvs *ipvs; 944 945 struct list_head destinations; /* real server d-linked list */ 946 __u32 num_dests; /* number of servers */ 947 struct ip_vs_stats stats; /* statistics for the service */ 948 949 /* for scheduling */ 950 struct ip_vs_scheduler __rcu *scheduler; /* bound scheduler object */ 951 spinlock_t sched_lock; /* lock sched_data */ 952 void *sched_data; /* scheduler application data */ 953 954 /* alternate persistence engine */ 955 struct ip_vs_pe __rcu *pe; 956 int conntrack_afmask; 957 958 struct rcu_head rcu_head; 959 }; 960 961 /* Information for cached dst */ 962 struct ip_vs_dest_dst { 963 struct dst_entry *dst_cache; /* destination cache entry */ 964 u32 dst_cookie; 965 union nf_inet_addr dst_saddr; 966 struct rcu_head rcu_head; 967 }; 968 969 /* The real server destination forwarding entry with ip address, port number, 970 * and so on. 971 */ 972 struct ip_vs_dest { 973 /* Cacheline for hash table nodes - read-mostly */ 974 975 struct list_head n_list; /* for the dests in the service */ 976 struct hlist_node d_list; /* for table with all the dests */ 977 978 atomic_t weight; /* server weight */ 979 atomic_t last_weight; /* server latest weight */ 980 981 /* connection thresholds */ 982 u32 l_threshold_val;/* used lower threshold */ 983 u32 u_threshold_val;/* used upper threshold */ 984 /* 32/48 */ 985 u32 l_threshold; /* lower threshold */ 986 u32 u_threshold; /* upper threshold */ 987 988 unsigned long cflags; /* config flags */ 989 990 /* 44/64 */ 991 atomic_t conn_flags; /* flags to copy to conn */ 992 993 __u16 tun_type; /* tunnel type */ 994 __be16 tun_port; /* tunnel port */ 995 __u16 tun_flags; /* tunnel flags */ 996 997 u16 af; /* address family */ 998 __be16 port; /* port number of the server */ 999 /* 60/80 */ 1000 union nf_inet_addr addr; /* IP address of the server */ 1001 1002 /* connection counters */ 1003 atomic_t totalconns; /* total connections */ 1004 atomic_t activeconns; /* active connections */ 1005 atomic_t persistconns; /* persistent connections */ 1006 1007 refcount_t refcnt; /* reference counter */ 1008 struct ip_vs_stats stats; /* statistics */ 1009 unsigned long idle_start; /* start time, jiffies */ 1010 1011 /* for destination cache */ 1012 spinlock_t dst_lock; /* lock of dst_cache */ 1013 struct ip_vs_dest_dst __rcu *dest_dst; /* cached dst info */ 1014 1015 /* for virtual service */ 1016 struct ip_vs_service __rcu *svc; /* service it belongs to */ 1017 __u16 protocol; /* which protocol (TCP/UDP) */ 1018 __be16 vport; /* virtual port number */ 1019 union nf_inet_addr vaddr; /* virtual IP address */ 1020 __u32 vfwmark; /* firewall mark of service */ 1021 1022 struct rcu_head rcu_head; 1023 struct list_head t_list; /* in dest_trash */ 1024 unsigned int in_rs_table:1; /* we are in rs_table */ 1025 }; 1026 1027 /* The scheduler object */ 1028 struct ip_vs_scheduler { 1029 struct list_head n_list; /* d-linked list head */ 1030 char *name; /* scheduler name */ 1031 atomic_t refcnt; /* reference counter */ 1032 struct module *module; /* THIS_MODULE/NULL */ 1033 1034 /* scheduler initializing service */ 1035 int (*init_service)(struct ip_vs_service *svc); 1036 /* scheduling service finish */ 1037 void (*done_service)(struct ip_vs_service *svc); 1038 /* dest is linked */ 1039 int (*add_dest)(struct ip_vs_service *svc, struct ip_vs_dest *dest); 1040 /* dest is unlinked */ 1041 int (*del_dest)(struct ip_vs_service *svc, struct ip_vs_dest *dest); 1042 /* dest is updated */ 1043 int (*upd_dest)(struct ip_vs_service *svc, struct ip_vs_dest *dest); 1044 1045 /* selecting a server from the given service */ 1046 struct ip_vs_dest* (*schedule)(struct ip_vs_service *svc, 1047 const struct sk_buff *skb, 1048 struct ip_vs_iphdr *iph); 1049 }; 1050 1051 /* The persistence engine object */ 1052 struct ip_vs_pe { 1053 struct list_head n_list; /* d-linked list head */ 1054 char *name; /* scheduler name */ 1055 atomic_t refcnt; /* reference counter */ 1056 struct module *module; /* THIS_MODULE/NULL */ 1057 1058 /* get the connection template, if any */ 1059 int (*fill_param)(struct ip_vs_conn_param *p, struct sk_buff *skb); 1060 bool (*ct_match)(const struct ip_vs_conn_param *p, 1061 struct ip_vs_conn *ct); 1062 u32 (*hashkey_raw)(const struct ip_vs_conn_param *p, 1063 struct ip_vs_rht *t, bool inverse); 1064 int (*show_pe_data)(const struct ip_vs_conn *cp, char *buf); 1065 /* create connections for real-server outgoing packets */ 1066 struct ip_vs_conn* (*conn_out)(struct ip_vs_service *svc, 1067 struct ip_vs_dest *dest, 1068 struct sk_buff *skb, 1069 const struct ip_vs_iphdr *iph, 1070 __be16 dport, __be16 cport); 1071 }; 1072 1073 /* The application module object (a.k.a. app incarnation) */ 1074 struct ip_vs_app { 1075 struct list_head a_list; /* member in app list */ 1076 int type; /* IP_VS_APP_TYPE_xxx */ 1077 char *name; /* application module name */ 1078 __u16 protocol; 1079 struct module *module; /* THIS_MODULE/NULL */ 1080 struct list_head incs_list; /* list of incarnations */ 1081 1082 /* members for application incarnations */ 1083 struct list_head p_list; /* member in proto app list */ 1084 struct ip_vs_app *app; /* its real application */ 1085 __be16 port; /* port number in net order */ 1086 atomic_t usecnt; /* usage counter */ 1087 struct rcu_head rcu_head; 1088 1089 /* output hook: Process packet in inout direction, diff set for TCP. 1090 * Return: 0=Error, 1=Payload Not Mangled/Mangled but checksum is ok, 1091 * 2=Mangled but checksum was not updated 1092 */ 1093 int (*pkt_out)(struct ip_vs_app *, struct ip_vs_conn *, 1094 struct sk_buff *, int *diff, struct ip_vs_iphdr *ipvsh); 1095 1096 /* input hook: Process packet in outin direction, diff set for TCP. 1097 * Return: 0=Error, 1=Payload Not Mangled/Mangled but checksum is ok, 1098 * 2=Mangled but checksum was not updated 1099 */ 1100 int (*pkt_in)(struct ip_vs_app *, struct ip_vs_conn *, 1101 struct sk_buff *, int *diff, struct ip_vs_iphdr *ipvsh); 1102 1103 /* ip_vs_app initializer */ 1104 int (*init_conn)(struct ip_vs_app *, struct ip_vs_conn *); 1105 1106 /* ip_vs_app finish */ 1107 int (*done_conn)(struct ip_vs_app *, struct ip_vs_conn *); 1108 1109 1110 /* not used now */ 1111 int (*bind_conn)(struct ip_vs_app *, struct ip_vs_conn *, 1112 struct ip_vs_protocol *); 1113 1114 void (*unbind_conn)(struct ip_vs_app *, struct ip_vs_conn *); 1115 1116 int * timeout_table; 1117 int * timeouts; 1118 int timeouts_size; 1119 1120 int (*conn_schedule)(struct sk_buff *skb, struct ip_vs_app *app, 1121 int *verdict, struct ip_vs_conn **cpp); 1122 1123 struct ip_vs_conn * 1124 (*conn_in_get)(const struct sk_buff *skb, struct ip_vs_app *app, 1125 const struct iphdr *iph, int inverse); 1126 1127 struct ip_vs_conn * 1128 (*conn_out_get)(const struct sk_buff *skb, struct ip_vs_app *app, 1129 const struct iphdr *iph, int inverse); 1130 1131 int (*state_transition)(struct ip_vs_conn *cp, int direction, 1132 const struct sk_buff *skb, 1133 struct ip_vs_app *app); 1134 1135 void (*timeout_change)(struct ip_vs_app *app, int flags); 1136 }; 1137 1138 struct ipvs_master_sync_state { 1139 struct list_head sync_queue; 1140 struct ip_vs_sync_buff *sync_buff; 1141 unsigned long sync_queue_len; 1142 unsigned int sync_queue_delay; 1143 struct delayed_work master_wakeup_work; 1144 struct netns_ipvs *ipvs; 1145 }; 1146 1147 struct ip_vs_sync_thread_data; 1148 1149 /* How much time to keep dests in trash */ 1150 #define IP_VS_DEST_TRASH_PERIOD (120 * HZ) 1151 1152 struct ipvs_sync_daemon_cfg { 1153 union nf_inet_addr mcast_group; 1154 int syncid; 1155 u16 sync_maxlen; 1156 u16 mcast_port; 1157 u8 mcast_af; 1158 u8 mcast_ttl; 1159 /* multicast interface name */ 1160 char mcast_ifn[IP_VS_IFNAME_MAXLEN]; 1161 }; 1162 1163 /* IPVS in network namespace */ 1164 struct netns_ipvs { 1165 int gen; /* Generation */ 1166 int enable; /* enable like nf_hooks do */ 1167 /* Hash table: for real service lookups */ 1168 #define IP_VS_RTAB_BITS 4 1169 #define IP_VS_RTAB_SIZE (1 << IP_VS_RTAB_BITS) 1170 #define IP_VS_RTAB_MASK (IP_VS_RTAB_SIZE - 1) 1171 1172 struct hlist_head rs_table[IP_VS_RTAB_SIZE]; 1173 /* ip_vs_app */ 1174 struct list_head app_list; 1175 /* ip_vs_proto */ 1176 #define IP_VS_PROTO_TAB_SIZE 32 /* must be power of 2 */ 1177 struct ip_vs_proto_data *proto_data_table[IP_VS_PROTO_TAB_SIZE]; 1178 /* ip_vs_proto_tcp */ 1179 #ifdef CONFIG_IP_VS_PROTO_TCP 1180 #define TCP_APP_TAB_BITS 4 1181 #define TCP_APP_TAB_SIZE (1 << TCP_APP_TAB_BITS) 1182 #define TCP_APP_TAB_MASK (TCP_APP_TAB_SIZE - 1) 1183 struct list_head tcp_apps[TCP_APP_TAB_SIZE]; 1184 #endif 1185 /* ip_vs_proto_udp */ 1186 #ifdef CONFIG_IP_VS_PROTO_UDP 1187 #define UDP_APP_TAB_BITS 4 1188 #define UDP_APP_TAB_SIZE (1 << UDP_APP_TAB_BITS) 1189 #define UDP_APP_TAB_MASK (UDP_APP_TAB_SIZE - 1) 1190 struct list_head udp_apps[UDP_APP_TAB_SIZE]; 1191 #endif 1192 /* ip_vs_proto_sctp */ 1193 #ifdef CONFIG_IP_VS_PROTO_SCTP 1194 #define SCTP_APP_TAB_BITS 4 1195 #define SCTP_APP_TAB_SIZE (1 << SCTP_APP_TAB_BITS) 1196 #define SCTP_APP_TAB_MASK (SCTP_APP_TAB_SIZE - 1) 1197 /* Hash table for SCTP application incarnations */ 1198 struct list_head sctp_apps[SCTP_APP_TAB_SIZE]; 1199 #endif 1200 /* ip_vs_conn */ 1201 atomic_t conn_count; /* connection counter */ 1202 atomic_t no_cport_conns[IP_VS_AF_MAX]; 1203 struct delayed_work conn_resize_work;/* resize conn_tab */ 1204 1205 /* ip_vs_ctl */ 1206 struct ip_vs_stats_rcu *tot_stats; /* Statistics & est. */ 1207 1208 /* Trash for destinations */ 1209 struct list_head dest_trash; 1210 spinlock_t dest_trash_lock; 1211 struct timer_list dest_trash_timer; /* expiration timer */ 1212 struct mutex service_mutex; /* service reconfig */ 1213 struct rw_semaphore svc_resize_sem; /* svc_table resizing */ 1214 struct rw_semaphore svc_replace_sem; /* svc_table replace */ 1215 struct delayed_work svc_resize_work; /* resize svc_table */ 1216 atomic_t svc_table_changes;/* ++ on table changes */ 1217 /* Service counters */ 1218 atomic_t num_services[IP_VS_AF_MAX]; /* Services */ 1219 atomic_t fwm_services[IP_VS_AF_MAX]; /* Services */ 1220 atomic_t nonfwm_services[IP_VS_AF_MAX];/* Services */ 1221 atomic_t ftpsvc_counter[IP_VS_AF_MAX]; /* FTPPORT */ 1222 atomic_t nullsvc_counter[IP_VS_AF_MAX];/* Zero port */ 1223 atomic_t conn_out_counter[IP_VS_AF_MAX];/* out conn */ 1224 1225 #ifdef CONFIG_SYSCTL 1226 /* delayed work for expiring no dest connections */ 1227 struct delayed_work expire_nodest_conn_work; 1228 /* 1/rate drop and drop-entry variables */ 1229 struct delayed_work defense_work; /* Work handler */ 1230 int drop_rate; 1231 int drop_counter; 1232 int old_secure_tcp; 1233 atomic_t dropentry; 1234 s8 dropentry_counters[8]; 1235 /* locks in ctl.c */ 1236 spinlock_t dropentry_lock; /* drop entry handling */ 1237 spinlock_t droppacket_lock; /* drop packet handling */ 1238 spinlock_t securetcp_lock; /* state and timeout tables */ 1239 1240 /* sys-ctl struct */ 1241 struct ctl_table_header *sysctl_hdr; 1242 struct ctl_table *sysctl_tbl; 1243 #endif 1244 1245 /* sysctl variables */ 1246 int sysctl_amemthresh; 1247 int sysctl_am_droprate; 1248 #ifdef CONFIG_SYSCTL 1249 int sysctl_conn_max;/* soft limit for conns */ 1250 int conn_max_limit; /* hard limit for conn_max */ 1251 #endif 1252 int sysctl_drop_entry; 1253 int sysctl_drop_packet; 1254 int sysctl_secure_tcp; 1255 #ifdef CONFIG_IP_VS_NFCT 1256 int sysctl_conntrack; 1257 #endif 1258 int sysctl_snat_reroute; 1259 int sysctl_sync_ver; 1260 int sysctl_sync_ports; 1261 int sysctl_sync_persist_mode; 1262 unsigned long sysctl_sync_qlen_max; 1263 int sysctl_sync_sock_size; 1264 int sysctl_cache_bypass; 1265 int sysctl_expire_nodest_conn; 1266 int sysctl_sloppy_tcp; 1267 int sysctl_sloppy_sctp; 1268 int sysctl_expire_quiescent_template; 1269 int sysctl_sync_threshold[2]; 1270 unsigned int sysctl_sync_refresh_period; 1271 int sysctl_sync_retries; 1272 int sysctl_nat_icmp_send; 1273 int sysctl_pmtu_disc; 1274 int sysctl_backup_only; 1275 int sysctl_conn_reuse_mode; 1276 int sysctl_schedule_icmp; 1277 int sysctl_ignore_tunneled; 1278 int sysctl_run_estimation; 1279 #ifdef CONFIG_SYSCTL 1280 cpumask_var_t sysctl_est_cpulist; /* kthread cpumask */ 1281 int est_cpulist_valid; /* cpulist set */ 1282 int sysctl_est_nice; /* kthread nice */ 1283 int est_stopped; /* stop tasks */ 1284 #endif 1285 int sysctl_conn_lfactor; 1286 int sysctl_svc_lfactor; 1287 1288 /* ip_vs_lblc */ 1289 int sysctl_lblc_expiration; 1290 struct ctl_table_header *lblc_ctl_header; 1291 struct ctl_table *lblc_ctl_table; 1292 /* ip_vs_lblcr */ 1293 int sysctl_lblcr_expiration; 1294 struct ctl_table_header *lblcr_ctl_header; 1295 struct ctl_table *lblcr_ctl_table; 1296 unsigned long work_flags; /* IP_VS_WORK_* flags */ 1297 /* ip_vs_est */ 1298 struct delayed_work est_reload_work;/* Reload kthread tasks */ 1299 struct mutex est_mutex; /* protect kthread tasks */ 1300 struct hlist_head est_temp_list; /* Ests during calc phase */ 1301 struct ip_vs_est_kt_data **est_kt_arr; /* Array of kthread data ptrs */ 1302 unsigned long est_max_threads;/* Hard limit of kthreads */ 1303 int est_calc_phase; /* Calculation phase */ 1304 int est_chain_max; /* Calculated chain_max */ 1305 int est_kt_count; /* Allocated ptrs */ 1306 int est_add_ktid; /* ktid where to add ests */ 1307 atomic_t est_genid; /* kthreads reload genid */ 1308 atomic_t est_genid_done; /* applied genid */ 1309 /* ip_vs_sync */ 1310 spinlock_t sync_lock; 1311 struct ipvs_master_sync_state *ms; 1312 spinlock_t sync_buff_lock; 1313 struct ip_vs_sync_thread_data *master_tinfo; 1314 struct ip_vs_sync_thread_data *backup_tinfo; 1315 int threads_mask; 1316 volatile int sync_state; 1317 struct mutex sync_mutex; 1318 struct ipvs_sync_daemon_cfg mcfg; /* Master Configuration */ 1319 struct ipvs_sync_daemon_cfg bcfg; /* Backup Configuration */ 1320 /* net name space ptr */ 1321 struct net *net; /* Needed by timer routines */ 1322 /* Number of heterogeneous destinations, needed because heterogeneous 1323 * are not supported when synchronization is enabled. 1324 */ 1325 unsigned int mixed_address_family_dests; 1326 unsigned int hooks_afmask; /* &1=AF_INET, &2=AF_INET6 */ 1327 1328 struct ip_vs_rht __rcu *svc_table; /* Services */ 1329 struct ip_vs_rht __rcu *conn_tab; /* Connections */ 1330 atomic_t conn_tab_changes;/* ++ on new table */ 1331 }; 1332 1333 #define DEFAULT_SYNC_THRESHOLD 3 1334 #define DEFAULT_SYNC_PERIOD 50 1335 #define DEFAULT_SYNC_VER 1 1336 #define DEFAULT_SLOPPY_TCP 0 1337 #define DEFAULT_SLOPPY_SCTP 0 1338 #define DEFAULT_SYNC_REFRESH_PERIOD (0U * HZ) 1339 #define DEFAULT_SYNC_RETRIES 0 1340 #define IPVS_SYNC_WAKEUP_RATE 8 1341 #define IPVS_SYNC_QLEN_MAX (IPVS_SYNC_WAKEUP_RATE * 4) 1342 #define IPVS_SYNC_SEND_DELAY (HZ / 50) 1343 #define IPVS_SYNC_CHECK_PERIOD HZ 1344 #define IPVS_SYNC_FLUSH_TIME (HZ * 2) 1345 #define IPVS_SYNC_PORTS_MAX (1 << 6) 1346 1347 #ifdef CONFIG_SYSCTL 1348 1349 static inline int sysctl_conn_max(struct netns_ipvs *ipvs) 1350 { 1351 return READ_ONCE(ipvs->sysctl_conn_max); 1352 } 1353 1354 static inline int sysctl_sync_threshold(struct netns_ipvs *ipvs) 1355 { 1356 return ipvs->sysctl_sync_threshold[0]; 1357 } 1358 1359 static inline int sysctl_sync_period(struct netns_ipvs *ipvs) 1360 { 1361 return READ_ONCE(ipvs->sysctl_sync_threshold[1]); 1362 } 1363 1364 static inline unsigned int sysctl_sync_refresh_period(struct netns_ipvs *ipvs) 1365 { 1366 return READ_ONCE(ipvs->sysctl_sync_refresh_period); 1367 } 1368 1369 static inline int sysctl_sync_retries(struct netns_ipvs *ipvs) 1370 { 1371 return ipvs->sysctl_sync_retries; 1372 } 1373 1374 static inline int sysctl_sync_ver(struct netns_ipvs *ipvs) 1375 { 1376 return ipvs->sysctl_sync_ver; 1377 } 1378 1379 static inline int sysctl_sloppy_tcp(struct netns_ipvs *ipvs) 1380 { 1381 return ipvs->sysctl_sloppy_tcp; 1382 } 1383 1384 static inline int sysctl_sloppy_sctp(struct netns_ipvs *ipvs) 1385 { 1386 return ipvs->sysctl_sloppy_sctp; 1387 } 1388 1389 static inline int sysctl_sync_ports(struct netns_ipvs *ipvs) 1390 { 1391 return READ_ONCE(ipvs->sysctl_sync_ports); 1392 } 1393 1394 static inline int sysctl_sync_persist_mode(struct netns_ipvs *ipvs) 1395 { 1396 return ipvs->sysctl_sync_persist_mode; 1397 } 1398 1399 static inline unsigned long sysctl_sync_qlen_max(struct netns_ipvs *ipvs) 1400 { 1401 return ipvs->sysctl_sync_qlen_max; 1402 } 1403 1404 static inline int sysctl_sync_sock_size(struct netns_ipvs *ipvs) 1405 { 1406 return ipvs->sysctl_sync_sock_size; 1407 } 1408 1409 static inline int sysctl_pmtu_disc(struct netns_ipvs *ipvs) 1410 { 1411 return ipvs->sysctl_pmtu_disc; 1412 } 1413 1414 static inline int sysctl_backup_only(struct netns_ipvs *ipvs) 1415 { 1416 return ipvs->sync_state & IP_VS_STATE_BACKUP && 1417 ipvs->sysctl_backup_only; 1418 } 1419 1420 static inline int sysctl_conn_reuse_mode(struct netns_ipvs *ipvs) 1421 { 1422 return ipvs->sysctl_conn_reuse_mode; 1423 } 1424 1425 static inline int sysctl_expire_nodest_conn(struct netns_ipvs *ipvs) 1426 { 1427 return ipvs->sysctl_expire_nodest_conn; 1428 } 1429 1430 static inline int sysctl_schedule_icmp(struct netns_ipvs *ipvs) 1431 { 1432 return ipvs->sysctl_schedule_icmp; 1433 } 1434 1435 static inline int sysctl_ignore_tunneled(struct netns_ipvs *ipvs) 1436 { 1437 return ipvs->sysctl_ignore_tunneled; 1438 } 1439 1440 static inline int sysctl_cache_bypass(struct netns_ipvs *ipvs) 1441 { 1442 return ipvs->sysctl_cache_bypass; 1443 } 1444 1445 static inline int sysctl_run_estimation(struct netns_ipvs *ipvs) 1446 { 1447 return ipvs->sysctl_run_estimation; 1448 } 1449 1450 static inline const struct cpumask *__sysctl_est_cpulist(struct netns_ipvs *ipvs) 1451 { 1452 if (ipvs->est_cpulist_valid) 1453 return ipvs->sysctl_est_cpulist; 1454 else 1455 return housekeeping_cpumask(HK_TYPE_KTHREAD); 1456 } 1457 1458 static inline const struct cpumask *sysctl_est_preferred_cpulist(struct netns_ipvs *ipvs) 1459 { 1460 if (ipvs->est_cpulist_valid) 1461 return ipvs->sysctl_est_cpulist; 1462 else 1463 return NULL; 1464 } 1465 1466 static inline int sysctl_est_nice(struct netns_ipvs *ipvs) 1467 { 1468 return ipvs->sysctl_est_nice; 1469 } 1470 1471 #else 1472 1473 static inline int sysctl_conn_max(struct netns_ipvs *ipvs) 1474 { 1475 return IP_VS_CONN_MAX; 1476 } 1477 1478 static inline int sysctl_sync_threshold(struct netns_ipvs *ipvs) 1479 { 1480 return DEFAULT_SYNC_THRESHOLD; 1481 } 1482 1483 static inline int sysctl_sync_period(struct netns_ipvs *ipvs) 1484 { 1485 return DEFAULT_SYNC_PERIOD; 1486 } 1487 1488 static inline unsigned int sysctl_sync_refresh_period(struct netns_ipvs *ipvs) 1489 { 1490 return DEFAULT_SYNC_REFRESH_PERIOD; 1491 } 1492 1493 static inline int sysctl_sync_retries(struct netns_ipvs *ipvs) 1494 { 1495 return DEFAULT_SYNC_RETRIES & 3; 1496 } 1497 1498 static inline int sysctl_sync_ver(struct netns_ipvs *ipvs) 1499 { 1500 return DEFAULT_SYNC_VER; 1501 } 1502 1503 static inline int sysctl_sloppy_tcp(struct netns_ipvs *ipvs) 1504 { 1505 return DEFAULT_SLOPPY_TCP; 1506 } 1507 1508 static inline int sysctl_sloppy_sctp(struct netns_ipvs *ipvs) 1509 { 1510 return DEFAULT_SLOPPY_SCTP; 1511 } 1512 1513 static inline int sysctl_sync_ports(struct netns_ipvs *ipvs) 1514 { 1515 return 1; 1516 } 1517 1518 static inline int sysctl_sync_persist_mode(struct netns_ipvs *ipvs) 1519 { 1520 return 0; 1521 } 1522 1523 static inline unsigned long sysctl_sync_qlen_max(struct netns_ipvs *ipvs) 1524 { 1525 return IPVS_SYNC_QLEN_MAX; 1526 } 1527 1528 static inline int sysctl_sync_sock_size(struct netns_ipvs *ipvs) 1529 { 1530 return 0; 1531 } 1532 1533 static inline int sysctl_pmtu_disc(struct netns_ipvs *ipvs) 1534 { 1535 return 1; 1536 } 1537 1538 static inline int sysctl_backup_only(struct netns_ipvs *ipvs) 1539 { 1540 return 0; 1541 } 1542 1543 static inline int sysctl_conn_reuse_mode(struct netns_ipvs *ipvs) 1544 { 1545 return 1; 1546 } 1547 1548 static inline int sysctl_expire_nodest_conn(struct netns_ipvs *ipvs) 1549 { 1550 return 0; 1551 } 1552 1553 static inline int sysctl_schedule_icmp(struct netns_ipvs *ipvs) 1554 { 1555 return 0; 1556 } 1557 1558 static inline int sysctl_ignore_tunneled(struct netns_ipvs *ipvs) 1559 { 1560 return 0; 1561 } 1562 1563 static inline int sysctl_cache_bypass(struct netns_ipvs *ipvs) 1564 { 1565 return 0; 1566 } 1567 1568 static inline int sysctl_run_estimation(struct netns_ipvs *ipvs) 1569 { 1570 return 1; 1571 } 1572 1573 static inline const struct cpumask *__sysctl_est_cpulist(struct netns_ipvs *ipvs) 1574 { 1575 return housekeeping_cpumask(HK_TYPE_KTHREAD); 1576 } 1577 1578 static inline const struct cpumask *sysctl_est_preferred_cpulist(struct netns_ipvs *ipvs) 1579 { 1580 return NULL; 1581 } 1582 1583 static inline int sysctl_est_nice(struct netns_ipvs *ipvs) 1584 { 1585 return IPVS_EST_NICE; 1586 } 1587 1588 #endif 1589 1590 /* Get load factor to map conn_count/u_thresh to t->size */ 1591 static inline int sysctl_conn_lfactor(struct netns_ipvs *ipvs) 1592 { 1593 return READ_ONCE(ipvs->sysctl_conn_lfactor); 1594 } 1595 1596 /* Get load factor to map num_services/u_thresh to t->size 1597 * Smaller value decreases u_thresh to reduce collisions but increases 1598 * the table size 1599 * Returns factor where: 1600 * - <0: u_thresh = size >> -factor, eg. lfactor -2 = 25% load 1601 * - >=0: u_thresh = size << factor, eg. lfactor 1 = 200% load 1602 */ 1603 static inline int sysctl_svc_lfactor(struct netns_ipvs *ipvs) 1604 { 1605 return READ_ONCE(ipvs->sysctl_svc_lfactor); 1606 } 1607 1608 static inline bool sysctl_est_cpulist_empty(struct netns_ipvs *ipvs) 1609 { 1610 guard(rcu)(); 1611 return cpumask_empty(__sysctl_est_cpulist(ipvs)); 1612 } 1613 1614 static inline unsigned int sysctl_est_cpulist_weight(struct netns_ipvs *ipvs) 1615 { 1616 guard(rcu)(); 1617 return cpumask_weight(__sysctl_est_cpulist(ipvs)); 1618 } 1619 1620 /* IPVS core functions 1621 * (from ip_vs_core.c) 1622 */ 1623 const char *ip_vs_proto_name(unsigned int proto); 1624 void ip_vs_init_hash_table(struct list_head *table, int rows); 1625 struct ip_vs_conn *ip_vs_new_conn_out(struct ip_vs_service *svc, 1626 struct ip_vs_dest *dest, 1627 struct sk_buff *skb, 1628 const struct ip_vs_iphdr *iph, 1629 __be16 dport, 1630 __be16 cport); 1631 #define IP_VS_INIT_HASH_TABLE(t) ip_vs_init_hash_table((t), ARRAY_SIZE((t))) 1632 1633 #define IP_VS_APP_TYPE_FTP 1 1634 1635 /* ip_vs_conn handling functions 1636 * (from ip_vs_conn.c) 1637 */ 1638 enum { 1639 IP_VS_DIR_INPUT = 0, 1640 IP_VS_DIR_OUTPUT, 1641 IP_VS_DIR_INPUT_ONLY, 1642 IP_VS_DIR_LAST, 1643 }; 1644 1645 static inline void ip_vs_conn_fill_param(struct netns_ipvs *ipvs, int af, int protocol, 1646 const union nf_inet_addr *caddr, 1647 __be16 cport, 1648 const union nf_inet_addr *vaddr, 1649 __be16 vport, 1650 struct ip_vs_conn_param *p) 1651 { 1652 p->ipvs = ipvs; 1653 p->af = af; 1654 p->protocol = protocol; 1655 p->caddr = caddr; 1656 p->cport = cport; 1657 p->vaddr = vaddr; 1658 p->vport = vport; 1659 p->pe = NULL; 1660 p->pe_data = NULL; 1661 } 1662 1663 struct ip_vs_conn *ip_vs_conn_in_get(const struct ip_vs_conn_param *p); 1664 struct ip_vs_conn *ip_vs_ct_in_get(const struct ip_vs_conn_param *p); 1665 1666 struct ip_vs_conn * ip_vs_conn_in_get_proto(struct netns_ipvs *ipvs, int af, 1667 const struct sk_buff *skb, 1668 const struct ip_vs_iphdr *iph); 1669 1670 struct ip_vs_conn *ip_vs_conn_out_get(const struct ip_vs_conn_param *p); 1671 1672 struct ip_vs_conn * ip_vs_conn_out_get_proto(struct netns_ipvs *ipvs, int af, 1673 const struct sk_buff *skb, 1674 const struct ip_vs_iphdr *iph); 1675 1676 /* Get reference to gain full access to conn. 1677 * By default, RCU read-side critical sections have access only to 1678 * conn fields and its PE data, see ip_vs_conn_rcu_free() for reference. 1679 */ 1680 static inline bool __ip_vs_conn_get(struct ip_vs_conn *cp) 1681 { 1682 return refcount_inc_not_zero(&cp->refcnt); 1683 } 1684 1685 /* put back the conn without restarting its timer */ 1686 static inline void __ip_vs_conn_put(struct ip_vs_conn *cp) 1687 { 1688 smp_mb__before_atomic(); 1689 refcount_dec(&cp->refcnt); 1690 } 1691 void ip_vs_conn_put(struct ip_vs_conn *cp); 1692 void ip_vs_conn_fill_cport(struct ip_vs_conn *cp, __be16 cport); 1693 int ip_vs_conn_desired_size(struct netns_ipvs *ipvs, struct ip_vs_rht *t, 1694 int lfactor); 1695 struct ip_vs_rht *ip_vs_conn_tab_alloc(struct netns_ipvs *ipvs, int buckets, 1696 int lfactor); 1697 1698 static inline struct ip_vs_conn * 1699 ip_vs_hn0_to_conn(struct ip_vs_conn_hnode *hn) 1700 { 1701 return container_of(hn, struct ip_vs_conn, hn0); 1702 } 1703 1704 static inline struct ip_vs_conn * 1705 ip_vs_hn_to_conn(struct ip_vs_conn_hnode *hn) 1706 { 1707 return hn->dir ? container_of(hn, struct ip_vs_conn, hn1) : 1708 container_of(hn, struct ip_vs_conn, hn0); 1709 } 1710 1711 struct ip_vs_conn *ip_vs_conn_new(const struct ip_vs_conn_param *p, int dest_af, 1712 const union nf_inet_addr *daddr, 1713 __be16 dport, unsigned int flags, 1714 struct ip_vs_dest *dest, __u32 fwmark); 1715 void ip_vs_conn_expire_now(struct ip_vs_conn *cp); 1716 1717 const char *ip_vs_state_name(const struct ip_vs_conn *cp); 1718 1719 void ip_vs_tcp_conn_listen(struct ip_vs_conn *cp); 1720 int ip_vs_check_template(struct ip_vs_conn *ct, struct ip_vs_dest *cdest); 1721 void ip_vs_random_dropentry(struct netns_ipvs *ipvs); 1722 int ip_vs_conn_init(void); 1723 void ip_vs_conn_cleanup(void); 1724 1725 static inline void ip_vs_control_del(struct ip_vs_conn *cp) 1726 { 1727 struct ip_vs_conn *ctl_cp = cp->control; 1728 if (!ctl_cp) { 1729 IP_VS_ERR_BUF("request control DEL for uncontrolled: " 1730 "%s:%d to %s:%d\n", 1731 IP_VS_DBG_ADDR(cp->af, &cp->caddr), 1732 ntohs(cp->cport), 1733 IP_VS_DBG_ADDR(cp->af, &cp->vaddr), 1734 ntohs(cp->vport)); 1735 1736 return; 1737 } 1738 1739 IP_VS_DBG_BUF(7, "DELeting control for: " 1740 "cp.dst=%s:%d ctl_cp.dst=%s:%d\n", 1741 IP_VS_DBG_ADDR(cp->af, &cp->caddr), 1742 ntohs(cp->cport), 1743 IP_VS_DBG_ADDR(cp->af, &ctl_cp->caddr), 1744 ntohs(ctl_cp->cport)); 1745 1746 cp->control = NULL; 1747 if (atomic_read(&ctl_cp->n_control) == 0) { 1748 IP_VS_ERR_BUF("BUG control DEL with n=0 : " 1749 "%s:%d to %s:%d\n", 1750 IP_VS_DBG_ADDR(cp->af, &cp->caddr), 1751 ntohs(cp->cport), 1752 IP_VS_DBG_ADDR(cp->af, &cp->vaddr), 1753 ntohs(cp->vport)); 1754 1755 return; 1756 } 1757 atomic_dec(&ctl_cp->n_control); 1758 } 1759 1760 static inline void 1761 ip_vs_control_add(struct ip_vs_conn *cp, struct ip_vs_conn *ctl_cp) 1762 { 1763 if (cp->control) { 1764 IP_VS_ERR_BUF("request control ADD for already controlled: " 1765 "%s:%d to %s:%d\n", 1766 IP_VS_DBG_ADDR(cp->af, &cp->caddr), 1767 ntohs(cp->cport), 1768 IP_VS_DBG_ADDR(cp->af, &cp->vaddr), 1769 ntohs(cp->vport)); 1770 1771 ip_vs_control_del(cp); 1772 } 1773 1774 IP_VS_DBG_BUF(7, "ADDing control for: " 1775 "cp.dst=%s:%d ctl_cp.dst=%s:%d\n", 1776 IP_VS_DBG_ADDR(cp->af, &cp->caddr), 1777 ntohs(cp->cport), 1778 IP_VS_DBG_ADDR(cp->af, &ctl_cp->caddr), 1779 ntohs(ctl_cp->cport)); 1780 1781 cp->control = ctl_cp; 1782 atomic_inc(&ctl_cp->n_control); 1783 } 1784 1785 /* Mark our template as assured */ 1786 static inline void 1787 ip_vs_control_assure_ct(struct ip_vs_conn *cp) 1788 { 1789 struct ip_vs_conn *ct = cp->control; 1790 1791 if (ct && !(ct->state & IP_VS_CTPL_S_ASSURED) && 1792 (ct->flags & IP_VS_CONN_F_TEMPLATE)) 1793 ct->state |= IP_VS_CTPL_S_ASSURED; 1794 } 1795 1796 /* IPVS netns init & cleanup functions */ 1797 int ip_vs_estimator_net_init(struct netns_ipvs *ipvs); 1798 int ip_vs_control_net_init(struct netns_ipvs *ipvs); 1799 int ip_vs_protocol_net_init(struct netns_ipvs *ipvs); 1800 int ip_vs_app_net_init(struct netns_ipvs *ipvs); 1801 int ip_vs_conn_net_init(struct netns_ipvs *ipvs); 1802 int ip_vs_sync_net_init(struct netns_ipvs *ipvs); 1803 void ip_vs_conn_net_cleanup(struct netns_ipvs *ipvs); 1804 void ip_vs_app_net_cleanup(struct netns_ipvs *ipvs); 1805 void ip_vs_protocol_net_cleanup(struct netns_ipvs *ipvs); 1806 void ip_vs_control_net_cleanup(struct netns_ipvs *ipvs); 1807 void ip_vs_estimator_net_cleanup(struct netns_ipvs *ipvs); 1808 void ip_vs_sync_net_cleanup(struct netns_ipvs *ipvs); 1809 void ip_vs_service_nets_cleanup(struct list_head *net_list); 1810 1811 /* IPVS application functions 1812 * (from ip_vs_app.c) 1813 */ 1814 #define IP_VS_APP_MAX_PORTS 8 1815 struct ip_vs_app *register_ip_vs_app(struct netns_ipvs *ipvs, struct ip_vs_app *app); 1816 void unregister_ip_vs_app(struct netns_ipvs *ipvs, struct ip_vs_app *app); 1817 int ip_vs_bind_app(struct ip_vs_conn *cp, struct ip_vs_protocol *pp); 1818 void ip_vs_unbind_app(struct ip_vs_conn *cp); 1819 int register_ip_vs_app_inc(struct netns_ipvs *ipvs, struct ip_vs_app *app, __u16 proto, 1820 __u16 port); 1821 int ip_vs_app_inc_get(struct ip_vs_app *inc); 1822 void ip_vs_app_inc_put(struct ip_vs_app *inc); 1823 1824 int ip_vs_app_pkt_out(struct ip_vs_conn *, struct sk_buff *skb, 1825 struct ip_vs_iphdr *ipvsh); 1826 int ip_vs_app_pkt_in(struct ip_vs_conn *, struct sk_buff *skb, 1827 struct ip_vs_iphdr *ipvsh); 1828 1829 int register_ip_vs_pe(struct ip_vs_pe *pe); 1830 int unregister_ip_vs_pe(struct ip_vs_pe *pe); 1831 struct ip_vs_pe *ip_vs_pe_getbyname(const char *name); 1832 struct ip_vs_pe *__ip_vs_pe_getbyname(const char *pe_name); 1833 1834 /* Use a #define to avoid all of module.h just for these trivial ops */ 1835 #define ip_vs_pe_get(pe) \ 1836 if (pe && pe->module) \ 1837 __module_get(pe->module); 1838 1839 #define ip_vs_pe_put(pe) \ 1840 if (pe && pe->module) \ 1841 module_put(pe->module); 1842 1843 /* IPVS protocol functions (from ip_vs_proto.c) */ 1844 int ip_vs_protocol_init(void); 1845 void ip_vs_protocol_cleanup(void); 1846 void ip_vs_protocol_timeout_change(struct netns_ipvs *ipvs, int flags); 1847 int *ip_vs_create_timeout_table(int *table, int size); 1848 void ip_vs_tcpudp_debug_packet(int af, struct ip_vs_protocol *pp, 1849 const struct sk_buff *skb, int offset, 1850 const char *msg); 1851 1852 extern struct ip_vs_protocol ip_vs_protocol_tcp; 1853 extern struct ip_vs_protocol ip_vs_protocol_udp; 1854 extern struct ip_vs_protocol ip_vs_protocol_icmp; 1855 extern struct ip_vs_protocol ip_vs_protocol_esp; 1856 extern struct ip_vs_protocol ip_vs_protocol_ah; 1857 extern struct ip_vs_protocol ip_vs_protocol_sctp; 1858 1859 /* Registering/unregistering scheduler functions 1860 * (from ip_vs_sched.c) 1861 */ 1862 int register_ip_vs_scheduler(struct ip_vs_scheduler *scheduler); 1863 int unregister_ip_vs_scheduler(struct ip_vs_scheduler *scheduler); 1864 int ip_vs_bind_scheduler(struct ip_vs_service *svc, 1865 struct ip_vs_scheduler *scheduler); 1866 void ip_vs_unbind_scheduler(struct ip_vs_service *svc); 1867 struct ip_vs_scheduler *ip_vs_scheduler_get(const char *sched_name); 1868 void ip_vs_scheduler_put(struct ip_vs_scheduler *scheduler); 1869 struct ip_vs_conn * 1870 ip_vs_schedule(struct ip_vs_service *svc, struct sk_buff *skb, 1871 struct ip_vs_proto_data *pd, int *ignored, 1872 struct ip_vs_iphdr *iph); 1873 int ip_vs_leave(struct ip_vs_service *svc, struct sk_buff *skb, 1874 struct ip_vs_proto_data *pd, struct ip_vs_iphdr *iph); 1875 1876 void ip_vs_scheduler_err(struct ip_vs_service *svc, const char *msg); 1877 1878 /* IPVS control data and functions (from ip_vs_ctl.c) */ 1879 extern struct ip_vs_stats ip_vs_stats; 1880 extern int sysctl_ip_vs_sync_ver; 1881 1882 struct ip_vs_service * 1883 ip_vs_service_find(struct netns_ipvs *ipvs, int af, __u32 fwmark, __u16 protocol, 1884 const union nf_inet_addr *vaddr, __be16 vport); 1885 1886 bool ip_vs_has_real_service(struct netns_ipvs *ipvs, int af, __u16 protocol, 1887 const union nf_inet_addr *daddr, __be16 dport); 1888 1889 struct ip_vs_dest * 1890 ip_vs_find_real_service(struct netns_ipvs *ipvs, int af, __u16 protocol, 1891 const union nf_inet_addr *daddr, __be16 dport); 1892 struct ip_vs_dest *ip_vs_find_tunnel(struct netns_ipvs *ipvs, int af, 1893 const union nf_inet_addr *daddr, 1894 __be16 tun_port); 1895 1896 int ip_vs_use_count_inc(void); 1897 void ip_vs_use_count_dec(void); 1898 int ip_vs_register_nl_ioctl(void); 1899 void ip_vs_unregister_nl_ioctl(void); 1900 int ip_vs_control_init(void); 1901 void ip_vs_control_cleanup(void); 1902 struct ip_vs_dest * 1903 ip_vs_find_dest(struct netns_ipvs *ipvs, int svc_af, int dest_af, 1904 const union nf_inet_addr *daddr, __be16 dport, 1905 const union nf_inet_addr *vaddr, __be16 vport, 1906 __u16 protocol, __u32 fwmark, __u32 flags); 1907 void ip_vs_try_bind_dest(struct ip_vs_conn *cp); 1908 1909 static inline void ip_vs_dest_hold(struct ip_vs_dest *dest) 1910 { 1911 refcount_inc(&dest->refcnt); 1912 } 1913 1914 static inline void ip_vs_dest_put(struct ip_vs_dest *dest) 1915 { 1916 smp_mb__before_atomic(); 1917 refcount_dec(&dest->refcnt); 1918 } 1919 1920 static inline void ip_vs_dest_put_and_free(struct ip_vs_dest *dest) 1921 { 1922 if (refcount_dec_and_test(&dest->refcnt)) 1923 kfree(dest); 1924 } 1925 1926 void ip_vs_dest_update_overload(struct ip_vs_dest *dest, bool overload); 1927 1928 /* Check if dest is in overloaded state */ 1929 static inline bool ip_vs_dest_is_overloaded(const struct ip_vs_dest *dest) 1930 { 1931 return READ_ONCE(dest->l_threshold_val); 1932 } 1933 1934 /* IPVS sync daemon data and function prototypes 1935 * (from ip_vs_sync.c) 1936 */ 1937 int start_sync_thread(struct netns_ipvs *ipvs, struct ipvs_sync_daemon_cfg *cfg, 1938 int state); 1939 int stop_sync_thread(struct netns_ipvs *ipvs, int state); 1940 void ip_vs_sync_conn(struct netns_ipvs *ipvs, struct ip_vs_conn *cp, int pkts); 1941 1942 /* IPVS rate estimator prototypes (from ip_vs_est.c) */ 1943 int ip_vs_start_estimator(struct netns_ipvs *ipvs, struct ip_vs_stats *stats); 1944 void ip_vs_stop_estimator(struct netns_ipvs *ipvs, struct ip_vs_stats *stats); 1945 void ip_vs_zero_estimator(struct ip_vs_stats *stats); 1946 void ip_vs_read_estimator(struct ip_vs_kstats *dst, struct ip_vs_stats *stats); 1947 void ip_vs_est_reload_start(struct netns_ipvs *ipvs, bool restart); 1948 int ip_vs_est_kthread_start(struct netns_ipvs *ipvs, 1949 struct ip_vs_est_kt_data *kd); 1950 void ip_vs_est_kthread_stop(struct ip_vs_est_kt_data *kd); 1951 1952 static inline void ip_vs_stop_estimator_tot_stats(struct netns_ipvs *ipvs) 1953 { 1954 #ifdef CONFIG_SYSCTL 1955 ip_vs_stop_estimator(ipvs, &ipvs->tot_stats->s); 1956 ipvs->tot_stats->s.est.ktid = -2; 1957 #endif 1958 } 1959 1960 static inline void ip_vs_est_stopped_recalc(struct netns_ipvs *ipvs) 1961 { 1962 #ifdef CONFIG_SYSCTL 1963 /* Stop tasks while cpulist is empty or if disabled with flag */ 1964 ipvs->est_stopped = !sysctl_run_estimation(ipvs) || 1965 (ipvs->est_cpulist_valid && 1966 sysctl_est_cpulist_empty(ipvs)); 1967 #endif 1968 } 1969 1970 static inline bool ip_vs_est_stopped(struct netns_ipvs *ipvs) 1971 { 1972 #ifdef CONFIG_SYSCTL 1973 return ipvs->est_stopped; 1974 #else 1975 return false; 1976 #endif 1977 } 1978 1979 static inline int ip_vs_est_max_threads(struct netns_ipvs *ipvs) 1980 { 1981 unsigned int limit = IPVS_EST_CPU_KTHREADS * 1982 sysctl_est_cpulist_weight(ipvs); 1983 1984 return max(1U, limit); 1985 } 1986 1987 /* Various IPVS packet transmitters (from ip_vs_xmit.c) */ 1988 int ip_vs_null_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 1989 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1990 int ip_vs_bypass_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 1991 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1992 int ip_vs_nat_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 1993 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1994 int ip_vs_tunnel_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 1995 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1996 int ip_vs_dr_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 1997 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1998 int ip_vs_icmp_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 1999 struct ip_vs_protocol *pp, unsigned int toff, 2000 unsigned int hooknum, struct ip_vs_iphdr *ciph); 2001 void ip_vs_dest_dst_rcu_free(struct rcu_head *head); 2002 2003 #ifdef CONFIG_IP_VS_IPV6 2004 int ip_vs_bypass_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, 2005 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 2006 int ip_vs_nat_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, 2007 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 2008 int ip_vs_tunnel_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, 2009 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 2010 int ip_vs_dr_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, 2011 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 2012 int ip_vs_icmp_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, 2013 struct ip_vs_protocol *pp, unsigned int toff, 2014 unsigned int hooknum, struct ip_vs_iphdr *ciph); 2015 #endif 2016 2017 #ifdef CONFIG_SYSCTL 2018 /* This is a simple mechanism to ignore packets when 2019 * we are loaded. Just set ip_vs_drop_rate to 'n' and 2020 * we start to drop 1/rate of the packets 2021 */ 2022 static inline int ip_vs_todrop(struct netns_ipvs *ipvs) 2023 { 2024 if (!ipvs->drop_rate) 2025 return 0; 2026 if (--ipvs->drop_counter > 0) 2027 return 0; 2028 ipvs->drop_counter = ipvs->drop_rate; 2029 return 1; 2030 } 2031 #else 2032 static inline int ip_vs_todrop(struct netns_ipvs *ipvs) { return 0; } 2033 #endif 2034 2035 #ifdef CONFIG_SYSCTL 2036 /* Enqueue delayed work for expiring no dest connections 2037 * Only run when sysctl_expire_nodest=1 2038 */ 2039 static inline void ip_vs_enqueue_expire_nodest_conns(struct netns_ipvs *ipvs) 2040 { 2041 if (sysctl_expire_nodest_conn(ipvs)) 2042 queue_delayed_work(system_long_wq, 2043 &ipvs->expire_nodest_conn_work, 1); 2044 } 2045 2046 void ip_vs_expire_nodest_conn_flush(struct netns_ipvs *ipvs); 2047 #else 2048 static inline void ip_vs_enqueue_expire_nodest_conns(struct netns_ipvs *ipvs) {} 2049 #endif 2050 2051 #define IP_VS_DFWD_METHOD(dest) (atomic_read(&(dest)->conn_flags) & \ 2052 IP_VS_CONN_F_FWD_MASK) 2053 2054 /* ip_vs_fwd_tag returns the forwarding tag of the connection */ 2055 #define IP_VS_FWD_METHOD(cp) (cp->flags & IP_VS_CONN_F_FWD_MASK) 2056 2057 static inline char ip_vs_fwd_tag(struct ip_vs_conn *cp) 2058 { 2059 char fwd; 2060 2061 switch (IP_VS_FWD_METHOD(cp)) { 2062 case IP_VS_CONN_F_MASQ: 2063 fwd = 'M'; break; 2064 case IP_VS_CONN_F_LOCALNODE: 2065 fwd = 'L'; break; 2066 case IP_VS_CONN_F_TUNNEL: 2067 fwd = 'T'; break; 2068 case IP_VS_CONN_F_DROUTE: 2069 fwd = 'R'; break; 2070 case IP_VS_CONN_F_BYPASS: 2071 fwd = 'B'; break; 2072 default: 2073 fwd = '?'; break; 2074 } 2075 return fwd; 2076 } 2077 2078 /* Check if connection uses double hashing */ 2079 static inline bool ip_vs_conn_use_hash2(struct ip_vs_conn *cp) 2080 { 2081 return IP_VS_FWD_METHOD(cp) == IP_VS_CONN_F_MASQ && 2082 !(cp->flags & IP_VS_CONN_F_TEMPLATE); 2083 } 2084 2085 bool ip_vs_nat_icmp(struct sk_buff *skb, struct ip_vs_protocol *pp, 2086 struct ip_vs_conn *cp, int dir, unsigned int toff, 2087 bool has_ports, struct ip_vs_iphdr *ciph); 2088 2089 #ifdef CONFIG_IP_VS_IPV6 2090 void ip_vs_nat_icmp_v6(struct sk_buff *skb, struct ip_vs_protocol *pp, 2091 struct ip_vs_conn *cp, int dir, unsigned int toff, 2092 bool has_ports, struct ip_vs_iphdr *ciph); 2093 #endif 2094 2095 static inline __wsum ip_vs_check_diff4(__be32 old, __be32 new, __wsum oldsum) 2096 { 2097 __be32 diff[2] = { ~old, new }; 2098 2099 return csum_partial(diff, sizeof(diff), oldsum); 2100 } 2101 2102 #ifdef CONFIG_IP_VS_IPV6 2103 static inline __wsum ip_vs_check_diff16(const __be32 *old, const __be32 *new, 2104 __wsum oldsum) 2105 { 2106 __be32 diff[8] = { ~old[3], ~old[2], ~old[1], ~old[0], 2107 new[3], new[2], new[1], new[0] }; 2108 2109 return csum_partial(diff, sizeof(diff), oldsum); 2110 } 2111 #endif 2112 2113 static inline __wsum ip_vs_check_diff2(__be16 old, __be16 new, __wsum oldsum) 2114 { 2115 __be16 diff[2] = { ~old, new }; 2116 2117 return csum_partial(diff, sizeof(diff), oldsum); 2118 } 2119 2120 static inline bool ip_vs_checksum_needed(struct sk_buff *skb) 2121 { 2122 /* Checksum unnecessary or already validated? */ 2123 if (skb_csum_unnecessary(skb)) 2124 return false; 2125 /* Locally generated ? */ 2126 if (!skb->dev) 2127 return false; 2128 return true; 2129 } 2130 2131 static inline bool ip_vs_checksum_common_check(struct sk_buff *skb, 2132 int offset, int proto, int af) 2133 { 2134 if (!ip_vs_checksum_needed(skb)) 2135 return true; 2136 /* Validate csum even for FORWARD */ 2137 return !nf_checksum(skb, NF_INET_LOCAL_IN, offset, proto, af); 2138 } 2139 2140 /* Forget current conntrack (unconfirmed) and attach notrack entry */ 2141 static inline void ip_vs_notrack(struct sk_buff *skb) 2142 { 2143 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE) 2144 enum ip_conntrack_info ctinfo; 2145 struct nf_conn *ct = nf_ct_get(skb, &ctinfo); 2146 2147 if (ct) { 2148 nf_reset_ct(skb); 2149 nf_ct_set(skb, NULL, IP_CT_UNTRACKED); 2150 } 2151 #endif 2152 } 2153 2154 #ifdef CONFIG_IP_VS_NFCT 2155 /* Netfilter connection tracking 2156 * (from ip_vs_nfct.c) 2157 */ 2158 static inline int ip_vs_conntrack_enabled(struct netns_ipvs *ipvs) 2159 { 2160 #ifdef CONFIG_SYSCTL 2161 return ipvs->sysctl_conntrack; 2162 #else 2163 return 0; 2164 #endif 2165 } 2166 2167 void ip_vs_update_conntrack(struct sk_buff *skb, struct ip_vs_conn *cp, 2168 int outin); 2169 int ip_vs_confirm_conntrack(struct sk_buff *skb); 2170 void ip_vs_nfct_expect_related(struct sk_buff *skb, struct nf_conn *ct, 2171 struct ip_vs_conn *cp, u8 proto, 2172 const __be16 port, int from_rs); 2173 void ip_vs_conn_drop_conntrack(struct ip_vs_conn *cp); 2174 2175 #else 2176 2177 static inline int ip_vs_conntrack_enabled(struct netns_ipvs *ipvs) 2178 { 2179 return 0; 2180 } 2181 2182 static inline void ip_vs_update_conntrack(struct sk_buff *skb, 2183 struct ip_vs_conn *cp, int outin) 2184 { 2185 } 2186 2187 static inline int ip_vs_confirm_conntrack(struct sk_buff *skb) 2188 { 2189 return NF_ACCEPT; 2190 } 2191 2192 static inline void ip_vs_conn_drop_conntrack(struct ip_vs_conn *cp) 2193 { 2194 } 2195 #endif /* CONFIG_IP_VS_NFCT */ 2196 2197 /* Using old conntrack that can not be redirected to another real server? */ 2198 static inline bool ip_vs_conn_uses_old_conntrack(struct ip_vs_conn *cp, 2199 struct sk_buff *skb) 2200 { 2201 #ifdef CONFIG_IP_VS_NFCT 2202 enum ip_conntrack_info ctinfo; 2203 struct nf_conn *ct; 2204 2205 ct = nf_ct_get(skb, &ctinfo); 2206 if (ct && nf_ct_is_confirmed(ct)) 2207 return true; 2208 #endif 2209 return false; 2210 } 2211 2212 static inline int ip_vs_register_conntrack(struct ip_vs_service *svc) 2213 { 2214 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 2215 int afmask = (svc->af == AF_INET6) ? 2 : 1; 2216 int ret = 0; 2217 2218 if (!(svc->conntrack_afmask & afmask)) { 2219 ret = nf_ct_netns_get(svc->ipvs->net, svc->af); 2220 if (ret >= 0) 2221 svc->conntrack_afmask |= afmask; 2222 } 2223 return ret; 2224 #else 2225 return 0; 2226 #endif 2227 } 2228 2229 static inline void ip_vs_unregister_conntrack(struct ip_vs_service *svc) 2230 { 2231 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 2232 int afmask = (svc->af == AF_INET6) ? 2 : 1; 2233 2234 if (svc->conntrack_afmask & afmask) { 2235 nf_ct_netns_put(svc->ipvs->net, svc->af); 2236 svc->conntrack_afmask &= ~afmask; 2237 } 2238 #endif 2239 } 2240 2241 int ip_vs_register_hooks(struct netns_ipvs *ipvs, unsigned int af); 2242 void ip_vs_unregister_hooks(struct netns_ipvs *ipvs, unsigned int af); 2243 2244 static inline int 2245 ip_vs_dest_conn_overhead(struct ip_vs_dest *dest) 2246 { 2247 /* We think the overhead of processing active connections is 257 2248 * times higher than that of inactive connections in average. (This 2249 * 257 times might not be accurate, we will change it later) We 2250 * use the following formula to estimate the overhead now: 2251 * dest->activeconns*256 + dest->totalconns 2252 */ 2253 return (atomic_read(&dest->activeconns) << 8) + 2254 atomic_read(&dest->totalconns); 2255 } 2256 2257 static inline int 2258 ip_vs_dest_inactconns(const struct ip_vs_dest *dest) 2259 { 2260 return max(atomic_read(&dest->totalconns) - 2261 atomic_read(&dest->activeconns), 0); 2262 } 2263 2264 #ifdef CONFIG_IP_VS_PROTO_TCP 2265 INDIRECT_CALLABLE_DECLARE(int 2266 tcp_snat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, 2267 struct ip_vs_conn *cp, struct ip_vs_iphdr *iph)); 2268 #endif 2269 2270 #ifdef CONFIG_IP_VS_PROTO_UDP 2271 INDIRECT_CALLABLE_DECLARE(int 2272 udp_snat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, 2273 struct ip_vs_conn *cp, struct ip_vs_iphdr *iph)); 2274 #endif 2275 #endif /* _NET_IP_VS_H */ 2276