1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* IP Virtual Server 3 * data structure and functionality definitions 4 */ 5 6 #ifndef _NET_IP_VS_H 7 #define _NET_IP_VS_H 8 9 #include <linux/ip_vs.h> /* definitions shared with userland */ 10 11 #include <asm/types.h> /* for __uXX types */ 12 13 #include <linux/list.h> /* for struct list_head */ 14 #include <linux/rculist_bl.h> /* for struct hlist_bl_head */ 15 #include <linux/spinlock.h> /* for struct rwlock_t */ 16 #include <linux/atomic.h> /* for struct atomic_t */ 17 #include <linux/refcount.h> /* for struct refcount_t */ 18 #include <linux/workqueue.h> 19 20 #include <linux/compiler.h> 21 #include <linux/timer.h> 22 #include <linux/bug.h> 23 24 #include <net/checksum.h> 25 #include <linux/netfilter.h> /* for union nf_inet_addr */ 26 #include <linux/ip.h> 27 #include <linux/ipv6.h> /* for struct ipv6hdr */ 28 #include <net/ipv6.h> 29 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 30 #include <net/netfilter/nf_conntrack.h> 31 #endif 32 #include <net/net_namespace.h> /* Netw namespace */ 33 #include <linux/sched/isolation.h> 34 #include <linux/siphash.h> 35 36 #define IP_VS_HDR_INVERSE 1 37 #define IP_VS_HDR_ICMP 2 38 39 /* conn_tab limits (as per Kconfig) */ 40 #define IP_VS_CONN_TAB_MIN_BITS 8 41 #if BITS_PER_LONG > 32 42 #define IP_VS_CONN_TAB_MAX_BITS 27 43 #else 44 #define IP_VS_CONN_TAB_MAX_BITS 20 45 #endif 46 47 /* conn_max limits */ 48 #if BITS_PER_LONG > 32 49 /* Limit of atomic_t but restricted by roundup_pow_of_two() in ip_vs_core.c */ 50 #define IP_VS_CONN_MAX (1 << 30) 51 #else 52 #define IP_VS_CONN_MAX (1 << 24) 53 #endif 54 55 /* svc_table limits */ 56 #define IP_VS_SVC_TAB_MIN_BITS 4 57 #define IP_VS_SVC_TAB_MAX_BITS 20 58 59 /* Generic access of ipvs struct */ 60 static inline struct netns_ipvs *net_ipvs(struct net* net) 61 { 62 return net->ipvs; 63 } 64 65 /* Connections' size value needed by ip_vs_ctl.c */ 66 extern int ip_vs_conn_tab_size; 67 68 struct ip_vs_iphdr { 69 int hdr_flags; /* ipvs flags */ 70 __u32 off; /* Where IP or IPv4 header starts */ 71 __u32 len; /* IPv4 simply where L4 starts 72 * IPv6 where L4 Transport Header starts */ 73 __u16 fragoffs; /* IPv6 fragment offset, 0 if first frag (or not frag)*/ 74 __s16 protocol; 75 __s32 flags; 76 union nf_inet_addr saddr; 77 union nf_inet_addr daddr; 78 }; 79 80 static inline void *frag_safe_skb_hp(const struct sk_buff *skb, int offset, 81 int len, void *buffer) 82 { 83 return skb_header_pointer(skb, offset, len, buffer); 84 } 85 86 /* This function handles filling *ip_vs_iphdr, both for IPv4 and IPv6. 87 * IPv6 requires some extra work, as finding proper header position, 88 * depend on the IPv6 extension headers. 89 */ 90 static inline int 91 ip_vs_fill_iph_skb_off(int af, const struct sk_buff *skb, int offset, 92 int hdr_flags, struct ip_vs_iphdr *iphdr) 93 { 94 iphdr->hdr_flags = hdr_flags; 95 iphdr->off = offset; 96 97 #ifdef CONFIG_IP_VS_IPV6 98 if (af == AF_INET6) { 99 struct ipv6hdr _iph; 100 const struct ipv6hdr *iph = skb_header_pointer( 101 skb, offset, sizeof(_iph), &_iph); 102 if (!iph) 103 return 0; 104 105 iphdr->saddr.in6 = iph->saddr; 106 iphdr->daddr.in6 = iph->daddr; 107 /* ipv6_find_hdr() updates len, flags */ 108 iphdr->len = offset; 109 iphdr->flags = 0; 110 iphdr->protocol = ipv6_find_hdr(skb, &iphdr->len, -1, 111 &iphdr->fragoffs, 112 &iphdr->flags); 113 if (iphdr->protocol < 0) 114 return 0; 115 } else 116 #endif 117 { 118 struct iphdr _iph; 119 const struct iphdr *iph = skb_header_pointer( 120 skb, offset, sizeof(_iph), &_iph); 121 if (!iph) 122 return 0; 123 124 iphdr->len = offset + iph->ihl * 4; 125 iphdr->fragoffs = 0; 126 iphdr->protocol = iph->protocol; 127 iphdr->saddr.ip = iph->saddr; 128 iphdr->daddr.ip = iph->daddr; 129 } 130 131 return 1; 132 } 133 134 static inline int 135 ip_vs_fill_iph_skb_icmp(int af, const struct sk_buff *skb, int offset, 136 bool inverse, struct ip_vs_iphdr *iphdr) 137 { 138 int hdr_flags = IP_VS_HDR_ICMP; 139 140 if (inverse) 141 hdr_flags |= IP_VS_HDR_INVERSE; 142 143 return ip_vs_fill_iph_skb_off(af, skb, offset, hdr_flags, iphdr); 144 } 145 146 static inline int 147 ip_vs_fill_iph_skb(int af, const struct sk_buff *skb, bool inverse, 148 struct ip_vs_iphdr *iphdr) 149 { 150 int hdr_flags = 0; 151 152 if (inverse) 153 hdr_flags |= IP_VS_HDR_INVERSE; 154 155 return ip_vs_fill_iph_skb_off(af, skb, skb_network_offset(skb), 156 hdr_flags, iphdr); 157 } 158 159 static inline bool 160 ip_vs_iph_inverse(const struct ip_vs_iphdr *iph) 161 { 162 return !!(iph->hdr_flags & IP_VS_HDR_INVERSE); 163 } 164 165 static inline bool 166 ip_vs_iph_icmp(const struct ip_vs_iphdr *iph) 167 { 168 return !!(iph->hdr_flags & IP_VS_HDR_ICMP); 169 } 170 171 static inline void ip_vs_addr_copy(int af, union nf_inet_addr *dst, 172 const union nf_inet_addr *src) 173 { 174 #ifdef CONFIG_IP_VS_IPV6 175 if (af == AF_INET6) 176 dst->in6 = src->in6; 177 else 178 #endif 179 dst->ip = src->ip; 180 } 181 182 static inline void ip_vs_addr_set(int af, union nf_inet_addr *dst, 183 const union nf_inet_addr *src) 184 { 185 #ifdef CONFIG_IP_VS_IPV6 186 if (af == AF_INET6) { 187 dst->in6 = src->in6; 188 return; 189 } 190 #endif 191 dst->ip = src->ip; 192 dst->all[1] = 0; 193 dst->all[2] = 0; 194 dst->all[3] = 0; 195 } 196 197 static inline int ip_vs_addr_equal(int af, const union nf_inet_addr *a, 198 const union nf_inet_addr *b) 199 { 200 #ifdef CONFIG_IP_VS_IPV6 201 if (af == AF_INET6) 202 return ipv6_addr_equal(&a->in6, &b->in6); 203 #endif 204 return a->ip == b->ip; 205 } 206 207 #ifdef CONFIG_IP_VS_DEBUG 208 #include <linux/net.h> 209 210 int ip_vs_get_debug_level(void); 211 212 static inline const char *ip_vs_dbg_addr(int af, char *buf, size_t buf_len, 213 const union nf_inet_addr *addr, 214 int *idx) 215 { 216 int len; 217 #ifdef CONFIG_IP_VS_IPV6 218 if (af == AF_INET6) 219 len = snprintf(&buf[*idx], buf_len - *idx, "[%pI6c]", 220 &addr->in6) + 1; 221 else 222 #endif 223 len = snprintf(&buf[*idx], buf_len - *idx, "%pI4", 224 &addr->ip) + 1; 225 226 *idx += len; 227 BUG_ON(*idx > buf_len + 1); 228 return &buf[*idx - len]; 229 } 230 231 #define IP_VS_DBG_BUF(level, msg, ...) \ 232 do { \ 233 char ip_vs_dbg_buf[160]; \ 234 int ip_vs_dbg_idx = 0; \ 235 if (level <= ip_vs_get_debug_level()) \ 236 printk(KERN_DEBUG pr_fmt(msg), ##__VA_ARGS__); \ 237 } while (0) 238 #define IP_VS_ERR_BUF(msg...) \ 239 do { \ 240 char ip_vs_dbg_buf[160]; \ 241 int ip_vs_dbg_idx = 0; \ 242 pr_err(msg); \ 243 } while (0) 244 245 /* Only use from within IP_VS_DBG_BUF() or IP_VS_ERR_BUF macros */ 246 #define IP_VS_DBG_ADDR(af, addr) \ 247 ip_vs_dbg_addr(af, ip_vs_dbg_buf, \ 248 sizeof(ip_vs_dbg_buf), addr, \ 249 &ip_vs_dbg_idx) 250 251 #define IP_VS_DBG(level, msg, ...) \ 252 do { \ 253 if (level <= ip_vs_get_debug_level()) \ 254 printk(KERN_DEBUG pr_fmt(msg), ##__VA_ARGS__); \ 255 } while (0) 256 #define IP_VS_DBG_RL(msg, ...) \ 257 do { \ 258 if (net_ratelimit()) \ 259 printk(KERN_DEBUG pr_fmt(msg), ##__VA_ARGS__); \ 260 } while (0) 261 #define IP_VS_DBG_PKT(level, af, pp, skb, ofs, msg) \ 262 do { \ 263 if (level <= ip_vs_get_debug_level()) \ 264 pp->debug_packet(af, pp, skb, ofs, msg); \ 265 } while (0) 266 #define IP_VS_DBG_RL_PKT(level, af, pp, skb, ofs, msg) \ 267 do { \ 268 if (level <= ip_vs_get_debug_level() && \ 269 net_ratelimit()) \ 270 pp->debug_packet(af, pp, skb, ofs, msg); \ 271 } while (0) 272 #else /* NO DEBUGGING at ALL */ 273 #define IP_VS_DBG_BUF(level, msg...) do {} while (0) 274 #define IP_VS_ERR_BUF(msg...) do {} while (0) 275 #define IP_VS_DBG(level, msg...) do {} while (0) 276 #define IP_VS_DBG_RL(msg...) do {} while (0) 277 #define IP_VS_DBG_PKT(level, af, pp, skb, ofs, msg) do {} while (0) 278 #define IP_VS_DBG_RL_PKT(level, af, pp, skb, ofs, msg) do {} while (0) 279 #endif 280 281 #define IP_VS_BUG() BUG() 282 #define IP_VS_ERR_RL(msg, ...) \ 283 do { \ 284 if (net_ratelimit()) \ 285 pr_err(msg, ##__VA_ARGS__); \ 286 } while (0) 287 288 struct ip_vs_aligned_lock { 289 spinlock_t l; /* Protect buckets */ 290 } ____cacheline_aligned_in_smp; 291 292 /* For arrays per family */ 293 enum { 294 IP_VS_AF_INET, 295 IP_VS_AF_INET6, 296 IP_VS_AF_MAX 297 }; 298 299 static inline int ip_vs_af_index(int af) 300 { 301 return af == AF_INET6 ? IP_VS_AF_INET6 : IP_VS_AF_INET; 302 } 303 304 /* work_flags */ 305 enum { 306 IP_VS_WORK_SVC_RESIZE, /* Schedule svc_resize_work */ 307 IP_VS_WORK_SVC_NORESIZE, /* Stopping svc_resize_work */ 308 IP_VS_WORK_CONN_RESIZE, /* Schedule conn_resize_work */ 309 }; 310 311 /* The port number of FTP service (in network order). */ 312 #define FTPPORT cpu_to_be16(21) 313 #define FTPDATA cpu_to_be16(20) 314 315 /* TCP State Values */ 316 enum { 317 IP_VS_TCP_S_NONE = 0, 318 IP_VS_TCP_S_ESTABLISHED, 319 IP_VS_TCP_S_SYN_SENT, 320 IP_VS_TCP_S_SYN_RECV, 321 IP_VS_TCP_S_FIN_WAIT, 322 IP_VS_TCP_S_TIME_WAIT, 323 IP_VS_TCP_S_CLOSE, 324 IP_VS_TCP_S_CLOSE_WAIT, 325 IP_VS_TCP_S_LAST_ACK, 326 IP_VS_TCP_S_LISTEN, 327 IP_VS_TCP_S_SYNACK, 328 IP_VS_TCP_S_LAST 329 }; 330 331 /* UDP State Values */ 332 enum { 333 IP_VS_UDP_S_NORMAL, 334 IP_VS_UDP_S_LAST, 335 }; 336 337 /* ICMP State Values */ 338 enum { 339 IP_VS_ICMP_S_NORMAL, 340 IP_VS_ICMP_S_LAST, 341 }; 342 343 /* SCTP State Values */ 344 enum ip_vs_sctp_states { 345 IP_VS_SCTP_S_NONE, 346 IP_VS_SCTP_S_INIT1, 347 IP_VS_SCTP_S_INIT, 348 IP_VS_SCTP_S_COOKIE_SENT, 349 IP_VS_SCTP_S_COOKIE_REPLIED, 350 IP_VS_SCTP_S_COOKIE_WAIT, 351 IP_VS_SCTP_S_COOKIE, 352 IP_VS_SCTP_S_COOKIE_ECHOED, 353 IP_VS_SCTP_S_ESTABLISHED, 354 IP_VS_SCTP_S_SHUTDOWN_SENT, 355 IP_VS_SCTP_S_SHUTDOWN_RECEIVED, 356 IP_VS_SCTP_S_SHUTDOWN_ACK_SENT, 357 IP_VS_SCTP_S_REJECTED, 358 IP_VS_SCTP_S_CLOSED, 359 IP_VS_SCTP_S_LAST 360 }; 361 362 /* Connection templates use bits from state */ 363 #define IP_VS_CTPL_S_NONE 0x0000 364 #define IP_VS_CTPL_S_ASSURED 0x0001 365 #define IP_VS_CTPL_S_LAST 0x0002 366 367 /* Delta sequence info structure 368 * Each ip_vs_conn has 2 (output AND input seq. changes). 369 * Only used in the VS/NAT. 370 */ 371 struct ip_vs_seq { 372 __u32 init_seq; /* Add delta from this seq */ 373 __u32 delta; /* Delta in sequence numbers */ 374 __u32 previous_delta; /* Delta in sequence numbers 375 * before last resized pkt */ 376 }; 377 378 /* counters per cpu */ 379 struct ip_vs_counters { 380 u64_stats_t conns; /* connections scheduled */ 381 u64_stats_t inpkts; /* incoming packets */ 382 u64_stats_t outpkts; /* outgoing packets */ 383 u64_stats_t inbytes; /* incoming bytes */ 384 u64_stats_t outbytes; /* outgoing bytes */ 385 }; 386 /* Stats per cpu */ 387 struct ip_vs_cpu_stats { 388 struct ip_vs_counters cnt; 389 struct u64_stats_sync syncp; 390 }; 391 392 /* Default nice for estimator kthreads */ 393 #define IPVS_EST_NICE 0 394 395 /* IPVS statistics objects */ 396 struct ip_vs_estimator { 397 struct hlist_node list; 398 399 u64 last_inbytes; 400 u64 last_outbytes; 401 u64 last_conns; 402 u64 last_inpkts; 403 u64 last_outpkts; 404 405 u64 cps; 406 u64 inpps; 407 u64 outpps; 408 u64 inbps; 409 u64 outbps; 410 411 s32 ktid:16, /* kthread ID, -1=temp list */ 412 ktrow:8, /* row/tick ID for kthread */ 413 ktcid:8; /* chain ID for kthread tick */ 414 }; 415 416 /* 417 * IPVS statistics object, 64-bit kernel version of struct ip_vs_stats_user 418 */ 419 struct ip_vs_kstats { 420 u64 conns; /* connections scheduled */ 421 u64 inpkts; /* incoming packets */ 422 u64 outpkts; /* outgoing packets */ 423 u64 inbytes; /* incoming bytes */ 424 u64 outbytes; /* outgoing bytes */ 425 426 u64 cps; /* current connection rate */ 427 u64 inpps; /* current in packet rate */ 428 u64 outpps; /* current out packet rate */ 429 u64 inbps; /* current in byte rate */ 430 u64 outbps; /* current out byte rate */ 431 }; 432 433 struct ip_vs_stats { 434 struct ip_vs_kstats kstats; /* kernel statistics */ 435 struct ip_vs_estimator est; /* estimator */ 436 struct ip_vs_cpu_stats __percpu *cpustats; /* per cpu counters */ 437 spinlock_t lock; /* spin lock */ 438 struct ip_vs_kstats kstats0; /* reset values */ 439 }; 440 441 struct ip_vs_stats_rcu { 442 struct ip_vs_stats s; 443 struct rcu_head rcu_head; 444 }; 445 446 int ip_vs_stats_init_alloc(struct ip_vs_stats *s); 447 struct ip_vs_stats *ip_vs_stats_alloc(void); 448 void ip_vs_stats_release(struct ip_vs_stats *stats); 449 void ip_vs_stats_free(struct ip_vs_stats *stats); 450 451 /* Process estimators in multiple timer ticks (20/50/100, see ktrow) */ 452 #define IPVS_EST_NTICKS 50 453 /* Estimation uses a 2-second period containing ticks (in jiffies) */ 454 #define IPVS_EST_TICK ((2 * HZ) / IPVS_EST_NTICKS) 455 456 /* Limit of CPU load per kthread (8 for 12.5%), ratio of CPU capacity (1/C). 457 * Value of 4 and above ensures kthreads will take work without exceeding 458 * the CPU capacity under different circumstances. 459 */ 460 #define IPVS_EST_LOAD_DIVISOR 8 461 462 /* Kthreads should not have work that exceeds the CPU load above 50% */ 463 #define IPVS_EST_CPU_KTHREADS (IPVS_EST_LOAD_DIVISOR / 2) 464 465 /* Desired number of chains per timer tick (chain load factor in 100us units), 466 * 48=4.8ms of 40ms tick (12% CPU usage): 467 * 2 sec * 1000 ms in sec * 10 (100us in ms) / 8 (12.5%) / 50 468 */ 469 #define IPVS_EST_CHAIN_FACTOR \ 470 ALIGN_DOWN(2 * 1000 * 10 / IPVS_EST_LOAD_DIVISOR / IPVS_EST_NTICKS, 8) 471 472 /* Compiled number of chains per tick 473 * The defines should match cond_resched_rcu 474 */ 475 #if defined(CONFIG_DEBUG_ATOMIC_SLEEP) || !defined(CONFIG_PREEMPT_RCU) 476 #define IPVS_EST_TICK_CHAINS IPVS_EST_CHAIN_FACTOR 477 #else 478 #define IPVS_EST_TICK_CHAINS 1 479 #endif 480 481 #if IPVS_EST_NTICKS > 127 482 #error Too many timer ticks for ktrow 483 #endif 484 485 /* Multiple chains processed in same tick */ 486 struct ip_vs_est_tick_data { 487 struct rcu_head rcu_head; 488 struct hlist_head chains[IPVS_EST_TICK_CHAINS]; 489 DECLARE_BITMAP(present, IPVS_EST_TICK_CHAINS); 490 DECLARE_BITMAP(full, IPVS_EST_TICK_CHAINS); 491 int chain_len[IPVS_EST_TICK_CHAINS]; 492 }; 493 494 /* Context for estimation kthread */ 495 struct ip_vs_est_kt_data { 496 struct netns_ipvs *ipvs; 497 struct task_struct *task; /* task if running */ 498 struct ip_vs_est_tick_data __rcu *ticks[IPVS_EST_NTICKS]; 499 DECLARE_BITMAP(avail, IPVS_EST_NTICKS); /* tick has space for ests */ 500 unsigned long est_timer; /* estimation timer (jiffies) */ 501 struct ip_vs_stats *calc_stats; /* Used for calculation */ 502 int needed; /* task is needed */ 503 int tick_len[IPVS_EST_NTICKS]; /* est count */ 504 int id; /* ktid per netns */ 505 int chain_max; /* max ests per tick chain */ 506 int tick_max; /* max ests per tick */ 507 int est_count; /* attached ests to kthread */ 508 int est_max_count; /* max ests per kthread */ 509 int add_row; /* row for new ests */ 510 int est_row; /* estimated row */ 511 }; 512 513 /* IPVS resizable hash tables */ 514 struct ip_vs_rht { 515 struct hlist_bl_head *buckets; 516 struct ip_vs_rht __rcu *new_tbl; /* New/Same table */ 517 seqcount_t *seqc; /* Protects moves */ 518 struct ip_vs_aligned_lock *lock; /* Protect seqc */ 519 int mask; /* Buckets mask */ 520 int size; /* Buckets */ 521 int seqc_mask; /* seqc mask */ 522 int lock_mask; /* lock mask */ 523 u32 table_id; 524 int u_thresh; /* upper threshold */ 525 int l_thresh; /* lower threshold */ 526 int lfactor; /* Load Factor (shift)*/ 527 int bits; /* size = 1 << bits */ 528 siphash_key_t hash_key; 529 struct rcu_head rcu_head; 530 }; 531 532 /** 533 * ip_vs_rht_for_each_table() - Walk the hash tables 534 * @table: struct ip_vs_rht __rcu *table 535 * @t: current table, used as cursor, struct ip_vs_rht *var 536 * @p: previous table, temp struct ip_vs_rht *var 537 * 538 * Walk tables assuming others can not change the installed tables 539 */ 540 #define ip_vs_rht_for_each_table(table, t, p) \ 541 for (p = NULL, t = rcu_dereference_protected(table, 1); \ 542 t != p; \ 543 p = t, t = rcu_dereference_protected(t->new_tbl, 1)) 544 545 /** 546 * ip_vs_rht_for_each_table_rcu() - Walk the hash tables under RCU reader lock 547 * @table: struct ip_vs_rht __rcu *table 548 * @t: current table, used as cursor, struct ip_vs_rht *var 549 * @p: previous table, temp struct ip_vs_rht *var 550 * 551 * We usually search in one table and also in second table on resizing 552 */ 553 #define ip_vs_rht_for_each_table_rcu(table, t, p) \ 554 for (p = NULL, t = rcu_dereference(table); \ 555 t != p; \ 556 p = t, t = rcu_dereference(t->new_tbl)) 557 558 /** 559 * ip_vs_rht_for_each_bucket() - Walk all table buckets 560 * @t: current table, used as cursor, struct ip_vs_rht *var 561 * @bucket: bucket index, used as cursor, u32 var 562 * @head: bucket address, used as cursor, struct hlist_bl_head *var 563 */ 564 #define ip_vs_rht_for_each_bucket(t, bucket, head) \ 565 for (bucket = 0, head = (t)->buckets; \ 566 bucket < t->size; bucket++, head++) 567 568 /** 569 * ip_vs_rht_for_bucket_retry() - Retry bucket if entries are moved 570 * @t: current table, used as cursor, struct ip_vs_rht *var 571 * @bucket: index of current bucket or hash key 572 * @sc: temp seqcount_t *var 573 * @seq: temp unsigned int var for sequence count 574 * @retry: temp int var 575 */ 576 #define ip_vs_rht_for_bucket_retry(t, bucket, sc, seq, retry) \ 577 for (retry = 1, sc = &(t)->seqc[(bucket) & (t)->seqc_mask]; \ 578 retry && ({ seq = read_seqcount_begin(sc); 1; }); \ 579 retry = read_seqcount_retry(sc, seq)) 580 581 /** 582 * DECLARE_IP_VS_RHT_WALK_BUCKETS_RCU() - Declare variables 583 * 584 * Variables for ip_vs_rht_walk_buckets_rcu 585 */ 586 #define DECLARE_IP_VS_RHT_WALK_BUCKETS_RCU() \ 587 struct ip_vs_rht *_t, *_p; \ 588 unsigned int _seq; \ 589 seqcount_t *_sc; \ 590 u32 _bucket; \ 591 int _retry 592 /** 593 * ip_vs_rht_walk_buckets_rcu() - Walk all buckets under RCU read lock 594 * @table: struct ip_vs_rht __rcu *table 595 * @head: bucket address, used as cursor, struct hlist_bl_head *var 596 * 597 * Can be used while others add/delete/move entries 598 * Not suitable if duplicates are not desired 599 * Possible cases for reader that uses cond_resched_rcu() in the loop: 600 * - new table can not be installed, no need to repeat 601 * - new table can be installed => check and repeat if new table is 602 * installed, needed for !PREEMPT_RCU 603 */ 604 #define ip_vs_rht_walk_buckets_rcu(table, head) \ 605 ip_vs_rht_for_each_table_rcu(table, _t, _p) \ 606 ip_vs_rht_for_each_bucket(_t, _bucket, head) \ 607 ip_vs_rht_for_bucket_retry(_t, _bucket, _sc, \ 608 _seq, _retry) 609 610 /** 611 * DECLARE_IP_VS_RHT_WALK_BUCKET_RCU() - Declare variables 612 * 613 * Variables for ip_vs_rht_walk_bucket_rcu 614 */ 615 #define DECLARE_IP_VS_RHT_WALK_BUCKET_RCU() \ 616 unsigned int _seq; \ 617 seqcount_t *_sc; \ 618 int _retry 619 /** 620 * ip_vs_rht_walk_bucket_rcu() - Walk bucket under RCU read lock 621 * @t: current table, struct ip_vs_rht *var 622 * @bucket: index of current bucket or hash key 623 * @head: bucket address, used as cursor, struct hlist_bl_head *var 624 * 625 * Can be used while others add/delete/move entries 626 * Not suitable if duplicates are not desired 627 * Possible cases for reader that uses cond_resched_rcu() in the loop: 628 * - new table can not be installed, no need to repeat 629 * - new table can be installed => check and repeat if new table is 630 * installed, needed for !PREEMPT_RCU 631 */ 632 #define ip_vs_rht_walk_bucket_rcu(t, bucket, head) \ 633 if (({ head = (t)->buckets + ((bucket) & (t)->mask); 0; })) \ 634 {} \ 635 else \ 636 ip_vs_rht_for_bucket_retry(t, (bucket), _sc, _seq, _retry) 637 638 /** 639 * DECLARE_IP_VS_RHT_WALK_BUCKETS_SAFE_RCU() - Declare variables 640 * 641 * Variables for ip_vs_rht_walk_buckets_safe_rcu 642 */ 643 #define DECLARE_IP_VS_RHT_WALK_BUCKETS_SAFE_RCU() \ 644 struct ip_vs_rht *_t, *_p; \ 645 u32 _bucket 646 /** 647 * ip_vs_rht_walk_buckets_safe_rcu() - Walk all buckets under RCU read lock 648 * @table: struct ip_vs_rht __rcu *table 649 * @head: bucket address, used as cursor, struct hlist_bl_head *var 650 * 651 * Can be used while others add/delete entries but moving is disabled 652 * Using cond_resched_rcu() should be safe if tables do not change 653 */ 654 #define ip_vs_rht_walk_buckets_safe_rcu(table, head) \ 655 ip_vs_rht_for_each_table_rcu(table, _t, _p) \ 656 ip_vs_rht_for_each_bucket(_t, _bucket, head) 657 658 /** 659 * DECLARE_IP_VS_RHT_WALK_BUCKETS() - Declare variables 660 * 661 * Variables for ip_vs_rht_walk_buckets 662 */ 663 #define DECLARE_IP_VS_RHT_WALK_BUCKETS() \ 664 struct ip_vs_rht *_t, *_p; \ 665 u32 _bucket 666 667 /** 668 * ip_vs_rht_walk_buckets() - Walk all buckets 669 * @table: struct ip_vs_rht __rcu *table 670 * @head: bucket address, used as cursor, struct hlist_bl_head *var 671 * 672 * Use if others can not add/delete/move entries 673 */ 674 #define ip_vs_rht_walk_buckets(table, head) \ 675 ip_vs_rht_for_each_table(table, _t, _p) \ 676 ip_vs_rht_for_each_bucket(_t, _bucket, head) 677 678 /* Entries can be in one of two tables, so we flip bit when new table is 679 * created and store it as highest bit in hash keys 680 */ 681 #define IP_VS_RHT_TABLE_ID_MASK BIT(31) 682 683 /* Check if hash key is from this table */ 684 static inline bool ip_vs_rht_same_table(struct ip_vs_rht *t, u32 hash_key) 685 { 686 return !((t->table_id ^ hash_key) & IP_VS_RHT_TABLE_ID_MASK); 687 } 688 689 /* Build per-table hash key from hash value */ 690 static inline u32 ip_vs_rht_build_hash_key(struct ip_vs_rht *t, u32 hash) 691 { 692 return t->table_id | (hash & ~IP_VS_RHT_TABLE_ID_MASK); 693 } 694 695 void ip_vs_rht_free(struct ip_vs_rht *t); 696 void ip_vs_rht_rcu_free(struct rcu_head *head); 697 struct ip_vs_rht *ip_vs_rht_alloc(int buckets, int scounts, int locks); 698 int ip_vs_rht_desired_size(struct netns_ipvs *ipvs, struct ip_vs_rht *t, int n, 699 int lfactor, int min_bits, int max_bits); 700 void ip_vs_rht_set_thresholds(struct ip_vs_rht *t, int size, int lfactor, 701 int min_bits, int max_bits); 702 u32 ip_vs_rht_hash_linfo(struct ip_vs_rht *t, int af, 703 const union nf_inet_addr *addr, u32 v1, u32 v2); 704 705 struct dst_entry; 706 struct iphdr; 707 struct ip_vs_conn; 708 struct ip_vs_app; 709 struct sk_buff; 710 struct ip_vs_proto_data; 711 712 struct ip_vs_protocol { 713 struct ip_vs_protocol *next; 714 char *name; 715 u16 protocol; 716 u16 num_states; 717 int dont_defrag; 718 719 void (*init)(struct ip_vs_protocol *pp); 720 721 void (*exit)(struct ip_vs_protocol *pp); 722 723 int (*init_netns)(struct netns_ipvs *ipvs, struct ip_vs_proto_data *pd); 724 725 void (*exit_netns)(struct netns_ipvs *ipvs, struct ip_vs_proto_data *pd); 726 727 int (*conn_schedule)(struct netns_ipvs *ipvs, 728 int af, struct sk_buff *skb, 729 struct ip_vs_proto_data *pd, 730 int *verdict, struct ip_vs_conn **cpp, 731 struct ip_vs_iphdr *iph); 732 733 struct ip_vs_conn * 734 (*conn_in_get)(struct netns_ipvs *ipvs, 735 int af, 736 const struct sk_buff *skb, 737 const struct ip_vs_iphdr *iph); 738 739 struct ip_vs_conn * 740 (*conn_out_get)(struct netns_ipvs *ipvs, 741 int af, 742 const struct sk_buff *skb, 743 const struct ip_vs_iphdr *iph); 744 745 int (*snat_handler)(struct sk_buff *skb, struct ip_vs_protocol *pp, 746 struct ip_vs_conn *cp, struct ip_vs_iphdr *iph); 747 748 int (*dnat_handler)(struct sk_buff *skb, struct ip_vs_protocol *pp, 749 struct ip_vs_conn *cp, struct ip_vs_iphdr *iph); 750 751 const char *(*state_name)(int state); 752 753 void (*state_transition)(struct ip_vs_conn *cp, int direction, 754 const struct sk_buff *skb, 755 struct ip_vs_proto_data *pd); 756 757 int (*register_app)(struct netns_ipvs *ipvs, struct ip_vs_app *inc); 758 759 void (*unregister_app)(struct netns_ipvs *ipvs, struct ip_vs_app *inc); 760 761 int (*app_conn_bind)(struct ip_vs_conn *cp); 762 763 void (*debug_packet)(int af, struct ip_vs_protocol *pp, 764 const struct sk_buff *skb, 765 int offset, 766 const char *msg); 767 768 void (*timeout_change)(struct ip_vs_proto_data *pd, int flags); 769 }; 770 771 /* protocol data per netns */ 772 struct ip_vs_proto_data { 773 struct ip_vs_proto_data *next; 774 struct ip_vs_protocol *pp; 775 int *timeout_table; /* protocol timeout table */ 776 atomic_t appcnt; /* counter of proto app incs. */ 777 struct tcp_states_t *tcp_state_table; 778 }; 779 780 struct ip_vs_protocol *ip_vs_proto_get(unsigned short proto); 781 struct ip_vs_proto_data *ip_vs_proto_data_get(struct netns_ipvs *ipvs, 782 unsigned short proto); 783 784 struct ip_vs_conn_param { 785 struct netns_ipvs *ipvs; 786 const union nf_inet_addr *caddr; 787 const union nf_inet_addr *vaddr; 788 __be16 cport; 789 __be16 vport; 790 __u16 protocol; 791 u16 af; 792 793 const struct ip_vs_pe *pe; 794 char *pe_data; 795 __u8 pe_data_len; 796 }; 797 798 /* Hash node in conn_tab */ 799 struct ip_vs_conn_hnode { 800 struct hlist_bl_node node; /* node in conn_tab */ 801 u32 hash_key; /* Key for the hash table */ 802 u8 dir; /* 0=out->in, 1=in->out */ 803 } __packed; 804 805 /* IP_VS structure allocated for each dynamically scheduled connection */ 806 struct ip_vs_conn { 807 /* Cacheline for hash table nodes - rarely modified */ 808 809 struct ip_vs_conn_hnode hn0; /* Original direction */ 810 u8 af; /* address family */ 811 __be16 cport; 812 struct ip_vs_conn_hnode hn1; /* Reply direction */ 813 u8 daf; /* Address family of the dest */ 814 __be16 dport; 815 struct ip_vs_dest *dest; /* real server */ 816 atomic_t n_control; /* Number of controlled ones */ 817 volatile __u32 flags; /* status flags */ 818 /* 44/64 */ 819 820 struct ip_vs_conn *control; /* Master control connection */ 821 const struct ip_vs_pe *pe; 822 char *pe_data; 823 __u8 pe_data_len; 824 volatile __u16 state; /* state info */ 825 volatile __u16 old_state; /* old state, to be used for 826 * state transition triggered 827 * synchronization 828 */ 829 /* 2-byte hole */ 830 /* 64/96 */ 831 832 union nf_inet_addr caddr; /* client address */ 833 union nf_inet_addr vaddr; /* virtual address */ 834 /* 96/128 */ 835 836 union nf_inet_addr daddr; /* destination address */ 837 __u32 fwmark; /* Fire wall mark from skb */ 838 __be16 vport; 839 __u16 protocol; /* Which protocol (TCP/UDP) */ 840 841 /* Note: we can group the following members into a structure, 842 * in order to save more space, and the following members are 843 * only used in VS/NAT anyway 844 */ 845 struct ip_vs_app *app; /* bound ip_vs_app object */ 846 void *app_data; /* Application private data */ 847 /* 128/168 */ 848 struct_group(sync_conn_opt, 849 struct ip_vs_seq in_seq; /* incoming seq. struct */ 850 struct ip_vs_seq out_seq; /* outgoing seq. struct */ 851 ); 852 /* 152/192 */ 853 854 struct timer_list timer; /* Expiration timer */ 855 volatile unsigned long timeout; /* timeout */ 856 spinlock_t lock; /* lock for state transition */ 857 refcount_t refcnt; /* reference count */ 858 atomic_t in_pkts; /* incoming packet counter */ 859 /* 64-bit: 4-byte gap */ 860 861 /* 188/256 */ 862 unsigned long sync_endtime; /* jiffies + sent_retries */ 863 struct netns_ipvs *ipvs; 864 865 /* Packet transmitter for different forwarding methods. If it 866 * mangles the packet, it must return NF_DROP or better NF_STOLEN, 867 * otherwise this must be changed to a sk_buff **. 868 * NF_ACCEPT can be returned when destination is local. 869 */ 870 int (*packet_xmit)(struct sk_buff *skb, struct ip_vs_conn *cp, 871 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 872 873 struct rcu_head rcu_head; 874 }; 875 876 /* Extended internal versions of struct ip_vs_service_user and ip_vs_dest_user 877 * for IPv6 support. 878 * 879 * We need these to conveniently pass around service and destination 880 * options, but unfortunately, we also need to keep the old definitions to 881 * maintain userspace backwards compatibility for the setsockopt interface. 882 */ 883 struct ip_vs_service_user_kern { 884 /* virtual service addresses */ 885 u16 af; 886 u16 protocol; 887 union nf_inet_addr addr; /* virtual ip address */ 888 __be16 port; 889 u32 fwmark; /* firewall mark of service */ 890 891 /* virtual service options */ 892 char *sched_name; 893 char *pe_name; 894 unsigned int flags; /* virtual service flags */ 895 unsigned int timeout; /* persistent timeout in sec */ 896 __be32 netmask; /* persistent netmask or plen */ 897 }; 898 899 900 struct ip_vs_dest_user_kern { 901 /* destination server address */ 902 union nf_inet_addr addr; 903 __be16 port; 904 905 /* real server options */ 906 unsigned int conn_flags; /* connection flags */ 907 int weight; /* destination weight */ 908 909 /* thresholds for active connections */ 910 u32 u_threshold; /* upper threshold */ 911 u32 l_threshold; /* lower threshold */ 912 913 /* Address family of addr */ 914 u16 af; 915 916 u16 tun_type; /* tunnel type */ 917 __be16 tun_port; /* tunnel port */ 918 u16 tun_flags; /* tunnel flags */ 919 }; 920 921 922 /* 923 * The information about the virtual service offered to the net and the 924 * forwarding entries. 925 */ 926 struct ip_vs_service { 927 struct hlist_bl_node s_list; /* node in service table */ 928 u32 hash_key; /* Key for the hash table */ 929 u16 af; /* address family */ 930 __u16 protocol; /* which protocol (TCP/UDP) */ 931 932 union nf_inet_addr addr; /* IP address for virtual service */ 933 __u32 fwmark; /* firewall mark of the service */ 934 atomic_t refcnt; /* reference counter */ 935 __be16 port; /* port number for the service */ 936 unsigned int flags; /* service status flags */ 937 unsigned int timeout; /* persistent timeout in ticks */ 938 __be32 netmask; /* grouping granularity, mask/plen */ 939 struct netns_ipvs *ipvs; 940 941 struct list_head destinations; /* real server d-linked list */ 942 __u32 num_dests; /* number of servers */ 943 struct ip_vs_stats stats; /* statistics for the service */ 944 945 /* for scheduling */ 946 struct ip_vs_scheduler __rcu *scheduler; /* bound scheduler object */ 947 spinlock_t sched_lock; /* lock sched_data */ 948 void *sched_data; /* scheduler application data */ 949 950 /* alternate persistence engine */ 951 struct ip_vs_pe __rcu *pe; 952 int conntrack_afmask; 953 954 struct rcu_head rcu_head; 955 }; 956 957 /* Information for cached dst */ 958 struct ip_vs_dest_dst { 959 struct dst_entry *dst_cache; /* destination cache entry */ 960 u32 dst_cookie; 961 union nf_inet_addr dst_saddr; 962 struct rcu_head rcu_head; 963 }; 964 965 /* The real server destination forwarding entry with ip address, port number, 966 * and so on. 967 */ 968 struct ip_vs_dest { 969 struct list_head n_list; /* for the dests in the service */ 970 struct hlist_node d_list; /* for table with all the dests */ 971 972 u16 af; /* address family */ 973 __be16 port; /* port number of the server */ 974 union nf_inet_addr addr; /* IP address of the server */ 975 volatile unsigned int flags; /* dest status flags */ 976 atomic_t conn_flags; /* flags to copy to conn */ 977 atomic_t weight; /* server weight */ 978 atomic_t last_weight; /* server latest weight */ 979 __u16 tun_type; /* tunnel type */ 980 __be16 tun_port; /* tunnel port */ 981 __u16 tun_flags; /* tunnel flags */ 982 983 refcount_t refcnt; /* reference counter */ 984 struct ip_vs_stats stats; /* statistics */ 985 unsigned long idle_start; /* start time, jiffies */ 986 987 /* connection counters and thresholds */ 988 atomic_t activeconns; /* active connections */ 989 atomic_t inactconns; /* inactive connections */ 990 atomic_t persistconns; /* persistent connections */ 991 __u32 u_threshold; /* upper threshold */ 992 __u32 l_threshold; /* lower threshold */ 993 994 /* for destination cache */ 995 spinlock_t dst_lock; /* lock of dst_cache */ 996 struct ip_vs_dest_dst __rcu *dest_dst; /* cached dst info */ 997 998 /* for virtual service */ 999 struct ip_vs_service __rcu *svc; /* service it belongs to */ 1000 __u16 protocol; /* which protocol (TCP/UDP) */ 1001 __be16 vport; /* virtual port number */ 1002 union nf_inet_addr vaddr; /* virtual IP address */ 1003 __u32 vfwmark; /* firewall mark of service */ 1004 1005 struct rcu_head rcu_head; 1006 struct list_head t_list; /* in dest_trash */ 1007 unsigned int in_rs_table:1; /* we are in rs_table */ 1008 }; 1009 1010 /* The scheduler object */ 1011 struct ip_vs_scheduler { 1012 struct list_head n_list; /* d-linked list head */ 1013 char *name; /* scheduler name */ 1014 atomic_t refcnt; /* reference counter */ 1015 struct module *module; /* THIS_MODULE/NULL */ 1016 1017 /* scheduler initializing service */ 1018 int (*init_service)(struct ip_vs_service *svc); 1019 /* scheduling service finish */ 1020 void (*done_service)(struct ip_vs_service *svc); 1021 /* dest is linked */ 1022 int (*add_dest)(struct ip_vs_service *svc, struct ip_vs_dest *dest); 1023 /* dest is unlinked */ 1024 int (*del_dest)(struct ip_vs_service *svc, struct ip_vs_dest *dest); 1025 /* dest is updated */ 1026 int (*upd_dest)(struct ip_vs_service *svc, struct ip_vs_dest *dest); 1027 1028 /* selecting a server from the given service */ 1029 struct ip_vs_dest* (*schedule)(struct ip_vs_service *svc, 1030 const struct sk_buff *skb, 1031 struct ip_vs_iphdr *iph); 1032 }; 1033 1034 /* The persistence engine object */ 1035 struct ip_vs_pe { 1036 struct list_head n_list; /* d-linked list head */ 1037 char *name; /* scheduler name */ 1038 atomic_t refcnt; /* reference counter */ 1039 struct module *module; /* THIS_MODULE/NULL */ 1040 1041 /* get the connection template, if any */ 1042 int (*fill_param)(struct ip_vs_conn_param *p, struct sk_buff *skb); 1043 bool (*ct_match)(const struct ip_vs_conn_param *p, 1044 struct ip_vs_conn *ct); 1045 u32 (*hashkey_raw)(const struct ip_vs_conn_param *p, 1046 struct ip_vs_rht *t, bool inverse); 1047 int (*show_pe_data)(const struct ip_vs_conn *cp, char *buf); 1048 /* create connections for real-server outgoing packets */ 1049 struct ip_vs_conn* (*conn_out)(struct ip_vs_service *svc, 1050 struct ip_vs_dest *dest, 1051 struct sk_buff *skb, 1052 const struct ip_vs_iphdr *iph, 1053 __be16 dport, __be16 cport); 1054 }; 1055 1056 /* The application module object (a.k.a. app incarnation) */ 1057 struct ip_vs_app { 1058 struct list_head a_list; /* member in app list */ 1059 int type; /* IP_VS_APP_TYPE_xxx */ 1060 char *name; /* application module name */ 1061 __u16 protocol; 1062 struct module *module; /* THIS_MODULE/NULL */ 1063 struct list_head incs_list; /* list of incarnations */ 1064 1065 /* members for application incarnations */ 1066 struct list_head p_list; /* member in proto app list */ 1067 struct ip_vs_app *app; /* its real application */ 1068 __be16 port; /* port number in net order */ 1069 atomic_t usecnt; /* usage counter */ 1070 struct rcu_head rcu_head; 1071 1072 /* output hook: Process packet in inout direction, diff set for TCP. 1073 * Return: 0=Error, 1=Payload Not Mangled/Mangled but checksum is ok, 1074 * 2=Mangled but checksum was not updated 1075 */ 1076 int (*pkt_out)(struct ip_vs_app *, struct ip_vs_conn *, 1077 struct sk_buff *, int *diff, struct ip_vs_iphdr *ipvsh); 1078 1079 /* input hook: Process packet in outin direction, diff set for TCP. 1080 * Return: 0=Error, 1=Payload Not Mangled/Mangled but checksum is ok, 1081 * 2=Mangled but checksum was not updated 1082 */ 1083 int (*pkt_in)(struct ip_vs_app *, struct ip_vs_conn *, 1084 struct sk_buff *, int *diff, struct ip_vs_iphdr *ipvsh); 1085 1086 /* ip_vs_app initializer */ 1087 int (*init_conn)(struct ip_vs_app *, struct ip_vs_conn *); 1088 1089 /* ip_vs_app finish */ 1090 int (*done_conn)(struct ip_vs_app *, struct ip_vs_conn *); 1091 1092 1093 /* not used now */ 1094 int (*bind_conn)(struct ip_vs_app *, struct ip_vs_conn *, 1095 struct ip_vs_protocol *); 1096 1097 void (*unbind_conn)(struct ip_vs_app *, struct ip_vs_conn *); 1098 1099 int * timeout_table; 1100 int * timeouts; 1101 int timeouts_size; 1102 1103 int (*conn_schedule)(struct sk_buff *skb, struct ip_vs_app *app, 1104 int *verdict, struct ip_vs_conn **cpp); 1105 1106 struct ip_vs_conn * 1107 (*conn_in_get)(const struct sk_buff *skb, struct ip_vs_app *app, 1108 const struct iphdr *iph, int inverse); 1109 1110 struct ip_vs_conn * 1111 (*conn_out_get)(const struct sk_buff *skb, struct ip_vs_app *app, 1112 const struct iphdr *iph, int inverse); 1113 1114 int (*state_transition)(struct ip_vs_conn *cp, int direction, 1115 const struct sk_buff *skb, 1116 struct ip_vs_app *app); 1117 1118 void (*timeout_change)(struct ip_vs_app *app, int flags); 1119 }; 1120 1121 struct ipvs_master_sync_state { 1122 struct list_head sync_queue; 1123 struct ip_vs_sync_buff *sync_buff; 1124 unsigned long sync_queue_len; 1125 unsigned int sync_queue_delay; 1126 struct delayed_work master_wakeup_work; 1127 struct netns_ipvs *ipvs; 1128 }; 1129 1130 struct ip_vs_sync_thread_data; 1131 1132 /* How much time to keep dests in trash */ 1133 #define IP_VS_DEST_TRASH_PERIOD (120 * HZ) 1134 1135 struct ipvs_sync_daemon_cfg { 1136 union nf_inet_addr mcast_group; 1137 int syncid; 1138 u16 sync_maxlen; 1139 u16 mcast_port; 1140 u8 mcast_af; 1141 u8 mcast_ttl; 1142 /* multicast interface name */ 1143 char mcast_ifn[IP_VS_IFNAME_MAXLEN]; 1144 }; 1145 1146 /* IPVS in network namespace */ 1147 struct netns_ipvs { 1148 int gen; /* Generation */ 1149 int enable; /* enable like nf_hooks do */ 1150 /* Hash table: for real service lookups */ 1151 #define IP_VS_RTAB_BITS 4 1152 #define IP_VS_RTAB_SIZE (1 << IP_VS_RTAB_BITS) 1153 #define IP_VS_RTAB_MASK (IP_VS_RTAB_SIZE - 1) 1154 1155 struct hlist_head rs_table[IP_VS_RTAB_SIZE]; 1156 /* ip_vs_app */ 1157 struct list_head app_list; 1158 /* ip_vs_proto */ 1159 #define IP_VS_PROTO_TAB_SIZE 32 /* must be power of 2 */ 1160 struct ip_vs_proto_data *proto_data_table[IP_VS_PROTO_TAB_SIZE]; 1161 /* ip_vs_proto_tcp */ 1162 #ifdef CONFIG_IP_VS_PROTO_TCP 1163 #define TCP_APP_TAB_BITS 4 1164 #define TCP_APP_TAB_SIZE (1 << TCP_APP_TAB_BITS) 1165 #define TCP_APP_TAB_MASK (TCP_APP_TAB_SIZE - 1) 1166 struct list_head tcp_apps[TCP_APP_TAB_SIZE]; 1167 #endif 1168 /* ip_vs_proto_udp */ 1169 #ifdef CONFIG_IP_VS_PROTO_UDP 1170 #define UDP_APP_TAB_BITS 4 1171 #define UDP_APP_TAB_SIZE (1 << UDP_APP_TAB_BITS) 1172 #define UDP_APP_TAB_MASK (UDP_APP_TAB_SIZE - 1) 1173 struct list_head udp_apps[UDP_APP_TAB_SIZE]; 1174 #endif 1175 /* ip_vs_proto_sctp */ 1176 #ifdef CONFIG_IP_VS_PROTO_SCTP 1177 #define SCTP_APP_TAB_BITS 4 1178 #define SCTP_APP_TAB_SIZE (1 << SCTP_APP_TAB_BITS) 1179 #define SCTP_APP_TAB_MASK (SCTP_APP_TAB_SIZE - 1) 1180 /* Hash table for SCTP application incarnations */ 1181 struct list_head sctp_apps[SCTP_APP_TAB_SIZE]; 1182 #endif 1183 /* ip_vs_conn */ 1184 atomic_t conn_count; /* connection counter */ 1185 atomic_t no_cport_conns[IP_VS_AF_MAX]; 1186 struct delayed_work conn_resize_work;/* resize conn_tab */ 1187 1188 /* ip_vs_ctl */ 1189 struct ip_vs_stats_rcu *tot_stats; /* Statistics & est. */ 1190 1191 /* Trash for destinations */ 1192 struct list_head dest_trash; 1193 spinlock_t dest_trash_lock; 1194 struct timer_list dest_trash_timer; /* expiration timer */ 1195 struct mutex service_mutex; /* service reconfig */ 1196 struct rw_semaphore svc_resize_sem; /* svc_table resizing */ 1197 struct rw_semaphore svc_replace_sem; /* svc_table replace */ 1198 struct delayed_work svc_resize_work; /* resize svc_table */ 1199 atomic_t svc_table_changes;/* ++ on table changes */ 1200 /* Service counters */ 1201 atomic_t num_services[IP_VS_AF_MAX]; /* Services */ 1202 atomic_t fwm_services[IP_VS_AF_MAX]; /* Services */ 1203 atomic_t nonfwm_services[IP_VS_AF_MAX];/* Services */ 1204 atomic_t ftpsvc_counter[IP_VS_AF_MAX]; /* FTPPORT */ 1205 atomic_t nullsvc_counter[IP_VS_AF_MAX];/* Zero port */ 1206 atomic_t conn_out_counter[IP_VS_AF_MAX];/* out conn */ 1207 1208 #ifdef CONFIG_SYSCTL 1209 /* delayed work for expiring no dest connections */ 1210 struct delayed_work expire_nodest_conn_work; 1211 /* 1/rate drop and drop-entry variables */ 1212 struct delayed_work defense_work; /* Work handler */ 1213 int drop_rate; 1214 int drop_counter; 1215 int old_secure_tcp; 1216 atomic_t dropentry; 1217 s8 dropentry_counters[8]; 1218 /* locks in ctl.c */ 1219 spinlock_t dropentry_lock; /* drop entry handling */ 1220 spinlock_t droppacket_lock; /* drop packet handling */ 1221 spinlock_t securetcp_lock; /* state and timeout tables */ 1222 1223 /* sys-ctl struct */ 1224 struct ctl_table_header *sysctl_hdr; 1225 struct ctl_table *sysctl_tbl; 1226 #endif 1227 1228 /* sysctl variables */ 1229 int sysctl_amemthresh; 1230 int sysctl_am_droprate; 1231 #ifdef CONFIG_SYSCTL 1232 int sysctl_conn_max;/* soft limit for conns */ 1233 int conn_max_limit; /* hard limit for conn_max */ 1234 #endif 1235 int sysctl_drop_entry; 1236 int sysctl_drop_packet; 1237 int sysctl_secure_tcp; 1238 #ifdef CONFIG_IP_VS_NFCT 1239 int sysctl_conntrack; 1240 #endif 1241 int sysctl_snat_reroute; 1242 int sysctl_sync_ver; 1243 int sysctl_sync_ports; 1244 int sysctl_sync_persist_mode; 1245 unsigned long sysctl_sync_qlen_max; 1246 int sysctl_sync_sock_size; 1247 int sysctl_cache_bypass; 1248 int sysctl_expire_nodest_conn; 1249 int sysctl_sloppy_tcp; 1250 int sysctl_sloppy_sctp; 1251 int sysctl_expire_quiescent_template; 1252 int sysctl_sync_threshold[2]; 1253 unsigned int sysctl_sync_refresh_period; 1254 int sysctl_sync_retries; 1255 int sysctl_nat_icmp_send; 1256 int sysctl_pmtu_disc; 1257 int sysctl_backup_only; 1258 int sysctl_conn_reuse_mode; 1259 int sysctl_schedule_icmp; 1260 int sysctl_ignore_tunneled; 1261 int sysctl_run_estimation; 1262 #ifdef CONFIG_SYSCTL 1263 cpumask_var_t sysctl_est_cpulist; /* kthread cpumask */ 1264 int est_cpulist_valid; /* cpulist set */ 1265 int sysctl_est_nice; /* kthread nice */ 1266 int est_stopped; /* stop tasks */ 1267 #endif 1268 int sysctl_conn_lfactor; 1269 int sysctl_svc_lfactor; 1270 1271 /* ip_vs_lblc */ 1272 int sysctl_lblc_expiration; 1273 struct ctl_table_header *lblc_ctl_header; 1274 struct ctl_table *lblc_ctl_table; 1275 /* ip_vs_lblcr */ 1276 int sysctl_lblcr_expiration; 1277 struct ctl_table_header *lblcr_ctl_header; 1278 struct ctl_table *lblcr_ctl_table; 1279 unsigned long work_flags; /* IP_VS_WORK_* flags */ 1280 /* ip_vs_est */ 1281 struct delayed_work est_reload_work;/* Reload kthread tasks */ 1282 struct mutex est_mutex; /* protect kthread tasks */ 1283 struct hlist_head est_temp_list; /* Ests during calc phase */ 1284 struct ip_vs_est_kt_data **est_kt_arr; /* Array of kthread data ptrs */ 1285 unsigned long est_max_threads;/* Hard limit of kthreads */ 1286 int est_calc_phase; /* Calculation phase */ 1287 int est_chain_max; /* Calculated chain_max */ 1288 int est_kt_count; /* Allocated ptrs */ 1289 int est_add_ktid; /* ktid where to add ests */ 1290 atomic_t est_genid; /* kthreads reload genid */ 1291 atomic_t est_genid_done; /* applied genid */ 1292 /* ip_vs_sync */ 1293 spinlock_t sync_lock; 1294 struct ipvs_master_sync_state *ms; 1295 spinlock_t sync_buff_lock; 1296 struct ip_vs_sync_thread_data *master_tinfo; 1297 struct ip_vs_sync_thread_data *backup_tinfo; 1298 int threads_mask; 1299 volatile int sync_state; 1300 struct mutex sync_mutex; 1301 struct ipvs_sync_daemon_cfg mcfg; /* Master Configuration */ 1302 struct ipvs_sync_daemon_cfg bcfg; /* Backup Configuration */ 1303 /* net name space ptr */ 1304 struct net *net; /* Needed by timer routines */ 1305 /* Number of heterogeneous destinations, needed because heterogeneous 1306 * are not supported when synchronization is enabled. 1307 */ 1308 unsigned int mixed_address_family_dests; 1309 unsigned int hooks_afmask; /* &1=AF_INET, &2=AF_INET6 */ 1310 1311 struct ip_vs_rht __rcu *svc_table; /* Services */ 1312 struct ip_vs_rht __rcu *conn_tab; /* Connections */ 1313 atomic_t conn_tab_changes;/* ++ on new table */ 1314 }; 1315 1316 #define DEFAULT_SYNC_THRESHOLD 3 1317 #define DEFAULT_SYNC_PERIOD 50 1318 #define DEFAULT_SYNC_VER 1 1319 #define DEFAULT_SLOPPY_TCP 0 1320 #define DEFAULT_SLOPPY_SCTP 0 1321 #define DEFAULT_SYNC_REFRESH_PERIOD (0U * HZ) 1322 #define DEFAULT_SYNC_RETRIES 0 1323 #define IPVS_SYNC_WAKEUP_RATE 8 1324 #define IPVS_SYNC_QLEN_MAX (IPVS_SYNC_WAKEUP_RATE * 4) 1325 #define IPVS_SYNC_SEND_DELAY (HZ / 50) 1326 #define IPVS_SYNC_CHECK_PERIOD HZ 1327 #define IPVS_SYNC_FLUSH_TIME (HZ * 2) 1328 #define IPVS_SYNC_PORTS_MAX (1 << 6) 1329 1330 #ifdef CONFIG_SYSCTL 1331 1332 static inline int sysctl_conn_max(struct netns_ipvs *ipvs) 1333 { 1334 return READ_ONCE(ipvs->sysctl_conn_max); 1335 } 1336 1337 static inline int sysctl_sync_threshold(struct netns_ipvs *ipvs) 1338 { 1339 return ipvs->sysctl_sync_threshold[0]; 1340 } 1341 1342 static inline int sysctl_sync_period(struct netns_ipvs *ipvs) 1343 { 1344 return READ_ONCE(ipvs->sysctl_sync_threshold[1]); 1345 } 1346 1347 static inline unsigned int sysctl_sync_refresh_period(struct netns_ipvs *ipvs) 1348 { 1349 return READ_ONCE(ipvs->sysctl_sync_refresh_period); 1350 } 1351 1352 static inline int sysctl_sync_retries(struct netns_ipvs *ipvs) 1353 { 1354 return ipvs->sysctl_sync_retries; 1355 } 1356 1357 static inline int sysctl_sync_ver(struct netns_ipvs *ipvs) 1358 { 1359 return ipvs->sysctl_sync_ver; 1360 } 1361 1362 static inline int sysctl_sloppy_tcp(struct netns_ipvs *ipvs) 1363 { 1364 return ipvs->sysctl_sloppy_tcp; 1365 } 1366 1367 static inline int sysctl_sloppy_sctp(struct netns_ipvs *ipvs) 1368 { 1369 return ipvs->sysctl_sloppy_sctp; 1370 } 1371 1372 static inline int sysctl_sync_ports(struct netns_ipvs *ipvs) 1373 { 1374 return READ_ONCE(ipvs->sysctl_sync_ports); 1375 } 1376 1377 static inline int sysctl_sync_persist_mode(struct netns_ipvs *ipvs) 1378 { 1379 return ipvs->sysctl_sync_persist_mode; 1380 } 1381 1382 static inline unsigned long sysctl_sync_qlen_max(struct netns_ipvs *ipvs) 1383 { 1384 return ipvs->sysctl_sync_qlen_max; 1385 } 1386 1387 static inline int sysctl_sync_sock_size(struct netns_ipvs *ipvs) 1388 { 1389 return ipvs->sysctl_sync_sock_size; 1390 } 1391 1392 static inline int sysctl_pmtu_disc(struct netns_ipvs *ipvs) 1393 { 1394 return ipvs->sysctl_pmtu_disc; 1395 } 1396 1397 static inline int sysctl_backup_only(struct netns_ipvs *ipvs) 1398 { 1399 return ipvs->sync_state & IP_VS_STATE_BACKUP && 1400 ipvs->sysctl_backup_only; 1401 } 1402 1403 static inline int sysctl_conn_reuse_mode(struct netns_ipvs *ipvs) 1404 { 1405 return ipvs->sysctl_conn_reuse_mode; 1406 } 1407 1408 static inline int sysctl_expire_nodest_conn(struct netns_ipvs *ipvs) 1409 { 1410 return ipvs->sysctl_expire_nodest_conn; 1411 } 1412 1413 static inline int sysctl_schedule_icmp(struct netns_ipvs *ipvs) 1414 { 1415 return ipvs->sysctl_schedule_icmp; 1416 } 1417 1418 static inline int sysctl_ignore_tunneled(struct netns_ipvs *ipvs) 1419 { 1420 return ipvs->sysctl_ignore_tunneled; 1421 } 1422 1423 static inline int sysctl_cache_bypass(struct netns_ipvs *ipvs) 1424 { 1425 return ipvs->sysctl_cache_bypass; 1426 } 1427 1428 static inline int sysctl_run_estimation(struct netns_ipvs *ipvs) 1429 { 1430 return ipvs->sysctl_run_estimation; 1431 } 1432 1433 static inline const struct cpumask *__sysctl_est_cpulist(struct netns_ipvs *ipvs) 1434 { 1435 if (ipvs->est_cpulist_valid) 1436 return ipvs->sysctl_est_cpulist; 1437 else 1438 return housekeeping_cpumask(HK_TYPE_KTHREAD); 1439 } 1440 1441 static inline const struct cpumask *sysctl_est_preferred_cpulist(struct netns_ipvs *ipvs) 1442 { 1443 if (ipvs->est_cpulist_valid) 1444 return ipvs->sysctl_est_cpulist; 1445 else 1446 return NULL; 1447 } 1448 1449 static inline int sysctl_est_nice(struct netns_ipvs *ipvs) 1450 { 1451 return ipvs->sysctl_est_nice; 1452 } 1453 1454 #else 1455 1456 static inline int sysctl_conn_max(struct netns_ipvs *ipvs) 1457 { 1458 return IP_VS_CONN_MAX; 1459 } 1460 1461 static inline int sysctl_sync_threshold(struct netns_ipvs *ipvs) 1462 { 1463 return DEFAULT_SYNC_THRESHOLD; 1464 } 1465 1466 static inline int sysctl_sync_period(struct netns_ipvs *ipvs) 1467 { 1468 return DEFAULT_SYNC_PERIOD; 1469 } 1470 1471 static inline unsigned int sysctl_sync_refresh_period(struct netns_ipvs *ipvs) 1472 { 1473 return DEFAULT_SYNC_REFRESH_PERIOD; 1474 } 1475 1476 static inline int sysctl_sync_retries(struct netns_ipvs *ipvs) 1477 { 1478 return DEFAULT_SYNC_RETRIES & 3; 1479 } 1480 1481 static inline int sysctl_sync_ver(struct netns_ipvs *ipvs) 1482 { 1483 return DEFAULT_SYNC_VER; 1484 } 1485 1486 static inline int sysctl_sloppy_tcp(struct netns_ipvs *ipvs) 1487 { 1488 return DEFAULT_SLOPPY_TCP; 1489 } 1490 1491 static inline int sysctl_sloppy_sctp(struct netns_ipvs *ipvs) 1492 { 1493 return DEFAULT_SLOPPY_SCTP; 1494 } 1495 1496 static inline int sysctl_sync_ports(struct netns_ipvs *ipvs) 1497 { 1498 return 1; 1499 } 1500 1501 static inline int sysctl_sync_persist_mode(struct netns_ipvs *ipvs) 1502 { 1503 return 0; 1504 } 1505 1506 static inline unsigned long sysctl_sync_qlen_max(struct netns_ipvs *ipvs) 1507 { 1508 return IPVS_SYNC_QLEN_MAX; 1509 } 1510 1511 static inline int sysctl_sync_sock_size(struct netns_ipvs *ipvs) 1512 { 1513 return 0; 1514 } 1515 1516 static inline int sysctl_pmtu_disc(struct netns_ipvs *ipvs) 1517 { 1518 return 1; 1519 } 1520 1521 static inline int sysctl_backup_only(struct netns_ipvs *ipvs) 1522 { 1523 return 0; 1524 } 1525 1526 static inline int sysctl_conn_reuse_mode(struct netns_ipvs *ipvs) 1527 { 1528 return 1; 1529 } 1530 1531 static inline int sysctl_expire_nodest_conn(struct netns_ipvs *ipvs) 1532 { 1533 return 0; 1534 } 1535 1536 static inline int sysctl_schedule_icmp(struct netns_ipvs *ipvs) 1537 { 1538 return 0; 1539 } 1540 1541 static inline int sysctl_ignore_tunneled(struct netns_ipvs *ipvs) 1542 { 1543 return 0; 1544 } 1545 1546 static inline int sysctl_cache_bypass(struct netns_ipvs *ipvs) 1547 { 1548 return 0; 1549 } 1550 1551 static inline int sysctl_run_estimation(struct netns_ipvs *ipvs) 1552 { 1553 return 1; 1554 } 1555 1556 static inline const struct cpumask *__sysctl_est_cpulist(struct netns_ipvs *ipvs) 1557 { 1558 return housekeeping_cpumask(HK_TYPE_KTHREAD); 1559 } 1560 1561 static inline const struct cpumask *sysctl_est_preferred_cpulist(struct netns_ipvs *ipvs) 1562 { 1563 return NULL; 1564 } 1565 1566 static inline int sysctl_est_nice(struct netns_ipvs *ipvs) 1567 { 1568 return IPVS_EST_NICE; 1569 } 1570 1571 #endif 1572 1573 /* Get load factor to map conn_count/u_thresh to t->size */ 1574 static inline int sysctl_conn_lfactor(struct netns_ipvs *ipvs) 1575 { 1576 return READ_ONCE(ipvs->sysctl_conn_lfactor); 1577 } 1578 1579 /* Get load factor to map num_services/u_thresh to t->size 1580 * Smaller value decreases u_thresh to reduce collisions but increases 1581 * the table size 1582 * Returns factor where: 1583 * - <0: u_thresh = size >> -factor, eg. lfactor -2 = 25% load 1584 * - >=0: u_thresh = size << factor, eg. lfactor 1 = 200% load 1585 */ 1586 static inline int sysctl_svc_lfactor(struct netns_ipvs *ipvs) 1587 { 1588 return READ_ONCE(ipvs->sysctl_svc_lfactor); 1589 } 1590 1591 static inline bool sysctl_est_cpulist_empty(struct netns_ipvs *ipvs) 1592 { 1593 guard(rcu)(); 1594 return cpumask_empty(__sysctl_est_cpulist(ipvs)); 1595 } 1596 1597 static inline unsigned int sysctl_est_cpulist_weight(struct netns_ipvs *ipvs) 1598 { 1599 guard(rcu)(); 1600 return cpumask_weight(__sysctl_est_cpulist(ipvs)); 1601 } 1602 1603 /* IPVS core functions 1604 * (from ip_vs_core.c) 1605 */ 1606 const char *ip_vs_proto_name(unsigned int proto); 1607 void ip_vs_init_hash_table(struct list_head *table, int rows); 1608 struct ip_vs_conn *ip_vs_new_conn_out(struct ip_vs_service *svc, 1609 struct ip_vs_dest *dest, 1610 struct sk_buff *skb, 1611 const struct ip_vs_iphdr *iph, 1612 __be16 dport, 1613 __be16 cport); 1614 #define IP_VS_INIT_HASH_TABLE(t) ip_vs_init_hash_table((t), ARRAY_SIZE((t))) 1615 1616 #define IP_VS_APP_TYPE_FTP 1 1617 1618 /* ip_vs_conn handling functions 1619 * (from ip_vs_conn.c) 1620 */ 1621 enum { 1622 IP_VS_DIR_INPUT = 0, 1623 IP_VS_DIR_OUTPUT, 1624 IP_VS_DIR_INPUT_ONLY, 1625 IP_VS_DIR_LAST, 1626 }; 1627 1628 static inline void ip_vs_conn_fill_param(struct netns_ipvs *ipvs, int af, int protocol, 1629 const union nf_inet_addr *caddr, 1630 __be16 cport, 1631 const union nf_inet_addr *vaddr, 1632 __be16 vport, 1633 struct ip_vs_conn_param *p) 1634 { 1635 p->ipvs = ipvs; 1636 p->af = af; 1637 p->protocol = protocol; 1638 p->caddr = caddr; 1639 p->cport = cport; 1640 p->vaddr = vaddr; 1641 p->vport = vport; 1642 p->pe = NULL; 1643 p->pe_data = NULL; 1644 } 1645 1646 struct ip_vs_conn *ip_vs_conn_in_get(const struct ip_vs_conn_param *p); 1647 struct ip_vs_conn *ip_vs_ct_in_get(const struct ip_vs_conn_param *p); 1648 1649 struct ip_vs_conn * ip_vs_conn_in_get_proto(struct netns_ipvs *ipvs, int af, 1650 const struct sk_buff *skb, 1651 const struct ip_vs_iphdr *iph); 1652 1653 struct ip_vs_conn *ip_vs_conn_out_get(const struct ip_vs_conn_param *p); 1654 1655 struct ip_vs_conn * ip_vs_conn_out_get_proto(struct netns_ipvs *ipvs, int af, 1656 const struct sk_buff *skb, 1657 const struct ip_vs_iphdr *iph); 1658 1659 /* Get reference to gain full access to conn. 1660 * By default, RCU read-side critical sections have access only to 1661 * conn fields and its PE data, see ip_vs_conn_rcu_free() for reference. 1662 */ 1663 static inline bool __ip_vs_conn_get(struct ip_vs_conn *cp) 1664 { 1665 return refcount_inc_not_zero(&cp->refcnt); 1666 } 1667 1668 /* put back the conn without restarting its timer */ 1669 static inline void __ip_vs_conn_put(struct ip_vs_conn *cp) 1670 { 1671 smp_mb__before_atomic(); 1672 refcount_dec(&cp->refcnt); 1673 } 1674 void ip_vs_conn_put(struct ip_vs_conn *cp); 1675 void ip_vs_conn_fill_cport(struct ip_vs_conn *cp, __be16 cport); 1676 int ip_vs_conn_desired_size(struct netns_ipvs *ipvs, struct ip_vs_rht *t, 1677 int lfactor); 1678 struct ip_vs_rht *ip_vs_conn_tab_alloc(struct netns_ipvs *ipvs, int buckets, 1679 int lfactor); 1680 1681 static inline struct ip_vs_conn * 1682 ip_vs_hn0_to_conn(struct ip_vs_conn_hnode *hn) 1683 { 1684 return container_of(hn, struct ip_vs_conn, hn0); 1685 } 1686 1687 static inline struct ip_vs_conn * 1688 ip_vs_hn_to_conn(struct ip_vs_conn_hnode *hn) 1689 { 1690 return hn->dir ? container_of(hn, struct ip_vs_conn, hn1) : 1691 container_of(hn, struct ip_vs_conn, hn0); 1692 } 1693 1694 struct ip_vs_conn *ip_vs_conn_new(const struct ip_vs_conn_param *p, int dest_af, 1695 const union nf_inet_addr *daddr, 1696 __be16 dport, unsigned int flags, 1697 struct ip_vs_dest *dest, __u32 fwmark); 1698 void ip_vs_conn_expire_now(struct ip_vs_conn *cp); 1699 1700 const char *ip_vs_state_name(const struct ip_vs_conn *cp); 1701 1702 void ip_vs_tcp_conn_listen(struct ip_vs_conn *cp); 1703 int ip_vs_check_template(struct ip_vs_conn *ct, struct ip_vs_dest *cdest); 1704 void ip_vs_random_dropentry(struct netns_ipvs *ipvs); 1705 int ip_vs_conn_init(void); 1706 void ip_vs_conn_cleanup(void); 1707 1708 static inline void ip_vs_control_del(struct ip_vs_conn *cp) 1709 { 1710 struct ip_vs_conn *ctl_cp = cp->control; 1711 if (!ctl_cp) { 1712 IP_VS_ERR_BUF("request control DEL for uncontrolled: " 1713 "%s:%d to %s:%d\n", 1714 IP_VS_DBG_ADDR(cp->af, &cp->caddr), 1715 ntohs(cp->cport), 1716 IP_VS_DBG_ADDR(cp->af, &cp->vaddr), 1717 ntohs(cp->vport)); 1718 1719 return; 1720 } 1721 1722 IP_VS_DBG_BUF(7, "DELeting control for: " 1723 "cp.dst=%s:%d ctl_cp.dst=%s:%d\n", 1724 IP_VS_DBG_ADDR(cp->af, &cp->caddr), 1725 ntohs(cp->cport), 1726 IP_VS_DBG_ADDR(cp->af, &ctl_cp->caddr), 1727 ntohs(ctl_cp->cport)); 1728 1729 cp->control = NULL; 1730 if (atomic_read(&ctl_cp->n_control) == 0) { 1731 IP_VS_ERR_BUF("BUG control DEL with n=0 : " 1732 "%s:%d to %s:%d\n", 1733 IP_VS_DBG_ADDR(cp->af, &cp->caddr), 1734 ntohs(cp->cport), 1735 IP_VS_DBG_ADDR(cp->af, &cp->vaddr), 1736 ntohs(cp->vport)); 1737 1738 return; 1739 } 1740 atomic_dec(&ctl_cp->n_control); 1741 } 1742 1743 static inline void 1744 ip_vs_control_add(struct ip_vs_conn *cp, struct ip_vs_conn *ctl_cp) 1745 { 1746 if (cp->control) { 1747 IP_VS_ERR_BUF("request control ADD for already controlled: " 1748 "%s:%d to %s:%d\n", 1749 IP_VS_DBG_ADDR(cp->af, &cp->caddr), 1750 ntohs(cp->cport), 1751 IP_VS_DBG_ADDR(cp->af, &cp->vaddr), 1752 ntohs(cp->vport)); 1753 1754 ip_vs_control_del(cp); 1755 } 1756 1757 IP_VS_DBG_BUF(7, "ADDing control for: " 1758 "cp.dst=%s:%d ctl_cp.dst=%s:%d\n", 1759 IP_VS_DBG_ADDR(cp->af, &cp->caddr), 1760 ntohs(cp->cport), 1761 IP_VS_DBG_ADDR(cp->af, &ctl_cp->caddr), 1762 ntohs(ctl_cp->cport)); 1763 1764 cp->control = ctl_cp; 1765 atomic_inc(&ctl_cp->n_control); 1766 } 1767 1768 /* Mark our template as assured */ 1769 static inline void 1770 ip_vs_control_assure_ct(struct ip_vs_conn *cp) 1771 { 1772 struct ip_vs_conn *ct = cp->control; 1773 1774 if (ct && !(ct->state & IP_VS_CTPL_S_ASSURED) && 1775 (ct->flags & IP_VS_CONN_F_TEMPLATE)) 1776 ct->state |= IP_VS_CTPL_S_ASSURED; 1777 } 1778 1779 /* IPVS netns init & cleanup functions */ 1780 int ip_vs_estimator_net_init(struct netns_ipvs *ipvs); 1781 int ip_vs_control_net_init(struct netns_ipvs *ipvs); 1782 int ip_vs_protocol_net_init(struct netns_ipvs *ipvs); 1783 int ip_vs_app_net_init(struct netns_ipvs *ipvs); 1784 int ip_vs_conn_net_init(struct netns_ipvs *ipvs); 1785 int ip_vs_sync_net_init(struct netns_ipvs *ipvs); 1786 void ip_vs_conn_net_cleanup(struct netns_ipvs *ipvs); 1787 void ip_vs_app_net_cleanup(struct netns_ipvs *ipvs); 1788 void ip_vs_protocol_net_cleanup(struct netns_ipvs *ipvs); 1789 void ip_vs_control_net_cleanup(struct netns_ipvs *ipvs); 1790 void ip_vs_estimator_net_cleanup(struct netns_ipvs *ipvs); 1791 void ip_vs_sync_net_cleanup(struct netns_ipvs *ipvs); 1792 void ip_vs_service_nets_cleanup(struct list_head *net_list); 1793 1794 /* IPVS application functions 1795 * (from ip_vs_app.c) 1796 */ 1797 #define IP_VS_APP_MAX_PORTS 8 1798 struct ip_vs_app *register_ip_vs_app(struct netns_ipvs *ipvs, struct ip_vs_app *app); 1799 void unregister_ip_vs_app(struct netns_ipvs *ipvs, struct ip_vs_app *app); 1800 int ip_vs_bind_app(struct ip_vs_conn *cp, struct ip_vs_protocol *pp); 1801 void ip_vs_unbind_app(struct ip_vs_conn *cp); 1802 int register_ip_vs_app_inc(struct netns_ipvs *ipvs, struct ip_vs_app *app, __u16 proto, 1803 __u16 port); 1804 int ip_vs_app_inc_get(struct ip_vs_app *inc); 1805 void ip_vs_app_inc_put(struct ip_vs_app *inc); 1806 1807 int ip_vs_app_pkt_out(struct ip_vs_conn *, struct sk_buff *skb, 1808 struct ip_vs_iphdr *ipvsh); 1809 int ip_vs_app_pkt_in(struct ip_vs_conn *, struct sk_buff *skb, 1810 struct ip_vs_iphdr *ipvsh); 1811 1812 int register_ip_vs_pe(struct ip_vs_pe *pe); 1813 int unregister_ip_vs_pe(struct ip_vs_pe *pe); 1814 struct ip_vs_pe *ip_vs_pe_getbyname(const char *name); 1815 struct ip_vs_pe *__ip_vs_pe_getbyname(const char *pe_name); 1816 1817 /* Use a #define to avoid all of module.h just for these trivial ops */ 1818 #define ip_vs_pe_get(pe) \ 1819 if (pe && pe->module) \ 1820 __module_get(pe->module); 1821 1822 #define ip_vs_pe_put(pe) \ 1823 if (pe && pe->module) \ 1824 module_put(pe->module); 1825 1826 /* IPVS protocol functions (from ip_vs_proto.c) */ 1827 int ip_vs_protocol_init(void); 1828 void ip_vs_protocol_cleanup(void); 1829 void ip_vs_protocol_timeout_change(struct netns_ipvs *ipvs, int flags); 1830 int *ip_vs_create_timeout_table(int *table, int size); 1831 void ip_vs_tcpudp_debug_packet(int af, struct ip_vs_protocol *pp, 1832 const struct sk_buff *skb, int offset, 1833 const char *msg); 1834 1835 extern struct ip_vs_protocol ip_vs_protocol_tcp; 1836 extern struct ip_vs_protocol ip_vs_protocol_udp; 1837 extern struct ip_vs_protocol ip_vs_protocol_icmp; 1838 extern struct ip_vs_protocol ip_vs_protocol_esp; 1839 extern struct ip_vs_protocol ip_vs_protocol_ah; 1840 extern struct ip_vs_protocol ip_vs_protocol_sctp; 1841 1842 /* Registering/unregistering scheduler functions 1843 * (from ip_vs_sched.c) 1844 */ 1845 int register_ip_vs_scheduler(struct ip_vs_scheduler *scheduler); 1846 int unregister_ip_vs_scheduler(struct ip_vs_scheduler *scheduler); 1847 int ip_vs_bind_scheduler(struct ip_vs_service *svc, 1848 struct ip_vs_scheduler *scheduler); 1849 void ip_vs_unbind_scheduler(struct ip_vs_service *svc); 1850 struct ip_vs_scheduler *ip_vs_scheduler_get(const char *sched_name); 1851 void ip_vs_scheduler_put(struct ip_vs_scheduler *scheduler); 1852 struct ip_vs_conn * 1853 ip_vs_schedule(struct ip_vs_service *svc, struct sk_buff *skb, 1854 struct ip_vs_proto_data *pd, int *ignored, 1855 struct ip_vs_iphdr *iph); 1856 int ip_vs_leave(struct ip_vs_service *svc, struct sk_buff *skb, 1857 struct ip_vs_proto_data *pd, struct ip_vs_iphdr *iph); 1858 1859 void ip_vs_scheduler_err(struct ip_vs_service *svc, const char *msg); 1860 1861 /* IPVS control data and functions (from ip_vs_ctl.c) */ 1862 extern struct ip_vs_stats ip_vs_stats; 1863 extern int sysctl_ip_vs_sync_ver; 1864 1865 struct ip_vs_service * 1866 ip_vs_service_find(struct netns_ipvs *ipvs, int af, __u32 fwmark, __u16 protocol, 1867 const union nf_inet_addr *vaddr, __be16 vport); 1868 1869 bool ip_vs_has_real_service(struct netns_ipvs *ipvs, int af, __u16 protocol, 1870 const union nf_inet_addr *daddr, __be16 dport); 1871 1872 struct ip_vs_dest * 1873 ip_vs_find_real_service(struct netns_ipvs *ipvs, int af, __u16 protocol, 1874 const union nf_inet_addr *daddr, __be16 dport); 1875 struct ip_vs_dest *ip_vs_find_tunnel(struct netns_ipvs *ipvs, int af, 1876 const union nf_inet_addr *daddr, 1877 __be16 tun_port); 1878 1879 int ip_vs_use_count_inc(void); 1880 void ip_vs_use_count_dec(void); 1881 int ip_vs_register_nl_ioctl(void); 1882 void ip_vs_unregister_nl_ioctl(void); 1883 int ip_vs_control_init(void); 1884 void ip_vs_control_cleanup(void); 1885 struct ip_vs_dest * 1886 ip_vs_find_dest(struct netns_ipvs *ipvs, int svc_af, int dest_af, 1887 const union nf_inet_addr *daddr, __be16 dport, 1888 const union nf_inet_addr *vaddr, __be16 vport, 1889 __u16 protocol, __u32 fwmark, __u32 flags); 1890 void ip_vs_try_bind_dest(struct ip_vs_conn *cp); 1891 1892 static inline void ip_vs_dest_hold(struct ip_vs_dest *dest) 1893 { 1894 refcount_inc(&dest->refcnt); 1895 } 1896 1897 static inline void ip_vs_dest_put(struct ip_vs_dest *dest) 1898 { 1899 smp_mb__before_atomic(); 1900 refcount_dec(&dest->refcnt); 1901 } 1902 1903 static inline void ip_vs_dest_put_and_free(struct ip_vs_dest *dest) 1904 { 1905 if (refcount_dec_and_test(&dest->refcnt)) 1906 kfree(dest); 1907 } 1908 1909 /* IPVS sync daemon data and function prototypes 1910 * (from ip_vs_sync.c) 1911 */ 1912 int start_sync_thread(struct netns_ipvs *ipvs, struct ipvs_sync_daemon_cfg *cfg, 1913 int state); 1914 int stop_sync_thread(struct netns_ipvs *ipvs, int state); 1915 void ip_vs_sync_conn(struct netns_ipvs *ipvs, struct ip_vs_conn *cp, int pkts); 1916 1917 /* IPVS rate estimator prototypes (from ip_vs_est.c) */ 1918 int ip_vs_start_estimator(struct netns_ipvs *ipvs, struct ip_vs_stats *stats); 1919 void ip_vs_stop_estimator(struct netns_ipvs *ipvs, struct ip_vs_stats *stats); 1920 void ip_vs_zero_estimator(struct ip_vs_stats *stats); 1921 void ip_vs_read_estimator(struct ip_vs_kstats *dst, struct ip_vs_stats *stats); 1922 void ip_vs_est_reload_start(struct netns_ipvs *ipvs, bool restart); 1923 int ip_vs_est_kthread_start(struct netns_ipvs *ipvs, 1924 struct ip_vs_est_kt_data *kd); 1925 void ip_vs_est_kthread_stop(struct ip_vs_est_kt_data *kd); 1926 1927 static inline void ip_vs_stop_estimator_tot_stats(struct netns_ipvs *ipvs) 1928 { 1929 #ifdef CONFIG_SYSCTL 1930 ip_vs_stop_estimator(ipvs, &ipvs->tot_stats->s); 1931 ipvs->tot_stats->s.est.ktid = -2; 1932 #endif 1933 } 1934 1935 static inline void ip_vs_est_stopped_recalc(struct netns_ipvs *ipvs) 1936 { 1937 #ifdef CONFIG_SYSCTL 1938 /* Stop tasks while cpulist is empty or if disabled with flag */ 1939 ipvs->est_stopped = !sysctl_run_estimation(ipvs) || 1940 (ipvs->est_cpulist_valid && 1941 sysctl_est_cpulist_empty(ipvs)); 1942 #endif 1943 } 1944 1945 static inline bool ip_vs_est_stopped(struct netns_ipvs *ipvs) 1946 { 1947 #ifdef CONFIG_SYSCTL 1948 return ipvs->est_stopped; 1949 #else 1950 return false; 1951 #endif 1952 } 1953 1954 static inline int ip_vs_est_max_threads(struct netns_ipvs *ipvs) 1955 { 1956 unsigned int limit = IPVS_EST_CPU_KTHREADS * 1957 sysctl_est_cpulist_weight(ipvs); 1958 1959 return max(1U, limit); 1960 } 1961 1962 /* Various IPVS packet transmitters (from ip_vs_xmit.c) */ 1963 int ip_vs_null_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 1964 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1965 int ip_vs_bypass_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 1966 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1967 int ip_vs_nat_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 1968 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1969 int ip_vs_tunnel_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 1970 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1971 int ip_vs_dr_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 1972 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1973 int ip_vs_icmp_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 1974 struct ip_vs_protocol *pp, int offset, 1975 unsigned int hooknum, struct ip_vs_iphdr *iph); 1976 void ip_vs_dest_dst_rcu_free(struct rcu_head *head); 1977 1978 #ifdef CONFIG_IP_VS_IPV6 1979 int ip_vs_bypass_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, 1980 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1981 int ip_vs_nat_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, 1982 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1983 int ip_vs_tunnel_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, 1984 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1985 int ip_vs_dr_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, 1986 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1987 int ip_vs_icmp_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, 1988 struct ip_vs_protocol *pp, int offset, 1989 unsigned int hooknum, struct ip_vs_iphdr *iph); 1990 #endif 1991 1992 #ifdef CONFIG_SYSCTL 1993 /* This is a simple mechanism to ignore packets when 1994 * we are loaded. Just set ip_vs_drop_rate to 'n' and 1995 * we start to drop 1/rate of the packets 1996 */ 1997 static inline int ip_vs_todrop(struct netns_ipvs *ipvs) 1998 { 1999 if (!ipvs->drop_rate) 2000 return 0; 2001 if (--ipvs->drop_counter > 0) 2002 return 0; 2003 ipvs->drop_counter = ipvs->drop_rate; 2004 return 1; 2005 } 2006 #else 2007 static inline int ip_vs_todrop(struct netns_ipvs *ipvs) { return 0; } 2008 #endif 2009 2010 #ifdef CONFIG_SYSCTL 2011 /* Enqueue delayed work for expiring no dest connections 2012 * Only run when sysctl_expire_nodest=1 2013 */ 2014 static inline void ip_vs_enqueue_expire_nodest_conns(struct netns_ipvs *ipvs) 2015 { 2016 if (sysctl_expire_nodest_conn(ipvs)) 2017 queue_delayed_work(system_long_wq, 2018 &ipvs->expire_nodest_conn_work, 1); 2019 } 2020 2021 void ip_vs_expire_nodest_conn_flush(struct netns_ipvs *ipvs); 2022 #else 2023 static inline void ip_vs_enqueue_expire_nodest_conns(struct netns_ipvs *ipvs) {} 2024 #endif 2025 2026 #define IP_VS_DFWD_METHOD(dest) (atomic_read(&(dest)->conn_flags) & \ 2027 IP_VS_CONN_F_FWD_MASK) 2028 2029 /* ip_vs_fwd_tag returns the forwarding tag of the connection */ 2030 #define IP_VS_FWD_METHOD(cp) (cp->flags & IP_VS_CONN_F_FWD_MASK) 2031 2032 static inline char ip_vs_fwd_tag(struct ip_vs_conn *cp) 2033 { 2034 char fwd; 2035 2036 switch (IP_VS_FWD_METHOD(cp)) { 2037 case IP_VS_CONN_F_MASQ: 2038 fwd = 'M'; break; 2039 case IP_VS_CONN_F_LOCALNODE: 2040 fwd = 'L'; break; 2041 case IP_VS_CONN_F_TUNNEL: 2042 fwd = 'T'; break; 2043 case IP_VS_CONN_F_DROUTE: 2044 fwd = 'R'; break; 2045 case IP_VS_CONN_F_BYPASS: 2046 fwd = 'B'; break; 2047 default: 2048 fwd = '?'; break; 2049 } 2050 return fwd; 2051 } 2052 2053 /* Check if connection uses double hashing */ 2054 static inline bool ip_vs_conn_use_hash2(struct ip_vs_conn *cp) 2055 { 2056 return IP_VS_FWD_METHOD(cp) == IP_VS_CONN_F_MASQ && 2057 !(cp->flags & IP_VS_CONN_F_TEMPLATE); 2058 } 2059 2060 void ip_vs_nat_icmp(struct sk_buff *skb, struct ip_vs_protocol *pp, 2061 struct ip_vs_conn *cp, int dir); 2062 2063 #ifdef CONFIG_IP_VS_IPV6 2064 void ip_vs_nat_icmp_v6(struct sk_buff *skb, struct ip_vs_protocol *pp, 2065 struct ip_vs_conn *cp, int dir); 2066 #endif 2067 2068 __sum16 ip_vs_checksum_complete(struct sk_buff *skb, int offset); 2069 2070 static inline __wsum ip_vs_check_diff4(__be32 old, __be32 new, __wsum oldsum) 2071 { 2072 __be32 diff[2] = { ~old, new }; 2073 2074 return csum_partial(diff, sizeof(diff), oldsum); 2075 } 2076 2077 #ifdef CONFIG_IP_VS_IPV6 2078 static inline __wsum ip_vs_check_diff16(const __be32 *old, const __be32 *new, 2079 __wsum oldsum) 2080 { 2081 __be32 diff[8] = { ~old[3], ~old[2], ~old[1], ~old[0], 2082 new[3], new[2], new[1], new[0] }; 2083 2084 return csum_partial(diff, sizeof(diff), oldsum); 2085 } 2086 #endif 2087 2088 static inline __wsum ip_vs_check_diff2(__be16 old, __be16 new, __wsum oldsum) 2089 { 2090 __be16 diff[2] = { ~old, new }; 2091 2092 return csum_partial(diff, sizeof(diff), oldsum); 2093 } 2094 2095 /* Forget current conntrack (unconfirmed) and attach notrack entry */ 2096 static inline void ip_vs_notrack(struct sk_buff *skb) 2097 { 2098 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE) 2099 enum ip_conntrack_info ctinfo; 2100 struct nf_conn *ct = nf_ct_get(skb, &ctinfo); 2101 2102 if (ct) { 2103 nf_conntrack_put(&ct->ct_general); 2104 nf_ct_set(skb, NULL, IP_CT_UNTRACKED); 2105 } 2106 #endif 2107 } 2108 2109 #ifdef CONFIG_IP_VS_NFCT 2110 /* Netfilter connection tracking 2111 * (from ip_vs_nfct.c) 2112 */ 2113 static inline int ip_vs_conntrack_enabled(struct netns_ipvs *ipvs) 2114 { 2115 #ifdef CONFIG_SYSCTL 2116 return ipvs->sysctl_conntrack; 2117 #else 2118 return 0; 2119 #endif 2120 } 2121 2122 void ip_vs_update_conntrack(struct sk_buff *skb, struct ip_vs_conn *cp, 2123 int outin); 2124 int ip_vs_confirm_conntrack(struct sk_buff *skb); 2125 void ip_vs_nfct_expect_related(struct sk_buff *skb, struct nf_conn *ct, 2126 struct ip_vs_conn *cp, u_int8_t proto, 2127 const __be16 port, int from_rs); 2128 void ip_vs_conn_drop_conntrack(struct ip_vs_conn *cp); 2129 2130 #else 2131 2132 static inline int ip_vs_conntrack_enabled(struct netns_ipvs *ipvs) 2133 { 2134 return 0; 2135 } 2136 2137 static inline void ip_vs_update_conntrack(struct sk_buff *skb, 2138 struct ip_vs_conn *cp, int outin) 2139 { 2140 } 2141 2142 static inline int ip_vs_confirm_conntrack(struct sk_buff *skb) 2143 { 2144 return NF_ACCEPT; 2145 } 2146 2147 static inline void ip_vs_conn_drop_conntrack(struct ip_vs_conn *cp) 2148 { 2149 } 2150 #endif /* CONFIG_IP_VS_NFCT */ 2151 2152 /* Using old conntrack that can not be redirected to another real server? */ 2153 static inline bool ip_vs_conn_uses_old_conntrack(struct ip_vs_conn *cp, 2154 struct sk_buff *skb) 2155 { 2156 #ifdef CONFIG_IP_VS_NFCT 2157 enum ip_conntrack_info ctinfo; 2158 struct nf_conn *ct; 2159 2160 ct = nf_ct_get(skb, &ctinfo); 2161 if (ct && nf_ct_is_confirmed(ct)) 2162 return true; 2163 #endif 2164 return false; 2165 } 2166 2167 static inline int ip_vs_register_conntrack(struct ip_vs_service *svc) 2168 { 2169 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 2170 int afmask = (svc->af == AF_INET6) ? 2 : 1; 2171 int ret = 0; 2172 2173 if (!(svc->conntrack_afmask & afmask)) { 2174 ret = nf_ct_netns_get(svc->ipvs->net, svc->af); 2175 if (ret >= 0) 2176 svc->conntrack_afmask |= afmask; 2177 } 2178 return ret; 2179 #else 2180 return 0; 2181 #endif 2182 } 2183 2184 static inline void ip_vs_unregister_conntrack(struct ip_vs_service *svc) 2185 { 2186 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 2187 int afmask = (svc->af == AF_INET6) ? 2 : 1; 2188 2189 if (svc->conntrack_afmask & afmask) { 2190 nf_ct_netns_put(svc->ipvs->net, svc->af); 2191 svc->conntrack_afmask &= ~afmask; 2192 } 2193 #endif 2194 } 2195 2196 int ip_vs_register_hooks(struct netns_ipvs *ipvs, unsigned int af); 2197 void ip_vs_unregister_hooks(struct netns_ipvs *ipvs, unsigned int af); 2198 2199 static inline int 2200 ip_vs_dest_conn_overhead(struct ip_vs_dest *dest) 2201 { 2202 /* We think the overhead of processing active connections is 256 2203 * times higher than that of inactive connections in average. (This 2204 * 256 times might not be accurate, we will change it later) We 2205 * use the following formula to estimate the overhead now: 2206 * dest->activeconns*256 + dest->inactconns 2207 */ 2208 return (atomic_read(&dest->activeconns) << 8) + 2209 atomic_read(&dest->inactconns); 2210 } 2211 2212 #ifdef CONFIG_IP_VS_PROTO_TCP 2213 INDIRECT_CALLABLE_DECLARE(int 2214 tcp_snat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, 2215 struct ip_vs_conn *cp, struct ip_vs_iphdr *iph)); 2216 #endif 2217 2218 #ifdef CONFIG_IP_VS_PROTO_UDP 2219 INDIRECT_CALLABLE_DECLARE(int 2220 udp_snat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, 2221 struct ip_vs_conn *cp, struct ip_vs_iphdr *iph)); 2222 #endif 2223 #endif /* _NET_IP_VS_H */ 2224