1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* IP Virtual Server 3 * data structure and functionality definitions 4 */ 5 6 #ifndef _NET_IP_VS_H 7 #define _NET_IP_VS_H 8 9 #include <linux/ip_vs.h> /* definitions shared with userland */ 10 11 #include <asm/types.h> /* for __uXX types */ 12 13 #include <linux/list.h> /* for struct list_head */ 14 #include <linux/rculist_bl.h> /* for struct hlist_bl_head */ 15 #include <linux/spinlock.h> /* for struct rwlock_t */ 16 #include <linux/atomic.h> /* for struct atomic_t */ 17 #include <linux/refcount.h> /* for struct refcount_t */ 18 #include <linux/workqueue.h> 19 20 #include <linux/compiler.h> 21 #include <linux/timer.h> 22 #include <linux/bug.h> 23 24 #include <net/checksum.h> 25 #include <linux/netfilter.h> /* for union nf_inet_addr */ 26 #include <linux/ip.h> 27 #include <linux/ipv6.h> /* for struct ipv6hdr */ 28 #include <net/ipv6.h> 29 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 30 #include <net/netfilter/nf_conntrack.h> 31 #endif 32 #include <net/net_namespace.h> /* Netw namespace */ 33 #include <linux/sched/isolation.h> 34 #include <linux/siphash.h> 35 36 #define IP_VS_HDR_INVERSE 1 37 #define IP_VS_HDR_ICMP 2 38 39 /* Destination Server Flags */ 40 #define IP_VS_DEST_F_OVERLOAD 0x0002 /* server is overloaded */ 41 42 /* Destination Server Config Flags */ 43 #define IP_VS_DEST_CF_AVAILABLE 0x0001 /* server is available */ 44 45 /* conn_tab limits (as per Kconfig) */ 46 #define IP_VS_CONN_TAB_MIN_BITS 8 47 #if BITS_PER_LONG > 32 48 #define IP_VS_CONN_TAB_MAX_BITS 27 49 #else 50 #define IP_VS_CONN_TAB_MAX_BITS 20 51 #endif 52 53 /* conn_max limits */ 54 #if BITS_PER_LONG > 32 55 /* Limit of atomic_t but restricted by roundup_pow_of_two() in ip_vs_core.c */ 56 #define IP_VS_CONN_MAX (1 << 30) 57 #else 58 #define IP_VS_CONN_MAX (1 << 24) 59 #endif 60 61 /* svc_table limits */ 62 #define IP_VS_SVC_TAB_MIN_BITS 4 63 #define IP_VS_SVC_TAB_MAX_BITS 20 64 65 /* Generic access of ipvs struct */ 66 static inline struct netns_ipvs *net_ipvs(struct net* net) 67 { 68 return net->ipvs; 69 } 70 71 /* Connections' size value needed by ip_vs_ctl.c */ 72 extern int ip_vs_conn_tab_size; 73 74 struct ip_vs_iphdr { 75 int hdr_flags; /* ipvs flags */ 76 __u32 off; /* Where IP or IPv4 header starts */ 77 __u32 len; /* IPv4 simply where L4 starts 78 * IPv6 where L4 Transport Header starts */ 79 __u16 fragoffs; /* IPv6 fragment offset, 0 if first frag (or not frag)*/ 80 __s16 protocol; 81 __s32 flags; 82 union nf_inet_addr saddr; 83 union nf_inet_addr daddr; 84 }; 85 86 static inline void *frag_safe_skb_hp(const struct sk_buff *skb, int offset, 87 int len, void *buffer) 88 { 89 return skb_header_pointer(skb, offset, len, buffer); 90 } 91 92 /* This function handles filling *ip_vs_iphdr, both for IPv4 and IPv6. 93 * IPv6 requires some extra work, as finding proper header position, 94 * depend on the IPv6 extension headers. 95 */ 96 static inline int 97 ip_vs_fill_iph_skb_off(int af, const struct sk_buff *skb, int offset, 98 int hdr_flags, struct ip_vs_iphdr *iphdr) 99 { 100 iphdr->hdr_flags = hdr_flags; 101 iphdr->off = offset; 102 103 #ifdef CONFIG_IP_VS_IPV6 104 if (af == AF_INET6) { 105 struct ipv6hdr _iph; 106 const struct ipv6hdr *iph = skb_header_pointer( 107 skb, offset, sizeof(_iph), &_iph); 108 if (!iph) 109 return 0; 110 111 iphdr->saddr.in6 = iph->saddr; 112 iphdr->daddr.in6 = iph->daddr; 113 /* ipv6_find_hdr() updates len, flags */ 114 iphdr->len = offset; 115 iphdr->flags = 0; 116 iphdr->protocol = ipv6_find_hdr(skb, &iphdr->len, -1, 117 &iphdr->fragoffs, 118 &iphdr->flags); 119 if (iphdr->protocol < 0) 120 return 0; 121 } else 122 #endif 123 { 124 struct iphdr _iph; 125 const struct iphdr *iph = skb_header_pointer( 126 skb, offset, sizeof(_iph), &_iph); 127 if (!iph) 128 return 0; 129 130 iphdr->len = offset + iph->ihl * 4; 131 iphdr->fragoffs = 0; 132 iphdr->protocol = iph->protocol; 133 iphdr->saddr.ip = iph->saddr; 134 iphdr->daddr.ip = iph->daddr; 135 } 136 137 return 1; 138 } 139 140 static inline int 141 ip_vs_fill_iph_skb_icmp(int af, const struct sk_buff *skb, int offset, 142 bool inverse, struct ip_vs_iphdr *iphdr) 143 { 144 int hdr_flags = IP_VS_HDR_ICMP; 145 146 if (inverse) 147 hdr_flags |= IP_VS_HDR_INVERSE; 148 149 return ip_vs_fill_iph_skb_off(af, skb, offset, hdr_flags, iphdr); 150 } 151 152 static inline int 153 ip_vs_fill_iph_skb(int af, const struct sk_buff *skb, bool inverse, 154 struct ip_vs_iphdr *iphdr) 155 { 156 int hdr_flags = 0; 157 158 if (inverse) 159 hdr_flags |= IP_VS_HDR_INVERSE; 160 161 return ip_vs_fill_iph_skb_off(af, skb, skb_network_offset(skb), 162 hdr_flags, iphdr); 163 } 164 165 static inline bool 166 ip_vs_iph_inverse(const struct ip_vs_iphdr *iph) 167 { 168 return !!(iph->hdr_flags & IP_VS_HDR_INVERSE); 169 } 170 171 static inline bool 172 ip_vs_iph_icmp(const struct ip_vs_iphdr *iph) 173 { 174 return !!(iph->hdr_flags & IP_VS_HDR_ICMP); 175 } 176 177 static inline void ip_vs_addr_copy(int af, union nf_inet_addr *dst, 178 const union nf_inet_addr *src) 179 { 180 #ifdef CONFIG_IP_VS_IPV6 181 if (af == AF_INET6) 182 dst->in6 = src->in6; 183 else 184 #endif 185 dst->ip = src->ip; 186 } 187 188 static inline void ip_vs_addr_set(int af, union nf_inet_addr *dst, 189 const union nf_inet_addr *src) 190 { 191 #ifdef CONFIG_IP_VS_IPV6 192 if (af == AF_INET6) { 193 dst->in6 = src->in6; 194 return; 195 } 196 #endif 197 dst->ip = src->ip; 198 dst->all[1] = 0; 199 dst->all[2] = 0; 200 dst->all[3] = 0; 201 } 202 203 static inline int ip_vs_addr_equal(int af, const union nf_inet_addr *a, 204 const union nf_inet_addr *b) 205 { 206 #ifdef CONFIG_IP_VS_IPV6 207 if (af == AF_INET6) 208 return ipv6_addr_equal(&a->in6, &b->in6); 209 #endif 210 return a->ip == b->ip; 211 } 212 213 #ifdef CONFIG_IP_VS_DEBUG 214 #include <linux/net.h> 215 216 int ip_vs_get_debug_level(void); 217 218 static inline const char *ip_vs_dbg_addr(int af, char *buf, size_t buf_len, 219 const union nf_inet_addr *addr, 220 int *idx) 221 { 222 int len; 223 #ifdef CONFIG_IP_VS_IPV6 224 if (af == AF_INET6) 225 len = snprintf(&buf[*idx], buf_len - *idx, "[%pI6c]", 226 &addr->in6) + 1; 227 else 228 #endif 229 len = snprintf(&buf[*idx], buf_len - *idx, "%pI4", 230 &addr->ip) + 1; 231 232 *idx += len; 233 BUG_ON(*idx > buf_len + 1); 234 return &buf[*idx - len]; 235 } 236 237 #define IP_VS_DBG_BUF(level, msg, ...) \ 238 do { \ 239 char ip_vs_dbg_buf[160]; \ 240 int ip_vs_dbg_idx = 0; \ 241 if (level <= ip_vs_get_debug_level()) \ 242 printk(KERN_DEBUG pr_fmt(msg), ##__VA_ARGS__); \ 243 } while (0) 244 #define IP_VS_ERR_BUF(msg...) \ 245 do { \ 246 char ip_vs_dbg_buf[160]; \ 247 int ip_vs_dbg_idx = 0; \ 248 pr_err(msg); \ 249 } while (0) 250 251 /* Only use from within IP_VS_DBG_BUF() or IP_VS_ERR_BUF macros */ 252 #define IP_VS_DBG_ADDR(af, addr) \ 253 ip_vs_dbg_addr(af, ip_vs_dbg_buf, \ 254 sizeof(ip_vs_dbg_buf), addr, \ 255 &ip_vs_dbg_idx) 256 257 #define IP_VS_DBG(level, msg, ...) \ 258 do { \ 259 if (level <= ip_vs_get_debug_level()) \ 260 printk(KERN_DEBUG pr_fmt(msg), ##__VA_ARGS__); \ 261 } while (0) 262 #define IP_VS_DBG_RL(msg, ...) \ 263 do { \ 264 if (net_ratelimit()) \ 265 printk(KERN_DEBUG pr_fmt(msg), ##__VA_ARGS__); \ 266 } while (0) 267 #define IP_VS_DBG_PKT(level, af, pp, skb, ofs, msg) \ 268 do { \ 269 if (level <= ip_vs_get_debug_level()) \ 270 pp->debug_packet(af, pp, skb, ofs, msg); \ 271 } while (0) 272 #define IP_VS_DBG_RL_PKT(level, af, pp, skb, ofs, msg) \ 273 do { \ 274 if (level <= ip_vs_get_debug_level() && \ 275 net_ratelimit()) \ 276 pp->debug_packet(af, pp, skb, ofs, msg); \ 277 } while (0) 278 #else /* NO DEBUGGING at ALL */ 279 #define IP_VS_DBG_BUF(level, msg...) do {} while (0) 280 #define IP_VS_ERR_BUF(msg...) do {} while (0) 281 #define IP_VS_DBG(level, msg...) do {} while (0) 282 #define IP_VS_DBG_RL(msg...) do {} while (0) 283 #define IP_VS_DBG_PKT(level, af, pp, skb, ofs, msg) do {} while (0) 284 #define IP_VS_DBG_RL_PKT(level, af, pp, skb, ofs, msg) do {} while (0) 285 #endif 286 287 #define IP_VS_BUG() BUG() 288 #define IP_VS_ERR_RL(msg, ...) \ 289 do { \ 290 if (net_ratelimit()) \ 291 pr_err(msg, ##__VA_ARGS__); \ 292 } while (0) 293 294 struct ip_vs_aligned_lock { 295 spinlock_t l; /* Protect buckets */ 296 } ____cacheline_aligned_in_smp; 297 298 /* For arrays per family */ 299 enum { 300 IP_VS_AF_INET, 301 IP_VS_AF_INET6, 302 IP_VS_AF_MAX 303 }; 304 305 static inline int ip_vs_af_index(int af) 306 { 307 return af == AF_INET6 ? IP_VS_AF_INET6 : IP_VS_AF_INET; 308 } 309 310 /* work_flags */ 311 enum { 312 IP_VS_WORK_SVC_RESIZE, /* Schedule svc_resize_work */ 313 IP_VS_WORK_SVC_NORESIZE, /* Stopping svc_resize_work */ 314 IP_VS_WORK_CONN_RESIZE, /* Schedule conn_resize_work */ 315 }; 316 317 /* The port number of FTP service (in network order). */ 318 #define FTPPORT cpu_to_be16(21) 319 #define FTPDATA cpu_to_be16(20) 320 321 /* TCP State Values */ 322 enum { 323 IP_VS_TCP_S_NONE = 0, 324 IP_VS_TCP_S_ESTABLISHED, 325 IP_VS_TCP_S_SYN_SENT, 326 IP_VS_TCP_S_SYN_RECV, 327 IP_VS_TCP_S_FIN_WAIT, 328 IP_VS_TCP_S_TIME_WAIT, 329 IP_VS_TCP_S_CLOSE, 330 IP_VS_TCP_S_CLOSE_WAIT, 331 IP_VS_TCP_S_LAST_ACK, 332 IP_VS_TCP_S_LISTEN, 333 IP_VS_TCP_S_SYNACK, 334 IP_VS_TCP_S_LAST 335 }; 336 337 /* UDP State Values */ 338 enum { 339 IP_VS_UDP_S_NORMAL, 340 IP_VS_UDP_S_LAST, 341 }; 342 343 /* ICMP State Values */ 344 enum { 345 IP_VS_ICMP_S_NORMAL, 346 IP_VS_ICMP_S_LAST, 347 }; 348 349 /* SCTP State Values */ 350 enum ip_vs_sctp_states { 351 IP_VS_SCTP_S_NONE, 352 IP_VS_SCTP_S_INIT1, 353 IP_VS_SCTP_S_INIT, 354 IP_VS_SCTP_S_COOKIE_SENT, 355 IP_VS_SCTP_S_COOKIE_REPLIED, 356 IP_VS_SCTP_S_COOKIE_WAIT, 357 IP_VS_SCTP_S_COOKIE, 358 IP_VS_SCTP_S_COOKIE_ECHOED, 359 IP_VS_SCTP_S_ESTABLISHED, 360 IP_VS_SCTP_S_SHUTDOWN_SENT, 361 IP_VS_SCTP_S_SHUTDOWN_RECEIVED, 362 IP_VS_SCTP_S_SHUTDOWN_ACK_SENT, 363 IP_VS_SCTP_S_REJECTED, 364 IP_VS_SCTP_S_CLOSED, 365 IP_VS_SCTP_S_LAST 366 }; 367 368 /* Connection templates use bits from state */ 369 #define IP_VS_CTPL_S_NONE 0x0000 370 #define IP_VS_CTPL_S_ASSURED 0x0001 371 #define IP_VS_CTPL_S_LAST 0x0002 372 373 /* Delta sequence info structure 374 * Each ip_vs_conn has 2 (output AND input seq. changes). 375 * Only used in the VS/NAT. 376 */ 377 struct ip_vs_seq { 378 __u32 init_seq; /* Add delta from this seq */ 379 __u32 delta; /* Delta in sequence numbers */ 380 __u32 previous_delta; /* Delta in sequence numbers 381 * before last resized pkt */ 382 }; 383 384 /* counters per cpu */ 385 struct ip_vs_counters { 386 u64_stats_t conns; /* connections scheduled */ 387 u64_stats_t inpkts; /* incoming packets */ 388 u64_stats_t outpkts; /* outgoing packets */ 389 u64_stats_t inbytes; /* incoming bytes */ 390 u64_stats_t outbytes; /* outgoing bytes */ 391 }; 392 /* Stats per cpu */ 393 struct ip_vs_cpu_stats { 394 struct ip_vs_counters cnt; 395 struct u64_stats_sync syncp; 396 }; 397 398 /* Default nice for estimator kthreads */ 399 #define IPVS_EST_NICE 0 400 401 /* IPVS statistics objects */ 402 struct ip_vs_estimator { 403 struct hlist_node list; 404 405 u64 last_inbytes; 406 u64 last_outbytes; 407 u64 last_conns; 408 u64 last_inpkts; 409 u64 last_outpkts; 410 411 u64 cps; 412 u64 inpps; 413 u64 outpps; 414 u64 inbps; 415 u64 outbps; 416 417 s32 ktid:16, /* kthread ID, -1=temp list */ 418 ktrow:8, /* row/tick ID for kthread */ 419 ktcid:8; /* chain ID for kthread tick */ 420 }; 421 422 /* 423 * IPVS statistics object, 64-bit kernel version of struct ip_vs_stats_user 424 */ 425 struct ip_vs_kstats { 426 u64 conns; /* connections scheduled */ 427 u64 inpkts; /* incoming packets */ 428 u64 outpkts; /* outgoing packets */ 429 u64 inbytes; /* incoming bytes */ 430 u64 outbytes; /* outgoing bytes */ 431 432 u64 cps; /* current connection rate */ 433 u64 inpps; /* current in packet rate */ 434 u64 outpps; /* current out packet rate */ 435 u64 inbps; /* current in byte rate */ 436 u64 outbps; /* current out byte rate */ 437 }; 438 439 struct ip_vs_stats { 440 struct ip_vs_kstats kstats; /* kernel statistics */ 441 struct ip_vs_estimator est; /* estimator */ 442 struct ip_vs_cpu_stats __percpu *cpustats; /* per cpu counters */ 443 spinlock_t lock; /* spin lock */ 444 struct ip_vs_kstats kstats0; /* reset values */ 445 }; 446 447 struct ip_vs_stats_rcu { 448 struct ip_vs_stats s; 449 struct rcu_head rcu_head; 450 }; 451 452 int ip_vs_stats_init_alloc(struct ip_vs_stats *s); 453 struct ip_vs_stats *ip_vs_stats_alloc(void); 454 void ip_vs_stats_release(struct ip_vs_stats *stats); 455 void ip_vs_stats_free(struct ip_vs_stats *stats); 456 457 /* Process estimators in multiple timer ticks (20/50/100, see ktrow) */ 458 #define IPVS_EST_NTICKS 50 459 /* Estimation uses a 2-second period containing ticks (in jiffies) */ 460 #define IPVS_EST_TICK ((2 * HZ) / IPVS_EST_NTICKS) 461 462 /* Limit of CPU load per kthread (8 for 12.5%), ratio of CPU capacity (1/C). 463 * Value of 4 and above ensures kthreads will take work without exceeding 464 * the CPU capacity under different circumstances. 465 */ 466 #define IPVS_EST_LOAD_DIVISOR 8 467 468 /* Kthreads should not have work that exceeds the CPU load above 50% */ 469 #define IPVS_EST_CPU_KTHREADS (IPVS_EST_LOAD_DIVISOR / 2) 470 471 /* Desired number of chains per timer tick (chain load factor in 100us units), 472 * 48=4.8ms of 40ms tick (12% CPU usage): 473 * 2 sec * 1000 ms in sec * 10 (100us in ms) / 8 (12.5%) / 50 474 */ 475 #define IPVS_EST_CHAIN_FACTOR \ 476 ALIGN_DOWN(2 * 1000 * 10 / IPVS_EST_LOAD_DIVISOR / IPVS_EST_NTICKS, 8) 477 478 /* Compiled number of chains per tick 479 * The defines should match cond_resched_rcu 480 */ 481 #if defined(CONFIG_DEBUG_ATOMIC_SLEEP) || !defined(CONFIG_PREEMPT_RCU) 482 #define IPVS_EST_TICK_CHAINS IPVS_EST_CHAIN_FACTOR 483 #else 484 #define IPVS_EST_TICK_CHAINS 1 485 #endif 486 487 #if IPVS_EST_NTICKS > 127 488 #error Too many timer ticks for ktrow 489 #endif 490 491 /* Multiple chains processed in same tick */ 492 struct ip_vs_est_tick_data { 493 struct rcu_head rcu_head; 494 struct hlist_head chains[IPVS_EST_TICK_CHAINS]; 495 DECLARE_BITMAP(present, IPVS_EST_TICK_CHAINS); 496 DECLARE_BITMAP(full, IPVS_EST_TICK_CHAINS); 497 int chain_len[IPVS_EST_TICK_CHAINS]; 498 }; 499 500 /* Context for estimation kthread */ 501 struct ip_vs_est_kt_data { 502 struct netns_ipvs *ipvs; 503 struct task_struct *task; /* task if running */ 504 struct ip_vs_est_tick_data __rcu *ticks[IPVS_EST_NTICKS]; 505 DECLARE_BITMAP(avail, IPVS_EST_NTICKS); /* tick has space for ests */ 506 unsigned long est_timer; /* estimation timer (jiffies) */ 507 struct ip_vs_stats *calc_stats; /* Used for calculation */ 508 int needed; /* task is needed */ 509 int tick_len[IPVS_EST_NTICKS]; /* est count */ 510 int id; /* ktid per netns */ 511 int chain_max; /* max ests per tick chain */ 512 int tick_max; /* max ests per tick */ 513 int est_count; /* attached ests to kthread */ 514 int est_max_count; /* max ests per kthread */ 515 int add_row; /* row for new ests */ 516 int est_row; /* estimated row */ 517 }; 518 519 /* IPVS resizable hash tables */ 520 struct ip_vs_rht { 521 struct hlist_bl_head *buckets; 522 struct ip_vs_rht __rcu *new_tbl; /* New/Same table */ 523 seqcount_t *seqc; /* Protects moves */ 524 struct ip_vs_aligned_lock *lock; /* Protect seqc */ 525 int mask; /* Buckets mask */ 526 int size; /* Buckets */ 527 int seqc_mask; /* seqc mask */ 528 int lock_mask; /* lock mask */ 529 u32 table_id; 530 int u_thresh; /* upper threshold */ 531 int l_thresh; /* lower threshold */ 532 int lfactor; /* Load Factor (shift)*/ 533 int bits; /* size = 1 << bits */ 534 siphash_key_t hash_key; 535 struct rcu_head rcu_head; 536 }; 537 538 /** 539 * ip_vs_rht_for_each_table() - Walk the hash tables 540 * @table: struct ip_vs_rht __rcu *table 541 * @t: current table, used as cursor, struct ip_vs_rht *var 542 * @p: previous table, temp struct ip_vs_rht *var 543 * 544 * Walk tables assuming others can not change the installed tables 545 */ 546 #define ip_vs_rht_for_each_table(table, t, p) \ 547 for (p = NULL, t = rcu_dereference_protected(table, 1); \ 548 t != p; \ 549 p = t, t = rcu_dereference_protected(t->new_tbl, 1)) 550 551 /** 552 * ip_vs_rht_for_each_table_rcu() - Walk the hash tables under RCU reader lock 553 * @table: struct ip_vs_rht __rcu *table 554 * @t: current table, used as cursor, struct ip_vs_rht *var 555 * @p: previous table, temp struct ip_vs_rht *var 556 * 557 * We usually search in one table and also in second table on resizing 558 */ 559 #define ip_vs_rht_for_each_table_rcu(table, t, p) \ 560 for (p = NULL, t = rcu_dereference(table); \ 561 t != p; \ 562 p = t, t = rcu_dereference(t->new_tbl)) 563 564 /** 565 * ip_vs_rht_for_each_bucket() - Walk all table buckets 566 * @t: current table, used as cursor, struct ip_vs_rht *var 567 * @bucket: bucket index, used as cursor, u32 var 568 * @head: bucket address, used as cursor, struct hlist_bl_head *var 569 */ 570 #define ip_vs_rht_for_each_bucket(t, bucket, head) \ 571 for (bucket = 0, head = (t)->buckets; \ 572 bucket < t->size; bucket++, head++) 573 574 /** 575 * ip_vs_rht_for_bucket_retry() - Retry bucket if entries are moved 576 * @t: current table, used as cursor, struct ip_vs_rht *var 577 * @bucket: index of current bucket or hash key 578 * @sc: temp seqcount_t *var 579 * @seq: temp unsigned int var for sequence count 580 * @retry: temp int var 581 */ 582 #define ip_vs_rht_for_bucket_retry(t, bucket, sc, seq, retry) \ 583 for (retry = 1, sc = &(t)->seqc[(bucket) & (t)->seqc_mask]; \ 584 retry && ({ seq = read_seqcount_begin(sc); 1; }); \ 585 retry = read_seqcount_retry(sc, seq)) 586 587 /** 588 * DECLARE_IP_VS_RHT_WALK_BUCKETS_RCU() - Declare variables 589 * 590 * Variables for ip_vs_rht_walk_buckets_rcu 591 */ 592 #define DECLARE_IP_VS_RHT_WALK_BUCKETS_RCU() \ 593 struct ip_vs_rht *_t, *_p; \ 594 unsigned int _seq; \ 595 seqcount_t *_sc; \ 596 u32 _bucket; \ 597 int _retry 598 /** 599 * ip_vs_rht_walk_buckets_rcu() - Walk all buckets under RCU read lock 600 * @table: struct ip_vs_rht __rcu *table 601 * @head: bucket address, used as cursor, struct hlist_bl_head *var 602 * 603 * Can be used while others add/delete/move entries 604 * Not suitable if duplicates are not desired 605 * Possible cases for reader that uses cond_resched_rcu() in the loop: 606 * - new table can not be installed, no need to repeat 607 * - new table can be installed => check and repeat if new table is 608 * installed, needed for !PREEMPT_RCU 609 */ 610 #define ip_vs_rht_walk_buckets_rcu(table, head) \ 611 ip_vs_rht_for_each_table_rcu(table, _t, _p) \ 612 ip_vs_rht_for_each_bucket(_t, _bucket, head) \ 613 ip_vs_rht_for_bucket_retry(_t, _bucket, _sc, \ 614 _seq, _retry) 615 616 /** 617 * DECLARE_IP_VS_RHT_WALK_BUCKET_RCU() - Declare variables 618 * 619 * Variables for ip_vs_rht_walk_bucket_rcu 620 */ 621 #define DECLARE_IP_VS_RHT_WALK_BUCKET_RCU() \ 622 unsigned int _seq; \ 623 seqcount_t *_sc; \ 624 int _retry 625 /** 626 * ip_vs_rht_walk_bucket_rcu() - Walk bucket under RCU read lock 627 * @t: current table, struct ip_vs_rht *var 628 * @bucket: index of current bucket or hash key 629 * @head: bucket address, used as cursor, struct hlist_bl_head *var 630 * 631 * Can be used while others add/delete/move entries 632 * Not suitable if duplicates are not desired 633 * Possible cases for reader that uses cond_resched_rcu() in the loop: 634 * - new table can not be installed, no need to repeat 635 * - new table can be installed => check and repeat if new table is 636 * installed, needed for !PREEMPT_RCU 637 */ 638 #define ip_vs_rht_walk_bucket_rcu(t, bucket, head) \ 639 if (({ head = (t)->buckets + ((bucket) & (t)->mask); 0; })) \ 640 {} \ 641 else \ 642 ip_vs_rht_for_bucket_retry(t, (bucket), _sc, _seq, _retry) 643 644 /** 645 * DECLARE_IP_VS_RHT_WALK_BUCKETS_SAFE_RCU() - Declare variables 646 * 647 * Variables for ip_vs_rht_walk_buckets_safe_rcu 648 */ 649 #define DECLARE_IP_VS_RHT_WALK_BUCKETS_SAFE_RCU() \ 650 struct ip_vs_rht *_t, *_p; \ 651 u32 _bucket 652 /** 653 * ip_vs_rht_walk_buckets_safe_rcu() - Walk all buckets under RCU read lock 654 * @table: struct ip_vs_rht __rcu *table 655 * @head: bucket address, used as cursor, struct hlist_bl_head *var 656 * 657 * Can be used while others add/delete entries but moving is disabled 658 * Using cond_resched_rcu() should be safe if tables do not change 659 */ 660 #define ip_vs_rht_walk_buckets_safe_rcu(table, head) \ 661 ip_vs_rht_for_each_table_rcu(table, _t, _p) \ 662 ip_vs_rht_for_each_bucket(_t, _bucket, head) 663 664 /** 665 * DECLARE_IP_VS_RHT_WALK_BUCKETS() - Declare variables 666 * 667 * Variables for ip_vs_rht_walk_buckets 668 */ 669 #define DECLARE_IP_VS_RHT_WALK_BUCKETS() \ 670 struct ip_vs_rht *_t, *_p; \ 671 u32 _bucket 672 673 /** 674 * ip_vs_rht_walk_buckets() - Walk all buckets 675 * @table: struct ip_vs_rht __rcu *table 676 * @head: bucket address, used as cursor, struct hlist_bl_head *var 677 * 678 * Use if others can not add/delete/move entries 679 */ 680 #define ip_vs_rht_walk_buckets(table, head) \ 681 ip_vs_rht_for_each_table(table, _t, _p) \ 682 ip_vs_rht_for_each_bucket(_t, _bucket, head) 683 684 /* Entries can be in one of two tables, so we flip bit when new table is 685 * created and store it as highest bit in hash keys 686 */ 687 #define IP_VS_RHT_TABLE_ID_MASK BIT(31) 688 689 /* Check if hash key is from this table */ 690 static inline bool ip_vs_rht_same_table(struct ip_vs_rht *t, u32 hash_key) 691 { 692 return !((t->table_id ^ hash_key) & IP_VS_RHT_TABLE_ID_MASK); 693 } 694 695 /* Build per-table hash key from hash value */ 696 static inline u32 ip_vs_rht_build_hash_key(struct ip_vs_rht *t, u32 hash) 697 { 698 return t->table_id | (hash & ~IP_VS_RHT_TABLE_ID_MASK); 699 } 700 701 void ip_vs_rht_free(struct ip_vs_rht *t); 702 void ip_vs_rht_rcu_free(struct rcu_head *head); 703 struct ip_vs_rht *ip_vs_rht_alloc(int buckets, int scounts, int locks); 704 int ip_vs_rht_desired_size(struct netns_ipvs *ipvs, struct ip_vs_rht *t, int n, 705 int lfactor, int min_bits, int max_bits); 706 void ip_vs_rht_set_thresholds(struct ip_vs_rht *t, int size, int lfactor, 707 int min_bits, int max_bits); 708 u32 ip_vs_rht_hash_linfo(struct ip_vs_rht *t, int af, 709 const union nf_inet_addr *addr, u32 v1, u32 v2); 710 711 struct dst_entry; 712 struct iphdr; 713 struct ip_vs_conn; 714 struct ip_vs_app; 715 struct sk_buff; 716 struct ip_vs_proto_data; 717 718 struct ip_vs_protocol { 719 struct ip_vs_protocol *next; 720 char *name; 721 u16 protocol; 722 u16 num_states; 723 int dont_defrag; 724 725 void (*init)(struct ip_vs_protocol *pp); 726 727 void (*exit)(struct ip_vs_protocol *pp); 728 729 int (*init_netns)(struct netns_ipvs *ipvs, struct ip_vs_proto_data *pd); 730 731 void (*exit_netns)(struct netns_ipvs *ipvs, struct ip_vs_proto_data *pd); 732 733 int (*conn_schedule)(struct netns_ipvs *ipvs, 734 int af, struct sk_buff *skb, 735 struct ip_vs_proto_data *pd, 736 int *verdict, struct ip_vs_conn **cpp, 737 struct ip_vs_iphdr *iph); 738 739 struct ip_vs_conn * 740 (*conn_in_get)(struct netns_ipvs *ipvs, 741 int af, 742 const struct sk_buff *skb, 743 const struct ip_vs_iphdr *iph); 744 745 struct ip_vs_conn * 746 (*conn_out_get)(struct netns_ipvs *ipvs, 747 int af, 748 const struct sk_buff *skb, 749 const struct ip_vs_iphdr *iph); 750 751 int (*snat_handler)(struct sk_buff *skb, struct ip_vs_protocol *pp, 752 struct ip_vs_conn *cp, struct ip_vs_iphdr *iph); 753 754 int (*dnat_handler)(struct sk_buff *skb, struct ip_vs_protocol *pp, 755 struct ip_vs_conn *cp, struct ip_vs_iphdr *iph); 756 757 const char *(*state_name)(int state); 758 759 void (*state_transition)(struct ip_vs_conn *cp, int direction, 760 const struct sk_buff *skb, 761 struct ip_vs_proto_data *pd, 762 unsigned int iph_len); 763 764 int (*register_app)(struct netns_ipvs *ipvs, struct ip_vs_app *inc); 765 766 void (*unregister_app)(struct netns_ipvs *ipvs, struct ip_vs_app *inc); 767 768 int (*app_conn_bind)(struct ip_vs_conn *cp); 769 770 void (*debug_packet)(int af, struct ip_vs_protocol *pp, 771 const struct sk_buff *skb, 772 int offset, 773 const char *msg); 774 775 void (*timeout_change)(struct ip_vs_proto_data *pd, int flags); 776 }; 777 778 /* protocol data per netns */ 779 struct ip_vs_proto_data { 780 struct ip_vs_proto_data *next; 781 struct ip_vs_protocol *pp; 782 int *timeout_table; /* protocol timeout table */ 783 atomic_t appcnt; /* counter of proto app incs. */ 784 struct tcp_states_t *tcp_state_table; 785 }; 786 787 struct ip_vs_protocol *ip_vs_proto_get(unsigned short proto); 788 struct ip_vs_proto_data *ip_vs_proto_data_get(struct netns_ipvs *ipvs, 789 unsigned short proto); 790 791 struct ip_vs_conn_param { 792 struct netns_ipvs *ipvs; 793 const union nf_inet_addr *caddr; 794 const union nf_inet_addr *vaddr; 795 __be16 cport; 796 __be16 vport; 797 __u16 protocol; 798 u16 af; 799 800 const struct ip_vs_pe *pe; 801 char *pe_data; 802 __u8 pe_data_len; 803 }; 804 805 /* Hash node in conn_tab */ 806 struct ip_vs_conn_hnode { 807 struct hlist_bl_node node; /* node in conn_tab */ 808 u32 hash_key; /* Key for the hash table */ 809 u8 dir; /* 0=out->in, 1=in->out */ 810 } __packed; 811 812 /* IP_VS structure allocated for each dynamically scheduled connection */ 813 struct ip_vs_conn { 814 /* Cacheline for hash table nodes - rarely modified */ 815 816 struct ip_vs_conn_hnode hn0; /* Original direction */ 817 u8 af; /* address family */ 818 __be16 cport; 819 struct ip_vs_conn_hnode hn1; /* Reply direction */ 820 u8 daf; /* Address family of the dest */ 821 __be16 dport; 822 struct ip_vs_dest *dest; /* real server */ 823 atomic_t n_control; /* Number of controlled ones */ 824 volatile __u32 flags; /* status flags */ 825 /* 44/64 */ 826 827 struct ip_vs_conn *control; /* Master control connection */ 828 const struct ip_vs_pe *pe; 829 char *pe_data; 830 __u8 pe_data_len; 831 volatile __u16 state; /* state info */ 832 volatile __u16 old_state; /* old state, to be used for 833 * state transition triggered 834 * synchronization 835 */ 836 /* 2-byte hole */ 837 /* 64/96 */ 838 839 union nf_inet_addr caddr; /* client address */ 840 union nf_inet_addr vaddr; /* virtual address */ 841 /* 96/128 */ 842 843 union nf_inet_addr daddr; /* destination address */ 844 __u32 fwmark; /* Fire wall mark from skb */ 845 __be16 vport; 846 __u16 protocol; /* Which protocol (TCP/UDP) */ 847 848 /* Note: we can group the following members into a structure, 849 * in order to save more space, and the following members are 850 * only used in VS/NAT anyway 851 */ 852 struct ip_vs_app *app; /* bound ip_vs_app object */ 853 void *app_data; /* Application private data */ 854 /* 128/168 */ 855 struct_group(sync_conn_opt, 856 struct ip_vs_seq in_seq; /* incoming seq. struct */ 857 struct ip_vs_seq out_seq; /* outgoing seq. struct */ 858 ); 859 /* 152/192 */ 860 861 struct timer_list timer; /* Expiration timer */ 862 volatile unsigned long timeout; /* timeout */ 863 spinlock_t lock; /* lock for state transition */ 864 refcount_t refcnt; /* reference count */ 865 atomic_t in_pkts; /* incoming packet counter */ 866 /* 64-bit: 4-byte gap */ 867 868 /* 188/256 */ 869 unsigned long sync_endtime; /* jiffies + sent_retries */ 870 struct netns_ipvs *ipvs; 871 872 /* Packet transmitter for different forwarding methods. If it 873 * mangles the packet, it must return NF_DROP or better NF_STOLEN, 874 * otherwise this must be changed to a sk_buff **. 875 * NF_ACCEPT can be returned when destination is local. 876 */ 877 int (*packet_xmit)(struct sk_buff *skb, struct ip_vs_conn *cp, 878 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 879 880 struct rcu_head rcu_head; 881 }; 882 883 /* Extended internal versions of struct ip_vs_service_user and ip_vs_dest_user 884 * for IPv6 support. 885 * 886 * We need these to conveniently pass around service and destination 887 * options, but unfortunately, we also need to keep the old definitions to 888 * maintain userspace backwards compatibility for the setsockopt interface. 889 */ 890 struct ip_vs_service_user_kern { 891 /* virtual service addresses */ 892 u16 af; 893 u16 protocol; 894 union nf_inet_addr addr; /* virtual ip address */ 895 __be16 port; 896 u32 fwmark; /* firewall mark of service */ 897 898 /* virtual service options */ 899 char *sched_name; 900 char *pe_name; 901 unsigned int flags; /* virtual service flags */ 902 unsigned int timeout; /* persistent timeout in sec */ 903 __be32 netmask; /* persistent netmask or plen */ 904 }; 905 906 907 struct ip_vs_dest_user_kern { 908 /* destination server address */ 909 union nf_inet_addr addr; 910 __be16 port; 911 912 /* real server options */ 913 unsigned int conn_flags; /* connection flags */ 914 int weight; /* destination weight */ 915 916 /* thresholds for active connections */ 917 u32 u_threshold; /* upper threshold */ 918 u32 l_threshold; /* lower threshold */ 919 920 /* Address family of addr */ 921 u16 af; 922 923 u16 tun_type; /* tunnel type */ 924 __be16 tun_port; /* tunnel port */ 925 u16 tun_flags; /* tunnel flags */ 926 }; 927 928 929 /* 930 * The information about the virtual service offered to the net and the 931 * forwarding entries. 932 */ 933 struct ip_vs_service { 934 struct hlist_bl_node s_list; /* node in service table */ 935 u32 hash_key; /* Key for the hash table */ 936 u16 af; /* address family */ 937 __u16 protocol; /* which protocol (TCP/UDP) */ 938 939 union nf_inet_addr addr; /* IP address for virtual service */ 940 __u32 fwmark; /* firewall mark of the service */ 941 atomic_t refcnt; /* reference counter */ 942 __be16 port; /* port number for the service */ 943 unsigned int flags; /* service status flags */ 944 unsigned int timeout; /* persistent timeout in ticks */ 945 __be32 netmask; /* grouping granularity, mask/plen */ 946 struct netns_ipvs *ipvs; 947 948 struct list_head destinations; /* real server d-linked list */ 949 __u32 num_dests; /* number of servers */ 950 struct ip_vs_stats stats; /* statistics for the service */ 951 952 /* for scheduling */ 953 struct ip_vs_scheduler __rcu *scheduler; /* bound scheduler object */ 954 spinlock_t sched_lock; /* lock sched_data */ 955 void *sched_data; /* scheduler application data */ 956 957 /* alternate persistence engine */ 958 struct ip_vs_pe __rcu *pe; 959 int conntrack_afmask; 960 961 struct rcu_head rcu_head; 962 }; 963 964 /* Information for cached dst */ 965 struct ip_vs_dest_dst { 966 struct dst_entry *dst_cache; /* destination cache entry */ 967 u32 dst_cookie; 968 union nf_inet_addr dst_saddr; 969 struct rcu_head rcu_head; 970 }; 971 972 /* The real server destination forwarding entry with ip address, port number, 973 * and so on. 974 */ 975 struct ip_vs_dest { 976 struct list_head n_list; /* for the dests in the service */ 977 struct hlist_node d_list; /* for table with all the dests */ 978 979 u16 af; /* address family */ 980 __be16 port; /* port number of the server */ 981 union nf_inet_addr addr; /* IP address of the server */ 982 volatile unsigned int flags; /* dest status flags */ 983 atomic_t conn_flags; /* flags to copy to conn */ 984 atomic_t weight; /* server weight */ 985 unsigned long cflags; /* config flags */ 986 atomic_t last_weight; /* server latest weight */ 987 __u16 tun_type; /* tunnel type */ 988 __be16 tun_port; /* tunnel port */ 989 __u16 tun_flags; /* tunnel flags */ 990 991 refcount_t refcnt; /* reference counter */ 992 struct ip_vs_stats stats; /* statistics */ 993 unsigned long idle_start; /* start time, jiffies */ 994 995 /* connection counters and thresholds */ 996 atomic_t activeconns; /* active connections */ 997 atomic_t totalconns; /* total connections */ 998 atomic_t persistconns; /* persistent connections */ 999 __u32 u_threshold; /* upper threshold */ 1000 __u32 l_threshold; /* lower threshold */ 1001 __u32 l_threshold_val;/* used lower threshold */ 1002 1003 /* for destination cache */ 1004 spinlock_t dst_lock; /* lock of dst_cache */ 1005 struct ip_vs_dest_dst __rcu *dest_dst; /* cached dst info */ 1006 1007 /* for virtual service */ 1008 struct ip_vs_service __rcu *svc; /* service it belongs to */ 1009 __u16 protocol; /* which protocol (TCP/UDP) */ 1010 __be16 vport; /* virtual port number */ 1011 union nf_inet_addr vaddr; /* virtual IP address */ 1012 __u32 vfwmark; /* firewall mark of service */ 1013 1014 struct rcu_head rcu_head; 1015 struct list_head t_list; /* in dest_trash */ 1016 unsigned int in_rs_table:1; /* we are in rs_table */ 1017 }; 1018 1019 /* The scheduler object */ 1020 struct ip_vs_scheduler { 1021 struct list_head n_list; /* d-linked list head */ 1022 char *name; /* scheduler name */ 1023 atomic_t refcnt; /* reference counter */ 1024 struct module *module; /* THIS_MODULE/NULL */ 1025 1026 /* scheduler initializing service */ 1027 int (*init_service)(struct ip_vs_service *svc); 1028 /* scheduling service finish */ 1029 void (*done_service)(struct ip_vs_service *svc); 1030 /* dest is linked */ 1031 int (*add_dest)(struct ip_vs_service *svc, struct ip_vs_dest *dest); 1032 /* dest is unlinked */ 1033 int (*del_dest)(struct ip_vs_service *svc, struct ip_vs_dest *dest); 1034 /* dest is updated */ 1035 int (*upd_dest)(struct ip_vs_service *svc, struct ip_vs_dest *dest); 1036 1037 /* selecting a server from the given service */ 1038 struct ip_vs_dest* (*schedule)(struct ip_vs_service *svc, 1039 const struct sk_buff *skb, 1040 struct ip_vs_iphdr *iph); 1041 }; 1042 1043 /* The persistence engine object */ 1044 struct ip_vs_pe { 1045 struct list_head n_list; /* d-linked list head */ 1046 char *name; /* scheduler name */ 1047 atomic_t refcnt; /* reference counter */ 1048 struct module *module; /* THIS_MODULE/NULL */ 1049 1050 /* get the connection template, if any */ 1051 int (*fill_param)(struct ip_vs_conn_param *p, struct sk_buff *skb); 1052 bool (*ct_match)(const struct ip_vs_conn_param *p, 1053 struct ip_vs_conn *ct); 1054 u32 (*hashkey_raw)(const struct ip_vs_conn_param *p, 1055 struct ip_vs_rht *t, bool inverse); 1056 int (*show_pe_data)(const struct ip_vs_conn *cp, char *buf); 1057 /* create connections for real-server outgoing packets */ 1058 struct ip_vs_conn* (*conn_out)(struct ip_vs_service *svc, 1059 struct ip_vs_dest *dest, 1060 struct sk_buff *skb, 1061 const struct ip_vs_iphdr *iph, 1062 __be16 dport, __be16 cport); 1063 }; 1064 1065 /* The application module object (a.k.a. app incarnation) */ 1066 struct ip_vs_app { 1067 struct list_head a_list; /* member in app list */ 1068 int type; /* IP_VS_APP_TYPE_xxx */ 1069 char *name; /* application module name */ 1070 __u16 protocol; 1071 struct module *module; /* THIS_MODULE/NULL */ 1072 struct list_head incs_list; /* list of incarnations */ 1073 1074 /* members for application incarnations */ 1075 struct list_head p_list; /* member in proto app list */ 1076 struct ip_vs_app *app; /* its real application */ 1077 __be16 port; /* port number in net order */ 1078 atomic_t usecnt; /* usage counter */ 1079 struct rcu_head rcu_head; 1080 1081 /* output hook: Process packet in inout direction, diff set for TCP. 1082 * Return: 0=Error, 1=Payload Not Mangled/Mangled but checksum is ok, 1083 * 2=Mangled but checksum was not updated 1084 */ 1085 int (*pkt_out)(struct ip_vs_app *, struct ip_vs_conn *, 1086 struct sk_buff *, int *diff, struct ip_vs_iphdr *ipvsh); 1087 1088 /* input hook: Process packet in outin direction, diff set for TCP. 1089 * Return: 0=Error, 1=Payload Not Mangled/Mangled but checksum is ok, 1090 * 2=Mangled but checksum was not updated 1091 */ 1092 int (*pkt_in)(struct ip_vs_app *, struct ip_vs_conn *, 1093 struct sk_buff *, int *diff, struct ip_vs_iphdr *ipvsh); 1094 1095 /* ip_vs_app initializer */ 1096 int (*init_conn)(struct ip_vs_app *, struct ip_vs_conn *); 1097 1098 /* ip_vs_app finish */ 1099 int (*done_conn)(struct ip_vs_app *, struct ip_vs_conn *); 1100 1101 1102 /* not used now */ 1103 int (*bind_conn)(struct ip_vs_app *, struct ip_vs_conn *, 1104 struct ip_vs_protocol *); 1105 1106 void (*unbind_conn)(struct ip_vs_app *, struct ip_vs_conn *); 1107 1108 int * timeout_table; 1109 int * timeouts; 1110 int timeouts_size; 1111 1112 int (*conn_schedule)(struct sk_buff *skb, struct ip_vs_app *app, 1113 int *verdict, struct ip_vs_conn **cpp); 1114 1115 struct ip_vs_conn * 1116 (*conn_in_get)(const struct sk_buff *skb, struct ip_vs_app *app, 1117 const struct iphdr *iph, int inverse); 1118 1119 struct ip_vs_conn * 1120 (*conn_out_get)(const struct sk_buff *skb, struct ip_vs_app *app, 1121 const struct iphdr *iph, int inverse); 1122 1123 int (*state_transition)(struct ip_vs_conn *cp, int direction, 1124 const struct sk_buff *skb, 1125 struct ip_vs_app *app); 1126 1127 void (*timeout_change)(struct ip_vs_app *app, int flags); 1128 }; 1129 1130 struct ipvs_master_sync_state { 1131 struct list_head sync_queue; 1132 struct ip_vs_sync_buff *sync_buff; 1133 unsigned long sync_queue_len; 1134 unsigned int sync_queue_delay; 1135 struct delayed_work master_wakeup_work; 1136 struct netns_ipvs *ipvs; 1137 }; 1138 1139 struct ip_vs_sync_thread_data; 1140 1141 /* How much time to keep dests in trash */ 1142 #define IP_VS_DEST_TRASH_PERIOD (120 * HZ) 1143 1144 struct ipvs_sync_daemon_cfg { 1145 union nf_inet_addr mcast_group; 1146 int syncid; 1147 u16 sync_maxlen; 1148 u16 mcast_port; 1149 u8 mcast_af; 1150 u8 mcast_ttl; 1151 /* multicast interface name */ 1152 char mcast_ifn[IP_VS_IFNAME_MAXLEN]; 1153 }; 1154 1155 /* IPVS in network namespace */ 1156 struct netns_ipvs { 1157 int gen; /* Generation */ 1158 int enable; /* enable like nf_hooks do */ 1159 /* Hash table: for real service lookups */ 1160 #define IP_VS_RTAB_BITS 4 1161 #define IP_VS_RTAB_SIZE (1 << IP_VS_RTAB_BITS) 1162 #define IP_VS_RTAB_MASK (IP_VS_RTAB_SIZE - 1) 1163 1164 struct hlist_head rs_table[IP_VS_RTAB_SIZE]; 1165 /* ip_vs_app */ 1166 struct list_head app_list; 1167 /* ip_vs_proto */ 1168 #define IP_VS_PROTO_TAB_SIZE 32 /* must be power of 2 */ 1169 struct ip_vs_proto_data *proto_data_table[IP_VS_PROTO_TAB_SIZE]; 1170 /* ip_vs_proto_tcp */ 1171 #ifdef CONFIG_IP_VS_PROTO_TCP 1172 #define TCP_APP_TAB_BITS 4 1173 #define TCP_APP_TAB_SIZE (1 << TCP_APP_TAB_BITS) 1174 #define TCP_APP_TAB_MASK (TCP_APP_TAB_SIZE - 1) 1175 struct list_head tcp_apps[TCP_APP_TAB_SIZE]; 1176 #endif 1177 /* ip_vs_proto_udp */ 1178 #ifdef CONFIG_IP_VS_PROTO_UDP 1179 #define UDP_APP_TAB_BITS 4 1180 #define UDP_APP_TAB_SIZE (1 << UDP_APP_TAB_BITS) 1181 #define UDP_APP_TAB_MASK (UDP_APP_TAB_SIZE - 1) 1182 struct list_head udp_apps[UDP_APP_TAB_SIZE]; 1183 #endif 1184 /* ip_vs_proto_sctp */ 1185 #ifdef CONFIG_IP_VS_PROTO_SCTP 1186 #define SCTP_APP_TAB_BITS 4 1187 #define SCTP_APP_TAB_SIZE (1 << SCTP_APP_TAB_BITS) 1188 #define SCTP_APP_TAB_MASK (SCTP_APP_TAB_SIZE - 1) 1189 /* Hash table for SCTP application incarnations */ 1190 struct list_head sctp_apps[SCTP_APP_TAB_SIZE]; 1191 #endif 1192 /* ip_vs_conn */ 1193 atomic_t conn_count; /* connection counter */ 1194 atomic_t no_cport_conns[IP_VS_AF_MAX]; 1195 struct delayed_work conn_resize_work;/* resize conn_tab */ 1196 1197 /* ip_vs_ctl */ 1198 struct ip_vs_stats_rcu *tot_stats; /* Statistics & est. */ 1199 1200 /* Trash for destinations */ 1201 struct list_head dest_trash; 1202 spinlock_t dest_trash_lock; 1203 struct timer_list dest_trash_timer; /* expiration timer */ 1204 struct mutex service_mutex; /* service reconfig */ 1205 struct rw_semaphore svc_resize_sem; /* svc_table resizing */ 1206 struct rw_semaphore svc_replace_sem; /* svc_table replace */ 1207 struct delayed_work svc_resize_work; /* resize svc_table */ 1208 atomic_t svc_table_changes;/* ++ on table changes */ 1209 /* Service counters */ 1210 atomic_t num_services[IP_VS_AF_MAX]; /* Services */ 1211 atomic_t fwm_services[IP_VS_AF_MAX]; /* Services */ 1212 atomic_t nonfwm_services[IP_VS_AF_MAX];/* Services */ 1213 atomic_t ftpsvc_counter[IP_VS_AF_MAX]; /* FTPPORT */ 1214 atomic_t nullsvc_counter[IP_VS_AF_MAX];/* Zero port */ 1215 atomic_t conn_out_counter[IP_VS_AF_MAX];/* out conn */ 1216 1217 #ifdef CONFIG_SYSCTL 1218 /* delayed work for expiring no dest connections */ 1219 struct delayed_work expire_nodest_conn_work; 1220 /* 1/rate drop and drop-entry variables */ 1221 struct delayed_work defense_work; /* Work handler */ 1222 int drop_rate; 1223 int drop_counter; 1224 int old_secure_tcp; 1225 atomic_t dropentry; 1226 s8 dropentry_counters[8]; 1227 /* locks in ctl.c */ 1228 spinlock_t dropentry_lock; /* drop entry handling */ 1229 spinlock_t droppacket_lock; /* drop packet handling */ 1230 spinlock_t securetcp_lock; /* state and timeout tables */ 1231 1232 /* sys-ctl struct */ 1233 struct ctl_table_header *sysctl_hdr; 1234 struct ctl_table *sysctl_tbl; 1235 #endif 1236 1237 /* sysctl variables */ 1238 int sysctl_amemthresh; 1239 int sysctl_am_droprate; 1240 #ifdef CONFIG_SYSCTL 1241 int sysctl_conn_max;/* soft limit for conns */ 1242 int conn_max_limit; /* hard limit for conn_max */ 1243 #endif 1244 int sysctl_drop_entry; 1245 int sysctl_drop_packet; 1246 int sysctl_secure_tcp; 1247 #ifdef CONFIG_IP_VS_NFCT 1248 int sysctl_conntrack; 1249 #endif 1250 int sysctl_snat_reroute; 1251 int sysctl_sync_ver; 1252 int sysctl_sync_ports; 1253 int sysctl_sync_persist_mode; 1254 unsigned long sysctl_sync_qlen_max; 1255 int sysctl_sync_sock_size; 1256 int sysctl_cache_bypass; 1257 int sysctl_expire_nodest_conn; 1258 int sysctl_sloppy_tcp; 1259 int sysctl_sloppy_sctp; 1260 int sysctl_expire_quiescent_template; 1261 int sysctl_sync_threshold[2]; 1262 unsigned int sysctl_sync_refresh_period; 1263 int sysctl_sync_retries; 1264 int sysctl_nat_icmp_send; 1265 int sysctl_pmtu_disc; 1266 int sysctl_backup_only; 1267 int sysctl_conn_reuse_mode; 1268 int sysctl_schedule_icmp; 1269 int sysctl_ignore_tunneled; 1270 int sysctl_run_estimation; 1271 #ifdef CONFIG_SYSCTL 1272 cpumask_var_t sysctl_est_cpulist; /* kthread cpumask */ 1273 int est_cpulist_valid; /* cpulist set */ 1274 int sysctl_est_nice; /* kthread nice */ 1275 int est_stopped; /* stop tasks */ 1276 #endif 1277 int sysctl_conn_lfactor; 1278 int sysctl_svc_lfactor; 1279 1280 /* ip_vs_lblc */ 1281 int sysctl_lblc_expiration; 1282 struct ctl_table_header *lblc_ctl_header; 1283 struct ctl_table *lblc_ctl_table; 1284 /* ip_vs_lblcr */ 1285 int sysctl_lblcr_expiration; 1286 struct ctl_table_header *lblcr_ctl_header; 1287 struct ctl_table *lblcr_ctl_table; 1288 unsigned long work_flags; /* IP_VS_WORK_* flags */ 1289 /* ip_vs_est */ 1290 struct delayed_work est_reload_work;/* Reload kthread tasks */ 1291 struct mutex est_mutex; /* protect kthread tasks */ 1292 struct hlist_head est_temp_list; /* Ests during calc phase */ 1293 struct ip_vs_est_kt_data **est_kt_arr; /* Array of kthread data ptrs */ 1294 unsigned long est_max_threads;/* Hard limit of kthreads */ 1295 int est_calc_phase; /* Calculation phase */ 1296 int est_chain_max; /* Calculated chain_max */ 1297 int est_kt_count; /* Allocated ptrs */ 1298 int est_add_ktid; /* ktid where to add ests */ 1299 atomic_t est_genid; /* kthreads reload genid */ 1300 atomic_t est_genid_done; /* applied genid */ 1301 /* ip_vs_sync */ 1302 spinlock_t sync_lock; 1303 struct ipvs_master_sync_state *ms; 1304 spinlock_t sync_buff_lock; 1305 struct ip_vs_sync_thread_data *master_tinfo; 1306 struct ip_vs_sync_thread_data *backup_tinfo; 1307 int threads_mask; 1308 volatile int sync_state; 1309 struct mutex sync_mutex; 1310 struct ipvs_sync_daemon_cfg mcfg; /* Master Configuration */ 1311 struct ipvs_sync_daemon_cfg bcfg; /* Backup Configuration */ 1312 /* net name space ptr */ 1313 struct net *net; /* Needed by timer routines */ 1314 /* Number of heterogeneous destinations, needed because heterogeneous 1315 * are not supported when synchronization is enabled. 1316 */ 1317 unsigned int mixed_address_family_dests; 1318 unsigned int hooks_afmask; /* &1=AF_INET, &2=AF_INET6 */ 1319 1320 struct ip_vs_rht __rcu *svc_table; /* Services */ 1321 struct ip_vs_rht __rcu *conn_tab; /* Connections */ 1322 atomic_t conn_tab_changes;/* ++ on new table */ 1323 }; 1324 1325 #define DEFAULT_SYNC_THRESHOLD 3 1326 #define DEFAULT_SYNC_PERIOD 50 1327 #define DEFAULT_SYNC_VER 1 1328 #define DEFAULT_SLOPPY_TCP 0 1329 #define DEFAULT_SLOPPY_SCTP 0 1330 #define DEFAULT_SYNC_REFRESH_PERIOD (0U * HZ) 1331 #define DEFAULT_SYNC_RETRIES 0 1332 #define IPVS_SYNC_WAKEUP_RATE 8 1333 #define IPVS_SYNC_QLEN_MAX (IPVS_SYNC_WAKEUP_RATE * 4) 1334 #define IPVS_SYNC_SEND_DELAY (HZ / 50) 1335 #define IPVS_SYNC_CHECK_PERIOD HZ 1336 #define IPVS_SYNC_FLUSH_TIME (HZ * 2) 1337 #define IPVS_SYNC_PORTS_MAX (1 << 6) 1338 1339 #ifdef CONFIG_SYSCTL 1340 1341 static inline int sysctl_conn_max(struct netns_ipvs *ipvs) 1342 { 1343 return READ_ONCE(ipvs->sysctl_conn_max); 1344 } 1345 1346 static inline int sysctl_sync_threshold(struct netns_ipvs *ipvs) 1347 { 1348 return ipvs->sysctl_sync_threshold[0]; 1349 } 1350 1351 static inline int sysctl_sync_period(struct netns_ipvs *ipvs) 1352 { 1353 return READ_ONCE(ipvs->sysctl_sync_threshold[1]); 1354 } 1355 1356 static inline unsigned int sysctl_sync_refresh_period(struct netns_ipvs *ipvs) 1357 { 1358 return READ_ONCE(ipvs->sysctl_sync_refresh_period); 1359 } 1360 1361 static inline int sysctl_sync_retries(struct netns_ipvs *ipvs) 1362 { 1363 return ipvs->sysctl_sync_retries; 1364 } 1365 1366 static inline int sysctl_sync_ver(struct netns_ipvs *ipvs) 1367 { 1368 return ipvs->sysctl_sync_ver; 1369 } 1370 1371 static inline int sysctl_sloppy_tcp(struct netns_ipvs *ipvs) 1372 { 1373 return ipvs->sysctl_sloppy_tcp; 1374 } 1375 1376 static inline int sysctl_sloppy_sctp(struct netns_ipvs *ipvs) 1377 { 1378 return ipvs->sysctl_sloppy_sctp; 1379 } 1380 1381 static inline int sysctl_sync_ports(struct netns_ipvs *ipvs) 1382 { 1383 return READ_ONCE(ipvs->sysctl_sync_ports); 1384 } 1385 1386 static inline int sysctl_sync_persist_mode(struct netns_ipvs *ipvs) 1387 { 1388 return ipvs->sysctl_sync_persist_mode; 1389 } 1390 1391 static inline unsigned long sysctl_sync_qlen_max(struct netns_ipvs *ipvs) 1392 { 1393 return ipvs->sysctl_sync_qlen_max; 1394 } 1395 1396 static inline int sysctl_sync_sock_size(struct netns_ipvs *ipvs) 1397 { 1398 return ipvs->sysctl_sync_sock_size; 1399 } 1400 1401 static inline int sysctl_pmtu_disc(struct netns_ipvs *ipvs) 1402 { 1403 return ipvs->sysctl_pmtu_disc; 1404 } 1405 1406 static inline int sysctl_backup_only(struct netns_ipvs *ipvs) 1407 { 1408 return ipvs->sync_state & IP_VS_STATE_BACKUP && 1409 ipvs->sysctl_backup_only; 1410 } 1411 1412 static inline int sysctl_conn_reuse_mode(struct netns_ipvs *ipvs) 1413 { 1414 return ipvs->sysctl_conn_reuse_mode; 1415 } 1416 1417 static inline int sysctl_expire_nodest_conn(struct netns_ipvs *ipvs) 1418 { 1419 return ipvs->sysctl_expire_nodest_conn; 1420 } 1421 1422 static inline int sysctl_schedule_icmp(struct netns_ipvs *ipvs) 1423 { 1424 return ipvs->sysctl_schedule_icmp; 1425 } 1426 1427 static inline int sysctl_ignore_tunneled(struct netns_ipvs *ipvs) 1428 { 1429 return ipvs->sysctl_ignore_tunneled; 1430 } 1431 1432 static inline int sysctl_cache_bypass(struct netns_ipvs *ipvs) 1433 { 1434 return ipvs->sysctl_cache_bypass; 1435 } 1436 1437 static inline int sysctl_run_estimation(struct netns_ipvs *ipvs) 1438 { 1439 return ipvs->sysctl_run_estimation; 1440 } 1441 1442 static inline const struct cpumask *__sysctl_est_cpulist(struct netns_ipvs *ipvs) 1443 { 1444 if (ipvs->est_cpulist_valid) 1445 return ipvs->sysctl_est_cpulist; 1446 else 1447 return housekeeping_cpumask(HK_TYPE_KTHREAD); 1448 } 1449 1450 static inline const struct cpumask *sysctl_est_preferred_cpulist(struct netns_ipvs *ipvs) 1451 { 1452 if (ipvs->est_cpulist_valid) 1453 return ipvs->sysctl_est_cpulist; 1454 else 1455 return NULL; 1456 } 1457 1458 static inline int sysctl_est_nice(struct netns_ipvs *ipvs) 1459 { 1460 return ipvs->sysctl_est_nice; 1461 } 1462 1463 #else 1464 1465 static inline int sysctl_conn_max(struct netns_ipvs *ipvs) 1466 { 1467 return IP_VS_CONN_MAX; 1468 } 1469 1470 static inline int sysctl_sync_threshold(struct netns_ipvs *ipvs) 1471 { 1472 return DEFAULT_SYNC_THRESHOLD; 1473 } 1474 1475 static inline int sysctl_sync_period(struct netns_ipvs *ipvs) 1476 { 1477 return DEFAULT_SYNC_PERIOD; 1478 } 1479 1480 static inline unsigned int sysctl_sync_refresh_period(struct netns_ipvs *ipvs) 1481 { 1482 return DEFAULT_SYNC_REFRESH_PERIOD; 1483 } 1484 1485 static inline int sysctl_sync_retries(struct netns_ipvs *ipvs) 1486 { 1487 return DEFAULT_SYNC_RETRIES & 3; 1488 } 1489 1490 static inline int sysctl_sync_ver(struct netns_ipvs *ipvs) 1491 { 1492 return DEFAULT_SYNC_VER; 1493 } 1494 1495 static inline int sysctl_sloppy_tcp(struct netns_ipvs *ipvs) 1496 { 1497 return DEFAULT_SLOPPY_TCP; 1498 } 1499 1500 static inline int sysctl_sloppy_sctp(struct netns_ipvs *ipvs) 1501 { 1502 return DEFAULT_SLOPPY_SCTP; 1503 } 1504 1505 static inline int sysctl_sync_ports(struct netns_ipvs *ipvs) 1506 { 1507 return 1; 1508 } 1509 1510 static inline int sysctl_sync_persist_mode(struct netns_ipvs *ipvs) 1511 { 1512 return 0; 1513 } 1514 1515 static inline unsigned long sysctl_sync_qlen_max(struct netns_ipvs *ipvs) 1516 { 1517 return IPVS_SYNC_QLEN_MAX; 1518 } 1519 1520 static inline int sysctl_sync_sock_size(struct netns_ipvs *ipvs) 1521 { 1522 return 0; 1523 } 1524 1525 static inline int sysctl_pmtu_disc(struct netns_ipvs *ipvs) 1526 { 1527 return 1; 1528 } 1529 1530 static inline int sysctl_backup_only(struct netns_ipvs *ipvs) 1531 { 1532 return 0; 1533 } 1534 1535 static inline int sysctl_conn_reuse_mode(struct netns_ipvs *ipvs) 1536 { 1537 return 1; 1538 } 1539 1540 static inline int sysctl_expire_nodest_conn(struct netns_ipvs *ipvs) 1541 { 1542 return 0; 1543 } 1544 1545 static inline int sysctl_schedule_icmp(struct netns_ipvs *ipvs) 1546 { 1547 return 0; 1548 } 1549 1550 static inline int sysctl_ignore_tunneled(struct netns_ipvs *ipvs) 1551 { 1552 return 0; 1553 } 1554 1555 static inline int sysctl_cache_bypass(struct netns_ipvs *ipvs) 1556 { 1557 return 0; 1558 } 1559 1560 static inline int sysctl_run_estimation(struct netns_ipvs *ipvs) 1561 { 1562 return 1; 1563 } 1564 1565 static inline const struct cpumask *__sysctl_est_cpulist(struct netns_ipvs *ipvs) 1566 { 1567 return housekeeping_cpumask(HK_TYPE_KTHREAD); 1568 } 1569 1570 static inline const struct cpumask *sysctl_est_preferred_cpulist(struct netns_ipvs *ipvs) 1571 { 1572 return NULL; 1573 } 1574 1575 static inline int sysctl_est_nice(struct netns_ipvs *ipvs) 1576 { 1577 return IPVS_EST_NICE; 1578 } 1579 1580 #endif 1581 1582 /* Get load factor to map conn_count/u_thresh to t->size */ 1583 static inline int sysctl_conn_lfactor(struct netns_ipvs *ipvs) 1584 { 1585 return READ_ONCE(ipvs->sysctl_conn_lfactor); 1586 } 1587 1588 /* Get load factor to map num_services/u_thresh to t->size 1589 * Smaller value decreases u_thresh to reduce collisions but increases 1590 * the table size 1591 * Returns factor where: 1592 * - <0: u_thresh = size >> -factor, eg. lfactor -2 = 25% load 1593 * - >=0: u_thresh = size << factor, eg. lfactor 1 = 200% load 1594 */ 1595 static inline int sysctl_svc_lfactor(struct netns_ipvs *ipvs) 1596 { 1597 return READ_ONCE(ipvs->sysctl_svc_lfactor); 1598 } 1599 1600 static inline bool sysctl_est_cpulist_empty(struct netns_ipvs *ipvs) 1601 { 1602 guard(rcu)(); 1603 return cpumask_empty(__sysctl_est_cpulist(ipvs)); 1604 } 1605 1606 static inline unsigned int sysctl_est_cpulist_weight(struct netns_ipvs *ipvs) 1607 { 1608 guard(rcu)(); 1609 return cpumask_weight(__sysctl_est_cpulist(ipvs)); 1610 } 1611 1612 /* IPVS core functions 1613 * (from ip_vs_core.c) 1614 */ 1615 const char *ip_vs_proto_name(unsigned int proto); 1616 void ip_vs_init_hash_table(struct list_head *table, int rows); 1617 struct ip_vs_conn *ip_vs_new_conn_out(struct ip_vs_service *svc, 1618 struct ip_vs_dest *dest, 1619 struct sk_buff *skb, 1620 const struct ip_vs_iphdr *iph, 1621 __be16 dport, 1622 __be16 cport); 1623 #define IP_VS_INIT_HASH_TABLE(t) ip_vs_init_hash_table((t), ARRAY_SIZE((t))) 1624 1625 #define IP_VS_APP_TYPE_FTP 1 1626 1627 /* ip_vs_conn handling functions 1628 * (from ip_vs_conn.c) 1629 */ 1630 enum { 1631 IP_VS_DIR_INPUT = 0, 1632 IP_VS_DIR_OUTPUT, 1633 IP_VS_DIR_INPUT_ONLY, 1634 IP_VS_DIR_LAST, 1635 }; 1636 1637 static inline void ip_vs_conn_fill_param(struct netns_ipvs *ipvs, int af, int protocol, 1638 const union nf_inet_addr *caddr, 1639 __be16 cport, 1640 const union nf_inet_addr *vaddr, 1641 __be16 vport, 1642 struct ip_vs_conn_param *p) 1643 { 1644 p->ipvs = ipvs; 1645 p->af = af; 1646 p->protocol = protocol; 1647 p->caddr = caddr; 1648 p->cport = cport; 1649 p->vaddr = vaddr; 1650 p->vport = vport; 1651 p->pe = NULL; 1652 p->pe_data = NULL; 1653 } 1654 1655 struct ip_vs_conn *ip_vs_conn_in_get(const struct ip_vs_conn_param *p); 1656 struct ip_vs_conn *ip_vs_ct_in_get(const struct ip_vs_conn_param *p); 1657 1658 struct ip_vs_conn * ip_vs_conn_in_get_proto(struct netns_ipvs *ipvs, int af, 1659 const struct sk_buff *skb, 1660 const struct ip_vs_iphdr *iph); 1661 1662 struct ip_vs_conn *ip_vs_conn_out_get(const struct ip_vs_conn_param *p); 1663 1664 struct ip_vs_conn * ip_vs_conn_out_get_proto(struct netns_ipvs *ipvs, int af, 1665 const struct sk_buff *skb, 1666 const struct ip_vs_iphdr *iph); 1667 1668 /* Get reference to gain full access to conn. 1669 * By default, RCU read-side critical sections have access only to 1670 * conn fields and its PE data, see ip_vs_conn_rcu_free() for reference. 1671 */ 1672 static inline bool __ip_vs_conn_get(struct ip_vs_conn *cp) 1673 { 1674 return refcount_inc_not_zero(&cp->refcnt); 1675 } 1676 1677 /* put back the conn without restarting its timer */ 1678 static inline void __ip_vs_conn_put(struct ip_vs_conn *cp) 1679 { 1680 smp_mb__before_atomic(); 1681 refcount_dec(&cp->refcnt); 1682 } 1683 void ip_vs_conn_put(struct ip_vs_conn *cp); 1684 void ip_vs_conn_fill_cport(struct ip_vs_conn *cp, __be16 cport); 1685 int ip_vs_conn_desired_size(struct netns_ipvs *ipvs, struct ip_vs_rht *t, 1686 int lfactor); 1687 struct ip_vs_rht *ip_vs_conn_tab_alloc(struct netns_ipvs *ipvs, int buckets, 1688 int lfactor); 1689 1690 static inline struct ip_vs_conn * 1691 ip_vs_hn0_to_conn(struct ip_vs_conn_hnode *hn) 1692 { 1693 return container_of(hn, struct ip_vs_conn, hn0); 1694 } 1695 1696 static inline struct ip_vs_conn * 1697 ip_vs_hn_to_conn(struct ip_vs_conn_hnode *hn) 1698 { 1699 return hn->dir ? container_of(hn, struct ip_vs_conn, hn1) : 1700 container_of(hn, struct ip_vs_conn, hn0); 1701 } 1702 1703 struct ip_vs_conn *ip_vs_conn_new(const struct ip_vs_conn_param *p, int dest_af, 1704 const union nf_inet_addr *daddr, 1705 __be16 dport, unsigned int flags, 1706 struct ip_vs_dest *dest, __u32 fwmark); 1707 void ip_vs_conn_expire_now(struct ip_vs_conn *cp); 1708 1709 const char *ip_vs_state_name(const struct ip_vs_conn *cp); 1710 1711 void ip_vs_tcp_conn_listen(struct ip_vs_conn *cp); 1712 int ip_vs_check_template(struct ip_vs_conn *ct, struct ip_vs_dest *cdest); 1713 void ip_vs_random_dropentry(struct netns_ipvs *ipvs); 1714 int ip_vs_conn_init(void); 1715 void ip_vs_conn_cleanup(void); 1716 1717 static inline void ip_vs_control_del(struct ip_vs_conn *cp) 1718 { 1719 struct ip_vs_conn *ctl_cp = cp->control; 1720 if (!ctl_cp) { 1721 IP_VS_ERR_BUF("request control DEL for uncontrolled: " 1722 "%s:%d to %s:%d\n", 1723 IP_VS_DBG_ADDR(cp->af, &cp->caddr), 1724 ntohs(cp->cport), 1725 IP_VS_DBG_ADDR(cp->af, &cp->vaddr), 1726 ntohs(cp->vport)); 1727 1728 return; 1729 } 1730 1731 IP_VS_DBG_BUF(7, "DELeting control for: " 1732 "cp.dst=%s:%d ctl_cp.dst=%s:%d\n", 1733 IP_VS_DBG_ADDR(cp->af, &cp->caddr), 1734 ntohs(cp->cport), 1735 IP_VS_DBG_ADDR(cp->af, &ctl_cp->caddr), 1736 ntohs(ctl_cp->cport)); 1737 1738 cp->control = NULL; 1739 if (atomic_read(&ctl_cp->n_control) == 0) { 1740 IP_VS_ERR_BUF("BUG control DEL with n=0 : " 1741 "%s:%d to %s:%d\n", 1742 IP_VS_DBG_ADDR(cp->af, &cp->caddr), 1743 ntohs(cp->cport), 1744 IP_VS_DBG_ADDR(cp->af, &cp->vaddr), 1745 ntohs(cp->vport)); 1746 1747 return; 1748 } 1749 atomic_dec(&ctl_cp->n_control); 1750 } 1751 1752 static inline void 1753 ip_vs_control_add(struct ip_vs_conn *cp, struct ip_vs_conn *ctl_cp) 1754 { 1755 if (cp->control) { 1756 IP_VS_ERR_BUF("request control ADD for already controlled: " 1757 "%s:%d to %s:%d\n", 1758 IP_VS_DBG_ADDR(cp->af, &cp->caddr), 1759 ntohs(cp->cport), 1760 IP_VS_DBG_ADDR(cp->af, &cp->vaddr), 1761 ntohs(cp->vport)); 1762 1763 ip_vs_control_del(cp); 1764 } 1765 1766 IP_VS_DBG_BUF(7, "ADDing control for: " 1767 "cp.dst=%s:%d ctl_cp.dst=%s:%d\n", 1768 IP_VS_DBG_ADDR(cp->af, &cp->caddr), 1769 ntohs(cp->cport), 1770 IP_VS_DBG_ADDR(cp->af, &ctl_cp->caddr), 1771 ntohs(ctl_cp->cport)); 1772 1773 cp->control = ctl_cp; 1774 atomic_inc(&ctl_cp->n_control); 1775 } 1776 1777 /* Mark our template as assured */ 1778 static inline void 1779 ip_vs_control_assure_ct(struct ip_vs_conn *cp) 1780 { 1781 struct ip_vs_conn *ct = cp->control; 1782 1783 if (ct && !(ct->state & IP_VS_CTPL_S_ASSURED) && 1784 (ct->flags & IP_VS_CONN_F_TEMPLATE)) 1785 ct->state |= IP_VS_CTPL_S_ASSURED; 1786 } 1787 1788 /* IPVS netns init & cleanup functions */ 1789 int ip_vs_estimator_net_init(struct netns_ipvs *ipvs); 1790 int ip_vs_control_net_init(struct netns_ipvs *ipvs); 1791 int ip_vs_protocol_net_init(struct netns_ipvs *ipvs); 1792 int ip_vs_app_net_init(struct netns_ipvs *ipvs); 1793 int ip_vs_conn_net_init(struct netns_ipvs *ipvs); 1794 int ip_vs_sync_net_init(struct netns_ipvs *ipvs); 1795 void ip_vs_conn_net_cleanup(struct netns_ipvs *ipvs); 1796 void ip_vs_app_net_cleanup(struct netns_ipvs *ipvs); 1797 void ip_vs_protocol_net_cleanup(struct netns_ipvs *ipvs); 1798 void ip_vs_control_net_cleanup(struct netns_ipvs *ipvs); 1799 void ip_vs_estimator_net_cleanup(struct netns_ipvs *ipvs); 1800 void ip_vs_sync_net_cleanup(struct netns_ipvs *ipvs); 1801 void ip_vs_service_nets_cleanup(struct list_head *net_list); 1802 1803 /* IPVS application functions 1804 * (from ip_vs_app.c) 1805 */ 1806 #define IP_VS_APP_MAX_PORTS 8 1807 struct ip_vs_app *register_ip_vs_app(struct netns_ipvs *ipvs, struct ip_vs_app *app); 1808 void unregister_ip_vs_app(struct netns_ipvs *ipvs, struct ip_vs_app *app); 1809 int ip_vs_bind_app(struct ip_vs_conn *cp, struct ip_vs_protocol *pp); 1810 void ip_vs_unbind_app(struct ip_vs_conn *cp); 1811 int register_ip_vs_app_inc(struct netns_ipvs *ipvs, struct ip_vs_app *app, __u16 proto, 1812 __u16 port); 1813 int ip_vs_app_inc_get(struct ip_vs_app *inc); 1814 void ip_vs_app_inc_put(struct ip_vs_app *inc); 1815 1816 int ip_vs_app_pkt_out(struct ip_vs_conn *, struct sk_buff *skb, 1817 struct ip_vs_iphdr *ipvsh); 1818 int ip_vs_app_pkt_in(struct ip_vs_conn *, struct sk_buff *skb, 1819 struct ip_vs_iphdr *ipvsh); 1820 1821 int register_ip_vs_pe(struct ip_vs_pe *pe); 1822 int unregister_ip_vs_pe(struct ip_vs_pe *pe); 1823 struct ip_vs_pe *ip_vs_pe_getbyname(const char *name); 1824 struct ip_vs_pe *__ip_vs_pe_getbyname(const char *pe_name); 1825 1826 /* Use a #define to avoid all of module.h just for these trivial ops */ 1827 #define ip_vs_pe_get(pe) \ 1828 if (pe && pe->module) \ 1829 __module_get(pe->module); 1830 1831 #define ip_vs_pe_put(pe) \ 1832 if (pe && pe->module) \ 1833 module_put(pe->module); 1834 1835 /* IPVS protocol functions (from ip_vs_proto.c) */ 1836 int ip_vs_protocol_init(void); 1837 void ip_vs_protocol_cleanup(void); 1838 void ip_vs_protocol_timeout_change(struct netns_ipvs *ipvs, int flags); 1839 int *ip_vs_create_timeout_table(int *table, int size); 1840 void ip_vs_tcpudp_debug_packet(int af, struct ip_vs_protocol *pp, 1841 const struct sk_buff *skb, int offset, 1842 const char *msg); 1843 1844 extern struct ip_vs_protocol ip_vs_protocol_tcp; 1845 extern struct ip_vs_protocol ip_vs_protocol_udp; 1846 extern struct ip_vs_protocol ip_vs_protocol_icmp; 1847 extern struct ip_vs_protocol ip_vs_protocol_esp; 1848 extern struct ip_vs_protocol ip_vs_protocol_ah; 1849 extern struct ip_vs_protocol ip_vs_protocol_sctp; 1850 1851 /* Registering/unregistering scheduler functions 1852 * (from ip_vs_sched.c) 1853 */ 1854 int register_ip_vs_scheduler(struct ip_vs_scheduler *scheduler); 1855 int unregister_ip_vs_scheduler(struct ip_vs_scheduler *scheduler); 1856 int ip_vs_bind_scheduler(struct ip_vs_service *svc, 1857 struct ip_vs_scheduler *scheduler); 1858 void ip_vs_unbind_scheduler(struct ip_vs_service *svc); 1859 struct ip_vs_scheduler *ip_vs_scheduler_get(const char *sched_name); 1860 void ip_vs_scheduler_put(struct ip_vs_scheduler *scheduler); 1861 struct ip_vs_conn * 1862 ip_vs_schedule(struct ip_vs_service *svc, struct sk_buff *skb, 1863 struct ip_vs_proto_data *pd, int *ignored, 1864 struct ip_vs_iphdr *iph); 1865 int ip_vs_leave(struct ip_vs_service *svc, struct sk_buff *skb, 1866 struct ip_vs_proto_data *pd, struct ip_vs_iphdr *iph); 1867 1868 void ip_vs_scheduler_err(struct ip_vs_service *svc, const char *msg); 1869 1870 /* IPVS control data and functions (from ip_vs_ctl.c) */ 1871 extern struct ip_vs_stats ip_vs_stats; 1872 extern int sysctl_ip_vs_sync_ver; 1873 1874 struct ip_vs_service * 1875 ip_vs_service_find(struct netns_ipvs *ipvs, int af, __u32 fwmark, __u16 protocol, 1876 const union nf_inet_addr *vaddr, __be16 vport); 1877 1878 bool ip_vs_has_real_service(struct netns_ipvs *ipvs, int af, __u16 protocol, 1879 const union nf_inet_addr *daddr, __be16 dport); 1880 1881 struct ip_vs_dest * 1882 ip_vs_find_real_service(struct netns_ipvs *ipvs, int af, __u16 protocol, 1883 const union nf_inet_addr *daddr, __be16 dport); 1884 struct ip_vs_dest *ip_vs_find_tunnel(struct netns_ipvs *ipvs, int af, 1885 const union nf_inet_addr *daddr, 1886 __be16 tun_port); 1887 1888 int ip_vs_use_count_inc(void); 1889 void ip_vs_use_count_dec(void); 1890 int ip_vs_register_nl_ioctl(void); 1891 void ip_vs_unregister_nl_ioctl(void); 1892 int ip_vs_control_init(void); 1893 void ip_vs_control_cleanup(void); 1894 struct ip_vs_dest * 1895 ip_vs_find_dest(struct netns_ipvs *ipvs, int svc_af, int dest_af, 1896 const union nf_inet_addr *daddr, __be16 dport, 1897 const union nf_inet_addr *vaddr, __be16 vport, 1898 __u16 protocol, __u32 fwmark, __u32 flags); 1899 void ip_vs_try_bind_dest(struct ip_vs_conn *cp); 1900 1901 static inline void ip_vs_dest_hold(struct ip_vs_dest *dest) 1902 { 1903 refcount_inc(&dest->refcnt); 1904 } 1905 1906 static inline void ip_vs_dest_put(struct ip_vs_dest *dest) 1907 { 1908 smp_mb__before_atomic(); 1909 refcount_dec(&dest->refcnt); 1910 } 1911 1912 static inline void ip_vs_dest_put_and_free(struct ip_vs_dest *dest) 1913 { 1914 if (refcount_dec_and_test(&dest->refcnt)) 1915 kfree(dest); 1916 } 1917 1918 void ip_vs_dest_update_overload(struct ip_vs_dest *dest, int mode); 1919 1920 /* IPVS sync daemon data and function prototypes 1921 * (from ip_vs_sync.c) 1922 */ 1923 int start_sync_thread(struct netns_ipvs *ipvs, struct ipvs_sync_daemon_cfg *cfg, 1924 int state); 1925 int stop_sync_thread(struct netns_ipvs *ipvs, int state); 1926 void ip_vs_sync_conn(struct netns_ipvs *ipvs, struct ip_vs_conn *cp, int pkts); 1927 1928 /* IPVS rate estimator prototypes (from ip_vs_est.c) */ 1929 int ip_vs_start_estimator(struct netns_ipvs *ipvs, struct ip_vs_stats *stats); 1930 void ip_vs_stop_estimator(struct netns_ipvs *ipvs, struct ip_vs_stats *stats); 1931 void ip_vs_zero_estimator(struct ip_vs_stats *stats); 1932 void ip_vs_read_estimator(struct ip_vs_kstats *dst, struct ip_vs_stats *stats); 1933 void ip_vs_est_reload_start(struct netns_ipvs *ipvs, bool restart); 1934 int ip_vs_est_kthread_start(struct netns_ipvs *ipvs, 1935 struct ip_vs_est_kt_data *kd); 1936 void ip_vs_est_kthread_stop(struct ip_vs_est_kt_data *kd); 1937 1938 static inline void ip_vs_stop_estimator_tot_stats(struct netns_ipvs *ipvs) 1939 { 1940 #ifdef CONFIG_SYSCTL 1941 ip_vs_stop_estimator(ipvs, &ipvs->tot_stats->s); 1942 ipvs->tot_stats->s.est.ktid = -2; 1943 #endif 1944 } 1945 1946 static inline void ip_vs_est_stopped_recalc(struct netns_ipvs *ipvs) 1947 { 1948 #ifdef CONFIG_SYSCTL 1949 /* Stop tasks while cpulist is empty or if disabled with flag */ 1950 ipvs->est_stopped = !sysctl_run_estimation(ipvs) || 1951 (ipvs->est_cpulist_valid && 1952 sysctl_est_cpulist_empty(ipvs)); 1953 #endif 1954 } 1955 1956 static inline bool ip_vs_est_stopped(struct netns_ipvs *ipvs) 1957 { 1958 #ifdef CONFIG_SYSCTL 1959 return ipvs->est_stopped; 1960 #else 1961 return false; 1962 #endif 1963 } 1964 1965 static inline int ip_vs_est_max_threads(struct netns_ipvs *ipvs) 1966 { 1967 unsigned int limit = IPVS_EST_CPU_KTHREADS * 1968 sysctl_est_cpulist_weight(ipvs); 1969 1970 return max(1U, limit); 1971 } 1972 1973 /* Various IPVS packet transmitters (from ip_vs_xmit.c) */ 1974 int ip_vs_null_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 1975 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1976 int ip_vs_bypass_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 1977 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1978 int ip_vs_nat_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 1979 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1980 int ip_vs_tunnel_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 1981 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1982 int ip_vs_dr_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 1983 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1984 int ip_vs_icmp_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 1985 struct ip_vs_protocol *pp, unsigned int toff, 1986 unsigned int hooknum, struct ip_vs_iphdr *ciph); 1987 void ip_vs_dest_dst_rcu_free(struct rcu_head *head); 1988 1989 #ifdef CONFIG_IP_VS_IPV6 1990 int ip_vs_bypass_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, 1991 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1992 int ip_vs_nat_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, 1993 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1994 int ip_vs_tunnel_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, 1995 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1996 int ip_vs_dr_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, 1997 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1998 int ip_vs_icmp_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, 1999 struct ip_vs_protocol *pp, unsigned int toff, 2000 unsigned int hooknum, struct ip_vs_iphdr *ciph); 2001 #endif 2002 2003 #ifdef CONFIG_SYSCTL 2004 /* This is a simple mechanism to ignore packets when 2005 * we are loaded. Just set ip_vs_drop_rate to 'n' and 2006 * we start to drop 1/rate of the packets 2007 */ 2008 static inline int ip_vs_todrop(struct netns_ipvs *ipvs) 2009 { 2010 if (!ipvs->drop_rate) 2011 return 0; 2012 if (--ipvs->drop_counter > 0) 2013 return 0; 2014 ipvs->drop_counter = ipvs->drop_rate; 2015 return 1; 2016 } 2017 #else 2018 static inline int ip_vs_todrop(struct netns_ipvs *ipvs) { return 0; } 2019 #endif 2020 2021 #ifdef CONFIG_SYSCTL 2022 /* Enqueue delayed work for expiring no dest connections 2023 * Only run when sysctl_expire_nodest=1 2024 */ 2025 static inline void ip_vs_enqueue_expire_nodest_conns(struct netns_ipvs *ipvs) 2026 { 2027 if (sysctl_expire_nodest_conn(ipvs)) 2028 queue_delayed_work(system_long_wq, 2029 &ipvs->expire_nodest_conn_work, 1); 2030 } 2031 2032 void ip_vs_expire_nodest_conn_flush(struct netns_ipvs *ipvs); 2033 #else 2034 static inline void ip_vs_enqueue_expire_nodest_conns(struct netns_ipvs *ipvs) {} 2035 #endif 2036 2037 #define IP_VS_DFWD_METHOD(dest) (atomic_read(&(dest)->conn_flags) & \ 2038 IP_VS_CONN_F_FWD_MASK) 2039 2040 /* ip_vs_fwd_tag returns the forwarding tag of the connection */ 2041 #define IP_VS_FWD_METHOD(cp) (cp->flags & IP_VS_CONN_F_FWD_MASK) 2042 2043 static inline char ip_vs_fwd_tag(struct ip_vs_conn *cp) 2044 { 2045 char fwd; 2046 2047 switch (IP_VS_FWD_METHOD(cp)) { 2048 case IP_VS_CONN_F_MASQ: 2049 fwd = 'M'; break; 2050 case IP_VS_CONN_F_LOCALNODE: 2051 fwd = 'L'; break; 2052 case IP_VS_CONN_F_TUNNEL: 2053 fwd = 'T'; break; 2054 case IP_VS_CONN_F_DROUTE: 2055 fwd = 'R'; break; 2056 case IP_VS_CONN_F_BYPASS: 2057 fwd = 'B'; break; 2058 default: 2059 fwd = '?'; break; 2060 } 2061 return fwd; 2062 } 2063 2064 /* Check if connection uses double hashing */ 2065 static inline bool ip_vs_conn_use_hash2(struct ip_vs_conn *cp) 2066 { 2067 return IP_VS_FWD_METHOD(cp) == IP_VS_CONN_F_MASQ && 2068 !(cp->flags & IP_VS_CONN_F_TEMPLATE); 2069 } 2070 2071 bool ip_vs_nat_icmp(struct sk_buff *skb, struct ip_vs_protocol *pp, 2072 struct ip_vs_conn *cp, int dir, unsigned int toff, 2073 bool has_ports, struct ip_vs_iphdr *ciph); 2074 2075 #ifdef CONFIG_IP_VS_IPV6 2076 void ip_vs_nat_icmp_v6(struct sk_buff *skb, struct ip_vs_protocol *pp, 2077 struct ip_vs_conn *cp, int dir, unsigned int toff, 2078 bool has_ports, struct ip_vs_iphdr *ciph); 2079 #endif 2080 2081 static inline __wsum ip_vs_check_diff4(__be32 old, __be32 new, __wsum oldsum) 2082 { 2083 __be32 diff[2] = { ~old, new }; 2084 2085 return csum_partial(diff, sizeof(diff), oldsum); 2086 } 2087 2088 #ifdef CONFIG_IP_VS_IPV6 2089 static inline __wsum ip_vs_check_diff16(const __be32 *old, const __be32 *new, 2090 __wsum oldsum) 2091 { 2092 __be32 diff[8] = { ~old[3], ~old[2], ~old[1], ~old[0], 2093 new[3], new[2], new[1], new[0] }; 2094 2095 return csum_partial(diff, sizeof(diff), oldsum); 2096 } 2097 #endif 2098 2099 static inline __wsum ip_vs_check_diff2(__be16 old, __be16 new, __wsum oldsum) 2100 { 2101 __be16 diff[2] = { ~old, new }; 2102 2103 return csum_partial(diff, sizeof(diff), oldsum); 2104 } 2105 2106 static inline bool ip_vs_checksum_needed(struct sk_buff *skb) 2107 { 2108 /* Checksum unnecessary or already validated? */ 2109 if (skb_csum_unnecessary(skb)) 2110 return false; 2111 /* Locally generated ? */ 2112 if (!skb->dev) 2113 return false; 2114 return true; 2115 } 2116 2117 static inline bool ip_vs_checksum_common_check(struct sk_buff *skb, 2118 int offset, int proto, int af) 2119 { 2120 if (!ip_vs_checksum_needed(skb)) 2121 return true; 2122 /* Validate csum even for FORWARD */ 2123 return !nf_checksum(skb, NF_INET_LOCAL_IN, offset, proto, af); 2124 } 2125 2126 /* Forget current conntrack (unconfirmed) and attach notrack entry */ 2127 static inline void ip_vs_notrack(struct sk_buff *skb) 2128 { 2129 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE) 2130 enum ip_conntrack_info ctinfo; 2131 struct nf_conn *ct = nf_ct_get(skb, &ctinfo); 2132 2133 if (ct) { 2134 nf_conntrack_put(&ct->ct_general); 2135 nf_ct_set(skb, NULL, IP_CT_UNTRACKED); 2136 } 2137 #endif 2138 } 2139 2140 #ifdef CONFIG_IP_VS_NFCT 2141 /* Netfilter connection tracking 2142 * (from ip_vs_nfct.c) 2143 */ 2144 static inline int ip_vs_conntrack_enabled(struct netns_ipvs *ipvs) 2145 { 2146 #ifdef CONFIG_SYSCTL 2147 return ipvs->sysctl_conntrack; 2148 #else 2149 return 0; 2150 #endif 2151 } 2152 2153 void ip_vs_update_conntrack(struct sk_buff *skb, struct ip_vs_conn *cp, 2154 int outin); 2155 int ip_vs_confirm_conntrack(struct sk_buff *skb); 2156 void ip_vs_nfct_expect_related(struct sk_buff *skb, struct nf_conn *ct, 2157 struct ip_vs_conn *cp, u_int8_t proto, 2158 const __be16 port, int from_rs); 2159 void ip_vs_conn_drop_conntrack(struct ip_vs_conn *cp); 2160 2161 #else 2162 2163 static inline int ip_vs_conntrack_enabled(struct netns_ipvs *ipvs) 2164 { 2165 return 0; 2166 } 2167 2168 static inline void ip_vs_update_conntrack(struct sk_buff *skb, 2169 struct ip_vs_conn *cp, int outin) 2170 { 2171 } 2172 2173 static inline int ip_vs_confirm_conntrack(struct sk_buff *skb) 2174 { 2175 return NF_ACCEPT; 2176 } 2177 2178 static inline void ip_vs_conn_drop_conntrack(struct ip_vs_conn *cp) 2179 { 2180 } 2181 #endif /* CONFIG_IP_VS_NFCT */ 2182 2183 /* Using old conntrack that can not be redirected to another real server? */ 2184 static inline bool ip_vs_conn_uses_old_conntrack(struct ip_vs_conn *cp, 2185 struct sk_buff *skb) 2186 { 2187 #ifdef CONFIG_IP_VS_NFCT 2188 enum ip_conntrack_info ctinfo; 2189 struct nf_conn *ct; 2190 2191 ct = nf_ct_get(skb, &ctinfo); 2192 if (ct && nf_ct_is_confirmed(ct)) 2193 return true; 2194 #endif 2195 return false; 2196 } 2197 2198 static inline int ip_vs_register_conntrack(struct ip_vs_service *svc) 2199 { 2200 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 2201 int afmask = (svc->af == AF_INET6) ? 2 : 1; 2202 int ret = 0; 2203 2204 if (!(svc->conntrack_afmask & afmask)) { 2205 ret = nf_ct_netns_get(svc->ipvs->net, svc->af); 2206 if (ret >= 0) 2207 svc->conntrack_afmask |= afmask; 2208 } 2209 return ret; 2210 #else 2211 return 0; 2212 #endif 2213 } 2214 2215 static inline void ip_vs_unregister_conntrack(struct ip_vs_service *svc) 2216 { 2217 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 2218 int afmask = (svc->af == AF_INET6) ? 2 : 1; 2219 2220 if (svc->conntrack_afmask & afmask) { 2221 nf_ct_netns_put(svc->ipvs->net, svc->af); 2222 svc->conntrack_afmask &= ~afmask; 2223 } 2224 #endif 2225 } 2226 2227 int ip_vs_register_hooks(struct netns_ipvs *ipvs, unsigned int af); 2228 void ip_vs_unregister_hooks(struct netns_ipvs *ipvs, unsigned int af); 2229 2230 static inline int 2231 ip_vs_dest_conn_overhead(struct ip_vs_dest *dest) 2232 { 2233 /* We think the overhead of processing active connections is 257 2234 * times higher than that of inactive connections in average. (This 2235 * 257 times might not be accurate, we will change it later) We 2236 * use the following formula to estimate the overhead now: 2237 * dest->activeconns*256 + dest->totalconns 2238 */ 2239 return (atomic_read(&dest->activeconns) << 8) + 2240 atomic_read(&dest->totalconns); 2241 } 2242 2243 static inline int 2244 ip_vs_dest_inactconns(const struct ip_vs_dest *dest) 2245 { 2246 return max(atomic_read(&dest->totalconns) - 2247 atomic_read(&dest->activeconns), 0); 2248 } 2249 2250 #ifdef CONFIG_IP_VS_PROTO_TCP 2251 INDIRECT_CALLABLE_DECLARE(int 2252 tcp_snat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, 2253 struct ip_vs_conn *cp, struct ip_vs_iphdr *iph)); 2254 #endif 2255 2256 #ifdef CONFIG_IP_VS_PROTO_UDP 2257 INDIRECT_CALLABLE_DECLARE(int 2258 udp_snat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, 2259 struct ip_vs_conn *cp, struct ip_vs_iphdr *iph)); 2260 #endif 2261 #endif /* _NET_IP_VS_H */ 2262