1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* IP Virtual Server 3 * data structure and functionality definitions 4 */ 5 6 #ifndef _NET_IP_VS_H 7 #define _NET_IP_VS_H 8 9 #include <linux/ip_vs.h> /* definitions shared with userland */ 10 11 #include <asm/types.h> /* for __uXX types */ 12 13 #include <linux/list.h> /* for struct list_head */ 14 #include <linux/rculist_bl.h> /* for struct hlist_bl_head */ 15 #include <linux/spinlock.h> /* for struct rwlock_t */ 16 #include <linux/atomic.h> /* for struct atomic_t */ 17 #include <linux/refcount.h> /* for struct refcount_t */ 18 #include <linux/workqueue.h> 19 20 #include <linux/compiler.h> 21 #include <linux/timer.h> 22 #include <linux/bug.h> 23 24 #include <net/checksum.h> 25 #include <linux/netfilter.h> /* for union nf_inet_addr */ 26 #include <linux/ip.h> 27 #include <linux/ipv6.h> /* for struct ipv6hdr */ 28 #include <net/route.h> 29 #include <net/ipv6.h> 30 #include <net/ip6_fib.h> 31 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 32 #include <net/netfilter/nf_conntrack.h> 33 #endif 34 #include <net/net_namespace.h> /* Netw namespace */ 35 #include <linux/sched/isolation.h> 36 #include <linux/siphash.h> 37 38 #define IP_VS_HDR_INVERSE 1 39 #define IP_VS_HDR_ICMP 2 40 41 /* conn_tab limits (as per Kconfig) */ 42 #define IP_VS_CONN_TAB_MIN_BITS 8 43 #if BITS_PER_LONG > 32 44 #define IP_VS_CONN_TAB_MAX_BITS 27 45 #else 46 #define IP_VS_CONN_TAB_MAX_BITS 20 47 #endif 48 49 /* conn_max limits */ 50 #if BITS_PER_LONG > 32 51 /* Limit of atomic_t but restricted by roundup_pow_of_two() in ip_vs_core.c */ 52 #define IP_VS_CONN_MAX (1 << 30) 53 #else 54 #define IP_VS_CONN_MAX (1 << 24) 55 #endif 56 57 /* svc_table limits */ 58 #define IP_VS_SVC_TAB_MIN_BITS 4 59 #define IP_VS_SVC_TAB_MAX_BITS 20 60 61 /* Generic access of ipvs struct */ 62 static inline struct netns_ipvs *net_ipvs(struct net* net) 63 { 64 return net->ipvs; 65 } 66 67 /* Connections' size value needed by ip_vs_ctl.c */ 68 extern int ip_vs_conn_tab_size; 69 70 struct ip_vs_iphdr { 71 int hdr_flags; /* ipvs flags */ 72 __u32 off; /* Where IP or IPv4 header starts */ 73 __u32 len; /* IPv4 simply where L4 starts 74 * IPv6 where L4 Transport Header starts */ 75 __u16 fragoffs; /* IPv6 fragment offset, 0 if first frag (or not frag)*/ 76 __s16 protocol; 77 __s32 flags; 78 union nf_inet_addr saddr; 79 union nf_inet_addr daddr; 80 }; 81 82 static inline void *frag_safe_skb_hp(const struct sk_buff *skb, int offset, 83 int len, void *buffer) 84 { 85 return skb_header_pointer(skb, offset, len, buffer); 86 } 87 88 /* This function handles filling *ip_vs_iphdr, both for IPv4 and IPv6. 89 * IPv6 requires some extra work, as finding proper header position, 90 * depend on the IPv6 extension headers. 91 */ 92 static inline int 93 ip_vs_fill_iph_skb_off(int af, const struct sk_buff *skb, int offset, 94 int hdr_flags, struct ip_vs_iphdr *iphdr) 95 { 96 iphdr->hdr_flags = hdr_flags; 97 iphdr->off = offset; 98 99 #ifdef CONFIG_IP_VS_IPV6 100 if (af == AF_INET6) { 101 struct ipv6hdr _iph; 102 const struct ipv6hdr *iph = skb_header_pointer( 103 skb, offset, sizeof(_iph), &_iph); 104 if (!iph) 105 return 0; 106 107 iphdr->saddr.in6 = iph->saddr; 108 iphdr->daddr.in6 = iph->daddr; 109 /* ipv6_find_hdr() updates len, flags */ 110 iphdr->len = offset; 111 iphdr->flags = 0; 112 iphdr->protocol = ipv6_find_hdr(skb, &iphdr->len, -1, 113 &iphdr->fragoffs, 114 &iphdr->flags); 115 if (iphdr->protocol < 0) 116 return 0; 117 } else 118 #endif 119 { 120 struct iphdr _iph; 121 const struct iphdr *iph = skb_header_pointer( 122 skb, offset, sizeof(_iph), &_iph); 123 if (!iph) 124 return 0; 125 126 iphdr->len = offset + iph->ihl * 4; 127 iphdr->fragoffs = 0; 128 iphdr->protocol = iph->protocol; 129 iphdr->saddr.ip = iph->saddr; 130 iphdr->daddr.ip = iph->daddr; 131 } 132 133 return 1; 134 } 135 136 static inline int 137 ip_vs_fill_iph_skb_icmp(int af, const struct sk_buff *skb, int offset, 138 bool inverse, struct ip_vs_iphdr *iphdr) 139 { 140 int hdr_flags = IP_VS_HDR_ICMP; 141 142 if (inverse) 143 hdr_flags |= IP_VS_HDR_INVERSE; 144 145 return ip_vs_fill_iph_skb_off(af, skb, offset, hdr_flags, iphdr); 146 } 147 148 static inline int 149 ip_vs_fill_iph_skb(int af, const struct sk_buff *skb, bool inverse, 150 struct ip_vs_iphdr *iphdr) 151 { 152 int hdr_flags = 0; 153 154 if (inverse) 155 hdr_flags |= IP_VS_HDR_INVERSE; 156 157 return ip_vs_fill_iph_skb_off(af, skb, skb_network_offset(skb), 158 hdr_flags, iphdr); 159 } 160 161 static inline bool 162 ip_vs_iph_inverse(const struct ip_vs_iphdr *iph) 163 { 164 return !!(iph->hdr_flags & IP_VS_HDR_INVERSE); 165 } 166 167 static inline bool 168 ip_vs_iph_icmp(const struct ip_vs_iphdr *iph) 169 { 170 return !!(iph->hdr_flags & IP_VS_HDR_ICMP); 171 } 172 173 static inline void ip_vs_addr_copy(int af, union nf_inet_addr *dst, 174 const union nf_inet_addr *src) 175 { 176 #ifdef CONFIG_IP_VS_IPV6 177 if (af == AF_INET6) 178 dst->in6 = src->in6; 179 else 180 #endif 181 dst->ip = src->ip; 182 } 183 184 static inline void ip_vs_addr_set(int af, union nf_inet_addr *dst, 185 const union nf_inet_addr *src) 186 { 187 #ifdef CONFIG_IP_VS_IPV6 188 if (af == AF_INET6) { 189 dst->in6 = src->in6; 190 return; 191 } 192 #endif 193 dst->ip = src->ip; 194 dst->all[1] = 0; 195 dst->all[2] = 0; 196 dst->all[3] = 0; 197 } 198 199 static inline int ip_vs_addr_equal(int af, const union nf_inet_addr *a, 200 const union nf_inet_addr *b) 201 { 202 #ifdef CONFIG_IP_VS_IPV6 203 if (af == AF_INET6) 204 return ipv6_addr_equal(&a->in6, &b->in6); 205 #endif 206 return a->ip == b->ip; 207 } 208 209 #ifdef CONFIG_IP_VS_DEBUG 210 #include <linux/net.h> 211 212 int ip_vs_get_debug_level(void); 213 214 static inline const char *ip_vs_dbg_addr(int af, char *buf, size_t buf_len, 215 const union nf_inet_addr *addr, 216 int *idx) 217 { 218 int len; 219 #ifdef CONFIG_IP_VS_IPV6 220 if (af == AF_INET6) 221 len = snprintf(&buf[*idx], buf_len - *idx, "[%pI6c]", 222 &addr->in6) + 1; 223 else 224 #endif 225 len = snprintf(&buf[*idx], buf_len - *idx, "%pI4", 226 &addr->ip) + 1; 227 228 *idx += len; 229 BUG_ON(*idx > buf_len + 1); 230 return &buf[*idx - len]; 231 } 232 233 #define IP_VS_DBG_BUF(level, msg, ...) \ 234 do { \ 235 char ip_vs_dbg_buf[160]; \ 236 int ip_vs_dbg_idx = 0; \ 237 if (level <= ip_vs_get_debug_level()) \ 238 printk(KERN_DEBUG pr_fmt(msg), ##__VA_ARGS__); \ 239 } while (0) 240 #define IP_VS_ERR_BUF(msg...) \ 241 do { \ 242 char ip_vs_dbg_buf[160]; \ 243 int ip_vs_dbg_idx = 0; \ 244 pr_err(msg); \ 245 } while (0) 246 247 /* Only use from within IP_VS_DBG_BUF() or IP_VS_ERR_BUF macros */ 248 #define IP_VS_DBG_ADDR(af, addr) \ 249 ip_vs_dbg_addr(af, ip_vs_dbg_buf, \ 250 sizeof(ip_vs_dbg_buf), addr, \ 251 &ip_vs_dbg_idx) 252 253 #define IP_VS_DBG(level, msg, ...) \ 254 do { \ 255 if (level <= ip_vs_get_debug_level()) \ 256 printk(KERN_DEBUG pr_fmt(msg), ##__VA_ARGS__); \ 257 } while (0) 258 #define IP_VS_DBG_RL(msg, ...) \ 259 do { \ 260 if (net_ratelimit()) \ 261 printk(KERN_DEBUG pr_fmt(msg), ##__VA_ARGS__); \ 262 } while (0) 263 #define IP_VS_DBG_PKT(level, af, pp, skb, ofs, msg) \ 264 do { \ 265 if (level <= ip_vs_get_debug_level()) \ 266 pp->debug_packet(af, pp, skb, ofs, msg); \ 267 } while (0) 268 #define IP_VS_DBG_RL_PKT(level, af, pp, skb, ofs, msg) \ 269 do { \ 270 if (level <= ip_vs_get_debug_level() && \ 271 net_ratelimit()) \ 272 pp->debug_packet(af, pp, skb, ofs, msg); \ 273 } while (0) 274 #else /* NO DEBUGGING at ALL */ 275 #define IP_VS_DBG_BUF(level, msg...) do {} while (0) 276 #define IP_VS_ERR_BUF(msg...) do {} while (0) 277 #define IP_VS_DBG(level, msg...) do {} while (0) 278 #define IP_VS_DBG_RL(msg...) do {} while (0) 279 #define IP_VS_DBG_PKT(level, af, pp, skb, ofs, msg) do {} while (0) 280 #define IP_VS_DBG_RL_PKT(level, af, pp, skb, ofs, msg) do {} while (0) 281 #endif 282 283 #define IP_VS_BUG() BUG() 284 #define IP_VS_ERR_RL(msg, ...) \ 285 do { \ 286 if (net_ratelimit()) \ 287 pr_err(msg, ##__VA_ARGS__); \ 288 } while (0) 289 290 struct ip_vs_aligned_lock { 291 spinlock_t l; /* Protect buckets */ 292 } ____cacheline_aligned_in_smp; 293 294 /* For arrays per family */ 295 enum { 296 IP_VS_AF_INET, 297 IP_VS_AF_INET6, 298 IP_VS_AF_MAX 299 }; 300 301 static inline int ip_vs_af_index(int af) 302 { 303 return af == AF_INET6 ? IP_VS_AF_INET6 : IP_VS_AF_INET; 304 } 305 306 /* work_flags */ 307 enum { 308 IP_VS_WORK_SVC_RESIZE, /* Schedule svc_resize_work */ 309 IP_VS_WORK_SVC_NORESIZE, /* Stopping svc_resize_work */ 310 IP_VS_WORK_CONN_RESIZE, /* Schedule conn_resize_work */ 311 }; 312 313 /* The port number of FTP service (in network order). */ 314 #define FTPPORT cpu_to_be16(21) 315 #define FTPDATA cpu_to_be16(20) 316 317 /* TCP State Values */ 318 enum { 319 IP_VS_TCP_S_NONE = 0, 320 IP_VS_TCP_S_ESTABLISHED, 321 IP_VS_TCP_S_SYN_SENT, 322 IP_VS_TCP_S_SYN_RECV, 323 IP_VS_TCP_S_FIN_WAIT, 324 IP_VS_TCP_S_TIME_WAIT, 325 IP_VS_TCP_S_CLOSE, 326 IP_VS_TCP_S_CLOSE_WAIT, 327 IP_VS_TCP_S_LAST_ACK, 328 IP_VS_TCP_S_LISTEN, 329 IP_VS_TCP_S_SYNACK, 330 IP_VS_TCP_S_LAST 331 }; 332 333 /* UDP State Values */ 334 enum { 335 IP_VS_UDP_S_NORMAL, 336 IP_VS_UDP_S_LAST, 337 }; 338 339 /* ICMP State Values */ 340 enum { 341 IP_VS_ICMP_S_NORMAL, 342 IP_VS_ICMP_S_LAST, 343 }; 344 345 /* SCTP State Values */ 346 enum ip_vs_sctp_states { 347 IP_VS_SCTP_S_NONE, 348 IP_VS_SCTP_S_INIT1, 349 IP_VS_SCTP_S_INIT, 350 IP_VS_SCTP_S_COOKIE_SENT, 351 IP_VS_SCTP_S_COOKIE_REPLIED, 352 IP_VS_SCTP_S_COOKIE_WAIT, 353 IP_VS_SCTP_S_COOKIE, 354 IP_VS_SCTP_S_COOKIE_ECHOED, 355 IP_VS_SCTP_S_ESTABLISHED, 356 IP_VS_SCTP_S_SHUTDOWN_SENT, 357 IP_VS_SCTP_S_SHUTDOWN_RECEIVED, 358 IP_VS_SCTP_S_SHUTDOWN_ACK_SENT, 359 IP_VS_SCTP_S_REJECTED, 360 IP_VS_SCTP_S_CLOSED, 361 IP_VS_SCTP_S_LAST 362 }; 363 364 /* Connection templates use bits from state */ 365 #define IP_VS_CTPL_S_NONE 0x0000 366 #define IP_VS_CTPL_S_ASSURED 0x0001 367 #define IP_VS_CTPL_S_LAST 0x0002 368 369 /* Delta sequence info structure 370 * Each ip_vs_conn has 2 (output AND input seq. changes). 371 * Only used in the VS/NAT. 372 */ 373 struct ip_vs_seq { 374 __u32 init_seq; /* Add delta from this seq */ 375 __u32 delta; /* Delta in sequence numbers */ 376 __u32 previous_delta; /* Delta in sequence numbers 377 * before last resized pkt */ 378 }; 379 380 /* counters per cpu */ 381 struct ip_vs_counters { 382 u64_stats_t conns; /* connections scheduled */ 383 u64_stats_t inpkts; /* incoming packets */ 384 u64_stats_t outpkts; /* outgoing packets */ 385 u64_stats_t inbytes; /* incoming bytes */ 386 u64_stats_t outbytes; /* outgoing bytes */ 387 }; 388 /* Stats per cpu */ 389 struct ip_vs_cpu_stats { 390 struct ip_vs_counters cnt; 391 struct u64_stats_sync syncp; 392 }; 393 394 /* Default nice for estimator kthreads */ 395 #define IPVS_EST_NICE 0 396 397 /* IPVS statistics objects */ 398 struct ip_vs_estimator { 399 struct hlist_node list; 400 401 u64 last_inbytes; 402 u64 last_outbytes; 403 u64 last_conns; 404 u64 last_inpkts; 405 u64 last_outpkts; 406 407 u64 cps; 408 u64 inpps; 409 u64 outpps; 410 u64 inbps; 411 u64 outbps; 412 413 s32 ktid:16, /* kthread ID, -1=temp list */ 414 ktrow:8, /* row/tick ID for kthread */ 415 ktcid:8; /* chain ID for kthread tick */ 416 }; 417 418 /* 419 * IPVS statistics object, 64-bit kernel version of struct ip_vs_stats_user 420 */ 421 struct ip_vs_kstats { 422 u64 conns; /* connections scheduled */ 423 u64 inpkts; /* incoming packets */ 424 u64 outpkts; /* outgoing packets */ 425 u64 inbytes; /* incoming bytes */ 426 u64 outbytes; /* outgoing bytes */ 427 428 u64 cps; /* current connection rate */ 429 u64 inpps; /* current in packet rate */ 430 u64 outpps; /* current out packet rate */ 431 u64 inbps; /* current in byte rate */ 432 u64 outbps; /* current out byte rate */ 433 }; 434 435 struct ip_vs_stats { 436 struct ip_vs_kstats kstats; /* kernel statistics */ 437 struct ip_vs_estimator est; /* estimator */ 438 struct ip_vs_cpu_stats __percpu *cpustats; /* per cpu counters */ 439 spinlock_t lock; /* spin lock */ 440 struct ip_vs_kstats kstats0; /* reset values */ 441 }; 442 443 struct ip_vs_stats_rcu { 444 struct ip_vs_stats s; 445 struct rcu_head rcu_head; 446 }; 447 448 int ip_vs_stats_init_alloc(struct ip_vs_stats *s); 449 struct ip_vs_stats *ip_vs_stats_alloc(void); 450 void ip_vs_stats_release(struct ip_vs_stats *stats); 451 void ip_vs_stats_free(struct ip_vs_stats *stats); 452 453 /* Process estimators in multiple timer ticks (20/50/100, see ktrow) */ 454 #define IPVS_EST_NTICKS 50 455 /* Estimation uses a 2-second period containing ticks (in jiffies) */ 456 #define IPVS_EST_TICK ((2 * HZ) / IPVS_EST_NTICKS) 457 458 /* Limit of CPU load per kthread (8 for 12.5%), ratio of CPU capacity (1/C). 459 * Value of 4 and above ensures kthreads will take work without exceeding 460 * the CPU capacity under different circumstances. 461 */ 462 #define IPVS_EST_LOAD_DIVISOR 8 463 464 /* Kthreads should not have work that exceeds the CPU load above 50% */ 465 #define IPVS_EST_CPU_KTHREADS (IPVS_EST_LOAD_DIVISOR / 2) 466 467 /* Desired number of chains per timer tick (chain load factor in 100us units), 468 * 48=4.8ms of 40ms tick (12% CPU usage): 469 * 2 sec * 1000 ms in sec * 10 (100us in ms) / 8 (12.5%) / 50 470 */ 471 #define IPVS_EST_CHAIN_FACTOR \ 472 ALIGN_DOWN(2 * 1000 * 10 / IPVS_EST_LOAD_DIVISOR / IPVS_EST_NTICKS, 8) 473 474 /* Compiled number of chains per tick 475 * The defines should match cond_resched_rcu 476 */ 477 #if defined(CONFIG_DEBUG_ATOMIC_SLEEP) || !defined(CONFIG_PREEMPT_RCU) 478 #define IPVS_EST_TICK_CHAINS IPVS_EST_CHAIN_FACTOR 479 #else 480 #define IPVS_EST_TICK_CHAINS 1 481 #endif 482 483 #if IPVS_EST_NTICKS > 127 484 #error Too many timer ticks for ktrow 485 #endif 486 487 /* Multiple chains processed in same tick */ 488 struct ip_vs_est_tick_data { 489 struct rcu_head rcu_head; 490 struct hlist_head chains[IPVS_EST_TICK_CHAINS]; 491 DECLARE_BITMAP(present, IPVS_EST_TICK_CHAINS); 492 DECLARE_BITMAP(full, IPVS_EST_TICK_CHAINS); 493 int chain_len[IPVS_EST_TICK_CHAINS]; 494 }; 495 496 /* Context for estimation kthread */ 497 struct ip_vs_est_kt_data { 498 struct netns_ipvs *ipvs; 499 struct task_struct *task; /* task if running */ 500 struct ip_vs_est_tick_data __rcu *ticks[IPVS_EST_NTICKS]; 501 DECLARE_BITMAP(avail, IPVS_EST_NTICKS); /* tick has space for ests */ 502 unsigned long est_timer; /* estimation timer (jiffies) */ 503 struct ip_vs_stats *calc_stats; /* Used for calculation */ 504 int needed; /* task is needed */ 505 int tick_len[IPVS_EST_NTICKS]; /* est count */ 506 int id; /* ktid per netns */ 507 int chain_max; /* max ests per tick chain */ 508 int tick_max; /* max ests per tick */ 509 int est_count; /* attached ests to kthread */ 510 int est_max_count; /* max ests per kthread */ 511 int add_row; /* row for new ests */ 512 int est_row; /* estimated row */ 513 }; 514 515 /* IPVS resizable hash tables */ 516 struct ip_vs_rht { 517 struct hlist_bl_head *buckets; 518 struct ip_vs_rht __rcu *new_tbl; /* New/Same table */ 519 seqcount_t *seqc; /* Protects moves */ 520 struct ip_vs_aligned_lock *lock; /* Protect seqc */ 521 int mask; /* Buckets mask */ 522 int size; /* Buckets */ 523 int seqc_mask; /* seqc mask */ 524 int lock_mask; /* lock mask */ 525 u32 table_id; 526 int u_thresh; /* upper threshold */ 527 int l_thresh; /* lower threshold */ 528 int lfactor; /* Load Factor (shift)*/ 529 int bits; /* size = 1 << bits */ 530 siphash_key_t hash_key; 531 struct rcu_head rcu_head; 532 }; 533 534 /** 535 * ip_vs_rht_for_each_table() - Walk the hash tables 536 * @table: struct ip_vs_rht __rcu *table 537 * @t: current table, used as cursor, struct ip_vs_rht *var 538 * @p: previous table, temp struct ip_vs_rht *var 539 * 540 * Walk tables assuming others can not change the installed tables 541 */ 542 #define ip_vs_rht_for_each_table(table, t, p) \ 543 for (p = NULL, t = rcu_dereference_protected(table, 1); \ 544 t != p; \ 545 p = t, t = rcu_dereference_protected(t->new_tbl, 1)) 546 547 /** 548 * ip_vs_rht_for_each_table_rcu() - Walk the hash tables under RCU reader lock 549 * @table: struct ip_vs_rht __rcu *table 550 * @t: current table, used as cursor, struct ip_vs_rht *var 551 * @p: previous table, temp struct ip_vs_rht *var 552 * 553 * We usually search in one table and also in second table on resizing 554 */ 555 #define ip_vs_rht_for_each_table_rcu(table, t, p) \ 556 for (p = NULL, t = rcu_dereference(table); \ 557 t != p; \ 558 p = t, t = rcu_dereference(t->new_tbl)) 559 560 /** 561 * ip_vs_rht_for_each_bucket() - Walk all table buckets 562 * @t: current table, used as cursor, struct ip_vs_rht *var 563 * @bucket: bucket index, used as cursor, u32 var 564 * @head: bucket address, used as cursor, struct hlist_bl_head *var 565 */ 566 #define ip_vs_rht_for_each_bucket(t, bucket, head) \ 567 for (bucket = 0, head = (t)->buckets; \ 568 bucket < t->size; bucket++, head++) 569 570 /** 571 * ip_vs_rht_for_bucket_retry() - Retry bucket if entries are moved 572 * @t: current table, used as cursor, struct ip_vs_rht *var 573 * @bucket: index of current bucket or hash key 574 * @sc: temp seqcount_t *var 575 * @seq: temp unsigned int var for sequence count 576 * @retry: temp int var 577 */ 578 #define ip_vs_rht_for_bucket_retry(t, bucket, sc, seq, retry) \ 579 for (retry = 1, sc = &(t)->seqc[(bucket) & (t)->seqc_mask]; \ 580 retry && ({ seq = read_seqcount_begin(sc); 1; }); \ 581 retry = read_seqcount_retry(sc, seq)) 582 583 /** 584 * DECLARE_IP_VS_RHT_WALK_BUCKETS_RCU() - Declare variables 585 * 586 * Variables for ip_vs_rht_walk_buckets_rcu 587 */ 588 #define DECLARE_IP_VS_RHT_WALK_BUCKETS_RCU() \ 589 struct ip_vs_rht *_t, *_p; \ 590 unsigned int _seq; \ 591 seqcount_t *_sc; \ 592 u32 _bucket; \ 593 int _retry 594 /** 595 * ip_vs_rht_walk_buckets_rcu() - Walk all buckets under RCU read lock 596 * @table: struct ip_vs_rht __rcu *table 597 * @head: bucket address, used as cursor, struct hlist_bl_head *var 598 * 599 * Can be used while others add/delete/move entries 600 * Not suitable if duplicates are not desired 601 * Possible cases for reader that uses cond_resched_rcu() in the loop: 602 * - new table can not be installed, no need to repeat 603 * - new table can be installed => check and repeat if new table is 604 * installed, needed for !PREEMPT_RCU 605 */ 606 #define ip_vs_rht_walk_buckets_rcu(table, head) \ 607 ip_vs_rht_for_each_table_rcu(table, _t, _p) \ 608 ip_vs_rht_for_each_bucket(_t, _bucket, head) \ 609 ip_vs_rht_for_bucket_retry(_t, _bucket, _sc, \ 610 _seq, _retry) 611 612 /** 613 * DECLARE_IP_VS_RHT_WALK_BUCKET_RCU() - Declare variables 614 * 615 * Variables for ip_vs_rht_walk_bucket_rcu 616 */ 617 #define DECLARE_IP_VS_RHT_WALK_BUCKET_RCU() \ 618 unsigned int _seq; \ 619 seqcount_t *_sc; \ 620 int _retry 621 /** 622 * ip_vs_rht_walk_bucket_rcu() - Walk bucket under RCU read lock 623 * @t: current table, struct ip_vs_rht *var 624 * @bucket: index of current bucket or hash key 625 * @head: bucket address, used as cursor, struct hlist_bl_head *var 626 * 627 * Can be used while others add/delete/move entries 628 * Not suitable if duplicates are not desired 629 * Possible cases for reader that uses cond_resched_rcu() in the loop: 630 * - new table can not be installed, no need to repeat 631 * - new table can be installed => check and repeat if new table is 632 * installed, needed for !PREEMPT_RCU 633 */ 634 #define ip_vs_rht_walk_bucket_rcu(t, bucket, head) \ 635 if (({ head = (t)->buckets + ((bucket) & (t)->mask); 0; })) \ 636 {} \ 637 else \ 638 ip_vs_rht_for_bucket_retry(t, (bucket), _sc, _seq, _retry) 639 640 /** 641 * DECLARE_IP_VS_RHT_WALK_BUCKETS_SAFE_RCU() - Declare variables 642 * 643 * Variables for ip_vs_rht_walk_buckets_safe_rcu 644 */ 645 #define DECLARE_IP_VS_RHT_WALK_BUCKETS_SAFE_RCU() \ 646 struct ip_vs_rht *_t, *_p; \ 647 u32 _bucket 648 /** 649 * ip_vs_rht_walk_buckets_safe_rcu() - Walk all buckets under RCU read lock 650 * @table: struct ip_vs_rht __rcu *table 651 * @head: bucket address, used as cursor, struct hlist_bl_head *var 652 * 653 * Can be used while others add/delete entries but moving is disabled 654 * Using cond_resched_rcu() should be safe if tables do not change 655 */ 656 #define ip_vs_rht_walk_buckets_safe_rcu(table, head) \ 657 ip_vs_rht_for_each_table_rcu(table, _t, _p) \ 658 ip_vs_rht_for_each_bucket(_t, _bucket, head) 659 660 /** 661 * DECLARE_IP_VS_RHT_WALK_BUCKETS() - Declare variables 662 * 663 * Variables for ip_vs_rht_walk_buckets 664 */ 665 #define DECLARE_IP_VS_RHT_WALK_BUCKETS() \ 666 struct ip_vs_rht *_t, *_p; \ 667 u32 _bucket 668 669 /** 670 * ip_vs_rht_walk_buckets() - Walk all buckets 671 * @table: struct ip_vs_rht __rcu *table 672 * @head: bucket address, used as cursor, struct hlist_bl_head *var 673 * 674 * Use if others can not add/delete/move entries 675 */ 676 #define ip_vs_rht_walk_buckets(table, head) \ 677 ip_vs_rht_for_each_table(table, _t, _p) \ 678 ip_vs_rht_for_each_bucket(_t, _bucket, head) 679 680 /* Entries can be in one of two tables, so we flip bit when new table is 681 * created and store it as highest bit in hash keys 682 */ 683 #define IP_VS_RHT_TABLE_ID_MASK BIT(31) 684 685 /* Check if hash key is from this table */ 686 static inline bool ip_vs_rht_same_table(struct ip_vs_rht *t, u32 hash_key) 687 { 688 return !((t->table_id ^ hash_key) & IP_VS_RHT_TABLE_ID_MASK); 689 } 690 691 /* Build per-table hash key from hash value */ 692 static inline u32 ip_vs_rht_build_hash_key(struct ip_vs_rht *t, u32 hash) 693 { 694 return t->table_id | (hash & ~IP_VS_RHT_TABLE_ID_MASK); 695 } 696 697 void ip_vs_rht_free(struct ip_vs_rht *t); 698 void ip_vs_rht_rcu_free(struct rcu_head *head); 699 struct ip_vs_rht *ip_vs_rht_alloc(int buckets, int scounts, int locks); 700 int ip_vs_rht_desired_size(struct netns_ipvs *ipvs, struct ip_vs_rht *t, int n, 701 int lfactor, int min_bits, int max_bits); 702 void ip_vs_rht_set_thresholds(struct ip_vs_rht *t, int size, int lfactor, 703 int min_bits, int max_bits); 704 u32 ip_vs_rht_hash_linfo(struct ip_vs_rht *t, int af, 705 const union nf_inet_addr *addr, u32 v1, u32 v2); 706 707 struct dst_entry; 708 struct iphdr; 709 struct ip_vs_conn; 710 struct ip_vs_app; 711 struct sk_buff; 712 struct ip_vs_proto_data; 713 714 struct ip_vs_protocol { 715 struct ip_vs_protocol *next; 716 char *name; 717 u16 protocol; 718 u16 num_states; 719 int dont_defrag; 720 721 void (*init)(struct ip_vs_protocol *pp); 722 723 void (*exit)(struct ip_vs_protocol *pp); 724 725 int (*init_netns)(struct netns_ipvs *ipvs, struct ip_vs_proto_data *pd); 726 727 void (*exit_netns)(struct netns_ipvs *ipvs, struct ip_vs_proto_data *pd); 728 729 int (*conn_schedule)(struct netns_ipvs *ipvs, 730 int af, struct sk_buff *skb, 731 struct ip_vs_proto_data *pd, 732 int *verdict, struct ip_vs_conn **cpp, 733 struct ip_vs_iphdr *iph); 734 735 struct ip_vs_conn * 736 (*conn_in_get)(struct netns_ipvs *ipvs, 737 int af, 738 const struct sk_buff *skb, 739 const struct ip_vs_iphdr *iph); 740 741 struct ip_vs_conn * 742 (*conn_out_get)(struct netns_ipvs *ipvs, 743 int af, 744 const struct sk_buff *skb, 745 const struct ip_vs_iphdr *iph); 746 747 int (*snat_handler)(struct sk_buff *skb, struct ip_vs_protocol *pp, 748 struct ip_vs_conn *cp, struct ip_vs_iphdr *iph); 749 750 int (*dnat_handler)(struct sk_buff *skb, struct ip_vs_protocol *pp, 751 struct ip_vs_conn *cp, struct ip_vs_iphdr *iph); 752 753 const char *(*state_name)(int state); 754 755 void (*state_transition)(struct ip_vs_conn *cp, int direction, 756 const struct sk_buff *skb, 757 struct ip_vs_proto_data *pd, 758 unsigned int iph_len); 759 760 int (*register_app)(struct netns_ipvs *ipvs, struct ip_vs_app *inc); 761 762 void (*unregister_app)(struct netns_ipvs *ipvs, struct ip_vs_app *inc); 763 764 int (*app_conn_bind)(struct ip_vs_conn *cp); 765 766 void (*debug_packet)(int af, struct ip_vs_protocol *pp, 767 const struct sk_buff *skb, 768 int offset, 769 const char *msg); 770 771 void (*timeout_change)(struct ip_vs_proto_data *pd, int flags); 772 }; 773 774 /* protocol data per netns */ 775 struct ip_vs_proto_data { 776 struct ip_vs_proto_data *next; 777 struct ip_vs_protocol *pp; 778 int *timeout_table; /* protocol timeout table */ 779 atomic_t appcnt; /* counter of proto app incs. */ 780 struct tcp_states_t *tcp_state_table; 781 }; 782 783 struct ip_vs_protocol *ip_vs_proto_get(unsigned short proto); 784 struct ip_vs_proto_data *ip_vs_proto_data_get(struct netns_ipvs *ipvs, 785 unsigned short proto); 786 787 struct ip_vs_conn_param { 788 struct netns_ipvs *ipvs; 789 const union nf_inet_addr *caddr; 790 const union nf_inet_addr *vaddr; 791 __be16 cport; 792 __be16 vport; 793 __u16 protocol; 794 u16 af; 795 796 const struct ip_vs_pe *pe; 797 char *pe_data; 798 __u8 pe_data_len; 799 }; 800 801 /* Hash node in conn_tab */ 802 struct ip_vs_conn_hnode { 803 struct hlist_bl_node node; /* node in conn_tab */ 804 u32 hash_key; /* Key for the hash table */ 805 u8 dir; /* 0=out->in, 1=in->out */ 806 } __packed; 807 808 /* IP_VS structure allocated for each dynamically scheduled connection */ 809 struct ip_vs_conn { 810 /* Cacheline for hash table nodes - rarely modified */ 811 812 struct ip_vs_conn_hnode hn0; /* Original direction */ 813 u8 af; /* address family */ 814 __be16 cport; 815 struct ip_vs_conn_hnode hn1; /* Reply direction */ 816 u8 daf; /* Address family of the dest */ 817 __be16 dport; 818 struct ip_vs_dest *dest; /* real server */ 819 atomic_t n_control; /* Number of controlled ones */ 820 volatile __u32 flags; /* status flags */ 821 /* 44/64 */ 822 823 struct ip_vs_conn *control; /* Master control connection */ 824 const struct ip_vs_pe *pe; 825 char *pe_data; 826 __u8 pe_data_len; 827 volatile __u16 state; /* state info */ 828 volatile __u16 old_state; /* old state, to be used for 829 * state transition triggered 830 * synchronization 831 */ 832 /* 2-byte hole */ 833 /* 64/96 */ 834 835 union nf_inet_addr caddr; /* client address */ 836 union nf_inet_addr vaddr; /* virtual address */ 837 /* 96/128 */ 838 839 union nf_inet_addr daddr; /* destination address */ 840 __u32 fwmark; /* Fire wall mark from skb */ 841 __be16 vport; 842 __u16 protocol; /* Which protocol (TCP/UDP) */ 843 844 /* Note: we can group the following members into a structure, 845 * in order to save more space, and the following members are 846 * only used in VS/NAT anyway 847 */ 848 struct ip_vs_app *app; /* bound ip_vs_app object */ 849 void *app_data; /* Application private data */ 850 /* 128/168 */ 851 struct_group(sync_conn_opt, 852 struct ip_vs_seq in_seq; /* incoming seq. struct */ 853 struct ip_vs_seq out_seq; /* outgoing seq. struct */ 854 ); 855 /* 152/192 */ 856 857 struct timer_list timer; /* Expiration timer */ 858 volatile unsigned long timeout; /* timeout */ 859 spinlock_t lock; /* lock for state transition */ 860 refcount_t refcnt; /* reference count */ 861 atomic_t in_pkts; /* incoming packet counter */ 862 /* 64-bit: 4-byte gap */ 863 864 /* 188/256 */ 865 unsigned long sync_endtime; /* jiffies + sent_retries */ 866 struct netns_ipvs *ipvs; 867 868 /* Packet transmitter for different forwarding methods. If it 869 * mangles the packet, it must return NF_DROP or better NF_STOLEN, 870 * otherwise this must be changed to a sk_buff **. 871 * NF_ACCEPT can be returned when destination is local. 872 */ 873 int (*packet_xmit)(struct sk_buff *skb, struct ip_vs_conn *cp, 874 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 875 876 struct rcu_head rcu_head; 877 }; 878 879 /* Extended internal versions of struct ip_vs_service_user and ip_vs_dest_user 880 * for IPv6 support. 881 * 882 * We need these to conveniently pass around service and destination 883 * options, but unfortunately, we also need to keep the old definitions to 884 * maintain userspace backwards compatibility for the setsockopt interface. 885 */ 886 struct ip_vs_service_user_kern { 887 /* virtual service addresses */ 888 u16 af; 889 u16 protocol; 890 union nf_inet_addr addr; /* virtual ip address */ 891 __be16 port; 892 u32 fwmark; /* firewall mark of service */ 893 894 /* virtual service options */ 895 char *sched_name; 896 char *pe_name; 897 unsigned int flags; /* virtual service flags */ 898 unsigned int timeout; /* persistent timeout in sec */ 899 __be32 netmask; /* persistent netmask or plen */ 900 }; 901 902 903 struct ip_vs_dest_user_kern { 904 /* destination server address */ 905 union nf_inet_addr addr; 906 __be16 port; 907 908 /* real server options */ 909 unsigned int conn_flags; /* connection flags */ 910 int weight; /* destination weight */ 911 912 /* thresholds for active connections */ 913 u32 u_threshold; /* upper threshold */ 914 u32 l_threshold; /* lower threshold */ 915 916 /* Address family of addr */ 917 u16 af; 918 919 u16 tun_type; /* tunnel type */ 920 __be16 tun_port; /* tunnel port */ 921 u16 tun_flags; /* tunnel flags */ 922 }; 923 924 925 /* 926 * The information about the virtual service offered to the net and the 927 * forwarding entries. 928 */ 929 struct ip_vs_service { 930 struct hlist_bl_node s_list; /* node in service table */ 931 u32 hash_key; /* Key for the hash table */ 932 u16 af; /* address family */ 933 __u16 protocol; /* which protocol (TCP/UDP) */ 934 935 union nf_inet_addr addr; /* IP address for virtual service */ 936 __u32 fwmark; /* firewall mark of the service */ 937 atomic_t refcnt; /* reference counter */ 938 __be16 port; /* port number for the service */ 939 unsigned int flags; /* service status flags */ 940 unsigned int timeout; /* persistent timeout in ticks */ 941 __be32 netmask; /* grouping granularity, mask/plen */ 942 struct netns_ipvs *ipvs; 943 944 struct list_head destinations; /* real server d-linked list */ 945 __u32 num_dests; /* number of servers */ 946 struct ip_vs_stats stats; /* statistics for the service */ 947 948 /* for scheduling */ 949 struct ip_vs_scheduler __rcu *scheduler; /* bound scheduler object */ 950 spinlock_t sched_lock; /* lock sched_data */ 951 void *sched_data; /* scheduler application data */ 952 953 /* alternate persistence engine */ 954 struct ip_vs_pe __rcu *pe; 955 int conntrack_afmask; 956 957 struct rcu_head rcu_head; 958 }; 959 960 /* Information for cached dst */ 961 struct ip_vs_dest_dst { 962 struct dst_entry *dst_cache; /* destination cache entry */ 963 u32 dst_cookie; 964 union nf_inet_addr dst_saddr; 965 struct rcu_head rcu_head; 966 }; 967 968 /* The real server destination forwarding entry with ip address, port number, 969 * and so on. 970 */ 971 struct ip_vs_dest { 972 struct list_head n_list; /* for the dests in the service */ 973 struct hlist_node d_list; /* for table with all the dests */ 974 975 u16 af; /* address family */ 976 __be16 port; /* port number of the server */ 977 union nf_inet_addr addr; /* IP address of the server */ 978 volatile unsigned int flags; /* dest status flags */ 979 atomic_t conn_flags; /* flags to copy to conn */ 980 atomic_t weight; /* server weight */ 981 atomic_t last_weight; /* server latest weight */ 982 __u16 tun_type; /* tunnel type */ 983 __be16 tun_port; /* tunnel port */ 984 __u16 tun_flags; /* tunnel flags */ 985 986 refcount_t refcnt; /* reference counter */ 987 struct ip_vs_stats stats; /* statistics */ 988 unsigned long idle_start; /* start time, jiffies */ 989 990 /* connection counters and thresholds */ 991 atomic_t activeconns; /* active connections */ 992 atomic_t inactconns; /* inactive connections */ 993 atomic_t persistconns; /* persistent connections */ 994 __u32 u_threshold; /* upper threshold */ 995 __u32 l_threshold; /* lower threshold */ 996 997 /* for destination cache */ 998 spinlock_t dst_lock; /* lock of dst_cache */ 999 struct ip_vs_dest_dst __rcu *dest_dst; /* cached dst info */ 1000 1001 /* for virtual service */ 1002 struct ip_vs_service __rcu *svc; /* service it belongs to */ 1003 __u16 protocol; /* which protocol (TCP/UDP) */ 1004 __be16 vport; /* virtual port number */ 1005 union nf_inet_addr vaddr; /* virtual IP address */ 1006 __u32 vfwmark; /* firewall mark of service */ 1007 1008 struct rcu_head rcu_head; 1009 struct list_head t_list; /* in dest_trash */ 1010 unsigned int in_rs_table:1; /* we are in rs_table */ 1011 }; 1012 1013 /* The scheduler object */ 1014 struct ip_vs_scheduler { 1015 struct list_head n_list; /* d-linked list head */ 1016 char *name; /* scheduler name */ 1017 atomic_t refcnt; /* reference counter */ 1018 struct module *module; /* THIS_MODULE/NULL */ 1019 1020 /* scheduler initializing service */ 1021 int (*init_service)(struct ip_vs_service *svc); 1022 /* scheduling service finish */ 1023 void (*done_service)(struct ip_vs_service *svc); 1024 /* dest is linked */ 1025 int (*add_dest)(struct ip_vs_service *svc, struct ip_vs_dest *dest); 1026 /* dest is unlinked */ 1027 int (*del_dest)(struct ip_vs_service *svc, struct ip_vs_dest *dest); 1028 /* dest is updated */ 1029 int (*upd_dest)(struct ip_vs_service *svc, struct ip_vs_dest *dest); 1030 1031 /* selecting a server from the given service */ 1032 struct ip_vs_dest* (*schedule)(struct ip_vs_service *svc, 1033 const struct sk_buff *skb, 1034 struct ip_vs_iphdr *iph); 1035 }; 1036 1037 /* The persistence engine object */ 1038 struct ip_vs_pe { 1039 struct list_head n_list; /* d-linked list head */ 1040 char *name; /* scheduler name */ 1041 atomic_t refcnt; /* reference counter */ 1042 struct module *module; /* THIS_MODULE/NULL */ 1043 1044 /* get the connection template, if any */ 1045 int (*fill_param)(struct ip_vs_conn_param *p, struct sk_buff *skb); 1046 bool (*ct_match)(const struct ip_vs_conn_param *p, 1047 struct ip_vs_conn *ct); 1048 u32 (*hashkey_raw)(const struct ip_vs_conn_param *p, 1049 struct ip_vs_rht *t, bool inverse); 1050 int (*show_pe_data)(const struct ip_vs_conn *cp, char *buf); 1051 /* create connections for real-server outgoing packets */ 1052 struct ip_vs_conn* (*conn_out)(struct ip_vs_service *svc, 1053 struct ip_vs_dest *dest, 1054 struct sk_buff *skb, 1055 const struct ip_vs_iphdr *iph, 1056 __be16 dport, __be16 cport); 1057 }; 1058 1059 /* The application module object (a.k.a. app incarnation) */ 1060 struct ip_vs_app { 1061 struct list_head a_list; /* member in app list */ 1062 int type; /* IP_VS_APP_TYPE_xxx */ 1063 char *name; /* application module name */ 1064 __u16 protocol; 1065 struct module *module; /* THIS_MODULE/NULL */ 1066 struct list_head incs_list; /* list of incarnations */ 1067 1068 /* members for application incarnations */ 1069 struct list_head p_list; /* member in proto app list */ 1070 struct ip_vs_app *app; /* its real application */ 1071 __be16 port; /* port number in net order */ 1072 atomic_t usecnt; /* usage counter */ 1073 struct rcu_head rcu_head; 1074 1075 /* output hook: Process packet in inout direction, diff set for TCP. 1076 * Return: 0=Error, 1=Payload Not Mangled/Mangled but checksum is ok, 1077 * 2=Mangled but checksum was not updated 1078 */ 1079 int (*pkt_out)(struct ip_vs_app *, struct ip_vs_conn *, 1080 struct sk_buff *, int *diff, struct ip_vs_iphdr *ipvsh); 1081 1082 /* input hook: Process packet in outin direction, diff set for TCP. 1083 * Return: 0=Error, 1=Payload Not Mangled/Mangled but checksum is ok, 1084 * 2=Mangled but checksum was not updated 1085 */ 1086 int (*pkt_in)(struct ip_vs_app *, struct ip_vs_conn *, 1087 struct sk_buff *, int *diff, struct ip_vs_iphdr *ipvsh); 1088 1089 /* ip_vs_app initializer */ 1090 int (*init_conn)(struct ip_vs_app *, struct ip_vs_conn *); 1091 1092 /* ip_vs_app finish */ 1093 int (*done_conn)(struct ip_vs_app *, struct ip_vs_conn *); 1094 1095 1096 /* not used now */ 1097 int (*bind_conn)(struct ip_vs_app *, struct ip_vs_conn *, 1098 struct ip_vs_protocol *); 1099 1100 void (*unbind_conn)(struct ip_vs_app *, struct ip_vs_conn *); 1101 1102 int * timeout_table; 1103 int * timeouts; 1104 int timeouts_size; 1105 1106 int (*conn_schedule)(struct sk_buff *skb, struct ip_vs_app *app, 1107 int *verdict, struct ip_vs_conn **cpp); 1108 1109 struct ip_vs_conn * 1110 (*conn_in_get)(const struct sk_buff *skb, struct ip_vs_app *app, 1111 const struct iphdr *iph, int inverse); 1112 1113 struct ip_vs_conn * 1114 (*conn_out_get)(const struct sk_buff *skb, struct ip_vs_app *app, 1115 const struct iphdr *iph, int inverse); 1116 1117 int (*state_transition)(struct ip_vs_conn *cp, int direction, 1118 const struct sk_buff *skb, 1119 struct ip_vs_app *app); 1120 1121 void (*timeout_change)(struct ip_vs_app *app, int flags); 1122 }; 1123 1124 struct ipvs_master_sync_state { 1125 struct list_head sync_queue; 1126 struct ip_vs_sync_buff *sync_buff; 1127 unsigned long sync_queue_len; 1128 unsigned int sync_queue_delay; 1129 struct delayed_work master_wakeup_work; 1130 struct netns_ipvs *ipvs; 1131 }; 1132 1133 struct ip_vs_sync_thread_data; 1134 1135 /* How much time to keep dests in trash */ 1136 #define IP_VS_DEST_TRASH_PERIOD (120 * HZ) 1137 1138 struct ipvs_sync_daemon_cfg { 1139 union nf_inet_addr mcast_group; 1140 int syncid; 1141 u16 sync_maxlen; 1142 u16 mcast_port; 1143 u8 mcast_af; 1144 u8 mcast_ttl; 1145 /* multicast interface name */ 1146 char mcast_ifn[IP_VS_IFNAME_MAXLEN]; 1147 }; 1148 1149 /* IPVS in network namespace */ 1150 struct netns_ipvs { 1151 int gen; /* Generation */ 1152 int enable; /* enable like nf_hooks do */ 1153 /* Hash table: for real service lookups */ 1154 #define IP_VS_RTAB_BITS 4 1155 #define IP_VS_RTAB_SIZE (1 << IP_VS_RTAB_BITS) 1156 #define IP_VS_RTAB_MASK (IP_VS_RTAB_SIZE - 1) 1157 1158 struct hlist_head rs_table[IP_VS_RTAB_SIZE]; 1159 /* ip_vs_app */ 1160 struct list_head app_list; 1161 /* ip_vs_proto */ 1162 #define IP_VS_PROTO_TAB_SIZE 32 /* must be power of 2 */ 1163 struct ip_vs_proto_data *proto_data_table[IP_VS_PROTO_TAB_SIZE]; 1164 /* ip_vs_proto_tcp */ 1165 #ifdef CONFIG_IP_VS_PROTO_TCP 1166 #define TCP_APP_TAB_BITS 4 1167 #define TCP_APP_TAB_SIZE (1 << TCP_APP_TAB_BITS) 1168 #define TCP_APP_TAB_MASK (TCP_APP_TAB_SIZE - 1) 1169 struct list_head tcp_apps[TCP_APP_TAB_SIZE]; 1170 #endif 1171 /* ip_vs_proto_udp */ 1172 #ifdef CONFIG_IP_VS_PROTO_UDP 1173 #define UDP_APP_TAB_BITS 4 1174 #define UDP_APP_TAB_SIZE (1 << UDP_APP_TAB_BITS) 1175 #define UDP_APP_TAB_MASK (UDP_APP_TAB_SIZE - 1) 1176 struct list_head udp_apps[UDP_APP_TAB_SIZE]; 1177 #endif 1178 /* ip_vs_proto_sctp */ 1179 #ifdef CONFIG_IP_VS_PROTO_SCTP 1180 #define SCTP_APP_TAB_BITS 4 1181 #define SCTP_APP_TAB_SIZE (1 << SCTP_APP_TAB_BITS) 1182 #define SCTP_APP_TAB_MASK (SCTP_APP_TAB_SIZE - 1) 1183 /* Hash table for SCTP application incarnations */ 1184 struct list_head sctp_apps[SCTP_APP_TAB_SIZE]; 1185 #endif 1186 /* ip_vs_conn */ 1187 atomic_t conn_count; /* connection counter */ 1188 atomic_t no_cport_conns[IP_VS_AF_MAX]; 1189 struct delayed_work conn_resize_work;/* resize conn_tab */ 1190 1191 /* ip_vs_ctl */ 1192 struct ip_vs_stats_rcu *tot_stats; /* Statistics & est. */ 1193 1194 /* Trash for destinations */ 1195 struct list_head dest_trash; 1196 spinlock_t dest_trash_lock; 1197 struct timer_list dest_trash_timer; /* expiration timer */ 1198 struct mutex service_mutex; /* service reconfig */ 1199 struct rw_semaphore svc_resize_sem; /* svc_table resizing */ 1200 struct rw_semaphore svc_replace_sem; /* svc_table replace */ 1201 struct delayed_work svc_resize_work; /* resize svc_table */ 1202 atomic_t svc_table_changes;/* ++ on table changes */ 1203 /* Service counters */ 1204 atomic_t num_services[IP_VS_AF_MAX]; /* Services */ 1205 atomic_t fwm_services[IP_VS_AF_MAX]; /* Services */ 1206 atomic_t nonfwm_services[IP_VS_AF_MAX];/* Services */ 1207 atomic_t ftpsvc_counter[IP_VS_AF_MAX]; /* FTPPORT */ 1208 atomic_t nullsvc_counter[IP_VS_AF_MAX];/* Zero port */ 1209 atomic_t conn_out_counter[IP_VS_AF_MAX];/* out conn */ 1210 1211 #ifdef CONFIG_SYSCTL 1212 /* delayed work for expiring no dest connections */ 1213 struct delayed_work expire_nodest_conn_work; 1214 /* 1/rate drop and drop-entry variables */ 1215 struct delayed_work defense_work; /* Work handler */ 1216 int drop_rate; 1217 int drop_counter; 1218 int old_secure_tcp; 1219 atomic_t dropentry; 1220 s8 dropentry_counters[8]; 1221 /* locks in ctl.c */ 1222 spinlock_t dropentry_lock; /* drop entry handling */ 1223 spinlock_t droppacket_lock; /* drop packet handling */ 1224 spinlock_t securetcp_lock; /* state and timeout tables */ 1225 1226 /* sys-ctl struct */ 1227 struct ctl_table_header *sysctl_hdr; 1228 struct ctl_table *sysctl_tbl; 1229 #endif 1230 1231 /* sysctl variables */ 1232 int sysctl_amemthresh; 1233 int sysctl_am_droprate; 1234 #ifdef CONFIG_SYSCTL 1235 int sysctl_conn_max;/* soft limit for conns */ 1236 int conn_max_limit; /* hard limit for conn_max */ 1237 #endif 1238 int sysctl_drop_entry; 1239 int sysctl_drop_packet; 1240 int sysctl_secure_tcp; 1241 #ifdef CONFIG_IP_VS_NFCT 1242 int sysctl_conntrack; 1243 #endif 1244 int sysctl_snat_reroute; 1245 int sysctl_sync_ver; 1246 int sysctl_sync_ports; 1247 int sysctl_sync_persist_mode; 1248 unsigned long sysctl_sync_qlen_max; 1249 int sysctl_sync_sock_size; 1250 int sysctl_cache_bypass; 1251 int sysctl_expire_nodest_conn; 1252 int sysctl_sloppy_tcp; 1253 int sysctl_sloppy_sctp; 1254 int sysctl_expire_quiescent_template; 1255 int sysctl_sync_threshold[2]; 1256 unsigned int sysctl_sync_refresh_period; 1257 int sysctl_sync_retries; 1258 int sysctl_nat_icmp_send; 1259 int sysctl_pmtu_disc; 1260 int sysctl_backup_only; 1261 int sysctl_conn_reuse_mode; 1262 int sysctl_schedule_icmp; 1263 int sysctl_ignore_tunneled; 1264 int sysctl_run_estimation; 1265 #ifdef CONFIG_SYSCTL 1266 cpumask_var_t sysctl_est_cpulist; /* kthread cpumask */ 1267 int est_cpulist_valid; /* cpulist set */ 1268 int sysctl_est_nice; /* kthread nice */ 1269 int est_stopped; /* stop tasks */ 1270 #endif 1271 int sysctl_conn_lfactor; 1272 int sysctl_svc_lfactor; 1273 1274 /* ip_vs_lblc */ 1275 int sysctl_lblc_expiration; 1276 struct ctl_table_header *lblc_ctl_header; 1277 struct ctl_table *lblc_ctl_table; 1278 /* ip_vs_lblcr */ 1279 int sysctl_lblcr_expiration; 1280 struct ctl_table_header *lblcr_ctl_header; 1281 struct ctl_table *lblcr_ctl_table; 1282 unsigned long work_flags; /* IP_VS_WORK_* flags */ 1283 /* ip_vs_est */ 1284 struct delayed_work est_reload_work;/* Reload kthread tasks */ 1285 struct mutex est_mutex; /* protect kthread tasks */ 1286 struct hlist_head est_temp_list; /* Ests during calc phase */ 1287 struct ip_vs_est_kt_data **est_kt_arr; /* Array of kthread data ptrs */ 1288 unsigned long est_max_threads;/* Hard limit of kthreads */ 1289 int est_calc_phase; /* Calculation phase */ 1290 int est_chain_max; /* Calculated chain_max */ 1291 int est_kt_count; /* Allocated ptrs */ 1292 int est_add_ktid; /* ktid where to add ests */ 1293 atomic_t est_genid; /* kthreads reload genid */ 1294 atomic_t est_genid_done; /* applied genid */ 1295 /* ip_vs_sync */ 1296 spinlock_t sync_lock; 1297 struct ipvs_master_sync_state *ms; 1298 spinlock_t sync_buff_lock; 1299 struct ip_vs_sync_thread_data *master_tinfo; 1300 struct ip_vs_sync_thread_data *backup_tinfo; 1301 int threads_mask; 1302 volatile int sync_state; 1303 struct mutex sync_mutex; 1304 struct ipvs_sync_daemon_cfg mcfg; /* Master Configuration */ 1305 struct ipvs_sync_daemon_cfg bcfg; /* Backup Configuration */ 1306 /* net name space ptr */ 1307 struct net *net; /* Needed by timer routines */ 1308 /* Number of heterogeneous destinations, needed because heterogeneous 1309 * are not supported when synchronization is enabled. 1310 */ 1311 unsigned int mixed_address_family_dests; 1312 unsigned int hooks_afmask; /* &1=AF_INET, &2=AF_INET6 */ 1313 1314 struct ip_vs_rht __rcu *svc_table; /* Services */ 1315 struct ip_vs_rht __rcu *conn_tab; /* Connections */ 1316 atomic_t conn_tab_changes;/* ++ on new table */ 1317 }; 1318 1319 #define DEFAULT_SYNC_THRESHOLD 3 1320 #define DEFAULT_SYNC_PERIOD 50 1321 #define DEFAULT_SYNC_VER 1 1322 #define DEFAULT_SLOPPY_TCP 0 1323 #define DEFAULT_SLOPPY_SCTP 0 1324 #define DEFAULT_SYNC_REFRESH_PERIOD (0U * HZ) 1325 #define DEFAULT_SYNC_RETRIES 0 1326 #define IPVS_SYNC_WAKEUP_RATE 8 1327 #define IPVS_SYNC_QLEN_MAX (IPVS_SYNC_WAKEUP_RATE * 4) 1328 #define IPVS_SYNC_SEND_DELAY (HZ / 50) 1329 #define IPVS_SYNC_CHECK_PERIOD HZ 1330 #define IPVS_SYNC_FLUSH_TIME (HZ * 2) 1331 #define IPVS_SYNC_PORTS_MAX (1 << 6) 1332 1333 #ifdef CONFIG_SYSCTL 1334 1335 static inline int sysctl_conn_max(struct netns_ipvs *ipvs) 1336 { 1337 return READ_ONCE(ipvs->sysctl_conn_max); 1338 } 1339 1340 static inline int sysctl_sync_threshold(struct netns_ipvs *ipvs) 1341 { 1342 return ipvs->sysctl_sync_threshold[0]; 1343 } 1344 1345 static inline int sysctl_sync_period(struct netns_ipvs *ipvs) 1346 { 1347 return READ_ONCE(ipvs->sysctl_sync_threshold[1]); 1348 } 1349 1350 static inline unsigned int sysctl_sync_refresh_period(struct netns_ipvs *ipvs) 1351 { 1352 return READ_ONCE(ipvs->sysctl_sync_refresh_period); 1353 } 1354 1355 static inline int sysctl_sync_retries(struct netns_ipvs *ipvs) 1356 { 1357 return ipvs->sysctl_sync_retries; 1358 } 1359 1360 static inline int sysctl_sync_ver(struct netns_ipvs *ipvs) 1361 { 1362 return ipvs->sysctl_sync_ver; 1363 } 1364 1365 static inline int sysctl_sloppy_tcp(struct netns_ipvs *ipvs) 1366 { 1367 return ipvs->sysctl_sloppy_tcp; 1368 } 1369 1370 static inline int sysctl_sloppy_sctp(struct netns_ipvs *ipvs) 1371 { 1372 return ipvs->sysctl_sloppy_sctp; 1373 } 1374 1375 static inline int sysctl_sync_ports(struct netns_ipvs *ipvs) 1376 { 1377 return READ_ONCE(ipvs->sysctl_sync_ports); 1378 } 1379 1380 static inline int sysctl_sync_persist_mode(struct netns_ipvs *ipvs) 1381 { 1382 return ipvs->sysctl_sync_persist_mode; 1383 } 1384 1385 static inline unsigned long sysctl_sync_qlen_max(struct netns_ipvs *ipvs) 1386 { 1387 return ipvs->sysctl_sync_qlen_max; 1388 } 1389 1390 static inline int sysctl_sync_sock_size(struct netns_ipvs *ipvs) 1391 { 1392 return ipvs->sysctl_sync_sock_size; 1393 } 1394 1395 static inline int sysctl_pmtu_disc(struct netns_ipvs *ipvs) 1396 { 1397 return ipvs->sysctl_pmtu_disc; 1398 } 1399 1400 static inline int sysctl_backup_only(struct netns_ipvs *ipvs) 1401 { 1402 return ipvs->sync_state & IP_VS_STATE_BACKUP && 1403 ipvs->sysctl_backup_only; 1404 } 1405 1406 static inline int sysctl_conn_reuse_mode(struct netns_ipvs *ipvs) 1407 { 1408 return ipvs->sysctl_conn_reuse_mode; 1409 } 1410 1411 static inline int sysctl_expire_nodest_conn(struct netns_ipvs *ipvs) 1412 { 1413 return ipvs->sysctl_expire_nodest_conn; 1414 } 1415 1416 static inline int sysctl_schedule_icmp(struct netns_ipvs *ipvs) 1417 { 1418 return ipvs->sysctl_schedule_icmp; 1419 } 1420 1421 static inline int sysctl_ignore_tunneled(struct netns_ipvs *ipvs) 1422 { 1423 return ipvs->sysctl_ignore_tunneled; 1424 } 1425 1426 static inline int sysctl_cache_bypass(struct netns_ipvs *ipvs) 1427 { 1428 return ipvs->sysctl_cache_bypass; 1429 } 1430 1431 static inline int sysctl_run_estimation(struct netns_ipvs *ipvs) 1432 { 1433 return ipvs->sysctl_run_estimation; 1434 } 1435 1436 static inline const struct cpumask *__sysctl_est_cpulist(struct netns_ipvs *ipvs) 1437 { 1438 if (ipvs->est_cpulist_valid) 1439 return ipvs->sysctl_est_cpulist; 1440 else 1441 return housekeeping_cpumask(HK_TYPE_KTHREAD); 1442 } 1443 1444 static inline const struct cpumask *sysctl_est_preferred_cpulist(struct netns_ipvs *ipvs) 1445 { 1446 if (ipvs->est_cpulist_valid) 1447 return ipvs->sysctl_est_cpulist; 1448 else 1449 return NULL; 1450 } 1451 1452 static inline int sysctl_est_nice(struct netns_ipvs *ipvs) 1453 { 1454 return ipvs->sysctl_est_nice; 1455 } 1456 1457 #else 1458 1459 static inline int sysctl_conn_max(struct netns_ipvs *ipvs) 1460 { 1461 return IP_VS_CONN_MAX; 1462 } 1463 1464 static inline int sysctl_sync_threshold(struct netns_ipvs *ipvs) 1465 { 1466 return DEFAULT_SYNC_THRESHOLD; 1467 } 1468 1469 static inline int sysctl_sync_period(struct netns_ipvs *ipvs) 1470 { 1471 return DEFAULT_SYNC_PERIOD; 1472 } 1473 1474 static inline unsigned int sysctl_sync_refresh_period(struct netns_ipvs *ipvs) 1475 { 1476 return DEFAULT_SYNC_REFRESH_PERIOD; 1477 } 1478 1479 static inline int sysctl_sync_retries(struct netns_ipvs *ipvs) 1480 { 1481 return DEFAULT_SYNC_RETRIES & 3; 1482 } 1483 1484 static inline int sysctl_sync_ver(struct netns_ipvs *ipvs) 1485 { 1486 return DEFAULT_SYNC_VER; 1487 } 1488 1489 static inline int sysctl_sloppy_tcp(struct netns_ipvs *ipvs) 1490 { 1491 return DEFAULT_SLOPPY_TCP; 1492 } 1493 1494 static inline int sysctl_sloppy_sctp(struct netns_ipvs *ipvs) 1495 { 1496 return DEFAULT_SLOPPY_SCTP; 1497 } 1498 1499 static inline int sysctl_sync_ports(struct netns_ipvs *ipvs) 1500 { 1501 return 1; 1502 } 1503 1504 static inline int sysctl_sync_persist_mode(struct netns_ipvs *ipvs) 1505 { 1506 return 0; 1507 } 1508 1509 static inline unsigned long sysctl_sync_qlen_max(struct netns_ipvs *ipvs) 1510 { 1511 return IPVS_SYNC_QLEN_MAX; 1512 } 1513 1514 static inline int sysctl_sync_sock_size(struct netns_ipvs *ipvs) 1515 { 1516 return 0; 1517 } 1518 1519 static inline int sysctl_pmtu_disc(struct netns_ipvs *ipvs) 1520 { 1521 return 1; 1522 } 1523 1524 static inline int sysctl_backup_only(struct netns_ipvs *ipvs) 1525 { 1526 return 0; 1527 } 1528 1529 static inline int sysctl_conn_reuse_mode(struct netns_ipvs *ipvs) 1530 { 1531 return 1; 1532 } 1533 1534 static inline int sysctl_expire_nodest_conn(struct netns_ipvs *ipvs) 1535 { 1536 return 0; 1537 } 1538 1539 static inline int sysctl_schedule_icmp(struct netns_ipvs *ipvs) 1540 { 1541 return 0; 1542 } 1543 1544 static inline int sysctl_ignore_tunneled(struct netns_ipvs *ipvs) 1545 { 1546 return 0; 1547 } 1548 1549 static inline int sysctl_cache_bypass(struct netns_ipvs *ipvs) 1550 { 1551 return 0; 1552 } 1553 1554 static inline int sysctl_run_estimation(struct netns_ipvs *ipvs) 1555 { 1556 return 1; 1557 } 1558 1559 static inline const struct cpumask *__sysctl_est_cpulist(struct netns_ipvs *ipvs) 1560 { 1561 return housekeeping_cpumask(HK_TYPE_KTHREAD); 1562 } 1563 1564 static inline const struct cpumask *sysctl_est_preferred_cpulist(struct netns_ipvs *ipvs) 1565 { 1566 return NULL; 1567 } 1568 1569 static inline int sysctl_est_nice(struct netns_ipvs *ipvs) 1570 { 1571 return IPVS_EST_NICE; 1572 } 1573 1574 #endif 1575 1576 /* Get load factor to map conn_count/u_thresh to t->size */ 1577 static inline int sysctl_conn_lfactor(struct netns_ipvs *ipvs) 1578 { 1579 return READ_ONCE(ipvs->sysctl_conn_lfactor); 1580 } 1581 1582 /* Get load factor to map num_services/u_thresh to t->size 1583 * Smaller value decreases u_thresh to reduce collisions but increases 1584 * the table size 1585 * Returns factor where: 1586 * - <0: u_thresh = size >> -factor, eg. lfactor -2 = 25% load 1587 * - >=0: u_thresh = size << factor, eg. lfactor 1 = 200% load 1588 */ 1589 static inline int sysctl_svc_lfactor(struct netns_ipvs *ipvs) 1590 { 1591 return READ_ONCE(ipvs->sysctl_svc_lfactor); 1592 } 1593 1594 static inline bool sysctl_est_cpulist_empty(struct netns_ipvs *ipvs) 1595 { 1596 guard(rcu)(); 1597 return cpumask_empty(__sysctl_est_cpulist(ipvs)); 1598 } 1599 1600 static inline unsigned int sysctl_est_cpulist_weight(struct netns_ipvs *ipvs) 1601 { 1602 guard(rcu)(); 1603 return cpumask_weight(__sysctl_est_cpulist(ipvs)); 1604 } 1605 1606 /* IPVS core functions 1607 * (from ip_vs_core.c) 1608 */ 1609 const char *ip_vs_proto_name(unsigned int proto); 1610 void ip_vs_init_hash_table(struct list_head *table, int rows); 1611 struct ip_vs_conn *ip_vs_new_conn_out(struct ip_vs_service *svc, 1612 struct ip_vs_dest *dest, 1613 struct sk_buff *skb, 1614 const struct ip_vs_iphdr *iph, 1615 __be16 dport, 1616 __be16 cport); 1617 #define IP_VS_INIT_HASH_TABLE(t) ip_vs_init_hash_table((t), ARRAY_SIZE((t))) 1618 1619 #define IP_VS_APP_TYPE_FTP 1 1620 1621 /* ip_vs_conn handling functions 1622 * (from ip_vs_conn.c) 1623 */ 1624 enum { 1625 IP_VS_DIR_INPUT = 0, 1626 IP_VS_DIR_OUTPUT, 1627 IP_VS_DIR_INPUT_ONLY, 1628 IP_VS_DIR_LAST, 1629 }; 1630 1631 static inline void ip_vs_conn_fill_param(struct netns_ipvs *ipvs, int af, int protocol, 1632 const union nf_inet_addr *caddr, 1633 __be16 cport, 1634 const union nf_inet_addr *vaddr, 1635 __be16 vport, 1636 struct ip_vs_conn_param *p) 1637 { 1638 p->ipvs = ipvs; 1639 p->af = af; 1640 p->protocol = protocol; 1641 p->caddr = caddr; 1642 p->cport = cport; 1643 p->vaddr = vaddr; 1644 p->vport = vport; 1645 p->pe = NULL; 1646 p->pe_data = NULL; 1647 } 1648 1649 struct ip_vs_conn *ip_vs_conn_in_get(const struct ip_vs_conn_param *p); 1650 struct ip_vs_conn *ip_vs_ct_in_get(const struct ip_vs_conn_param *p); 1651 1652 struct ip_vs_conn * ip_vs_conn_in_get_proto(struct netns_ipvs *ipvs, int af, 1653 const struct sk_buff *skb, 1654 const struct ip_vs_iphdr *iph); 1655 1656 struct ip_vs_conn *ip_vs_conn_out_get(const struct ip_vs_conn_param *p); 1657 1658 struct ip_vs_conn * ip_vs_conn_out_get_proto(struct netns_ipvs *ipvs, int af, 1659 const struct sk_buff *skb, 1660 const struct ip_vs_iphdr *iph); 1661 1662 /* Get reference to gain full access to conn. 1663 * By default, RCU read-side critical sections have access only to 1664 * conn fields and its PE data, see ip_vs_conn_rcu_free() for reference. 1665 */ 1666 static inline bool __ip_vs_conn_get(struct ip_vs_conn *cp) 1667 { 1668 return refcount_inc_not_zero(&cp->refcnt); 1669 } 1670 1671 /* put back the conn without restarting its timer */ 1672 static inline void __ip_vs_conn_put(struct ip_vs_conn *cp) 1673 { 1674 smp_mb__before_atomic(); 1675 refcount_dec(&cp->refcnt); 1676 } 1677 void ip_vs_conn_put(struct ip_vs_conn *cp); 1678 void ip_vs_conn_fill_cport(struct ip_vs_conn *cp, __be16 cport); 1679 int ip_vs_conn_desired_size(struct netns_ipvs *ipvs, struct ip_vs_rht *t, 1680 int lfactor); 1681 struct ip_vs_rht *ip_vs_conn_tab_alloc(struct netns_ipvs *ipvs, int buckets, 1682 int lfactor); 1683 1684 static inline struct ip_vs_conn * 1685 ip_vs_hn0_to_conn(struct ip_vs_conn_hnode *hn) 1686 { 1687 return container_of(hn, struct ip_vs_conn, hn0); 1688 } 1689 1690 static inline struct ip_vs_conn * 1691 ip_vs_hn_to_conn(struct ip_vs_conn_hnode *hn) 1692 { 1693 return hn->dir ? container_of(hn, struct ip_vs_conn, hn1) : 1694 container_of(hn, struct ip_vs_conn, hn0); 1695 } 1696 1697 struct ip_vs_conn *ip_vs_conn_new(const struct ip_vs_conn_param *p, int dest_af, 1698 const union nf_inet_addr *daddr, 1699 __be16 dport, unsigned int flags, 1700 struct ip_vs_dest *dest, __u32 fwmark); 1701 void ip_vs_conn_expire_now(struct ip_vs_conn *cp); 1702 1703 const char *ip_vs_state_name(const struct ip_vs_conn *cp); 1704 1705 void ip_vs_tcp_conn_listen(struct ip_vs_conn *cp); 1706 int ip_vs_check_template(struct ip_vs_conn *ct, struct ip_vs_dest *cdest); 1707 void ip_vs_random_dropentry(struct netns_ipvs *ipvs); 1708 int ip_vs_conn_init(void); 1709 void ip_vs_conn_cleanup(void); 1710 1711 static inline void ip_vs_control_del(struct ip_vs_conn *cp) 1712 { 1713 struct ip_vs_conn *ctl_cp = cp->control; 1714 if (!ctl_cp) { 1715 IP_VS_ERR_BUF("request control DEL for uncontrolled: " 1716 "%s:%d to %s:%d\n", 1717 IP_VS_DBG_ADDR(cp->af, &cp->caddr), 1718 ntohs(cp->cport), 1719 IP_VS_DBG_ADDR(cp->af, &cp->vaddr), 1720 ntohs(cp->vport)); 1721 1722 return; 1723 } 1724 1725 IP_VS_DBG_BUF(7, "DELeting control for: " 1726 "cp.dst=%s:%d ctl_cp.dst=%s:%d\n", 1727 IP_VS_DBG_ADDR(cp->af, &cp->caddr), 1728 ntohs(cp->cport), 1729 IP_VS_DBG_ADDR(cp->af, &ctl_cp->caddr), 1730 ntohs(ctl_cp->cport)); 1731 1732 cp->control = NULL; 1733 if (atomic_read(&ctl_cp->n_control) == 0) { 1734 IP_VS_ERR_BUF("BUG control DEL with n=0 : " 1735 "%s:%d to %s:%d\n", 1736 IP_VS_DBG_ADDR(cp->af, &cp->caddr), 1737 ntohs(cp->cport), 1738 IP_VS_DBG_ADDR(cp->af, &cp->vaddr), 1739 ntohs(cp->vport)); 1740 1741 return; 1742 } 1743 atomic_dec(&ctl_cp->n_control); 1744 } 1745 1746 static inline void 1747 ip_vs_control_add(struct ip_vs_conn *cp, struct ip_vs_conn *ctl_cp) 1748 { 1749 if (cp->control) { 1750 IP_VS_ERR_BUF("request control ADD for already controlled: " 1751 "%s:%d to %s:%d\n", 1752 IP_VS_DBG_ADDR(cp->af, &cp->caddr), 1753 ntohs(cp->cport), 1754 IP_VS_DBG_ADDR(cp->af, &cp->vaddr), 1755 ntohs(cp->vport)); 1756 1757 ip_vs_control_del(cp); 1758 } 1759 1760 IP_VS_DBG_BUF(7, "ADDing control for: " 1761 "cp.dst=%s:%d ctl_cp.dst=%s:%d\n", 1762 IP_VS_DBG_ADDR(cp->af, &cp->caddr), 1763 ntohs(cp->cport), 1764 IP_VS_DBG_ADDR(cp->af, &ctl_cp->caddr), 1765 ntohs(ctl_cp->cport)); 1766 1767 cp->control = ctl_cp; 1768 atomic_inc(&ctl_cp->n_control); 1769 } 1770 1771 /* Mark our template as assured */ 1772 static inline void 1773 ip_vs_control_assure_ct(struct ip_vs_conn *cp) 1774 { 1775 struct ip_vs_conn *ct = cp->control; 1776 1777 if (ct && !(ct->state & IP_VS_CTPL_S_ASSURED) && 1778 (ct->flags & IP_VS_CONN_F_TEMPLATE)) 1779 ct->state |= IP_VS_CTPL_S_ASSURED; 1780 } 1781 1782 /* IPVS netns init & cleanup functions */ 1783 int ip_vs_estimator_net_init(struct netns_ipvs *ipvs); 1784 int ip_vs_control_net_init(struct netns_ipvs *ipvs); 1785 int ip_vs_protocol_net_init(struct netns_ipvs *ipvs); 1786 int ip_vs_app_net_init(struct netns_ipvs *ipvs); 1787 int ip_vs_conn_net_init(struct netns_ipvs *ipvs); 1788 int ip_vs_sync_net_init(struct netns_ipvs *ipvs); 1789 void ip_vs_conn_net_cleanup(struct netns_ipvs *ipvs); 1790 void ip_vs_app_net_cleanup(struct netns_ipvs *ipvs); 1791 void ip_vs_protocol_net_cleanup(struct netns_ipvs *ipvs); 1792 void ip_vs_control_net_cleanup(struct netns_ipvs *ipvs); 1793 void ip_vs_estimator_net_cleanup(struct netns_ipvs *ipvs); 1794 void ip_vs_sync_net_cleanup(struct netns_ipvs *ipvs); 1795 void ip_vs_service_nets_cleanup(struct list_head *net_list); 1796 1797 /* IPVS application functions 1798 * (from ip_vs_app.c) 1799 */ 1800 #define IP_VS_APP_MAX_PORTS 8 1801 struct ip_vs_app *register_ip_vs_app(struct netns_ipvs *ipvs, struct ip_vs_app *app); 1802 void unregister_ip_vs_app(struct netns_ipvs *ipvs, struct ip_vs_app *app); 1803 int ip_vs_bind_app(struct ip_vs_conn *cp, struct ip_vs_protocol *pp); 1804 void ip_vs_unbind_app(struct ip_vs_conn *cp); 1805 int register_ip_vs_app_inc(struct netns_ipvs *ipvs, struct ip_vs_app *app, __u16 proto, 1806 __u16 port); 1807 int ip_vs_app_inc_get(struct ip_vs_app *inc); 1808 void ip_vs_app_inc_put(struct ip_vs_app *inc); 1809 1810 int ip_vs_app_pkt_out(struct ip_vs_conn *, struct sk_buff *skb, 1811 struct ip_vs_iphdr *ipvsh); 1812 int ip_vs_app_pkt_in(struct ip_vs_conn *, struct sk_buff *skb, 1813 struct ip_vs_iphdr *ipvsh); 1814 1815 int register_ip_vs_pe(struct ip_vs_pe *pe); 1816 int unregister_ip_vs_pe(struct ip_vs_pe *pe); 1817 struct ip_vs_pe *ip_vs_pe_getbyname(const char *name); 1818 struct ip_vs_pe *__ip_vs_pe_getbyname(const char *pe_name); 1819 1820 /* Use a #define to avoid all of module.h just for these trivial ops */ 1821 #define ip_vs_pe_get(pe) \ 1822 if (pe && pe->module) \ 1823 __module_get(pe->module); 1824 1825 #define ip_vs_pe_put(pe) \ 1826 if (pe && pe->module) \ 1827 module_put(pe->module); 1828 1829 /* IPVS protocol functions (from ip_vs_proto.c) */ 1830 int ip_vs_protocol_init(void); 1831 void ip_vs_protocol_cleanup(void); 1832 void ip_vs_protocol_timeout_change(struct netns_ipvs *ipvs, int flags); 1833 int *ip_vs_create_timeout_table(int *table, int size); 1834 void ip_vs_tcpudp_debug_packet(int af, struct ip_vs_protocol *pp, 1835 const struct sk_buff *skb, int offset, 1836 const char *msg); 1837 1838 extern struct ip_vs_protocol ip_vs_protocol_tcp; 1839 extern struct ip_vs_protocol ip_vs_protocol_udp; 1840 extern struct ip_vs_protocol ip_vs_protocol_icmp; 1841 extern struct ip_vs_protocol ip_vs_protocol_esp; 1842 extern struct ip_vs_protocol ip_vs_protocol_ah; 1843 extern struct ip_vs_protocol ip_vs_protocol_sctp; 1844 1845 /* Registering/unregistering scheduler functions 1846 * (from ip_vs_sched.c) 1847 */ 1848 int register_ip_vs_scheduler(struct ip_vs_scheduler *scheduler); 1849 int unregister_ip_vs_scheduler(struct ip_vs_scheduler *scheduler); 1850 int ip_vs_bind_scheduler(struct ip_vs_service *svc, 1851 struct ip_vs_scheduler *scheduler); 1852 void ip_vs_unbind_scheduler(struct ip_vs_service *svc); 1853 struct ip_vs_scheduler *ip_vs_scheduler_get(const char *sched_name); 1854 void ip_vs_scheduler_put(struct ip_vs_scheduler *scheduler); 1855 struct ip_vs_conn * 1856 ip_vs_schedule(struct ip_vs_service *svc, struct sk_buff *skb, 1857 struct ip_vs_proto_data *pd, int *ignored, 1858 struct ip_vs_iphdr *iph); 1859 int ip_vs_leave(struct ip_vs_service *svc, struct sk_buff *skb, 1860 struct ip_vs_proto_data *pd, struct ip_vs_iphdr *iph); 1861 1862 void ip_vs_scheduler_err(struct ip_vs_service *svc, const char *msg); 1863 1864 /* IPVS control data and functions (from ip_vs_ctl.c) */ 1865 extern struct ip_vs_stats ip_vs_stats; 1866 extern int sysctl_ip_vs_sync_ver; 1867 1868 struct ip_vs_service * 1869 ip_vs_service_find(struct netns_ipvs *ipvs, int af, __u32 fwmark, __u16 protocol, 1870 const union nf_inet_addr *vaddr, __be16 vport); 1871 1872 bool ip_vs_has_real_service(struct netns_ipvs *ipvs, int af, __u16 protocol, 1873 const union nf_inet_addr *daddr, __be16 dport); 1874 1875 struct ip_vs_dest * 1876 ip_vs_find_real_service(struct netns_ipvs *ipvs, int af, __u16 protocol, 1877 const union nf_inet_addr *daddr, __be16 dport); 1878 struct ip_vs_dest *ip_vs_find_tunnel(struct netns_ipvs *ipvs, int af, 1879 const union nf_inet_addr *daddr, 1880 __be16 tun_port); 1881 1882 int ip_vs_use_count_inc(void); 1883 void ip_vs_use_count_dec(void); 1884 int ip_vs_register_nl_ioctl(void); 1885 void ip_vs_unregister_nl_ioctl(void); 1886 int ip_vs_control_init(void); 1887 void ip_vs_control_cleanup(void); 1888 struct ip_vs_dest * 1889 ip_vs_find_dest(struct netns_ipvs *ipvs, int svc_af, int dest_af, 1890 const union nf_inet_addr *daddr, __be16 dport, 1891 const union nf_inet_addr *vaddr, __be16 vport, 1892 __u16 protocol, __u32 fwmark, __u32 flags); 1893 void ip_vs_try_bind_dest(struct ip_vs_conn *cp); 1894 1895 static inline void ip_vs_dest_hold(struct ip_vs_dest *dest) 1896 { 1897 refcount_inc(&dest->refcnt); 1898 } 1899 1900 static inline void ip_vs_dest_put(struct ip_vs_dest *dest) 1901 { 1902 smp_mb__before_atomic(); 1903 refcount_dec(&dest->refcnt); 1904 } 1905 1906 static inline void ip_vs_dest_put_and_free(struct ip_vs_dest *dest) 1907 { 1908 if (refcount_dec_and_test(&dest->refcnt)) 1909 kfree(dest); 1910 } 1911 1912 /* IPVS sync daemon data and function prototypes 1913 * (from ip_vs_sync.c) 1914 */ 1915 int start_sync_thread(struct netns_ipvs *ipvs, struct ipvs_sync_daemon_cfg *cfg, 1916 int state); 1917 int stop_sync_thread(struct netns_ipvs *ipvs, int state); 1918 void ip_vs_sync_conn(struct netns_ipvs *ipvs, struct ip_vs_conn *cp, int pkts); 1919 1920 /* IPVS rate estimator prototypes (from ip_vs_est.c) */ 1921 int ip_vs_start_estimator(struct netns_ipvs *ipvs, struct ip_vs_stats *stats); 1922 void ip_vs_stop_estimator(struct netns_ipvs *ipvs, struct ip_vs_stats *stats); 1923 void ip_vs_zero_estimator(struct ip_vs_stats *stats); 1924 void ip_vs_read_estimator(struct ip_vs_kstats *dst, struct ip_vs_stats *stats); 1925 void ip_vs_est_reload_start(struct netns_ipvs *ipvs, bool restart); 1926 int ip_vs_est_kthread_start(struct netns_ipvs *ipvs, 1927 struct ip_vs_est_kt_data *kd); 1928 void ip_vs_est_kthread_stop(struct ip_vs_est_kt_data *kd); 1929 1930 static inline void ip_vs_stop_estimator_tot_stats(struct netns_ipvs *ipvs) 1931 { 1932 #ifdef CONFIG_SYSCTL 1933 ip_vs_stop_estimator(ipvs, &ipvs->tot_stats->s); 1934 ipvs->tot_stats->s.est.ktid = -2; 1935 #endif 1936 } 1937 1938 static inline void ip_vs_est_stopped_recalc(struct netns_ipvs *ipvs) 1939 { 1940 #ifdef CONFIG_SYSCTL 1941 /* Stop tasks while cpulist is empty or if disabled with flag */ 1942 ipvs->est_stopped = !sysctl_run_estimation(ipvs) || 1943 (ipvs->est_cpulist_valid && 1944 sysctl_est_cpulist_empty(ipvs)); 1945 #endif 1946 } 1947 1948 static inline bool ip_vs_est_stopped(struct netns_ipvs *ipvs) 1949 { 1950 #ifdef CONFIG_SYSCTL 1951 return ipvs->est_stopped; 1952 #else 1953 return false; 1954 #endif 1955 } 1956 1957 static inline int ip_vs_est_max_threads(struct netns_ipvs *ipvs) 1958 { 1959 unsigned int limit = IPVS_EST_CPU_KTHREADS * 1960 sysctl_est_cpulist_weight(ipvs); 1961 1962 return max(1U, limit); 1963 } 1964 1965 /* Various IPVS packet transmitters (from ip_vs_xmit.c) */ 1966 int ip_vs_null_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 1967 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1968 int ip_vs_bypass_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 1969 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1970 int ip_vs_nat_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 1971 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1972 int ip_vs_tunnel_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 1973 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1974 int ip_vs_dr_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 1975 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1976 int ip_vs_icmp_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 1977 struct ip_vs_protocol *pp, unsigned int toff, 1978 unsigned int hooknum, struct ip_vs_iphdr *ciph); 1979 void ip_vs_dest_dst_rcu_free(struct rcu_head *head); 1980 1981 #ifdef CONFIG_IP_VS_IPV6 1982 int ip_vs_bypass_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, 1983 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1984 int ip_vs_nat_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, 1985 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1986 int ip_vs_tunnel_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, 1987 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1988 int ip_vs_dr_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, 1989 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1990 int ip_vs_icmp_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, 1991 struct ip_vs_protocol *pp, unsigned int toff, 1992 unsigned int hooknum, struct ip_vs_iphdr *ciph); 1993 #endif 1994 1995 #ifdef CONFIG_SYSCTL 1996 /* This is a simple mechanism to ignore packets when 1997 * we are loaded. Just set ip_vs_drop_rate to 'n' and 1998 * we start to drop 1/rate of the packets 1999 */ 2000 static inline int ip_vs_todrop(struct netns_ipvs *ipvs) 2001 { 2002 if (!ipvs->drop_rate) 2003 return 0; 2004 if (--ipvs->drop_counter > 0) 2005 return 0; 2006 ipvs->drop_counter = ipvs->drop_rate; 2007 return 1; 2008 } 2009 #else 2010 static inline int ip_vs_todrop(struct netns_ipvs *ipvs) { return 0; } 2011 #endif 2012 2013 #ifdef CONFIG_SYSCTL 2014 /* Enqueue delayed work for expiring no dest connections 2015 * Only run when sysctl_expire_nodest=1 2016 */ 2017 static inline void ip_vs_enqueue_expire_nodest_conns(struct netns_ipvs *ipvs) 2018 { 2019 if (sysctl_expire_nodest_conn(ipvs)) 2020 queue_delayed_work(system_long_wq, 2021 &ipvs->expire_nodest_conn_work, 1); 2022 } 2023 2024 void ip_vs_expire_nodest_conn_flush(struct netns_ipvs *ipvs); 2025 #else 2026 static inline void ip_vs_enqueue_expire_nodest_conns(struct netns_ipvs *ipvs) {} 2027 #endif 2028 2029 #define IP_VS_DFWD_METHOD(dest) (atomic_read(&(dest)->conn_flags) & \ 2030 IP_VS_CONN_F_FWD_MASK) 2031 2032 /* ip_vs_fwd_tag returns the forwarding tag of the connection */ 2033 #define IP_VS_FWD_METHOD(cp) (cp->flags & IP_VS_CONN_F_FWD_MASK) 2034 2035 static inline char ip_vs_fwd_tag(struct ip_vs_conn *cp) 2036 { 2037 char fwd; 2038 2039 switch (IP_VS_FWD_METHOD(cp)) { 2040 case IP_VS_CONN_F_MASQ: 2041 fwd = 'M'; break; 2042 case IP_VS_CONN_F_LOCALNODE: 2043 fwd = 'L'; break; 2044 case IP_VS_CONN_F_TUNNEL: 2045 fwd = 'T'; break; 2046 case IP_VS_CONN_F_DROUTE: 2047 fwd = 'R'; break; 2048 case IP_VS_CONN_F_BYPASS: 2049 fwd = 'B'; break; 2050 default: 2051 fwd = '?'; break; 2052 } 2053 return fwd; 2054 } 2055 2056 /* Check if connection uses double hashing */ 2057 static inline bool ip_vs_conn_use_hash2(struct ip_vs_conn *cp) 2058 { 2059 return IP_VS_FWD_METHOD(cp) == IP_VS_CONN_F_MASQ && 2060 !(cp->flags & IP_VS_CONN_F_TEMPLATE); 2061 } 2062 2063 void ip_vs_nat_icmp(struct sk_buff *skb, struct ip_vs_protocol *pp, 2064 struct ip_vs_conn *cp, int dir, unsigned int toff, 2065 bool has_ports); 2066 2067 #ifdef CONFIG_IP_VS_IPV6 2068 void ip_vs_nat_icmp_v6(struct sk_buff *skb, struct ip_vs_protocol *pp, 2069 struct ip_vs_conn *cp, int dir, unsigned int toff, 2070 bool has_ports, struct ip_vs_iphdr *ciph); 2071 #endif 2072 2073 static inline __wsum ip_vs_check_diff4(__be32 old, __be32 new, __wsum oldsum) 2074 { 2075 __be32 diff[2] = { ~old, new }; 2076 2077 return csum_partial(diff, sizeof(diff), oldsum); 2078 } 2079 2080 #ifdef CONFIG_IP_VS_IPV6 2081 static inline __wsum ip_vs_check_diff16(const __be32 *old, const __be32 *new, 2082 __wsum oldsum) 2083 { 2084 __be32 diff[8] = { ~old[3], ~old[2], ~old[1], ~old[0], 2085 new[3], new[2], new[1], new[0] }; 2086 2087 return csum_partial(diff, sizeof(diff), oldsum); 2088 } 2089 #endif 2090 2091 static inline __wsum ip_vs_check_diff2(__be16 old, __be16 new, __wsum oldsum) 2092 { 2093 __be16 diff[2] = { ~old, new }; 2094 2095 return csum_partial(diff, sizeof(diff), oldsum); 2096 } 2097 2098 static inline bool ip_vs_checksum_needed(struct sk_buff *skb, int af) 2099 { 2100 /* Checksum unnecessary or already validated? */ 2101 if (skb_csum_unnecessary(skb)) 2102 return false; 2103 /* LOCAL_OUT ? */ 2104 if (!skb->dev || skb->dev->flags & IFF_LOOPBACK) 2105 return false; 2106 /* !LOCAL_IN (FORWARD) ? */ 2107 if (af == AF_INET6) { 2108 if (!(dst_rt6_info(skb_dst(skb))->rt6i_flags & RTF_LOCAL)) 2109 return false; 2110 } else { 2111 if (!(skb_rtable(skb)->rt_flags & RTCF_LOCAL)) 2112 return false; 2113 } 2114 return true; 2115 } 2116 2117 static inline bool ip_vs_checksum_common_check(struct sk_buff *skb, 2118 int offset, int proto, int af) 2119 { 2120 if (!ip_vs_checksum_needed(skb, af)) 2121 return true; 2122 return !nf_checksum(skb, NF_INET_LOCAL_IN, offset, proto, af); 2123 } 2124 2125 /* Forget current conntrack (unconfirmed) and attach notrack entry */ 2126 static inline void ip_vs_notrack(struct sk_buff *skb) 2127 { 2128 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE) 2129 enum ip_conntrack_info ctinfo; 2130 struct nf_conn *ct = nf_ct_get(skb, &ctinfo); 2131 2132 if (ct) { 2133 nf_conntrack_put(&ct->ct_general); 2134 nf_ct_set(skb, NULL, IP_CT_UNTRACKED); 2135 } 2136 #endif 2137 } 2138 2139 #ifdef CONFIG_IP_VS_NFCT 2140 /* Netfilter connection tracking 2141 * (from ip_vs_nfct.c) 2142 */ 2143 static inline int ip_vs_conntrack_enabled(struct netns_ipvs *ipvs) 2144 { 2145 #ifdef CONFIG_SYSCTL 2146 return ipvs->sysctl_conntrack; 2147 #else 2148 return 0; 2149 #endif 2150 } 2151 2152 void ip_vs_update_conntrack(struct sk_buff *skb, struct ip_vs_conn *cp, 2153 int outin); 2154 int ip_vs_confirm_conntrack(struct sk_buff *skb); 2155 void ip_vs_nfct_expect_related(struct sk_buff *skb, struct nf_conn *ct, 2156 struct ip_vs_conn *cp, u_int8_t proto, 2157 const __be16 port, int from_rs); 2158 void ip_vs_conn_drop_conntrack(struct ip_vs_conn *cp); 2159 2160 #else 2161 2162 static inline int ip_vs_conntrack_enabled(struct netns_ipvs *ipvs) 2163 { 2164 return 0; 2165 } 2166 2167 static inline void ip_vs_update_conntrack(struct sk_buff *skb, 2168 struct ip_vs_conn *cp, int outin) 2169 { 2170 } 2171 2172 static inline int ip_vs_confirm_conntrack(struct sk_buff *skb) 2173 { 2174 return NF_ACCEPT; 2175 } 2176 2177 static inline void ip_vs_conn_drop_conntrack(struct ip_vs_conn *cp) 2178 { 2179 } 2180 #endif /* CONFIG_IP_VS_NFCT */ 2181 2182 /* Using old conntrack that can not be redirected to another real server? */ 2183 static inline bool ip_vs_conn_uses_old_conntrack(struct ip_vs_conn *cp, 2184 struct sk_buff *skb) 2185 { 2186 #ifdef CONFIG_IP_VS_NFCT 2187 enum ip_conntrack_info ctinfo; 2188 struct nf_conn *ct; 2189 2190 ct = nf_ct_get(skb, &ctinfo); 2191 if (ct && nf_ct_is_confirmed(ct)) 2192 return true; 2193 #endif 2194 return false; 2195 } 2196 2197 static inline int ip_vs_register_conntrack(struct ip_vs_service *svc) 2198 { 2199 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 2200 int afmask = (svc->af == AF_INET6) ? 2 : 1; 2201 int ret = 0; 2202 2203 if (!(svc->conntrack_afmask & afmask)) { 2204 ret = nf_ct_netns_get(svc->ipvs->net, svc->af); 2205 if (ret >= 0) 2206 svc->conntrack_afmask |= afmask; 2207 } 2208 return ret; 2209 #else 2210 return 0; 2211 #endif 2212 } 2213 2214 static inline void ip_vs_unregister_conntrack(struct ip_vs_service *svc) 2215 { 2216 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 2217 int afmask = (svc->af == AF_INET6) ? 2 : 1; 2218 2219 if (svc->conntrack_afmask & afmask) { 2220 nf_ct_netns_put(svc->ipvs->net, svc->af); 2221 svc->conntrack_afmask &= ~afmask; 2222 } 2223 #endif 2224 } 2225 2226 int ip_vs_register_hooks(struct netns_ipvs *ipvs, unsigned int af); 2227 void ip_vs_unregister_hooks(struct netns_ipvs *ipvs, unsigned int af); 2228 2229 static inline int 2230 ip_vs_dest_conn_overhead(struct ip_vs_dest *dest) 2231 { 2232 /* We think the overhead of processing active connections is 256 2233 * times higher than that of inactive connections in average. (This 2234 * 256 times might not be accurate, we will change it later) We 2235 * use the following formula to estimate the overhead now: 2236 * dest->activeconns*256 + dest->inactconns 2237 */ 2238 return (atomic_read(&dest->activeconns) << 8) + 2239 atomic_read(&dest->inactconns); 2240 } 2241 2242 #ifdef CONFIG_IP_VS_PROTO_TCP 2243 INDIRECT_CALLABLE_DECLARE(int 2244 tcp_snat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, 2245 struct ip_vs_conn *cp, struct ip_vs_iphdr *iph)); 2246 #endif 2247 2248 #ifdef CONFIG_IP_VS_PROTO_UDP 2249 INDIRECT_CALLABLE_DECLARE(int 2250 udp_snat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, 2251 struct ip_vs_conn *cp, struct ip_vs_iphdr *iph)); 2252 #endif 2253 #endif /* _NET_IP_VS_H */ 2254