1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* IP Virtual Server 3 * data structure and functionality definitions 4 */ 5 6 #ifndef _NET_IP_VS_H 7 #define _NET_IP_VS_H 8 9 #include <linux/ip_vs.h> /* definitions shared with userland */ 10 11 #include <asm/types.h> /* for __uXX types */ 12 13 #include <linux/list.h> /* for struct list_head */ 14 #include <linux/rculist_bl.h> /* for struct hlist_bl_head */ 15 #include <linux/spinlock.h> /* for struct rwlock_t */ 16 #include <linux/atomic.h> /* for struct atomic_t */ 17 #include <linux/refcount.h> /* for struct refcount_t */ 18 #include <linux/workqueue.h> 19 20 #include <linux/compiler.h> 21 #include <linux/timer.h> 22 #include <linux/bug.h> 23 24 #include <net/checksum.h> 25 #include <linux/netfilter.h> /* for union nf_inet_addr */ 26 #include <linux/ip.h> 27 #include <linux/ipv6.h> /* for struct ipv6hdr */ 28 #include <net/ipv6.h> 29 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 30 #include <net/netfilter/nf_conntrack.h> 31 #endif 32 #include <net/net_namespace.h> /* Netw namespace */ 33 #include <linux/sched/isolation.h> 34 #include <linux/siphash.h> 35 36 #define IP_VS_HDR_INVERSE 1 37 #define IP_VS_HDR_ICMP 2 38 39 /* conn_tab limits (as per Kconfig) */ 40 #define IP_VS_CONN_TAB_MIN_BITS 8 41 #if BITS_PER_LONG > 32 42 #define IP_VS_CONN_TAB_MAX_BITS 27 43 #else 44 #define IP_VS_CONN_TAB_MAX_BITS 20 45 #endif 46 47 /* svc_table limits */ 48 #define IP_VS_SVC_TAB_MIN_BITS 4 49 #define IP_VS_SVC_TAB_MAX_BITS 20 50 51 /* Generic access of ipvs struct */ 52 static inline struct netns_ipvs *net_ipvs(struct net* net) 53 { 54 return net->ipvs; 55 } 56 57 /* Connections' size value needed by ip_vs_ctl.c */ 58 extern int ip_vs_conn_tab_size; 59 60 struct ip_vs_iphdr { 61 int hdr_flags; /* ipvs flags */ 62 __u32 off; /* Where IP or IPv4 header starts */ 63 __u32 len; /* IPv4 simply where L4 starts 64 * IPv6 where L4 Transport Header starts */ 65 __u16 fragoffs; /* IPv6 fragment offset, 0 if first frag (or not frag)*/ 66 __s16 protocol; 67 __s32 flags; 68 union nf_inet_addr saddr; 69 union nf_inet_addr daddr; 70 }; 71 72 static inline void *frag_safe_skb_hp(const struct sk_buff *skb, int offset, 73 int len, void *buffer) 74 { 75 return skb_header_pointer(skb, offset, len, buffer); 76 } 77 78 /* This function handles filling *ip_vs_iphdr, both for IPv4 and IPv6. 79 * IPv6 requires some extra work, as finding proper header position, 80 * depend on the IPv6 extension headers. 81 */ 82 static inline int 83 ip_vs_fill_iph_skb_off(int af, const struct sk_buff *skb, int offset, 84 int hdr_flags, struct ip_vs_iphdr *iphdr) 85 { 86 iphdr->hdr_flags = hdr_flags; 87 iphdr->off = offset; 88 89 #ifdef CONFIG_IP_VS_IPV6 90 if (af == AF_INET6) { 91 struct ipv6hdr _iph; 92 const struct ipv6hdr *iph = skb_header_pointer( 93 skb, offset, sizeof(_iph), &_iph); 94 if (!iph) 95 return 0; 96 97 iphdr->saddr.in6 = iph->saddr; 98 iphdr->daddr.in6 = iph->daddr; 99 /* ipv6_find_hdr() updates len, flags */ 100 iphdr->len = offset; 101 iphdr->flags = 0; 102 iphdr->protocol = ipv6_find_hdr(skb, &iphdr->len, -1, 103 &iphdr->fragoffs, 104 &iphdr->flags); 105 if (iphdr->protocol < 0) 106 return 0; 107 } else 108 #endif 109 { 110 struct iphdr _iph; 111 const struct iphdr *iph = skb_header_pointer( 112 skb, offset, sizeof(_iph), &_iph); 113 if (!iph) 114 return 0; 115 116 iphdr->len = offset + iph->ihl * 4; 117 iphdr->fragoffs = 0; 118 iphdr->protocol = iph->protocol; 119 iphdr->saddr.ip = iph->saddr; 120 iphdr->daddr.ip = iph->daddr; 121 } 122 123 return 1; 124 } 125 126 static inline int 127 ip_vs_fill_iph_skb_icmp(int af, const struct sk_buff *skb, int offset, 128 bool inverse, struct ip_vs_iphdr *iphdr) 129 { 130 int hdr_flags = IP_VS_HDR_ICMP; 131 132 if (inverse) 133 hdr_flags |= IP_VS_HDR_INVERSE; 134 135 return ip_vs_fill_iph_skb_off(af, skb, offset, hdr_flags, iphdr); 136 } 137 138 static inline int 139 ip_vs_fill_iph_skb(int af, const struct sk_buff *skb, bool inverse, 140 struct ip_vs_iphdr *iphdr) 141 { 142 int hdr_flags = 0; 143 144 if (inverse) 145 hdr_flags |= IP_VS_HDR_INVERSE; 146 147 return ip_vs_fill_iph_skb_off(af, skb, skb_network_offset(skb), 148 hdr_flags, iphdr); 149 } 150 151 static inline bool 152 ip_vs_iph_inverse(const struct ip_vs_iphdr *iph) 153 { 154 return !!(iph->hdr_flags & IP_VS_HDR_INVERSE); 155 } 156 157 static inline bool 158 ip_vs_iph_icmp(const struct ip_vs_iphdr *iph) 159 { 160 return !!(iph->hdr_flags & IP_VS_HDR_ICMP); 161 } 162 163 static inline void ip_vs_addr_copy(int af, union nf_inet_addr *dst, 164 const union nf_inet_addr *src) 165 { 166 #ifdef CONFIG_IP_VS_IPV6 167 if (af == AF_INET6) 168 dst->in6 = src->in6; 169 else 170 #endif 171 dst->ip = src->ip; 172 } 173 174 static inline void ip_vs_addr_set(int af, union nf_inet_addr *dst, 175 const union nf_inet_addr *src) 176 { 177 #ifdef CONFIG_IP_VS_IPV6 178 if (af == AF_INET6) { 179 dst->in6 = src->in6; 180 return; 181 } 182 #endif 183 dst->ip = src->ip; 184 dst->all[1] = 0; 185 dst->all[2] = 0; 186 dst->all[3] = 0; 187 } 188 189 static inline int ip_vs_addr_equal(int af, const union nf_inet_addr *a, 190 const union nf_inet_addr *b) 191 { 192 #ifdef CONFIG_IP_VS_IPV6 193 if (af == AF_INET6) 194 return ipv6_addr_equal(&a->in6, &b->in6); 195 #endif 196 return a->ip == b->ip; 197 } 198 199 #ifdef CONFIG_IP_VS_DEBUG 200 #include <linux/net.h> 201 202 int ip_vs_get_debug_level(void); 203 204 static inline const char *ip_vs_dbg_addr(int af, char *buf, size_t buf_len, 205 const union nf_inet_addr *addr, 206 int *idx) 207 { 208 int len; 209 #ifdef CONFIG_IP_VS_IPV6 210 if (af == AF_INET6) 211 len = snprintf(&buf[*idx], buf_len - *idx, "[%pI6c]", 212 &addr->in6) + 1; 213 else 214 #endif 215 len = snprintf(&buf[*idx], buf_len - *idx, "%pI4", 216 &addr->ip) + 1; 217 218 *idx += len; 219 BUG_ON(*idx > buf_len + 1); 220 return &buf[*idx - len]; 221 } 222 223 #define IP_VS_DBG_BUF(level, msg, ...) \ 224 do { \ 225 char ip_vs_dbg_buf[160]; \ 226 int ip_vs_dbg_idx = 0; \ 227 if (level <= ip_vs_get_debug_level()) \ 228 printk(KERN_DEBUG pr_fmt(msg), ##__VA_ARGS__); \ 229 } while (0) 230 #define IP_VS_ERR_BUF(msg...) \ 231 do { \ 232 char ip_vs_dbg_buf[160]; \ 233 int ip_vs_dbg_idx = 0; \ 234 pr_err(msg); \ 235 } while (0) 236 237 /* Only use from within IP_VS_DBG_BUF() or IP_VS_ERR_BUF macros */ 238 #define IP_VS_DBG_ADDR(af, addr) \ 239 ip_vs_dbg_addr(af, ip_vs_dbg_buf, \ 240 sizeof(ip_vs_dbg_buf), addr, \ 241 &ip_vs_dbg_idx) 242 243 #define IP_VS_DBG(level, msg, ...) \ 244 do { \ 245 if (level <= ip_vs_get_debug_level()) \ 246 printk(KERN_DEBUG pr_fmt(msg), ##__VA_ARGS__); \ 247 } while (0) 248 #define IP_VS_DBG_RL(msg, ...) \ 249 do { \ 250 if (net_ratelimit()) \ 251 printk(KERN_DEBUG pr_fmt(msg), ##__VA_ARGS__); \ 252 } while (0) 253 #define IP_VS_DBG_PKT(level, af, pp, skb, ofs, msg) \ 254 do { \ 255 if (level <= ip_vs_get_debug_level()) \ 256 pp->debug_packet(af, pp, skb, ofs, msg); \ 257 } while (0) 258 #define IP_VS_DBG_RL_PKT(level, af, pp, skb, ofs, msg) \ 259 do { \ 260 if (level <= ip_vs_get_debug_level() && \ 261 net_ratelimit()) \ 262 pp->debug_packet(af, pp, skb, ofs, msg); \ 263 } while (0) 264 #else /* NO DEBUGGING at ALL */ 265 #define IP_VS_DBG_BUF(level, msg...) do {} while (0) 266 #define IP_VS_ERR_BUF(msg...) do {} while (0) 267 #define IP_VS_DBG(level, msg...) do {} while (0) 268 #define IP_VS_DBG_RL(msg...) do {} while (0) 269 #define IP_VS_DBG_PKT(level, af, pp, skb, ofs, msg) do {} while (0) 270 #define IP_VS_DBG_RL_PKT(level, af, pp, skb, ofs, msg) do {} while (0) 271 #endif 272 273 #define IP_VS_BUG() BUG() 274 #define IP_VS_ERR_RL(msg, ...) \ 275 do { \ 276 if (net_ratelimit()) \ 277 pr_err(msg, ##__VA_ARGS__); \ 278 } while (0) 279 280 struct ip_vs_aligned_lock { 281 spinlock_t l; /* Protect buckets */ 282 } ____cacheline_aligned_in_smp; 283 284 /* For arrays per family */ 285 enum { 286 IP_VS_AF_INET, 287 IP_VS_AF_INET6, 288 IP_VS_AF_MAX 289 }; 290 291 static inline int ip_vs_af_index(int af) 292 { 293 return af == AF_INET6 ? IP_VS_AF_INET6 : IP_VS_AF_INET; 294 } 295 296 /* work_flags */ 297 enum { 298 IP_VS_WORK_SVC_RESIZE, /* Schedule svc_resize_work */ 299 IP_VS_WORK_SVC_NORESIZE, /* Stopping svc_resize_work */ 300 IP_VS_WORK_CONN_RESIZE, /* Schedule conn_resize_work */ 301 }; 302 303 /* The port number of FTP service (in network order). */ 304 #define FTPPORT cpu_to_be16(21) 305 #define FTPDATA cpu_to_be16(20) 306 307 /* TCP State Values */ 308 enum { 309 IP_VS_TCP_S_NONE = 0, 310 IP_VS_TCP_S_ESTABLISHED, 311 IP_VS_TCP_S_SYN_SENT, 312 IP_VS_TCP_S_SYN_RECV, 313 IP_VS_TCP_S_FIN_WAIT, 314 IP_VS_TCP_S_TIME_WAIT, 315 IP_VS_TCP_S_CLOSE, 316 IP_VS_TCP_S_CLOSE_WAIT, 317 IP_VS_TCP_S_LAST_ACK, 318 IP_VS_TCP_S_LISTEN, 319 IP_VS_TCP_S_SYNACK, 320 IP_VS_TCP_S_LAST 321 }; 322 323 /* UDP State Values */ 324 enum { 325 IP_VS_UDP_S_NORMAL, 326 IP_VS_UDP_S_LAST, 327 }; 328 329 /* ICMP State Values */ 330 enum { 331 IP_VS_ICMP_S_NORMAL, 332 IP_VS_ICMP_S_LAST, 333 }; 334 335 /* SCTP State Values */ 336 enum ip_vs_sctp_states { 337 IP_VS_SCTP_S_NONE, 338 IP_VS_SCTP_S_INIT1, 339 IP_VS_SCTP_S_INIT, 340 IP_VS_SCTP_S_COOKIE_SENT, 341 IP_VS_SCTP_S_COOKIE_REPLIED, 342 IP_VS_SCTP_S_COOKIE_WAIT, 343 IP_VS_SCTP_S_COOKIE, 344 IP_VS_SCTP_S_COOKIE_ECHOED, 345 IP_VS_SCTP_S_ESTABLISHED, 346 IP_VS_SCTP_S_SHUTDOWN_SENT, 347 IP_VS_SCTP_S_SHUTDOWN_RECEIVED, 348 IP_VS_SCTP_S_SHUTDOWN_ACK_SENT, 349 IP_VS_SCTP_S_REJECTED, 350 IP_VS_SCTP_S_CLOSED, 351 IP_VS_SCTP_S_LAST 352 }; 353 354 /* Connection templates use bits from state */ 355 #define IP_VS_CTPL_S_NONE 0x0000 356 #define IP_VS_CTPL_S_ASSURED 0x0001 357 #define IP_VS_CTPL_S_LAST 0x0002 358 359 /* Delta sequence info structure 360 * Each ip_vs_conn has 2 (output AND input seq. changes). 361 * Only used in the VS/NAT. 362 */ 363 struct ip_vs_seq { 364 __u32 init_seq; /* Add delta from this seq */ 365 __u32 delta; /* Delta in sequence numbers */ 366 __u32 previous_delta; /* Delta in sequence numbers 367 * before last resized pkt */ 368 }; 369 370 /* counters per cpu */ 371 struct ip_vs_counters { 372 u64_stats_t conns; /* connections scheduled */ 373 u64_stats_t inpkts; /* incoming packets */ 374 u64_stats_t outpkts; /* outgoing packets */ 375 u64_stats_t inbytes; /* incoming bytes */ 376 u64_stats_t outbytes; /* outgoing bytes */ 377 }; 378 /* Stats per cpu */ 379 struct ip_vs_cpu_stats { 380 struct ip_vs_counters cnt; 381 struct u64_stats_sync syncp; 382 }; 383 384 /* Default nice for estimator kthreads */ 385 #define IPVS_EST_NICE 0 386 387 /* IPVS statistics objects */ 388 struct ip_vs_estimator { 389 struct hlist_node list; 390 391 u64 last_inbytes; 392 u64 last_outbytes; 393 u64 last_conns; 394 u64 last_inpkts; 395 u64 last_outpkts; 396 397 u64 cps; 398 u64 inpps; 399 u64 outpps; 400 u64 inbps; 401 u64 outbps; 402 403 s32 ktid:16, /* kthread ID, -1=temp list */ 404 ktrow:8, /* row/tick ID for kthread */ 405 ktcid:8; /* chain ID for kthread tick */ 406 }; 407 408 /* 409 * IPVS statistics object, 64-bit kernel version of struct ip_vs_stats_user 410 */ 411 struct ip_vs_kstats { 412 u64 conns; /* connections scheduled */ 413 u64 inpkts; /* incoming packets */ 414 u64 outpkts; /* outgoing packets */ 415 u64 inbytes; /* incoming bytes */ 416 u64 outbytes; /* outgoing bytes */ 417 418 u64 cps; /* current connection rate */ 419 u64 inpps; /* current in packet rate */ 420 u64 outpps; /* current out packet rate */ 421 u64 inbps; /* current in byte rate */ 422 u64 outbps; /* current out byte rate */ 423 }; 424 425 struct ip_vs_stats { 426 struct ip_vs_kstats kstats; /* kernel statistics */ 427 struct ip_vs_estimator est; /* estimator */ 428 struct ip_vs_cpu_stats __percpu *cpustats; /* per cpu counters */ 429 spinlock_t lock; /* spin lock */ 430 struct ip_vs_kstats kstats0; /* reset values */ 431 }; 432 433 struct ip_vs_stats_rcu { 434 struct ip_vs_stats s; 435 struct rcu_head rcu_head; 436 }; 437 438 int ip_vs_stats_init_alloc(struct ip_vs_stats *s); 439 struct ip_vs_stats *ip_vs_stats_alloc(void); 440 void ip_vs_stats_release(struct ip_vs_stats *stats); 441 void ip_vs_stats_free(struct ip_vs_stats *stats); 442 443 /* Process estimators in multiple timer ticks (20/50/100, see ktrow) */ 444 #define IPVS_EST_NTICKS 50 445 /* Estimation uses a 2-second period containing ticks (in jiffies) */ 446 #define IPVS_EST_TICK ((2 * HZ) / IPVS_EST_NTICKS) 447 448 /* Limit of CPU load per kthread (8 for 12.5%), ratio of CPU capacity (1/C). 449 * Value of 4 and above ensures kthreads will take work without exceeding 450 * the CPU capacity under different circumstances. 451 */ 452 #define IPVS_EST_LOAD_DIVISOR 8 453 454 /* Kthreads should not have work that exceeds the CPU load above 50% */ 455 #define IPVS_EST_CPU_KTHREADS (IPVS_EST_LOAD_DIVISOR / 2) 456 457 /* Desired number of chains per timer tick (chain load factor in 100us units), 458 * 48=4.8ms of 40ms tick (12% CPU usage): 459 * 2 sec * 1000 ms in sec * 10 (100us in ms) / 8 (12.5%) / 50 460 */ 461 #define IPVS_EST_CHAIN_FACTOR \ 462 ALIGN_DOWN(2 * 1000 * 10 / IPVS_EST_LOAD_DIVISOR / IPVS_EST_NTICKS, 8) 463 464 /* Compiled number of chains per tick 465 * The defines should match cond_resched_rcu 466 */ 467 #if defined(CONFIG_DEBUG_ATOMIC_SLEEP) || !defined(CONFIG_PREEMPT_RCU) 468 #define IPVS_EST_TICK_CHAINS IPVS_EST_CHAIN_FACTOR 469 #else 470 #define IPVS_EST_TICK_CHAINS 1 471 #endif 472 473 #if IPVS_EST_NTICKS > 127 474 #error Too many timer ticks for ktrow 475 #endif 476 477 /* Multiple chains processed in same tick */ 478 struct ip_vs_est_tick_data { 479 struct rcu_head rcu_head; 480 struct hlist_head chains[IPVS_EST_TICK_CHAINS]; 481 DECLARE_BITMAP(present, IPVS_EST_TICK_CHAINS); 482 DECLARE_BITMAP(full, IPVS_EST_TICK_CHAINS); 483 int chain_len[IPVS_EST_TICK_CHAINS]; 484 }; 485 486 /* Context for estimation kthread */ 487 struct ip_vs_est_kt_data { 488 struct netns_ipvs *ipvs; 489 struct task_struct *task; /* task if running */ 490 struct ip_vs_est_tick_data __rcu *ticks[IPVS_EST_NTICKS]; 491 DECLARE_BITMAP(avail, IPVS_EST_NTICKS); /* tick has space for ests */ 492 unsigned long est_timer; /* estimation timer (jiffies) */ 493 struct ip_vs_stats *calc_stats; /* Used for calculation */ 494 int needed; /* task is needed */ 495 int tick_len[IPVS_EST_NTICKS]; /* est count */ 496 int id; /* ktid per netns */ 497 int chain_max; /* max ests per tick chain */ 498 int tick_max; /* max ests per tick */ 499 int est_count; /* attached ests to kthread */ 500 int est_max_count; /* max ests per kthread */ 501 int add_row; /* row for new ests */ 502 int est_row; /* estimated row */ 503 }; 504 505 /* IPVS resizable hash tables */ 506 struct ip_vs_rht { 507 struct hlist_bl_head *buckets; 508 struct ip_vs_rht __rcu *new_tbl; /* New/Same table */ 509 seqcount_t *seqc; /* Protects moves */ 510 struct ip_vs_aligned_lock *lock; /* Protect seqc */ 511 int mask; /* Buckets mask */ 512 int size; /* Buckets */ 513 int seqc_mask; /* seqc mask */ 514 int lock_mask; /* lock mask */ 515 u32 table_id; 516 int u_thresh; /* upper threshold */ 517 int l_thresh; /* lower threshold */ 518 int lfactor; /* Load Factor (shift)*/ 519 int bits; /* size = 1 << bits */ 520 siphash_key_t hash_key; 521 struct rcu_head rcu_head; 522 }; 523 524 /** 525 * ip_vs_rht_for_each_table() - Walk the hash tables 526 * @table: struct ip_vs_rht __rcu *table 527 * @t: current table, used as cursor, struct ip_vs_rht *var 528 * @p: previous table, temp struct ip_vs_rht *var 529 * 530 * Walk tables assuming others can not change the installed tables 531 */ 532 #define ip_vs_rht_for_each_table(table, t, p) \ 533 for (p = NULL, t = rcu_dereference_protected(table, 1); \ 534 t != p; \ 535 p = t, t = rcu_dereference_protected(t->new_tbl, 1)) 536 537 /** 538 * ip_vs_rht_for_each_table_rcu() - Walk the hash tables under RCU reader lock 539 * @table: struct ip_vs_rht __rcu *table 540 * @t: current table, used as cursor, struct ip_vs_rht *var 541 * @p: previous table, temp struct ip_vs_rht *var 542 * 543 * We usually search in one table and also in second table on resizing 544 */ 545 #define ip_vs_rht_for_each_table_rcu(table, t, p) \ 546 for (p = NULL, t = rcu_dereference(table); \ 547 t != p; \ 548 p = t, t = rcu_dereference(t->new_tbl)) 549 550 /** 551 * ip_vs_rht_for_each_bucket() - Walk all table buckets 552 * @t: current table, used as cursor, struct ip_vs_rht *var 553 * @bucket: bucket index, used as cursor, u32 var 554 * @head: bucket address, used as cursor, struct hlist_bl_head *var 555 */ 556 #define ip_vs_rht_for_each_bucket(t, bucket, head) \ 557 for (bucket = 0, head = (t)->buckets; \ 558 bucket < t->size; bucket++, head++) 559 560 /** 561 * ip_vs_rht_for_bucket_retry() - Retry bucket if entries are moved 562 * @t: current table, used as cursor, struct ip_vs_rht *var 563 * @bucket: index of current bucket or hash key 564 * @sc: temp seqcount_t *var 565 * @seq: temp unsigned int var for sequence count 566 * @retry: temp int var 567 */ 568 #define ip_vs_rht_for_bucket_retry(t, bucket, sc, seq, retry) \ 569 for (retry = 1, sc = &(t)->seqc[(bucket) & (t)->seqc_mask]; \ 570 retry && ({ seq = read_seqcount_begin(sc); 1; }); \ 571 retry = read_seqcount_retry(sc, seq)) 572 573 /** 574 * DECLARE_IP_VS_RHT_WALK_BUCKETS_RCU() - Declare variables 575 * 576 * Variables for ip_vs_rht_walk_buckets_rcu 577 */ 578 #define DECLARE_IP_VS_RHT_WALK_BUCKETS_RCU() \ 579 struct ip_vs_rht *_t, *_p; \ 580 unsigned int _seq; \ 581 seqcount_t *_sc; \ 582 u32 _bucket; \ 583 int _retry 584 /** 585 * ip_vs_rht_walk_buckets_rcu() - Walk all buckets under RCU read lock 586 * @table: struct ip_vs_rht __rcu *table 587 * @head: bucket address, used as cursor, struct hlist_bl_head *var 588 * 589 * Can be used while others add/delete/move entries 590 * Not suitable if duplicates are not desired 591 * Possible cases for reader that uses cond_resched_rcu() in the loop: 592 * - new table can not be installed, no need to repeat 593 * - new table can be installed => check and repeat if new table is 594 * installed, needed for !PREEMPT_RCU 595 */ 596 #define ip_vs_rht_walk_buckets_rcu(table, head) \ 597 ip_vs_rht_for_each_table_rcu(table, _t, _p) \ 598 ip_vs_rht_for_each_bucket(_t, _bucket, head) \ 599 ip_vs_rht_for_bucket_retry(_t, _bucket, _sc, \ 600 _seq, _retry) 601 602 /** 603 * DECLARE_IP_VS_RHT_WALK_BUCKET_RCU() - Declare variables 604 * 605 * Variables for ip_vs_rht_walk_bucket_rcu 606 */ 607 #define DECLARE_IP_VS_RHT_WALK_BUCKET_RCU() \ 608 unsigned int _seq; \ 609 seqcount_t *_sc; \ 610 int _retry 611 /** 612 * ip_vs_rht_walk_bucket_rcu() - Walk bucket under RCU read lock 613 * @t: current table, struct ip_vs_rht *var 614 * @bucket: index of current bucket or hash key 615 * @head: bucket address, used as cursor, struct hlist_bl_head *var 616 * 617 * Can be used while others add/delete/move entries 618 * Not suitable if duplicates are not desired 619 * Possible cases for reader that uses cond_resched_rcu() in the loop: 620 * - new table can not be installed, no need to repeat 621 * - new table can be installed => check and repeat if new table is 622 * installed, needed for !PREEMPT_RCU 623 */ 624 #define ip_vs_rht_walk_bucket_rcu(t, bucket, head) \ 625 if (({ head = (t)->buckets + ((bucket) & (t)->mask); 0; })) \ 626 {} \ 627 else \ 628 ip_vs_rht_for_bucket_retry(t, (bucket), _sc, _seq, _retry) 629 630 /** 631 * DECLARE_IP_VS_RHT_WALK_BUCKETS_SAFE_RCU() - Declare variables 632 * 633 * Variables for ip_vs_rht_walk_buckets_safe_rcu 634 */ 635 #define DECLARE_IP_VS_RHT_WALK_BUCKETS_SAFE_RCU() \ 636 struct ip_vs_rht *_t, *_p; \ 637 u32 _bucket 638 /** 639 * ip_vs_rht_walk_buckets_safe_rcu() - Walk all buckets under RCU read lock 640 * @table: struct ip_vs_rht __rcu *table 641 * @head: bucket address, used as cursor, struct hlist_bl_head *var 642 * 643 * Can be used while others add/delete entries but moving is disabled 644 * Using cond_resched_rcu() should be safe if tables do not change 645 */ 646 #define ip_vs_rht_walk_buckets_safe_rcu(table, head) \ 647 ip_vs_rht_for_each_table_rcu(table, _t, _p) \ 648 ip_vs_rht_for_each_bucket(_t, _bucket, head) 649 650 /** 651 * DECLARE_IP_VS_RHT_WALK_BUCKETS() - Declare variables 652 * 653 * Variables for ip_vs_rht_walk_buckets 654 */ 655 #define DECLARE_IP_VS_RHT_WALK_BUCKETS() \ 656 struct ip_vs_rht *_t, *_p; \ 657 u32 _bucket 658 659 /** 660 * ip_vs_rht_walk_buckets() - Walk all buckets 661 * @table: struct ip_vs_rht __rcu *table 662 * @head: bucket address, used as cursor, struct hlist_bl_head *var 663 * 664 * Use if others can not add/delete/move entries 665 */ 666 #define ip_vs_rht_walk_buckets(table, head) \ 667 ip_vs_rht_for_each_table(table, _t, _p) \ 668 ip_vs_rht_for_each_bucket(_t, _bucket, head) 669 670 /* Entries can be in one of two tables, so we flip bit when new table is 671 * created and store it as highest bit in hash keys 672 */ 673 #define IP_VS_RHT_TABLE_ID_MASK BIT(31) 674 675 /* Check if hash key is from this table */ 676 static inline bool ip_vs_rht_same_table(struct ip_vs_rht *t, u32 hash_key) 677 { 678 return !((t->table_id ^ hash_key) & IP_VS_RHT_TABLE_ID_MASK); 679 } 680 681 /* Build per-table hash key from hash value */ 682 static inline u32 ip_vs_rht_build_hash_key(struct ip_vs_rht *t, u32 hash) 683 { 684 return t->table_id | (hash & ~IP_VS_RHT_TABLE_ID_MASK); 685 } 686 687 void ip_vs_rht_free(struct ip_vs_rht *t); 688 void ip_vs_rht_rcu_free(struct rcu_head *head); 689 struct ip_vs_rht *ip_vs_rht_alloc(int buckets, int scounts, int locks); 690 int ip_vs_rht_desired_size(struct netns_ipvs *ipvs, struct ip_vs_rht *t, int n, 691 int lfactor, int min_bits, int max_bits); 692 void ip_vs_rht_set_thresholds(struct ip_vs_rht *t, int size, int lfactor, 693 int min_bits, int max_bits); 694 u32 ip_vs_rht_hash_linfo(struct ip_vs_rht *t, int af, 695 const union nf_inet_addr *addr, u32 v1, u32 v2); 696 697 struct dst_entry; 698 struct iphdr; 699 struct ip_vs_conn; 700 struct ip_vs_app; 701 struct sk_buff; 702 struct ip_vs_proto_data; 703 704 struct ip_vs_protocol { 705 struct ip_vs_protocol *next; 706 char *name; 707 u16 protocol; 708 u16 num_states; 709 int dont_defrag; 710 711 void (*init)(struct ip_vs_protocol *pp); 712 713 void (*exit)(struct ip_vs_protocol *pp); 714 715 int (*init_netns)(struct netns_ipvs *ipvs, struct ip_vs_proto_data *pd); 716 717 void (*exit_netns)(struct netns_ipvs *ipvs, struct ip_vs_proto_data *pd); 718 719 int (*conn_schedule)(struct netns_ipvs *ipvs, 720 int af, struct sk_buff *skb, 721 struct ip_vs_proto_data *pd, 722 int *verdict, struct ip_vs_conn **cpp, 723 struct ip_vs_iphdr *iph); 724 725 struct ip_vs_conn * 726 (*conn_in_get)(struct netns_ipvs *ipvs, 727 int af, 728 const struct sk_buff *skb, 729 const struct ip_vs_iphdr *iph); 730 731 struct ip_vs_conn * 732 (*conn_out_get)(struct netns_ipvs *ipvs, 733 int af, 734 const struct sk_buff *skb, 735 const struct ip_vs_iphdr *iph); 736 737 int (*snat_handler)(struct sk_buff *skb, struct ip_vs_protocol *pp, 738 struct ip_vs_conn *cp, struct ip_vs_iphdr *iph); 739 740 int (*dnat_handler)(struct sk_buff *skb, struct ip_vs_protocol *pp, 741 struct ip_vs_conn *cp, struct ip_vs_iphdr *iph); 742 743 const char *(*state_name)(int state); 744 745 void (*state_transition)(struct ip_vs_conn *cp, int direction, 746 const struct sk_buff *skb, 747 struct ip_vs_proto_data *pd); 748 749 int (*register_app)(struct netns_ipvs *ipvs, struct ip_vs_app *inc); 750 751 void (*unregister_app)(struct netns_ipvs *ipvs, struct ip_vs_app *inc); 752 753 int (*app_conn_bind)(struct ip_vs_conn *cp); 754 755 void (*debug_packet)(int af, struct ip_vs_protocol *pp, 756 const struct sk_buff *skb, 757 int offset, 758 const char *msg); 759 760 void (*timeout_change)(struct ip_vs_proto_data *pd, int flags); 761 }; 762 763 /* protocol data per netns */ 764 struct ip_vs_proto_data { 765 struct ip_vs_proto_data *next; 766 struct ip_vs_protocol *pp; 767 int *timeout_table; /* protocol timeout table */ 768 atomic_t appcnt; /* counter of proto app incs. */ 769 struct tcp_states_t *tcp_state_table; 770 }; 771 772 struct ip_vs_protocol *ip_vs_proto_get(unsigned short proto); 773 struct ip_vs_proto_data *ip_vs_proto_data_get(struct netns_ipvs *ipvs, 774 unsigned short proto); 775 776 struct ip_vs_conn_param { 777 struct netns_ipvs *ipvs; 778 const union nf_inet_addr *caddr; 779 const union nf_inet_addr *vaddr; 780 __be16 cport; 781 __be16 vport; 782 __u16 protocol; 783 u16 af; 784 785 const struct ip_vs_pe *pe; 786 char *pe_data; 787 __u8 pe_data_len; 788 }; 789 790 /* Hash node in conn_tab */ 791 struct ip_vs_conn_hnode { 792 struct hlist_bl_node node; /* node in conn_tab */ 793 u32 hash_key; /* Key for the hash table */ 794 u8 dir; /* 0=out->in, 1=in->out */ 795 } __packed; 796 797 /* IP_VS structure allocated for each dynamically scheduled connection */ 798 struct ip_vs_conn { 799 /* Cacheline for hash table nodes - rarely modified */ 800 801 struct ip_vs_conn_hnode hn0; /* Original direction */ 802 u8 af; /* address family */ 803 __be16 cport; 804 struct ip_vs_conn_hnode hn1; /* Reply direction */ 805 u8 daf; /* Address family of the dest */ 806 __be16 dport; 807 struct ip_vs_dest *dest; /* real server */ 808 atomic_t n_control; /* Number of controlled ones */ 809 volatile __u32 flags; /* status flags */ 810 /* 44/64 */ 811 812 struct ip_vs_conn *control; /* Master control connection */ 813 const struct ip_vs_pe *pe; 814 char *pe_data; 815 __u8 pe_data_len; 816 volatile __u16 state; /* state info */ 817 volatile __u16 old_state; /* old state, to be used for 818 * state transition triggered 819 * synchronization 820 */ 821 /* 2-byte hole */ 822 /* 64/96 */ 823 824 union nf_inet_addr caddr; /* client address */ 825 union nf_inet_addr vaddr; /* virtual address */ 826 /* 96/128 */ 827 828 union nf_inet_addr daddr; /* destination address */ 829 __u32 fwmark; /* Fire wall mark from skb */ 830 __be16 vport; 831 __u16 protocol; /* Which protocol (TCP/UDP) */ 832 833 /* Note: we can group the following members into a structure, 834 * in order to save more space, and the following members are 835 * only used in VS/NAT anyway 836 */ 837 struct ip_vs_app *app; /* bound ip_vs_app object */ 838 void *app_data; /* Application private data */ 839 /* 128/168 */ 840 struct_group(sync_conn_opt, 841 struct ip_vs_seq in_seq; /* incoming seq. struct */ 842 struct ip_vs_seq out_seq; /* outgoing seq. struct */ 843 ); 844 /* 152/192 */ 845 846 struct timer_list timer; /* Expiration timer */ 847 volatile unsigned long timeout; /* timeout */ 848 spinlock_t lock; /* lock for state transition */ 849 refcount_t refcnt; /* reference count */ 850 atomic_t in_pkts; /* incoming packet counter */ 851 /* 64-bit: 4-byte gap */ 852 853 /* 188/256 */ 854 unsigned long sync_endtime; /* jiffies + sent_retries */ 855 struct netns_ipvs *ipvs; 856 857 /* Packet transmitter for different forwarding methods. If it 858 * mangles the packet, it must return NF_DROP or better NF_STOLEN, 859 * otherwise this must be changed to a sk_buff **. 860 * NF_ACCEPT can be returned when destination is local. 861 */ 862 int (*packet_xmit)(struct sk_buff *skb, struct ip_vs_conn *cp, 863 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 864 865 struct rcu_head rcu_head; 866 }; 867 868 /* Extended internal versions of struct ip_vs_service_user and ip_vs_dest_user 869 * for IPv6 support. 870 * 871 * We need these to conveniently pass around service and destination 872 * options, but unfortunately, we also need to keep the old definitions to 873 * maintain userspace backwards compatibility for the setsockopt interface. 874 */ 875 struct ip_vs_service_user_kern { 876 /* virtual service addresses */ 877 u16 af; 878 u16 protocol; 879 union nf_inet_addr addr; /* virtual ip address */ 880 __be16 port; 881 u32 fwmark; /* firewall mark of service */ 882 883 /* virtual service options */ 884 char *sched_name; 885 char *pe_name; 886 unsigned int flags; /* virtual service flags */ 887 unsigned int timeout; /* persistent timeout in sec */ 888 __be32 netmask; /* persistent netmask or plen */ 889 }; 890 891 892 struct ip_vs_dest_user_kern { 893 /* destination server address */ 894 union nf_inet_addr addr; 895 __be16 port; 896 897 /* real server options */ 898 unsigned int conn_flags; /* connection flags */ 899 int weight; /* destination weight */ 900 901 /* thresholds for active connections */ 902 u32 u_threshold; /* upper threshold */ 903 u32 l_threshold; /* lower threshold */ 904 905 /* Address family of addr */ 906 u16 af; 907 908 u16 tun_type; /* tunnel type */ 909 __be16 tun_port; /* tunnel port */ 910 u16 tun_flags; /* tunnel flags */ 911 }; 912 913 914 /* 915 * The information about the virtual service offered to the net and the 916 * forwarding entries. 917 */ 918 struct ip_vs_service { 919 struct hlist_bl_node s_list; /* node in service table */ 920 u32 hash_key; /* Key for the hash table */ 921 u16 af; /* address family */ 922 __u16 protocol; /* which protocol (TCP/UDP) */ 923 924 union nf_inet_addr addr; /* IP address for virtual service */ 925 __u32 fwmark; /* firewall mark of the service */ 926 atomic_t refcnt; /* reference counter */ 927 __be16 port; /* port number for the service */ 928 unsigned int flags; /* service status flags */ 929 unsigned int timeout; /* persistent timeout in ticks */ 930 __be32 netmask; /* grouping granularity, mask/plen */ 931 struct netns_ipvs *ipvs; 932 933 struct list_head destinations; /* real server d-linked list */ 934 __u32 num_dests; /* number of servers */ 935 struct ip_vs_stats stats; /* statistics for the service */ 936 937 /* for scheduling */ 938 struct ip_vs_scheduler __rcu *scheduler; /* bound scheduler object */ 939 spinlock_t sched_lock; /* lock sched_data */ 940 void *sched_data; /* scheduler application data */ 941 942 /* alternate persistence engine */ 943 struct ip_vs_pe __rcu *pe; 944 int conntrack_afmask; 945 946 struct rcu_head rcu_head; 947 }; 948 949 /* Information for cached dst */ 950 struct ip_vs_dest_dst { 951 struct dst_entry *dst_cache; /* destination cache entry */ 952 u32 dst_cookie; 953 union nf_inet_addr dst_saddr; 954 struct rcu_head rcu_head; 955 }; 956 957 /* The real server destination forwarding entry with ip address, port number, 958 * and so on. 959 */ 960 struct ip_vs_dest { 961 struct list_head n_list; /* for the dests in the service */ 962 struct hlist_node d_list; /* for table with all the dests */ 963 964 u16 af; /* address family */ 965 __be16 port; /* port number of the server */ 966 union nf_inet_addr addr; /* IP address of the server */ 967 volatile unsigned int flags; /* dest status flags */ 968 atomic_t conn_flags; /* flags to copy to conn */ 969 atomic_t weight; /* server weight */ 970 atomic_t last_weight; /* server latest weight */ 971 __u16 tun_type; /* tunnel type */ 972 __be16 tun_port; /* tunnel port */ 973 __u16 tun_flags; /* tunnel flags */ 974 975 refcount_t refcnt; /* reference counter */ 976 struct ip_vs_stats stats; /* statistics */ 977 unsigned long idle_start; /* start time, jiffies */ 978 979 /* connection counters and thresholds */ 980 atomic_t activeconns; /* active connections */ 981 atomic_t inactconns; /* inactive connections */ 982 atomic_t persistconns; /* persistent connections */ 983 __u32 u_threshold; /* upper threshold */ 984 __u32 l_threshold; /* lower threshold */ 985 986 /* for destination cache */ 987 spinlock_t dst_lock; /* lock of dst_cache */ 988 struct ip_vs_dest_dst __rcu *dest_dst; /* cached dst info */ 989 990 /* for virtual service */ 991 struct ip_vs_service __rcu *svc; /* service it belongs to */ 992 __u16 protocol; /* which protocol (TCP/UDP) */ 993 __be16 vport; /* virtual port number */ 994 union nf_inet_addr vaddr; /* virtual IP address */ 995 __u32 vfwmark; /* firewall mark of service */ 996 997 struct rcu_head rcu_head; 998 struct list_head t_list; /* in dest_trash */ 999 unsigned int in_rs_table:1; /* we are in rs_table */ 1000 }; 1001 1002 /* The scheduler object */ 1003 struct ip_vs_scheduler { 1004 struct list_head n_list; /* d-linked list head */ 1005 char *name; /* scheduler name */ 1006 atomic_t refcnt; /* reference counter */ 1007 struct module *module; /* THIS_MODULE/NULL */ 1008 1009 /* scheduler initializing service */ 1010 int (*init_service)(struct ip_vs_service *svc); 1011 /* scheduling service finish */ 1012 void (*done_service)(struct ip_vs_service *svc); 1013 /* dest is linked */ 1014 int (*add_dest)(struct ip_vs_service *svc, struct ip_vs_dest *dest); 1015 /* dest is unlinked */ 1016 int (*del_dest)(struct ip_vs_service *svc, struct ip_vs_dest *dest); 1017 /* dest is updated */ 1018 int (*upd_dest)(struct ip_vs_service *svc, struct ip_vs_dest *dest); 1019 1020 /* selecting a server from the given service */ 1021 struct ip_vs_dest* (*schedule)(struct ip_vs_service *svc, 1022 const struct sk_buff *skb, 1023 struct ip_vs_iphdr *iph); 1024 }; 1025 1026 /* The persistence engine object */ 1027 struct ip_vs_pe { 1028 struct list_head n_list; /* d-linked list head */ 1029 char *name; /* scheduler name */ 1030 atomic_t refcnt; /* reference counter */ 1031 struct module *module; /* THIS_MODULE/NULL */ 1032 1033 /* get the connection template, if any */ 1034 int (*fill_param)(struct ip_vs_conn_param *p, struct sk_buff *skb); 1035 bool (*ct_match)(const struct ip_vs_conn_param *p, 1036 struct ip_vs_conn *ct); 1037 u32 (*hashkey_raw)(const struct ip_vs_conn_param *p, 1038 struct ip_vs_rht *t, bool inverse); 1039 int (*show_pe_data)(const struct ip_vs_conn *cp, char *buf); 1040 /* create connections for real-server outgoing packets */ 1041 struct ip_vs_conn* (*conn_out)(struct ip_vs_service *svc, 1042 struct ip_vs_dest *dest, 1043 struct sk_buff *skb, 1044 const struct ip_vs_iphdr *iph, 1045 __be16 dport, __be16 cport); 1046 }; 1047 1048 /* The application module object (a.k.a. app incarnation) */ 1049 struct ip_vs_app { 1050 struct list_head a_list; /* member in app list */ 1051 int type; /* IP_VS_APP_TYPE_xxx */ 1052 char *name; /* application module name */ 1053 __u16 protocol; 1054 struct module *module; /* THIS_MODULE/NULL */ 1055 struct list_head incs_list; /* list of incarnations */ 1056 1057 /* members for application incarnations */ 1058 struct list_head p_list; /* member in proto app list */ 1059 struct ip_vs_app *app; /* its real application */ 1060 __be16 port; /* port number in net order */ 1061 atomic_t usecnt; /* usage counter */ 1062 struct rcu_head rcu_head; 1063 1064 /* output hook: Process packet in inout direction, diff set for TCP. 1065 * Return: 0=Error, 1=Payload Not Mangled/Mangled but checksum is ok, 1066 * 2=Mangled but checksum was not updated 1067 */ 1068 int (*pkt_out)(struct ip_vs_app *, struct ip_vs_conn *, 1069 struct sk_buff *, int *diff, struct ip_vs_iphdr *ipvsh); 1070 1071 /* input hook: Process packet in outin direction, diff set for TCP. 1072 * Return: 0=Error, 1=Payload Not Mangled/Mangled but checksum is ok, 1073 * 2=Mangled but checksum was not updated 1074 */ 1075 int (*pkt_in)(struct ip_vs_app *, struct ip_vs_conn *, 1076 struct sk_buff *, int *diff, struct ip_vs_iphdr *ipvsh); 1077 1078 /* ip_vs_app initializer */ 1079 int (*init_conn)(struct ip_vs_app *, struct ip_vs_conn *); 1080 1081 /* ip_vs_app finish */ 1082 int (*done_conn)(struct ip_vs_app *, struct ip_vs_conn *); 1083 1084 1085 /* not used now */ 1086 int (*bind_conn)(struct ip_vs_app *, struct ip_vs_conn *, 1087 struct ip_vs_protocol *); 1088 1089 void (*unbind_conn)(struct ip_vs_app *, struct ip_vs_conn *); 1090 1091 int * timeout_table; 1092 int * timeouts; 1093 int timeouts_size; 1094 1095 int (*conn_schedule)(struct sk_buff *skb, struct ip_vs_app *app, 1096 int *verdict, struct ip_vs_conn **cpp); 1097 1098 struct ip_vs_conn * 1099 (*conn_in_get)(const struct sk_buff *skb, struct ip_vs_app *app, 1100 const struct iphdr *iph, int inverse); 1101 1102 struct ip_vs_conn * 1103 (*conn_out_get)(const struct sk_buff *skb, struct ip_vs_app *app, 1104 const struct iphdr *iph, int inverse); 1105 1106 int (*state_transition)(struct ip_vs_conn *cp, int direction, 1107 const struct sk_buff *skb, 1108 struct ip_vs_app *app); 1109 1110 void (*timeout_change)(struct ip_vs_app *app, int flags); 1111 }; 1112 1113 struct ipvs_master_sync_state { 1114 struct list_head sync_queue; 1115 struct ip_vs_sync_buff *sync_buff; 1116 unsigned long sync_queue_len; 1117 unsigned int sync_queue_delay; 1118 struct delayed_work master_wakeup_work; 1119 struct netns_ipvs *ipvs; 1120 }; 1121 1122 struct ip_vs_sync_thread_data; 1123 1124 /* How much time to keep dests in trash */ 1125 #define IP_VS_DEST_TRASH_PERIOD (120 * HZ) 1126 1127 struct ipvs_sync_daemon_cfg { 1128 union nf_inet_addr mcast_group; 1129 int syncid; 1130 u16 sync_maxlen; 1131 u16 mcast_port; 1132 u8 mcast_af; 1133 u8 mcast_ttl; 1134 /* multicast interface name */ 1135 char mcast_ifn[IP_VS_IFNAME_MAXLEN]; 1136 }; 1137 1138 /* IPVS in network namespace */ 1139 struct netns_ipvs { 1140 int gen; /* Generation */ 1141 int enable; /* enable like nf_hooks do */ 1142 /* Hash table: for real service lookups */ 1143 #define IP_VS_RTAB_BITS 4 1144 #define IP_VS_RTAB_SIZE (1 << IP_VS_RTAB_BITS) 1145 #define IP_VS_RTAB_MASK (IP_VS_RTAB_SIZE - 1) 1146 1147 struct hlist_head rs_table[IP_VS_RTAB_SIZE]; 1148 /* ip_vs_app */ 1149 struct list_head app_list; 1150 /* ip_vs_proto */ 1151 #define IP_VS_PROTO_TAB_SIZE 32 /* must be power of 2 */ 1152 struct ip_vs_proto_data *proto_data_table[IP_VS_PROTO_TAB_SIZE]; 1153 /* ip_vs_proto_tcp */ 1154 #ifdef CONFIG_IP_VS_PROTO_TCP 1155 #define TCP_APP_TAB_BITS 4 1156 #define TCP_APP_TAB_SIZE (1 << TCP_APP_TAB_BITS) 1157 #define TCP_APP_TAB_MASK (TCP_APP_TAB_SIZE - 1) 1158 struct list_head tcp_apps[TCP_APP_TAB_SIZE]; 1159 #endif 1160 /* ip_vs_proto_udp */ 1161 #ifdef CONFIG_IP_VS_PROTO_UDP 1162 #define UDP_APP_TAB_BITS 4 1163 #define UDP_APP_TAB_SIZE (1 << UDP_APP_TAB_BITS) 1164 #define UDP_APP_TAB_MASK (UDP_APP_TAB_SIZE - 1) 1165 struct list_head udp_apps[UDP_APP_TAB_SIZE]; 1166 #endif 1167 /* ip_vs_proto_sctp */ 1168 #ifdef CONFIG_IP_VS_PROTO_SCTP 1169 #define SCTP_APP_TAB_BITS 4 1170 #define SCTP_APP_TAB_SIZE (1 << SCTP_APP_TAB_BITS) 1171 #define SCTP_APP_TAB_MASK (SCTP_APP_TAB_SIZE - 1) 1172 /* Hash table for SCTP application incarnations */ 1173 struct list_head sctp_apps[SCTP_APP_TAB_SIZE]; 1174 #endif 1175 /* ip_vs_conn */ 1176 atomic_t conn_count; /* connection counter */ 1177 atomic_t no_cport_conns[IP_VS_AF_MAX]; 1178 struct delayed_work conn_resize_work;/* resize conn_tab */ 1179 1180 /* ip_vs_ctl */ 1181 struct ip_vs_stats_rcu *tot_stats; /* Statistics & est. */ 1182 1183 /* Trash for destinations */ 1184 struct list_head dest_trash; 1185 spinlock_t dest_trash_lock; 1186 struct timer_list dest_trash_timer; /* expiration timer */ 1187 struct mutex service_mutex; /* service reconfig */ 1188 struct rw_semaphore svc_resize_sem; /* svc_table resizing */ 1189 struct rw_semaphore svc_replace_sem; /* svc_table replace */ 1190 struct delayed_work svc_resize_work; /* resize svc_table */ 1191 atomic_t svc_table_changes;/* ++ on table changes */ 1192 /* Service counters */ 1193 atomic_t num_services[IP_VS_AF_MAX]; /* Services */ 1194 atomic_t fwm_services[IP_VS_AF_MAX]; /* Services */ 1195 atomic_t nonfwm_services[IP_VS_AF_MAX];/* Services */ 1196 atomic_t ftpsvc_counter[IP_VS_AF_MAX]; /* FTPPORT */ 1197 atomic_t nullsvc_counter[IP_VS_AF_MAX];/* Zero port */ 1198 atomic_t conn_out_counter[IP_VS_AF_MAX];/* out conn */ 1199 1200 #ifdef CONFIG_SYSCTL 1201 /* delayed work for expiring no dest connections */ 1202 struct delayed_work expire_nodest_conn_work; 1203 /* 1/rate drop and drop-entry variables */ 1204 struct delayed_work defense_work; /* Work handler */ 1205 int drop_rate; 1206 int drop_counter; 1207 int old_secure_tcp; 1208 atomic_t dropentry; 1209 s8 dropentry_counters[8]; 1210 /* locks in ctl.c */ 1211 spinlock_t dropentry_lock; /* drop entry handling */ 1212 spinlock_t droppacket_lock; /* drop packet handling */ 1213 spinlock_t securetcp_lock; /* state and timeout tables */ 1214 1215 /* sys-ctl struct */ 1216 struct ctl_table_header *sysctl_hdr; 1217 struct ctl_table *sysctl_tbl; 1218 #endif 1219 1220 /* sysctl variables */ 1221 int sysctl_amemthresh; 1222 int sysctl_am_droprate; 1223 int sysctl_drop_entry; 1224 int sysctl_drop_packet; 1225 int sysctl_secure_tcp; 1226 #ifdef CONFIG_IP_VS_NFCT 1227 int sysctl_conntrack; 1228 #endif 1229 int sysctl_snat_reroute; 1230 int sysctl_sync_ver; 1231 int sysctl_sync_ports; 1232 int sysctl_sync_persist_mode; 1233 unsigned long sysctl_sync_qlen_max; 1234 int sysctl_sync_sock_size; 1235 int sysctl_cache_bypass; 1236 int sysctl_expire_nodest_conn; 1237 int sysctl_sloppy_tcp; 1238 int sysctl_sloppy_sctp; 1239 int sysctl_expire_quiescent_template; 1240 int sysctl_sync_threshold[2]; 1241 unsigned int sysctl_sync_refresh_period; 1242 int sysctl_sync_retries; 1243 int sysctl_nat_icmp_send; 1244 int sysctl_pmtu_disc; 1245 int sysctl_backup_only; 1246 int sysctl_conn_reuse_mode; 1247 int sysctl_schedule_icmp; 1248 int sysctl_ignore_tunneled; 1249 int sysctl_run_estimation; 1250 #ifdef CONFIG_SYSCTL 1251 cpumask_var_t sysctl_est_cpulist; /* kthread cpumask */ 1252 int est_cpulist_valid; /* cpulist set */ 1253 int sysctl_est_nice; /* kthread nice */ 1254 int est_stopped; /* stop tasks */ 1255 #endif 1256 int sysctl_conn_lfactor; 1257 int sysctl_svc_lfactor; 1258 1259 /* ip_vs_lblc */ 1260 int sysctl_lblc_expiration; 1261 struct ctl_table_header *lblc_ctl_header; 1262 struct ctl_table *lblc_ctl_table; 1263 /* ip_vs_lblcr */ 1264 int sysctl_lblcr_expiration; 1265 struct ctl_table_header *lblcr_ctl_header; 1266 struct ctl_table *lblcr_ctl_table; 1267 unsigned long work_flags; /* IP_VS_WORK_* flags */ 1268 /* ip_vs_est */ 1269 struct delayed_work est_reload_work;/* Reload kthread tasks */ 1270 struct mutex est_mutex; /* protect kthread tasks */ 1271 struct hlist_head est_temp_list; /* Ests during calc phase */ 1272 struct ip_vs_est_kt_data **est_kt_arr; /* Array of kthread data ptrs */ 1273 unsigned long est_max_threads;/* Hard limit of kthreads */ 1274 int est_calc_phase; /* Calculation phase */ 1275 int est_chain_max; /* Calculated chain_max */ 1276 int est_kt_count; /* Allocated ptrs */ 1277 int est_add_ktid; /* ktid where to add ests */ 1278 atomic_t est_genid; /* kthreads reload genid */ 1279 atomic_t est_genid_done; /* applied genid */ 1280 /* ip_vs_sync */ 1281 spinlock_t sync_lock; 1282 struct ipvs_master_sync_state *ms; 1283 spinlock_t sync_buff_lock; 1284 struct ip_vs_sync_thread_data *master_tinfo; 1285 struct ip_vs_sync_thread_data *backup_tinfo; 1286 int threads_mask; 1287 volatile int sync_state; 1288 struct mutex sync_mutex; 1289 struct ipvs_sync_daemon_cfg mcfg; /* Master Configuration */ 1290 struct ipvs_sync_daemon_cfg bcfg; /* Backup Configuration */ 1291 /* net name space ptr */ 1292 struct net *net; /* Needed by timer routines */ 1293 /* Number of heterogeneous destinations, needed because heterogeneous 1294 * are not supported when synchronization is enabled. 1295 */ 1296 unsigned int mixed_address_family_dests; 1297 unsigned int hooks_afmask; /* &1=AF_INET, &2=AF_INET6 */ 1298 1299 struct ip_vs_rht __rcu *svc_table; /* Services */ 1300 struct ip_vs_rht __rcu *conn_tab; /* Connections */ 1301 atomic_t conn_tab_changes;/* ++ on new table */ 1302 }; 1303 1304 #define DEFAULT_SYNC_THRESHOLD 3 1305 #define DEFAULT_SYNC_PERIOD 50 1306 #define DEFAULT_SYNC_VER 1 1307 #define DEFAULT_SLOPPY_TCP 0 1308 #define DEFAULT_SLOPPY_SCTP 0 1309 #define DEFAULT_SYNC_REFRESH_PERIOD (0U * HZ) 1310 #define DEFAULT_SYNC_RETRIES 0 1311 #define IPVS_SYNC_WAKEUP_RATE 8 1312 #define IPVS_SYNC_QLEN_MAX (IPVS_SYNC_WAKEUP_RATE * 4) 1313 #define IPVS_SYNC_SEND_DELAY (HZ / 50) 1314 #define IPVS_SYNC_CHECK_PERIOD HZ 1315 #define IPVS_SYNC_FLUSH_TIME (HZ * 2) 1316 #define IPVS_SYNC_PORTS_MAX (1 << 6) 1317 1318 #ifdef CONFIG_SYSCTL 1319 1320 static inline int sysctl_sync_threshold(struct netns_ipvs *ipvs) 1321 { 1322 return ipvs->sysctl_sync_threshold[0]; 1323 } 1324 1325 static inline int sysctl_sync_period(struct netns_ipvs *ipvs) 1326 { 1327 return READ_ONCE(ipvs->sysctl_sync_threshold[1]); 1328 } 1329 1330 static inline unsigned int sysctl_sync_refresh_period(struct netns_ipvs *ipvs) 1331 { 1332 return READ_ONCE(ipvs->sysctl_sync_refresh_period); 1333 } 1334 1335 static inline int sysctl_sync_retries(struct netns_ipvs *ipvs) 1336 { 1337 return ipvs->sysctl_sync_retries; 1338 } 1339 1340 static inline int sysctl_sync_ver(struct netns_ipvs *ipvs) 1341 { 1342 return ipvs->sysctl_sync_ver; 1343 } 1344 1345 static inline int sysctl_sloppy_tcp(struct netns_ipvs *ipvs) 1346 { 1347 return ipvs->sysctl_sloppy_tcp; 1348 } 1349 1350 static inline int sysctl_sloppy_sctp(struct netns_ipvs *ipvs) 1351 { 1352 return ipvs->sysctl_sloppy_sctp; 1353 } 1354 1355 static inline int sysctl_sync_ports(struct netns_ipvs *ipvs) 1356 { 1357 return READ_ONCE(ipvs->sysctl_sync_ports); 1358 } 1359 1360 static inline int sysctl_sync_persist_mode(struct netns_ipvs *ipvs) 1361 { 1362 return ipvs->sysctl_sync_persist_mode; 1363 } 1364 1365 static inline unsigned long sysctl_sync_qlen_max(struct netns_ipvs *ipvs) 1366 { 1367 return ipvs->sysctl_sync_qlen_max; 1368 } 1369 1370 static inline int sysctl_sync_sock_size(struct netns_ipvs *ipvs) 1371 { 1372 return ipvs->sysctl_sync_sock_size; 1373 } 1374 1375 static inline int sysctl_pmtu_disc(struct netns_ipvs *ipvs) 1376 { 1377 return ipvs->sysctl_pmtu_disc; 1378 } 1379 1380 static inline int sysctl_backup_only(struct netns_ipvs *ipvs) 1381 { 1382 return ipvs->sync_state & IP_VS_STATE_BACKUP && 1383 ipvs->sysctl_backup_only; 1384 } 1385 1386 static inline int sysctl_conn_reuse_mode(struct netns_ipvs *ipvs) 1387 { 1388 return ipvs->sysctl_conn_reuse_mode; 1389 } 1390 1391 static inline int sysctl_expire_nodest_conn(struct netns_ipvs *ipvs) 1392 { 1393 return ipvs->sysctl_expire_nodest_conn; 1394 } 1395 1396 static inline int sysctl_schedule_icmp(struct netns_ipvs *ipvs) 1397 { 1398 return ipvs->sysctl_schedule_icmp; 1399 } 1400 1401 static inline int sysctl_ignore_tunneled(struct netns_ipvs *ipvs) 1402 { 1403 return ipvs->sysctl_ignore_tunneled; 1404 } 1405 1406 static inline int sysctl_cache_bypass(struct netns_ipvs *ipvs) 1407 { 1408 return ipvs->sysctl_cache_bypass; 1409 } 1410 1411 static inline int sysctl_run_estimation(struct netns_ipvs *ipvs) 1412 { 1413 return ipvs->sysctl_run_estimation; 1414 } 1415 1416 static inline const struct cpumask *__sysctl_est_cpulist(struct netns_ipvs *ipvs) 1417 { 1418 if (ipvs->est_cpulist_valid) 1419 return ipvs->sysctl_est_cpulist; 1420 else 1421 return housekeeping_cpumask(HK_TYPE_KTHREAD); 1422 } 1423 1424 static inline const struct cpumask *sysctl_est_preferred_cpulist(struct netns_ipvs *ipvs) 1425 { 1426 if (ipvs->est_cpulist_valid) 1427 return ipvs->sysctl_est_cpulist; 1428 else 1429 return NULL; 1430 } 1431 1432 static inline int sysctl_est_nice(struct netns_ipvs *ipvs) 1433 { 1434 return ipvs->sysctl_est_nice; 1435 } 1436 1437 #else 1438 1439 static inline int sysctl_sync_threshold(struct netns_ipvs *ipvs) 1440 { 1441 return DEFAULT_SYNC_THRESHOLD; 1442 } 1443 1444 static inline int sysctl_sync_period(struct netns_ipvs *ipvs) 1445 { 1446 return DEFAULT_SYNC_PERIOD; 1447 } 1448 1449 static inline unsigned int sysctl_sync_refresh_period(struct netns_ipvs *ipvs) 1450 { 1451 return DEFAULT_SYNC_REFRESH_PERIOD; 1452 } 1453 1454 static inline int sysctl_sync_retries(struct netns_ipvs *ipvs) 1455 { 1456 return DEFAULT_SYNC_RETRIES & 3; 1457 } 1458 1459 static inline int sysctl_sync_ver(struct netns_ipvs *ipvs) 1460 { 1461 return DEFAULT_SYNC_VER; 1462 } 1463 1464 static inline int sysctl_sloppy_tcp(struct netns_ipvs *ipvs) 1465 { 1466 return DEFAULT_SLOPPY_TCP; 1467 } 1468 1469 static inline int sysctl_sloppy_sctp(struct netns_ipvs *ipvs) 1470 { 1471 return DEFAULT_SLOPPY_SCTP; 1472 } 1473 1474 static inline int sysctl_sync_ports(struct netns_ipvs *ipvs) 1475 { 1476 return 1; 1477 } 1478 1479 static inline int sysctl_sync_persist_mode(struct netns_ipvs *ipvs) 1480 { 1481 return 0; 1482 } 1483 1484 static inline unsigned long sysctl_sync_qlen_max(struct netns_ipvs *ipvs) 1485 { 1486 return IPVS_SYNC_QLEN_MAX; 1487 } 1488 1489 static inline int sysctl_sync_sock_size(struct netns_ipvs *ipvs) 1490 { 1491 return 0; 1492 } 1493 1494 static inline int sysctl_pmtu_disc(struct netns_ipvs *ipvs) 1495 { 1496 return 1; 1497 } 1498 1499 static inline int sysctl_backup_only(struct netns_ipvs *ipvs) 1500 { 1501 return 0; 1502 } 1503 1504 static inline int sysctl_conn_reuse_mode(struct netns_ipvs *ipvs) 1505 { 1506 return 1; 1507 } 1508 1509 static inline int sysctl_expire_nodest_conn(struct netns_ipvs *ipvs) 1510 { 1511 return 0; 1512 } 1513 1514 static inline int sysctl_schedule_icmp(struct netns_ipvs *ipvs) 1515 { 1516 return 0; 1517 } 1518 1519 static inline int sysctl_ignore_tunneled(struct netns_ipvs *ipvs) 1520 { 1521 return 0; 1522 } 1523 1524 static inline int sysctl_cache_bypass(struct netns_ipvs *ipvs) 1525 { 1526 return 0; 1527 } 1528 1529 static inline int sysctl_run_estimation(struct netns_ipvs *ipvs) 1530 { 1531 return 1; 1532 } 1533 1534 static inline const struct cpumask *__sysctl_est_cpulist(struct netns_ipvs *ipvs) 1535 { 1536 return housekeeping_cpumask(HK_TYPE_KTHREAD); 1537 } 1538 1539 static inline const struct cpumask *sysctl_est_preferred_cpulist(struct netns_ipvs *ipvs) 1540 { 1541 return NULL; 1542 } 1543 1544 static inline int sysctl_est_nice(struct netns_ipvs *ipvs) 1545 { 1546 return IPVS_EST_NICE; 1547 } 1548 1549 #endif 1550 1551 /* Get load factor to map conn_count/u_thresh to t->size */ 1552 static inline int sysctl_conn_lfactor(struct netns_ipvs *ipvs) 1553 { 1554 return READ_ONCE(ipvs->sysctl_conn_lfactor); 1555 } 1556 1557 /* Get load factor to map num_services/u_thresh to t->size 1558 * Smaller value decreases u_thresh to reduce collisions but increases 1559 * the table size 1560 * Returns factor where: 1561 * - <0: u_thresh = size >> -factor, eg. lfactor -2 = 25% load 1562 * - >=0: u_thresh = size << factor, eg. lfactor 1 = 200% load 1563 */ 1564 static inline int sysctl_svc_lfactor(struct netns_ipvs *ipvs) 1565 { 1566 return READ_ONCE(ipvs->sysctl_svc_lfactor); 1567 } 1568 1569 static inline bool sysctl_est_cpulist_empty(struct netns_ipvs *ipvs) 1570 { 1571 guard(rcu)(); 1572 return cpumask_empty(__sysctl_est_cpulist(ipvs)); 1573 } 1574 1575 static inline unsigned int sysctl_est_cpulist_weight(struct netns_ipvs *ipvs) 1576 { 1577 guard(rcu)(); 1578 return cpumask_weight(__sysctl_est_cpulist(ipvs)); 1579 } 1580 1581 /* IPVS core functions 1582 * (from ip_vs_core.c) 1583 */ 1584 const char *ip_vs_proto_name(unsigned int proto); 1585 void ip_vs_init_hash_table(struct list_head *table, int rows); 1586 struct ip_vs_conn *ip_vs_new_conn_out(struct ip_vs_service *svc, 1587 struct ip_vs_dest *dest, 1588 struct sk_buff *skb, 1589 const struct ip_vs_iphdr *iph, 1590 __be16 dport, 1591 __be16 cport); 1592 #define IP_VS_INIT_HASH_TABLE(t) ip_vs_init_hash_table((t), ARRAY_SIZE((t))) 1593 1594 #define IP_VS_APP_TYPE_FTP 1 1595 1596 /* ip_vs_conn handling functions 1597 * (from ip_vs_conn.c) 1598 */ 1599 enum { 1600 IP_VS_DIR_INPUT = 0, 1601 IP_VS_DIR_OUTPUT, 1602 IP_VS_DIR_INPUT_ONLY, 1603 IP_VS_DIR_LAST, 1604 }; 1605 1606 static inline void ip_vs_conn_fill_param(struct netns_ipvs *ipvs, int af, int protocol, 1607 const union nf_inet_addr *caddr, 1608 __be16 cport, 1609 const union nf_inet_addr *vaddr, 1610 __be16 vport, 1611 struct ip_vs_conn_param *p) 1612 { 1613 p->ipvs = ipvs; 1614 p->af = af; 1615 p->protocol = protocol; 1616 p->caddr = caddr; 1617 p->cport = cport; 1618 p->vaddr = vaddr; 1619 p->vport = vport; 1620 p->pe = NULL; 1621 p->pe_data = NULL; 1622 } 1623 1624 struct ip_vs_conn *ip_vs_conn_in_get(const struct ip_vs_conn_param *p); 1625 struct ip_vs_conn *ip_vs_ct_in_get(const struct ip_vs_conn_param *p); 1626 1627 struct ip_vs_conn * ip_vs_conn_in_get_proto(struct netns_ipvs *ipvs, int af, 1628 const struct sk_buff *skb, 1629 const struct ip_vs_iphdr *iph); 1630 1631 struct ip_vs_conn *ip_vs_conn_out_get(const struct ip_vs_conn_param *p); 1632 1633 struct ip_vs_conn * ip_vs_conn_out_get_proto(struct netns_ipvs *ipvs, int af, 1634 const struct sk_buff *skb, 1635 const struct ip_vs_iphdr *iph); 1636 1637 /* Get reference to gain full access to conn. 1638 * By default, RCU read-side critical sections have access only to 1639 * conn fields and its PE data, see ip_vs_conn_rcu_free() for reference. 1640 */ 1641 static inline bool __ip_vs_conn_get(struct ip_vs_conn *cp) 1642 { 1643 return refcount_inc_not_zero(&cp->refcnt); 1644 } 1645 1646 /* put back the conn without restarting its timer */ 1647 static inline void __ip_vs_conn_put(struct ip_vs_conn *cp) 1648 { 1649 smp_mb__before_atomic(); 1650 refcount_dec(&cp->refcnt); 1651 } 1652 void ip_vs_conn_put(struct ip_vs_conn *cp); 1653 void ip_vs_conn_fill_cport(struct ip_vs_conn *cp, __be16 cport); 1654 int ip_vs_conn_desired_size(struct netns_ipvs *ipvs, struct ip_vs_rht *t, 1655 int lfactor); 1656 struct ip_vs_rht *ip_vs_conn_tab_alloc(struct netns_ipvs *ipvs, int buckets, 1657 int lfactor); 1658 1659 static inline struct ip_vs_conn * 1660 ip_vs_hn0_to_conn(struct ip_vs_conn_hnode *hn) 1661 { 1662 return container_of(hn, struct ip_vs_conn, hn0); 1663 } 1664 1665 static inline struct ip_vs_conn * 1666 ip_vs_hn_to_conn(struct ip_vs_conn_hnode *hn) 1667 { 1668 return hn->dir ? container_of(hn, struct ip_vs_conn, hn1) : 1669 container_of(hn, struct ip_vs_conn, hn0); 1670 } 1671 1672 struct ip_vs_conn *ip_vs_conn_new(const struct ip_vs_conn_param *p, int dest_af, 1673 const union nf_inet_addr *daddr, 1674 __be16 dport, unsigned int flags, 1675 struct ip_vs_dest *dest, __u32 fwmark); 1676 void ip_vs_conn_expire_now(struct ip_vs_conn *cp); 1677 1678 const char *ip_vs_state_name(const struct ip_vs_conn *cp); 1679 1680 void ip_vs_tcp_conn_listen(struct ip_vs_conn *cp); 1681 int ip_vs_check_template(struct ip_vs_conn *ct, struct ip_vs_dest *cdest); 1682 void ip_vs_random_dropentry(struct netns_ipvs *ipvs); 1683 int ip_vs_conn_init(void); 1684 void ip_vs_conn_cleanup(void); 1685 1686 static inline void ip_vs_control_del(struct ip_vs_conn *cp) 1687 { 1688 struct ip_vs_conn *ctl_cp = cp->control; 1689 if (!ctl_cp) { 1690 IP_VS_ERR_BUF("request control DEL for uncontrolled: " 1691 "%s:%d to %s:%d\n", 1692 IP_VS_DBG_ADDR(cp->af, &cp->caddr), 1693 ntohs(cp->cport), 1694 IP_VS_DBG_ADDR(cp->af, &cp->vaddr), 1695 ntohs(cp->vport)); 1696 1697 return; 1698 } 1699 1700 IP_VS_DBG_BUF(7, "DELeting control for: " 1701 "cp.dst=%s:%d ctl_cp.dst=%s:%d\n", 1702 IP_VS_DBG_ADDR(cp->af, &cp->caddr), 1703 ntohs(cp->cport), 1704 IP_VS_DBG_ADDR(cp->af, &ctl_cp->caddr), 1705 ntohs(ctl_cp->cport)); 1706 1707 cp->control = NULL; 1708 if (atomic_read(&ctl_cp->n_control) == 0) { 1709 IP_VS_ERR_BUF("BUG control DEL with n=0 : " 1710 "%s:%d to %s:%d\n", 1711 IP_VS_DBG_ADDR(cp->af, &cp->caddr), 1712 ntohs(cp->cport), 1713 IP_VS_DBG_ADDR(cp->af, &cp->vaddr), 1714 ntohs(cp->vport)); 1715 1716 return; 1717 } 1718 atomic_dec(&ctl_cp->n_control); 1719 } 1720 1721 static inline void 1722 ip_vs_control_add(struct ip_vs_conn *cp, struct ip_vs_conn *ctl_cp) 1723 { 1724 if (cp->control) { 1725 IP_VS_ERR_BUF("request control ADD for already controlled: " 1726 "%s:%d to %s:%d\n", 1727 IP_VS_DBG_ADDR(cp->af, &cp->caddr), 1728 ntohs(cp->cport), 1729 IP_VS_DBG_ADDR(cp->af, &cp->vaddr), 1730 ntohs(cp->vport)); 1731 1732 ip_vs_control_del(cp); 1733 } 1734 1735 IP_VS_DBG_BUF(7, "ADDing control for: " 1736 "cp.dst=%s:%d ctl_cp.dst=%s:%d\n", 1737 IP_VS_DBG_ADDR(cp->af, &cp->caddr), 1738 ntohs(cp->cport), 1739 IP_VS_DBG_ADDR(cp->af, &ctl_cp->caddr), 1740 ntohs(ctl_cp->cport)); 1741 1742 cp->control = ctl_cp; 1743 atomic_inc(&ctl_cp->n_control); 1744 } 1745 1746 /* Mark our template as assured */ 1747 static inline void 1748 ip_vs_control_assure_ct(struct ip_vs_conn *cp) 1749 { 1750 struct ip_vs_conn *ct = cp->control; 1751 1752 if (ct && !(ct->state & IP_VS_CTPL_S_ASSURED) && 1753 (ct->flags & IP_VS_CONN_F_TEMPLATE)) 1754 ct->state |= IP_VS_CTPL_S_ASSURED; 1755 } 1756 1757 /* IPVS netns init & cleanup functions */ 1758 int ip_vs_estimator_net_init(struct netns_ipvs *ipvs); 1759 int ip_vs_control_net_init(struct netns_ipvs *ipvs); 1760 int ip_vs_protocol_net_init(struct netns_ipvs *ipvs); 1761 int ip_vs_app_net_init(struct netns_ipvs *ipvs); 1762 int ip_vs_conn_net_init(struct netns_ipvs *ipvs); 1763 int ip_vs_sync_net_init(struct netns_ipvs *ipvs); 1764 void ip_vs_conn_net_cleanup(struct netns_ipvs *ipvs); 1765 void ip_vs_app_net_cleanup(struct netns_ipvs *ipvs); 1766 void ip_vs_protocol_net_cleanup(struct netns_ipvs *ipvs); 1767 void ip_vs_control_net_cleanup(struct netns_ipvs *ipvs); 1768 void ip_vs_estimator_net_cleanup(struct netns_ipvs *ipvs); 1769 void ip_vs_sync_net_cleanup(struct netns_ipvs *ipvs); 1770 void ip_vs_service_nets_cleanup(struct list_head *net_list); 1771 1772 /* IPVS application functions 1773 * (from ip_vs_app.c) 1774 */ 1775 #define IP_VS_APP_MAX_PORTS 8 1776 struct ip_vs_app *register_ip_vs_app(struct netns_ipvs *ipvs, struct ip_vs_app *app); 1777 void unregister_ip_vs_app(struct netns_ipvs *ipvs, struct ip_vs_app *app); 1778 int ip_vs_bind_app(struct ip_vs_conn *cp, struct ip_vs_protocol *pp); 1779 void ip_vs_unbind_app(struct ip_vs_conn *cp); 1780 int register_ip_vs_app_inc(struct netns_ipvs *ipvs, struct ip_vs_app *app, __u16 proto, 1781 __u16 port); 1782 int ip_vs_app_inc_get(struct ip_vs_app *inc); 1783 void ip_vs_app_inc_put(struct ip_vs_app *inc); 1784 1785 int ip_vs_app_pkt_out(struct ip_vs_conn *, struct sk_buff *skb, 1786 struct ip_vs_iphdr *ipvsh); 1787 int ip_vs_app_pkt_in(struct ip_vs_conn *, struct sk_buff *skb, 1788 struct ip_vs_iphdr *ipvsh); 1789 1790 int register_ip_vs_pe(struct ip_vs_pe *pe); 1791 int unregister_ip_vs_pe(struct ip_vs_pe *pe); 1792 struct ip_vs_pe *ip_vs_pe_getbyname(const char *name); 1793 struct ip_vs_pe *__ip_vs_pe_getbyname(const char *pe_name); 1794 1795 /* Use a #define to avoid all of module.h just for these trivial ops */ 1796 #define ip_vs_pe_get(pe) \ 1797 if (pe && pe->module) \ 1798 __module_get(pe->module); 1799 1800 #define ip_vs_pe_put(pe) \ 1801 if (pe && pe->module) \ 1802 module_put(pe->module); 1803 1804 /* IPVS protocol functions (from ip_vs_proto.c) */ 1805 int ip_vs_protocol_init(void); 1806 void ip_vs_protocol_cleanup(void); 1807 void ip_vs_protocol_timeout_change(struct netns_ipvs *ipvs, int flags); 1808 int *ip_vs_create_timeout_table(int *table, int size); 1809 void ip_vs_tcpudp_debug_packet(int af, struct ip_vs_protocol *pp, 1810 const struct sk_buff *skb, int offset, 1811 const char *msg); 1812 1813 extern struct ip_vs_protocol ip_vs_protocol_tcp; 1814 extern struct ip_vs_protocol ip_vs_protocol_udp; 1815 extern struct ip_vs_protocol ip_vs_protocol_icmp; 1816 extern struct ip_vs_protocol ip_vs_protocol_esp; 1817 extern struct ip_vs_protocol ip_vs_protocol_ah; 1818 extern struct ip_vs_protocol ip_vs_protocol_sctp; 1819 1820 /* Registering/unregistering scheduler functions 1821 * (from ip_vs_sched.c) 1822 */ 1823 int register_ip_vs_scheduler(struct ip_vs_scheduler *scheduler); 1824 int unregister_ip_vs_scheduler(struct ip_vs_scheduler *scheduler); 1825 int ip_vs_bind_scheduler(struct ip_vs_service *svc, 1826 struct ip_vs_scheduler *scheduler); 1827 void ip_vs_unbind_scheduler(struct ip_vs_service *svc, 1828 struct ip_vs_scheduler *sched); 1829 struct ip_vs_scheduler *ip_vs_scheduler_get(const char *sched_name); 1830 void ip_vs_scheduler_put(struct ip_vs_scheduler *scheduler); 1831 struct ip_vs_conn * 1832 ip_vs_schedule(struct ip_vs_service *svc, struct sk_buff *skb, 1833 struct ip_vs_proto_data *pd, int *ignored, 1834 struct ip_vs_iphdr *iph); 1835 int ip_vs_leave(struct ip_vs_service *svc, struct sk_buff *skb, 1836 struct ip_vs_proto_data *pd, struct ip_vs_iphdr *iph); 1837 1838 void ip_vs_scheduler_err(struct ip_vs_service *svc, const char *msg); 1839 1840 /* IPVS control data and functions (from ip_vs_ctl.c) */ 1841 extern struct ip_vs_stats ip_vs_stats; 1842 extern int sysctl_ip_vs_sync_ver; 1843 1844 struct ip_vs_service * 1845 ip_vs_service_find(struct netns_ipvs *ipvs, int af, __u32 fwmark, __u16 protocol, 1846 const union nf_inet_addr *vaddr, __be16 vport); 1847 1848 bool ip_vs_has_real_service(struct netns_ipvs *ipvs, int af, __u16 protocol, 1849 const union nf_inet_addr *daddr, __be16 dport); 1850 1851 struct ip_vs_dest * 1852 ip_vs_find_real_service(struct netns_ipvs *ipvs, int af, __u16 protocol, 1853 const union nf_inet_addr *daddr, __be16 dport); 1854 struct ip_vs_dest *ip_vs_find_tunnel(struct netns_ipvs *ipvs, int af, 1855 const union nf_inet_addr *daddr, 1856 __be16 tun_port); 1857 1858 int ip_vs_use_count_inc(void); 1859 void ip_vs_use_count_dec(void); 1860 int ip_vs_register_nl_ioctl(void); 1861 void ip_vs_unregister_nl_ioctl(void); 1862 int ip_vs_control_init(void); 1863 void ip_vs_control_cleanup(void); 1864 struct ip_vs_dest * 1865 ip_vs_find_dest(struct netns_ipvs *ipvs, int svc_af, int dest_af, 1866 const union nf_inet_addr *daddr, __be16 dport, 1867 const union nf_inet_addr *vaddr, __be16 vport, 1868 __u16 protocol, __u32 fwmark, __u32 flags); 1869 void ip_vs_try_bind_dest(struct ip_vs_conn *cp); 1870 1871 static inline void ip_vs_dest_hold(struct ip_vs_dest *dest) 1872 { 1873 refcount_inc(&dest->refcnt); 1874 } 1875 1876 static inline void ip_vs_dest_put(struct ip_vs_dest *dest) 1877 { 1878 smp_mb__before_atomic(); 1879 refcount_dec(&dest->refcnt); 1880 } 1881 1882 static inline void ip_vs_dest_put_and_free(struct ip_vs_dest *dest) 1883 { 1884 if (refcount_dec_and_test(&dest->refcnt)) 1885 kfree(dest); 1886 } 1887 1888 /* IPVS sync daemon data and function prototypes 1889 * (from ip_vs_sync.c) 1890 */ 1891 int start_sync_thread(struct netns_ipvs *ipvs, struct ipvs_sync_daemon_cfg *cfg, 1892 int state); 1893 int stop_sync_thread(struct netns_ipvs *ipvs, int state); 1894 void ip_vs_sync_conn(struct netns_ipvs *ipvs, struct ip_vs_conn *cp, int pkts); 1895 1896 /* IPVS rate estimator prototypes (from ip_vs_est.c) */ 1897 int ip_vs_start_estimator(struct netns_ipvs *ipvs, struct ip_vs_stats *stats); 1898 void ip_vs_stop_estimator(struct netns_ipvs *ipvs, struct ip_vs_stats *stats); 1899 void ip_vs_zero_estimator(struct ip_vs_stats *stats); 1900 void ip_vs_read_estimator(struct ip_vs_kstats *dst, struct ip_vs_stats *stats); 1901 void ip_vs_est_reload_start(struct netns_ipvs *ipvs, bool restart); 1902 int ip_vs_est_kthread_start(struct netns_ipvs *ipvs, 1903 struct ip_vs_est_kt_data *kd); 1904 void ip_vs_est_kthread_stop(struct ip_vs_est_kt_data *kd); 1905 1906 static inline void ip_vs_stop_estimator_tot_stats(struct netns_ipvs *ipvs) 1907 { 1908 #ifdef CONFIG_SYSCTL 1909 ip_vs_stop_estimator(ipvs, &ipvs->tot_stats->s); 1910 ipvs->tot_stats->s.est.ktid = -2; 1911 #endif 1912 } 1913 1914 static inline void ip_vs_est_stopped_recalc(struct netns_ipvs *ipvs) 1915 { 1916 #ifdef CONFIG_SYSCTL 1917 /* Stop tasks while cpulist is empty or if disabled with flag */ 1918 ipvs->est_stopped = !sysctl_run_estimation(ipvs) || 1919 (ipvs->est_cpulist_valid && 1920 sysctl_est_cpulist_empty(ipvs)); 1921 #endif 1922 } 1923 1924 static inline bool ip_vs_est_stopped(struct netns_ipvs *ipvs) 1925 { 1926 #ifdef CONFIG_SYSCTL 1927 return ipvs->est_stopped; 1928 #else 1929 return false; 1930 #endif 1931 } 1932 1933 static inline int ip_vs_est_max_threads(struct netns_ipvs *ipvs) 1934 { 1935 unsigned int limit = IPVS_EST_CPU_KTHREADS * 1936 sysctl_est_cpulist_weight(ipvs); 1937 1938 return max(1U, limit); 1939 } 1940 1941 /* Various IPVS packet transmitters (from ip_vs_xmit.c) */ 1942 int ip_vs_null_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 1943 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1944 int ip_vs_bypass_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 1945 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1946 int ip_vs_nat_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 1947 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1948 int ip_vs_tunnel_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 1949 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1950 int ip_vs_dr_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 1951 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1952 int ip_vs_icmp_xmit(struct sk_buff *skb, struct ip_vs_conn *cp, 1953 struct ip_vs_protocol *pp, int offset, 1954 unsigned int hooknum, struct ip_vs_iphdr *iph); 1955 void ip_vs_dest_dst_rcu_free(struct rcu_head *head); 1956 1957 #ifdef CONFIG_IP_VS_IPV6 1958 int ip_vs_bypass_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, 1959 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1960 int ip_vs_nat_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, 1961 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1962 int ip_vs_tunnel_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, 1963 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1964 int ip_vs_dr_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, 1965 struct ip_vs_protocol *pp, struct ip_vs_iphdr *iph); 1966 int ip_vs_icmp_xmit_v6(struct sk_buff *skb, struct ip_vs_conn *cp, 1967 struct ip_vs_protocol *pp, int offset, 1968 unsigned int hooknum, struct ip_vs_iphdr *iph); 1969 #endif 1970 1971 #ifdef CONFIG_SYSCTL 1972 /* This is a simple mechanism to ignore packets when 1973 * we are loaded. Just set ip_vs_drop_rate to 'n' and 1974 * we start to drop 1/rate of the packets 1975 */ 1976 static inline int ip_vs_todrop(struct netns_ipvs *ipvs) 1977 { 1978 if (!ipvs->drop_rate) 1979 return 0; 1980 if (--ipvs->drop_counter > 0) 1981 return 0; 1982 ipvs->drop_counter = ipvs->drop_rate; 1983 return 1; 1984 } 1985 #else 1986 static inline int ip_vs_todrop(struct netns_ipvs *ipvs) { return 0; } 1987 #endif 1988 1989 #ifdef CONFIG_SYSCTL 1990 /* Enqueue delayed work for expiring no dest connections 1991 * Only run when sysctl_expire_nodest=1 1992 */ 1993 static inline void ip_vs_enqueue_expire_nodest_conns(struct netns_ipvs *ipvs) 1994 { 1995 if (sysctl_expire_nodest_conn(ipvs)) 1996 queue_delayed_work(system_long_wq, 1997 &ipvs->expire_nodest_conn_work, 1); 1998 } 1999 2000 void ip_vs_expire_nodest_conn_flush(struct netns_ipvs *ipvs); 2001 #else 2002 static inline void ip_vs_enqueue_expire_nodest_conns(struct netns_ipvs *ipvs) {} 2003 #endif 2004 2005 #define IP_VS_DFWD_METHOD(dest) (atomic_read(&(dest)->conn_flags) & \ 2006 IP_VS_CONN_F_FWD_MASK) 2007 2008 /* ip_vs_fwd_tag returns the forwarding tag of the connection */ 2009 #define IP_VS_FWD_METHOD(cp) (cp->flags & IP_VS_CONN_F_FWD_MASK) 2010 2011 static inline char ip_vs_fwd_tag(struct ip_vs_conn *cp) 2012 { 2013 char fwd; 2014 2015 switch (IP_VS_FWD_METHOD(cp)) { 2016 case IP_VS_CONN_F_MASQ: 2017 fwd = 'M'; break; 2018 case IP_VS_CONN_F_LOCALNODE: 2019 fwd = 'L'; break; 2020 case IP_VS_CONN_F_TUNNEL: 2021 fwd = 'T'; break; 2022 case IP_VS_CONN_F_DROUTE: 2023 fwd = 'R'; break; 2024 case IP_VS_CONN_F_BYPASS: 2025 fwd = 'B'; break; 2026 default: 2027 fwd = '?'; break; 2028 } 2029 return fwd; 2030 } 2031 2032 /* Check if connection uses double hashing */ 2033 static inline bool ip_vs_conn_use_hash2(struct ip_vs_conn *cp) 2034 { 2035 return IP_VS_FWD_METHOD(cp) == IP_VS_CONN_F_MASQ && 2036 !(cp->flags & IP_VS_CONN_F_TEMPLATE); 2037 } 2038 2039 void ip_vs_nat_icmp(struct sk_buff *skb, struct ip_vs_protocol *pp, 2040 struct ip_vs_conn *cp, int dir); 2041 2042 #ifdef CONFIG_IP_VS_IPV6 2043 void ip_vs_nat_icmp_v6(struct sk_buff *skb, struct ip_vs_protocol *pp, 2044 struct ip_vs_conn *cp, int dir); 2045 #endif 2046 2047 __sum16 ip_vs_checksum_complete(struct sk_buff *skb, int offset); 2048 2049 static inline __wsum ip_vs_check_diff4(__be32 old, __be32 new, __wsum oldsum) 2050 { 2051 __be32 diff[2] = { ~old, new }; 2052 2053 return csum_partial(diff, sizeof(diff), oldsum); 2054 } 2055 2056 #ifdef CONFIG_IP_VS_IPV6 2057 static inline __wsum ip_vs_check_diff16(const __be32 *old, const __be32 *new, 2058 __wsum oldsum) 2059 { 2060 __be32 diff[8] = { ~old[3], ~old[2], ~old[1], ~old[0], 2061 new[3], new[2], new[1], new[0] }; 2062 2063 return csum_partial(diff, sizeof(diff), oldsum); 2064 } 2065 #endif 2066 2067 static inline __wsum ip_vs_check_diff2(__be16 old, __be16 new, __wsum oldsum) 2068 { 2069 __be16 diff[2] = { ~old, new }; 2070 2071 return csum_partial(diff, sizeof(diff), oldsum); 2072 } 2073 2074 /* Forget current conntrack (unconfirmed) and attach notrack entry */ 2075 static inline void ip_vs_notrack(struct sk_buff *skb) 2076 { 2077 #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE) 2078 enum ip_conntrack_info ctinfo; 2079 struct nf_conn *ct = nf_ct_get(skb, &ctinfo); 2080 2081 if (ct) { 2082 nf_conntrack_put(&ct->ct_general); 2083 nf_ct_set(skb, NULL, IP_CT_UNTRACKED); 2084 } 2085 #endif 2086 } 2087 2088 #ifdef CONFIG_IP_VS_NFCT 2089 /* Netfilter connection tracking 2090 * (from ip_vs_nfct.c) 2091 */ 2092 static inline int ip_vs_conntrack_enabled(struct netns_ipvs *ipvs) 2093 { 2094 #ifdef CONFIG_SYSCTL 2095 return ipvs->sysctl_conntrack; 2096 #else 2097 return 0; 2098 #endif 2099 } 2100 2101 void ip_vs_update_conntrack(struct sk_buff *skb, struct ip_vs_conn *cp, 2102 int outin); 2103 int ip_vs_confirm_conntrack(struct sk_buff *skb); 2104 void ip_vs_nfct_expect_related(struct sk_buff *skb, struct nf_conn *ct, 2105 struct ip_vs_conn *cp, u_int8_t proto, 2106 const __be16 port, int from_rs); 2107 void ip_vs_conn_drop_conntrack(struct ip_vs_conn *cp); 2108 2109 #else 2110 2111 static inline int ip_vs_conntrack_enabled(struct netns_ipvs *ipvs) 2112 { 2113 return 0; 2114 } 2115 2116 static inline void ip_vs_update_conntrack(struct sk_buff *skb, 2117 struct ip_vs_conn *cp, int outin) 2118 { 2119 } 2120 2121 static inline int ip_vs_confirm_conntrack(struct sk_buff *skb) 2122 { 2123 return NF_ACCEPT; 2124 } 2125 2126 static inline void ip_vs_conn_drop_conntrack(struct ip_vs_conn *cp) 2127 { 2128 } 2129 #endif /* CONFIG_IP_VS_NFCT */ 2130 2131 /* Using old conntrack that can not be redirected to another real server? */ 2132 static inline bool ip_vs_conn_uses_old_conntrack(struct ip_vs_conn *cp, 2133 struct sk_buff *skb) 2134 { 2135 #ifdef CONFIG_IP_VS_NFCT 2136 enum ip_conntrack_info ctinfo; 2137 struct nf_conn *ct; 2138 2139 ct = nf_ct_get(skb, &ctinfo); 2140 if (ct && nf_ct_is_confirmed(ct)) 2141 return true; 2142 #endif 2143 return false; 2144 } 2145 2146 static inline int ip_vs_register_conntrack(struct ip_vs_service *svc) 2147 { 2148 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 2149 int afmask = (svc->af == AF_INET6) ? 2 : 1; 2150 int ret = 0; 2151 2152 if (!(svc->conntrack_afmask & afmask)) { 2153 ret = nf_ct_netns_get(svc->ipvs->net, svc->af); 2154 if (ret >= 0) 2155 svc->conntrack_afmask |= afmask; 2156 } 2157 return ret; 2158 #else 2159 return 0; 2160 #endif 2161 } 2162 2163 static inline void ip_vs_unregister_conntrack(struct ip_vs_service *svc) 2164 { 2165 #if IS_ENABLED(CONFIG_NF_CONNTRACK) 2166 int afmask = (svc->af == AF_INET6) ? 2 : 1; 2167 2168 if (svc->conntrack_afmask & afmask) { 2169 nf_ct_netns_put(svc->ipvs->net, svc->af); 2170 svc->conntrack_afmask &= ~afmask; 2171 } 2172 #endif 2173 } 2174 2175 int ip_vs_register_hooks(struct netns_ipvs *ipvs, unsigned int af); 2176 void ip_vs_unregister_hooks(struct netns_ipvs *ipvs, unsigned int af); 2177 2178 static inline int 2179 ip_vs_dest_conn_overhead(struct ip_vs_dest *dest) 2180 { 2181 /* We think the overhead of processing active connections is 256 2182 * times higher than that of inactive connections in average. (This 2183 * 256 times might not be accurate, we will change it later) We 2184 * use the following formula to estimate the overhead now: 2185 * dest->activeconns*256 + dest->inactconns 2186 */ 2187 return (atomic_read(&dest->activeconns) << 8) + 2188 atomic_read(&dest->inactconns); 2189 } 2190 2191 #ifdef CONFIG_IP_VS_PROTO_TCP 2192 INDIRECT_CALLABLE_DECLARE(int 2193 tcp_snat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, 2194 struct ip_vs_conn *cp, struct ip_vs_iphdr *iph)); 2195 #endif 2196 2197 #ifdef CONFIG_IP_VS_PROTO_UDP 2198 INDIRECT_CALLABLE_DECLARE(int 2199 udp_snat_handler(struct sk_buff *skb, struct ip_vs_protocol *pp, 2200 struct ip_vs_conn *cp, struct ip_vs_iphdr *iph)); 2201 #endif 2202 #endif /* _NET_IP_VS_H */ 2203