net/netfilter/ipvs/ip_vs_conn.c

   1 /*
   2  * IPVS         An implementation of the IP virtual server support for the
   3  *              LINUX operating system.  IPVS is now implemented as a module
   4  *              over the Netfilter framework. IPVS can be used to build a
   5  *              high-performance and highly available server based on a
   6  *              cluster of servers.
   7  *
   8  * Authors:     Wensong Zhang <wensong@linuxvirtualserver.org>
   9  *              Peter Kese <peter.kese@ijs.si>
  10  *              Julian Anastasov <ja@ssi.bg>
  11  *
  12  *              This program is free software; you can redistribute it and/or
  13  *              modify it under the terms of the GNU General Public License
  14  *              as published by the Free Software Foundation; either version
  15  *              2 of the License, or (at your option) any later version.
  16  *
  17  * The IPVS code for kernel 2.2 was done by Wensong Zhang and Peter Kese,
  18  * with changes/fixes from Julian Anastasov, Lars Marowsky-Bree, Horms
  19  * and others. Many code here is taken from IP MASQ code of kernel 2.2.
  20  *
  21  * Changes:
  22  *
  23  */
  24
  25 #define KMSG_COMPONENT "IPVS"
  26 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  27
  28 #include <linux/interrupt.h>
  29 #include <linux/in.h>
  30 #include <linux/net.h>
  31 #include <linux/kernel.h>
  32 #include <linux/module.h>
  33 #include <linux/vmalloc.h>
  34 #include <linux/proc_fs.h>              /* for proc_net_* */
  35 #include <linux/slab.h>
  36 #include <linux/seq_file.h>
  37 #include <linux/jhash.h>
  38 #include <linux/random.h>
  39
  40 #include <net/net_namespace.h>
  41 #include <net/ip_vs.h>
  42
  43
  44 #ifndef CONFIG_IP_VS_TAB_BITS
  45 #define CONFIG_IP_VS_TAB_BITS   12
  46 #endif
  47
  48 /*
  49  * Connection hash size. Default is what was selected at compile time.
  50 */
  51 int ip_vs_conn_tab_bits = CONFIG_IP_VS_TAB_BITS;
  52 module_param_named(conn_tab_bits, ip_vs_conn_tab_bits, int, 0444);
  53 MODULE_PARM_DESC(conn_tab_bits, "Set connections' hash size");
  54
  55 /* size and mask values */
  56 int ip_vs_conn_tab_size;
  57 int ip_vs_conn_tab_mask;
  58
  59 /*
  60  *  Connection hash table: for input and output packets lookups of IPVS
  61  */
  62 static struct list_head *ip_vs_conn_tab;
  63
  64 /*  SLAB cache for IPVS connections */
  65 static struct kmem_cache *ip_vs_conn_cachep __read_mostly;
  66
  67 /*  counter for current IPVS connections */
  68 static atomic_t ip_vs_conn_count = ATOMIC_INIT(0);
  69
  70 /*  counter for no client port connections */
  71 static atomic_t ip_vs_conn_no_cport_cnt = ATOMIC_INIT(0);
  72
  73 /* random value for IPVS connection hash */
  74 static unsigned int ip_vs_conn_rnd;
  75
  76 /*
  77  *  Fine locking granularity for big connection hash table
  78  */
  79 #define CT_LOCKARRAY_BITS  4
  80 #define CT_LOCKARRAY_SIZE  (1<<CT_LOCKARRAY_BITS)
  81 #define CT_LOCKARRAY_MASK  (CT_LOCKARRAY_SIZE-1)
  82
  83 struct ip_vs_aligned_lock
  84 {
  85         rwlock_t        l;
  86 } __attribute__((__aligned__(SMP_CACHE_BYTES)));
  87
  88 /* lock array for conn table */
  89 static struct ip_vs_aligned_lock
  90 __ip_vs_conntbl_lock_array[CT_LOCKARRAY_SIZE] __cacheline_aligned;
  91
  92 static inline void ct_read_lock(unsigned key)
  93 {
  94         read_lock(&__ip_vs_conntbl_lock_array[key&CT_LOCKARRAY_MASK].l);
  95 }
  96
  97 static inline void ct_read_unlock(unsigned key)
  98 {
  99         read_unlock(&__ip_vs_conntbl_lock_array[key&CT_LOCKARRAY_MASK].l);
 100 }
 101
 102 static inline void ct_write_lock(unsigned key)
 103 {
 104         write_lock(&__ip_vs_conntbl_lock_array[key&CT_LOCKARRAY_MASK].l);
 105 }
 106
 107 static inline void ct_write_unlock(unsigned key)
 108 {
 109         write_unlock(&__ip_vs_conntbl_lock_array[key&CT_LOCKARRAY_MASK].l);
 110 }
 111
 112 static inline void ct_read_lock_bh(unsigned key)
 113 {
 114         read_lock_bh(&__ip_vs_conntbl_lock_array[key&CT_LOCKARRAY_MASK].l);
 115 }
 116
 117 static inline void ct_read_unlock_bh(unsigned key)
 118 {
 119         read_unlock_bh(&__ip_vs_conntbl_lock_array[key&CT_LOCKARRAY_MASK].l);
 120 }
 121
 122 static inline void ct_write_lock_bh(unsigned key)
 123 {
 124         write_lock_bh(&__ip_vs_conntbl_lock_array[key&CT_LOCKARRAY_MASK].l);
 125 }
 126
 127 static inline void ct_write_unlock_bh(unsigned key)
 128 {
 129         write_unlock_bh(&__ip_vs_conntbl_lock_array[key&CT_LOCKARRAY_MASK].l);
 130 }
 131
 132
 133 /*
 134  *      Returns hash value for IPVS connection entry
 135  */
 136 static unsigned int ip_vs_conn_hashkey(int af, unsigned proto,
 137                                        const union nf_inet_addr *addr,
 138                                        __be16 port)
 139 {
 140 #ifdef CONFIG_IP_VS_IPV6
 141         if (af == AF_INET6)
 142                 return jhash_3words(jhash(addr, 16, ip_vs_conn_rnd),
 143                                     (__force u32)port, proto, ip_vs_conn_rnd)
 144                         & ip_vs_conn_tab_mask;
 145 #endif
 146         return jhash_3words((__force u32)addr->ip, (__force u32)port, proto,
 147                             ip_vs_conn_rnd)
 148                 & ip_vs_conn_tab_mask;
 149 }
 150
 151
 152 /*
 153  *      Hashes ip_vs_conn in ip_vs_conn_tab by proto,addr,port.
 154  *      returns bool success.
 155  */
 156 static inline int ip_vs_conn_hash(struct ip_vs_conn *cp)
 157 {
 158         unsigned hash;
 159         int ret;
 160
 161         /* Hash by protocol, client address and port */
 162         hash = ip_vs_conn_hashkey(cp->af, cp->protocol, &cp->caddr, cp->cport);
 163
 164         ct_write_lock(hash);
 165         spin_lock(&cp->lock);
 166
 167         if (!(cp->flags & IP_VS_CONN_F_HASHED)) {
 168                 list_add(&cp->c_list, &ip_vs_conn_tab[hash]);
 169                 cp->flags |= IP_VS_CONN_F_HASHED;
 170                 atomic_inc(&cp->refcnt);
 171                 ret = 1;
 172         } else {
 173                 pr_err("%s(): request for already hashed, called from %pF\n",
 174                        __func__, __builtin_return_address(0));
 175                 ret = 0;
 176         }
 177
 178         spin_unlock(&cp->lock);
 179         ct_write_unlock(hash);
 180
 181         return ret;
 182 }
 183
 184
 185 /*
 186  *      UNhashes ip_vs_conn from ip_vs_conn_tab.
 187  *      returns bool success.
 188  */
 189 static inline int ip_vs_conn_unhash(struct ip_vs_conn *cp)
 190 {
 191         unsigned hash;
 192         int ret;
 193
 194         /* unhash it and decrease its reference counter */
 195         hash = ip_vs_conn_hashkey(cp->af, cp->protocol, &cp->caddr, cp->cport);
 196
 197         ct_write_lock(hash);
 198         spin_lock(&cp->lock);
 199
 200         if (cp->flags & IP_VS_CONN_F_HASHED) {
 201                 list_del(&cp->c_list);
 202                 cp->flags &= ~IP_VS_CONN_F_HASHED;
 203                 atomic_dec(&cp->refcnt);
 204                 ret = 1;
 205         } else
 206                 ret = 0;
 207
 208         spin_unlock(&cp->lock);
 209         ct_write_unlock(hash);
 210
 211         return ret;
 212 }
 213
 214
 215 /*
 216  *  Gets ip_vs_conn associated with supplied parameters in the ip_vs_conn_tab.
 217  *  Called for pkts coming from OUTside-to-INside.
 218  *      s_addr, s_port: pkt source address (foreign host)
 219  *      d_addr, d_port: pkt dest address (load balancer)
 220  */
 221 static inline struct ip_vs_conn *__ip_vs_conn_in_get
 222 (int af, int protocol, const union nf_inet_addr *s_addr, __be16 s_port,
 223  const union nf_inet_addr *d_addr, __be16 d_port)
 224 {
 225         unsigned hash;
 226         struct ip_vs_conn *cp;
 227
 228         hash = ip_vs_conn_hashkey(af, protocol, s_addr, s_port);
 229
 230         ct_read_lock(hash);
 231
 232         list_for_each_entry(cp, &ip_vs_conn_tab[hash], c_list) {
 233                 if (cp->af == af &&
 234                     ip_vs_addr_equal(af, s_addr, &cp->caddr) &&
 235                     ip_vs_addr_equal(af, d_addr, &cp->vaddr) &&
 236                     s_port == cp->cport && d_port == cp->vport &&
 237                     ((!s_port) ^ (!(cp->flags & IP_VS_CONN_F_NO_CPORT))) &&
 238                     protocol == cp->protocol) {
 239                         /* HIT */
 240                         atomic_inc(&cp->refcnt);
 241                         ct_read_unlock(hash);
 242                         return cp;
 243                 }
 244         }
 245
 246         ct_read_unlock(hash);
 247
 248         return NULL;
 249 }
 250
 251 struct ip_vs_conn *ip_vs_conn_in_get
 252 (int af, int protocol, const union nf_inet_addr *s_addr, __be16 s_port,
 253  const union nf_inet_addr *d_addr, __be16 d_port)
 254 {
 255         struct ip_vs_conn *cp;
 256
 257         cp = __ip_vs_conn_in_get(af, protocol, s_addr, s_port, d_addr, d_port);
 258         if (!cp && atomic_read(&ip_vs_conn_no_cport_cnt))
 259                 cp = __ip_vs_conn_in_get(af, protocol, s_addr, 0, d_addr,
 260                                          d_port);
 261
 262         IP_VS_DBG_BUF(9, "lookup/in %s %s:%d->%s:%d %s\n",
 263                       ip_vs_proto_name(protocol),
 264                       IP_VS_DBG_ADDR(af, s_addr), ntohs(s_port),
 265                       IP_VS_DBG_ADDR(af, d_addr), ntohs(d_port),
 266                       cp ? "hit" : "not hit");
 267
 268         return cp;
 269 }
 270
 271 /* Get reference to connection template */
 272 struct ip_vs_conn *ip_vs_ct_in_get
 273 (int af, int protocol, const union nf_inet_addr *s_addr, __be16 s_port,
 274  const union nf_inet_addr *d_addr, __be16 d_port)
 275 {
 276         unsigned hash;
 277         struct ip_vs_conn *cp;
 278
 279         hash = ip_vs_conn_hashkey(af, protocol, s_addr, s_port);
 280
 281         ct_read_lock(hash);
 282
 283         list_for_each_entry(cp, &ip_vs_conn_tab[hash], c_list) {
 284                 if (cp->af == af &&
 285                     ip_vs_addr_equal(af, s_addr, &cp->caddr) &&
 286                     /* protocol should only be IPPROTO_IP if
 287                      * d_addr is a fwmark */
 288                     ip_vs_addr_equal(protocol == IPPROTO_IP ? AF_UNSPEC : af,
 289                                      d_addr, &cp->vaddr) &&
 290                     s_port == cp->cport && d_port == cp->vport &&
 291                     cp->flags & IP_VS_CONN_F_TEMPLATE &&
 292                     protocol == cp->protocol) {
 293                         /* HIT */
 294                         atomic_inc(&cp->refcnt);
 295                         goto out;
 296                 }
 297         }
 298         cp = NULL;
 299
 300   out:
 301         ct_read_unlock(hash);
 302
 303         IP_VS_DBG_BUF(9, "template lookup/in %s %s:%d->%s:%d %s\n",
 304                       ip_vs_proto_name(protocol),
 305                       IP_VS_DBG_ADDR(af, s_addr), ntohs(s_port),
 306                       IP_VS_DBG_ADDR(af, d_addr), ntohs(d_port),
 307                       cp ? "hit" : "not hit");
 308
 309         return cp;
 310 }
 311
 312 /*
 313  *  Gets ip_vs_conn associated with supplied parameters in the ip_vs_conn_tab.
 314  *  Called for pkts coming from inside-to-OUTside.
 315  *      s_addr, s_port: pkt source address (inside host)
 316  *      d_addr, d_port: pkt dest address (foreign host)
 317  */
 318 struct ip_vs_conn *ip_vs_conn_out_get
 319 (int af, int protocol, const union nf_inet_addr *s_addr, __be16 s_port,
 320  const union nf_inet_addr *d_addr, __be16 d_port)
 321 {
 322         unsigned hash;
 323         struct ip_vs_conn *cp, *ret=NULL;
 324
 325         /*
 326          *      Check for "full" addressed entries
 327          */
 328         hash = ip_vs_conn_hashkey(af, protocol, d_addr, d_port);
 329
 330         ct_read_lock(hash);
 331
 332         list_for_each_entry(cp, &ip_vs_conn_tab[hash], c_list) {
 333                 if (cp->af == af &&
 334                     ip_vs_addr_equal(af, d_addr, &cp->caddr) &&
 335                     ip_vs_addr_equal(af, s_addr, &cp->daddr) &&
 336                     d_port == cp->cport && s_port == cp->dport &&
 337                     protocol == cp->protocol) {
 338                         /* HIT */
 339                         atomic_inc(&cp->refcnt);
 340                         ret = cp;
 341                         break;
 342                 }
 343         }
 344
 345         ct_read_unlock(hash);
 346
 347         IP_VS_DBG_BUF(9, "lookup/out %s %s:%d->%s:%d %s\n",
 348                       ip_vs_proto_name(protocol),
 349                       IP_VS_DBG_ADDR(af, s_addr), ntohs(s_port),
 350                       IP_VS_DBG_ADDR(af, d_addr), ntohs(d_port),
 351                       ret ? "hit" : "not hit");
 352
 353         return ret;
 354 }
 355
 356
 357 /*
 358  *      Put back the conn and restart its timer with its timeout
 359  */
 360 void ip_vs_conn_put(struct ip_vs_conn *cp)
 361 {
 362         /* reset it expire in its timeout */
 363         mod_timer(&cp->timer, jiffies+cp->timeout);
 364
 365         __ip_vs_conn_put(cp);
 366 }
 367
 368
 369 /*
 370  *      Fill a no_client_port connection with a client port number
 371  */
 372 void ip_vs_conn_fill_cport(struct ip_vs_conn *cp, __be16 cport)
 373 {
 374         if (ip_vs_conn_unhash(cp)) {
 375                 spin_lock(&cp->lock);
 376                 if (cp->flags & IP_VS_CONN_F_NO_CPORT) {
 377                         atomic_dec(&ip_vs_conn_no_cport_cnt);
 378                         cp->flags &= ~IP_VS_CONN_F_NO_CPORT;
 379                         cp->cport = cport;
 380                 }
 381                 spin_unlock(&cp->lock);
 382
 383                 /* hash on new dport */
 384                 ip_vs_conn_hash(cp);
 385         }
 386 }
 387
 388
 389 /*
 390  *      Bind a connection entry with the corresponding packet_xmit.
 391  *      Called by ip_vs_conn_new.
 392  */
 393 static inline void ip_vs_bind_xmit(struct ip_vs_conn *cp)
 394 {
 395         switch (IP_VS_FWD_METHOD(cp)) {
 396         case IP_VS_CONN_F_MASQ:
 397                 cp->packet_xmit = ip_vs_nat_xmit;
 398                 break;
 399
 400         case IP_VS_CONN_F_TUNNEL:
 401                 cp->packet_xmit = ip_vs_tunnel_xmit;
 402                 break;
 403
 404         case IP_VS_CONN_F_DROUTE:
 405                 cp->packet_xmit = ip_vs_dr_xmit;
 406                 break;
 407
 408         case IP_VS_CONN_F_LOCALNODE:
 409                 cp->packet_xmit = ip_vs_null_xmit;
 410                 break;
 411
 412         case IP_VS_CONN_F_BYPASS:
 413                 cp->packet_xmit = ip_vs_bypass_xmit;
 414                 break;
 415         }
 416 }
 417
 418 #ifdef CONFIG_IP_VS_IPV6
 419 static inline void ip_vs_bind_xmit_v6(struct ip_vs_conn *cp)
 420 {
 421         switch (IP_VS_FWD_METHOD(cp)) {
 422         case IP_VS_CONN_F_MASQ:
 423                 cp->packet_xmit = ip_vs_nat_xmit_v6;
 424                 break;
 425
 426         case IP_VS_CONN_F_TUNNEL:
 427                 cp->packet_xmit = ip_vs_tunnel_xmit_v6;
 428                 break;
 429
 430         case IP_VS_CONN_F_DROUTE:
 431                 cp->packet_xmit = ip_vs_dr_xmit_v6;
 432                 break;
 433
 434         case IP_VS_CONN_F_LOCALNODE:
 435                 cp->packet_xmit = ip_vs_null_xmit;
 436                 break;
 437
 438         case IP_VS_CONN_F_BYPASS:
 439                 cp->packet_xmit = ip_vs_bypass_xmit_v6;
 440                 break;
 441         }
 442 }
 443 #endif
 444
 445
 446 static inline int ip_vs_dest_totalconns(struct ip_vs_dest *dest)
 447 {
 448         return atomic_read(&dest->activeconns)
 449                 + atomic_read(&dest->inactconns);
 450 }
 451
 452 /*
 453  *      Bind a connection entry with a virtual service destination
 454  *      Called just after a new connection entry is created.
 455  */
 456 static inline void
 457 ip_vs_bind_dest(struct ip_vs_conn *cp, struct ip_vs_dest *dest)
 458 {
 459         /* if dest is NULL, then return directly */
 460         if (!dest)
 461                 return;
 462
 463         /* Increase the refcnt counter of the dest */
 464         atomic_inc(&dest->refcnt);
 465
 466         /* Bind with the destination and its corresponding transmitter */
 467         if ((cp->flags & IP_VS_CONN_F_SYNC) &&
 468             (!(cp->flags & IP_VS_CONN_F_TEMPLATE)))
 469                 /* if the connection is not template and is created
 470                  * by sync, preserve the activity flag.
 471                  */
 472                 cp->flags |= atomic_read(&dest->conn_flags) &
 473                              (~IP_VS_CONN_F_INACTIVE);
 474         else
 475                 cp->flags |= atomic_read(&dest->conn_flags);
 476         cp->dest = dest;
 477
 478         IP_VS_DBG_BUF(7, "Bind-dest %s c:%s:%d v:%s:%d "
 479                       "d:%s:%d fwd:%c s:%u conn->flags:%X conn->refcnt:%d "
 480                       "dest->refcnt:%d\n",
 481                       ip_vs_proto_name(cp->protocol),
 482                       IP_VS_DBG_ADDR(cp->af, &cp->caddr), ntohs(cp->cport),
 483                       IP_VS_DBG_ADDR(cp->af, &cp->vaddr), ntohs(cp->vport),
 484                       IP_VS_DBG_ADDR(cp->af, &cp->daddr), ntohs(cp->dport),
 485                       ip_vs_fwd_tag(cp), cp->state,
 486                       cp->flags, atomic_read(&cp->refcnt),
 487                       atomic_read(&dest->refcnt));
 488
 489         /* Update the connection counters */
 490         if (!(cp->flags & IP_VS_CONN_F_TEMPLATE)) {
 491                 /* It is a normal connection, so increase the inactive
 492                    connection counter because it is in TCP SYNRECV
 493                    state (inactive) or other protocol inacive state */
 494                 if ((cp->flags & IP_VS_CONN_F_SYNC) &&
 495                     (!(cp->flags & IP_VS_CONN_F_INACTIVE)))
 496                         atomic_inc(&dest->activeconns);
 497                 else
 498                         atomic_inc(&dest->inactconns);
 499         } else {
 500                 /* It is a persistent connection/template, so increase
 501                    the peristent connection counter */
 502                 atomic_inc(&dest->persistconns);
 503         }
 504
 505         if (dest->u_threshold != 0 &&
 506             ip_vs_dest_totalconns(dest) >= dest->u_threshold)
 507                 dest->flags |= IP_VS_DEST_F_OVERLOAD;
 508 }
 509
 510
 511 /*
 512  * Check if there is a destination for the connection, if so
 513  * bind the connection to the destination.
 514  */
 515 struct ip_vs_dest *ip_vs_try_bind_dest(struct ip_vs_conn *cp)
 516 {
 517         struct ip_vs_dest *dest;
 518
 519         if ((cp) && (!cp->dest)) {
 520                 dest = ip_vs_find_dest(cp->af, &cp->daddr, cp->dport,
 521                                        &cp->vaddr, cp->vport,
 522                                        cp->protocol);
 523                 ip_vs_bind_dest(cp, dest);
 524                 return dest;
 525         } else
 526                 return NULL;
 527 }
 528
 529
 530 /*
 531  *      Unbind a connection entry with its VS destination
 532  *      Called by the ip_vs_conn_expire function.
 533  */
 534 static inline void ip_vs_unbind_dest(struct ip_vs_conn *cp)
 535 {
 536         struct ip_vs_dest *dest = cp->dest;
 537
 538         if (!dest)
 539                 return;
 540
 541         IP_VS_DBG_BUF(7, "Unbind-dest %s c:%s:%d v:%s:%d "
 542                       "d:%s:%d fwd:%c s:%u conn->flags:%X conn->refcnt:%d "
 543                       "dest->refcnt:%d\n",
 544                       ip_vs_proto_name(cp->protocol),
 545                       IP_VS_DBG_ADDR(cp->af, &cp->caddr), ntohs(cp->cport),
 546                       IP_VS_DBG_ADDR(cp->af, &cp->vaddr), ntohs(cp->vport),
 547                       IP_VS_DBG_ADDR(cp->af, &cp->daddr), ntohs(cp->dport),
 548                       ip_vs_fwd_tag(cp), cp->state,
 549                       cp->flags, atomic_read(&cp->refcnt),
 550                       atomic_read(&dest->refcnt));
 551
 552         /* Update the connection counters */
 553         if (!(cp->flags & IP_VS_CONN_F_TEMPLATE)) {
 554                 /* It is a normal connection, so decrease the inactconns
 555                    or activeconns counter */
 556                 if (cp->flags & IP_VS_CONN_F_INACTIVE) {
 557                         atomic_dec(&dest->inactconns);
 558                 } else {
 559                         atomic_dec(&dest->activeconns);
 560                 }
 561         } else {
 562                 /* It is a persistent connection/template, so decrease
 563                    the peristent connection counter */
 564                 atomic_dec(&dest->persistconns);
 565         }
 566
 567         if (dest->l_threshold != 0) {
 568                 if (ip_vs_dest_totalconns(dest) < dest->l_threshold)
 569                         dest->flags &= ~IP_VS_DEST_F_OVERLOAD;
 570         } else if (dest->u_threshold != 0) {
 571                 if (ip_vs_dest_totalconns(dest) * 4 < dest->u_threshold * 3)
 572                         dest->flags &= ~IP_VS_DEST_F_OVERLOAD;
 573         } else {
 574                 if (dest->flags & IP_VS_DEST_F_OVERLOAD)
 575                         dest->flags &= ~IP_VS_DEST_F_OVERLOAD;
 576         }
 577
 578         /*
 579          * Simply decrease the refcnt of the dest, because the
 580          * dest will be either in service's destination list
 581          * or in the trash.
 582          */
 583         atomic_dec(&dest->refcnt);
 584 }
 585
 586
 587 /*
 588  *      Checking if the destination of a connection template is available.
 589  *      If available, return 1, otherwise invalidate this connection
 590  *      template and return 0.
 591  */
 592 int ip_vs_check_template(struct ip_vs_conn *ct)
 593 {
 594         struct ip_vs_dest *dest = ct->dest;
 595
 596         /*
 597          * Checking the dest server status.
 598          */
 599         if ((dest == NULL) ||
 600             !(dest->flags & IP_VS_DEST_F_AVAILABLE) ||
 601             (sysctl_ip_vs_expire_quiescent_template &&
 602              (atomic_read(&dest->weight) == 0))) {
 603                 IP_VS_DBG_BUF(9, "check_template: dest not available for "
 604                               "protocol %s s:%s:%d v:%s:%d "
 605                               "-> d:%s:%d\n",
 606                               ip_vs_proto_name(ct->protocol),
 607                               IP_VS_DBG_ADDR(ct->af, &ct->caddr),
 608                               ntohs(ct->cport),
 609                               IP_VS_DBG_ADDR(ct->af, &ct->vaddr),
 610                               ntohs(ct->vport),
 611                               IP_VS_DBG_ADDR(ct->af, &ct->daddr),
 612                               ntohs(ct->dport));
 613
 614                 /*
 615                  * Invalidate the connection template
 616                  */
 617                 if (ct->vport != htons(0xffff)) {
 618                         if (ip_vs_conn_unhash(ct)) {
 619                                 ct->dport = htons(0xffff);
 620                                 ct->vport = htons(0xffff);
 621                                 ct->cport = 0;
 622                                 ip_vs_conn_hash(ct);
 623                         }
 624                 }
 625
 626                 /*
 627                  * Simply decrease the refcnt of the template,
 628                  * don't restart its timer.
 629                  */
 630                 atomic_dec(&ct->refcnt);
 631                 return 0;
 632         }
 633         return 1;
 634 }
 635
 636 static void ip_vs_conn_expire(unsigned long data)
 637 {
 638         struct ip_vs_conn *cp = (struct ip_vs_conn *)data;
 639
 640         cp->timeout = 60*HZ;
 641
 642         /*
 643          *      hey, I'm using it
 644          */
 645         atomic_inc(&cp->refcnt);
 646
 647         /*
 648          *      do I control anybody?
 649          */
 650         if (atomic_read(&cp->n_control))
 651                 goto expire_later;
 652
 653         /*
 654          *      unhash it if it is hashed in the conn table
 655          */
 656         if (!ip_vs_conn_unhash(cp))
 657                 goto expire_later;
 658
 659         /*
 660          *      refcnt==1 implies I'm the only one referrer
 661          */
 662         if (likely(atomic_read(&cp->refcnt) == 1)) {
 663                 /* delete the timer if it is activated by other users */
 664                 if (timer_pending(&cp->timer))
 665                         del_timer(&cp->timer);
 666
 667                 /* does anybody control me? */
 668                 if (cp->control)
 669                         ip_vs_control_del(cp);
 670
 671                 if (unlikely(cp->app != NULL))
 672                         ip_vs_unbind_app(cp);
 673                 ip_vs_unbind_dest(cp);
 674                 if (cp->flags & IP_VS_CONN_F_NO_CPORT)
 675                         atomic_dec(&ip_vs_conn_no_cport_cnt);
 676                 atomic_dec(&ip_vs_conn_count);
 677
 678                 kmem_cache_free(ip_vs_conn_cachep, cp);
 679                 return;
 680         }
 681
 682         /* hash it back to the table */
 683         ip_vs_conn_hash(cp);
 684
 685   expire_later:
 686         IP_VS_DBG(7, "delayed: conn->refcnt-1=%d conn->n_control=%d\n",
 687                   atomic_read(&cp->refcnt)-1,
 688                   atomic_read(&cp->n_control));
 689
 690         ip_vs_conn_put(cp);
 691 }
 692
 693
 694 void ip_vs_conn_expire_now(struct ip_vs_conn *cp)
 695 {
 696         if (del_timer(&cp->timer))
 697                 mod_timer(&cp->timer, jiffies);
 698 }
 699
 700
 701 /*
 702  *      Create a new connection entry and hash it into the ip_vs_conn_tab
 703  */
 704 struct ip_vs_conn *
 705 ip_vs_conn_new(int af, int proto, const union nf_inet_addr *caddr, __be16 cport,
 706                const union nf_inet_addr *vaddr, __be16 vport,
 707                const union nf_inet_addr *daddr, __be16 dport, unsigned flags,
 708                struct ip_vs_dest *dest)
 709 {
 710         struct ip_vs_conn *cp;
 711         struct ip_vs_protocol *pp = ip_vs_proto_get(proto);
 712
 713         cp = kmem_cache_zalloc(ip_vs_conn_cachep, GFP_ATOMIC);
 714         if (cp == NULL) {
 715                 IP_VS_ERR_RL("%s(): no memory\n", __func__);
 716                 return NULL;
 717         }
 718
 719         INIT_LIST_HEAD(&cp->c_list);
 720         setup_timer(&cp->timer, ip_vs_conn_expire, (unsigned long)cp);
 721         cp->af             = af;
 722         cp->protocol       = proto;
 723         ip_vs_addr_copy(af, &cp->caddr, caddr);
 724         cp->cport          = cport;
 725         ip_vs_addr_copy(af, &cp->vaddr, vaddr);
 726         cp->vport          = vport;
 727         /* proto should only be IPPROTO_IP if d_addr is a fwmark */
 728         ip_vs_addr_copy(proto == IPPROTO_IP ? AF_UNSPEC : af,
 729                         &cp->daddr, daddr);
 730         cp->dport          = dport;
 731         cp->flags          = flags;
 732         spin_lock_init(&cp->lock);
 733
 734         /*
 735          * Set the entry is referenced by the current thread before hashing
 736          * it in the table, so that other thread run ip_vs_random_dropentry
 737          * but cannot drop this entry.
 738          */
 739         atomic_set(&cp->refcnt, 1);
 740
 741         atomic_set(&cp->n_control, 0);
 742         atomic_set(&cp->in_pkts, 0);
 743
 744         atomic_inc(&ip_vs_conn_count);
 745         if (flags & IP_VS_CONN_F_NO_CPORT)
 746                 atomic_inc(&ip_vs_conn_no_cport_cnt);
 747
 748         /* Bind the connection with a destination server */
 749         ip_vs_bind_dest(cp, dest);
 750
 751         /* Set its state and timeout */
 752         cp->state = 0;
 753         cp->timeout = 3*HZ;
 754
 755         /* Bind its packet transmitter */
 756 #ifdef CONFIG_IP_VS_IPV6
 757         if (af == AF_INET6)
 758                 ip_vs_bind_xmit_v6(cp);
 759         else
 760 #endif
 761                 ip_vs_bind_xmit(cp);
 762
 763         if (unlikely(pp && atomic_read(&pp->appcnt)))
 764                 ip_vs_bind_app(cp, pp);
 765
 766         /* Hash it in the ip_vs_conn_tab finally */
 767         ip_vs_conn_hash(cp);
 768
 769         return cp;
 770 }
 771
 772
 773 /*
 774  *      /proc/net/ip_vs_conn entries
 775  */
 776 #ifdef CONFIG_PROC_FS
 777
 778 static void *ip_vs_conn_array(struct seq_file *seq, loff_t pos)
 779 {
 780         int idx;
 781         struct ip_vs_conn *cp;
 782
 783         for (idx = 0; idx < ip_vs_conn_tab_size; idx++) {
 784                 ct_read_lock_bh(idx);
 785                 list_for_each_entry(cp, &ip_vs_conn_tab[idx], c_list) {
 786                         if (pos-- == 0) {
 787                                 seq->private = &ip_vs_conn_tab[idx];
 788                                 return cp;
 789                         }
 790                 }
 791                 ct_read_unlock_bh(idx);
 792         }
 793
 794         return NULL;
 795 }
 796
 797 static void *ip_vs_conn_seq_start(struct seq_file *seq, loff_t *pos)
 798 {
 799         seq->private = NULL;
 800         return *pos ? ip_vs_conn_array(seq, *pos - 1) :SEQ_START_TOKEN;
 801 }
 802
 803 static void *ip_vs_conn_seq_next(struct seq_file *seq, void *v, loff_t *pos)
 804 {
 805         struct ip_vs_conn *cp = v;
 806         struct list_head *e, *l = seq->private;
 807         int idx;
 808
 809         ++*pos;
 810         if (v == SEQ_START_TOKEN)
 811                 return ip_vs_conn_array(seq, 0);
 812
 813         /* more on same hash chain? */
 814         if ((e = cp->c_list.next) != l)
 815                 return list_entry(e, struct ip_vs_conn, c_list);
 816
 817         idx = l - ip_vs_conn_tab;
 818         ct_read_unlock_bh(idx);
 819
 820         while (++idx < ip_vs_conn_tab_size) {
 821                 ct_read_lock_bh(idx);
 822                 list_for_each_entry(cp, &ip_vs_conn_tab[idx], c_list) {
 823                         seq->private = &ip_vs_conn_tab[idx];
 824                         return cp;
 825                 }
 826                 ct_read_unlock_bh(idx);
 827         }
 828         seq->private = NULL;
 829         return NULL;
 830 }
 831
 832 static void ip_vs_conn_seq_stop(struct seq_file *seq, void *v)
 833 {
 834         struct list_head *l = seq->private;
 835
 836         if (l)
 837                 ct_read_unlock_bh(l - ip_vs_conn_tab);
 838 }
 839
 840 static int ip_vs_conn_seq_show(struct seq_file *seq, void *v)
 841 {
 842
 843         if (v == SEQ_START_TOKEN)
 844                 seq_puts(seq,
 845    "Pro FromIP   FPrt ToIP     TPrt DestIP   DPrt State       Expires\n");
 846         else {
 847                 const struct ip_vs_conn *cp = v;
 848
 849 #ifdef CONFIG_IP_VS_IPV6
 850                 if (cp->af == AF_INET6)
 851                         seq_printf(seq, "%-3s %pI6 %04X %pI6 %04X %pI6 %04X %-11s %7lu\n",
 852                                 ip_vs_proto_name(cp->protocol),
 853                                 &cp->caddr.in6, ntohs(cp->cport),
 854                                 &cp->vaddr.in6, ntohs(cp->vport),
 855                                 &cp->daddr.in6, ntohs(cp->dport),
 856                                 ip_vs_state_name(cp->protocol, cp->state),
 857                                 (cp->timer.expires-jiffies)/HZ);
 858                 else
 859 #endif
 860                         seq_printf(seq,
 861                                 "%-3s %08X %04X %08X %04X"
 862                                 " %08X %04X %-11s %7lu\n",
 863                                 ip_vs_proto_name(cp->protocol),
 864                                 ntohl(cp->caddr.ip), ntohs(cp->cport),
 865                                 ntohl(cp->vaddr.ip), ntohs(cp->vport),
 866                                 ntohl(cp->daddr.ip), ntohs(cp->dport),
 867                                 ip_vs_state_name(cp->protocol, cp->state),
 868                                 (cp->timer.expires-jiffies)/HZ);
 869         }
 870         return 0;
 871 }
 872
 873 static const struct seq_operations ip_vs_conn_seq_ops = {
 874         .start = ip_vs_conn_seq_start,
 875         .next  = ip_vs_conn_seq_next,
 876         .stop  = ip_vs_conn_seq_stop,
 877         .show  = ip_vs_conn_seq_show,
 878 };
 879
 880 static int ip_vs_conn_open(struct inode *inode, struct file *file)
 881 {
 882         return seq_open(file, &ip_vs_conn_seq_ops);
 883 }
 884
 885 static const struct file_operations ip_vs_conn_fops = {
 886         .owner   = THIS_MODULE,
 887         .open    = ip_vs_conn_open,
 888         .read    = seq_read,
 889         .llseek  = seq_lseek,
 890         .release = seq_release,
 891 };
 892
 893 static const char *ip_vs_origin_name(unsigned flags)
 894 {
 895         if (flags & IP_VS_CONN_F_SYNC)
 896                 return "SYNC";
 897         else
 898                 return "LOCAL";
 899 }
 900
 901 static int ip_vs_conn_sync_seq_show(struct seq_file *seq, void *v)
 902 {
 903
 904         if (v == SEQ_START_TOKEN)
 905                 seq_puts(seq,
 906    "Pro FromIP   FPrt ToIP     TPrt DestIP   DPrt State       Origin Expires\n");
 907         else {
 908                 const struct ip_vs_conn *cp = v;
 909
 910 #ifdef CONFIG_IP_VS_IPV6
 911                 if (cp->af == AF_INET6)
 912                         seq_printf(seq, "%-3s %pI6 %04X %pI6 %04X %pI6 %04X %-11s %-6s %7lu\n",
 913                                 ip_vs_proto_name(cp->protocol),
 914                                 &cp->caddr.in6, ntohs(cp->cport),
 915                                 &cp->vaddr.in6, ntohs(cp->vport),
 916                                 &cp->daddr.in6, ntohs(cp->dport),
 917                                 ip_vs_state_name(cp->protocol, cp->state),
 918                                 ip_vs_origin_name(cp->flags),
 919                                 (cp->timer.expires-jiffies)/HZ);
 920                 else
 921 #endif
 922                         seq_printf(seq,
 923                                 "%-3s %08X %04X %08X %04X "
 924                                 "%08X %04X %-11s %-6s %7lu\n",
 925                                 ip_vs_proto_name(cp->protocol),
 926                                 ntohl(cp->caddr.ip), ntohs(cp->cport),
 927                                 ntohl(cp->vaddr.ip), ntohs(cp->vport),
 928                                 ntohl(cp->daddr.ip), ntohs(cp->dport),
 929                                 ip_vs_state_name(cp->protocol, cp->state),
 930                                 ip_vs_origin_name(cp->flags),
 931                                 (cp->timer.expires-jiffies)/HZ);
 932         }
 933         return 0;
 934 }
 935
 936 static const struct seq_operations ip_vs_conn_sync_seq_ops = {
 937         .start = ip_vs_conn_seq_start,
 938         .next  = ip_vs_conn_seq_next,
 939         .stop  = ip_vs_conn_seq_stop,
 940         .show  = ip_vs_conn_sync_seq_show,
 941 };
 942
 943 static int ip_vs_conn_sync_open(struct inode *inode, struct file *file)
 944 {
 945         return seq_open(file, &ip_vs_conn_sync_seq_ops);
 946 }
 947
 948 static const struct file_operations ip_vs_conn_sync_fops = {
 949         .owner   = THIS_MODULE,
 950         .open    = ip_vs_conn_sync_open,
 951         .read    = seq_read,
 952         .llseek  = seq_lseek,
 953         .release = seq_release,
 954 };
 955
 956 #endif
 957
 958
 959 /*
 960  *      Randomly drop connection entries before running out of memory
 961  */
 962 static inline int todrop_entry(struct ip_vs_conn *cp)
 963 {
 964         /*
 965          * The drop rate array needs tuning for real environments.
 966          * Called from timer bh only => no locking
 967          */
 968         static const char todrop_rate[9] = {0, 1, 2, 3, 4, 5, 6, 7, 8};
 969         static char todrop_counter[9] = {0};
 970         int i;
 971
 972         /* if the conn entry hasn't lasted for 60 seconds, don't drop it.
 973            This will leave enough time for normal connection to get
 974            through. */
 975         if (time_before(cp->timeout + jiffies, cp->timer.expires + 60*HZ))
 976                 return 0;
 977
 978         /* Don't drop the entry if its number of incoming packets is not
 979            located in [0, 8] */
 980         i = atomic_read(&cp->in_pkts);
 981         if (i > 8 || i < 0) return 0;
 982
 983         if (!todrop_rate[i]) return 0;
 984         if (--todrop_counter[i] > 0) return 0;
 985
 986         todrop_counter[i] = todrop_rate[i];
 987         return 1;
 988 }
 989
 990 /* Called from keventd and must protect itself from softirqs */
 991 void ip_vs_random_dropentry(void)
 992 {
 993         int idx;
 994         struct ip_vs_conn *cp;
 995
 996         /*
 997          * Randomly scan 1/32 of the whole table every second
 998          */
 999         for (idx = 0; idx < (ip_vs_conn_tab_size>>5); idx++) {
1000                 unsigned hash = net_random() & ip_vs_conn_tab_mask;
1001
1002                 /*
1003                  *  Lock is actually needed in this loop.
1004                  */
1005                 ct_write_lock_bh(hash);
1006
1007                 list_for_each_entry(cp, &ip_vs_conn_tab[hash], c_list) {
1008                         if (cp->flags & IP_VS_CONN_F_TEMPLATE)
1009                                 /* connection template */
1010                                 continue;
1011
1012                         if (cp->protocol == IPPROTO_TCP) {
1013                                 switch(cp->state) {
1014                                 case IP_VS_TCP_S_SYN_RECV:
1015                                 case IP_VS_TCP_S_SYNACK:
1016                                         break;
1017
1018                                 case IP_VS_TCP_S_ESTABLISHED:
1019                                         if (todrop_entry(cp))
1020                                                 break;
1021                                         continue;
1022
1023                                 default:
1024                                         continue;
1025                                 }
1026                         } else {
1027                                 if (!todrop_entry(cp))
1028                                         continue;
1029                         }
1030
1031                         IP_VS_DBG(4, "del connection\n");
1032                         ip_vs_conn_expire_now(cp);
1033                         if (cp->control) {
1034                                 IP_VS_DBG(4, "del conn template\n");
1035                                 ip_vs_conn_expire_now(cp->control);
1036                         }
1037                 }
1038                 ct_write_unlock_bh(hash);
1039         }
1040 }
1041
1042
1043 /*
1044  *      Flush all the connection entries in the ip_vs_conn_tab
1045  */
1046 static void ip_vs_conn_flush(void)
1047 {
1048         int idx;
1049         struct ip_vs_conn *cp;
1050
1051   flush_again:
1052         for (idx = 0; idx < ip_vs_conn_tab_size; idx++) {
1053                 /*
1054                  *  Lock is actually needed in this loop.
1055                  */
1056                 ct_write_lock_bh(idx);
1057
1058                 list_for_each_entry(cp, &ip_vs_conn_tab[idx], c_list) {
1059
1060                         IP_VS_DBG(4, "del connection\n");
1061                         ip_vs_conn_expire_now(cp);
1062                         if (cp->control) {
1063                                 IP_VS_DBG(4, "del conn template\n");
1064                                 ip_vs_conn_expire_now(cp->control);
1065                         }
1066                 }
1067                 ct_write_unlock_bh(idx);
1068         }
1069
1070         /* the counter may be not NULL, because maybe some conn entries
1071            are run by slow timer handler or unhashed but still referred */
1072         if (atomic_read(&ip_vs_conn_count) != 0) {
1073                 schedule();
1074                 goto flush_again;
1075         }
1076 }
1077
1078
1079 int __init ip_vs_conn_init(void)
1080 {
1081         int idx;
1082
1083         /* Compute size and mask */
1084         ip_vs_conn_tab_size = 1 << ip_vs_conn_tab_bits;
1085         ip_vs_conn_tab_mask = ip_vs_conn_tab_size - 1;
1086
1087         /*
1088          * Allocate the connection hash table and initialize its list heads
1089          */
1090         ip_vs_conn_tab = vmalloc(ip_vs_conn_tab_size *
1091                                  sizeof(struct list_head));
1092         if (!ip_vs_conn_tab)
1093                 return -ENOMEM;
1094
1095         /* Allocate ip_vs_conn slab cache */
1096         ip_vs_conn_cachep = kmem_cache_create("ip_vs_conn",
1097                                               sizeof(struct ip_vs_conn), 0,
1098                                               SLAB_HWCACHE_ALIGN, NULL);
1099         if (!ip_vs_conn_cachep) {
1100                 vfree(ip_vs_conn_tab);
1101                 return -ENOMEM;
1102         }
1103
1104         pr_info("Connection hash table configured "
1105                 "(size=%d, memory=%ldKbytes)\n",
1106                 ip_vs_conn_tab_size,
1107                 (long)(ip_vs_conn_tab_size*sizeof(struct list_head))/1024);
1108         IP_VS_DBG(0, "Each connection entry needs %Zd bytes at least\n",
1109                   sizeof(struct ip_vs_conn));
1110
1111         for (idx = 0; idx < ip_vs_conn_tab_size; idx++) {
1112                 INIT_LIST_HEAD(&ip_vs_conn_tab[idx]);
1113         }
1114
1115         for (idx = 0; idx < CT_LOCKARRAY_SIZE; idx++)  {
1116                 rwlock_init(&__ip_vs_conntbl_lock_array[idx].l);
1117         }
1118
1119         proc_net_fops_create(&init_net, "ip_vs_conn", 0, &ip_vs_conn_fops);
1120         proc_net_fops_create(&init_net, "ip_vs_conn_sync", 0, &ip_vs_conn_sync_fops);
1121
1122         /* calculate the random value for connection hash */
1123         get_random_bytes(&ip_vs_conn_rnd, sizeof(ip_vs_conn_rnd));
1124
1125         return 0;
1126 }
1127
1128
1129 void ip_vs_conn_cleanup(void)
1130 {
1131         /* flush all the connection entries first */
1132         ip_vs_conn_flush();
1133
1134         /* Release the empty cache */
1135         kmem_cache_destroy(ip_vs_conn_cachep);
1136         proc_net_remove(&init_net, "ip_vs_conn");
1137         proc_net_remove(&init_net, "ip_vs_conn_sync");
1138         vfree(ip_vs_conn_tab);
1139 }