1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  23  * Copyright 2019 Joyent, Inc.
  24  * Copyright (c) 2013, OmniTI Computer Consulting, Inc. All rights reserved.
  25  * Copyright (c) 2013, 2017 by Delphix. All rights reserved.
  26  */
  27 
  28 #ifndef _INET_TCP_IMPL_H
  29 #define _INET_TCP_IMPL_H
  30 
  31 /*
  32  * TCP implementation private declarations.  These interfaces are
  33  * used to build the IP module and are not meant to be accessed
  34  * by any modules except IP itself.  They are undocumented and are
  35  * subject to change without notice.
  36  */
  37 
  38 #ifdef  __cplusplus
  39 extern "C" {
  40 #endif
  41 
  42 #ifdef _KERNEL
  43 
  44 #include <sys/cpuvar.h>
  45 #include <sys/clock_impl.h>       /* For LBOLT_FASTPATH{,64} */
  46 #include <inet/optcom.h>
  47 #include <inet/tcp.h>
  48 #include <inet/tunables.h>
  49 
  50 #define TCP_MOD_ID      5105
  51 
  52 extern struct qinit     tcp_sock_winit;
  53 extern struct qinit     tcp_winit;
  54 
  55 extern sock_downcalls_t sock_tcp_downcalls;
  56 
  57 /*
  58  * Note that by default, the _snd_lowat_fraction tunable controls the value of
  59  * the transmit low water mark.  TCP_XMIT_LOWATER (and thus the _xmit_lowat
  60  * property) is only used if the administrator has disabled _snd_lowat_fraction
  61  * by setting it to 0.
  62  */
  63 #define TCP_XMIT_LOWATER        4096
  64 #define TCP_XMIT_HIWATER        49152
  65 #define TCP_RECV_LOWATER        2048
  66 #define TCP_RECV_HIWATER        128000
  67 
  68 /*
  69  * Bind hash list size and has function.  It has to be a power of 2 for
  70  * hashing.
  71  */
  72 #define TCP_BIND_FANOUT_SIZE    1024
  73 #define TCP_BIND_HASH(lport) (ntohs(lport) & (TCP_BIND_FANOUT_SIZE - 1))
  74 
  75 /*
  76  * This implementation follows the 4.3BSD interpretation of the urgent
  77  * pointer and not RFC 1122. Switching to RFC 1122 behavior would cause
  78  * incompatible changes in protocols like telnet and rlogin.
  79  */
  80 #define TCP_OLD_URP_INTERPRETATION      1
  81 
  82 /* TCP option length */
  83 #define TCPOPT_NOP_LEN          1
  84 #define TCPOPT_MAXSEG_LEN       4
  85 #define TCPOPT_WS_LEN           3
  86 #define TCPOPT_REAL_WS_LEN      (TCPOPT_WS_LEN+1)
  87 #define TCPOPT_TSTAMP_LEN       10
  88 #define TCPOPT_REAL_TS_LEN      (TCPOPT_TSTAMP_LEN+2)
  89 #define TCPOPT_SACK_OK_LEN      2
  90 #define TCPOPT_REAL_SACK_OK_LEN (TCPOPT_SACK_OK_LEN+2)
  91 #define TCPOPT_REAL_SACK_LEN    4
  92 #define TCPOPT_MAX_SACK_LEN     36
  93 #define TCPOPT_HEADER_LEN       2
  94 
  95 /* Round up the value to the nearest mss. */
  96 #define MSS_ROUNDUP(value, mss)         ((((value) - 1) / (mss) + 1) * (mss))
  97 
  98 /*
  99  * Was this tcp created via socket() interface?
 100  */
 101 #define TCP_IS_SOCKET(tcp)      ((tcp)->tcp_issocket)
 102 
 103 /*
 104  * Is this tcp not attached to any upper client?
 105  */
 106 #define TCP_IS_DETACHED(tcp)    ((tcp)->tcp_detached)
 107 
 108 /* TCP timers related data structures.  Refer to tcp_timers.c. */
 109 typedef struct tcp_timer_s {
 110         conn_t  *connp;
 111         void    (*tcpt_proc)(void *);
 112         callout_id_t   tcpt_tid;
 113 } tcp_timer_t;
 114 
 115 extern kmem_cache_t *tcp_timercache;
 116 
 117 /*
 118  * Macro for starting various timers.  Retransmission timer has its own macro,
 119  * TCP_TIMER_RESTART().  tim is in millisec.
 120  */
 121 #define TCP_TIMER(tcp, f, tim)          \
 122         tcp_timeout(tcp->tcp_connp, f, tim)
 123 #define TCP_TIMER_CANCEL(tcp, id)       \
 124         tcp_timeout_cancel(tcp->tcp_connp, id)
 125 
 126 /*
 127  * To restart the TCP retransmission timer.  intvl is in millisec.
 128  */
 129 #define TCP_TIMER_RESTART(tcp, intvl) {                                 \
 130         if ((tcp)->tcp_timer_tid != 0)                                       \
 131                 (void) TCP_TIMER_CANCEL((tcp), (tcp)->tcp_timer_tid);        \
 132         (tcp)->tcp_timer_tid = TCP_TIMER((tcp), tcp_timer, (intvl)); \
 133 }
 134 
 135 
 136 /*
 137  * Maximum TIME_WAIT timeout.  It is defined here (instead of tcp_tunables.c)
 138  * so that other parameters can be derived from it.
 139  */
 140 #define TCP_TIME_WAIT_MAX       (10 * MINUTES)
 141 
 142 /*
 143  * TCP_TIME_WAIT_DELAY governs how often the time_wait_collector runs.
 144  * Running it every 5 seconds seems to yield a reasonable balance between
 145  * cleanup liveliness and system load.
 146  */
 147 #define TCP_TIME_WAIT_DELAY     (5 * SECONDS)
 148 
 149 #define TCP_TIME_WAIT_BUCKETS   ((TCP_TIME_WAIT_MAX / TCP_TIME_WAIT_DELAY) + 1)
 150 
 151 /*
 152  * For scalability, we must not run a timer for every TCP connection
 153  * in TIME_WAIT state.  To see why, consider (for time wait interval of
 154  * 1 minutes):
 155  *      10,000 connections/sec * 60 seconds/time wait = 600,000 active conn's
 156  *
 157  * Since TIME_WAIT expiration occurs on a per-squeue basis, handling
 158  * connections from all netstacks on the system, a simple queue is inadequate
 159  * for pending entries.  This is because tcp_time_wait_interval may differ
 160  * between connections, causing tail insertion to violate expiration order.
 161  *
 162  * Instead of performing expensive sorting or unnecessary list traversal to
 163  * counteract interval variance between netstacks, a timing wheel structure is
 164  * used.  The duration covered by each bucket in the wheel is determined by the
 165  * TCP_TIME_WAIT_DELAY (5 seconds).  The number of buckets in the wheel is
 166  * determined by dividing the maximum TIME_WAIT interval (10 minutes) by
 167  * TCP_TIME_WAIT_DELAY, with one added bucket for rollover protection.
 168  * (Yielding 121 buckets with the current parameters)  When items are inserted
 169  * into the set of buckets, they are indexed by using their expiration time
 170  * divided by the bucket size, modulo the number of buckets.  This means that
 171  * when each bucket is processed, all items within should have expired within
 172  * the last TCP_TIME_WAIT_DELAY interval.
 173  *
 174  * Since bucket timer schedules are rounded to the nearest TCP_TIME_WAIT_DELAY
 175  * interval to ensure all connections in the pending bucket will be expired, a
 176  * per-squeue offset is used when doing TIME_WAIT scheduling.  This offset is
 177  * between 0 and the TCP_TIME_WAIT_DELAY and is designed to avoid scheduling
 178  * all of the tcp_time_wait_collector threads to run in lock-step.  The offset
 179  * is fixed while there are any connections present in the buckets.
 180  *
 181  * When a tcp_t enters TIME_WAIT state, a timer is started (timeout is
 182  * tcps_time_wait_interval).  When the tcp_t is detached (upper layer closes
 183  * the end point), it is scheduled to be cleaned up by the squeue-driving
 184  * tcp_time_wait_collector (also using tcps_time_wait_interval).  This means
 185  * that the TIME_WAIT state can be extended (up to doubled) if the tcp_t
 186  * doesn't become detached for a long time.
 187  *
 188  * The list manipulations (including tcp_time_wait_next/prev)
 189  * are protected by the tcp_time_wait_lock. The content of the
 190  * detached TIME_WAIT connections is protected by the normal perimeters.
 191  *
 192  * These connection lists are per squeue and squeues are shared across the
 193  * tcp_stack_t instances.  Things in a tcp_time_wait_bucket remain associated
 194  * with the tcp_stack_t and conn_netstack.  Any tcp_t connections stored in the
 195  * tcp_free_list are disassociated and have NULL tcp_tcps and conn_netstack
 196  * pointers.
 197  */
 198 typedef struct tcp_squeue_priv_s {
 199         kmutex_t        tcp_time_wait_lock;
 200         boolean_t       tcp_time_wait_collector_active;
 201         callout_id_t    tcp_time_wait_tid;
 202         uint64_t        tcp_time_wait_cnt;
 203         int64_t         tcp_time_wait_schedule;
 204         int64_t         tcp_time_wait_offset;
 205         tcp_t           *tcp_time_wait_bucket[TCP_TIME_WAIT_BUCKETS];
 206         tcp_t           *tcp_free_list;
 207         uint_t          tcp_free_list_cnt;
 208 } tcp_squeue_priv_t;
 209 
 210 /*
 211  * Parameters for TCP Initial Send Sequence number (ISS) generation.  When
 212  * tcp_strong_iss is set to 1, which is the default, the ISS is calculated
 213  * by adding three components: a time component which grows by 1 every 4096
 214  * nanoseconds (versus every 4 microseconds suggested by RFC 793, page 27);
 215  * a per-connection component which grows by 125000 for every new connection;
 216  * and an "extra" component that grows by a random amount centered
 217  * approximately on 64000.  This causes the ISS generator to cycle every
 218  * 4.89 hours if no TCP connections are made, and faster if connections are
 219  * made.
 220  *
 221  * When tcp_strong_iss is set to 0, ISS is calculated by adding two
 222  * components: a time component which grows by 250000 every second; and
 223  * a per-connection component which grows by 125000 for every new connections.
 224  *
 225  * A third method, when tcp_strong_iss is set to 2, for generating ISS is
 226  * prescribed by Steve Bellovin.  This involves adding time, the 125000 per
 227  * connection, and a one-way hash (MD5) of the connection ID <sport, dport,
 228  * src, dst>, a "truly" random (per RFC 1750) number, and a console-entered
 229  * password.
 230  */
 231 #define ISS_INCR        250000
 232 #define ISS_NSEC_SHT    12
 233 
 234 /* Macros for timestamp comparisons */
 235 #define TSTMP_GEQ(a, b) ((int32_t)((a)-(b)) >= 0)
 236 #define TSTMP_LT(a, b)  ((int32_t)((a)-(b)) < 0)
 237 
 238 /*
 239  * Initialize cwnd according to RFC 3390.  def_max_init_cwnd is
 240  * either tcp_slow_start_initial or tcp_slow_start_after idle
 241  * depending on the caller.  If the upper layer has not used the
 242  * TCP_INIT_CWND option to change the initial cwnd, tcp_init_cwnd
 243  * should be 0 and we use the formula in RFC 3390 to set tcp_cwnd.
 244  * If the upper layer has changed set the tcp_init_cwnd, just use
 245  * it to calculate the tcp_cwnd.
 246  *
 247  * "An Argument for Increasing TCP's Initial Congestion Window"
 248  * ACM SIGCOMM Computer Communications Review, vol. 40 (2010), pp. 27-33
 249  *  -- Nandita Dukkipati, Tiziana Refice, Yuchung Cheng,
 250  *     Hsiao-keng Jerry Chu, Tom Herbert, Amit Agarwal,
 251  *     Arvind Jain, Natalia Sutin
 252  *
 253  *   "Based on the results from our experiments, we believe the
 254  *    initial congestion window should be at least ten segments
 255  *    and the same be investigated for standardization by the IETF."
 256  *
 257  * As such, the def_max_init_cwnd argument with which this macro is
 258  * invoked is either the tcps_slow_start_initial or
 259  * tcps_slow_start_after_idle which both default to 0 and will respect
 260  * RFC 3390 exactly.  If the tunables are explicitly set by the operator,
 261  * then the initial congestion window should be set as the operator
 262  * demands, within reason. We shall arbitrarily define reason as a
 263  * maximum of 16 (same as used by the TCP_INIT_CWND setsockopt).
 264  */
 265 
 266 /* Maximum TCP initial cwin (start/restart). */
 267 #define TCP_MAX_INIT_CWND       16
 268 
 269 #define TCP_SET_INIT_CWND(tcp, mss, def_max_init_cwnd)                  \
 270 {                                                                       \
 271         if ((tcp)->tcp_init_cwnd == 0) {                             \
 272                 if (def_max_init_cwnd == 0) {                           \
 273                         (tcp)->tcp_cwnd = MIN(4 * (mss),             \
 274                             MAX(2 * (mss), 4380 / (mss) * (mss)));      \
 275                 } else {                                                \
 276                         (tcp)->tcp_cwnd = MIN(TCP_MAX_INIT_CWND * (mss),\
 277                             def_max_init_cwnd * (mss));                 \
 278                 }                                                       \
 279         } else {                                                        \
 280                 (tcp)->tcp_cwnd = (tcp)->tcp_init_cwnd * (mss);           \
 281         }                                                               \
 282         tcp->tcp_cwnd_cnt = 0;                                               \
 283 }
 284 
 285 /*
 286  * Set ECN capable transport (ECT) code point in IP header.
 287  *
 288  * Note that there are 2 ECT code points '01' and '10', which are called
 289  * ECT(1) and ECT(0) respectively.  Here we follow the original ECT code
 290  * point ECT(0) for TCP as described in RFC 2481.
 291  */
 292 #define TCP_SET_ECT(tcp, iph) \
 293         if ((tcp)->tcp_connp->conn_ipversion == IPV4_VERSION) { \
 294                 /* We need to clear the code point first. */ \
 295                 ((ipha_t *)(iph))->ipha_type_of_service &= 0xFC; \
 296                 ((ipha_t *)(iph))->ipha_type_of_service |= IPH_ECN_ECT0; \
 297         } else { \
 298                 ((ip6_t *)(iph))->ip6_vcf &= htonl(0xFFCFFFFF); \
 299                 ((ip6_t *)(iph))->ip6_vcf |= htonl(IPH_ECN_ECT0 << 20); \
 300         }
 301 
 302 /*
 303  * TCP options struct returned from tcp_parse_options.
 304  */
 305 typedef struct tcp_opt_s {
 306         uint32_t        tcp_opt_mss;
 307         uint32_t        tcp_opt_wscale;
 308         uint32_t        tcp_opt_ts_val;
 309         uint32_t        tcp_opt_ts_ecr;
 310         tcp_t           *tcp;
 311 } tcp_opt_t;
 312 
 313 /*
 314  * Flags returned from tcp_parse_options.
 315  */
 316 #define TCP_OPT_MSS_PRESENT     1
 317 #define TCP_OPT_WSCALE_PRESENT  2
 318 #define TCP_OPT_TSTAMP_PRESENT  4
 319 #define TCP_OPT_SACK_OK_PRESENT 8
 320 #define TCP_OPT_SACK_PRESENT    16
 321 
 322 /*
 323  * Write-side flow-control is implemented via the per instance STREAMS
 324  * write-side Q by explicitly setting QFULL to stop the flow of mblk_t(s)
 325  * and clearing QFULL and calling qbackenable() to restart the flow based
 326  * on the number of TCP unsent bytes (i.e. those not on the wire waiting
 327  * for a remote ACK).
 328  *
 329  * This is different than a standard STREAMS kmod which when using the
 330  * STREAMS Q the framework would automatictly flow-control based on the
 331  * defined hiwat/lowat values as mblk_t's are enqueued/dequeued.
 332  *
 333  * As of FireEngine TCP write-side flow-control needs to take into account
 334  * both the unsent tcp_xmit list bytes but also any squeue_t enqueued bytes
 335  * (i.e. from tcp_wput() -> tcp_output()).
 336  *
 337  * This is accomplished by adding a new tcp_t fields, tcp_squeue_bytes, to
 338  * count the number of bytes enqueued by tcp_wput() and the number of bytes
 339  * dequeued and processed by tcp_output().
 340  *
 341  * So, the total number of bytes unsent is (squeue_bytes + unsent) with all
 342  * flow-control uses of unsent replaced with the macro TCP_UNSENT_BYTES.
 343  */
 344 extern void     tcp_clrqfull(tcp_t *);
 345 extern void     tcp_setqfull(tcp_t *);
 346 
 347 #define TCP_UNSENT_BYTES(tcp) \
 348         ((tcp)->tcp_squeue_bytes + (tcp)->tcp_unsent)
 349 
 350 /*
 351  * Linked list struct to store listener connection limit configuration per
 352  * IP stack.  The list is stored at tcps_listener_conf in tcp_stack_t.
 353  *
 354  * tl_port: the listener port of this limit configuration
 355  * tl_ratio: the maximum amount of memory consumed by all concurrent TCP
 356  *           connections created by a listener does not exceed 1/tl_ratio
 357  *           of the total system memory.  Note that this is only an
 358  *           approximation.
 359  * tl_link: linked list struct
 360  */
 361 typedef struct tcp_listener_s {
 362         in_port_t       tl_port;
 363         uint32_t        tl_ratio;
 364         list_node_t     tl_link;
 365 } tcp_listener_t;
 366 
 367 /*
 368  * If there is a limit set on the number of connections allowed per each
 369  * listener, the following struct is used to store that counter.  It keeps
 370  * the number of TCP connection created by a listener.  Note that this needs
 371  * to be separated from the listener since the listener can go away before
 372  * all the connections are gone.
 373  *
 374  * When the struct is allocated, tlc_cnt is set to 1.  When a new connection
 375  * is created by the listener, tlc_cnt is incremented by 1.  When a connection
 376  * created by the listener goes away, tlc_count is decremented by 1.  When the
 377  * listener itself goes away, tlc_cnt is decremented  by one.  The last
 378  * connection (or the listener) which decrements tlc_cnt to zero frees the
 379  * struct.
 380  *
 381  * tlc_max is the maximum number of concurrent TCP connections created from a
 382  * listner.  It is calculated when the tcp_listen_cnt_t is allocated.
 383  *
 384  * tlc_report_time stores the time when cmn_err() is called to report that the
 385  * max has been exceeeded.  Report is done at most once every
 386  * TCP_TLC_REPORT_INTERVAL mins for a listener.
 387  *
 388  * tlc_drop stores the number of connection attempt dropped because the
 389  * limit has reached.
 390  */
 391 typedef struct tcp_listen_cnt_s {
 392         uint32_t        tlc_max;
 393         uint32_t        tlc_cnt;
 394         int64_t         tlc_report_time;
 395         uint32_t        tlc_drop;
 396 } tcp_listen_cnt_t;
 397 
 398 #define TCP_TLC_REPORT_INTERVAL (30 * MINUTES)
 399 
 400 #define TCP_DECR_LISTEN_CNT(tcp)                                        \
 401 {                                                                       \
 402         ASSERT((tcp)->tcp_listen_cnt->tlc_cnt > 0);                    \
 403         if (atomic_dec_32_nv(&(tcp)->tcp_listen_cnt->tlc_cnt) == 0) \
 404                 kmem_free((tcp)->tcp_listen_cnt, sizeof (tcp_listen_cnt_t)); \
 405         (tcp)->tcp_listen_cnt = NULL;                                        \
 406 }
 407 
 408 /* Increment and decrement the number of connections in tcp_stack_t. */
 409 #define TCPS_CONN_INC(tcps)                                             \
 410         atomic_inc_64(                                                  \
 411             (uint64_t *)&(tcps)->tcps_sc[CPU->cpu_seqid]->tcp_sc_conn_cnt)
 412 
 413 #define TCPS_CONN_DEC(tcps)                                             \
 414         atomic_dec_64(                                                  \
 415             (uint64_t *)&(tcps)->tcps_sc[CPU->cpu_seqid]->tcp_sc_conn_cnt)
 416 
 417 /*
 418  * When the system is under memory pressure, stack variable tcps_reclaim is
 419  * true, we shorten the connection timeout abort interval to tcp_early_abort
 420  * seconds.  Defined in tcp.c.
 421  */
 422 extern uint32_t tcp_early_abort;
 423 
 424 /*
 425  * To reach to an eager in Q0 which can be dropped due to an incoming
 426  * new SYN request when Q0 is full, a new doubly linked list is
 427  * introduced. This list allows to select an eager from Q0 in O(1) time.
 428  * This is needed to avoid spending too much time walking through the
 429  * long list of eagers in Q0 when tcp_drop_q0() is called. Each member of
 430  * this new list has to be a member of Q0.
 431  * This list is headed by listener's tcp_t. When the list is empty,
 432  * both the pointers - tcp_eager_next_drop_q0 and tcp_eager_prev_drop_q0,
 433  * of listener's tcp_t point to listener's tcp_t itself.
 434  *
 435  * Given an eager in Q0 and a listener, MAKE_DROPPABLE() puts the eager
 436  * in the list. MAKE_UNDROPPABLE() takes the eager out of the list.
 437  * These macros do not affect the eager's membership to Q0.
 438  */
 439 #define MAKE_DROPPABLE(listener, eager)                                 \
 440         if ((eager)->tcp_eager_next_drop_q0 == NULL) {                       \
 441                 (listener)->tcp_eager_next_drop_q0->tcp_eager_prev_drop_q0\
 442                     = (eager);                                          \
 443                 (eager)->tcp_eager_prev_drop_q0 = (listener);                \
 444                 (eager)->tcp_eager_next_drop_q0 =                    \
 445                     (listener)->tcp_eager_next_drop_q0;                      \
 446                 (listener)->tcp_eager_next_drop_q0 = (eager);                \
 447         }
 448 
 449 #define MAKE_UNDROPPABLE(eager)                                         \
 450         if ((eager)->tcp_eager_next_drop_q0 != NULL) {                       \
 451                 (eager)->tcp_eager_next_drop_q0->tcp_eager_prev_drop_q0   \
 452                     = (eager)->tcp_eager_prev_drop_q0;                       \
 453                 (eager)->tcp_eager_prev_drop_q0->tcp_eager_next_drop_q0   \
 454                     = (eager)->tcp_eager_next_drop_q0;                       \
 455                 (eager)->tcp_eager_prev_drop_q0 = NULL;                      \
 456                 (eager)->tcp_eager_next_drop_q0 = NULL;                      \
 457         }
 458 
 459 /*
 460  * The format argument to pass to tcp_display().
 461  * DISP_PORT_ONLY means that the returned string has only port info.
 462  * DISP_ADDR_AND_PORT means that the returned string also contains the
 463  * remote and local IP address.
 464  */
 465 #define DISP_PORT_ONLY          1
 466 #define DISP_ADDR_AND_PORT      2
 467 
 468 #define IP_ADDR_CACHE_SIZE      2048
 469 #define IP_ADDR_CACHE_HASH(faddr)                                       \
 470         (ntohl(faddr) & (IP_ADDR_CACHE_SIZE -1))
 471 
 472 /*
 473  * TCP reassembly macros.  We hide starting and ending sequence numbers in
 474  * b_next and b_prev of messages on the reassembly queue.  The messages are
 475  * chained using b_cont.  These macros are used in tcp_reass() so we don't
 476  * have to see the ugly casts and assignments.
 477  */
 478 #define TCP_REASS_SEQ(mp)               ((uint32_t)(uintptr_t)((mp)->b_next))
 479 #define TCP_REASS_SET_SEQ(mp, u)        ((mp)->b_next = \
 480                                         (mblk_t *)(uintptr_t)(u))
 481 #define TCP_REASS_END(mp)               ((uint32_t)(uintptr_t)((mp)->b_prev))
 482 #define TCP_REASS_SET_END(mp, u)        ((mp)->b_prev = \
 483                                         (mblk_t *)(uintptr_t)(u))
 484 
 485 #define tcps_time_wait_interval         tcps_propinfo_tbl[0].prop_cur_uval
 486 #define tcps_conn_req_max_q             tcps_propinfo_tbl[1].prop_cur_uval
 487 #define tcps_conn_req_max_q0            tcps_propinfo_tbl[2].prop_cur_uval
 488 #define tcps_conn_req_min               tcps_propinfo_tbl[3].prop_cur_uval
 489 #define tcps_conn_grace_period          tcps_propinfo_tbl[4].prop_cur_uval
 490 #define tcps_cwnd_max_                  tcps_propinfo_tbl[5].prop_cur_uval
 491 #define tcps_dbg                        tcps_propinfo_tbl[6].prop_cur_uval
 492 #define tcps_smallest_nonpriv_port      tcps_propinfo_tbl[7].prop_cur_uval
 493 #define tcps_ip_abort_cinterval         tcps_propinfo_tbl[8].prop_cur_uval
 494 #define tcps_ip_abort_linterval         tcps_propinfo_tbl[9].prop_cur_uval
 495 #define tcps_ip_abort_interval          tcps_propinfo_tbl[10].prop_cur_uval
 496 #define tcps_ip_notify_cinterval        tcps_propinfo_tbl[11].prop_cur_uval
 497 #define tcps_ip_notify_interval         tcps_propinfo_tbl[12].prop_cur_uval
 498 #define tcps_ipv4_ttl                   tcps_propinfo_tbl[13].prop_cur_uval
 499 #define tcps_keepalive_interval_high    tcps_propinfo_tbl[14].prop_max_uval
 500 #define tcps_keepalive_interval         tcps_propinfo_tbl[14].prop_cur_uval
 501 #define tcps_keepalive_interval_low     tcps_propinfo_tbl[14].prop_min_uval
 502 #define tcps_maxpsz_multiplier          tcps_propinfo_tbl[15].prop_cur_uval
 503 #define tcps_mss_def_ipv4               tcps_propinfo_tbl[16].prop_cur_uval
 504 #define tcps_mss_max_ipv4               tcps_propinfo_tbl[17].prop_cur_uval
 505 #define tcps_mss_min                    tcps_propinfo_tbl[18].prop_cur_uval
 506 #define tcps_naglim_def                 tcps_propinfo_tbl[19].prop_cur_uval
 507 #define tcps_rexmit_interval_initial_high       \
 508                                         tcps_propinfo_tbl[20].prop_max_uval
 509 #define tcps_rexmit_interval_initial    tcps_propinfo_tbl[20].prop_cur_uval
 510 #define tcps_rexmit_interval_initial_low        \
 511                                         tcps_propinfo_tbl[20].prop_min_uval
 512 #define tcps_rexmit_interval_max_high   tcps_propinfo_tbl[21].prop_max_uval
 513 #define tcps_rexmit_interval_max        tcps_propinfo_tbl[21].prop_cur_uval
 514 #define tcps_rexmit_interval_max_low    tcps_propinfo_tbl[21].prop_min_uval
 515 #define tcps_rexmit_interval_min_high   tcps_propinfo_tbl[22].prop_max_uval
 516 #define tcps_rexmit_interval_min        tcps_propinfo_tbl[22].prop_cur_uval
 517 #define tcps_rexmit_interval_min_low    tcps_propinfo_tbl[22].prop_min_uval
 518 #define tcps_deferred_ack_interval      tcps_propinfo_tbl[23].prop_cur_uval
 519 #define tcps_snd_lowat_fraction         tcps_propinfo_tbl[24].prop_cur_uval
 520 #define tcps_dupack_fast_retransmit     tcps_propinfo_tbl[25].prop_cur_uval
 521 #define tcps_ignore_path_mtu            tcps_propinfo_tbl[26].prop_cur_bval
 522 #define tcps_smallest_anon_port         tcps_propinfo_tbl[27].prop_cur_uval
 523 #define tcps_largest_anon_port          tcps_propinfo_tbl[28].prop_cur_uval
 524 #define tcps_xmit_hiwat                 tcps_propinfo_tbl[29].prop_cur_uval
 525 #define tcps_xmit_lowat                 tcps_propinfo_tbl[30].prop_cur_uval
 526 #define tcps_recv_hiwat                 tcps_propinfo_tbl[31].prop_cur_uval
 527 #define tcps_recv_hiwat_minmss          tcps_propinfo_tbl[32].prop_cur_uval
 528 #define tcps_fin_wait_2_flush_interval_high     \
 529                                         tcps_propinfo_tbl[33].prop_max_uval
 530 #define tcps_fin_wait_2_flush_interval  tcps_propinfo_tbl[33].prop_cur_uval
 531 #define tcps_fin_wait_2_flush_interval_low      \
 532                                         tcps_propinfo_tbl[33].prop_min_uval
 533 #define tcps_max_buf                    tcps_propinfo_tbl[34].prop_cur_uval
 534 #define tcps_strong_iss                 tcps_propinfo_tbl[35].prop_cur_uval
 535 #define tcps_rtt_updates                tcps_propinfo_tbl[36].prop_cur_uval
 536 #define tcps_wscale_always              tcps_propinfo_tbl[37].prop_cur_bval
 537 #define tcps_tstamp_always              tcps_propinfo_tbl[38].prop_cur_bval
 538 #define tcps_tstamp_if_wscale           tcps_propinfo_tbl[39].prop_cur_bval
 539 #define tcps_rexmit_interval_extra      tcps_propinfo_tbl[40].prop_cur_uval
 540 #define tcps_deferred_acks_max          tcps_propinfo_tbl[41].prop_cur_uval
 541 #define tcps_slow_start_after_idle      tcps_propinfo_tbl[42].prop_cur_uval
 542 #define tcps_slow_start_initial         tcps_propinfo_tbl[43].prop_cur_uval
 543 #define tcps_sack_permitted             tcps_propinfo_tbl[44].prop_cur_uval
 544 #define tcps_ipv6_hoplimit              tcps_propinfo_tbl[45].prop_cur_uval
 545 #define tcps_mss_def_ipv6               tcps_propinfo_tbl[46].prop_cur_uval
 546 #define tcps_mss_max_ipv6               tcps_propinfo_tbl[47].prop_cur_uval
 547 #define tcps_rev_src_routes             tcps_propinfo_tbl[48].prop_cur_bval
 548 #define tcps_local_dack_interval        tcps_propinfo_tbl[49].prop_cur_uval
 549 #define tcps_local_dacks_max            tcps_propinfo_tbl[50].prop_cur_uval
 550 #define tcps_ecn_permitted              tcps_propinfo_tbl[51].prop_cur_uval
 551 #define tcps_rst_sent_rate_enabled      tcps_propinfo_tbl[52].prop_cur_bval
 552 #define tcps_rst_sent_rate              tcps_propinfo_tbl[53].prop_cur_uval
 553 #define tcps_push_timer_interval        tcps_propinfo_tbl[54].prop_cur_uval
 554 #define tcps_use_smss_as_mss_opt        tcps_propinfo_tbl[55].prop_cur_bval
 555 #define tcps_keepalive_abort_interval_high \
 556                                         tcps_propinfo_tbl[56].prop_max_uval
 557 #define tcps_keepalive_abort_interval \
 558                                         tcps_propinfo_tbl[56].prop_cur_uval
 559 #define tcps_keepalive_abort_interval_low \
 560                                         tcps_propinfo_tbl[56].prop_min_uval
 561 #define tcps_wroff_xtra                 tcps_propinfo_tbl[57].prop_cur_uval
 562 #define tcps_dev_flow_ctl               tcps_propinfo_tbl[58].prop_cur_bval
 563 #define tcps_reass_timeout              tcps_propinfo_tbl[59].prop_cur_uval
 564 #define tcps_iss_incr                   tcps_propinfo_tbl[65].prop_cur_uval
 565 #define tcps_abc                        tcps_propinfo_tbl[67].prop_cur_bval
 566 #define tcps_abc_l_var                  tcps_propinfo_tbl[68].prop_cur_uval
 567 
 568 
 569 /*
 570  * As defined in RFC 6298, the RTO is the average estimates (SRTT) plus a
 571  * multiple of the deviation estimates (K * RTTVAR):
 572  *
 573  * RTO = SRTT + max(G, K * RTTVAR)
 574  *
 575  * K is defined in the RFC as 4, and G is the clock granularity. We constrain
 576  * the minimum mean deviation to TCP_SD_MIN when processing new RTTs, so this
 577  * becomes:
 578  *
 579  * RTO = SRTT + 4 * RTTVAR
 580  *
 581  * In practice, however, we make several additions to it. As we use a finer
 582  * grained clock than BSD and update RTO for every ACK, we add in another 1/4 of
 583  * RTT to the deviation of RTO to accommodate burstiness of 1/4 of window size:
 584  *
 585  * RTO = SRTT + (SRTT / 4) + 4 * RTTVAR
 586  *
 587  * Since tcp_rtt_sa is 8 times the SRTT, and tcp_rtt_sd is 4 times the RTTVAR,
 588  * this becomes:
 589  *
 590  * RTO = (tcp_rtt_sa / 8) + ((tcp_rtt_sa / 8) / 4) + tcp_rtt_sd
 591  * RTO = (tcp_rtt_sa / 2^3) + (tcp_rtt_sa / 2^5) + tcp_rtt_sd
 592  * RTO = (tcp_rtt_sa >> 3) + (tcp_rtt_sa >> 5) + tcp_rtt_sd
 593  *
 594  * The "tcp_rexmit_interval_extra" and "tcp_conn_grace_period" tunables are
 595  * used to help account for extreme environments where the algorithm fails to
 596  * work; by default they should be 0. (The latter tunable is only used for
 597  * calculating the intial RTO, and so is optionally passed in as "extra".) We
 598  * add them here:
 599  *
 600  * RTO = (tcp_rtt_sa >> 3) + (tcp_rtt_sa >> 5) + tcp_rtt_sd +
 601  *     tcps_rexmit_interval_extra + tcps_conn_grace_period
 602  *
 603  * We then pin the RTO within our configured boundaries (sections 2.4 and 2.5
 604  * of RFC 6298).
 605  */
 606 static __GNU_INLINE clock_t
 607 tcp_calculate_rto(tcp_t *tcp, tcp_stack_t *tcps, uint32_t extra)
 608 {
 609         clock_t rto;
 610 
 611         rto = NSEC2MSEC((tcp->tcp_rtt_sa >> 3) + (tcp->tcp_rtt_sa >> 5) +
 612             tcp->tcp_rtt_sd) + tcps->tcps_rexmit_interval_extra + extra;
 613 
 614         if (rto < tcp->tcp_rto_min) {
 615                 rto = tcp->tcp_rto_min;
 616         } else if (rto > tcp->tcp_rto_max) {
 617                 rto = tcp->tcp_rto_max;
 618         }
 619 
 620         return (rto);
 621 }
 622 
 623 extern struct qinit tcp_rinitv4, tcp_rinitv6;
 624 extern boolean_t do_tcp_fusion;
 625 
 626 /*
 627  * Object to represent database of options to search passed to
 628  * {sock,tpi}optcom_req() interface routine to take care of option
 629  * management and associated methods.
 630  */
 631 extern optdb_obj_t      tcp_opt_obj;
 632 extern uint_t           tcp_max_optsize;
 633 
 634 extern int tcp_squeue_flag;
 635 
 636 extern uint_t tcp_free_list_max_cnt;
 637 
 638 /*
 639  * Functions in tcp.c.
 640  */
 641 extern void     tcp_acceptor_hash_insert(t_uscalar_t, tcp_t *);
 642 extern tcp_t    *tcp_acceptor_hash_lookup(t_uscalar_t, tcp_stack_t *);
 643 extern void     tcp_acceptor_hash_remove(tcp_t *);
 644 extern mblk_t   *tcp_ack_mp(tcp_t *);
 645 extern int      tcp_build_hdrs(tcp_t *);
 646 extern void     tcp_cleanup(tcp_t *);
 647 extern int      tcp_clean_death(tcp_t *, int);
 648 extern void     tcp_clean_death_wrapper(void *, mblk_t *, void *,
 649                     ip_recv_attr_t *);
 650 extern void     tcp_close_common(conn_t *, int);
 651 extern void     tcp_close_detached(tcp_t *);
 652 extern void     tcp_close_mpp(mblk_t **);
 653 extern void     tcp_closei_local(tcp_t *);
 654 extern sock_lower_handle_t tcp_create(int, int, int, sock_downcalls_t **,
 655                     uint_t *, int *, int, cred_t *);
 656 extern conn_t   *tcp_create_common(cred_t *, boolean_t, boolean_t, int *);
 657 extern void     tcp_disconnect(tcp_t *, mblk_t *);
 658 extern char     *tcp_display(tcp_t *, char *, char);
 659 extern int      tcp_do_bind(conn_t *, struct sockaddr *, socklen_t, cred_t *,
 660                     boolean_t);
 661 extern int      tcp_do_connect(conn_t *, const struct sockaddr *, socklen_t,
 662                     cred_t *, pid_t);
 663 extern int      tcp_do_listen(conn_t *, struct sockaddr *, socklen_t, int,
 664                     cred_t *, boolean_t);
 665 extern int      tcp_do_unbind(conn_t *);
 666 extern boolean_t        tcp_eager_blowoff(tcp_t *, t_scalar_t);
 667 extern void     tcp_eager_cleanup(tcp_t *, boolean_t);
 668 extern void     tcp_eager_kill(void *, mblk_t *, void *, ip_recv_attr_t *);
 669 extern void     tcp_eager_unlink(tcp_t *);
 670 extern void     tcp_init_values(tcp_t *, tcp_t *);
 671 extern void     tcp_ipsec_cleanup(tcp_t *);
 672 extern int      tcp_maxpsz_set(tcp_t *, boolean_t);
 673 extern void     tcp_mss_set(tcp_t *, uint32_t);
 674 extern void     tcp_reinput(conn_t *, mblk_t *, ip_recv_attr_t *, ip_stack_t *);
 675 extern int      tcp_rsrv(queue_t *);
 676 extern uint_t   tcp_rwnd_reopen(tcp_t *);
 677 extern int      tcp_rwnd_set(tcp_t *, uint32_t);
 678 extern int      tcp_set_destination(tcp_t *);
 679 extern void     tcp_set_ws_value(tcp_t *);
 680 extern void     tcp_stop_lingering(tcp_t *);
 681 extern void     tcp_update_pmtu(tcp_t *, boolean_t);
 682 extern mblk_t   *tcp_zcopy_backoff(tcp_t *, mblk_t *, boolean_t);
 683 extern boolean_t        tcp_zcopy_check(tcp_t *);
 684 extern void     tcp_zcopy_notify(tcp_t *);
 685 extern void     tcp_get_proto_props(tcp_t *, struct sock_proto_props *);
 686 
 687 /*
 688  * Bind related functions in tcp_bind.c
 689  */
 690 extern int      tcp_bind_check(conn_t *, struct sockaddr *, socklen_t,
 691                     cred_t *, boolean_t);
 692 extern void     tcp_bind_hash_insert(tf_t *, tcp_t *, int);
 693 extern void     tcp_bind_hash_remove(tcp_t *);
 694 extern in_port_t        tcp_bindi(tcp_t *, in_port_t, const in6_addr_t *,
 695                             int, boolean_t, boolean_t, boolean_t);
 696 extern in_port_t        tcp_update_next_port(in_port_t, const tcp_t *,
 697                             boolean_t);
 698 
 699 /*
 700  * Fusion related functions in tcp_fusion.c.
 701  */
 702 extern void     tcp_fuse(tcp_t *, uchar_t *, tcpha_t *);
 703 extern void     tcp_unfuse(tcp_t *);
 704 extern boolean_t tcp_fuse_output(tcp_t *, mblk_t *, uint32_t);
 705 extern void     tcp_fuse_output_urg(tcp_t *, mblk_t *);
 706 extern boolean_t tcp_fuse_rcv_drain(queue_t *, tcp_t *, mblk_t **);
 707 extern size_t   tcp_fuse_set_rcv_hiwat(tcp_t *, size_t);
 708 extern int      tcp_fuse_maxpsz(tcp_t *);
 709 extern void     tcp_fuse_backenable(tcp_t *);
 710 extern void     tcp_iss_key_init(uint8_t *, int, tcp_stack_t *);
 711 
 712 /*
 713  * Output related functions in tcp_output.c.
 714  */
 715 extern void     tcp_close_output(void *, mblk_t *, void *, ip_recv_attr_t *);
 716 extern void     tcp_output(void *, mblk_t *, void *, ip_recv_attr_t *);
 717 extern void     tcp_output_urgent(void *, mblk_t *, void *, ip_recv_attr_t *);
 718 extern void     tcp_rexmit_after_error(tcp_t *);
 719 extern void     tcp_sack_rexmit(tcp_t *, uint_t *);
 720 extern void     tcp_send_data(tcp_t *, mblk_t *);
 721 extern void     tcp_send_synack(void *, mblk_t *, void *, ip_recv_attr_t *);
 722 extern void     tcp_shutdown_output(void *, mblk_t *, void *, ip_recv_attr_t *);
 723 extern void     tcp_ss_rexmit(tcp_t *);
 724 extern void     tcp_update_xmit_tail(tcp_t *, uint32_t);
 725 extern int      tcp_wput(queue_t *, mblk_t *);
 726 extern void     tcp_wput_data(tcp_t *, mblk_t *, boolean_t);
 727 extern int      tcp_wput_sock(queue_t *, mblk_t *);
 728 extern int      tcp_wput_fallback(queue_t *, mblk_t *);
 729 extern void     tcp_xmit_ctl(char *, tcp_t *, uint32_t, uint32_t, int);
 730 extern void     tcp_xmit_listeners_reset(mblk_t *, ip_recv_attr_t *,
 731                     ip_stack_t *i, conn_t *);
 732 extern mblk_t   *tcp_xmit_mp(tcp_t *, mblk_t *, int32_t, int32_t *,
 733                     mblk_t **, uint32_t, boolean_t, uint32_t *, boolean_t);
 734 
 735 /*
 736  * Input related functions in tcp_input.c.
 737  */
 738 extern void     cc_cong_signal(tcp_t *, uint32_t, uint32_t);
 739 extern void     tcp_icmp_input(void *, mblk_t *, void *, ip_recv_attr_t *);
 740 extern void     tcp_input_data(void *, mblk_t *, void *, ip_recv_attr_t *);
 741 extern void     tcp_input_listener_unbound(void *, mblk_t *, void *,
 742                     ip_recv_attr_t *);
 743 extern boolean_t        tcp_paws_check(tcp_t *, const tcp_opt_t *);
 744 extern int      tcp_parse_options(tcpha_t *, tcp_opt_t *);
 745 extern uint_t   tcp_rcv_drain(tcp_t *);
 746 extern void     tcp_rcv_enqueue(tcp_t *, mblk_t *, uint_t, cred_t *);
 747 extern boolean_t        tcp_verifyicmp(conn_t *, void *, icmph_t *, icmp6_t *,
 748                             ip_recv_attr_t *);
 749 
 750 /*
 751  * Kernel socket related functions in tcp_socket.c.
 752  */
 753 extern int      tcp_fallback(sock_lower_handle_t, queue_t *, boolean_t,
 754                     so_proto_quiesced_cb_t, sock_quiesce_arg_t *);
 755 extern boolean_t tcp_newconn_notify(tcp_t *, ip_recv_attr_t *);
 756 
 757 /*
 758  * Timer related functions in tcp_timers.c.
 759  */
 760 extern void     tcp_ack_timer(void *);
 761 extern void     tcp_close_linger_timeout(void *);
 762 extern void     tcp_keepalive_timer(void *);
 763 extern void     tcp_push_timer(void *);
 764 extern void     tcp_reass_timer(void *);
 765 extern mblk_t   *tcp_timermp_alloc(int);
 766 extern void     tcp_timermp_free(tcp_t *);
 767 extern timeout_id_t tcp_timeout(conn_t *, void (*)(void *), hrtime_t);
 768 extern clock_t  tcp_timeout_cancel(conn_t *, timeout_id_t);
 769 extern void     tcp_timer(void *arg);
 770 extern void     tcp_timers_stop(tcp_t *);
 771 
 772 /*
 773  * TCP TPI related functions in tcp_tpi.c.
 774  */
 775 extern void     tcp_addr_req(tcp_t *, mblk_t *);
 776 extern void     tcp_capability_req(tcp_t *, mblk_t *);
 777 extern boolean_t        tcp_conn_con(tcp_t *, uchar_t *, mblk_t *,
 778                             mblk_t **, ip_recv_attr_t *);
 779 extern void     tcp_err_ack(tcp_t *, mblk_t *, int, int);
 780 extern void     tcp_err_ack_prim(tcp_t *, mblk_t *, int, int, int);
 781 extern void     tcp_info_req(tcp_t *, mblk_t *);
 782 extern void     tcp_send_conn_ind(void *, mblk_t *, void *);
 783 extern void     tcp_send_pending(void *, mblk_t *, void *, ip_recv_attr_t *);
 784 extern int      tcp_tpi_accept(queue_t *, mblk_t *);
 785 extern void     tcp_tpi_bind(tcp_t *, mblk_t *);
 786 extern int      tcp_tpi_close(queue_t *, int, cred_t *);
 787 extern int      tcp_tpi_close_accept(queue_t *, int, cred_t *);
 788 extern void     tcp_tpi_connect(tcp_t *, mblk_t *);
 789 extern int      tcp_tpi_opt_get(queue_t *, t_scalar_t, t_scalar_t, uchar_t *);
 790 extern int      tcp_tpi_opt_set(queue_t *, uint_t, int, int, uint_t, uchar_t *,
 791                     uint_t *, uchar_t *, void *, cred_t *);
 792 extern void     tcp_tpi_unbind(tcp_t *, mblk_t *);
 793 extern void     tcp_tli_accept(tcp_t *, mblk_t *);
 794 extern void     tcp_use_pure_tpi(tcp_t *);
 795 extern void     tcp_do_capability_ack(tcp_t *, struct T_capability_ack *,
 796                     t_uscalar_t);
 797 
 798 /*
 799  * TCP option processing related functions in tcp_opt_data.c
 800  */
 801 extern int      tcp_opt_get(conn_t *, int, int, uchar_t *);
 802 extern int      tcp_opt_set(conn_t *, uint_t, int, int, uint_t, uchar_t *,
 803                     uint_t *, uchar_t *, void *, cred_t *);
 804 
 805 /*
 806  * TCP time wait processing related functions in tcp_time_wait.c.
 807  */
 808 extern void             tcp_time_wait_append(tcp_t *);
 809 extern void             tcp_time_wait_collector(void *);
 810 extern boolean_t        tcp_time_wait_remove(tcp_t *, tcp_squeue_priv_t *);
 811 extern void             tcp_time_wait_processing(tcp_t *, mblk_t *, uint32_t,
 812                             uint32_t, int, tcpha_t *, ip_recv_attr_t *);
 813 
 814 /*
 815  * Misc functions in tcp_misc.c.
 816  */
 817 extern uint32_t tcp_find_listener_conf(tcp_stack_t *, in_port_t);
 818 extern void     tcp_ioctl_abort_conn(queue_t *, mblk_t *);
 819 extern void     tcp_listener_conf_cleanup(tcp_stack_t *);
 820 extern void     tcp_stack_cpu_add(tcp_stack_t *, processorid_t);
 821 
 822 #endif  /* _KERNEL */
 823 
 824 #ifdef  __cplusplus
 825 }
 826 #endif
 827 
 828 #endif  /* _INET_TCP_IMPL_H */