net/ipv4/tcp.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * INET         An implementation of the TCP/IP protocol suite for the LINUX
   4  *              operating system.  INET is implemented using the  BSD Socket
   5  *              interface as the means of communication with the user level.
   6  *
   7  *              Implementation of the Transmission Control Protocol(TCP).
   8  *
   9  * Authors:     Ross Biro
  10  *              Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
  11  *              Mark Evans, <evansmp@uhura.aston.ac.uk>
  12  *              Corey Minyard <wf-rch!minyard@relay.EU.net>
  13  *              Florian La Roche, <flla@stud.uni-sb.de>
  14  *              Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
  15  *              Linus Torvalds, <torvalds@cs.helsinki.fi>
  16  *              Alan Cox, <gw4pts@gw4pts.ampr.org>
  17  *              Matthew Dillon, <dillon@apollo.west.oic.com>
  18  *              Arnt Gulbrandsen, <agulbra@nvg.unit.no>
  19  *              Jorge Cwik, <jorge@laser.satlink.net>
  20  *
  21  * Fixes:
  22  *              Alan Cox        :       Numerous verify_area() calls
  23  *              Alan Cox        :       Set the ACK bit on a reset
  24  *              Alan Cox        :       Stopped it crashing if it closed while
  25  *                                      sk->inuse=1 and was trying to connect
  26  *                                      (tcp_err()).
  27  *              Alan Cox        :       All icmp error handling was broken
  28  *                                      pointers passed where wrong and the
  29  *                                      socket was looked up backwards. Nobody
  30  *                                      tested any icmp error code obviously.
  31  *              Alan Cox        :       tcp_err() now handled properly. It
  32  *                                      wakes people on errors. poll
  33  *                                      behaves and the icmp error race
  34  *                                      has gone by moving it into sock.c
  35  *              Alan Cox        :       tcp_send_reset() fixed to work for
  36  *                                      everything not just packets for
  37  *                                      unknown sockets.
  38  *              Alan Cox        :       tcp option processing.
  39  *              Alan Cox        :       Reset tweaked (still not 100%) [Had
  40  *                                      syn rule wrong]
  41  *              Herp Rosmanith  :       More reset fixes
  42  *              Alan Cox        :       No longer acks invalid rst frames.
  43  *                                      Acking any kind of RST is right out.
  44  *              Alan Cox        :       Sets an ignore me flag on an rst
  45  *                                      receive otherwise odd bits of prattle
  46  *                                      escape still
  47  *              Alan Cox        :       Fixed another acking RST frame bug.
  48  *                                      Should stop LAN workplace lockups.
  49  *              Alan Cox        :       Some tidyups using the new skb list
  50  *                                      facilities
  51  *              Alan Cox        :       sk->keepopen now seems to work
  52  *              Alan Cox        :       Pulls options out correctly on accepts
  53  *              Alan Cox        :       Fixed assorted sk->rqueue->next errors
  54  *              Alan Cox        :       PSH doesn't end a TCP read. Switched a
  55  *                                      bit to skb ops.
  56  *              Alan Cox        :       Tidied tcp_data to avoid a potential
  57  *                                      nasty.
  58  *              Alan Cox        :       Added some better commenting, as the
  59  *                                      tcp is hard to follow
  60  *              Alan Cox        :       Removed incorrect check for 20 * psh
  61  *      Michael O'Reilly        :       ack < copied bug fix.
  62  *      Johannes Stille         :       Misc tcp fixes (not all in yet).
  63  *              Alan Cox        :       FIN with no memory -> CRASH
  64  *              Alan Cox        :       Added socket option proto entries.
  65  *                                      Also added awareness of them to accept.
  66  *              Alan Cox        :       Added TCP options (SOL_TCP)
  67  *              Alan Cox        :       Switched wakeup calls to callbacks,
  68  *                                      so the kernel can layer network
  69  *                                      sockets.
  70  *              Alan Cox        :       Use ip_tos/ip_ttl settings.
  71  *              Alan Cox        :       Handle FIN (more) properly (we hope).
  72  *              Alan Cox        :       RST frames sent on unsynchronised
  73  *                                      state ack error.
  74  *              Alan Cox        :       Put in missing check for SYN bit.
  75  *              Alan Cox        :       Added tcp_select_window() aka NET2E
  76  *                                      window non shrink trick.
  77  *              Alan Cox        :       Added a couple of small NET2E timer
  78  *                                      fixes
  79  *              Charles Hedrick :       TCP fixes
  80  *              Toomas Tamm     :       TCP window fixes
  81  *              Alan Cox        :       Small URG fix to rlogin ^C ack fight
  82  *              Charles Hedrick :       Rewrote most of it to actually work
  83  *              Linus           :       Rewrote tcp_read() and URG handling
  84  *                                      completely
  85  *              Gerhard Koerting:       Fixed some missing timer handling
  86  *              Matthew Dillon  :       Reworked TCP machine states as per RFC
  87  *              Gerhard Koerting:       PC/TCP workarounds
  88  *              Adam Caldwell   :       Assorted timer/timing errors
  89  *              Matthew Dillon  :       Fixed another RST bug
  90  *              Alan Cox        :       Move to kernel side addressing changes.
  91  *              Alan Cox        :       Beginning work on TCP fastpathing
  92  *                                      (not yet usable)
  93  *              Arnt Gulbrandsen:       Turbocharged tcp_check() routine.
  94  *              Alan Cox        :       TCP fast path debugging
  95  *              Alan Cox        :       Window clamping
  96  *              Michael Riepe   :       Bug in tcp_check()
  97  *              Matt Dillon     :       More TCP improvements and RST bug fixes
  98  *              Matt Dillon     :       Yet more small nasties remove from the
  99  *                                      TCP code (Be very nice to this man if
 100  *                                      tcp finally works 100%) 8)
 101  *              Alan Cox        :       BSD accept semantics.
 102  *              Alan Cox        :       Reset on closedown bug.
 103  *      Peter De Schrijver      :       ENOTCONN check missing in tcp_sendto().
 104  *              Michael Pall    :       Handle poll() after URG properly in
 105  *                                      all cases.
 106  *              Michael Pall    :       Undo the last fix in tcp_read_urg()
 107  *                                      (multi URG PUSH broke rlogin).
 108  *              Michael Pall    :       Fix the multi URG PUSH problem in
 109  *                                      tcp_readable(), poll() after URG
 110  *                                      works now.
 111  *              Michael Pall    :       recv(...,MSG_OOB) never blocks in the
 112  *                                      BSD api.
 113  *              Alan Cox        :       Changed the semantics of sk->socket to
 114  *                                      fix a race and a signal problem with
 115  *                                      accept() and async I/O.
 116  *              Alan Cox        :       Relaxed the rules on tcp_sendto().
 117  *              Yury Shevchuk   :       Really fixed accept() blocking problem.
 118  *              Craig I. Hagan  :       Allow for BSD compatible TIME_WAIT for
 119  *                                      clients/servers which listen in on
 120  *                                      fixed ports.
 121  *              Alan Cox        :       Cleaned the above up and shrank it to
 122  *                                      a sensible code size.
 123  *              Alan Cox        :       Self connect lockup fix.
 124  *              Alan Cox        :       No connect to multicast.
 125  *              Ross Biro       :       Close unaccepted children on master
 126  *                                      socket close.
 127  *              Alan Cox        :       Reset tracing code.
 128  *              Alan Cox        :       Spurious resets on shutdown.
 129  *              Alan Cox        :       Giant 15 minute/60 second timer error
 130  *              Alan Cox        :       Small whoops in polling before an
 131  *                                      accept.
 132  *              Alan Cox        :       Kept the state trace facility since
 133  *                                      it's handy for debugging.
 134  *              Alan Cox        :       More reset handler fixes.
 135  *              Alan Cox        :       Started rewriting the code based on
 136  *                                      the RFC's for other useful protocol
 137  *                                      references see: Comer, KA9Q NOS, and
 138  *                                      for a reference on the difference
 139  *                                      between specifications and how BSD
 140  *                                      works see the 4.4lite source.
 141  *              A.N.Kuznetsov   :       Don't time wait on completion of tidy
 142  *                                      close.
 143  *              Linus Torvalds  :       Fin/Shutdown & copied_seq changes.
 144  *              Linus Torvalds  :       Fixed BSD port reuse to work first syn
 145  *              Alan Cox        :       Reimplemented timers as per the RFC
 146  *                                      and using multiple timers for sanity.
 147  *              Alan Cox        :       Small bug fixes, and a lot of new
 148  *                                      comments.
 149  *              Alan Cox        :       Fixed dual reader crash by locking
 150  *                                      the buffers (much like datagram.c)
 151  *              Alan Cox        :       Fixed stuck sockets in probe. A probe
 152  *                                      now gets fed up of retrying without
 153  *                                      (even a no space) answer.
 154  *              Alan Cox        :       Extracted closing code better
 155  *              Alan Cox        :       Fixed the closing state machine to
 156  *                                      resemble the RFC.
 157  *              Alan Cox        :       More 'per spec' fixes.
 158  *              Jorge Cwik      :       Even faster checksumming.
 159  *              Alan Cox        :       tcp_data() doesn't ack illegal PSH
 160  *                                      only frames. At least one pc tcp stack
 161  *                                      generates them.
 162  *              Alan Cox        :       Cache last socket.
 163  *              Alan Cox        :       Per route irtt.
 164  *              Matt Day        :       poll()->select() match BSD precisely on error
 165  *              Alan Cox        :       New buffers
 166  *              Marc Tamsky     :       Various sk->prot->retransmits and
 167  *                                      sk->retransmits misupdating fixed.
 168  *                                      Fixed tcp_write_timeout: stuck close,
 169  *                                      and TCP syn retries gets used now.
 170  *              Mark Yarvis     :       In tcp_read_wakeup(), don't send an
 171  *                                      ack if state is TCP_CLOSED.
 172  *              Alan Cox        :       Look up device on a retransmit - routes may
 173  *                                      change. Doesn't yet cope with MSS shrink right
 174  *                                      but it's a start!
 175  *              Marc Tamsky     :       Closing in closing fixes.
 176  *              Mike Shaver     :       RFC1122 verifications.
 177  *              Alan Cox        :       rcv_saddr errors.
 178  *              Alan Cox        :       Block double connect().
 179  *              Alan Cox        :       Small hooks for enSKIP.
 180  *              Alexey Kuznetsov:       Path MTU discovery.
 181  *              Alan Cox        :       Support soft errors.
 182  *              Alan Cox        :       Fix MTU discovery pathological case
 183  *                                      when the remote claims no mtu!
 184  *              Marc Tamsky     :       TCP_CLOSE fix.
 185  *              Colin (G3TNE)   :       Send a reset on syn ack replies in
 186  *                                      window but wrong (fixes NT lpd problems)
 187  *              Pedro Roque     :       Better TCP window handling, delayed ack.
 188  *              Joerg Reuter    :       No modification of locked buffers in
 189  *                                      tcp_do_retransmit()
 190  *              Eric Schenk     :       Changed receiver side silly window
 191  *                                      avoidance algorithm to BSD style
 192  *                                      algorithm. This doubles throughput
 193  *                                      against machines running Solaris,
 194  *                                      and seems to result in general
 195  *                                      improvement.
 196  *      Stefan Magdalinski      :       adjusted tcp_readable() to fix FIONREAD
 197  *      Willy Konynenberg       :       Transparent proxying support.
 198  *      Mike McLagan            :       Routing by source
 199  *              Keith Owens     :       Do proper merging with partial SKB's in
 200  *                                      tcp_do_sendmsg to avoid burstiness.
 201  *              Eric Schenk     :       Fix fast close down bug with
 202  *                                      shutdown() followed by close().
 203  *              Andi Kleen      :       Make poll agree with SIGIO
 204  *      Salvatore Sanfilippo    :       Support SO_LINGER with linger == 1 and
 205  *                                      lingertime == 0 (RFC 793 ABORT Call)
 206  *      Hirokazu Takahashi      :       Use copy_from_user() instead of
 207  *                                      csum_and_copy_from_user() if possible.
 208  *
 209  * Description of States:
 210  *
 211  *      TCP_SYN_SENT            sent a connection request, waiting for ack
 212  *
 213  *      TCP_SYN_RECV            received a connection request, sent ack,
 214  *                              waiting for final ack in three-way handshake.
 215  *
 216  *      TCP_ESTABLISHED         connection established
 217  *
 218  *      TCP_FIN_WAIT1           our side has shutdown, waiting to complete
 219  *                              transmission of remaining buffered data
 220  *
 221  *      TCP_FIN_WAIT2           all buffered data sent, waiting for remote
 222  *                              to shutdown
 223  *
 224  *      TCP_CLOSING             both sides have shutdown but we still have
 225  *                              data we have to finish sending
 226  *
 227  *      TCP_TIME_WAIT           timeout to catch resent junk before entering
 228  *                              closed, can only be entered from FIN_WAIT2
 229  *                              or CLOSING.  Required because the other end
 230  *                              may not have gotten our last ACK causing it
 231  *                              to retransmit the data packet (which we ignore)
 232  *
 233  *      TCP_CLOSE_WAIT          remote side has shutdown and is waiting for
 234  *                              us to finish writing our data and to shutdown
 235  *                              (we have to close() to move on to LAST_ACK)
 236  *
 237  *      TCP_LAST_ACK            out side has shutdown after remote has
 238  *                              shutdown.  There may still be data in our
 239  *                              buffer that we have to finish sending
 240  *
 241  *      TCP_CLOSE               socket is finished
 242  */
 243
 244 #define pr_fmt(fmt) "TCP: " fmt
 245
 246 #include <crypto/hash.h>
 247 #include <linux/kernel.h>
 248 #include <linux/module.h>
 249 #include <linux/types.h>
 250 #include <linux/fcntl.h>
 251 #include <linux/poll.h>
 252 #include <linux/inet_diag.h>
 253 #include <linux/init.h>
 254 #include <linux/fs.h>
 255 #include <linux/skbuff.h>
 256 #include <linux/scatterlist.h>
 257 #include <linux/splice.h>
 258 #include <linux/net.h>
 259 #include <linux/socket.h>
 260 #include <linux/random.h>
 261 #include <linux/memblock.h>
 262 #include <linux/highmem.h>
 263 #include <linux/swap.h>
 264 #include <linux/cache.h>
 265 #include <linux/err.h>
 266 #include <linux/time.h>
 267 #include <linux/slab.h>
 268 #include <linux/errqueue.h>
 269 #include <linux/static_key.h>
 270
 271 #include <net/icmp.h>
 272 #include <net/inet_common.h>
 273 #include <net/tcp.h>
 274 #include <net/xfrm.h>
 275 #include <net/ip.h>
 276 #include <net/sock.h>
 277
 278 #include <linux/uaccess.h>
 279 #include <asm/ioctls.h>
 280 #include <net/busy_poll.h>
 281
 282 struct percpu_counter tcp_orphan_count;
 283 EXPORT_SYMBOL_GPL(tcp_orphan_count);
 284
 285 long sysctl_tcp_mem[3] __read_mostly;
 286 EXPORT_SYMBOL(sysctl_tcp_mem);
 287
 288 atomic_long_t tcp_memory_allocated;     /* Current allocated memory. */
 289 EXPORT_SYMBOL(tcp_memory_allocated);
 290
 291 #if IS_ENABLED(CONFIG_SMC)
 292 DEFINE_STATIC_KEY_FALSE(tcp_have_smc);
 293 EXPORT_SYMBOL(tcp_have_smc);
 294 #endif
 295
 296 /*
 297  * Current number of TCP sockets.
 298  */
 299 struct percpu_counter tcp_sockets_allocated;
 300 EXPORT_SYMBOL(tcp_sockets_allocated);
 301
 302 /*
 303  * TCP splice context
 304  */
 305 struct tcp_splice_state {
 306         struct pipe_inode_info *pipe;
 307         size_t len;
 308         unsigned int flags;
 309 };
 310
 311 /*
 312  * Pressure flag: try to collapse.
 313  * Technical note: it is used by multiple contexts non atomically.
 314  * All the __sk_mem_schedule() is of this nature: accounting
 315  * is strict, actions are advisory and have some latency.
 316  */
 317 unsigned long tcp_memory_pressure __read_mostly;
 318 EXPORT_SYMBOL_GPL(tcp_memory_pressure);
 319
 320 DEFINE_STATIC_KEY_FALSE(tcp_rx_skb_cache_key);
 321 EXPORT_SYMBOL(tcp_rx_skb_cache_key);
 322
 323 DEFINE_STATIC_KEY_FALSE(tcp_tx_skb_cache_key);
 324
 325 void tcp_enter_memory_pressure(struct sock *sk)
 326 {
 327         unsigned long val;
 328
 329         if (READ_ONCE(tcp_memory_pressure))
 330                 return;
 331         val = jiffies;
 332
 333         if (!val)
 334                 val--;
 335         if (!cmpxchg(&tcp_memory_pressure, 0, val))
 336                 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMEMORYPRESSURES);
 337 }
 338 EXPORT_SYMBOL_GPL(tcp_enter_memory_pressure);
 339
 340 void tcp_leave_memory_pressure(struct sock *sk)
 341 {
 342         unsigned long val;
 343
 344         if (!READ_ONCE(tcp_memory_pressure))
 345                 return;
 346         val = xchg(&tcp_memory_pressure, 0);
 347         if (val)
 348                 NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPMEMORYPRESSURESCHRONO,
 349                               jiffies_to_msecs(jiffies - val));
 350 }
 351 EXPORT_SYMBOL_GPL(tcp_leave_memory_pressure);
 352
 353 /* Convert seconds to retransmits based on initial and max timeout */
 354 static u8 secs_to_retrans(int seconds, int timeout, int rto_max)
 355 {
 356         u8 res = 0;
 357
 358         if (seconds > 0) {
 359                 int period = timeout;
 360
 361                 res = 1;
 362                 while (seconds > period && res < 255) {
 363                         res++;
 364                         timeout <<= 1;
 365                         if (timeout > rto_max)
 366                                 timeout = rto_max;
 367                         period += timeout;
 368                 }
 369         }
 370         return res;
 371 }
 372
 373 /* Convert retransmits to seconds based on initial and max timeout */
 374 static int retrans_to_secs(u8 retrans, int timeout, int rto_max)
 375 {
 376         int period = 0;
 377
 378         if (retrans > 0) {
 379                 period = timeout;
 380                 while (--retrans) {
 381                         timeout <<= 1;
 382                         if (timeout > rto_max)
 383                                 timeout = rto_max;
 384                         period += timeout;
 385                 }
 386         }
 387         return period;
 388 }
 389
 390 static u64 tcp_compute_delivery_rate(const struct tcp_sock *tp)
 391 {
 392         u32 rate = READ_ONCE(tp->rate_delivered);
 393         u32 intv = READ_ONCE(tp->rate_interval_us);
 394         u64 rate64 = 0;
 395
 396         if (rate && intv) {
 397                 rate64 = (u64)rate * tp->mss_cache * USEC_PER_SEC;
 398                 do_div(rate64, intv);
 399         }
 400         return rate64;
 401 }
 402
 403 /* Address-family independent initialization for a tcp_sock.
 404  *
 405  * NOTE: A lot of things set to zero explicitly by call to
 406  *       sk_alloc() so need not be done here.
 407  */
 408 void tcp_init_sock(struct sock *sk)
 409 {
 410         struct inet_connection_sock *icsk = inet_csk(sk);
 411         struct tcp_sock *tp = tcp_sk(sk);
 412
 413         tp->out_of_order_queue = RB_ROOT;
 414         sk->tcp_rtx_queue = RB_ROOT;
 415         tcp_init_xmit_timers(sk);
 416         INIT_LIST_HEAD(&tp->tsq_node);
 417         INIT_LIST_HEAD(&tp->tsorted_sent_queue);
 418
 419         icsk->icsk_rto = TCP_TIMEOUT_INIT;
 420         tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT);
 421         minmax_reset(&tp->rtt_min, tcp_jiffies32, ~0U);
 422
 423         /* So many TCP implementations out there (incorrectly) count the
 424          * initial SYN frame in their delayed-ACK and congestion control
 425          * algorithms that we must have the following bandaid to talk
 426          * efficiently to them.  -DaveM
 427          */
 428         tp->snd_cwnd = TCP_INIT_CWND;
 429
 430         /* There's a bubble in the pipe until at least the first ACK. */
 431         tp->app_limited = ~0U;
 432
 433         /* See draft-stevens-tcpca-spec-01 for discussion of the
 434          * initialization of these values.
 435          */
 436         tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;
 437         tp->snd_cwnd_clamp = ~0;
 438         tp->mss_cache = TCP_MSS_DEFAULT;
 439
 440         tp->reordering = sock_net(sk)->ipv4.sysctl_tcp_reordering;
 441         tcp_assign_congestion_control(sk);
 442
 443         tp->tsoffset = 0;
 444         tp->rack.reo_wnd_steps = 1;
 445
 446         sk->sk_state = TCP_CLOSE;
 447
 448         sk->sk_write_space = sk_stream_write_space;
 449         sock_set_flag(sk, SOCK_USE_WRITE_QUEUE);
 450
 451         icsk->icsk_sync_mss = tcp_sync_mss;
 452
 453         sk->sk_sndbuf = sock_net(sk)->ipv4.sysctl_tcp_wmem[1];
 454         sk->sk_rcvbuf = sock_net(sk)->ipv4.sysctl_tcp_rmem[1];
 455
 456         sk_sockets_allocated_inc(sk);
 457         sk->sk_route_forced_caps = NETIF_F_GSO;
 458 }
 459 EXPORT_SYMBOL(tcp_init_sock);
 460
 461 static void tcp_tx_timestamp(struct sock *sk, u16 tsflags)
 462 {
 463         struct sk_buff *skb = tcp_write_queue_tail(sk);
 464
 465         if (tsflags && skb) {
 466                 struct skb_shared_info *shinfo = skb_shinfo(skb);
 467                 struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
 468
 469                 sock_tx_timestamp(sk, tsflags, &shinfo->tx_flags);
 470                 if (tsflags & SOF_TIMESTAMPING_TX_ACK)
 471                         tcb->txstamp_ack = 1;
 472                 if (tsflags & SOF_TIMESTAMPING_TX_RECORD_MASK)
 473                         shinfo->tskey = TCP_SKB_CB(skb)->seq + skb->len - 1;
 474         }
 475 }
 476
 477 static inline bool tcp_stream_is_readable(const struct tcp_sock *tp,
 478                                           int target, struct sock *sk)
 479 {
 480         return (READ_ONCE(tp->rcv_nxt) - READ_ONCE(tp->copied_seq) >= target) ||
 481                 (sk->sk_prot->stream_memory_read ?
 482                 sk->sk_prot->stream_memory_read(sk) : false);
 483 }
 484
 485 /*
 486  *      Wait for a TCP event.
 487  *
 488  *      Note that we don't need to lock the socket, as the upper poll layers
 489  *      take care of normal races (between the test and the event) and we don't
 490  *      go look at any of the socket buffers directly.
 491  */
 492 __poll_t tcp_poll(struct file *file, struct socket *sock, poll_table *wait)
 493 {
 494         __poll_t mask;
 495         struct sock *sk = sock->sk;
 496         const struct tcp_sock *tp = tcp_sk(sk);
 497         int state;
 498
 499         sock_poll_wait(file, sock, wait);
 500
 501         state = inet_sk_state_load(sk);
 502         if (state == TCP_LISTEN)
 503                 return inet_csk_listen_poll(sk);
 504
 505         /* Socket is not locked. We are protected from async events
 506          * by poll logic and correct handling of state changes
 507          * made by other threads is impossible in any case.
 508          */
 509
 510         mask = 0;
 511
 512         /*
 513          * EPOLLHUP is certainly not done right. But poll() doesn't
 514          * have a notion of HUP in just one direction, and for a
 515          * socket the read side is more interesting.
 516          *
 517          * Some poll() documentation says that EPOLLHUP is incompatible
 518          * with the EPOLLOUT/POLLWR flags, so somebody should check this
 519          * all. But careful, it tends to be safer to return too many
 520          * bits than too few, and you can easily break real applications
 521          * if you don't tell them that something has hung up!
 522          *
 523          * Check-me.
 524          *
 525          * Check number 1. EPOLLHUP is _UNMASKABLE_ event (see UNIX98 and
 526          * our fs/select.c). It means that after we received EOF,
 527          * poll always returns immediately, making impossible poll() on write()
 528          * in state CLOSE_WAIT. One solution is evident --- to set EPOLLHUP
 529          * if and only if shutdown has been made in both directions.
 530          * Actually, it is interesting to look how Solaris and DUX
 531          * solve this dilemma. I would prefer, if EPOLLHUP were maskable,
 532          * then we could set it on SND_SHUTDOWN. BTW examples given
 533          * in Stevens' books assume exactly this behaviour, it explains
 534          * why EPOLLHUP is incompatible with EPOLLOUT.  --ANK
 535          *
 536          * NOTE. Check for TCP_CLOSE is added. The goal is to prevent
 537          * blocking on fresh not-connected or disconnected socket. --ANK
 538          */
 539         if (sk->sk_shutdown == SHUTDOWN_MASK || state == TCP_CLOSE)
 540                 mask |= EPOLLHUP;
 541         if (sk->sk_shutdown & RCV_SHUTDOWN)
 542                 mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
 543
 544         /* Connected or passive Fast Open socket? */
 545         if (state != TCP_SYN_SENT &&
 546             (state != TCP_SYN_RECV || rcu_access_pointer(tp->fastopen_rsk))) {
 547                 int target = sock_rcvlowat(sk, 0, INT_MAX);
 548
 549                 if (tp->urg_seq == READ_ONCE(tp->copied_seq) &&
 550                     !sock_flag(sk, SOCK_URGINLINE) &&
 551                     tp->urg_data)
 552                         target++;
 553
 554                 if (tcp_stream_is_readable(tp, target, sk))
 555                         mask |= EPOLLIN | EPOLLRDNORM;
 556
 557                 if (!(sk->sk_shutdown & SEND_SHUTDOWN)) {
 558                         if (sk_stream_is_writeable(sk)) {
 559                                 mask |= EPOLLOUT | EPOLLWRNORM;
 560                         } else {  /* send SIGIO later */
 561                                 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
 562                                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 563
 564                                 /* Race breaker. If space is freed after
 565                                  * wspace test but before the flags are set,
 566                                  * IO signal will be lost. Memory barrier
 567                                  * pairs with the input side.
 568                                  */
 569                                 smp_mb__after_atomic();
 570                                 if (sk_stream_is_writeable(sk))
 571                                         mask |= EPOLLOUT | EPOLLWRNORM;
 572                         }
 573                 } else
 574                         mask |= EPOLLOUT | EPOLLWRNORM;
 575
 576                 if (tp->urg_data & TCP_URG_VALID)
 577                         mask |= EPOLLPRI;
 578         } else if (state == TCP_SYN_SENT && inet_sk(sk)->defer_connect) {
 579                 /* Active TCP fastopen socket with defer_connect
 580                  * Return EPOLLOUT so application can call write()
 581                  * in order for kernel to generate SYN+data
 582                  */
 583                 mask |= EPOLLOUT | EPOLLWRNORM;
 584         }
 585         /* This barrier is coupled with smp_wmb() in tcp_reset() */
 586         smp_rmb();
 587         if (sk->sk_err || !skb_queue_empty(&sk->sk_error_queue))
 588                 mask |= EPOLLERR;
 589
 590         return mask;
 591 }
 592 EXPORT_SYMBOL(tcp_poll);
 593
 594 int tcp_ioctl(struct sock *sk, int cmd, unsigned long arg)
 595 {
 596         struct tcp_sock *tp = tcp_sk(sk);
 597         int answ;
 598         bool slow;
 599
 600         switch (cmd) {
 601         case SIOCINQ:
 602                 if (sk->sk_state == TCP_LISTEN)
 603                         return -EINVAL;
 604
 605                 slow = lock_sock_fast(sk);
 606                 answ = tcp_inq(sk);
 607                 unlock_sock_fast(sk, slow);
 608                 break;
 609         case SIOCATMARK:
 610                 answ = tp->urg_data && tp->urg_seq == READ_ONCE(tp->copied_seq);
 611                 break;
 612         case SIOCOUTQ:
 613                 if (sk->sk_state == TCP_LISTEN)
 614                         return -EINVAL;
 615
 616                 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
 617                         answ = 0;
 618                 else
 619                         answ = READ_ONCE(tp->write_seq) - tp->snd_una;
 620                 break;
 621         case SIOCOUTQNSD:
 622                 if (sk->sk_state == TCP_LISTEN)
 623                         return -EINVAL;
 624
 625                 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
 626                         answ = 0;
 627                 else
 628                         answ = READ_ONCE(tp->write_seq) -
 629                                READ_ONCE(tp->snd_nxt);
 630                 break;
 631         default:
 632                 return -ENOIOCTLCMD;
 633         }
 634
 635         return put_user(answ, (int __user *)arg);
 636 }
 637 EXPORT_SYMBOL(tcp_ioctl);
 638
 639 static inline void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb)
 640 {
 641         TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
 642         tp->pushed_seq = tp->write_seq;
 643 }
 644
 645 static inline bool forced_push(const struct tcp_sock *tp)
 646 {
 647         return after(tp->write_seq, tp->pushed_seq + (tp->max_window >> 1));
 648 }
 649
 650 static void skb_entail(struct sock *sk, struct sk_buff *skb)
 651 {
 652         struct tcp_sock *tp = tcp_sk(sk);
 653         struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
 654
 655         skb->csum    = 0;
 656         tcb->seq     = tcb->end_seq = tp->write_seq;
 657         tcb->tcp_flags = TCPHDR_ACK;
 658         tcb->sacked  = 0;
 659         __skb_header_release(skb);
 660         tcp_add_write_queue_tail(sk, skb);
 661         sk->sk_wmem_queued += skb->truesize;
 662         sk_mem_charge(sk, skb->truesize);
 663         if (tp->nonagle & TCP_NAGLE_PUSH)
 664                 tp->nonagle &= ~TCP_NAGLE_PUSH;
 665
 666         tcp_slow_start_after_idle_check(sk);
 667 }
 668
 669 static inline void tcp_mark_urg(struct tcp_sock *tp, int flags)
 670 {
 671         if (flags & MSG_OOB)
 672                 tp->snd_up = tp->write_seq;
 673 }
 674
 675 /* If a not yet filled skb is pushed, do not send it if
 676  * we have data packets in Qdisc or NIC queues :
 677  * Because TX completion will happen shortly, it gives a chance
 678  * to coalesce future sendmsg() payload into this skb, without
 679  * need for a timer, and with no latency trade off.
 680  * As packets containing data payload have a bigger truesize
 681  * than pure acks (dataless) packets, the last checks prevent
 682  * autocorking if we only have an ACK in Qdisc/NIC queues,
 683  * or if TX completion was delayed after we processed ACK packet.
 684  */
 685 static bool tcp_should_autocork(struct sock *sk, struct sk_buff *skb,
 686                                 int size_goal)
 687 {
 688         return skb->len < size_goal &&
 689                sock_net(sk)->ipv4.sysctl_tcp_autocorking &&
 690                !tcp_rtx_queue_empty(sk) &&
 691                refcount_read(&sk->sk_wmem_alloc) > skb->truesize;
 692 }
 693
 694 static void tcp_push(struct sock *sk, int flags, int mss_now,
 695                      int nonagle, int size_goal)
 696 {
 697         struct tcp_sock *tp = tcp_sk(sk);
 698         struct sk_buff *skb;
 699
 700         skb = tcp_write_queue_tail(sk);
 701         if (!skb)
 702                 return;
 703         if (!(flags & MSG_MORE) || forced_push(tp))
 704                 tcp_mark_push(tp, skb);
 705
 706         tcp_mark_urg(tp, flags);
 707
 708         if (tcp_should_autocork(sk, skb, size_goal)) {
 709
 710                 /* avoid atomic op if TSQ_THROTTLED bit is already set */
 711                 if (!test_bit(TSQ_THROTTLED, &sk->sk_tsq_flags)) {
 712                         NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAUTOCORKING);
 713                         set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags);
 714                 }
 715                 /* It is possible TX completion already happened
 716                  * before we set TSQ_THROTTLED.
 717                  */
 718                 if (refcount_read(&sk->sk_wmem_alloc) > skb->truesize)
 719                         return;
 720         }
 721
 722         if (flags & MSG_MORE)
 723                 nonagle = TCP_NAGLE_CORK;
 724
 725         __tcp_push_pending_frames(sk, mss_now, nonagle);
 726 }
 727
 728 static int tcp_splice_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
 729                                 unsigned int offset, size_t len)
 730 {
 731         struct tcp_splice_state *tss = rd_desc->arg.data;
 732         int ret;
 733
 734         ret = skb_splice_bits(skb, skb->sk, offset, tss->pipe,
 735                               min(rd_desc->count, len), tss->flags);
 736         if (ret > 0)
 737                 rd_desc->count -= ret;
 738         return ret;
 739 }
 740
 741 static int __tcp_splice_read(struct sock *sk, struct tcp_splice_state *tss)
 742 {
 743         /* Store TCP splice context information in read_descriptor_t. */
 744         read_descriptor_t rd_desc = {
 745                 .arg.data = tss,
 746                 .count    = tss->len,
 747         };
 748
 749         return tcp_read_sock(sk, &rd_desc, tcp_splice_data_recv);
 750 }
 751
 752 /**
 753  *  tcp_splice_read - splice data from TCP socket to a pipe
 754  * @sock:       socket to splice from
 755  * @ppos:       position (not valid)
 756  * @pipe:       pipe to splice to
 757  * @len:        number of bytes to splice
 758  * @flags:      splice modifier flags
 759  *
 760  * Description:
 761  *    Will read pages from given socket and fill them into a pipe.
 762  *
 763  **/
 764 ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos,
 765                         struct pipe_inode_info *pipe, size_t len,
 766                         unsigned int flags)
 767 {
 768         struct sock *sk = sock->sk;
 769         struct tcp_splice_state tss = {
 770                 .pipe = pipe,
 771                 .len = len,
 772                 .flags = flags,
 773         };
 774         long timeo;
 775         ssize_t spliced;
 776         int ret;
 777
 778         sock_rps_record_flow(sk);
 779         /*
 780          * We can't seek on a socket input
 781          */
 782         if (unlikely(*ppos))
 783                 return -ESPIPE;
 784
 785         ret = spliced = 0;
 786
 787         lock_sock(sk);
 788
 789         timeo = sock_rcvtimeo(sk, sock->file->f_flags & O_NONBLOCK);
 790         while (tss.len) {
 791                 ret = __tcp_splice_read(sk, &tss);
 792                 if (ret < 0)
 793                         break;
 794                 else if (!ret) {
 795                         if (spliced)
 796                                 break;
 797                         if (sock_flag(sk, SOCK_DONE))
 798                                 break;
 799                         if (sk->sk_err) {
 800                                 ret = sock_error(sk);
 801                                 break;
 802                         }
 803                         if (sk->sk_shutdown & RCV_SHUTDOWN)
 804                                 break;
 805                         if (sk->sk_state == TCP_CLOSE) {
 806                                 /*
 807                                  * This occurs when user tries to read
 808                                  * from never connected socket.
 809                                  */
 810                                 ret = -ENOTCONN;
 811                                 break;
 812                         }
 813                         if (!timeo) {
 814                                 ret = -EAGAIN;
 815                                 break;
 816                         }
 817                         /* if __tcp_splice_read() got nothing while we have
 818                          * an skb in receive queue, we do not want to loop.
 819                          * This might happen with URG data.
 820                          */
 821                         if (!skb_queue_empty(&sk->sk_receive_queue))
 822                                 break;
 823                         sk_wait_data(sk, &timeo, NULL);
 824                         if (signal_pending(current)) {
 825                                 ret = sock_intr_errno(timeo);
 826                                 break;
 827                         }
 828                         continue;
 829                 }
 830                 tss.len -= ret;
 831                 spliced += ret;
 832
 833                 if (!timeo)
 834                         break;
 835                 release_sock(sk);
 836                 lock_sock(sk);
 837
 838                 if (sk->sk_err || sk->sk_state == TCP_CLOSE ||
 839                     (sk->sk_shutdown & RCV_SHUTDOWN) ||
 840                     signal_pending(current))
 841                         break;
 842         }
 843
 844         release_sock(sk);
 845
 846         if (spliced)
 847                 return spliced;
 848
 849         return ret;
 850 }
 851 EXPORT_SYMBOL(tcp_splice_read);
 852
 853 struct sk_buff *sk_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp,
 854                                     bool force_schedule)
 855 {
 856         struct sk_buff *skb;
 857
 858         if (likely(!size)) {
 859                 skb = sk->sk_tx_skb_cache;
 860                 if (skb) {
 861                         skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
 862                         sk->sk_tx_skb_cache = NULL;
 863                         pskb_trim(skb, 0);
 864                         INIT_LIST_HEAD(&skb->tcp_tsorted_anchor);
 865                         skb_shinfo(skb)->tx_flags = 0;
 866                         memset(TCP_SKB_CB(skb), 0, sizeof(struct tcp_skb_cb));
 867                         return skb;
 868                 }
 869         }
 870         /* The TCP header must be at least 32-bit aligned.  */
 871         size = ALIGN(size, 4);
 872
 873         if (unlikely(tcp_under_memory_pressure(sk)))
 874                 sk_mem_reclaim_partial(sk);
 875
 876         skb = alloc_skb_fclone(size + sk->sk_prot->max_header, gfp);
 877         if (likely(skb)) {
 878                 bool mem_scheduled;
 879
 880                 if (force_schedule) {
 881                         mem_scheduled = true;
 882                         sk_forced_mem_schedule(sk, skb->truesize);
 883                 } else {
 884                         mem_scheduled = sk_wmem_schedule(sk, skb->truesize);
 885                 }
 886                 if (likely(mem_scheduled)) {
 887                         skb_reserve(skb, sk->sk_prot->max_header);
 888                         /*
 889                          * Make sure that we have exactly size bytes
 890                          * available to the caller, no more, no less.
 891                          */
 892                         skb->reserved_tailroom = skb->end - skb->tail - size;
 893                         INIT_LIST_HEAD(&skb->tcp_tsorted_anchor);
 894                         return skb;
 895                 }
 896                 __kfree_skb(skb);
 897         } else {
 898                 sk->sk_prot->enter_memory_pressure(sk);
 899                 sk_stream_moderate_sndbuf(sk);
 900         }
 901         return NULL;
 902 }
 903
 904 static unsigned int tcp_xmit_size_goal(struct sock *sk, u32 mss_now,
 905                                        int large_allowed)
 906 {
 907         struct tcp_sock *tp = tcp_sk(sk);
 908         u32 new_size_goal, size_goal;
 909
 910         if (!large_allowed)
 911                 return mss_now;
 912
 913         /* Note : tcp_tso_autosize() will eventually split this later */
 914         new_size_goal = sk->sk_gso_max_size - 1 - MAX_TCP_HEADER;
 915         new_size_goal = tcp_bound_to_half_wnd(tp, new_size_goal);
 916
 917         /* We try hard to avoid divides here */
 918         size_goal = tp->gso_segs * mss_now;
 919         if (unlikely(new_size_goal < size_goal ||
 920                      new_size_goal >= size_goal + mss_now)) {
 921                 tp->gso_segs = min_t(u16, new_size_goal / mss_now,
 922                                      sk->sk_gso_max_segs);
 923                 size_goal = tp->gso_segs * mss_now;
 924         }
 925
 926         return max(size_goal, mss_now);
 927 }
 928
 929 static int tcp_send_mss(struct sock *sk, int *size_goal, int flags)
 930 {
 931         int mss_now;
 932
 933         mss_now = tcp_current_mss(sk);
 934         *size_goal = tcp_xmit_size_goal(sk, mss_now, !(flags & MSG_OOB));
 935
 936         return mss_now;
 937 }
 938
 939 /* In some cases, both sendpage() and sendmsg() could have added
 940  * an skb to the write queue, but failed adding payload on it.
 941  * We need to remove it to consume less memory, but more
 942  * importantly be able to generate EPOLLOUT for Edge Trigger epoll()
 943  * users.
 944  */
 945 static void tcp_remove_empty_skb(struct sock *sk, struct sk_buff *skb)
 946 {
 947         if (skb && !skb->len) {
 948                 tcp_unlink_write_queue(skb, sk);
 949                 if (tcp_write_queue_empty(sk))
 950                         tcp_chrono_stop(sk, TCP_CHRONO_BUSY);
 951                 sk_wmem_free_skb(sk, skb);
 952         }
 953 }
 954
 955 ssize_t do_tcp_sendpages(struct sock *sk, struct page *page, int offset,
 956                          size_t size, int flags)
 957 {
 958         struct tcp_sock *tp = tcp_sk(sk);
 959         int mss_now, size_goal;
 960         int err;
 961         ssize_t copied;
 962         long timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
 963
 964         if (IS_ENABLED(CONFIG_DEBUG_VM) &&
 965             WARN_ONCE(PageSlab(page), "page must not be a Slab one"))
 966                 return -EINVAL;
 967
 968         /* Wait for a connection to finish. One exception is TCP Fast Open
 969          * (passive side) where data is allowed to be sent before a connection
 970          * is fully established.
 971          */
 972         if (((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) &&
 973             !tcp_passive_fastopen(sk)) {
 974                 err = sk_stream_wait_connect(sk, &timeo);
 975                 if (err != 0)
 976                         goto out_err;
 977         }
 978
 979         sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
 980
 981         mss_now = tcp_send_mss(sk, &size_goal, flags);
 982         copied = 0;
 983
 984         err = -EPIPE;
 985         if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
 986                 goto out_err;
 987
 988         while (size > 0) {
 989                 struct sk_buff *skb = tcp_write_queue_tail(sk);
 990                 int copy, i;
 991                 bool can_coalesce;
 992
 993                 if (!skb || (copy = size_goal - skb->len) <= 0 ||
 994                     !tcp_skb_can_collapse_to(skb)) {
 995 new_segment:
 996                         if (!sk_stream_memory_free(sk))
 997                                 goto wait_for_sndbuf;
 998
 999                         skb = sk_stream_alloc_skb(sk, 0, sk->sk_allocation,
1000                                         tcp_rtx_and_write_queues_empty(sk));
1001                         if (!skb)
1002                                 goto wait_for_memory;
1003
1004 #ifdef CONFIG_TLS_DEVICE
1005                         skb->decrypted = !!(flags & MSG_SENDPAGE_DECRYPTED);
1006 #endif
1007                         skb_entail(sk, skb);
1008                         copy = size_goal;
1009                 }
1010
1011                 if (copy > size)
1012                         copy = size;
1013
1014                 i = skb_shinfo(skb)->nr_frags;
1015                 can_coalesce = skb_can_coalesce(skb, i, page, offset);
1016                 if (!can_coalesce && i >= sysctl_max_skb_frags) {
1017                         tcp_mark_push(tp, skb);
1018                         goto new_segment;
1019                 }
1020                 if (!sk_wmem_schedule(sk, copy))
1021                         goto wait_for_memory;
1022
1023                 if (can_coalesce) {
1024                         skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
1025                 } else {
1026                         get_page(page);
1027                         skb_fill_page_desc(skb, i, page, offset, copy);
1028                 }
1029
1030                 if (!(flags & MSG_NO_SHARED_FRAGS))
1031                         skb_shinfo(skb)->tx_flags |= SKBTX_SHARED_FRAG;
1032
1033                 skb->len += copy;
1034                 skb->data_len += copy;
1035                 skb->truesize += copy;
1036                 sk->sk_wmem_queued += copy;
1037                 sk_mem_charge(sk, copy);
1038                 skb->ip_summed = CHECKSUM_PARTIAL;
1039                 WRITE_ONCE(tp->write_seq, tp->write_seq + copy);
1040                 TCP_SKB_CB(skb)->end_seq += copy;
1041                 tcp_skb_pcount_set(skb, 0);
1042
1043                 if (!copied)
1044                         TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH;
1045
1046                 copied += copy;
1047                 offset += copy;
1048                 size -= copy;
1049                 if (!size)
1050                         goto out;
1051
1052                 if (skb->len < size_goal || (flags & MSG_OOB))
1053                         continue;
1054
1055                 if (forced_push(tp)) {
1056                         tcp_mark_push(tp, skb);
1057                         __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH);
1058                 } else if (skb == tcp_send_head(sk))
1059                         tcp_push_one(sk, mss_now);
1060                 continue;
1061
1062 wait_for_sndbuf:
1063                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1064 wait_for_memory:
1065                 tcp_push(sk, flags & ~MSG_MORE, mss_now,
1066                          TCP_NAGLE_PUSH, size_goal);
1067
1068                 err = sk_stream_wait_memory(sk, &timeo);
1069                 if (err != 0)
1070                         goto do_error;
1071
1072                 mss_now = tcp_send_mss(sk, &size_goal, flags);
1073         }
1074
1075 out:
1076         if (copied) {
1077                 tcp_tx_timestamp(sk, sk->sk_tsflags);
1078                 if (!(flags & MSG_SENDPAGE_NOTLAST))
1079                         tcp_push(sk, flags, mss_now, tp->nonagle, size_goal);
1080         }
1081         return copied;
1082
1083 do_error:
1084         tcp_remove_empty_skb(sk, tcp_write_queue_tail(sk));
1085         if (copied)
1086                 goto out;
1087 out_err:
1088         /* make sure we wake any epoll edge trigger waiter */
1089         if (unlikely(skb_queue_len(&sk->sk_write_queue) == 0 &&
1090                      err == -EAGAIN)) {
1091                 sk->sk_write_space(sk);
1092                 tcp_chrono_stop(sk, TCP_CHRONO_SNDBUF_LIMITED);
1093         }
1094         return sk_stream_error(sk, flags, err);
1095 }
1096 EXPORT_SYMBOL_GPL(do_tcp_sendpages);
1097
1098 int tcp_sendpage_locked(struct sock *sk, struct page *page, int offset,
1099                         size_t size, int flags)
1100 {
1101         if (!(sk->sk_route_caps & NETIF_F_SG))
1102                 return sock_no_sendpage_locked(sk, page, offset, size, flags);
1103
1104         tcp_rate_check_app_limited(sk);  /* is sending application-limited? */
1105
1106         return do_tcp_sendpages(sk, page, offset, size, flags);
1107 }
1108 EXPORT_SYMBOL_GPL(tcp_sendpage_locked);
1109
1110 int tcp_sendpage(struct sock *sk, struct page *page, int offset,
1111                  size_t size, int flags)
1112 {
1113         int ret;
1114
1115         lock_sock(sk);
1116         ret = tcp_sendpage_locked(sk, page, offset, size, flags);
1117         release_sock(sk);
1118
1119         return ret;
1120 }
1121 EXPORT_SYMBOL(tcp_sendpage);
1122
1123 void tcp_free_fastopen_req(struct tcp_sock *tp)
1124 {
1125         if (tp->fastopen_req) {
1126                 kfree(tp->fastopen_req);
1127                 tp->fastopen_req = NULL;
1128         }
1129 }
1130
1131 static int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg,
1132                                 int *copied, size_t size,
1133                                 struct ubuf_info *uarg)
1134 {
1135         struct tcp_sock *tp = tcp_sk(sk);
1136         struct inet_sock *inet = inet_sk(sk);
1137         struct sockaddr *uaddr = msg->msg_name;
1138         int err, flags;
1139
1140         if (!(sock_net(sk)->ipv4.sysctl_tcp_fastopen & TFO_CLIENT_ENABLE) ||
1141             (uaddr && msg->msg_namelen >= sizeof(uaddr->sa_family) &&
1142              uaddr->sa_family == AF_UNSPEC))
1143                 return -EOPNOTSUPP;
1144         if (tp->fastopen_req)
1145                 return -EALREADY; /* Another Fast Open is in progress */
1146
1147         tp->fastopen_req = kzalloc(sizeof(struct tcp_fastopen_request),
1148                                    sk->sk_allocation);
1149         if (unlikely(!tp->fastopen_req))
1150                 return -ENOBUFS;
1151         tp->fastopen_req->data = msg;
1152         tp->fastopen_req->size = size;
1153         tp->fastopen_req->uarg = uarg;
1154
1155         if (inet->defer_connect) {
1156                 err = tcp_connect(sk);
1157                 /* Same failure procedure as in tcp_v4/6_connect */
1158                 if (err) {
1159                         tcp_set_state(sk, TCP_CLOSE);
1160                         inet->inet_dport = 0;
1161                         sk->sk_route_caps = 0;
1162                 }
1163         }
1164         flags = (msg->msg_flags & MSG_DONTWAIT) ? O_NONBLOCK : 0;
1165         err = __inet_stream_connect(sk->sk_socket, uaddr,
1166                                     msg->msg_namelen, flags, 1);
1167         /* fastopen_req could already be freed in __inet_stream_connect
1168          * if the connection times out or gets rst
1169          */
1170         if (tp->fastopen_req) {
1171                 *copied = tp->fastopen_req->copied;
1172                 tcp_free_fastopen_req(tp);
1173                 inet->defer_connect = 0;
1174         }
1175         return err;
1176 }
1177
1178 int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size)
1179 {
1180         struct tcp_sock *tp = tcp_sk(sk);
1181         struct ubuf_info *uarg = NULL;
1182         struct sk_buff *skb;
1183         struct sockcm_cookie sockc;
1184         int flags, err, copied = 0;
1185         int mss_now = 0, size_goal, copied_syn = 0;
1186         int process_backlog = 0;
1187         bool zc = false;
1188         long timeo;
1189
1190         flags = msg->msg_flags;
1191
1192         if (flags & MSG_ZEROCOPY && size && sock_flag(sk, SOCK_ZEROCOPY)) {
1193                 skb = tcp_write_queue_tail(sk);
1194                 uarg = sock_zerocopy_realloc(sk, size, skb_zcopy(skb));
1195                 if (!uarg) {
1196                         err = -ENOBUFS;
1197                         goto out_err;
1198                 }
1199
1200                 zc = sk->sk_route_caps & NETIF_F_SG;
1201                 if (!zc)
1202                         uarg->zerocopy = 0;
1203         }
1204
1205         if (unlikely(flags & MSG_FASTOPEN || inet_sk(sk)->defer_connect) &&
1206             !tp->repair) {
1207                 err = tcp_sendmsg_fastopen(sk, msg, &copied_syn, size, uarg);
1208                 if (err == -EINPROGRESS && copied_syn > 0)
1209                         goto out;
1210                 else if (err)
1211                         goto out_err;
1212         }
1213
1214         timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
1215
1216         tcp_rate_check_app_limited(sk);  /* is sending application-limited? */
1217
1218         /* Wait for a connection to finish. One exception is TCP Fast Open
1219          * (passive side) where data is allowed to be sent before a connection
1220          * is fully established.
1221          */
1222         if (((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) &&
1223             !tcp_passive_fastopen(sk)) {
1224                 err = sk_stream_wait_connect(sk, &timeo);
1225                 if (err != 0)
1226                         goto do_error;
1227         }
1228
1229         if (unlikely(tp->repair)) {
1230                 if (tp->repair_queue == TCP_RECV_QUEUE) {
1231                         copied = tcp_send_rcvq(sk, msg, size);
1232                         goto out_nopush;
1233                 }
1234
1235                 err = -EINVAL;
1236                 if (tp->repair_queue == TCP_NO_QUEUE)
1237                         goto out_err;
1238
1239                 /* 'common' sending to sendq */
1240         }
1241
1242         sockcm_init(&sockc, sk);
1243         if (msg->msg_controllen) {
1244                 err = sock_cmsg_send(sk, msg, &sockc);
1245                 if (unlikely(err)) {
1246                         err = -EINVAL;
1247                         goto out_err;
1248                 }
1249         }
1250
1251         /* This should be in poll */
1252         sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1253
1254         /* Ok commence sending. */
1255         copied = 0;
1256
1257 restart:
1258         mss_now = tcp_send_mss(sk, &size_goal, flags);
1259
1260         err = -EPIPE;
1261         if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
1262                 goto do_error;
1263
1264         while (msg_data_left(msg)) {
1265                 int copy = 0;
1266
1267                 skb = tcp_write_queue_tail(sk);
1268                 if (skb)
1269                         copy = size_goal - skb->len;
1270
1271                 if (copy <= 0 || !tcp_skb_can_collapse_to(skb)) {
1272                         bool first_skb;
1273
1274 new_segment:
1275                         if (!sk_stream_memory_free(sk))
1276                                 goto wait_for_sndbuf;
1277
1278                         if (unlikely(process_backlog >= 16)) {
1279                                 process_backlog = 0;
1280                                 if (sk_flush_backlog(sk))
1281                                         goto restart;
1282                         }
1283                         first_skb = tcp_rtx_and_write_queues_empty(sk);
1284                         skb = sk_stream_alloc_skb(sk, 0, sk->sk_allocation,
1285                                                   first_skb);
1286                         if (!skb)
1287                                 goto wait_for_memory;
1288
1289                         process_backlog++;
1290                         skb->ip_summed = CHECKSUM_PARTIAL;
1291
1292                         skb_entail(sk, skb);
1293                         copy = size_goal;
1294
1295                         /* All packets are restored as if they have
1296                          * already been sent. skb_mstamp_ns isn't set to
1297                          * avoid wrong rtt estimation.
1298                          */
1299                         if (tp->repair)
1300                                 TCP_SKB_CB(skb)->sacked |= TCPCB_REPAIRED;
1301                 }
1302
1303                 /* Try to append data to the end of skb. */
1304                 if (copy > msg_data_left(msg))
1305                         copy = msg_data_left(msg);
1306
1307                 /* Where to copy to? */
1308                 if (skb_availroom(skb) > 0 && !zc) {
1309                         /* We have some space in skb head. Superb! */
1310                         copy = min_t(int, copy, skb_availroom(skb));
1311                         err = skb_add_data_nocache(sk, skb, &msg->msg_iter, copy);
1312                         if (err)
1313                                 goto do_fault;
1314                 } else if (!zc) {
1315                         bool merge = true;
1316                         int i = skb_shinfo(skb)->nr_frags;
1317                         struct page_frag *pfrag = sk_page_frag(sk);
1318
1319                         if (!sk_page_frag_refill(sk, pfrag))
1320                                 goto wait_for_memory;
1321
1322                         if (!skb_can_coalesce(skb, i, pfrag->page,
1323                                               pfrag->offset)) {
1324                                 if (i >= sysctl_max_skb_frags) {
1325                                         tcp_mark_push(tp, skb);
1326                                         goto new_segment;
1327                                 }
1328                                 merge = false;
1329                         }
1330
1331                         copy = min_t(int, copy, pfrag->size - pfrag->offset);
1332
1333                         if (!sk_wmem_schedule(sk, copy))
1334                                 goto wait_for_memory;
1335
1336                         err = skb_copy_to_page_nocache(sk, &msg->msg_iter, skb,
1337                                                        pfrag->page,
1338                                                        pfrag->offset,
1339                                                        copy);
1340                         if (err)
1341                                 goto do_error;
1342
1343                         /* Update the skb. */
1344                         if (merge) {
1345                                 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
1346                         } else {
1347                                 skb_fill_page_desc(skb, i, pfrag->page,
1348                                                    pfrag->offset, copy);
1349                                 page_ref_inc(pfrag->page);
1350                         }
1351                         pfrag->offset += copy;
1352                 } else {
1353                         err = skb_zerocopy_iter_stream(sk, skb, msg, copy, uarg);
1354                         if (err == -EMSGSIZE || err == -EEXIST) {
1355                                 tcp_mark_push(tp, skb);
1356                                 goto new_segment;
1357                         }
1358                         if (err < 0)
1359                                 goto do_error;
1360                         copy = err;
1361                 }
1362
1363                 if (!copied)
1364                         TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH;
1365
1366                 WRITE_ONCE(tp->write_seq, tp->write_seq + copy);
1367                 TCP_SKB_CB(skb)->end_seq += copy;
1368                 tcp_skb_pcount_set(skb, 0);
1369
1370                 copied += copy;
1371                 if (!msg_data_left(msg)) {
1372                         if (unlikely(flags & MSG_EOR))
1373                                 TCP_SKB_CB(skb)->eor = 1;
1374                         goto out;
1375                 }
1376
1377                 if (skb->len < size_goal || (flags & MSG_OOB) || unlikely(tp->repair))
1378                         continue;
1379
1380                 if (forced_push(tp)) {
1381                         tcp_mark_push(tp, skb);
1382                         __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH);
1383                 } else if (skb == tcp_send_head(sk))
1384                         tcp_push_one(sk, mss_now);
1385                 continue;
1386
1387 wait_for_sndbuf:
1388                 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1389 wait_for_memory:
1390                 if (copied)
1391                         tcp_push(sk, flags & ~MSG_MORE, mss_now,
1392                                  TCP_NAGLE_PUSH, size_goal);
1393
1394                 err = sk_stream_wait_memory(sk, &timeo);
1395                 if (err != 0)
1396                         goto do_error;
1397
1398                 mss_now = tcp_send_mss(sk, &size_goal, flags);
1399         }
1400
1401 out:
1402         if (copied) {
1403                 tcp_tx_timestamp(sk, sockc.tsflags);
1404                 tcp_push(sk, flags, mss_now, tp->nonagle, size_goal);
1405         }
1406 out_nopush:
1407         sock_zerocopy_put(uarg);
1408         return copied + copied_syn;
1409
1410 do_error:
1411         skb = tcp_write_queue_tail(sk);
1412 do_fault:
1413         tcp_remove_empty_skb(sk, skb);
1414
1415         if (copied + copied_syn)
1416                 goto out;
1417 out_err:
1418         sock_zerocopy_put_abort(uarg, true);
1419         err = sk_stream_error(sk, flags, err);
1420         /* make sure we wake any epoll edge trigger waiter */
1421         if (unlikely(skb_queue_len(&sk->sk_write_queue) == 0 &&
1422                      err == -EAGAIN)) {
1423                 sk->sk_write_space(sk);
1424                 tcp_chrono_stop(sk, TCP_CHRONO_SNDBUF_LIMITED);
1425         }
1426         return err;
1427 }
1428 EXPORT_SYMBOL_GPL(tcp_sendmsg_locked);
1429
1430 int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
1431 {
1432         int ret;
1433
1434         lock_sock(sk);
1435         ret = tcp_sendmsg_locked(sk, msg, size);
1436         release_sock(sk);
1437
1438         return ret;
1439 }
1440 EXPORT_SYMBOL(tcp_sendmsg);
1441
1442 /*
1443  *      Handle reading urgent data. BSD has very simple semantics for
1444  *      this, no blocking and very strange errors 8)
1445  */
1446
1447 static int tcp_recv_urg(struct sock *sk, struct msghdr *msg, int len, int flags)
1448 {
1449         struct tcp_sock *tp = tcp_sk(sk);
1450
1451         /* No URG data to read. */
1452         if (sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data ||
1453             tp->urg_data == TCP_URG_READ)
1454                 return -EINVAL; /* Yes this is right ! */
1455
1456         if (sk->sk_state == TCP_CLOSE && !sock_flag(sk, SOCK_DONE))
1457                 return -ENOTCONN;
1458
1459         if (tp->urg_data & TCP_URG_VALID) {
1460                 int err = 0;
1461                 char c = tp->urg_data;
1462
1463                 if (!(flags & MSG_PEEK))
1464                         tp->urg_data = TCP_URG_READ;
1465
1466                 /* Read urgent data. */
1467                 msg->msg_flags |= MSG_OOB;
1468
1469                 if (len > 0) {
1470                         if (!(flags & MSG_TRUNC))
1471                                 err = memcpy_to_msg(msg, &c, 1);
1472                         len = 1;
1473                 } else
1474                         msg->msg_flags |= MSG_TRUNC;
1475
1476                 return err ? -EFAULT : len;
1477         }
1478
1479         if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN))
1480                 return 0;
1481
1482         /* Fixed the recv(..., MSG_OOB) behaviour.  BSD docs and
1483          * the available implementations agree in this case:
1484          * this call should never block, independent of the
1485          * blocking state of the socket.
1486          * Mike <pall@rz.uni-karlsruhe.de>
1487          */
1488         return -EAGAIN;
1489 }
1490
1491 static int tcp_peek_sndq(struct sock *sk, struct msghdr *msg, int len)
1492 {
1493         struct sk_buff *skb;
1494         int copied = 0, err = 0;
1495
1496         /* XXX -- need to support SO_PEEK_OFF */
1497
1498         skb_rbtree_walk(skb, &sk->tcp_rtx_queue) {
1499                 err = skb_copy_datagram_msg(skb, 0, msg, skb->len);
1500                 if (err)
1501                         return err;
1502                 copied += skb->len;
1503         }
1504
1505         skb_queue_walk(&sk->sk_write_queue, skb) {
1506                 err = skb_copy_datagram_msg(skb, 0, msg, skb->len);
1507                 if (err)
1508                         break;
1509
1510                 copied += skb->len;
1511         }
1512
1513         return err ?: copied;
1514 }
1515
1516 /* Clean up the receive buffer for full frames taken by the user,
1517  * then send an ACK if necessary.  COPIED is the number of bytes
1518  * tcp_recvmsg has given to the user so far, it speeds up the
1519  * calculation of whether or not we must ACK for the sake of
1520  * a window update.
1521  */
1522 static void tcp_cleanup_rbuf(struct sock *sk, int copied)
1523 {
1524         struct tcp_sock *tp = tcp_sk(sk);
1525         bool time_to_ack = false;
1526
1527         struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);
1528
1529         WARN(skb && !before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq),
1530              "cleanup rbuf bug: copied %X seq %X rcvnxt %X\n",
1531              tp->copied_seq, TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt);
1532
1533         if (inet_csk_ack_scheduled(sk)) {
1534                 const struct inet_connection_sock *icsk = inet_csk(sk);
1535                    /* Delayed ACKs frequently hit locked sockets during bulk
1536                     * receive. */
1537                 if (icsk->icsk_ack.blocked ||
1538                     /* Once-per-two-segments ACK was not sent by tcp_input.c */
1539                     tp->rcv_nxt - tp->rcv_wup > icsk->icsk_ack.rcv_mss ||
1540                     /*
1541                      * If this read emptied read buffer, we send ACK, if
1542                      * connection is not bidirectional, user drained
1543                      * receive buffer and there was a small segment
1544                      * in queue.
1545                      */
1546                     (copied > 0 &&
1547                      ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED2) ||
1548                       ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED) &&
1549                        !inet_csk_in_pingpong_mode(sk))) &&
1550                       !atomic_read(&sk->sk_rmem_alloc)))
1551                         time_to_ack = true;
1552         }
1553
1554         /* We send an ACK if we can now advertise a non-zero window
1555          * which has been raised "significantly".
1556          *
1557          * Even if window raised up to infinity, do not send window open ACK
1558          * in states, where we will not receive more. It is useless.
1559          */
1560         if (copied > 0 && !time_to_ack && !(sk->sk_shutdown & RCV_SHUTDOWN)) {
1561                 __u32 rcv_window_now = tcp_receive_window(tp);
1562
1563                 /* Optimize, __tcp_select_window() is not cheap. */
1564                 if (2*rcv_window_now <= tp->window_clamp) {
1565                         __u32 new_window = __tcp_select_window(sk);
1566
1567                         /* Send ACK now, if this read freed lots of space
1568                          * in our buffer. Certainly, new_window is new window.
1569                          * We can advertise it now, if it is not less than current one.
1570                          * "Lots" means "at least twice" here.
1571                          */
1572                         if (new_window && new_window >= 2 * rcv_window_now)
1573                                 time_to_ack = true;
1574                 }
1575         }
1576         if (time_to_ack)
1577                 tcp_send_ack(sk);
1578 }
1579
1580 static struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off)
1581 {
1582         struct sk_buff *skb;
1583         u32 offset;
1584
1585         while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) {
1586                 offset = seq - TCP_SKB_CB(skb)->seq;
1587                 if (unlikely(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
1588                         pr_err_once("%s: found a SYN, please report !\n", __func__);
1589                         offset--;
1590                 }
1591                 if (offset < skb->len || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)) {
1592                         *off = offset;
1593                         return skb;
1594                 }
1595                 /* This looks weird, but this can happen if TCP collapsing
1596                  * splitted a fat GRO packet, while we released socket lock
1597                  * in skb_splice_bits()
1598                  */
1599                 sk_eat_skb(sk, skb);
1600         }
1601         return NULL;
1602 }
1603
1604 /*
1605  * This routine provides an alternative to tcp_recvmsg() for routines
1606  * that would like to handle copying from skbuffs directly in 'sendfile'
1607  * fashion.
1608  * Note:
1609  *      - It is assumed that the socket was locked by the caller.
1610  *      - The routine does not block.
1611  *      - At present, there is no support for reading OOB data
1612  *        or for 'peeking' the socket using this routine
1613  *        (although both would be easy to implement).
1614  */
1615 int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
1616                   sk_read_actor_t recv_actor)
1617 {
1618         struct sk_buff *skb;
1619         struct tcp_sock *tp = tcp_sk(sk);
1620         u32 seq = tp->copied_seq;
1621         u32 offset;
1622         int copied = 0;
1623
1624         if (sk->sk_state == TCP_LISTEN)
1625                 return -ENOTCONN;
1626         while ((skb = tcp_recv_skb(sk, seq, &offset)) != NULL) {
1627                 if (offset < skb->len) {
1628                         int used;
1629                         size_t len;
1630
1631                         len = skb->len - offset;
1632                         /* Stop reading if we hit a patch of urgent data */
1633                         if (tp->urg_data) {
1634                                 u32 urg_offset = tp->urg_seq - seq;
1635                                 if (urg_offset < len)
1636                                         len = urg_offset;
1637                                 if (!len)
1638                                         break;
1639                         }
1640                         used = recv_actor(desc, skb, offset, len);
1641                         if (used <= 0) {
1642                                 if (!copied)
1643                                         copied = used;
1644                                 break;
1645                         } else if (used <= len) {
1646                                 seq += used;
1647                                 copied += used;
1648                                 offset += used;
1649                         }
1650                         /* If recv_actor drops the lock (e.g. TCP splice
1651                          * receive) the skb pointer might be invalid when
1652                          * getting here: tcp_collapse might have deleted it
1653                          * while aggregating skbs from the socket queue.
1654                          */
1655                         skb = tcp_recv_skb(sk, seq - 1, &offset);
1656                         if (!skb)
1657                                 break;
1658                         /* TCP coalescing might have appended data to the skb.
1659                          * Try to splice more frags
1660                          */
1661                         if (offset + 1 != skb->len)
1662                                 continue;
1663                 }
1664                 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) {
1665                         sk_eat_skb(sk, skb);
1666                         ++seq;
1667                         break;
1668                 }
1669                 sk_eat_skb(sk, skb);
1670                 if (!desc->count)
1671                         break;
1672                 WRITE_ONCE(tp->copied_seq, seq);
1673         }
1674         WRITE_ONCE(tp->copied_seq, seq);
1675
1676         tcp_rcv_space_adjust(sk);
1677
1678         /* Clean up data we have read: This will do ACK frames. */
1679         if (copied > 0) {
1680                 tcp_recv_skb(sk, seq, &offset);
1681                 tcp_cleanup_rbuf(sk, copied);
1682         }
1683         return copied;
1684 }
1685 EXPORT_SYMBOL(tcp_read_sock);
1686
1687 int tcp_peek_len(struct socket *sock)
1688 {
1689         return tcp_inq(sock->sk);
1690 }
1691 EXPORT_SYMBOL(tcp_peek_len);
1692
1693 /* Make sure sk_rcvbuf is big enough to satisfy SO_RCVLOWAT hint */
1694 int tcp_set_rcvlowat(struct sock *sk, int val)
1695 {
1696         int cap;
1697
1698         if (sk->sk_userlocks & SOCK_RCVBUF_LOCK)
1699                 cap = sk->sk_rcvbuf >> 1;
1700         else
1701                 cap = sock_net(sk)->ipv4.sysctl_tcp_rmem[2] >> 1;
1702         val = min(val, cap);
1703         WRITE_ONCE(sk->sk_rcvlowat, val ? : 1);
1704
1705         /* Check if we need to signal EPOLLIN right now */
1706         tcp_data_ready(sk);
1707
1708         if (sk->sk_userlocks & SOCK_RCVBUF_LOCK)
1709                 return 0;
1710
1711         val <<= 1;
1712         if (val > sk->sk_rcvbuf) {
1713                 sk->sk_rcvbuf = val;
1714                 tcp_sk(sk)->window_clamp = tcp_win_from_space(sk, val);
1715         }
1716         return 0;
1717 }
1718 EXPORT_SYMBOL(tcp_set_rcvlowat);
1719
1720 #ifdef CONFIG_MMU
1721 static const struct vm_operations_struct tcp_vm_ops = {
1722 };
1723
1724 int tcp_mmap(struct file *file, struct socket *sock,
1725              struct vm_area_struct *vma)
1726 {
1727         if (vma->vm_flags & (VM_WRITE | VM_EXEC))
1728                 return -EPERM;
1729         vma->vm_flags &= ~(VM_MAYWRITE | VM_MAYEXEC);
1730
1731         /* Instruct vm_insert_page() to not down_read(mmap_sem) */
1732         vma->vm_flags |= VM_MIXEDMAP;
1733
1734         vma->vm_ops = &tcp_vm_ops;
1735         return 0;
1736 }
1737 EXPORT_SYMBOL(tcp_mmap);
1738
1739 static int tcp_zerocopy_receive(struct sock *sk,
1740                                 struct tcp_zerocopy_receive *zc)
1741 {
1742         unsigned long address = (unsigned long)zc->address;
1743         const skb_frag_t *frags = NULL;
1744         u32 length = 0, seq, offset;
1745         struct vm_area_struct *vma;
1746         struct sk_buff *skb = NULL;
1747         struct tcp_sock *tp;
1748         int inq;
1749         int ret;
1750
1751         if (address & (PAGE_SIZE - 1) || address != zc->address)
1752                 return -EINVAL;
1753
1754         if (sk->sk_state == TCP_LISTEN)
1755                 return -ENOTCONN;
1756
1757         sock_rps_record_flow(sk);
1758
1759         down_read(&current->mm->mmap_sem);
1760
1761         ret = -EINVAL;
1762         vma = find_vma(current->mm, address);
1763         if (!vma || vma->vm_start > address || vma->vm_ops != &tcp_vm_ops)
1764                 goto out;
1765         zc->length = min_t(unsigned long, zc->length, vma->vm_end - address);
1766
1767         tp = tcp_sk(sk);
1768         seq = tp->copied_seq;
1769         inq = tcp_inq(sk);
1770         zc->length = min_t(u32, zc->length, inq);
1771         zc->length &= ~(PAGE_SIZE - 1);
1772         if (zc->length) {
1773                 zap_page_range(vma, address, zc->length);
1774                 zc->recv_skip_hint = 0;
1775         } else {
1776                 zc->recv_skip_hint = inq;
1777         }
1778         ret = 0;
1779         while (length + PAGE_SIZE <= zc->length) {
1780                 if (zc->recv_skip_hint < PAGE_SIZE) {
1781                         if (skb) {
1782                                 skb = skb->next;
1783                                 offset = seq - TCP_SKB_CB(skb)->seq;
1784                         } else {
1785                                 skb = tcp_recv_skb(sk, seq, &offset);
1786                         }
1787
1788                         zc->recv_skip_hint = skb->len - offset;
1789                         offset -= skb_headlen(skb);
1790                         if ((int)offset < 0 || skb_has_frag_list(skb))
1791                                 break;
1792                         frags = skb_shinfo(skb)->frags;
1793                         while (offset) {
1794                                 if (skb_frag_size(frags) > offset)
1795                                         goto out;
1796                                 offset -= skb_frag_size(frags);
1797                                 frags++;
1798                         }
1799                 }
1800                 if (skb_frag_size(frags) != PAGE_SIZE || skb_frag_off(frags)) {
1801                         int remaining = zc->recv_skip_hint;
1802
1803                         while (remaining && (skb_frag_size(frags) != PAGE_SIZE ||
1804                                              skb_frag_off(frags))) {
1805                                 remaining -= skb_frag_size(frags);
1806                                 frags++;
1807                         }
1808                         zc->recv_skip_hint -= remaining;
1809                         break;
1810                 }
1811                 ret = vm_insert_page(vma, address + length,
1812                                      skb_frag_page(frags));
1813                 if (ret)
1814                         break;
1815                 length += PAGE_SIZE;
1816                 seq += PAGE_SIZE;
1817                 zc->recv_skip_hint -= PAGE_SIZE;
1818                 frags++;
1819         }
1820 out:
1821         up_read(&current->mm->mmap_sem);
1822         if (length) {
1823                 WRITE_ONCE(tp->copied_seq, seq);
1824                 tcp_rcv_space_adjust(sk);
1825
1826                 /* Clean up data we have read: This will do ACK frames. */
1827                 tcp_recv_skb(sk, seq, &offset);
1828                 tcp_cleanup_rbuf(sk, length);
1829                 ret = 0;
1830                 if (length == zc->length)
1831                         zc->recv_skip_hint = 0;
1832         } else {
1833                 if (!zc->recv_skip_hint && sock_flag(sk, SOCK_DONE))
1834                         ret = -EIO;
1835         }
1836         zc->length = length;
1837         return ret;
1838 }
1839 #endif
1840
1841 static void tcp_update_recv_tstamps(struct sk_buff *skb,
1842                                     struct scm_timestamping_internal *tss)
1843 {
1844         if (skb->tstamp)
1845                 tss->ts[0] = ktime_to_timespec64(skb->tstamp);
1846         else
1847                 tss->ts[0] = (struct timespec64) {0};
1848
1849         if (skb_hwtstamps(skb)->hwtstamp)
1850                 tss->ts[2] = ktime_to_timespec64(skb_hwtstamps(skb)->hwtstamp);
1851         else
1852                 tss->ts[2] = (struct timespec64) {0};
1853 }
1854
1855 /* Similar to __sock_recv_timestamp, but does not require an skb */
1856 static void tcp_recv_timestamp(struct msghdr *msg, const struct sock *sk,
1857                                struct scm_timestamping_internal *tss)
1858 {
1859         int new_tstamp = sock_flag(sk, SOCK_TSTAMP_NEW);
1860         bool has_timestamping = false;
1861
1862         if (tss->ts[0].tv_sec || tss->ts[0].tv_nsec) {
1863                 if (sock_flag(sk, SOCK_RCVTSTAMP)) {
1864                         if (sock_flag(sk, SOCK_RCVTSTAMPNS)) {
1865                                 if (new_tstamp) {
1866                                         struct __kernel_timespec kts = {tss->ts[0].tv_sec, tss->ts[0].tv_nsec};
1867
1868                                         put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMPNS_NEW,
1869                                                  sizeof(kts), &kts);
1870                                 } else {
1871                                         struct timespec ts_old = timespec64_to_timespec(tss->ts[0]);
1872
1873                                         put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMPNS_OLD,
1874                                                  sizeof(ts_old), &ts_old);
1875                                 }
1876                         } else {
1877                                 if (new_tstamp) {
1878                                         struct __kernel_sock_timeval stv;
1879
1880                                         stv.tv_sec = tss->ts[0].tv_sec;
1881                                         stv.tv_usec = tss->ts[0].tv_nsec / 1000;
1882                                         put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMP_NEW,
1883                                                  sizeof(stv), &stv);
1884                                 } else {
1885                                         struct __kernel_old_timeval tv;
1886
1887                                         tv.tv_sec = tss->ts[0].tv_sec;
1888                                         tv.tv_usec = tss->ts[0].tv_nsec / 1000;
1889                                         put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMP_OLD,
1890                                                  sizeof(tv), &tv);
1891                                 }
1892                         }
1893                 }
1894
1895                 if (sk->sk_tsflags & SOF_TIMESTAMPING_SOFTWARE)
1896                         has_timestamping = true;
1897                 else
1898                         tss->ts[0] = (struct timespec64) {0};
1899         }
1900
1901         if (tss->ts[2].tv_sec || tss->ts[2].tv_nsec) {
1902                 if (sk->sk_tsflags & SOF_TIMESTAMPING_RAW_HARDWARE)
1903                         has_timestamping = true;
1904                 else
1905                         tss->ts[2] = (struct timespec64) {0};
1906         }
1907
1908         if (has_timestamping) {
1909                 tss->ts[1] = (struct timespec64) {0};
1910                 if (sock_flag(sk, SOCK_TSTAMP_NEW))
1911                         put_cmsg_scm_timestamping64(msg, tss);
1912                 else
1913                         put_cmsg_scm_timestamping(msg, tss);
1914         }
1915 }
1916
1917 static int tcp_inq_hint(struct sock *sk)
1918 {
1919         const struct tcp_sock *tp = tcp_sk(sk);
1920         u32 copied_seq = READ_ONCE(tp->copied_seq);
1921         u32 rcv_nxt = READ_ONCE(tp->rcv_nxt);
1922         int inq;
1923
1924         inq = rcv_nxt - copied_seq;
1925         if (unlikely(inq < 0 || copied_seq != READ_ONCE(tp->copied_seq))) {
1926                 lock_sock(sk);
1927                 inq = tp->rcv_nxt - tp->copied_seq;
1928                 release_sock(sk);
1929         }
1930         /* After receiving a FIN, tell the user-space to continue reading
1931          * by returning a non-zero inq.
1932          */
1933         if (inq == 0 && sock_flag(sk, SOCK_DONE))
1934                 inq = 1;
1935         return inq;
1936 }
1937
1938 /*
1939  *      This routine copies from a sock struct into the user buffer.
1940  *
1941  *      Technical note: in 2.3 we work on _locked_ socket, so that
1942  *      tricks with *seq access order and skb->users are not required.
1943  *      Probably, code can be easily improved even more.
1944  */
1945
1946 int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int nonblock,
1947                 int flags, int *addr_len)
1948 {
1949         struct tcp_sock *tp = tcp_sk(sk);
1950         int copied = 0;
1951         u32 peek_seq;
1952         u32 *seq;
1953         unsigned long used;
1954         int err, inq;
1955         int target;             /* Read at least this many bytes */
1956         long timeo;
1957         struct sk_buff *skb, *last;
1958         u32 urg_hole = 0;
1959         struct scm_timestamping_internal tss;
1960         bool has_tss = false;
1961         bool has_cmsg;
1962
1963         if (unlikely(flags & MSG_ERRQUEUE))
1964                 return inet_recv_error(sk, msg, len, addr_len);
1965
1966         if (sk_can_busy_loop(sk) && skb_queue_empty(&sk->sk_receive_queue) &&
1967             (sk->sk_state == TCP_ESTABLISHED))
1968                 sk_busy_loop(sk, nonblock);
1969
1970         lock_sock(sk);
1971
1972         err = -ENOTCONN;
1973         if (sk->sk_state == TCP_LISTEN)
1974                 goto out;
1975
1976         has_cmsg = tp->recvmsg_inq;
1977         timeo = sock_rcvtimeo(sk, nonblock);
1978
1979         /* Urgent data needs to be handled specially. */
1980         if (flags & MSG_OOB)
1981                 goto recv_urg;
1982
1983         if (unlikely(tp->repair)) {
1984                 err = -EPERM;
1985                 if (!(flags & MSG_PEEK))
1986                         goto out;
1987
1988                 if (tp->repair_queue == TCP_SEND_QUEUE)
1989                         goto recv_sndq;
1990
1991                 err = -EINVAL;
1992                 if (tp->repair_queue == TCP_NO_QUEUE)
1993                         goto out;
1994
1995                 /* 'common' recv queue MSG_PEEK-ing */
1996         }
1997
1998         seq = &tp->copied_seq;
1999         if (flags & MSG_PEEK) {
2000                 peek_seq = tp->copied_seq;
2001                 seq = &peek_seq;
2002         }
2003
2004         target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
2005
2006         do {
2007                 u32 offset;
2008
2009                 /* Are we at urgent data? Stop if we have read anything or have SIGURG pending. */
2010                 if (tp->urg_data && tp->urg_seq == *seq) {
2011                         if (copied)
2012                                 break;
2013                         if (signal_pending(current)) {
2014                                 copied = timeo ? sock_intr_errno(timeo) : -EAGAIN;
2015                                 break;
2016                         }
2017                 }
2018
2019                 /* Next get a buffer. */
2020
2021                 last = skb_peek_tail(&sk->sk_receive_queue);
2022                 skb_queue_walk(&sk->sk_receive_queue, skb) {
2023                         last = skb;
2024                         /* Now that we have two receive queues this
2025                          * shouldn't happen.
2026                          */
2027                         if (WARN(before(*seq, TCP_SKB_CB(skb)->seq),
2028                                  "TCP recvmsg seq # bug: copied %X, seq %X, rcvnxt %X, fl %X\n",
2029                                  *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt,
2030                                  flags))
2031                                 break;
2032
2033                         offset = *seq - TCP_SKB_CB(skb)->seq;
2034                         if (unlikely(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
2035                                 pr_err_once("%s: found a SYN, please report !\n", __func__);
2036                                 offset--;
2037                         }
2038                         if (offset < skb->len)
2039                                 goto found_ok_skb;
2040                         if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
2041                                 goto found_fin_ok;
2042                         WARN(!(flags & MSG_PEEK),
2043                              "TCP recvmsg seq # bug 2: copied %X, seq %X, rcvnxt %X, fl %X\n",
2044                              *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt, flags);
2045                 }
2046
2047                 /* Well, if we have backlog, try to process it now yet. */
2048
2049                 if (copied >= target && !sk->sk_backlog.tail)
2050                         break;
2051
2052                 if (copied) {
2053                         if (sk->sk_err ||
2054                             sk->sk_state == TCP_CLOSE ||
2055                             (sk->sk_shutdown & RCV_SHUTDOWN) ||
2056                             !timeo ||
2057                             signal_pending(current))
2058                                 break;
2059                 } else {
2060                         if (sock_flag(sk, SOCK_DONE))
2061                                 break;
2062
2063                         if (sk->sk_err) {
2064                                 copied = sock_error(sk);
2065                                 break;
2066                         }
2067
2068                         if (sk->sk_shutdown & RCV_SHUTDOWN)
2069                                 break;
2070
2071                         if (sk->sk_state == TCP_CLOSE) {
2072                                 /* This occurs when user tries to read
2073                                  * from never connected socket.
2074                                  */
2075                                 copied = -ENOTCONN;
2076                                 break;
2077                         }
2078
2079                         if (!timeo) {
2080                                 copied = -EAGAIN;
2081                                 break;
2082                         }
2083
2084                         if (signal_pending(current)) {
2085                                 copied = sock_intr_errno(timeo);
2086                                 break;
2087                         }
2088                 }
2089
2090                 tcp_cleanup_rbuf(sk, copied);
2091
2092                 if (copied >= target) {
2093                         /* Do not sleep, just process backlog. */
2094                         release_sock(sk);
2095                         lock_sock(sk);
2096                 } else {
2097                         sk_wait_data(sk, &timeo, last);
2098                 }
2099
2100                 if ((flags & MSG_PEEK) &&
2101                     (peek_seq - copied - urg_hole != tp->copied_seq)) {
2102                         net_dbg_ratelimited("TCP(%s:%d): Application bug, race in MSG_PEEK\n",
2103                                             current->comm,
2104                                             task_pid_nr(current));
2105                         peek_seq = tp->copied_seq;
2106                 }
2107                 continue;
2108
2109 found_ok_skb:
2110                 /* Ok so how much can we use? */
2111                 used = skb->len - offset;
2112                 if (len < used)
2113                         used = len;
2114
2115                 /* Do we have urgent data here? */
2116                 if (tp->urg_data) {
2117                         u32 urg_offset = tp->urg_seq - *seq;
2118                         if (urg_offset < used) {
2119                                 if (!urg_offset) {
2120                                         if (!sock_flag(sk, SOCK_URGINLINE)) {
2121                                                 WRITE_ONCE(*seq, *seq + 1);
2122                                                 urg_hole++;
2123                                                 offset++;
2124                                                 used--;
2125                                                 if (!used)
2126                                                         goto skip_copy;
2127                                         }
2128                                 } else
2129                                         used = urg_offset;
2130                         }
2131                 }
2132
2133                 if (!(flags & MSG_TRUNC)) {
2134                         err = skb_copy_datagram_msg(skb, offset, msg, used);
2135                         if (err) {
2136                                 /* Exception. Bailout! */
2137                                 if (!copied)
2138                                         copied = -EFAULT;
2139                                 break;
2140                         }
2141                 }
2142
2143                 WRITE_ONCE(*seq, *seq + used);
2144                 copied += used;
2145                 len -= used;
2146
2147                 tcp_rcv_space_adjust(sk);
2148
2149 skip_copy:
2150                 if (tp->urg_data && after(tp->copied_seq, tp->urg_seq)) {
2151                         tp->urg_data = 0;
2152                         tcp_fast_path_check(sk);
2153                 }
2154                 if (used + offset < skb->len)
2155                         continue;
2156
2157                 if (TCP_SKB_CB(skb)->has_rxtstamp) {
2158                         tcp_update_recv_tstamps(skb, &tss);
2159                         has_tss = true;
2160                         has_cmsg = true;
2161                 }
2162                 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
2163                         goto found_fin_ok;
2164                 if (!(flags & MSG_PEEK))
2165                         sk_eat_skb(sk, skb);
2166                 continue;
2167
2168 found_fin_ok:
2169                 /* Process the FIN. */
2170                 WRITE_ONCE(*seq, *seq + 1);
2171                 if (!(flags & MSG_PEEK))
2172                         sk_eat_skb(sk, skb);
2173                 break;
2174         } while (len > 0);
2175
2176         /* According to UNIX98, msg_name/msg_namelen are ignored
2177          * on connected socket. I was just happy when found this 8) --ANK
2178          */
2179
2180         /* Clean up data we have read: This will do ACK frames. */
2181         tcp_cleanup_rbuf(sk, copied);
2182
2183         release_sock(sk);
2184
2185         if (has_cmsg) {
2186                 if (has_tss)
2187                         tcp_recv_timestamp(msg, sk, &tss);
2188                 if (tp->recvmsg_inq) {
2189                         inq = tcp_inq_hint(sk);
2190                         put_cmsg(msg, SOL_TCP, TCP_CM_INQ, sizeof(inq), &inq);
2191                 }
2192         }
2193
2194         return copied;
2195
2196 out:
2197         release_sock(sk);
2198         return err;
2199
2200 recv_urg:
2201         err = tcp_recv_urg(sk, msg, len, flags);
2202         goto out;
2203
2204 recv_sndq:
2205         err = tcp_peek_sndq(sk, msg, len);
2206         goto out;
2207 }
2208 EXPORT_SYMBOL(tcp_recvmsg);
2209
2210 void tcp_set_state(struct sock *sk, int state)
2211 {
2212         int oldstate = sk->sk_state;
2213
2214         /* We defined a new enum for TCP states that are exported in BPF
2215          * so as not force the internal TCP states to be frozen. The
2216          * following checks will detect if an internal state value ever
2217          * differs from the BPF value. If this ever happens, then we will
2218          * need to remap the internal value to the BPF value before calling
2219          * tcp_call_bpf_2arg.
2220          */
2221         BUILD_BUG_ON((int)BPF_TCP_ESTABLISHED != (int)TCP_ESTABLISHED);
2222         BUILD_BUG_ON((int)BPF_TCP_SYN_SENT != (int)TCP_SYN_SENT);
2223         BUILD_BUG_ON((int)BPF_TCP_SYN_RECV != (int)TCP_SYN_RECV);
2224         BUILD_BUG_ON((int)BPF_TCP_FIN_WAIT1 != (int)TCP_FIN_WAIT1);
2225         BUILD_BUG_ON((int)BPF_TCP_FIN_WAIT2 != (int)TCP_FIN_WAIT2);
2226         BUILD_BUG_ON((int)BPF_TCP_TIME_WAIT != (int)TCP_TIME_WAIT);
2227         BUILD_BUG_ON((int)BPF_TCP_CLOSE != (int)TCP_CLOSE);
2228         BUILD_BUG_ON((int)BPF_TCP_CLOSE_WAIT != (int)TCP_CLOSE_WAIT);
2229         BUILD_BUG_ON((int)BPF_TCP_LAST_ACK != (int)TCP_LAST_ACK);
2230         BUILD_BUG_ON((int)BPF_TCP_LISTEN != (int)TCP_LISTEN);
2231         BUILD_BUG_ON((int)BPF_TCP_CLOSING != (int)TCP_CLOSING);
2232         BUILD_BUG_ON((int)BPF_TCP_NEW_SYN_RECV != (int)TCP_NEW_SYN_RECV);
2233         BUILD_BUG_ON((int)BPF_TCP_MAX_STATES != (int)TCP_MAX_STATES);
2234
2235         if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_STATE_CB_FLAG))
2236                 tcp_call_bpf_2arg(sk, BPF_SOCK_OPS_STATE_CB, oldstate, state);
2237
2238         switch (state) {
2239         case TCP_ESTABLISHED:
2240                 if (oldstate != TCP_ESTABLISHED)
2241                         TCP_INC_STATS(sock_net(sk), TCP_MIB_CURRESTAB);
2242                 break;
2243
2244         case TCP_CLOSE:
2245                 if (oldstate == TCP_CLOSE_WAIT || oldstate == TCP_ESTABLISHED)
2246                         TCP_INC_STATS(sock_net(sk), TCP_MIB_ESTABRESETS);
2247
2248                 sk->sk_prot->unhash(sk);
2249                 if (inet_csk(sk)->icsk_bind_hash &&
2250                     !(sk->sk_userlocks & SOCK_BINDPORT_LOCK))
2251                         inet_put_port(sk);
2252                 /* fall through */
2253         default:
2254                 if (oldstate == TCP_ESTABLISHED)
2255                         TCP_DEC_STATS(sock_net(sk), TCP_MIB_CURRESTAB);
2256         }
2257
2258         /* Change state AFTER socket is unhashed to avoid closed
2259          * socket sitting in hash tables.
2260          */
2261         inet_sk_state_store(sk, state);
2262 }
2263 EXPORT_SYMBOL_GPL(tcp_set_state);
2264
2265 /*
2266  *      State processing on a close. This implements the state shift for
2267  *      sending our FIN frame. Note that we only send a FIN for some
2268  *      states. A shutdown() may have already sent the FIN, or we may be
2269  *      closed.
2270  */
2271
2272 static const unsigned char new_state[16] = {
2273   /* current state:        new state:      action:      */
2274   [0 /* (Invalid) */]   = TCP_CLOSE,
2275   [TCP_ESTABLISHED]     = TCP_FIN_WAIT1 | TCP_ACTION_FIN,
2276   [TCP_SYN_SENT]        = TCP_CLOSE,
2277   [TCP_SYN_RECV]        = TCP_FIN_WAIT1 | TCP_ACTION_FIN,
2278   [TCP_FIN_WAIT1]       = TCP_FIN_WAIT1,
2279   [TCP_FIN_WAIT2]       = TCP_FIN_WAIT2,
2280   [TCP_TIME_WAIT]       = TCP_CLOSE,
2281   [TCP_CLOSE]           = TCP_CLOSE,
2282   [TCP_CLOSE_WAIT]      = TCP_LAST_ACK  | TCP_ACTION_FIN,
2283   [TCP_LAST_ACK]        = TCP_LAST_ACK,
2284   [TCP_LISTEN]          = TCP_CLOSE,
2285   [TCP_CLOSING]         = TCP_CLOSING,
2286   [TCP_NEW_SYN_RECV]    = TCP_CLOSE,    /* should not happen ! */
2287 };
2288
2289 static int tcp_close_state(struct sock *sk)
2290 {
2291         int next = (int)new_state[sk->sk_state];
2292         int ns = next & TCP_STATE_MASK;
2293
2294         tcp_set_state(sk, ns);
2295
2296         return next & TCP_ACTION_FIN;
2297 }
2298
2299 /*
2300  *      Shutdown the sending side of a connection. Much like close except
2301  *      that we don't receive shut down or sock_set_flag(sk, SOCK_DEAD).
2302  */
2303
2304 void tcp_shutdown(struct sock *sk, int how)
2305 {
2306         /*      We need to grab some memory, and put together a FIN,
2307          *      and then put it into the queue to be sent.
2308          *              Tim MacKenzie(tym@dibbler.cs.monash.edu.au) 4 Dec '92.
2309          */
2310         if (!(how & SEND_SHUTDOWN))
2311                 return;
2312
2313         /* If we've already sent a FIN, or it's a closed state, skip this. */
2314         if ((1 << sk->sk_state) &
2315             (TCPF_ESTABLISHED | TCPF_SYN_SENT |
2316              TCPF_SYN_RECV | TCPF_CLOSE_WAIT)) {
2317                 /* Clear out any half completed packets.  FIN if needed. */
2318                 if (tcp_close_state(sk))
2319                         tcp_send_fin(sk);
2320         }
2321 }
2322 EXPORT_SYMBOL(tcp_shutdown);
2323
2324 bool tcp_check_oom(struct sock *sk, int shift)
2325 {
2326         bool too_many_orphans, out_of_socket_memory;
2327
2328         too_many_orphans = tcp_too_many_orphans(sk, shift);
2329         out_of_socket_memory = tcp_out_of_memory(sk);
2330
2331         if (too_many_orphans)
2332                 net_info_ratelimited("too many orphaned sockets\n");
2333         if (out_of_socket_memory)
2334                 net_info_ratelimited("out of memory -- consider tuning tcp_mem\n");
2335         return too_many_orphans || out_of_socket_memory;
2336 }
2337
2338 void tcp_close(struct sock *sk, long timeout)
2339 {
2340         struct sk_buff *skb;
2341         int data_was_unread = 0;
2342         int state;
2343
2344         lock_sock(sk);
2345         sk->sk_shutdown = SHUTDOWN_MASK;
2346
2347         if (sk->sk_state == TCP_LISTEN) {
2348                 tcp_set_state(sk, TCP_CLOSE);
2349
2350                 /* Special case. */
2351                 inet_csk_listen_stop(sk);
2352
2353                 goto adjudge_to_death;
2354         }
2355
2356         /*  We need to flush the recv. buffs.  We do this only on the
2357          *  descriptor close, not protocol-sourced closes, because the
2358          *  reader process may not have drained the data yet!
2359          */
2360         while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) {
2361                 u32 len = TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq;
2362
2363                 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
2364                         len--;
2365                 data_was_unread += len;
2366                 __kfree_skb(skb);
2367         }
2368
2369         sk_mem_reclaim(sk);
2370
2371         /* If socket has been already reset (e.g. in tcp_reset()) - kill it. */
2372         if (sk->sk_state == TCP_CLOSE)
2373                 goto adjudge_to_death;
2374
2375         /* As outlined in RFC 2525, section 2.17, we send a RST here because
2376          * data was lost. To witness the awful effects of the old behavior of
2377          * always doing a FIN, run an older 2.1.x kernel or 2.0.x, start a bulk
2378          * GET in an FTP client, suspend the process, wait for the client to
2379          * advertise a zero window, then kill -9 the FTP client, wheee...
2380          * Note: timeout is always zero in such a case.
2381          */
2382         if (unlikely(tcp_sk(sk)->repair)) {
2383                 sk->sk_prot->disconnect(sk, 0);
2384         } else if (data_was_unread) {
2385                 /* Unread data was tossed, zap the connection. */
2386                 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONCLOSE);
2387                 tcp_set_state(sk, TCP_CLOSE);
2388                 tcp_send_active_reset(sk, sk->sk_allocation);
2389         } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) {
2390                 /* Check zero linger _after_ checking for unread data. */
2391                 sk->sk_prot->disconnect(sk, 0);
2392                 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONDATA);
2393         } else if (tcp_close_state(sk)) {
2394                 /* We FIN if the application ate all the data before
2395                  * zapping the connection.
2396                  */
2397
2398                 /* RED-PEN. Formally speaking, we have broken TCP state
2399                  * machine. State transitions:
2400                  *
2401                  * TCP_ESTABLISHED -> TCP_FIN_WAIT1
2402                  * TCP_SYN_RECV -> TCP_FIN_WAIT1 (forget it, it's impossible)
2403                  * TCP_CLOSE_WAIT -> TCP_LAST_ACK
2404                  *
2405                  * are legal only when FIN has been sent (i.e. in window),
2406                  * rather than queued out of window. Purists blame.
2407                  *
2408                  * F.e. "RFC state" is ESTABLISHED,
2409                  * if Linux state is FIN-WAIT-1, but FIN is still not sent.
2410                  *
2411                  * The visible declinations are that sometimes
2412                  * we enter time-wait state, when it is not required really
2413                  * (harmless), do not send active resets, when they are
2414                  * required by specs (TCP_ESTABLISHED, TCP_CLOSE_WAIT, when
2415                  * they look as CLOSING or LAST_ACK for Linux)
2416                  * Probably, I missed some more holelets.
2417                  *                                              --ANK
2418                  * XXX (TFO) - To start off we don't support SYN+ACK+FIN
2419                  * in a single packet! (May consider it later but will
2420                  * probably need API support or TCP_CORK SYN-ACK until
2421                  * data is written and socket is closed.)
2422                  */
2423                 tcp_send_fin(sk);
2424         }
2425
2426         sk_stream_wait_close(sk, timeout);
2427
2428 adjudge_to_death:
2429         state = sk->sk_state;
2430         sock_hold(sk);
2431         sock_orphan(sk);
2432
2433         local_bh_disable();
2434         bh_lock_sock(sk);
2435         /* remove backlog if any, without releasing ownership. */
2436         __release_sock(sk);
2437
2438         percpu_counter_inc(sk->sk_prot->orphan_count);
2439
2440         /* Have we already been destroyed by a softirq or backlog? */
2441         if (state != TCP_CLOSE && sk->sk_state == TCP_CLOSE)
2442                 goto out;
2443
2444         /*      This is a (useful) BSD violating of the RFC. There is a
2445          *      problem with TCP as specified in that the other end could
2446          *      keep a socket open forever with no application left this end.
2447          *      We use a 1 minute timeout (about the same as BSD) then kill
2448          *      our end. If they send after that then tough - BUT: long enough
2449          *      that we won't make the old 4*rto = almost no time - whoops
2450          *      reset mistake.
2451          *
2452          *      Nope, it was not mistake. It is really desired behaviour
2453          *      f.e. on http servers, when such sockets are useless, but
2454          *      consume significant resources. Let's do it with special
2455          *      linger2 option.                                 --ANK
2456          */
2457
2458         if (sk->sk_state == TCP_FIN_WAIT2) {
2459                 struct tcp_sock *tp = tcp_sk(sk);
2460                 if (tp->linger2 < 0) {
2461                         tcp_set_state(sk, TCP_CLOSE);
2462                         tcp_send_active_reset(sk, GFP_ATOMIC);
2463                         __NET_INC_STATS(sock_net(sk),
2464                                         LINUX_MIB_TCPABORTONLINGER);
2465                 } else {
2466                         const int tmo = tcp_fin_time(sk);
2467
2468                         if (tmo > TCP_TIMEWAIT_LEN) {
2469                                 inet_csk_reset_keepalive_timer(sk,
2470                                                 tmo - TCP_TIMEWAIT_LEN);
2471                         } else {
2472                                 tcp_time_wait(sk, TCP_FIN_WAIT2, tmo);
2473                                 goto out;
2474                         }
2475                 }
2476         }
2477         if (sk->sk_state != TCP_CLOSE) {
2478                 sk_mem_reclaim(sk);
2479                 if (tcp_check_oom(sk, 0)) {
2480                         tcp_set_state(sk, TCP_CLOSE);
2481                         tcp_send_active_reset(sk, GFP_ATOMIC);
2482                         __NET_INC_STATS(sock_net(sk),
2483                                         LINUX_MIB_TCPABORTONMEMORY);
2484                 } else if (!check_net(sock_net(sk))) {
2485                         /* Not possible to send reset; just close */
2486                         tcp_set_state(sk, TCP_CLOSE);
2487                 }
2488         }
2489
2490         if (sk->sk_state == TCP_CLOSE) {
2491                 struct request_sock *req;
2492
2493                 req = rcu_dereference_protected(tcp_sk(sk)->fastopen_rsk,
2494                                                 lockdep_sock_is_held(sk));
2495                 /* We could get here with a non-NULL req if the socket is
2496                  * aborted (e.g., closed with unread data) before 3WHS
2497                  * finishes.
2498                  */
2499                 if (req)
2500                         reqsk_fastopen_remove(sk, req, false);
2501                 inet_csk_destroy_sock(sk);
2502         }
2503         /* Otherwise, socket is reprieved until protocol close. */
2504
2505 out:
2506         bh_unlock_sock(sk);
2507         local_bh_enable();
2508         release_sock(sk);
2509         sock_put(sk);
2510 }
2511 EXPORT_SYMBOL(tcp_close);
2512
2513 /* These states need RST on ABORT according to RFC793 */
2514
2515 static inline bool tcp_need_reset(int state)
2516 {
2517         return (1 << state) &
2518                (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT | TCPF_FIN_WAIT1 |
2519                 TCPF_FIN_WAIT2 | TCPF_SYN_RECV);
2520 }
2521
2522 static void tcp_rtx_queue_purge(struct sock *sk)
2523 {
2524         struct rb_node *p = rb_first(&sk->tcp_rtx_queue);
2525
2526         while (p) {
2527                 struct sk_buff *skb = rb_to_skb(p);
2528
2529                 p = rb_next(p);
2530                 /* Since we are deleting whole queue, no need to
2531                  * list_del(&skb->tcp_tsorted_anchor)
2532                  */
2533                 tcp_rtx_queue_unlink(skb, sk);
2534                 sk_wmem_free_skb(sk, skb);
2535         }
2536 }
2537
2538 void tcp_write_queue_purge(struct sock *sk)
2539 {
2540         struct sk_buff *skb;
2541
2542         tcp_chrono_stop(sk, TCP_CHRONO_BUSY);
2543         while ((skb = __skb_dequeue(&sk->sk_write_queue)) != NULL) {
2544                 tcp_skb_tsorted_anchor_cleanup(skb);
2545                 sk_wmem_free_skb(sk, skb);
2546         }
2547         tcp_rtx_queue_purge(sk);
2548         skb = sk->sk_tx_skb_cache;
2549         if (skb) {
2550                 __kfree_skb(skb);
2551                 sk->sk_tx_skb_cache = NULL;
2552         }
2553         INIT_LIST_HEAD(&tcp_sk(sk)->tsorted_sent_queue);
2554         sk_mem_reclaim(sk);
2555         tcp_clear_all_retrans_hints(tcp_sk(sk));
2556         tcp_sk(sk)->packets_out = 0;
2557         inet_csk(sk)->icsk_backoff = 0;
2558 }
2559
2560 int tcp_disconnect(struct sock *sk, int flags)
2561 {
2562         struct inet_sock *inet = inet_sk(sk);
2563         struct inet_connection_sock *icsk = inet_csk(sk);
2564         struct tcp_sock *tp = tcp_sk(sk);
2565         int old_state = sk->sk_state;
2566         u32 seq;
2567
2568         if (old_state != TCP_CLOSE)
2569                 tcp_set_state(sk, TCP_CLOSE);
2570
2571         /* ABORT function of RFC793 */
2572         if (old_state == TCP_LISTEN) {
2573                 inet_csk_listen_stop(sk);
2574         } else if (unlikely(tp->repair)) {
2575                 sk->sk_err = ECONNABORTED;
2576         } else if (tcp_need_reset(old_state) ||
2577                    (tp->snd_nxt != tp->write_seq &&
2578                     (1 << old_state) & (TCPF_CLOSING | TCPF_LAST_ACK))) {
2579                 /* The last check adjusts for discrepancy of Linux wrt. RFC
2580                  * states
2581                  */
2582                 tcp_send_active_reset(sk, gfp_any());
2583                 sk->sk_err = ECONNRESET;
2584         } else if (old_state == TCP_SYN_SENT)
2585                 sk->sk_err = ECONNRESET;
2586
2587         tcp_clear_xmit_timers(sk);
2588         __skb_queue_purge(&sk->sk_receive_queue);
2589         if (sk->sk_rx_skb_cache) {
2590                 __kfree_skb(sk->sk_rx_skb_cache);
2591                 sk->sk_rx_skb_cache = NULL;
2592         }
2593         WRITE_ONCE(tp->copied_seq, tp->rcv_nxt);
2594         tp->urg_data = 0;
2595         tcp_write_queue_purge(sk);
2596         tcp_fastopen_active_disable_ofo_check(sk);
2597         skb_rbtree_purge(&tp->out_of_order_queue);
2598
2599         inet->inet_dport = 0;
2600
2601         if (!(sk->sk_userlocks & SOCK_BINDADDR_LOCK))
2602                 inet_reset_saddr(sk);
2603
2604         sk->sk_shutdown = 0;
2605         sock_reset_flag(sk, SOCK_DONE);
2606         tp->srtt_us = 0;
2607         tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT);
2608         tp->rcv_rtt_last_tsecr = 0;
2609
2610         seq = tp->write_seq + tp->max_window + 2;
2611         if (!seq)
2612                 seq = 1;
2613         WRITE_ONCE(tp->write_seq, seq);
2614
2615         icsk->icsk_backoff = 0;
2616         tp->snd_cwnd = 2;
2617         icsk->icsk_probes_out = 0;
2618         icsk->icsk_rto = TCP_TIMEOUT_INIT;
2619         tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;
2620         tp->snd_cwnd = TCP_INIT_CWND;
2621         tp->snd_cwnd_cnt = 0;
2622         tp->window_clamp = 0;
2623         tp->delivered_ce = 0;
2624         tcp_set_ca_state(sk, TCP_CA_Open);
2625         tp->is_sack_reneg = 0;
2626         tcp_clear_retrans(tp);
2627         inet_csk_delack_init(sk);
2628         /* Initialize rcv_mss to TCP_MIN_MSS to avoid division by 0
2629          * issue in __tcp_select_window()
2630          */
2631         icsk->icsk_ack.rcv_mss = TCP_MIN_MSS;
2632         memset(&tp->rx_opt, 0, sizeof(tp->rx_opt));
2633         __sk_dst_reset(sk);
2634         dst_release(sk->sk_rx_dst);
2635         sk->sk_rx_dst = NULL;
2636         tcp_saved_syn_free(tp);
2637         tp->compressed_ack = 0;
2638         tp->bytes_sent = 0;
2639         tp->bytes_acked = 0;
2640         tp->bytes_received = 0;
2641         tp->bytes_retrans = 0;
2642         tp->duplicate_sack[0].start_seq = 0;
2643         tp->duplicate_sack[0].end_seq = 0;
2644         tp->dsack_dups = 0;
2645         tp->reord_seen = 0;
2646         tp->retrans_out = 0;
2647         tp->sacked_out = 0;
2648         tp->tlp_high_seq = 0;
2649         tp->last_oow_ack_time = 0;
2650         /* There's a bubble in the pipe until at least the first ACK. */
2651         tp->app_limited = ~0U;
2652         tp->rack.mstamp = 0;
2653         tp->rack.advanced = 0;
2654         tp->rack.reo_wnd_steps = 1;
2655         tp->rack.last_delivered = 0;
2656         tp->rack.reo_wnd_persist = 0;
2657         tp->rack.dsack_seen = 0;
2658         tp->syn_data_acked = 0;
2659         tp->rx_opt.saw_tstamp = 0;
2660         tp->rx_opt.dsack = 0;
2661         tp->rx_opt.num_sacks = 0;
2662         tp->rcv_ooopack = 0;
2663
2664
2665         /* Clean up fastopen related fields */
2666         tcp_free_fastopen_req(tp);
2667         inet->defer_connect = 0;
2668
2669         WARN_ON(inet->inet_num && !icsk->icsk_bind_hash);
2670
2671         if (sk->sk_frag.page) {
2672                 put_page(sk->sk_frag.page);
2673                 sk->sk_frag.page = NULL;
2674                 sk->sk_frag.offset = 0;
2675         }
2676
2677         sk->sk_error_report(sk);
2678         return 0;
2679 }
2680 EXPORT_SYMBOL(tcp_disconnect);
2681
2682 static inline bool tcp_can_repair_sock(const struct sock *sk)
2683 {
2684         return ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN) &&
2685                 (sk->sk_state != TCP_LISTEN);
2686 }
2687
2688 static int tcp_repair_set_window(struct tcp_sock *tp, char __user *optbuf, int len)
2689 {
2690         struct tcp_repair_window opt;
2691
2692         if (!tp->repair)
2693                 return -EPERM;
2694
2695         if (len != sizeof(opt))
2696                 return -EINVAL;
2697
2698         if (copy_from_user(&opt, optbuf, sizeof(opt)))
2699                 return -EFAULT;
2700
2701         if (opt.max_window < opt.snd_wnd)
2702                 return -EINVAL;
2703
2704         if (after(opt.snd_wl1, tp->rcv_nxt + opt.rcv_wnd))
2705                 return -EINVAL;
2706
2707         if (after(opt.rcv_wup, tp->rcv_nxt))
2708                 return -EINVAL;
2709
2710         tp->snd_wl1     = opt.snd_wl1;
2711         tp->snd_wnd     = opt.snd_wnd;
2712         tp->max_window  = opt.max_window;
2713
2714         tp->rcv_wnd     = opt.rcv_wnd;
2715         tp->rcv_wup     = opt.rcv_wup;
2716
2717         return 0;
2718 }
2719
2720 static int tcp_repair_options_est(struct sock *sk,
2721                 struct tcp_repair_opt __user *optbuf, unsigned int len)
2722 {
2723         struct tcp_sock *tp = tcp_sk(sk);
2724         struct tcp_repair_opt opt;
2725
2726         while (len >= sizeof(opt)) {
2727                 if (copy_from_user(&opt, optbuf, sizeof(opt)))
2728                         return -EFAULT;
2729
2730                 optbuf++;
2731                 len -= sizeof(opt);
2732
2733                 switch (opt.opt_code) {
2734                 case TCPOPT_MSS:
2735                         tp->rx_opt.mss_clamp = opt.opt_val;
2736                         tcp_mtup_init(sk);
2737                         break;
2738                 case TCPOPT_WINDOW:
2739                         {
2740                                 u16 snd_wscale = opt.opt_val & 0xFFFF;
2741                                 u16 rcv_wscale = opt.opt_val >> 16;
2742
2743                                 if (snd_wscale > TCP_MAX_WSCALE || rcv_wscale > TCP_MAX_WSCALE)
2744                                         return -EFBIG;
2745
2746                                 tp->rx_opt.snd_wscale = snd_wscale;
2747                                 tp->rx_opt.rcv_wscale = rcv_wscale;
2748                                 tp->rx_opt.wscale_ok = 1;
2749                         }
2750                         break;
2751                 case TCPOPT_SACK_PERM:
2752                         if (opt.opt_val != 0)
2753                                 return -EINVAL;
2754
2755                         tp->rx_opt.sack_ok |= TCP_SACK_SEEN;
2756                         break;
2757                 case TCPOPT_TIMESTAMP:
2758                         if (opt.opt_val != 0)
2759                                 return -EINVAL;
2760
2761                         tp->rx_opt.tstamp_ok = 1;
2762                         break;
2763                 }
2764         }
2765
2766         return 0;
2767 }
2768
2769 DEFINE_STATIC_KEY_FALSE(tcp_tx_delay_enabled);
2770 EXPORT_SYMBOL(tcp_tx_delay_enabled);
2771
2772 static void tcp_enable_tx_delay(void)
2773 {
2774         if (!static_branch_unlikely(&tcp_tx_delay_enabled)) {
2775                 static int __tcp_tx_delay_enabled = 0;
2776
2777                 if (cmpxchg(&__tcp_tx_delay_enabled, 0, 1) == 0) {
2778                         static_branch_enable(&tcp_tx_delay_enabled);
2779                         pr_info("TCP_TX_DELAY enabled\n");
2780                 }
2781         }
2782 }
2783
2784 /*
2785  *      Socket option code for TCP.
2786  */
2787 static int do_tcp_setsockopt(struct sock *sk, int level,
2788                 int optname, char __user *optval, unsigned int optlen)
2789 {
2790         struct tcp_sock *tp = tcp_sk(sk);
2791         struct inet_connection_sock *icsk = inet_csk(sk);
2792         struct net *net = sock_net(sk);
2793         int val;
2794         int err = 0;
2795
2796         /* These are data/string values, all the others are ints */
2797         switch (optname) {
2798         case TCP_CONGESTION: {
2799                 char name[TCP_CA_NAME_MAX];
2800
2801                 if (optlen < 1)
2802                         return -EINVAL;
2803
2804                 val = strncpy_from_user(name, optval,
2805                                         min_t(long, TCP_CA_NAME_MAX-1, optlen));
2806                 if (val < 0)
2807                         return -EFAULT;
2808                 name[val] = 0;
2809
2810                 lock_sock(sk);
2811                 err = tcp_set_congestion_control(sk, name, true, true,
2812                                                  ns_capable(sock_net(sk)->user_ns,
2813                                                             CAP_NET_ADMIN));
2814                 release_sock(sk);
2815                 return err;
2816         }
2817         case TCP_ULP: {
2818                 char name[TCP_ULP_NAME_MAX];
2819
2820                 if (optlen < 1)
2821                         return -EINVAL;
2822
2823                 val = strncpy_from_user(name, optval,
2824                                         min_t(long, TCP_ULP_NAME_MAX - 1,
2825                                               optlen));
2826                 if (val < 0)
2827                         return -EFAULT;
2828                 name[val] = 0;
2829
2830                 lock_sock(sk);
2831                 err = tcp_set_ulp(sk, name);
2832                 release_sock(sk);
2833                 return err;
2834         }
2835         case TCP_FASTOPEN_KEY: {
2836                 __u8 key[TCP_FASTOPEN_KEY_BUF_LENGTH];
2837                 __u8 *backup_key = NULL;
2838
2839                 /* Allow a backup key as well to facilitate key rotation
2840                  * First key is the active one.
2841                  */
2842                 if (optlen != TCP_FASTOPEN_KEY_LENGTH &&
2843                     optlen != TCP_FASTOPEN_KEY_BUF_LENGTH)
2844                         return -EINVAL;
2845
2846                 if (copy_from_user(key, optval, optlen))
2847                         return -EFAULT;
2848
2849                 if (optlen == TCP_FASTOPEN_KEY_BUF_LENGTH)
2850                         backup_key = key + TCP_FASTOPEN_KEY_LENGTH;
2851
2852                 return tcp_fastopen_reset_cipher(net, sk, key, backup_key);
2853         }
2854         default:
2855                 /* fallthru */
2856                 break;
2857         }
2858
2859         if (optlen < sizeof(int))
2860                 return -EINVAL;
2861
2862         if (get_user(val, (int __user *)optval))
2863                 return -EFAULT;
2864
2865         lock_sock(sk);
2866
2867         switch (optname) {
2868         case TCP_MAXSEG:
2869                 /* Values greater than interface MTU won't take effect. However
2870                  * at the point when this call is done we typically don't yet
2871                  * know which interface is going to be used
2872                  */
2873                 if (val && (val < TCP_MIN_MSS || val > MAX_TCP_WINDOW)) {
2874                         err = -EINVAL;
2875                         break;
2876                 }
2877                 tp->rx_opt.user_mss = val;
2878                 break;
2879
2880         case TCP_NODELAY:
2881                 if (val) {
2882                         /* TCP_NODELAY is weaker than TCP_CORK, so that
2883                          * this option on corked socket is remembered, but
2884                          * it is not activated until cork is cleared.
2885                          *
2886                          * However, when TCP_NODELAY is set we make
2887                          * an explicit push, which overrides even TCP_CORK
2888                          * for currently queued segments.
2889                          */
2890                         tp->nonagle |= TCP_NAGLE_OFF|TCP_NAGLE_PUSH;
2891                         tcp_push_pending_frames(sk);
2892                 } else {
2893                         tp->nonagle &= ~TCP_NAGLE_OFF;
2894                 }
2895                 break;
2896
2897         case TCP_THIN_LINEAR_TIMEOUTS:
2898                 if (val < 0 || val > 1)
2899                         err = -EINVAL;
2900                 else
2901                         tp->thin_lto = val;
2902                 break;
2903
2904         case TCP_THIN_DUPACK:
2905                 if (val < 0 || val > 1)
2906                         err = -EINVAL;
2907                 break;
2908
2909         case TCP_REPAIR:
2910                 if (!tcp_can_repair_sock(sk))
2911                         err = -EPERM;
2912                 else if (val == TCP_REPAIR_ON) {
2913                         tp->repair = 1;
2914                         sk->sk_reuse = SK_FORCE_REUSE;
2915                         tp->repair_queue = TCP_NO_QUEUE;
2916                 } else if (val == TCP_REPAIR_OFF) {
2917                         tp->repair = 0;
2918                         sk->sk_reuse = SK_NO_REUSE;
2919                         tcp_send_window_probe(sk);
2920                 } else if (val == TCP_REPAIR_OFF_NO_WP) {
2921                         tp->repair = 0;
2922                         sk->sk_reuse = SK_NO_REUSE;
2923                 } else
2924                         err = -EINVAL;
2925
2926                 break;
2927
2928         case TCP_REPAIR_QUEUE:
2929                 if (!tp->repair)
2930                         err = -EPERM;
2931                 else if ((unsigned int)val < TCP_QUEUES_NR)
2932                         tp->repair_queue = val;
2933                 else
2934                         err = -EINVAL;
2935                 break;
2936
2937         case TCP_QUEUE_SEQ:
2938                 if (sk->sk_state != TCP_CLOSE)
2939                         err = -EPERM;
2940                 else if (tp->repair_queue == TCP_SEND_QUEUE)
2941                         WRITE_ONCE(tp->write_seq, val);
2942                 else if (tp->repair_queue == TCP_RECV_QUEUE)
2943                         WRITE_ONCE(tp->rcv_nxt, val);
2944                 else
2945                         err = -EINVAL;
2946                 break;
2947
2948         case TCP_REPAIR_OPTIONS:
2949                 if (!tp->repair)
2950                         err = -EINVAL;
2951                 else if (sk->sk_state == TCP_ESTABLISHED)
2952                         err = tcp_repair_options_est(sk,
2953                                         (struct tcp_repair_opt __user *)optval,
2954                                         optlen);
2955                 else
2956                         err = -EPERM;
2957                 break;
2958
2959         case TCP_CORK:
2960                 /* When set indicates to always queue non-full frames.
2961                  * Later the user clears this option and we transmit
2962                  * any pending partial frames in the queue.  This is
2963                  * meant to be used alongside sendfile() to get properly
2964                  * filled frames when the user (for example) must write
2965                  * out headers with a write() call first and then use
2966                  * sendfile to send out the data parts.
2967                  *
2968                  * TCP_CORK can be set together with TCP_NODELAY and it is
2969                  * stronger than TCP_NODELAY.
2970                  */
2971                 if (val) {
2972                         tp->nonagle |= TCP_NAGLE_CORK;
2973                 } else {
2974                         tp->nonagle &= ~TCP_NAGLE_CORK;
2975                         if (tp->nonagle&TCP_NAGLE_OFF)
2976                                 tp->nonagle |= TCP_NAGLE_PUSH;
2977                         tcp_push_pending_frames(sk);
2978                 }
2979                 break;
2980
2981         case TCP_KEEPIDLE:
2982                 if (val < 1 || val > MAX_TCP_KEEPIDLE)
2983                         err = -EINVAL;
2984                 else {
2985                         tp->keepalive_time = val * HZ;
2986                         if (sock_flag(sk, SOCK_KEEPOPEN) &&
2987                             !((1 << sk->sk_state) &
2988                               (TCPF_CLOSE | TCPF_LISTEN))) {
2989                                 u32 elapsed = keepalive_time_elapsed(tp);
2990                                 if (tp->keepalive_time > elapsed)
2991                                         elapsed = tp->keepalive_time - elapsed;
2992                                 else
2993                                         elapsed = 0;
2994                                 inet_csk_reset_keepalive_timer(sk, elapsed);
2995                         }
2996                 }
2997                 break;
2998         case TCP_KEEPINTVL:
2999                 if (val < 1 || val > MAX_TCP_KEEPINTVL)
3000                         err = -EINVAL;
3001                 else
3002                         tp->keepalive_intvl = val * HZ;
3003                 break;
3004         case TCP_KEEPCNT:
3005                 if (val < 1 || val > MAX_TCP_KEEPCNT)
3006                         err = -EINVAL;
3007                 else
3008                         tp->keepalive_probes = val;
3009                 break;
3010         case TCP_SYNCNT:
3011                 if (val < 1 || val > MAX_TCP_SYNCNT)
3012                         err = -EINVAL;
3013                 else
3014                         icsk->icsk_syn_retries = val;
3015                 break;
3016
3017         case TCP_SAVE_SYN:
3018                 if (val < 0 || val > 1)
3019                         err = -EINVAL;
3020                 else
3021                         tp->save_syn = val;
3022                 break;
3023
3024         case TCP_LINGER2:
3025                 if (val < 0)
3026                         tp->linger2 = -1;
3027                 else if (val > net->ipv4.sysctl_tcp_fin_timeout / HZ)
3028                         tp->linger2 = 0;
3029                 else
3030                         tp->linger2 = val * HZ;
3031                 break;
3032
3033         case TCP_DEFER_ACCEPT:
3034                 /* Translate value in seconds to number of retransmits */
3035                 icsk->icsk_accept_queue.rskq_defer_accept =
3036                         secs_to_retrans(val, TCP_TIMEOUT_INIT / HZ,
3037                                         TCP_RTO_MAX / HZ);
3038                 break;
3039
3040         case TCP_WINDOW_CLAMP:
3041                 if (!val) {
3042                         if (sk->sk_state != TCP_CLOSE) {
3043                                 err = -EINVAL;
3044                                 break;
3045                         }
3046                         tp->window_clamp = 0;
3047                 } else
3048                         tp->window_clamp = val < SOCK_MIN_RCVBUF / 2 ?
3049                                                 SOCK_MIN_RCVBUF / 2 : val;
3050                 break;
3051
3052         case TCP_QUICKACK:
3053                 if (!val) {
3054                         inet_csk_enter_pingpong_mode(sk);
3055                 } else {
3056                         inet_csk_exit_pingpong_mode(sk);
3057                         if ((1 << sk->sk_state) &
3058                             (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT) &&
3059                             inet_csk_ack_scheduled(sk)) {
3060                                 icsk->icsk_ack.pending |= ICSK_ACK_PUSHED;
3061                                 tcp_cleanup_rbuf(sk, 1);
3062                                 if (!(val & 1))
3063                                         inet_csk_enter_pingpong_mode(sk);
3064                         }
3065                 }
3066                 break;
3067
3068 #ifdef CONFIG_TCP_MD5SIG
3069         case TCP_MD5SIG:
3070         case TCP_MD5SIG_EXT:
3071                 if ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))
3072                         err = tp->af_specific->md5_parse(sk, optname, optval, optlen);
3073                 else
3074                         err = -EINVAL;
3075                 break;
3076 #endif
3077         case TCP_USER_TIMEOUT:
3078                 /* Cap the max time in ms TCP will retry or probe the window
3079                  * before giving up and aborting (ETIMEDOUT) a connection.
3080                  */
3081                 if (val < 0)
3082                         err = -EINVAL;
3083                 else
3084                         icsk->icsk_user_timeout = val;
3085                 break;
3086
3087         case TCP_FASTOPEN:
3088                 if (val >= 0 && ((1 << sk->sk_state) & (TCPF_CLOSE |
3089                     TCPF_LISTEN))) {
3090                         tcp_fastopen_init_key_once(net);
3091
3092                         fastopen_queue_tune(sk, val);
3093                 } else {
3094                         err = -EINVAL;
3095                 }
3096                 break;
3097         case TCP_FASTOPEN_CONNECT:
3098                 if (val > 1 || val < 0) {
3099                         err = -EINVAL;
3100                 } else if (net->ipv4.sysctl_tcp_fastopen & TFO_CLIENT_ENABLE) {
3101                         if (sk->sk_state == TCP_CLOSE)
3102                                 tp->fastopen_connect = val;
3103                         else
3104                                 err = -EINVAL;
3105                 } else {
3106                         err = -EOPNOTSUPP;
3107                 }
3108                 break;
3109         case TCP_FASTOPEN_NO_COOKIE:
3110                 if (val > 1 || val < 0)
3111                         err = -EINVAL;
3112                 else if (!((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)))
3113                         err = -EINVAL;
3114                 else
3115                         tp->fastopen_no_cookie = val;
3116                 break;
3117         case TCP_TIMESTAMP:
3118                 if (!tp->repair)
3119                         err = -EPERM;
3120                 else
3121                         tp->tsoffset = val - tcp_time_stamp_raw();
3122                 break;
3123         case TCP_REPAIR_WINDOW:
3124                 err = tcp_repair_set_window(tp, optval, optlen);
3125                 break;
3126         case TCP_NOTSENT_LOWAT:
3127                 tp->notsent_lowat = val;
3128                 sk->sk_write_space(sk);
3129                 break;
3130         case TCP_INQ:
3131                 if (val > 1 || val < 0)
3132                         err = -EINVAL;
3133                 else
3134                         tp->recvmsg_inq = val;
3135                 break;
3136         case TCP_TX_DELAY:
3137                 if (val)
3138                         tcp_enable_tx_delay();
3139                 tp->tcp_tx_delay = val;
3140                 break;
3141         default:
3142                 err = -ENOPROTOOPT;
3143                 break;
3144         }
3145
3146         release_sock(sk);
3147         return err;
3148 }
3149
3150 int tcp_setsockopt(struct sock *sk, int level, int optname, char __user *optval,
3151                    unsigned int optlen)
3152 {
3153         const struct inet_connection_sock *icsk = inet_csk(sk);
3154
3155         if (level != SOL_TCP)
3156                 return icsk->icsk_af_ops->setsockopt(sk, level, optname,
3157                                                      optval, optlen);
3158         return do_tcp_setsockopt(sk, level, optname, optval, optlen);
3159 }
3160 EXPORT_SYMBOL(tcp_setsockopt);
3161
3162 #ifdef CONFIG_COMPAT
3163 int compat_tcp_setsockopt(struct sock *sk, int level, int optname,
3164                           char __user *optval, unsigned int optlen)
3165 {
3166         if (level != SOL_TCP)
3167                 return inet_csk_compat_setsockopt(sk, level, optname,
3168                                                   optval, optlen);
3169         return do_tcp_setsockopt(sk, level, optname, optval, optlen);
3170 }
3171 EXPORT_SYMBOL(compat_tcp_setsockopt);
3172 #endif
3173
3174 static void tcp_get_info_chrono_stats(const struct tcp_sock *tp,
3175                                       struct tcp_info *info)
3176 {
3177         u64 stats[__TCP_CHRONO_MAX], total = 0;
3178         enum tcp_chrono i;
3179
3180         for (i = TCP_CHRONO_BUSY; i < __TCP_CHRONO_MAX; ++i) {
3181                 stats[i] = tp->chrono_stat[i - 1];
3182                 if (i == tp->chrono_type)
3183                         stats[i] += tcp_jiffies32 - tp->chrono_start;
3184                 stats[i] *= USEC_PER_SEC / HZ;
3185                 total += stats[i];
3186         }
3187
3188         info->tcpi_busy_time = total;
3189         info->tcpi_rwnd_limited = stats[TCP_CHRONO_RWND_LIMITED];
3190         info->tcpi_sndbuf_limited = stats[TCP_CHRONO_SNDBUF_LIMITED];
3191 }
3192
3193 /* Return information about state of tcp endpoint in API format. */
3194 void tcp_get_info(struct sock *sk, struct tcp_info *info)
3195 {
3196         const struct tcp_sock *tp = tcp_sk(sk); /* iff sk_type == SOCK_STREAM */
3197         const struct inet_connection_sock *icsk = inet_csk(sk);
3198         unsigned long rate;
3199         u32 now;
3200         u64 rate64;
3201         bool slow;
3202
3203         memset(info, 0, sizeof(*info));
3204         if (sk->sk_type != SOCK_STREAM)
3205                 return;
3206
3207         info->tcpi_state = inet_sk_state_load(sk);
3208
3209         /* Report meaningful fields for all TCP states, including listeners */
3210         rate = READ_ONCE(sk->sk_pacing_rate);
3211         rate64 = (rate != ~0UL) ? rate : ~0ULL;
3212         info->tcpi_pacing_rate = rate64;
3213
3214         rate = READ_ONCE(sk->sk_max_pacing_rate);
3215         rate64 = (rate != ~0UL) ? rate : ~0ULL;
3216         info->tcpi_max_pacing_rate = rate64;
3217
3218         info->tcpi_reordering = tp->reordering;
3219         info->tcpi_snd_cwnd = tp->snd_cwnd;
3220
3221         if (info->tcpi_state == TCP_LISTEN) {
3222                 /* listeners aliased fields :
3223                  * tcpi_unacked -> Number of children ready for accept()
3224                  * tcpi_sacked  -> max backlog
3225                  */
3226                 info->tcpi_unacked = sk->sk_ack_backlog;
3227                 info->tcpi_sacked = sk->sk_max_ack_backlog;
3228                 return;
3229         }
3230
3231         slow = lock_sock_fast(sk);
3232
3233         info->tcpi_ca_state = icsk->icsk_ca_state;
3234         info->tcpi_retransmits = icsk->icsk_retransmits;
3235         info->tcpi_probes = icsk->icsk_probes_out;
3236         info->tcpi_backoff = icsk->icsk_backoff;
3237
3238         if (tp->rx_opt.tstamp_ok)
3239                 info->tcpi_options |= TCPI_OPT_TIMESTAMPS;
3240         if (tcp_is_sack(tp))
3241                 info->tcpi_options |= TCPI_OPT_SACK;
3242         if (tp->rx_opt.wscale_ok) {
3243                 info->tcpi_options |= TCPI_OPT_WSCALE;
3244                 info->tcpi_snd_wscale = tp->rx_opt.snd_wscale;
3245                 info->tcpi_rcv_wscale = tp->rx_opt.rcv_wscale;
3246         }
3247
3248         if (tp->ecn_flags & TCP_ECN_OK)
3249                 info->tcpi_options |= TCPI_OPT_ECN;
3250         if (tp->ecn_flags & TCP_ECN_SEEN)
3251                 info->tcpi_options |= TCPI_OPT_ECN_SEEN;
3252         if (tp->syn_data_acked)
3253                 info->tcpi_options |= TCPI_OPT_SYN_DATA;
3254
3255         info->tcpi_rto = jiffies_to_usecs(icsk->icsk_rto);
3256         info->tcpi_ato = jiffies_to_usecs(icsk->icsk_ack.ato);
3257         info->tcpi_snd_mss = tp->mss_cache;
3258         info->tcpi_rcv_mss = icsk->icsk_ack.rcv_mss;
3259
3260         info->tcpi_unacked = tp->packets_out;
3261         info->tcpi_sacked = tp->sacked_out;
3262
3263         info->tcpi_lost = tp->lost_out;
3264         info->tcpi_retrans = tp->retrans_out;
3265
3266         now = tcp_jiffies32;
3267         info->tcpi_last_data_sent = jiffies_to_msecs(now - tp->lsndtime);
3268         info->tcpi_last_data_recv = jiffies_to_msecs(now - icsk->icsk_ack.lrcvtime);
3269         info->tcpi_last_ack_recv = jiffies_to_msecs(now - tp->rcv_tstamp);
3270
3271         info->tcpi_pmtu = icsk->icsk_pmtu_cookie;
3272         info->tcpi_rcv_ssthresh = tp->rcv_ssthresh;
3273         info->tcpi_rtt = tp->srtt_us >> 3;
3274         info->tcpi_rttvar = tp->mdev_us >> 2;
3275         info->tcpi_snd_ssthresh = tp->snd_ssthresh;
3276         info->tcpi_advmss = tp->advmss;
3277
3278         info->tcpi_rcv_rtt = tp->rcv_rtt_est.rtt_us >> 3;
3279         info->tcpi_rcv_space = tp->rcvq_space.space;
3280
3281         info->tcpi_total_retrans = tp->total_retrans;
3282
3283         info->tcpi_bytes_acked = tp->bytes_acked;
3284         info->tcpi_bytes_received = tp->bytes_received;
3285         info->tcpi_notsent_bytes = max_t(int, 0, tp->write_seq - tp->snd_nxt);
3286         tcp_get_info_chrono_stats(tp, info);
3287
3288         info->tcpi_segs_out = tp->segs_out;
3289         info->tcpi_segs_in = tp->segs_in;
3290
3291         info->tcpi_min_rtt = tcp_min_rtt(tp);
3292         info->tcpi_data_segs_in = tp->data_segs_in;
3293         info->tcpi_data_segs_out = tp->data_segs_out;
3294
3295         info->tcpi_delivery_rate_app_limited = tp->rate_app_limited ? 1 : 0;
3296         rate64 = tcp_compute_delivery_rate(tp);
3297         if (rate64)
3298                 info->tcpi_delivery_rate = rate64;
3299         info->tcpi_delivered = tp->delivered;
3300         info->tcpi_delivered_ce = tp->delivered_ce;
3301         info->tcpi_bytes_sent = tp->bytes_sent;
3302         info->tcpi_bytes_retrans = tp->bytes_retrans;
3303         info->tcpi_dsack_dups = tp->dsack_dups;
3304         info->tcpi_reord_seen = tp->reord_seen;
3305         info->tcpi_rcv_ooopack = tp->rcv_ooopack;
3306         info->tcpi_snd_wnd = tp->snd_wnd;
3307         unlock_sock_fast(sk, slow);
3308 }
3309 EXPORT_SYMBOL_GPL(tcp_get_info);
3310
3311 static size_t tcp_opt_stats_get_size(void)
3312 {
3313         return
3314                 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_BUSY */
3315                 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_RWND_LIMITED */
3316                 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_SNDBUF_LIMITED */
3317                 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_DATA_SEGS_OUT */
3318                 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_TOTAL_RETRANS */
3319                 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_PACING_RATE */
3320                 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_DELIVERY_RATE */
3321                 nla_total_size(sizeof(u32)) + /* TCP_NLA_SND_CWND */
3322                 nla_total_size(sizeof(u32)) + /* TCP_NLA_REORDERING */
3323                 nla_total_size(sizeof(u32)) + /* TCP_NLA_MIN_RTT */
3324                 nla_total_size(sizeof(u8)) + /* TCP_NLA_RECUR_RETRANS */
3325                 nla_total_size(sizeof(u8)) + /* TCP_NLA_DELIVERY_RATE_APP_LMT */
3326                 nla_total_size(sizeof(u32)) + /* TCP_NLA_SNDQ_SIZE */
3327                 nla_total_size(sizeof(u8)) + /* TCP_NLA_CA_STATE */
3328                 nla_total_size(sizeof(u32)) + /* TCP_NLA_SND_SSTHRESH */
3329                 nla_total_size(sizeof(u32)) + /* TCP_NLA_DELIVERED */
3330                 nla_total_size(sizeof(u32)) + /* TCP_NLA_DELIVERED_CE */
3331                 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_BYTES_SENT */
3332                 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_BYTES_RETRANS */
3333                 nla_total_size(sizeof(u32)) + /* TCP_NLA_DSACK_DUPS */
3334                 nla_total_size(sizeof(u32)) + /* TCP_NLA_REORD_SEEN */
3335                 nla_total_size(sizeof(u32)) + /* TCP_NLA_SRTT */
3336                 0;
3337 }
3338
3339 struct sk_buff *tcp_get_timestamping_opt_stats(const struct sock *sk)
3340 {
3341         const struct tcp_sock *tp = tcp_sk(sk);
3342         struct sk_buff *stats;
3343         struct tcp_info info;
3344         unsigned long rate;
3345         u64 rate64;
3346
3347         stats = alloc_skb(tcp_opt_stats_get_size(), GFP_ATOMIC);
3348         if (!stats)
3349                 return NULL;
3350
3351         tcp_get_info_chrono_stats(tp, &info);
3352         nla_put_u64_64bit(stats, TCP_NLA_BUSY,
3353                           info.tcpi_busy_time, TCP_NLA_PAD);
3354         nla_put_u64_64bit(stats, TCP_NLA_RWND_LIMITED,
3355                           info.tcpi_rwnd_limited, TCP_NLA_PAD);
3356         nla_put_u64_64bit(stats, TCP_NLA_SNDBUF_LIMITED,
3357                           info.tcpi_sndbuf_limited, TCP_NLA_PAD);
3358         nla_put_u64_64bit(stats, TCP_NLA_DATA_SEGS_OUT,
3359                           tp->data_segs_out, TCP_NLA_PAD);
3360         nla_put_u64_64bit(stats, TCP_NLA_TOTAL_RETRANS,
3361                           tp->total_retrans, TCP_NLA_PAD);
3362
3363         rate = READ_ONCE(sk->sk_pacing_rate);
3364         rate64 = (rate != ~0UL) ? rate : ~0ULL;
3365         nla_put_u64_64bit(stats, TCP_NLA_PACING_RATE, rate64, TCP_NLA_PAD);
3366
3367         rate64 = tcp_compute_delivery_rate(tp);
3368         nla_put_u64_64bit(stats, TCP_NLA_DELIVERY_RATE, rate64, TCP_NLA_PAD);
3369
3370         nla_put_u32(stats, TCP_NLA_SND_CWND, tp->snd_cwnd);
3371         nla_put_u32(stats, TCP_NLA_REORDERING, tp->reordering);
3372         nla_put_u32(stats, TCP_NLA_MIN_RTT, tcp_min_rtt(tp));
3373
3374         nla_put_u8(stats, TCP_NLA_RECUR_RETRANS, inet_csk(sk)->icsk_retransmits);
3375         nla_put_u8(stats, TCP_NLA_DELIVERY_RATE_APP_LMT, !!tp->rate_app_limited);
3376         nla_put_u32(stats, TCP_NLA_SND_SSTHRESH, tp->snd_ssthresh);
3377         nla_put_u32(stats, TCP_NLA_DELIVERED, tp->delivered);
3378         nla_put_u32(stats, TCP_NLA_DELIVERED_CE, tp->delivered_ce);
3379
3380         nla_put_u32(stats, TCP_NLA_SNDQ_SIZE, tp->write_seq - tp->snd_una);
3381         nla_put_u8(stats, TCP_NLA_CA_STATE, inet_csk(sk)->icsk_ca_state);
3382
3383         nla_put_u64_64bit(stats, TCP_NLA_BYTES_SENT, tp->bytes_sent,
3384                           TCP_NLA_PAD);
3385         nla_put_u64_64bit(stats, TCP_NLA_BYTES_RETRANS, tp->bytes_retrans,
3386                           TCP_NLA_PAD);
3387         nla_put_u32(stats, TCP_NLA_DSACK_DUPS, tp->dsack_dups);
3388         nla_put_u32(stats, TCP_NLA_REORD_SEEN, tp->reord_seen);
3389         nla_put_u32(stats, TCP_NLA_SRTT, tp->srtt_us >> 3);
3390
3391         return stats;
3392 }
3393
3394 static int do_tcp_getsockopt(struct sock *sk, int level,
3395                 int optname, char __user *optval, int __user *optlen)
3396 {
3397         struct inet_connection_sock *icsk = inet_csk(sk);
3398         struct tcp_sock *tp = tcp_sk(sk);
3399         struct net *net = sock_net(sk);
3400         int val, len;
3401
3402         if (get_user(len, optlen))
3403                 return -EFAULT;
3404
3405         len = min_t(unsigned int, len, sizeof(int));
3406
3407         if (len < 0)
3408                 return -EINVAL;
3409
3410         switch (optname) {
3411         case TCP_MAXSEG:
3412                 val = tp->mss_cache;
3413                 if (!val && ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)))
3414                         val = tp->rx_opt.user_mss;
3415                 if (tp->repair)
3416                         val = tp->rx_opt.mss_clamp;
3417                 break;
3418         case TCP_NODELAY:
3419                 val = !!(tp->nonagle&TCP_NAGLE_OFF);
3420                 break;
3421         case TCP_CORK:
3422                 val = !!(tp->nonagle&TCP_NAGLE_CORK);
3423                 break;
3424         case TCP_KEEPIDLE:
3425                 val = keepalive_time_when(tp) / HZ;
3426                 break;
3427         case TCP_KEEPINTVL:
3428                 val = keepalive_intvl_when(tp) / HZ;
3429                 break;
3430         case TCP_KEEPCNT:
3431                 val = keepalive_probes(tp);
3432                 break;
3433         case TCP_SYNCNT:
3434                 val = icsk->icsk_syn_retries ? : net->ipv4.sysctl_tcp_syn_retries;
3435                 break;
3436         case TCP_LINGER2:
3437                 val = tp->linger2;
3438                 if (val >= 0)
3439                         val = (val ? : net->ipv4.sysctl_tcp_fin_timeout) / HZ;
3440                 break;
3441         case TCP_DEFER_ACCEPT:
3442                 val = retrans_to_secs(icsk->icsk_accept_queue.rskq_defer_accept,
3443                                       TCP_TIMEOUT_INIT / HZ, TCP_RTO_MAX / HZ);
3444                 break;
3445         case TCP_WINDOW_CLAMP:
3446                 val = tp->window_clamp;
3447                 break;
3448         case TCP_INFO: {
3449                 struct tcp_info info;
3450
3451                 if (get_user(len, optlen))
3452                         return -EFAULT;
3453
3454                 tcp_get_info(sk, &info);
3455
3456                 len = min_t(unsigned int, len, sizeof(info));
3457                 if (put_user(len, optlen))
3458                         return -EFAULT;
3459                 if (copy_to_user(optval, &info, len))
3460                         return -EFAULT;
3461                 return 0;
3462         }
3463         case TCP_CC_INFO: {
3464                 const struct tcp_congestion_ops *ca_ops;
3465                 union tcp_cc_info info;
3466                 size_t sz = 0;
3467                 int attr;
3468
3469                 if (get_user(len, optlen))
3470                         return -EFAULT;
3471
3472                 ca_ops = icsk->icsk_ca_ops;
3473                 if (ca_ops && ca_ops->get_info)
3474                         sz = ca_ops->get_info(sk, ~0U, &attr, &info);
3475
3476                 len = min_t(unsigned int, len, sz);
3477                 if (put_user(len, optlen))
3478                         return -EFAULT;
3479                 if (copy_to_user(optval, &info, len))
3480                         return -EFAULT;
3481                 return 0;
3482         }
3483         case TCP_QUICKACK:
3484                 val = !inet_csk_in_pingpong_mode(sk);
3485                 break;
3486
3487         case TCP_CONGESTION:
3488                 if (get_user(len, optlen))
3489                         return -EFAULT;
3490                 len = min_t(unsigned int, len, TCP_CA_NAME_MAX);
3491                 if (put_user(len, optlen))
3492                         return -EFAULT;
3493                 if (copy_to_user(optval, icsk->icsk_ca_ops->name, len))
3494                         return -EFAULT;
3495                 return 0;
3496
3497         case TCP_ULP:
3498                 if (get_user(len, optlen))
3499                         return -EFAULT;
3500                 len = min_t(unsigned int, len, TCP_ULP_NAME_MAX);
3501                 if (!icsk->icsk_ulp_ops) {
3502                         if (put_user(0, optlen))
3503                                 return -EFAULT;
3504                         return 0;
3505                 }
3506                 if (put_user(len, optlen))
3507                         return -EFAULT;
3508                 if (copy_to_user(optval, icsk->icsk_ulp_ops->name, len))
3509                         return -EFAULT;
3510                 return 0;
3511
3512         case TCP_FASTOPEN_KEY: {
3513                 __u8 key[TCP_FASTOPEN_KEY_BUF_LENGTH];
3514                 struct tcp_fastopen_context *ctx;
3515                 unsigned int key_len = 0;
3516
3517                 if (get_user(len, optlen))
3518                         return -EFAULT;
3519
3520                 rcu_read_lock();
3521                 ctx = rcu_dereference(icsk->icsk_accept_queue.fastopenq.ctx);
3522                 if (ctx) {
3523                         key_len = tcp_fastopen_context_len(ctx) *
3524                                         TCP_FASTOPEN_KEY_LENGTH;
3525                         memcpy(&key[0], &ctx->key[0], key_len);
3526                 }
3527                 rcu_read_unlock();
3528
3529                 len = min_t(unsigned int, len, key_len);
3530                 if (put_user(len, optlen))
3531                         return -EFAULT;
3532                 if (copy_to_user(optval, key, len))
3533                         return -EFAULT;
3534                 return 0;
3535         }
3536         case TCP_THIN_LINEAR_TIMEOUTS:
3537                 val = tp->thin_lto;
3538                 break;
3539
3540         case TCP_THIN_DUPACK:
3541                 val = 0;
3542                 break;
3543
3544         case TCP_REPAIR:
3545                 val = tp->repair;
3546                 break;
3547
3548         case TCP_REPAIR_QUEUE:
3549                 if (tp->repair)
3550                         val = tp->repair_queue;
3551                 else
3552                         return -EINVAL;
3553                 break;
3554
3555         case TCP_REPAIR_WINDOW: {
3556                 struct tcp_repair_window opt;
3557
3558                 if (get_user(len, optlen))
3559                         return -EFAULT;
3560
3561                 if (len != sizeof(opt))
3562                         return -EINVAL;
3563
3564                 if (!tp->repair)
3565                         return -EPERM;
3566
3567                 opt.snd_wl1     = tp->snd_wl1;
3568                 opt.snd_wnd     = tp->snd_wnd;
3569                 opt.max_window  = tp->max_window;
3570                 opt.rcv_wnd     = tp->rcv_wnd;
3571                 opt.rcv_wup     = tp->rcv_wup;
3572
3573                 if (copy_to_user(optval, &opt, len))
3574                         return -EFAULT;
3575                 return 0;
3576         }
3577         case TCP_QUEUE_SEQ:
3578                 if (tp->repair_queue == TCP_SEND_QUEUE)
3579                         val = tp->write_seq;
3580                 else if (tp->repair_queue == TCP_RECV_QUEUE)
3581                         val = tp->rcv_nxt;
3582                 else
3583                         return -EINVAL;
3584                 break;
3585
3586         case TCP_USER_TIMEOUT:
3587                 val = icsk->icsk_user_timeout;
3588                 break;
3589
3590         case TCP_FASTOPEN:
3591                 val = icsk->icsk_accept_queue.fastopenq.max_qlen;
3592                 break;
3593
3594         case TCP_FASTOPEN_CONNECT:
3595                 val = tp->fastopen_connect;
3596                 break;
3597
3598         case TCP_FASTOPEN_NO_COOKIE:
3599                 val = tp->fastopen_no_cookie;
3600                 break;
3601
3602         case TCP_TX_DELAY:
3603                 val = tp->tcp_tx_delay;
3604                 break;
3605
3606         case TCP_TIMESTAMP:
3607                 val = tcp_time_stamp_raw() + tp->tsoffset;
3608                 break;
3609         case TCP_NOTSENT_LOWAT:
3610                 val = tp->notsent_lowat;
3611                 break;
3612         case TCP_INQ:
3613                 val = tp->recvmsg_inq;
3614                 break;
3615         case TCP_SAVE_SYN:
3616                 val = tp->save_syn;
3617                 break;
3618         case TCP_SAVED_SYN: {
3619                 if (get_user(len, optlen))
3620                         return -EFAULT;
3621
3622                 lock_sock(sk);
3623                 if (tp->saved_syn) {
3624                         if (len < tp->saved_syn[0]) {
3625                                 if (put_user(tp->saved_syn[0], optlen)) {
3626                                         release_sock(sk);
3627                                         return -EFAULT;
3628                                 }
3629                                 release_sock(sk);
3630                                 return -EINVAL;
3631                         }
3632                         len = tp->saved_syn[0];
3633                         if (put_user(len, optlen)) {
3634                                 release_sock(sk);
3635                                 return -EFAULT;
3636                         }
3637                         if (copy_to_user(optval, tp->saved_syn + 1, len)) {
3638                                 release_sock(sk);
3639                                 return -EFAULT;
3640                         }
3641                         tcp_saved_syn_free(tp);
3642                         release_sock(sk);
3643                 } else {
3644                         release_sock(sk);
3645                         len = 0;
3646                         if (put_user(len, optlen))
3647                                 return -EFAULT;
3648                 }
3649                 return 0;
3650         }
3651 #ifdef CONFIG_MMU
3652         case TCP_ZEROCOPY_RECEIVE: {
3653                 struct tcp_zerocopy_receive zc;
3654                 int err;
3655
3656                 if (get_user(len, optlen))
3657                         return -EFAULT;
3658                 if (len != sizeof(zc))
3659                         return -EINVAL;
3660                 if (copy_from_user(&zc, optval, len))
3661                         return -EFAULT;
3662                 lock_sock(sk);
3663                 err = tcp_zerocopy_receive(sk, &zc);
3664                 release_sock(sk);
3665                 if (!err && copy_to_user(optval, &zc, len))
3666                         err = -EFAULT;
3667                 return err;
3668         }
3669 #endif
3670         default:
3671                 return -ENOPROTOOPT;
3672         }
3673
3674         if (put_user(len, optlen))
3675                 return -EFAULT;
3676         if (copy_to_user(optval, &val, len))
3677                 return -EFAULT;
3678         return 0;
3679 }
3680
3681 int tcp_getsockopt(struct sock *sk, int level, int optname, char __user *optval,
3682                    int __user *optlen)
3683 {
3684         struct inet_connection_sock *icsk = inet_csk(sk);
3685
3686         if (level != SOL_TCP)
3687                 return icsk->icsk_af_ops->getsockopt(sk, level, optname,
3688                                                      optval, optlen);
3689         return do_tcp_getsockopt(sk, level, optname, optval, optlen);
3690 }
3691 EXPORT_SYMBOL(tcp_getsockopt);
3692
3693 #ifdef CONFIG_COMPAT
3694 int compat_tcp_getsockopt(struct sock *sk, int level, int optname,
3695                           char __user *optval, int __user *optlen)
3696 {
3697         if (level != SOL_TCP)
3698                 return inet_csk_compat_getsockopt(sk, level, optname,
3699                                                   optval, optlen);
3700         return do_tcp_getsockopt(sk, level, optname, optval, optlen);
3701 }
3702 EXPORT_SYMBOL(compat_tcp_getsockopt);
3703 #endif
3704
3705 #ifdef CONFIG_TCP_MD5SIG
3706 static DEFINE_PER_CPU(struct tcp_md5sig_pool, tcp_md5sig_pool);
3707 static DEFINE_MUTEX(tcp_md5sig_mutex);
3708 static bool tcp_md5sig_pool_populated = false;
3709
3710 static void __tcp_alloc_md5sig_pool(void)
3711 {
3712         struct crypto_ahash *hash;
3713         int cpu;
3714
3715         hash = crypto_alloc_ahash("md5", 0, CRYPTO_ALG_ASYNC);
3716         if (IS_ERR(hash))
3717                 return;
3718
3719         for_each_possible_cpu(cpu) {
3720                 void *scratch = per_cpu(tcp_md5sig_pool, cpu).scratch;
3721                 struct ahash_request *req;
3722
3723                 if (!scratch) {
3724                         scratch = kmalloc_node(sizeof(union tcp_md5sum_block) +
3725                                                sizeof(struct tcphdr),
3726                                                GFP_KERNEL,
3727                                                cpu_to_node(cpu));
3728                         if (!scratch)
3729                                 return;
3730                         per_cpu(tcp_md5sig_pool, cpu).scratch = scratch;
3731                 }
3732                 if (per_cpu(tcp_md5sig_pool, cpu).md5_req)
3733                         continue;
3734
3735                 req = ahash_request_alloc(hash, GFP_KERNEL);
3736                 if (!req)
3737                         return;
3738
3739                 ahash_request_set_callback(req, 0, NULL, NULL);
3740
3741                 per_cpu(tcp_md5sig_pool, cpu).md5_req = req;
3742         }
3743         /* before setting tcp_md5sig_pool_populated, we must commit all writes
3744          * to memory. See smp_rmb() in tcp_get_md5sig_pool()
3745          */
3746         smp_wmb();
3747         tcp_md5sig_pool_populated = true;
3748 }
3749
3750 bool tcp_alloc_md5sig_pool(void)
3751 {
3752         if (unlikely(!tcp_md5sig_pool_populated)) {
3753                 mutex_lock(&tcp_md5sig_mutex);
3754
3755                 if (!tcp_md5sig_pool_populated) {
3756                         __tcp_alloc_md5sig_pool();
3757                         if (tcp_md5sig_pool_populated)
3758                                 static_branch_inc(&tcp_md5_needed);
3759                 }
3760
3761                 mutex_unlock(&tcp_md5sig_mutex);
3762         }
3763         return tcp_md5sig_pool_populated;
3764 }
3765 EXPORT_SYMBOL(tcp_alloc_md5sig_pool);
3766
3767
3768 /**
3769  *      tcp_get_md5sig_pool - get md5sig_pool for this user
3770  *
3771  *      We use percpu structure, so if we succeed, we exit with preemption
3772  *      and BH disabled, to make sure another thread or softirq handling
3773  *      wont try to get same context.
3774  */
3775 struct tcp_md5sig_pool *tcp_get_md5sig_pool(void)
3776 {
3777         local_bh_disable();
3778
3779         if (tcp_md5sig_pool_populated) {
3780                 /* coupled with smp_wmb() in __tcp_alloc_md5sig_pool() */
3781                 smp_rmb();
3782                 return this_cpu_ptr(&tcp_md5sig_pool);
3783         }
3784         local_bh_enable();
3785         return NULL;
3786 }
3787 EXPORT_SYMBOL(tcp_get_md5sig_pool);
3788
3789 int tcp_md5_hash_skb_data(struct tcp_md5sig_pool *hp,
3790                           const struct sk_buff *skb, unsigned int header_len)
3791 {
3792         struct scatterlist sg;
3793         const struct tcphdr *tp = tcp_hdr(skb);
3794         struct ahash_request *req = hp->md5_req;
3795         unsigned int i;
3796         const unsigned int head_data_len = skb_headlen(skb) > header_len ?
3797                                            skb_headlen(skb) - header_len : 0;
3798         const struct skb_shared_info *shi = skb_shinfo(skb);
3799         struct sk_buff *frag_iter;
3800
3801         sg_init_table(&sg, 1);
3802
3803         sg_set_buf(&sg, ((u8 *) tp) + header_len, head_data_len);
3804         ahash_request_set_crypt(req, &sg, NULL, head_data_len);
3805         if (crypto_ahash_update(req))
3806                 return 1;
3807
3808         for (i = 0; i < shi->nr_frags; ++i) {
3809                 const skb_frag_t *f = &shi->frags[i];
3810                 unsigned int offset = skb_frag_off(f);
3811                 struct page *page = skb_frag_page(f) + (offset >> PAGE_SHIFT);
3812
3813                 sg_set_page(&sg, page, skb_frag_size(f),
3814                             offset_in_page(offset));
3815                 ahash_request_set_crypt(req, &sg, NULL, skb_frag_size(f));
3816                 if (crypto_ahash_update(req))
3817                         return 1;
3818         }
3819
3820         skb_walk_frags(skb, frag_iter)
3821                 if (tcp_md5_hash_skb_data(hp, frag_iter, 0))
3822                         return 1;
3823
3824         return 0;
3825 }
3826 EXPORT_SYMBOL(tcp_md5_hash_skb_data);
3827
3828 int tcp_md5_hash_key(struct tcp_md5sig_pool *hp, const struct tcp_md5sig_key *key)
3829 {
3830         struct scatterlist sg;
3831
3832         sg_init_one(&sg, key->key, key->keylen);
3833         ahash_request_set_crypt(hp->md5_req, &sg, NULL, key->keylen);
3834         return crypto_ahash_update(hp->md5_req);
3835 }
3836 EXPORT_SYMBOL(tcp_md5_hash_key);
3837
3838 #endif
3839
3840 void tcp_done(struct sock *sk)
3841 {
3842         struct request_sock *req;
3843
3844         req = rcu_dereference_protected(tcp_sk(sk)->fastopen_rsk,
3845                                         lockdep_sock_is_held(sk));
3846         if (sk->sk_state == TCP_SYN_SENT || sk->sk_state == TCP_SYN_RECV)
3847                 TCP_INC_STATS(sock_net(sk), TCP_MIB_ATTEMPTFAILS);
3848
3849         tcp_set_state(sk, TCP_CLOSE);
3850         tcp_clear_xmit_timers(sk);
3851         if (req)
3852                 reqsk_fastopen_remove(sk, req, false);
3853
3854         sk->sk_shutdown = SHUTDOWN_MASK;
3855
3856         if (!sock_flag(sk, SOCK_DEAD))
3857                 sk->sk_state_change(sk);
3858         else
3859                 inet_csk_destroy_sock(sk);
3860 }
3861 EXPORT_SYMBOL_GPL(tcp_done);
3862
3863 int tcp_abort(struct sock *sk, int err)
3864 {
3865         if (!sk_fullsock(sk)) {
3866                 if (sk->sk_state == TCP_NEW_SYN_RECV) {
3867                         struct request_sock *req = inet_reqsk(sk);
3868
3869                         local_bh_disable();
3870                         inet_csk_reqsk_queue_drop(req->rsk_listener, req);
3871                         local_bh_enable();
3872                         return 0;
3873                 }
3874                 return -EOPNOTSUPP;
3875         }
3876
3877         /* Don't race with userspace socket closes such as tcp_close. */
3878         lock_sock(sk);
3879
3880         if (sk->sk_state == TCP_LISTEN) {
3881                 tcp_set_state(sk, TCP_CLOSE);
3882                 inet_csk_listen_stop(sk);
3883         }
3884
3885         /* Don't race with BH socket closes such as inet_csk_listen_stop. */
3886         local_bh_disable();
3887         bh_lock_sock(sk);
3888
3889         if (!sock_flag(sk, SOCK_DEAD)) {
3890                 sk->sk_err = err;
3891                 /* This barrier is coupled with smp_rmb() in tcp_poll() */
3892                 smp_wmb();
3893                 sk->sk_error_report(sk);
3894                 if (tcp_need_reset(sk->sk_state))
3895                         tcp_send_active_reset(sk, GFP_ATOMIC);
3896                 tcp_done(sk);
3897         }
3898
3899         bh_unlock_sock(sk);
3900         local_bh_enable();
3901         tcp_write_queue_purge(sk);
3902         release_sock(sk);
3903         return 0;
3904 }
3905 EXPORT_SYMBOL_GPL(tcp_abort);
3906
3907 extern struct tcp_congestion_ops tcp_reno;
3908
3909 static __initdata unsigned long thash_entries;
3910 static int __init set_thash_entries(char *str)
3911 {
3912         ssize_t ret;
3913
3914         if (!str)
3915                 return 0;
3916
3917         ret = kstrtoul(str, 0, &thash_entries);
3918         if (ret)
3919                 return 0;
3920
3921         return 1;
3922 }
3923 __setup("thash_entries=", set_thash_entries);
3924
3925 static void __init tcp_init_mem(void)
3926 {
3927         unsigned long limit = nr_free_buffer_pages() / 16;
3928
3929         limit = max(limit, 128UL);
3930         sysctl_tcp_mem[0] = limit / 4 * 3;              /* 4.68 % */
3931         sysctl_tcp_mem[1] = limit;                      /* 6.25 % */
3932         sysctl_tcp_mem[2] = sysctl_tcp_mem[0] * 2;      /* 9.37 % */
3933 }
3934
3935 void __init tcp_init(void)
3936 {
3937         int max_rshare, max_wshare, cnt;
3938         unsigned long limit;
3939         unsigned int i;
3940
3941         BUILD_BUG_ON(TCP_MIN_SND_MSS <= MAX_TCP_OPTION_SPACE);
3942         BUILD_BUG_ON(sizeof(struct tcp_skb_cb) >
3943                      FIELD_SIZEOF(struct sk_buff, cb));
3944
3945         percpu_counter_init(&tcp_sockets_allocated, 0, GFP_KERNEL);
3946         percpu_counter_init(&tcp_orphan_count, 0, GFP_KERNEL);
3947         inet_hashinfo_init(&tcp_hashinfo);
3948         inet_hashinfo2_init(&tcp_hashinfo, "tcp_listen_portaddr_hash",
3949                             thash_entries, 21,  /* one slot per 2 MB*/
3950                             0, 64 * 1024);
3951         tcp_hashinfo.bind_bucket_cachep =
3952                 kmem_cache_create("tcp_bind_bucket",
3953                                   sizeof(struct inet_bind_bucket), 0,
3954                                   SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
3955
3956         /* Size and allocate the main established and bind bucket
3957          * hash tables.
3958          *
3959          * The methodology is similar to that of the buffer cache.
3960          */
3961         tcp_hashinfo.ehash =
3962                 alloc_large_system_hash("TCP established",
3963                                         sizeof(struct inet_ehash_bucket),
3964                                         thash_entries,
3965                                         17, /* one slot per 128 KB of memory */
3966                                         0,
3967                                         NULL,
3968                                         &tcp_hashinfo.ehash_mask,
3969                                         0,
3970                                         thash_entries ? 0 : 512 * 1024);
3971         for (i = 0; i <= tcp_hashinfo.ehash_mask; i++)
3972                 INIT_HLIST_NULLS_HEAD(&tcp_hashinfo.ehash[i].chain, i);
3973
3974         if (inet_ehash_locks_alloc(&tcp_hashinfo))
3975                 panic("TCP: failed to alloc ehash_locks");
3976         tcp_hashinfo.bhash =
3977                 alloc_large_system_hash("TCP bind",
3978                                         sizeof(struct inet_bind_hashbucket),
3979                                         tcp_hashinfo.ehash_mask + 1,
3980                                         17, /* one slot per 128 KB of memory */
3981                                         0,
3982                                         &tcp_hashinfo.bhash_size,
3983                                         NULL,
3984                                         0,
3985                                         64 * 1024);
3986         tcp_hashinfo.bhash_size = 1U << tcp_hashinfo.bhash_size;
3987         for (i = 0; i < tcp_hashinfo.bhash_size; i++) {
3988                 spin_lock_init(&tcp_hashinfo.bhash[i].lock);
3989                 INIT_HLIST_HEAD(&tcp_hashinfo.bhash[i].chain);
3990         }
3991
3992
3993         cnt = tcp_hashinfo.ehash_mask + 1;
3994         sysctl_tcp_max_orphans = cnt / 2;
3995
3996         tcp_init_mem();
3997         /* Set per-socket limits to no more than 1/128 the pressure threshold */
3998         limit = nr_free_buffer_pages() << (PAGE_SHIFT - 7);
3999         max_wshare = min(4UL*1024*1024, limit);
4000         max_rshare = min(6UL*1024*1024, limit);
4001
4002         init_net.ipv4.sysctl_tcp_wmem[0] = SK_MEM_QUANTUM;
4003         init_net.ipv4.sysctl_tcp_wmem[1] = 16*1024;
4004         init_net.ipv4.sysctl_tcp_wmem[2] = max(64*1024, max_wshare);
4005
4006         init_net.ipv4.sysctl_tcp_rmem[0] = SK_MEM_QUANTUM;
4007         init_net.ipv4.sysctl_tcp_rmem[1] = 131072;
4008         init_net.ipv4.sysctl_tcp_rmem[2] = max(131072, max_rshare);
4009
4010         pr_info("Hash tables configured (established %u bind %u)\n",
4011                 tcp_hashinfo.ehash_mask + 1, tcp_hashinfo.bhash_size);
4012
4013         tcp_v4_init();
4014         tcp_metrics_init();
4015         BUG_ON(tcp_register_congestion_control(&tcp_reno) != 0);
4016         tcp_tasklet_init();
4017 }