1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35#ifndef _SOCK_H
36#define _SOCK_H
37
38#include <linux/hardirq.h>
39#include <linux/kernel.h>
40#include <linux/list.h>
41#include <linux/list_nulls.h>
42#include <linux/timer.h>
43#include <linux/cache.h>
44#include <linux/bitops.h>
45#include <linux/lockdep.h>
46#include <linux/netdevice.h>
47#include <linux/skbuff.h>
48#include <linux/mm.h>
49#include <linux/security.h>
50#include <linux/slab.h>
51#include <linux/uaccess.h>
52#include <linux/page_counter.h>
53#include <linux/memcontrol.h>
54#include <linux/static_key.h>
55#include <linux/sched.h>
56#include <linux/wait.h>
57#include <linux/cgroup-defs.h>
58#include <linux/rbtree.h>
59#include <linux/filter.h>
60#include <linux/rculist_nulls.h>
61#include <linux/poll.h>
62#include <linux/sockptr.h>
63#include <linux/indirect_call_wrapper.h>
64#include <linux/atomic.h>
65#include <linux/refcount.h>
66#include <net/dst.h>
67#include <net/checksum.h>
68#include <net/tcp_states.h>
69#include <linux/net_tstamp.h>
70#include <net/l3mdev.h>
71#include <uapi/linux/socket.h>
72
73
74
75
76
77
78
79
80#define SOCK_DEBUGGING
81#ifdef SOCK_DEBUGGING
82#define SOCK_DEBUG(sk, msg...) do { if ((sk) && sock_flag((sk), SOCK_DBG)) \
83 printk(KERN_DEBUG msg); } while (0)
84#else
85
86static inline __printf(2, 3)
87void SOCK_DEBUG(const struct sock *sk, const char *msg, ...)
88{
89}
90#endif
91
92
93
94
95
96typedef struct {
97 spinlock_t slock;
98 int owned;
99 wait_queue_head_t wq;
100
101
102
103
104
105
106#ifdef CONFIG_DEBUG_LOCK_ALLOC
107 struct lockdep_map dep_map;
108#endif
109} socket_lock_t;
110
111struct sock;
112struct proto;
113struct net;
114
115typedef __u32 __bitwise __portpair;
116typedef __u64 __bitwise __addrpair;
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163struct sock_common {
164
165
166
167 union {
168 __addrpair skc_addrpair;
169 struct {
170 __be32 skc_daddr;
171 __be32 skc_rcv_saddr;
172 };
173 };
174 union {
175 unsigned int skc_hash;
176 __u16 skc_u16hashes[2];
177 };
178
179 union {
180 __portpair skc_portpair;
181 struct {
182 __be16 skc_dport;
183 __u16 skc_num;
184 };
185 };
186
187 unsigned short skc_family;
188 volatile unsigned char skc_state;
189 unsigned char skc_reuse:4;
190 unsigned char skc_reuseport:1;
191 unsigned char skc_ipv6only:1;
192 unsigned char skc_net_refcnt:1;
193 int skc_bound_dev_if;
194 union {
195 struct hlist_node skc_bind_node;
196 struct hlist_node skc_portaddr_node;
197 };
198 struct proto *skc_prot;
199 possible_net_t skc_net;
200
201#if IS_ENABLED(CONFIG_IPV6)
202 struct in6_addr skc_v6_daddr;
203 struct in6_addr skc_v6_rcv_saddr;
204#endif
205
206 atomic64_t skc_cookie;
207
208
209
210
211
212
213 union {
214 unsigned long skc_flags;
215 struct sock *skc_listener;
216 struct inet_timewait_death_row *skc_tw_dr;
217 };
218
219
220
221
222
223 int skc_dontcopy_begin[0];
224
225 union {
226 struct hlist_node skc_node;
227 struct hlist_nulls_node skc_nulls_node;
228 };
229 unsigned short skc_tx_queue_mapping;
230#ifdef CONFIG_SOCK_RX_QUEUE_MAPPING
231 unsigned short skc_rx_queue_mapping;
232#endif
233 union {
234 int skc_incoming_cpu;
235 u32 skc_rcv_wnd;
236 u32 skc_tw_rcv_nxt;
237 };
238
239 refcount_t skc_refcnt;
240
241 int skc_dontcopy_end[0];
242 union {
243 u32 skc_rxhash;
244 u32 skc_window_clamp;
245 u32 skc_tw_snd_nxt;
246 };
247
248};
249
250struct bpf_local_storage;
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353struct sock {
354
355
356
357
358 struct sock_common __sk_common;
359#define sk_node __sk_common.skc_node
360#define sk_nulls_node __sk_common.skc_nulls_node
361#define sk_refcnt __sk_common.skc_refcnt
362#define sk_tx_queue_mapping __sk_common.skc_tx_queue_mapping
363#ifdef CONFIG_SOCK_RX_QUEUE_MAPPING
364#define sk_rx_queue_mapping __sk_common.skc_rx_queue_mapping
365#endif
366
367#define sk_dontcopy_begin __sk_common.skc_dontcopy_begin
368#define sk_dontcopy_end __sk_common.skc_dontcopy_end
369#define sk_hash __sk_common.skc_hash
370#define sk_portpair __sk_common.skc_portpair
371#define sk_num __sk_common.skc_num
372#define sk_dport __sk_common.skc_dport
373#define sk_addrpair __sk_common.skc_addrpair
374#define sk_daddr __sk_common.skc_daddr
375#define sk_rcv_saddr __sk_common.skc_rcv_saddr
376#define sk_family __sk_common.skc_family
377#define sk_state __sk_common.skc_state
378#define sk_reuse __sk_common.skc_reuse
379#define sk_reuseport __sk_common.skc_reuseport
380#define sk_ipv6only __sk_common.skc_ipv6only
381#define sk_net_refcnt __sk_common.skc_net_refcnt
382#define sk_bound_dev_if __sk_common.skc_bound_dev_if
383#define sk_bind_node __sk_common.skc_bind_node
384#define sk_prot __sk_common.skc_prot
385#define sk_net __sk_common.skc_net
386#define sk_v6_daddr __sk_common.skc_v6_daddr
387#define sk_v6_rcv_saddr __sk_common.skc_v6_rcv_saddr
388#define sk_cookie __sk_common.skc_cookie
389#define sk_incoming_cpu __sk_common.skc_incoming_cpu
390#define sk_flags __sk_common.skc_flags
391#define sk_rxhash __sk_common.skc_rxhash
392
393 socket_lock_t sk_lock;
394 atomic_t sk_drops;
395 int sk_rcvlowat;
396 struct sk_buff_head sk_error_queue;
397 struct sk_buff *sk_rx_skb_cache;
398 struct sk_buff_head sk_receive_queue;
399
400
401
402
403
404
405
406
407 struct {
408 atomic_t rmem_alloc;
409 int len;
410 struct sk_buff *head;
411 struct sk_buff *tail;
412 } sk_backlog;
413#define sk_rmem_alloc sk_backlog.rmem_alloc
414
415 int sk_forward_alloc;
416#ifdef CONFIG_NET_RX_BUSY_POLL
417 unsigned int sk_ll_usec;
418
419 unsigned int sk_napi_id;
420#endif
421 int sk_rcvbuf;
422
423 struct sk_filter __rcu *sk_filter;
424 union {
425 struct socket_wq __rcu *sk_wq;
426
427 struct socket_wq *sk_wq_raw;
428
429 };
430#ifdef CONFIG_XFRM
431 struct xfrm_policy __rcu *sk_policy[2];
432#endif
433 struct dst_entry *sk_rx_dst;
434 struct dst_entry __rcu *sk_dst_cache;
435 atomic_t sk_omem_alloc;
436 int sk_sndbuf;
437
438
439 int sk_wmem_queued;
440 refcount_t sk_wmem_alloc;
441 unsigned long sk_tsq_flags;
442 union {
443 struct sk_buff *sk_send_head;
444 struct rb_root tcp_rtx_queue;
445 };
446 struct sk_buff *sk_tx_skb_cache;
447 struct sk_buff_head sk_write_queue;
448 __s32 sk_peek_off;
449 int sk_write_pending;
450 __u32 sk_dst_pending_confirm;
451 u32 sk_pacing_status;
452 long sk_sndtimeo;
453 struct timer_list sk_timer;
454 __u32 sk_priority;
455 __u32 sk_mark;
456 unsigned long sk_pacing_rate;
457 unsigned long sk_max_pacing_rate;
458 struct page_frag sk_frag;
459 netdev_features_t sk_route_caps;
460 netdev_features_t sk_route_nocaps;
461 netdev_features_t sk_route_forced_caps;
462 int sk_gso_type;
463 unsigned int sk_gso_max_size;
464 gfp_t sk_allocation;
465 __u32 sk_txhash;
466
467
468
469
470
471 u8 sk_padding : 1,
472 sk_kern_sock : 1,
473 sk_no_check_tx : 1,
474 sk_no_check_rx : 1,
475 sk_userlocks : 4;
476 u8 sk_pacing_shift;
477 u16 sk_type;
478 u16 sk_protocol;
479 u16 sk_gso_max_segs;
480 unsigned long sk_lingertime;
481 struct proto *sk_prot_creator;
482 rwlock_t sk_callback_lock;
483 int sk_err,
484 sk_err_soft;
485 u32 sk_ack_backlog;
486 u32 sk_max_ack_backlog;
487 kuid_t sk_uid;
488#ifdef CONFIG_NET_RX_BUSY_POLL
489 u8 sk_prefer_busy_poll;
490 u16 sk_busy_poll_budget;
491#endif
492 spinlock_t sk_peer_lock;
493 struct pid *sk_peer_pid;
494 const struct cred *sk_peer_cred;
495
496 long sk_rcvtimeo;
497 ktime_t sk_stamp;
498#if BITS_PER_LONG==32
499 seqlock_t sk_stamp_seq;
500#endif
501 u16 sk_tsflags;
502 int sk_bind_phc;
503 u8 sk_shutdown;
504 u32 sk_tskey;
505 atomic_t sk_zckey;
506
507 u8 sk_clockid;
508 u8 sk_txtime_deadline_mode : 1,
509 sk_txtime_report_errors : 1,
510 sk_txtime_unused : 6;
511
512 struct socket *sk_socket;
513 void *sk_user_data;
514#ifdef CONFIG_SECURITY
515 void *sk_security;
516#endif
517 struct sock_cgroup_data sk_cgrp_data;
518 struct mem_cgroup *sk_memcg;
519 void (*sk_state_change)(struct sock *sk);
520 void (*sk_data_ready)(struct sock *sk);
521 void (*sk_write_space)(struct sock *sk);
522 void (*sk_error_report)(struct sock *sk);
523 int (*sk_backlog_rcv)(struct sock *sk,
524 struct sk_buff *skb);
525#ifdef CONFIG_SOCK_VALIDATE_XMIT
526 struct sk_buff* (*sk_validate_xmit_skb)(struct sock *sk,
527 struct net_device *dev,
528 struct sk_buff *skb);
529#endif
530 void (*sk_destruct)(struct sock *sk);
531 struct sock_reuseport __rcu *sk_reuseport_cb;
532#ifdef CONFIG_BPF_SYSCALL
533 struct bpf_local_storage __rcu *sk_bpf_storage;
534#endif
535 struct rcu_head sk_rcu;
536};
537
538enum sk_pacing {
539 SK_PACING_NONE = 0,
540 SK_PACING_NEEDED = 1,
541 SK_PACING_FQ = 2,
542};
543
544
545
546
547
548
549#define SK_USER_DATA_NOCOPY 1UL
550#define SK_USER_DATA_BPF 2UL
551#define SK_USER_DATA_PTRMASK ~(SK_USER_DATA_NOCOPY | SK_USER_DATA_BPF)
552
553
554
555
556
557static inline bool sk_user_data_is_nocopy(const struct sock *sk)
558{
559 return ((uintptr_t)sk->sk_user_data & SK_USER_DATA_NOCOPY);
560}
561
562#define __sk_user_data(sk) ((*((void __rcu **)&(sk)->sk_user_data)))
563
564#define rcu_dereference_sk_user_data(sk) \
565({ \
566 void *__tmp = rcu_dereference(__sk_user_data((sk))); \
567 (void *)((uintptr_t)__tmp & SK_USER_DATA_PTRMASK); \
568})
569#define rcu_assign_sk_user_data(sk, ptr) \
570({ \
571 uintptr_t __tmp = (uintptr_t)(ptr); \
572 WARN_ON_ONCE(__tmp & ~SK_USER_DATA_PTRMASK); \
573 rcu_assign_pointer(__sk_user_data((sk)), __tmp); \
574})
575#define rcu_assign_sk_user_data_nocopy(sk, ptr) \
576({ \
577 uintptr_t __tmp = (uintptr_t)(ptr); \
578 WARN_ON_ONCE(__tmp & ~SK_USER_DATA_PTRMASK); \
579 rcu_assign_pointer(__sk_user_data((sk)), \
580 __tmp | SK_USER_DATA_NOCOPY); \
581})
582
583
584
585
586
587
588
589
590#define SK_NO_REUSE 0
591#define SK_CAN_REUSE 1
592#define SK_FORCE_REUSE 2
593
594int sk_set_peek_off(struct sock *sk, int val);
595
596static inline int sk_peek_offset(struct sock *sk, int flags)
597{
598 if (unlikely(flags & MSG_PEEK)) {
599 return READ_ONCE(sk->sk_peek_off);
600 }
601
602 return 0;
603}
604
605static inline void sk_peek_offset_bwd(struct sock *sk, int val)
606{
607 s32 off = READ_ONCE(sk->sk_peek_off);
608
609 if (unlikely(off >= 0)) {
610 off = max_t(s32, off - val, 0);
611 WRITE_ONCE(sk->sk_peek_off, off);
612 }
613}
614
615static inline void sk_peek_offset_fwd(struct sock *sk, int val)
616{
617 sk_peek_offset_bwd(sk, -val);
618}
619
620
621
622
623static inline struct sock *sk_entry(const struct hlist_node *node)
624{
625 return hlist_entry(node, struct sock, sk_node);
626}
627
628static inline struct sock *__sk_head(const struct hlist_head *head)
629{
630 return hlist_entry(head->first, struct sock, sk_node);
631}
632
633static inline struct sock *sk_head(const struct hlist_head *head)
634{
635 return hlist_empty(head) ? NULL : __sk_head(head);
636}
637
638static inline struct sock *__sk_nulls_head(const struct hlist_nulls_head *head)
639{
640 return hlist_nulls_entry(head->first, struct sock, sk_nulls_node);
641}
642
643static inline struct sock *sk_nulls_head(const struct hlist_nulls_head *head)
644{
645 return hlist_nulls_empty(head) ? NULL : __sk_nulls_head(head);
646}
647
648static inline struct sock *sk_next(const struct sock *sk)
649{
650 return hlist_entry_safe(sk->sk_node.next, struct sock, sk_node);
651}
652
653static inline struct sock *sk_nulls_next(const struct sock *sk)
654{
655 return (!is_a_nulls(sk->sk_nulls_node.next)) ?
656 hlist_nulls_entry(sk->sk_nulls_node.next,
657 struct sock, sk_nulls_node) :
658 NULL;
659}
660
661static inline bool sk_unhashed(const struct sock *sk)
662{
663 return hlist_unhashed(&sk->sk_node);
664}
665
666static inline bool sk_hashed(const struct sock *sk)
667{
668 return !sk_unhashed(sk);
669}
670
671static inline void sk_node_init(struct hlist_node *node)
672{
673 node->pprev = NULL;
674}
675
676static inline void sk_nulls_node_init(struct hlist_nulls_node *node)
677{
678 node->pprev = NULL;
679}
680
681static inline void __sk_del_node(struct sock *sk)
682{
683 __hlist_del(&sk->sk_node);
684}
685
686
687static inline bool __sk_del_node_init(struct sock *sk)
688{
689 if (sk_hashed(sk)) {
690 __sk_del_node(sk);
691 sk_node_init(&sk->sk_node);
692 return true;
693 }
694 return false;
695}
696
697
698
699
700
701
702
703static __always_inline void sock_hold(struct sock *sk)
704{
705 refcount_inc(&sk->sk_refcnt);
706}
707
708
709
710
711static __always_inline void __sock_put(struct sock *sk)
712{
713 refcount_dec(&sk->sk_refcnt);
714}
715
716static inline bool sk_del_node_init(struct sock *sk)
717{
718 bool rc = __sk_del_node_init(sk);
719
720 if (rc) {
721
722 WARN_ON(refcount_read(&sk->sk_refcnt) == 1);
723 __sock_put(sk);
724 }
725 return rc;
726}
727#define sk_del_node_init_rcu(sk) sk_del_node_init(sk)
728
729static inline bool __sk_nulls_del_node_init_rcu(struct sock *sk)
730{
731 if (sk_hashed(sk)) {
732 hlist_nulls_del_init_rcu(&sk->sk_nulls_node);
733 return true;
734 }
735 return false;
736}
737
738static inline bool sk_nulls_del_node_init_rcu(struct sock *sk)
739{
740 bool rc = __sk_nulls_del_node_init_rcu(sk);
741
742 if (rc) {
743
744 WARN_ON(refcount_read(&sk->sk_refcnt) == 1);
745 __sock_put(sk);
746 }
747 return rc;
748}
749
750static inline void __sk_add_node(struct sock *sk, struct hlist_head *list)
751{
752 hlist_add_head(&sk->sk_node, list);
753}
754
755static inline void sk_add_node(struct sock *sk, struct hlist_head *list)
756{
757 sock_hold(sk);
758 __sk_add_node(sk, list);
759}
760
761static inline void sk_add_node_rcu(struct sock *sk, struct hlist_head *list)
762{
763 sock_hold(sk);
764 if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport &&
765 sk->sk_family == AF_INET6)
766 hlist_add_tail_rcu(&sk->sk_node, list);
767 else
768 hlist_add_head_rcu(&sk->sk_node, list);
769}
770
771static inline void sk_add_node_tail_rcu(struct sock *sk, struct hlist_head *list)
772{
773 sock_hold(sk);
774 hlist_add_tail_rcu(&sk->sk_node, list);
775}
776
777static inline void __sk_nulls_add_node_rcu(struct sock *sk, struct hlist_nulls_head *list)
778{
779 hlist_nulls_add_head_rcu(&sk->sk_nulls_node, list);
780}
781
782static inline void __sk_nulls_add_node_tail_rcu(struct sock *sk, struct hlist_nulls_head *list)
783{
784 hlist_nulls_add_tail_rcu(&sk->sk_nulls_node, list);
785}
786
787static inline void sk_nulls_add_node_rcu(struct sock *sk, struct hlist_nulls_head *list)
788{
789 sock_hold(sk);
790 __sk_nulls_add_node_rcu(sk, list);
791}
792
793static inline void __sk_del_bind_node(struct sock *sk)
794{
795 __hlist_del(&sk->sk_bind_node);
796}
797
798static inline void sk_add_bind_node(struct sock *sk,
799 struct hlist_head *list)
800{
801 hlist_add_head(&sk->sk_bind_node, list);
802}
803
804#define sk_for_each(__sk, list) \
805 hlist_for_each_entry(__sk, list, sk_node)
806#define sk_for_each_rcu(__sk, list) \
807 hlist_for_each_entry_rcu(__sk, list, sk_node)
808#define sk_nulls_for_each(__sk, node, list) \
809 hlist_nulls_for_each_entry(__sk, node, list, sk_nulls_node)
810#define sk_nulls_for_each_rcu(__sk, node, list) \
811 hlist_nulls_for_each_entry_rcu(__sk, node, list, sk_nulls_node)
812#define sk_for_each_from(__sk) \
813 hlist_for_each_entry_from(__sk, sk_node)
814#define sk_nulls_for_each_from(__sk, node) \
815 if (__sk && ({ node = &(__sk)->sk_nulls_node; 1; })) \
816 hlist_nulls_for_each_entry_from(__sk, node, sk_nulls_node)
817#define sk_for_each_safe(__sk, tmp, list) \
818 hlist_for_each_entry_safe(__sk, tmp, list, sk_node)
819#define sk_for_each_bound(__sk, list) \
820 hlist_for_each_entry(__sk, list, sk_bind_node)
821
822
823
824
825
826
827
828
829
830#define sk_for_each_entry_offset_rcu(tpos, pos, head, offset) \
831 for (pos = rcu_dereference(hlist_first_rcu(head)); \
832 pos != NULL && \
833 ({ tpos = (typeof(*tpos) *)((void *)pos - offset); 1;}); \
834 pos = rcu_dereference(hlist_next_rcu(pos)))
835
836static inline struct user_namespace *sk_user_ns(struct sock *sk)
837{
838
839
840
841
842 return sk->sk_socket->file->f_cred->user_ns;
843}
844
845
846enum sock_flags {
847 SOCK_DEAD,
848 SOCK_DONE,
849 SOCK_URGINLINE,
850 SOCK_KEEPOPEN,
851 SOCK_LINGER,
852 SOCK_DESTROY,
853 SOCK_BROADCAST,
854 SOCK_TIMESTAMP,
855 SOCK_ZAPPED,
856 SOCK_USE_WRITE_QUEUE,
857 SOCK_DBG,
858 SOCK_RCVTSTAMP,
859 SOCK_RCVTSTAMPNS,
860 SOCK_LOCALROUTE,
861 SOCK_MEMALLOC,
862 SOCK_TIMESTAMPING_RX_SOFTWARE,
863 SOCK_FASYNC,
864 SOCK_RXQ_OVFL,
865 SOCK_ZEROCOPY,
866 SOCK_WIFI_STATUS,
867 SOCK_NOFCS,
868
869
870
871 SOCK_FILTER_LOCKED,
872 SOCK_SELECT_ERR_QUEUE,
873 SOCK_RCU_FREE,
874 SOCK_TXTIME,
875 SOCK_XDP,
876 SOCK_TSTAMP_NEW,
877};
878
879#define SK_FLAGS_TIMESTAMP ((1UL << SOCK_TIMESTAMP) | (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE))
880
881static inline void sock_copy_flags(struct sock *nsk, struct sock *osk)
882{
883 nsk->sk_flags = osk->sk_flags;
884}
885
886static inline void sock_set_flag(struct sock *sk, enum sock_flags flag)
887{
888 __set_bit(flag, &sk->sk_flags);
889}
890
891static inline void sock_reset_flag(struct sock *sk, enum sock_flags flag)
892{
893 __clear_bit(flag, &sk->sk_flags);
894}
895
896static inline void sock_valbool_flag(struct sock *sk, enum sock_flags bit,
897 int valbool)
898{
899 if (valbool)
900 sock_set_flag(sk, bit);
901 else
902 sock_reset_flag(sk, bit);
903}
904
905static inline bool sock_flag(const struct sock *sk, enum sock_flags flag)
906{
907 return test_bit(flag, &sk->sk_flags);
908}
909
910#ifdef CONFIG_NET
911DECLARE_STATIC_KEY_FALSE(memalloc_socks_key);
912static inline int sk_memalloc_socks(void)
913{
914 return static_branch_unlikely(&memalloc_socks_key);
915}
916
917void __receive_sock(struct file *file);
918#else
919
920static inline int sk_memalloc_socks(void)
921{
922 return 0;
923}
924
925static inline void __receive_sock(struct file *file)
926{ }
927#endif
928
929static inline gfp_t sk_gfp_mask(const struct sock *sk, gfp_t gfp_mask)
930{
931 return gfp_mask | (sk->sk_allocation & __GFP_MEMALLOC);
932}
933
934static inline void sk_acceptq_removed(struct sock *sk)
935{
936 WRITE_ONCE(sk->sk_ack_backlog, sk->sk_ack_backlog - 1);
937}
938
939static inline void sk_acceptq_added(struct sock *sk)
940{
941 WRITE_ONCE(sk->sk_ack_backlog, sk->sk_ack_backlog + 1);
942}
943
944
945
946
947
948static inline bool sk_acceptq_is_full(const struct sock *sk)
949{
950 return READ_ONCE(sk->sk_ack_backlog) > READ_ONCE(sk->sk_max_ack_backlog);
951}
952
953
954
955
956static inline int sk_stream_min_wspace(const struct sock *sk)
957{
958 return READ_ONCE(sk->sk_wmem_queued) >> 1;
959}
960
961static inline int sk_stream_wspace(const struct sock *sk)
962{
963 return READ_ONCE(sk->sk_sndbuf) - READ_ONCE(sk->sk_wmem_queued);
964}
965
966static inline void sk_wmem_queued_add(struct sock *sk, int val)
967{
968 WRITE_ONCE(sk->sk_wmem_queued, sk->sk_wmem_queued + val);
969}
970
971void sk_stream_write_space(struct sock *sk);
972
973
974static inline void __sk_add_backlog(struct sock *sk, struct sk_buff *skb)
975{
976
977 skb_dst_force(skb);
978
979 if (!sk->sk_backlog.tail)
980 WRITE_ONCE(sk->sk_backlog.head, skb);
981 else
982 sk->sk_backlog.tail->next = skb;
983
984 WRITE_ONCE(sk->sk_backlog.tail, skb);
985 skb->next = NULL;
986}
987
988
989
990
991
992
993static inline bool sk_rcvqueues_full(const struct sock *sk, unsigned int limit)
994{
995 unsigned int qsize = sk->sk_backlog.len + atomic_read(&sk->sk_rmem_alloc);
996
997 return qsize > limit;
998}
999
1000
1001static inline __must_check int sk_add_backlog(struct sock *sk, struct sk_buff *skb,
1002 unsigned int limit)
1003{
1004 if (sk_rcvqueues_full(sk, limit))
1005 return -ENOBUFS;
1006
1007
1008
1009
1010
1011
1012 if (skb_pfmemalloc(skb) && !sock_flag(sk, SOCK_MEMALLOC))
1013 return -ENOMEM;
1014
1015 __sk_add_backlog(sk, skb);
1016 sk->sk_backlog.len += skb->truesize;
1017 return 0;
1018}
1019
1020int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb);
1021
1022static inline int sk_backlog_rcv(struct sock *sk, struct sk_buff *skb)
1023{
1024 if (sk_memalloc_socks() && skb_pfmemalloc(skb))
1025 return __sk_backlog_rcv(sk, skb);
1026
1027 return sk->sk_backlog_rcv(sk, skb);
1028}
1029
1030static inline void sk_incoming_cpu_update(struct sock *sk)
1031{
1032 int cpu = raw_smp_processor_id();
1033
1034 if (unlikely(READ_ONCE(sk->sk_incoming_cpu) != cpu))
1035 WRITE_ONCE(sk->sk_incoming_cpu, cpu);
1036}
1037
1038static inline void sock_rps_record_flow_hash(__u32 hash)
1039{
1040#ifdef CONFIG_RPS
1041 struct rps_sock_flow_table *sock_flow_table;
1042
1043 rcu_read_lock();
1044 sock_flow_table = rcu_dereference(rps_sock_flow_table);
1045 rps_record_sock_flow(sock_flow_table, hash);
1046 rcu_read_unlock();
1047#endif
1048}
1049
1050static inline void sock_rps_record_flow(const struct sock *sk)
1051{
1052#ifdef CONFIG_RPS
1053 if (static_branch_unlikely(&rfs_needed)) {
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064 if (sk->sk_state == TCP_ESTABLISHED)
1065 sock_rps_record_flow_hash(sk->sk_rxhash);
1066 }
1067#endif
1068}
1069
1070static inline void sock_rps_save_rxhash(struct sock *sk,
1071 const struct sk_buff *skb)
1072{
1073#ifdef CONFIG_RPS
1074 if (unlikely(sk->sk_rxhash != skb->hash))
1075 sk->sk_rxhash = skb->hash;
1076#endif
1077}
1078
1079static inline void sock_rps_reset_rxhash(struct sock *sk)
1080{
1081#ifdef CONFIG_RPS
1082 sk->sk_rxhash = 0;
1083#endif
1084}
1085
1086#define sk_wait_event(__sk, __timeo, __condition, __wait) \
1087 ({ int __rc; \
1088 release_sock(__sk); \
1089 __rc = __condition; \
1090 if (!__rc) { \
1091 *(__timeo) = wait_woken(__wait, \
1092 TASK_INTERRUPTIBLE, \
1093 *(__timeo)); \
1094 } \
1095 sched_annotate_sleep(); \
1096 lock_sock(__sk); \
1097 __rc = __condition; \
1098 __rc; \
1099 })
1100
1101int sk_stream_wait_connect(struct sock *sk, long *timeo_p);
1102int sk_stream_wait_memory(struct sock *sk, long *timeo_p);
1103void sk_stream_wait_close(struct sock *sk, long timeo_p);
1104int sk_stream_error(struct sock *sk, int flags, int err);
1105void sk_stream_kill_queues(struct sock *sk);
1106void sk_set_memalloc(struct sock *sk);
1107void sk_clear_memalloc(struct sock *sk);
1108
1109void __sk_flush_backlog(struct sock *sk);
1110
1111static inline bool sk_flush_backlog(struct sock *sk)
1112{
1113 if (unlikely(READ_ONCE(sk->sk_backlog.tail))) {
1114 __sk_flush_backlog(sk);
1115 return true;
1116 }
1117 return false;
1118}
1119
1120int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb);
1121
1122struct request_sock_ops;
1123struct timewait_sock_ops;
1124struct inet_hashinfo;
1125struct raw_hashinfo;
1126struct smc_hashinfo;
1127struct module;
1128struct sk_psock;
1129
1130
1131
1132
1133
1134static inline void sk_prot_clear_nulls(struct sock *sk, int size)
1135{
1136 if (offsetof(struct sock, sk_node.next) != 0)
1137 memset(sk, 0, offsetof(struct sock, sk_node.next));
1138 memset(&sk->sk_node.pprev, 0,
1139 size - offsetof(struct sock, sk_node.pprev));
1140}
1141
1142
1143
1144
1145struct proto {
1146 void (*close)(struct sock *sk,
1147 long timeout);
1148 int (*pre_connect)(struct sock *sk,
1149 struct sockaddr *uaddr,
1150 int addr_len);
1151 int (*connect)(struct sock *sk,
1152 struct sockaddr *uaddr,
1153 int addr_len);
1154 int (*disconnect)(struct sock *sk, int flags);
1155
1156 struct sock * (*accept)(struct sock *sk, int flags, int *err,
1157 bool kern);
1158
1159 int (*ioctl)(struct sock *sk, int cmd,
1160 unsigned long arg);
1161 int (*init)(struct sock *sk);
1162 void (*destroy)(struct sock *sk);
1163 void (*shutdown)(struct sock *sk, int how);
1164 int (*setsockopt)(struct sock *sk, int level,
1165 int optname, sockptr_t optval,
1166 unsigned int optlen);
1167 int (*getsockopt)(struct sock *sk, int level,
1168 int optname, char __user *optval,
1169 int __user *option);
1170 void (*keepalive)(struct sock *sk, int valbool);
1171#ifdef CONFIG_COMPAT
1172 int (*compat_ioctl)(struct sock *sk,
1173 unsigned int cmd, unsigned long arg);
1174#endif
1175 int (*sendmsg)(struct sock *sk, struct msghdr *msg,
1176 size_t len);
1177 int (*recvmsg)(struct sock *sk, struct msghdr *msg,
1178 size_t len, int noblock, int flags,
1179 int *addr_len);
1180 int (*sendpage)(struct sock *sk, struct page *page,
1181 int offset, size_t size, int flags);
1182 int (*bind)(struct sock *sk,
1183 struct sockaddr *addr, int addr_len);
1184 int (*bind_add)(struct sock *sk,
1185 struct sockaddr *addr, int addr_len);
1186
1187 int (*backlog_rcv) (struct sock *sk,
1188 struct sk_buff *skb);
1189 bool (*bpf_bypass_getsockopt)(int level,
1190 int optname);
1191
1192 void (*release_cb)(struct sock *sk);
1193
1194
1195 int (*hash)(struct sock *sk);
1196 void (*unhash)(struct sock *sk);
1197 void (*rehash)(struct sock *sk);
1198 int (*get_port)(struct sock *sk, unsigned short snum);
1199#ifdef CONFIG_BPF_SYSCALL
1200 int (*psock_update_sk_prot)(struct sock *sk,
1201 struct sk_psock *psock,
1202 bool restore);
1203#endif
1204
1205
1206#ifdef CONFIG_PROC_FS
1207 unsigned int inuse_idx;
1208#endif
1209
1210 bool (*stream_memory_free)(const struct sock *sk, int wake);
1211 bool (*sock_is_readable)(struct sock *sk);
1212
1213 void (*enter_memory_pressure)(struct sock *sk);
1214 void (*leave_memory_pressure)(struct sock *sk);
1215 atomic_long_t *memory_allocated;
1216 struct percpu_counter *sockets_allocated;
1217
1218
1219
1220
1221
1222
1223 unsigned long *memory_pressure;
1224 long *sysctl_mem;
1225
1226 int *sysctl_wmem;
1227 int *sysctl_rmem;
1228 u32 sysctl_wmem_offset;
1229 u32 sysctl_rmem_offset;
1230
1231 int max_header;
1232 bool no_autobind;
1233
1234 struct kmem_cache *slab;
1235 unsigned int obj_size;
1236 slab_flags_t slab_flags;
1237 unsigned int useroffset;
1238 unsigned int usersize;
1239
1240 struct percpu_counter *orphan_count;
1241
1242 struct request_sock_ops *rsk_prot;
1243 struct timewait_sock_ops *twsk_prot;
1244
1245 union {
1246 struct inet_hashinfo *hashinfo;
1247 struct udp_table *udp_table;
1248 struct raw_hashinfo *raw_hash;
1249 struct smc_hashinfo *smc_hash;
1250 } h;
1251
1252 struct module *owner;
1253
1254 char name[32];
1255
1256 struct list_head node;
1257#ifdef SOCK_REFCNT_DEBUG
1258 atomic_t socks;
1259#endif
1260 int (*diag_destroy)(struct sock *sk, int err);
1261} __randomize_layout;
1262
1263int proto_register(struct proto *prot, int alloc_slab);
1264void proto_unregister(struct proto *prot);
1265int sock_load_diag_module(int family, int protocol);
1266
1267#ifdef SOCK_REFCNT_DEBUG
1268static inline void sk_refcnt_debug_inc(struct sock *sk)
1269{
1270 atomic_inc(&sk->sk_prot->socks);
1271}
1272
1273static inline void sk_refcnt_debug_dec(struct sock *sk)
1274{
1275 atomic_dec(&sk->sk_prot->socks);
1276 printk(KERN_DEBUG "%s socket %p released, %d are still alive\n",
1277 sk->sk_prot->name, sk, atomic_read(&sk->sk_prot->socks));
1278}
1279
1280static inline void sk_refcnt_debug_release(const struct sock *sk)
1281{
1282 if (refcount_read(&sk->sk_refcnt) != 1)
1283 printk(KERN_DEBUG "Destruction of the %s socket %p delayed, refcnt=%d\n",
1284 sk->sk_prot->name, sk, refcount_read(&sk->sk_refcnt));
1285}
1286#else
1287#define sk_refcnt_debug_inc(sk) do { } while (0)
1288#define sk_refcnt_debug_dec(sk) do { } while (0)
1289#define sk_refcnt_debug_release(sk) do { } while (0)
1290#endif
1291
1292INDIRECT_CALLABLE_DECLARE(bool tcp_stream_memory_free(const struct sock *sk, int wake));
1293
1294static inline bool __sk_stream_memory_free(const struct sock *sk, int wake)
1295{
1296 if (READ_ONCE(sk->sk_wmem_queued) >= READ_ONCE(sk->sk_sndbuf))
1297 return false;
1298
1299#ifdef CONFIG_INET
1300 return sk->sk_prot->stream_memory_free ?
1301 INDIRECT_CALL_1(sk->sk_prot->stream_memory_free,
1302 tcp_stream_memory_free,
1303 sk, wake) : true;
1304#else
1305 return sk->sk_prot->stream_memory_free ?
1306 sk->sk_prot->stream_memory_free(sk, wake) : true;
1307#endif
1308}
1309
1310static inline bool sk_stream_memory_free(const struct sock *sk)
1311{
1312 return __sk_stream_memory_free(sk, 0);
1313}
1314
1315static inline bool __sk_stream_is_writeable(const struct sock *sk, int wake)
1316{
1317 return sk_stream_wspace(sk) >= sk_stream_min_wspace(sk) &&
1318 __sk_stream_memory_free(sk, wake);
1319}
1320
1321static inline bool sk_stream_is_writeable(const struct sock *sk)
1322{
1323 return __sk_stream_is_writeable(sk, 0);
1324}
1325
1326static inline int sk_under_cgroup_hierarchy(struct sock *sk,
1327 struct cgroup *ancestor)
1328{
1329#ifdef CONFIG_SOCK_CGROUP_DATA
1330 return cgroup_is_descendant(sock_cgroup_ptr(&sk->sk_cgrp_data),
1331 ancestor);
1332#else
1333 return -ENOTSUPP;
1334#endif
1335}
1336
1337static inline bool sk_has_memory_pressure(const struct sock *sk)
1338{
1339 return sk->sk_prot->memory_pressure != NULL;
1340}
1341
1342static inline bool sk_under_memory_pressure(const struct sock *sk)
1343{
1344 if (!sk->sk_prot->memory_pressure)
1345 return false;
1346
1347 if (mem_cgroup_sockets_enabled && sk->sk_memcg &&
1348 mem_cgroup_under_socket_pressure(sk->sk_memcg))
1349 return true;
1350
1351 return !!*sk->sk_prot->memory_pressure;
1352}
1353
1354static inline long
1355sk_memory_allocated(const struct sock *sk)
1356{
1357 return atomic_long_read(sk->sk_prot->memory_allocated);
1358}
1359
1360static inline long
1361sk_memory_allocated_add(struct sock *sk, int amt)
1362{
1363 return atomic_long_add_return(amt, sk->sk_prot->memory_allocated);
1364}
1365
1366static inline void
1367sk_memory_allocated_sub(struct sock *sk, int amt)
1368{
1369 atomic_long_sub(amt, sk->sk_prot->memory_allocated);
1370}
1371
1372#define SK_ALLOC_PERCPU_COUNTER_BATCH 16
1373
1374static inline void sk_sockets_allocated_dec(struct sock *sk)
1375{
1376 percpu_counter_add_batch(sk->sk_prot->sockets_allocated, -1,
1377 SK_ALLOC_PERCPU_COUNTER_BATCH);
1378}
1379
1380static inline void sk_sockets_allocated_inc(struct sock *sk)
1381{
1382 percpu_counter_add_batch(sk->sk_prot->sockets_allocated, 1,
1383 SK_ALLOC_PERCPU_COUNTER_BATCH);
1384}
1385
1386static inline u64
1387sk_sockets_allocated_read_positive(struct sock *sk)
1388{
1389 return percpu_counter_read_positive(sk->sk_prot->sockets_allocated);
1390}
1391
1392static inline int
1393proto_sockets_allocated_sum_positive(struct proto *prot)
1394{
1395 return percpu_counter_sum_positive(prot->sockets_allocated);
1396}
1397
1398static inline long
1399proto_memory_allocated(struct proto *prot)
1400{
1401 return atomic_long_read(prot->memory_allocated);
1402}
1403
1404static inline bool
1405proto_memory_pressure(struct proto *prot)
1406{
1407 if (!prot->memory_pressure)
1408 return false;
1409 return !!*prot->memory_pressure;
1410}
1411
1412
1413#ifdef CONFIG_PROC_FS
1414
1415void sock_prot_inuse_add(struct net *net, struct proto *prot, int inc);
1416int sock_prot_inuse_get(struct net *net, struct proto *proto);
1417int sock_inuse_get(struct net *net);
1418#else
1419static inline void sock_prot_inuse_add(struct net *net, struct proto *prot,
1420 int inc)
1421{
1422}
1423#endif
1424
1425
1426
1427
1428
1429static inline int __sk_prot_rehash(struct sock *sk)
1430{
1431 sk->sk_prot->unhash(sk);
1432 return sk->sk_prot->hash(sk);
1433}
1434
1435
1436#define SOCK_DESTROY_TIME (10*HZ)
1437
1438
1439#define PROT_SOCK 1024
1440
1441#define SHUTDOWN_MASK 3
1442#define RCV_SHUTDOWN 1
1443#define SEND_SHUTDOWN 2
1444
1445#define SOCK_BINDADDR_LOCK 4
1446#define SOCK_BINDPORT_LOCK 8
1447
1448struct socket_alloc {
1449 struct socket socket;
1450 struct inode vfs_inode;
1451};
1452
1453static inline struct socket *SOCKET_I(struct inode *inode)
1454{
1455 return &container_of(inode, struct socket_alloc, vfs_inode)->socket;
1456}
1457
1458static inline struct inode *SOCK_INODE(struct socket *socket)
1459{
1460 return &container_of(socket, struct socket_alloc, socket)->vfs_inode;
1461}
1462
1463
1464
1465
1466int __sk_mem_raise_allocated(struct sock *sk, int size, int amt, int kind);
1467int __sk_mem_schedule(struct sock *sk, int size, int kind);
1468void __sk_mem_reduce_allocated(struct sock *sk, int amount);
1469void __sk_mem_reclaim(struct sock *sk, int amount);
1470
1471
1472
1473
1474#define SK_MEM_QUANTUM 4096
1475#define SK_MEM_QUANTUM_SHIFT ilog2(SK_MEM_QUANTUM)
1476#define SK_MEM_SEND 0
1477#define SK_MEM_RECV 1
1478
1479
1480static inline long sk_prot_mem_limits(const struct sock *sk, int index)
1481{
1482 long val = sk->sk_prot->sysctl_mem[index];
1483
1484#if PAGE_SIZE > SK_MEM_QUANTUM
1485 val <<= PAGE_SHIFT - SK_MEM_QUANTUM_SHIFT;
1486#elif PAGE_SIZE < SK_MEM_QUANTUM
1487 val >>= SK_MEM_QUANTUM_SHIFT - PAGE_SHIFT;
1488#endif
1489 return val;
1490}
1491
1492static inline int sk_mem_pages(int amt)
1493{
1494 return (amt + SK_MEM_QUANTUM - 1) >> SK_MEM_QUANTUM_SHIFT;
1495}
1496
1497static inline bool sk_has_account(struct sock *sk)
1498{
1499
1500 return !!sk->sk_prot->memory_allocated;
1501}
1502
1503static inline bool sk_wmem_schedule(struct sock *sk, int size)
1504{
1505 if (!sk_has_account(sk))
1506 return true;
1507 return size <= sk->sk_forward_alloc ||
1508 __sk_mem_schedule(sk, size, SK_MEM_SEND);
1509}
1510
1511static inline bool
1512sk_rmem_schedule(struct sock *sk, struct sk_buff *skb, int size)
1513{
1514 if (!sk_has_account(sk))
1515 return true;
1516 return size <= sk->sk_forward_alloc ||
1517 __sk_mem_schedule(sk, size, SK_MEM_RECV) ||
1518 skb_pfmemalloc(skb);
1519}
1520
1521static inline void sk_mem_reclaim(struct sock *sk)
1522{
1523 if (!sk_has_account(sk))
1524 return;
1525 if (sk->sk_forward_alloc >= SK_MEM_QUANTUM)
1526 __sk_mem_reclaim(sk, sk->sk_forward_alloc);
1527}
1528
1529static inline void sk_mem_reclaim_partial(struct sock *sk)
1530{
1531 if (!sk_has_account(sk))
1532 return;
1533 if (sk->sk_forward_alloc > SK_MEM_QUANTUM)
1534 __sk_mem_reclaim(sk, sk->sk_forward_alloc - 1);
1535}
1536
1537static inline void sk_mem_charge(struct sock *sk, int size)
1538{
1539 if (!sk_has_account(sk))
1540 return;
1541 sk->sk_forward_alloc -= size;
1542}
1543
1544static inline void sk_mem_uncharge(struct sock *sk, int size)
1545{
1546 if (!sk_has_account(sk))
1547 return;
1548 sk->sk_forward_alloc += size;
1549
1550
1551
1552
1553
1554
1555
1556
1557 if (unlikely(sk->sk_forward_alloc >= 1 << 21))
1558 __sk_mem_reclaim(sk, 1 << 20);
1559}
1560
1561DECLARE_STATIC_KEY_FALSE(tcp_tx_skb_cache_key);
1562static inline void sk_wmem_free_skb(struct sock *sk, struct sk_buff *skb)
1563{
1564 sk_wmem_queued_add(sk, -skb->truesize);
1565 sk_mem_uncharge(sk, skb->truesize);
1566 if (static_branch_unlikely(&tcp_tx_skb_cache_key) &&
1567 !sk->sk_tx_skb_cache && !skb_cloned(skb)) {
1568 skb_ext_reset(skb);
1569 skb_zcopy_clear(skb, true);
1570 sk->sk_tx_skb_cache = skb;
1571 return;
1572 }
1573 __kfree_skb(skb);
1574}
1575
1576static inline void sock_release_ownership(struct sock *sk)
1577{
1578 if (sk->sk_lock.owned) {
1579 sk->sk_lock.owned = 0;
1580
1581
1582 mutex_release(&sk->sk_lock.dep_map, _RET_IP_);
1583 }
1584}
1585
1586
1587
1588
1589
1590
1591
1592
1593#define sock_lock_init_class_and_name(sk, sname, skey, name, key) \
1594do { \
1595 sk->sk_lock.owned = 0; \
1596 init_waitqueue_head(&sk->sk_lock.wq); \
1597 spin_lock_init(&(sk)->sk_lock.slock); \
1598 debug_check_no_locks_freed((void *)&(sk)->sk_lock, \
1599 sizeof((sk)->sk_lock)); \
1600 lockdep_set_class_and_name(&(sk)->sk_lock.slock, \
1601 (skey), (sname)); \
1602 lockdep_init_map(&(sk)->sk_lock.dep_map, (name), (key), 0); \
1603} while (0)
1604
1605static inline bool lockdep_sock_is_held(const struct sock *sk)
1606{
1607 return lockdep_is_held(&sk->sk_lock) ||
1608 lockdep_is_held(&sk->sk_lock.slock);
1609}
1610
1611void lock_sock_nested(struct sock *sk, int subclass);
1612
1613static inline void lock_sock(struct sock *sk)
1614{
1615 lock_sock_nested(sk, 0);
1616}
1617
1618void __lock_sock(struct sock *sk);
1619void __release_sock(struct sock *sk);
1620void release_sock(struct sock *sk);
1621
1622
1623#define bh_lock_sock(__sk) spin_lock(&((__sk)->sk_lock.slock))
1624#define bh_lock_sock_nested(__sk) \
1625 spin_lock_nested(&((__sk)->sk_lock.slock), \
1626 SINGLE_DEPTH_NESTING)
1627#define bh_unlock_sock(__sk) spin_unlock(&((__sk)->sk_lock.slock))
1628
1629bool __lock_sock_fast(struct sock *sk) __acquires(&sk->sk_lock.slock);
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644static inline bool lock_sock_fast(struct sock *sk)
1645{
1646
1647 mutex_acquire(&sk->sk_lock.dep_map, 0, 0, _RET_IP_);
1648
1649 return __lock_sock_fast(sk);
1650}
1651
1652
1653static inline bool lock_sock_fast_nested(struct sock *sk)
1654{
1655 mutex_acquire(&sk->sk_lock.dep_map, SINGLE_DEPTH_NESTING, 0, _RET_IP_);
1656
1657 return __lock_sock_fast(sk);
1658}
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668static inline void unlock_sock_fast(struct sock *sk, bool slow)
1669 __releases(&sk->sk_lock.slock)
1670{
1671 if (slow) {
1672 release_sock(sk);
1673 __release(&sk->sk_lock.slock);
1674 } else {
1675 mutex_release(&sk->sk_lock.dep_map, _RET_IP_);
1676 spin_unlock_bh(&sk->sk_lock.slock);
1677 }
1678}
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694static inline void sock_owned_by_me(const struct sock *sk)
1695{
1696#ifdef CONFIG_LOCKDEP
1697 WARN_ON_ONCE(!lockdep_sock_is_held(sk) && debug_locks);
1698#endif
1699}
1700
1701static inline bool sock_owned_by_user(const struct sock *sk)
1702{
1703 sock_owned_by_me(sk);
1704 return sk->sk_lock.owned;
1705}
1706
1707static inline bool sock_owned_by_user_nocheck(const struct sock *sk)
1708{
1709 return sk->sk_lock.owned;
1710}
1711
1712
1713static inline bool sock_allow_reclassification(const struct sock *csk)
1714{
1715 struct sock *sk = (struct sock *)csk;
1716
1717 return !sk->sk_lock.owned && !spin_is_locked(&sk->sk_lock.slock);
1718}
1719
1720struct sock *sk_alloc(struct net *net, int family, gfp_t priority,
1721 struct proto *prot, int kern);
1722void sk_free(struct sock *sk);
1723void sk_destruct(struct sock *sk);
1724struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority);
1725void sk_free_unlock_clone(struct sock *sk);
1726
1727struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force,
1728 gfp_t priority);
1729void __sock_wfree(struct sk_buff *skb);
1730void sock_wfree(struct sk_buff *skb);
1731struct sk_buff *sock_omalloc(struct sock *sk, unsigned long size,
1732 gfp_t priority);
1733void skb_orphan_partial(struct sk_buff *skb);
1734void sock_rfree(struct sk_buff *skb);
1735void sock_efree(struct sk_buff *skb);
1736#ifdef CONFIG_INET
1737void sock_edemux(struct sk_buff *skb);
1738void sock_pfree(struct sk_buff *skb);
1739#else
1740#define sock_edemux sock_efree
1741#endif
1742
1743int sock_setsockopt(struct socket *sock, int level, int op,
1744 sockptr_t optval, unsigned int optlen);
1745
1746int sock_getsockopt(struct socket *sock, int level, int op,
1747 char __user *optval, int __user *optlen);
1748int sock_gettstamp(struct socket *sock, void __user *userstamp,
1749 bool timeval, bool time32);
1750struct sk_buff *sock_alloc_send_skb(struct sock *sk, unsigned long size,
1751 int noblock, int *errcode);
1752struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len,
1753 unsigned long data_len, int noblock,
1754 int *errcode, int max_page_order);
1755void *sock_kmalloc(struct sock *sk, int size, gfp_t priority);
1756void sock_kfree_s(struct sock *sk, void *mem, int size);
1757void sock_kzfree_s(struct sock *sk, void *mem, int size);
1758void sk_send_sigurg(struct sock *sk);
1759
1760struct sockcm_cookie {
1761 u64 transmit_time;
1762 u32 mark;
1763 u16 tsflags;
1764};
1765
1766static inline void sockcm_init(struct sockcm_cookie *sockc,
1767 const struct sock *sk)
1768{
1769 *sockc = (struct sockcm_cookie) { .tsflags = sk->sk_tsflags };
1770}
1771
1772int __sock_cmsg_send(struct sock *sk, struct msghdr *msg, struct cmsghdr *cmsg,
1773 struct sockcm_cookie *sockc);
1774int sock_cmsg_send(struct sock *sk, struct msghdr *msg,
1775 struct sockcm_cookie *sockc);
1776
1777
1778
1779
1780
1781int sock_no_bind(struct socket *, struct sockaddr *, int);
1782int sock_no_connect(struct socket *, struct sockaddr *, int, int);
1783int sock_no_socketpair(struct socket *, struct socket *);
1784int sock_no_accept(struct socket *, struct socket *, int, bool);
1785int sock_no_getname(struct socket *, struct sockaddr *, int);
1786int sock_no_ioctl(struct socket *, unsigned int, unsigned long);
1787int sock_no_listen(struct socket *, int);
1788int sock_no_shutdown(struct socket *, int);
1789int sock_no_sendmsg(struct socket *, struct msghdr *, size_t);
1790int sock_no_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t len);
1791int sock_no_recvmsg(struct socket *, struct msghdr *, size_t, int);
1792int sock_no_mmap(struct file *file, struct socket *sock,
1793 struct vm_area_struct *vma);
1794ssize_t sock_no_sendpage(struct socket *sock, struct page *page, int offset,
1795 size_t size, int flags);
1796ssize_t sock_no_sendpage_locked(struct sock *sk, struct page *page,
1797 int offset, size_t size, int flags);
1798
1799
1800
1801
1802
1803int sock_common_getsockopt(struct socket *sock, int level, int optname,
1804 char __user *optval, int __user *optlen);
1805int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
1806 int flags);
1807int sock_common_setsockopt(struct socket *sock, int level, int optname,
1808 sockptr_t optval, unsigned int optlen);
1809
1810void sk_common_release(struct sock *sk);
1811
1812
1813
1814
1815
1816
1817void sock_init_data(struct socket *sock, struct sock *sk);
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845static inline void sock_put(struct sock *sk)
1846{
1847 if (refcount_dec_and_test(&sk->sk_refcnt))
1848 sk_free(sk);
1849}
1850
1851
1852
1853void sock_gen_put(struct sock *sk);
1854
1855int __sk_receive_skb(struct sock *sk, struct sk_buff *skb, const int nested,
1856 unsigned int trim_cap, bool refcounted);
1857static inline int sk_receive_skb(struct sock *sk, struct sk_buff *skb,
1858 const int nested)
1859{
1860 return __sk_receive_skb(sk, skb, nested, 1, true);
1861}
1862
1863static inline void sk_tx_queue_set(struct sock *sk, int tx_queue)
1864{
1865
1866 if (WARN_ON_ONCE((unsigned short)tx_queue >= USHRT_MAX))
1867 return;
1868 sk->sk_tx_queue_mapping = tx_queue;
1869}
1870
1871#define NO_QUEUE_MAPPING USHRT_MAX
1872
1873static inline void sk_tx_queue_clear(struct sock *sk)
1874{
1875 sk->sk_tx_queue_mapping = NO_QUEUE_MAPPING;
1876}
1877
1878static inline int sk_tx_queue_get(const struct sock *sk)
1879{
1880 if (sk && sk->sk_tx_queue_mapping != NO_QUEUE_MAPPING)
1881 return sk->sk_tx_queue_mapping;
1882
1883 return -1;
1884}
1885
1886static inline void sk_rx_queue_set(struct sock *sk, const struct sk_buff *skb)
1887{
1888#ifdef CONFIG_SOCK_RX_QUEUE_MAPPING
1889 if (skb_rx_queue_recorded(skb)) {
1890 u16 rx_queue = skb_get_rx_queue(skb);
1891
1892 if (WARN_ON_ONCE(rx_queue == NO_QUEUE_MAPPING))
1893 return;
1894
1895 sk->sk_rx_queue_mapping = rx_queue;
1896 }
1897#endif
1898}
1899
1900static inline void sk_rx_queue_clear(struct sock *sk)
1901{
1902#ifdef CONFIG_SOCK_RX_QUEUE_MAPPING
1903 sk->sk_rx_queue_mapping = NO_QUEUE_MAPPING;
1904#endif
1905}
1906
1907static inline int sk_rx_queue_get(const struct sock *sk)
1908{
1909#ifdef CONFIG_SOCK_RX_QUEUE_MAPPING
1910 if (sk && sk->sk_rx_queue_mapping != NO_QUEUE_MAPPING)
1911 return sk->sk_rx_queue_mapping;
1912#endif
1913
1914 return -1;
1915}
1916
1917static inline void sk_set_socket(struct sock *sk, struct socket *sock)
1918{
1919 sk->sk_socket = sock;
1920}
1921
1922static inline wait_queue_head_t *sk_sleep(struct sock *sk)
1923{
1924 BUILD_BUG_ON(offsetof(struct socket_wq, wait) != 0);
1925 return &rcu_dereference_raw(sk->sk_wq)->wait;
1926}
1927
1928
1929
1930
1931
1932
1933
1934static inline void sock_orphan(struct sock *sk)
1935{
1936 write_lock_bh(&sk->sk_callback_lock);
1937 sock_set_flag(sk, SOCK_DEAD);
1938 sk_set_socket(sk, NULL);
1939 sk->sk_wq = NULL;
1940 write_unlock_bh(&sk->sk_callback_lock);
1941}
1942
1943static inline void sock_graft(struct sock *sk, struct socket *parent)
1944{
1945 WARN_ON(parent->sk);
1946 write_lock_bh(&sk->sk_callback_lock);
1947 rcu_assign_pointer(sk->sk_wq, &parent->wq);
1948 parent->sk = sk;
1949 sk_set_socket(sk, parent);
1950 sk->sk_uid = SOCK_INODE(parent)->i_uid;
1951 security_sock_graft(sk, parent);
1952 write_unlock_bh(&sk->sk_callback_lock);
1953}
1954
1955kuid_t sock_i_uid(struct sock *sk);
1956unsigned long sock_i_ino(struct sock *sk);
1957
1958static inline kuid_t sock_net_uid(const struct net *net, const struct sock *sk)
1959{
1960 return sk ? sk->sk_uid : make_kuid(net->user_ns, 0);
1961}
1962
1963static inline u32 net_tx_rndhash(void)
1964{
1965 u32 v = prandom_u32();
1966
1967 return v ?: 1;
1968}
1969
1970static inline void sk_set_txhash(struct sock *sk)
1971{
1972
1973 WRITE_ONCE(sk->sk_txhash, net_tx_rndhash());
1974}
1975
1976static inline bool sk_rethink_txhash(struct sock *sk)
1977{
1978 if (sk->sk_txhash) {
1979 sk_set_txhash(sk);
1980 return true;
1981 }
1982 return false;
1983}
1984
1985static inline struct dst_entry *
1986__sk_dst_get(struct sock *sk)
1987{
1988 return rcu_dereference_check(sk->sk_dst_cache,
1989 lockdep_sock_is_held(sk));
1990}
1991
1992static inline struct dst_entry *
1993sk_dst_get(struct sock *sk)
1994{
1995 struct dst_entry *dst;
1996
1997 rcu_read_lock();
1998 dst = rcu_dereference(sk->sk_dst_cache);
1999 if (dst && !atomic_inc_not_zero(&dst->__refcnt))
2000 dst = NULL;
2001 rcu_read_unlock();
2002 return dst;
2003}
2004
2005static inline void __dst_negative_advice(struct sock *sk)
2006{
2007 struct dst_entry *ndst, *dst = __sk_dst_get(sk);
2008
2009 if (dst && dst->ops->negative_advice) {
2010 ndst = dst->ops->negative_advice(dst);
2011
2012 if (ndst != dst) {
2013 rcu_assign_pointer(sk->sk_dst_cache, ndst);
2014 sk_tx_queue_clear(sk);
2015 sk->sk_dst_pending_confirm = 0;
2016 }
2017 }
2018}
2019
2020static inline void dst_negative_advice(struct sock *sk)
2021{
2022 sk_rethink_txhash(sk);
2023 __dst_negative_advice(sk);
2024}
2025
2026static inline void
2027__sk_dst_set(struct sock *sk, struct dst_entry *dst)
2028{
2029 struct dst_entry *old_dst;
2030
2031 sk_tx_queue_clear(sk);
2032 sk->sk_dst_pending_confirm = 0;
2033 old_dst = rcu_dereference_protected(sk->sk_dst_cache,
2034 lockdep_sock_is_held(sk));
2035 rcu_assign_pointer(sk->sk_dst_cache, dst);
2036 dst_release(old_dst);
2037}
2038
2039static inline void
2040sk_dst_set(struct sock *sk, struct dst_entry *dst)
2041{
2042 struct dst_entry *old_dst;
2043
2044 sk_tx_queue_clear(sk);
2045 sk->sk_dst_pending_confirm = 0;
2046 old_dst = xchg((__force struct dst_entry **)&sk->sk_dst_cache, dst);
2047 dst_release(old_dst);
2048}
2049
2050static inline void
2051__sk_dst_reset(struct sock *sk)
2052{
2053 __sk_dst_set(sk, NULL);
2054}
2055
2056static inline void
2057sk_dst_reset(struct sock *sk)
2058{
2059 sk_dst_set(sk, NULL);
2060}
2061
2062struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie);
2063
2064struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie);
2065
2066static inline void sk_dst_confirm(struct sock *sk)
2067{
2068 if (!READ_ONCE(sk->sk_dst_pending_confirm))
2069 WRITE_ONCE(sk->sk_dst_pending_confirm, 1);
2070}
2071
2072static inline void sock_confirm_neigh(struct sk_buff *skb, struct neighbour *n)
2073{
2074 if (skb_get_dst_pending_confirm(skb)) {
2075 struct sock *sk = skb->sk;
2076 unsigned long now = jiffies;
2077
2078
2079 if (READ_ONCE(n->confirmed) != now)
2080 WRITE_ONCE(n->confirmed, now);
2081 if (sk && READ_ONCE(sk->sk_dst_pending_confirm))
2082 WRITE_ONCE(sk->sk_dst_pending_confirm, 0);
2083 }
2084}
2085
2086bool sk_mc_loop(struct sock *sk);
2087
2088static inline bool sk_can_gso(const struct sock *sk)
2089{
2090 return net_gso_ok(sk->sk_route_caps, sk->sk_gso_type);
2091}
2092
2093void sk_setup_caps(struct sock *sk, struct dst_entry *dst);
2094
2095static inline void sk_nocaps_add(struct sock *sk, netdev_features_t flags)
2096{
2097 sk->sk_route_nocaps |= flags;
2098 sk->sk_route_caps &= ~flags;
2099}
2100
2101static inline int skb_do_copy_data_nocache(struct sock *sk, struct sk_buff *skb,
2102 struct iov_iter *from, char *to,
2103 int copy, int offset)
2104{
2105 if (skb->ip_summed == CHECKSUM_NONE) {
2106 __wsum csum = 0;
2107 if (!csum_and_copy_from_iter_full(to, copy, &csum, from))
2108 return -EFAULT;
2109 skb->csum = csum_block_add(skb->csum, csum, offset);
2110 } else if (sk->sk_route_caps & NETIF_F_NOCACHE_COPY) {
2111 if (!copy_from_iter_full_nocache(to, copy, from))
2112 return -EFAULT;
2113 } else if (!copy_from_iter_full(to, copy, from))
2114 return -EFAULT;
2115
2116 return 0;
2117}
2118
2119static inline int skb_add_data_nocache(struct sock *sk, struct sk_buff *skb,
2120 struct iov_iter *from, int copy)
2121{
2122 int err, offset = skb->len;
2123
2124 err = skb_do_copy_data_nocache(sk, skb, from, skb_put(skb, copy),
2125 copy, offset);
2126 if (err)
2127 __skb_trim(skb, offset);
2128
2129 return err;
2130}
2131
2132static inline int skb_copy_to_page_nocache(struct sock *sk, struct iov_iter *from,
2133 struct sk_buff *skb,
2134 struct page *page,
2135 int off, int copy)
2136{
2137 int err;
2138
2139 err = skb_do_copy_data_nocache(sk, skb, from, page_address(page) + off,
2140 copy, skb->len);
2141 if (err)
2142 return err;
2143
2144 skb->len += copy;
2145 skb->data_len += copy;
2146 skb->truesize += copy;
2147 sk_wmem_queued_add(sk, copy);
2148 sk_mem_charge(sk, copy);
2149 return 0;
2150}
2151
2152
2153
2154
2155
2156
2157
2158static inline int sk_wmem_alloc_get(const struct sock *sk)
2159{
2160 return refcount_read(&sk->sk_wmem_alloc) - 1;
2161}
2162
2163
2164
2165
2166
2167
2168
2169static inline int sk_rmem_alloc_get(const struct sock *sk)
2170{
2171 return atomic_read(&sk->sk_rmem_alloc);
2172}
2173
2174
2175
2176
2177
2178
2179
2180static inline bool sk_has_allocations(const struct sock *sk)
2181{
2182 return sk_wmem_alloc_get(sk) || sk_rmem_alloc_get(sk);
2183}
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216static inline bool skwq_has_sleeper(struct socket_wq *wq)
2217{
2218 return wq && wq_has_sleeper(&wq->wait);
2219}
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229static inline void sock_poll_wait(struct file *filp, struct socket *sock,
2230 poll_table *p)
2231{
2232 if (!poll_does_not_wait(p)) {
2233 poll_wait(filp, &sock->wq.wait, p);
2234
2235
2236
2237
2238
2239 smp_mb();
2240 }
2241}
2242
2243static inline void skb_set_hash_from_sk(struct sk_buff *skb, struct sock *sk)
2244{
2245
2246 u32 txhash = READ_ONCE(sk->sk_txhash);
2247
2248 if (txhash) {
2249 skb->l4_hash = 1;
2250 skb->hash = txhash;
2251 }
2252}
2253
2254void skb_set_owner_w(struct sk_buff *skb, struct sock *sk);
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264static inline void skb_set_owner_r(struct sk_buff *skb, struct sock *sk)
2265{
2266 skb_orphan(skb);
2267 skb->sk = sk;
2268 skb->destructor = sock_rfree;
2269 atomic_add(skb->truesize, &sk->sk_rmem_alloc);
2270 sk_mem_charge(sk, skb->truesize);
2271}
2272
2273static inline __must_check bool skb_set_owner_sk_safe(struct sk_buff *skb, struct sock *sk)
2274{
2275 if (sk && refcount_inc_not_zero(&sk->sk_refcnt)) {
2276 skb_orphan(skb);
2277 skb->destructor = sock_efree;
2278 skb->sk = sk;
2279 return true;
2280 }
2281 return false;
2282}
2283
2284static inline void skb_prepare_for_gro(struct sk_buff *skb)
2285{
2286 if (skb->destructor != sock_wfree) {
2287 skb_orphan(skb);
2288 return;
2289 }
2290 skb->slow_gro = 1;
2291}
2292
2293void sk_reset_timer(struct sock *sk, struct timer_list *timer,
2294 unsigned long expires);
2295
2296void sk_stop_timer(struct sock *sk, struct timer_list *timer);
2297
2298void sk_stop_timer_sync(struct sock *sk, struct timer_list *timer);
2299
2300int __sk_queue_drop_skb(struct sock *sk, struct sk_buff_head *sk_queue,
2301 struct sk_buff *skb, unsigned int flags,
2302 void (*destructor)(struct sock *sk,
2303 struct sk_buff *skb));
2304int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb);
2305int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb);
2306
2307int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb);
2308struct sk_buff *sock_dequeue_err_skb(struct sock *sk);
2309
2310
2311
2312
2313
2314static inline int sock_error(struct sock *sk)
2315{
2316 int err;
2317
2318
2319
2320
2321 if (likely(data_race(!sk->sk_err)))
2322 return 0;
2323
2324 err = xchg(&sk->sk_err, 0);
2325 return -err;
2326}
2327
2328void sk_error_report(struct sock *sk);
2329
2330static inline unsigned long sock_wspace(struct sock *sk)
2331{
2332 int amt = 0;
2333
2334 if (!(sk->sk_shutdown & SEND_SHUTDOWN)) {
2335 amt = sk->sk_sndbuf - refcount_read(&sk->sk_wmem_alloc);
2336 if (amt < 0)
2337 amt = 0;
2338 }
2339 return amt;
2340}
2341
2342
2343
2344
2345
2346static inline void sk_set_bit(int nr, struct sock *sk)
2347{
2348 if ((nr == SOCKWQ_ASYNC_NOSPACE || nr == SOCKWQ_ASYNC_WAITDATA) &&
2349 !sock_flag(sk, SOCK_FASYNC))
2350 return;
2351
2352 set_bit(nr, &sk->sk_wq_raw->flags);
2353}
2354
2355static inline void sk_clear_bit(int nr, struct sock *sk)
2356{
2357 if ((nr == SOCKWQ_ASYNC_NOSPACE || nr == SOCKWQ_ASYNC_WAITDATA) &&
2358 !sock_flag(sk, SOCK_FASYNC))
2359 return;
2360
2361 clear_bit(nr, &sk->sk_wq_raw->flags);
2362}
2363
2364static inline void sk_wake_async(const struct sock *sk, int how, int band)
2365{
2366 if (sock_flag(sk, SOCK_FASYNC)) {
2367 rcu_read_lock();
2368 sock_wake_async(rcu_dereference(sk->sk_wq), how, band);
2369 rcu_read_unlock();
2370 }
2371}
2372
2373
2374
2375
2376
2377
2378#define TCP_SKB_MIN_TRUESIZE (2048 + SKB_DATA_ALIGN(sizeof(struct sk_buff)))
2379
2380#define SOCK_MIN_SNDBUF (TCP_SKB_MIN_TRUESIZE * 2)
2381#define SOCK_MIN_RCVBUF TCP_SKB_MIN_TRUESIZE
2382
2383static inline void sk_stream_moderate_sndbuf(struct sock *sk)
2384{
2385 u32 val;
2386
2387 if (sk->sk_userlocks & SOCK_SNDBUF_LOCK)
2388 return;
2389
2390 val = min(sk->sk_sndbuf, sk->sk_wmem_queued >> 1);
2391
2392 WRITE_ONCE(sk->sk_sndbuf, max_t(u32, val, SOCK_MIN_SNDBUF));
2393}
2394
2395struct sk_buff *sk_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp,
2396 bool force_schedule);
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413static inline struct page_frag *sk_page_frag(struct sock *sk)
2414{
2415 if (gfpflags_normal_context(sk->sk_allocation))
2416 return ¤t->task_frag;
2417
2418 return &sk->sk_frag;
2419}
2420
2421bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag);
2422
2423
2424
2425
2426static inline bool sock_writeable(const struct sock *sk)
2427{
2428 return refcount_read(&sk->sk_wmem_alloc) < (READ_ONCE(sk->sk_sndbuf) >> 1);
2429}
2430
2431static inline gfp_t gfp_any(void)
2432{
2433 return in_softirq() ? GFP_ATOMIC : GFP_KERNEL;
2434}
2435
2436static inline gfp_t gfp_memcg_charge(void)
2437{
2438 return in_softirq() ? GFP_NOWAIT : GFP_KERNEL;
2439}
2440
2441static inline long sock_rcvtimeo(const struct sock *sk, bool noblock)
2442{
2443 return noblock ? 0 : sk->sk_rcvtimeo;
2444}
2445
2446static inline long sock_sndtimeo(const struct sock *sk, bool noblock)
2447{
2448 return noblock ? 0 : sk->sk_sndtimeo;
2449}
2450
2451static inline int sock_rcvlowat(const struct sock *sk, int waitall, int len)
2452{
2453 int v = waitall ? len : min_t(int, READ_ONCE(sk->sk_rcvlowat), len);
2454
2455 return v ?: 1;
2456}
2457
2458
2459
2460
2461static inline int sock_intr_errno(long timeo)
2462{
2463 return timeo == MAX_SCHEDULE_TIMEOUT ? -ERESTARTSYS : -EINTR;
2464}
2465
2466struct sock_skb_cb {
2467 u32 dropcount;
2468};
2469
2470
2471
2472
2473
2474#define SOCK_SKB_CB_OFFSET ((sizeof_field(struct sk_buff, cb) - \
2475 sizeof(struct sock_skb_cb)))
2476
2477#define SOCK_SKB_CB(__skb) ((struct sock_skb_cb *)((__skb)->cb + \
2478 SOCK_SKB_CB_OFFSET))
2479
2480#define sock_skb_cb_check_size(size) \
2481 BUILD_BUG_ON((size) > SOCK_SKB_CB_OFFSET)
2482
2483static inline void
2484sock_skb_set_dropcount(const struct sock *sk, struct sk_buff *skb)
2485{
2486 SOCK_SKB_CB(skb)->dropcount = sock_flag(sk, SOCK_RXQ_OVFL) ?
2487 atomic_read(&sk->sk_drops) : 0;
2488}
2489
2490static inline void sk_drops_add(struct sock *sk, const struct sk_buff *skb)
2491{
2492 int segs = max_t(u16, 1, skb_shinfo(skb)->gso_segs);
2493
2494 atomic_add(segs, &sk->sk_drops);
2495}
2496
2497static inline ktime_t sock_read_timestamp(struct sock *sk)
2498{
2499#if BITS_PER_LONG==32
2500 unsigned int seq;
2501 ktime_t kt;
2502
2503 do {
2504 seq = read_seqbegin(&sk->sk_stamp_seq);
2505 kt = sk->sk_stamp;
2506 } while (read_seqretry(&sk->sk_stamp_seq, seq));
2507
2508 return kt;
2509#else
2510 return READ_ONCE(sk->sk_stamp);
2511#endif
2512}
2513
2514static inline void sock_write_timestamp(struct sock *sk, ktime_t kt)
2515{
2516#if BITS_PER_LONG==32
2517 write_seqlock(&sk->sk_stamp_seq);
2518 sk->sk_stamp = kt;
2519 write_sequnlock(&sk->sk_stamp_seq);
2520#else
2521 WRITE_ONCE(sk->sk_stamp, kt);
2522#endif
2523}
2524
2525void __sock_recv_timestamp(struct msghdr *msg, struct sock *sk,
2526 struct sk_buff *skb);
2527void __sock_recv_wifi_status(struct msghdr *msg, struct sock *sk,
2528 struct sk_buff *skb);
2529
2530static inline void
2531sock_recv_timestamp(struct msghdr *msg, struct sock *sk, struct sk_buff *skb)
2532{
2533 ktime_t kt = skb->tstamp;
2534 struct skb_shared_hwtstamps *hwtstamps = skb_hwtstamps(skb);
2535
2536
2537
2538
2539
2540
2541
2542 if (sock_flag(sk, SOCK_RCVTSTAMP) ||
2543 (sk->sk_tsflags & SOF_TIMESTAMPING_RX_SOFTWARE) ||
2544 (kt && sk->sk_tsflags & SOF_TIMESTAMPING_SOFTWARE) ||
2545 (hwtstamps->hwtstamp &&
2546 (sk->sk_tsflags & SOF_TIMESTAMPING_RAW_HARDWARE)))
2547 __sock_recv_timestamp(msg, sk, skb);
2548 else
2549 sock_write_timestamp(sk, kt);
2550
2551 if (sock_flag(sk, SOCK_WIFI_STATUS) && skb->wifi_acked_valid)
2552 __sock_recv_wifi_status(msg, sk, skb);
2553}
2554
2555void __sock_recv_ts_and_drops(struct msghdr *msg, struct sock *sk,
2556 struct sk_buff *skb);
2557
2558#define SK_DEFAULT_STAMP (-1L * NSEC_PER_SEC)
2559static inline void sock_recv_ts_and_drops(struct msghdr *msg, struct sock *sk,
2560 struct sk_buff *skb)
2561{
2562#define FLAGS_TS_OR_DROPS ((1UL << SOCK_RXQ_OVFL) | \
2563 (1UL << SOCK_RCVTSTAMP))
2564#define TSFLAGS_ANY (SOF_TIMESTAMPING_SOFTWARE | \
2565 SOF_TIMESTAMPING_RAW_HARDWARE)
2566
2567 if (sk->sk_flags & FLAGS_TS_OR_DROPS || sk->sk_tsflags & TSFLAGS_ANY)
2568 __sock_recv_ts_and_drops(msg, sk, skb);
2569 else if (unlikely(sock_flag(sk, SOCK_TIMESTAMP)))
2570 sock_write_timestamp(sk, skb->tstamp);
2571 else if (unlikely(sk->sk_stamp == SK_DEFAULT_STAMP))
2572 sock_write_timestamp(sk, 0);
2573}
2574
2575void __sock_tx_timestamp(__u16 tsflags, __u8 *tx_flags);
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586static inline void _sock_tx_timestamp(struct sock *sk, __u16 tsflags,
2587 __u8 *tx_flags, __u32 *tskey)
2588{
2589 if (unlikely(tsflags)) {
2590 __sock_tx_timestamp(tsflags, tx_flags);
2591 if (tsflags & SOF_TIMESTAMPING_OPT_ID && tskey &&
2592 tsflags & SOF_TIMESTAMPING_TX_RECORD_MASK)
2593 *tskey = sk->sk_tskey++;
2594 }
2595 if (unlikely(sock_flag(sk, SOCK_WIFI_STATUS)))
2596 *tx_flags |= SKBTX_WIFI_STATUS;
2597}
2598
2599static inline void sock_tx_timestamp(struct sock *sk, __u16 tsflags,
2600 __u8 *tx_flags)
2601{
2602 _sock_tx_timestamp(sk, tsflags, tx_flags, NULL);
2603}
2604
2605static inline void skb_setup_tx_timestamp(struct sk_buff *skb, __u16 tsflags)
2606{
2607 _sock_tx_timestamp(skb->sk, tsflags, &skb_shinfo(skb)->tx_flags,
2608 &skb_shinfo(skb)->tskey);
2609}
2610
2611DECLARE_STATIC_KEY_FALSE(tcp_rx_skb_cache_key);
2612
2613
2614
2615
2616
2617
2618
2619
2620static inline void sk_eat_skb(struct sock *sk, struct sk_buff *skb)
2621{
2622 __skb_unlink(skb, &sk->sk_receive_queue);
2623 if (static_branch_unlikely(&tcp_rx_skb_cache_key) &&
2624 !sk->sk_rx_skb_cache) {
2625 sk->sk_rx_skb_cache = skb;
2626 skb_orphan(skb);
2627 return;
2628 }
2629 __kfree_skb(skb);
2630}
2631
2632static inline
2633struct net *sock_net(const struct sock *sk)
2634{
2635 return read_pnet(&sk->sk_net);
2636}
2637
2638static inline
2639void sock_net_set(struct sock *sk, struct net *net)
2640{
2641 write_pnet(&sk->sk_net, net);
2642}
2643
2644static inline bool
2645skb_sk_is_prefetched(struct sk_buff *skb)
2646{
2647#ifdef CONFIG_INET
2648 return skb->destructor == sock_pfree;
2649#else
2650 return false;
2651#endif
2652}
2653
2654
2655
2656
2657static inline bool sk_fullsock(const struct sock *sk)
2658{
2659 return (1 << sk->sk_state) & ~(TCPF_TIME_WAIT | TCPF_NEW_SYN_RECV);
2660}
2661
2662static inline bool
2663sk_is_refcounted(struct sock *sk)
2664{
2665
2666 return !sk_fullsock(sk) || !sock_flag(sk, SOCK_RCU_FREE);
2667}
2668
2669
2670
2671
2672
2673
2674static inline struct sock *
2675skb_steal_sock(struct sk_buff *skb, bool *refcounted)
2676{
2677 if (skb->sk) {
2678 struct sock *sk = skb->sk;
2679
2680 *refcounted = true;
2681 if (skb_sk_is_prefetched(skb))
2682 *refcounted = sk_is_refcounted(sk);
2683 skb->destructor = NULL;
2684 skb->sk = NULL;
2685 return sk;
2686 }
2687 *refcounted = false;
2688 return NULL;
2689}
2690
2691
2692
2693
2694
2695static inline struct sk_buff *sk_validate_xmit_skb(struct sk_buff *skb,
2696 struct net_device *dev)
2697{
2698#ifdef CONFIG_SOCK_VALIDATE_XMIT
2699 struct sock *sk = skb->sk;
2700
2701 if (sk && sk_fullsock(sk) && sk->sk_validate_xmit_skb) {
2702 skb = sk->sk_validate_xmit_skb(sk, dev, skb);
2703#ifdef CONFIG_TLS_DEVICE
2704 } else if (unlikely(skb->decrypted)) {
2705 pr_warn_ratelimited("unencrypted skb with no associated socket - dropping\n");
2706 kfree_skb(skb);
2707 skb = NULL;
2708#endif
2709 }
2710#endif
2711
2712 return skb;
2713}
2714
2715
2716
2717
2718static inline bool sk_listener(const struct sock *sk)
2719{
2720 return (1 << sk->sk_state) & (TCPF_LISTEN | TCPF_NEW_SYN_RECV);
2721}
2722
2723void sock_enable_timestamp(struct sock *sk, enum sock_flags flag);
2724int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len, int level,
2725 int type);
2726
2727bool sk_ns_capable(const struct sock *sk,
2728 struct user_namespace *user_ns, int cap);
2729bool sk_capable(const struct sock *sk, int cap);
2730bool sk_net_capable(const struct sock *sk, int cap);
2731
2732void sk_get_meminfo(const struct sock *sk, u32 *meminfo);
2733
2734
2735
2736
2737
2738
2739#define _SK_MEM_PACKETS 256
2740#define _SK_MEM_OVERHEAD SKB_TRUESIZE(256)
2741#define SK_WMEM_MAX (_SK_MEM_OVERHEAD * _SK_MEM_PACKETS)
2742#define SK_RMEM_MAX (_SK_MEM_OVERHEAD * _SK_MEM_PACKETS)
2743
2744extern __u32 sysctl_wmem_max;
2745extern __u32 sysctl_rmem_max;
2746
2747extern int sysctl_tstamp_allow_data;
2748extern int sysctl_optmem_max;
2749
2750extern __u32 sysctl_wmem_default;
2751extern __u32 sysctl_rmem_default;
2752
2753#define SKB_FRAG_PAGE_ORDER get_order(32768)
2754DECLARE_STATIC_KEY_FALSE(net_high_order_alloc_disable_key);
2755
2756static inline int sk_get_wmem0(const struct sock *sk, const struct proto *proto)
2757{
2758
2759 if (proto->sysctl_wmem_offset)
2760 return *(int *)((void *)sock_net(sk) + proto->sysctl_wmem_offset);
2761
2762 return *proto->sysctl_wmem;
2763}
2764
2765static inline int sk_get_rmem0(const struct sock *sk, const struct proto *proto)
2766{
2767
2768 if (proto->sysctl_rmem_offset)
2769 return *(int *)((void *)sock_net(sk) + proto->sysctl_rmem_offset);
2770
2771 return *proto->sysctl_rmem;
2772}
2773
2774
2775
2776
2777
2778static inline void sk_pacing_shift_update(struct sock *sk, int val)
2779{
2780 if (!sk || !sk_fullsock(sk) || READ_ONCE(sk->sk_pacing_shift) == val)
2781 return;
2782 WRITE_ONCE(sk->sk_pacing_shift, val);
2783}
2784
2785
2786
2787
2788
2789
2790static inline bool sk_dev_equal_l3scope(struct sock *sk, int dif)
2791{
2792 int mdif;
2793
2794 if (!sk->sk_bound_dev_if || sk->sk_bound_dev_if == dif)
2795 return true;
2796
2797 mdif = l3mdev_master_ifindex_by_index(sock_net(sk), dif);
2798 if (mdif && mdif == sk->sk_bound_dev_if)
2799 return true;
2800
2801 return false;
2802}
2803
2804void sock_def_readable(struct sock *sk);
2805
2806int sock_bindtoindex(struct sock *sk, int ifindex, bool lock_sk);
2807void sock_set_timestamp(struct sock *sk, int optname, bool valbool);
2808int sock_set_timestamping(struct sock *sk, int optname,
2809 struct so_timestamping timestamping);
2810
2811void sock_enable_timestamps(struct sock *sk);
2812void sock_no_linger(struct sock *sk);
2813void sock_set_keepalive(struct sock *sk);
2814void sock_set_priority(struct sock *sk, u32 priority);
2815void sock_set_rcvbuf(struct sock *sk, int val);
2816void sock_set_mark(struct sock *sk, u32 val);
2817void sock_set_reuseaddr(struct sock *sk);
2818void sock_set_reuseport(struct sock *sk);
2819void sock_set_sndtimeo(struct sock *sk, s64 secs);
2820
2821int sock_bind_add(struct sock *sk, struct sockaddr *addr, int addr_len);
2822
2823static inline bool sk_is_readable(struct sock *sk)
2824{
2825 if (sk->sk_prot->sock_is_readable)
2826 return sk->sk_prot->sock_is_readable(sk);
2827 return false;
2828}
2829#endif
2830