1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35#ifndef _SOCK_H
36#define _SOCK_H
37
38#include <linux/hardirq.h>
39#include <linux/kernel.h>
40#include <linux/list.h>
41#include <linux/list_nulls.h>
42#include <linux/timer.h>
43#include <linux/cache.h>
44#include <linux/bitops.h>
45#include <linux/lockdep.h>
46#include <linux/netdevice.h>
47#include <linux/skbuff.h>
48#include <linux/mm.h>
49#include <linux/security.h>
50#include <linux/slab.h>
51#include <linux/uaccess.h>
52#include <linux/page_counter.h>
53#include <linux/memcontrol.h>
54#include <linux/static_key.h>
55#include <linux/sched.h>
56#include <linux/wait.h>
57#include <linux/cgroup-defs.h>
58#include <linux/rbtree.h>
59#include <linux/filter.h>
60#include <linux/rculist_nulls.h>
61#include <linux/poll.h>
62#include <linux/sockptr.h>
63#include <linux/indirect_call_wrapper.h>
64#include <linux/atomic.h>
65#include <linux/refcount.h>
66#include <net/dst.h>
67#include <net/checksum.h>
68#include <net/tcp_states.h>
69#include <linux/net_tstamp.h>
70#include <net/l3mdev.h>
71
72
73
74
75
76
77
78
79#define SOCK_DEBUGGING
80#ifdef SOCK_DEBUGGING
81#define SOCK_DEBUG(sk, msg...) do { if ((sk) && sock_flag((sk), SOCK_DBG)) \
82 printk(KERN_DEBUG msg); } while (0)
83#else
84
85static inline __printf(2, 3)
86void SOCK_DEBUG(const struct sock *sk, const char *msg, ...)
87{
88}
89#endif
90
91
92
93
94
95typedef struct {
96 spinlock_t slock;
97 int owned;
98 wait_queue_head_t wq;
99
100
101
102
103
104
105#ifdef CONFIG_DEBUG_LOCK_ALLOC
106 struct lockdep_map dep_map;
107#endif
108} socket_lock_t;
109
110struct sock;
111struct proto;
112struct net;
113
114typedef __u32 __bitwise __portpair;
115typedef __u64 __bitwise __addrpair;
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162struct sock_common {
163
164
165
166 union {
167 __addrpair skc_addrpair;
168 struct {
169 __be32 skc_daddr;
170 __be32 skc_rcv_saddr;
171 };
172 };
173 union {
174 unsigned int skc_hash;
175 __u16 skc_u16hashes[2];
176 };
177
178 union {
179 __portpair skc_portpair;
180 struct {
181 __be16 skc_dport;
182 __u16 skc_num;
183 };
184 };
185
186 unsigned short skc_family;
187 volatile unsigned char skc_state;
188 unsigned char skc_reuse:4;
189 unsigned char skc_reuseport:1;
190 unsigned char skc_ipv6only:1;
191 unsigned char skc_net_refcnt:1;
192 int skc_bound_dev_if;
193 union {
194 struct hlist_node skc_bind_node;
195 struct hlist_node skc_portaddr_node;
196 };
197 struct proto *skc_prot;
198 possible_net_t skc_net;
199
200#if IS_ENABLED(CONFIG_IPV6)
201 struct in6_addr skc_v6_daddr;
202 struct in6_addr skc_v6_rcv_saddr;
203#endif
204
205 atomic64_t skc_cookie;
206
207
208
209
210
211
212 union {
213 unsigned long skc_flags;
214 struct sock *skc_listener;
215 struct inet_timewait_death_row *skc_tw_dr;
216 };
217
218
219
220
221
222 int skc_dontcopy_begin[0];
223
224 union {
225 struct hlist_node skc_node;
226 struct hlist_nulls_node skc_nulls_node;
227 };
228 unsigned short skc_tx_queue_mapping;
229#ifdef CONFIG_SOCK_RX_QUEUE_MAPPING
230 unsigned short skc_rx_queue_mapping;
231#endif
232 union {
233 int skc_incoming_cpu;
234 u32 skc_rcv_wnd;
235 u32 skc_tw_rcv_nxt;
236 };
237
238 refcount_t skc_refcnt;
239
240 int skc_dontcopy_end[0];
241 union {
242 u32 skc_rxhash;
243 u32 skc_window_clamp;
244 u32 skc_tw_snd_nxt;
245 };
246
247};
248
249struct bpf_local_storage;
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351struct sock {
352
353
354
355
356 struct sock_common __sk_common;
357#define sk_node __sk_common.skc_node
358#define sk_nulls_node __sk_common.skc_nulls_node
359#define sk_refcnt __sk_common.skc_refcnt
360#define sk_tx_queue_mapping __sk_common.skc_tx_queue_mapping
361#ifdef CONFIG_SOCK_RX_QUEUE_MAPPING
362#define sk_rx_queue_mapping __sk_common.skc_rx_queue_mapping
363#endif
364
365#define sk_dontcopy_begin __sk_common.skc_dontcopy_begin
366#define sk_dontcopy_end __sk_common.skc_dontcopy_end
367#define sk_hash __sk_common.skc_hash
368#define sk_portpair __sk_common.skc_portpair
369#define sk_num __sk_common.skc_num
370#define sk_dport __sk_common.skc_dport
371#define sk_addrpair __sk_common.skc_addrpair
372#define sk_daddr __sk_common.skc_daddr
373#define sk_rcv_saddr __sk_common.skc_rcv_saddr
374#define sk_family __sk_common.skc_family
375#define sk_state __sk_common.skc_state
376#define sk_reuse __sk_common.skc_reuse
377#define sk_reuseport __sk_common.skc_reuseport
378#define sk_ipv6only __sk_common.skc_ipv6only
379#define sk_net_refcnt __sk_common.skc_net_refcnt
380#define sk_bound_dev_if __sk_common.skc_bound_dev_if
381#define sk_bind_node __sk_common.skc_bind_node
382#define sk_prot __sk_common.skc_prot
383#define sk_net __sk_common.skc_net
384#define sk_v6_daddr __sk_common.skc_v6_daddr
385#define sk_v6_rcv_saddr __sk_common.skc_v6_rcv_saddr
386#define sk_cookie __sk_common.skc_cookie
387#define sk_incoming_cpu __sk_common.skc_incoming_cpu
388#define sk_flags __sk_common.skc_flags
389#define sk_rxhash __sk_common.skc_rxhash
390
391 socket_lock_t sk_lock;
392 atomic_t sk_drops;
393 int sk_rcvlowat;
394 struct sk_buff_head sk_error_queue;
395 struct sk_buff *sk_rx_skb_cache;
396 struct sk_buff_head sk_receive_queue;
397
398
399
400
401
402
403
404
405 struct {
406 atomic_t rmem_alloc;
407 int len;
408 struct sk_buff *head;
409 struct sk_buff *tail;
410 } sk_backlog;
411#define sk_rmem_alloc sk_backlog.rmem_alloc
412
413 int sk_forward_alloc;
414#ifdef CONFIG_NET_RX_BUSY_POLL
415 unsigned int sk_ll_usec;
416
417 unsigned int sk_napi_id;
418#endif
419 int sk_rcvbuf;
420
421 struct sk_filter __rcu *sk_filter;
422 union {
423 struct socket_wq __rcu *sk_wq;
424
425 struct socket_wq *sk_wq_raw;
426
427 };
428#ifdef CONFIG_XFRM
429 struct xfrm_policy __rcu *sk_policy[2];
430#endif
431 struct dst_entry *sk_rx_dst;
432 struct dst_entry __rcu *sk_dst_cache;
433 atomic_t sk_omem_alloc;
434 int sk_sndbuf;
435
436
437 int sk_wmem_queued;
438 refcount_t sk_wmem_alloc;
439 unsigned long sk_tsq_flags;
440 union {
441 struct sk_buff *sk_send_head;
442 struct rb_root tcp_rtx_queue;
443 };
444 struct sk_buff *sk_tx_skb_cache;
445 struct sk_buff_head sk_write_queue;
446 __s32 sk_peek_off;
447 int sk_write_pending;
448 __u32 sk_dst_pending_confirm;
449 u32 sk_pacing_status;
450 long sk_sndtimeo;
451 struct timer_list sk_timer;
452 __u32 sk_priority;
453 __u32 sk_mark;
454 unsigned long sk_pacing_rate;
455 unsigned long sk_max_pacing_rate;
456 struct page_frag sk_frag;
457 netdev_features_t sk_route_caps;
458 netdev_features_t sk_route_nocaps;
459 netdev_features_t sk_route_forced_caps;
460 int sk_gso_type;
461 unsigned int sk_gso_max_size;
462 gfp_t sk_allocation;
463 __u32 sk_txhash;
464
465
466
467
468
469 u8 sk_padding : 1,
470 sk_kern_sock : 1,
471 sk_no_check_tx : 1,
472 sk_no_check_rx : 1,
473 sk_userlocks : 4;
474 u8 sk_pacing_shift;
475 u16 sk_type;
476 u16 sk_protocol;
477 u16 sk_gso_max_segs;
478 unsigned long sk_lingertime;
479 struct proto *sk_prot_creator;
480 rwlock_t sk_callback_lock;
481 int sk_err,
482 sk_err_soft;
483 u32 sk_ack_backlog;
484 u32 sk_max_ack_backlog;
485 kuid_t sk_uid;
486#ifdef CONFIG_NET_RX_BUSY_POLL
487 u8 sk_prefer_busy_poll;
488 u16 sk_busy_poll_budget;
489#endif
490 struct pid *sk_peer_pid;
491 const struct cred *sk_peer_cred;
492 long sk_rcvtimeo;
493 ktime_t sk_stamp;
494#if BITS_PER_LONG==32
495 seqlock_t sk_stamp_seq;
496#endif
497 u16 sk_tsflags;
498 int sk_bind_phc;
499 u8 sk_shutdown;
500 u32 sk_tskey;
501 atomic_t sk_zckey;
502
503 u8 sk_clockid;
504 u8 sk_txtime_deadline_mode : 1,
505 sk_txtime_report_errors : 1,
506 sk_txtime_unused : 6;
507
508 struct socket *sk_socket;
509 void *sk_user_data;
510#ifdef CONFIG_SECURITY
511 void *sk_security;
512#endif
513 struct sock_cgroup_data sk_cgrp_data;
514 struct mem_cgroup *sk_memcg;
515 void (*sk_state_change)(struct sock *sk);
516 void (*sk_data_ready)(struct sock *sk);
517 void (*sk_write_space)(struct sock *sk);
518 void (*sk_error_report)(struct sock *sk);
519 int (*sk_backlog_rcv)(struct sock *sk,
520 struct sk_buff *skb);
521#ifdef CONFIG_SOCK_VALIDATE_XMIT
522 struct sk_buff* (*sk_validate_xmit_skb)(struct sock *sk,
523 struct net_device *dev,
524 struct sk_buff *skb);
525#endif
526 void (*sk_destruct)(struct sock *sk);
527 struct sock_reuseport __rcu *sk_reuseport_cb;
528#ifdef CONFIG_BPF_SYSCALL
529 struct bpf_local_storage __rcu *sk_bpf_storage;
530#endif
531 struct rcu_head sk_rcu;
532};
533
534enum sk_pacing {
535 SK_PACING_NONE = 0,
536 SK_PACING_NEEDED = 1,
537 SK_PACING_FQ = 2,
538};
539
540
541
542
543
544
545#define SK_USER_DATA_NOCOPY 1UL
546#define SK_USER_DATA_BPF 2UL
547#define SK_USER_DATA_PTRMASK ~(SK_USER_DATA_NOCOPY | SK_USER_DATA_BPF)
548
549
550
551
552
553static inline bool sk_user_data_is_nocopy(const struct sock *sk)
554{
555 return ((uintptr_t)sk->sk_user_data & SK_USER_DATA_NOCOPY);
556}
557
558#define __sk_user_data(sk) ((*((void __rcu **)&(sk)->sk_user_data)))
559
560#define rcu_dereference_sk_user_data(sk) \
561({ \
562 void *__tmp = rcu_dereference(__sk_user_data((sk))); \
563 (void *)((uintptr_t)__tmp & SK_USER_DATA_PTRMASK); \
564})
565#define rcu_assign_sk_user_data(sk, ptr) \
566({ \
567 uintptr_t __tmp = (uintptr_t)(ptr); \
568 WARN_ON_ONCE(__tmp & ~SK_USER_DATA_PTRMASK); \
569 rcu_assign_pointer(__sk_user_data((sk)), __tmp); \
570})
571#define rcu_assign_sk_user_data_nocopy(sk, ptr) \
572({ \
573 uintptr_t __tmp = (uintptr_t)(ptr); \
574 WARN_ON_ONCE(__tmp & ~SK_USER_DATA_PTRMASK); \
575 rcu_assign_pointer(__sk_user_data((sk)), \
576 __tmp | SK_USER_DATA_NOCOPY); \
577})
578
579
580
581
582
583
584
585
586#define SK_NO_REUSE 0
587#define SK_CAN_REUSE 1
588#define SK_FORCE_REUSE 2
589
590int sk_set_peek_off(struct sock *sk, int val);
591
592static inline int sk_peek_offset(struct sock *sk, int flags)
593{
594 if (unlikely(flags & MSG_PEEK)) {
595 return READ_ONCE(sk->sk_peek_off);
596 }
597
598 return 0;
599}
600
601static inline void sk_peek_offset_bwd(struct sock *sk, int val)
602{
603 s32 off = READ_ONCE(sk->sk_peek_off);
604
605 if (unlikely(off >= 0)) {
606 off = max_t(s32, off - val, 0);
607 WRITE_ONCE(sk->sk_peek_off, off);
608 }
609}
610
611static inline void sk_peek_offset_fwd(struct sock *sk, int val)
612{
613 sk_peek_offset_bwd(sk, -val);
614}
615
616
617
618
619static inline struct sock *sk_entry(const struct hlist_node *node)
620{
621 return hlist_entry(node, struct sock, sk_node);
622}
623
624static inline struct sock *__sk_head(const struct hlist_head *head)
625{
626 return hlist_entry(head->first, struct sock, sk_node);
627}
628
629static inline struct sock *sk_head(const struct hlist_head *head)
630{
631 return hlist_empty(head) ? NULL : __sk_head(head);
632}
633
634static inline struct sock *__sk_nulls_head(const struct hlist_nulls_head *head)
635{
636 return hlist_nulls_entry(head->first, struct sock, sk_nulls_node);
637}
638
639static inline struct sock *sk_nulls_head(const struct hlist_nulls_head *head)
640{
641 return hlist_nulls_empty(head) ? NULL : __sk_nulls_head(head);
642}
643
644static inline struct sock *sk_next(const struct sock *sk)
645{
646 return hlist_entry_safe(sk->sk_node.next, struct sock, sk_node);
647}
648
649static inline struct sock *sk_nulls_next(const struct sock *sk)
650{
651 return (!is_a_nulls(sk->sk_nulls_node.next)) ?
652 hlist_nulls_entry(sk->sk_nulls_node.next,
653 struct sock, sk_nulls_node) :
654 NULL;
655}
656
657static inline bool sk_unhashed(const struct sock *sk)
658{
659 return hlist_unhashed(&sk->sk_node);
660}
661
662static inline bool sk_hashed(const struct sock *sk)
663{
664 return !sk_unhashed(sk);
665}
666
667static inline void sk_node_init(struct hlist_node *node)
668{
669 node->pprev = NULL;
670}
671
672static inline void sk_nulls_node_init(struct hlist_nulls_node *node)
673{
674 node->pprev = NULL;
675}
676
677static inline void __sk_del_node(struct sock *sk)
678{
679 __hlist_del(&sk->sk_node);
680}
681
682
683static inline bool __sk_del_node_init(struct sock *sk)
684{
685 if (sk_hashed(sk)) {
686 __sk_del_node(sk);
687 sk_node_init(&sk->sk_node);
688 return true;
689 }
690 return false;
691}
692
693
694
695
696
697
698
699static __always_inline void sock_hold(struct sock *sk)
700{
701 refcount_inc(&sk->sk_refcnt);
702}
703
704
705
706
707static __always_inline void __sock_put(struct sock *sk)
708{
709 refcount_dec(&sk->sk_refcnt);
710}
711
712static inline bool sk_del_node_init(struct sock *sk)
713{
714 bool rc = __sk_del_node_init(sk);
715
716 if (rc) {
717
718 WARN_ON(refcount_read(&sk->sk_refcnt) == 1);
719 __sock_put(sk);
720 }
721 return rc;
722}
723#define sk_del_node_init_rcu(sk) sk_del_node_init(sk)
724
725static inline bool __sk_nulls_del_node_init_rcu(struct sock *sk)
726{
727 if (sk_hashed(sk)) {
728 hlist_nulls_del_init_rcu(&sk->sk_nulls_node);
729 return true;
730 }
731 return false;
732}
733
734static inline bool sk_nulls_del_node_init_rcu(struct sock *sk)
735{
736 bool rc = __sk_nulls_del_node_init_rcu(sk);
737
738 if (rc) {
739
740 WARN_ON(refcount_read(&sk->sk_refcnt) == 1);
741 __sock_put(sk);
742 }
743 return rc;
744}
745
746static inline void __sk_add_node(struct sock *sk, struct hlist_head *list)
747{
748 hlist_add_head(&sk->sk_node, list);
749}
750
751static inline void sk_add_node(struct sock *sk, struct hlist_head *list)
752{
753 sock_hold(sk);
754 __sk_add_node(sk, list);
755}
756
757static inline void sk_add_node_rcu(struct sock *sk, struct hlist_head *list)
758{
759 sock_hold(sk);
760 if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport &&
761 sk->sk_family == AF_INET6)
762 hlist_add_tail_rcu(&sk->sk_node, list);
763 else
764 hlist_add_head_rcu(&sk->sk_node, list);
765}
766
767static inline void sk_add_node_tail_rcu(struct sock *sk, struct hlist_head *list)
768{
769 sock_hold(sk);
770 hlist_add_tail_rcu(&sk->sk_node, list);
771}
772
773static inline void __sk_nulls_add_node_rcu(struct sock *sk, struct hlist_nulls_head *list)
774{
775 hlist_nulls_add_head_rcu(&sk->sk_nulls_node, list);
776}
777
778static inline void __sk_nulls_add_node_tail_rcu(struct sock *sk, struct hlist_nulls_head *list)
779{
780 hlist_nulls_add_tail_rcu(&sk->sk_nulls_node, list);
781}
782
783static inline void sk_nulls_add_node_rcu(struct sock *sk, struct hlist_nulls_head *list)
784{
785 sock_hold(sk);
786 __sk_nulls_add_node_rcu(sk, list);
787}
788
789static inline void __sk_del_bind_node(struct sock *sk)
790{
791 __hlist_del(&sk->sk_bind_node);
792}
793
794static inline void sk_add_bind_node(struct sock *sk,
795 struct hlist_head *list)
796{
797 hlist_add_head(&sk->sk_bind_node, list);
798}
799
800#define sk_for_each(__sk, list) \
801 hlist_for_each_entry(__sk, list, sk_node)
802#define sk_for_each_rcu(__sk, list) \
803 hlist_for_each_entry_rcu(__sk, list, sk_node)
804#define sk_nulls_for_each(__sk, node, list) \
805 hlist_nulls_for_each_entry(__sk, node, list, sk_nulls_node)
806#define sk_nulls_for_each_rcu(__sk, node, list) \
807 hlist_nulls_for_each_entry_rcu(__sk, node, list, sk_nulls_node)
808#define sk_for_each_from(__sk) \
809 hlist_for_each_entry_from(__sk, sk_node)
810#define sk_nulls_for_each_from(__sk, node) \
811 if (__sk && ({ node = &(__sk)->sk_nulls_node; 1; })) \
812 hlist_nulls_for_each_entry_from(__sk, node, sk_nulls_node)
813#define sk_for_each_safe(__sk, tmp, list) \
814 hlist_for_each_entry_safe(__sk, tmp, list, sk_node)
815#define sk_for_each_bound(__sk, list) \
816 hlist_for_each_entry(__sk, list, sk_bind_node)
817
818
819
820
821
822
823
824
825
826#define sk_for_each_entry_offset_rcu(tpos, pos, head, offset) \
827 for (pos = rcu_dereference(hlist_first_rcu(head)); \
828 pos != NULL && \
829 ({ tpos = (typeof(*tpos) *)((void *)pos - offset); 1;}); \
830 pos = rcu_dereference(hlist_next_rcu(pos)))
831
832static inline struct user_namespace *sk_user_ns(struct sock *sk)
833{
834
835
836
837
838 return sk->sk_socket->file->f_cred->user_ns;
839}
840
841
842enum sock_flags {
843 SOCK_DEAD,
844 SOCK_DONE,
845 SOCK_URGINLINE,
846 SOCK_KEEPOPEN,
847 SOCK_LINGER,
848 SOCK_DESTROY,
849 SOCK_BROADCAST,
850 SOCK_TIMESTAMP,
851 SOCK_ZAPPED,
852 SOCK_USE_WRITE_QUEUE,
853 SOCK_DBG,
854 SOCK_RCVTSTAMP,
855 SOCK_RCVTSTAMPNS,
856 SOCK_LOCALROUTE,
857 SOCK_MEMALLOC,
858 SOCK_TIMESTAMPING_RX_SOFTWARE,
859 SOCK_FASYNC,
860 SOCK_RXQ_OVFL,
861 SOCK_ZEROCOPY,
862 SOCK_WIFI_STATUS,
863 SOCK_NOFCS,
864
865
866
867 SOCK_FILTER_LOCKED,
868 SOCK_SELECT_ERR_QUEUE,
869 SOCK_RCU_FREE,
870 SOCK_TXTIME,
871 SOCK_XDP,
872 SOCK_TSTAMP_NEW,
873};
874
875#define SK_FLAGS_TIMESTAMP ((1UL << SOCK_TIMESTAMP) | (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE))
876
877static inline void sock_copy_flags(struct sock *nsk, struct sock *osk)
878{
879 nsk->sk_flags = osk->sk_flags;
880}
881
882static inline void sock_set_flag(struct sock *sk, enum sock_flags flag)
883{
884 __set_bit(flag, &sk->sk_flags);
885}
886
887static inline void sock_reset_flag(struct sock *sk, enum sock_flags flag)
888{
889 __clear_bit(flag, &sk->sk_flags);
890}
891
892static inline void sock_valbool_flag(struct sock *sk, enum sock_flags bit,
893 int valbool)
894{
895 if (valbool)
896 sock_set_flag(sk, bit);
897 else
898 sock_reset_flag(sk, bit);
899}
900
901static inline bool sock_flag(const struct sock *sk, enum sock_flags flag)
902{
903 return test_bit(flag, &sk->sk_flags);
904}
905
906#ifdef CONFIG_NET
907DECLARE_STATIC_KEY_FALSE(memalloc_socks_key);
908static inline int sk_memalloc_socks(void)
909{
910 return static_branch_unlikely(&memalloc_socks_key);
911}
912
913void __receive_sock(struct file *file);
914#else
915
916static inline int sk_memalloc_socks(void)
917{
918 return 0;
919}
920
921static inline void __receive_sock(struct file *file)
922{ }
923#endif
924
925static inline gfp_t sk_gfp_mask(const struct sock *sk, gfp_t gfp_mask)
926{
927 return gfp_mask | (sk->sk_allocation & __GFP_MEMALLOC);
928}
929
930static inline void sk_acceptq_removed(struct sock *sk)
931{
932 WRITE_ONCE(sk->sk_ack_backlog, sk->sk_ack_backlog - 1);
933}
934
935static inline void sk_acceptq_added(struct sock *sk)
936{
937 WRITE_ONCE(sk->sk_ack_backlog, sk->sk_ack_backlog + 1);
938}
939
940
941
942
943
944static inline bool sk_acceptq_is_full(const struct sock *sk)
945{
946 return READ_ONCE(sk->sk_ack_backlog) > READ_ONCE(sk->sk_max_ack_backlog);
947}
948
949
950
951
952static inline int sk_stream_min_wspace(const struct sock *sk)
953{
954 return READ_ONCE(sk->sk_wmem_queued) >> 1;
955}
956
957static inline int sk_stream_wspace(const struct sock *sk)
958{
959 return READ_ONCE(sk->sk_sndbuf) - READ_ONCE(sk->sk_wmem_queued);
960}
961
962static inline void sk_wmem_queued_add(struct sock *sk, int val)
963{
964 WRITE_ONCE(sk->sk_wmem_queued, sk->sk_wmem_queued + val);
965}
966
967void sk_stream_write_space(struct sock *sk);
968
969
970static inline void __sk_add_backlog(struct sock *sk, struct sk_buff *skb)
971{
972
973 skb_dst_force(skb);
974
975 if (!sk->sk_backlog.tail)
976 WRITE_ONCE(sk->sk_backlog.head, skb);
977 else
978 sk->sk_backlog.tail->next = skb;
979
980 WRITE_ONCE(sk->sk_backlog.tail, skb);
981 skb->next = NULL;
982}
983
984
985
986
987
988
989static inline bool sk_rcvqueues_full(const struct sock *sk, unsigned int limit)
990{
991 unsigned int qsize = sk->sk_backlog.len + atomic_read(&sk->sk_rmem_alloc);
992
993 return qsize > limit;
994}
995
996
997static inline __must_check int sk_add_backlog(struct sock *sk, struct sk_buff *skb,
998 unsigned int limit)
999{
1000 if (sk_rcvqueues_full(sk, limit))
1001 return -ENOBUFS;
1002
1003
1004
1005
1006
1007
1008 if (skb_pfmemalloc(skb) && !sock_flag(sk, SOCK_MEMALLOC))
1009 return -ENOMEM;
1010
1011 __sk_add_backlog(sk, skb);
1012 sk->sk_backlog.len += skb->truesize;
1013 return 0;
1014}
1015
1016int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb);
1017
1018static inline int sk_backlog_rcv(struct sock *sk, struct sk_buff *skb)
1019{
1020 if (sk_memalloc_socks() && skb_pfmemalloc(skb))
1021 return __sk_backlog_rcv(sk, skb);
1022
1023 return sk->sk_backlog_rcv(sk, skb);
1024}
1025
1026static inline void sk_incoming_cpu_update(struct sock *sk)
1027{
1028 int cpu = raw_smp_processor_id();
1029
1030 if (unlikely(READ_ONCE(sk->sk_incoming_cpu) != cpu))
1031 WRITE_ONCE(sk->sk_incoming_cpu, cpu);
1032}
1033
1034static inline void sock_rps_record_flow_hash(__u32 hash)
1035{
1036#ifdef CONFIG_RPS
1037 struct rps_sock_flow_table *sock_flow_table;
1038
1039 rcu_read_lock();
1040 sock_flow_table = rcu_dereference(rps_sock_flow_table);
1041 rps_record_sock_flow(sock_flow_table, hash);
1042 rcu_read_unlock();
1043#endif
1044}
1045
1046static inline void sock_rps_record_flow(const struct sock *sk)
1047{
1048#ifdef CONFIG_RPS
1049 if (static_branch_unlikely(&rfs_needed)) {
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060 if (sk->sk_state == TCP_ESTABLISHED)
1061 sock_rps_record_flow_hash(sk->sk_rxhash);
1062 }
1063#endif
1064}
1065
1066static inline void sock_rps_save_rxhash(struct sock *sk,
1067 const struct sk_buff *skb)
1068{
1069#ifdef CONFIG_RPS
1070 if (unlikely(sk->sk_rxhash != skb->hash))
1071 sk->sk_rxhash = skb->hash;
1072#endif
1073}
1074
1075static inline void sock_rps_reset_rxhash(struct sock *sk)
1076{
1077#ifdef CONFIG_RPS
1078 sk->sk_rxhash = 0;
1079#endif
1080}
1081
1082#define sk_wait_event(__sk, __timeo, __condition, __wait) \
1083 ({ int __rc; \
1084 release_sock(__sk); \
1085 __rc = __condition; \
1086 if (!__rc) { \
1087 *(__timeo) = wait_woken(__wait, \
1088 TASK_INTERRUPTIBLE, \
1089 *(__timeo)); \
1090 } \
1091 sched_annotate_sleep(); \
1092 lock_sock(__sk); \
1093 __rc = __condition; \
1094 __rc; \
1095 })
1096
1097int sk_stream_wait_connect(struct sock *sk, long *timeo_p);
1098int sk_stream_wait_memory(struct sock *sk, long *timeo_p);
1099void sk_stream_wait_close(struct sock *sk, long timeo_p);
1100int sk_stream_error(struct sock *sk, int flags, int err);
1101void sk_stream_kill_queues(struct sock *sk);
1102void sk_set_memalloc(struct sock *sk);
1103void sk_clear_memalloc(struct sock *sk);
1104
1105void __sk_flush_backlog(struct sock *sk);
1106
1107static inline bool sk_flush_backlog(struct sock *sk)
1108{
1109 if (unlikely(READ_ONCE(sk->sk_backlog.tail))) {
1110 __sk_flush_backlog(sk);
1111 return true;
1112 }
1113 return false;
1114}
1115
1116int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb);
1117
1118struct request_sock_ops;
1119struct timewait_sock_ops;
1120struct inet_hashinfo;
1121struct raw_hashinfo;
1122struct smc_hashinfo;
1123struct module;
1124struct sk_psock;
1125
1126
1127
1128
1129
1130static inline void sk_prot_clear_nulls(struct sock *sk, int size)
1131{
1132 if (offsetof(struct sock, sk_node.next) != 0)
1133 memset(sk, 0, offsetof(struct sock, sk_node.next));
1134 memset(&sk->sk_node.pprev, 0,
1135 size - offsetof(struct sock, sk_node.pprev));
1136}
1137
1138
1139
1140
1141struct proto {
1142 void (*close)(struct sock *sk,
1143 long timeout);
1144 int (*pre_connect)(struct sock *sk,
1145 struct sockaddr *uaddr,
1146 int addr_len);
1147 int (*connect)(struct sock *sk,
1148 struct sockaddr *uaddr,
1149 int addr_len);
1150 int (*disconnect)(struct sock *sk, int flags);
1151
1152 struct sock * (*accept)(struct sock *sk, int flags, int *err,
1153 bool kern);
1154
1155 int (*ioctl)(struct sock *sk, int cmd,
1156 unsigned long arg);
1157 int (*init)(struct sock *sk);
1158 void (*destroy)(struct sock *sk);
1159 void (*shutdown)(struct sock *sk, int how);
1160 int (*setsockopt)(struct sock *sk, int level,
1161 int optname, sockptr_t optval,
1162 unsigned int optlen);
1163 int (*getsockopt)(struct sock *sk, int level,
1164 int optname, char __user *optval,
1165 int __user *option);
1166 void (*keepalive)(struct sock *sk, int valbool);
1167#ifdef CONFIG_COMPAT
1168 int (*compat_ioctl)(struct sock *sk,
1169 unsigned int cmd, unsigned long arg);
1170#endif
1171 int (*sendmsg)(struct sock *sk, struct msghdr *msg,
1172 size_t len);
1173 int (*recvmsg)(struct sock *sk, struct msghdr *msg,
1174 size_t len, int noblock, int flags,
1175 int *addr_len);
1176 int (*sendpage)(struct sock *sk, struct page *page,
1177 int offset, size_t size, int flags);
1178 int (*bind)(struct sock *sk,
1179 struct sockaddr *addr, int addr_len);
1180 int (*bind_add)(struct sock *sk,
1181 struct sockaddr *addr, int addr_len);
1182
1183 int (*backlog_rcv) (struct sock *sk,
1184 struct sk_buff *skb);
1185 bool (*bpf_bypass_getsockopt)(int level,
1186 int optname);
1187
1188 void (*release_cb)(struct sock *sk);
1189
1190
1191 int (*hash)(struct sock *sk);
1192 void (*unhash)(struct sock *sk);
1193 void (*rehash)(struct sock *sk);
1194 int (*get_port)(struct sock *sk, unsigned short snum);
1195#ifdef CONFIG_BPF_SYSCALL
1196 int (*psock_update_sk_prot)(struct sock *sk,
1197 struct sk_psock *psock,
1198 bool restore);
1199#endif
1200
1201
1202#ifdef CONFIG_PROC_FS
1203 unsigned int inuse_idx;
1204#endif
1205
1206 bool (*stream_memory_free)(const struct sock *sk, int wake);
1207 bool (*stream_memory_read)(const struct sock *sk);
1208
1209 void (*enter_memory_pressure)(struct sock *sk);
1210 void (*leave_memory_pressure)(struct sock *sk);
1211 atomic_long_t *memory_allocated;
1212 struct percpu_counter *sockets_allocated;
1213
1214
1215
1216
1217
1218
1219 unsigned long *memory_pressure;
1220 long *sysctl_mem;
1221
1222 int *sysctl_wmem;
1223 int *sysctl_rmem;
1224 u32 sysctl_wmem_offset;
1225 u32 sysctl_rmem_offset;
1226
1227 int max_header;
1228 bool no_autobind;
1229
1230 struct kmem_cache *slab;
1231 unsigned int obj_size;
1232 slab_flags_t slab_flags;
1233 unsigned int useroffset;
1234 unsigned int usersize;
1235
1236 struct percpu_counter *orphan_count;
1237
1238 struct request_sock_ops *rsk_prot;
1239 struct timewait_sock_ops *twsk_prot;
1240
1241 union {
1242 struct inet_hashinfo *hashinfo;
1243 struct udp_table *udp_table;
1244 struct raw_hashinfo *raw_hash;
1245 struct smc_hashinfo *smc_hash;
1246 } h;
1247
1248 struct module *owner;
1249
1250 char name[32];
1251
1252 struct list_head node;
1253#ifdef SOCK_REFCNT_DEBUG
1254 atomic_t socks;
1255#endif
1256 int (*diag_destroy)(struct sock *sk, int err);
1257} __randomize_layout;
1258
1259int proto_register(struct proto *prot, int alloc_slab);
1260void proto_unregister(struct proto *prot);
1261int sock_load_diag_module(int family, int protocol);
1262
1263#ifdef SOCK_REFCNT_DEBUG
1264static inline void sk_refcnt_debug_inc(struct sock *sk)
1265{
1266 atomic_inc(&sk->sk_prot->socks);
1267}
1268
1269static inline void sk_refcnt_debug_dec(struct sock *sk)
1270{
1271 atomic_dec(&sk->sk_prot->socks);
1272 printk(KERN_DEBUG "%s socket %p released, %d are still alive\n",
1273 sk->sk_prot->name, sk, atomic_read(&sk->sk_prot->socks));
1274}
1275
1276static inline void sk_refcnt_debug_release(const struct sock *sk)
1277{
1278 if (refcount_read(&sk->sk_refcnt) != 1)
1279 printk(KERN_DEBUG "Destruction of the %s socket %p delayed, refcnt=%d\n",
1280 sk->sk_prot->name, sk, refcount_read(&sk->sk_refcnt));
1281}
1282#else
1283#define sk_refcnt_debug_inc(sk) do { } while (0)
1284#define sk_refcnt_debug_dec(sk) do { } while (0)
1285#define sk_refcnt_debug_release(sk) do { } while (0)
1286#endif
1287
1288INDIRECT_CALLABLE_DECLARE(bool tcp_stream_memory_free(const struct sock *sk, int wake));
1289
1290static inline bool __sk_stream_memory_free(const struct sock *sk, int wake)
1291{
1292 if (READ_ONCE(sk->sk_wmem_queued) >= READ_ONCE(sk->sk_sndbuf))
1293 return false;
1294
1295#ifdef CONFIG_INET
1296 return sk->sk_prot->stream_memory_free ?
1297 INDIRECT_CALL_1(sk->sk_prot->stream_memory_free,
1298 tcp_stream_memory_free,
1299 sk, wake) : true;
1300#else
1301 return sk->sk_prot->stream_memory_free ?
1302 sk->sk_prot->stream_memory_free(sk, wake) : true;
1303#endif
1304}
1305
1306static inline bool sk_stream_memory_free(const struct sock *sk)
1307{
1308 return __sk_stream_memory_free(sk, 0);
1309}
1310
1311static inline bool __sk_stream_is_writeable(const struct sock *sk, int wake)
1312{
1313 return sk_stream_wspace(sk) >= sk_stream_min_wspace(sk) &&
1314 __sk_stream_memory_free(sk, wake);
1315}
1316
1317static inline bool sk_stream_is_writeable(const struct sock *sk)
1318{
1319 return __sk_stream_is_writeable(sk, 0);
1320}
1321
1322static inline int sk_under_cgroup_hierarchy(struct sock *sk,
1323 struct cgroup *ancestor)
1324{
1325#ifdef CONFIG_SOCK_CGROUP_DATA
1326 return cgroup_is_descendant(sock_cgroup_ptr(&sk->sk_cgrp_data),
1327 ancestor);
1328#else
1329 return -ENOTSUPP;
1330#endif
1331}
1332
1333static inline bool sk_has_memory_pressure(const struct sock *sk)
1334{
1335 return sk->sk_prot->memory_pressure != NULL;
1336}
1337
1338static inline bool sk_under_memory_pressure(const struct sock *sk)
1339{
1340 if (!sk->sk_prot->memory_pressure)
1341 return false;
1342
1343 if (mem_cgroup_sockets_enabled && sk->sk_memcg &&
1344 mem_cgroup_under_socket_pressure(sk->sk_memcg))
1345 return true;
1346
1347 return !!*sk->sk_prot->memory_pressure;
1348}
1349
1350static inline long
1351sk_memory_allocated(const struct sock *sk)
1352{
1353 return atomic_long_read(sk->sk_prot->memory_allocated);
1354}
1355
1356static inline long
1357sk_memory_allocated_add(struct sock *sk, int amt)
1358{
1359 return atomic_long_add_return(amt, sk->sk_prot->memory_allocated);
1360}
1361
1362static inline void
1363sk_memory_allocated_sub(struct sock *sk, int amt)
1364{
1365 atomic_long_sub(amt, sk->sk_prot->memory_allocated);
1366}
1367
1368#define SK_ALLOC_PERCPU_COUNTER_BATCH 16
1369
1370static inline void sk_sockets_allocated_dec(struct sock *sk)
1371{
1372 percpu_counter_add_batch(sk->sk_prot->sockets_allocated, -1,
1373 SK_ALLOC_PERCPU_COUNTER_BATCH);
1374}
1375
1376static inline void sk_sockets_allocated_inc(struct sock *sk)
1377{
1378 percpu_counter_add_batch(sk->sk_prot->sockets_allocated, 1,
1379 SK_ALLOC_PERCPU_COUNTER_BATCH);
1380}
1381
1382static inline u64
1383sk_sockets_allocated_read_positive(struct sock *sk)
1384{
1385 return percpu_counter_read_positive(sk->sk_prot->sockets_allocated);
1386}
1387
1388static inline int
1389proto_sockets_allocated_sum_positive(struct proto *prot)
1390{
1391 return percpu_counter_sum_positive(prot->sockets_allocated);
1392}
1393
1394static inline long
1395proto_memory_allocated(struct proto *prot)
1396{
1397 return atomic_long_read(prot->memory_allocated);
1398}
1399
1400static inline bool
1401proto_memory_pressure(struct proto *prot)
1402{
1403 if (!prot->memory_pressure)
1404 return false;
1405 return !!*prot->memory_pressure;
1406}
1407
1408
1409#ifdef CONFIG_PROC_FS
1410
1411void sock_prot_inuse_add(struct net *net, struct proto *prot, int inc);
1412int sock_prot_inuse_get(struct net *net, struct proto *proto);
1413int sock_inuse_get(struct net *net);
1414#else
1415static inline void sock_prot_inuse_add(struct net *net, struct proto *prot,
1416 int inc)
1417{
1418}
1419#endif
1420
1421
1422
1423
1424
1425static inline int __sk_prot_rehash(struct sock *sk)
1426{
1427 sk->sk_prot->unhash(sk);
1428 return sk->sk_prot->hash(sk);
1429}
1430
1431
1432#define SOCK_DESTROY_TIME (10*HZ)
1433
1434
1435#define PROT_SOCK 1024
1436
1437#define SHUTDOWN_MASK 3
1438#define RCV_SHUTDOWN 1
1439#define SEND_SHUTDOWN 2
1440
1441#define SOCK_SNDBUF_LOCK 1
1442#define SOCK_RCVBUF_LOCK 2
1443#define SOCK_BINDADDR_LOCK 4
1444#define SOCK_BINDPORT_LOCK 8
1445
1446struct socket_alloc {
1447 struct socket socket;
1448 struct inode vfs_inode;
1449};
1450
1451static inline struct socket *SOCKET_I(struct inode *inode)
1452{
1453 return &container_of(inode, struct socket_alloc, vfs_inode)->socket;
1454}
1455
1456static inline struct inode *SOCK_INODE(struct socket *socket)
1457{
1458 return &container_of(socket, struct socket_alloc, socket)->vfs_inode;
1459}
1460
1461
1462
1463
1464int __sk_mem_raise_allocated(struct sock *sk, int size, int amt, int kind);
1465int __sk_mem_schedule(struct sock *sk, int size, int kind);
1466void __sk_mem_reduce_allocated(struct sock *sk, int amount);
1467void __sk_mem_reclaim(struct sock *sk, int amount);
1468
1469
1470
1471
1472#define SK_MEM_QUANTUM 4096
1473#define SK_MEM_QUANTUM_SHIFT ilog2(SK_MEM_QUANTUM)
1474#define SK_MEM_SEND 0
1475#define SK_MEM_RECV 1
1476
1477
1478static inline long sk_prot_mem_limits(const struct sock *sk, int index)
1479{
1480 long val = sk->sk_prot->sysctl_mem[index];
1481
1482#if PAGE_SIZE > SK_MEM_QUANTUM
1483 val <<= PAGE_SHIFT - SK_MEM_QUANTUM_SHIFT;
1484#elif PAGE_SIZE < SK_MEM_QUANTUM
1485 val >>= SK_MEM_QUANTUM_SHIFT - PAGE_SHIFT;
1486#endif
1487 return val;
1488}
1489
1490static inline int sk_mem_pages(int amt)
1491{
1492 return (amt + SK_MEM_QUANTUM - 1) >> SK_MEM_QUANTUM_SHIFT;
1493}
1494
1495static inline bool sk_has_account(struct sock *sk)
1496{
1497
1498 return !!sk->sk_prot->memory_allocated;
1499}
1500
1501static inline bool sk_wmem_schedule(struct sock *sk, int size)
1502{
1503 if (!sk_has_account(sk))
1504 return true;
1505 return size <= sk->sk_forward_alloc ||
1506 __sk_mem_schedule(sk, size, SK_MEM_SEND);
1507}
1508
1509static inline bool
1510sk_rmem_schedule(struct sock *sk, struct sk_buff *skb, int size)
1511{
1512 if (!sk_has_account(sk))
1513 return true;
1514 return size <= sk->sk_forward_alloc ||
1515 __sk_mem_schedule(sk, size, SK_MEM_RECV) ||
1516 skb_pfmemalloc(skb);
1517}
1518
1519static inline void sk_mem_reclaim(struct sock *sk)
1520{
1521 if (!sk_has_account(sk))
1522 return;
1523 if (sk->sk_forward_alloc >= SK_MEM_QUANTUM)
1524 __sk_mem_reclaim(sk, sk->sk_forward_alloc);
1525}
1526
1527static inline void sk_mem_reclaim_partial(struct sock *sk)
1528{
1529 if (!sk_has_account(sk))
1530 return;
1531 if (sk->sk_forward_alloc > SK_MEM_QUANTUM)
1532 __sk_mem_reclaim(sk, sk->sk_forward_alloc - 1);
1533}
1534
1535static inline void sk_mem_charge(struct sock *sk, int size)
1536{
1537 if (!sk_has_account(sk))
1538 return;
1539 sk->sk_forward_alloc -= size;
1540}
1541
1542static inline void sk_mem_uncharge(struct sock *sk, int size)
1543{
1544 if (!sk_has_account(sk))
1545 return;
1546 sk->sk_forward_alloc += size;
1547
1548
1549
1550
1551
1552
1553
1554
1555 if (unlikely(sk->sk_forward_alloc >= 1 << 21))
1556 __sk_mem_reclaim(sk, 1 << 20);
1557}
1558
1559DECLARE_STATIC_KEY_FALSE(tcp_tx_skb_cache_key);
1560static inline void sk_wmem_free_skb(struct sock *sk, struct sk_buff *skb)
1561{
1562 sk_wmem_queued_add(sk, -skb->truesize);
1563 sk_mem_uncharge(sk, skb->truesize);
1564 if (static_branch_unlikely(&tcp_tx_skb_cache_key) &&
1565 !sk->sk_tx_skb_cache && !skb_cloned(skb)) {
1566 skb_ext_reset(skb);
1567 skb_zcopy_clear(skb, true);
1568 sk->sk_tx_skb_cache = skb;
1569 return;
1570 }
1571 __kfree_skb(skb);
1572}
1573
1574static inline void sock_release_ownership(struct sock *sk)
1575{
1576 if (sk->sk_lock.owned) {
1577 sk->sk_lock.owned = 0;
1578
1579
1580 mutex_release(&sk->sk_lock.dep_map, _RET_IP_);
1581 }
1582}
1583
1584
1585
1586
1587
1588
1589
1590
1591#define sock_lock_init_class_and_name(sk, sname, skey, name, key) \
1592do { \
1593 sk->sk_lock.owned = 0; \
1594 init_waitqueue_head(&sk->sk_lock.wq); \
1595 spin_lock_init(&(sk)->sk_lock.slock); \
1596 debug_check_no_locks_freed((void *)&(sk)->sk_lock, \
1597 sizeof((sk)->sk_lock)); \
1598 lockdep_set_class_and_name(&(sk)->sk_lock.slock, \
1599 (skey), (sname)); \
1600 lockdep_init_map(&(sk)->sk_lock.dep_map, (name), (key), 0); \
1601} while (0)
1602
1603static inline bool lockdep_sock_is_held(const struct sock *sk)
1604{
1605 return lockdep_is_held(&sk->sk_lock) ||
1606 lockdep_is_held(&sk->sk_lock.slock);
1607}
1608
1609void lock_sock_nested(struct sock *sk, int subclass);
1610
1611static inline void lock_sock(struct sock *sk)
1612{
1613 lock_sock_nested(sk, 0);
1614}
1615
1616void __lock_sock(struct sock *sk);
1617void __release_sock(struct sock *sk);
1618void release_sock(struct sock *sk);
1619
1620
1621#define bh_lock_sock(__sk) spin_lock(&((__sk)->sk_lock.slock))
1622#define bh_lock_sock_nested(__sk) \
1623 spin_lock_nested(&((__sk)->sk_lock.slock), \
1624 SINGLE_DEPTH_NESTING)
1625#define bh_unlock_sock(__sk) spin_unlock(&((__sk)->sk_lock.slock))
1626
1627bool lock_sock_fast(struct sock *sk) __acquires(&sk->sk_lock.slock);
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637static inline void unlock_sock_fast(struct sock *sk, bool slow)
1638 __releases(&sk->sk_lock.slock)
1639{
1640 if (slow) {
1641 release_sock(sk);
1642 __release(&sk->sk_lock.slock);
1643 } else {
1644 spin_unlock_bh(&sk->sk_lock.slock);
1645 }
1646}
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662static inline void sock_owned_by_me(const struct sock *sk)
1663{
1664#ifdef CONFIG_LOCKDEP
1665 WARN_ON_ONCE(!lockdep_sock_is_held(sk) && debug_locks);
1666#endif
1667}
1668
1669static inline bool sock_owned_by_user(const struct sock *sk)
1670{
1671 sock_owned_by_me(sk);
1672 return sk->sk_lock.owned;
1673}
1674
1675static inline bool sock_owned_by_user_nocheck(const struct sock *sk)
1676{
1677 return sk->sk_lock.owned;
1678}
1679
1680
1681static inline bool sock_allow_reclassification(const struct sock *csk)
1682{
1683 struct sock *sk = (struct sock *)csk;
1684
1685 return !sk->sk_lock.owned && !spin_is_locked(&sk->sk_lock.slock);
1686}
1687
1688struct sock *sk_alloc(struct net *net, int family, gfp_t priority,
1689 struct proto *prot, int kern);
1690void sk_free(struct sock *sk);
1691void sk_destruct(struct sock *sk);
1692struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority);
1693void sk_free_unlock_clone(struct sock *sk);
1694
1695struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force,
1696 gfp_t priority);
1697void __sock_wfree(struct sk_buff *skb);
1698void sock_wfree(struct sk_buff *skb);
1699struct sk_buff *sock_omalloc(struct sock *sk, unsigned long size,
1700 gfp_t priority);
1701void skb_orphan_partial(struct sk_buff *skb);
1702void sock_rfree(struct sk_buff *skb);
1703void sock_efree(struct sk_buff *skb);
1704#ifdef CONFIG_INET
1705void sock_edemux(struct sk_buff *skb);
1706void sock_pfree(struct sk_buff *skb);
1707#else
1708#define sock_edemux sock_efree
1709#endif
1710
1711int sock_setsockopt(struct socket *sock, int level, int op,
1712 sockptr_t optval, unsigned int optlen);
1713
1714int sock_getsockopt(struct socket *sock, int level, int op,
1715 char __user *optval, int __user *optlen);
1716int sock_gettstamp(struct socket *sock, void __user *userstamp,
1717 bool timeval, bool time32);
1718struct sk_buff *sock_alloc_send_skb(struct sock *sk, unsigned long size,
1719 int noblock, int *errcode);
1720struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len,
1721 unsigned long data_len, int noblock,
1722 int *errcode, int max_page_order);
1723void *sock_kmalloc(struct sock *sk, int size, gfp_t priority);
1724void sock_kfree_s(struct sock *sk, void *mem, int size);
1725void sock_kzfree_s(struct sock *sk, void *mem, int size);
1726void sk_send_sigurg(struct sock *sk);
1727
1728struct sockcm_cookie {
1729 u64 transmit_time;
1730 u32 mark;
1731 u16 tsflags;
1732};
1733
1734static inline void sockcm_init(struct sockcm_cookie *sockc,
1735 const struct sock *sk)
1736{
1737 *sockc = (struct sockcm_cookie) { .tsflags = sk->sk_tsflags };
1738}
1739
1740int __sock_cmsg_send(struct sock *sk, struct msghdr *msg, struct cmsghdr *cmsg,
1741 struct sockcm_cookie *sockc);
1742int sock_cmsg_send(struct sock *sk, struct msghdr *msg,
1743 struct sockcm_cookie *sockc);
1744
1745
1746
1747
1748
1749int sock_no_bind(struct socket *, struct sockaddr *, int);
1750int sock_no_connect(struct socket *, struct sockaddr *, int, int);
1751int sock_no_socketpair(struct socket *, struct socket *);
1752int sock_no_accept(struct socket *, struct socket *, int, bool);
1753int sock_no_getname(struct socket *, struct sockaddr *, int);
1754int sock_no_ioctl(struct socket *, unsigned int, unsigned long);
1755int sock_no_listen(struct socket *, int);
1756int sock_no_shutdown(struct socket *, int);
1757int sock_no_sendmsg(struct socket *, struct msghdr *, size_t);
1758int sock_no_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t len);
1759int sock_no_recvmsg(struct socket *, struct msghdr *, size_t, int);
1760int sock_no_mmap(struct file *file, struct socket *sock,
1761 struct vm_area_struct *vma);
1762ssize_t sock_no_sendpage(struct socket *sock, struct page *page, int offset,
1763 size_t size, int flags);
1764ssize_t sock_no_sendpage_locked(struct sock *sk, struct page *page,
1765 int offset, size_t size, int flags);
1766
1767
1768
1769
1770
1771int sock_common_getsockopt(struct socket *sock, int level, int optname,
1772 char __user *optval, int __user *optlen);
1773int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
1774 int flags);
1775int sock_common_setsockopt(struct socket *sock, int level, int optname,
1776 sockptr_t optval, unsigned int optlen);
1777
1778void sk_common_release(struct sock *sk);
1779
1780
1781
1782
1783
1784
1785void sock_init_data(struct socket *sock, struct sock *sk);
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813static inline void sock_put(struct sock *sk)
1814{
1815 if (refcount_dec_and_test(&sk->sk_refcnt))
1816 sk_free(sk);
1817}
1818
1819
1820
1821void sock_gen_put(struct sock *sk);
1822
1823int __sk_receive_skb(struct sock *sk, struct sk_buff *skb, const int nested,
1824 unsigned int trim_cap, bool refcounted);
1825static inline int sk_receive_skb(struct sock *sk, struct sk_buff *skb,
1826 const int nested)
1827{
1828 return __sk_receive_skb(sk, skb, nested, 1, true);
1829}
1830
1831static inline void sk_tx_queue_set(struct sock *sk, int tx_queue)
1832{
1833
1834 if (WARN_ON_ONCE((unsigned short)tx_queue >= USHRT_MAX))
1835 return;
1836 sk->sk_tx_queue_mapping = tx_queue;
1837}
1838
1839#define NO_QUEUE_MAPPING USHRT_MAX
1840
1841static inline void sk_tx_queue_clear(struct sock *sk)
1842{
1843 sk->sk_tx_queue_mapping = NO_QUEUE_MAPPING;
1844}
1845
1846static inline int sk_tx_queue_get(const struct sock *sk)
1847{
1848 if (sk && sk->sk_tx_queue_mapping != NO_QUEUE_MAPPING)
1849 return sk->sk_tx_queue_mapping;
1850
1851 return -1;
1852}
1853
1854static inline void sk_rx_queue_set(struct sock *sk, const struct sk_buff *skb)
1855{
1856#ifdef CONFIG_SOCK_RX_QUEUE_MAPPING
1857 if (skb_rx_queue_recorded(skb)) {
1858 u16 rx_queue = skb_get_rx_queue(skb);
1859
1860 if (WARN_ON_ONCE(rx_queue == NO_QUEUE_MAPPING))
1861 return;
1862
1863 sk->sk_rx_queue_mapping = rx_queue;
1864 }
1865#endif
1866}
1867
1868static inline void sk_rx_queue_clear(struct sock *sk)
1869{
1870#ifdef CONFIG_SOCK_RX_QUEUE_MAPPING
1871 sk->sk_rx_queue_mapping = NO_QUEUE_MAPPING;
1872#endif
1873}
1874
1875static inline int sk_rx_queue_get(const struct sock *sk)
1876{
1877#ifdef CONFIG_SOCK_RX_QUEUE_MAPPING
1878 if (sk && sk->sk_rx_queue_mapping != NO_QUEUE_MAPPING)
1879 return sk->sk_rx_queue_mapping;
1880#endif
1881
1882 return -1;
1883}
1884
1885static inline void sk_set_socket(struct sock *sk, struct socket *sock)
1886{
1887 sk->sk_socket = sock;
1888}
1889
1890static inline wait_queue_head_t *sk_sleep(struct sock *sk)
1891{
1892 BUILD_BUG_ON(offsetof(struct socket_wq, wait) != 0);
1893 return &rcu_dereference_raw(sk->sk_wq)->wait;
1894}
1895
1896
1897
1898
1899
1900
1901
1902static inline void sock_orphan(struct sock *sk)
1903{
1904 write_lock_bh(&sk->sk_callback_lock);
1905 sock_set_flag(sk, SOCK_DEAD);
1906 sk_set_socket(sk, NULL);
1907 sk->sk_wq = NULL;
1908 write_unlock_bh(&sk->sk_callback_lock);
1909}
1910
1911static inline void sock_graft(struct sock *sk, struct socket *parent)
1912{
1913 WARN_ON(parent->sk);
1914 write_lock_bh(&sk->sk_callback_lock);
1915 rcu_assign_pointer(sk->sk_wq, &parent->wq);
1916 parent->sk = sk;
1917 sk_set_socket(sk, parent);
1918 sk->sk_uid = SOCK_INODE(parent)->i_uid;
1919 security_sock_graft(sk, parent);
1920 write_unlock_bh(&sk->sk_callback_lock);
1921}
1922
1923kuid_t sock_i_uid(struct sock *sk);
1924unsigned long sock_i_ino(struct sock *sk);
1925
1926static inline kuid_t sock_net_uid(const struct net *net, const struct sock *sk)
1927{
1928 return sk ? sk->sk_uid : make_kuid(net->user_ns, 0);
1929}
1930
1931static inline u32 net_tx_rndhash(void)
1932{
1933 u32 v = prandom_u32();
1934
1935 return v ?: 1;
1936}
1937
1938static inline void sk_set_txhash(struct sock *sk)
1939{
1940
1941 WRITE_ONCE(sk->sk_txhash, net_tx_rndhash());
1942}
1943
1944static inline bool sk_rethink_txhash(struct sock *sk)
1945{
1946 if (sk->sk_txhash) {
1947 sk_set_txhash(sk);
1948 return true;
1949 }
1950 return false;
1951}
1952
1953static inline struct dst_entry *
1954__sk_dst_get(struct sock *sk)
1955{
1956 return rcu_dereference_check(sk->sk_dst_cache,
1957 lockdep_sock_is_held(sk));
1958}
1959
1960static inline struct dst_entry *
1961sk_dst_get(struct sock *sk)
1962{
1963 struct dst_entry *dst;
1964
1965 rcu_read_lock();
1966 dst = rcu_dereference(sk->sk_dst_cache);
1967 if (dst && !atomic_inc_not_zero(&dst->__refcnt))
1968 dst = NULL;
1969 rcu_read_unlock();
1970 return dst;
1971}
1972
1973static inline void __dst_negative_advice(struct sock *sk)
1974{
1975 struct dst_entry *ndst, *dst = __sk_dst_get(sk);
1976
1977 if (dst && dst->ops->negative_advice) {
1978 ndst = dst->ops->negative_advice(dst);
1979
1980 if (ndst != dst) {
1981 rcu_assign_pointer(sk->sk_dst_cache, ndst);
1982 sk_tx_queue_clear(sk);
1983 sk->sk_dst_pending_confirm = 0;
1984 }
1985 }
1986}
1987
1988static inline void dst_negative_advice(struct sock *sk)
1989{
1990 sk_rethink_txhash(sk);
1991 __dst_negative_advice(sk);
1992}
1993
1994static inline void
1995__sk_dst_set(struct sock *sk, struct dst_entry *dst)
1996{
1997 struct dst_entry *old_dst;
1998
1999 sk_tx_queue_clear(sk);
2000 sk->sk_dst_pending_confirm = 0;
2001 old_dst = rcu_dereference_protected(sk->sk_dst_cache,
2002 lockdep_sock_is_held(sk));
2003 rcu_assign_pointer(sk->sk_dst_cache, dst);
2004 dst_release(old_dst);
2005}
2006
2007static inline void
2008sk_dst_set(struct sock *sk, struct dst_entry *dst)
2009{
2010 struct dst_entry *old_dst;
2011
2012 sk_tx_queue_clear(sk);
2013 sk->sk_dst_pending_confirm = 0;
2014 old_dst = xchg((__force struct dst_entry **)&sk->sk_dst_cache, dst);
2015 dst_release(old_dst);
2016}
2017
2018static inline void
2019__sk_dst_reset(struct sock *sk)
2020{
2021 __sk_dst_set(sk, NULL);
2022}
2023
2024static inline void
2025sk_dst_reset(struct sock *sk)
2026{
2027 sk_dst_set(sk, NULL);
2028}
2029
2030struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie);
2031
2032struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie);
2033
2034static inline void sk_dst_confirm(struct sock *sk)
2035{
2036 if (!READ_ONCE(sk->sk_dst_pending_confirm))
2037 WRITE_ONCE(sk->sk_dst_pending_confirm, 1);
2038}
2039
2040static inline void sock_confirm_neigh(struct sk_buff *skb, struct neighbour *n)
2041{
2042 if (skb_get_dst_pending_confirm(skb)) {
2043 struct sock *sk = skb->sk;
2044 unsigned long now = jiffies;
2045
2046
2047 if (READ_ONCE(n->confirmed) != now)
2048 WRITE_ONCE(n->confirmed, now);
2049 if (sk && READ_ONCE(sk->sk_dst_pending_confirm))
2050 WRITE_ONCE(sk->sk_dst_pending_confirm, 0);
2051 }
2052}
2053
2054bool sk_mc_loop(struct sock *sk);
2055
2056static inline bool sk_can_gso(const struct sock *sk)
2057{
2058 return net_gso_ok(sk->sk_route_caps, sk->sk_gso_type);
2059}
2060
2061void sk_setup_caps(struct sock *sk, struct dst_entry *dst);
2062
2063static inline void sk_nocaps_add(struct sock *sk, netdev_features_t flags)
2064{
2065 sk->sk_route_nocaps |= flags;
2066 sk->sk_route_caps &= ~flags;
2067}
2068
2069static inline int skb_do_copy_data_nocache(struct sock *sk, struct sk_buff *skb,
2070 struct iov_iter *from, char *to,
2071 int copy, int offset)
2072{
2073 if (skb->ip_summed == CHECKSUM_NONE) {
2074 __wsum csum = 0;
2075 if (!csum_and_copy_from_iter_full(to, copy, &csum, from))
2076 return -EFAULT;
2077 skb->csum = csum_block_add(skb->csum, csum, offset);
2078 } else if (sk->sk_route_caps & NETIF_F_NOCACHE_COPY) {
2079 if (!copy_from_iter_full_nocache(to, copy, from))
2080 return -EFAULT;
2081 } else if (!copy_from_iter_full(to, copy, from))
2082 return -EFAULT;
2083
2084 return 0;
2085}
2086
2087static inline int skb_add_data_nocache(struct sock *sk, struct sk_buff *skb,
2088 struct iov_iter *from, int copy)
2089{
2090 int err, offset = skb->len;
2091
2092 err = skb_do_copy_data_nocache(sk, skb, from, skb_put(skb, copy),
2093 copy, offset);
2094 if (err)
2095 __skb_trim(skb, offset);
2096
2097 return err;
2098}
2099
2100static inline int skb_copy_to_page_nocache(struct sock *sk, struct iov_iter *from,
2101 struct sk_buff *skb,
2102 struct page *page,
2103 int off, int copy)
2104{
2105 int err;
2106
2107 err = skb_do_copy_data_nocache(sk, skb, from, page_address(page) + off,
2108 copy, skb->len);
2109 if (err)
2110 return err;
2111
2112 skb->len += copy;
2113 skb->data_len += copy;
2114 skb->truesize += copy;
2115 sk_wmem_queued_add(sk, copy);
2116 sk_mem_charge(sk, copy);
2117 return 0;
2118}
2119
2120
2121
2122
2123
2124
2125
2126static inline int sk_wmem_alloc_get(const struct sock *sk)
2127{
2128 return refcount_read(&sk->sk_wmem_alloc) - 1;
2129}
2130
2131
2132
2133
2134
2135
2136
2137static inline int sk_rmem_alloc_get(const struct sock *sk)
2138{
2139 return atomic_read(&sk->sk_rmem_alloc);
2140}
2141
2142
2143
2144
2145
2146
2147
2148static inline bool sk_has_allocations(const struct sock *sk)
2149{
2150 return sk_wmem_alloc_get(sk) || sk_rmem_alloc_get(sk);
2151}
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184static inline bool skwq_has_sleeper(struct socket_wq *wq)
2185{
2186 return wq && wq_has_sleeper(&wq->wait);
2187}
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197static inline void sock_poll_wait(struct file *filp, struct socket *sock,
2198 poll_table *p)
2199{
2200 if (!poll_does_not_wait(p)) {
2201 poll_wait(filp, &sock->wq.wait, p);
2202
2203
2204
2205
2206
2207 smp_mb();
2208 }
2209}
2210
2211static inline void skb_set_hash_from_sk(struct sk_buff *skb, struct sock *sk)
2212{
2213
2214 u32 txhash = READ_ONCE(sk->sk_txhash);
2215
2216 if (txhash) {
2217 skb->l4_hash = 1;
2218 skb->hash = txhash;
2219 }
2220}
2221
2222void skb_set_owner_w(struct sk_buff *skb, struct sock *sk);
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232static inline void skb_set_owner_r(struct sk_buff *skb, struct sock *sk)
2233{
2234 skb_orphan(skb);
2235 skb->sk = sk;
2236 skb->destructor = sock_rfree;
2237 atomic_add(skb->truesize, &sk->sk_rmem_alloc);
2238 sk_mem_charge(sk, skb->truesize);
2239}
2240
2241static inline __must_check bool skb_set_owner_sk_safe(struct sk_buff *skb, struct sock *sk)
2242{
2243 if (sk && refcount_inc_not_zero(&sk->sk_refcnt)) {
2244 skb_orphan(skb);
2245 skb->destructor = sock_efree;
2246 skb->sk = sk;
2247 return true;
2248 }
2249 return false;
2250}
2251
2252void sk_reset_timer(struct sock *sk, struct timer_list *timer,
2253 unsigned long expires);
2254
2255void sk_stop_timer(struct sock *sk, struct timer_list *timer);
2256
2257void sk_stop_timer_sync(struct sock *sk, struct timer_list *timer);
2258
2259int __sk_queue_drop_skb(struct sock *sk, struct sk_buff_head *sk_queue,
2260 struct sk_buff *skb, unsigned int flags,
2261 void (*destructor)(struct sock *sk,
2262 struct sk_buff *skb));
2263int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb);
2264int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb);
2265
2266int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb);
2267struct sk_buff *sock_dequeue_err_skb(struct sock *sk);
2268
2269
2270
2271
2272
2273static inline int sock_error(struct sock *sk)
2274{
2275 int err;
2276
2277
2278
2279
2280 if (likely(data_race(!sk->sk_err)))
2281 return 0;
2282
2283 err = xchg(&sk->sk_err, 0);
2284 return -err;
2285}
2286
2287void sk_error_report(struct sock *sk);
2288
2289static inline unsigned long sock_wspace(struct sock *sk)
2290{
2291 int amt = 0;
2292
2293 if (!(sk->sk_shutdown & SEND_SHUTDOWN)) {
2294 amt = sk->sk_sndbuf - refcount_read(&sk->sk_wmem_alloc);
2295 if (amt < 0)
2296 amt = 0;
2297 }
2298 return amt;
2299}
2300
2301
2302
2303
2304
2305static inline void sk_set_bit(int nr, struct sock *sk)
2306{
2307 if ((nr == SOCKWQ_ASYNC_NOSPACE || nr == SOCKWQ_ASYNC_WAITDATA) &&
2308 !sock_flag(sk, SOCK_FASYNC))
2309 return;
2310
2311 set_bit(nr, &sk->sk_wq_raw->flags);
2312}
2313
2314static inline void sk_clear_bit(int nr, struct sock *sk)
2315{
2316 if ((nr == SOCKWQ_ASYNC_NOSPACE || nr == SOCKWQ_ASYNC_WAITDATA) &&
2317 !sock_flag(sk, SOCK_FASYNC))
2318 return;
2319
2320 clear_bit(nr, &sk->sk_wq_raw->flags);
2321}
2322
2323static inline void sk_wake_async(const struct sock *sk, int how, int band)
2324{
2325 if (sock_flag(sk, SOCK_FASYNC)) {
2326 rcu_read_lock();
2327 sock_wake_async(rcu_dereference(sk->sk_wq), how, band);
2328 rcu_read_unlock();
2329 }
2330}
2331
2332
2333
2334
2335
2336
2337#define TCP_SKB_MIN_TRUESIZE (2048 + SKB_DATA_ALIGN(sizeof(struct sk_buff)))
2338
2339#define SOCK_MIN_SNDBUF (TCP_SKB_MIN_TRUESIZE * 2)
2340#define SOCK_MIN_RCVBUF TCP_SKB_MIN_TRUESIZE
2341
2342static inline void sk_stream_moderate_sndbuf(struct sock *sk)
2343{
2344 u32 val;
2345
2346 if (sk->sk_userlocks & SOCK_SNDBUF_LOCK)
2347 return;
2348
2349 val = min(sk->sk_sndbuf, sk->sk_wmem_queued >> 1);
2350
2351 WRITE_ONCE(sk->sk_sndbuf, max_t(u32, val, SOCK_MIN_SNDBUF));
2352}
2353
2354struct sk_buff *sk_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp,
2355 bool force_schedule);
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372static inline struct page_frag *sk_page_frag(struct sock *sk)
2373{
2374 if (gfpflags_normal_context(sk->sk_allocation))
2375 return ¤t->task_frag;
2376
2377 return &sk->sk_frag;
2378}
2379
2380bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag);
2381
2382
2383
2384
2385static inline bool sock_writeable(const struct sock *sk)
2386{
2387 return refcount_read(&sk->sk_wmem_alloc) < (READ_ONCE(sk->sk_sndbuf) >> 1);
2388}
2389
2390static inline gfp_t gfp_any(void)
2391{
2392 return in_softirq() ? GFP_ATOMIC : GFP_KERNEL;
2393}
2394
2395static inline long sock_rcvtimeo(const struct sock *sk, bool noblock)
2396{
2397 return noblock ? 0 : sk->sk_rcvtimeo;
2398}
2399
2400static inline long sock_sndtimeo(const struct sock *sk, bool noblock)
2401{
2402 return noblock ? 0 : sk->sk_sndtimeo;
2403}
2404
2405static inline int sock_rcvlowat(const struct sock *sk, int waitall, int len)
2406{
2407 int v = waitall ? len : min_t(int, READ_ONCE(sk->sk_rcvlowat), len);
2408
2409 return v ?: 1;
2410}
2411
2412
2413
2414
2415static inline int sock_intr_errno(long timeo)
2416{
2417 return timeo == MAX_SCHEDULE_TIMEOUT ? -ERESTARTSYS : -EINTR;
2418}
2419
2420struct sock_skb_cb {
2421 u32 dropcount;
2422};
2423
2424
2425
2426
2427
2428#define SOCK_SKB_CB_OFFSET ((sizeof_field(struct sk_buff, cb) - \
2429 sizeof(struct sock_skb_cb)))
2430
2431#define SOCK_SKB_CB(__skb) ((struct sock_skb_cb *)((__skb)->cb + \
2432 SOCK_SKB_CB_OFFSET))
2433
2434#define sock_skb_cb_check_size(size) \
2435 BUILD_BUG_ON((size) > SOCK_SKB_CB_OFFSET)
2436
2437static inline void
2438sock_skb_set_dropcount(const struct sock *sk, struct sk_buff *skb)
2439{
2440 SOCK_SKB_CB(skb)->dropcount = sock_flag(sk, SOCK_RXQ_OVFL) ?
2441 atomic_read(&sk->sk_drops) : 0;
2442}
2443
2444static inline void sk_drops_add(struct sock *sk, const struct sk_buff *skb)
2445{
2446 int segs = max_t(u16, 1, skb_shinfo(skb)->gso_segs);
2447
2448 atomic_add(segs, &sk->sk_drops);
2449}
2450
2451static inline ktime_t sock_read_timestamp(struct sock *sk)
2452{
2453#if BITS_PER_LONG==32
2454 unsigned int seq;
2455 ktime_t kt;
2456
2457 do {
2458 seq = read_seqbegin(&sk->sk_stamp_seq);
2459 kt = sk->sk_stamp;
2460 } while (read_seqretry(&sk->sk_stamp_seq, seq));
2461
2462 return kt;
2463#else
2464 return READ_ONCE(sk->sk_stamp);
2465#endif
2466}
2467
2468static inline void sock_write_timestamp(struct sock *sk, ktime_t kt)
2469{
2470#if BITS_PER_LONG==32
2471 write_seqlock(&sk->sk_stamp_seq);
2472 sk->sk_stamp = kt;
2473 write_sequnlock(&sk->sk_stamp_seq);
2474#else
2475 WRITE_ONCE(sk->sk_stamp, kt);
2476#endif
2477}
2478
2479void __sock_recv_timestamp(struct msghdr *msg, struct sock *sk,
2480 struct sk_buff *skb);
2481void __sock_recv_wifi_status(struct msghdr *msg, struct sock *sk,
2482 struct sk_buff *skb);
2483
2484static inline void
2485sock_recv_timestamp(struct msghdr *msg, struct sock *sk, struct sk_buff *skb)
2486{
2487 ktime_t kt = skb->tstamp;
2488 struct skb_shared_hwtstamps *hwtstamps = skb_hwtstamps(skb);
2489
2490
2491
2492
2493
2494
2495
2496 if (sock_flag(sk, SOCK_RCVTSTAMP) ||
2497 (sk->sk_tsflags & SOF_TIMESTAMPING_RX_SOFTWARE) ||
2498 (kt && sk->sk_tsflags & SOF_TIMESTAMPING_SOFTWARE) ||
2499 (hwtstamps->hwtstamp &&
2500 (sk->sk_tsflags & SOF_TIMESTAMPING_RAW_HARDWARE)))
2501 __sock_recv_timestamp(msg, sk, skb);
2502 else
2503 sock_write_timestamp(sk, kt);
2504
2505 if (sock_flag(sk, SOCK_WIFI_STATUS) && skb->wifi_acked_valid)
2506 __sock_recv_wifi_status(msg, sk, skb);
2507}
2508
2509void __sock_recv_ts_and_drops(struct msghdr *msg, struct sock *sk,
2510 struct sk_buff *skb);
2511
2512#define SK_DEFAULT_STAMP (-1L * NSEC_PER_SEC)
2513static inline void sock_recv_ts_and_drops(struct msghdr *msg, struct sock *sk,
2514 struct sk_buff *skb)
2515{
2516#define FLAGS_TS_OR_DROPS ((1UL << SOCK_RXQ_OVFL) | \
2517 (1UL << SOCK_RCVTSTAMP))
2518#define TSFLAGS_ANY (SOF_TIMESTAMPING_SOFTWARE | \
2519 SOF_TIMESTAMPING_RAW_HARDWARE)
2520
2521 if (sk->sk_flags & FLAGS_TS_OR_DROPS || sk->sk_tsflags & TSFLAGS_ANY)
2522 __sock_recv_ts_and_drops(msg, sk, skb);
2523 else if (unlikely(sock_flag(sk, SOCK_TIMESTAMP)))
2524 sock_write_timestamp(sk, skb->tstamp);
2525 else if (unlikely(sk->sk_stamp == SK_DEFAULT_STAMP))
2526 sock_write_timestamp(sk, 0);
2527}
2528
2529void __sock_tx_timestamp(__u16 tsflags, __u8 *tx_flags);
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540static inline void _sock_tx_timestamp(struct sock *sk, __u16 tsflags,
2541 __u8 *tx_flags, __u32 *tskey)
2542{
2543 if (unlikely(tsflags)) {
2544 __sock_tx_timestamp(tsflags, tx_flags);
2545 if (tsflags & SOF_TIMESTAMPING_OPT_ID && tskey &&
2546 tsflags & SOF_TIMESTAMPING_TX_RECORD_MASK)
2547 *tskey = sk->sk_tskey++;
2548 }
2549 if (unlikely(sock_flag(sk, SOCK_WIFI_STATUS)))
2550 *tx_flags |= SKBTX_WIFI_STATUS;
2551}
2552
2553static inline void sock_tx_timestamp(struct sock *sk, __u16 tsflags,
2554 __u8 *tx_flags)
2555{
2556 _sock_tx_timestamp(sk, tsflags, tx_flags, NULL);
2557}
2558
2559static inline void skb_setup_tx_timestamp(struct sk_buff *skb, __u16 tsflags)
2560{
2561 _sock_tx_timestamp(skb->sk, tsflags, &skb_shinfo(skb)->tx_flags,
2562 &skb_shinfo(skb)->tskey);
2563}
2564
2565DECLARE_STATIC_KEY_FALSE(tcp_rx_skb_cache_key);
2566
2567
2568
2569
2570
2571
2572
2573
2574static inline void sk_eat_skb(struct sock *sk, struct sk_buff *skb)
2575{
2576 __skb_unlink(skb, &sk->sk_receive_queue);
2577 if (static_branch_unlikely(&tcp_rx_skb_cache_key) &&
2578 !sk->sk_rx_skb_cache) {
2579 sk->sk_rx_skb_cache = skb;
2580 skb_orphan(skb);
2581 return;
2582 }
2583 __kfree_skb(skb);
2584}
2585
2586static inline
2587struct net *sock_net(const struct sock *sk)
2588{
2589 return read_pnet(&sk->sk_net);
2590}
2591
2592static inline
2593void sock_net_set(struct sock *sk, struct net *net)
2594{
2595 write_pnet(&sk->sk_net, net);
2596}
2597
2598static inline bool
2599skb_sk_is_prefetched(struct sk_buff *skb)
2600{
2601#ifdef CONFIG_INET
2602 return skb->destructor == sock_pfree;
2603#else
2604 return false;
2605#endif
2606}
2607
2608
2609
2610
2611static inline bool sk_fullsock(const struct sock *sk)
2612{
2613 return (1 << sk->sk_state) & ~(TCPF_TIME_WAIT | TCPF_NEW_SYN_RECV);
2614}
2615
2616static inline bool
2617sk_is_refcounted(struct sock *sk)
2618{
2619
2620 return !sk_fullsock(sk) || !sock_flag(sk, SOCK_RCU_FREE);
2621}
2622
2623
2624
2625
2626
2627
2628static inline struct sock *
2629skb_steal_sock(struct sk_buff *skb, bool *refcounted)
2630{
2631 if (skb->sk) {
2632 struct sock *sk = skb->sk;
2633
2634 *refcounted = true;
2635 if (skb_sk_is_prefetched(skb))
2636 *refcounted = sk_is_refcounted(sk);
2637 skb->destructor = NULL;
2638 skb->sk = NULL;
2639 return sk;
2640 }
2641 *refcounted = false;
2642 return NULL;
2643}
2644
2645
2646
2647
2648
2649static inline struct sk_buff *sk_validate_xmit_skb(struct sk_buff *skb,
2650 struct net_device *dev)
2651{
2652#ifdef CONFIG_SOCK_VALIDATE_XMIT
2653 struct sock *sk = skb->sk;
2654
2655 if (sk && sk_fullsock(sk) && sk->sk_validate_xmit_skb) {
2656 skb = sk->sk_validate_xmit_skb(sk, dev, skb);
2657#ifdef CONFIG_TLS_DEVICE
2658 } else if (unlikely(skb->decrypted)) {
2659 pr_warn_ratelimited("unencrypted skb with no associated socket - dropping\n");
2660 kfree_skb(skb);
2661 skb = NULL;
2662#endif
2663 }
2664#endif
2665
2666 return skb;
2667}
2668
2669
2670
2671
2672static inline bool sk_listener(const struct sock *sk)
2673{
2674 return (1 << sk->sk_state) & (TCPF_LISTEN | TCPF_NEW_SYN_RECV);
2675}
2676
2677void sock_enable_timestamp(struct sock *sk, enum sock_flags flag);
2678int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len, int level,
2679 int type);
2680
2681bool sk_ns_capable(const struct sock *sk,
2682 struct user_namespace *user_ns, int cap);
2683bool sk_capable(const struct sock *sk, int cap);
2684bool sk_net_capable(const struct sock *sk, int cap);
2685
2686void sk_get_meminfo(const struct sock *sk, u32 *meminfo);
2687
2688
2689
2690
2691
2692
2693#define _SK_MEM_PACKETS 256
2694#define _SK_MEM_OVERHEAD SKB_TRUESIZE(256)
2695#define SK_WMEM_MAX (_SK_MEM_OVERHEAD * _SK_MEM_PACKETS)
2696#define SK_RMEM_MAX (_SK_MEM_OVERHEAD * _SK_MEM_PACKETS)
2697
2698extern __u32 sysctl_wmem_max;
2699extern __u32 sysctl_rmem_max;
2700
2701extern int sysctl_tstamp_allow_data;
2702extern int sysctl_optmem_max;
2703
2704extern __u32 sysctl_wmem_default;
2705extern __u32 sysctl_rmem_default;
2706
2707DECLARE_STATIC_KEY_FALSE(net_high_order_alloc_disable_key);
2708
2709static inline int sk_get_wmem0(const struct sock *sk, const struct proto *proto)
2710{
2711
2712 if (proto->sysctl_wmem_offset)
2713 return *(int *)((void *)sock_net(sk) + proto->sysctl_wmem_offset);
2714
2715 return *proto->sysctl_wmem;
2716}
2717
2718static inline int sk_get_rmem0(const struct sock *sk, const struct proto *proto)
2719{
2720
2721 if (proto->sysctl_rmem_offset)
2722 return *(int *)((void *)sock_net(sk) + proto->sysctl_rmem_offset);
2723
2724 return *proto->sysctl_rmem;
2725}
2726
2727
2728
2729
2730
2731static inline void sk_pacing_shift_update(struct sock *sk, int val)
2732{
2733 if (!sk || !sk_fullsock(sk) || READ_ONCE(sk->sk_pacing_shift) == val)
2734 return;
2735 WRITE_ONCE(sk->sk_pacing_shift, val);
2736}
2737
2738
2739
2740
2741
2742
2743static inline bool sk_dev_equal_l3scope(struct sock *sk, int dif)
2744{
2745 int mdif;
2746
2747 if (!sk->sk_bound_dev_if || sk->sk_bound_dev_if == dif)
2748 return true;
2749
2750 mdif = l3mdev_master_ifindex_by_index(sock_net(sk), dif);
2751 if (mdif && mdif == sk->sk_bound_dev_if)
2752 return true;
2753
2754 return false;
2755}
2756
2757void sock_def_readable(struct sock *sk);
2758
2759int sock_bindtoindex(struct sock *sk, int ifindex, bool lock_sk);
2760void sock_set_timestamp(struct sock *sk, int optname, bool valbool);
2761int sock_set_timestamping(struct sock *sk, int optname,
2762 struct so_timestamping timestamping);
2763
2764void sock_enable_timestamps(struct sock *sk);
2765void sock_no_linger(struct sock *sk);
2766void sock_set_keepalive(struct sock *sk);
2767void sock_set_priority(struct sock *sk, u32 priority);
2768void sock_set_rcvbuf(struct sock *sk, int val);
2769void sock_set_mark(struct sock *sk, u32 val);
2770void sock_set_reuseaddr(struct sock *sk);
2771void sock_set_reuseport(struct sock *sk);
2772void sock_set_sndtimeo(struct sock *sk, s64 secs);
2773
2774int sock_bind_add(struct sock *sk, struct sockaddr *addr, int addr_len);
2775
2776#endif
2777