1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40#ifndef _SOCK_H
41#define _SOCK_H
42
43#include <linux/hardirq.h>
44#include <linux/kernel.h>
45#include <linux/list.h>
46#include <linux/list_nulls.h>
47#include <linux/timer.h>
48#include <linux/cache.h>
49#include <linux/bitops.h>
50#include <linux/lockdep.h>
51#include <linux/netdevice.h>
52#include <linux/skbuff.h>
53#include <linux/mm.h>
54#include <linux/security.h>
55#include <linux/slab.h>
56#include <linux/uaccess.h>
57#include <linux/page_counter.h>
58#include <linux/memcontrol.h>
59#include <linux/static_key.h>
60#include <linux/sched.h>
61#include <linux/wait.h>
62#include <linux/cgroup-defs.h>
63
64#include <linux/filter.h>
65#include <linux/rculist_nulls.h>
66#include <linux/poll.h>
67
68#include <linux/atomic.h>
69#include <net/dst.h>
70#include <net/checksum.h>
71#include <net/tcp_states.h>
72#include <linux/net_tstamp.h>
73#include <net/smc.h>
74
75
76
77
78
79
80
81
82#define SOCK_DEBUGGING
83#ifdef SOCK_DEBUGGING
84#define SOCK_DEBUG(sk, msg...) do { if ((sk) && sock_flag((sk), SOCK_DBG)) \
85 printk(KERN_DEBUG msg); } while (0)
86#else
87
88static inline __printf(2, 3)
89void SOCK_DEBUG(const struct sock *sk, const char *msg, ...)
90{
91}
92#endif
93
94
95
96
97
98typedef struct {
99 spinlock_t slock;
100 int owned;
101 wait_queue_head_t wq;
102
103
104
105
106
107
108#ifdef CONFIG_DEBUG_LOCK_ALLOC
109 struct lockdep_map dep_map;
110#endif
111} socket_lock_t;
112
113struct sock;
114struct proto;
115struct net;
116
117typedef __u32 __bitwise __portpair;
118typedef __u64 __bitwise __addrpair;
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149struct sock_common {
150
151
152
153 union {
154 __addrpair skc_addrpair;
155 struct {
156 __be32 skc_daddr;
157 __be32 skc_rcv_saddr;
158 };
159 };
160 union {
161 unsigned int skc_hash;
162 __u16 skc_u16hashes[2];
163 };
164
165 union {
166 __portpair skc_portpair;
167 struct {
168 __be16 skc_dport;
169 __u16 skc_num;
170 };
171 };
172
173 unsigned short skc_family;
174 volatile unsigned char skc_state;
175 unsigned char skc_reuse:4;
176 unsigned char skc_reuseport:1;
177 unsigned char skc_ipv6only:1;
178 unsigned char skc_net_refcnt:1;
179 int skc_bound_dev_if;
180 union {
181 struct hlist_node skc_bind_node;
182 struct hlist_node skc_portaddr_node;
183 };
184 struct proto *skc_prot;
185 possible_net_t skc_net;
186
187#if IS_ENABLED(CONFIG_IPV6)
188 struct in6_addr skc_v6_daddr;
189 struct in6_addr skc_v6_rcv_saddr;
190#endif
191
192 atomic64_t skc_cookie;
193
194
195
196
197
198
199 union {
200 unsigned long skc_flags;
201 struct sock *skc_listener;
202 struct inet_timewait_death_row *skc_tw_dr;
203 };
204
205
206
207
208
209 int skc_dontcopy_begin[0];
210
211 union {
212 struct hlist_node skc_node;
213 struct hlist_nulls_node skc_nulls_node;
214 };
215 int skc_tx_queue_mapping;
216 union {
217 int skc_incoming_cpu;
218 u32 skc_rcv_wnd;
219 u32 skc_tw_rcv_nxt;
220 };
221
222 atomic_t skc_refcnt;
223
224 int skc_dontcopy_end[0];
225 union {
226 u32 skc_rxhash;
227 u32 skc_window_clamp;
228 u32 skc_tw_snd_nxt;
229 };
230
231};
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311struct sock {
312
313
314
315
316 struct sock_common __sk_common;
317#define sk_node __sk_common.skc_node
318#define sk_nulls_node __sk_common.skc_nulls_node
319#define sk_refcnt __sk_common.skc_refcnt
320#define sk_tx_queue_mapping __sk_common.skc_tx_queue_mapping
321
322#define sk_dontcopy_begin __sk_common.skc_dontcopy_begin
323#define sk_dontcopy_end __sk_common.skc_dontcopy_end
324#define sk_hash __sk_common.skc_hash
325#define sk_portpair __sk_common.skc_portpair
326#define sk_num __sk_common.skc_num
327#define sk_dport __sk_common.skc_dport
328#define sk_addrpair __sk_common.skc_addrpair
329#define sk_daddr __sk_common.skc_daddr
330#define sk_rcv_saddr __sk_common.skc_rcv_saddr
331#define sk_family __sk_common.skc_family
332#define sk_state __sk_common.skc_state
333#define sk_reuse __sk_common.skc_reuse
334#define sk_reuseport __sk_common.skc_reuseport
335#define sk_ipv6only __sk_common.skc_ipv6only
336#define sk_net_refcnt __sk_common.skc_net_refcnt
337#define sk_bound_dev_if __sk_common.skc_bound_dev_if
338#define sk_bind_node __sk_common.skc_bind_node
339#define sk_prot __sk_common.skc_prot
340#define sk_net __sk_common.skc_net
341#define sk_v6_daddr __sk_common.skc_v6_daddr
342#define sk_v6_rcv_saddr __sk_common.skc_v6_rcv_saddr
343#define sk_cookie __sk_common.skc_cookie
344#define sk_incoming_cpu __sk_common.skc_incoming_cpu
345#define sk_flags __sk_common.skc_flags
346#define sk_rxhash __sk_common.skc_rxhash
347
348 socket_lock_t sk_lock;
349 atomic_t sk_drops;
350 int sk_rcvlowat;
351 struct sk_buff_head sk_error_queue;
352 struct sk_buff_head sk_receive_queue;
353
354
355
356
357
358
359
360
361 struct {
362 atomic_t rmem_alloc;
363 int len;
364 struct sk_buff *head;
365 struct sk_buff *tail;
366 } sk_backlog;
367#define sk_rmem_alloc sk_backlog.rmem_alloc
368
369 int sk_forward_alloc;
370#ifdef CONFIG_NET_RX_BUSY_POLL
371 unsigned int sk_ll_usec;
372
373 unsigned int sk_napi_id;
374#endif
375 int sk_rcvbuf;
376
377 struct sk_filter __rcu *sk_filter;
378 union {
379 struct socket_wq __rcu *sk_wq;
380 struct socket_wq *sk_wq_raw;
381 };
382#ifdef CONFIG_XFRM
383 struct xfrm_policy __rcu *sk_policy[2];
384#endif
385 struct dst_entry *sk_rx_dst;
386 struct dst_entry __rcu *sk_dst_cache;
387 atomic_t sk_omem_alloc;
388 int sk_sndbuf;
389
390
391 int sk_wmem_queued;
392 atomic_t sk_wmem_alloc;
393 unsigned long sk_tsq_flags;
394 struct sk_buff *sk_send_head;
395 struct sk_buff_head sk_write_queue;
396 __s32 sk_peek_off;
397 int sk_write_pending;
398 __u32 sk_dst_pending_confirm;
399
400 long sk_sndtimeo;
401 struct timer_list sk_timer;
402 __u32 sk_priority;
403 __u32 sk_mark;
404 u32 sk_pacing_rate;
405 u32 sk_max_pacing_rate;
406 struct page_frag sk_frag;
407 netdev_features_t sk_route_caps;
408 netdev_features_t sk_route_nocaps;
409 int sk_gso_type;
410 unsigned int sk_gso_max_size;
411 gfp_t sk_allocation;
412 __u32 sk_txhash;
413
414
415
416
417
418 unsigned int __sk_flags_offset[0];
419#ifdef __BIG_ENDIAN_BITFIELD
420#define SK_FL_PROTO_SHIFT 16
421#define SK_FL_PROTO_MASK 0x00ff0000
422
423#define SK_FL_TYPE_SHIFT 0
424#define SK_FL_TYPE_MASK 0x0000ffff
425#else
426#define SK_FL_PROTO_SHIFT 8
427#define SK_FL_PROTO_MASK 0x0000ff00
428
429#define SK_FL_TYPE_SHIFT 16
430#define SK_FL_TYPE_MASK 0xffff0000
431#endif
432
433 kmemcheck_bitfield_begin(flags);
434 unsigned int sk_padding : 1,
435 sk_kern_sock : 1,
436 sk_no_check_tx : 1,
437 sk_no_check_rx : 1,
438 sk_userlocks : 4,
439 sk_protocol : 8,
440 sk_type : 16;
441#define SK_PROTOCOL_MAX U8_MAX
442 kmemcheck_bitfield_end(flags);
443
444 u16 sk_gso_max_segs;
445 unsigned long sk_lingertime;
446 struct proto *sk_prot_creator;
447 rwlock_t sk_callback_lock;
448 int sk_err,
449 sk_err_soft;
450 u32 sk_ack_backlog;
451 u32 sk_max_ack_backlog;
452 kuid_t sk_uid;
453 struct pid *sk_peer_pid;
454 const struct cred *sk_peer_cred;
455 long sk_rcvtimeo;
456 ktime_t sk_stamp;
457 u16 sk_tsflags;
458 u8 sk_shutdown;
459 u32 sk_tskey;
460 struct socket *sk_socket;
461 void *sk_user_data;
462#ifdef CONFIG_SECURITY
463 void *sk_security;
464#endif
465 struct sock_cgroup_data sk_cgrp_data;
466 struct mem_cgroup *sk_memcg;
467 void (*sk_state_change)(struct sock *sk);
468 void (*sk_data_ready)(struct sock *sk);
469 void (*sk_write_space)(struct sock *sk);
470 void (*sk_error_report)(struct sock *sk);
471 int (*sk_backlog_rcv)(struct sock *sk,
472 struct sk_buff *skb);
473 void (*sk_destruct)(struct sock *sk);
474 struct sock_reuseport __rcu *sk_reuseport_cb;
475 struct rcu_head sk_rcu;
476};
477
478#define __sk_user_data(sk) ((*((void __rcu **)&(sk)->sk_user_data)))
479
480#define rcu_dereference_sk_user_data(sk) rcu_dereference(__sk_user_data((sk)))
481#define rcu_assign_sk_user_data(sk, ptr) rcu_assign_pointer(__sk_user_data((sk)), ptr)
482
483
484
485
486
487
488
489
490#define SK_NO_REUSE 0
491#define SK_CAN_REUSE 1
492#define SK_FORCE_REUSE 2
493
494int sk_set_peek_off(struct sock *sk, int val);
495
496static inline int sk_peek_offset(struct sock *sk, int flags)
497{
498 if (unlikely(flags & MSG_PEEK)) {
499 s32 off = READ_ONCE(sk->sk_peek_off);
500 if (off >= 0)
501 return off;
502 }
503
504 return 0;
505}
506
507static inline void sk_peek_offset_bwd(struct sock *sk, int val)
508{
509 s32 off = READ_ONCE(sk->sk_peek_off);
510
511 if (unlikely(off >= 0)) {
512 off = max_t(s32, off - val, 0);
513 WRITE_ONCE(sk->sk_peek_off, off);
514 }
515}
516
517static inline void sk_peek_offset_fwd(struct sock *sk, int val)
518{
519 sk_peek_offset_bwd(sk, -val);
520}
521
522
523
524
525static inline struct sock *sk_entry(const struct hlist_node *node)
526{
527 return hlist_entry(node, struct sock, sk_node);
528}
529
530static inline struct sock *__sk_head(const struct hlist_head *head)
531{
532 return hlist_entry(head->first, struct sock, sk_node);
533}
534
535static inline struct sock *sk_head(const struct hlist_head *head)
536{
537 return hlist_empty(head) ? NULL : __sk_head(head);
538}
539
540static inline struct sock *__sk_nulls_head(const struct hlist_nulls_head *head)
541{
542 return hlist_nulls_entry(head->first, struct sock, sk_nulls_node);
543}
544
545static inline struct sock *sk_nulls_head(const struct hlist_nulls_head *head)
546{
547 return hlist_nulls_empty(head) ? NULL : __sk_nulls_head(head);
548}
549
550static inline struct sock *sk_next(const struct sock *sk)
551{
552 return hlist_entry_safe(sk->sk_node.next, struct sock, sk_node);
553}
554
555static inline struct sock *sk_nulls_next(const struct sock *sk)
556{
557 return (!is_a_nulls(sk->sk_nulls_node.next)) ?
558 hlist_nulls_entry(sk->sk_nulls_node.next,
559 struct sock, sk_nulls_node) :
560 NULL;
561}
562
563static inline bool sk_unhashed(const struct sock *sk)
564{
565 return hlist_unhashed(&sk->sk_node);
566}
567
568static inline bool sk_hashed(const struct sock *sk)
569{
570 return !sk_unhashed(sk);
571}
572
573static inline void sk_node_init(struct hlist_node *node)
574{
575 node->pprev = NULL;
576}
577
578static inline void sk_nulls_node_init(struct hlist_nulls_node *node)
579{
580 node->pprev = NULL;
581}
582
583static inline void __sk_del_node(struct sock *sk)
584{
585 __hlist_del(&sk->sk_node);
586}
587
588
589static inline bool __sk_del_node_init(struct sock *sk)
590{
591 if (sk_hashed(sk)) {
592 __sk_del_node(sk);
593 sk_node_init(&sk->sk_node);
594 return true;
595 }
596 return false;
597}
598
599
600
601
602
603
604
605static __always_inline void sock_hold(struct sock *sk)
606{
607 atomic_inc(&sk->sk_refcnt);
608}
609
610
611
612
613static __always_inline void __sock_put(struct sock *sk)
614{
615 atomic_dec(&sk->sk_refcnt);
616}
617
618static inline bool sk_del_node_init(struct sock *sk)
619{
620 bool rc = __sk_del_node_init(sk);
621
622 if (rc) {
623
624 WARN_ON(atomic_read(&sk->sk_refcnt) == 1);
625 __sock_put(sk);
626 }
627 return rc;
628}
629#define sk_del_node_init_rcu(sk) sk_del_node_init(sk)
630
631static inline bool __sk_nulls_del_node_init_rcu(struct sock *sk)
632{
633 if (sk_hashed(sk)) {
634 hlist_nulls_del_init_rcu(&sk->sk_nulls_node);
635 return true;
636 }
637 return false;
638}
639
640static inline bool sk_nulls_del_node_init_rcu(struct sock *sk)
641{
642 bool rc = __sk_nulls_del_node_init_rcu(sk);
643
644 if (rc) {
645
646 WARN_ON(atomic_read(&sk->sk_refcnt) == 1);
647 __sock_put(sk);
648 }
649 return rc;
650}
651
652static inline void __sk_add_node(struct sock *sk, struct hlist_head *list)
653{
654 hlist_add_head(&sk->sk_node, list);
655}
656
657static inline void sk_add_node(struct sock *sk, struct hlist_head *list)
658{
659 sock_hold(sk);
660 __sk_add_node(sk, list);
661}
662
663static inline void sk_add_node_rcu(struct sock *sk, struct hlist_head *list)
664{
665 sock_hold(sk);
666 if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport &&
667 sk->sk_family == AF_INET6)
668 hlist_add_tail_rcu(&sk->sk_node, list);
669 else
670 hlist_add_head_rcu(&sk->sk_node, list);
671}
672
673static inline void __sk_nulls_add_node_rcu(struct sock *sk, struct hlist_nulls_head *list)
674{
675 if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport &&
676 sk->sk_family == AF_INET6)
677 hlist_nulls_add_tail_rcu(&sk->sk_nulls_node, list);
678 else
679 hlist_nulls_add_head_rcu(&sk->sk_nulls_node, list);
680}
681
682static inline void sk_nulls_add_node_rcu(struct sock *sk, struct hlist_nulls_head *list)
683{
684 sock_hold(sk);
685 __sk_nulls_add_node_rcu(sk, list);
686}
687
688static inline void __sk_del_bind_node(struct sock *sk)
689{
690 __hlist_del(&sk->sk_bind_node);
691}
692
693static inline void sk_add_bind_node(struct sock *sk,
694 struct hlist_head *list)
695{
696 hlist_add_head(&sk->sk_bind_node, list);
697}
698
699#define sk_for_each(__sk, list) \
700 hlist_for_each_entry(__sk, list, sk_node)
701#define sk_for_each_rcu(__sk, list) \
702 hlist_for_each_entry_rcu(__sk, list, sk_node)
703#define sk_nulls_for_each(__sk, node, list) \
704 hlist_nulls_for_each_entry(__sk, node, list, sk_nulls_node)
705#define sk_nulls_for_each_rcu(__sk, node, list) \
706 hlist_nulls_for_each_entry_rcu(__sk, node, list, sk_nulls_node)
707#define sk_for_each_from(__sk) \
708 hlist_for_each_entry_from(__sk, sk_node)
709#define sk_nulls_for_each_from(__sk, node) \
710 if (__sk && ({ node = &(__sk)->sk_nulls_node; 1; })) \
711 hlist_nulls_for_each_entry_from(__sk, node, sk_nulls_node)
712#define sk_for_each_safe(__sk, tmp, list) \
713 hlist_for_each_entry_safe(__sk, tmp, list, sk_node)
714#define sk_for_each_bound(__sk, list) \
715 hlist_for_each_entry(__sk, list, sk_bind_node)
716
717
718
719
720
721
722
723
724
725#define sk_for_each_entry_offset_rcu(tpos, pos, head, offset) \
726 for (pos = rcu_dereference((head)->first); \
727 pos != NULL && \
728 ({ tpos = (typeof(*tpos) *)((void *)pos - offset); 1;}); \
729 pos = rcu_dereference(pos->next))
730
731static inline struct user_namespace *sk_user_ns(struct sock *sk)
732{
733
734
735
736
737 return sk->sk_socket->file->f_cred->user_ns;
738}
739
740
741enum sock_flags {
742 SOCK_DEAD,
743 SOCK_DONE,
744 SOCK_URGINLINE,
745 SOCK_KEEPOPEN,
746 SOCK_LINGER,
747 SOCK_DESTROY,
748 SOCK_BROADCAST,
749 SOCK_TIMESTAMP,
750 SOCK_ZAPPED,
751 SOCK_USE_WRITE_QUEUE,
752 SOCK_DBG,
753 SOCK_RCVTSTAMP,
754 SOCK_RCVTSTAMPNS,
755 SOCK_LOCALROUTE,
756 SOCK_QUEUE_SHRUNK,
757 SOCK_MEMALLOC,
758 SOCK_TIMESTAMPING_RX_SOFTWARE,
759 SOCK_FASYNC,
760 SOCK_RXQ_OVFL,
761 SOCK_ZEROCOPY,
762 SOCK_WIFI_STATUS,
763 SOCK_NOFCS,
764
765
766
767 SOCK_FILTER_LOCKED,
768 SOCK_SELECT_ERR_QUEUE,
769 SOCK_RCU_FREE,
770};
771
772#define SK_FLAGS_TIMESTAMP ((1UL << SOCK_TIMESTAMP) | (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE))
773
774static inline void sock_copy_flags(struct sock *nsk, struct sock *osk)
775{
776 nsk->sk_flags = osk->sk_flags;
777}
778
779static inline void sock_set_flag(struct sock *sk, enum sock_flags flag)
780{
781 __set_bit(flag, &sk->sk_flags);
782}
783
784static inline void sock_reset_flag(struct sock *sk, enum sock_flags flag)
785{
786 __clear_bit(flag, &sk->sk_flags);
787}
788
789static inline bool sock_flag(const struct sock *sk, enum sock_flags flag)
790{
791 return test_bit(flag, &sk->sk_flags);
792}
793
794#ifdef CONFIG_NET
795extern struct static_key memalloc_socks;
796static inline int sk_memalloc_socks(void)
797{
798 return static_key_false(&memalloc_socks);
799}
800#else
801
802static inline int sk_memalloc_socks(void)
803{
804 return 0;
805}
806
807#endif
808
809static inline gfp_t sk_gfp_mask(const struct sock *sk, gfp_t gfp_mask)
810{
811 return gfp_mask | (sk->sk_allocation & __GFP_MEMALLOC);
812}
813
814static inline void sk_acceptq_removed(struct sock *sk)
815{
816 sk->sk_ack_backlog--;
817}
818
819static inline void sk_acceptq_added(struct sock *sk)
820{
821 sk->sk_ack_backlog++;
822}
823
824static inline bool sk_acceptq_is_full(const struct sock *sk)
825{
826 return sk->sk_ack_backlog > sk->sk_max_ack_backlog;
827}
828
829
830
831
832static inline int sk_stream_min_wspace(const struct sock *sk)
833{
834 return sk->sk_wmem_queued >> 1;
835}
836
837static inline int sk_stream_wspace(const struct sock *sk)
838{
839 return sk->sk_sndbuf - sk->sk_wmem_queued;
840}
841
842void sk_stream_write_space(struct sock *sk);
843
844
845static inline void __sk_add_backlog(struct sock *sk, struct sk_buff *skb)
846{
847
848 skb_dst_force_safe(skb);
849
850 if (!sk->sk_backlog.tail)
851 sk->sk_backlog.head = skb;
852 else
853 sk->sk_backlog.tail->next = skb;
854
855 sk->sk_backlog.tail = skb;
856 skb->next = NULL;
857}
858
859
860
861
862
863
864static inline bool sk_rcvqueues_full(const struct sock *sk, unsigned int limit)
865{
866 unsigned int qsize = sk->sk_backlog.len + atomic_read(&sk->sk_rmem_alloc);
867
868 return qsize > limit;
869}
870
871
872static inline __must_check int sk_add_backlog(struct sock *sk, struct sk_buff *skb,
873 unsigned int limit)
874{
875 if (sk_rcvqueues_full(sk, limit))
876 return -ENOBUFS;
877
878
879
880
881
882
883 if (skb_pfmemalloc(skb) && !sock_flag(sk, SOCK_MEMALLOC))
884 return -ENOMEM;
885
886 __sk_add_backlog(sk, skb);
887 sk->sk_backlog.len += skb->truesize;
888 return 0;
889}
890
891int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb);
892
893static inline int sk_backlog_rcv(struct sock *sk, struct sk_buff *skb)
894{
895 if (sk_memalloc_socks() && skb_pfmemalloc(skb))
896 return __sk_backlog_rcv(sk, skb);
897
898 return sk->sk_backlog_rcv(sk, skb);
899}
900
901static inline void sk_incoming_cpu_update(struct sock *sk)
902{
903 sk->sk_incoming_cpu = raw_smp_processor_id();
904}
905
906static inline void sock_rps_record_flow_hash(__u32 hash)
907{
908#ifdef CONFIG_RPS
909 struct rps_sock_flow_table *sock_flow_table;
910
911 rcu_read_lock();
912 sock_flow_table = rcu_dereference(rps_sock_flow_table);
913 rps_record_sock_flow(sock_flow_table, hash);
914 rcu_read_unlock();
915#endif
916}
917
918static inline void sock_rps_record_flow(const struct sock *sk)
919{
920#ifdef CONFIG_RPS
921 if (static_key_false(&rfs_needed)) {
922
923
924
925
926
927
928
929
930
931
932 if (sk->sk_state == TCP_ESTABLISHED)
933 sock_rps_record_flow_hash(sk->sk_rxhash);
934 }
935#endif
936}
937
938static inline void sock_rps_save_rxhash(struct sock *sk,
939 const struct sk_buff *skb)
940{
941#ifdef CONFIG_RPS
942 if (unlikely(sk->sk_rxhash != skb->hash))
943 sk->sk_rxhash = skb->hash;
944#endif
945}
946
947static inline void sock_rps_reset_rxhash(struct sock *sk)
948{
949#ifdef CONFIG_RPS
950 sk->sk_rxhash = 0;
951#endif
952}
953
954#define sk_wait_event(__sk, __timeo, __condition, __wait) \
955 ({ int __rc; \
956 release_sock(__sk); \
957 __rc = __condition; \
958 if (!__rc) { \
959 *(__timeo) = wait_woken(__wait, \
960 TASK_INTERRUPTIBLE, \
961 *(__timeo)); \
962 } \
963 sched_annotate_sleep(); \
964 lock_sock(__sk); \
965 __rc = __condition; \
966 __rc; \
967 })
968
969int sk_stream_wait_connect(struct sock *sk, long *timeo_p);
970int sk_stream_wait_memory(struct sock *sk, long *timeo_p);
971void sk_stream_wait_close(struct sock *sk, long timeo_p);
972int sk_stream_error(struct sock *sk, int flags, int err);
973void sk_stream_kill_queues(struct sock *sk);
974void sk_set_memalloc(struct sock *sk);
975void sk_clear_memalloc(struct sock *sk);
976
977void __sk_flush_backlog(struct sock *sk);
978
979static inline bool sk_flush_backlog(struct sock *sk)
980{
981 if (unlikely(READ_ONCE(sk->sk_backlog.tail))) {
982 __sk_flush_backlog(sk);
983 return true;
984 }
985 return false;
986}
987
988int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb);
989
990struct request_sock_ops;
991struct timewait_sock_ops;
992struct inet_hashinfo;
993struct raw_hashinfo;
994struct smc_hashinfo;
995struct module;
996
997
998
999
1000
1001static inline void sk_prot_clear_nulls(struct sock *sk, int size)
1002{
1003 if (offsetof(struct sock, sk_node.next) != 0)
1004 memset(sk, 0, offsetof(struct sock, sk_node.next));
1005 memset(&sk->sk_node.pprev, 0,
1006 size - offsetof(struct sock, sk_node.pprev));
1007}
1008
1009
1010
1011
1012struct proto {
1013 void (*close)(struct sock *sk,
1014 long timeout);
1015 int (*connect)(struct sock *sk,
1016 struct sockaddr *uaddr,
1017 int addr_len);
1018 int (*disconnect)(struct sock *sk, int flags);
1019
1020 struct sock * (*accept)(struct sock *sk, int flags, int *err,
1021 bool kern);
1022
1023 int (*ioctl)(struct sock *sk, int cmd,
1024 unsigned long arg);
1025 int (*init)(struct sock *sk);
1026 void (*destroy)(struct sock *sk);
1027 void (*shutdown)(struct sock *sk, int how);
1028 int (*setsockopt)(struct sock *sk, int level,
1029 int optname, char __user *optval,
1030 unsigned int optlen);
1031 int (*getsockopt)(struct sock *sk, int level,
1032 int optname, char __user *optval,
1033 int __user *option);
1034 void (*keepalive)(struct sock *sk, int valbool);
1035#ifdef CONFIG_COMPAT
1036 int (*compat_setsockopt)(struct sock *sk,
1037 int level,
1038 int optname, char __user *optval,
1039 unsigned int optlen);
1040 int (*compat_getsockopt)(struct sock *sk,
1041 int level,
1042 int optname, char __user *optval,
1043 int __user *option);
1044 int (*compat_ioctl)(struct sock *sk,
1045 unsigned int cmd, unsigned long arg);
1046#endif
1047 int (*sendmsg)(struct sock *sk, struct msghdr *msg,
1048 size_t len);
1049 int (*recvmsg)(struct sock *sk, struct msghdr *msg,
1050 size_t len, int noblock, int flags,
1051 int *addr_len);
1052 int (*sendpage)(struct sock *sk, struct page *page,
1053 int offset, size_t size, int flags);
1054 int (*bind)(struct sock *sk,
1055 struct sockaddr *uaddr, int addr_len);
1056
1057 int (*backlog_rcv) (struct sock *sk,
1058 struct sk_buff *skb);
1059
1060 void (*release_cb)(struct sock *sk);
1061
1062
1063 int (*hash)(struct sock *sk);
1064 void (*unhash)(struct sock *sk);
1065 void (*rehash)(struct sock *sk);
1066 int (*get_port)(struct sock *sk, unsigned short snum);
1067
1068
1069#ifdef CONFIG_PROC_FS
1070 unsigned int inuse_idx;
1071#endif
1072
1073 bool (*stream_memory_free)(const struct sock *sk);
1074
1075 void (*enter_memory_pressure)(struct sock *sk);
1076 atomic_long_t *memory_allocated;
1077 struct percpu_counter *sockets_allocated;
1078
1079
1080
1081
1082
1083
1084 int *memory_pressure;
1085 long *sysctl_mem;
1086 int *sysctl_wmem;
1087 int *sysctl_rmem;
1088 int max_header;
1089 bool no_autobind;
1090
1091 struct kmem_cache *slab;
1092 unsigned int obj_size;
1093 int slab_flags;
1094
1095 struct percpu_counter *orphan_count;
1096
1097 struct request_sock_ops *rsk_prot;
1098 struct timewait_sock_ops *twsk_prot;
1099
1100 union {
1101 struct inet_hashinfo *hashinfo;
1102 struct udp_table *udp_table;
1103 struct raw_hashinfo *raw_hash;
1104 struct smc_hashinfo *smc_hash;
1105 } h;
1106
1107 struct module *owner;
1108
1109 char name[32];
1110
1111 struct list_head node;
1112#ifdef SOCK_REFCNT_DEBUG
1113 atomic_t socks;
1114#endif
1115 int (*diag_destroy)(struct sock *sk, int err);
1116};
1117
1118int proto_register(struct proto *prot, int alloc_slab);
1119void proto_unregister(struct proto *prot);
1120
1121#ifdef SOCK_REFCNT_DEBUG
1122static inline void sk_refcnt_debug_inc(struct sock *sk)
1123{
1124 atomic_inc(&sk->sk_prot->socks);
1125}
1126
1127static inline void sk_refcnt_debug_dec(struct sock *sk)
1128{
1129 atomic_dec(&sk->sk_prot->socks);
1130 printk(KERN_DEBUG "%s socket %p released, %d are still alive\n",
1131 sk->sk_prot->name, sk, atomic_read(&sk->sk_prot->socks));
1132}
1133
1134static inline void sk_refcnt_debug_release(const struct sock *sk)
1135{
1136 if (atomic_read(&sk->sk_refcnt) != 1)
1137 printk(KERN_DEBUG "Destruction of the %s socket %p delayed, refcnt=%d\n",
1138 sk->sk_prot->name, sk, atomic_read(&sk->sk_refcnt));
1139}
1140#else
1141#define sk_refcnt_debug_inc(sk) do { } while (0)
1142#define sk_refcnt_debug_dec(sk) do { } while (0)
1143#define sk_refcnt_debug_release(sk) do { } while (0)
1144#endif
1145
1146static inline bool sk_stream_memory_free(const struct sock *sk)
1147{
1148 if (sk->sk_wmem_queued >= sk->sk_sndbuf)
1149 return false;
1150
1151 return sk->sk_prot->stream_memory_free ?
1152 sk->sk_prot->stream_memory_free(sk) : true;
1153}
1154
1155static inline bool sk_stream_is_writeable(const struct sock *sk)
1156{
1157 return sk_stream_wspace(sk) >= sk_stream_min_wspace(sk) &&
1158 sk_stream_memory_free(sk);
1159}
1160
1161static inline int sk_under_cgroup_hierarchy(struct sock *sk,
1162 struct cgroup *ancestor)
1163{
1164#ifdef CONFIG_SOCK_CGROUP_DATA
1165 return cgroup_is_descendant(sock_cgroup_ptr(&sk->sk_cgrp_data),
1166 ancestor);
1167#else
1168 return -ENOTSUPP;
1169#endif
1170}
1171
1172static inline bool sk_has_memory_pressure(const struct sock *sk)
1173{
1174 return sk->sk_prot->memory_pressure != NULL;
1175}
1176
1177static inline bool sk_under_memory_pressure(const struct sock *sk)
1178{
1179 if (!sk->sk_prot->memory_pressure)
1180 return false;
1181
1182 if (mem_cgroup_sockets_enabled && sk->sk_memcg &&
1183 mem_cgroup_under_socket_pressure(sk->sk_memcg))
1184 return true;
1185
1186 return !!*sk->sk_prot->memory_pressure;
1187}
1188
1189static inline void sk_leave_memory_pressure(struct sock *sk)
1190{
1191 int *memory_pressure = sk->sk_prot->memory_pressure;
1192
1193 if (!memory_pressure)
1194 return;
1195
1196 if (*memory_pressure)
1197 *memory_pressure = 0;
1198}
1199
1200static inline void sk_enter_memory_pressure(struct sock *sk)
1201{
1202 if (!sk->sk_prot->enter_memory_pressure)
1203 return;
1204
1205 sk->sk_prot->enter_memory_pressure(sk);
1206}
1207
1208static inline long
1209sk_memory_allocated(const struct sock *sk)
1210{
1211 return atomic_long_read(sk->sk_prot->memory_allocated);
1212}
1213
1214static inline long
1215sk_memory_allocated_add(struct sock *sk, int amt)
1216{
1217 return atomic_long_add_return(amt, sk->sk_prot->memory_allocated);
1218}
1219
1220static inline void
1221sk_memory_allocated_sub(struct sock *sk, int amt)
1222{
1223 atomic_long_sub(amt, sk->sk_prot->memory_allocated);
1224}
1225
1226static inline void sk_sockets_allocated_dec(struct sock *sk)
1227{
1228 percpu_counter_dec(sk->sk_prot->sockets_allocated);
1229}
1230
1231static inline void sk_sockets_allocated_inc(struct sock *sk)
1232{
1233 percpu_counter_inc(sk->sk_prot->sockets_allocated);
1234}
1235
1236static inline int
1237sk_sockets_allocated_read_positive(struct sock *sk)
1238{
1239 return percpu_counter_read_positive(sk->sk_prot->sockets_allocated);
1240}
1241
1242static inline int
1243proto_sockets_allocated_sum_positive(struct proto *prot)
1244{
1245 return percpu_counter_sum_positive(prot->sockets_allocated);
1246}
1247
1248static inline long
1249proto_memory_allocated(struct proto *prot)
1250{
1251 return atomic_long_read(prot->memory_allocated);
1252}
1253
1254static inline bool
1255proto_memory_pressure(struct proto *prot)
1256{
1257 if (!prot->memory_pressure)
1258 return false;
1259 return !!*prot->memory_pressure;
1260}
1261
1262
1263#ifdef CONFIG_PROC_FS
1264
1265void sock_prot_inuse_add(struct net *net, struct proto *prot, int inc);
1266int sock_prot_inuse_get(struct net *net, struct proto *proto);
1267#else
1268static inline void sock_prot_inuse_add(struct net *net, struct proto *prot,
1269 int inc)
1270{
1271}
1272#endif
1273
1274
1275
1276
1277
1278static inline int __sk_prot_rehash(struct sock *sk)
1279{
1280 sk->sk_prot->unhash(sk);
1281 return sk->sk_prot->hash(sk);
1282}
1283
1284
1285#define SOCK_DESTROY_TIME (10*HZ)
1286
1287
1288#define PROT_SOCK 1024
1289
1290#define SHUTDOWN_MASK 3
1291#define RCV_SHUTDOWN 1
1292#define SEND_SHUTDOWN 2
1293
1294#define SOCK_SNDBUF_LOCK 1
1295#define SOCK_RCVBUF_LOCK 2
1296#define SOCK_BINDADDR_LOCK 4
1297#define SOCK_BINDPORT_LOCK 8
1298
1299struct socket_alloc {
1300 struct socket socket;
1301 struct inode vfs_inode;
1302};
1303
1304static inline struct socket *SOCKET_I(struct inode *inode)
1305{
1306 return &container_of(inode, struct socket_alloc, vfs_inode)->socket;
1307}
1308
1309static inline struct inode *SOCK_INODE(struct socket *socket)
1310{
1311 return &container_of(socket, struct socket_alloc, socket)->vfs_inode;
1312}
1313
1314
1315
1316
1317int __sk_mem_raise_allocated(struct sock *sk, int size, int amt, int kind);
1318int __sk_mem_schedule(struct sock *sk, int size, int kind);
1319void __sk_mem_reduce_allocated(struct sock *sk, int amount);
1320void __sk_mem_reclaim(struct sock *sk, int amount);
1321
1322
1323
1324
1325#define SK_MEM_QUANTUM 4096
1326#define SK_MEM_QUANTUM_SHIFT ilog2(SK_MEM_QUANTUM)
1327#define SK_MEM_SEND 0
1328#define SK_MEM_RECV 1
1329
1330
1331static inline long sk_prot_mem_limits(const struct sock *sk, int index)
1332{
1333 long val = sk->sk_prot->sysctl_mem[index];
1334
1335#if PAGE_SIZE > SK_MEM_QUANTUM
1336 val <<= PAGE_SHIFT - SK_MEM_QUANTUM_SHIFT;
1337#elif PAGE_SIZE < SK_MEM_QUANTUM
1338 val >>= SK_MEM_QUANTUM_SHIFT - PAGE_SHIFT;
1339#endif
1340 return val;
1341}
1342
1343static inline int sk_mem_pages(int amt)
1344{
1345 return (amt + SK_MEM_QUANTUM - 1) >> SK_MEM_QUANTUM_SHIFT;
1346}
1347
1348static inline bool sk_has_account(struct sock *sk)
1349{
1350
1351 return !!sk->sk_prot->memory_allocated;
1352}
1353
1354static inline bool sk_wmem_schedule(struct sock *sk, int size)
1355{
1356 if (!sk_has_account(sk))
1357 return true;
1358 return size <= sk->sk_forward_alloc ||
1359 __sk_mem_schedule(sk, size, SK_MEM_SEND);
1360}
1361
1362static inline bool
1363sk_rmem_schedule(struct sock *sk, struct sk_buff *skb, int size)
1364{
1365 if (!sk_has_account(sk))
1366 return true;
1367 return size<= sk->sk_forward_alloc ||
1368 __sk_mem_schedule(sk, size, SK_MEM_RECV) ||
1369 skb_pfmemalloc(skb);
1370}
1371
1372static inline void sk_mem_reclaim(struct sock *sk)
1373{
1374 if (!sk_has_account(sk))
1375 return;
1376 if (sk->sk_forward_alloc >= SK_MEM_QUANTUM)
1377 __sk_mem_reclaim(sk, sk->sk_forward_alloc);
1378}
1379
1380static inline void sk_mem_reclaim_partial(struct sock *sk)
1381{
1382 if (!sk_has_account(sk))
1383 return;
1384 if (sk->sk_forward_alloc > SK_MEM_QUANTUM)
1385 __sk_mem_reclaim(sk, sk->sk_forward_alloc - 1);
1386}
1387
1388static inline void sk_mem_charge(struct sock *sk, int size)
1389{
1390 if (!sk_has_account(sk))
1391 return;
1392 sk->sk_forward_alloc -= size;
1393}
1394
1395static inline void sk_mem_uncharge(struct sock *sk, int size)
1396{
1397 if (!sk_has_account(sk))
1398 return;
1399 sk->sk_forward_alloc += size;
1400
1401
1402
1403
1404
1405
1406
1407
1408 if (unlikely(sk->sk_forward_alloc >= 1 << 21))
1409 __sk_mem_reclaim(sk, 1 << 20);
1410}
1411
1412static inline void sk_wmem_free_skb(struct sock *sk, struct sk_buff *skb)
1413{
1414 sock_set_flag(sk, SOCK_QUEUE_SHRUNK);
1415 sk->sk_wmem_queued -= skb->truesize;
1416 sk_mem_uncharge(sk, skb->truesize);
1417 __kfree_skb(skb);
1418}
1419
1420static inline void sock_release_ownership(struct sock *sk)
1421{
1422 if (sk->sk_lock.owned) {
1423 sk->sk_lock.owned = 0;
1424
1425
1426 mutex_release(&sk->sk_lock.dep_map, 1, _RET_IP_);
1427 }
1428}
1429
1430
1431
1432
1433
1434
1435
1436
1437#define sock_lock_init_class_and_name(sk, sname, skey, name, key) \
1438do { \
1439 sk->sk_lock.owned = 0; \
1440 init_waitqueue_head(&sk->sk_lock.wq); \
1441 spin_lock_init(&(sk)->sk_lock.slock); \
1442 debug_check_no_locks_freed((void *)&(sk)->sk_lock, \
1443 sizeof((sk)->sk_lock)); \
1444 lockdep_set_class_and_name(&(sk)->sk_lock.slock, \
1445 (skey), (sname)); \
1446 lockdep_init_map(&(sk)->sk_lock.dep_map, (name), (key), 0); \
1447} while (0)
1448
1449#ifdef CONFIG_LOCKDEP
1450static inline bool lockdep_sock_is_held(const struct sock *csk)
1451{
1452 struct sock *sk = (struct sock *)csk;
1453
1454 return lockdep_is_held(&sk->sk_lock) ||
1455 lockdep_is_held(&sk->sk_lock.slock);
1456}
1457#endif
1458
1459void lock_sock_nested(struct sock *sk, int subclass);
1460
1461static inline void lock_sock(struct sock *sk)
1462{
1463 lock_sock_nested(sk, 0);
1464}
1465
1466void release_sock(struct sock *sk);
1467
1468
1469#define bh_lock_sock(__sk) spin_lock(&((__sk)->sk_lock.slock))
1470#define bh_lock_sock_nested(__sk) \
1471 spin_lock_nested(&((__sk)->sk_lock.slock), \
1472 SINGLE_DEPTH_NESTING)
1473#define bh_unlock_sock(__sk) spin_unlock(&((__sk)->sk_lock.slock))
1474
1475bool lock_sock_fast(struct sock *sk);
1476
1477
1478
1479
1480
1481
1482
1483
1484static inline void unlock_sock_fast(struct sock *sk, bool slow)
1485{
1486 if (slow)
1487 release_sock(sk);
1488 else
1489 spin_unlock_bh(&sk->sk_lock.slock);
1490}
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506static inline void sock_owned_by_me(const struct sock *sk)
1507{
1508#ifdef CONFIG_LOCKDEP
1509 WARN_ON_ONCE(!lockdep_sock_is_held(sk) && debug_locks);
1510#endif
1511}
1512
1513static inline bool sock_owned_by_user(const struct sock *sk)
1514{
1515 sock_owned_by_me(sk);
1516 return sk->sk_lock.owned;
1517}
1518
1519
1520static inline bool sock_allow_reclassification(const struct sock *csk)
1521{
1522 struct sock *sk = (struct sock *)csk;
1523
1524 return !sk->sk_lock.owned && !spin_is_locked(&sk->sk_lock.slock);
1525}
1526
1527struct sock *sk_alloc(struct net *net, int family, gfp_t priority,
1528 struct proto *prot, int kern);
1529void sk_free(struct sock *sk);
1530void sk_destruct(struct sock *sk);
1531struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority);
1532void sk_free_unlock_clone(struct sock *sk);
1533
1534struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force,
1535 gfp_t priority);
1536void __sock_wfree(struct sk_buff *skb);
1537void sock_wfree(struct sk_buff *skb);
1538void skb_orphan_partial(struct sk_buff *skb);
1539void sock_rfree(struct sk_buff *skb);
1540void sock_efree(struct sk_buff *skb);
1541#ifdef CONFIG_INET
1542void sock_edemux(struct sk_buff *skb);
1543#else
1544#define sock_edemux sock_efree
1545#endif
1546
1547int sock_setsockopt(struct socket *sock, int level, int op,
1548 char __user *optval, unsigned int optlen);
1549
1550int sock_getsockopt(struct socket *sock, int level, int op,
1551 char __user *optval, int __user *optlen);
1552struct sk_buff *sock_alloc_send_skb(struct sock *sk, unsigned long size,
1553 int noblock, int *errcode);
1554struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len,
1555 unsigned long data_len, int noblock,
1556 int *errcode, int max_page_order);
1557void *sock_kmalloc(struct sock *sk, int size, gfp_t priority);
1558void sock_kfree_s(struct sock *sk, void *mem, int size);
1559void sock_kzfree_s(struct sock *sk, void *mem, int size);
1560void sk_send_sigurg(struct sock *sk);
1561
1562struct sockcm_cookie {
1563 u32 mark;
1564 u16 tsflags;
1565};
1566
1567int __sock_cmsg_send(struct sock *sk, struct msghdr *msg, struct cmsghdr *cmsg,
1568 struct sockcm_cookie *sockc);
1569int sock_cmsg_send(struct sock *sk, struct msghdr *msg,
1570 struct sockcm_cookie *sockc);
1571
1572
1573
1574
1575
1576int sock_no_bind(struct socket *, struct sockaddr *, int);
1577int sock_no_connect(struct socket *, struct sockaddr *, int, int);
1578int sock_no_socketpair(struct socket *, struct socket *);
1579int sock_no_accept(struct socket *, struct socket *, int, bool);
1580int sock_no_getname(struct socket *, struct sockaddr *, int *, int);
1581unsigned int sock_no_poll(struct file *, struct socket *,
1582 struct poll_table_struct *);
1583int sock_no_ioctl(struct socket *, unsigned int, unsigned long);
1584int sock_no_listen(struct socket *, int);
1585int sock_no_shutdown(struct socket *, int);
1586int sock_no_getsockopt(struct socket *, int , int, char __user *, int __user *);
1587int sock_no_setsockopt(struct socket *, int, int, char __user *, unsigned int);
1588int sock_no_sendmsg(struct socket *, struct msghdr *, size_t);
1589int sock_no_recvmsg(struct socket *, struct msghdr *, size_t, int);
1590int sock_no_mmap(struct file *file, struct socket *sock,
1591 struct vm_area_struct *vma);
1592ssize_t sock_no_sendpage(struct socket *sock, struct page *page, int offset,
1593 size_t size, int flags);
1594
1595
1596
1597
1598
1599int sock_common_getsockopt(struct socket *sock, int level, int optname,
1600 char __user *optval, int __user *optlen);
1601int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
1602 int flags);
1603int sock_common_setsockopt(struct socket *sock, int level, int optname,
1604 char __user *optval, unsigned int optlen);
1605int compat_sock_common_getsockopt(struct socket *sock, int level,
1606 int optname, char __user *optval, int __user *optlen);
1607int compat_sock_common_setsockopt(struct socket *sock, int level,
1608 int optname, char __user *optval, unsigned int optlen);
1609
1610void sk_common_release(struct sock *sk);
1611
1612
1613
1614
1615
1616
1617void sock_init_data(struct socket *sock, struct sock *sk);
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645static inline void sock_put(struct sock *sk)
1646{
1647 if (atomic_dec_and_test(&sk->sk_refcnt))
1648 sk_free(sk);
1649}
1650
1651
1652
1653void sock_gen_put(struct sock *sk);
1654
1655int __sk_receive_skb(struct sock *sk, struct sk_buff *skb, const int nested,
1656 unsigned int trim_cap, bool refcounted);
1657static inline int sk_receive_skb(struct sock *sk, struct sk_buff *skb,
1658 const int nested)
1659{
1660 return __sk_receive_skb(sk, skb, nested, 1, true);
1661}
1662
1663static inline void sk_tx_queue_set(struct sock *sk, int tx_queue)
1664{
1665 sk->sk_tx_queue_mapping = tx_queue;
1666}
1667
1668static inline void sk_tx_queue_clear(struct sock *sk)
1669{
1670 sk->sk_tx_queue_mapping = -1;
1671}
1672
1673static inline int sk_tx_queue_get(const struct sock *sk)
1674{
1675 return sk ? sk->sk_tx_queue_mapping : -1;
1676}
1677
1678static inline void sk_set_socket(struct sock *sk, struct socket *sock)
1679{
1680 sk_tx_queue_clear(sk);
1681 sk->sk_socket = sock;
1682}
1683
1684static inline wait_queue_head_t *sk_sleep(struct sock *sk)
1685{
1686 BUILD_BUG_ON(offsetof(struct socket_wq, wait) != 0);
1687 return &rcu_dereference_raw(sk->sk_wq)->wait;
1688}
1689
1690
1691
1692
1693
1694
1695
1696static inline void sock_orphan(struct sock *sk)
1697{
1698 write_lock_bh(&sk->sk_callback_lock);
1699 sock_set_flag(sk, SOCK_DEAD);
1700 sk_set_socket(sk, NULL);
1701 sk->sk_wq = NULL;
1702 write_unlock_bh(&sk->sk_callback_lock);
1703}
1704
1705static inline void sock_graft(struct sock *sk, struct socket *parent)
1706{
1707 write_lock_bh(&sk->sk_callback_lock);
1708 sk->sk_wq = parent->wq;
1709 parent->sk = sk;
1710 sk_set_socket(sk, parent);
1711 sk->sk_uid = SOCK_INODE(parent)->i_uid;
1712 security_sock_graft(sk, parent);
1713 write_unlock_bh(&sk->sk_callback_lock);
1714}
1715
1716kuid_t sock_i_uid(struct sock *sk);
1717unsigned long sock_i_ino(struct sock *sk);
1718
1719static inline kuid_t sock_net_uid(const struct net *net, const struct sock *sk)
1720{
1721 return sk ? sk->sk_uid : make_kuid(net->user_ns, 0);
1722}
1723
1724static inline u32 net_tx_rndhash(void)
1725{
1726 u32 v = prandom_u32();
1727
1728 return v ?: 1;
1729}
1730
1731static inline void sk_set_txhash(struct sock *sk)
1732{
1733 sk->sk_txhash = net_tx_rndhash();
1734}
1735
1736static inline void sk_rethink_txhash(struct sock *sk)
1737{
1738 if (sk->sk_txhash)
1739 sk_set_txhash(sk);
1740}
1741
1742static inline struct dst_entry *
1743__sk_dst_get(struct sock *sk)
1744{
1745 return rcu_dereference_check(sk->sk_dst_cache,
1746 lockdep_sock_is_held(sk));
1747}
1748
1749static inline struct dst_entry *
1750sk_dst_get(struct sock *sk)
1751{
1752 struct dst_entry *dst;
1753
1754 rcu_read_lock();
1755 dst = rcu_dereference(sk->sk_dst_cache);
1756 if (dst && !atomic_inc_not_zero(&dst->__refcnt))
1757 dst = NULL;
1758 rcu_read_unlock();
1759 return dst;
1760}
1761
1762static inline void dst_negative_advice(struct sock *sk)
1763{
1764 struct dst_entry *ndst, *dst = __sk_dst_get(sk);
1765
1766 sk_rethink_txhash(sk);
1767
1768 if (dst && dst->ops->negative_advice) {
1769 ndst = dst->ops->negative_advice(dst);
1770
1771 if (ndst != dst) {
1772 rcu_assign_pointer(sk->sk_dst_cache, ndst);
1773 sk_tx_queue_clear(sk);
1774 sk->sk_dst_pending_confirm = 0;
1775 }
1776 }
1777}
1778
1779static inline void
1780__sk_dst_set(struct sock *sk, struct dst_entry *dst)
1781{
1782 struct dst_entry *old_dst;
1783
1784 sk_tx_queue_clear(sk);
1785 sk->sk_dst_pending_confirm = 0;
1786 old_dst = rcu_dereference_protected(sk->sk_dst_cache,
1787 lockdep_sock_is_held(sk));
1788 rcu_assign_pointer(sk->sk_dst_cache, dst);
1789 dst_release(old_dst);
1790}
1791
1792static inline void
1793sk_dst_set(struct sock *sk, struct dst_entry *dst)
1794{
1795 struct dst_entry *old_dst;
1796
1797 sk_tx_queue_clear(sk);
1798 sk->sk_dst_pending_confirm = 0;
1799 old_dst = xchg((__force struct dst_entry **)&sk->sk_dst_cache, dst);
1800 dst_release(old_dst);
1801}
1802
1803static inline void
1804__sk_dst_reset(struct sock *sk)
1805{
1806 __sk_dst_set(sk, NULL);
1807}
1808
1809static inline void
1810sk_dst_reset(struct sock *sk)
1811{
1812 sk_dst_set(sk, NULL);
1813}
1814
1815struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie);
1816
1817struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie);
1818
1819static inline void sk_dst_confirm(struct sock *sk)
1820{
1821 if (!sk->sk_dst_pending_confirm)
1822 sk->sk_dst_pending_confirm = 1;
1823}
1824
1825static inline void sock_confirm_neigh(struct sk_buff *skb, struct neighbour *n)
1826{
1827 if (skb_get_dst_pending_confirm(skb)) {
1828 struct sock *sk = skb->sk;
1829 unsigned long now = jiffies;
1830
1831
1832 if (n->confirmed != now)
1833 n->confirmed = now;
1834 if (sk && sk->sk_dst_pending_confirm)
1835 sk->sk_dst_pending_confirm = 0;
1836 }
1837}
1838
1839bool sk_mc_loop(struct sock *sk);
1840
1841static inline bool sk_can_gso(const struct sock *sk)
1842{
1843 return net_gso_ok(sk->sk_route_caps, sk->sk_gso_type);
1844}
1845
1846void sk_setup_caps(struct sock *sk, struct dst_entry *dst);
1847
1848static inline void sk_nocaps_add(struct sock *sk, netdev_features_t flags)
1849{
1850 sk->sk_route_nocaps |= flags;
1851 sk->sk_route_caps &= ~flags;
1852}
1853
1854static inline bool sk_check_csum_caps(struct sock *sk)
1855{
1856 return (sk->sk_route_caps & NETIF_F_HW_CSUM) ||
1857 (sk->sk_family == PF_INET &&
1858 (sk->sk_route_caps & NETIF_F_IP_CSUM)) ||
1859 (sk->sk_family == PF_INET6 &&
1860 (sk->sk_route_caps & NETIF_F_IPV6_CSUM));
1861}
1862
1863static inline int skb_do_copy_data_nocache(struct sock *sk, struct sk_buff *skb,
1864 struct iov_iter *from, char *to,
1865 int copy, int offset)
1866{
1867 if (skb->ip_summed == CHECKSUM_NONE) {
1868 __wsum csum = 0;
1869 if (!csum_and_copy_from_iter_full(to, copy, &csum, from))
1870 return -EFAULT;
1871 skb->csum = csum_block_add(skb->csum, csum, offset);
1872 } else if (sk->sk_route_caps & NETIF_F_NOCACHE_COPY) {
1873 if (!copy_from_iter_full_nocache(to, copy, from))
1874 return -EFAULT;
1875 } else if (!copy_from_iter_full(to, copy, from))
1876 return -EFAULT;
1877
1878 return 0;
1879}
1880
1881static inline int skb_add_data_nocache(struct sock *sk, struct sk_buff *skb,
1882 struct iov_iter *from, int copy)
1883{
1884 int err, offset = skb->len;
1885
1886 err = skb_do_copy_data_nocache(sk, skb, from, skb_put(skb, copy),
1887 copy, offset);
1888 if (err)
1889 __skb_trim(skb, offset);
1890
1891 return err;
1892}
1893
1894static inline int skb_copy_to_page_nocache(struct sock *sk, struct iov_iter *from,
1895 struct sk_buff *skb,
1896 struct page *page,
1897 int off, int copy)
1898{
1899 int err;
1900
1901 err = skb_do_copy_data_nocache(sk, skb, from, page_address(page) + off,
1902 copy, skb->len);
1903 if (err)
1904 return err;
1905
1906 skb->len += copy;
1907 skb->data_len += copy;
1908 skb->truesize += copy;
1909 sk->sk_wmem_queued += copy;
1910 sk_mem_charge(sk, copy);
1911 return 0;
1912}
1913
1914
1915
1916
1917
1918
1919
1920static inline int sk_wmem_alloc_get(const struct sock *sk)
1921{
1922 return atomic_read(&sk->sk_wmem_alloc) - 1;
1923}
1924
1925
1926
1927
1928
1929
1930
1931static inline int sk_rmem_alloc_get(const struct sock *sk)
1932{
1933 return atomic_read(&sk->sk_rmem_alloc);
1934}
1935
1936
1937
1938
1939
1940
1941
1942static inline bool sk_has_allocations(const struct sock *sk)
1943{
1944 return sk_wmem_alloc_get(sk) || sk_rmem_alloc_get(sk);
1945}
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979static inline bool skwq_has_sleeper(struct socket_wq *wq)
1980{
1981 return wq && wq_has_sleeper(&wq->wait);
1982}
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992static inline void sock_poll_wait(struct file *filp,
1993 wait_queue_head_t *wait_address, poll_table *p)
1994{
1995 if (!poll_does_not_wait(p) && wait_address) {
1996 poll_wait(filp, wait_address, p);
1997
1998
1999
2000
2001
2002 smp_mb();
2003 }
2004}
2005
2006static inline void skb_set_hash_from_sk(struct sk_buff *skb, struct sock *sk)
2007{
2008 if (sk->sk_txhash) {
2009 skb->l4_hash = 1;
2010 skb->hash = sk->sk_txhash;
2011 }
2012}
2013
2014void skb_set_owner_w(struct sk_buff *skb, struct sock *sk);
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024static inline void skb_set_owner_r(struct sk_buff *skb, struct sock *sk)
2025{
2026 skb_orphan(skb);
2027 skb->sk = sk;
2028 skb->destructor = sock_rfree;
2029 atomic_add(skb->truesize, &sk->sk_rmem_alloc);
2030 sk_mem_charge(sk, skb->truesize);
2031}
2032
2033void sk_reset_timer(struct sock *sk, struct timer_list *timer,
2034 unsigned long expires);
2035
2036void sk_stop_timer(struct sock *sk, struct timer_list *timer);
2037
2038int __sk_queue_drop_skb(struct sock *sk, struct sk_buff *skb,
2039 unsigned int flags,
2040 void (*destructor)(struct sock *sk,
2041 struct sk_buff *skb));
2042int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb);
2043int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb);
2044
2045int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb);
2046struct sk_buff *sock_dequeue_err_skb(struct sock *sk);
2047
2048
2049
2050
2051
2052static inline int sock_error(struct sock *sk)
2053{
2054 int err;
2055 if (likely(!sk->sk_err))
2056 return 0;
2057 err = xchg(&sk->sk_err, 0);
2058 return -err;
2059}
2060
2061static inline unsigned long sock_wspace(struct sock *sk)
2062{
2063 int amt = 0;
2064
2065 if (!(sk->sk_shutdown & SEND_SHUTDOWN)) {
2066 amt = sk->sk_sndbuf - atomic_read(&sk->sk_wmem_alloc);
2067 if (amt < 0)
2068 amt = 0;
2069 }
2070 return amt;
2071}
2072
2073
2074
2075
2076
2077static inline void sk_set_bit(int nr, struct sock *sk)
2078{
2079 if ((nr == SOCKWQ_ASYNC_NOSPACE || nr == SOCKWQ_ASYNC_WAITDATA) &&
2080 !sock_flag(sk, SOCK_FASYNC))
2081 return;
2082
2083 set_bit(nr, &sk->sk_wq_raw->flags);
2084}
2085
2086static inline void sk_clear_bit(int nr, struct sock *sk)
2087{
2088 if ((nr == SOCKWQ_ASYNC_NOSPACE || nr == SOCKWQ_ASYNC_WAITDATA) &&
2089 !sock_flag(sk, SOCK_FASYNC))
2090 return;
2091
2092 clear_bit(nr, &sk->sk_wq_raw->flags);
2093}
2094
2095static inline void sk_wake_async(const struct sock *sk, int how, int band)
2096{
2097 if (sock_flag(sk, SOCK_FASYNC)) {
2098 rcu_read_lock();
2099 sock_wake_async(rcu_dereference(sk->sk_wq), how, band);
2100 rcu_read_unlock();
2101 }
2102}
2103
2104
2105
2106
2107
2108
2109#define TCP_SKB_MIN_TRUESIZE (2048 + SKB_DATA_ALIGN(sizeof(struct sk_buff)))
2110
2111#define SOCK_MIN_SNDBUF (TCP_SKB_MIN_TRUESIZE * 2)
2112#define SOCK_MIN_RCVBUF TCP_SKB_MIN_TRUESIZE
2113
2114static inline void sk_stream_moderate_sndbuf(struct sock *sk)
2115{
2116 if (!(sk->sk_userlocks & SOCK_SNDBUF_LOCK)) {
2117 sk->sk_sndbuf = min(sk->sk_sndbuf, sk->sk_wmem_queued >> 1);
2118 sk->sk_sndbuf = max_t(u32, sk->sk_sndbuf, SOCK_MIN_SNDBUF);
2119 }
2120}
2121
2122struct sk_buff *sk_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp,
2123 bool force_schedule);
2124
2125
2126
2127
2128
2129
2130
2131
2132static inline struct page_frag *sk_page_frag(struct sock *sk)
2133{
2134 if (gfpflags_allow_blocking(sk->sk_allocation))
2135 return ¤t->task_frag;
2136
2137 return &sk->sk_frag;
2138}
2139
2140bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag);
2141
2142
2143
2144
2145static inline bool sock_writeable(const struct sock *sk)
2146{
2147 return atomic_read(&sk->sk_wmem_alloc) < (sk->sk_sndbuf >> 1);
2148}
2149
2150static inline gfp_t gfp_any(void)
2151{
2152 return in_softirq() ? GFP_ATOMIC : GFP_KERNEL;
2153}
2154
2155static inline long sock_rcvtimeo(const struct sock *sk, bool noblock)
2156{
2157 return noblock ? 0 : sk->sk_rcvtimeo;
2158}
2159
2160static inline long sock_sndtimeo(const struct sock *sk, bool noblock)
2161{
2162 return noblock ? 0 : sk->sk_sndtimeo;
2163}
2164
2165static inline int sock_rcvlowat(const struct sock *sk, int waitall, int len)
2166{
2167 return (waitall ? len : min_t(int, sk->sk_rcvlowat, len)) ? : 1;
2168}
2169
2170
2171
2172
2173static inline int sock_intr_errno(long timeo)
2174{
2175 return timeo == MAX_SCHEDULE_TIMEOUT ? -ERESTARTSYS : -EINTR;
2176}
2177
2178struct sock_skb_cb {
2179 u32 dropcount;
2180};
2181
2182
2183
2184
2185
2186#define SOCK_SKB_CB_OFFSET ((FIELD_SIZEOF(struct sk_buff, cb) - \
2187 sizeof(struct sock_skb_cb)))
2188
2189#define SOCK_SKB_CB(__skb) ((struct sock_skb_cb *)((__skb)->cb + \
2190 SOCK_SKB_CB_OFFSET))
2191
2192#define sock_skb_cb_check_size(size) \
2193 BUILD_BUG_ON((size) > SOCK_SKB_CB_OFFSET)
2194
2195static inline void
2196sock_skb_set_dropcount(const struct sock *sk, struct sk_buff *skb)
2197{
2198 SOCK_SKB_CB(skb)->dropcount = sock_flag(sk, SOCK_RXQ_OVFL) ?
2199 atomic_read(&sk->sk_drops) : 0;
2200}
2201
2202static inline void sk_drops_add(struct sock *sk, const struct sk_buff *skb)
2203{
2204 int segs = max_t(u16, 1, skb_shinfo(skb)->gso_segs);
2205
2206 atomic_add(segs, &sk->sk_drops);
2207}
2208
2209void __sock_recv_timestamp(struct msghdr *msg, struct sock *sk,
2210 struct sk_buff *skb);
2211void __sock_recv_wifi_status(struct msghdr *msg, struct sock *sk,
2212 struct sk_buff *skb);
2213
2214static inline void
2215sock_recv_timestamp(struct msghdr *msg, struct sock *sk, struct sk_buff *skb)
2216{
2217 ktime_t kt = skb->tstamp;
2218 struct skb_shared_hwtstamps *hwtstamps = skb_hwtstamps(skb);
2219
2220
2221
2222
2223
2224
2225
2226 if (sock_flag(sk, SOCK_RCVTSTAMP) ||
2227 (sk->sk_tsflags & SOF_TIMESTAMPING_RX_SOFTWARE) ||
2228 (kt && sk->sk_tsflags & SOF_TIMESTAMPING_SOFTWARE) ||
2229 (hwtstamps->hwtstamp &&
2230 (sk->sk_tsflags & SOF_TIMESTAMPING_RAW_HARDWARE)))
2231 __sock_recv_timestamp(msg, sk, skb);
2232 else
2233 sk->sk_stamp = kt;
2234
2235 if (sock_flag(sk, SOCK_WIFI_STATUS) && skb->wifi_acked_valid)
2236 __sock_recv_wifi_status(msg, sk, skb);
2237}
2238
2239void __sock_recv_ts_and_drops(struct msghdr *msg, struct sock *sk,
2240 struct sk_buff *skb);
2241
2242#define SK_DEFAULT_STAMP (-1L * NSEC_PER_SEC)
2243static inline void sock_recv_ts_and_drops(struct msghdr *msg, struct sock *sk,
2244 struct sk_buff *skb)
2245{
2246#define FLAGS_TS_OR_DROPS ((1UL << SOCK_RXQ_OVFL) | \
2247 (1UL << SOCK_RCVTSTAMP))
2248#define TSFLAGS_ANY (SOF_TIMESTAMPING_SOFTWARE | \
2249 SOF_TIMESTAMPING_RAW_HARDWARE)
2250
2251 if (sk->sk_flags & FLAGS_TS_OR_DROPS || sk->sk_tsflags & TSFLAGS_ANY)
2252 __sock_recv_ts_and_drops(msg, sk, skb);
2253 else if (unlikely(sock_flag(sk, SOCK_TIMESTAMP)))
2254 sk->sk_stamp = skb->tstamp;
2255 else if (unlikely(sk->sk_stamp == SK_DEFAULT_STAMP))
2256 sk->sk_stamp = 0;
2257}
2258
2259void __sock_tx_timestamp(__u16 tsflags, __u8 *tx_flags);
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269static inline void sock_tx_timestamp(const struct sock *sk, __u16 tsflags,
2270 __u8 *tx_flags)
2271{
2272 if (unlikely(tsflags))
2273 __sock_tx_timestamp(tsflags, tx_flags);
2274 if (unlikely(sock_flag(sk, SOCK_WIFI_STATUS)))
2275 *tx_flags |= SKBTX_WIFI_STATUS;
2276}
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286static inline void sk_eat_skb(struct sock *sk, struct sk_buff *skb)
2287{
2288 __skb_unlink(skb, &sk->sk_receive_queue);
2289 __kfree_skb(skb);
2290}
2291
2292static inline
2293struct net *sock_net(const struct sock *sk)
2294{
2295 return read_pnet(&sk->sk_net);
2296}
2297
2298static inline
2299void sock_net_set(struct sock *sk, struct net *net)
2300{
2301 write_pnet(&sk->sk_net, net);
2302}
2303
2304static inline struct sock *skb_steal_sock(struct sk_buff *skb)
2305{
2306 if (skb->sk) {
2307 struct sock *sk = skb->sk;
2308
2309 skb->destructor = NULL;
2310 skb->sk = NULL;
2311 return sk;
2312 }
2313 return NULL;
2314}
2315
2316
2317
2318
2319static inline bool sk_fullsock(const struct sock *sk)
2320{
2321 return (1 << sk->sk_state) & ~(TCPF_TIME_WAIT | TCPF_NEW_SYN_RECV);
2322}
2323
2324
2325
2326
2327static inline bool sk_listener(const struct sock *sk)
2328{
2329 return (1 << sk->sk_state) & (TCPF_LISTEN | TCPF_NEW_SYN_RECV);
2330}
2331
2332
2333
2334
2335
2336
2337
2338
2339static inline int sk_state_load(const struct sock *sk)
2340{
2341 return smp_load_acquire(&sk->sk_state);
2342}
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352static inline void sk_state_store(struct sock *sk, int newstate)
2353{
2354 smp_store_release(&sk->sk_state, newstate);
2355}
2356
2357void sock_enable_timestamp(struct sock *sk, int flag);
2358int sock_get_timestamp(struct sock *, struct timeval __user *);
2359int sock_get_timestampns(struct sock *, struct timespec __user *);
2360int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len, int level,
2361 int type);
2362
2363bool sk_ns_capable(const struct sock *sk,
2364 struct user_namespace *user_ns, int cap);
2365bool sk_capable(const struct sock *sk, int cap);
2366bool sk_net_capable(const struct sock *sk, int cap);
2367
2368void sk_get_meminfo(const struct sock *sk, u32 *meminfo);
2369
2370extern __u32 sysctl_wmem_max;
2371extern __u32 sysctl_rmem_max;
2372
2373extern int sysctl_tstamp_allow_data;
2374extern int sysctl_optmem_max;
2375
2376extern __u32 sysctl_wmem_default;
2377extern __u32 sysctl_rmem_default;
2378
2379#endif
2380