1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40#ifndef _SOCK_H
41#define _SOCK_H
42
43#include <linux/hardirq.h>
44#include <linux/kernel.h>
45#include <linux/list.h>
46#include <linux/list_nulls.h>
47#include <linux/timer.h>
48#include <linux/cache.h>
49#include <linux/bitops.h>
50#include <linux/lockdep.h>
51#include <linux/netdevice.h>
52#include <linux/skbuff.h>
53#include <linux/mm.h>
54#include <linux/security.h>
55#include <linux/slab.h>
56#include <linux/uaccess.h>
57#include <linux/page_counter.h>
58#include <linux/memcontrol.h>
59#include <linux/static_key.h>
60#include <linux/sched.h>
61#include <linux/wait.h>
62#include <linux/cgroup-defs.h>
63#include <linux/rbtree.h>
64#include <linux/filter.h>
65#include <linux/rculist_nulls.h>
66#include <linux/poll.h>
67
68#include <linux/atomic.h>
69#include <linux/refcount.h>
70#include <net/dst.h>
71#include <net/checksum.h>
72#include <net/tcp_states.h>
73#include <linux/net_tstamp.h>
74#include <net/smc.h>
75#include <net/l3mdev.h>
76
77
78
79
80
81
82
83
84#define SOCK_DEBUGGING
85#ifdef SOCK_DEBUGGING
86#define SOCK_DEBUG(sk, msg...) do { if ((sk) && sock_flag((sk), SOCK_DBG)) \
87 printk(KERN_DEBUG msg); } while (0)
88#else
89
90static inline __printf(2, 3)
91void SOCK_DEBUG(const struct sock *sk, const char *msg, ...)
92{
93}
94#endif
95
96
97
98
99
100typedef struct {
101 spinlock_t slock;
102 int owned;
103 wait_queue_head_t wq;
104
105
106
107
108
109
110#ifdef CONFIG_DEBUG_LOCK_ALLOC
111 struct lockdep_map dep_map;
112#endif
113} socket_lock_t;
114
115struct sock;
116struct proto;
117struct net;
118
119typedef __u32 __bitwise __portpair;
120typedef __u64 __bitwise __addrpair;
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151struct sock_common {
152
153
154
155 union {
156 __addrpair skc_addrpair;
157 struct {
158 __be32 skc_daddr;
159 __be32 skc_rcv_saddr;
160 };
161 };
162 union {
163 unsigned int skc_hash;
164 __u16 skc_u16hashes[2];
165 };
166
167 union {
168 __portpair skc_portpair;
169 struct {
170 __be16 skc_dport;
171 __u16 skc_num;
172 };
173 };
174
175 unsigned short skc_family;
176 volatile unsigned char skc_state;
177 unsigned char skc_reuse:4;
178 unsigned char skc_reuseport:1;
179 unsigned char skc_ipv6only:1;
180 unsigned char skc_net_refcnt:1;
181 int skc_bound_dev_if;
182 union {
183 struct hlist_node skc_bind_node;
184 struct hlist_node skc_portaddr_node;
185 };
186 struct proto *skc_prot;
187 possible_net_t skc_net;
188
189#if IS_ENABLED(CONFIG_IPV6)
190 struct in6_addr skc_v6_daddr;
191 struct in6_addr skc_v6_rcv_saddr;
192#endif
193
194 atomic64_t skc_cookie;
195
196
197
198
199
200
201 union {
202 unsigned long skc_flags;
203 struct sock *skc_listener;
204 struct inet_timewait_death_row *skc_tw_dr;
205 };
206
207
208
209
210
211 int skc_dontcopy_begin[0];
212
213 union {
214 struct hlist_node skc_node;
215 struct hlist_nulls_node skc_nulls_node;
216 };
217 int skc_tx_queue_mapping;
218 union {
219 int skc_incoming_cpu;
220 u32 skc_rcv_wnd;
221 u32 skc_tw_rcv_nxt;
222 };
223
224 refcount_t skc_refcnt;
225
226 int skc_dontcopy_end[0];
227 union {
228 u32 skc_rxhash;
229 u32 skc_window_clamp;
230 u32 skc_tw_snd_nxt;
231 };
232
233};
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319struct sock {
320
321
322
323
324 struct sock_common __sk_common;
325#define sk_node __sk_common.skc_node
326#define sk_nulls_node __sk_common.skc_nulls_node
327#define sk_refcnt __sk_common.skc_refcnt
328#define sk_tx_queue_mapping __sk_common.skc_tx_queue_mapping
329
330#define sk_dontcopy_begin __sk_common.skc_dontcopy_begin
331#define sk_dontcopy_end __sk_common.skc_dontcopy_end
332#define sk_hash __sk_common.skc_hash
333#define sk_portpair __sk_common.skc_portpair
334#define sk_num __sk_common.skc_num
335#define sk_dport __sk_common.skc_dport
336#define sk_addrpair __sk_common.skc_addrpair
337#define sk_daddr __sk_common.skc_daddr
338#define sk_rcv_saddr __sk_common.skc_rcv_saddr
339#define sk_family __sk_common.skc_family
340#define sk_state __sk_common.skc_state
341#define sk_reuse __sk_common.skc_reuse
342#define sk_reuseport __sk_common.skc_reuseport
343#define sk_ipv6only __sk_common.skc_ipv6only
344#define sk_net_refcnt __sk_common.skc_net_refcnt
345#define sk_bound_dev_if __sk_common.skc_bound_dev_if
346#define sk_bind_node __sk_common.skc_bind_node
347#define sk_prot __sk_common.skc_prot
348#define sk_net __sk_common.skc_net
349#define sk_v6_daddr __sk_common.skc_v6_daddr
350#define sk_v6_rcv_saddr __sk_common.skc_v6_rcv_saddr
351#define sk_cookie __sk_common.skc_cookie
352#define sk_incoming_cpu __sk_common.skc_incoming_cpu
353#define sk_flags __sk_common.skc_flags
354#define sk_rxhash __sk_common.skc_rxhash
355
356 socket_lock_t sk_lock;
357 atomic_t sk_drops;
358 int sk_rcvlowat;
359 struct sk_buff_head sk_error_queue;
360 struct sk_buff_head sk_receive_queue;
361
362
363
364
365
366
367
368
369 struct {
370 atomic_t rmem_alloc;
371 int len;
372 struct sk_buff *head;
373 struct sk_buff *tail;
374 } sk_backlog;
375#define sk_rmem_alloc sk_backlog.rmem_alloc
376
377 int sk_forward_alloc;
378#ifdef CONFIG_NET_RX_BUSY_POLL
379 unsigned int sk_ll_usec;
380
381 unsigned int sk_napi_id;
382#endif
383 int sk_rcvbuf;
384
385 struct sk_filter __rcu *sk_filter;
386 union {
387 struct socket_wq __rcu *sk_wq;
388 struct socket_wq *sk_wq_raw;
389 };
390#ifdef CONFIG_XFRM
391 struct xfrm_policy __rcu *sk_policy[2];
392#endif
393 struct dst_entry *sk_rx_dst;
394 struct dst_entry __rcu *sk_dst_cache;
395 atomic_t sk_omem_alloc;
396 int sk_sndbuf;
397
398
399 int sk_wmem_queued;
400 refcount_t sk_wmem_alloc;
401 unsigned long sk_tsq_flags;
402 union {
403 struct sk_buff *sk_send_head;
404 struct rb_root tcp_rtx_queue;
405 };
406 struct sk_buff_head sk_write_queue;
407 __s32 sk_peek_off;
408 int sk_write_pending;
409 __u32 sk_dst_pending_confirm;
410 u32 sk_pacing_status;
411 long sk_sndtimeo;
412 struct timer_list sk_timer;
413 __u32 sk_priority;
414 __u32 sk_mark;
415 u32 sk_pacing_rate;
416 u32 sk_max_pacing_rate;
417 struct page_frag sk_frag;
418 netdev_features_t sk_route_caps;
419 netdev_features_t sk_route_nocaps;
420 netdev_features_t sk_route_forced_caps;
421 int sk_gso_type;
422 unsigned int sk_gso_max_size;
423 gfp_t sk_allocation;
424 __u32 sk_txhash;
425
426
427
428
429
430 unsigned int __sk_flags_offset[0];
431#ifdef __BIG_ENDIAN_BITFIELD
432#define SK_FL_PROTO_SHIFT 16
433#define SK_FL_PROTO_MASK 0x00ff0000
434
435#define SK_FL_TYPE_SHIFT 0
436#define SK_FL_TYPE_MASK 0x0000ffff
437#else
438#define SK_FL_PROTO_SHIFT 8
439#define SK_FL_PROTO_MASK 0x0000ff00
440
441#define SK_FL_TYPE_SHIFT 16
442#define SK_FL_TYPE_MASK 0xffff0000
443#endif
444
445 unsigned int sk_padding : 1,
446 sk_kern_sock : 1,
447 sk_no_check_tx : 1,
448 sk_no_check_rx : 1,
449 sk_userlocks : 4,
450 sk_protocol : 8,
451 sk_type : 16;
452#define SK_PROTOCOL_MAX U8_MAX
453 u16 sk_gso_max_segs;
454 u8 sk_pacing_shift;
455 unsigned long sk_lingertime;
456 struct proto *sk_prot_creator;
457 rwlock_t sk_callback_lock;
458 int sk_err,
459 sk_err_soft;
460 u32 sk_ack_backlog;
461 u32 sk_max_ack_backlog;
462 kuid_t sk_uid;
463 struct pid *sk_peer_pid;
464 const struct cred *sk_peer_cred;
465 long sk_rcvtimeo;
466 ktime_t sk_stamp;
467 u16 sk_tsflags;
468 u8 sk_shutdown;
469 u32 sk_tskey;
470 atomic_t sk_zckey;
471 struct socket *sk_socket;
472 void *sk_user_data;
473#ifdef CONFIG_SECURITY
474 void *sk_security;
475#endif
476 struct sock_cgroup_data sk_cgrp_data;
477 struct mem_cgroup *sk_memcg;
478 void (*sk_state_change)(struct sock *sk);
479 void (*sk_data_ready)(struct sock *sk);
480 void (*sk_write_space)(struct sock *sk);
481 void (*sk_error_report)(struct sock *sk);
482 int (*sk_backlog_rcv)(struct sock *sk,
483 struct sk_buff *skb);
484 void (*sk_destruct)(struct sock *sk);
485 struct sock_reuseport __rcu *sk_reuseport_cb;
486 struct rcu_head sk_rcu;
487};
488
489enum sk_pacing {
490 SK_PACING_NONE = 0,
491 SK_PACING_NEEDED = 1,
492 SK_PACING_FQ = 2,
493};
494
495#define __sk_user_data(sk) ((*((void __rcu **)&(sk)->sk_user_data)))
496
497#define rcu_dereference_sk_user_data(sk) rcu_dereference(__sk_user_data((sk)))
498#define rcu_assign_sk_user_data(sk, ptr) rcu_assign_pointer(__sk_user_data((sk)), ptr)
499
500
501
502
503
504
505
506
507#define SK_NO_REUSE 0
508#define SK_CAN_REUSE 1
509#define SK_FORCE_REUSE 2
510
511int sk_set_peek_off(struct sock *sk, int val);
512
513static inline int sk_peek_offset(struct sock *sk, int flags)
514{
515 if (unlikely(flags & MSG_PEEK)) {
516 return READ_ONCE(sk->sk_peek_off);
517 }
518
519 return 0;
520}
521
522static inline void sk_peek_offset_bwd(struct sock *sk, int val)
523{
524 s32 off = READ_ONCE(sk->sk_peek_off);
525
526 if (unlikely(off >= 0)) {
527 off = max_t(s32, off - val, 0);
528 WRITE_ONCE(sk->sk_peek_off, off);
529 }
530}
531
532static inline void sk_peek_offset_fwd(struct sock *sk, int val)
533{
534 sk_peek_offset_bwd(sk, -val);
535}
536
537
538
539
540static inline struct sock *sk_entry(const struct hlist_node *node)
541{
542 return hlist_entry(node, struct sock, sk_node);
543}
544
545static inline struct sock *__sk_head(const struct hlist_head *head)
546{
547 return hlist_entry(head->first, struct sock, sk_node);
548}
549
550static inline struct sock *sk_head(const struct hlist_head *head)
551{
552 return hlist_empty(head) ? NULL : __sk_head(head);
553}
554
555static inline struct sock *__sk_nulls_head(const struct hlist_nulls_head *head)
556{
557 return hlist_nulls_entry(head->first, struct sock, sk_nulls_node);
558}
559
560static inline struct sock *sk_nulls_head(const struct hlist_nulls_head *head)
561{
562 return hlist_nulls_empty(head) ? NULL : __sk_nulls_head(head);
563}
564
565static inline struct sock *sk_next(const struct sock *sk)
566{
567 return hlist_entry_safe(sk->sk_node.next, struct sock, sk_node);
568}
569
570static inline struct sock *sk_nulls_next(const struct sock *sk)
571{
572 return (!is_a_nulls(sk->sk_nulls_node.next)) ?
573 hlist_nulls_entry(sk->sk_nulls_node.next,
574 struct sock, sk_nulls_node) :
575 NULL;
576}
577
578static inline bool sk_unhashed(const struct sock *sk)
579{
580 return hlist_unhashed(&sk->sk_node);
581}
582
583static inline bool sk_hashed(const struct sock *sk)
584{
585 return !sk_unhashed(sk);
586}
587
588static inline void sk_node_init(struct hlist_node *node)
589{
590 node->pprev = NULL;
591}
592
593static inline void sk_nulls_node_init(struct hlist_nulls_node *node)
594{
595 node->pprev = NULL;
596}
597
598static inline void __sk_del_node(struct sock *sk)
599{
600 __hlist_del(&sk->sk_node);
601}
602
603
604static inline bool __sk_del_node_init(struct sock *sk)
605{
606 if (sk_hashed(sk)) {
607 __sk_del_node(sk);
608 sk_node_init(&sk->sk_node);
609 return true;
610 }
611 return false;
612}
613
614
615
616
617
618
619
620static __always_inline void sock_hold(struct sock *sk)
621{
622 refcount_inc(&sk->sk_refcnt);
623}
624
625
626
627
628static __always_inline void __sock_put(struct sock *sk)
629{
630 refcount_dec(&sk->sk_refcnt);
631}
632
633static inline bool sk_del_node_init(struct sock *sk)
634{
635 bool rc = __sk_del_node_init(sk);
636
637 if (rc) {
638
639 WARN_ON(refcount_read(&sk->sk_refcnt) == 1);
640 __sock_put(sk);
641 }
642 return rc;
643}
644#define sk_del_node_init_rcu(sk) sk_del_node_init(sk)
645
646static inline bool __sk_nulls_del_node_init_rcu(struct sock *sk)
647{
648 if (sk_hashed(sk)) {
649 hlist_nulls_del_init_rcu(&sk->sk_nulls_node);
650 return true;
651 }
652 return false;
653}
654
655static inline bool sk_nulls_del_node_init_rcu(struct sock *sk)
656{
657 bool rc = __sk_nulls_del_node_init_rcu(sk);
658
659 if (rc) {
660
661 WARN_ON(refcount_read(&sk->sk_refcnt) == 1);
662 __sock_put(sk);
663 }
664 return rc;
665}
666
667static inline void __sk_add_node(struct sock *sk, struct hlist_head *list)
668{
669 hlist_add_head(&sk->sk_node, list);
670}
671
672static inline void sk_add_node(struct sock *sk, struct hlist_head *list)
673{
674 sock_hold(sk);
675 __sk_add_node(sk, list);
676}
677
678static inline void sk_add_node_rcu(struct sock *sk, struct hlist_head *list)
679{
680 sock_hold(sk);
681 if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport &&
682 sk->sk_family == AF_INET6)
683 hlist_add_tail_rcu(&sk->sk_node, list);
684 else
685 hlist_add_head_rcu(&sk->sk_node, list);
686}
687
688static inline void __sk_nulls_add_node_rcu(struct sock *sk, struct hlist_nulls_head *list)
689{
690 hlist_nulls_add_head_rcu(&sk->sk_nulls_node, list);
691}
692
693static inline void sk_nulls_add_node_rcu(struct sock *sk, struct hlist_nulls_head *list)
694{
695 sock_hold(sk);
696 __sk_nulls_add_node_rcu(sk, list);
697}
698
699static inline void __sk_del_bind_node(struct sock *sk)
700{
701 __hlist_del(&sk->sk_bind_node);
702}
703
704static inline void sk_add_bind_node(struct sock *sk,
705 struct hlist_head *list)
706{
707 hlist_add_head(&sk->sk_bind_node, list);
708}
709
710#define sk_for_each(__sk, list) \
711 hlist_for_each_entry(__sk, list, sk_node)
712#define sk_for_each_rcu(__sk, list) \
713 hlist_for_each_entry_rcu(__sk, list, sk_node)
714#define sk_nulls_for_each(__sk, node, list) \
715 hlist_nulls_for_each_entry(__sk, node, list, sk_nulls_node)
716#define sk_nulls_for_each_rcu(__sk, node, list) \
717 hlist_nulls_for_each_entry_rcu(__sk, node, list, sk_nulls_node)
718#define sk_for_each_from(__sk) \
719 hlist_for_each_entry_from(__sk, sk_node)
720#define sk_nulls_for_each_from(__sk, node) \
721 if (__sk && ({ node = &(__sk)->sk_nulls_node; 1; })) \
722 hlist_nulls_for_each_entry_from(__sk, node, sk_nulls_node)
723#define sk_for_each_safe(__sk, tmp, list) \
724 hlist_for_each_entry_safe(__sk, tmp, list, sk_node)
725#define sk_for_each_bound(__sk, list) \
726 hlist_for_each_entry(__sk, list, sk_bind_node)
727
728
729
730
731
732
733
734
735
736#define sk_for_each_entry_offset_rcu(tpos, pos, head, offset) \
737 for (pos = rcu_dereference(hlist_first_rcu(head)); \
738 pos != NULL && \
739 ({ tpos = (typeof(*tpos) *)((void *)pos - offset); 1;}); \
740 pos = rcu_dereference(hlist_next_rcu(pos)))
741
742static inline struct user_namespace *sk_user_ns(struct sock *sk)
743{
744
745
746
747
748 return sk->sk_socket->file->f_cred->user_ns;
749}
750
751
752enum sock_flags {
753 SOCK_DEAD,
754 SOCK_DONE,
755 SOCK_URGINLINE,
756 SOCK_KEEPOPEN,
757 SOCK_LINGER,
758 SOCK_DESTROY,
759 SOCK_BROADCAST,
760 SOCK_TIMESTAMP,
761 SOCK_ZAPPED,
762 SOCK_USE_WRITE_QUEUE,
763 SOCK_DBG,
764 SOCK_RCVTSTAMP,
765 SOCK_RCVTSTAMPNS,
766 SOCK_LOCALROUTE,
767 SOCK_QUEUE_SHRUNK,
768 SOCK_MEMALLOC,
769 SOCK_TIMESTAMPING_RX_SOFTWARE,
770 SOCK_FASYNC,
771 SOCK_RXQ_OVFL,
772 SOCK_ZEROCOPY,
773 SOCK_WIFI_STATUS,
774 SOCK_NOFCS,
775
776
777
778 SOCK_FILTER_LOCKED,
779 SOCK_SELECT_ERR_QUEUE,
780 SOCK_RCU_FREE,
781};
782
783#define SK_FLAGS_TIMESTAMP ((1UL << SOCK_TIMESTAMP) | (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE))
784
785static inline void sock_copy_flags(struct sock *nsk, struct sock *osk)
786{
787 nsk->sk_flags = osk->sk_flags;
788}
789
790static inline void sock_set_flag(struct sock *sk, enum sock_flags flag)
791{
792 __set_bit(flag, &sk->sk_flags);
793}
794
795static inline void sock_reset_flag(struct sock *sk, enum sock_flags flag)
796{
797 __clear_bit(flag, &sk->sk_flags);
798}
799
800static inline bool sock_flag(const struct sock *sk, enum sock_flags flag)
801{
802 return test_bit(flag, &sk->sk_flags);
803}
804
805#ifdef CONFIG_NET
806extern struct static_key memalloc_socks;
807static inline int sk_memalloc_socks(void)
808{
809 return static_key_false(&memalloc_socks);
810}
811#else
812
813static inline int sk_memalloc_socks(void)
814{
815 return 0;
816}
817
818#endif
819
820static inline gfp_t sk_gfp_mask(const struct sock *sk, gfp_t gfp_mask)
821{
822 return gfp_mask | (sk->sk_allocation & __GFP_MEMALLOC);
823}
824
825static inline void sk_acceptq_removed(struct sock *sk)
826{
827 sk->sk_ack_backlog--;
828}
829
830static inline void sk_acceptq_added(struct sock *sk)
831{
832 sk->sk_ack_backlog++;
833}
834
835static inline bool sk_acceptq_is_full(const struct sock *sk)
836{
837 return sk->sk_ack_backlog > sk->sk_max_ack_backlog;
838}
839
840
841
842
843static inline int sk_stream_min_wspace(const struct sock *sk)
844{
845 return sk->sk_wmem_queued >> 1;
846}
847
848static inline int sk_stream_wspace(const struct sock *sk)
849{
850 return sk->sk_sndbuf - sk->sk_wmem_queued;
851}
852
853void sk_stream_write_space(struct sock *sk);
854
855
856static inline void __sk_add_backlog(struct sock *sk, struct sk_buff *skb)
857{
858
859 skb_dst_force(skb);
860
861 if (!sk->sk_backlog.tail)
862 sk->sk_backlog.head = skb;
863 else
864 sk->sk_backlog.tail->next = skb;
865
866 sk->sk_backlog.tail = skb;
867 skb->next = NULL;
868}
869
870
871
872
873
874
875static inline bool sk_rcvqueues_full(const struct sock *sk, unsigned int limit)
876{
877 unsigned int qsize = sk->sk_backlog.len + atomic_read(&sk->sk_rmem_alloc);
878
879 return qsize > limit;
880}
881
882
883static inline __must_check int sk_add_backlog(struct sock *sk, struct sk_buff *skb,
884 unsigned int limit)
885{
886 if (sk_rcvqueues_full(sk, limit))
887 return -ENOBUFS;
888
889
890
891
892
893
894 if (skb_pfmemalloc(skb) && !sock_flag(sk, SOCK_MEMALLOC))
895 return -ENOMEM;
896
897 __sk_add_backlog(sk, skb);
898 sk->sk_backlog.len += skb->truesize;
899 return 0;
900}
901
902int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb);
903
904static inline int sk_backlog_rcv(struct sock *sk, struct sk_buff *skb)
905{
906 if (sk_memalloc_socks() && skb_pfmemalloc(skb))
907 return __sk_backlog_rcv(sk, skb);
908
909 return sk->sk_backlog_rcv(sk, skb);
910}
911
912static inline void sk_incoming_cpu_update(struct sock *sk)
913{
914 int cpu = raw_smp_processor_id();
915
916 if (unlikely(sk->sk_incoming_cpu != cpu))
917 sk->sk_incoming_cpu = cpu;
918}
919
920static inline void sock_rps_record_flow_hash(__u32 hash)
921{
922#ifdef CONFIG_RPS
923 struct rps_sock_flow_table *sock_flow_table;
924
925 rcu_read_lock();
926 sock_flow_table = rcu_dereference(rps_sock_flow_table);
927 rps_record_sock_flow(sock_flow_table, hash);
928 rcu_read_unlock();
929#endif
930}
931
932static inline void sock_rps_record_flow(const struct sock *sk)
933{
934#ifdef CONFIG_RPS
935 if (static_key_false(&rfs_needed)) {
936
937
938
939
940
941
942
943
944
945
946 if (sk->sk_state == TCP_ESTABLISHED)
947 sock_rps_record_flow_hash(sk->sk_rxhash);
948 }
949#endif
950}
951
952static inline void sock_rps_save_rxhash(struct sock *sk,
953 const struct sk_buff *skb)
954{
955#ifdef CONFIG_RPS
956 if (unlikely(sk->sk_rxhash != skb->hash))
957 sk->sk_rxhash = skb->hash;
958#endif
959}
960
961static inline void sock_rps_reset_rxhash(struct sock *sk)
962{
963#ifdef CONFIG_RPS
964 sk->sk_rxhash = 0;
965#endif
966}
967
968#define sk_wait_event(__sk, __timeo, __condition, __wait) \
969 ({ int __rc; \
970 release_sock(__sk); \
971 __rc = __condition; \
972 if (!__rc) { \
973 *(__timeo) = wait_woken(__wait, \
974 TASK_INTERRUPTIBLE, \
975 *(__timeo)); \
976 } \
977 sched_annotate_sleep(); \
978 lock_sock(__sk); \
979 __rc = __condition; \
980 __rc; \
981 })
982
983int sk_stream_wait_connect(struct sock *sk, long *timeo_p);
984int sk_stream_wait_memory(struct sock *sk, long *timeo_p);
985void sk_stream_wait_close(struct sock *sk, long timeo_p);
986int sk_stream_error(struct sock *sk, int flags, int err);
987void sk_stream_kill_queues(struct sock *sk);
988void sk_set_memalloc(struct sock *sk);
989void sk_clear_memalloc(struct sock *sk);
990
991void __sk_flush_backlog(struct sock *sk);
992
993static inline bool sk_flush_backlog(struct sock *sk)
994{
995 if (unlikely(READ_ONCE(sk->sk_backlog.tail))) {
996 __sk_flush_backlog(sk);
997 return true;
998 }
999 return false;
1000}
1001
1002int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb);
1003
1004struct request_sock_ops;
1005struct timewait_sock_ops;
1006struct inet_hashinfo;
1007struct raw_hashinfo;
1008struct smc_hashinfo;
1009struct module;
1010
1011
1012
1013
1014
1015static inline void sk_prot_clear_nulls(struct sock *sk, int size)
1016{
1017 if (offsetof(struct sock, sk_node.next) != 0)
1018 memset(sk, 0, offsetof(struct sock, sk_node.next));
1019 memset(&sk->sk_node.pprev, 0,
1020 size - offsetof(struct sock, sk_node.pprev));
1021}
1022
1023
1024
1025
1026struct proto {
1027 void (*close)(struct sock *sk,
1028 long timeout);
1029 int (*pre_connect)(struct sock *sk,
1030 struct sockaddr *uaddr,
1031 int addr_len);
1032 int (*connect)(struct sock *sk,
1033 struct sockaddr *uaddr,
1034 int addr_len);
1035 int (*disconnect)(struct sock *sk, int flags);
1036
1037 struct sock * (*accept)(struct sock *sk, int flags, int *err,
1038 bool kern);
1039
1040 int (*ioctl)(struct sock *sk, int cmd,
1041 unsigned long arg);
1042 int (*init)(struct sock *sk);
1043 void (*destroy)(struct sock *sk);
1044 void (*shutdown)(struct sock *sk, int how);
1045 int (*setsockopt)(struct sock *sk, int level,
1046 int optname, char __user *optval,
1047 unsigned int optlen);
1048 int (*getsockopt)(struct sock *sk, int level,
1049 int optname, char __user *optval,
1050 int __user *option);
1051 void (*keepalive)(struct sock *sk, int valbool);
1052#ifdef CONFIG_COMPAT
1053 int (*compat_setsockopt)(struct sock *sk,
1054 int level,
1055 int optname, char __user *optval,
1056 unsigned int optlen);
1057 int (*compat_getsockopt)(struct sock *sk,
1058 int level,
1059 int optname, char __user *optval,
1060 int __user *option);
1061 int (*compat_ioctl)(struct sock *sk,
1062 unsigned int cmd, unsigned long arg);
1063#endif
1064 int (*sendmsg)(struct sock *sk, struct msghdr *msg,
1065 size_t len);
1066 int (*recvmsg)(struct sock *sk, struct msghdr *msg,
1067 size_t len, int noblock, int flags,
1068 int *addr_len);
1069 int (*sendpage)(struct sock *sk, struct page *page,
1070 int offset, size_t size, int flags);
1071 int (*bind)(struct sock *sk,
1072 struct sockaddr *uaddr, int addr_len);
1073
1074 int (*backlog_rcv) (struct sock *sk,
1075 struct sk_buff *skb);
1076
1077 void (*release_cb)(struct sock *sk);
1078
1079
1080 int (*hash)(struct sock *sk);
1081 void (*unhash)(struct sock *sk);
1082 void (*rehash)(struct sock *sk);
1083 int (*get_port)(struct sock *sk, unsigned short snum);
1084
1085
1086#ifdef CONFIG_PROC_FS
1087 unsigned int inuse_idx;
1088#endif
1089
1090 bool (*stream_memory_free)(const struct sock *sk);
1091 bool (*stream_memory_read)(const struct sock *sk);
1092
1093 void (*enter_memory_pressure)(struct sock *sk);
1094 void (*leave_memory_pressure)(struct sock *sk);
1095 atomic_long_t *memory_allocated;
1096 struct percpu_counter *sockets_allocated;
1097
1098
1099
1100
1101
1102
1103 unsigned long *memory_pressure;
1104 long *sysctl_mem;
1105
1106 int *sysctl_wmem;
1107 int *sysctl_rmem;
1108 u32 sysctl_wmem_offset;
1109 u32 sysctl_rmem_offset;
1110
1111 int max_header;
1112 bool no_autobind;
1113
1114 struct kmem_cache *slab;
1115 unsigned int obj_size;
1116 slab_flags_t slab_flags;
1117 unsigned int useroffset;
1118 unsigned int usersize;
1119
1120 struct percpu_counter *orphan_count;
1121
1122 struct request_sock_ops *rsk_prot;
1123 struct timewait_sock_ops *twsk_prot;
1124
1125 union {
1126 struct inet_hashinfo *hashinfo;
1127 struct udp_table *udp_table;
1128 struct raw_hashinfo *raw_hash;
1129 struct smc_hashinfo *smc_hash;
1130 } h;
1131
1132 struct module *owner;
1133
1134 char name[32];
1135
1136 struct list_head node;
1137#ifdef SOCK_REFCNT_DEBUG
1138 atomic_t socks;
1139#endif
1140 int (*diag_destroy)(struct sock *sk, int err);
1141} __randomize_layout;
1142
1143int proto_register(struct proto *prot, int alloc_slab);
1144void proto_unregister(struct proto *prot);
1145int sock_load_diag_module(int family, int protocol);
1146
1147#ifdef SOCK_REFCNT_DEBUG
1148static inline void sk_refcnt_debug_inc(struct sock *sk)
1149{
1150 atomic_inc(&sk->sk_prot->socks);
1151}
1152
1153static inline void sk_refcnt_debug_dec(struct sock *sk)
1154{
1155 atomic_dec(&sk->sk_prot->socks);
1156 printk(KERN_DEBUG "%s socket %p released, %d are still alive\n",
1157 sk->sk_prot->name, sk, atomic_read(&sk->sk_prot->socks));
1158}
1159
1160static inline void sk_refcnt_debug_release(const struct sock *sk)
1161{
1162 if (refcount_read(&sk->sk_refcnt) != 1)
1163 printk(KERN_DEBUG "Destruction of the %s socket %p delayed, refcnt=%d\n",
1164 sk->sk_prot->name, sk, refcount_read(&sk->sk_refcnt));
1165}
1166#else
1167#define sk_refcnt_debug_inc(sk) do { } while (0)
1168#define sk_refcnt_debug_dec(sk) do { } while (0)
1169#define sk_refcnt_debug_release(sk) do { } while (0)
1170#endif
1171
1172static inline bool sk_stream_memory_free(const struct sock *sk)
1173{
1174 if (sk->sk_wmem_queued >= sk->sk_sndbuf)
1175 return false;
1176
1177 return sk->sk_prot->stream_memory_free ?
1178 sk->sk_prot->stream_memory_free(sk) : true;
1179}
1180
1181static inline bool sk_stream_is_writeable(const struct sock *sk)
1182{
1183 return sk_stream_wspace(sk) >= sk_stream_min_wspace(sk) &&
1184 sk_stream_memory_free(sk);
1185}
1186
1187static inline int sk_under_cgroup_hierarchy(struct sock *sk,
1188 struct cgroup *ancestor)
1189{
1190#ifdef CONFIG_SOCK_CGROUP_DATA
1191 return cgroup_is_descendant(sock_cgroup_ptr(&sk->sk_cgrp_data),
1192 ancestor);
1193#else
1194 return -ENOTSUPP;
1195#endif
1196}
1197
1198static inline bool sk_has_memory_pressure(const struct sock *sk)
1199{
1200 return sk->sk_prot->memory_pressure != NULL;
1201}
1202
1203static inline bool sk_under_memory_pressure(const struct sock *sk)
1204{
1205 if (!sk->sk_prot->memory_pressure)
1206 return false;
1207
1208 if (mem_cgroup_sockets_enabled && sk->sk_memcg &&
1209 mem_cgroup_under_socket_pressure(sk->sk_memcg))
1210 return true;
1211
1212 return !!*sk->sk_prot->memory_pressure;
1213}
1214
1215static inline long
1216sk_memory_allocated(const struct sock *sk)
1217{
1218 return atomic_long_read(sk->sk_prot->memory_allocated);
1219}
1220
1221static inline long
1222sk_memory_allocated_add(struct sock *sk, int amt)
1223{
1224 return atomic_long_add_return(amt, sk->sk_prot->memory_allocated);
1225}
1226
1227static inline void
1228sk_memory_allocated_sub(struct sock *sk, int amt)
1229{
1230 atomic_long_sub(amt, sk->sk_prot->memory_allocated);
1231}
1232
1233static inline void sk_sockets_allocated_dec(struct sock *sk)
1234{
1235 percpu_counter_dec(sk->sk_prot->sockets_allocated);
1236}
1237
1238static inline void sk_sockets_allocated_inc(struct sock *sk)
1239{
1240 percpu_counter_inc(sk->sk_prot->sockets_allocated);
1241}
1242
1243static inline int
1244sk_sockets_allocated_read_positive(struct sock *sk)
1245{
1246 return percpu_counter_read_positive(sk->sk_prot->sockets_allocated);
1247}
1248
1249static inline int
1250proto_sockets_allocated_sum_positive(struct proto *prot)
1251{
1252 return percpu_counter_sum_positive(prot->sockets_allocated);
1253}
1254
1255static inline long
1256proto_memory_allocated(struct proto *prot)
1257{
1258 return atomic_long_read(prot->memory_allocated);
1259}
1260
1261static inline bool
1262proto_memory_pressure(struct proto *prot)
1263{
1264 if (!prot->memory_pressure)
1265 return false;
1266 return !!*prot->memory_pressure;
1267}
1268
1269
1270#ifdef CONFIG_PROC_FS
1271
1272void sock_prot_inuse_add(struct net *net, struct proto *prot, int inc);
1273int sock_prot_inuse_get(struct net *net, struct proto *proto);
1274int sock_inuse_get(struct net *net);
1275#else
1276static inline void sock_prot_inuse_add(struct net *net, struct proto *prot,
1277 int inc)
1278{
1279}
1280#endif
1281
1282
1283
1284
1285
1286static inline int __sk_prot_rehash(struct sock *sk)
1287{
1288 sk->sk_prot->unhash(sk);
1289 return sk->sk_prot->hash(sk);
1290}
1291
1292
1293#define SOCK_DESTROY_TIME (10*HZ)
1294
1295
1296#define PROT_SOCK 1024
1297
1298#define SHUTDOWN_MASK 3
1299#define RCV_SHUTDOWN 1
1300#define SEND_SHUTDOWN 2
1301
1302#define SOCK_SNDBUF_LOCK 1
1303#define SOCK_RCVBUF_LOCK 2
1304#define SOCK_BINDADDR_LOCK 4
1305#define SOCK_BINDPORT_LOCK 8
1306
1307struct socket_alloc {
1308 struct socket socket;
1309 struct inode vfs_inode;
1310};
1311
1312static inline struct socket *SOCKET_I(struct inode *inode)
1313{
1314 return &container_of(inode, struct socket_alloc, vfs_inode)->socket;
1315}
1316
1317static inline struct inode *SOCK_INODE(struct socket *socket)
1318{
1319 return &container_of(socket, struct socket_alloc, socket)->vfs_inode;
1320}
1321
1322
1323
1324
1325int __sk_mem_raise_allocated(struct sock *sk, int size, int amt, int kind);
1326int __sk_mem_schedule(struct sock *sk, int size, int kind);
1327void __sk_mem_reduce_allocated(struct sock *sk, int amount);
1328void __sk_mem_reclaim(struct sock *sk, int amount);
1329
1330
1331
1332
1333#define SK_MEM_QUANTUM 4096
1334#define SK_MEM_QUANTUM_SHIFT ilog2(SK_MEM_QUANTUM)
1335#define SK_MEM_SEND 0
1336#define SK_MEM_RECV 1
1337
1338
1339static inline long sk_prot_mem_limits(const struct sock *sk, int index)
1340{
1341 long val = sk->sk_prot->sysctl_mem[index];
1342
1343#if PAGE_SIZE > SK_MEM_QUANTUM
1344 val <<= PAGE_SHIFT - SK_MEM_QUANTUM_SHIFT;
1345#elif PAGE_SIZE < SK_MEM_QUANTUM
1346 val >>= SK_MEM_QUANTUM_SHIFT - PAGE_SHIFT;
1347#endif
1348 return val;
1349}
1350
1351static inline int sk_mem_pages(int amt)
1352{
1353 return (amt + SK_MEM_QUANTUM - 1) >> SK_MEM_QUANTUM_SHIFT;
1354}
1355
1356static inline bool sk_has_account(struct sock *sk)
1357{
1358
1359 return !!sk->sk_prot->memory_allocated;
1360}
1361
1362static inline bool sk_wmem_schedule(struct sock *sk, int size)
1363{
1364 if (!sk_has_account(sk))
1365 return true;
1366 return size <= sk->sk_forward_alloc ||
1367 __sk_mem_schedule(sk, size, SK_MEM_SEND);
1368}
1369
1370static inline bool
1371sk_rmem_schedule(struct sock *sk, struct sk_buff *skb, int size)
1372{
1373 if (!sk_has_account(sk))
1374 return true;
1375 return size<= sk->sk_forward_alloc ||
1376 __sk_mem_schedule(sk, size, SK_MEM_RECV) ||
1377 skb_pfmemalloc(skb);
1378}
1379
1380static inline void sk_mem_reclaim(struct sock *sk)
1381{
1382 if (!sk_has_account(sk))
1383 return;
1384 if (sk->sk_forward_alloc >= SK_MEM_QUANTUM)
1385 __sk_mem_reclaim(sk, sk->sk_forward_alloc);
1386}
1387
1388static inline void sk_mem_reclaim_partial(struct sock *sk)
1389{
1390 if (!sk_has_account(sk))
1391 return;
1392 if (sk->sk_forward_alloc > SK_MEM_QUANTUM)
1393 __sk_mem_reclaim(sk, sk->sk_forward_alloc - 1);
1394}
1395
1396static inline void sk_mem_charge(struct sock *sk, int size)
1397{
1398 if (!sk_has_account(sk))
1399 return;
1400 sk->sk_forward_alloc -= size;
1401}
1402
1403static inline void sk_mem_uncharge(struct sock *sk, int size)
1404{
1405 if (!sk_has_account(sk))
1406 return;
1407 sk->sk_forward_alloc += size;
1408
1409
1410
1411
1412
1413
1414
1415
1416 if (unlikely(sk->sk_forward_alloc >= 1 << 21))
1417 __sk_mem_reclaim(sk, 1 << 20);
1418}
1419
1420static inline void sk_wmem_free_skb(struct sock *sk, struct sk_buff *skb)
1421{
1422 sock_set_flag(sk, SOCK_QUEUE_SHRUNK);
1423 sk->sk_wmem_queued -= skb->truesize;
1424 sk_mem_uncharge(sk, skb->truesize);
1425 __kfree_skb(skb);
1426}
1427
1428static inline void sock_release_ownership(struct sock *sk)
1429{
1430 if (sk->sk_lock.owned) {
1431 sk->sk_lock.owned = 0;
1432
1433
1434 mutex_release(&sk->sk_lock.dep_map, 1, _RET_IP_);
1435 }
1436}
1437
1438
1439
1440
1441
1442
1443
1444
1445#define sock_lock_init_class_and_name(sk, sname, skey, name, key) \
1446do { \
1447 sk->sk_lock.owned = 0; \
1448 init_waitqueue_head(&sk->sk_lock.wq); \
1449 spin_lock_init(&(sk)->sk_lock.slock); \
1450 debug_check_no_locks_freed((void *)&(sk)->sk_lock, \
1451 sizeof((sk)->sk_lock)); \
1452 lockdep_set_class_and_name(&(sk)->sk_lock.slock, \
1453 (skey), (sname)); \
1454 lockdep_init_map(&(sk)->sk_lock.dep_map, (name), (key), 0); \
1455} while (0)
1456
1457#ifdef CONFIG_LOCKDEP
1458static inline bool lockdep_sock_is_held(const struct sock *sk)
1459{
1460 return lockdep_is_held(&sk->sk_lock) ||
1461 lockdep_is_held(&sk->sk_lock.slock);
1462}
1463#endif
1464
1465void lock_sock_nested(struct sock *sk, int subclass);
1466
1467static inline void lock_sock(struct sock *sk)
1468{
1469 lock_sock_nested(sk, 0);
1470}
1471
1472void release_sock(struct sock *sk);
1473
1474
1475#define bh_lock_sock(__sk) spin_lock(&((__sk)->sk_lock.slock))
1476#define bh_lock_sock_nested(__sk) \
1477 spin_lock_nested(&((__sk)->sk_lock.slock), \
1478 SINGLE_DEPTH_NESTING)
1479#define bh_unlock_sock(__sk) spin_unlock(&((__sk)->sk_lock.slock))
1480
1481bool lock_sock_fast(struct sock *sk);
1482
1483
1484
1485
1486
1487
1488
1489
1490static inline void unlock_sock_fast(struct sock *sk, bool slow)
1491{
1492 if (slow)
1493 release_sock(sk);
1494 else
1495 spin_unlock_bh(&sk->sk_lock.slock);
1496}
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512static inline void sock_owned_by_me(const struct sock *sk)
1513{
1514#ifdef CONFIG_LOCKDEP
1515 WARN_ON_ONCE(!lockdep_sock_is_held(sk) && debug_locks);
1516#endif
1517}
1518
1519static inline bool sock_owned_by_user(const struct sock *sk)
1520{
1521 sock_owned_by_me(sk);
1522 return sk->sk_lock.owned;
1523}
1524
1525static inline bool sock_owned_by_user_nocheck(const struct sock *sk)
1526{
1527 return sk->sk_lock.owned;
1528}
1529
1530
1531static inline bool sock_allow_reclassification(const struct sock *csk)
1532{
1533 struct sock *sk = (struct sock *)csk;
1534
1535 return !sk->sk_lock.owned && !spin_is_locked(&sk->sk_lock.slock);
1536}
1537
1538struct sock *sk_alloc(struct net *net, int family, gfp_t priority,
1539 struct proto *prot, int kern);
1540void sk_free(struct sock *sk);
1541void sk_destruct(struct sock *sk);
1542struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority);
1543void sk_free_unlock_clone(struct sock *sk);
1544
1545struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force,
1546 gfp_t priority);
1547void __sock_wfree(struct sk_buff *skb);
1548void sock_wfree(struct sk_buff *skb);
1549struct sk_buff *sock_omalloc(struct sock *sk, unsigned long size,
1550 gfp_t priority);
1551void skb_orphan_partial(struct sk_buff *skb);
1552void sock_rfree(struct sk_buff *skb);
1553void sock_efree(struct sk_buff *skb);
1554#ifdef CONFIG_INET
1555void sock_edemux(struct sk_buff *skb);
1556#else
1557#define sock_edemux sock_efree
1558#endif
1559
1560int sock_setsockopt(struct socket *sock, int level, int op,
1561 char __user *optval, unsigned int optlen);
1562
1563int sock_getsockopt(struct socket *sock, int level, int op,
1564 char __user *optval, int __user *optlen);
1565struct sk_buff *sock_alloc_send_skb(struct sock *sk, unsigned long size,
1566 int noblock, int *errcode);
1567struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len,
1568 unsigned long data_len, int noblock,
1569 int *errcode, int max_page_order);
1570void *sock_kmalloc(struct sock *sk, int size, gfp_t priority);
1571void sock_kfree_s(struct sock *sk, void *mem, int size);
1572void sock_kzfree_s(struct sock *sk, void *mem, int size);
1573void sk_send_sigurg(struct sock *sk);
1574
1575struct sockcm_cookie {
1576 u32 mark;
1577 u16 tsflags;
1578};
1579
1580int __sock_cmsg_send(struct sock *sk, struct msghdr *msg, struct cmsghdr *cmsg,
1581 struct sockcm_cookie *sockc);
1582int sock_cmsg_send(struct sock *sk, struct msghdr *msg,
1583 struct sockcm_cookie *sockc);
1584
1585
1586
1587
1588
1589int sock_no_bind(struct socket *, struct sockaddr *, int);
1590int sock_no_connect(struct socket *, struct sockaddr *, int, int);
1591int sock_no_socketpair(struct socket *, struct socket *);
1592int sock_no_accept(struct socket *, struct socket *, int, bool);
1593int sock_no_getname(struct socket *, struct sockaddr *, int);
1594__poll_t sock_no_poll(struct file *, struct socket *,
1595 struct poll_table_struct *);
1596int sock_no_ioctl(struct socket *, unsigned int, unsigned long);
1597int sock_no_listen(struct socket *, int);
1598int sock_no_shutdown(struct socket *, int);
1599int sock_no_getsockopt(struct socket *, int , int, char __user *, int __user *);
1600int sock_no_setsockopt(struct socket *, int, int, char __user *, unsigned int);
1601int sock_no_sendmsg(struct socket *, struct msghdr *, size_t);
1602int sock_no_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t len);
1603int sock_no_recvmsg(struct socket *, struct msghdr *, size_t, int);
1604int sock_no_mmap(struct file *file, struct socket *sock,
1605 struct vm_area_struct *vma);
1606ssize_t sock_no_sendpage(struct socket *sock, struct page *page, int offset,
1607 size_t size, int flags);
1608ssize_t sock_no_sendpage_locked(struct sock *sk, struct page *page,
1609 int offset, size_t size, int flags);
1610
1611
1612
1613
1614
1615int sock_common_getsockopt(struct socket *sock, int level, int optname,
1616 char __user *optval, int __user *optlen);
1617int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
1618 int flags);
1619int sock_common_setsockopt(struct socket *sock, int level, int optname,
1620 char __user *optval, unsigned int optlen);
1621int compat_sock_common_getsockopt(struct socket *sock, int level,
1622 int optname, char __user *optval, int __user *optlen);
1623int compat_sock_common_setsockopt(struct socket *sock, int level,
1624 int optname, char __user *optval, unsigned int optlen);
1625
1626void sk_common_release(struct sock *sk);
1627
1628
1629
1630
1631
1632
1633void sock_init_data(struct socket *sock, struct sock *sk);
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661static inline void sock_put(struct sock *sk)
1662{
1663 if (refcount_dec_and_test(&sk->sk_refcnt))
1664 sk_free(sk);
1665}
1666
1667
1668
1669void sock_gen_put(struct sock *sk);
1670
1671int __sk_receive_skb(struct sock *sk, struct sk_buff *skb, const int nested,
1672 unsigned int trim_cap, bool refcounted);
1673static inline int sk_receive_skb(struct sock *sk, struct sk_buff *skb,
1674 const int nested)
1675{
1676 return __sk_receive_skb(sk, skb, nested, 1, true);
1677}
1678
1679static inline void sk_tx_queue_set(struct sock *sk, int tx_queue)
1680{
1681 sk->sk_tx_queue_mapping = tx_queue;
1682}
1683
1684static inline void sk_tx_queue_clear(struct sock *sk)
1685{
1686 sk->sk_tx_queue_mapping = -1;
1687}
1688
1689static inline int sk_tx_queue_get(const struct sock *sk)
1690{
1691 return sk ? sk->sk_tx_queue_mapping : -1;
1692}
1693
1694static inline void sk_set_socket(struct sock *sk, struct socket *sock)
1695{
1696 sk_tx_queue_clear(sk);
1697 sk->sk_socket = sock;
1698}
1699
1700static inline wait_queue_head_t *sk_sleep(struct sock *sk)
1701{
1702 BUILD_BUG_ON(offsetof(struct socket_wq, wait) != 0);
1703 return &rcu_dereference_raw(sk->sk_wq)->wait;
1704}
1705
1706
1707
1708
1709
1710
1711
1712static inline void sock_orphan(struct sock *sk)
1713{
1714 write_lock_bh(&sk->sk_callback_lock);
1715 sock_set_flag(sk, SOCK_DEAD);
1716 sk_set_socket(sk, NULL);
1717 sk->sk_wq = NULL;
1718 write_unlock_bh(&sk->sk_callback_lock);
1719}
1720
1721static inline void sock_graft(struct sock *sk, struct socket *parent)
1722{
1723 WARN_ON(parent->sk);
1724 write_lock_bh(&sk->sk_callback_lock);
1725 sk->sk_wq = parent->wq;
1726 parent->sk = sk;
1727 sk_set_socket(sk, parent);
1728 sk->sk_uid = SOCK_INODE(parent)->i_uid;
1729 security_sock_graft(sk, parent);
1730 write_unlock_bh(&sk->sk_callback_lock);
1731}
1732
1733kuid_t sock_i_uid(struct sock *sk);
1734unsigned long sock_i_ino(struct sock *sk);
1735
1736static inline kuid_t sock_net_uid(const struct net *net, const struct sock *sk)
1737{
1738 return sk ? sk->sk_uid : make_kuid(net->user_ns, 0);
1739}
1740
1741static inline u32 net_tx_rndhash(void)
1742{
1743 u32 v = prandom_u32();
1744
1745 return v ?: 1;
1746}
1747
1748static inline void sk_set_txhash(struct sock *sk)
1749{
1750 sk->sk_txhash = net_tx_rndhash();
1751}
1752
1753static inline void sk_rethink_txhash(struct sock *sk)
1754{
1755 if (sk->sk_txhash)
1756 sk_set_txhash(sk);
1757}
1758
1759static inline struct dst_entry *
1760__sk_dst_get(struct sock *sk)
1761{
1762 return rcu_dereference_check(sk->sk_dst_cache,
1763 lockdep_sock_is_held(sk));
1764}
1765
1766static inline struct dst_entry *
1767sk_dst_get(struct sock *sk)
1768{
1769 struct dst_entry *dst;
1770
1771 rcu_read_lock();
1772 dst = rcu_dereference(sk->sk_dst_cache);
1773 if (dst && !atomic_inc_not_zero(&dst->__refcnt))
1774 dst = NULL;
1775 rcu_read_unlock();
1776 return dst;
1777}
1778
1779static inline void dst_negative_advice(struct sock *sk)
1780{
1781 struct dst_entry *ndst, *dst = __sk_dst_get(sk);
1782
1783 sk_rethink_txhash(sk);
1784
1785 if (dst && dst->ops->negative_advice) {
1786 ndst = dst->ops->negative_advice(dst);
1787
1788 if (ndst != dst) {
1789 rcu_assign_pointer(sk->sk_dst_cache, ndst);
1790 sk_tx_queue_clear(sk);
1791 sk->sk_dst_pending_confirm = 0;
1792 }
1793 }
1794}
1795
1796static inline void
1797__sk_dst_set(struct sock *sk, struct dst_entry *dst)
1798{
1799 struct dst_entry *old_dst;
1800
1801 sk_tx_queue_clear(sk);
1802 sk->sk_dst_pending_confirm = 0;
1803 old_dst = rcu_dereference_protected(sk->sk_dst_cache,
1804 lockdep_sock_is_held(sk));
1805 rcu_assign_pointer(sk->sk_dst_cache, dst);
1806 dst_release(old_dst);
1807}
1808
1809static inline void
1810sk_dst_set(struct sock *sk, struct dst_entry *dst)
1811{
1812 struct dst_entry *old_dst;
1813
1814 sk_tx_queue_clear(sk);
1815 sk->sk_dst_pending_confirm = 0;
1816 old_dst = xchg((__force struct dst_entry **)&sk->sk_dst_cache, dst);
1817 dst_release(old_dst);
1818}
1819
1820static inline void
1821__sk_dst_reset(struct sock *sk)
1822{
1823 __sk_dst_set(sk, NULL);
1824}
1825
1826static inline void
1827sk_dst_reset(struct sock *sk)
1828{
1829 sk_dst_set(sk, NULL);
1830}
1831
1832struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie);
1833
1834struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie);
1835
1836static inline void sk_dst_confirm(struct sock *sk)
1837{
1838 if (!sk->sk_dst_pending_confirm)
1839 sk->sk_dst_pending_confirm = 1;
1840}
1841
1842static inline void sock_confirm_neigh(struct sk_buff *skb, struct neighbour *n)
1843{
1844 if (skb_get_dst_pending_confirm(skb)) {
1845 struct sock *sk = skb->sk;
1846 unsigned long now = jiffies;
1847
1848
1849 if (n->confirmed != now)
1850 n->confirmed = now;
1851 if (sk && sk->sk_dst_pending_confirm)
1852 sk->sk_dst_pending_confirm = 0;
1853 }
1854}
1855
1856bool sk_mc_loop(struct sock *sk);
1857
1858static inline bool sk_can_gso(const struct sock *sk)
1859{
1860 return net_gso_ok(sk->sk_route_caps, sk->sk_gso_type);
1861}
1862
1863void sk_setup_caps(struct sock *sk, struct dst_entry *dst);
1864
1865static inline void sk_nocaps_add(struct sock *sk, netdev_features_t flags)
1866{
1867 sk->sk_route_nocaps |= flags;
1868 sk->sk_route_caps &= ~flags;
1869}
1870
1871static inline int skb_do_copy_data_nocache(struct sock *sk, struct sk_buff *skb,
1872 struct iov_iter *from, char *to,
1873 int copy, int offset)
1874{
1875 if (skb->ip_summed == CHECKSUM_NONE) {
1876 __wsum csum = 0;
1877 if (!csum_and_copy_from_iter_full(to, copy, &csum, from))
1878 return -EFAULT;
1879 skb->csum = csum_block_add(skb->csum, csum, offset);
1880 } else if (sk->sk_route_caps & NETIF_F_NOCACHE_COPY) {
1881 if (!copy_from_iter_full_nocache(to, copy, from))
1882 return -EFAULT;
1883 } else if (!copy_from_iter_full(to, copy, from))
1884 return -EFAULT;
1885
1886 return 0;
1887}
1888
1889static inline int skb_add_data_nocache(struct sock *sk, struct sk_buff *skb,
1890 struct iov_iter *from, int copy)
1891{
1892 int err, offset = skb->len;
1893
1894 err = skb_do_copy_data_nocache(sk, skb, from, skb_put(skb, copy),
1895 copy, offset);
1896 if (err)
1897 __skb_trim(skb, offset);
1898
1899 return err;
1900}
1901
1902static inline int skb_copy_to_page_nocache(struct sock *sk, struct iov_iter *from,
1903 struct sk_buff *skb,
1904 struct page *page,
1905 int off, int copy)
1906{
1907 int err;
1908
1909 err = skb_do_copy_data_nocache(sk, skb, from, page_address(page) + off,
1910 copy, skb->len);
1911 if (err)
1912 return err;
1913
1914 skb->len += copy;
1915 skb->data_len += copy;
1916 skb->truesize += copy;
1917 sk->sk_wmem_queued += copy;
1918 sk_mem_charge(sk, copy);
1919 return 0;
1920}
1921
1922
1923
1924
1925
1926
1927
1928static inline int sk_wmem_alloc_get(const struct sock *sk)
1929{
1930 return refcount_read(&sk->sk_wmem_alloc) - 1;
1931}
1932
1933
1934
1935
1936
1937
1938
1939static inline int sk_rmem_alloc_get(const struct sock *sk)
1940{
1941 return atomic_read(&sk->sk_rmem_alloc);
1942}
1943
1944
1945
1946
1947
1948
1949
1950static inline bool sk_has_allocations(const struct sock *sk)
1951{
1952 return sk_wmem_alloc_get(sk) || sk_rmem_alloc_get(sk);
1953}
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986static inline bool skwq_has_sleeper(struct socket_wq *wq)
1987{
1988 return wq && wq_has_sleeper(&wq->wait);
1989}
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999static inline void sock_poll_wait(struct file *filp,
2000 wait_queue_head_t *wait_address, poll_table *p)
2001{
2002 if (!poll_does_not_wait(p) && wait_address) {
2003 poll_wait(filp, wait_address, p);
2004
2005
2006
2007
2008
2009 smp_mb();
2010 }
2011}
2012
2013static inline void skb_set_hash_from_sk(struct sk_buff *skb, struct sock *sk)
2014{
2015 if (sk->sk_txhash) {
2016 skb->l4_hash = 1;
2017 skb->hash = sk->sk_txhash;
2018 }
2019}
2020
2021void skb_set_owner_w(struct sk_buff *skb, struct sock *sk);
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031static inline void skb_set_owner_r(struct sk_buff *skb, struct sock *sk)
2032{
2033 skb_orphan(skb);
2034 skb->sk = sk;
2035 skb->destructor = sock_rfree;
2036 atomic_add(skb->truesize, &sk->sk_rmem_alloc);
2037 sk_mem_charge(sk, skb->truesize);
2038}
2039
2040void sk_reset_timer(struct sock *sk, struct timer_list *timer,
2041 unsigned long expires);
2042
2043void sk_stop_timer(struct sock *sk, struct timer_list *timer);
2044
2045int __sk_queue_drop_skb(struct sock *sk, struct sk_buff_head *sk_queue,
2046 struct sk_buff *skb, unsigned int flags,
2047 void (*destructor)(struct sock *sk,
2048 struct sk_buff *skb));
2049int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb);
2050int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb);
2051
2052int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb);
2053struct sk_buff *sock_dequeue_err_skb(struct sock *sk);
2054
2055
2056
2057
2058
2059static inline int sock_error(struct sock *sk)
2060{
2061 int err;
2062 if (likely(!sk->sk_err))
2063 return 0;
2064 err = xchg(&sk->sk_err, 0);
2065 return -err;
2066}
2067
2068static inline unsigned long sock_wspace(struct sock *sk)
2069{
2070 int amt = 0;
2071
2072 if (!(sk->sk_shutdown & SEND_SHUTDOWN)) {
2073 amt = sk->sk_sndbuf - refcount_read(&sk->sk_wmem_alloc);
2074 if (amt < 0)
2075 amt = 0;
2076 }
2077 return amt;
2078}
2079
2080
2081
2082
2083
2084static inline void sk_set_bit(int nr, struct sock *sk)
2085{
2086 if ((nr == SOCKWQ_ASYNC_NOSPACE || nr == SOCKWQ_ASYNC_WAITDATA) &&
2087 !sock_flag(sk, SOCK_FASYNC))
2088 return;
2089
2090 set_bit(nr, &sk->sk_wq_raw->flags);
2091}
2092
2093static inline void sk_clear_bit(int nr, struct sock *sk)
2094{
2095 if ((nr == SOCKWQ_ASYNC_NOSPACE || nr == SOCKWQ_ASYNC_WAITDATA) &&
2096 !sock_flag(sk, SOCK_FASYNC))
2097 return;
2098
2099 clear_bit(nr, &sk->sk_wq_raw->flags);
2100}
2101
2102static inline void sk_wake_async(const struct sock *sk, int how, int band)
2103{
2104 if (sock_flag(sk, SOCK_FASYNC)) {
2105 rcu_read_lock();
2106 sock_wake_async(rcu_dereference(sk->sk_wq), how, band);
2107 rcu_read_unlock();
2108 }
2109}
2110
2111
2112
2113
2114
2115
2116#define TCP_SKB_MIN_TRUESIZE (2048 + SKB_DATA_ALIGN(sizeof(struct sk_buff)))
2117
2118#define SOCK_MIN_SNDBUF (TCP_SKB_MIN_TRUESIZE * 2)
2119#define SOCK_MIN_RCVBUF TCP_SKB_MIN_TRUESIZE
2120
2121static inline void sk_stream_moderate_sndbuf(struct sock *sk)
2122{
2123 if (!(sk->sk_userlocks & SOCK_SNDBUF_LOCK)) {
2124 sk->sk_sndbuf = min(sk->sk_sndbuf, sk->sk_wmem_queued >> 1);
2125 sk->sk_sndbuf = max_t(u32, sk->sk_sndbuf, SOCK_MIN_SNDBUF);
2126 }
2127}
2128
2129struct sk_buff *sk_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp,
2130 bool force_schedule);
2131
2132
2133
2134
2135
2136
2137
2138
2139static inline struct page_frag *sk_page_frag(struct sock *sk)
2140{
2141 if (gfpflags_allow_blocking(sk->sk_allocation))
2142 return ¤t->task_frag;
2143
2144 return &sk->sk_frag;
2145}
2146
2147bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag);
2148
2149int sk_alloc_sg(struct sock *sk, int len, struct scatterlist *sg,
2150 int sg_start, int *sg_curr, unsigned int *sg_size,
2151 int first_coalesce);
2152
2153
2154
2155
2156static inline bool sock_writeable(const struct sock *sk)
2157{
2158 return refcount_read(&sk->sk_wmem_alloc) < (sk->sk_sndbuf >> 1);
2159}
2160
2161static inline gfp_t gfp_any(void)
2162{
2163 return in_softirq() ? GFP_ATOMIC : GFP_KERNEL;
2164}
2165
2166static inline long sock_rcvtimeo(const struct sock *sk, bool noblock)
2167{
2168 return noblock ? 0 : sk->sk_rcvtimeo;
2169}
2170
2171static inline long sock_sndtimeo(const struct sock *sk, bool noblock)
2172{
2173 return noblock ? 0 : sk->sk_sndtimeo;
2174}
2175
2176static inline int sock_rcvlowat(const struct sock *sk, int waitall, int len)
2177{
2178 return (waitall ? len : min_t(int, sk->sk_rcvlowat, len)) ? : 1;
2179}
2180
2181
2182
2183
2184static inline int sock_intr_errno(long timeo)
2185{
2186 return timeo == MAX_SCHEDULE_TIMEOUT ? -ERESTARTSYS : -EINTR;
2187}
2188
2189struct sock_skb_cb {
2190 u32 dropcount;
2191};
2192
2193
2194
2195
2196
2197#define SOCK_SKB_CB_OFFSET ((FIELD_SIZEOF(struct sk_buff, cb) - \
2198 sizeof(struct sock_skb_cb)))
2199
2200#define SOCK_SKB_CB(__skb) ((struct sock_skb_cb *)((__skb)->cb + \
2201 SOCK_SKB_CB_OFFSET))
2202
2203#define sock_skb_cb_check_size(size) \
2204 BUILD_BUG_ON((size) > SOCK_SKB_CB_OFFSET)
2205
2206static inline void
2207sock_skb_set_dropcount(const struct sock *sk, struct sk_buff *skb)
2208{
2209 SOCK_SKB_CB(skb)->dropcount = sock_flag(sk, SOCK_RXQ_OVFL) ?
2210 atomic_read(&sk->sk_drops) : 0;
2211}
2212
2213static inline void sk_drops_add(struct sock *sk, const struct sk_buff *skb)
2214{
2215 int segs = max_t(u16, 1, skb_shinfo(skb)->gso_segs);
2216
2217 atomic_add(segs, &sk->sk_drops);
2218}
2219
2220void __sock_recv_timestamp(struct msghdr *msg, struct sock *sk,
2221 struct sk_buff *skb);
2222void __sock_recv_wifi_status(struct msghdr *msg, struct sock *sk,
2223 struct sk_buff *skb);
2224
2225static inline void
2226sock_recv_timestamp(struct msghdr *msg, struct sock *sk, struct sk_buff *skb)
2227{
2228 ktime_t kt = skb->tstamp;
2229 struct skb_shared_hwtstamps *hwtstamps = skb_hwtstamps(skb);
2230
2231
2232
2233
2234
2235
2236
2237 if (sock_flag(sk, SOCK_RCVTSTAMP) ||
2238 (sk->sk_tsflags & SOF_TIMESTAMPING_RX_SOFTWARE) ||
2239 (kt && sk->sk_tsflags & SOF_TIMESTAMPING_SOFTWARE) ||
2240 (hwtstamps->hwtstamp &&
2241 (sk->sk_tsflags & SOF_TIMESTAMPING_RAW_HARDWARE)))
2242 __sock_recv_timestamp(msg, sk, skb);
2243 else
2244 sk->sk_stamp = kt;
2245
2246 if (sock_flag(sk, SOCK_WIFI_STATUS) && skb->wifi_acked_valid)
2247 __sock_recv_wifi_status(msg, sk, skb);
2248}
2249
2250void __sock_recv_ts_and_drops(struct msghdr *msg, struct sock *sk,
2251 struct sk_buff *skb);
2252
2253#define SK_DEFAULT_STAMP (-1L * NSEC_PER_SEC)
2254static inline void sock_recv_ts_and_drops(struct msghdr *msg, struct sock *sk,
2255 struct sk_buff *skb)
2256{
2257#define FLAGS_TS_OR_DROPS ((1UL << SOCK_RXQ_OVFL) | \
2258 (1UL << SOCK_RCVTSTAMP))
2259#define TSFLAGS_ANY (SOF_TIMESTAMPING_SOFTWARE | \
2260 SOF_TIMESTAMPING_RAW_HARDWARE)
2261
2262 if (sk->sk_flags & FLAGS_TS_OR_DROPS || sk->sk_tsflags & TSFLAGS_ANY)
2263 __sock_recv_ts_and_drops(msg, sk, skb);
2264 else if (unlikely(sock_flag(sk, SOCK_TIMESTAMP)))
2265 sk->sk_stamp = skb->tstamp;
2266 else if (unlikely(sk->sk_stamp == SK_DEFAULT_STAMP))
2267 sk->sk_stamp = 0;
2268}
2269
2270void __sock_tx_timestamp(__u16 tsflags, __u8 *tx_flags);
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280static inline void sock_tx_timestamp(const struct sock *sk, __u16 tsflags,
2281 __u8 *tx_flags)
2282{
2283 if (unlikely(tsflags))
2284 __sock_tx_timestamp(tsflags, tx_flags);
2285 if (unlikely(sock_flag(sk, SOCK_WIFI_STATUS)))
2286 *tx_flags |= SKBTX_WIFI_STATUS;
2287}
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297static inline void sk_eat_skb(struct sock *sk, struct sk_buff *skb)
2298{
2299 __skb_unlink(skb, &sk->sk_receive_queue);
2300 __kfree_skb(skb);
2301}
2302
2303static inline
2304struct net *sock_net(const struct sock *sk)
2305{
2306 return read_pnet(&sk->sk_net);
2307}
2308
2309static inline
2310void sock_net_set(struct sock *sk, struct net *net)
2311{
2312 write_pnet(&sk->sk_net, net);
2313}
2314
2315static inline struct sock *skb_steal_sock(struct sk_buff *skb)
2316{
2317 if (skb->sk) {
2318 struct sock *sk = skb->sk;
2319
2320 skb->destructor = NULL;
2321 skb->sk = NULL;
2322 return sk;
2323 }
2324 return NULL;
2325}
2326
2327
2328
2329
2330static inline bool sk_fullsock(const struct sock *sk)
2331{
2332 return (1 << sk->sk_state) & ~(TCPF_TIME_WAIT | TCPF_NEW_SYN_RECV);
2333}
2334
2335
2336
2337
2338static inline bool sk_listener(const struct sock *sk)
2339{
2340 return (1 << sk->sk_state) & (TCPF_LISTEN | TCPF_NEW_SYN_RECV);
2341}
2342
2343void sock_enable_timestamp(struct sock *sk, int flag);
2344int sock_get_timestamp(struct sock *, struct timeval __user *);
2345int sock_get_timestampns(struct sock *, struct timespec __user *);
2346int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len, int level,
2347 int type);
2348
2349bool sk_ns_capable(const struct sock *sk,
2350 struct user_namespace *user_ns, int cap);
2351bool sk_capable(const struct sock *sk, int cap);
2352bool sk_net_capable(const struct sock *sk, int cap);
2353
2354void sk_get_meminfo(const struct sock *sk, u32 *meminfo);
2355
2356
2357
2358
2359
2360
2361#define _SK_MEM_PACKETS 256
2362#define _SK_MEM_OVERHEAD SKB_TRUESIZE(256)
2363#define SK_WMEM_MAX (_SK_MEM_OVERHEAD * _SK_MEM_PACKETS)
2364#define SK_RMEM_MAX (_SK_MEM_OVERHEAD * _SK_MEM_PACKETS)
2365
2366extern __u32 sysctl_wmem_max;
2367extern __u32 sysctl_rmem_max;
2368
2369extern int sysctl_tstamp_allow_data;
2370extern int sysctl_optmem_max;
2371
2372extern __u32 sysctl_wmem_default;
2373extern __u32 sysctl_rmem_default;
2374
2375static inline int sk_get_wmem0(const struct sock *sk, const struct proto *proto)
2376{
2377
2378 if (proto->sysctl_wmem_offset)
2379 return *(int *)((void *)sock_net(sk) + proto->sysctl_wmem_offset);
2380
2381 return *proto->sysctl_wmem;
2382}
2383
2384static inline int sk_get_rmem0(const struct sock *sk, const struct proto *proto)
2385{
2386
2387 if (proto->sysctl_rmem_offset)
2388 return *(int *)((void *)sock_net(sk) + proto->sysctl_rmem_offset);
2389
2390 return *proto->sysctl_rmem;
2391}
2392
2393
2394
2395
2396
2397static inline void sk_pacing_shift_update(struct sock *sk, int val)
2398{
2399 if (!sk || !sk_fullsock(sk) || sk->sk_pacing_shift == val)
2400 return;
2401 sk->sk_pacing_shift = val;
2402}
2403
2404
2405
2406
2407
2408
2409static inline bool sk_dev_equal_l3scope(struct sock *sk, int dif)
2410{
2411 int mdif;
2412
2413 if (!sk->sk_bound_dev_if || sk->sk_bound_dev_if == dif)
2414 return true;
2415
2416 mdif = l3mdev_master_ifindex_by_index(sock_net(sk), dif);
2417 if (mdif && mdif == sk->sk_bound_dev_if)
2418 return true;
2419
2420 return false;
2421}
2422
2423#endif
2424