1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40#ifndef _SOCK_H
41#define _SOCK_H
42
43#include <linux/hardirq.h>
44#include <linux/kernel.h>
45#include <linux/list.h>
46#include <linux/list_nulls.h>
47#include <linux/timer.h>
48#include <linux/cache.h>
49#include <linux/bitops.h>
50#include <linux/lockdep.h>
51#include <linux/netdevice.h>
52#include <linux/skbuff.h>
53#include <linux/mm.h>
54#include <linux/security.h>
55#include <linux/slab.h>
56#include <linux/uaccess.h>
57#ifdef __GENKSYMS__
58#include <linux/res_counter.h>
59#else
60#include <linux/page_counter.h>
61#endif
62#include <linux/memcontrol.h>
63#include <linux/static_key.h>
64#include <linux/aio.h>
65#include <linux/sched.h>
66
67#include <linux/filter.h>
68#include <linux/rculist_nulls.h>
69#include <linux/poll.h>
70
71#include <linux/atomic.h>
72#include <net/dst.h>
73#include <net/checksum.h>
74#include <linux/net_tstamp.h>
75
76#include <linux/rh_kabi.h>
77#include <net/tcp_states.h>
78
79struct cgroup;
80struct cgroup_subsys;
81#ifdef CONFIG_NET
82int mem_cgroup_sockets_init(struct mem_cgroup *memcg, struct cgroup_subsys *ss);
83void mem_cgroup_sockets_destroy(struct mem_cgroup *memcg);
84#else
85static inline
86int mem_cgroup_sockets_init(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
87{
88 return 0;
89}
90static inline
91void mem_cgroup_sockets_destroy(struct mem_cgroup *memcg)
92{
93}
94#endif
95
96
97
98
99
100
101
102#define SOCK_DEBUGGING
103#ifdef SOCK_DEBUGGING
104#define SOCK_DEBUG(sk, msg...) do { if ((sk) && sock_flag((sk), SOCK_DBG)) \
105 printk(KERN_DEBUG msg); } while (0)
106#else
107
108static inline __printf(2, 3)
109void SOCK_DEBUG(const struct sock *sk, const char *msg, ...)
110{
111}
112#endif
113
114
115
116
117
118typedef struct {
119 spinlock_t slock;
120 int owned;
121 wait_queue_head_t wq;
122
123
124
125
126
127
128#ifdef CONFIG_DEBUG_LOCK_ALLOC
129 struct lockdep_map dep_map;
130#endif
131} socket_lock_t;
132
133struct sock;
134struct proto;
135struct net;
136
137typedef __u32 __bitwise __portpair;
138typedef __u64 __bitwise __addrpair;
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165struct sock_common {
166
167
168
169 union {
170 __addrpair skc_addrpair;
171 struct {
172 __be32 skc_daddr;
173 __be32 skc_rcv_saddr;
174 };
175 };
176 union {
177 unsigned int skc_hash;
178 __u16 skc_u16hashes[2];
179 };
180
181 union {
182 __portpair skc_portpair;
183 struct {
184 __be16 skc_dport;
185 __u16 skc_num;
186 };
187 };
188
189 unsigned short skc_family;
190 volatile unsigned char skc_state;
191 unsigned char skc_reuse:4;
192 unsigned char skc_reuseport:4;
193 int skc_bound_dev_if;
194 union {
195 struct hlist_node skc_bind_node;
196 RH_KABI_REPLACE(struct hlist_nulls_node skc_portaddr_node, struct hlist_node skc_portaddr_node)
197 };
198 struct proto *skc_prot;
199 possible_net_t skc_net;
200
201#if IS_ENABLED(CONFIG_IPV6)
202 struct in6_addr skc_v6_daddr;
203 struct in6_addr skc_v6_rcv_saddr;
204#endif
205
206
207
208
209
210
211 int skc_dontcopy_begin[0];
212
213 union {
214 struct hlist_node skc_node;
215 struct hlist_nulls_node skc_nulls_node;
216 };
217 int skc_tx_queue_mapping;
218 atomic_t skc_refcnt;
219
220 int skc_dontcopy_end[0];
221
222};
223
224struct cg_proto;
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303struct sock {
304
305
306
307
308 struct sock_common __sk_common;
309#define sk_node __sk_common.skc_node
310#define sk_nulls_node __sk_common.skc_nulls_node
311#define sk_refcnt __sk_common.skc_refcnt
312#define sk_tx_queue_mapping __sk_common.skc_tx_queue_mapping
313
314#define sk_dontcopy_begin __sk_common.skc_dontcopy_begin
315#define sk_dontcopy_end __sk_common.skc_dontcopy_end
316#define sk_hash __sk_common.skc_hash
317#define sk_portpair __sk_common.skc_portpair
318#define sk_num __sk_common.skc_num
319#define sk_dport __sk_common.skc_dport
320#define sk_addrpair __sk_common.skc_addrpair
321#define sk_daddr __sk_common.skc_daddr
322#define sk_rcv_saddr __sk_common.skc_rcv_saddr
323#define sk_family __sk_common.skc_family
324#define sk_state __sk_common.skc_state
325#define sk_reuse __sk_common.skc_reuse
326#define sk_reuseport __sk_common.skc_reuseport
327#define sk_bound_dev_if __sk_common.skc_bound_dev_if
328#define sk_bind_node __sk_common.skc_bind_node
329#define sk_prot __sk_common.skc_prot
330#define sk_net __sk_common.skc_net
331#define sk_v6_daddr __sk_common.skc_v6_daddr
332#define sk_v6_rcv_saddr __sk_common.skc_v6_rcv_saddr
333
334 socket_lock_t sk_lock;
335 struct sk_buff_head sk_receive_queue;
336
337
338
339
340
341
342
343
344 struct {
345 atomic_t rmem_alloc;
346 int len;
347 struct sk_buff *head;
348 struct sk_buff *tail;
349 } sk_backlog;
350#define sk_rmem_alloc sk_backlog.rmem_alloc
351 int sk_forward_alloc;
352#ifdef CONFIG_RPS
353 __u32 sk_rxhash;
354#endif
355#ifdef CONFIG_NET_RX_BUSY_POLL
356 unsigned int sk_napi_id;
357 unsigned int sk_ll_usec;
358#endif
359 atomic_t sk_drops;
360 int sk_rcvbuf;
361
362 struct sk_filter __rcu *sk_filter;
363 struct socket_wq __rcu *sk_wq;
364
365#ifdef CONFIG_NET_DMA_RH_KABI
366 RH_KABI_DEPRECATE(struct sk_buff_head, sk_async_wait_queue)
367#endif
368
369#ifdef CONFIG_XFRM
370 struct xfrm_policy *sk_policy[2];
371#endif
372 unsigned long sk_flags;
373 struct dst_entry *sk_rx_dst;
374 struct dst_entry __rcu *sk_dst_cache;
375 RH_KABI_DEPRECATE(spinlock_t, sk_dst_lock)
376 atomic_t sk_wmem_alloc;
377 atomic_t sk_omem_alloc;
378 int sk_sndbuf;
379 struct sk_buff_head sk_write_queue;
380 kmemcheck_bitfield_begin(flags);
381 unsigned int sk_shutdown : 2,
382#ifdef __GENKSYMS__
383 sk_no_check : 2,
384#else
385 sk_no_check_tx : 1,
386 sk_no_check_rx : 1,
387#endif
388 sk_userlocks : 4,
389 sk_protocol : 8,
390#define SK_PROTOCOL_MAX U8_MAX
391 sk_type : 16;
392 kmemcheck_bitfield_end(flags);
393 int sk_wmem_queued;
394 gfp_t sk_allocation;
395 u32 sk_pacing_rate;
396 netdev_features_t sk_route_caps;
397 netdev_features_t sk_route_nocaps;
398 int sk_gso_type;
399 unsigned int sk_gso_max_size;
400 u16 sk_gso_max_segs;
401 int sk_rcvlowat;
402 unsigned long sk_lingertime;
403 struct sk_buff_head sk_error_queue;
404 struct proto *sk_prot_creator;
405 rwlock_t sk_callback_lock;
406 int sk_err,
407 sk_err_soft;
408 unsigned short sk_ack_backlog;
409 unsigned short sk_max_ack_backlog;
410 __u32 sk_priority;
411#if IS_ENABLED(CONFIG_NETPRIO_CGROUP)
412 __u32 sk_cgrp_prioidx;
413#endif
414 struct pid *sk_peer_pid;
415 const struct cred *sk_peer_cred;
416 long sk_rcvtimeo;
417 long sk_sndtimeo;
418 void *sk_protinfo;
419 struct timer_list sk_timer;
420 ktime_t sk_stamp;
421 struct socket *sk_socket;
422 void *sk_user_data;
423 struct page_frag sk_frag;
424 struct sk_buff *sk_send_head;
425 __s32 sk_peek_off;
426 int sk_write_pending;
427#ifdef CONFIG_SECURITY
428 void *sk_security;
429#endif
430 __u32 sk_mark;
431 u32 sk_classid;
432 struct cg_proto *sk_cgrp;
433 void (*sk_state_change)(struct sock *sk);
434 void (*sk_data_ready)(struct sock *sk, int bytes);
435 void (*sk_write_space)(struct sock *sk);
436 void (*sk_error_report)(struct sock *sk);
437 int (*sk_backlog_rcv)(struct sock *sk,
438 struct sk_buff *skb);
439 void (*sk_destruct)(struct sock *sk);
440
441
442
443
444
445
446
447
448 RH_KABI_USE2_P(1, __u32 sk_txhash, u32 sk_max_pacing_rate)
449 RH_KABI_USE2_P(2, u16 sk_tsflags, __u32 sk_dst_pending_confirm)
450 RH_KABI_USE_P(3, struct sock_reuseport __rcu *sk_reuseport_cb)
451#ifndef __GENKSYMS__
452 struct rcu_head sk_rcu;
453#else
454 RH_KABI_USE_P(4, sk_rcu.next)
455 RH_KABI_USE_P(5, sk_rcu.func)
456#endif
457 RH_KABI_RESERVE_P(6)
458 RH_KABI_RESERVE_P(7)
459 RH_KABI_RESERVE_P(8)
460};
461
462#define __sk_user_data(sk) ((*((void __rcu **)&(sk)->sk_user_data)))
463
464#define rcu_dereference_sk_user_data(sk) rcu_dereference(__sk_user_data((sk)))
465#define rcu_assign_sk_user_data(sk, ptr) rcu_assign_pointer(__sk_user_data((sk)), ptr)
466
467
468
469
470
471
472
473
474#define SK_NO_REUSE 0
475#define SK_CAN_REUSE 1
476#define SK_FORCE_REUSE 2
477
478static inline int sk_peek_offset(struct sock *sk, int flags)
479{
480 if (unlikely(flags & MSG_PEEK)) {
481 s32 off = READ_ONCE(sk->sk_peek_off);
482 if (off >= 0)
483 return off;
484 }
485
486 return 0;
487}
488
489static inline void sk_peek_offset_bwd(struct sock *sk, int val)
490{
491 s32 off = READ_ONCE(sk->sk_peek_off);
492
493 if (unlikely(off >= 0)) {
494 off = max_t(s32, off - val, 0);
495 WRITE_ONCE(sk->sk_peek_off, off);
496 }
497}
498
499static inline void sk_peek_offset_fwd(struct sock *sk, int val)
500{
501 sk_peek_offset_bwd(sk, -val);
502}
503
504
505
506
507static inline struct sock *sk_entry(const struct hlist_node *node)
508{
509 return hlist_entry(node, struct sock, sk_node);
510}
511
512static inline struct sock *__sk_head(const struct hlist_head *head)
513{
514 return hlist_entry(head->first, struct sock, sk_node);
515}
516
517static inline struct sock *sk_head(const struct hlist_head *head)
518{
519 return hlist_empty(head) ? NULL : __sk_head(head);
520}
521
522static inline struct sock *__sk_nulls_head(const struct hlist_nulls_head *head)
523{
524 return hlist_nulls_entry(head->first, struct sock, sk_nulls_node);
525}
526
527static inline struct sock *sk_nulls_head(const struct hlist_nulls_head *head)
528{
529 return hlist_nulls_empty(head) ? NULL : __sk_nulls_head(head);
530}
531
532static inline struct sock *sk_next(const struct sock *sk)
533{
534 return sk->sk_node.next ?
535 hlist_entry(sk->sk_node.next, struct sock, sk_node) : NULL;
536}
537
538static inline struct sock *sk_nulls_next(const struct sock *sk)
539{
540 return (!is_a_nulls(sk->sk_nulls_node.next)) ?
541 hlist_nulls_entry(sk->sk_nulls_node.next,
542 struct sock, sk_nulls_node) :
543 NULL;
544}
545
546static inline bool sk_unhashed(const struct sock *sk)
547{
548 return hlist_unhashed(&sk->sk_node);
549}
550
551static inline bool sk_hashed(const struct sock *sk)
552{
553 return !sk_unhashed(sk);
554}
555
556static inline void sk_node_init(struct hlist_node *node)
557{
558 node->pprev = NULL;
559}
560
561static inline void sk_nulls_node_init(struct hlist_nulls_node *node)
562{
563 node->pprev = NULL;
564}
565
566static inline void __sk_del_node(struct sock *sk)
567{
568 __hlist_del(&sk->sk_node);
569}
570
571
572static inline bool __sk_del_node_init(struct sock *sk)
573{
574 if (sk_hashed(sk)) {
575 __sk_del_node(sk);
576 sk_node_init(&sk->sk_node);
577 return true;
578 }
579 return false;
580}
581
582
583
584
585
586
587
588static inline void sock_hold(struct sock *sk)
589{
590 atomic_inc(&sk->sk_refcnt);
591}
592
593
594
595
596static inline void __sock_put(struct sock *sk)
597{
598 atomic_dec(&sk->sk_refcnt);
599}
600
601static inline bool sk_del_node_init(struct sock *sk)
602{
603 bool rc = __sk_del_node_init(sk);
604
605 if (rc) {
606
607 WARN_ON(atomic_read(&sk->sk_refcnt) == 1);
608 __sock_put(sk);
609 }
610 return rc;
611}
612#define sk_del_node_init_rcu(sk) sk_del_node_init(sk)
613
614static inline bool __sk_nulls_del_node_init_rcu(struct sock *sk)
615{
616 if (sk_hashed(sk)) {
617 hlist_nulls_del_init_rcu(&sk->sk_nulls_node);
618 return true;
619 }
620 return false;
621}
622
623static inline bool sk_nulls_del_node_init_rcu(struct sock *sk)
624{
625 bool rc = __sk_nulls_del_node_init_rcu(sk);
626
627 if (rc) {
628
629 WARN_ON(atomic_read(&sk->sk_refcnt) == 1);
630 __sock_put(sk);
631 }
632 return rc;
633}
634
635static inline void __sk_add_node(struct sock *sk, struct hlist_head *list)
636{
637 hlist_add_head(&sk->sk_node, list);
638}
639
640static inline void sk_add_node(struct sock *sk, struct hlist_head *list)
641{
642 sock_hold(sk);
643 __sk_add_node(sk, list);
644}
645
646static inline void sk_add_node_rcu(struct sock *sk, struct hlist_head *list)
647{
648 sock_hold(sk);
649 if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport &&
650 sk->sk_family == AF_INET6)
651 hlist_add_tail_rcu(&sk->sk_node, list);
652 else
653 hlist_add_head_rcu(&sk->sk_node, list);
654}
655
656static inline void __sk_nulls_add_node_rcu(struct sock *sk, struct hlist_nulls_head *list)
657{
658 if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport &&
659 sk->sk_family == AF_INET6)
660 hlist_nulls_add_tail_rcu(&sk->sk_nulls_node, list);
661 else
662 hlist_nulls_add_head_rcu(&sk->sk_nulls_node, list);
663}
664
665static inline void sk_nulls_add_node_rcu(struct sock *sk, struct hlist_nulls_head *list)
666{
667 sock_hold(sk);
668 __sk_nulls_add_node_rcu(sk, list);
669}
670
671static inline void __sk_del_bind_node(struct sock *sk)
672{
673 __hlist_del(&sk->sk_bind_node);
674}
675
676static inline void sk_add_bind_node(struct sock *sk,
677 struct hlist_head *list)
678{
679 hlist_add_head(&sk->sk_bind_node, list);
680}
681
682#define sk_for_each(__sk, list) \
683 hlist_for_each_entry(__sk, list, sk_node)
684#define sk_for_each_rcu(__sk, list) \
685 hlist_for_each_entry_rcu(__sk, list, sk_node)
686#define sk_nulls_for_each(__sk, node, list) \
687 hlist_nulls_for_each_entry(__sk, node, list, sk_nulls_node)
688#define sk_nulls_for_each_rcu(__sk, node, list) \
689 hlist_nulls_for_each_entry_rcu(__sk, node, list, sk_nulls_node)
690#define sk_for_each_from(__sk) \
691 hlist_for_each_entry_from(__sk, sk_node)
692#define sk_nulls_for_each_from(__sk, node) \
693 if (__sk && ({ node = &(__sk)->sk_nulls_node; 1; })) \
694 hlist_nulls_for_each_entry_from(__sk, node, sk_nulls_node)
695#define sk_for_each_safe(__sk, tmp, list) \
696 hlist_for_each_entry_safe(__sk, tmp, list, sk_node)
697#define sk_for_each_bound(__sk, list) \
698 hlist_for_each_entry(__sk, list, sk_bind_node)
699
700
701
702
703
704
705
706
707
708#define sk_for_each_entry_offset_rcu(tpos, pos, head, offset) \
709 for (pos = rcu_dereference((head)->first); \
710 pos != NULL && \
711 ({ tpos = (typeof(*tpos) *)((void *)pos - offset); 1;}); \
712 pos = rcu_dereference(pos->next))
713
714static inline struct user_namespace *sk_user_ns(struct sock *sk)
715{
716
717
718
719
720 return sk->sk_socket->file->f_cred->user_ns;
721}
722
723
724enum sock_flags {
725 SOCK_DEAD,
726 SOCK_DONE,
727 SOCK_URGINLINE,
728 SOCK_KEEPOPEN,
729 SOCK_LINGER,
730 SOCK_DESTROY,
731 SOCK_BROADCAST,
732 SOCK_TIMESTAMP,
733 SOCK_ZAPPED,
734 SOCK_USE_WRITE_QUEUE,
735 SOCK_DBG,
736 SOCK_RCVTSTAMP,
737 SOCK_RCVTSTAMPNS,
738 SOCK_LOCALROUTE,
739 SOCK_QUEUE_SHRUNK,
740 SOCK_MEMALLOC,
741 SOCK_RH_RESERVED_1,
742
743
744 SOCK_RH_RESERVED_2,
745 SOCK_RH_RESERVED_3,
746 SOCK_TIMESTAMPING_RX_SOFTWARE,
747 SOCK_RH_RESERVED_4,
748 SOCK_RH_RESERVED_5,
749 SOCK_RH_RESERVED_6,
750 SOCK_FASYNC,
751 SOCK_RXQ_OVFL,
752 SOCK_ZEROCOPY,
753 SOCK_WIFI_STATUS,
754 SOCK_NOFCS,
755
756
757
758 SOCK_FILTER_LOCKED,
759 SOCK_SELECT_ERR_QUEUE,
760 SOCK_RCU_FREE,
761};
762
763#define SK_FLAGS_TIMESTAMP ((1UL << SOCK_TIMESTAMP) | (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE))
764
765static inline void sock_copy_flags(struct sock *nsk, struct sock *osk)
766{
767 nsk->sk_flags = osk->sk_flags;
768}
769
770static inline void sock_set_flag(struct sock *sk, enum sock_flags flag)
771{
772 __set_bit(flag, &sk->sk_flags);
773}
774
775static inline void sock_reset_flag(struct sock *sk, enum sock_flags flag)
776{
777 __clear_bit(flag, &sk->sk_flags);
778}
779
780static inline bool sock_flag(const struct sock *sk, enum sock_flags flag)
781{
782 return test_bit(flag, &sk->sk_flags);
783}
784
785#ifdef CONFIG_NET
786extern struct static_key memalloc_socks;
787static inline int sk_memalloc_socks(void)
788{
789 return static_key_false(&memalloc_socks);
790}
791#else
792
793static inline int sk_memalloc_socks(void)
794{
795 return 0;
796}
797
798#endif
799
800static inline gfp_t sk_gfp_atomic(struct sock *sk, gfp_t gfp_mask)
801{
802 return GFP_ATOMIC | (sk->sk_allocation & __GFP_MEMALLOC);
803}
804
805static inline void sk_acceptq_removed(struct sock *sk)
806{
807 sk->sk_ack_backlog--;
808}
809
810static inline void sk_acceptq_added(struct sock *sk)
811{
812 sk->sk_ack_backlog++;
813}
814
815static inline bool sk_acceptq_is_full(const struct sock *sk)
816{
817 return sk->sk_ack_backlog > sk->sk_max_ack_backlog;
818}
819
820
821
822
823static inline int sk_stream_min_wspace(const struct sock *sk)
824{
825 return sk->sk_wmem_queued >> 1;
826}
827
828static inline int sk_stream_wspace(const struct sock *sk)
829{
830 return sk->sk_sndbuf - sk->sk_wmem_queued;
831}
832
833extern void sk_stream_write_space(struct sock *sk);
834
835
836static inline void __sk_add_backlog(struct sock *sk, struct sk_buff *skb)
837{
838
839 skb_dst_force_safe(skb);
840
841 if (!sk->sk_backlog.tail)
842 sk->sk_backlog.head = skb;
843 else
844 sk->sk_backlog.tail->next = skb;
845
846 sk->sk_backlog.tail = skb;
847 skb->next = NULL;
848}
849
850
851
852
853
854
855static inline bool sk_rcvqueues_full(const struct sock *sk, const struct sk_buff *skb,
856 unsigned int limit)
857{
858 unsigned int qsize = sk->sk_backlog.len + atomic_read(&sk->sk_rmem_alloc);
859
860 return qsize > limit;
861}
862
863
864static inline __must_check int sk_add_backlog(struct sock *sk, struct sk_buff *skb,
865 unsigned int limit)
866{
867 if (sk_rcvqueues_full(sk, skb, limit))
868 return -ENOBUFS;
869
870
871
872
873
874
875 if (skb_pfmemalloc(skb) && !sock_flag(sk, SOCK_MEMALLOC))
876 return -ENOMEM;
877
878 __sk_add_backlog(sk, skb);
879 sk->sk_backlog.len += skb->truesize;
880 return 0;
881}
882
883extern int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb);
884
885static inline int sk_backlog_rcv(struct sock *sk, struct sk_buff *skb)
886{
887 if (sk_memalloc_socks() && skb_pfmemalloc(skb))
888 return __sk_backlog_rcv(sk, skb);
889
890 return sk->sk_backlog_rcv(sk, skb);
891}
892
893static inline void sock_rps_record_flow(const struct sock *sk)
894{
895#ifdef CONFIG_RPS
896 struct rps_sock_flow_table *sock_flow_table;
897
898 rcu_read_lock();
899 sock_flow_table = rcu_dereference(rps_sock_flow_table);
900 rps_record_sock_flow(sock_flow_table, sk->sk_rxhash);
901 rcu_read_unlock();
902#endif
903}
904
905static inline void sock_rps_reset_flow(const struct sock *sk)
906{
907#ifdef CONFIG_RPS
908 struct rps_sock_flow_table *sock_flow_table;
909
910 rcu_read_lock();
911 sock_flow_table = rcu_dereference(rps_sock_flow_table);
912 rps_reset_sock_flow(sock_flow_table, sk->sk_rxhash);
913 rcu_read_unlock();
914#endif
915}
916
917static inline void sock_rps_save_rxhash(struct sock *sk,
918 const struct sk_buff *skb)
919{
920#ifdef CONFIG_RPS
921 if (unlikely(sk->sk_rxhash != skb->hash)) {
922 sock_rps_reset_flow(sk);
923 sk->sk_rxhash = skb->hash;
924 }
925#endif
926}
927
928static inline void sock_rps_reset_rxhash(struct sock *sk)
929{
930#ifdef CONFIG_RPS
931 sock_rps_reset_flow(sk);
932 sk->sk_rxhash = 0;
933#endif
934}
935
936#define sk_wait_event(__sk, __timeo, __condition) \
937 ({ int __rc; \
938 release_sock(__sk); \
939 __rc = __condition; \
940 if (!__rc) { \
941 *(__timeo) = schedule_timeout(*(__timeo)); \
942 } \
943 lock_sock(__sk); \
944 __rc = __condition; \
945 __rc; \
946 })
947
948extern int sk_stream_wait_connect(struct sock *sk, long *timeo_p);
949extern int sk_stream_wait_memory(struct sock *sk, long *timeo_p);
950extern void sk_stream_wait_close(struct sock *sk, long timeo_p);
951extern int sk_stream_error(struct sock *sk, int flags, int err);
952extern void sk_stream_kill_queues(struct sock *sk);
953extern void sk_set_memalloc(struct sock *sk);
954extern void sk_clear_memalloc(struct sock *sk);
955
956int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb);
957
958struct request_sock_ops;
959struct timewait_sock_ops;
960struct inet_hashinfo;
961struct raw_hashinfo;
962struct module;
963
964
965
966
967
968static inline void sk_prot_clear_nulls(struct sock *sk, int size)
969{
970 if (offsetof(struct sock, sk_node.next) != 0)
971 memset(sk, 0, offsetof(struct sock, sk_node.next));
972 memset(&sk->sk_node.pprev, 0,
973 size - offsetof(struct sock, sk_node.pprev));
974}
975
976
977
978
979
980struct proto {
981 void (*close)(struct sock *sk,
982 long timeout);
983 int (*connect)(struct sock *sk,
984 struct sockaddr *uaddr,
985 int addr_len);
986 int (*disconnect)(struct sock *sk, int flags);
987
988 struct sock * (*accept)(struct sock *sk, int flags, int *err);
989
990 int (*ioctl)(struct sock *sk, int cmd,
991 unsigned long arg);
992 int (*init)(struct sock *sk);
993 void (*destroy)(struct sock *sk);
994 void (*shutdown)(struct sock *sk, int how);
995 int (*setsockopt)(struct sock *sk, int level,
996 int optname, char __user *optval,
997 unsigned int optlen);
998 int (*getsockopt)(struct sock *sk, int level,
999 int optname, char __user *optval,
1000 int __user *option);
1001#ifdef CONFIG_COMPAT
1002 int (*compat_setsockopt)(struct sock *sk,
1003 int level,
1004 int optname, char __user *optval,
1005 unsigned int optlen);
1006 int (*compat_getsockopt)(struct sock *sk,
1007 int level,
1008 int optname, char __user *optval,
1009 int __user *option);
1010 int (*compat_ioctl)(struct sock *sk,
1011 unsigned int cmd, unsigned long arg);
1012#endif
1013 int (*sendmsg)(struct kiocb *iocb, struct sock *sk,
1014 struct msghdr *msg, size_t len);
1015 int (*recvmsg)(struct kiocb *iocb, struct sock *sk,
1016 struct msghdr *msg,
1017 size_t len, int noblock, int flags,
1018 int *addr_len);
1019 int (*sendpage)(struct sock *sk, struct page *page,
1020 int offset, size_t size, int flags);
1021 int (*bind)(struct sock *sk,
1022 struct sockaddr *uaddr, int addr_len);
1023
1024 int (*backlog_rcv) (struct sock *sk,
1025 struct sk_buff *skb);
1026
1027 void (*release_cb)(struct sock *sk);
1028 void (*mtu_reduced)(struct sock *sk);
1029
1030
1031 void (*hash)(struct sock *sk);
1032 void (*unhash)(struct sock *sk);
1033 void (*rehash)(struct sock *sk);
1034 int (*get_port)(struct sock *sk, unsigned short snum);
1035 void (*clear_sk)(struct sock *sk, int size);
1036
1037
1038#ifdef CONFIG_PROC_FS
1039 unsigned int inuse_idx;
1040#endif
1041
1042#ifndef __GENKSYMS__
1043 bool (*stream_memory_free)(const struct sock *sk);
1044#endif
1045
1046
1047 void (*enter_memory_pressure)(struct sock *sk);
1048 atomic_long_t *memory_allocated;
1049 struct percpu_counter *sockets_allocated;
1050
1051
1052
1053
1054
1055
1056 int *memory_pressure;
1057 long *sysctl_mem;
1058 int *sysctl_wmem;
1059 int *sysctl_rmem;
1060 int max_header;
1061 bool no_autobind;
1062
1063 struct kmem_cache *slab;
1064 unsigned int obj_size;
1065 int slab_flags;
1066
1067 struct percpu_counter *orphan_count;
1068
1069 struct request_sock_ops *rsk_prot;
1070 struct timewait_sock_ops *twsk_prot;
1071
1072 union {
1073 struct inet_hashinfo *hashinfo;
1074 struct udp_table *udp_table;
1075 struct raw_hashinfo *raw_hash;
1076 } h;
1077
1078 struct module *owner;
1079
1080 char name[32];
1081
1082 struct list_head node;
1083#ifdef SOCK_REFCNT_DEBUG
1084 atomic_t socks;
1085#endif
1086#ifdef CONFIG_MEMCG_KMEM
1087
1088
1089
1090
1091
1092
1093 int (*init_cgroup)(struct mem_cgroup *memcg,
1094 struct cgroup_subsys *ss);
1095 void (*destroy_cgroup)(struct mem_cgroup *memcg);
1096 struct cg_proto *(*proto_cgroup)(struct mem_cgroup *memcg);
1097#endif
1098};
1099
1100
1101
1102
1103enum cg_proto_flags {
1104
1105 MEMCG_SOCK_ACTIVE,
1106
1107 MEMCG_SOCK_ACTIVATED,
1108};
1109
1110struct cg_proto {
1111 void (*enter_memory_pressure)(struct sock *sk);
1112
1113 RH_KABI_REPLACE(struct res_counter *memory_allocated,
1114 struct page_counter *memory_allocated)
1115 RH_KABI_DEPRECATE(struct percpu_counter*, sockets_allocated)
1116 int *memory_pressure;
1117 long *sysctl_mem;
1118 unsigned long flags;
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128 struct mem_cgroup *memcg;
1129};
1130
1131extern int proto_register(struct proto *prot, int alloc_slab);
1132extern void proto_unregister(struct proto *prot);
1133
1134static inline bool memcg_proto_active(struct cg_proto *cg_proto)
1135{
1136 return test_bit(MEMCG_SOCK_ACTIVE, &cg_proto->flags);
1137}
1138
1139static inline bool memcg_proto_activated(struct cg_proto *cg_proto)
1140{
1141 return test_bit(MEMCG_SOCK_ACTIVATED, &cg_proto->flags);
1142}
1143
1144int sock_load_diag_module(int family, int protocol);
1145
1146#ifdef SOCK_REFCNT_DEBUG
1147static inline void sk_refcnt_debug_inc(struct sock *sk)
1148{
1149 atomic_inc(&sk->sk_prot->socks);
1150}
1151
1152static inline void sk_refcnt_debug_dec(struct sock *sk)
1153{
1154 atomic_dec(&sk->sk_prot->socks);
1155 printk(KERN_DEBUG "%s socket %p released, %d are still alive\n",
1156 sk->sk_prot->name, sk, atomic_read(&sk->sk_prot->socks));
1157}
1158
1159static inline void sk_refcnt_debug_release(const struct sock *sk)
1160{
1161 if (atomic_read(&sk->sk_refcnt) != 1)
1162 printk(KERN_DEBUG "Destruction of the %s socket %p delayed, refcnt=%d\n",
1163 sk->sk_prot->name, sk, atomic_read(&sk->sk_refcnt));
1164}
1165#else
1166#define sk_refcnt_debug_inc(sk) do { } while (0)
1167#define sk_refcnt_debug_dec(sk) do { } while (0)
1168#define sk_refcnt_debug_release(sk) do { } while (0)
1169#endif
1170
1171#if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_NET)
1172extern struct static_key memcg_socket_limit_enabled;
1173#define mem_cgroup_sockets_enabled static_key_false(&memcg_socket_limit_enabled)
1174#else
1175#define mem_cgroup_sockets_enabled 0
1176#endif
1177
1178static inline bool sk_stream_memory_free(const struct sock *sk)
1179{
1180 if (sk->sk_wmem_queued >= sk->sk_sndbuf)
1181 return false;
1182
1183 return sk->sk_prot->stream_memory_free ?
1184 sk->sk_prot->stream_memory_free(sk) : true;
1185}
1186
1187static inline bool sk_stream_is_writeable(const struct sock *sk)
1188{
1189 return sk_stream_wspace(sk) >= sk_stream_min_wspace(sk) &&
1190 sk_stream_memory_free(sk);
1191}
1192
1193
1194static inline bool sk_has_memory_pressure(const struct sock *sk)
1195{
1196 return sk->sk_prot->memory_pressure != NULL;
1197}
1198
1199static inline bool sk_under_memory_pressure(const struct sock *sk)
1200{
1201 if (!sk->sk_prot->memory_pressure)
1202 return false;
1203
1204 if (mem_cgroup_sockets_enabled && sk->sk_cgrp)
1205 return !!*sk->sk_cgrp->memory_pressure;
1206
1207 return !!*sk->sk_prot->memory_pressure;
1208}
1209
1210static inline void sk_leave_memory_pressure(struct sock *sk)
1211{
1212 int *memory_pressure = sk->sk_prot->memory_pressure;
1213
1214 if (!memory_pressure)
1215 return;
1216
1217 if (*memory_pressure)
1218 *memory_pressure = 0;
1219
1220 if (mem_cgroup_sockets_enabled && sk->sk_cgrp)
1221 if (*sk->sk_cgrp->memory_pressure)
1222 *sk->sk_cgrp->memory_pressure = 0;
1223}
1224
1225static inline void sk_enter_memory_pressure(struct sock *sk)
1226{
1227 if (!sk->sk_prot->enter_memory_pressure)
1228 return;
1229
1230 if (mem_cgroup_sockets_enabled && sk->sk_cgrp)
1231 sk->sk_cgrp->enter_memory_pressure(sk);
1232
1233 sk->sk_prot->enter_memory_pressure(sk);
1234}
1235
1236static inline long sk_prot_mem_limits(const struct sock *sk, int index)
1237{
1238 long *prot = sk->sk_prot->sysctl_mem;
1239 if (mem_cgroup_sockets_enabled && sk->sk_cgrp)
1240 prot = sk->sk_cgrp->sysctl_mem;
1241 return prot[index];
1242}
1243
1244static inline void memcg_memory_allocated_add(struct cg_proto *prot,
1245 unsigned long amt,
1246 int *parent_status)
1247{
1248 struct page_counter *counter;
1249
1250 if (page_counter_try_charge(prot->memory_allocated, amt, &counter))
1251 return;
1252
1253 page_counter_charge(prot->memory_allocated, amt);
1254 *parent_status = OVER_LIMIT;
1255}
1256
1257static inline void memcg_memory_allocated_sub(struct cg_proto *prot,
1258 unsigned long amt)
1259{
1260 page_counter_uncharge(prot->memory_allocated, amt);
1261}
1262
1263static inline long
1264sk_memory_allocated(const struct sock *sk)
1265{
1266 struct proto *prot = sk->sk_prot;
1267
1268 if (mem_cgroup_sockets_enabled && sk->sk_cgrp)
1269 return page_counter_read(sk->sk_cgrp->memory_allocated);
1270
1271 return atomic_long_read(prot->memory_allocated);
1272}
1273
1274static inline long
1275sk_memory_allocated_add(struct sock *sk, int amt, int *parent_status)
1276{
1277 struct proto *prot = sk->sk_prot;
1278
1279 if (mem_cgroup_sockets_enabled && sk->sk_cgrp) {
1280 memcg_memory_allocated_add(sk->sk_cgrp, amt, parent_status);
1281
1282 atomic_long_add_return(amt, prot->memory_allocated);
1283 return page_counter_read(sk->sk_cgrp->memory_allocated);
1284 }
1285
1286 return atomic_long_add_return(amt, prot->memory_allocated);
1287}
1288
1289static inline void
1290sk_memory_allocated_sub(struct sock *sk, int amt)
1291{
1292 struct proto *prot = sk->sk_prot;
1293
1294 if (mem_cgroup_sockets_enabled && sk->sk_cgrp)
1295 memcg_memory_allocated_sub(sk->sk_cgrp, amt);
1296
1297 atomic_long_sub(amt, prot->memory_allocated);
1298}
1299
1300static inline void sk_sockets_allocated_dec(struct sock *sk)
1301{
1302 percpu_counter_dec(sk->sk_prot->sockets_allocated);
1303}
1304
1305static inline void sk_sockets_allocated_inc(struct sock *sk)
1306{
1307 percpu_counter_inc(sk->sk_prot->sockets_allocated);
1308}
1309
1310static inline int
1311sk_sockets_allocated_read_positive(struct sock *sk)
1312{
1313 return percpu_counter_read_positive(sk->sk_prot->sockets_allocated);
1314}
1315
1316static inline int
1317proto_sockets_allocated_sum_positive(struct proto *prot)
1318{
1319 return percpu_counter_sum_positive(prot->sockets_allocated);
1320}
1321
1322static inline long
1323proto_memory_allocated(struct proto *prot)
1324{
1325 return atomic_long_read(prot->memory_allocated);
1326}
1327
1328static inline bool
1329proto_memory_pressure(struct proto *prot)
1330{
1331 if (!prot->memory_pressure)
1332 return false;
1333 return !!*prot->memory_pressure;
1334}
1335
1336
1337#ifdef CONFIG_PROC_FS
1338
1339extern void sock_prot_inuse_add(struct net *net, struct proto *prot, int inc);
1340extern int sock_prot_inuse_get(struct net *net, struct proto *proto);
1341#else
1342static inline void sock_prot_inuse_add(struct net *net, struct proto *prot,
1343 int inc)
1344{
1345}
1346#endif
1347
1348
1349
1350
1351
1352static inline void __sk_prot_rehash(struct sock *sk)
1353{
1354 sk->sk_prot->unhash(sk);
1355 sk->sk_prot->hash(sk);
1356}
1357
1358void sk_prot_clear_portaddr_nulls(struct sock *sk, int size);
1359
1360
1361#define SOCK_DESTROY_TIME (10*HZ)
1362
1363
1364#define PROT_SOCK 1024
1365
1366#define SHUTDOWN_MASK 3
1367#define RCV_SHUTDOWN 1
1368#define SEND_SHUTDOWN 2
1369
1370#define SOCK_SNDBUF_LOCK 1
1371#define SOCK_RCVBUF_LOCK 2
1372#define SOCK_BINDADDR_LOCK 4
1373#define SOCK_BINDPORT_LOCK 8
1374
1375
1376struct sock_iocb {
1377 struct list_head list;
1378
1379 int flags;
1380 int size;
1381 struct socket *sock;
1382 struct sock *sk;
1383 struct scm_cookie *scm;
1384 struct msghdr *msg, async_msg;
1385 struct kiocb *kiocb;
1386};
1387
1388static inline struct sock_iocb *kiocb_to_siocb(struct kiocb *iocb)
1389{
1390 return (struct sock_iocb *)iocb->private;
1391}
1392
1393static inline struct kiocb *siocb_to_kiocb(struct sock_iocb *si)
1394{
1395 return si->kiocb;
1396}
1397
1398struct socket_alloc {
1399 struct socket socket;
1400 struct inode vfs_inode;
1401};
1402
1403static inline struct socket *SOCKET_I(struct inode *inode)
1404{
1405 return &container_of(inode, struct socket_alloc, vfs_inode)->socket;
1406}
1407
1408static inline struct inode *SOCK_INODE(struct socket *socket)
1409{
1410 return &container_of(socket, struct socket_alloc, socket)->vfs_inode;
1411}
1412
1413
1414
1415
1416int __sk_mem_raise_allocated(struct sock *sk, int size, int amt, int kind);
1417extern int __sk_mem_schedule(struct sock *sk, int size, int kind);
1418void __sk_mem_reduce_allocated(struct sock *sk, int amount);
1419void __sk_mem_reclaim(struct sock *sk, int amount);
1420
1421#define SK_MEM_QUANTUM ((int)PAGE_SIZE)
1422#define SK_MEM_QUANTUM_SHIFT ilog2(SK_MEM_QUANTUM)
1423#define SK_MEM_SEND 0
1424#define SK_MEM_RECV 1
1425
1426static inline int sk_mem_pages(int amt)
1427{
1428 return (amt + SK_MEM_QUANTUM - 1) >> SK_MEM_QUANTUM_SHIFT;
1429}
1430
1431static inline bool sk_has_account(struct sock *sk)
1432{
1433
1434 return !!sk->sk_prot->memory_allocated;
1435}
1436
1437static inline bool sk_wmem_schedule(struct sock *sk, int size)
1438{
1439 if (!sk_has_account(sk))
1440 return true;
1441 return size <= sk->sk_forward_alloc ||
1442 __sk_mem_schedule(sk, size, SK_MEM_SEND);
1443}
1444
1445static inline bool
1446sk_rmem_schedule(struct sock *sk, struct sk_buff *skb, int size)
1447{
1448 if (!sk_has_account(sk))
1449 return true;
1450 return size<= sk->sk_forward_alloc ||
1451 __sk_mem_schedule(sk, size, SK_MEM_RECV) ||
1452 skb_pfmemalloc(skb);
1453}
1454
1455static inline void sk_mem_reclaim(struct sock *sk)
1456{
1457 if (!sk_has_account(sk))
1458 return;
1459 if (sk->sk_forward_alloc >= SK_MEM_QUANTUM)
1460 __sk_mem_reclaim(sk, sk->sk_forward_alloc);
1461}
1462
1463static inline void sk_mem_reclaim_partial(struct sock *sk)
1464{
1465 if (!sk_has_account(sk))
1466 return;
1467 if (sk->sk_forward_alloc > SK_MEM_QUANTUM)
1468 __sk_mem_reclaim(sk, sk->sk_forward_alloc - 1);
1469}
1470
1471static inline void sk_mem_charge(struct sock *sk, int size)
1472{
1473 if (!sk_has_account(sk))
1474 return;
1475 sk->sk_forward_alloc -= size;
1476}
1477
1478static inline void sk_mem_uncharge(struct sock *sk, int size)
1479{
1480 if (!sk_has_account(sk))
1481 return;
1482 sk->sk_forward_alloc += size;
1483}
1484
1485static inline void sk_wmem_free_skb(struct sock *sk, struct sk_buff *skb)
1486{
1487 sock_set_flag(sk, SOCK_QUEUE_SHRUNK);
1488 sk->sk_wmem_queued -= skb->truesize;
1489 sk_mem_uncharge(sk, skb->truesize);
1490 __kfree_skb(skb);
1491}
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506#define sock_owned_by_user(sk) ((sk)->sk_lock.owned)
1507
1508static inline void sock_release_ownership(struct sock *sk)
1509{
1510 if (sk->sk_lock.owned) {
1511 sk->sk_lock.owned = 0;
1512
1513
1514 mutex_release(&sk->sk_lock.dep_map, 1, _RET_IP_);
1515 }
1516}
1517
1518
1519
1520
1521
1522
1523
1524
1525#define sock_lock_init_class_and_name(sk, sname, skey, name, key) \
1526do { \
1527 sk->sk_lock.owned = 0; \
1528 init_waitqueue_head(&sk->sk_lock.wq); \
1529 spin_lock_init(&(sk)->sk_lock.slock); \
1530 debug_check_no_locks_freed((void *)&(sk)->sk_lock, \
1531 sizeof((sk)->sk_lock)); \
1532 lockdep_set_class_and_name(&(sk)->sk_lock.slock, \
1533 (skey), (sname)); \
1534 lockdep_init_map(&(sk)->sk_lock.dep_map, (name), (key), 0); \
1535} while (0)
1536
1537#ifdef CONFIG_LOCKDEP
1538static inline bool lockdep_sock_is_held(const struct sock *csk)
1539{
1540 struct sock *sk = (struct sock *)csk;
1541
1542 return lockdep_is_held(&sk->sk_lock) ||
1543 lockdep_is_held(&sk->sk_lock.slock);
1544}
1545#endif
1546
1547extern void lock_sock_nested(struct sock *sk, int subclass);
1548
1549static inline void lock_sock(struct sock *sk)
1550{
1551 lock_sock_nested(sk, 0);
1552}
1553
1554extern void release_sock(struct sock *sk);
1555
1556
1557#define bh_lock_sock(__sk) spin_lock(&((__sk)->sk_lock.slock))
1558#define bh_lock_sock_nested(__sk) \
1559 spin_lock_nested(&((__sk)->sk_lock.slock), \
1560 SINGLE_DEPTH_NESTING)
1561#define bh_unlock_sock(__sk) spin_unlock(&((__sk)->sk_lock.slock))
1562
1563extern bool lock_sock_fast(struct sock *sk);
1564
1565
1566
1567
1568
1569
1570
1571
1572static inline void unlock_sock_fast(struct sock *sk, bool slow)
1573{
1574 if (slow)
1575 release_sock(sk);
1576 else
1577 spin_unlock_bh(&sk->sk_lock.slock);
1578}
1579
1580
1581extern struct sock *sk_alloc(struct net *net, int family,
1582 gfp_t priority,
1583 struct proto *prot);
1584extern void sk_free(struct sock *sk);
1585extern void sk_release_kernel(struct sock *sk);
1586extern struct sock *sk_clone_lock(const struct sock *sk,
1587 const gfp_t priority);
1588
1589extern struct sk_buff *sock_wmalloc(struct sock *sk,
1590 unsigned long size, int force,
1591 gfp_t priority);
1592extern struct sk_buff *sock_rmalloc(struct sock *sk,
1593 unsigned long size, int force,
1594 gfp_t priority);
1595void __sock_wfree(struct sk_buff *skb);
1596extern void sock_wfree(struct sk_buff *skb);
1597extern void skb_orphan_partial(struct sk_buff *skb);
1598extern void sock_rfree(struct sk_buff *skb);
1599extern void sock_efree(struct sk_buff *skb);
1600#ifdef CONFIG_INET
1601void sock_edemux(struct sk_buff *skb);
1602#else
1603#define sock_edemux sock_efree
1604#endif
1605
1606extern int sock_setsockopt(struct socket *sock, int level,
1607 int op, char __user *optval,
1608 unsigned int optlen);
1609
1610extern int sock_getsockopt(struct socket *sock, int level,
1611 int op, char __user *optval,
1612 int __user *optlen);
1613extern struct sk_buff *sock_alloc_send_skb(struct sock *sk,
1614 unsigned long size,
1615 int noblock,
1616 int *errcode);
1617extern struct sk_buff *sock_alloc_send_pskb(struct sock *sk,
1618 unsigned long header_len,
1619 unsigned long data_len,
1620 int noblock,
1621 int *errcode,
1622 int max_page_order);
1623extern void *sock_kmalloc(struct sock *sk, int size,
1624 gfp_t priority);
1625extern void sock_kfree_s(struct sock *sk, void *mem, int size);
1626extern void sock_kzfree_s(struct sock *sk, void *mem, int size);
1627extern void sk_send_sigurg(struct sock *sk);
1628
1629
1630
1631
1632
1633extern int sock_no_bind(struct socket *,
1634 struct sockaddr *, int);
1635extern int sock_no_connect(struct socket *,
1636 struct sockaddr *, int, int);
1637extern int sock_no_socketpair(struct socket *,
1638 struct socket *);
1639extern int sock_no_accept(struct socket *,
1640 struct socket *, int);
1641extern int sock_no_getname(struct socket *,
1642 struct sockaddr *, int *, int);
1643extern unsigned int sock_no_poll(struct file *, struct socket *,
1644 struct poll_table_struct *);
1645extern int sock_no_ioctl(struct socket *, unsigned int,
1646 unsigned long);
1647extern int sock_no_listen(struct socket *, int);
1648extern int sock_no_shutdown(struct socket *, int);
1649extern int sock_no_getsockopt(struct socket *, int , int,
1650 char __user *, int __user *);
1651extern int sock_no_setsockopt(struct socket *, int, int,
1652 char __user *, unsigned int);
1653extern int sock_no_sendmsg(struct kiocb *, struct socket *,
1654 struct msghdr *, size_t);
1655extern int sock_no_recvmsg(struct kiocb *, struct socket *,
1656 struct msghdr *, size_t, int);
1657extern int sock_no_mmap(struct file *file,
1658 struct socket *sock,
1659 struct vm_area_struct *vma);
1660extern ssize_t sock_no_sendpage(struct socket *sock,
1661 struct page *page,
1662 int offset, size_t size,
1663 int flags);
1664
1665
1666
1667
1668
1669extern int sock_common_getsockopt(struct socket *sock, int level, int optname,
1670 char __user *optval, int __user *optlen);
1671extern int sock_common_recvmsg(struct kiocb *iocb, struct socket *sock,
1672 struct msghdr *msg, size_t size, int flags);
1673extern int sock_common_setsockopt(struct socket *sock, int level, int optname,
1674 char __user *optval, unsigned int optlen);
1675extern int compat_sock_common_getsockopt(struct socket *sock, int level,
1676 int optname, char __user *optval, int __user *optlen);
1677extern int compat_sock_common_setsockopt(struct socket *sock, int level,
1678 int optname, char __user *optval, unsigned int optlen);
1679
1680extern void sk_common_release(struct sock *sk);
1681
1682
1683
1684
1685
1686
1687extern void sock_init_data(struct socket *sock, struct sock *sk);
1688
1689extern void sk_filter_release_rcu(struct rcu_head *rcu);
1690
1691
1692
1693
1694
1695
1696
1697
1698static inline void sk_filter_release(struct sk_filter *fp)
1699{
1700 if (atomic_dec_and_test(&fp->refcnt))
1701 call_rcu(&fp->rcu, sk_filter_release_rcu);
1702}
1703
1704static inline void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp)
1705{
1706 unsigned int size = sk_filter_len(fp);
1707
1708 atomic_sub(size, &sk->sk_omem_alloc);
1709 sk_filter_release(fp);
1710}
1711
1712static inline void sk_filter_charge(struct sock *sk, struct sk_filter *fp)
1713{
1714 atomic_inc(&fp->refcnt);
1715 atomic_add(sk_filter_len(fp), &sk->sk_omem_alloc);
1716}
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744static inline void sock_put(struct sock *sk)
1745{
1746 if (atomic_dec_and_test(&sk->sk_refcnt))
1747 sk_free(sk);
1748}
1749
1750
1751
1752void sock_gen_put(struct sock *sk);
1753
1754extern int sk_receive_skb(struct sock *sk, struct sk_buff *skb,
1755 const int nested);
1756
1757static inline void sk_tx_queue_set(struct sock *sk, int tx_queue)
1758{
1759 sk->sk_tx_queue_mapping = tx_queue;
1760}
1761
1762static inline void sk_tx_queue_clear(struct sock *sk)
1763{
1764 sk->sk_tx_queue_mapping = -1;
1765}
1766
1767static inline int sk_tx_queue_get(const struct sock *sk)
1768{
1769 return sk ? sk->sk_tx_queue_mapping : -1;
1770}
1771
1772static inline void sk_set_socket(struct sock *sk, struct socket *sock)
1773{
1774 sk_tx_queue_clear(sk);
1775 sk->sk_socket = sock;
1776}
1777
1778static inline wait_queue_head_t *sk_sleep(struct sock *sk)
1779{
1780 BUILD_BUG_ON(offsetof(struct socket_wq, wait) != 0);
1781 return &rcu_dereference_raw(sk->sk_wq)->wait;
1782}
1783
1784
1785
1786
1787
1788
1789
1790static inline void sock_orphan(struct sock *sk)
1791{
1792 write_lock_bh(&sk->sk_callback_lock);
1793 sock_set_flag(sk, SOCK_DEAD);
1794 sk_set_socket(sk, NULL);
1795 sk->sk_wq = NULL;
1796 write_unlock_bh(&sk->sk_callback_lock);
1797}
1798
1799static inline void sock_graft(struct sock *sk, struct socket *parent)
1800{
1801 write_lock_bh(&sk->sk_callback_lock);
1802 sk->sk_wq = parent->wq;
1803 parent->sk = sk;
1804 sk_set_socket(sk, parent);
1805 security_sock_graft(sk, parent);
1806 write_unlock_bh(&sk->sk_callback_lock);
1807}
1808
1809extern kuid_t sock_i_uid(struct sock *sk);
1810extern unsigned long sock_i_ino(struct sock *sk);
1811
1812static inline void sk_set_txhash(struct sock *sk)
1813{
1814 sk->sk_txhash = prandom_u32();
1815
1816 if (unlikely(!sk->sk_txhash))
1817 sk->sk_txhash = 1;
1818}
1819
1820static inline struct dst_entry *
1821__sk_dst_get(struct sock *sk)
1822{
1823 return rcu_dereference_check(sk->sk_dst_cache,
1824 lockdep_sock_is_held(sk));
1825}
1826
1827static inline struct dst_entry *
1828sk_dst_get(struct sock *sk)
1829{
1830 struct dst_entry *dst;
1831
1832 rcu_read_lock();
1833 dst = rcu_dereference(sk->sk_dst_cache);
1834 if (dst && !atomic_inc_not_zero(&dst->__refcnt))
1835 dst = NULL;
1836 rcu_read_unlock();
1837 return dst;
1838}
1839
1840extern void sk_reset_txq(struct sock *sk);
1841
1842static inline void dst_negative_advice(struct sock *sk)
1843{
1844 struct dst_entry *ndst, *dst = __sk_dst_get(sk);
1845
1846 if (dst && dst->ops->negative_advice) {
1847 ndst = dst->ops->negative_advice(dst);
1848
1849 if (ndst != dst) {
1850 rcu_assign_pointer(sk->sk_dst_cache, ndst);
1851 sk_reset_txq(sk);
1852 sk->sk_dst_pending_confirm = 0;
1853 }
1854 }
1855}
1856
1857static inline void
1858__sk_dst_set(struct sock *sk, struct dst_entry *dst)
1859{
1860 struct dst_entry *old_dst;
1861
1862 sk_tx_queue_clear(sk);
1863 sk->sk_dst_pending_confirm = 0;
1864
1865
1866
1867
1868 old_dst = rcu_dereference_raw(sk->sk_dst_cache);
1869 rcu_assign_pointer(sk->sk_dst_cache, dst);
1870 dst_release(old_dst);
1871}
1872
1873static inline void
1874sk_dst_set(struct sock *sk, struct dst_entry *dst)
1875{
1876 struct dst_entry *old_dst;
1877
1878 sk_tx_queue_clear(sk);
1879 sk->sk_dst_pending_confirm = 0;
1880 old_dst = xchg(&sk->sk_dst_cache, dst);
1881 dst_release(old_dst);
1882}
1883
1884static inline void
1885__sk_dst_reset(struct sock *sk)
1886{
1887 __sk_dst_set(sk, NULL);
1888}
1889
1890static inline void
1891sk_dst_reset(struct sock *sk)
1892{
1893 sk_dst_set(sk, NULL);
1894}
1895
1896extern struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie);
1897
1898extern struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie);
1899
1900static inline void sk_dst_confirm(struct sock *sk)
1901{
1902 if (!sk->sk_dst_pending_confirm)
1903 sk->sk_dst_pending_confirm = 1;
1904}
1905
1906static inline void sock_confirm_neigh(struct sk_buff *skb, struct neighbour *n)
1907{
1908 if (skb_get_dst_pending_confirm(skb)) {
1909 struct sock *sk = skb->sk;
1910 unsigned long now = jiffies;
1911
1912
1913 if (n->confirmed != now)
1914 n->confirmed = now;
1915 if (sk && sk->sk_dst_pending_confirm)
1916 sk->sk_dst_pending_confirm = 0;
1917 }
1918}
1919
1920bool sk_mc_loop(struct sock *sk);
1921
1922static inline bool sk_can_gso(const struct sock *sk)
1923{
1924 return net_gso_ok(sk->sk_route_caps, sk->sk_gso_type);
1925}
1926
1927extern void sk_setup_caps(struct sock *sk, struct dst_entry *dst);
1928
1929static inline void sk_nocaps_add(struct sock *sk, netdev_features_t flags)
1930{
1931 sk->sk_route_nocaps |= flags;
1932 sk->sk_route_caps &= ~flags;
1933}
1934
1935static inline int skb_do_copy_data_nocache(struct sock *sk, struct sk_buff *skb,
1936 char __user *from, char *to,
1937 int copy, int offset)
1938{
1939 if (skb->ip_summed == CHECKSUM_NONE) {
1940 int err = 0;
1941 __wsum csum = csum_and_copy_from_user(from, to, copy, 0, &err);
1942 if (err)
1943 return err;
1944 skb->csum = csum_block_add(skb->csum, csum, offset);
1945 } else if (sk->sk_route_caps & NETIF_F_NOCACHE_COPY) {
1946 if (!access_ok(VERIFY_READ, from, copy) ||
1947 __copy_from_user_nocache(to, from, copy))
1948 return -EFAULT;
1949 } else if (copy_from_user(to, from, copy))
1950 return -EFAULT;
1951
1952 return 0;
1953}
1954
1955static inline int skb_add_data_nocache(struct sock *sk, struct sk_buff *skb,
1956 char __user *from, int copy)
1957{
1958 int err, offset = skb->len;
1959
1960 err = skb_do_copy_data_nocache(sk, skb, from, skb_put(skb, copy),
1961 copy, offset);
1962 if (err)
1963 __skb_trim(skb, offset);
1964
1965 return err;
1966}
1967
1968static inline int skb_copy_to_page_nocache(struct sock *sk, char __user *from,
1969 struct sk_buff *skb,
1970 struct page *page,
1971 int off, int copy)
1972{
1973 int err;
1974
1975 err = skb_do_copy_data_nocache(sk, skb, from, page_address(page) + off,
1976 copy, skb->len);
1977 if (err)
1978 return err;
1979
1980 skb->len += copy;
1981 skb->data_len += copy;
1982 skb->truesize += copy;
1983 sk->sk_wmem_queued += copy;
1984 sk_mem_charge(sk, copy);
1985 return 0;
1986}
1987
1988static inline int skb_copy_to_page(struct sock *sk, char __user *from,
1989 struct sk_buff *skb, struct page *page,
1990 int off, int copy)
1991{
1992 if (skb->ip_summed == CHECKSUM_NONE) {
1993 int err = 0;
1994 __wsum csum = csum_and_copy_from_user(from,
1995 page_address(page) + off,
1996 copy, 0, &err);
1997 if (err)
1998 return err;
1999 skb->csum = csum_block_add(skb->csum, csum, skb->len);
2000 } else if (copy_from_user(page_address(page) + off, from, copy))
2001 return -EFAULT;
2002
2003 skb->len += copy;
2004 skb->data_len += copy;
2005 skb->truesize += copy;
2006 sk->sk_wmem_queued += copy;
2007 sk_mem_charge(sk, copy);
2008 return 0;
2009}
2010
2011
2012
2013
2014
2015
2016
2017static inline int sk_wmem_alloc_get(const struct sock *sk)
2018{
2019 return atomic_read(&sk->sk_wmem_alloc) - 1;
2020}
2021
2022
2023
2024
2025
2026
2027
2028static inline int sk_rmem_alloc_get(const struct sock *sk)
2029{
2030 return atomic_read(&sk->sk_rmem_alloc);
2031}
2032
2033
2034
2035
2036
2037
2038
2039static inline bool sk_has_allocations(const struct sock *sk)
2040{
2041 return sk_wmem_alloc_get(sk) || sk_rmem_alloc_get(sk);
2042}
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076static inline bool wq_has_sleeper(struct socket_wq *wq)
2077{
2078
2079
2080
2081
2082
2083 smp_mb();
2084 return wq && waitqueue_active(&wq->wait);
2085}
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095static inline void sock_poll_wait(struct file *filp,
2096 wait_queue_head_t *wait_address, poll_table *p)
2097{
2098 if (!poll_does_not_wait(p) && wait_address) {
2099 poll_wait(filp, wait_address, p);
2100
2101
2102
2103
2104
2105 smp_mb();
2106 }
2107}
2108
2109static inline void skb_set_hash_from_sk(struct sk_buff *skb, struct sock *sk)
2110{
2111 if (sk->sk_txhash) {
2112 skb->l4_hash = 1;
2113 skb->hash = sk->sk_txhash;
2114 }
2115}
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126static inline void skb_set_owner_w(struct sk_buff *skb, struct sock *sk)
2127{
2128 skb_orphan(skb);
2129 skb->sk = sk;
2130 skb->destructor = sock_wfree;
2131 skb_set_hash_from_sk(skb, sk);
2132
2133
2134
2135
2136
2137 atomic_add(skb->truesize, &sk->sk_wmem_alloc);
2138}
2139
2140static inline void skb_set_owner_r(struct sk_buff *skb, struct sock *sk)
2141{
2142 skb_orphan(skb);
2143 skb->sk = sk;
2144 skb->destructor = sock_rfree;
2145 atomic_add(skb->truesize, &sk->sk_rmem_alloc);
2146 sk_mem_charge(sk, skb->truesize);
2147}
2148
2149extern void sk_reset_timer(struct sock *sk, struct timer_list *timer,
2150 unsigned long expires);
2151
2152extern void sk_stop_timer(struct sock *sk, struct timer_list *timer);
2153
2154int __sk_queue_drop_skb(struct sock *sk, struct sk_buff *skb,
2155 unsigned int flags,
2156 void (*destructor)(struct sock *sk,
2157 struct sk_buff *skb));
2158int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb);
2159extern int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb);
2160
2161extern int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb);
2162
2163
2164
2165
2166
2167static inline int sock_error(struct sock *sk)
2168{
2169 int err;
2170 if (likely(!sk->sk_err))
2171 return 0;
2172 err = xchg(&sk->sk_err, 0);
2173 return -err;
2174}
2175
2176static inline unsigned long sock_wspace(struct sock *sk)
2177{
2178 int amt = 0;
2179
2180 if (!(sk->sk_shutdown & SEND_SHUTDOWN)) {
2181 amt = sk->sk_sndbuf - atomic_read(&sk->sk_wmem_alloc);
2182 if (amt < 0)
2183 amt = 0;
2184 }
2185 return amt;
2186}
2187
2188static inline void sk_wake_async(struct sock *sk, int how, int band)
2189{
2190 if (sock_flag(sk, SOCK_FASYNC))
2191 sock_wake_async(sk->sk_socket, how, band);
2192}
2193
2194
2195
2196
2197
2198
2199#define TCP_SKB_MIN_TRUESIZE (2048 + SKB_DATA_ALIGN(sizeof(struct sk_buff)))
2200
2201#define SOCK_MIN_SNDBUF (TCP_SKB_MIN_TRUESIZE * 2)
2202#define SOCK_MIN_RCVBUF TCP_SKB_MIN_TRUESIZE
2203
2204static inline void sk_stream_moderate_sndbuf(struct sock *sk)
2205{
2206 if (!(sk->sk_userlocks & SOCK_SNDBUF_LOCK)) {
2207 sk->sk_sndbuf = min(sk->sk_sndbuf, sk->sk_wmem_queued >> 1);
2208 sk->sk_sndbuf = max_t(u32, sk->sk_sndbuf, SOCK_MIN_SNDBUF);
2209 }
2210}
2211
2212struct sk_buff *sk_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp);
2213
2214
2215
2216
2217
2218
2219
2220
2221static inline struct page_frag *sk_page_frag(struct sock *sk)
2222{
2223 if (sk->sk_allocation & __GFP_WAIT)
2224 return ¤t->task_frag;
2225
2226 return &sk->sk_frag;
2227}
2228
2229extern bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag);
2230
2231
2232
2233
2234static inline bool sock_writeable(const struct sock *sk)
2235{
2236 return atomic_read(&sk->sk_wmem_alloc) < (sk->sk_sndbuf >> 1);
2237}
2238
2239static inline gfp_t gfp_any(void)
2240{
2241 return in_softirq() ? GFP_ATOMIC : GFP_KERNEL;
2242}
2243
2244static inline long sock_rcvtimeo(const struct sock *sk, bool noblock)
2245{
2246 return noblock ? 0 : sk->sk_rcvtimeo;
2247}
2248
2249static inline long sock_sndtimeo(const struct sock *sk, bool noblock)
2250{
2251 return noblock ? 0 : sk->sk_sndtimeo;
2252}
2253
2254static inline int sock_rcvlowat(const struct sock *sk, int waitall, int len)
2255{
2256 return (waitall ? len : min_t(int, sk->sk_rcvlowat, len)) ? : 1;
2257}
2258
2259
2260
2261
2262static inline int sock_intr_errno(long timeo)
2263{
2264 return timeo == MAX_SCHEDULE_TIMEOUT ? -ERESTARTSYS : -EINTR;
2265}
2266
2267static inline void
2268sock_skb_set_dropcount(const struct sock *sk, struct sk_buff *skb)
2269{
2270 skb->dropcount = atomic_read(&sk->sk_drops);
2271}
2272
2273static inline void sk_drops_add(struct sock *sk, const struct sk_buff *skb)
2274{
2275 int segs = max_t(u16, 1, skb_shinfo(skb)->gso_segs);
2276
2277 atomic_add(segs, &sk->sk_drops);
2278}
2279
2280extern void __sock_recv_timestamp(struct msghdr *msg, struct sock *sk,
2281 struct sk_buff *skb);
2282extern void __sock_recv_wifi_status(struct msghdr *msg, struct sock *sk,
2283 struct sk_buff *skb);
2284
2285static inline void
2286sock_recv_timestamp(struct msghdr *msg, struct sock *sk, struct sk_buff *skb)
2287{
2288 ktime_t kt = skb->tstamp;
2289 struct skb_shared_hwtstamps *hwtstamps = skb_hwtstamps(skb);
2290
2291
2292
2293
2294
2295
2296
2297 if (sock_flag(sk, SOCK_RCVTSTAMP) ||
2298 (sk->sk_tsflags & SOF_TIMESTAMPING_RX_SOFTWARE) ||
2299 (kt.tv64 && sk->sk_tsflags & SOF_TIMESTAMPING_SOFTWARE) ||
2300 (hwtstamps->hwtstamp.tv64 &&
2301 (sk->sk_tsflags & SOF_TIMESTAMPING_RAW_HARDWARE)) ||
2302 (hwtstamps->syststamp.tv64 &&
2303 (sk->sk_tsflags & SOF_TIMESTAMPING_SYS_HARDWARE)))
2304 __sock_recv_timestamp(msg, sk, skb);
2305 else
2306 sk->sk_stamp = kt;
2307
2308 if (sock_flag(sk, SOCK_WIFI_STATUS) && skb->wifi_acked_valid)
2309 __sock_recv_wifi_status(msg, sk, skb);
2310}
2311
2312extern void __sock_recv_ts_and_drops(struct msghdr *msg, struct sock *sk,
2313 struct sk_buff *skb);
2314
2315static inline void sock_recv_ts_and_drops(struct msghdr *msg, struct sock *sk,
2316 struct sk_buff *skb)
2317{
2318#define FLAGS_TS_OR_DROPS ((1UL << SOCK_RXQ_OVFL) | \
2319 (1UL << SOCK_RCVTSTAMP))
2320#define TSFLAGS_ANY (SOF_TIMESTAMPING_SOFTWARE | \
2321 SOF_TIMESTAMPING_RAW_HARDWARE | \
2322 SOF_TIMESTAMPING_SYS_HARDWARE)
2323
2324 if (sk->sk_flags & FLAGS_TS_OR_DROPS || sk->sk_tsflags & TSFLAGS_ANY)
2325 __sock_recv_ts_and_drops(msg, sk, skb);
2326 else
2327 sk->sk_stamp = skb->tstamp;
2328}
2329
2330
2331
2332
2333
2334
2335extern void sock_tx_timestamp(struct sock *sk, __u8 *tx_flags);
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345static inline void sk_eat_skb(struct sock *sk, struct sk_buff *skb)
2346{
2347 __skb_unlink(skb, &sk->sk_receive_queue);
2348 __kfree_skb(skb);
2349}
2350
2351static inline
2352struct net *sock_net(const struct sock *sk)
2353{
2354 return read_pnet(&sk->sk_net);
2355}
2356
2357static inline
2358void sock_net_set(struct sock *sk, struct net *net)
2359{
2360 write_pnet(&sk->sk_net, net);
2361}
2362
2363
2364
2365
2366
2367
2368
2369static inline void sk_change_net(struct sock *sk, struct net *net)
2370{
2371 struct net *current_net = sock_net(sk);
2372
2373 if (!net_eq(current_net, net)) {
2374 put_net(current_net);
2375 sock_net_set(sk, net);
2376 }
2377}
2378
2379static inline struct sock *skb_steal_sock(struct sk_buff *skb)
2380{
2381 if (skb->sk) {
2382 struct sock *sk = skb->sk;
2383
2384 skb->destructor = NULL;
2385 skb->sk = NULL;
2386 return sk;
2387 }
2388 return NULL;
2389}
2390
2391extern void sock_enable_timestamp(struct sock *sk, int flag);
2392extern int sock_get_timestamp(struct sock *, struct timeval __user *);
2393extern int sock_get_timestampns(struct sock *, struct timespec __user *);
2394
2395
2396
2397
2398extern int net_msg_warn;
2399#define NETDEBUG(fmt, args...) \
2400 do { if (net_msg_warn) printk(fmt,##args); } while (0)
2401
2402#define LIMIT_NETDEBUG(fmt, args...) \
2403 do { if (net_msg_warn && net_ratelimit()) printk(fmt,##args); } while(0)
2404
2405
2406
2407
2408static inline bool sk_fullsock(const struct sock *sk)
2409{
2410 return (1 << sk->sk_state) & ~(TCPF_TIME_WAIT);
2411}
2412
2413bool sk_ns_capable(const struct sock *sk,
2414 struct user_namespace *user_ns, int cap);
2415bool sk_capable(const struct sock *sk, int cap);
2416bool sk_net_capable(const struct sock *sk, int cap);
2417
2418extern __u32 sysctl_wmem_max;
2419extern __u32 sysctl_rmem_max;
2420
2421extern int sysctl_optmem_max;
2422
2423extern __u32 sysctl_wmem_default;
2424extern __u32 sysctl_rmem_default;
2425
2426#endif
2427