1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40#ifndef _SOCK_H
41#define _SOCK_H
42
43#include <linux/hardirq.h>
44#include <linux/kernel.h>
45#include <linux/list.h>
46#include <linux/list_nulls.h>
47#include <linux/timer.h>
48#include <linux/cache.h>
49#include <linux/bitops.h>
50#include <linux/lockdep.h>
51#include <linux/netdevice.h>
52#include <linux/skbuff.h>
53#include <linux/mm.h>
54#include <linux/security.h>
55#include <linux/slab.h>
56#include <linux/uaccess.h>
57#include <linux/memcontrol.h>
58#include <linux/res_counter.h>
59#include <linux/static_key.h>
60#include <linux/aio.h>
61#include <linux/sched.h>
62
63#include <linux/filter.h>
64#include <linux/rculist_nulls.h>
65#include <linux/poll.h>
66
67#include <linux/atomic.h>
68#include <net/dst.h>
69#include <net/checksum.h>
70
71struct cgroup;
72struct cgroup_subsys;
73#ifdef CONFIG_NET
74int mem_cgroup_sockets_init(struct mem_cgroup *memcg, struct cgroup_subsys *ss);
75void mem_cgroup_sockets_destroy(struct mem_cgroup *memcg);
76#else
77static inline
78int mem_cgroup_sockets_init(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
79{
80 return 0;
81}
82static inline
83void mem_cgroup_sockets_destroy(struct mem_cgroup *memcg)
84{
85}
86#endif
87
88
89
90
91
92
93
94#define SOCK_DEBUGGING
95#ifdef SOCK_DEBUGGING
96#define SOCK_DEBUG(sk, msg...) do { if ((sk) && sock_flag((sk), SOCK_DBG)) \
97 printk(KERN_DEBUG msg); } while (0)
98#else
99
100static inline __printf(2, 3)
101void SOCK_DEBUG(const struct sock *sk, const char *msg, ...)
102{
103}
104#endif
105
106
107
108
109
110typedef struct {
111 spinlock_t slock;
112 int owned;
113 wait_queue_head_t wq;
114
115
116
117
118
119
120#ifdef CONFIG_DEBUG_LOCK_ALLOC
121 struct lockdep_map dep_map;
122#endif
123} socket_lock_t;
124
125struct sock;
126struct proto;
127struct net;
128
129typedef __u32 __bitwise __portpair;
130typedef __u64 __bitwise __addrpair;
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157struct sock_common {
158
159
160
161 union {
162 __addrpair skc_addrpair;
163 struct {
164 __be32 skc_daddr;
165 __be32 skc_rcv_saddr;
166 };
167 };
168 union {
169 unsigned int skc_hash;
170 __u16 skc_u16hashes[2];
171 };
172
173 union {
174 __portpair skc_portpair;
175 struct {
176 __be16 skc_dport;
177 __u16 skc_num;
178 };
179 };
180
181 unsigned short skc_family;
182 volatile unsigned char skc_state;
183 unsigned char skc_reuse:4;
184 unsigned char skc_reuseport:4;
185 int skc_bound_dev_if;
186 union {
187 struct hlist_node skc_bind_node;
188 struct hlist_nulls_node skc_portaddr_node;
189 };
190 struct proto *skc_prot;
191#ifdef CONFIG_NET_NS
192 struct net *skc_net;
193#endif
194
195
196
197
198
199 int skc_dontcopy_begin[0];
200
201 union {
202 struct hlist_node skc_node;
203 struct hlist_nulls_node skc_nulls_node;
204 };
205 int skc_tx_queue_mapping;
206 atomic_t skc_refcnt;
207
208 int skc_dontcopy_end[0];
209
210};
211
212struct cg_proto;
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288struct sock {
289
290
291
292
293 struct sock_common __sk_common;
294#define sk_node __sk_common.skc_node
295#define sk_nulls_node __sk_common.skc_nulls_node
296#define sk_refcnt __sk_common.skc_refcnt
297#define sk_tx_queue_mapping __sk_common.skc_tx_queue_mapping
298
299#define sk_dontcopy_begin __sk_common.skc_dontcopy_begin
300#define sk_dontcopy_end __sk_common.skc_dontcopy_end
301#define sk_hash __sk_common.skc_hash
302#define sk_family __sk_common.skc_family
303#define sk_state __sk_common.skc_state
304#define sk_reuse __sk_common.skc_reuse
305#define sk_reuseport __sk_common.skc_reuseport
306#define sk_bound_dev_if __sk_common.skc_bound_dev_if
307#define sk_bind_node __sk_common.skc_bind_node
308#define sk_prot __sk_common.skc_prot
309#define sk_net __sk_common.skc_net
310 socket_lock_t sk_lock;
311 struct sk_buff_head sk_receive_queue;
312
313
314
315
316
317
318
319
320 struct {
321 atomic_t rmem_alloc;
322 int len;
323 struct sk_buff *head;
324 struct sk_buff *tail;
325 } sk_backlog;
326#define sk_rmem_alloc sk_backlog.rmem_alloc
327 int sk_forward_alloc;
328#ifdef CONFIG_RPS
329 __u32 sk_rxhash;
330#endif
331#ifdef CONFIG_NET_RX_BUSY_POLL
332 unsigned int sk_napi_id;
333 unsigned int sk_ll_usec;
334#endif
335 atomic_t sk_drops;
336 int sk_rcvbuf;
337
338 struct sk_filter __rcu *sk_filter;
339 struct socket_wq __rcu *sk_wq;
340
341#ifdef CONFIG_NET_DMA
342 struct sk_buff_head sk_async_wait_queue;
343#endif
344
345#ifdef CONFIG_XFRM
346 struct xfrm_policy *sk_policy[2];
347#endif
348 unsigned long sk_flags;
349 struct dst_entry *sk_rx_dst;
350 struct dst_entry __rcu *sk_dst_cache;
351 spinlock_t sk_dst_lock;
352 atomic_t sk_wmem_alloc;
353 atomic_t sk_omem_alloc;
354 int sk_sndbuf;
355 struct sk_buff_head sk_write_queue;
356 kmemcheck_bitfield_begin(flags);
357 unsigned int sk_shutdown : 2,
358 sk_no_check : 2,
359 sk_userlocks : 4,
360 sk_protocol : 8,
361 sk_type : 16;
362 kmemcheck_bitfield_end(flags);
363 int sk_wmem_queued;
364 gfp_t sk_allocation;
365 u32 sk_pacing_rate;
366 netdev_features_t sk_route_caps;
367 netdev_features_t sk_route_nocaps;
368 int sk_gso_type;
369 unsigned int sk_gso_max_size;
370 u16 sk_gso_max_segs;
371 int sk_rcvlowat;
372 unsigned long sk_lingertime;
373 struct sk_buff_head sk_error_queue;
374 struct proto *sk_prot_creator;
375 rwlock_t sk_callback_lock;
376 int sk_err,
377 sk_err_soft;
378 unsigned short sk_ack_backlog;
379 unsigned short sk_max_ack_backlog;
380 __u32 sk_priority;
381#if IS_ENABLED(CONFIG_NETPRIO_CGROUP)
382 __u32 sk_cgrp_prioidx;
383#endif
384 struct pid *sk_peer_pid;
385 const struct cred *sk_peer_cred;
386 long sk_rcvtimeo;
387 long sk_sndtimeo;
388 void *sk_protinfo;
389 struct timer_list sk_timer;
390 ktime_t sk_stamp;
391 struct socket *sk_socket;
392 void *sk_user_data;
393 struct page_frag sk_frag;
394 struct sk_buff *sk_send_head;
395 __s32 sk_peek_off;
396 int sk_write_pending;
397#ifdef CONFIG_SECURITY
398 void *sk_security;
399#endif
400 __u32 sk_mark;
401 u32 sk_classid;
402 struct cg_proto *sk_cgrp;
403 void (*sk_state_change)(struct sock *sk);
404 void (*sk_data_ready)(struct sock *sk, int bytes);
405 void (*sk_write_space)(struct sock *sk);
406 void (*sk_error_report)(struct sock *sk);
407 int (*sk_backlog_rcv)(struct sock *sk,
408 struct sk_buff *skb);
409 void (*sk_destruct)(struct sock *sk);
410};
411
412#define __sk_user_data(sk) ((*((void __rcu **)&(sk)->sk_user_data)))
413
414#define rcu_dereference_sk_user_data(sk) rcu_dereference(__sk_user_data((sk)))
415#define rcu_assign_sk_user_data(sk, ptr) rcu_assign_pointer(__sk_user_data((sk)), ptr)
416
417
418
419
420
421
422
423
424#define SK_NO_REUSE 0
425#define SK_CAN_REUSE 1
426#define SK_FORCE_REUSE 2
427
428static inline int sk_peek_offset(struct sock *sk, int flags)
429{
430 if ((flags & MSG_PEEK) && (sk->sk_peek_off >= 0))
431 return sk->sk_peek_off;
432 else
433 return 0;
434}
435
436static inline void sk_peek_offset_bwd(struct sock *sk, int val)
437{
438 if (sk->sk_peek_off >= 0) {
439 if (sk->sk_peek_off >= val)
440 sk->sk_peek_off -= val;
441 else
442 sk->sk_peek_off = 0;
443 }
444}
445
446static inline void sk_peek_offset_fwd(struct sock *sk, int val)
447{
448 if (sk->sk_peek_off >= 0)
449 sk->sk_peek_off += val;
450}
451
452
453
454
455static inline struct sock *sk_entry(const struct hlist_node *node)
456{
457 return hlist_entry(node, struct sock, sk_node);
458}
459
460static inline struct sock *__sk_head(const struct hlist_head *head)
461{
462 return hlist_entry(head->first, struct sock, sk_node);
463}
464
465static inline struct sock *sk_head(const struct hlist_head *head)
466{
467 return hlist_empty(head) ? NULL : __sk_head(head);
468}
469
470static inline struct sock *__sk_nulls_head(const struct hlist_nulls_head *head)
471{
472 return hlist_nulls_entry(head->first, struct sock, sk_nulls_node);
473}
474
475static inline struct sock *sk_nulls_head(const struct hlist_nulls_head *head)
476{
477 return hlist_nulls_empty(head) ? NULL : __sk_nulls_head(head);
478}
479
480static inline struct sock *sk_next(const struct sock *sk)
481{
482 return sk->sk_node.next ?
483 hlist_entry(sk->sk_node.next, struct sock, sk_node) : NULL;
484}
485
486static inline struct sock *sk_nulls_next(const struct sock *sk)
487{
488 return (!is_a_nulls(sk->sk_nulls_node.next)) ?
489 hlist_nulls_entry(sk->sk_nulls_node.next,
490 struct sock, sk_nulls_node) :
491 NULL;
492}
493
494static inline bool sk_unhashed(const struct sock *sk)
495{
496 return hlist_unhashed(&sk->sk_node);
497}
498
499static inline bool sk_hashed(const struct sock *sk)
500{
501 return !sk_unhashed(sk);
502}
503
504static inline void sk_node_init(struct hlist_node *node)
505{
506 node->pprev = NULL;
507}
508
509static inline void sk_nulls_node_init(struct hlist_nulls_node *node)
510{
511 node->pprev = NULL;
512}
513
514static inline void __sk_del_node(struct sock *sk)
515{
516 __hlist_del(&sk->sk_node);
517}
518
519
520static inline bool __sk_del_node_init(struct sock *sk)
521{
522 if (sk_hashed(sk)) {
523 __sk_del_node(sk);
524 sk_node_init(&sk->sk_node);
525 return true;
526 }
527 return false;
528}
529
530
531
532
533
534
535
536static inline void sock_hold(struct sock *sk)
537{
538 atomic_inc(&sk->sk_refcnt);
539}
540
541
542
543
544static inline void __sock_put(struct sock *sk)
545{
546 atomic_dec(&sk->sk_refcnt);
547}
548
549static inline bool sk_del_node_init(struct sock *sk)
550{
551 bool rc = __sk_del_node_init(sk);
552
553 if (rc) {
554
555 WARN_ON(atomic_read(&sk->sk_refcnt) == 1);
556 __sock_put(sk);
557 }
558 return rc;
559}
560#define sk_del_node_init_rcu(sk) sk_del_node_init(sk)
561
562static inline bool __sk_nulls_del_node_init_rcu(struct sock *sk)
563{
564 if (sk_hashed(sk)) {
565 hlist_nulls_del_init_rcu(&sk->sk_nulls_node);
566 return true;
567 }
568 return false;
569}
570
571static inline bool sk_nulls_del_node_init_rcu(struct sock *sk)
572{
573 bool rc = __sk_nulls_del_node_init_rcu(sk);
574
575 if (rc) {
576
577 WARN_ON(atomic_read(&sk->sk_refcnt) == 1);
578 __sock_put(sk);
579 }
580 return rc;
581}
582
583static inline void __sk_add_node(struct sock *sk, struct hlist_head *list)
584{
585 hlist_add_head(&sk->sk_node, list);
586}
587
588static inline void sk_add_node(struct sock *sk, struct hlist_head *list)
589{
590 sock_hold(sk);
591 __sk_add_node(sk, list);
592}
593
594static inline void sk_add_node_rcu(struct sock *sk, struct hlist_head *list)
595{
596 sock_hold(sk);
597 hlist_add_head_rcu(&sk->sk_node, list);
598}
599
600static inline void __sk_nulls_add_node_rcu(struct sock *sk, struct hlist_nulls_head *list)
601{
602 hlist_nulls_add_head_rcu(&sk->sk_nulls_node, list);
603}
604
605static inline void sk_nulls_add_node_rcu(struct sock *sk, struct hlist_nulls_head *list)
606{
607 sock_hold(sk);
608 __sk_nulls_add_node_rcu(sk, list);
609}
610
611static inline void __sk_del_bind_node(struct sock *sk)
612{
613 __hlist_del(&sk->sk_bind_node);
614}
615
616static inline void sk_add_bind_node(struct sock *sk,
617 struct hlist_head *list)
618{
619 hlist_add_head(&sk->sk_bind_node, list);
620}
621
622#define sk_for_each(__sk, list) \
623 hlist_for_each_entry(__sk, list, sk_node)
624#define sk_for_each_rcu(__sk, list) \
625 hlist_for_each_entry_rcu(__sk, list, sk_node)
626#define sk_nulls_for_each(__sk, node, list) \
627 hlist_nulls_for_each_entry(__sk, node, list, sk_nulls_node)
628#define sk_nulls_for_each_rcu(__sk, node, list) \
629 hlist_nulls_for_each_entry_rcu(__sk, node, list, sk_nulls_node)
630#define sk_for_each_from(__sk) \
631 hlist_for_each_entry_from(__sk, sk_node)
632#define sk_nulls_for_each_from(__sk, node) \
633 if (__sk && ({ node = &(__sk)->sk_nulls_node; 1; })) \
634 hlist_nulls_for_each_entry_from(__sk, node, sk_nulls_node)
635#define sk_for_each_safe(__sk, tmp, list) \
636 hlist_for_each_entry_safe(__sk, tmp, list, sk_node)
637#define sk_for_each_bound(__sk, list) \
638 hlist_for_each_entry(__sk, list, sk_bind_node)
639
640static inline struct user_namespace *sk_user_ns(struct sock *sk)
641{
642
643
644
645
646 return sk->sk_socket->file->f_cred->user_ns;
647}
648
649
650enum sock_flags {
651 SOCK_DEAD,
652 SOCK_DONE,
653 SOCK_URGINLINE,
654 SOCK_KEEPOPEN,
655 SOCK_LINGER,
656 SOCK_DESTROY,
657 SOCK_BROADCAST,
658 SOCK_TIMESTAMP,
659 SOCK_ZAPPED,
660 SOCK_USE_WRITE_QUEUE,
661 SOCK_DBG,
662 SOCK_RCVTSTAMP,
663 SOCK_RCVTSTAMPNS,
664 SOCK_LOCALROUTE,
665 SOCK_QUEUE_SHRUNK,
666 SOCK_MEMALLOC,
667 SOCK_TIMESTAMPING_TX_HARDWARE,
668 SOCK_TIMESTAMPING_TX_SOFTWARE,
669 SOCK_TIMESTAMPING_RX_HARDWARE,
670 SOCK_TIMESTAMPING_RX_SOFTWARE,
671 SOCK_TIMESTAMPING_SOFTWARE,
672 SOCK_TIMESTAMPING_RAW_HARDWARE,
673 SOCK_TIMESTAMPING_SYS_HARDWARE,
674 SOCK_FASYNC,
675 SOCK_RXQ_OVFL,
676 SOCK_ZEROCOPY,
677 SOCK_WIFI_STATUS,
678 SOCK_NOFCS,
679
680
681
682 SOCK_FILTER_LOCKED,
683 SOCK_SELECT_ERR_QUEUE,
684};
685
686static inline void sock_copy_flags(struct sock *nsk, struct sock *osk)
687{
688 nsk->sk_flags = osk->sk_flags;
689}
690
691static inline void sock_set_flag(struct sock *sk, enum sock_flags flag)
692{
693 __set_bit(flag, &sk->sk_flags);
694}
695
696static inline void sock_reset_flag(struct sock *sk, enum sock_flags flag)
697{
698 __clear_bit(flag, &sk->sk_flags);
699}
700
701static inline bool sock_flag(const struct sock *sk, enum sock_flags flag)
702{
703 return test_bit(flag, &sk->sk_flags);
704}
705
706#ifdef CONFIG_NET
707extern struct static_key memalloc_socks;
708static inline int sk_memalloc_socks(void)
709{
710 return static_key_false(&memalloc_socks);
711}
712#else
713
714static inline int sk_memalloc_socks(void)
715{
716 return 0;
717}
718
719#endif
720
721static inline gfp_t sk_gfp_atomic(struct sock *sk, gfp_t gfp_mask)
722{
723 return GFP_ATOMIC | (sk->sk_allocation & __GFP_MEMALLOC);
724}
725
726static inline void sk_acceptq_removed(struct sock *sk)
727{
728 sk->sk_ack_backlog--;
729}
730
731static inline void sk_acceptq_added(struct sock *sk)
732{
733 sk->sk_ack_backlog++;
734}
735
736static inline bool sk_acceptq_is_full(const struct sock *sk)
737{
738 return sk->sk_ack_backlog > sk->sk_max_ack_backlog;
739}
740
741
742
743
744static inline int sk_stream_min_wspace(const struct sock *sk)
745{
746 return sk->sk_wmem_queued >> 1;
747}
748
749static inline int sk_stream_wspace(const struct sock *sk)
750{
751 return sk->sk_sndbuf - sk->sk_wmem_queued;
752}
753
754extern void sk_stream_write_space(struct sock *sk);
755
756
757static inline void __sk_add_backlog(struct sock *sk, struct sk_buff *skb)
758{
759
760 skb_dst_force(skb);
761
762 if (!sk->sk_backlog.tail)
763 sk->sk_backlog.head = skb;
764 else
765 sk->sk_backlog.tail->next = skb;
766
767 sk->sk_backlog.tail = skb;
768 skb->next = NULL;
769}
770
771
772
773
774
775
776static inline bool sk_rcvqueues_full(const struct sock *sk, const struct sk_buff *skb,
777 unsigned int limit)
778{
779 unsigned int qsize = sk->sk_backlog.len + atomic_read(&sk->sk_rmem_alloc);
780
781 return qsize > limit;
782}
783
784
785static inline __must_check int sk_add_backlog(struct sock *sk, struct sk_buff *skb,
786 unsigned int limit)
787{
788 if (sk_rcvqueues_full(sk, skb, limit))
789 return -ENOBUFS;
790
791 __sk_add_backlog(sk, skb);
792 sk->sk_backlog.len += skb->truesize;
793 return 0;
794}
795
796extern int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb);
797
798static inline int sk_backlog_rcv(struct sock *sk, struct sk_buff *skb)
799{
800 if (sk_memalloc_socks() && skb_pfmemalloc(skb))
801 return __sk_backlog_rcv(sk, skb);
802
803 return sk->sk_backlog_rcv(sk, skb);
804}
805
806static inline void sock_rps_record_flow(const struct sock *sk)
807{
808#ifdef CONFIG_RPS
809 struct rps_sock_flow_table *sock_flow_table;
810
811 rcu_read_lock();
812 sock_flow_table = rcu_dereference(rps_sock_flow_table);
813 rps_record_sock_flow(sock_flow_table, sk->sk_rxhash);
814 rcu_read_unlock();
815#endif
816}
817
818static inline void sock_rps_reset_flow(const struct sock *sk)
819{
820#ifdef CONFIG_RPS
821 struct rps_sock_flow_table *sock_flow_table;
822
823 rcu_read_lock();
824 sock_flow_table = rcu_dereference(rps_sock_flow_table);
825 rps_reset_sock_flow(sock_flow_table, sk->sk_rxhash);
826 rcu_read_unlock();
827#endif
828}
829
830static inline void sock_rps_save_rxhash(struct sock *sk,
831 const struct sk_buff *skb)
832{
833#ifdef CONFIG_RPS
834 if (unlikely(sk->sk_rxhash != skb->rxhash)) {
835 sock_rps_reset_flow(sk);
836 sk->sk_rxhash = skb->rxhash;
837 }
838#endif
839}
840
841static inline void sock_rps_reset_rxhash(struct sock *sk)
842{
843#ifdef CONFIG_RPS
844 sock_rps_reset_flow(sk);
845 sk->sk_rxhash = 0;
846#endif
847}
848
849#define sk_wait_event(__sk, __timeo, __condition) \
850 ({ int __rc; \
851 release_sock(__sk); \
852 __rc = __condition; \
853 if (!__rc) { \
854 *(__timeo) = schedule_timeout(*(__timeo)); \
855 } \
856 lock_sock(__sk); \
857 __rc = __condition; \
858 __rc; \
859 })
860
861extern int sk_stream_wait_connect(struct sock *sk, long *timeo_p);
862extern int sk_stream_wait_memory(struct sock *sk, long *timeo_p);
863extern void sk_stream_wait_close(struct sock *sk, long timeo_p);
864extern int sk_stream_error(struct sock *sk, int flags, int err);
865extern void sk_stream_kill_queues(struct sock *sk);
866extern void sk_set_memalloc(struct sock *sk);
867extern void sk_clear_memalloc(struct sock *sk);
868
869extern int sk_wait_data(struct sock *sk, long *timeo);
870
871struct request_sock_ops;
872struct timewait_sock_ops;
873struct inet_hashinfo;
874struct raw_hashinfo;
875struct module;
876
877
878
879
880
881static inline void sk_prot_clear_nulls(struct sock *sk, int size)
882{
883 if (offsetof(struct sock, sk_node.next) != 0)
884 memset(sk, 0, offsetof(struct sock, sk_node.next));
885 memset(&sk->sk_node.pprev, 0,
886 size - offsetof(struct sock, sk_node.pprev));
887}
888
889
890
891
892
893struct proto {
894 void (*close)(struct sock *sk,
895 long timeout);
896 int (*connect)(struct sock *sk,
897 struct sockaddr *uaddr,
898 int addr_len);
899 int (*disconnect)(struct sock *sk, int flags);
900
901 struct sock * (*accept)(struct sock *sk, int flags, int *err);
902
903 int (*ioctl)(struct sock *sk, int cmd,
904 unsigned long arg);
905 int (*init)(struct sock *sk);
906 void (*destroy)(struct sock *sk);
907 void (*shutdown)(struct sock *sk, int how);
908 int (*setsockopt)(struct sock *sk, int level,
909 int optname, char __user *optval,
910 unsigned int optlen);
911 int (*getsockopt)(struct sock *sk, int level,
912 int optname, char __user *optval,
913 int __user *option);
914#ifdef CONFIG_COMPAT
915 int (*compat_setsockopt)(struct sock *sk,
916 int level,
917 int optname, char __user *optval,
918 unsigned int optlen);
919 int (*compat_getsockopt)(struct sock *sk,
920 int level,
921 int optname, char __user *optval,
922 int __user *option);
923 int (*compat_ioctl)(struct sock *sk,
924 unsigned int cmd, unsigned long arg);
925#endif
926 int (*sendmsg)(struct kiocb *iocb, struct sock *sk,
927 struct msghdr *msg, size_t len);
928 int (*recvmsg)(struct kiocb *iocb, struct sock *sk,
929 struct msghdr *msg,
930 size_t len, int noblock, int flags,
931 int *addr_len);
932 int (*sendpage)(struct sock *sk, struct page *page,
933 int offset, size_t size, int flags);
934 int (*bind)(struct sock *sk,
935 struct sockaddr *uaddr, int addr_len);
936
937 int (*backlog_rcv) (struct sock *sk,
938 struct sk_buff *skb);
939
940 void (*release_cb)(struct sock *sk);
941 void (*mtu_reduced)(struct sock *sk);
942
943
944 void (*hash)(struct sock *sk);
945 void (*unhash)(struct sock *sk);
946 void (*rehash)(struct sock *sk);
947 int (*get_port)(struct sock *sk, unsigned short snum);
948 void (*clear_sk)(struct sock *sk, int size);
949
950
951#ifdef CONFIG_PROC_FS
952 unsigned int inuse_idx;
953#endif
954
955 bool (*stream_memory_free)(const struct sock *sk);
956
957 void (*enter_memory_pressure)(struct sock *sk);
958 atomic_long_t *memory_allocated;
959 struct percpu_counter *sockets_allocated;
960
961
962
963
964
965
966 int *memory_pressure;
967 long *sysctl_mem;
968 int *sysctl_wmem;
969 int *sysctl_rmem;
970 int max_header;
971 bool no_autobind;
972
973 struct kmem_cache *slab;
974 unsigned int obj_size;
975 int slab_flags;
976
977 struct percpu_counter *orphan_count;
978
979 struct request_sock_ops *rsk_prot;
980 struct timewait_sock_ops *twsk_prot;
981
982 union {
983 struct inet_hashinfo *hashinfo;
984 struct udp_table *udp_table;
985 struct raw_hashinfo *raw_hash;
986 } h;
987
988 struct module *owner;
989
990 char name[32];
991
992 struct list_head node;
993#ifdef SOCK_REFCNT_DEBUG
994 atomic_t socks;
995#endif
996#ifdef CONFIG_MEMCG_KMEM
997
998
999
1000
1001
1002
1003 int (*init_cgroup)(struct mem_cgroup *memcg,
1004 struct cgroup_subsys *ss);
1005 void (*destroy_cgroup)(struct mem_cgroup *memcg);
1006 struct cg_proto *(*proto_cgroup)(struct mem_cgroup *memcg);
1007#endif
1008};
1009
1010
1011
1012
1013enum cg_proto_flags {
1014
1015 MEMCG_SOCK_ACTIVE,
1016
1017 MEMCG_SOCK_ACTIVATED,
1018};
1019
1020struct cg_proto {
1021 void (*enter_memory_pressure)(struct sock *sk);
1022 struct res_counter *memory_allocated;
1023 struct percpu_counter *sockets_allocated;
1024 int *memory_pressure;
1025 long *sysctl_mem;
1026 unsigned long flags;
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036 struct mem_cgroup *memcg;
1037};
1038
1039extern int proto_register(struct proto *prot, int alloc_slab);
1040extern void proto_unregister(struct proto *prot);
1041
1042static inline bool memcg_proto_active(struct cg_proto *cg_proto)
1043{
1044 return test_bit(MEMCG_SOCK_ACTIVE, &cg_proto->flags);
1045}
1046
1047static inline bool memcg_proto_activated(struct cg_proto *cg_proto)
1048{
1049 return test_bit(MEMCG_SOCK_ACTIVATED, &cg_proto->flags);
1050}
1051
1052#ifdef SOCK_REFCNT_DEBUG
1053static inline void sk_refcnt_debug_inc(struct sock *sk)
1054{
1055 atomic_inc(&sk->sk_prot->socks);
1056}
1057
1058static inline void sk_refcnt_debug_dec(struct sock *sk)
1059{
1060 atomic_dec(&sk->sk_prot->socks);
1061 printk(KERN_DEBUG "%s socket %p released, %d are still alive\n",
1062 sk->sk_prot->name, sk, atomic_read(&sk->sk_prot->socks));
1063}
1064
1065static inline void sk_refcnt_debug_release(const struct sock *sk)
1066{
1067 if (atomic_read(&sk->sk_refcnt) != 1)
1068 printk(KERN_DEBUG "Destruction of the %s socket %p delayed, refcnt=%d\n",
1069 sk->sk_prot->name, sk, atomic_read(&sk->sk_refcnt));
1070}
1071#else
1072#define sk_refcnt_debug_inc(sk) do { } while (0)
1073#define sk_refcnt_debug_dec(sk) do { } while (0)
1074#define sk_refcnt_debug_release(sk) do { } while (0)
1075#endif
1076
1077#if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_NET)
1078extern struct static_key memcg_socket_limit_enabled;
1079static inline struct cg_proto *parent_cg_proto(struct proto *proto,
1080 struct cg_proto *cg_proto)
1081{
1082 return proto->proto_cgroup(parent_mem_cgroup(cg_proto->memcg));
1083}
1084#define mem_cgroup_sockets_enabled static_key_false(&memcg_socket_limit_enabled)
1085#else
1086#define mem_cgroup_sockets_enabled 0
1087static inline struct cg_proto *parent_cg_proto(struct proto *proto,
1088 struct cg_proto *cg_proto)
1089{
1090 return NULL;
1091}
1092#endif
1093
1094static inline bool sk_stream_memory_free(const struct sock *sk)
1095{
1096 if (sk->sk_wmem_queued >= sk->sk_sndbuf)
1097 return false;
1098
1099 return sk->sk_prot->stream_memory_free ?
1100 sk->sk_prot->stream_memory_free(sk) : true;
1101}
1102
1103static inline bool sk_stream_is_writeable(const struct sock *sk)
1104{
1105 return sk_stream_wspace(sk) >= sk_stream_min_wspace(sk) &&
1106 sk_stream_memory_free(sk);
1107}
1108
1109
1110static inline bool sk_has_memory_pressure(const struct sock *sk)
1111{
1112 return sk->sk_prot->memory_pressure != NULL;
1113}
1114
1115static inline bool sk_under_memory_pressure(const struct sock *sk)
1116{
1117 if (!sk->sk_prot->memory_pressure)
1118 return false;
1119
1120 if (mem_cgroup_sockets_enabled && sk->sk_cgrp)
1121 return !!*sk->sk_cgrp->memory_pressure;
1122
1123 return !!*sk->sk_prot->memory_pressure;
1124}
1125
1126static inline void sk_leave_memory_pressure(struct sock *sk)
1127{
1128 int *memory_pressure = sk->sk_prot->memory_pressure;
1129
1130 if (!memory_pressure)
1131 return;
1132
1133 if (*memory_pressure)
1134 *memory_pressure = 0;
1135
1136 if (mem_cgroup_sockets_enabled && sk->sk_cgrp) {
1137 struct cg_proto *cg_proto = sk->sk_cgrp;
1138 struct proto *prot = sk->sk_prot;
1139
1140 for (; cg_proto; cg_proto = parent_cg_proto(prot, cg_proto))
1141 if (*cg_proto->memory_pressure)
1142 *cg_proto->memory_pressure = 0;
1143 }
1144
1145}
1146
1147static inline void sk_enter_memory_pressure(struct sock *sk)
1148{
1149 if (!sk->sk_prot->enter_memory_pressure)
1150 return;
1151
1152 if (mem_cgroup_sockets_enabled && sk->sk_cgrp) {
1153 struct cg_proto *cg_proto = sk->sk_cgrp;
1154 struct proto *prot = sk->sk_prot;
1155
1156 for (; cg_proto; cg_proto = parent_cg_proto(prot, cg_proto))
1157 cg_proto->enter_memory_pressure(sk);
1158 }
1159
1160 sk->sk_prot->enter_memory_pressure(sk);
1161}
1162
1163static inline long sk_prot_mem_limits(const struct sock *sk, int index)
1164{
1165 long *prot = sk->sk_prot->sysctl_mem;
1166 if (mem_cgroup_sockets_enabled && sk->sk_cgrp)
1167 prot = sk->sk_cgrp->sysctl_mem;
1168 return prot[index];
1169}
1170
1171static inline void memcg_memory_allocated_add(struct cg_proto *prot,
1172 unsigned long amt,
1173 int *parent_status)
1174{
1175 struct res_counter *fail;
1176 int ret;
1177
1178 ret = res_counter_charge_nofail(prot->memory_allocated,
1179 amt << PAGE_SHIFT, &fail);
1180 if (ret < 0)
1181 *parent_status = OVER_LIMIT;
1182}
1183
1184static inline void memcg_memory_allocated_sub(struct cg_proto *prot,
1185 unsigned long amt)
1186{
1187 res_counter_uncharge(prot->memory_allocated, amt << PAGE_SHIFT);
1188}
1189
1190static inline u64 memcg_memory_allocated_read(struct cg_proto *prot)
1191{
1192 u64 ret;
1193 ret = res_counter_read_u64(prot->memory_allocated, RES_USAGE);
1194 return ret >> PAGE_SHIFT;
1195}
1196
1197static inline long
1198sk_memory_allocated(const struct sock *sk)
1199{
1200 struct proto *prot = sk->sk_prot;
1201 if (mem_cgroup_sockets_enabled && sk->sk_cgrp)
1202 return memcg_memory_allocated_read(sk->sk_cgrp);
1203
1204 return atomic_long_read(prot->memory_allocated);
1205}
1206
1207static inline long
1208sk_memory_allocated_add(struct sock *sk, int amt, int *parent_status)
1209{
1210 struct proto *prot = sk->sk_prot;
1211
1212 if (mem_cgroup_sockets_enabled && sk->sk_cgrp) {
1213 memcg_memory_allocated_add(sk->sk_cgrp, amt, parent_status);
1214
1215 atomic_long_add_return(amt, prot->memory_allocated);
1216 return memcg_memory_allocated_read(sk->sk_cgrp);
1217 }
1218
1219 return atomic_long_add_return(amt, prot->memory_allocated);
1220}
1221
1222static inline void
1223sk_memory_allocated_sub(struct sock *sk, int amt)
1224{
1225 struct proto *prot = sk->sk_prot;
1226
1227 if (mem_cgroup_sockets_enabled && sk->sk_cgrp)
1228 memcg_memory_allocated_sub(sk->sk_cgrp, amt);
1229
1230 atomic_long_sub(amt, prot->memory_allocated);
1231}
1232
1233static inline void sk_sockets_allocated_dec(struct sock *sk)
1234{
1235 struct proto *prot = sk->sk_prot;
1236
1237 if (mem_cgroup_sockets_enabled && sk->sk_cgrp) {
1238 struct cg_proto *cg_proto = sk->sk_cgrp;
1239
1240 for (; cg_proto; cg_proto = parent_cg_proto(prot, cg_proto))
1241 percpu_counter_dec(cg_proto->sockets_allocated);
1242 }
1243
1244 percpu_counter_dec(prot->sockets_allocated);
1245}
1246
1247static inline void sk_sockets_allocated_inc(struct sock *sk)
1248{
1249 struct proto *prot = sk->sk_prot;
1250
1251 if (mem_cgroup_sockets_enabled && sk->sk_cgrp) {
1252 struct cg_proto *cg_proto = sk->sk_cgrp;
1253
1254 for (; cg_proto; cg_proto = parent_cg_proto(prot, cg_proto))
1255 percpu_counter_inc(cg_proto->sockets_allocated);
1256 }
1257
1258 percpu_counter_inc(prot->sockets_allocated);
1259}
1260
1261static inline int
1262sk_sockets_allocated_read_positive(struct sock *sk)
1263{
1264 struct proto *prot = sk->sk_prot;
1265
1266 if (mem_cgroup_sockets_enabled && sk->sk_cgrp)
1267 return percpu_counter_read_positive(sk->sk_cgrp->sockets_allocated);
1268
1269 return percpu_counter_read_positive(prot->sockets_allocated);
1270}
1271
1272static inline int
1273proto_sockets_allocated_sum_positive(struct proto *prot)
1274{
1275 return percpu_counter_sum_positive(prot->sockets_allocated);
1276}
1277
1278static inline long
1279proto_memory_allocated(struct proto *prot)
1280{
1281 return atomic_long_read(prot->memory_allocated);
1282}
1283
1284static inline bool
1285proto_memory_pressure(struct proto *prot)
1286{
1287 if (!prot->memory_pressure)
1288 return false;
1289 return !!*prot->memory_pressure;
1290}
1291
1292
1293#ifdef CONFIG_PROC_FS
1294
1295extern void sock_prot_inuse_add(struct net *net, struct proto *prot, int inc);
1296extern int sock_prot_inuse_get(struct net *net, struct proto *proto);
1297#else
1298static inline void sock_prot_inuse_add(struct net *net, struct proto *prot,
1299 int inc)
1300{
1301}
1302#endif
1303
1304
1305
1306
1307
1308static inline void __sk_prot_rehash(struct sock *sk)
1309{
1310 sk->sk_prot->unhash(sk);
1311 sk->sk_prot->hash(sk);
1312}
1313
1314void sk_prot_clear_portaddr_nulls(struct sock *sk, int size);
1315
1316
1317#define SOCK_DESTROY_TIME (10*HZ)
1318
1319
1320#define PROT_SOCK 1024
1321
1322#define SHUTDOWN_MASK 3
1323#define RCV_SHUTDOWN 1
1324#define SEND_SHUTDOWN 2
1325
1326#define SOCK_SNDBUF_LOCK 1
1327#define SOCK_RCVBUF_LOCK 2
1328#define SOCK_BINDADDR_LOCK 4
1329#define SOCK_BINDPORT_LOCK 8
1330
1331
1332struct sock_iocb {
1333 struct list_head list;
1334
1335 int flags;
1336 int size;
1337 struct socket *sock;
1338 struct sock *sk;
1339 struct scm_cookie *scm;
1340 struct msghdr *msg, async_msg;
1341 struct kiocb *kiocb;
1342};
1343
1344static inline struct sock_iocb *kiocb_to_siocb(struct kiocb *iocb)
1345{
1346 return (struct sock_iocb *)iocb->private;
1347}
1348
1349static inline struct kiocb *siocb_to_kiocb(struct sock_iocb *si)
1350{
1351 return si->kiocb;
1352}
1353
1354struct socket_alloc {
1355 struct socket socket;
1356 struct inode vfs_inode;
1357};
1358
1359static inline struct socket *SOCKET_I(struct inode *inode)
1360{
1361 return &container_of(inode, struct socket_alloc, vfs_inode)->socket;
1362}
1363
1364static inline struct inode *SOCK_INODE(struct socket *socket)
1365{
1366 return &container_of(socket, struct socket_alloc, socket)->vfs_inode;
1367}
1368
1369
1370
1371
1372extern int __sk_mem_schedule(struct sock *sk, int size, int kind);
1373extern void __sk_mem_reclaim(struct sock *sk);
1374
1375#define SK_MEM_QUANTUM ((int)PAGE_SIZE)
1376#define SK_MEM_QUANTUM_SHIFT ilog2(SK_MEM_QUANTUM)
1377#define SK_MEM_SEND 0
1378#define SK_MEM_RECV 1
1379
1380static inline int sk_mem_pages(int amt)
1381{
1382 return (amt + SK_MEM_QUANTUM - 1) >> SK_MEM_QUANTUM_SHIFT;
1383}
1384
1385static inline bool sk_has_account(struct sock *sk)
1386{
1387
1388 return !!sk->sk_prot->memory_allocated;
1389}
1390
1391static inline bool sk_wmem_schedule(struct sock *sk, int size)
1392{
1393 if (!sk_has_account(sk))
1394 return true;
1395 return size <= sk->sk_forward_alloc ||
1396 __sk_mem_schedule(sk, size, SK_MEM_SEND);
1397}
1398
1399static inline bool
1400sk_rmem_schedule(struct sock *sk, struct sk_buff *skb, int size)
1401{
1402 if (!sk_has_account(sk))
1403 return true;
1404 return size<= sk->sk_forward_alloc ||
1405 __sk_mem_schedule(sk, size, SK_MEM_RECV) ||
1406 skb_pfmemalloc(skb);
1407}
1408
1409static inline void sk_mem_reclaim(struct sock *sk)
1410{
1411 if (!sk_has_account(sk))
1412 return;
1413 if (sk->sk_forward_alloc >= SK_MEM_QUANTUM)
1414 __sk_mem_reclaim(sk);
1415}
1416
1417static inline void sk_mem_reclaim_partial(struct sock *sk)
1418{
1419 if (!sk_has_account(sk))
1420 return;
1421 if (sk->sk_forward_alloc > SK_MEM_QUANTUM)
1422 __sk_mem_reclaim(sk);
1423}
1424
1425static inline void sk_mem_charge(struct sock *sk, int size)
1426{
1427 if (!sk_has_account(sk))
1428 return;
1429 sk->sk_forward_alloc -= size;
1430}
1431
1432static inline void sk_mem_uncharge(struct sock *sk, int size)
1433{
1434 if (!sk_has_account(sk))
1435 return;
1436 sk->sk_forward_alloc += size;
1437}
1438
1439static inline void sk_wmem_free_skb(struct sock *sk, struct sk_buff *skb)
1440{
1441 sock_set_flag(sk, SOCK_QUEUE_SHRUNK);
1442 sk->sk_wmem_queued -= skb->truesize;
1443 sk_mem_uncharge(sk, skb->truesize);
1444 __kfree_skb(skb);
1445}
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460#define sock_owned_by_user(sk) ((sk)->sk_lock.owned)
1461
1462
1463
1464
1465
1466
1467
1468
1469#define sock_lock_init_class_and_name(sk, sname, skey, name, key) \
1470do { \
1471 sk->sk_lock.owned = 0; \
1472 init_waitqueue_head(&sk->sk_lock.wq); \
1473 spin_lock_init(&(sk)->sk_lock.slock); \
1474 debug_check_no_locks_freed((void *)&(sk)->sk_lock, \
1475 sizeof((sk)->sk_lock)); \
1476 lockdep_set_class_and_name(&(sk)->sk_lock.slock, \
1477 (skey), (sname)); \
1478 lockdep_init_map(&(sk)->sk_lock.dep_map, (name), (key), 0); \
1479} while (0)
1480
1481extern void lock_sock_nested(struct sock *sk, int subclass);
1482
1483static inline void lock_sock(struct sock *sk)
1484{
1485 lock_sock_nested(sk, 0);
1486}
1487
1488extern void release_sock(struct sock *sk);
1489
1490
1491#define bh_lock_sock(__sk) spin_lock(&((__sk)->sk_lock.slock))
1492#define bh_lock_sock_nested(__sk) \
1493 spin_lock_nested(&((__sk)->sk_lock.slock), \
1494 SINGLE_DEPTH_NESTING)
1495#define bh_unlock_sock(__sk) spin_unlock(&((__sk)->sk_lock.slock))
1496
1497extern bool lock_sock_fast(struct sock *sk);
1498
1499
1500
1501
1502
1503
1504
1505
1506static inline void unlock_sock_fast(struct sock *sk, bool slow)
1507{
1508 if (slow)
1509 release_sock(sk);
1510 else
1511 spin_unlock_bh(&sk->sk_lock.slock);
1512}
1513
1514
1515extern struct sock *sk_alloc(struct net *net, int family,
1516 gfp_t priority,
1517 struct proto *prot);
1518extern void sk_free(struct sock *sk);
1519extern void sk_release_kernel(struct sock *sk);
1520extern struct sock *sk_clone_lock(const struct sock *sk,
1521 const gfp_t priority);
1522
1523extern struct sk_buff *sock_wmalloc(struct sock *sk,
1524 unsigned long size, int force,
1525 gfp_t priority);
1526extern struct sk_buff *sock_rmalloc(struct sock *sk,
1527 unsigned long size, int force,
1528 gfp_t priority);
1529extern void sock_wfree(struct sk_buff *skb);
1530extern void skb_orphan_partial(struct sk_buff *skb);
1531extern void sock_rfree(struct sk_buff *skb);
1532extern void sock_edemux(struct sk_buff *skb);
1533
1534extern int sock_setsockopt(struct socket *sock, int level,
1535 int op, char __user *optval,
1536 unsigned int optlen);
1537
1538extern int sock_getsockopt(struct socket *sock, int level,
1539 int op, char __user *optval,
1540 int __user *optlen);
1541extern struct sk_buff *sock_alloc_send_skb(struct sock *sk,
1542 unsigned long size,
1543 int noblock,
1544 int *errcode);
1545extern struct sk_buff *sock_alloc_send_pskb(struct sock *sk,
1546 unsigned long header_len,
1547 unsigned long data_len,
1548 int noblock,
1549 int *errcode,
1550 int max_page_order);
1551extern void *sock_kmalloc(struct sock *sk, int size,
1552 gfp_t priority);
1553extern void sock_kfree_s(struct sock *sk, void *mem, int size);
1554extern void sk_send_sigurg(struct sock *sk);
1555
1556
1557
1558
1559
1560extern int sock_no_bind(struct socket *,
1561 struct sockaddr *, int);
1562extern int sock_no_connect(struct socket *,
1563 struct sockaddr *, int, int);
1564extern int sock_no_socketpair(struct socket *,
1565 struct socket *);
1566extern int sock_no_accept(struct socket *,
1567 struct socket *, int);
1568extern int sock_no_getname(struct socket *,
1569 struct sockaddr *, int *, int);
1570extern unsigned int sock_no_poll(struct file *, struct socket *,
1571 struct poll_table_struct *);
1572extern int sock_no_ioctl(struct socket *, unsigned int,
1573 unsigned long);
1574extern int sock_no_listen(struct socket *, int);
1575extern int sock_no_shutdown(struct socket *, int);
1576extern int sock_no_getsockopt(struct socket *, int , int,
1577 char __user *, int __user *);
1578extern int sock_no_setsockopt(struct socket *, int, int,
1579 char __user *, unsigned int);
1580extern int sock_no_sendmsg(struct kiocb *, struct socket *,
1581 struct msghdr *, size_t);
1582extern int sock_no_recvmsg(struct kiocb *, struct socket *,
1583 struct msghdr *, size_t, int);
1584extern int sock_no_mmap(struct file *file,
1585 struct socket *sock,
1586 struct vm_area_struct *vma);
1587extern ssize_t sock_no_sendpage(struct socket *sock,
1588 struct page *page,
1589 int offset, size_t size,
1590 int flags);
1591
1592
1593
1594
1595
1596extern int sock_common_getsockopt(struct socket *sock, int level, int optname,
1597 char __user *optval, int __user *optlen);
1598extern int sock_common_recvmsg(struct kiocb *iocb, struct socket *sock,
1599 struct msghdr *msg, size_t size, int flags);
1600extern int sock_common_setsockopt(struct socket *sock, int level, int optname,
1601 char __user *optval, unsigned int optlen);
1602extern int compat_sock_common_getsockopt(struct socket *sock, int level,
1603 int optname, char __user *optval, int __user *optlen);
1604extern int compat_sock_common_setsockopt(struct socket *sock, int level,
1605 int optname, char __user *optval, unsigned int optlen);
1606
1607extern void sk_common_release(struct sock *sk);
1608
1609
1610
1611
1612
1613
1614extern void sock_init_data(struct socket *sock, struct sock *sk);
1615
1616extern void sk_filter_release_rcu(struct rcu_head *rcu);
1617
1618
1619
1620
1621
1622
1623
1624
1625static inline void sk_filter_release(struct sk_filter *fp)
1626{
1627 if (atomic_dec_and_test(&fp->refcnt))
1628 call_rcu(&fp->rcu, sk_filter_release_rcu);
1629}
1630
1631static inline void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp)
1632{
1633 atomic_sub(sk_filter_size(fp->len), &sk->sk_omem_alloc);
1634 sk_filter_release(fp);
1635}
1636
1637static inline void sk_filter_charge(struct sock *sk, struct sk_filter *fp)
1638{
1639 atomic_inc(&fp->refcnt);
1640 atomic_add(sk_filter_size(fp->len), &sk->sk_omem_alloc);
1641}
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669static inline void sock_put(struct sock *sk)
1670{
1671 if (atomic_dec_and_test(&sk->sk_refcnt))
1672 sk_free(sk);
1673}
1674
1675extern int sk_receive_skb(struct sock *sk, struct sk_buff *skb,
1676 const int nested);
1677
1678static inline void sk_tx_queue_set(struct sock *sk, int tx_queue)
1679{
1680 sk->sk_tx_queue_mapping = tx_queue;
1681}
1682
1683static inline void sk_tx_queue_clear(struct sock *sk)
1684{
1685 sk->sk_tx_queue_mapping = -1;
1686}
1687
1688static inline int sk_tx_queue_get(const struct sock *sk)
1689{
1690 return sk ? sk->sk_tx_queue_mapping : -1;
1691}
1692
1693static inline void sk_set_socket(struct sock *sk, struct socket *sock)
1694{
1695 sk_tx_queue_clear(sk);
1696 sk->sk_socket = sock;
1697}
1698
1699static inline wait_queue_head_t *sk_sleep(struct sock *sk)
1700{
1701 BUILD_BUG_ON(offsetof(struct socket_wq, wait) != 0);
1702 return &rcu_dereference_raw(sk->sk_wq)->wait;
1703}
1704
1705
1706
1707
1708
1709
1710
1711static inline void sock_orphan(struct sock *sk)
1712{
1713 write_lock_bh(&sk->sk_callback_lock);
1714 sock_set_flag(sk, SOCK_DEAD);
1715 sk_set_socket(sk, NULL);
1716 sk->sk_wq = NULL;
1717 write_unlock_bh(&sk->sk_callback_lock);
1718}
1719
1720static inline void sock_graft(struct sock *sk, struct socket *parent)
1721{
1722 write_lock_bh(&sk->sk_callback_lock);
1723 sk->sk_wq = parent->wq;
1724 parent->sk = sk;
1725 sk_set_socket(sk, parent);
1726 security_sock_graft(sk, parent);
1727 write_unlock_bh(&sk->sk_callback_lock);
1728}
1729
1730extern kuid_t sock_i_uid(struct sock *sk);
1731extern unsigned long sock_i_ino(struct sock *sk);
1732
1733static inline struct dst_entry *
1734__sk_dst_get(struct sock *sk)
1735{
1736 return rcu_dereference_check(sk->sk_dst_cache, sock_owned_by_user(sk) ||
1737 lockdep_is_held(&sk->sk_lock.slock));
1738}
1739
1740static inline struct dst_entry *
1741sk_dst_get(struct sock *sk)
1742{
1743 struct dst_entry *dst;
1744
1745 rcu_read_lock();
1746 dst = rcu_dereference(sk->sk_dst_cache);
1747 if (dst)
1748 dst_hold(dst);
1749 rcu_read_unlock();
1750 return dst;
1751}
1752
1753extern void sk_reset_txq(struct sock *sk);
1754
1755static inline void dst_negative_advice(struct sock *sk)
1756{
1757 struct dst_entry *ndst, *dst = __sk_dst_get(sk);
1758
1759 if (dst && dst->ops->negative_advice) {
1760 ndst = dst->ops->negative_advice(dst);
1761
1762 if (ndst != dst) {
1763 rcu_assign_pointer(sk->sk_dst_cache, ndst);
1764 sk_reset_txq(sk);
1765 }
1766 }
1767}
1768
1769static inline void
1770__sk_dst_set(struct sock *sk, struct dst_entry *dst)
1771{
1772 struct dst_entry *old_dst;
1773
1774 sk_tx_queue_clear(sk);
1775
1776
1777
1778
1779 old_dst = rcu_dereference_raw(sk->sk_dst_cache);
1780 rcu_assign_pointer(sk->sk_dst_cache, dst);
1781 dst_release(old_dst);
1782}
1783
1784static inline void
1785sk_dst_set(struct sock *sk, struct dst_entry *dst)
1786{
1787 spin_lock(&sk->sk_dst_lock);
1788 __sk_dst_set(sk, dst);
1789 spin_unlock(&sk->sk_dst_lock);
1790}
1791
1792static inline void
1793__sk_dst_reset(struct sock *sk)
1794{
1795 __sk_dst_set(sk, NULL);
1796}
1797
1798static inline void
1799sk_dst_reset(struct sock *sk)
1800{
1801 spin_lock(&sk->sk_dst_lock);
1802 __sk_dst_reset(sk);
1803 spin_unlock(&sk->sk_dst_lock);
1804}
1805
1806extern struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie);
1807
1808extern struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie);
1809
1810static inline bool sk_can_gso(const struct sock *sk)
1811{
1812 return net_gso_ok(sk->sk_route_caps, sk->sk_gso_type);
1813}
1814
1815extern void sk_setup_caps(struct sock *sk, struct dst_entry *dst);
1816
1817static inline void sk_nocaps_add(struct sock *sk, netdev_features_t flags)
1818{
1819 sk->sk_route_nocaps |= flags;
1820 sk->sk_route_caps &= ~flags;
1821}
1822
1823static inline int skb_do_copy_data_nocache(struct sock *sk, struct sk_buff *skb,
1824 char __user *from, char *to,
1825 int copy, int offset)
1826{
1827 if (skb->ip_summed == CHECKSUM_NONE) {
1828 int err = 0;
1829 __wsum csum = csum_and_copy_from_user(from, to, copy, 0, &err);
1830 if (err)
1831 return err;
1832 skb->csum = csum_block_add(skb->csum, csum, offset);
1833 } else if (sk->sk_route_caps & NETIF_F_NOCACHE_COPY) {
1834 if (!access_ok(VERIFY_READ, from, copy) ||
1835 __copy_from_user_nocache(to, from, copy))
1836 return -EFAULT;
1837 } else if (copy_from_user(to, from, copy))
1838 return -EFAULT;
1839
1840 return 0;
1841}
1842
1843static inline int skb_add_data_nocache(struct sock *sk, struct sk_buff *skb,
1844 char __user *from, int copy)
1845{
1846 int err, offset = skb->len;
1847
1848 err = skb_do_copy_data_nocache(sk, skb, from, skb_put(skb, copy),
1849 copy, offset);
1850 if (err)
1851 __skb_trim(skb, offset);
1852
1853 return err;
1854}
1855
1856static inline int skb_copy_to_page_nocache(struct sock *sk, char __user *from,
1857 struct sk_buff *skb,
1858 struct page *page,
1859 int off, int copy)
1860{
1861 int err;
1862
1863 err = skb_do_copy_data_nocache(sk, skb, from, page_address(page) + off,
1864 copy, skb->len);
1865 if (err)
1866 return err;
1867
1868 skb->len += copy;
1869 skb->data_len += copy;
1870 skb->truesize += copy;
1871 sk->sk_wmem_queued += copy;
1872 sk_mem_charge(sk, copy);
1873 return 0;
1874}
1875
1876static inline int skb_copy_to_page(struct sock *sk, char __user *from,
1877 struct sk_buff *skb, struct page *page,
1878 int off, int copy)
1879{
1880 if (skb->ip_summed == CHECKSUM_NONE) {
1881 int err = 0;
1882 __wsum csum = csum_and_copy_from_user(from,
1883 page_address(page) + off,
1884 copy, 0, &err);
1885 if (err)
1886 return err;
1887 skb->csum = csum_block_add(skb->csum, csum, skb->len);
1888 } else if (copy_from_user(page_address(page) + off, from, copy))
1889 return -EFAULT;
1890
1891 skb->len += copy;
1892 skb->data_len += copy;
1893 skb->truesize += copy;
1894 sk->sk_wmem_queued += copy;
1895 sk_mem_charge(sk, copy);
1896 return 0;
1897}
1898
1899
1900
1901
1902
1903
1904
1905static inline int sk_wmem_alloc_get(const struct sock *sk)
1906{
1907 return atomic_read(&sk->sk_wmem_alloc) - 1;
1908}
1909
1910
1911
1912
1913
1914
1915
1916static inline int sk_rmem_alloc_get(const struct sock *sk)
1917{
1918 return atomic_read(&sk->sk_rmem_alloc);
1919}
1920
1921
1922
1923
1924
1925
1926
1927static inline bool sk_has_allocations(const struct sock *sk)
1928{
1929 return sk_wmem_alloc_get(sk) || sk_rmem_alloc_get(sk);
1930}
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964static inline bool wq_has_sleeper(struct socket_wq *wq)
1965{
1966
1967
1968
1969
1970
1971 smp_mb();
1972 return wq && waitqueue_active(&wq->wait);
1973}
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983static inline void sock_poll_wait(struct file *filp,
1984 wait_queue_head_t *wait_address, poll_table *p)
1985{
1986 if (!poll_does_not_wait(p) && wait_address) {
1987 poll_wait(filp, wait_address, p);
1988
1989
1990
1991
1992
1993 smp_mb();
1994 }
1995}
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006static inline void skb_set_owner_w(struct sk_buff *skb, struct sock *sk)
2007{
2008 skb_orphan(skb);
2009 skb->sk = sk;
2010 skb->destructor = sock_wfree;
2011
2012
2013
2014
2015
2016 atomic_add(skb->truesize, &sk->sk_wmem_alloc);
2017}
2018
2019static inline void skb_set_owner_r(struct sk_buff *skb, struct sock *sk)
2020{
2021 skb_orphan(skb);
2022 skb->sk = sk;
2023 skb->destructor = sock_rfree;
2024 atomic_add(skb->truesize, &sk->sk_rmem_alloc);
2025 sk_mem_charge(sk, skb->truesize);
2026}
2027
2028extern void sk_reset_timer(struct sock *sk, struct timer_list *timer,
2029 unsigned long expires);
2030
2031extern void sk_stop_timer(struct sock *sk, struct timer_list *timer);
2032
2033extern int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb);
2034
2035extern int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb);
2036
2037
2038
2039
2040
2041static inline int sock_error(struct sock *sk)
2042{
2043 int err;
2044 if (likely(!sk->sk_err))
2045 return 0;
2046 err = xchg(&sk->sk_err, 0);
2047 return -err;
2048}
2049
2050static inline unsigned long sock_wspace(struct sock *sk)
2051{
2052 int amt = 0;
2053
2054 if (!(sk->sk_shutdown & SEND_SHUTDOWN)) {
2055 amt = sk->sk_sndbuf - atomic_read(&sk->sk_wmem_alloc);
2056 if (amt < 0)
2057 amt = 0;
2058 }
2059 return amt;
2060}
2061
2062static inline void sk_wake_async(struct sock *sk, int how, int band)
2063{
2064 if (sock_flag(sk, SOCK_FASYNC))
2065 sock_wake_async(sk->sk_socket, how, band);
2066}
2067
2068
2069
2070
2071
2072
2073#define TCP_SKB_MIN_TRUESIZE (2048 + SKB_DATA_ALIGN(sizeof(struct sk_buff)))
2074
2075#define SOCK_MIN_SNDBUF (TCP_SKB_MIN_TRUESIZE * 2)
2076#define SOCK_MIN_RCVBUF TCP_SKB_MIN_TRUESIZE
2077
2078static inline void sk_stream_moderate_sndbuf(struct sock *sk)
2079{
2080 if (!(sk->sk_userlocks & SOCK_SNDBUF_LOCK)) {
2081 sk->sk_sndbuf = min(sk->sk_sndbuf, sk->sk_wmem_queued >> 1);
2082 sk->sk_sndbuf = max_t(u32, sk->sk_sndbuf, SOCK_MIN_SNDBUF);
2083 }
2084}
2085
2086struct sk_buff *sk_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp);
2087
2088
2089
2090
2091
2092
2093
2094
2095static inline struct page_frag *sk_page_frag(struct sock *sk)
2096{
2097 if (sk->sk_allocation & __GFP_WAIT)
2098 return ¤t->task_frag;
2099
2100 return &sk->sk_frag;
2101}
2102
2103extern bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag);
2104
2105
2106
2107
2108static inline bool sock_writeable(const struct sock *sk)
2109{
2110 return atomic_read(&sk->sk_wmem_alloc) < (sk->sk_sndbuf >> 1);
2111}
2112
2113static inline gfp_t gfp_any(void)
2114{
2115 return in_softirq() ? GFP_ATOMIC : GFP_KERNEL;
2116}
2117
2118static inline long sock_rcvtimeo(const struct sock *sk, bool noblock)
2119{
2120 return noblock ? 0 : sk->sk_rcvtimeo;
2121}
2122
2123static inline long sock_sndtimeo(const struct sock *sk, bool noblock)
2124{
2125 return noblock ? 0 : sk->sk_sndtimeo;
2126}
2127
2128static inline int sock_rcvlowat(const struct sock *sk, int waitall, int len)
2129{
2130 return (waitall ? len : min_t(int, sk->sk_rcvlowat, len)) ? : 1;
2131}
2132
2133
2134
2135
2136static inline int sock_intr_errno(long timeo)
2137{
2138 return timeo == MAX_SCHEDULE_TIMEOUT ? -ERESTARTSYS : -EINTR;
2139}
2140
2141extern void __sock_recv_timestamp(struct msghdr *msg, struct sock *sk,
2142 struct sk_buff *skb);
2143extern void __sock_recv_wifi_status(struct msghdr *msg, struct sock *sk,
2144 struct sk_buff *skb);
2145
2146static inline void
2147sock_recv_timestamp(struct msghdr *msg, struct sock *sk, struct sk_buff *skb)
2148{
2149 ktime_t kt = skb->tstamp;
2150 struct skb_shared_hwtstamps *hwtstamps = skb_hwtstamps(skb);
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162 if (sock_flag(sk, SOCK_RCVTSTAMP) ||
2163 sock_flag(sk, SOCK_TIMESTAMPING_RX_SOFTWARE) ||
2164 (kt.tv64 && sock_flag(sk, SOCK_TIMESTAMPING_SOFTWARE)) ||
2165 (hwtstamps->hwtstamp.tv64 &&
2166 sock_flag(sk, SOCK_TIMESTAMPING_RAW_HARDWARE)) ||
2167 (hwtstamps->syststamp.tv64 &&
2168 sock_flag(sk, SOCK_TIMESTAMPING_SYS_HARDWARE)))
2169 __sock_recv_timestamp(msg, sk, skb);
2170 else
2171 sk->sk_stamp = kt;
2172
2173 if (sock_flag(sk, SOCK_WIFI_STATUS) && skb->wifi_acked_valid)
2174 __sock_recv_wifi_status(msg, sk, skb);
2175}
2176
2177extern void __sock_recv_ts_and_drops(struct msghdr *msg, struct sock *sk,
2178 struct sk_buff *skb);
2179
2180static inline void sock_recv_ts_and_drops(struct msghdr *msg, struct sock *sk,
2181 struct sk_buff *skb)
2182{
2183#define FLAGS_TS_OR_DROPS ((1UL << SOCK_RXQ_OVFL) | \
2184 (1UL << SOCK_RCVTSTAMP) | \
2185 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \
2186 (1UL << SOCK_TIMESTAMPING_SOFTWARE) | \
2187 (1UL << SOCK_TIMESTAMPING_RAW_HARDWARE) | \
2188 (1UL << SOCK_TIMESTAMPING_SYS_HARDWARE))
2189
2190 if (sk->sk_flags & FLAGS_TS_OR_DROPS)
2191 __sock_recv_ts_and_drops(msg, sk, skb);
2192 else
2193 sk->sk_stamp = skb->tstamp;
2194}
2195
2196
2197
2198
2199
2200
2201
2202
2203extern void sock_tx_timestamp(struct sock *sk, __u8 *tx_flags);
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214#ifdef CONFIG_NET_DMA
2215static inline void sk_eat_skb(struct sock *sk, struct sk_buff *skb, bool copied_early)
2216{
2217 __skb_unlink(skb, &sk->sk_receive_queue);
2218 if (!copied_early)
2219 __kfree_skb(skb);
2220 else
2221 __skb_queue_tail(&sk->sk_async_wait_queue, skb);
2222}
2223#else
2224static inline void sk_eat_skb(struct sock *sk, struct sk_buff *skb, bool copied_early)
2225{
2226 __skb_unlink(skb, &sk->sk_receive_queue);
2227 __kfree_skb(skb);
2228}
2229#endif
2230
2231static inline
2232struct net *sock_net(const struct sock *sk)
2233{
2234 return read_pnet(&sk->sk_net);
2235}
2236
2237static inline
2238void sock_net_set(struct sock *sk, struct net *net)
2239{
2240 write_pnet(&sk->sk_net, net);
2241}
2242
2243
2244
2245
2246
2247
2248
2249static inline void sk_change_net(struct sock *sk, struct net *net)
2250{
2251 put_net(sock_net(sk));
2252 sock_net_set(sk, hold_net(net));
2253}
2254
2255static inline struct sock *skb_steal_sock(struct sk_buff *skb)
2256{
2257 if (skb->sk) {
2258 struct sock *sk = skb->sk;
2259
2260 skb->destructor = NULL;
2261 skb->sk = NULL;
2262 return sk;
2263 }
2264 return NULL;
2265}
2266
2267extern void sock_enable_timestamp(struct sock *sk, int flag);
2268extern int sock_get_timestamp(struct sock *, struct timeval __user *);
2269extern int sock_get_timestampns(struct sock *, struct timespec __user *);
2270extern int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len,
2271 int level, int type);
2272
2273
2274
2275
2276extern int net_msg_warn;
2277#define NETDEBUG(fmt, args...) \
2278 do { if (net_msg_warn) printk(fmt,##args); } while (0)
2279
2280#define LIMIT_NETDEBUG(fmt, args...) \
2281 do { if (net_msg_warn && net_ratelimit()) printk(fmt,##args); } while(0)
2282
2283extern __u32 sysctl_wmem_max;
2284extern __u32 sysctl_rmem_max;
2285
2286extern int sysctl_optmem_max;
2287
2288extern __u32 sysctl_wmem_default;
2289extern __u32 sysctl_rmem_default;
2290
2291#endif
2292