1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40#ifndef _SOCK_H
41#define _SOCK_H
42
43#include <linux/hardirq.h>
44#include <linux/kernel.h>
45#include <linux/list.h>
46#include <linux/list_nulls.h>
47#include <linux/timer.h>
48#include <linux/cache.h>
49#include <linux/bitops.h>
50#include <linux/lockdep.h>
51#include <linux/netdevice.h>
52#include <linux/skbuff.h>
53#include <linux/mm.h>
54#include <linux/security.h>
55#include <linux/slab.h>
56#include <linux/uaccess.h>
57#include <linux/memcontrol.h>
58#include <linux/res_counter.h>
59#include <linux/static_key.h>
60#include <linux/aio.h>
61#include <linux/sched.h>
62
63#include <linux/filter.h>
64#include <linux/rculist_nulls.h>
65#include <linux/poll.h>
66
67#include <linux/atomic.h>
68#include <net/dst.h>
69#include <net/checksum.h>
70
71struct cgroup;
72struct cgroup_subsys;
73#ifdef CONFIG_NET
74int mem_cgroup_sockets_init(struct mem_cgroup *memcg, struct cgroup_subsys *ss);
75void mem_cgroup_sockets_destroy(struct mem_cgroup *memcg);
76#else
77static inline
78int mem_cgroup_sockets_init(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
79{
80 return 0;
81}
82static inline
83void mem_cgroup_sockets_destroy(struct mem_cgroup *memcg)
84{
85}
86#endif
87
88
89
90
91
92
93
94#define SOCK_DEBUGGING
95#ifdef SOCK_DEBUGGING
96#define SOCK_DEBUG(sk, msg...) do { if ((sk) && sock_flag((sk), SOCK_DBG)) \
97 printk(KERN_DEBUG msg); } while (0)
98#else
99
100static inline __printf(2, 3)
101void SOCK_DEBUG(const struct sock *sk, const char *msg, ...)
102{
103}
104#endif
105
106
107
108
109
110typedef struct {
111 spinlock_t slock;
112 int owned;
113 wait_queue_head_t wq;
114
115
116
117
118
119
120#ifdef CONFIG_DEBUG_LOCK_ALLOC
121 struct lockdep_map dep_map;
122#endif
123} socket_lock_t;
124
125struct sock;
126struct proto;
127struct net;
128
129typedef __u32 __bitwise __portpair;
130typedef __u64 __bitwise __addrpair;
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157struct sock_common {
158
159
160
161 union {
162 __addrpair skc_addrpair;
163 struct {
164 __be32 skc_daddr;
165 __be32 skc_rcv_saddr;
166 };
167 };
168 union {
169 unsigned int skc_hash;
170 __u16 skc_u16hashes[2];
171 };
172
173 union {
174 __portpair skc_portpair;
175 struct {
176 __be16 skc_dport;
177 __u16 skc_num;
178 };
179 };
180
181 unsigned short skc_family;
182 volatile unsigned char skc_state;
183 unsigned char skc_reuse:4;
184 unsigned char skc_reuseport:4;
185 int skc_bound_dev_if;
186 union {
187 struct hlist_node skc_bind_node;
188 struct hlist_nulls_node skc_portaddr_node;
189 };
190 struct proto *skc_prot;
191#ifdef CONFIG_NET_NS
192 struct net *skc_net;
193#endif
194
195
196
197
198
199 int skc_dontcopy_begin[0];
200
201 union {
202 struct hlist_node skc_node;
203 struct hlist_nulls_node skc_nulls_node;
204 };
205 int skc_tx_queue_mapping;
206 atomic_t skc_refcnt;
207
208 int skc_dontcopy_end[0];
209
210};
211
212struct cg_proto;
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287struct sock {
288
289
290
291
292 struct sock_common __sk_common;
293#define sk_node __sk_common.skc_node
294#define sk_nulls_node __sk_common.skc_nulls_node
295#define sk_refcnt __sk_common.skc_refcnt
296#define sk_tx_queue_mapping __sk_common.skc_tx_queue_mapping
297
298#define sk_dontcopy_begin __sk_common.skc_dontcopy_begin
299#define sk_dontcopy_end __sk_common.skc_dontcopy_end
300#define sk_hash __sk_common.skc_hash
301#define sk_family __sk_common.skc_family
302#define sk_state __sk_common.skc_state
303#define sk_reuse __sk_common.skc_reuse
304#define sk_reuseport __sk_common.skc_reuseport
305#define sk_bound_dev_if __sk_common.skc_bound_dev_if
306#define sk_bind_node __sk_common.skc_bind_node
307#define sk_prot __sk_common.skc_prot
308#define sk_net __sk_common.skc_net
309 socket_lock_t sk_lock;
310 struct sk_buff_head sk_receive_queue;
311
312
313
314
315
316
317
318
319 struct {
320 atomic_t rmem_alloc;
321 int len;
322 struct sk_buff *head;
323 struct sk_buff *tail;
324 } sk_backlog;
325#define sk_rmem_alloc sk_backlog.rmem_alloc
326 int sk_forward_alloc;
327#ifdef CONFIG_RPS
328 __u32 sk_rxhash;
329#endif
330#ifdef CONFIG_NET_RX_BUSY_POLL
331 unsigned int sk_napi_id;
332 unsigned int sk_ll_usec;
333#endif
334 atomic_t sk_drops;
335 int sk_rcvbuf;
336
337 struct sk_filter __rcu *sk_filter;
338 struct socket_wq __rcu *sk_wq;
339
340#ifdef CONFIG_NET_DMA
341 struct sk_buff_head sk_async_wait_queue;
342#endif
343
344#ifdef CONFIG_XFRM
345 struct xfrm_policy *sk_policy[2];
346#endif
347 unsigned long sk_flags;
348 struct dst_entry *sk_rx_dst;
349 struct dst_entry __rcu *sk_dst_cache;
350 spinlock_t sk_dst_lock;
351 atomic_t sk_wmem_alloc;
352 atomic_t sk_omem_alloc;
353 int sk_sndbuf;
354 struct sk_buff_head sk_write_queue;
355 kmemcheck_bitfield_begin(flags);
356 unsigned int sk_shutdown : 2,
357 sk_no_check : 2,
358 sk_userlocks : 4,
359 sk_protocol : 8,
360 sk_type : 16;
361 kmemcheck_bitfield_end(flags);
362 int sk_wmem_queued;
363 gfp_t sk_allocation;
364 netdev_features_t sk_route_caps;
365 netdev_features_t sk_route_nocaps;
366 int sk_gso_type;
367 unsigned int sk_gso_max_size;
368 u16 sk_gso_max_segs;
369 int sk_rcvlowat;
370 unsigned long sk_lingertime;
371 struct sk_buff_head sk_error_queue;
372 struct proto *sk_prot_creator;
373 rwlock_t sk_callback_lock;
374 int sk_err,
375 sk_err_soft;
376 unsigned short sk_ack_backlog;
377 unsigned short sk_max_ack_backlog;
378 __u32 sk_priority;
379#if IS_ENABLED(CONFIG_NETPRIO_CGROUP)
380 __u32 sk_cgrp_prioidx;
381#endif
382 struct pid *sk_peer_pid;
383 const struct cred *sk_peer_cred;
384 long sk_rcvtimeo;
385 long sk_sndtimeo;
386 void *sk_protinfo;
387 struct timer_list sk_timer;
388 ktime_t sk_stamp;
389 struct socket *sk_socket;
390 void *sk_user_data;
391 struct page_frag sk_frag;
392 struct sk_buff *sk_send_head;
393 __s32 sk_peek_off;
394 int sk_write_pending;
395#ifdef CONFIG_SECURITY
396 void *sk_security;
397#endif
398 __u32 sk_mark;
399 u32 sk_classid;
400 struct cg_proto *sk_cgrp;
401 void (*sk_state_change)(struct sock *sk);
402 void (*sk_data_ready)(struct sock *sk, int bytes);
403 void (*sk_write_space)(struct sock *sk);
404 void (*sk_error_report)(struct sock *sk);
405 int (*sk_backlog_rcv)(struct sock *sk,
406 struct sk_buff *skb);
407 void (*sk_destruct)(struct sock *sk);
408};
409
410
411
412
413
414
415
416
417#define SK_NO_REUSE 0
418#define SK_CAN_REUSE 1
419#define SK_FORCE_REUSE 2
420
421static inline int sk_peek_offset(struct sock *sk, int flags)
422{
423 if ((flags & MSG_PEEK) && (sk->sk_peek_off >= 0))
424 return sk->sk_peek_off;
425 else
426 return 0;
427}
428
429static inline void sk_peek_offset_bwd(struct sock *sk, int val)
430{
431 if (sk->sk_peek_off >= 0) {
432 if (sk->sk_peek_off >= val)
433 sk->sk_peek_off -= val;
434 else
435 sk->sk_peek_off = 0;
436 }
437}
438
439static inline void sk_peek_offset_fwd(struct sock *sk, int val)
440{
441 if (sk->sk_peek_off >= 0)
442 sk->sk_peek_off += val;
443}
444
445
446
447
448static inline struct sock *sk_entry(const struct hlist_node *node)
449{
450 return hlist_entry(node, struct sock, sk_node);
451}
452
453static inline struct sock *__sk_head(const struct hlist_head *head)
454{
455 return hlist_entry(head->first, struct sock, sk_node);
456}
457
458static inline struct sock *sk_head(const struct hlist_head *head)
459{
460 return hlist_empty(head) ? NULL : __sk_head(head);
461}
462
463static inline struct sock *__sk_nulls_head(const struct hlist_nulls_head *head)
464{
465 return hlist_nulls_entry(head->first, struct sock, sk_nulls_node);
466}
467
468static inline struct sock *sk_nulls_head(const struct hlist_nulls_head *head)
469{
470 return hlist_nulls_empty(head) ? NULL : __sk_nulls_head(head);
471}
472
473static inline struct sock *sk_next(const struct sock *sk)
474{
475 return sk->sk_node.next ?
476 hlist_entry(sk->sk_node.next, struct sock, sk_node) : NULL;
477}
478
479static inline struct sock *sk_nulls_next(const struct sock *sk)
480{
481 return (!is_a_nulls(sk->sk_nulls_node.next)) ?
482 hlist_nulls_entry(sk->sk_nulls_node.next,
483 struct sock, sk_nulls_node) :
484 NULL;
485}
486
487static inline bool sk_unhashed(const struct sock *sk)
488{
489 return hlist_unhashed(&sk->sk_node);
490}
491
492static inline bool sk_hashed(const struct sock *sk)
493{
494 return !sk_unhashed(sk);
495}
496
497static inline void sk_node_init(struct hlist_node *node)
498{
499 node->pprev = NULL;
500}
501
502static inline void sk_nulls_node_init(struct hlist_nulls_node *node)
503{
504 node->pprev = NULL;
505}
506
507static inline void __sk_del_node(struct sock *sk)
508{
509 __hlist_del(&sk->sk_node);
510}
511
512
513static inline bool __sk_del_node_init(struct sock *sk)
514{
515 if (sk_hashed(sk)) {
516 __sk_del_node(sk);
517 sk_node_init(&sk->sk_node);
518 return true;
519 }
520 return false;
521}
522
523
524
525
526
527
528
529static inline void sock_hold(struct sock *sk)
530{
531 atomic_inc(&sk->sk_refcnt);
532}
533
534
535
536
537static inline void __sock_put(struct sock *sk)
538{
539 atomic_dec(&sk->sk_refcnt);
540}
541
542static inline bool sk_del_node_init(struct sock *sk)
543{
544 bool rc = __sk_del_node_init(sk);
545
546 if (rc) {
547
548 WARN_ON(atomic_read(&sk->sk_refcnt) == 1);
549 __sock_put(sk);
550 }
551 return rc;
552}
553#define sk_del_node_init_rcu(sk) sk_del_node_init(sk)
554
555static inline bool __sk_nulls_del_node_init_rcu(struct sock *sk)
556{
557 if (sk_hashed(sk)) {
558 hlist_nulls_del_init_rcu(&sk->sk_nulls_node);
559 return true;
560 }
561 return false;
562}
563
564static inline bool sk_nulls_del_node_init_rcu(struct sock *sk)
565{
566 bool rc = __sk_nulls_del_node_init_rcu(sk);
567
568 if (rc) {
569
570 WARN_ON(atomic_read(&sk->sk_refcnt) == 1);
571 __sock_put(sk);
572 }
573 return rc;
574}
575
576static inline void __sk_add_node(struct sock *sk, struct hlist_head *list)
577{
578 hlist_add_head(&sk->sk_node, list);
579}
580
581static inline void sk_add_node(struct sock *sk, struct hlist_head *list)
582{
583 sock_hold(sk);
584 __sk_add_node(sk, list);
585}
586
587static inline void sk_add_node_rcu(struct sock *sk, struct hlist_head *list)
588{
589 sock_hold(sk);
590 hlist_add_head_rcu(&sk->sk_node, list);
591}
592
593static inline void __sk_nulls_add_node_rcu(struct sock *sk, struct hlist_nulls_head *list)
594{
595 hlist_nulls_add_head_rcu(&sk->sk_nulls_node, list);
596}
597
598static inline void sk_nulls_add_node_rcu(struct sock *sk, struct hlist_nulls_head *list)
599{
600 sock_hold(sk);
601 __sk_nulls_add_node_rcu(sk, list);
602}
603
604static inline void __sk_del_bind_node(struct sock *sk)
605{
606 __hlist_del(&sk->sk_bind_node);
607}
608
609static inline void sk_add_bind_node(struct sock *sk,
610 struct hlist_head *list)
611{
612 hlist_add_head(&sk->sk_bind_node, list);
613}
614
615#define sk_for_each(__sk, list) \
616 hlist_for_each_entry(__sk, list, sk_node)
617#define sk_for_each_rcu(__sk, list) \
618 hlist_for_each_entry_rcu(__sk, list, sk_node)
619#define sk_nulls_for_each(__sk, node, list) \
620 hlist_nulls_for_each_entry(__sk, node, list, sk_nulls_node)
621#define sk_nulls_for_each_rcu(__sk, node, list) \
622 hlist_nulls_for_each_entry_rcu(__sk, node, list, sk_nulls_node)
623#define sk_for_each_from(__sk) \
624 hlist_for_each_entry_from(__sk, sk_node)
625#define sk_nulls_for_each_from(__sk, node) \
626 if (__sk && ({ node = &(__sk)->sk_nulls_node; 1; })) \
627 hlist_nulls_for_each_entry_from(__sk, node, sk_nulls_node)
628#define sk_for_each_safe(__sk, tmp, list) \
629 hlist_for_each_entry_safe(__sk, tmp, list, sk_node)
630#define sk_for_each_bound(__sk, list) \
631 hlist_for_each_entry(__sk, list, sk_bind_node)
632
633static inline struct user_namespace *sk_user_ns(struct sock *sk)
634{
635
636
637
638
639 return sk->sk_socket->file->f_cred->user_ns;
640}
641
642
643enum sock_flags {
644 SOCK_DEAD,
645 SOCK_DONE,
646 SOCK_URGINLINE,
647 SOCK_KEEPOPEN,
648 SOCK_LINGER,
649 SOCK_DESTROY,
650 SOCK_BROADCAST,
651 SOCK_TIMESTAMP,
652 SOCK_ZAPPED,
653 SOCK_USE_WRITE_QUEUE,
654 SOCK_DBG,
655 SOCK_RCVTSTAMP,
656 SOCK_RCVTSTAMPNS,
657 SOCK_LOCALROUTE,
658 SOCK_QUEUE_SHRUNK,
659 SOCK_MEMALLOC,
660 SOCK_TIMESTAMPING_TX_HARDWARE,
661 SOCK_TIMESTAMPING_TX_SOFTWARE,
662 SOCK_TIMESTAMPING_RX_HARDWARE,
663 SOCK_TIMESTAMPING_RX_SOFTWARE,
664 SOCK_TIMESTAMPING_SOFTWARE,
665 SOCK_TIMESTAMPING_RAW_HARDWARE,
666 SOCK_TIMESTAMPING_SYS_HARDWARE,
667 SOCK_FASYNC,
668 SOCK_RXQ_OVFL,
669 SOCK_ZEROCOPY,
670 SOCK_WIFI_STATUS,
671 SOCK_NOFCS,
672
673
674
675 SOCK_FILTER_LOCKED,
676 SOCK_SELECT_ERR_QUEUE,
677};
678
679static inline void sock_copy_flags(struct sock *nsk, struct sock *osk)
680{
681 nsk->sk_flags = osk->sk_flags;
682}
683
684static inline void sock_set_flag(struct sock *sk, enum sock_flags flag)
685{
686 __set_bit(flag, &sk->sk_flags);
687}
688
689static inline void sock_reset_flag(struct sock *sk, enum sock_flags flag)
690{
691 __clear_bit(flag, &sk->sk_flags);
692}
693
694static inline bool sock_flag(const struct sock *sk, enum sock_flags flag)
695{
696 return test_bit(flag, &sk->sk_flags);
697}
698
699#ifdef CONFIG_NET
700extern struct static_key memalloc_socks;
701static inline int sk_memalloc_socks(void)
702{
703 return static_key_false(&memalloc_socks);
704}
705#else
706
707static inline int sk_memalloc_socks(void)
708{
709 return 0;
710}
711
712#endif
713
714static inline gfp_t sk_gfp_atomic(struct sock *sk, gfp_t gfp_mask)
715{
716 return GFP_ATOMIC | (sk->sk_allocation & __GFP_MEMALLOC);
717}
718
719static inline void sk_acceptq_removed(struct sock *sk)
720{
721 sk->sk_ack_backlog--;
722}
723
724static inline void sk_acceptq_added(struct sock *sk)
725{
726 sk->sk_ack_backlog++;
727}
728
729static inline bool sk_acceptq_is_full(const struct sock *sk)
730{
731 return sk->sk_ack_backlog > sk->sk_max_ack_backlog;
732}
733
734
735
736
737static inline int sk_stream_min_wspace(const struct sock *sk)
738{
739 return sk->sk_wmem_queued >> 1;
740}
741
742static inline int sk_stream_wspace(const struct sock *sk)
743{
744 return sk->sk_sndbuf - sk->sk_wmem_queued;
745}
746
747extern void sk_stream_write_space(struct sock *sk);
748
749static inline bool sk_stream_memory_free(const struct sock *sk)
750{
751 return sk->sk_wmem_queued < sk->sk_sndbuf;
752}
753
754
755static inline void __sk_add_backlog(struct sock *sk, struct sk_buff *skb)
756{
757
758 skb_dst_force(skb);
759
760 if (!sk->sk_backlog.tail)
761 sk->sk_backlog.head = skb;
762 else
763 sk->sk_backlog.tail->next = skb;
764
765 sk->sk_backlog.tail = skb;
766 skb->next = NULL;
767}
768
769
770
771
772
773
774static inline bool sk_rcvqueues_full(const struct sock *sk, const struct sk_buff *skb,
775 unsigned int limit)
776{
777 unsigned int qsize = sk->sk_backlog.len + atomic_read(&sk->sk_rmem_alloc);
778
779 return qsize > limit;
780}
781
782
783static inline __must_check int sk_add_backlog(struct sock *sk, struct sk_buff *skb,
784 unsigned int limit)
785{
786 if (sk_rcvqueues_full(sk, skb, limit))
787 return -ENOBUFS;
788
789 __sk_add_backlog(sk, skb);
790 sk->sk_backlog.len += skb->truesize;
791 return 0;
792}
793
794extern int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb);
795
796static inline int sk_backlog_rcv(struct sock *sk, struct sk_buff *skb)
797{
798 if (sk_memalloc_socks() && skb_pfmemalloc(skb))
799 return __sk_backlog_rcv(sk, skb);
800
801 return sk->sk_backlog_rcv(sk, skb);
802}
803
804static inline void sock_rps_record_flow(const struct sock *sk)
805{
806#ifdef CONFIG_RPS
807 struct rps_sock_flow_table *sock_flow_table;
808
809 rcu_read_lock();
810 sock_flow_table = rcu_dereference(rps_sock_flow_table);
811 rps_record_sock_flow(sock_flow_table, sk->sk_rxhash);
812 rcu_read_unlock();
813#endif
814}
815
816static inline void sock_rps_reset_flow(const struct sock *sk)
817{
818#ifdef CONFIG_RPS
819 struct rps_sock_flow_table *sock_flow_table;
820
821 rcu_read_lock();
822 sock_flow_table = rcu_dereference(rps_sock_flow_table);
823 rps_reset_sock_flow(sock_flow_table, sk->sk_rxhash);
824 rcu_read_unlock();
825#endif
826}
827
828static inline void sock_rps_save_rxhash(struct sock *sk,
829 const struct sk_buff *skb)
830{
831#ifdef CONFIG_RPS
832 if (unlikely(sk->sk_rxhash != skb->rxhash)) {
833 sock_rps_reset_flow(sk);
834 sk->sk_rxhash = skb->rxhash;
835 }
836#endif
837}
838
839static inline void sock_rps_reset_rxhash(struct sock *sk)
840{
841#ifdef CONFIG_RPS
842 sock_rps_reset_flow(sk);
843 sk->sk_rxhash = 0;
844#endif
845}
846
847#define sk_wait_event(__sk, __timeo, __condition) \
848 ({ int __rc; \
849 release_sock(__sk); \
850 __rc = __condition; \
851 if (!__rc) { \
852 *(__timeo) = schedule_timeout(*(__timeo)); \
853 } \
854 lock_sock(__sk); \
855 __rc = __condition; \
856 __rc; \
857 })
858
859extern int sk_stream_wait_connect(struct sock *sk, long *timeo_p);
860extern int sk_stream_wait_memory(struct sock *sk, long *timeo_p);
861extern void sk_stream_wait_close(struct sock *sk, long timeo_p);
862extern int sk_stream_error(struct sock *sk, int flags, int err);
863extern void sk_stream_kill_queues(struct sock *sk);
864extern void sk_set_memalloc(struct sock *sk);
865extern void sk_clear_memalloc(struct sock *sk);
866
867extern int sk_wait_data(struct sock *sk, long *timeo);
868
869struct request_sock_ops;
870struct timewait_sock_ops;
871struct inet_hashinfo;
872struct raw_hashinfo;
873struct module;
874
875
876
877
878
879static inline void sk_prot_clear_nulls(struct sock *sk, int size)
880{
881 if (offsetof(struct sock, sk_node.next) != 0)
882 memset(sk, 0, offsetof(struct sock, sk_node.next));
883 memset(&sk->sk_node.pprev, 0,
884 size - offsetof(struct sock, sk_node.pprev));
885}
886
887
888
889
890
891struct proto {
892 void (*close)(struct sock *sk,
893 long timeout);
894 int (*connect)(struct sock *sk,
895 struct sockaddr *uaddr,
896 int addr_len);
897 int (*disconnect)(struct sock *sk, int flags);
898
899 struct sock * (*accept)(struct sock *sk, int flags, int *err);
900
901 int (*ioctl)(struct sock *sk, int cmd,
902 unsigned long arg);
903 int (*init)(struct sock *sk);
904 void (*destroy)(struct sock *sk);
905 void (*shutdown)(struct sock *sk, int how);
906 int (*setsockopt)(struct sock *sk, int level,
907 int optname, char __user *optval,
908 unsigned int optlen);
909 int (*getsockopt)(struct sock *sk, int level,
910 int optname, char __user *optval,
911 int __user *option);
912#ifdef CONFIG_COMPAT
913 int (*compat_setsockopt)(struct sock *sk,
914 int level,
915 int optname, char __user *optval,
916 unsigned int optlen);
917 int (*compat_getsockopt)(struct sock *sk,
918 int level,
919 int optname, char __user *optval,
920 int __user *option);
921 int (*compat_ioctl)(struct sock *sk,
922 unsigned int cmd, unsigned long arg);
923#endif
924 int (*sendmsg)(struct kiocb *iocb, struct sock *sk,
925 struct msghdr *msg, size_t len);
926 int (*recvmsg)(struct kiocb *iocb, struct sock *sk,
927 struct msghdr *msg,
928 size_t len, int noblock, int flags,
929 int *addr_len);
930 int (*sendpage)(struct sock *sk, struct page *page,
931 int offset, size_t size, int flags);
932 int (*bind)(struct sock *sk,
933 struct sockaddr *uaddr, int addr_len);
934
935 int (*backlog_rcv) (struct sock *sk,
936 struct sk_buff *skb);
937
938 void (*release_cb)(struct sock *sk);
939 void (*mtu_reduced)(struct sock *sk);
940
941
942 void (*hash)(struct sock *sk);
943 void (*unhash)(struct sock *sk);
944 void (*rehash)(struct sock *sk);
945 int (*get_port)(struct sock *sk, unsigned short snum);
946 void (*clear_sk)(struct sock *sk, int size);
947
948
949#ifdef CONFIG_PROC_FS
950 unsigned int inuse_idx;
951#endif
952
953
954 void (*enter_memory_pressure)(struct sock *sk);
955 atomic_long_t *memory_allocated;
956 struct percpu_counter *sockets_allocated;
957
958
959
960
961
962
963 int *memory_pressure;
964 long *sysctl_mem;
965 int *sysctl_wmem;
966 int *sysctl_rmem;
967 int max_header;
968 bool no_autobind;
969
970 struct kmem_cache *slab;
971 unsigned int obj_size;
972 int slab_flags;
973
974 struct percpu_counter *orphan_count;
975
976 struct request_sock_ops *rsk_prot;
977 struct timewait_sock_ops *twsk_prot;
978
979 union {
980 struct inet_hashinfo *hashinfo;
981 struct udp_table *udp_table;
982 struct raw_hashinfo *raw_hash;
983 } h;
984
985 struct module *owner;
986
987 char name[32];
988
989 struct list_head node;
990#ifdef SOCK_REFCNT_DEBUG
991 atomic_t socks;
992#endif
993#ifdef CONFIG_MEMCG_KMEM
994
995
996
997
998
999
1000 int (*init_cgroup)(struct mem_cgroup *memcg,
1001 struct cgroup_subsys *ss);
1002 void (*destroy_cgroup)(struct mem_cgroup *memcg);
1003 struct cg_proto *(*proto_cgroup)(struct mem_cgroup *memcg);
1004#endif
1005};
1006
1007
1008
1009
1010enum cg_proto_flags {
1011
1012 MEMCG_SOCK_ACTIVE,
1013
1014 MEMCG_SOCK_ACTIVATED,
1015};
1016
1017struct cg_proto {
1018 void (*enter_memory_pressure)(struct sock *sk);
1019 struct res_counter *memory_allocated;
1020 struct percpu_counter *sockets_allocated;
1021 int *memory_pressure;
1022 long *sysctl_mem;
1023 unsigned long flags;
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033 struct mem_cgroup *memcg;
1034};
1035
1036extern int proto_register(struct proto *prot, int alloc_slab);
1037extern void proto_unregister(struct proto *prot);
1038
1039static inline bool memcg_proto_active(struct cg_proto *cg_proto)
1040{
1041 return test_bit(MEMCG_SOCK_ACTIVE, &cg_proto->flags);
1042}
1043
1044static inline bool memcg_proto_activated(struct cg_proto *cg_proto)
1045{
1046 return test_bit(MEMCG_SOCK_ACTIVATED, &cg_proto->flags);
1047}
1048
1049#ifdef SOCK_REFCNT_DEBUG
1050static inline void sk_refcnt_debug_inc(struct sock *sk)
1051{
1052 atomic_inc(&sk->sk_prot->socks);
1053}
1054
1055static inline void sk_refcnt_debug_dec(struct sock *sk)
1056{
1057 atomic_dec(&sk->sk_prot->socks);
1058 printk(KERN_DEBUG "%s socket %p released, %d are still alive\n",
1059 sk->sk_prot->name, sk, atomic_read(&sk->sk_prot->socks));
1060}
1061
1062static inline void sk_refcnt_debug_release(const struct sock *sk)
1063{
1064 if (atomic_read(&sk->sk_refcnt) != 1)
1065 printk(KERN_DEBUG "Destruction of the %s socket %p delayed, refcnt=%d\n",
1066 sk->sk_prot->name, sk, atomic_read(&sk->sk_refcnt));
1067}
1068#else
1069#define sk_refcnt_debug_inc(sk) do { } while (0)
1070#define sk_refcnt_debug_dec(sk) do { } while (0)
1071#define sk_refcnt_debug_release(sk) do { } while (0)
1072#endif
1073
1074#if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_NET)
1075extern struct static_key memcg_socket_limit_enabled;
1076static inline struct cg_proto *parent_cg_proto(struct proto *proto,
1077 struct cg_proto *cg_proto)
1078{
1079 return proto->proto_cgroup(parent_mem_cgroup(cg_proto->memcg));
1080}
1081#define mem_cgroup_sockets_enabled static_key_false(&memcg_socket_limit_enabled)
1082#else
1083#define mem_cgroup_sockets_enabled 0
1084static inline struct cg_proto *parent_cg_proto(struct proto *proto,
1085 struct cg_proto *cg_proto)
1086{
1087 return NULL;
1088}
1089#endif
1090
1091
1092static inline bool sk_has_memory_pressure(const struct sock *sk)
1093{
1094 return sk->sk_prot->memory_pressure != NULL;
1095}
1096
1097static inline bool sk_under_memory_pressure(const struct sock *sk)
1098{
1099 if (!sk->sk_prot->memory_pressure)
1100 return false;
1101
1102 if (mem_cgroup_sockets_enabled && sk->sk_cgrp)
1103 return !!*sk->sk_cgrp->memory_pressure;
1104
1105 return !!*sk->sk_prot->memory_pressure;
1106}
1107
1108static inline void sk_leave_memory_pressure(struct sock *sk)
1109{
1110 int *memory_pressure = sk->sk_prot->memory_pressure;
1111
1112 if (!memory_pressure)
1113 return;
1114
1115 if (*memory_pressure)
1116 *memory_pressure = 0;
1117
1118 if (mem_cgroup_sockets_enabled && sk->sk_cgrp) {
1119 struct cg_proto *cg_proto = sk->sk_cgrp;
1120 struct proto *prot = sk->sk_prot;
1121
1122 for (; cg_proto; cg_proto = parent_cg_proto(prot, cg_proto))
1123 if (*cg_proto->memory_pressure)
1124 *cg_proto->memory_pressure = 0;
1125 }
1126
1127}
1128
1129static inline void sk_enter_memory_pressure(struct sock *sk)
1130{
1131 if (!sk->sk_prot->enter_memory_pressure)
1132 return;
1133
1134 if (mem_cgroup_sockets_enabled && sk->sk_cgrp) {
1135 struct cg_proto *cg_proto = sk->sk_cgrp;
1136 struct proto *prot = sk->sk_prot;
1137
1138 for (; cg_proto; cg_proto = parent_cg_proto(prot, cg_proto))
1139 cg_proto->enter_memory_pressure(sk);
1140 }
1141
1142 sk->sk_prot->enter_memory_pressure(sk);
1143}
1144
1145static inline long sk_prot_mem_limits(const struct sock *sk, int index)
1146{
1147 long *prot = sk->sk_prot->sysctl_mem;
1148 if (mem_cgroup_sockets_enabled && sk->sk_cgrp)
1149 prot = sk->sk_cgrp->sysctl_mem;
1150 return prot[index];
1151}
1152
1153static inline void memcg_memory_allocated_add(struct cg_proto *prot,
1154 unsigned long amt,
1155 int *parent_status)
1156{
1157 struct res_counter *fail;
1158 int ret;
1159
1160 ret = res_counter_charge_nofail(prot->memory_allocated,
1161 amt << PAGE_SHIFT, &fail);
1162 if (ret < 0)
1163 *parent_status = OVER_LIMIT;
1164}
1165
1166static inline void memcg_memory_allocated_sub(struct cg_proto *prot,
1167 unsigned long amt)
1168{
1169 res_counter_uncharge(prot->memory_allocated, amt << PAGE_SHIFT);
1170}
1171
1172static inline u64 memcg_memory_allocated_read(struct cg_proto *prot)
1173{
1174 u64 ret;
1175 ret = res_counter_read_u64(prot->memory_allocated, RES_USAGE);
1176 return ret >> PAGE_SHIFT;
1177}
1178
1179static inline long
1180sk_memory_allocated(const struct sock *sk)
1181{
1182 struct proto *prot = sk->sk_prot;
1183 if (mem_cgroup_sockets_enabled && sk->sk_cgrp)
1184 return memcg_memory_allocated_read(sk->sk_cgrp);
1185
1186 return atomic_long_read(prot->memory_allocated);
1187}
1188
1189static inline long
1190sk_memory_allocated_add(struct sock *sk, int amt, int *parent_status)
1191{
1192 struct proto *prot = sk->sk_prot;
1193
1194 if (mem_cgroup_sockets_enabled && sk->sk_cgrp) {
1195 memcg_memory_allocated_add(sk->sk_cgrp, amt, parent_status);
1196
1197 atomic_long_add_return(amt, prot->memory_allocated);
1198 return memcg_memory_allocated_read(sk->sk_cgrp);
1199 }
1200
1201 return atomic_long_add_return(amt, prot->memory_allocated);
1202}
1203
1204static inline void
1205sk_memory_allocated_sub(struct sock *sk, int amt)
1206{
1207 struct proto *prot = sk->sk_prot;
1208
1209 if (mem_cgroup_sockets_enabled && sk->sk_cgrp)
1210 memcg_memory_allocated_sub(sk->sk_cgrp, amt);
1211
1212 atomic_long_sub(amt, prot->memory_allocated);
1213}
1214
1215static inline void sk_sockets_allocated_dec(struct sock *sk)
1216{
1217 struct proto *prot = sk->sk_prot;
1218
1219 if (mem_cgroup_sockets_enabled && sk->sk_cgrp) {
1220 struct cg_proto *cg_proto = sk->sk_cgrp;
1221
1222 for (; cg_proto; cg_proto = parent_cg_proto(prot, cg_proto))
1223 percpu_counter_dec(cg_proto->sockets_allocated);
1224 }
1225
1226 percpu_counter_dec(prot->sockets_allocated);
1227}
1228
1229static inline void sk_sockets_allocated_inc(struct sock *sk)
1230{
1231 struct proto *prot = sk->sk_prot;
1232
1233 if (mem_cgroup_sockets_enabled && sk->sk_cgrp) {
1234 struct cg_proto *cg_proto = sk->sk_cgrp;
1235
1236 for (; cg_proto; cg_proto = parent_cg_proto(prot, cg_proto))
1237 percpu_counter_inc(cg_proto->sockets_allocated);
1238 }
1239
1240 percpu_counter_inc(prot->sockets_allocated);
1241}
1242
1243static inline int
1244sk_sockets_allocated_read_positive(struct sock *sk)
1245{
1246 struct proto *prot = sk->sk_prot;
1247
1248 if (mem_cgroup_sockets_enabled && sk->sk_cgrp)
1249 return percpu_counter_read_positive(sk->sk_cgrp->sockets_allocated);
1250
1251 return percpu_counter_read_positive(prot->sockets_allocated);
1252}
1253
1254static inline int
1255proto_sockets_allocated_sum_positive(struct proto *prot)
1256{
1257 return percpu_counter_sum_positive(prot->sockets_allocated);
1258}
1259
1260static inline long
1261proto_memory_allocated(struct proto *prot)
1262{
1263 return atomic_long_read(prot->memory_allocated);
1264}
1265
1266static inline bool
1267proto_memory_pressure(struct proto *prot)
1268{
1269 if (!prot->memory_pressure)
1270 return false;
1271 return !!*prot->memory_pressure;
1272}
1273
1274
1275#ifdef CONFIG_PROC_FS
1276
1277extern void sock_prot_inuse_add(struct net *net, struct proto *prot, int inc);
1278extern int sock_prot_inuse_get(struct net *net, struct proto *proto);
1279#else
1280static inline void sock_prot_inuse_add(struct net *net, struct proto *prot,
1281 int inc)
1282{
1283}
1284#endif
1285
1286
1287
1288
1289
1290static inline void __sk_prot_rehash(struct sock *sk)
1291{
1292 sk->sk_prot->unhash(sk);
1293 sk->sk_prot->hash(sk);
1294}
1295
1296void sk_prot_clear_portaddr_nulls(struct sock *sk, int size);
1297
1298
1299#define SOCK_DESTROY_TIME (10*HZ)
1300
1301
1302#define PROT_SOCK 1024
1303
1304#define SHUTDOWN_MASK 3
1305#define RCV_SHUTDOWN 1
1306#define SEND_SHUTDOWN 2
1307
1308#define SOCK_SNDBUF_LOCK 1
1309#define SOCK_RCVBUF_LOCK 2
1310#define SOCK_BINDADDR_LOCK 4
1311#define SOCK_BINDPORT_LOCK 8
1312
1313
1314struct sock_iocb {
1315 struct list_head list;
1316
1317 int flags;
1318 int size;
1319 struct socket *sock;
1320 struct sock *sk;
1321 struct scm_cookie *scm;
1322 struct msghdr *msg, async_msg;
1323 struct kiocb *kiocb;
1324};
1325
1326static inline struct sock_iocb *kiocb_to_siocb(struct kiocb *iocb)
1327{
1328 return (struct sock_iocb *)iocb->private;
1329}
1330
1331static inline struct kiocb *siocb_to_kiocb(struct sock_iocb *si)
1332{
1333 return si->kiocb;
1334}
1335
1336struct socket_alloc {
1337 struct socket socket;
1338 struct inode vfs_inode;
1339};
1340
1341static inline struct socket *SOCKET_I(struct inode *inode)
1342{
1343 return &container_of(inode, struct socket_alloc, vfs_inode)->socket;
1344}
1345
1346static inline struct inode *SOCK_INODE(struct socket *socket)
1347{
1348 return &container_of(socket, struct socket_alloc, socket)->vfs_inode;
1349}
1350
1351
1352
1353
1354extern int __sk_mem_schedule(struct sock *sk, int size, int kind);
1355extern void __sk_mem_reclaim(struct sock *sk);
1356
1357#define SK_MEM_QUANTUM ((int)PAGE_SIZE)
1358#define SK_MEM_QUANTUM_SHIFT ilog2(SK_MEM_QUANTUM)
1359#define SK_MEM_SEND 0
1360#define SK_MEM_RECV 1
1361
1362static inline int sk_mem_pages(int amt)
1363{
1364 return (amt + SK_MEM_QUANTUM - 1) >> SK_MEM_QUANTUM_SHIFT;
1365}
1366
1367static inline bool sk_has_account(struct sock *sk)
1368{
1369
1370 return !!sk->sk_prot->memory_allocated;
1371}
1372
1373static inline bool sk_wmem_schedule(struct sock *sk, int size)
1374{
1375 if (!sk_has_account(sk))
1376 return true;
1377 return size <= sk->sk_forward_alloc ||
1378 __sk_mem_schedule(sk, size, SK_MEM_SEND);
1379}
1380
1381static inline bool
1382sk_rmem_schedule(struct sock *sk, struct sk_buff *skb, int size)
1383{
1384 if (!sk_has_account(sk))
1385 return true;
1386 return size<= sk->sk_forward_alloc ||
1387 __sk_mem_schedule(sk, size, SK_MEM_RECV) ||
1388 skb_pfmemalloc(skb);
1389}
1390
1391static inline void sk_mem_reclaim(struct sock *sk)
1392{
1393 if (!sk_has_account(sk))
1394 return;
1395 if (sk->sk_forward_alloc >= SK_MEM_QUANTUM)
1396 __sk_mem_reclaim(sk);
1397}
1398
1399static inline void sk_mem_reclaim_partial(struct sock *sk)
1400{
1401 if (!sk_has_account(sk))
1402 return;
1403 if (sk->sk_forward_alloc > SK_MEM_QUANTUM)
1404 __sk_mem_reclaim(sk);
1405}
1406
1407static inline void sk_mem_charge(struct sock *sk, int size)
1408{
1409 if (!sk_has_account(sk))
1410 return;
1411 sk->sk_forward_alloc -= size;
1412}
1413
1414static inline void sk_mem_uncharge(struct sock *sk, int size)
1415{
1416 if (!sk_has_account(sk))
1417 return;
1418 sk->sk_forward_alloc += size;
1419}
1420
1421static inline void sk_wmem_free_skb(struct sock *sk, struct sk_buff *skb)
1422{
1423 sock_set_flag(sk, SOCK_QUEUE_SHRUNK);
1424 sk->sk_wmem_queued -= skb->truesize;
1425 sk_mem_uncharge(sk, skb->truesize);
1426 __kfree_skb(skb);
1427}
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442#define sock_owned_by_user(sk) ((sk)->sk_lock.owned)
1443
1444
1445
1446
1447
1448
1449
1450
1451#define sock_lock_init_class_and_name(sk, sname, skey, name, key) \
1452do { \
1453 sk->sk_lock.owned = 0; \
1454 init_waitqueue_head(&sk->sk_lock.wq); \
1455 spin_lock_init(&(sk)->sk_lock.slock); \
1456 debug_check_no_locks_freed((void *)&(sk)->sk_lock, \
1457 sizeof((sk)->sk_lock)); \
1458 lockdep_set_class_and_name(&(sk)->sk_lock.slock, \
1459 (skey), (sname)); \
1460 lockdep_init_map(&(sk)->sk_lock.dep_map, (name), (key), 0); \
1461} while (0)
1462
1463extern void lock_sock_nested(struct sock *sk, int subclass);
1464
1465static inline void lock_sock(struct sock *sk)
1466{
1467 lock_sock_nested(sk, 0);
1468}
1469
1470extern void release_sock(struct sock *sk);
1471
1472
1473#define bh_lock_sock(__sk) spin_lock(&((__sk)->sk_lock.slock))
1474#define bh_lock_sock_nested(__sk) \
1475 spin_lock_nested(&((__sk)->sk_lock.slock), \
1476 SINGLE_DEPTH_NESTING)
1477#define bh_unlock_sock(__sk) spin_unlock(&((__sk)->sk_lock.slock))
1478
1479extern bool lock_sock_fast(struct sock *sk);
1480
1481
1482
1483
1484
1485
1486
1487
1488static inline void unlock_sock_fast(struct sock *sk, bool slow)
1489{
1490 if (slow)
1491 release_sock(sk);
1492 else
1493 spin_unlock_bh(&sk->sk_lock.slock);
1494}
1495
1496
1497extern struct sock *sk_alloc(struct net *net, int family,
1498 gfp_t priority,
1499 struct proto *prot);
1500extern void sk_free(struct sock *sk);
1501extern void sk_release_kernel(struct sock *sk);
1502extern struct sock *sk_clone_lock(const struct sock *sk,
1503 const gfp_t priority);
1504
1505extern struct sk_buff *sock_wmalloc(struct sock *sk,
1506 unsigned long size, int force,
1507 gfp_t priority);
1508extern struct sk_buff *sock_rmalloc(struct sock *sk,
1509 unsigned long size, int force,
1510 gfp_t priority);
1511extern void sock_wfree(struct sk_buff *skb);
1512extern void sock_rfree(struct sk_buff *skb);
1513extern void sock_edemux(struct sk_buff *skb);
1514
1515extern int sock_setsockopt(struct socket *sock, int level,
1516 int op, char __user *optval,
1517 unsigned int optlen);
1518
1519extern int sock_getsockopt(struct socket *sock, int level,
1520 int op, char __user *optval,
1521 int __user *optlen);
1522extern struct sk_buff *sock_alloc_send_skb(struct sock *sk,
1523 unsigned long size,
1524 int noblock,
1525 int *errcode);
1526extern struct sk_buff *sock_alloc_send_pskb(struct sock *sk,
1527 unsigned long header_len,
1528 unsigned long data_len,
1529 int noblock,
1530 int *errcode);
1531extern void *sock_kmalloc(struct sock *sk, int size,
1532 gfp_t priority);
1533extern void sock_kfree_s(struct sock *sk, void *mem, int size);
1534extern void sk_send_sigurg(struct sock *sk);
1535
1536
1537
1538
1539
1540extern int sock_no_bind(struct socket *,
1541 struct sockaddr *, int);
1542extern int sock_no_connect(struct socket *,
1543 struct sockaddr *, int, int);
1544extern int sock_no_socketpair(struct socket *,
1545 struct socket *);
1546extern int sock_no_accept(struct socket *,
1547 struct socket *, int);
1548extern int sock_no_getname(struct socket *,
1549 struct sockaddr *, int *, int);
1550extern unsigned int sock_no_poll(struct file *, struct socket *,
1551 struct poll_table_struct *);
1552extern int sock_no_ioctl(struct socket *, unsigned int,
1553 unsigned long);
1554extern int sock_no_listen(struct socket *, int);
1555extern int sock_no_shutdown(struct socket *, int);
1556extern int sock_no_getsockopt(struct socket *, int , int,
1557 char __user *, int __user *);
1558extern int sock_no_setsockopt(struct socket *, int, int,
1559 char __user *, unsigned int);
1560extern int sock_no_sendmsg(struct kiocb *, struct socket *,
1561 struct msghdr *, size_t);
1562extern int sock_no_recvmsg(struct kiocb *, struct socket *,
1563 struct msghdr *, size_t, int);
1564extern int sock_no_mmap(struct file *file,
1565 struct socket *sock,
1566 struct vm_area_struct *vma);
1567extern ssize_t sock_no_sendpage(struct socket *sock,
1568 struct page *page,
1569 int offset, size_t size,
1570 int flags);
1571
1572
1573
1574
1575
1576extern int sock_common_getsockopt(struct socket *sock, int level, int optname,
1577 char __user *optval, int __user *optlen);
1578extern int sock_common_recvmsg(struct kiocb *iocb, struct socket *sock,
1579 struct msghdr *msg, size_t size, int flags);
1580extern int sock_common_setsockopt(struct socket *sock, int level, int optname,
1581 char __user *optval, unsigned int optlen);
1582extern int compat_sock_common_getsockopt(struct socket *sock, int level,
1583 int optname, char __user *optval, int __user *optlen);
1584extern int compat_sock_common_setsockopt(struct socket *sock, int level,
1585 int optname, char __user *optval, unsigned int optlen);
1586
1587extern void sk_common_release(struct sock *sk);
1588
1589
1590
1591
1592
1593
1594extern void sock_init_data(struct socket *sock, struct sock *sk);
1595
1596extern void sk_filter_release_rcu(struct rcu_head *rcu);
1597
1598
1599
1600
1601
1602
1603
1604
1605static inline void sk_filter_release(struct sk_filter *fp)
1606{
1607 if (atomic_dec_and_test(&fp->refcnt))
1608 call_rcu(&fp->rcu, sk_filter_release_rcu);
1609}
1610
1611static inline void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp)
1612{
1613 unsigned int size = sk_filter_len(fp);
1614
1615 atomic_sub(size, &sk->sk_omem_alloc);
1616 sk_filter_release(fp);
1617}
1618
1619static inline void sk_filter_charge(struct sock *sk, struct sk_filter *fp)
1620{
1621 atomic_inc(&fp->refcnt);
1622 atomic_add(sk_filter_len(fp), &sk->sk_omem_alloc);
1623}
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651static inline void sock_put(struct sock *sk)
1652{
1653 if (atomic_dec_and_test(&sk->sk_refcnt))
1654 sk_free(sk);
1655}
1656
1657extern int sk_receive_skb(struct sock *sk, struct sk_buff *skb,
1658 const int nested);
1659
1660static inline void sk_tx_queue_set(struct sock *sk, int tx_queue)
1661{
1662 sk->sk_tx_queue_mapping = tx_queue;
1663}
1664
1665static inline void sk_tx_queue_clear(struct sock *sk)
1666{
1667 sk->sk_tx_queue_mapping = -1;
1668}
1669
1670static inline int sk_tx_queue_get(const struct sock *sk)
1671{
1672 return sk ? sk->sk_tx_queue_mapping : -1;
1673}
1674
1675static inline void sk_set_socket(struct sock *sk, struct socket *sock)
1676{
1677 sk_tx_queue_clear(sk);
1678 sk->sk_socket = sock;
1679}
1680
1681static inline wait_queue_head_t *sk_sleep(struct sock *sk)
1682{
1683 BUILD_BUG_ON(offsetof(struct socket_wq, wait) != 0);
1684 return &rcu_dereference_raw(sk->sk_wq)->wait;
1685}
1686
1687
1688
1689
1690
1691
1692
1693static inline void sock_orphan(struct sock *sk)
1694{
1695 write_lock_bh(&sk->sk_callback_lock);
1696 sock_set_flag(sk, SOCK_DEAD);
1697 sk_set_socket(sk, NULL);
1698 sk->sk_wq = NULL;
1699 write_unlock_bh(&sk->sk_callback_lock);
1700}
1701
1702static inline void sock_graft(struct sock *sk, struct socket *parent)
1703{
1704 write_lock_bh(&sk->sk_callback_lock);
1705 sk->sk_wq = parent->wq;
1706 parent->sk = sk;
1707 sk_set_socket(sk, parent);
1708 security_sock_graft(sk, parent);
1709 write_unlock_bh(&sk->sk_callback_lock);
1710}
1711
1712extern kuid_t sock_i_uid(struct sock *sk);
1713extern unsigned long sock_i_ino(struct sock *sk);
1714
1715static inline struct dst_entry *
1716__sk_dst_get(struct sock *sk)
1717{
1718 return rcu_dereference_check(sk->sk_dst_cache, sock_owned_by_user(sk) ||
1719 lockdep_is_held(&sk->sk_lock.slock));
1720}
1721
1722static inline struct dst_entry *
1723sk_dst_get(struct sock *sk)
1724{
1725 struct dst_entry *dst;
1726
1727 rcu_read_lock();
1728 dst = rcu_dereference(sk->sk_dst_cache);
1729 if (dst)
1730 dst_hold(dst);
1731 rcu_read_unlock();
1732 return dst;
1733}
1734
1735extern void sk_reset_txq(struct sock *sk);
1736
1737static inline void dst_negative_advice(struct sock *sk)
1738{
1739 struct dst_entry *ndst, *dst = __sk_dst_get(sk);
1740
1741 if (dst && dst->ops->negative_advice) {
1742 ndst = dst->ops->negative_advice(dst);
1743
1744 if (ndst != dst) {
1745 rcu_assign_pointer(sk->sk_dst_cache, ndst);
1746 sk_reset_txq(sk);
1747 }
1748 }
1749}
1750
1751static inline void
1752__sk_dst_set(struct sock *sk, struct dst_entry *dst)
1753{
1754 struct dst_entry *old_dst;
1755
1756 sk_tx_queue_clear(sk);
1757
1758
1759
1760
1761 old_dst = rcu_dereference_raw(sk->sk_dst_cache);
1762 rcu_assign_pointer(sk->sk_dst_cache, dst);
1763 dst_release(old_dst);
1764}
1765
1766static inline void
1767sk_dst_set(struct sock *sk, struct dst_entry *dst)
1768{
1769 spin_lock(&sk->sk_dst_lock);
1770 __sk_dst_set(sk, dst);
1771 spin_unlock(&sk->sk_dst_lock);
1772}
1773
1774static inline void
1775__sk_dst_reset(struct sock *sk)
1776{
1777 __sk_dst_set(sk, NULL);
1778}
1779
1780static inline void
1781sk_dst_reset(struct sock *sk)
1782{
1783 spin_lock(&sk->sk_dst_lock);
1784 __sk_dst_reset(sk);
1785 spin_unlock(&sk->sk_dst_lock);
1786}
1787
1788extern struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie);
1789
1790extern struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie);
1791
1792static inline bool sk_can_gso(const struct sock *sk)
1793{
1794 return net_gso_ok(sk->sk_route_caps, sk->sk_gso_type);
1795}
1796
1797extern void sk_setup_caps(struct sock *sk, struct dst_entry *dst);
1798
1799static inline void sk_nocaps_add(struct sock *sk, netdev_features_t flags)
1800{
1801 sk->sk_route_nocaps |= flags;
1802 sk->sk_route_caps &= ~flags;
1803}
1804
1805static inline int skb_do_copy_data_nocache(struct sock *sk, struct sk_buff *skb,
1806 char __user *from, char *to,
1807 int copy, int offset)
1808{
1809 if (skb->ip_summed == CHECKSUM_NONE) {
1810 int err = 0;
1811 __wsum csum = csum_and_copy_from_user(from, to, copy, 0, &err);
1812 if (err)
1813 return err;
1814 skb->csum = csum_block_add(skb->csum, csum, offset);
1815 } else if (sk->sk_route_caps & NETIF_F_NOCACHE_COPY) {
1816 if (!access_ok(VERIFY_READ, from, copy) ||
1817 __copy_from_user_nocache(to, from, copy))
1818 return -EFAULT;
1819 } else if (copy_from_user(to, from, copy))
1820 return -EFAULT;
1821
1822 return 0;
1823}
1824
1825static inline int skb_add_data_nocache(struct sock *sk, struct sk_buff *skb,
1826 char __user *from, int copy)
1827{
1828 int err, offset = skb->len;
1829
1830 err = skb_do_copy_data_nocache(sk, skb, from, skb_put(skb, copy),
1831 copy, offset);
1832 if (err)
1833 __skb_trim(skb, offset);
1834
1835 return err;
1836}
1837
1838static inline int skb_copy_to_page_nocache(struct sock *sk, char __user *from,
1839 struct sk_buff *skb,
1840 struct page *page,
1841 int off, int copy)
1842{
1843 int err;
1844
1845 err = skb_do_copy_data_nocache(sk, skb, from, page_address(page) + off,
1846 copy, skb->len);
1847 if (err)
1848 return err;
1849
1850 skb->len += copy;
1851 skb->data_len += copy;
1852 skb->truesize += copy;
1853 sk->sk_wmem_queued += copy;
1854 sk_mem_charge(sk, copy);
1855 return 0;
1856}
1857
1858static inline int skb_copy_to_page(struct sock *sk, char __user *from,
1859 struct sk_buff *skb, struct page *page,
1860 int off, int copy)
1861{
1862 if (skb->ip_summed == CHECKSUM_NONE) {
1863 int err = 0;
1864 __wsum csum = csum_and_copy_from_user(from,
1865 page_address(page) + off,
1866 copy, 0, &err);
1867 if (err)
1868 return err;
1869 skb->csum = csum_block_add(skb->csum, csum, skb->len);
1870 } else if (copy_from_user(page_address(page) + off, from, copy))
1871 return -EFAULT;
1872
1873 skb->len += copy;
1874 skb->data_len += copy;
1875 skb->truesize += copy;
1876 sk->sk_wmem_queued += copy;
1877 sk_mem_charge(sk, copy);
1878 return 0;
1879}
1880
1881
1882
1883
1884
1885
1886
1887static inline int sk_wmem_alloc_get(const struct sock *sk)
1888{
1889 return atomic_read(&sk->sk_wmem_alloc) - 1;
1890}
1891
1892
1893
1894
1895
1896
1897
1898static inline int sk_rmem_alloc_get(const struct sock *sk)
1899{
1900 return atomic_read(&sk->sk_rmem_alloc);
1901}
1902
1903
1904
1905
1906
1907
1908
1909static inline bool sk_has_allocations(const struct sock *sk)
1910{
1911 return sk_wmem_alloc_get(sk) || sk_rmem_alloc_get(sk);
1912}
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946static inline bool wq_has_sleeper(struct socket_wq *wq)
1947{
1948
1949
1950
1951
1952
1953 smp_mb();
1954 return wq && waitqueue_active(&wq->wait);
1955}
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965static inline void sock_poll_wait(struct file *filp,
1966 wait_queue_head_t *wait_address, poll_table *p)
1967{
1968 if (!poll_does_not_wait(p) && wait_address) {
1969 poll_wait(filp, wait_address, p);
1970
1971
1972
1973
1974
1975 smp_mb();
1976 }
1977}
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988static inline void skb_set_owner_w(struct sk_buff *skb, struct sock *sk)
1989{
1990 skb_orphan(skb);
1991 skb->sk = sk;
1992 skb->destructor = sock_wfree;
1993
1994
1995
1996
1997
1998 atomic_add(skb->truesize, &sk->sk_wmem_alloc);
1999}
2000
2001static inline void skb_set_owner_r(struct sk_buff *skb, struct sock *sk)
2002{
2003 skb_orphan(skb);
2004 skb->sk = sk;
2005 skb->destructor = sock_rfree;
2006 atomic_add(skb->truesize, &sk->sk_rmem_alloc);
2007 sk_mem_charge(sk, skb->truesize);
2008}
2009
2010extern void sk_reset_timer(struct sock *sk, struct timer_list *timer,
2011 unsigned long expires);
2012
2013extern void sk_stop_timer(struct sock *sk, struct timer_list *timer);
2014
2015extern int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb);
2016
2017extern int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb);
2018
2019
2020
2021
2022
2023static inline int sock_error(struct sock *sk)
2024{
2025 int err;
2026 if (likely(!sk->sk_err))
2027 return 0;
2028 err = xchg(&sk->sk_err, 0);
2029 return -err;
2030}
2031
2032static inline unsigned long sock_wspace(struct sock *sk)
2033{
2034 int amt = 0;
2035
2036 if (!(sk->sk_shutdown & SEND_SHUTDOWN)) {
2037 amt = sk->sk_sndbuf - atomic_read(&sk->sk_wmem_alloc);
2038 if (amt < 0)
2039 amt = 0;
2040 }
2041 return amt;
2042}
2043
2044static inline void sk_wake_async(struct sock *sk, int how, int band)
2045{
2046 if (sock_flag(sk, SOCK_FASYNC))
2047 sock_wake_async(sk->sk_socket, how, band);
2048}
2049
2050
2051
2052
2053
2054
2055#define TCP_SKB_MIN_TRUESIZE (2048 + SKB_DATA_ALIGN(sizeof(struct sk_buff)))
2056
2057#define SOCK_MIN_SNDBUF (TCP_SKB_MIN_TRUESIZE * 2)
2058#define SOCK_MIN_RCVBUF TCP_SKB_MIN_TRUESIZE
2059
2060static inline void sk_stream_moderate_sndbuf(struct sock *sk)
2061{
2062 if (!(sk->sk_userlocks & SOCK_SNDBUF_LOCK)) {
2063 sk->sk_sndbuf = min(sk->sk_sndbuf, sk->sk_wmem_queued >> 1);
2064 sk->sk_sndbuf = max_t(u32, sk->sk_sndbuf, SOCK_MIN_SNDBUF);
2065 }
2066}
2067
2068struct sk_buff *sk_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp);
2069
2070
2071
2072
2073
2074
2075
2076
2077static inline struct page_frag *sk_page_frag(struct sock *sk)
2078{
2079 if (sk->sk_allocation & __GFP_WAIT)
2080 return ¤t->task_frag;
2081
2082 return &sk->sk_frag;
2083}
2084
2085extern bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag);
2086
2087
2088
2089
2090static inline bool sock_writeable(const struct sock *sk)
2091{
2092 return atomic_read(&sk->sk_wmem_alloc) < (sk->sk_sndbuf >> 1);
2093}
2094
2095static inline gfp_t gfp_any(void)
2096{
2097 return in_softirq() ? GFP_ATOMIC : GFP_KERNEL;
2098}
2099
2100static inline long sock_rcvtimeo(const struct sock *sk, bool noblock)
2101{
2102 return noblock ? 0 : sk->sk_rcvtimeo;
2103}
2104
2105static inline long sock_sndtimeo(const struct sock *sk, bool noblock)
2106{
2107 return noblock ? 0 : sk->sk_sndtimeo;
2108}
2109
2110static inline int sock_rcvlowat(const struct sock *sk, int waitall, int len)
2111{
2112 return (waitall ? len : min_t(int, sk->sk_rcvlowat, len)) ? : 1;
2113}
2114
2115
2116
2117
2118static inline int sock_intr_errno(long timeo)
2119{
2120 return timeo == MAX_SCHEDULE_TIMEOUT ? -ERESTARTSYS : -EINTR;
2121}
2122
2123extern void __sock_recv_timestamp(struct msghdr *msg, struct sock *sk,
2124 struct sk_buff *skb);
2125extern void __sock_recv_wifi_status(struct msghdr *msg, struct sock *sk,
2126 struct sk_buff *skb);
2127
2128static inline void
2129sock_recv_timestamp(struct msghdr *msg, struct sock *sk, struct sk_buff *skb)
2130{
2131 ktime_t kt = skb->tstamp;
2132 struct skb_shared_hwtstamps *hwtstamps = skb_hwtstamps(skb);
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144 if (sock_flag(sk, SOCK_RCVTSTAMP) ||
2145 sock_flag(sk, SOCK_TIMESTAMPING_RX_SOFTWARE) ||
2146 (kt.tv64 && sock_flag(sk, SOCK_TIMESTAMPING_SOFTWARE)) ||
2147 (hwtstamps->hwtstamp.tv64 &&
2148 sock_flag(sk, SOCK_TIMESTAMPING_RAW_HARDWARE)) ||
2149 (hwtstamps->syststamp.tv64 &&
2150 sock_flag(sk, SOCK_TIMESTAMPING_SYS_HARDWARE)))
2151 __sock_recv_timestamp(msg, sk, skb);
2152 else
2153 sk->sk_stamp = kt;
2154
2155 if (sock_flag(sk, SOCK_WIFI_STATUS) && skb->wifi_acked_valid)
2156 __sock_recv_wifi_status(msg, sk, skb);
2157}
2158
2159extern void __sock_recv_ts_and_drops(struct msghdr *msg, struct sock *sk,
2160 struct sk_buff *skb);
2161
2162static inline void sock_recv_ts_and_drops(struct msghdr *msg, struct sock *sk,
2163 struct sk_buff *skb)
2164{
2165#define FLAGS_TS_OR_DROPS ((1UL << SOCK_RXQ_OVFL) | \
2166 (1UL << SOCK_RCVTSTAMP) | \
2167 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \
2168 (1UL << SOCK_TIMESTAMPING_SOFTWARE) | \
2169 (1UL << SOCK_TIMESTAMPING_RAW_HARDWARE) | \
2170 (1UL << SOCK_TIMESTAMPING_SYS_HARDWARE))
2171
2172 if (sk->sk_flags & FLAGS_TS_OR_DROPS)
2173 __sock_recv_ts_and_drops(msg, sk, skb);
2174 else
2175 sk->sk_stamp = skb->tstamp;
2176}
2177
2178
2179
2180
2181
2182
2183
2184
2185extern void sock_tx_timestamp(struct sock *sk, __u8 *tx_flags);
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196#ifdef CONFIG_NET_DMA
2197static inline void sk_eat_skb(struct sock *sk, struct sk_buff *skb, bool copied_early)
2198{
2199 __skb_unlink(skb, &sk->sk_receive_queue);
2200 if (!copied_early)
2201 __kfree_skb(skb);
2202 else
2203 __skb_queue_tail(&sk->sk_async_wait_queue, skb);
2204}
2205#else
2206static inline void sk_eat_skb(struct sock *sk, struct sk_buff *skb, bool copied_early)
2207{
2208 __skb_unlink(skb, &sk->sk_receive_queue);
2209 __kfree_skb(skb);
2210}
2211#endif
2212
2213static inline
2214struct net *sock_net(const struct sock *sk)
2215{
2216 return read_pnet(&sk->sk_net);
2217}
2218
2219static inline
2220void sock_net_set(struct sock *sk, struct net *net)
2221{
2222 write_pnet(&sk->sk_net, net);
2223}
2224
2225
2226
2227
2228
2229
2230
2231static inline void sk_change_net(struct sock *sk, struct net *net)
2232{
2233 put_net(sock_net(sk));
2234 sock_net_set(sk, hold_net(net));
2235}
2236
2237static inline struct sock *skb_steal_sock(struct sk_buff *skb)
2238{
2239 if (skb->sk) {
2240 struct sock *sk = skb->sk;
2241
2242 skb->destructor = NULL;
2243 skb->sk = NULL;
2244 return sk;
2245 }
2246 return NULL;
2247}
2248
2249extern void sock_enable_timestamp(struct sock *sk, int flag);
2250extern int sock_get_timestamp(struct sock *, struct timeval __user *);
2251extern int sock_get_timestampns(struct sock *, struct timespec __user *);
2252
2253
2254
2255
2256extern int net_msg_warn;
2257#define NETDEBUG(fmt, args...) \
2258 do { if (net_msg_warn) printk(fmt,##args); } while (0)
2259
2260#define LIMIT_NETDEBUG(fmt, args...) \
2261 do { if (net_msg_warn && net_ratelimit()) printk(fmt,##args); } while(0)
2262
2263extern __u32 sysctl_wmem_max;
2264extern __u32 sysctl_rmem_max;
2265
2266extern int sysctl_optmem_max;
2267
2268extern __u32 sysctl_wmem_default;
2269extern __u32 sysctl_rmem_default;
2270
2271#endif
2272