1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40#ifndef _SOCK_H
41#define _SOCK_H
42
43#include <linux/hardirq.h>
44#include <linux/kernel.h>
45#include <linux/list.h>
46#include <linux/list_nulls.h>
47#include <linux/timer.h>
48#include <linux/cache.h>
49#include <linux/bitops.h>
50#include <linux/lockdep.h>
51#include <linux/netdevice.h>
52#include <linux/skbuff.h>
53#include <linux/mm.h>
54#include <linux/security.h>
55#include <linux/slab.h>
56#include <linux/uaccess.h>
57#include <linux/memcontrol.h>
58#include <linux/res_counter.h>
59#include <linux/static_key.h>
60#include <linux/aio.h>
61#include <linux/sched.h>
62
63#include <linux/filter.h>
64#include <linux/rculist_nulls.h>
65#include <linux/poll.h>
66
67#include <linux/atomic.h>
68#include <net/dst.h>
69#include <net/checksum.h>
70
71struct cgroup;
72struct cgroup_subsys;
73#ifdef CONFIG_NET
74int mem_cgroup_sockets_init(struct mem_cgroup *memcg, struct cgroup_subsys *ss);
75void mem_cgroup_sockets_destroy(struct mem_cgroup *memcg);
76#else
77static inline
78int mem_cgroup_sockets_init(struct mem_cgroup *memcg, struct cgroup_subsys *ss)
79{
80 return 0;
81}
82static inline
83void mem_cgroup_sockets_destroy(struct mem_cgroup *memcg)
84{
85}
86#endif
87
88
89
90
91
92
93
94#define SOCK_DEBUGGING
95#ifdef SOCK_DEBUGGING
96#define SOCK_DEBUG(sk, msg...) do { if ((sk) && sock_flag((sk), SOCK_DBG)) \
97 printk(KERN_DEBUG msg); } while (0)
98#else
99
100static inline __printf(2, 3)
101void SOCK_DEBUG(const struct sock *sk, const char *msg, ...)
102{
103}
104#endif
105
106
107
108
109
110typedef struct {
111 spinlock_t slock;
112 int owned;
113 wait_queue_head_t wq;
114
115
116
117
118
119
120#ifdef CONFIG_DEBUG_LOCK_ALLOC
121 struct lockdep_map dep_map;
122#endif
123} socket_lock_t;
124
125struct sock;
126struct proto;
127struct net;
128
129typedef __u32 __bitwise __portpair;
130typedef __u64 __bitwise __addrpair;
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157struct sock_common {
158
159
160
161 union {
162 __addrpair skc_addrpair;
163 struct {
164 __be32 skc_daddr;
165 __be32 skc_rcv_saddr;
166 };
167 };
168 union {
169 unsigned int skc_hash;
170 __u16 skc_u16hashes[2];
171 };
172
173 union {
174 __portpair skc_portpair;
175 struct {
176 __be16 skc_dport;
177 __u16 skc_num;
178 };
179 };
180
181 unsigned short skc_family;
182 volatile unsigned char skc_state;
183 unsigned char skc_reuse:4;
184 unsigned char skc_reuseport:4;
185 int skc_bound_dev_if;
186 union {
187 struct hlist_node skc_bind_node;
188 struct hlist_nulls_node skc_portaddr_node;
189 };
190 struct proto *skc_prot;
191#ifdef CONFIG_NET_NS
192 struct net *skc_net;
193#endif
194
195
196
197
198
199 int skc_dontcopy_begin[0];
200
201 union {
202 struct hlist_node skc_node;
203 struct hlist_nulls_node skc_nulls_node;
204 };
205 int skc_tx_queue_mapping;
206 atomic_t skc_refcnt;
207
208 int skc_dontcopy_end[0];
209
210};
211
212struct cg_proto;
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285struct sock {
286
287
288
289
290 struct sock_common __sk_common;
291#define sk_node __sk_common.skc_node
292#define sk_nulls_node __sk_common.skc_nulls_node
293#define sk_refcnt __sk_common.skc_refcnt
294#define sk_tx_queue_mapping __sk_common.skc_tx_queue_mapping
295
296#define sk_dontcopy_begin __sk_common.skc_dontcopy_begin
297#define sk_dontcopy_end __sk_common.skc_dontcopy_end
298#define sk_hash __sk_common.skc_hash
299#define sk_family __sk_common.skc_family
300#define sk_state __sk_common.skc_state
301#define sk_reuse __sk_common.skc_reuse
302#define sk_reuseport __sk_common.skc_reuseport
303#define sk_bound_dev_if __sk_common.skc_bound_dev_if
304#define sk_bind_node __sk_common.skc_bind_node
305#define sk_prot __sk_common.skc_prot
306#define sk_net __sk_common.skc_net
307 socket_lock_t sk_lock;
308 struct sk_buff_head sk_receive_queue;
309
310
311
312
313
314
315
316
317 struct {
318 atomic_t rmem_alloc;
319 int len;
320 struct sk_buff *head;
321 struct sk_buff *tail;
322 } sk_backlog;
323#define sk_rmem_alloc sk_backlog.rmem_alloc
324 int sk_forward_alloc;
325#ifdef CONFIG_RPS
326 __u32 sk_rxhash;
327#endif
328 atomic_t sk_drops;
329 int sk_rcvbuf;
330
331 struct sk_filter __rcu *sk_filter;
332 struct socket_wq __rcu *sk_wq;
333
334#ifdef CONFIG_NET_DMA
335 struct sk_buff_head sk_async_wait_queue;
336#endif
337
338#ifdef CONFIG_XFRM
339 struct xfrm_policy *sk_policy[2];
340#endif
341 unsigned long sk_flags;
342 struct dst_entry *sk_rx_dst;
343 struct dst_entry __rcu *sk_dst_cache;
344 spinlock_t sk_dst_lock;
345 atomic_t sk_wmem_alloc;
346 atomic_t sk_omem_alloc;
347 int sk_sndbuf;
348 struct sk_buff_head sk_write_queue;
349 kmemcheck_bitfield_begin(flags);
350 unsigned int sk_shutdown : 2,
351 sk_no_check : 2,
352 sk_userlocks : 4,
353 sk_protocol : 8,
354 sk_type : 16;
355 kmemcheck_bitfield_end(flags);
356 int sk_wmem_queued;
357 gfp_t sk_allocation;
358 netdev_features_t sk_route_caps;
359 netdev_features_t sk_route_nocaps;
360 int sk_gso_type;
361 unsigned int sk_gso_max_size;
362 u16 sk_gso_max_segs;
363 int sk_rcvlowat;
364 unsigned long sk_lingertime;
365 struct sk_buff_head sk_error_queue;
366 struct proto *sk_prot_creator;
367 rwlock_t sk_callback_lock;
368 int sk_err,
369 sk_err_soft;
370 unsigned short sk_ack_backlog;
371 unsigned short sk_max_ack_backlog;
372 __u32 sk_priority;
373#if IS_ENABLED(CONFIG_NETPRIO_CGROUP)
374 __u32 sk_cgrp_prioidx;
375#endif
376 struct pid *sk_peer_pid;
377 const struct cred *sk_peer_cred;
378 long sk_rcvtimeo;
379 long sk_sndtimeo;
380 void *sk_protinfo;
381 struct timer_list sk_timer;
382 ktime_t sk_stamp;
383 struct socket *sk_socket;
384 void *sk_user_data;
385 struct page_frag sk_frag;
386 struct sk_buff *sk_send_head;
387 __s32 sk_peek_off;
388 int sk_write_pending;
389#ifdef CONFIG_SECURITY
390 void *sk_security;
391#endif
392 __u32 sk_mark;
393 u32 sk_classid;
394 struct cg_proto *sk_cgrp;
395 void (*sk_state_change)(struct sock *sk);
396 void (*sk_data_ready)(struct sock *sk, int bytes);
397 void (*sk_write_space)(struct sock *sk);
398 void (*sk_error_report)(struct sock *sk);
399 int (*sk_backlog_rcv)(struct sock *sk,
400 struct sk_buff *skb);
401 void (*sk_destruct)(struct sock *sk);
402};
403
404
405
406
407
408
409
410
411#define SK_NO_REUSE 0
412#define SK_CAN_REUSE 1
413#define SK_FORCE_REUSE 2
414
415static inline int sk_peek_offset(struct sock *sk, int flags)
416{
417 if ((flags & MSG_PEEK) && (sk->sk_peek_off >= 0))
418 return sk->sk_peek_off;
419 else
420 return 0;
421}
422
423static inline void sk_peek_offset_bwd(struct sock *sk, int val)
424{
425 if (sk->sk_peek_off >= 0) {
426 if (sk->sk_peek_off >= val)
427 sk->sk_peek_off -= val;
428 else
429 sk->sk_peek_off = 0;
430 }
431}
432
433static inline void sk_peek_offset_fwd(struct sock *sk, int val)
434{
435 if (sk->sk_peek_off >= 0)
436 sk->sk_peek_off += val;
437}
438
439
440
441
442static inline struct sock *sk_entry(const struct hlist_node *node)
443{
444 return hlist_entry(node, struct sock, sk_node);
445}
446
447static inline struct sock *__sk_head(const struct hlist_head *head)
448{
449 return hlist_entry(head->first, struct sock, sk_node);
450}
451
452static inline struct sock *sk_head(const struct hlist_head *head)
453{
454 return hlist_empty(head) ? NULL : __sk_head(head);
455}
456
457static inline struct sock *__sk_nulls_head(const struct hlist_nulls_head *head)
458{
459 return hlist_nulls_entry(head->first, struct sock, sk_nulls_node);
460}
461
462static inline struct sock *sk_nulls_head(const struct hlist_nulls_head *head)
463{
464 return hlist_nulls_empty(head) ? NULL : __sk_nulls_head(head);
465}
466
467static inline struct sock *sk_next(const struct sock *sk)
468{
469 return sk->sk_node.next ?
470 hlist_entry(sk->sk_node.next, struct sock, sk_node) : NULL;
471}
472
473static inline struct sock *sk_nulls_next(const struct sock *sk)
474{
475 return (!is_a_nulls(sk->sk_nulls_node.next)) ?
476 hlist_nulls_entry(sk->sk_nulls_node.next,
477 struct sock, sk_nulls_node) :
478 NULL;
479}
480
481static inline bool sk_unhashed(const struct sock *sk)
482{
483 return hlist_unhashed(&sk->sk_node);
484}
485
486static inline bool sk_hashed(const struct sock *sk)
487{
488 return !sk_unhashed(sk);
489}
490
491static inline void sk_node_init(struct hlist_node *node)
492{
493 node->pprev = NULL;
494}
495
496static inline void sk_nulls_node_init(struct hlist_nulls_node *node)
497{
498 node->pprev = NULL;
499}
500
501static inline void __sk_del_node(struct sock *sk)
502{
503 __hlist_del(&sk->sk_node);
504}
505
506
507static inline bool __sk_del_node_init(struct sock *sk)
508{
509 if (sk_hashed(sk)) {
510 __sk_del_node(sk);
511 sk_node_init(&sk->sk_node);
512 return true;
513 }
514 return false;
515}
516
517
518
519
520
521
522
523static inline void sock_hold(struct sock *sk)
524{
525 atomic_inc(&sk->sk_refcnt);
526}
527
528
529
530
531static inline void __sock_put(struct sock *sk)
532{
533 atomic_dec(&sk->sk_refcnt);
534}
535
536static inline bool sk_del_node_init(struct sock *sk)
537{
538 bool rc = __sk_del_node_init(sk);
539
540 if (rc) {
541
542 WARN_ON(atomic_read(&sk->sk_refcnt) == 1);
543 __sock_put(sk);
544 }
545 return rc;
546}
547#define sk_del_node_init_rcu(sk) sk_del_node_init(sk)
548
549static inline bool __sk_nulls_del_node_init_rcu(struct sock *sk)
550{
551 if (sk_hashed(sk)) {
552 hlist_nulls_del_init_rcu(&sk->sk_nulls_node);
553 return true;
554 }
555 return false;
556}
557
558static inline bool sk_nulls_del_node_init_rcu(struct sock *sk)
559{
560 bool rc = __sk_nulls_del_node_init_rcu(sk);
561
562 if (rc) {
563
564 WARN_ON(atomic_read(&sk->sk_refcnt) == 1);
565 __sock_put(sk);
566 }
567 return rc;
568}
569
570static inline void __sk_add_node(struct sock *sk, struct hlist_head *list)
571{
572 hlist_add_head(&sk->sk_node, list);
573}
574
575static inline void sk_add_node(struct sock *sk, struct hlist_head *list)
576{
577 sock_hold(sk);
578 __sk_add_node(sk, list);
579}
580
581static inline void sk_add_node_rcu(struct sock *sk, struct hlist_head *list)
582{
583 sock_hold(sk);
584 hlist_add_head_rcu(&sk->sk_node, list);
585}
586
587static inline void __sk_nulls_add_node_rcu(struct sock *sk, struct hlist_nulls_head *list)
588{
589 hlist_nulls_add_head_rcu(&sk->sk_nulls_node, list);
590}
591
592static inline void sk_nulls_add_node_rcu(struct sock *sk, struct hlist_nulls_head *list)
593{
594 sock_hold(sk);
595 __sk_nulls_add_node_rcu(sk, list);
596}
597
598static inline void __sk_del_bind_node(struct sock *sk)
599{
600 __hlist_del(&sk->sk_bind_node);
601}
602
603static inline void sk_add_bind_node(struct sock *sk,
604 struct hlist_head *list)
605{
606 hlist_add_head(&sk->sk_bind_node, list);
607}
608
609#define sk_for_each(__sk, list) \
610 hlist_for_each_entry(__sk, list, sk_node)
611#define sk_for_each_rcu(__sk, list) \
612 hlist_for_each_entry_rcu(__sk, list, sk_node)
613#define sk_nulls_for_each(__sk, node, list) \
614 hlist_nulls_for_each_entry(__sk, node, list, sk_nulls_node)
615#define sk_nulls_for_each_rcu(__sk, node, list) \
616 hlist_nulls_for_each_entry_rcu(__sk, node, list, sk_nulls_node)
617#define sk_for_each_from(__sk) \
618 hlist_for_each_entry_from(__sk, sk_node)
619#define sk_nulls_for_each_from(__sk, node) \
620 if (__sk && ({ node = &(__sk)->sk_nulls_node; 1; })) \
621 hlist_nulls_for_each_entry_from(__sk, node, sk_nulls_node)
622#define sk_for_each_safe(__sk, tmp, list) \
623 hlist_for_each_entry_safe(__sk, tmp, list, sk_node)
624#define sk_for_each_bound(__sk, list) \
625 hlist_for_each_entry(__sk, list, sk_bind_node)
626
627static inline struct user_namespace *sk_user_ns(struct sock *sk)
628{
629
630
631
632
633 return sk->sk_socket->file->f_cred->user_ns;
634}
635
636
637enum sock_flags {
638 SOCK_DEAD,
639 SOCK_DONE,
640 SOCK_URGINLINE,
641 SOCK_KEEPOPEN,
642 SOCK_LINGER,
643 SOCK_DESTROY,
644 SOCK_BROADCAST,
645 SOCK_TIMESTAMP,
646 SOCK_ZAPPED,
647 SOCK_USE_WRITE_QUEUE,
648 SOCK_DBG,
649 SOCK_RCVTSTAMP,
650 SOCK_RCVTSTAMPNS,
651 SOCK_LOCALROUTE,
652 SOCK_QUEUE_SHRUNK,
653 SOCK_MEMALLOC,
654 SOCK_TIMESTAMPING_TX_HARDWARE,
655 SOCK_TIMESTAMPING_TX_SOFTWARE,
656 SOCK_TIMESTAMPING_RX_HARDWARE,
657 SOCK_TIMESTAMPING_RX_SOFTWARE,
658 SOCK_TIMESTAMPING_SOFTWARE,
659 SOCK_TIMESTAMPING_RAW_HARDWARE,
660 SOCK_TIMESTAMPING_SYS_HARDWARE,
661 SOCK_FASYNC,
662 SOCK_RXQ_OVFL,
663 SOCK_ZEROCOPY,
664 SOCK_WIFI_STATUS,
665 SOCK_NOFCS,
666
667
668
669 SOCK_FILTER_LOCKED,
670 SOCK_SELECT_ERR_QUEUE,
671};
672
673static inline void sock_copy_flags(struct sock *nsk, struct sock *osk)
674{
675 nsk->sk_flags = osk->sk_flags;
676}
677
678static inline void sock_set_flag(struct sock *sk, enum sock_flags flag)
679{
680 __set_bit(flag, &sk->sk_flags);
681}
682
683static inline void sock_reset_flag(struct sock *sk, enum sock_flags flag)
684{
685 __clear_bit(flag, &sk->sk_flags);
686}
687
688static inline bool sock_flag(const struct sock *sk, enum sock_flags flag)
689{
690 return test_bit(flag, &sk->sk_flags);
691}
692
693#ifdef CONFIG_NET
694extern struct static_key memalloc_socks;
695static inline int sk_memalloc_socks(void)
696{
697 return static_key_false(&memalloc_socks);
698}
699#else
700
701static inline int sk_memalloc_socks(void)
702{
703 return 0;
704}
705
706#endif
707
708static inline gfp_t sk_gfp_atomic(struct sock *sk, gfp_t gfp_mask)
709{
710 return GFP_ATOMIC | (sk->sk_allocation & __GFP_MEMALLOC);
711}
712
713static inline void sk_acceptq_removed(struct sock *sk)
714{
715 sk->sk_ack_backlog--;
716}
717
718static inline void sk_acceptq_added(struct sock *sk)
719{
720 sk->sk_ack_backlog++;
721}
722
723static inline bool sk_acceptq_is_full(const struct sock *sk)
724{
725 return sk->sk_ack_backlog > sk->sk_max_ack_backlog;
726}
727
728
729
730
731static inline int sk_stream_min_wspace(const struct sock *sk)
732{
733 return sk->sk_wmem_queued >> 1;
734}
735
736static inline int sk_stream_wspace(const struct sock *sk)
737{
738 return sk->sk_sndbuf - sk->sk_wmem_queued;
739}
740
741extern void sk_stream_write_space(struct sock *sk);
742
743static inline bool sk_stream_memory_free(const struct sock *sk)
744{
745 return sk->sk_wmem_queued < sk->sk_sndbuf;
746}
747
748
749static inline void __sk_add_backlog(struct sock *sk, struct sk_buff *skb)
750{
751
752 skb_dst_force(skb);
753
754 if (!sk->sk_backlog.tail)
755 sk->sk_backlog.head = skb;
756 else
757 sk->sk_backlog.tail->next = skb;
758
759 sk->sk_backlog.tail = skb;
760 skb->next = NULL;
761}
762
763
764
765
766
767
768static inline bool sk_rcvqueues_full(const struct sock *sk, const struct sk_buff *skb,
769 unsigned int limit)
770{
771 unsigned int qsize = sk->sk_backlog.len + atomic_read(&sk->sk_rmem_alloc);
772
773 return qsize > limit;
774}
775
776
777static inline __must_check int sk_add_backlog(struct sock *sk, struct sk_buff *skb,
778 unsigned int limit)
779{
780 if (sk_rcvqueues_full(sk, skb, limit))
781 return -ENOBUFS;
782
783 __sk_add_backlog(sk, skb);
784 sk->sk_backlog.len += skb->truesize;
785 return 0;
786}
787
788extern int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb);
789
790static inline int sk_backlog_rcv(struct sock *sk, struct sk_buff *skb)
791{
792 if (sk_memalloc_socks() && skb_pfmemalloc(skb))
793 return __sk_backlog_rcv(sk, skb);
794
795 return sk->sk_backlog_rcv(sk, skb);
796}
797
798static inline void sock_rps_record_flow(const struct sock *sk)
799{
800#ifdef CONFIG_RPS
801 struct rps_sock_flow_table *sock_flow_table;
802
803 rcu_read_lock();
804 sock_flow_table = rcu_dereference(rps_sock_flow_table);
805 rps_record_sock_flow(sock_flow_table, sk->sk_rxhash);
806 rcu_read_unlock();
807#endif
808}
809
810static inline void sock_rps_reset_flow(const struct sock *sk)
811{
812#ifdef CONFIG_RPS
813 struct rps_sock_flow_table *sock_flow_table;
814
815 rcu_read_lock();
816 sock_flow_table = rcu_dereference(rps_sock_flow_table);
817 rps_reset_sock_flow(sock_flow_table, sk->sk_rxhash);
818 rcu_read_unlock();
819#endif
820}
821
822static inline void sock_rps_save_rxhash(struct sock *sk,
823 const struct sk_buff *skb)
824{
825#ifdef CONFIG_RPS
826 if (unlikely(sk->sk_rxhash != skb->rxhash)) {
827 sock_rps_reset_flow(sk);
828 sk->sk_rxhash = skb->rxhash;
829 }
830#endif
831}
832
833static inline void sock_rps_reset_rxhash(struct sock *sk)
834{
835#ifdef CONFIG_RPS
836 sock_rps_reset_flow(sk);
837 sk->sk_rxhash = 0;
838#endif
839}
840
841#define sk_wait_event(__sk, __timeo, __condition) \
842 ({ int __rc; \
843 release_sock(__sk); \
844 __rc = __condition; \
845 if (!__rc) { \
846 *(__timeo) = schedule_timeout(*(__timeo)); \
847 } \
848 lock_sock(__sk); \
849 __rc = __condition; \
850 __rc; \
851 })
852
853extern int sk_stream_wait_connect(struct sock *sk, long *timeo_p);
854extern int sk_stream_wait_memory(struct sock *sk, long *timeo_p);
855extern void sk_stream_wait_close(struct sock *sk, long timeo_p);
856extern int sk_stream_error(struct sock *sk, int flags, int err);
857extern void sk_stream_kill_queues(struct sock *sk);
858extern void sk_set_memalloc(struct sock *sk);
859extern void sk_clear_memalloc(struct sock *sk);
860
861extern int sk_wait_data(struct sock *sk, long *timeo);
862
863struct request_sock_ops;
864struct timewait_sock_ops;
865struct inet_hashinfo;
866struct raw_hashinfo;
867struct module;
868
869
870
871
872
873static inline void sk_prot_clear_nulls(struct sock *sk, int size)
874{
875 if (offsetof(struct sock, sk_node.next) != 0)
876 memset(sk, 0, offsetof(struct sock, sk_node.next));
877 memset(&sk->sk_node.pprev, 0,
878 size - offsetof(struct sock, sk_node.pprev));
879}
880
881
882
883
884
885struct proto {
886 void (*close)(struct sock *sk,
887 long timeout);
888 int (*connect)(struct sock *sk,
889 struct sockaddr *uaddr,
890 int addr_len);
891 int (*disconnect)(struct sock *sk, int flags);
892
893 struct sock * (*accept)(struct sock *sk, int flags, int *err);
894
895 int (*ioctl)(struct sock *sk, int cmd,
896 unsigned long arg);
897 int (*init)(struct sock *sk);
898 void (*destroy)(struct sock *sk);
899 void (*shutdown)(struct sock *sk, int how);
900 int (*setsockopt)(struct sock *sk, int level,
901 int optname, char __user *optval,
902 unsigned int optlen);
903 int (*getsockopt)(struct sock *sk, int level,
904 int optname, char __user *optval,
905 int __user *option);
906#ifdef CONFIG_COMPAT
907 int (*compat_setsockopt)(struct sock *sk,
908 int level,
909 int optname, char __user *optval,
910 unsigned int optlen);
911 int (*compat_getsockopt)(struct sock *sk,
912 int level,
913 int optname, char __user *optval,
914 int __user *option);
915 int (*compat_ioctl)(struct sock *sk,
916 unsigned int cmd, unsigned long arg);
917#endif
918 int (*sendmsg)(struct kiocb *iocb, struct sock *sk,
919 struct msghdr *msg, size_t len);
920 int (*recvmsg)(struct kiocb *iocb, struct sock *sk,
921 struct msghdr *msg,
922 size_t len, int noblock, int flags,
923 int *addr_len);
924 int (*sendpage)(struct sock *sk, struct page *page,
925 int offset, size_t size, int flags);
926 int (*bind)(struct sock *sk,
927 struct sockaddr *uaddr, int addr_len);
928
929 int (*backlog_rcv) (struct sock *sk,
930 struct sk_buff *skb);
931
932 void (*release_cb)(struct sock *sk);
933 void (*mtu_reduced)(struct sock *sk);
934
935
936 void (*hash)(struct sock *sk);
937 void (*unhash)(struct sock *sk);
938 void (*rehash)(struct sock *sk);
939 int (*get_port)(struct sock *sk, unsigned short snum);
940 void (*clear_sk)(struct sock *sk, int size);
941
942
943#ifdef CONFIG_PROC_FS
944 unsigned int inuse_idx;
945#endif
946
947
948 void (*enter_memory_pressure)(struct sock *sk);
949 atomic_long_t *memory_allocated;
950 struct percpu_counter *sockets_allocated;
951
952
953
954
955
956
957 int *memory_pressure;
958 long *sysctl_mem;
959 int *sysctl_wmem;
960 int *sysctl_rmem;
961 int max_header;
962 bool no_autobind;
963
964 struct kmem_cache *slab;
965 unsigned int obj_size;
966 int slab_flags;
967
968 struct percpu_counter *orphan_count;
969
970 struct request_sock_ops *rsk_prot;
971 struct timewait_sock_ops *twsk_prot;
972
973 union {
974 struct inet_hashinfo *hashinfo;
975 struct udp_table *udp_table;
976 struct raw_hashinfo *raw_hash;
977 } h;
978
979 struct module *owner;
980
981 char name[32];
982
983 struct list_head node;
984#ifdef SOCK_REFCNT_DEBUG
985 atomic_t socks;
986#endif
987#ifdef CONFIG_MEMCG_KMEM
988
989
990
991
992
993
994 int (*init_cgroup)(struct mem_cgroup *memcg,
995 struct cgroup_subsys *ss);
996 void (*destroy_cgroup)(struct mem_cgroup *memcg);
997 struct cg_proto *(*proto_cgroup)(struct mem_cgroup *memcg);
998#endif
999};
1000
1001
1002
1003
1004enum cg_proto_flags {
1005
1006 MEMCG_SOCK_ACTIVE,
1007
1008 MEMCG_SOCK_ACTIVATED,
1009};
1010
1011struct cg_proto {
1012 void (*enter_memory_pressure)(struct sock *sk);
1013 struct res_counter *memory_allocated;
1014 struct percpu_counter *sockets_allocated;
1015 int *memory_pressure;
1016 long *sysctl_mem;
1017 unsigned long flags;
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027 struct mem_cgroup *memcg;
1028};
1029
1030extern int proto_register(struct proto *prot, int alloc_slab);
1031extern void proto_unregister(struct proto *prot);
1032
1033static inline bool memcg_proto_active(struct cg_proto *cg_proto)
1034{
1035 return test_bit(MEMCG_SOCK_ACTIVE, &cg_proto->flags);
1036}
1037
1038static inline bool memcg_proto_activated(struct cg_proto *cg_proto)
1039{
1040 return test_bit(MEMCG_SOCK_ACTIVATED, &cg_proto->flags);
1041}
1042
1043#ifdef SOCK_REFCNT_DEBUG
1044static inline void sk_refcnt_debug_inc(struct sock *sk)
1045{
1046 atomic_inc(&sk->sk_prot->socks);
1047}
1048
1049static inline void sk_refcnt_debug_dec(struct sock *sk)
1050{
1051 atomic_dec(&sk->sk_prot->socks);
1052 printk(KERN_DEBUG "%s socket %p released, %d are still alive\n",
1053 sk->sk_prot->name, sk, atomic_read(&sk->sk_prot->socks));
1054}
1055
1056static inline void sk_refcnt_debug_release(const struct sock *sk)
1057{
1058 if (atomic_read(&sk->sk_refcnt) != 1)
1059 printk(KERN_DEBUG "Destruction of the %s socket %p delayed, refcnt=%d\n",
1060 sk->sk_prot->name, sk, atomic_read(&sk->sk_refcnt));
1061}
1062#else
1063#define sk_refcnt_debug_inc(sk) do { } while (0)
1064#define sk_refcnt_debug_dec(sk) do { } while (0)
1065#define sk_refcnt_debug_release(sk) do { } while (0)
1066#endif
1067
1068#if defined(CONFIG_MEMCG_KMEM) && defined(CONFIG_NET)
1069extern struct static_key memcg_socket_limit_enabled;
1070static inline struct cg_proto *parent_cg_proto(struct proto *proto,
1071 struct cg_proto *cg_proto)
1072{
1073 return proto->proto_cgroup(parent_mem_cgroup(cg_proto->memcg));
1074}
1075#define mem_cgroup_sockets_enabled static_key_false(&memcg_socket_limit_enabled)
1076#else
1077#define mem_cgroup_sockets_enabled 0
1078static inline struct cg_proto *parent_cg_proto(struct proto *proto,
1079 struct cg_proto *cg_proto)
1080{
1081 return NULL;
1082}
1083#endif
1084
1085
1086static inline bool sk_has_memory_pressure(const struct sock *sk)
1087{
1088 return sk->sk_prot->memory_pressure != NULL;
1089}
1090
1091static inline bool sk_under_memory_pressure(const struct sock *sk)
1092{
1093 if (!sk->sk_prot->memory_pressure)
1094 return false;
1095
1096 if (mem_cgroup_sockets_enabled && sk->sk_cgrp)
1097 return !!*sk->sk_cgrp->memory_pressure;
1098
1099 return !!*sk->sk_prot->memory_pressure;
1100}
1101
1102static inline void sk_leave_memory_pressure(struct sock *sk)
1103{
1104 int *memory_pressure = sk->sk_prot->memory_pressure;
1105
1106 if (!memory_pressure)
1107 return;
1108
1109 if (*memory_pressure)
1110 *memory_pressure = 0;
1111
1112 if (mem_cgroup_sockets_enabled && sk->sk_cgrp) {
1113 struct cg_proto *cg_proto = sk->sk_cgrp;
1114 struct proto *prot = sk->sk_prot;
1115
1116 for (; cg_proto; cg_proto = parent_cg_proto(prot, cg_proto))
1117 if (*cg_proto->memory_pressure)
1118 *cg_proto->memory_pressure = 0;
1119 }
1120
1121}
1122
1123static inline void sk_enter_memory_pressure(struct sock *sk)
1124{
1125 if (!sk->sk_prot->enter_memory_pressure)
1126 return;
1127
1128 if (mem_cgroup_sockets_enabled && sk->sk_cgrp) {
1129 struct cg_proto *cg_proto = sk->sk_cgrp;
1130 struct proto *prot = sk->sk_prot;
1131
1132 for (; cg_proto; cg_proto = parent_cg_proto(prot, cg_proto))
1133 cg_proto->enter_memory_pressure(sk);
1134 }
1135
1136 sk->sk_prot->enter_memory_pressure(sk);
1137}
1138
1139static inline long sk_prot_mem_limits(const struct sock *sk, int index)
1140{
1141 long *prot = sk->sk_prot->sysctl_mem;
1142 if (mem_cgroup_sockets_enabled && sk->sk_cgrp)
1143 prot = sk->sk_cgrp->sysctl_mem;
1144 return prot[index];
1145}
1146
1147static inline void memcg_memory_allocated_add(struct cg_proto *prot,
1148 unsigned long amt,
1149 int *parent_status)
1150{
1151 struct res_counter *fail;
1152 int ret;
1153
1154 ret = res_counter_charge_nofail(prot->memory_allocated,
1155 amt << PAGE_SHIFT, &fail);
1156 if (ret < 0)
1157 *parent_status = OVER_LIMIT;
1158}
1159
1160static inline void memcg_memory_allocated_sub(struct cg_proto *prot,
1161 unsigned long amt)
1162{
1163 res_counter_uncharge(prot->memory_allocated, amt << PAGE_SHIFT);
1164}
1165
1166static inline u64 memcg_memory_allocated_read(struct cg_proto *prot)
1167{
1168 u64 ret;
1169 ret = res_counter_read_u64(prot->memory_allocated, RES_USAGE);
1170 return ret >> PAGE_SHIFT;
1171}
1172
1173static inline long
1174sk_memory_allocated(const struct sock *sk)
1175{
1176 struct proto *prot = sk->sk_prot;
1177 if (mem_cgroup_sockets_enabled && sk->sk_cgrp)
1178 return memcg_memory_allocated_read(sk->sk_cgrp);
1179
1180 return atomic_long_read(prot->memory_allocated);
1181}
1182
1183static inline long
1184sk_memory_allocated_add(struct sock *sk, int amt, int *parent_status)
1185{
1186 struct proto *prot = sk->sk_prot;
1187
1188 if (mem_cgroup_sockets_enabled && sk->sk_cgrp) {
1189 memcg_memory_allocated_add(sk->sk_cgrp, amt, parent_status);
1190
1191 atomic_long_add_return(amt, prot->memory_allocated);
1192 return memcg_memory_allocated_read(sk->sk_cgrp);
1193 }
1194
1195 return atomic_long_add_return(amt, prot->memory_allocated);
1196}
1197
1198static inline void
1199sk_memory_allocated_sub(struct sock *sk, int amt)
1200{
1201 struct proto *prot = sk->sk_prot;
1202
1203 if (mem_cgroup_sockets_enabled && sk->sk_cgrp)
1204 memcg_memory_allocated_sub(sk->sk_cgrp, amt);
1205
1206 atomic_long_sub(amt, prot->memory_allocated);
1207}
1208
1209static inline void sk_sockets_allocated_dec(struct sock *sk)
1210{
1211 struct proto *prot = sk->sk_prot;
1212
1213 if (mem_cgroup_sockets_enabled && sk->sk_cgrp) {
1214 struct cg_proto *cg_proto = sk->sk_cgrp;
1215
1216 for (; cg_proto; cg_proto = parent_cg_proto(prot, cg_proto))
1217 percpu_counter_dec(cg_proto->sockets_allocated);
1218 }
1219
1220 percpu_counter_dec(prot->sockets_allocated);
1221}
1222
1223static inline void sk_sockets_allocated_inc(struct sock *sk)
1224{
1225 struct proto *prot = sk->sk_prot;
1226
1227 if (mem_cgroup_sockets_enabled && sk->sk_cgrp) {
1228 struct cg_proto *cg_proto = sk->sk_cgrp;
1229
1230 for (; cg_proto; cg_proto = parent_cg_proto(prot, cg_proto))
1231 percpu_counter_inc(cg_proto->sockets_allocated);
1232 }
1233
1234 percpu_counter_inc(prot->sockets_allocated);
1235}
1236
1237static inline int
1238sk_sockets_allocated_read_positive(struct sock *sk)
1239{
1240 struct proto *prot = sk->sk_prot;
1241
1242 if (mem_cgroup_sockets_enabled && sk->sk_cgrp)
1243 return percpu_counter_read_positive(sk->sk_cgrp->sockets_allocated);
1244
1245 return percpu_counter_read_positive(prot->sockets_allocated);
1246}
1247
1248static inline int
1249proto_sockets_allocated_sum_positive(struct proto *prot)
1250{
1251 return percpu_counter_sum_positive(prot->sockets_allocated);
1252}
1253
1254static inline long
1255proto_memory_allocated(struct proto *prot)
1256{
1257 return atomic_long_read(prot->memory_allocated);
1258}
1259
1260static inline bool
1261proto_memory_pressure(struct proto *prot)
1262{
1263 if (!prot->memory_pressure)
1264 return false;
1265 return !!*prot->memory_pressure;
1266}
1267
1268
1269#ifdef CONFIG_PROC_FS
1270
1271extern void sock_prot_inuse_add(struct net *net, struct proto *prot, int inc);
1272extern int sock_prot_inuse_get(struct net *net, struct proto *proto);
1273#else
1274static inline void sock_prot_inuse_add(struct net *net, struct proto *prot,
1275 int inc)
1276{
1277}
1278#endif
1279
1280
1281
1282
1283
1284static inline void __sk_prot_rehash(struct sock *sk)
1285{
1286 sk->sk_prot->unhash(sk);
1287 sk->sk_prot->hash(sk);
1288}
1289
1290void sk_prot_clear_portaddr_nulls(struct sock *sk, int size);
1291
1292
1293#define SOCK_DESTROY_TIME (10*HZ)
1294
1295
1296#define PROT_SOCK 1024
1297
1298#define SHUTDOWN_MASK 3
1299#define RCV_SHUTDOWN 1
1300#define SEND_SHUTDOWN 2
1301
1302#define SOCK_SNDBUF_LOCK 1
1303#define SOCK_RCVBUF_LOCK 2
1304#define SOCK_BINDADDR_LOCK 4
1305#define SOCK_BINDPORT_LOCK 8
1306
1307
1308struct sock_iocb {
1309 struct list_head list;
1310
1311 int flags;
1312 int size;
1313 struct socket *sock;
1314 struct sock *sk;
1315 struct scm_cookie *scm;
1316 struct msghdr *msg, async_msg;
1317 struct kiocb *kiocb;
1318};
1319
1320static inline struct sock_iocb *kiocb_to_siocb(struct kiocb *iocb)
1321{
1322 return (struct sock_iocb *)iocb->private;
1323}
1324
1325static inline struct kiocb *siocb_to_kiocb(struct sock_iocb *si)
1326{
1327 return si->kiocb;
1328}
1329
1330struct socket_alloc {
1331 struct socket socket;
1332 struct inode vfs_inode;
1333};
1334
1335static inline struct socket *SOCKET_I(struct inode *inode)
1336{
1337 return &container_of(inode, struct socket_alloc, vfs_inode)->socket;
1338}
1339
1340static inline struct inode *SOCK_INODE(struct socket *socket)
1341{
1342 return &container_of(socket, struct socket_alloc, socket)->vfs_inode;
1343}
1344
1345
1346
1347
1348extern int __sk_mem_schedule(struct sock *sk, int size, int kind);
1349extern void __sk_mem_reclaim(struct sock *sk);
1350
1351#define SK_MEM_QUANTUM ((int)PAGE_SIZE)
1352#define SK_MEM_QUANTUM_SHIFT ilog2(SK_MEM_QUANTUM)
1353#define SK_MEM_SEND 0
1354#define SK_MEM_RECV 1
1355
1356static inline int sk_mem_pages(int amt)
1357{
1358 return (amt + SK_MEM_QUANTUM - 1) >> SK_MEM_QUANTUM_SHIFT;
1359}
1360
1361static inline bool sk_has_account(struct sock *sk)
1362{
1363
1364 return !!sk->sk_prot->memory_allocated;
1365}
1366
1367static inline bool sk_wmem_schedule(struct sock *sk, int size)
1368{
1369 if (!sk_has_account(sk))
1370 return true;
1371 return size <= sk->sk_forward_alloc ||
1372 __sk_mem_schedule(sk, size, SK_MEM_SEND);
1373}
1374
1375static inline bool
1376sk_rmem_schedule(struct sock *sk, struct sk_buff *skb, int size)
1377{
1378 if (!sk_has_account(sk))
1379 return true;
1380 return size<= sk->sk_forward_alloc ||
1381 __sk_mem_schedule(sk, size, SK_MEM_RECV) ||
1382 skb_pfmemalloc(skb);
1383}
1384
1385static inline void sk_mem_reclaim(struct sock *sk)
1386{
1387 if (!sk_has_account(sk))
1388 return;
1389 if (sk->sk_forward_alloc >= SK_MEM_QUANTUM)
1390 __sk_mem_reclaim(sk);
1391}
1392
1393static inline void sk_mem_reclaim_partial(struct sock *sk)
1394{
1395 if (!sk_has_account(sk))
1396 return;
1397 if (sk->sk_forward_alloc > SK_MEM_QUANTUM)
1398 __sk_mem_reclaim(sk);
1399}
1400
1401static inline void sk_mem_charge(struct sock *sk, int size)
1402{
1403 if (!sk_has_account(sk))
1404 return;
1405 sk->sk_forward_alloc -= size;
1406}
1407
1408static inline void sk_mem_uncharge(struct sock *sk, int size)
1409{
1410 if (!sk_has_account(sk))
1411 return;
1412 sk->sk_forward_alloc += size;
1413}
1414
1415static inline void sk_wmem_free_skb(struct sock *sk, struct sk_buff *skb)
1416{
1417 sock_set_flag(sk, SOCK_QUEUE_SHRUNK);
1418 sk->sk_wmem_queued -= skb->truesize;
1419 sk_mem_uncharge(sk, skb->truesize);
1420 __kfree_skb(skb);
1421}
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436#define sock_owned_by_user(sk) ((sk)->sk_lock.owned)
1437
1438
1439
1440
1441
1442
1443
1444
1445#define sock_lock_init_class_and_name(sk, sname, skey, name, key) \
1446do { \
1447 sk->sk_lock.owned = 0; \
1448 init_waitqueue_head(&sk->sk_lock.wq); \
1449 spin_lock_init(&(sk)->sk_lock.slock); \
1450 debug_check_no_locks_freed((void *)&(sk)->sk_lock, \
1451 sizeof((sk)->sk_lock)); \
1452 lockdep_set_class_and_name(&(sk)->sk_lock.slock, \
1453 (skey), (sname)); \
1454 lockdep_init_map(&(sk)->sk_lock.dep_map, (name), (key), 0); \
1455} while (0)
1456
1457extern void lock_sock_nested(struct sock *sk, int subclass);
1458
1459static inline void lock_sock(struct sock *sk)
1460{
1461 lock_sock_nested(sk, 0);
1462}
1463
1464extern void release_sock(struct sock *sk);
1465
1466
1467#define bh_lock_sock(__sk) spin_lock(&((__sk)->sk_lock.slock))
1468#define bh_lock_sock_nested(__sk) \
1469 spin_lock_nested(&((__sk)->sk_lock.slock), \
1470 SINGLE_DEPTH_NESTING)
1471#define bh_unlock_sock(__sk) spin_unlock(&((__sk)->sk_lock.slock))
1472
1473extern bool lock_sock_fast(struct sock *sk);
1474
1475
1476
1477
1478
1479
1480
1481
1482static inline void unlock_sock_fast(struct sock *sk, bool slow)
1483{
1484 if (slow)
1485 release_sock(sk);
1486 else
1487 spin_unlock_bh(&sk->sk_lock.slock);
1488}
1489
1490
1491extern struct sock *sk_alloc(struct net *net, int family,
1492 gfp_t priority,
1493 struct proto *prot);
1494extern void sk_free(struct sock *sk);
1495extern void sk_release_kernel(struct sock *sk);
1496extern struct sock *sk_clone_lock(const struct sock *sk,
1497 const gfp_t priority);
1498
1499extern struct sk_buff *sock_wmalloc(struct sock *sk,
1500 unsigned long size, int force,
1501 gfp_t priority);
1502extern struct sk_buff *sock_rmalloc(struct sock *sk,
1503 unsigned long size, int force,
1504 gfp_t priority);
1505extern void sock_wfree(struct sk_buff *skb);
1506extern void sock_rfree(struct sk_buff *skb);
1507extern void sock_edemux(struct sk_buff *skb);
1508
1509extern int sock_setsockopt(struct socket *sock, int level,
1510 int op, char __user *optval,
1511 unsigned int optlen);
1512
1513extern int sock_getsockopt(struct socket *sock, int level,
1514 int op, char __user *optval,
1515 int __user *optlen);
1516extern struct sk_buff *sock_alloc_send_skb(struct sock *sk,
1517 unsigned long size,
1518 int noblock,
1519 int *errcode);
1520extern struct sk_buff *sock_alloc_send_pskb(struct sock *sk,
1521 unsigned long header_len,
1522 unsigned long data_len,
1523 int noblock,
1524 int *errcode);
1525extern void *sock_kmalloc(struct sock *sk, int size,
1526 gfp_t priority);
1527extern void sock_kfree_s(struct sock *sk, void *mem, int size);
1528extern void sk_send_sigurg(struct sock *sk);
1529
1530
1531
1532
1533
1534extern int sock_no_bind(struct socket *,
1535 struct sockaddr *, int);
1536extern int sock_no_connect(struct socket *,
1537 struct sockaddr *, int, int);
1538extern int sock_no_socketpair(struct socket *,
1539 struct socket *);
1540extern int sock_no_accept(struct socket *,
1541 struct socket *, int);
1542extern int sock_no_getname(struct socket *,
1543 struct sockaddr *, int *, int);
1544extern unsigned int sock_no_poll(struct file *, struct socket *,
1545 struct poll_table_struct *);
1546extern int sock_no_ioctl(struct socket *, unsigned int,
1547 unsigned long);
1548extern int sock_no_listen(struct socket *, int);
1549extern int sock_no_shutdown(struct socket *, int);
1550extern int sock_no_getsockopt(struct socket *, int , int,
1551 char __user *, int __user *);
1552extern int sock_no_setsockopt(struct socket *, int, int,
1553 char __user *, unsigned int);
1554extern int sock_no_sendmsg(struct kiocb *, struct socket *,
1555 struct msghdr *, size_t);
1556extern int sock_no_recvmsg(struct kiocb *, struct socket *,
1557 struct msghdr *, size_t, int);
1558extern int sock_no_mmap(struct file *file,
1559 struct socket *sock,
1560 struct vm_area_struct *vma);
1561extern ssize_t sock_no_sendpage(struct socket *sock,
1562 struct page *page,
1563 int offset, size_t size,
1564 int flags);
1565
1566
1567
1568
1569
1570extern int sock_common_getsockopt(struct socket *sock, int level, int optname,
1571 char __user *optval, int __user *optlen);
1572extern int sock_common_recvmsg(struct kiocb *iocb, struct socket *sock,
1573 struct msghdr *msg, size_t size, int flags);
1574extern int sock_common_setsockopt(struct socket *sock, int level, int optname,
1575 char __user *optval, unsigned int optlen);
1576extern int compat_sock_common_getsockopt(struct socket *sock, int level,
1577 int optname, char __user *optval, int __user *optlen);
1578extern int compat_sock_common_setsockopt(struct socket *sock, int level,
1579 int optname, char __user *optval, unsigned int optlen);
1580
1581extern void sk_common_release(struct sock *sk);
1582
1583
1584
1585
1586
1587
1588extern void sock_init_data(struct socket *sock, struct sock *sk);
1589
1590extern void sk_filter_release_rcu(struct rcu_head *rcu);
1591
1592
1593
1594
1595
1596
1597
1598
1599static inline void sk_filter_release(struct sk_filter *fp)
1600{
1601 if (atomic_dec_and_test(&fp->refcnt))
1602 call_rcu(&fp->rcu, sk_filter_release_rcu);
1603}
1604
1605static inline void sk_filter_uncharge(struct sock *sk, struct sk_filter *fp)
1606{
1607 unsigned int size = sk_filter_len(fp);
1608
1609 atomic_sub(size, &sk->sk_omem_alloc);
1610 sk_filter_release(fp);
1611}
1612
1613static inline void sk_filter_charge(struct sock *sk, struct sk_filter *fp)
1614{
1615 atomic_inc(&fp->refcnt);
1616 atomic_add(sk_filter_len(fp), &sk->sk_omem_alloc);
1617}
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645static inline void sock_put(struct sock *sk)
1646{
1647 if (atomic_dec_and_test(&sk->sk_refcnt))
1648 sk_free(sk);
1649}
1650
1651extern int sk_receive_skb(struct sock *sk, struct sk_buff *skb,
1652 const int nested);
1653
1654static inline void sk_tx_queue_set(struct sock *sk, int tx_queue)
1655{
1656 sk->sk_tx_queue_mapping = tx_queue;
1657}
1658
1659static inline void sk_tx_queue_clear(struct sock *sk)
1660{
1661 sk->sk_tx_queue_mapping = -1;
1662}
1663
1664static inline int sk_tx_queue_get(const struct sock *sk)
1665{
1666 return sk ? sk->sk_tx_queue_mapping : -1;
1667}
1668
1669static inline void sk_set_socket(struct sock *sk, struct socket *sock)
1670{
1671 sk_tx_queue_clear(sk);
1672 sk->sk_socket = sock;
1673}
1674
1675static inline wait_queue_head_t *sk_sleep(struct sock *sk)
1676{
1677 BUILD_BUG_ON(offsetof(struct socket_wq, wait) != 0);
1678 return &rcu_dereference_raw(sk->sk_wq)->wait;
1679}
1680
1681
1682
1683
1684
1685
1686
1687static inline void sock_orphan(struct sock *sk)
1688{
1689 write_lock_bh(&sk->sk_callback_lock);
1690 sock_set_flag(sk, SOCK_DEAD);
1691 sk_set_socket(sk, NULL);
1692 sk->sk_wq = NULL;
1693 write_unlock_bh(&sk->sk_callback_lock);
1694}
1695
1696static inline void sock_graft(struct sock *sk, struct socket *parent)
1697{
1698 write_lock_bh(&sk->sk_callback_lock);
1699 sk->sk_wq = parent->wq;
1700 parent->sk = sk;
1701 sk_set_socket(sk, parent);
1702 security_sock_graft(sk, parent);
1703 write_unlock_bh(&sk->sk_callback_lock);
1704}
1705
1706extern kuid_t sock_i_uid(struct sock *sk);
1707extern unsigned long sock_i_ino(struct sock *sk);
1708
1709static inline struct dst_entry *
1710__sk_dst_get(struct sock *sk)
1711{
1712 return rcu_dereference_check(sk->sk_dst_cache, sock_owned_by_user(sk) ||
1713 lockdep_is_held(&sk->sk_lock.slock));
1714}
1715
1716static inline struct dst_entry *
1717sk_dst_get(struct sock *sk)
1718{
1719 struct dst_entry *dst;
1720
1721 rcu_read_lock();
1722 dst = rcu_dereference(sk->sk_dst_cache);
1723 if (dst)
1724 dst_hold(dst);
1725 rcu_read_unlock();
1726 return dst;
1727}
1728
1729extern void sk_reset_txq(struct sock *sk);
1730
1731static inline void dst_negative_advice(struct sock *sk)
1732{
1733 struct dst_entry *ndst, *dst = __sk_dst_get(sk);
1734
1735 if (dst && dst->ops->negative_advice) {
1736 ndst = dst->ops->negative_advice(dst);
1737
1738 if (ndst != dst) {
1739 rcu_assign_pointer(sk->sk_dst_cache, ndst);
1740 sk_reset_txq(sk);
1741 }
1742 }
1743}
1744
1745static inline void
1746__sk_dst_set(struct sock *sk, struct dst_entry *dst)
1747{
1748 struct dst_entry *old_dst;
1749
1750 sk_tx_queue_clear(sk);
1751
1752
1753
1754
1755 old_dst = rcu_dereference_raw(sk->sk_dst_cache);
1756 rcu_assign_pointer(sk->sk_dst_cache, dst);
1757 dst_release(old_dst);
1758}
1759
1760static inline void
1761sk_dst_set(struct sock *sk, struct dst_entry *dst)
1762{
1763 spin_lock(&sk->sk_dst_lock);
1764 __sk_dst_set(sk, dst);
1765 spin_unlock(&sk->sk_dst_lock);
1766}
1767
1768static inline void
1769__sk_dst_reset(struct sock *sk)
1770{
1771 __sk_dst_set(sk, NULL);
1772}
1773
1774static inline void
1775sk_dst_reset(struct sock *sk)
1776{
1777 spin_lock(&sk->sk_dst_lock);
1778 __sk_dst_reset(sk);
1779 spin_unlock(&sk->sk_dst_lock);
1780}
1781
1782extern struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie);
1783
1784extern struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie);
1785
1786static inline bool sk_can_gso(const struct sock *sk)
1787{
1788 return net_gso_ok(sk->sk_route_caps, sk->sk_gso_type);
1789}
1790
1791extern void sk_setup_caps(struct sock *sk, struct dst_entry *dst);
1792
1793static inline void sk_nocaps_add(struct sock *sk, netdev_features_t flags)
1794{
1795 sk->sk_route_nocaps |= flags;
1796 sk->sk_route_caps &= ~flags;
1797}
1798
1799static inline int skb_do_copy_data_nocache(struct sock *sk, struct sk_buff *skb,
1800 char __user *from, char *to,
1801 int copy, int offset)
1802{
1803 if (skb->ip_summed == CHECKSUM_NONE) {
1804 int err = 0;
1805 __wsum csum = csum_and_copy_from_user(from, to, copy, 0, &err);
1806 if (err)
1807 return err;
1808 skb->csum = csum_block_add(skb->csum, csum, offset);
1809 } else if (sk->sk_route_caps & NETIF_F_NOCACHE_COPY) {
1810 if (!access_ok(VERIFY_READ, from, copy) ||
1811 __copy_from_user_nocache(to, from, copy))
1812 return -EFAULT;
1813 } else if (copy_from_user(to, from, copy))
1814 return -EFAULT;
1815
1816 return 0;
1817}
1818
1819static inline int skb_add_data_nocache(struct sock *sk, struct sk_buff *skb,
1820 char __user *from, int copy)
1821{
1822 int err, offset = skb->len;
1823
1824 err = skb_do_copy_data_nocache(sk, skb, from, skb_put(skb, copy),
1825 copy, offset);
1826 if (err)
1827 __skb_trim(skb, offset);
1828
1829 return err;
1830}
1831
1832static inline int skb_copy_to_page_nocache(struct sock *sk, char __user *from,
1833 struct sk_buff *skb,
1834 struct page *page,
1835 int off, int copy)
1836{
1837 int err;
1838
1839 err = skb_do_copy_data_nocache(sk, skb, from, page_address(page) + off,
1840 copy, skb->len);
1841 if (err)
1842 return err;
1843
1844 skb->len += copy;
1845 skb->data_len += copy;
1846 skb->truesize += copy;
1847 sk->sk_wmem_queued += copy;
1848 sk_mem_charge(sk, copy);
1849 return 0;
1850}
1851
1852static inline int skb_copy_to_page(struct sock *sk, char __user *from,
1853 struct sk_buff *skb, struct page *page,
1854 int off, int copy)
1855{
1856 if (skb->ip_summed == CHECKSUM_NONE) {
1857 int err = 0;
1858 __wsum csum = csum_and_copy_from_user(from,
1859 page_address(page) + off,
1860 copy, 0, &err);
1861 if (err)
1862 return err;
1863 skb->csum = csum_block_add(skb->csum, csum, skb->len);
1864 } else if (copy_from_user(page_address(page) + off, from, copy))
1865 return -EFAULT;
1866
1867 skb->len += copy;
1868 skb->data_len += copy;
1869 skb->truesize += copy;
1870 sk->sk_wmem_queued += copy;
1871 sk_mem_charge(sk, copy);
1872 return 0;
1873}
1874
1875
1876
1877
1878
1879
1880
1881static inline int sk_wmem_alloc_get(const struct sock *sk)
1882{
1883 return atomic_read(&sk->sk_wmem_alloc) - 1;
1884}
1885
1886
1887
1888
1889
1890
1891
1892static inline int sk_rmem_alloc_get(const struct sock *sk)
1893{
1894 return atomic_read(&sk->sk_rmem_alloc);
1895}
1896
1897
1898
1899
1900
1901
1902
1903static inline bool sk_has_allocations(const struct sock *sk)
1904{
1905 return sk_wmem_alloc_get(sk) || sk_rmem_alloc_get(sk);
1906}
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940static inline bool wq_has_sleeper(struct socket_wq *wq)
1941{
1942
1943
1944
1945
1946
1947 smp_mb();
1948 return wq && waitqueue_active(&wq->wait);
1949}
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959static inline void sock_poll_wait(struct file *filp,
1960 wait_queue_head_t *wait_address, poll_table *p)
1961{
1962 if (!poll_does_not_wait(p) && wait_address) {
1963 poll_wait(filp, wait_address, p);
1964
1965
1966
1967
1968
1969 smp_mb();
1970 }
1971}
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982static inline void skb_set_owner_w(struct sk_buff *skb, struct sock *sk)
1983{
1984 skb_orphan(skb);
1985 skb->sk = sk;
1986 skb->destructor = sock_wfree;
1987
1988
1989
1990
1991
1992 atomic_add(skb->truesize, &sk->sk_wmem_alloc);
1993}
1994
1995static inline void skb_set_owner_r(struct sk_buff *skb, struct sock *sk)
1996{
1997 skb_orphan(skb);
1998 skb->sk = sk;
1999 skb->destructor = sock_rfree;
2000 atomic_add(skb->truesize, &sk->sk_rmem_alloc);
2001 sk_mem_charge(sk, skb->truesize);
2002}
2003
2004extern void sk_reset_timer(struct sock *sk, struct timer_list *timer,
2005 unsigned long expires);
2006
2007extern void sk_stop_timer(struct sock *sk, struct timer_list *timer);
2008
2009extern int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb);
2010
2011extern int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb);
2012
2013
2014
2015
2016
2017static inline int sock_error(struct sock *sk)
2018{
2019 int err;
2020 if (likely(!sk->sk_err))
2021 return 0;
2022 err = xchg(&sk->sk_err, 0);
2023 return -err;
2024}
2025
2026static inline unsigned long sock_wspace(struct sock *sk)
2027{
2028 int amt = 0;
2029
2030 if (!(sk->sk_shutdown & SEND_SHUTDOWN)) {
2031 amt = sk->sk_sndbuf - atomic_read(&sk->sk_wmem_alloc);
2032 if (amt < 0)
2033 amt = 0;
2034 }
2035 return amt;
2036}
2037
2038static inline void sk_wake_async(struct sock *sk, int how, int band)
2039{
2040 if (sock_flag(sk, SOCK_FASYNC))
2041 sock_wake_async(sk->sk_socket, how, band);
2042}
2043
2044#define SOCK_MIN_SNDBUF 2048
2045
2046
2047
2048
2049#define SOCK_MIN_RCVBUF (2048 + sizeof(struct sk_buff))
2050
2051static inline void sk_stream_moderate_sndbuf(struct sock *sk)
2052{
2053 if (!(sk->sk_userlocks & SOCK_SNDBUF_LOCK)) {
2054 sk->sk_sndbuf = min(sk->sk_sndbuf, sk->sk_wmem_queued >> 1);
2055 sk->sk_sndbuf = max(sk->sk_sndbuf, SOCK_MIN_SNDBUF);
2056 }
2057}
2058
2059struct sk_buff *sk_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp);
2060
2061
2062
2063
2064
2065
2066
2067
2068static inline struct page_frag *sk_page_frag(struct sock *sk)
2069{
2070 if (sk->sk_allocation & __GFP_WAIT)
2071 return ¤t->task_frag;
2072
2073 return &sk->sk_frag;
2074}
2075
2076extern bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag);
2077
2078
2079
2080
2081static inline bool sock_writeable(const struct sock *sk)
2082{
2083 return atomic_read(&sk->sk_wmem_alloc) < (sk->sk_sndbuf >> 1);
2084}
2085
2086static inline gfp_t gfp_any(void)
2087{
2088 return in_softirq() ? GFP_ATOMIC : GFP_KERNEL;
2089}
2090
2091static inline long sock_rcvtimeo(const struct sock *sk, bool noblock)
2092{
2093 return noblock ? 0 : sk->sk_rcvtimeo;
2094}
2095
2096static inline long sock_sndtimeo(const struct sock *sk, bool noblock)
2097{
2098 return noblock ? 0 : sk->sk_sndtimeo;
2099}
2100
2101static inline int sock_rcvlowat(const struct sock *sk, int waitall, int len)
2102{
2103 return (waitall ? len : min_t(int, sk->sk_rcvlowat, len)) ? : 1;
2104}
2105
2106
2107
2108
2109static inline int sock_intr_errno(long timeo)
2110{
2111 return timeo == MAX_SCHEDULE_TIMEOUT ? -ERESTARTSYS : -EINTR;
2112}
2113
2114extern void __sock_recv_timestamp(struct msghdr *msg, struct sock *sk,
2115 struct sk_buff *skb);
2116extern void __sock_recv_wifi_status(struct msghdr *msg, struct sock *sk,
2117 struct sk_buff *skb);
2118
2119static inline void
2120sock_recv_timestamp(struct msghdr *msg, struct sock *sk, struct sk_buff *skb)
2121{
2122 ktime_t kt = skb->tstamp;
2123 struct skb_shared_hwtstamps *hwtstamps = skb_hwtstamps(skb);
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135 if (sock_flag(sk, SOCK_RCVTSTAMP) ||
2136 sock_flag(sk, SOCK_TIMESTAMPING_RX_SOFTWARE) ||
2137 (kt.tv64 && sock_flag(sk, SOCK_TIMESTAMPING_SOFTWARE)) ||
2138 (hwtstamps->hwtstamp.tv64 &&
2139 sock_flag(sk, SOCK_TIMESTAMPING_RAW_HARDWARE)) ||
2140 (hwtstamps->syststamp.tv64 &&
2141 sock_flag(sk, SOCK_TIMESTAMPING_SYS_HARDWARE)))
2142 __sock_recv_timestamp(msg, sk, skb);
2143 else
2144 sk->sk_stamp = kt;
2145
2146 if (sock_flag(sk, SOCK_WIFI_STATUS) && skb->wifi_acked_valid)
2147 __sock_recv_wifi_status(msg, sk, skb);
2148}
2149
2150extern void __sock_recv_ts_and_drops(struct msghdr *msg, struct sock *sk,
2151 struct sk_buff *skb);
2152
2153static inline void sock_recv_ts_and_drops(struct msghdr *msg, struct sock *sk,
2154 struct sk_buff *skb)
2155{
2156#define FLAGS_TS_OR_DROPS ((1UL << SOCK_RXQ_OVFL) | \
2157 (1UL << SOCK_RCVTSTAMP) | \
2158 (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE) | \
2159 (1UL << SOCK_TIMESTAMPING_SOFTWARE) | \
2160 (1UL << SOCK_TIMESTAMPING_RAW_HARDWARE) | \
2161 (1UL << SOCK_TIMESTAMPING_SYS_HARDWARE))
2162
2163 if (sk->sk_flags & FLAGS_TS_OR_DROPS)
2164 __sock_recv_ts_and_drops(msg, sk, skb);
2165 else
2166 sk->sk_stamp = skb->tstamp;
2167}
2168
2169
2170
2171
2172
2173
2174
2175
2176extern void sock_tx_timestamp(struct sock *sk, __u8 *tx_flags);
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187#ifdef CONFIG_NET_DMA
2188static inline void sk_eat_skb(struct sock *sk, struct sk_buff *skb, bool copied_early)
2189{
2190 __skb_unlink(skb, &sk->sk_receive_queue);
2191 if (!copied_early)
2192 __kfree_skb(skb);
2193 else
2194 __skb_queue_tail(&sk->sk_async_wait_queue, skb);
2195}
2196#else
2197static inline void sk_eat_skb(struct sock *sk, struct sk_buff *skb, bool copied_early)
2198{
2199 __skb_unlink(skb, &sk->sk_receive_queue);
2200 __kfree_skb(skb);
2201}
2202#endif
2203
2204static inline
2205struct net *sock_net(const struct sock *sk)
2206{
2207 return read_pnet(&sk->sk_net);
2208}
2209
2210static inline
2211void sock_net_set(struct sock *sk, struct net *net)
2212{
2213 write_pnet(&sk->sk_net, net);
2214}
2215
2216
2217
2218
2219
2220
2221
2222static inline void sk_change_net(struct sock *sk, struct net *net)
2223{
2224 put_net(sock_net(sk));
2225 sock_net_set(sk, hold_net(net));
2226}
2227
2228static inline struct sock *skb_steal_sock(struct sk_buff *skb)
2229{
2230 if (skb->sk) {
2231 struct sock *sk = skb->sk;
2232
2233 skb->destructor = NULL;
2234 skb->sk = NULL;
2235 return sk;
2236 }
2237 return NULL;
2238}
2239
2240extern void sock_enable_timestamp(struct sock *sk, int flag);
2241extern int sock_get_timestamp(struct sock *, struct timeval __user *);
2242extern int sock_get_timestampns(struct sock *, struct timespec __user *);
2243
2244
2245
2246
2247extern int net_msg_warn;
2248#define NETDEBUG(fmt, args...) \
2249 do { if (net_msg_warn) printk(fmt,##args); } while (0)
2250
2251#define LIMIT_NETDEBUG(fmt, args...) \
2252 do { if (net_msg_warn && net_ratelimit()) printk(fmt,##args); } while(0)
2253
2254extern __u32 sysctl_wmem_max;
2255extern __u32 sysctl_rmem_max;
2256
2257extern int sysctl_optmem_max;
2258
2259extern __u32 sysctl_wmem_default;
2260extern __u32 sysctl_rmem_default;
2261
2262#endif
2263