1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40#ifndef _SOCK_H
41#define _SOCK_H
42
43#include <linux/hardirq.h>
44#include <linux/kernel.h>
45#include <linux/list.h>
46#include <linux/list_nulls.h>
47#include <linux/timer.h>
48#include <linux/cache.h>
49#include <linux/bitops.h>
50#include <linux/lockdep.h>
51#include <linux/netdevice.h>
52#include <linux/skbuff.h>
53#include <linux/mm.h>
54#include <linux/security.h>
55#include <linux/slab.h>
56#include <linux/uaccess.h>
57#include <linux/page_counter.h>
58#include <linux/memcontrol.h>
59#include <linux/static_key.h>
60#include <linux/sched.h>
61#include <linux/wait.h>
62#include <linux/cgroup-defs.h>
63
64#include <linux/filter.h>
65#include <linux/rculist_nulls.h>
66#include <linux/poll.h>
67
68#include <linux/atomic.h>
69#include <net/dst.h>
70#include <net/checksum.h>
71#include <net/tcp_states.h>
72#include <linux/net_tstamp.h>
73
74
75
76
77
78
79
80
81#define SOCK_DEBUGGING
82#ifdef SOCK_DEBUGGING
83#define SOCK_DEBUG(sk, msg...) do { if ((sk) && sock_flag((sk), SOCK_DBG)) \
84 printk(KERN_DEBUG msg); } while (0)
85#else
86
87static inline __printf(2, 3)
88void SOCK_DEBUG(const struct sock *sk, const char *msg, ...)
89{
90}
91#endif
92
93
94
95
96
97typedef struct {
98 spinlock_t slock;
99 int owned;
100 wait_queue_head_t wq;
101
102
103
104
105
106
107#ifdef CONFIG_DEBUG_LOCK_ALLOC
108 struct lockdep_map dep_map;
109#endif
110} socket_lock_t;
111
112struct sock;
113struct proto;
114struct net;
115
116typedef __u32 __bitwise __portpair;
117typedef __u64 __bitwise __addrpair;
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148struct sock_common {
149
150
151
152 union {
153 __addrpair skc_addrpair;
154 struct {
155 __be32 skc_daddr;
156 __be32 skc_rcv_saddr;
157 };
158 };
159 union {
160 unsigned int skc_hash;
161 __u16 skc_u16hashes[2];
162 };
163
164 union {
165 __portpair skc_portpair;
166 struct {
167 __be16 skc_dport;
168 __u16 skc_num;
169 };
170 };
171
172 unsigned short skc_family;
173 volatile unsigned char skc_state;
174 unsigned char skc_reuse:4;
175 unsigned char skc_reuseport:1;
176 unsigned char skc_ipv6only:1;
177 unsigned char skc_net_refcnt:1;
178 int skc_bound_dev_if;
179 union {
180 struct hlist_node skc_bind_node;
181 struct hlist_nulls_node skc_portaddr_node;
182 };
183 struct proto *skc_prot;
184 possible_net_t skc_net;
185
186#if IS_ENABLED(CONFIG_IPV6)
187 struct in6_addr skc_v6_daddr;
188 struct in6_addr skc_v6_rcv_saddr;
189#endif
190
191 atomic64_t skc_cookie;
192
193
194
195
196
197
198 union {
199 unsigned long skc_flags;
200 struct sock *skc_listener;
201 struct inet_timewait_death_row *skc_tw_dr;
202 };
203
204
205
206
207
208 int skc_dontcopy_begin[0];
209
210 union {
211 struct hlist_node skc_node;
212 struct hlist_nulls_node skc_nulls_node;
213 };
214 int skc_tx_queue_mapping;
215 union {
216 int skc_incoming_cpu;
217 u32 skc_rcv_wnd;
218 u32 skc_tw_rcv_nxt;
219 };
220
221 atomic_t skc_refcnt;
222
223 int skc_dontcopy_end[0];
224 union {
225 u32 skc_rxhash;
226 u32 skc_window_clamp;
227 u32 skc_tw_snd_nxt;
228 };
229
230};
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306struct sock {
307
308
309
310
311 struct sock_common __sk_common;
312#define sk_node __sk_common.skc_node
313#define sk_nulls_node __sk_common.skc_nulls_node
314#define sk_refcnt __sk_common.skc_refcnt
315#define sk_tx_queue_mapping __sk_common.skc_tx_queue_mapping
316
317#define sk_dontcopy_begin __sk_common.skc_dontcopy_begin
318#define sk_dontcopy_end __sk_common.skc_dontcopy_end
319#define sk_hash __sk_common.skc_hash
320#define sk_portpair __sk_common.skc_portpair
321#define sk_num __sk_common.skc_num
322#define sk_dport __sk_common.skc_dport
323#define sk_addrpair __sk_common.skc_addrpair
324#define sk_daddr __sk_common.skc_daddr
325#define sk_rcv_saddr __sk_common.skc_rcv_saddr
326#define sk_family __sk_common.skc_family
327#define sk_state __sk_common.skc_state
328#define sk_reuse __sk_common.skc_reuse
329#define sk_reuseport __sk_common.skc_reuseport
330#define sk_ipv6only __sk_common.skc_ipv6only
331#define sk_net_refcnt __sk_common.skc_net_refcnt
332#define sk_bound_dev_if __sk_common.skc_bound_dev_if
333#define sk_bind_node __sk_common.skc_bind_node
334#define sk_prot __sk_common.skc_prot
335#define sk_net __sk_common.skc_net
336#define sk_v6_daddr __sk_common.skc_v6_daddr
337#define sk_v6_rcv_saddr __sk_common.skc_v6_rcv_saddr
338#define sk_cookie __sk_common.skc_cookie
339#define sk_incoming_cpu __sk_common.skc_incoming_cpu
340#define sk_flags __sk_common.skc_flags
341#define sk_rxhash __sk_common.skc_rxhash
342
343 socket_lock_t sk_lock;
344 struct sk_buff_head sk_receive_queue;
345
346
347
348
349
350
351
352
353 struct {
354 atomic_t rmem_alloc;
355 int len;
356 struct sk_buff *head;
357 struct sk_buff *tail;
358 } sk_backlog;
359#define sk_rmem_alloc sk_backlog.rmem_alloc
360 int sk_forward_alloc;
361
362 __u32 sk_txhash;
363#ifdef CONFIG_NET_RX_BUSY_POLL
364 unsigned int sk_napi_id;
365 unsigned int sk_ll_usec;
366#endif
367 atomic_t sk_drops;
368 int sk_rcvbuf;
369
370 struct sk_filter __rcu *sk_filter;
371 union {
372 struct socket_wq __rcu *sk_wq;
373 struct socket_wq *sk_wq_raw;
374 };
375#ifdef CONFIG_XFRM
376 struct xfrm_policy __rcu *sk_policy[2];
377#endif
378 struct dst_entry *sk_rx_dst;
379 struct dst_entry __rcu *sk_dst_cache;
380
381 atomic_t sk_wmem_alloc;
382 atomic_t sk_omem_alloc;
383 int sk_sndbuf;
384 struct sk_buff_head sk_write_queue;
385 kmemcheck_bitfield_begin(flags);
386 unsigned int sk_shutdown : 2,
387 sk_no_check_tx : 1,
388 sk_no_check_rx : 1,
389 sk_userlocks : 4,
390 sk_protocol : 8,
391 sk_type : 16;
392#define SK_PROTOCOL_MAX U8_MAX
393 kmemcheck_bitfield_end(flags);
394 int sk_wmem_queued;
395 gfp_t sk_allocation;
396 u32 sk_pacing_rate;
397 u32 sk_max_pacing_rate;
398 netdev_features_t sk_route_caps;
399 netdev_features_t sk_route_nocaps;
400 int sk_gso_type;
401 unsigned int sk_gso_max_size;
402 u16 sk_gso_max_segs;
403 int sk_rcvlowat;
404 unsigned long sk_lingertime;
405 struct sk_buff_head sk_error_queue;
406 struct proto *sk_prot_creator;
407 rwlock_t sk_callback_lock;
408 int sk_err,
409 sk_err_soft;
410 u32 sk_ack_backlog;
411 u32 sk_max_ack_backlog;
412 __u32 sk_priority;
413 __u32 sk_mark;
414 struct pid *sk_peer_pid;
415 const struct cred *sk_peer_cred;
416 long sk_rcvtimeo;
417 long sk_sndtimeo;
418 struct timer_list sk_timer;
419 ktime_t sk_stamp;
420 u16 sk_tsflags;
421 u32 sk_tskey;
422 struct socket *sk_socket;
423 void *sk_user_data;
424 struct page_frag sk_frag;
425 struct sk_buff *sk_send_head;
426 __s32 sk_peek_off;
427 int sk_write_pending;
428#ifdef CONFIG_SECURITY
429 void *sk_security;
430#endif
431 struct sock_cgroup_data sk_cgrp_data;
432 struct mem_cgroup *sk_memcg;
433 void (*sk_state_change)(struct sock *sk);
434 void (*sk_data_ready)(struct sock *sk);
435 void (*sk_write_space)(struct sock *sk);
436 void (*sk_error_report)(struct sock *sk);
437 int (*sk_backlog_rcv)(struct sock *sk,
438 struct sk_buff *skb);
439 void (*sk_destruct)(struct sock *sk);
440 struct sock_reuseport __rcu *sk_reuseport_cb;
441};
442
443#define __sk_user_data(sk) ((*((void __rcu **)&(sk)->sk_user_data)))
444
445#define rcu_dereference_sk_user_data(sk) rcu_dereference(__sk_user_data((sk)))
446#define rcu_assign_sk_user_data(sk, ptr) rcu_assign_pointer(__sk_user_data((sk)), ptr)
447
448
449
450
451
452
453
454
455#define SK_NO_REUSE 0
456#define SK_CAN_REUSE 1
457#define SK_FORCE_REUSE 2
458
459static inline int sk_peek_offset(struct sock *sk, int flags)
460{
461 if ((flags & MSG_PEEK) && (sk->sk_peek_off >= 0))
462 return sk->sk_peek_off;
463 else
464 return 0;
465}
466
467static inline void sk_peek_offset_bwd(struct sock *sk, int val)
468{
469 if (sk->sk_peek_off >= 0) {
470 if (sk->sk_peek_off >= val)
471 sk->sk_peek_off -= val;
472 else
473 sk->sk_peek_off = 0;
474 }
475}
476
477static inline void sk_peek_offset_fwd(struct sock *sk, int val)
478{
479 if (sk->sk_peek_off >= 0)
480 sk->sk_peek_off += val;
481}
482
483
484
485
486static inline struct sock *sk_entry(const struct hlist_node *node)
487{
488 return hlist_entry(node, struct sock, sk_node);
489}
490
491static inline struct sock *__sk_head(const struct hlist_head *head)
492{
493 return hlist_entry(head->first, struct sock, sk_node);
494}
495
496static inline struct sock *sk_head(const struct hlist_head *head)
497{
498 return hlist_empty(head) ? NULL : __sk_head(head);
499}
500
501static inline struct sock *__sk_nulls_head(const struct hlist_nulls_head *head)
502{
503 return hlist_nulls_entry(head->first, struct sock, sk_nulls_node);
504}
505
506static inline struct sock *sk_nulls_head(const struct hlist_nulls_head *head)
507{
508 return hlist_nulls_empty(head) ? NULL : __sk_nulls_head(head);
509}
510
511static inline struct sock *sk_next(const struct sock *sk)
512{
513 return sk->sk_node.next ?
514 hlist_entry(sk->sk_node.next, struct sock, sk_node) : NULL;
515}
516
517static inline struct sock *sk_nulls_next(const struct sock *sk)
518{
519 return (!is_a_nulls(sk->sk_nulls_node.next)) ?
520 hlist_nulls_entry(sk->sk_nulls_node.next,
521 struct sock, sk_nulls_node) :
522 NULL;
523}
524
525static inline bool sk_unhashed(const struct sock *sk)
526{
527 return hlist_unhashed(&sk->sk_node);
528}
529
530static inline bool sk_hashed(const struct sock *sk)
531{
532 return !sk_unhashed(sk);
533}
534
535static inline void sk_node_init(struct hlist_node *node)
536{
537 node->pprev = NULL;
538}
539
540static inline void sk_nulls_node_init(struct hlist_nulls_node *node)
541{
542 node->pprev = NULL;
543}
544
545static inline void __sk_del_node(struct sock *sk)
546{
547 __hlist_del(&sk->sk_node);
548}
549
550
551static inline bool __sk_del_node_init(struct sock *sk)
552{
553 if (sk_hashed(sk)) {
554 __sk_del_node(sk);
555 sk_node_init(&sk->sk_node);
556 return true;
557 }
558 return false;
559}
560
561
562
563
564
565
566
567static inline void sock_hold(struct sock *sk)
568{
569 atomic_inc(&sk->sk_refcnt);
570}
571
572
573
574
575static inline void __sock_put(struct sock *sk)
576{
577 atomic_dec(&sk->sk_refcnt);
578}
579
580static inline bool sk_del_node_init(struct sock *sk)
581{
582 bool rc = __sk_del_node_init(sk);
583
584 if (rc) {
585
586 WARN_ON(atomic_read(&sk->sk_refcnt) == 1);
587 __sock_put(sk);
588 }
589 return rc;
590}
591#define sk_del_node_init_rcu(sk) sk_del_node_init(sk)
592
593static inline bool __sk_nulls_del_node_init_rcu(struct sock *sk)
594{
595 if (sk_hashed(sk)) {
596 hlist_nulls_del_init_rcu(&sk->sk_nulls_node);
597 return true;
598 }
599 return false;
600}
601
602static inline bool sk_nulls_del_node_init_rcu(struct sock *sk)
603{
604 bool rc = __sk_nulls_del_node_init_rcu(sk);
605
606 if (rc) {
607
608 WARN_ON(atomic_read(&sk->sk_refcnt) == 1);
609 __sock_put(sk);
610 }
611 return rc;
612}
613
614static inline void __sk_add_node(struct sock *sk, struct hlist_head *list)
615{
616 hlist_add_head(&sk->sk_node, list);
617}
618
619static inline void sk_add_node(struct sock *sk, struct hlist_head *list)
620{
621 sock_hold(sk);
622 __sk_add_node(sk, list);
623}
624
625static inline void sk_add_node_rcu(struct sock *sk, struct hlist_head *list)
626{
627 sock_hold(sk);
628 hlist_add_head_rcu(&sk->sk_node, list);
629}
630
631static inline void __sk_nulls_add_node_rcu(struct sock *sk, struct hlist_nulls_head *list)
632{
633 if (IS_ENABLED(CONFIG_IPV6) && sk->sk_reuseport &&
634 sk->sk_family == AF_INET6)
635 hlist_nulls_add_tail_rcu(&sk->sk_nulls_node, list);
636 else
637 hlist_nulls_add_head_rcu(&sk->sk_nulls_node, list);
638}
639
640static inline void sk_nulls_add_node_rcu(struct sock *sk, struct hlist_nulls_head *list)
641{
642 sock_hold(sk);
643 __sk_nulls_add_node_rcu(sk, list);
644}
645
646static inline void __sk_del_bind_node(struct sock *sk)
647{
648 __hlist_del(&sk->sk_bind_node);
649}
650
651static inline void sk_add_bind_node(struct sock *sk,
652 struct hlist_head *list)
653{
654 hlist_add_head(&sk->sk_bind_node, list);
655}
656
657#define sk_for_each(__sk, list) \
658 hlist_for_each_entry(__sk, list, sk_node)
659#define sk_for_each_rcu(__sk, list) \
660 hlist_for_each_entry_rcu(__sk, list, sk_node)
661#define sk_nulls_for_each(__sk, node, list) \
662 hlist_nulls_for_each_entry(__sk, node, list, sk_nulls_node)
663#define sk_nulls_for_each_rcu(__sk, node, list) \
664 hlist_nulls_for_each_entry_rcu(__sk, node, list, sk_nulls_node)
665#define sk_for_each_from(__sk) \
666 hlist_for_each_entry_from(__sk, sk_node)
667#define sk_nulls_for_each_from(__sk, node) \
668 if (__sk && ({ node = &(__sk)->sk_nulls_node; 1; })) \
669 hlist_nulls_for_each_entry_from(__sk, node, sk_nulls_node)
670#define sk_for_each_safe(__sk, tmp, list) \
671 hlist_for_each_entry_safe(__sk, tmp, list, sk_node)
672#define sk_for_each_bound(__sk, list) \
673 hlist_for_each_entry(__sk, list, sk_bind_node)
674
675
676
677
678
679
680
681
682
683#define sk_nulls_for_each_entry_offset(tpos, pos, head, offset) \
684 for (pos = (head)->first; \
685 (!is_a_nulls(pos)) && \
686 ({ tpos = (typeof(*tpos) *)((void *)pos - offset); 1;}); \
687 pos = pos->next)
688
689static inline struct user_namespace *sk_user_ns(struct sock *sk)
690{
691
692
693
694
695 return sk->sk_socket->file->f_cred->user_ns;
696}
697
698
699enum sock_flags {
700 SOCK_DEAD,
701 SOCK_DONE,
702 SOCK_URGINLINE,
703 SOCK_KEEPOPEN,
704 SOCK_LINGER,
705 SOCK_DESTROY,
706 SOCK_BROADCAST,
707 SOCK_TIMESTAMP,
708 SOCK_ZAPPED,
709 SOCK_USE_WRITE_QUEUE,
710 SOCK_DBG,
711 SOCK_RCVTSTAMP,
712 SOCK_RCVTSTAMPNS,
713 SOCK_LOCALROUTE,
714 SOCK_QUEUE_SHRUNK,
715 SOCK_MEMALLOC,
716 SOCK_TIMESTAMPING_RX_SOFTWARE,
717 SOCK_FASYNC,
718 SOCK_RXQ_OVFL,
719 SOCK_ZEROCOPY,
720 SOCK_WIFI_STATUS,
721 SOCK_NOFCS,
722
723
724
725 SOCK_FILTER_LOCKED,
726 SOCK_SELECT_ERR_QUEUE,
727};
728
729#define SK_FLAGS_TIMESTAMP ((1UL << SOCK_TIMESTAMP) | (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE))
730
731static inline void sock_copy_flags(struct sock *nsk, struct sock *osk)
732{
733 nsk->sk_flags = osk->sk_flags;
734}
735
736static inline void sock_set_flag(struct sock *sk, enum sock_flags flag)
737{
738 __set_bit(flag, &sk->sk_flags);
739}
740
741static inline void sock_reset_flag(struct sock *sk, enum sock_flags flag)
742{
743 __clear_bit(flag, &sk->sk_flags);
744}
745
746static inline bool sock_flag(const struct sock *sk, enum sock_flags flag)
747{
748 return test_bit(flag, &sk->sk_flags);
749}
750
751#ifdef CONFIG_NET
752extern struct static_key memalloc_socks;
753static inline int sk_memalloc_socks(void)
754{
755 return static_key_false(&memalloc_socks);
756}
757#else
758
759static inline int sk_memalloc_socks(void)
760{
761 return 0;
762}
763
764#endif
765
766static inline gfp_t sk_gfp_mask(const struct sock *sk, gfp_t gfp_mask)
767{
768 return gfp_mask | (sk->sk_allocation & __GFP_MEMALLOC);
769}
770
771static inline void sk_acceptq_removed(struct sock *sk)
772{
773 sk->sk_ack_backlog--;
774}
775
776static inline void sk_acceptq_added(struct sock *sk)
777{
778 sk->sk_ack_backlog++;
779}
780
781static inline bool sk_acceptq_is_full(const struct sock *sk)
782{
783 return sk->sk_ack_backlog > sk->sk_max_ack_backlog;
784}
785
786
787
788
789static inline int sk_stream_min_wspace(const struct sock *sk)
790{
791 return sk->sk_wmem_queued >> 1;
792}
793
794static inline int sk_stream_wspace(const struct sock *sk)
795{
796 return sk->sk_sndbuf - sk->sk_wmem_queued;
797}
798
799void sk_stream_write_space(struct sock *sk);
800
801
802static inline void __sk_add_backlog(struct sock *sk, struct sk_buff *skb)
803{
804
805 skb_dst_force_safe(skb);
806
807 if (!sk->sk_backlog.tail)
808 sk->sk_backlog.head = skb;
809 else
810 sk->sk_backlog.tail->next = skb;
811
812 sk->sk_backlog.tail = skb;
813 skb->next = NULL;
814}
815
816
817
818
819
820
821static inline bool sk_rcvqueues_full(const struct sock *sk, unsigned int limit)
822{
823 unsigned int qsize = sk->sk_backlog.len + atomic_read(&sk->sk_rmem_alloc);
824
825 return qsize > limit;
826}
827
828
829static inline __must_check int sk_add_backlog(struct sock *sk, struct sk_buff *skb,
830 unsigned int limit)
831{
832 if (sk_rcvqueues_full(sk, limit))
833 return -ENOBUFS;
834
835
836
837
838
839
840 if (skb_pfmemalloc(skb) && !sock_flag(sk, SOCK_MEMALLOC))
841 return -ENOMEM;
842
843 __sk_add_backlog(sk, skb);
844 sk->sk_backlog.len += skb->truesize;
845 return 0;
846}
847
848int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb);
849
850static inline int sk_backlog_rcv(struct sock *sk, struct sk_buff *skb)
851{
852 if (sk_memalloc_socks() && skb_pfmemalloc(skb))
853 return __sk_backlog_rcv(sk, skb);
854
855 return sk->sk_backlog_rcv(sk, skb);
856}
857
858static inline void sk_incoming_cpu_update(struct sock *sk)
859{
860 sk->sk_incoming_cpu = raw_smp_processor_id();
861}
862
863static inline void sock_rps_record_flow_hash(__u32 hash)
864{
865#ifdef CONFIG_RPS
866 struct rps_sock_flow_table *sock_flow_table;
867
868 rcu_read_lock();
869 sock_flow_table = rcu_dereference(rps_sock_flow_table);
870 rps_record_sock_flow(sock_flow_table, hash);
871 rcu_read_unlock();
872#endif
873}
874
875static inline void sock_rps_record_flow(const struct sock *sk)
876{
877#ifdef CONFIG_RPS
878 sock_rps_record_flow_hash(sk->sk_rxhash);
879#endif
880}
881
882static inline void sock_rps_save_rxhash(struct sock *sk,
883 const struct sk_buff *skb)
884{
885#ifdef CONFIG_RPS
886 if (unlikely(sk->sk_rxhash != skb->hash))
887 sk->sk_rxhash = skb->hash;
888#endif
889}
890
891static inline void sock_rps_reset_rxhash(struct sock *sk)
892{
893#ifdef CONFIG_RPS
894 sk->sk_rxhash = 0;
895#endif
896}
897
898#define sk_wait_event(__sk, __timeo, __condition) \
899 ({ int __rc; \
900 release_sock(__sk); \
901 __rc = __condition; \
902 if (!__rc) { \
903 *(__timeo) = schedule_timeout(*(__timeo)); \
904 } \
905 sched_annotate_sleep(); \
906 lock_sock(__sk); \
907 __rc = __condition; \
908 __rc; \
909 })
910
911int sk_stream_wait_connect(struct sock *sk, long *timeo_p);
912int sk_stream_wait_memory(struct sock *sk, long *timeo_p);
913void sk_stream_wait_close(struct sock *sk, long timeo_p);
914int sk_stream_error(struct sock *sk, int flags, int err);
915void sk_stream_kill_queues(struct sock *sk);
916void sk_set_memalloc(struct sock *sk);
917void sk_clear_memalloc(struct sock *sk);
918
919int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb);
920
921struct request_sock_ops;
922struct timewait_sock_ops;
923struct inet_hashinfo;
924struct raw_hashinfo;
925struct module;
926
927
928
929
930
931static inline void sk_prot_clear_nulls(struct sock *sk, int size)
932{
933 if (offsetof(struct sock, sk_node.next) != 0)
934 memset(sk, 0, offsetof(struct sock, sk_node.next));
935 memset(&sk->sk_node.pprev, 0,
936 size - offsetof(struct sock, sk_node.pprev));
937}
938
939
940
941
942struct proto {
943 void (*close)(struct sock *sk,
944 long timeout);
945 int (*connect)(struct sock *sk,
946 struct sockaddr *uaddr,
947 int addr_len);
948 int (*disconnect)(struct sock *sk, int flags);
949
950 struct sock * (*accept)(struct sock *sk, int flags, int *err);
951
952 int (*ioctl)(struct sock *sk, int cmd,
953 unsigned long arg);
954 int (*init)(struct sock *sk);
955 void (*destroy)(struct sock *sk);
956 void (*shutdown)(struct sock *sk, int how);
957 int (*setsockopt)(struct sock *sk, int level,
958 int optname, char __user *optval,
959 unsigned int optlen);
960 int (*getsockopt)(struct sock *sk, int level,
961 int optname, char __user *optval,
962 int __user *option);
963#ifdef CONFIG_COMPAT
964 int (*compat_setsockopt)(struct sock *sk,
965 int level,
966 int optname, char __user *optval,
967 unsigned int optlen);
968 int (*compat_getsockopt)(struct sock *sk,
969 int level,
970 int optname, char __user *optval,
971 int __user *option);
972 int (*compat_ioctl)(struct sock *sk,
973 unsigned int cmd, unsigned long arg);
974#endif
975 int (*sendmsg)(struct sock *sk, struct msghdr *msg,
976 size_t len);
977 int (*recvmsg)(struct sock *sk, struct msghdr *msg,
978 size_t len, int noblock, int flags,
979 int *addr_len);
980 int (*sendpage)(struct sock *sk, struct page *page,
981 int offset, size_t size, int flags);
982 int (*bind)(struct sock *sk,
983 struct sockaddr *uaddr, int addr_len);
984
985 int (*backlog_rcv) (struct sock *sk,
986 struct sk_buff *skb);
987
988 void (*release_cb)(struct sock *sk);
989
990
991 int (*hash)(struct sock *sk);
992 void (*unhash)(struct sock *sk);
993 void (*rehash)(struct sock *sk);
994 int (*get_port)(struct sock *sk, unsigned short snum);
995 void (*clear_sk)(struct sock *sk, int size);
996
997
998#ifdef CONFIG_PROC_FS
999 unsigned int inuse_idx;
1000#endif
1001
1002 bool (*stream_memory_free)(const struct sock *sk);
1003
1004 void (*enter_memory_pressure)(struct sock *sk);
1005 atomic_long_t *memory_allocated;
1006 struct percpu_counter *sockets_allocated;
1007
1008
1009
1010
1011
1012
1013 int *memory_pressure;
1014 long *sysctl_mem;
1015 int *sysctl_wmem;
1016 int *sysctl_rmem;
1017 int max_header;
1018 bool no_autobind;
1019
1020 struct kmem_cache *slab;
1021 unsigned int obj_size;
1022 int slab_flags;
1023
1024 struct percpu_counter *orphan_count;
1025
1026 struct request_sock_ops *rsk_prot;
1027 struct timewait_sock_ops *twsk_prot;
1028
1029 union {
1030 struct inet_hashinfo *hashinfo;
1031 struct udp_table *udp_table;
1032 struct raw_hashinfo *raw_hash;
1033 } h;
1034
1035 struct module *owner;
1036
1037 char name[32];
1038
1039 struct list_head node;
1040#ifdef SOCK_REFCNT_DEBUG
1041 atomic_t socks;
1042#endif
1043 int (*diag_destroy)(struct sock *sk, int err);
1044};
1045
1046int proto_register(struct proto *prot, int alloc_slab);
1047void proto_unregister(struct proto *prot);
1048
1049#ifdef SOCK_REFCNT_DEBUG
1050static inline void sk_refcnt_debug_inc(struct sock *sk)
1051{
1052 atomic_inc(&sk->sk_prot->socks);
1053}
1054
1055static inline void sk_refcnt_debug_dec(struct sock *sk)
1056{
1057 atomic_dec(&sk->sk_prot->socks);
1058 printk(KERN_DEBUG "%s socket %p released, %d are still alive\n",
1059 sk->sk_prot->name, sk, atomic_read(&sk->sk_prot->socks));
1060}
1061
1062static inline void sk_refcnt_debug_release(const struct sock *sk)
1063{
1064 if (atomic_read(&sk->sk_refcnt) != 1)
1065 printk(KERN_DEBUG "Destruction of the %s socket %p delayed, refcnt=%d\n",
1066 sk->sk_prot->name, sk, atomic_read(&sk->sk_refcnt));
1067}
1068#else
1069#define sk_refcnt_debug_inc(sk) do { } while (0)
1070#define sk_refcnt_debug_dec(sk) do { } while (0)
1071#define sk_refcnt_debug_release(sk) do { } while (0)
1072#endif
1073
1074static inline bool sk_stream_memory_free(const struct sock *sk)
1075{
1076 if (sk->sk_wmem_queued >= sk->sk_sndbuf)
1077 return false;
1078
1079 return sk->sk_prot->stream_memory_free ?
1080 sk->sk_prot->stream_memory_free(sk) : true;
1081}
1082
1083static inline bool sk_stream_is_writeable(const struct sock *sk)
1084{
1085 return sk_stream_wspace(sk) >= sk_stream_min_wspace(sk) &&
1086 sk_stream_memory_free(sk);
1087}
1088
1089
1090static inline bool sk_has_memory_pressure(const struct sock *sk)
1091{
1092 return sk->sk_prot->memory_pressure != NULL;
1093}
1094
1095static inline bool sk_under_memory_pressure(const struct sock *sk)
1096{
1097 if (!sk->sk_prot->memory_pressure)
1098 return false;
1099
1100 if (mem_cgroup_sockets_enabled && sk->sk_memcg &&
1101 mem_cgroup_under_socket_pressure(sk->sk_memcg))
1102 return true;
1103
1104 return !!*sk->sk_prot->memory_pressure;
1105}
1106
1107static inline void sk_leave_memory_pressure(struct sock *sk)
1108{
1109 int *memory_pressure = sk->sk_prot->memory_pressure;
1110
1111 if (!memory_pressure)
1112 return;
1113
1114 if (*memory_pressure)
1115 *memory_pressure = 0;
1116}
1117
1118static inline void sk_enter_memory_pressure(struct sock *sk)
1119{
1120 if (!sk->sk_prot->enter_memory_pressure)
1121 return;
1122
1123 sk->sk_prot->enter_memory_pressure(sk);
1124}
1125
1126static inline long sk_prot_mem_limits(const struct sock *sk, int index)
1127{
1128 return sk->sk_prot->sysctl_mem[index];
1129}
1130
1131static inline long
1132sk_memory_allocated(const struct sock *sk)
1133{
1134 return atomic_long_read(sk->sk_prot->memory_allocated);
1135}
1136
1137static inline long
1138sk_memory_allocated_add(struct sock *sk, int amt)
1139{
1140 return atomic_long_add_return(amt, sk->sk_prot->memory_allocated);
1141}
1142
1143static inline void
1144sk_memory_allocated_sub(struct sock *sk, int amt)
1145{
1146 atomic_long_sub(amt, sk->sk_prot->memory_allocated);
1147}
1148
1149static inline void sk_sockets_allocated_dec(struct sock *sk)
1150{
1151 percpu_counter_dec(sk->sk_prot->sockets_allocated);
1152}
1153
1154static inline void sk_sockets_allocated_inc(struct sock *sk)
1155{
1156 percpu_counter_inc(sk->sk_prot->sockets_allocated);
1157}
1158
1159static inline int
1160sk_sockets_allocated_read_positive(struct sock *sk)
1161{
1162 return percpu_counter_read_positive(sk->sk_prot->sockets_allocated);
1163}
1164
1165static inline int
1166proto_sockets_allocated_sum_positive(struct proto *prot)
1167{
1168 return percpu_counter_sum_positive(prot->sockets_allocated);
1169}
1170
1171static inline long
1172proto_memory_allocated(struct proto *prot)
1173{
1174 return atomic_long_read(prot->memory_allocated);
1175}
1176
1177static inline bool
1178proto_memory_pressure(struct proto *prot)
1179{
1180 if (!prot->memory_pressure)
1181 return false;
1182 return !!*prot->memory_pressure;
1183}
1184
1185
1186#ifdef CONFIG_PROC_FS
1187
1188void sock_prot_inuse_add(struct net *net, struct proto *prot, int inc);
1189int sock_prot_inuse_get(struct net *net, struct proto *proto);
1190#else
1191static inline void sock_prot_inuse_add(struct net *net, struct proto *prot,
1192 int inc)
1193{
1194}
1195#endif
1196
1197
1198
1199
1200
1201static inline int __sk_prot_rehash(struct sock *sk)
1202{
1203 sk->sk_prot->unhash(sk);
1204 return sk->sk_prot->hash(sk);
1205}
1206
1207void sk_prot_clear_portaddr_nulls(struct sock *sk, int size);
1208
1209
1210#define SOCK_DESTROY_TIME (10*HZ)
1211
1212
1213#define PROT_SOCK 1024
1214
1215#define SHUTDOWN_MASK 3
1216#define RCV_SHUTDOWN 1
1217#define SEND_SHUTDOWN 2
1218
1219#define SOCK_SNDBUF_LOCK 1
1220#define SOCK_RCVBUF_LOCK 2
1221#define SOCK_BINDADDR_LOCK 4
1222#define SOCK_BINDPORT_LOCK 8
1223
1224struct socket_alloc {
1225 struct socket socket;
1226 struct inode vfs_inode;
1227};
1228
1229static inline struct socket *SOCKET_I(struct inode *inode)
1230{
1231 return &container_of(inode, struct socket_alloc, vfs_inode)->socket;
1232}
1233
1234static inline struct inode *SOCK_INODE(struct socket *socket)
1235{
1236 return &container_of(socket, struct socket_alloc, socket)->vfs_inode;
1237}
1238
1239
1240
1241
1242int __sk_mem_schedule(struct sock *sk, int size, int kind);
1243void __sk_mem_reclaim(struct sock *sk, int amount);
1244
1245#define SK_MEM_QUANTUM ((int)PAGE_SIZE)
1246#define SK_MEM_QUANTUM_SHIFT ilog2(SK_MEM_QUANTUM)
1247#define SK_MEM_SEND 0
1248#define SK_MEM_RECV 1
1249
1250static inline int sk_mem_pages(int amt)
1251{
1252 return (amt + SK_MEM_QUANTUM - 1) >> SK_MEM_QUANTUM_SHIFT;
1253}
1254
1255static inline bool sk_has_account(struct sock *sk)
1256{
1257
1258 return !!sk->sk_prot->memory_allocated;
1259}
1260
1261static inline bool sk_wmem_schedule(struct sock *sk, int size)
1262{
1263 if (!sk_has_account(sk))
1264 return true;
1265 return size <= sk->sk_forward_alloc ||
1266 __sk_mem_schedule(sk, size, SK_MEM_SEND);
1267}
1268
1269static inline bool
1270sk_rmem_schedule(struct sock *sk, struct sk_buff *skb, int size)
1271{
1272 if (!sk_has_account(sk))
1273 return true;
1274 return size<= sk->sk_forward_alloc ||
1275 __sk_mem_schedule(sk, size, SK_MEM_RECV) ||
1276 skb_pfmemalloc(skb);
1277}
1278
1279static inline void sk_mem_reclaim(struct sock *sk)
1280{
1281 if (!sk_has_account(sk))
1282 return;
1283 if (sk->sk_forward_alloc >= SK_MEM_QUANTUM)
1284 __sk_mem_reclaim(sk, sk->sk_forward_alloc);
1285}
1286
1287static inline void sk_mem_reclaim_partial(struct sock *sk)
1288{
1289 if (!sk_has_account(sk))
1290 return;
1291 if (sk->sk_forward_alloc > SK_MEM_QUANTUM)
1292 __sk_mem_reclaim(sk, sk->sk_forward_alloc - 1);
1293}
1294
1295static inline void sk_mem_charge(struct sock *sk, int size)
1296{
1297 if (!sk_has_account(sk))
1298 return;
1299 sk->sk_forward_alloc -= size;
1300}
1301
1302static inline void sk_mem_uncharge(struct sock *sk, int size)
1303{
1304 if (!sk_has_account(sk))
1305 return;
1306 sk->sk_forward_alloc += size;
1307}
1308
1309static inline void sk_wmem_free_skb(struct sock *sk, struct sk_buff *skb)
1310{
1311 sock_set_flag(sk, SOCK_QUEUE_SHRUNK);
1312 sk->sk_wmem_queued -= skb->truesize;
1313 sk_mem_uncharge(sk, skb->truesize);
1314 __kfree_skb(skb);
1315}
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330#define sock_owned_by_user(sk) ((sk)->sk_lock.owned)
1331
1332static inline void sock_release_ownership(struct sock *sk)
1333{
1334 sk->sk_lock.owned = 0;
1335}
1336
1337
1338
1339
1340
1341
1342
1343
1344#define sock_lock_init_class_and_name(sk, sname, skey, name, key) \
1345do { \
1346 sk->sk_lock.owned = 0; \
1347 init_waitqueue_head(&sk->sk_lock.wq); \
1348 spin_lock_init(&(sk)->sk_lock.slock); \
1349 debug_check_no_locks_freed((void *)&(sk)->sk_lock, \
1350 sizeof((sk)->sk_lock)); \
1351 lockdep_set_class_and_name(&(sk)->sk_lock.slock, \
1352 (skey), (sname)); \
1353 lockdep_init_map(&(sk)->sk_lock.dep_map, (name), (key), 0); \
1354} while (0)
1355
1356void lock_sock_nested(struct sock *sk, int subclass);
1357
1358static inline void lock_sock(struct sock *sk)
1359{
1360 lock_sock_nested(sk, 0);
1361}
1362
1363void release_sock(struct sock *sk);
1364
1365
1366#define bh_lock_sock(__sk) spin_lock(&((__sk)->sk_lock.slock))
1367#define bh_lock_sock_nested(__sk) \
1368 spin_lock_nested(&((__sk)->sk_lock.slock), \
1369 SINGLE_DEPTH_NESTING)
1370#define bh_unlock_sock(__sk) spin_unlock(&((__sk)->sk_lock.slock))
1371
1372bool lock_sock_fast(struct sock *sk);
1373
1374
1375
1376
1377
1378
1379
1380
1381static inline void unlock_sock_fast(struct sock *sk, bool slow)
1382{
1383 if (slow)
1384 release_sock(sk);
1385 else
1386 spin_unlock_bh(&sk->sk_lock.slock);
1387}
1388
1389
1390struct sock *sk_alloc(struct net *net, int family, gfp_t priority,
1391 struct proto *prot, int kern);
1392void sk_free(struct sock *sk);
1393void sk_destruct(struct sock *sk);
1394struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority);
1395
1396struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force,
1397 gfp_t priority);
1398void sock_wfree(struct sk_buff *skb);
1399void skb_orphan_partial(struct sk_buff *skb);
1400void sock_rfree(struct sk_buff *skb);
1401void sock_efree(struct sk_buff *skb);
1402#ifdef CONFIG_INET
1403void sock_edemux(struct sk_buff *skb);
1404#else
1405#define sock_edemux(skb) sock_efree(skb)
1406#endif
1407
1408int sock_setsockopt(struct socket *sock, int level, int op,
1409 char __user *optval, unsigned int optlen);
1410
1411int sock_getsockopt(struct socket *sock, int level, int op,
1412 char __user *optval, int __user *optlen);
1413struct sk_buff *sock_alloc_send_skb(struct sock *sk, unsigned long size,
1414 int noblock, int *errcode);
1415struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len,
1416 unsigned long data_len, int noblock,
1417 int *errcode, int max_page_order);
1418void *sock_kmalloc(struct sock *sk, int size, gfp_t priority);
1419void sock_kfree_s(struct sock *sk, void *mem, int size);
1420void sock_kzfree_s(struct sock *sk, void *mem, int size);
1421void sk_send_sigurg(struct sock *sk);
1422
1423struct sockcm_cookie {
1424 u32 mark;
1425};
1426
1427int sock_cmsg_send(struct sock *sk, struct msghdr *msg,
1428 struct sockcm_cookie *sockc);
1429
1430
1431
1432
1433
1434int sock_no_bind(struct socket *, struct sockaddr *, int);
1435int sock_no_connect(struct socket *, struct sockaddr *, int, int);
1436int sock_no_socketpair(struct socket *, struct socket *);
1437int sock_no_accept(struct socket *, struct socket *, int);
1438int sock_no_getname(struct socket *, struct sockaddr *, int *, int);
1439unsigned int sock_no_poll(struct file *, struct socket *,
1440 struct poll_table_struct *);
1441int sock_no_ioctl(struct socket *, unsigned int, unsigned long);
1442int sock_no_listen(struct socket *, int);
1443int sock_no_shutdown(struct socket *, int);
1444int sock_no_getsockopt(struct socket *, int , int, char __user *, int __user *);
1445int sock_no_setsockopt(struct socket *, int, int, char __user *, unsigned int);
1446int sock_no_sendmsg(struct socket *, struct msghdr *, size_t);
1447int sock_no_recvmsg(struct socket *, struct msghdr *, size_t, int);
1448int sock_no_mmap(struct file *file, struct socket *sock,
1449 struct vm_area_struct *vma);
1450ssize_t sock_no_sendpage(struct socket *sock, struct page *page, int offset,
1451 size_t size, int flags);
1452
1453
1454
1455
1456
1457int sock_common_getsockopt(struct socket *sock, int level, int optname,
1458 char __user *optval, int __user *optlen);
1459int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
1460 int flags);
1461int sock_common_setsockopt(struct socket *sock, int level, int optname,
1462 char __user *optval, unsigned int optlen);
1463int compat_sock_common_getsockopt(struct socket *sock, int level,
1464 int optname, char __user *optval, int __user *optlen);
1465int compat_sock_common_setsockopt(struct socket *sock, int level,
1466 int optname, char __user *optval, unsigned int optlen);
1467
1468void sk_common_release(struct sock *sk);
1469
1470
1471
1472
1473
1474
1475void sock_init_data(struct socket *sock, struct sock *sk);
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503static inline void sock_put(struct sock *sk)
1504{
1505 if (atomic_dec_and_test(&sk->sk_refcnt))
1506 sk_free(sk);
1507}
1508
1509
1510
1511void sock_gen_put(struct sock *sk);
1512
1513int sk_receive_skb(struct sock *sk, struct sk_buff *skb, const int nested);
1514
1515static inline void sk_tx_queue_set(struct sock *sk, int tx_queue)
1516{
1517 sk->sk_tx_queue_mapping = tx_queue;
1518}
1519
1520static inline void sk_tx_queue_clear(struct sock *sk)
1521{
1522 sk->sk_tx_queue_mapping = -1;
1523}
1524
1525static inline int sk_tx_queue_get(const struct sock *sk)
1526{
1527 return sk ? sk->sk_tx_queue_mapping : -1;
1528}
1529
1530static inline void sk_set_socket(struct sock *sk, struct socket *sock)
1531{
1532 sk_tx_queue_clear(sk);
1533 sk->sk_socket = sock;
1534}
1535
1536static inline wait_queue_head_t *sk_sleep(struct sock *sk)
1537{
1538 BUILD_BUG_ON(offsetof(struct socket_wq, wait) != 0);
1539 return &rcu_dereference_raw(sk->sk_wq)->wait;
1540}
1541
1542
1543
1544
1545
1546
1547
1548static inline void sock_orphan(struct sock *sk)
1549{
1550 write_lock_bh(&sk->sk_callback_lock);
1551 sock_set_flag(sk, SOCK_DEAD);
1552 sk_set_socket(sk, NULL);
1553 sk->sk_wq = NULL;
1554 write_unlock_bh(&sk->sk_callback_lock);
1555}
1556
1557static inline void sock_graft(struct sock *sk, struct socket *parent)
1558{
1559 write_lock_bh(&sk->sk_callback_lock);
1560 sk->sk_wq = parent->wq;
1561 parent->sk = sk;
1562 sk_set_socket(sk, parent);
1563 security_sock_graft(sk, parent);
1564 write_unlock_bh(&sk->sk_callback_lock);
1565}
1566
1567kuid_t sock_i_uid(struct sock *sk);
1568unsigned long sock_i_ino(struct sock *sk);
1569
1570static inline u32 net_tx_rndhash(void)
1571{
1572 u32 v = prandom_u32();
1573
1574 return v ?: 1;
1575}
1576
1577static inline void sk_set_txhash(struct sock *sk)
1578{
1579 sk->sk_txhash = net_tx_rndhash();
1580}
1581
1582static inline void sk_rethink_txhash(struct sock *sk)
1583{
1584 if (sk->sk_txhash)
1585 sk_set_txhash(sk);
1586}
1587
1588static inline struct dst_entry *
1589__sk_dst_get(struct sock *sk)
1590{
1591 return rcu_dereference_check(sk->sk_dst_cache, sock_owned_by_user(sk) ||
1592 lockdep_is_held(&sk->sk_lock.slock));
1593}
1594
1595static inline struct dst_entry *
1596sk_dst_get(struct sock *sk)
1597{
1598 struct dst_entry *dst;
1599
1600 rcu_read_lock();
1601 dst = rcu_dereference(sk->sk_dst_cache);
1602 if (dst && !atomic_inc_not_zero(&dst->__refcnt))
1603 dst = NULL;
1604 rcu_read_unlock();
1605 return dst;
1606}
1607
1608static inline void dst_negative_advice(struct sock *sk)
1609{
1610 struct dst_entry *ndst, *dst = __sk_dst_get(sk);
1611
1612 sk_rethink_txhash(sk);
1613
1614 if (dst && dst->ops->negative_advice) {
1615 ndst = dst->ops->negative_advice(dst);
1616
1617 if (ndst != dst) {
1618 rcu_assign_pointer(sk->sk_dst_cache, ndst);
1619 sk_tx_queue_clear(sk);
1620 }
1621 }
1622}
1623
1624static inline void
1625__sk_dst_set(struct sock *sk, struct dst_entry *dst)
1626{
1627 struct dst_entry *old_dst;
1628
1629 sk_tx_queue_clear(sk);
1630
1631
1632
1633
1634 old_dst = rcu_dereference_raw(sk->sk_dst_cache);
1635 rcu_assign_pointer(sk->sk_dst_cache, dst);
1636 dst_release(old_dst);
1637}
1638
1639static inline void
1640sk_dst_set(struct sock *sk, struct dst_entry *dst)
1641{
1642 struct dst_entry *old_dst;
1643
1644 sk_tx_queue_clear(sk);
1645 old_dst = xchg((__force struct dst_entry **)&sk->sk_dst_cache, dst);
1646 dst_release(old_dst);
1647}
1648
1649static inline void
1650__sk_dst_reset(struct sock *sk)
1651{
1652 __sk_dst_set(sk, NULL);
1653}
1654
1655static inline void
1656sk_dst_reset(struct sock *sk)
1657{
1658 sk_dst_set(sk, NULL);
1659}
1660
1661struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie);
1662
1663struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie);
1664
1665bool sk_mc_loop(struct sock *sk);
1666
1667static inline bool sk_can_gso(const struct sock *sk)
1668{
1669 return net_gso_ok(sk->sk_route_caps, sk->sk_gso_type);
1670}
1671
1672void sk_setup_caps(struct sock *sk, struct dst_entry *dst);
1673
1674static inline void sk_nocaps_add(struct sock *sk, netdev_features_t flags)
1675{
1676 sk->sk_route_nocaps |= flags;
1677 sk->sk_route_caps &= ~flags;
1678}
1679
1680static inline bool sk_check_csum_caps(struct sock *sk)
1681{
1682 return (sk->sk_route_caps & NETIF_F_HW_CSUM) ||
1683 (sk->sk_family == PF_INET &&
1684 (sk->sk_route_caps & NETIF_F_IP_CSUM)) ||
1685 (sk->sk_family == PF_INET6 &&
1686 (sk->sk_route_caps & NETIF_F_IPV6_CSUM));
1687}
1688
1689static inline int skb_do_copy_data_nocache(struct sock *sk, struct sk_buff *skb,
1690 struct iov_iter *from, char *to,
1691 int copy, int offset)
1692{
1693 if (skb->ip_summed == CHECKSUM_NONE) {
1694 __wsum csum = 0;
1695 if (csum_and_copy_from_iter(to, copy, &csum, from) != copy)
1696 return -EFAULT;
1697 skb->csum = csum_block_add(skb->csum, csum, offset);
1698 } else if (sk->sk_route_caps & NETIF_F_NOCACHE_COPY) {
1699 if (copy_from_iter_nocache(to, copy, from) != copy)
1700 return -EFAULT;
1701 } else if (copy_from_iter(to, copy, from) != copy)
1702 return -EFAULT;
1703
1704 return 0;
1705}
1706
1707static inline int skb_add_data_nocache(struct sock *sk, struct sk_buff *skb,
1708 struct iov_iter *from, int copy)
1709{
1710 int err, offset = skb->len;
1711
1712 err = skb_do_copy_data_nocache(sk, skb, from, skb_put(skb, copy),
1713 copy, offset);
1714 if (err)
1715 __skb_trim(skb, offset);
1716
1717 return err;
1718}
1719
1720static inline int skb_copy_to_page_nocache(struct sock *sk, struct iov_iter *from,
1721 struct sk_buff *skb,
1722 struct page *page,
1723 int off, int copy)
1724{
1725 int err;
1726
1727 err = skb_do_copy_data_nocache(sk, skb, from, page_address(page) + off,
1728 copy, skb->len);
1729 if (err)
1730 return err;
1731
1732 skb->len += copy;
1733 skb->data_len += copy;
1734 skb->truesize += copy;
1735 sk->sk_wmem_queued += copy;
1736 sk_mem_charge(sk, copy);
1737 return 0;
1738}
1739
1740
1741
1742
1743
1744
1745
1746static inline int sk_wmem_alloc_get(const struct sock *sk)
1747{
1748 return atomic_read(&sk->sk_wmem_alloc) - 1;
1749}
1750
1751
1752
1753
1754
1755
1756
1757static inline int sk_rmem_alloc_get(const struct sock *sk)
1758{
1759 return atomic_read(&sk->sk_rmem_alloc);
1760}
1761
1762
1763
1764
1765
1766
1767
1768static inline bool sk_has_allocations(const struct sock *sk)
1769{
1770 return sk_wmem_alloc_get(sk) || sk_rmem_alloc_get(sk);
1771}
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805static inline bool skwq_has_sleeper(struct socket_wq *wq)
1806{
1807 return wq && wq_has_sleeper(&wq->wait);
1808}
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818static inline void sock_poll_wait(struct file *filp,
1819 wait_queue_head_t *wait_address, poll_table *p)
1820{
1821 if (!poll_does_not_wait(p) && wait_address) {
1822 poll_wait(filp, wait_address, p);
1823
1824
1825
1826
1827
1828 smp_mb();
1829 }
1830}
1831
1832static inline void skb_set_hash_from_sk(struct sk_buff *skb, struct sock *sk)
1833{
1834 if (sk->sk_txhash) {
1835 skb->l4_hash = 1;
1836 skb->hash = sk->sk_txhash;
1837 }
1838}
1839
1840void skb_set_owner_w(struct sk_buff *skb, struct sock *sk);
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850static inline void skb_set_owner_r(struct sk_buff *skb, struct sock *sk)
1851{
1852 skb_orphan(skb);
1853 skb->sk = sk;
1854 skb->destructor = sock_rfree;
1855 atomic_add(skb->truesize, &sk->sk_rmem_alloc);
1856 sk_mem_charge(sk, skb->truesize);
1857}
1858
1859void sk_reset_timer(struct sock *sk, struct timer_list *timer,
1860 unsigned long expires);
1861
1862void sk_stop_timer(struct sock *sk, struct timer_list *timer);
1863
1864int sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb);
1865
1866int sock_queue_err_skb(struct sock *sk, struct sk_buff *skb);
1867struct sk_buff *sock_dequeue_err_skb(struct sock *sk);
1868
1869
1870
1871
1872
1873static inline int sock_error(struct sock *sk)
1874{
1875 int err;
1876 if (likely(!sk->sk_err))
1877 return 0;
1878 err = xchg(&sk->sk_err, 0);
1879 return -err;
1880}
1881
1882static inline unsigned long sock_wspace(struct sock *sk)
1883{
1884 int amt = 0;
1885
1886 if (!(sk->sk_shutdown & SEND_SHUTDOWN)) {
1887 amt = sk->sk_sndbuf - atomic_read(&sk->sk_wmem_alloc);
1888 if (amt < 0)
1889 amt = 0;
1890 }
1891 return amt;
1892}
1893
1894
1895
1896
1897
1898static inline void sk_set_bit(int nr, struct sock *sk)
1899{
1900 set_bit(nr, &sk->sk_wq_raw->flags);
1901}
1902
1903static inline void sk_clear_bit(int nr, struct sock *sk)
1904{
1905 clear_bit(nr, &sk->sk_wq_raw->flags);
1906}
1907
1908static inline void sk_wake_async(const struct sock *sk, int how, int band)
1909{
1910 if (sock_flag(sk, SOCK_FASYNC)) {
1911 rcu_read_lock();
1912 sock_wake_async(rcu_dereference(sk->sk_wq), how, band);
1913 rcu_read_unlock();
1914 }
1915}
1916
1917
1918
1919
1920
1921
1922#define TCP_SKB_MIN_TRUESIZE (2048 + SKB_DATA_ALIGN(sizeof(struct sk_buff)))
1923
1924#define SOCK_MIN_SNDBUF (TCP_SKB_MIN_TRUESIZE * 2)
1925#define SOCK_MIN_RCVBUF TCP_SKB_MIN_TRUESIZE
1926
1927static inline void sk_stream_moderate_sndbuf(struct sock *sk)
1928{
1929 if (!(sk->sk_userlocks & SOCK_SNDBUF_LOCK)) {
1930 sk->sk_sndbuf = min(sk->sk_sndbuf, sk->sk_wmem_queued >> 1);
1931 sk->sk_sndbuf = max_t(u32, sk->sk_sndbuf, SOCK_MIN_SNDBUF);
1932 }
1933}
1934
1935struct sk_buff *sk_stream_alloc_skb(struct sock *sk, int size, gfp_t gfp,
1936 bool force_schedule);
1937
1938
1939
1940
1941
1942
1943
1944
1945static inline struct page_frag *sk_page_frag(struct sock *sk)
1946{
1947 if (gfpflags_allow_blocking(sk->sk_allocation))
1948 return ¤t->task_frag;
1949
1950 return &sk->sk_frag;
1951}
1952
1953bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag);
1954
1955
1956
1957
1958static inline bool sock_writeable(const struct sock *sk)
1959{
1960 return atomic_read(&sk->sk_wmem_alloc) < (sk->sk_sndbuf >> 1);
1961}
1962
1963static inline gfp_t gfp_any(void)
1964{
1965 return in_softirq() ? GFP_ATOMIC : GFP_KERNEL;
1966}
1967
1968static inline long sock_rcvtimeo(const struct sock *sk, bool noblock)
1969{
1970 return noblock ? 0 : sk->sk_rcvtimeo;
1971}
1972
1973static inline long sock_sndtimeo(const struct sock *sk, bool noblock)
1974{
1975 return noblock ? 0 : sk->sk_sndtimeo;
1976}
1977
1978static inline int sock_rcvlowat(const struct sock *sk, int waitall, int len)
1979{
1980 return (waitall ? len : min_t(int, sk->sk_rcvlowat, len)) ? : 1;
1981}
1982
1983
1984
1985
1986static inline int sock_intr_errno(long timeo)
1987{
1988 return timeo == MAX_SCHEDULE_TIMEOUT ? -ERESTARTSYS : -EINTR;
1989}
1990
1991struct sock_skb_cb {
1992 u32 dropcount;
1993};
1994
1995
1996
1997
1998
1999#define SOCK_SKB_CB_OFFSET ((FIELD_SIZEOF(struct sk_buff, cb) - \
2000 sizeof(struct sock_skb_cb)))
2001
2002#define SOCK_SKB_CB(__skb) ((struct sock_skb_cb *)((__skb)->cb + \
2003 SOCK_SKB_CB_OFFSET))
2004
2005#define sock_skb_cb_check_size(size) \
2006 BUILD_BUG_ON((size) > SOCK_SKB_CB_OFFSET)
2007
2008static inline void
2009sock_skb_set_dropcount(const struct sock *sk, struct sk_buff *skb)
2010{
2011 SOCK_SKB_CB(skb)->dropcount = atomic_read(&sk->sk_drops);
2012}
2013
2014void __sock_recv_timestamp(struct msghdr *msg, struct sock *sk,
2015 struct sk_buff *skb);
2016void __sock_recv_wifi_status(struct msghdr *msg, struct sock *sk,
2017 struct sk_buff *skb);
2018
2019static inline void
2020sock_recv_timestamp(struct msghdr *msg, struct sock *sk, struct sk_buff *skb)
2021{
2022 ktime_t kt = skb->tstamp;
2023 struct skb_shared_hwtstamps *hwtstamps = skb_hwtstamps(skb);
2024
2025
2026
2027
2028
2029
2030
2031 if (sock_flag(sk, SOCK_RCVTSTAMP) ||
2032 (sk->sk_tsflags & SOF_TIMESTAMPING_RX_SOFTWARE) ||
2033 (kt.tv64 && sk->sk_tsflags & SOF_TIMESTAMPING_SOFTWARE) ||
2034 (hwtstamps->hwtstamp.tv64 &&
2035 (sk->sk_tsflags & SOF_TIMESTAMPING_RAW_HARDWARE)))
2036 __sock_recv_timestamp(msg, sk, skb);
2037 else
2038 sk->sk_stamp = kt;
2039
2040 if (sock_flag(sk, SOCK_WIFI_STATUS) && skb->wifi_acked_valid)
2041 __sock_recv_wifi_status(msg, sk, skb);
2042}
2043
2044void __sock_recv_ts_and_drops(struct msghdr *msg, struct sock *sk,
2045 struct sk_buff *skb);
2046
2047static inline void sock_recv_ts_and_drops(struct msghdr *msg, struct sock *sk,
2048 struct sk_buff *skb)
2049{
2050#define FLAGS_TS_OR_DROPS ((1UL << SOCK_RXQ_OVFL) | \
2051 (1UL << SOCK_RCVTSTAMP))
2052#define TSFLAGS_ANY (SOF_TIMESTAMPING_SOFTWARE | \
2053 SOF_TIMESTAMPING_RAW_HARDWARE)
2054
2055 if (sk->sk_flags & FLAGS_TS_OR_DROPS || sk->sk_tsflags & TSFLAGS_ANY)
2056 __sock_recv_ts_and_drops(msg, sk, skb);
2057 else
2058 sk->sk_stamp = skb->tstamp;
2059}
2060
2061void __sock_tx_timestamp(const struct sock *sk, __u8 *tx_flags);
2062
2063
2064
2065
2066
2067
2068
2069
2070static inline void sock_tx_timestamp(const struct sock *sk, __u8 *tx_flags)
2071{
2072 if (unlikely(sk->sk_tsflags))
2073 __sock_tx_timestamp(sk, tx_flags);
2074 if (unlikely(sock_flag(sk, SOCK_WIFI_STATUS)))
2075 *tx_flags |= SKBTX_WIFI_STATUS;
2076}
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086static inline void sk_eat_skb(struct sock *sk, struct sk_buff *skb)
2087{
2088 __skb_unlink(skb, &sk->sk_receive_queue);
2089 __kfree_skb(skb);
2090}
2091
2092static inline
2093struct net *sock_net(const struct sock *sk)
2094{
2095 return read_pnet(&sk->sk_net);
2096}
2097
2098static inline
2099void sock_net_set(struct sock *sk, struct net *net)
2100{
2101 write_pnet(&sk->sk_net, net);
2102}
2103
2104static inline struct sock *skb_steal_sock(struct sk_buff *skb)
2105{
2106 if (skb->sk) {
2107 struct sock *sk = skb->sk;
2108
2109 skb->destructor = NULL;
2110 skb->sk = NULL;
2111 return sk;
2112 }
2113 return NULL;
2114}
2115
2116
2117
2118
2119static inline bool sk_fullsock(const struct sock *sk)
2120{
2121 return (1 << sk->sk_state) & ~(TCPF_TIME_WAIT | TCPF_NEW_SYN_RECV);
2122}
2123
2124
2125
2126
2127static inline bool sk_listener(const struct sock *sk)
2128{
2129 return (1 << sk->sk_state) & (TCPF_LISTEN | TCPF_NEW_SYN_RECV);
2130}
2131
2132
2133
2134
2135
2136
2137
2138
2139static inline int sk_state_load(const struct sock *sk)
2140{
2141 return smp_load_acquire(&sk->sk_state);
2142}
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152static inline void sk_state_store(struct sock *sk, int newstate)
2153{
2154 smp_store_release(&sk->sk_state, newstate);
2155}
2156
2157void sock_enable_timestamp(struct sock *sk, int flag);
2158int sock_get_timestamp(struct sock *, struct timeval __user *);
2159int sock_get_timestampns(struct sock *, struct timespec __user *);
2160int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len, int level,
2161 int type);
2162
2163bool sk_ns_capable(const struct sock *sk,
2164 struct user_namespace *user_ns, int cap);
2165bool sk_capable(const struct sock *sk, int cap);
2166bool sk_net_capable(const struct sock *sk, int cap);
2167
2168extern __u32 sysctl_wmem_max;
2169extern __u32 sysctl_rmem_max;
2170
2171extern int sysctl_tstamp_allow_data;
2172extern int sysctl_optmem_max;
2173
2174extern __u32 sysctl_wmem_default;
2175extern __u32 sysctl_rmem_default;
2176
2177#endif
2178