1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23#define pr_fmt(fmt) "MPTCP: " fmt
24
25#include <linux/kernel.h>
26#include <linux/module.h>
27#include <linux/memblock.h>
28#include <linux/ip.h>
29#include <linux/tcp.h>
30#include <net/sock.h>
31#include <net/inet_common.h>
32#include <net/protocol.h>
33#include <net/mptcp.h>
34#include "protocol.h"
35
36#define TOKEN_MAX_CHAIN_LEN 4
37
38struct token_bucket {
39 spinlock_t lock;
40 int chain_len;
41 struct hlist_nulls_head req_chain;
42 struct hlist_nulls_head msk_chain;
43};
44
45static struct token_bucket *token_hash __read_mostly;
46static unsigned int token_mask __read_mostly;
47
48static struct token_bucket *token_bucket(u32 token)
49{
50 return &token_hash[token & token_mask];
51}
52
53
54static struct mptcp_subflow_request_sock *
55__token_lookup_req(struct token_bucket *t, u32 token)
56{
57 struct mptcp_subflow_request_sock *req;
58 struct hlist_nulls_node *pos;
59
60 hlist_nulls_for_each_entry_rcu(req, pos, &t->req_chain, token_node)
61 if (req->token == token)
62 return req;
63 return NULL;
64}
65
66
67static struct mptcp_sock *
68__token_lookup_msk(struct token_bucket *t, u32 token)
69{
70 struct hlist_nulls_node *pos;
71 struct sock *sk;
72
73 sk_nulls_for_each_rcu(sk, pos, &t->msk_chain)
74 if (mptcp_sk(sk)->token == token)
75 return mptcp_sk(sk);
76 return NULL;
77}
78
79static bool __token_bucket_busy(struct token_bucket *t, u32 token)
80{
81 return !token || t->chain_len >= TOKEN_MAX_CHAIN_LEN ||
82 __token_lookup_req(t, token) || __token_lookup_msk(t, token);
83}
84
85static void mptcp_crypto_key_gen_sha(u64 *key, u32 *token, u64 *idsn)
86{
87
88
89
90
91
92
93 get_random_bytes(key, sizeof(u64));
94 mptcp_crypto_key_sha(*key, token, idsn);
95}
96
97
98
99
100
101
102
103
104
105
106
107
108int mptcp_token_new_request(struct request_sock *req)
109{
110 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
111 struct token_bucket *bucket;
112 u32 token;
113
114 mptcp_crypto_key_sha(subflow_req->local_key,
115 &subflow_req->token,
116 &subflow_req->idsn);
117 pr_debug("req=%p local_key=%llu, token=%u, idsn=%llu\n",
118 req, subflow_req->local_key, subflow_req->token,
119 subflow_req->idsn);
120
121 token = subflow_req->token;
122 bucket = token_bucket(token);
123 spin_lock_bh(&bucket->lock);
124 if (__token_bucket_busy(bucket, token)) {
125 spin_unlock_bh(&bucket->lock);
126 return -EBUSY;
127 }
128
129 hlist_nulls_add_head_rcu(&subflow_req->token_node, &bucket->req_chain);
130 bucket->chain_len++;
131 spin_unlock_bh(&bucket->lock);
132 return 0;
133}
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151int mptcp_token_new_connect(struct sock *sk)
152{
153 struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
154 struct mptcp_sock *msk = mptcp_sk(subflow->conn);
155 int retries = MPTCP_TOKEN_MAX_RETRIES;
156 struct token_bucket *bucket;
157
158again:
159 mptcp_crypto_key_gen_sha(&subflow->local_key, &subflow->token,
160 &subflow->idsn);
161
162 bucket = token_bucket(subflow->token);
163 spin_lock_bh(&bucket->lock);
164 if (__token_bucket_busy(bucket, subflow->token)) {
165 spin_unlock_bh(&bucket->lock);
166 if (!--retries)
167 return -EBUSY;
168 goto again;
169 }
170
171 pr_debug("ssk=%p, local_key=%llu, token=%u, idsn=%llu\n",
172 sk, subflow->local_key, subflow->token, subflow->idsn);
173
174 WRITE_ONCE(msk->token, subflow->token);
175 __sk_nulls_add_node_rcu((struct sock *)msk, &bucket->msk_chain);
176 bucket->chain_len++;
177 spin_unlock_bh(&bucket->lock);
178 return 0;
179}
180
181
182
183
184
185
186
187
188
189void mptcp_token_accept(struct mptcp_subflow_request_sock *req,
190 struct mptcp_sock *msk)
191{
192 struct mptcp_subflow_request_sock *pos;
193 struct token_bucket *bucket;
194
195 bucket = token_bucket(req->token);
196 spin_lock_bh(&bucket->lock);
197
198
199 pos = __token_lookup_req(bucket, req->token);
200 if (!WARN_ON_ONCE(pos != req))
201 hlist_nulls_del_init_rcu(&req->token_node);
202 __sk_nulls_add_node_rcu((struct sock *)msk, &bucket->msk_chain);
203 spin_unlock_bh(&bucket->lock);
204}
205
206bool mptcp_token_exists(u32 token)
207{
208 struct hlist_nulls_node *pos;
209 struct token_bucket *bucket;
210 struct mptcp_sock *msk;
211 struct sock *sk;
212
213 rcu_read_lock();
214 bucket = token_bucket(token);
215
216again:
217 sk_nulls_for_each_rcu(sk, pos, &bucket->msk_chain) {
218 msk = mptcp_sk(sk);
219 if (READ_ONCE(msk->token) == token)
220 goto found;
221 }
222 if (get_nulls_value(pos) != (token & token_mask))
223 goto again;
224
225 rcu_read_unlock();
226 return false;
227found:
228 rcu_read_unlock();
229 return true;
230}
231
232
233
234
235
236
237
238
239
240
241struct mptcp_sock *mptcp_token_get_sock(u32 token)
242{
243 struct hlist_nulls_node *pos;
244 struct token_bucket *bucket;
245 struct mptcp_sock *msk;
246 struct sock *sk;
247
248 rcu_read_lock();
249 bucket = token_bucket(token);
250
251again:
252 sk_nulls_for_each_rcu(sk, pos, &bucket->msk_chain) {
253 msk = mptcp_sk(sk);
254 if (READ_ONCE(msk->token) != token)
255 continue;
256 if (!refcount_inc_not_zero(&sk->sk_refcnt))
257 goto not_found;
258 if (READ_ONCE(msk->token) != token) {
259 sock_put(sk);
260 goto again;
261 }
262 goto found;
263 }
264 if (get_nulls_value(pos) != (token & token_mask))
265 goto again;
266
267not_found:
268 msk = NULL;
269
270found:
271 rcu_read_unlock();
272 return msk;
273}
274EXPORT_SYMBOL_GPL(mptcp_token_get_sock);
275
276
277
278
279
280
281
282
283
284
285
286
287
288struct mptcp_sock *mptcp_token_iter_next(const struct net *net, long *s_slot,
289 long *s_num)
290{
291 struct mptcp_sock *ret = NULL;
292 struct hlist_nulls_node *pos;
293 int slot, num = 0;
294
295 for (slot = *s_slot; slot <= token_mask; *s_num = 0, slot++) {
296 struct token_bucket *bucket = &token_hash[slot];
297 struct sock *sk;
298
299 num = 0;
300
301 if (hlist_nulls_empty(&bucket->msk_chain))
302 continue;
303
304 rcu_read_lock();
305 sk_nulls_for_each_rcu(sk, pos, &bucket->msk_chain) {
306 ++num;
307 if (!net_eq(sock_net(sk), net))
308 continue;
309
310 if (num <= *s_num)
311 continue;
312
313 if (!refcount_inc_not_zero(&sk->sk_refcnt))
314 continue;
315
316 if (!net_eq(sock_net(sk), net)) {
317 sock_put(sk);
318 continue;
319 }
320
321 ret = mptcp_sk(sk);
322 rcu_read_unlock();
323 goto out;
324 }
325 rcu_read_unlock();
326 }
327
328out:
329 *s_slot = slot;
330 *s_num = num;
331 return ret;
332}
333EXPORT_SYMBOL_GPL(mptcp_token_iter_next);
334
335
336
337
338
339
340
341void mptcp_token_destroy_request(struct request_sock *req)
342{
343 struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
344 struct mptcp_subflow_request_sock *pos;
345 struct token_bucket *bucket;
346
347 if (hlist_nulls_unhashed(&subflow_req->token_node))
348 return;
349
350 bucket = token_bucket(subflow_req->token);
351 spin_lock_bh(&bucket->lock);
352 pos = __token_lookup_req(bucket, subflow_req->token);
353 if (!WARN_ON_ONCE(pos != subflow_req)) {
354 hlist_nulls_del_init_rcu(&pos->token_node);
355 bucket->chain_len--;
356 }
357 spin_unlock_bh(&bucket->lock);
358}
359
360
361
362
363
364
365
366void mptcp_token_destroy(struct mptcp_sock *msk)
367{
368 struct token_bucket *bucket;
369 struct mptcp_sock *pos;
370
371 if (sk_unhashed((struct sock *)msk))
372 return;
373
374 bucket = token_bucket(msk->token);
375 spin_lock_bh(&bucket->lock);
376 pos = __token_lookup_msk(bucket, msk->token);
377 if (!WARN_ON_ONCE(pos != msk)) {
378 __sk_nulls_del_node_init_rcu((struct sock *)pos);
379 bucket->chain_len--;
380 }
381 spin_unlock_bh(&bucket->lock);
382}
383
384void __init mptcp_token_init(void)
385{
386 int i;
387
388 token_hash = alloc_large_system_hash("MPTCP token",
389 sizeof(struct token_bucket),
390 0,
391 20,
392 HASH_ZERO,
393 NULL,
394 &token_mask,
395 0,
396 64 * 1024);
397 for (i = 0; i < token_mask + 1; ++i) {
398 INIT_HLIST_NULLS_HEAD(&token_hash[i].req_chain, i);
399 INIT_HLIST_NULLS_HEAD(&token_hash[i].msk_chain, i);
400 spin_lock_init(&token_hash[i].lock);
401 }
402}
403
404#if IS_MODULE(CONFIG_MPTCP_KUNIT_TEST)
405EXPORT_SYMBOL_GPL(mptcp_token_new_request);
406EXPORT_SYMBOL_GPL(mptcp_token_new_connect);
407EXPORT_SYMBOL_GPL(mptcp_token_accept);
408EXPORT_SYMBOL_GPL(mptcp_token_destroy_request);
409EXPORT_SYMBOL_GPL(mptcp_token_destroy);
410#endif
411