1
2
3
4
5
6
7
8
9
10
11
12#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
13
14#include <linux/module.h>
15#include <linux/net.h>
16#include <linux/skbuff.h>
17#include <linux/errqueue.h>
18#include <linux/udp.h>
19#include <linux/in.h>
20#include <linux/in6.h>
21#include <linux/icmp.h>
22#include <linux/gfp.h>
23#include <linux/circ_buf.h>
24#include <net/sock.h>
25#include <net/af_rxrpc.h>
26#include <net/ip.h>
27#include "ar-internal.h"
28
29
30
31
32
33static int rxrpc_service_prealloc_one(struct rxrpc_sock *rx,
34 struct rxrpc_backlog *b,
35 rxrpc_notify_rx_t notify_rx,
36 rxrpc_user_attach_call_t user_attach_call,
37 unsigned long user_call_ID, gfp_t gfp)
38{
39 const void *here = __builtin_return_address(0);
40 struct rxrpc_call *call;
41 struct rxrpc_net *rxnet = rxrpc_net(sock_net(&rx->sk));
42 int max, tmp;
43 unsigned int size = RXRPC_BACKLOG_MAX;
44 unsigned int head, tail, call_head, call_tail;
45
46 max = rx->sk.sk_max_ack_backlog;
47 tmp = rx->sk.sk_ack_backlog;
48 if (tmp >= max) {
49 _leave(" = -ENOBUFS [full %u]", max);
50 return -ENOBUFS;
51 }
52 max -= tmp;
53
54
55
56
57
58 call_head = b->call_backlog_head;
59 call_tail = READ_ONCE(b->call_backlog_tail);
60 tmp = CIRC_CNT(call_head, call_tail, size);
61 if (tmp >= max) {
62 _leave(" = -ENOBUFS [enough %u]", tmp);
63 return -ENOBUFS;
64 }
65 max = tmp + 1;
66
67 head = b->peer_backlog_head;
68 tail = READ_ONCE(b->peer_backlog_tail);
69 if (CIRC_CNT(head, tail, size) < max) {
70 struct rxrpc_peer *peer = rxrpc_alloc_peer(rx->local, gfp);
71 if (!peer)
72 return -ENOMEM;
73 b->peer_backlog[head] = peer;
74 smp_store_release(&b->peer_backlog_head,
75 (head + 1) & (size - 1));
76 }
77
78 head = b->conn_backlog_head;
79 tail = READ_ONCE(b->conn_backlog_tail);
80 if (CIRC_CNT(head, tail, size) < max) {
81 struct rxrpc_connection *conn;
82
83 conn = rxrpc_prealloc_service_connection(rxnet, gfp);
84 if (!conn)
85 return -ENOMEM;
86 b->conn_backlog[head] = conn;
87 smp_store_release(&b->conn_backlog_head,
88 (head + 1) & (size - 1));
89
90 trace_rxrpc_conn(conn, rxrpc_conn_new_service,
91 atomic_read(&conn->usage), here);
92 }
93
94
95
96
97 call = rxrpc_alloc_call(gfp);
98 if (!call)
99 return -ENOMEM;
100 call->flags |= (1 << RXRPC_CALL_IS_SERVICE);
101 call->state = RXRPC_CALL_SERVER_PREALLOC;
102
103 trace_rxrpc_call(call, rxrpc_call_new_service,
104 atomic_read(&call->usage),
105 here, (const void *)user_call_ID);
106
107 write_lock(&rx->call_lock);
108 if (user_attach_call) {
109 struct rxrpc_call *xcall;
110 struct rb_node *parent, **pp;
111
112
113 pp = &rx->calls.rb_node;
114 parent = NULL;
115 while (*pp) {
116 parent = *pp;
117 xcall = rb_entry(parent, struct rxrpc_call, sock_node);
118 if (user_call_ID < call->user_call_ID)
119 pp = &(*pp)->rb_left;
120 else if (user_call_ID > call->user_call_ID)
121 pp = &(*pp)->rb_right;
122 else
123 goto id_in_use;
124 }
125
126 call->user_call_ID = user_call_ID;
127 call->notify_rx = notify_rx;
128 rxrpc_get_call(call, rxrpc_call_got_kernel);
129 user_attach_call(call, user_call_ID);
130 rxrpc_get_call(call, rxrpc_call_got_userid);
131 rb_link_node(&call->sock_node, parent, pp);
132 rb_insert_color(&call->sock_node, &rx->calls);
133 set_bit(RXRPC_CALL_HAS_USERID, &call->flags);
134 }
135
136 list_add(&call->sock_link, &rx->sock_calls);
137
138 write_unlock(&rx->call_lock);
139
140 write_lock(&rxnet->call_lock);
141 list_add_tail(&call->link, &rxnet->calls);
142 write_unlock(&rxnet->call_lock);
143
144 b->call_backlog[call_head] = call;
145 smp_store_release(&b->call_backlog_head, (call_head + 1) & (size - 1));
146 _leave(" = 0 [%d -> %lx]", call->debug_id, user_call_ID);
147 return 0;
148
149id_in_use:
150 write_unlock(&rx->call_lock);
151 rxrpc_cleanup_call(call);
152 _leave(" = -EBADSLT");
153 return -EBADSLT;
154}
155
156
157
158
159
160
161
162
163int rxrpc_service_prealloc(struct rxrpc_sock *rx, gfp_t gfp)
164{
165 struct rxrpc_backlog *b = rx->backlog;
166
167 if (!b) {
168 b = kzalloc(sizeof(struct rxrpc_backlog), gfp);
169 if (!b)
170 return -ENOMEM;
171 rx->backlog = b;
172 }
173
174 if (rx->discard_new_call)
175 return 0;
176
177 while (rxrpc_service_prealloc_one(rx, b, NULL, NULL, 0, gfp) == 0)
178 ;
179
180 return 0;
181}
182
183
184
185
186void rxrpc_discard_prealloc(struct rxrpc_sock *rx)
187{
188 struct rxrpc_backlog *b = rx->backlog;
189 struct rxrpc_net *rxnet = rxrpc_net(sock_net(&rx->sk));
190 unsigned int size = RXRPC_BACKLOG_MAX, head, tail;
191
192 if (!b)
193 return;
194 rx->backlog = NULL;
195
196
197
198
199 spin_lock_bh(&rx->incoming_lock);
200 spin_unlock_bh(&rx->incoming_lock);
201
202 head = b->peer_backlog_head;
203 tail = b->peer_backlog_tail;
204 while (CIRC_CNT(head, tail, size) > 0) {
205 struct rxrpc_peer *peer = b->peer_backlog[tail];
206 kfree(peer);
207 tail = (tail + 1) & (size - 1);
208 }
209
210 head = b->conn_backlog_head;
211 tail = b->conn_backlog_tail;
212 while (CIRC_CNT(head, tail, size) > 0) {
213 struct rxrpc_connection *conn = b->conn_backlog[tail];
214 write_lock(&rxnet->conn_lock);
215 list_del(&conn->link);
216 list_del(&conn->proc_link);
217 write_unlock(&rxnet->conn_lock);
218 kfree(conn);
219 tail = (tail + 1) & (size - 1);
220 }
221
222 head = b->call_backlog_head;
223 tail = b->call_backlog_tail;
224 while (CIRC_CNT(head, tail, size) > 0) {
225 struct rxrpc_call *call = b->call_backlog[tail];
226 call->socket = rx;
227 if (rx->discard_new_call) {
228 _debug("discard %lx", call->user_call_ID);
229 rx->discard_new_call(call, call->user_call_ID);
230 rxrpc_put_call(call, rxrpc_call_put_kernel);
231 }
232 rxrpc_call_completed(call);
233 rxrpc_release_call(rx, call);
234 rxrpc_put_call(call, rxrpc_call_put);
235 tail = (tail + 1) & (size - 1);
236 }
237
238 kfree(b);
239}
240
241
242
243
244
245static struct rxrpc_call *rxrpc_alloc_incoming_call(struct rxrpc_sock *rx,
246 struct rxrpc_local *local,
247 struct rxrpc_connection *conn,
248 struct sk_buff *skb)
249{
250 struct rxrpc_backlog *b = rx->backlog;
251 struct rxrpc_peer *peer, *xpeer;
252 struct rxrpc_call *call;
253 unsigned short call_head, conn_head, peer_head;
254 unsigned short call_tail, conn_tail, peer_tail;
255 unsigned short call_count, conn_count;
256
257
258 call_head = smp_load_acquire(&b->call_backlog_head);
259 call_tail = b->call_backlog_tail;
260 call_count = CIRC_CNT(call_head, call_tail, RXRPC_BACKLOG_MAX);
261 conn_head = smp_load_acquire(&b->conn_backlog_head);
262 conn_tail = b->conn_backlog_tail;
263 conn_count = CIRC_CNT(conn_head, conn_tail, RXRPC_BACKLOG_MAX);
264 ASSERTCMP(conn_count, >=, call_count);
265 peer_head = smp_load_acquire(&b->peer_backlog_head);
266 peer_tail = b->peer_backlog_tail;
267 ASSERTCMP(CIRC_CNT(peer_head, peer_tail, RXRPC_BACKLOG_MAX), >=,
268 conn_count);
269
270 if (call_count == 0)
271 return NULL;
272
273 if (!conn) {
274
275
276
277
278
279 xpeer = b->peer_backlog[peer_tail];
280 if (rxrpc_extract_addr_from_skb(local, &xpeer->srx, skb) < 0)
281 return NULL;
282
283 peer = rxrpc_lookup_incoming_peer(local, xpeer);
284 if (peer == xpeer) {
285 b->peer_backlog[peer_tail] = NULL;
286 smp_store_release(&b->peer_backlog_tail,
287 (peer_tail + 1) &
288 (RXRPC_BACKLOG_MAX - 1));
289 }
290
291
292 conn = b->conn_backlog[conn_tail];
293 b->conn_backlog[conn_tail] = NULL;
294 smp_store_release(&b->conn_backlog_tail,
295 (conn_tail + 1) & (RXRPC_BACKLOG_MAX - 1));
296 rxrpc_get_local(local);
297 conn->params.local = local;
298 conn->params.peer = peer;
299 rxrpc_see_connection(conn);
300 rxrpc_new_incoming_connection(rx, conn, skb);
301 } else {
302 rxrpc_get_connection(conn);
303 }
304
305
306 call = b->call_backlog[call_tail];
307 b->call_backlog[call_tail] = NULL;
308 smp_store_release(&b->call_backlog_tail,
309 (call_tail + 1) & (RXRPC_BACKLOG_MAX - 1));
310
311 rxrpc_see_call(call);
312 call->conn = conn;
313 call->peer = rxrpc_get_peer(conn->params.peer);
314 call->cong_cwnd = call->peer->cong_cwnd;
315 return call;
316}
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333struct rxrpc_call *rxrpc_new_incoming_call(struct rxrpc_local *local,
334 struct rxrpc_connection *conn,
335 struct sk_buff *skb)
336{
337 struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
338 struct rxrpc_sock *rx;
339 struct rxrpc_call *call;
340 u16 service_id = sp->hdr.serviceId;
341
342 _enter("");
343
344
345 rx = rcu_dereference(local->service);
346 if (rx && (service_id == rx->srx.srx_service ||
347 service_id == rx->second_service))
348 goto found_service;
349
350 trace_rxrpc_abort("INV", sp->hdr.cid, sp->hdr.callNumber, sp->hdr.seq,
351 RX_INVALID_OPERATION, EOPNOTSUPP);
352 skb->mark = RXRPC_SKB_MARK_LOCAL_ABORT;
353 skb->priority = RX_INVALID_OPERATION;
354 _leave(" = NULL [service]");
355 return NULL;
356
357found_service:
358 spin_lock(&rx->incoming_lock);
359 if (rx->sk.sk_state == RXRPC_SERVER_LISTEN_DISABLED ||
360 rx->sk.sk_state == RXRPC_CLOSE) {
361 trace_rxrpc_abort("CLS", sp->hdr.cid, sp->hdr.callNumber,
362 sp->hdr.seq, RX_INVALID_OPERATION, ESHUTDOWN);
363 skb->mark = RXRPC_SKB_MARK_LOCAL_ABORT;
364 skb->priority = RX_INVALID_OPERATION;
365 _leave(" = NULL [close]");
366 call = NULL;
367 goto out;
368 }
369
370 call = rxrpc_alloc_incoming_call(rx, local, conn, skb);
371 if (!call) {
372 skb->mark = RXRPC_SKB_MARK_BUSY;
373 _leave(" = NULL [busy]");
374 call = NULL;
375 goto out;
376 }
377
378 trace_rxrpc_receive(call, rxrpc_receive_incoming,
379 sp->hdr.serial, sp->hdr.seq);
380
381
382
383
384
385
386
387
388
389
390 if (!mutex_trylock(&call->user_mutex))
391 BUG();
392
393
394 rxrpc_incoming_call(rx, call, skb);
395 conn = call->conn;
396
397 if (rx->notify_new_call)
398 rx->notify_new_call(&rx->sk, call, call->user_call_ID);
399 else
400 sk_acceptq_added(&rx->sk);
401
402 spin_lock(&conn->state_lock);
403 switch (conn->state) {
404 case RXRPC_CONN_SERVICE_UNSECURED:
405 conn->state = RXRPC_CONN_SERVICE_CHALLENGING;
406 set_bit(RXRPC_CONN_EV_CHALLENGE, &call->conn->events);
407 rxrpc_queue_conn(call->conn);
408 break;
409
410 case RXRPC_CONN_SERVICE:
411 write_lock(&call->state_lock);
412 if (rx->discard_new_call)
413 call->state = RXRPC_CALL_SERVER_RECV_REQUEST;
414 else
415 call->state = RXRPC_CALL_SERVER_ACCEPTING;
416 write_unlock(&call->state_lock);
417 break;
418
419 case RXRPC_CONN_REMOTELY_ABORTED:
420 rxrpc_set_call_completion(call, RXRPC_CALL_REMOTELY_ABORTED,
421 conn->remote_abort, -ECONNABORTED);
422 break;
423 case RXRPC_CONN_LOCALLY_ABORTED:
424 rxrpc_abort_call("CON", call, sp->hdr.seq,
425 conn->local_abort, -ECONNABORTED);
426 break;
427 default:
428 BUG();
429 }
430 spin_unlock(&conn->state_lock);
431
432 if (call->state == RXRPC_CALL_SERVER_ACCEPTING)
433 rxrpc_notify_socket(call);
434
435
436
437
438
439
440 rxrpc_put_call(call, rxrpc_call_put);
441
442 _leave(" = %p{%d}", call, call->debug_id);
443out:
444 spin_unlock(&rx->incoming_lock);
445 return call;
446}
447
448
449
450
451
452
453struct rxrpc_call *rxrpc_accept_call(struct rxrpc_sock *rx,
454 unsigned long user_call_ID,
455 rxrpc_notify_rx_t notify_rx)
456 __releases(&rx->sk.sk_lock.slock)
457{
458 struct rxrpc_call *call;
459 struct rb_node *parent, **pp;
460 int ret;
461
462 _enter(",%lx", user_call_ID);
463
464 ASSERT(!irqs_disabled());
465
466 write_lock(&rx->call_lock);
467
468 if (list_empty(&rx->to_be_accepted)) {
469 write_unlock(&rx->call_lock);
470 release_sock(&rx->sk);
471 kleave(" = -ENODATA [empty]");
472 return ERR_PTR(-ENODATA);
473 }
474
475
476 pp = &rx->calls.rb_node;
477 parent = NULL;
478 while (*pp) {
479 parent = *pp;
480 call = rb_entry(parent, struct rxrpc_call, sock_node);
481
482 if (user_call_ID < call->user_call_ID)
483 pp = &(*pp)->rb_left;
484 else if (user_call_ID > call->user_call_ID)
485 pp = &(*pp)->rb_right;
486 else
487 goto id_in_use;
488 }
489
490
491
492
493 call = list_entry(rx->to_be_accepted.next,
494 struct rxrpc_call, accept_link);
495 write_unlock(&rx->call_lock);
496
497
498
499
500
501
502 if (mutex_lock_interruptible(&call->user_mutex) < 0) {
503 release_sock(&rx->sk);
504 kleave(" = -ERESTARTSYS");
505 return ERR_PTR(-ERESTARTSYS);
506 }
507
508 write_lock(&rx->call_lock);
509 list_del_init(&call->accept_link);
510 sk_acceptq_removed(&rx->sk);
511 rxrpc_see_call(call);
512
513
514 pp = &rx->calls.rb_node;
515 parent = NULL;
516 while (*pp) {
517 parent = *pp;
518 call = rb_entry(parent, struct rxrpc_call, sock_node);
519
520 if (user_call_ID < call->user_call_ID)
521 pp = &(*pp)->rb_left;
522 else if (user_call_ID > call->user_call_ID)
523 pp = &(*pp)->rb_right;
524 else
525 BUG();
526 }
527
528 write_lock_bh(&call->state_lock);
529 switch (call->state) {
530 case RXRPC_CALL_SERVER_ACCEPTING:
531 call->state = RXRPC_CALL_SERVER_RECV_REQUEST;
532 break;
533 case RXRPC_CALL_COMPLETE:
534 ret = call->error;
535 goto out_release;
536 default:
537 BUG();
538 }
539
540
541 call->notify_rx = notify_rx;
542 call->user_call_ID = user_call_ID;
543 rxrpc_get_call(call, rxrpc_call_got_userid);
544 rb_link_node(&call->sock_node, parent, pp);
545 rb_insert_color(&call->sock_node, &rx->calls);
546 if (test_and_set_bit(RXRPC_CALL_HAS_USERID, &call->flags))
547 BUG();
548
549 write_unlock_bh(&call->state_lock);
550 write_unlock(&rx->call_lock);
551 rxrpc_notify_socket(call);
552 rxrpc_service_prealloc(rx, GFP_KERNEL);
553 release_sock(&rx->sk);
554 _leave(" = %p{%d}", call, call->debug_id);
555 return call;
556
557out_release:
558 _debug("release %p", call);
559 write_unlock_bh(&call->state_lock);
560 write_unlock(&rx->call_lock);
561 rxrpc_release_call(rx, call);
562 rxrpc_put_call(call, rxrpc_call_put);
563 goto out;
564
565id_in_use:
566 ret = -EBADSLT;
567 write_unlock(&rx->call_lock);
568out:
569 rxrpc_service_prealloc(rx, GFP_KERNEL);
570 release_sock(&rx->sk);
571 _leave(" = %d", ret);
572 return ERR_PTR(ret);
573}
574
575
576
577
578
579int rxrpc_reject_call(struct rxrpc_sock *rx)
580{
581 struct rxrpc_call *call;
582 bool abort = false;
583 int ret;
584
585 _enter("");
586
587 ASSERT(!irqs_disabled());
588
589 write_lock(&rx->call_lock);
590
591 if (list_empty(&rx->to_be_accepted)) {
592 write_unlock(&rx->call_lock);
593 return -ENODATA;
594 }
595
596
597
598
599 call = list_entry(rx->to_be_accepted.next,
600 struct rxrpc_call, accept_link);
601 list_del_init(&call->accept_link);
602 sk_acceptq_removed(&rx->sk);
603 rxrpc_see_call(call);
604
605 write_lock_bh(&call->state_lock);
606 switch (call->state) {
607 case RXRPC_CALL_SERVER_ACCEPTING:
608 __rxrpc_abort_call("REJ", call, 1, RX_USER_ABORT, -ECONNABORTED);
609 abort = true;
610
611 case RXRPC_CALL_COMPLETE:
612 ret = call->error;
613 goto out_discard;
614 default:
615 BUG();
616 }
617
618out_discard:
619 write_unlock_bh(&call->state_lock);
620 write_unlock(&rx->call_lock);
621 if (abort) {
622 rxrpc_send_abort_packet(call);
623 rxrpc_release_call(rx, call);
624 rxrpc_put_call(call, rxrpc_call_put);
625 }
626 rxrpc_service_prealloc(rx, GFP_KERNEL);
627 _leave(" = %d", ret);
628 return ret;
629}
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645int rxrpc_kernel_charge_accept(struct socket *sock,
646 rxrpc_notify_rx_t notify_rx,
647 rxrpc_user_attach_call_t user_attach_call,
648 unsigned long user_call_ID, gfp_t gfp)
649{
650 struct rxrpc_sock *rx = rxrpc_sk(sock->sk);
651 struct rxrpc_backlog *b = rx->backlog;
652
653 if (sock->sk->sk_state == RXRPC_CLOSE)
654 return -ESHUTDOWN;
655
656 return rxrpc_service_prealloc_one(rx, b, notify_rx,
657 user_attach_call, user_call_ID,
658 gfp);
659}
660EXPORT_SYMBOL(rxrpc_kernel_charge_accept);
661