1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40#include <linux/module.h>
41
42#include <linux/types.h>
43#include <linux/interrupt.h>
44#include <linux/workqueue.h>
45#include <linux/net.h>
46#include <linux/ktime.h>
47
48#include <linux/sunrpc/clnt.h>
49#include <linux/sunrpc/metrics.h>
50#include <linux/sunrpc/bc_xprt.h>
51#include <linux/rcupdate.h>
52
53#include <trace/events/sunrpc.h>
54
55#include "sunrpc.h"
56
57
58
59
60
61#if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
62# define RPCDBG_FACILITY RPCDBG_XPRT
63#endif
64
65
66
67
68static void xprt_init(struct rpc_xprt *xprt, struct net *net);
69static void xprt_request_init(struct rpc_task *, struct rpc_xprt *);
70static void xprt_connect_status(struct rpc_task *task);
71static int __xprt_get_cong(struct rpc_xprt *, struct rpc_task *);
72static void __xprt_put_cong(struct rpc_xprt *, struct rpc_rqst *);
73static void xprt_destroy(struct rpc_xprt *xprt);
74
75static DEFINE_SPINLOCK(xprt_list_lock);
76static LIST_HEAD(xprt_list);
77
78
79
80
81
82
83
84
85
86
87
88
89
90int xprt_register_transport(struct xprt_class *transport)
91{
92 struct xprt_class *t;
93 int result;
94
95 result = -EEXIST;
96 spin_lock(&xprt_list_lock);
97 list_for_each_entry(t, &xprt_list, list) {
98
99 if (t->ident == transport->ident)
100 goto out;
101 }
102
103 list_add_tail(&transport->list, &xprt_list);
104 printk(KERN_INFO "RPC: Registered %s transport module.\n",
105 transport->name);
106 result = 0;
107
108out:
109 spin_unlock(&xprt_list_lock);
110 return result;
111}
112EXPORT_SYMBOL_GPL(xprt_register_transport);
113
114
115
116
117
118
119
120
121
122int xprt_unregister_transport(struct xprt_class *transport)
123{
124 struct xprt_class *t;
125 int result;
126
127 result = 0;
128 spin_lock(&xprt_list_lock);
129 list_for_each_entry(t, &xprt_list, list) {
130 if (t == transport) {
131 printk(KERN_INFO
132 "RPC: Unregistered %s transport module.\n",
133 transport->name);
134 list_del_init(&transport->list);
135 goto out;
136 }
137 }
138 result = -ENOENT;
139
140out:
141 spin_unlock(&xprt_list_lock);
142 return result;
143}
144EXPORT_SYMBOL_GPL(xprt_unregister_transport);
145
146
147
148
149
150
151
152
153
154int xprt_load_transport(const char *transport_name)
155{
156 struct xprt_class *t;
157 int result;
158
159 result = 0;
160 spin_lock(&xprt_list_lock);
161 list_for_each_entry(t, &xprt_list, list) {
162 if (strcmp(t->name, transport_name) == 0) {
163 spin_unlock(&xprt_list_lock);
164 goto out;
165 }
166 }
167 spin_unlock(&xprt_list_lock);
168 result = request_module("xprt%s", transport_name);
169out:
170 return result;
171}
172EXPORT_SYMBOL_GPL(xprt_load_transport);
173
174
175
176
177
178
179
180
181
182
183int xprt_reserve_xprt(struct rpc_xprt *xprt, struct rpc_task *task)
184{
185 struct rpc_rqst *req = task->tk_rqstp;
186 int priority;
187
188 if (test_and_set_bit(XPRT_LOCKED, &xprt->state)) {
189 if (task == xprt->snd_task)
190 return 1;
191 goto out_sleep;
192 }
193 xprt->snd_task = task;
194 if (req != NULL)
195 req->rq_ntrans++;
196
197 return 1;
198
199out_sleep:
200 dprintk("RPC: %5u failed to lock transport %p\n",
201 task->tk_pid, xprt);
202 task->tk_timeout = 0;
203 task->tk_status = -EAGAIN;
204 if (req == NULL)
205 priority = RPC_PRIORITY_LOW;
206 else if (!req->rq_ntrans)
207 priority = RPC_PRIORITY_NORMAL;
208 else
209 priority = RPC_PRIORITY_HIGH;
210 rpc_sleep_on_priority(&xprt->sending, task, NULL, priority);
211 return 0;
212}
213EXPORT_SYMBOL_GPL(xprt_reserve_xprt);
214
215static void xprt_clear_locked(struct rpc_xprt *xprt)
216{
217 xprt->snd_task = NULL;
218 if (!test_bit(XPRT_CLOSE_WAIT, &xprt->state)) {
219 smp_mb__before_atomic();
220 clear_bit(XPRT_LOCKED, &xprt->state);
221 smp_mb__after_atomic();
222 } else
223 queue_work(rpciod_workqueue, &xprt->task_cleanup);
224}
225
226
227
228
229
230
231
232
233
234int xprt_reserve_xprt_cong(struct rpc_xprt *xprt, struct rpc_task *task)
235{
236 struct rpc_rqst *req = task->tk_rqstp;
237 int priority;
238
239 if (test_and_set_bit(XPRT_LOCKED, &xprt->state)) {
240 if (task == xprt->snd_task)
241 return 1;
242 goto out_sleep;
243 }
244 if (req == NULL) {
245 xprt->snd_task = task;
246 return 1;
247 }
248 if (__xprt_get_cong(xprt, task)) {
249 xprt->snd_task = task;
250 req->rq_ntrans++;
251 return 1;
252 }
253 xprt_clear_locked(xprt);
254out_sleep:
255 if (req)
256 __xprt_put_cong(xprt, req);
257 dprintk("RPC: %5u failed to lock transport %p\n", task->tk_pid, xprt);
258 task->tk_timeout = 0;
259 task->tk_status = -EAGAIN;
260 if (req == NULL)
261 priority = RPC_PRIORITY_LOW;
262 else if (!req->rq_ntrans)
263 priority = RPC_PRIORITY_NORMAL;
264 else
265 priority = RPC_PRIORITY_HIGH;
266 rpc_sleep_on_priority(&xprt->sending, task, NULL, priority);
267 return 0;
268}
269EXPORT_SYMBOL_GPL(xprt_reserve_xprt_cong);
270
271static inline int xprt_lock_write(struct rpc_xprt *xprt, struct rpc_task *task)
272{
273 int retval;
274
275 spin_lock_bh(&xprt->transport_lock);
276 retval = xprt->ops->reserve_xprt(xprt, task);
277 spin_unlock_bh(&xprt->transport_lock);
278 return retval;
279}
280
281static bool __xprt_lock_write_func(struct rpc_task *task, void *data)
282{
283 struct rpc_xprt *xprt = data;
284 struct rpc_rqst *req;
285
286 req = task->tk_rqstp;
287 xprt->snd_task = task;
288 if (req)
289 req->rq_ntrans++;
290 return true;
291}
292
293static void __xprt_lock_write_next(struct rpc_xprt *xprt)
294{
295 if (test_and_set_bit(XPRT_LOCKED, &xprt->state))
296 return;
297
298 if (rpc_wake_up_first(&xprt->sending, __xprt_lock_write_func, xprt))
299 return;
300 xprt_clear_locked(xprt);
301}
302
303static bool __xprt_lock_write_cong_func(struct rpc_task *task, void *data)
304{
305 struct rpc_xprt *xprt = data;
306 struct rpc_rqst *req;
307
308 req = task->tk_rqstp;
309 if (req == NULL) {
310 xprt->snd_task = task;
311 return true;
312 }
313 if (__xprt_get_cong(xprt, task)) {
314 xprt->snd_task = task;
315 req->rq_ntrans++;
316 return true;
317 }
318 return false;
319}
320
321static void __xprt_lock_write_next_cong(struct rpc_xprt *xprt)
322{
323 if (test_and_set_bit(XPRT_LOCKED, &xprt->state))
324 return;
325 if (RPCXPRT_CONGESTED(xprt))
326 goto out_unlock;
327 if (rpc_wake_up_first(&xprt->sending, __xprt_lock_write_cong_func, xprt))
328 return;
329out_unlock:
330 xprt_clear_locked(xprt);
331}
332
333static void xprt_task_clear_bytes_sent(struct rpc_task *task)
334{
335 if (task != NULL) {
336 struct rpc_rqst *req = task->tk_rqstp;
337 if (req != NULL)
338 req->rq_bytes_sent = 0;
339 }
340}
341
342
343
344
345
346
347
348
349void xprt_release_xprt(struct rpc_xprt *xprt, struct rpc_task *task)
350{
351 if (xprt->snd_task == task) {
352 xprt_task_clear_bytes_sent(task);
353 xprt_clear_locked(xprt);
354 __xprt_lock_write_next(xprt);
355 }
356}
357EXPORT_SYMBOL_GPL(xprt_release_xprt);
358
359
360
361
362
363
364
365
366
367void xprt_release_xprt_cong(struct rpc_xprt *xprt, struct rpc_task *task)
368{
369 if (xprt->snd_task == task) {
370 xprt_task_clear_bytes_sent(task);
371 xprt_clear_locked(xprt);
372 __xprt_lock_write_next_cong(xprt);
373 }
374}
375EXPORT_SYMBOL_GPL(xprt_release_xprt_cong);
376
377static inline void xprt_release_write(struct rpc_xprt *xprt, struct rpc_task *task)
378{
379 spin_lock_bh(&xprt->transport_lock);
380 xprt->ops->release_xprt(xprt, task);
381 spin_unlock_bh(&xprt->transport_lock);
382}
383
384
385
386
387
388static int
389__xprt_get_cong(struct rpc_xprt *xprt, struct rpc_task *task)
390{
391 struct rpc_rqst *req = task->tk_rqstp;
392
393 if (req->rq_cong)
394 return 1;
395 dprintk("RPC: %5u xprt_cwnd_limited cong = %lu cwnd = %lu\n",
396 task->tk_pid, xprt->cong, xprt->cwnd);
397 if (RPCXPRT_CONGESTED(xprt))
398 return 0;
399 req->rq_cong = 1;
400 xprt->cong += RPC_CWNDSCALE;
401 return 1;
402}
403
404
405
406
407
408static void
409__xprt_put_cong(struct rpc_xprt *xprt, struct rpc_rqst *req)
410{
411 if (!req->rq_cong)
412 return;
413 req->rq_cong = 0;
414 xprt->cong -= RPC_CWNDSCALE;
415 __xprt_lock_write_next_cong(xprt);
416}
417
418
419
420
421
422
423
424void xprt_release_rqst_cong(struct rpc_task *task)
425{
426 struct rpc_rqst *req = task->tk_rqstp;
427
428 __xprt_put_cong(req->rq_xprt, req);
429}
430EXPORT_SYMBOL_GPL(xprt_release_rqst_cong);
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448void xprt_adjust_cwnd(struct rpc_xprt *xprt, struct rpc_task *task, int result)
449{
450 struct rpc_rqst *req = task->tk_rqstp;
451 unsigned long cwnd = xprt->cwnd;
452
453 if (result >= 0 && cwnd <= xprt->cong) {
454
455
456 cwnd += (RPC_CWNDSCALE * RPC_CWNDSCALE + (cwnd >> 1)) / cwnd;
457 if (cwnd > RPC_MAXCWND(xprt))
458 cwnd = RPC_MAXCWND(xprt);
459 __xprt_lock_write_next_cong(xprt);
460 } else if (result == -ETIMEDOUT) {
461 cwnd >>= 1;
462 if (cwnd < RPC_CWNDSCALE)
463 cwnd = RPC_CWNDSCALE;
464 }
465 dprintk("RPC: cong %ld, cwnd was %ld, now %ld\n",
466 xprt->cong, xprt->cwnd, cwnd);
467 xprt->cwnd = cwnd;
468 __xprt_put_cong(xprt, req);
469}
470EXPORT_SYMBOL_GPL(xprt_adjust_cwnd);
471
472
473
474
475
476
477
478void xprt_wake_pending_tasks(struct rpc_xprt *xprt, int status)
479{
480 if (status < 0)
481 rpc_wake_up_status(&xprt->pending, status);
482 else
483 rpc_wake_up(&xprt->pending);
484}
485EXPORT_SYMBOL_GPL(xprt_wake_pending_tasks);
486
487
488
489
490
491
492
493
494
495
496void xprt_wait_for_buffer_space(struct rpc_task *task, rpc_action action)
497{
498 struct rpc_rqst *req = task->tk_rqstp;
499 struct rpc_xprt *xprt = req->rq_xprt;
500
501 task->tk_timeout = RPC_IS_SOFT(task) ? req->rq_timeout : 0;
502 rpc_sleep_on(&xprt->pending, task, action);
503}
504EXPORT_SYMBOL_GPL(xprt_wait_for_buffer_space);
505
506
507
508
509
510
511
512void xprt_write_space(struct rpc_xprt *xprt)
513{
514 spin_lock_bh(&xprt->transport_lock);
515 if (xprt->snd_task) {
516 dprintk("RPC: write space: waking waiting task on "
517 "xprt %p\n", xprt);
518 rpc_wake_up_queued_task(&xprt->pending, xprt->snd_task);
519 }
520 spin_unlock_bh(&xprt->transport_lock);
521}
522EXPORT_SYMBOL_GPL(xprt_write_space);
523
524
525
526
527
528
529
530
531
532void xprt_set_retrans_timeout_def(struct rpc_task *task)
533{
534 task->tk_timeout = task->tk_rqstp->rq_timeout;
535}
536EXPORT_SYMBOL_GPL(xprt_set_retrans_timeout_def);
537
538
539
540
541
542
543
544void xprt_set_retrans_timeout_rtt(struct rpc_task *task)
545{
546 int timer = task->tk_msg.rpc_proc->p_timer;
547 struct rpc_clnt *clnt = task->tk_client;
548 struct rpc_rtt *rtt = clnt->cl_rtt;
549 struct rpc_rqst *req = task->tk_rqstp;
550 unsigned long max_timeout = clnt->cl_timeout->to_maxval;
551
552 task->tk_timeout = rpc_calc_rto(rtt, timer);
553 task->tk_timeout <<= rpc_ntimeo(rtt, timer) + req->rq_retries;
554 if (task->tk_timeout > max_timeout || task->tk_timeout == 0)
555 task->tk_timeout = max_timeout;
556}
557EXPORT_SYMBOL_GPL(xprt_set_retrans_timeout_rtt);
558
559static void xprt_reset_majortimeo(struct rpc_rqst *req)
560{
561 const struct rpc_timeout *to = req->rq_task->tk_client->cl_timeout;
562
563 req->rq_majortimeo = req->rq_timeout;
564 if (to->to_exponential)
565 req->rq_majortimeo <<= to->to_retries;
566 else
567 req->rq_majortimeo += to->to_increment * to->to_retries;
568 if (req->rq_majortimeo > to->to_maxval || req->rq_majortimeo == 0)
569 req->rq_majortimeo = to->to_maxval;
570 req->rq_majortimeo += jiffies;
571}
572
573
574
575
576
577
578int xprt_adjust_timeout(struct rpc_rqst *req)
579{
580 struct rpc_xprt *xprt = req->rq_xprt;
581 const struct rpc_timeout *to = req->rq_task->tk_client->cl_timeout;
582 int status = 0;
583
584 if (time_before(jiffies, req->rq_majortimeo)) {
585 if (to->to_exponential)
586 req->rq_timeout <<= 1;
587 else
588 req->rq_timeout += to->to_increment;
589 if (to->to_maxval && req->rq_timeout >= to->to_maxval)
590 req->rq_timeout = to->to_maxval;
591 req->rq_retries++;
592 } else {
593 req->rq_timeout = to->to_initval;
594 req->rq_retries = 0;
595 xprt_reset_majortimeo(req);
596
597 spin_lock_bh(&xprt->transport_lock);
598 rpc_init_rtt(req->rq_task->tk_client->cl_rtt, to->to_initval);
599 spin_unlock_bh(&xprt->transport_lock);
600 status = -ETIMEDOUT;
601 }
602
603 if (req->rq_timeout == 0) {
604 printk(KERN_WARNING "xprt_adjust_timeout: rq_timeout = 0!\n");
605 req->rq_timeout = 5 * HZ;
606 }
607 return status;
608}
609
610static void xprt_autoclose(struct work_struct *work)
611{
612 struct rpc_xprt *xprt =
613 container_of(work, struct rpc_xprt, task_cleanup);
614
615 clear_bit(XPRT_CLOSE_WAIT, &xprt->state);
616 xprt->ops->close(xprt);
617 xprt_release_write(xprt, NULL);
618 wake_up_bit(&xprt->state, XPRT_LOCKED);
619}
620
621
622
623
624
625
626void xprt_disconnect_done(struct rpc_xprt *xprt)
627{
628 dprintk("RPC: disconnected transport %p\n", xprt);
629 spin_lock_bh(&xprt->transport_lock);
630 xprt_clear_connected(xprt);
631 xprt_wake_pending_tasks(xprt, -EAGAIN);
632 spin_unlock_bh(&xprt->transport_lock);
633}
634EXPORT_SYMBOL_GPL(xprt_disconnect_done);
635
636
637
638
639
640
641void xprt_force_disconnect(struct rpc_xprt *xprt)
642{
643
644 spin_lock_bh(&xprt->transport_lock);
645 set_bit(XPRT_CLOSE_WAIT, &xprt->state);
646
647 if (test_and_set_bit(XPRT_LOCKED, &xprt->state) == 0)
648 queue_work(rpciod_workqueue, &xprt->task_cleanup);
649 xprt_wake_pending_tasks(xprt, -EAGAIN);
650 spin_unlock_bh(&xprt->transport_lock);
651}
652
653
654
655
656
657
658
659
660
661
662
663
664void xprt_conditional_disconnect(struct rpc_xprt *xprt, unsigned int cookie)
665{
666
667 spin_lock_bh(&xprt->transport_lock);
668 if (cookie != xprt->connect_cookie)
669 goto out;
670 if (test_bit(XPRT_CLOSING, &xprt->state) || !xprt_connected(xprt))
671 goto out;
672 set_bit(XPRT_CLOSE_WAIT, &xprt->state);
673
674 if (test_and_set_bit(XPRT_LOCKED, &xprt->state) == 0)
675 queue_work(rpciod_workqueue, &xprt->task_cleanup);
676 xprt_wake_pending_tasks(xprt, -EAGAIN);
677out:
678 spin_unlock_bh(&xprt->transport_lock);
679}
680
681static void
682xprt_init_autodisconnect(unsigned long data)
683{
684 struct rpc_xprt *xprt = (struct rpc_xprt *)data;
685
686 spin_lock(&xprt->transport_lock);
687 if (!list_empty(&xprt->recv))
688 goto out_abort;
689 if (test_and_set_bit(XPRT_LOCKED, &xprt->state))
690 goto out_abort;
691 spin_unlock(&xprt->transport_lock);
692 queue_work(rpciod_workqueue, &xprt->task_cleanup);
693 return;
694out_abort:
695 spin_unlock(&xprt->transport_lock);
696}
697
698bool xprt_lock_connect(struct rpc_xprt *xprt,
699 struct rpc_task *task,
700 void *cookie)
701{
702 bool ret = false;
703
704 spin_lock_bh(&xprt->transport_lock);
705 if (!test_bit(XPRT_LOCKED, &xprt->state))
706 goto out;
707 if (xprt->snd_task != task)
708 goto out;
709 xprt_task_clear_bytes_sent(task);
710 xprt->snd_task = cookie;
711 ret = true;
712out:
713 spin_unlock_bh(&xprt->transport_lock);
714 return ret;
715}
716
717void xprt_unlock_connect(struct rpc_xprt *xprt, void *cookie)
718{
719 spin_lock_bh(&xprt->transport_lock);
720 if (xprt->snd_task != cookie)
721 goto out;
722 if (!test_bit(XPRT_LOCKED, &xprt->state))
723 goto out;
724 xprt->snd_task =NULL;
725 xprt->ops->release_xprt(xprt, NULL);
726out:
727 spin_unlock_bh(&xprt->transport_lock);
728 wake_up_bit(&xprt->state, XPRT_LOCKED);
729}
730
731
732
733
734
735
736void xprt_connect(struct rpc_task *task)
737{
738 struct rpc_xprt *xprt = task->tk_rqstp->rq_xprt;
739
740 dprintk("RPC: %5u xprt_connect xprt %p %s connected\n", task->tk_pid,
741 xprt, (xprt_connected(xprt) ? "is" : "is not"));
742
743 if (!xprt_bound(xprt)) {
744 task->tk_status = -EAGAIN;
745 return;
746 }
747 if (!xprt_lock_write(xprt, task))
748 return;
749
750 if (test_and_clear_bit(XPRT_CLOSE_WAIT, &xprt->state))
751 xprt->ops->close(xprt);
752
753 if (!xprt_connected(xprt)) {
754 task->tk_rqstp->rq_bytes_sent = 0;
755 task->tk_timeout = task->tk_rqstp->rq_timeout;
756 rpc_sleep_on(&xprt->pending, task, xprt_connect_status);
757
758 if (test_bit(XPRT_CLOSING, &xprt->state))
759 return;
760 if (xprt_test_and_set_connecting(xprt))
761 return;
762 xprt->stat.connect_start = jiffies;
763 xprt->ops->connect(xprt, task);
764 }
765 xprt_release_write(xprt, task);
766}
767
768static void xprt_connect_status(struct rpc_task *task)
769{
770 struct rpc_xprt *xprt = task->tk_rqstp->rq_xprt;
771
772 if (task->tk_status == 0) {
773 xprt->stat.connect_count++;
774 xprt->stat.connect_time += (long)jiffies - xprt->stat.connect_start;
775 dprintk("RPC: %5u xprt_connect_status: connection established\n",
776 task->tk_pid);
777 return;
778 }
779
780 switch (task->tk_status) {
781 case -ECONNREFUSED:
782 case -ECONNRESET:
783 case -ECONNABORTED:
784 case -ENETUNREACH:
785 case -EHOSTUNREACH:
786 case -EPIPE:
787 case -EAGAIN:
788 dprintk("RPC: %5u xprt_connect_status: retrying\n", task->tk_pid);
789 break;
790 case -ETIMEDOUT:
791 dprintk("RPC: %5u xprt_connect_status: connect attempt timed "
792 "out\n", task->tk_pid);
793 break;
794 default:
795 dprintk("RPC: %5u xprt_connect_status: error %d connecting to "
796 "server %s\n", task->tk_pid, -task->tk_status,
797 xprt->servername);
798 task->tk_status = -EIO;
799 }
800}
801
802
803
804
805
806
807
808struct rpc_rqst *xprt_lookup_rqst(struct rpc_xprt *xprt, __be32 xid)
809{
810 struct rpc_rqst *entry;
811
812 list_for_each_entry(entry, &xprt->recv, rq_list)
813 if (entry->rq_xid == xid) {
814 trace_xprt_lookup_rqst(xprt, xid, 0);
815 return entry;
816 }
817
818 dprintk("RPC: xprt_lookup_rqst did not find xid %08x\n",
819 ntohl(xid));
820 trace_xprt_lookup_rqst(xprt, xid, -ENOENT);
821 xprt->stat.bad_xids++;
822 return NULL;
823}
824EXPORT_SYMBOL_GPL(xprt_lookup_rqst);
825
826static void xprt_update_rtt(struct rpc_task *task)
827{
828 struct rpc_rqst *req = task->tk_rqstp;
829 struct rpc_rtt *rtt = task->tk_client->cl_rtt;
830 unsigned int timer = task->tk_msg.rpc_proc->p_timer;
831 long m = usecs_to_jiffies(ktime_to_us(req->rq_rtt));
832
833 if (timer) {
834 if (req->rq_ntrans == 1)
835 rpc_update_rtt(rtt, timer, m);
836 rpc_set_timeo(rtt, timer, req->rq_ntrans - 1);
837 }
838}
839
840
841
842
843
844
845
846
847void xprt_complete_rqst(struct rpc_task *task, int copied)
848{
849 struct rpc_rqst *req = task->tk_rqstp;
850 struct rpc_xprt *xprt = req->rq_xprt;
851
852 dprintk("RPC: %5u xid %08x complete (%d bytes received)\n",
853 task->tk_pid, ntohl(req->rq_xid), copied);
854 trace_xprt_complete_rqst(xprt, req->rq_xid, copied);
855
856 xprt->stat.recvs++;
857 req->rq_rtt = ktime_sub(ktime_get(), req->rq_xtime);
858 if (xprt->ops->timer != NULL)
859 xprt_update_rtt(task);
860
861 list_del_init(&req->rq_list);
862 req->rq_private_buf.len = copied;
863
864
865 smp_wmb();
866 req->rq_reply_bytes_recvd = copied;
867 rpc_wake_up_queued_task(&xprt->pending, task);
868}
869EXPORT_SYMBOL_GPL(xprt_complete_rqst);
870
871static void xprt_timer(struct rpc_task *task)
872{
873 struct rpc_rqst *req = task->tk_rqstp;
874 struct rpc_xprt *xprt = req->rq_xprt;
875
876 if (task->tk_status != -ETIMEDOUT)
877 return;
878 dprintk("RPC: %5u xprt_timer\n", task->tk_pid);
879
880 spin_lock_bh(&xprt->transport_lock);
881 if (!req->rq_reply_bytes_recvd) {
882 if (xprt->ops->timer)
883 xprt->ops->timer(xprt, task);
884 } else
885 task->tk_status = 0;
886 spin_unlock_bh(&xprt->transport_lock);
887}
888
889static inline int xprt_has_timer(struct rpc_xprt *xprt)
890{
891 return xprt->idle_timeout != 0;
892}
893
894
895
896
897
898
899bool xprt_prepare_transmit(struct rpc_task *task)
900{
901 struct rpc_rqst *req = task->tk_rqstp;
902 struct rpc_xprt *xprt = req->rq_xprt;
903 bool ret = false;
904
905 dprintk("RPC: %5u xprt_prepare_transmit\n", task->tk_pid);
906
907 spin_lock_bh(&xprt->transport_lock);
908 if (!req->rq_bytes_sent) {
909 if (req->rq_reply_bytes_recvd) {
910 task->tk_status = req->rq_reply_bytes_recvd;
911 goto out_unlock;
912 }
913 if ((task->tk_flags & RPC_TASK_NO_RETRANS_TIMEOUT)
914 && xprt_connected(xprt)
915 && req->rq_connect_cookie == xprt->connect_cookie) {
916 xprt->ops->set_retrans_timeout(task);
917 rpc_sleep_on(&xprt->pending, task, xprt_timer);
918 goto out_unlock;
919 }
920 }
921 if (!xprt->ops->reserve_xprt(xprt, task)) {
922 task->tk_status = -EAGAIN;
923 goto out_unlock;
924 }
925 ret = true;
926out_unlock:
927 spin_unlock_bh(&xprt->transport_lock);
928 return ret;
929}
930
931void xprt_end_transmit(struct rpc_task *task)
932{
933 xprt_release_write(task->tk_rqstp->rq_xprt, task);
934}
935
936
937
938
939
940
941
942void xprt_transmit(struct rpc_task *task)
943{
944 struct rpc_rqst *req = task->tk_rqstp;
945 struct rpc_xprt *xprt = req->rq_xprt;
946 int status, numreqs;
947
948 dprintk("RPC: %5u xprt_transmit(%u)\n", task->tk_pid, req->rq_slen);
949
950 if (!req->rq_reply_bytes_recvd) {
951 if (list_empty(&req->rq_list) && rpc_reply_expected(task)) {
952
953
954
955 spin_lock_bh(&xprt->transport_lock);
956
957 memcpy(&req->rq_private_buf, &req->rq_rcv_buf,
958 sizeof(req->rq_private_buf));
959
960 list_add_tail(&req->rq_list, &xprt->recv);
961 spin_unlock_bh(&xprt->transport_lock);
962 xprt_reset_majortimeo(req);
963
964 del_singleshot_timer_sync(&xprt->timer);
965 }
966 } else if (!req->rq_bytes_sent)
967 return;
968
969 req->rq_xtime = ktime_get();
970 status = xprt->ops->send_request(task);
971 trace_xprt_transmit(xprt, req->rq_xid, status);
972 if (status != 0) {
973 task->tk_status = status;
974 return;
975 }
976 xprt_inject_disconnect(xprt);
977
978 dprintk("RPC: %5u xmit complete\n", task->tk_pid);
979 task->tk_flags |= RPC_TASK_SENT;
980 spin_lock_bh(&xprt->transport_lock);
981
982 xprt->ops->set_retrans_timeout(task);
983
984 numreqs = atomic_read(&xprt->num_reqs);
985 if (numreqs > xprt->stat.max_slots)
986 xprt->stat.max_slots = numreqs;
987 xprt->stat.sends++;
988 xprt->stat.req_u += xprt->stat.sends - xprt->stat.recvs;
989 xprt->stat.bklog_u += xprt->backlog.qlen;
990 xprt->stat.sending_u += xprt->sending.qlen;
991 xprt->stat.pending_u += xprt->pending.qlen;
992
993
994 if (!xprt_connected(xprt))
995 task->tk_status = -ENOTCONN;
996 else {
997
998
999
1000
1001 if (!req->rq_reply_bytes_recvd && rpc_reply_expected(task))
1002 rpc_sleep_on(&xprt->pending, task, xprt_timer);
1003 req->rq_connect_cookie = xprt->connect_cookie;
1004 }
1005 spin_unlock_bh(&xprt->transport_lock);
1006}
1007
1008static void xprt_add_backlog(struct rpc_xprt *xprt, struct rpc_task *task)
1009{
1010 set_bit(XPRT_CONGESTED, &xprt->state);
1011 rpc_sleep_on(&xprt->backlog, task, NULL);
1012}
1013
1014static void xprt_wake_up_backlog(struct rpc_xprt *xprt)
1015{
1016 if (rpc_wake_up_next(&xprt->backlog) == NULL)
1017 clear_bit(XPRT_CONGESTED, &xprt->state);
1018}
1019
1020static bool xprt_throttle_congested(struct rpc_xprt *xprt, struct rpc_task *task)
1021{
1022 bool ret = false;
1023
1024 if (!test_bit(XPRT_CONGESTED, &xprt->state))
1025 goto out;
1026 spin_lock(&xprt->reserve_lock);
1027 if (test_bit(XPRT_CONGESTED, &xprt->state)) {
1028 rpc_sleep_on(&xprt->backlog, task, NULL);
1029 ret = true;
1030 }
1031 spin_unlock(&xprt->reserve_lock);
1032out:
1033 return ret;
1034}
1035
1036static struct rpc_rqst *xprt_dynamic_alloc_slot(struct rpc_xprt *xprt, gfp_t gfp_flags)
1037{
1038 struct rpc_rqst *req = ERR_PTR(-EAGAIN);
1039
1040 if (!atomic_add_unless(&xprt->num_reqs, 1, xprt->max_reqs))
1041 goto out;
1042 req = kzalloc(sizeof(struct rpc_rqst), gfp_flags);
1043 if (req != NULL)
1044 goto out;
1045 atomic_dec(&xprt->num_reqs);
1046 req = ERR_PTR(-ENOMEM);
1047out:
1048 return req;
1049}
1050
1051static bool xprt_dynamic_free_slot(struct rpc_xprt *xprt, struct rpc_rqst *req)
1052{
1053 if (atomic_add_unless(&xprt->num_reqs, -1, xprt->min_reqs)) {
1054 kfree(req);
1055 return true;
1056 }
1057 return false;
1058}
1059
1060void xprt_alloc_slot(struct rpc_xprt *xprt, struct rpc_task *task)
1061{
1062 struct rpc_rqst *req;
1063
1064 spin_lock(&xprt->reserve_lock);
1065 if (!list_empty(&xprt->free)) {
1066 req = list_entry(xprt->free.next, struct rpc_rqst, rq_list);
1067 list_del(&req->rq_list);
1068 goto out_init_req;
1069 }
1070 req = xprt_dynamic_alloc_slot(xprt, GFP_NOWAIT|__GFP_NOWARN);
1071 if (!IS_ERR(req))
1072 goto out_init_req;
1073 switch (PTR_ERR(req)) {
1074 case -ENOMEM:
1075 dprintk("RPC: dynamic allocation of request slot "
1076 "failed! Retrying\n");
1077 task->tk_status = -ENOMEM;
1078 break;
1079 case -EAGAIN:
1080 xprt_add_backlog(xprt, task);
1081 dprintk("RPC: waiting for request slot\n");
1082 default:
1083 task->tk_status = -EAGAIN;
1084 }
1085 spin_unlock(&xprt->reserve_lock);
1086 return;
1087out_init_req:
1088 task->tk_status = 0;
1089 task->tk_rqstp = req;
1090 xprt_request_init(task, xprt);
1091 spin_unlock(&xprt->reserve_lock);
1092}
1093EXPORT_SYMBOL_GPL(xprt_alloc_slot);
1094
1095void xprt_lock_and_alloc_slot(struct rpc_xprt *xprt, struct rpc_task *task)
1096{
1097
1098
1099
1100
1101
1102 if (xprt_lock_write(xprt, task)) {
1103 xprt_alloc_slot(xprt, task);
1104 xprt_release_write(xprt, task);
1105 }
1106}
1107EXPORT_SYMBOL_GPL(xprt_lock_and_alloc_slot);
1108
1109static void xprt_free_slot(struct rpc_xprt *xprt, struct rpc_rqst *req)
1110{
1111 spin_lock(&xprt->reserve_lock);
1112 if (!xprt_dynamic_free_slot(xprt, req)) {
1113 memset(req, 0, sizeof(*req));
1114 list_add(&req->rq_list, &xprt->free);
1115 }
1116 xprt_wake_up_backlog(xprt);
1117 spin_unlock(&xprt->reserve_lock);
1118}
1119
1120static void xprt_free_all_slots(struct rpc_xprt *xprt)
1121{
1122 struct rpc_rqst *req;
1123 while (!list_empty(&xprt->free)) {
1124 req = list_first_entry(&xprt->free, struct rpc_rqst, rq_list);
1125 list_del(&req->rq_list);
1126 kfree(req);
1127 }
1128}
1129
1130struct rpc_xprt *xprt_alloc(struct net *net, size_t size,
1131 unsigned int num_prealloc,
1132 unsigned int max_alloc)
1133{
1134 struct rpc_xprt *xprt;
1135 struct rpc_rqst *req;
1136 int i;
1137
1138 xprt = kzalloc(size, GFP_KERNEL);
1139 if (xprt == NULL)
1140 goto out;
1141
1142 xprt_init(xprt, net);
1143
1144 for (i = 0; i < num_prealloc; i++) {
1145 req = kzalloc(sizeof(struct rpc_rqst), GFP_KERNEL);
1146 if (!req)
1147 goto out_free;
1148 list_add(&req->rq_list, &xprt->free);
1149 }
1150 if (max_alloc > num_prealloc)
1151 xprt->max_reqs = max_alloc;
1152 else
1153 xprt->max_reqs = num_prealloc;
1154 xprt->min_reqs = num_prealloc;
1155 atomic_set(&xprt->num_reqs, num_prealloc);
1156
1157 return xprt;
1158
1159out_free:
1160 xprt_free(xprt);
1161out:
1162 return NULL;
1163}
1164EXPORT_SYMBOL_GPL(xprt_alloc);
1165
1166void xprt_free(struct rpc_xprt *xprt)
1167{
1168 put_net(xprt->xprt_net);
1169 xprt_free_all_slots(xprt);
1170 kfree_rcu(xprt, rcu);
1171}
1172EXPORT_SYMBOL_GPL(xprt_free);
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182void xprt_reserve(struct rpc_task *task)
1183{
1184 struct rpc_xprt *xprt = task->tk_xprt;
1185
1186 task->tk_status = 0;
1187 if (task->tk_rqstp != NULL)
1188 return;
1189
1190 task->tk_timeout = 0;
1191 task->tk_status = -EAGAIN;
1192 if (!xprt_throttle_congested(xprt, task))
1193 xprt->ops->alloc_slot(xprt, task);
1194}
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205void xprt_retry_reserve(struct rpc_task *task)
1206{
1207 struct rpc_xprt *xprt = task->tk_xprt;
1208
1209 task->tk_status = 0;
1210 if (task->tk_rqstp != NULL)
1211 return;
1212
1213 task->tk_timeout = 0;
1214 task->tk_status = -EAGAIN;
1215 xprt->ops->alloc_slot(xprt, task);
1216}
1217
1218static inline __be32 xprt_alloc_xid(struct rpc_xprt *xprt)
1219{
1220 return (__force __be32)xprt->xid++;
1221}
1222
1223static inline void xprt_init_xid(struct rpc_xprt *xprt)
1224{
1225 xprt->xid = prandom_u32();
1226}
1227
1228static void xprt_request_init(struct rpc_task *task, struct rpc_xprt *xprt)
1229{
1230 struct rpc_rqst *req = task->tk_rqstp;
1231
1232 INIT_LIST_HEAD(&req->rq_list);
1233 req->rq_timeout = task->tk_client->cl_timeout->to_initval;
1234 req->rq_task = task;
1235 req->rq_xprt = xprt;
1236 req->rq_buffer = NULL;
1237 req->rq_xid = xprt_alloc_xid(xprt);
1238 req->rq_connect_cookie = xprt->connect_cookie - 1;
1239 req->rq_bytes_sent = 0;
1240 req->rq_snd_buf.len = 0;
1241 req->rq_snd_buf.buflen = 0;
1242 req->rq_rcv_buf.len = 0;
1243 req->rq_rcv_buf.buflen = 0;
1244 req->rq_release_snd_buf = NULL;
1245 xprt_reset_majortimeo(req);
1246 dprintk("RPC: %5u reserved req %p xid %08x\n", task->tk_pid,
1247 req, ntohl(req->rq_xid));
1248}
1249
1250
1251
1252
1253
1254
1255void xprt_release(struct rpc_task *task)
1256{
1257 struct rpc_xprt *xprt;
1258 struct rpc_rqst *req = task->tk_rqstp;
1259
1260 if (req == NULL) {
1261 if (task->tk_client) {
1262 xprt = task->tk_xprt;
1263 if (xprt->snd_task == task)
1264 xprt_release_write(xprt, task);
1265 }
1266 return;
1267 }
1268
1269 xprt = req->rq_xprt;
1270 if (task->tk_ops->rpc_count_stats != NULL)
1271 task->tk_ops->rpc_count_stats(task, task->tk_calldata);
1272 else if (task->tk_client)
1273 rpc_count_iostats(task, task->tk_client->cl_metrics);
1274 spin_lock_bh(&xprt->transport_lock);
1275 xprt->ops->release_xprt(xprt, task);
1276 if (xprt->ops->release_request)
1277 xprt->ops->release_request(task);
1278 if (!list_empty(&req->rq_list))
1279 list_del(&req->rq_list);
1280 xprt->last_used = jiffies;
1281 if (list_empty(&xprt->recv) && xprt_has_timer(xprt))
1282 mod_timer(&xprt->timer,
1283 xprt->last_used + xprt->idle_timeout);
1284 spin_unlock_bh(&xprt->transport_lock);
1285 if (req->rq_buffer)
1286 xprt->ops->buf_free(req->rq_buffer);
1287 xprt_inject_disconnect(xprt);
1288 if (req->rq_cred != NULL)
1289 put_rpccred(req->rq_cred);
1290 task->tk_rqstp = NULL;
1291 if (req->rq_release_snd_buf)
1292 req->rq_release_snd_buf(req);
1293
1294 dprintk("RPC: %5u release request %p\n", task->tk_pid, req);
1295 if (likely(!bc_prealloc(req)))
1296 xprt_free_slot(xprt, req);
1297 else
1298 xprt_free_bc_request(req);
1299}
1300
1301static void xprt_init(struct rpc_xprt *xprt, struct net *net)
1302{
1303 kref_init(&xprt->kref);
1304
1305 spin_lock_init(&xprt->transport_lock);
1306 spin_lock_init(&xprt->reserve_lock);
1307
1308 INIT_LIST_HEAD(&xprt->free);
1309 INIT_LIST_HEAD(&xprt->recv);
1310#if defined(CONFIG_SUNRPC_BACKCHANNEL)
1311 spin_lock_init(&xprt->bc_pa_lock);
1312 INIT_LIST_HEAD(&xprt->bc_pa_list);
1313#endif
1314 INIT_LIST_HEAD(&xprt->xprt_switch);
1315
1316 xprt->last_used = jiffies;
1317 xprt->cwnd = RPC_INITCWND;
1318 xprt->bind_index = 0;
1319
1320 rpc_init_wait_queue(&xprt->binding, "xprt_binding");
1321 rpc_init_wait_queue(&xprt->pending, "xprt_pending");
1322 rpc_init_priority_wait_queue(&xprt->sending, "xprt_sending");
1323 rpc_init_priority_wait_queue(&xprt->backlog, "xprt_backlog");
1324
1325 xprt_init_xid(xprt);
1326
1327 xprt->xprt_net = get_net(net);
1328}
1329
1330
1331
1332
1333
1334
1335struct rpc_xprt *xprt_create_transport(struct xprt_create *args)
1336{
1337 struct rpc_xprt *xprt;
1338 struct xprt_class *t;
1339
1340 spin_lock(&xprt_list_lock);
1341 list_for_each_entry(t, &xprt_list, list) {
1342 if (t->ident == args->ident) {
1343 spin_unlock(&xprt_list_lock);
1344 goto found;
1345 }
1346 }
1347 spin_unlock(&xprt_list_lock);
1348 dprintk("RPC: transport (%d) not supported\n", args->ident);
1349 return ERR_PTR(-EIO);
1350
1351found:
1352 xprt = t->setup(args);
1353 if (IS_ERR(xprt)) {
1354 dprintk("RPC: xprt_create_transport: failed, %ld\n",
1355 -PTR_ERR(xprt));
1356 goto out;
1357 }
1358 if (args->flags & XPRT_CREATE_NO_IDLE_TIMEOUT)
1359 xprt->idle_timeout = 0;
1360 INIT_WORK(&xprt->task_cleanup, xprt_autoclose);
1361 if (xprt_has_timer(xprt))
1362 setup_timer(&xprt->timer, xprt_init_autodisconnect,
1363 (unsigned long)xprt);
1364 else
1365 init_timer(&xprt->timer);
1366
1367 if (strlen(args->servername) > RPC_MAXNETNAMELEN) {
1368 xprt_destroy(xprt);
1369 return ERR_PTR(-EINVAL);
1370 }
1371 xprt->servername = kstrdup(args->servername, GFP_KERNEL);
1372 if (xprt->servername == NULL) {
1373 xprt_destroy(xprt);
1374 return ERR_PTR(-ENOMEM);
1375 }
1376
1377 rpc_xprt_debugfs_register(xprt);
1378
1379 dprintk("RPC: created transport %p with %u slots\n", xprt,
1380 xprt->max_reqs);
1381out:
1382 return xprt;
1383}
1384
1385
1386
1387
1388
1389
1390static void xprt_destroy(struct rpc_xprt *xprt)
1391{
1392 dprintk("RPC: destroying transport %p\n", xprt);
1393
1394
1395 wait_on_bit_lock(&xprt->state, XPRT_LOCKED, TASK_UNINTERRUPTIBLE);
1396
1397 del_timer_sync(&xprt->timer);
1398
1399 rpc_xprt_debugfs_unregister(xprt);
1400 rpc_destroy_wait_queue(&xprt->binding);
1401 rpc_destroy_wait_queue(&xprt->pending);
1402 rpc_destroy_wait_queue(&xprt->sending);
1403 rpc_destroy_wait_queue(&xprt->backlog);
1404 cancel_work_sync(&xprt->task_cleanup);
1405 kfree(xprt->servername);
1406
1407
1408
1409 xprt->ops->destroy(xprt);
1410}
1411
1412static void xprt_destroy_kref(struct kref *kref)
1413{
1414 xprt_destroy(container_of(kref, struct rpc_xprt, kref));
1415}
1416
1417
1418
1419
1420
1421
1422struct rpc_xprt *xprt_get(struct rpc_xprt *xprt)
1423{
1424 if (xprt != NULL && kref_get_unless_zero(&xprt->kref))
1425 return xprt;
1426 return NULL;
1427}
1428EXPORT_SYMBOL_GPL(xprt_get);
1429
1430
1431
1432
1433
1434
1435void xprt_put(struct rpc_xprt *xprt)
1436{
1437 if (xprt != NULL)
1438 kref_put(&xprt->kref, xprt_destroy_kref);
1439}
1440EXPORT_SYMBOL_GPL(xprt_put);
1441