1
2
3
4
5
6
7
8
9
10
11
12
13
14#include <linux/linkage.h>
15#include <linux/sched/signal.h>
16#include <linux/errno.h>
17#include <linux/net.h>
18#include <linux/in.h>
19#include <linux/mm.h>
20#include <linux/interrupt.h>
21#include <linux/module.h>
22#include <linux/kthread.h>
23#include <linux/slab.h>
24
25#include <linux/sunrpc/types.h>
26#include <linux/sunrpc/xdr.h>
27#include <linux/sunrpc/stats.h>
28#include <linux/sunrpc/svcsock.h>
29#include <linux/sunrpc/clnt.h>
30#include <linux/sunrpc/bc_xprt.h>
31
32#include <trace/events/sunrpc.h>
33
34#include "fail.h"
35
36#define RPCDBG_FACILITY RPCDBG_SVCDSP
37
38static void svc_unregister(const struct svc_serv *serv, struct net *net);
39
40#define svc_serv_is_pooled(serv) ((serv)->sv_ops->svo_function)
41
42#define SVC_POOL_DEFAULT SVC_POOL_GLOBAL
43
44
45
46
47
48struct svc_pool_map svc_pool_map = {
49 .mode = SVC_POOL_DEFAULT
50};
51EXPORT_SYMBOL_GPL(svc_pool_map);
52
53static DEFINE_MUTEX(svc_pool_map_mutex);
54
55static int
56param_set_pool_mode(const char *val, const struct kernel_param *kp)
57{
58 int *ip = (int *)kp->arg;
59 struct svc_pool_map *m = &svc_pool_map;
60 int err;
61
62 mutex_lock(&svc_pool_map_mutex);
63
64 err = -EBUSY;
65 if (m->count)
66 goto out;
67
68 err = 0;
69 if (!strncmp(val, "auto", 4))
70 *ip = SVC_POOL_AUTO;
71 else if (!strncmp(val, "global", 6))
72 *ip = SVC_POOL_GLOBAL;
73 else if (!strncmp(val, "percpu", 6))
74 *ip = SVC_POOL_PERCPU;
75 else if (!strncmp(val, "pernode", 7))
76 *ip = SVC_POOL_PERNODE;
77 else
78 err = -EINVAL;
79
80out:
81 mutex_unlock(&svc_pool_map_mutex);
82 return err;
83}
84
85static int
86param_get_pool_mode(char *buf, const struct kernel_param *kp)
87{
88 int *ip = (int *)kp->arg;
89
90 switch (*ip)
91 {
92 case SVC_POOL_AUTO:
93 return strlcpy(buf, "auto\n", 20);
94 case SVC_POOL_GLOBAL:
95 return strlcpy(buf, "global\n", 20);
96 case SVC_POOL_PERCPU:
97 return strlcpy(buf, "percpu\n", 20);
98 case SVC_POOL_PERNODE:
99 return strlcpy(buf, "pernode\n", 20);
100 default:
101 return sprintf(buf, "%d\n", *ip);
102 }
103}
104
105module_param_call(pool_mode, param_set_pool_mode, param_get_pool_mode,
106 &svc_pool_map.mode, 0644);
107
108
109
110
111
112static int
113svc_pool_map_choose_mode(void)
114{
115 unsigned int node;
116
117 if (nr_online_nodes > 1) {
118
119
120
121
122 return SVC_POOL_PERNODE;
123 }
124
125 node = first_online_node;
126 if (nr_cpus_node(node) > 2) {
127
128
129
130
131
132
133 return SVC_POOL_PERCPU;
134 }
135
136
137 return SVC_POOL_GLOBAL;
138}
139
140
141
142
143
144static int
145svc_pool_map_alloc_arrays(struct svc_pool_map *m, unsigned int maxpools)
146{
147 m->to_pool = kcalloc(maxpools, sizeof(unsigned int), GFP_KERNEL);
148 if (!m->to_pool)
149 goto fail;
150 m->pool_to = kcalloc(maxpools, sizeof(unsigned int), GFP_KERNEL);
151 if (!m->pool_to)
152 goto fail_free;
153
154 return 0;
155
156fail_free:
157 kfree(m->to_pool);
158 m->to_pool = NULL;
159fail:
160 return -ENOMEM;
161}
162
163
164
165
166
167static int
168svc_pool_map_init_percpu(struct svc_pool_map *m)
169{
170 unsigned int maxpools = nr_cpu_ids;
171 unsigned int pidx = 0;
172 unsigned int cpu;
173 int err;
174
175 err = svc_pool_map_alloc_arrays(m, maxpools);
176 if (err)
177 return err;
178
179 for_each_online_cpu(cpu) {
180 BUG_ON(pidx >= maxpools);
181 m->to_pool[cpu] = pidx;
182 m->pool_to[pidx] = cpu;
183 pidx++;
184 }
185
186
187 return pidx;
188};
189
190
191
192
193
194
195static int
196svc_pool_map_init_pernode(struct svc_pool_map *m)
197{
198 unsigned int maxpools = nr_node_ids;
199 unsigned int pidx = 0;
200 unsigned int node;
201 int err;
202
203 err = svc_pool_map_alloc_arrays(m, maxpools);
204 if (err)
205 return err;
206
207 for_each_node_with_cpus(node) {
208
209 BUG_ON(pidx > maxpools);
210 m->to_pool[node] = pidx;
211 m->pool_to[pidx] = node;
212 pidx++;
213 }
214
215
216 return pidx;
217}
218
219
220
221
222
223
224
225unsigned int
226svc_pool_map_get(void)
227{
228 struct svc_pool_map *m = &svc_pool_map;
229 int npools = -1;
230
231 mutex_lock(&svc_pool_map_mutex);
232
233 if (m->count++) {
234 mutex_unlock(&svc_pool_map_mutex);
235 return m->npools;
236 }
237
238 if (m->mode == SVC_POOL_AUTO)
239 m->mode = svc_pool_map_choose_mode();
240
241 switch (m->mode) {
242 case SVC_POOL_PERCPU:
243 npools = svc_pool_map_init_percpu(m);
244 break;
245 case SVC_POOL_PERNODE:
246 npools = svc_pool_map_init_pernode(m);
247 break;
248 }
249
250 if (npools < 0) {
251
252 npools = 1;
253 m->mode = SVC_POOL_GLOBAL;
254 }
255 m->npools = npools;
256
257 mutex_unlock(&svc_pool_map_mutex);
258 return m->npools;
259}
260EXPORT_SYMBOL_GPL(svc_pool_map_get);
261
262
263
264
265
266
267
268
269void
270svc_pool_map_put(void)
271{
272 struct svc_pool_map *m = &svc_pool_map;
273
274 mutex_lock(&svc_pool_map_mutex);
275
276 if (!--m->count) {
277 kfree(m->to_pool);
278 m->to_pool = NULL;
279 kfree(m->pool_to);
280 m->pool_to = NULL;
281 m->npools = 0;
282 }
283
284 mutex_unlock(&svc_pool_map_mutex);
285}
286EXPORT_SYMBOL_GPL(svc_pool_map_put);
287
288static int svc_pool_map_get_node(unsigned int pidx)
289{
290 const struct svc_pool_map *m = &svc_pool_map;
291
292 if (m->count) {
293 if (m->mode == SVC_POOL_PERCPU)
294 return cpu_to_node(m->pool_to[pidx]);
295 if (m->mode == SVC_POOL_PERNODE)
296 return m->pool_to[pidx];
297 }
298 return NUMA_NO_NODE;
299}
300
301
302
303
304static inline void
305svc_pool_map_set_cpumask(struct task_struct *task, unsigned int pidx)
306{
307 struct svc_pool_map *m = &svc_pool_map;
308 unsigned int node = m->pool_to[pidx];
309
310
311
312
313
314 WARN_ON_ONCE(m->count == 0);
315 if (m->count == 0)
316 return;
317
318 switch (m->mode) {
319 case SVC_POOL_PERCPU:
320 {
321 set_cpus_allowed_ptr(task, cpumask_of(node));
322 break;
323 }
324 case SVC_POOL_PERNODE:
325 {
326 set_cpus_allowed_ptr(task, cpumask_of_node(node));
327 break;
328 }
329 }
330}
331
332
333
334
335
336
337struct svc_pool *
338svc_pool_for_cpu(struct svc_serv *serv, int cpu)
339{
340 struct svc_pool_map *m = &svc_pool_map;
341 unsigned int pidx = 0;
342
343
344
345
346
347
348 if (svc_serv_is_pooled(serv)) {
349 switch (m->mode) {
350 case SVC_POOL_PERCPU:
351 pidx = m->to_pool[cpu];
352 break;
353 case SVC_POOL_PERNODE:
354 pidx = m->to_pool[cpu_to_node(cpu)];
355 break;
356 }
357 }
358 return &serv->sv_pools[pidx % serv->sv_nrpools];
359}
360
361int svc_rpcb_setup(struct svc_serv *serv, struct net *net)
362{
363 int err;
364
365 err = rpcb_create_local(net);
366 if (err)
367 return err;
368
369
370 svc_unregister(serv, net);
371 return 0;
372}
373EXPORT_SYMBOL_GPL(svc_rpcb_setup);
374
375void svc_rpcb_cleanup(struct svc_serv *serv, struct net *net)
376{
377 svc_unregister(serv, net);
378 rpcb_put_local(net);
379}
380EXPORT_SYMBOL_GPL(svc_rpcb_cleanup);
381
382static int svc_uses_rpcbind(struct svc_serv *serv)
383{
384 struct svc_program *progp;
385 unsigned int i;
386
387 for (progp = serv->sv_program; progp; progp = progp->pg_next) {
388 for (i = 0; i < progp->pg_nvers; i++) {
389 if (progp->pg_vers[i] == NULL)
390 continue;
391 if (!progp->pg_vers[i]->vs_hidden)
392 return 1;
393 }
394 }
395
396 return 0;
397}
398
399int svc_bind(struct svc_serv *serv, struct net *net)
400{
401 if (!svc_uses_rpcbind(serv))
402 return 0;
403 return svc_rpcb_setup(serv, net);
404}
405EXPORT_SYMBOL_GPL(svc_bind);
406
407#if defined(CONFIG_SUNRPC_BACKCHANNEL)
408static void
409__svc_init_bc(struct svc_serv *serv)
410{
411 INIT_LIST_HEAD(&serv->sv_cb_list);
412 spin_lock_init(&serv->sv_cb_lock);
413 init_waitqueue_head(&serv->sv_cb_waitq);
414}
415#else
416static void
417__svc_init_bc(struct svc_serv *serv)
418{
419}
420#endif
421
422
423
424
425static struct svc_serv *
426__svc_create(struct svc_program *prog, unsigned int bufsize, int npools,
427 const struct svc_serv_ops *ops)
428{
429 struct svc_serv *serv;
430 unsigned int vers;
431 unsigned int xdrsize;
432 unsigned int i;
433
434 if (!(serv = kzalloc(sizeof(*serv), GFP_KERNEL)))
435 return NULL;
436 serv->sv_name = prog->pg_name;
437 serv->sv_program = prog;
438 serv->sv_nrthreads = 1;
439 serv->sv_stats = prog->pg_stats;
440 if (bufsize > RPCSVC_MAXPAYLOAD)
441 bufsize = RPCSVC_MAXPAYLOAD;
442 serv->sv_max_payload = bufsize? bufsize : 4096;
443 serv->sv_max_mesg = roundup(serv->sv_max_payload + PAGE_SIZE, PAGE_SIZE);
444 serv->sv_ops = ops;
445 xdrsize = 0;
446 while (prog) {
447 prog->pg_lovers = prog->pg_nvers-1;
448 for (vers=0; vers<prog->pg_nvers ; vers++)
449 if (prog->pg_vers[vers]) {
450 prog->pg_hivers = vers;
451 if (prog->pg_lovers > vers)
452 prog->pg_lovers = vers;
453 if (prog->pg_vers[vers]->vs_xdrsize > xdrsize)
454 xdrsize = prog->pg_vers[vers]->vs_xdrsize;
455 }
456 prog = prog->pg_next;
457 }
458 serv->sv_xdrsize = xdrsize;
459 INIT_LIST_HEAD(&serv->sv_tempsocks);
460 INIT_LIST_HEAD(&serv->sv_permsocks);
461 timer_setup(&serv->sv_temptimer, NULL, 0);
462 spin_lock_init(&serv->sv_lock);
463
464 __svc_init_bc(serv);
465
466 serv->sv_nrpools = npools;
467 serv->sv_pools =
468 kcalloc(serv->sv_nrpools, sizeof(struct svc_pool),
469 GFP_KERNEL);
470 if (!serv->sv_pools) {
471 kfree(serv);
472 return NULL;
473 }
474
475 for (i = 0; i < serv->sv_nrpools; i++) {
476 struct svc_pool *pool = &serv->sv_pools[i];
477
478 dprintk("svc: initialising pool %u for %s\n",
479 i, serv->sv_name);
480
481 pool->sp_id = i;
482 INIT_LIST_HEAD(&pool->sp_sockets);
483 INIT_LIST_HEAD(&pool->sp_all_threads);
484 spin_lock_init(&pool->sp_lock);
485 }
486
487 return serv;
488}
489
490struct svc_serv *
491svc_create(struct svc_program *prog, unsigned int bufsize,
492 const struct svc_serv_ops *ops)
493{
494 return __svc_create(prog, bufsize, 1, ops);
495}
496EXPORT_SYMBOL_GPL(svc_create);
497
498struct svc_serv *
499svc_create_pooled(struct svc_program *prog, unsigned int bufsize,
500 const struct svc_serv_ops *ops)
501{
502 struct svc_serv *serv;
503 unsigned int npools = svc_pool_map_get();
504
505 serv = __svc_create(prog, bufsize, npools, ops);
506 if (!serv)
507 goto out_err;
508 return serv;
509out_err:
510 svc_pool_map_put();
511 return NULL;
512}
513EXPORT_SYMBOL_GPL(svc_create_pooled);
514
515void svc_shutdown_net(struct svc_serv *serv, struct net *net)
516{
517 svc_close_net(serv, net);
518
519 if (serv->sv_ops->svo_shutdown)
520 serv->sv_ops->svo_shutdown(serv, net);
521}
522EXPORT_SYMBOL_GPL(svc_shutdown_net);
523
524
525
526
527
528void
529svc_destroy(struct svc_serv *serv)
530{
531 dprintk("svc: svc_destroy(%s, %d)\n",
532 serv->sv_program->pg_name,
533 serv->sv_nrthreads);
534
535 if (serv->sv_nrthreads) {
536 if (--(serv->sv_nrthreads) != 0) {
537 svc_sock_update_bufs(serv);
538 return;
539 }
540 } else
541 printk("svc_destroy: no threads for serv=%p!\n", serv);
542
543 del_timer_sync(&serv->sv_temptimer);
544
545
546
547
548
549 BUG_ON(!list_empty(&serv->sv_permsocks));
550 BUG_ON(!list_empty(&serv->sv_tempsocks));
551
552 cache_clean_deferred(serv);
553
554 if (svc_serv_is_pooled(serv))
555 svc_pool_map_put();
556
557 kfree(serv->sv_pools);
558 kfree(serv);
559}
560EXPORT_SYMBOL_GPL(svc_destroy);
561
562
563
564
565
566static int
567svc_init_buffer(struct svc_rqst *rqstp, unsigned int size, int node)
568{
569 unsigned int pages, arghi;
570
571
572 if (svc_is_backchannel(rqstp))
573 return 1;
574
575 pages = size / PAGE_SIZE + 1;
576
577
578 arghi = 0;
579 WARN_ON_ONCE(pages > RPCSVC_MAXPAGES);
580 if (pages > RPCSVC_MAXPAGES)
581 pages = RPCSVC_MAXPAGES;
582 while (pages) {
583 struct page *p = alloc_pages_node(node, GFP_KERNEL, 0);
584 if (!p)
585 break;
586 rqstp->rq_pages[arghi++] = p;
587 pages--;
588 }
589 return pages == 0;
590}
591
592
593
594
595static void
596svc_release_buffer(struct svc_rqst *rqstp)
597{
598 unsigned int i;
599
600 for (i = 0; i < ARRAY_SIZE(rqstp->rq_pages); i++)
601 if (rqstp->rq_pages[i])
602 put_page(rqstp->rq_pages[i]);
603}
604
605struct svc_rqst *
606svc_rqst_alloc(struct svc_serv *serv, struct svc_pool *pool, int node)
607{
608 struct svc_rqst *rqstp;
609
610 rqstp = kzalloc_node(sizeof(*rqstp), GFP_KERNEL, node);
611 if (!rqstp)
612 return rqstp;
613
614 __set_bit(RQ_BUSY, &rqstp->rq_flags);
615 spin_lock_init(&rqstp->rq_lock);
616 rqstp->rq_server = serv;
617 rqstp->rq_pool = pool;
618
619 rqstp->rq_scratch_page = alloc_pages_node(node, GFP_KERNEL, 0);
620 if (!rqstp->rq_scratch_page)
621 goto out_enomem;
622
623 rqstp->rq_argp = kmalloc_node(serv->sv_xdrsize, GFP_KERNEL, node);
624 if (!rqstp->rq_argp)
625 goto out_enomem;
626
627 rqstp->rq_resp = kmalloc_node(serv->sv_xdrsize, GFP_KERNEL, node);
628 if (!rqstp->rq_resp)
629 goto out_enomem;
630
631 if (!svc_init_buffer(rqstp, serv->sv_max_mesg, node))
632 goto out_enomem;
633
634 return rqstp;
635out_enomem:
636 svc_rqst_free(rqstp);
637 return NULL;
638}
639EXPORT_SYMBOL_GPL(svc_rqst_alloc);
640
641struct svc_rqst *
642svc_prepare_thread(struct svc_serv *serv, struct svc_pool *pool, int node)
643{
644 struct svc_rqst *rqstp;
645
646 rqstp = svc_rqst_alloc(serv, pool, node);
647 if (!rqstp)
648 return ERR_PTR(-ENOMEM);
649
650 serv->sv_nrthreads++;
651 spin_lock_bh(&pool->sp_lock);
652 pool->sp_nrthreads++;
653 list_add_rcu(&rqstp->rq_all, &pool->sp_all_threads);
654 spin_unlock_bh(&pool->sp_lock);
655 return rqstp;
656}
657EXPORT_SYMBOL_GPL(svc_prepare_thread);
658
659
660
661
662static inline struct svc_pool *
663choose_pool(struct svc_serv *serv, struct svc_pool *pool, unsigned int *state)
664{
665 if (pool != NULL)
666 return pool;
667
668 return &serv->sv_pools[(*state)++ % serv->sv_nrpools];
669}
670
671
672
673
674static inline struct task_struct *
675choose_victim(struct svc_serv *serv, struct svc_pool *pool, unsigned int *state)
676{
677 unsigned int i;
678 struct task_struct *task = NULL;
679
680 if (pool != NULL) {
681 spin_lock_bh(&pool->sp_lock);
682 } else {
683
684 for (i = 0; i < serv->sv_nrpools; i++) {
685 pool = &serv->sv_pools[--(*state) % serv->sv_nrpools];
686 spin_lock_bh(&pool->sp_lock);
687 if (!list_empty(&pool->sp_all_threads))
688 goto found_pool;
689 spin_unlock_bh(&pool->sp_lock);
690 }
691 return NULL;
692 }
693
694found_pool:
695 if (!list_empty(&pool->sp_all_threads)) {
696 struct svc_rqst *rqstp;
697
698
699
700
701
702 rqstp = list_entry(pool->sp_all_threads.next, struct svc_rqst, rq_all);
703 set_bit(RQ_VICTIM, &rqstp->rq_flags);
704 list_del_rcu(&rqstp->rq_all);
705 task = rqstp->rq_task;
706 }
707 spin_unlock_bh(&pool->sp_lock);
708
709 return task;
710}
711
712
713static int
714svc_start_kthreads(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
715{
716 struct svc_rqst *rqstp;
717 struct task_struct *task;
718 struct svc_pool *chosen_pool;
719 unsigned int state = serv->sv_nrthreads-1;
720 int node;
721
722 do {
723 nrservs--;
724 chosen_pool = choose_pool(serv, pool, &state);
725
726 node = svc_pool_map_get_node(chosen_pool->sp_id);
727 rqstp = svc_prepare_thread(serv, chosen_pool, node);
728 if (IS_ERR(rqstp))
729 return PTR_ERR(rqstp);
730
731 __module_get(serv->sv_ops->svo_module);
732 task = kthread_create_on_node(serv->sv_ops->svo_function, rqstp,
733 node, "%s", serv->sv_name);
734 if (IS_ERR(task)) {
735 module_put(serv->sv_ops->svo_module);
736 svc_exit_thread(rqstp);
737 return PTR_ERR(task);
738 }
739
740 rqstp->rq_task = task;
741 if (serv->sv_nrpools > 1)
742 svc_pool_map_set_cpumask(task, chosen_pool->sp_id);
743
744 svc_sock_update_bufs(serv);
745 wake_up_process(task);
746 } while (nrservs > 0);
747
748 return 0;
749}
750
751
752
753static int
754svc_signal_kthreads(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
755{
756 struct task_struct *task;
757 unsigned int state = serv->sv_nrthreads-1;
758
759
760 do {
761 task = choose_victim(serv, pool, &state);
762 if (task == NULL)
763 break;
764 send_sig(SIGINT, task, 1);
765 nrservs++;
766 } while (nrservs < 0);
767
768 return 0;
769}
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785int
786svc_set_num_threads(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
787{
788 if (pool == NULL) {
789
790 nrservs -= (serv->sv_nrthreads-1);
791 } else {
792 spin_lock_bh(&pool->sp_lock);
793 nrservs -= pool->sp_nrthreads;
794 spin_unlock_bh(&pool->sp_lock);
795 }
796
797 if (nrservs > 0)
798 return svc_start_kthreads(serv, pool, nrservs);
799 if (nrservs < 0)
800 return svc_signal_kthreads(serv, pool, nrservs);
801 return 0;
802}
803EXPORT_SYMBOL_GPL(svc_set_num_threads);
804
805
806static int
807svc_stop_kthreads(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
808{
809 struct task_struct *task;
810 unsigned int state = serv->sv_nrthreads-1;
811
812
813 do {
814 task = choose_victim(serv, pool, &state);
815 if (task == NULL)
816 break;
817 kthread_stop(task);
818 nrservs++;
819 } while (nrservs < 0);
820 return 0;
821}
822
823int
824svc_set_num_threads_sync(struct svc_serv *serv, struct svc_pool *pool, int nrservs)
825{
826 if (pool == NULL) {
827
828 nrservs -= (serv->sv_nrthreads-1);
829 } else {
830 spin_lock_bh(&pool->sp_lock);
831 nrservs -= pool->sp_nrthreads;
832 spin_unlock_bh(&pool->sp_lock);
833 }
834
835 if (nrservs > 0)
836 return svc_start_kthreads(serv, pool, nrservs);
837 if (nrservs < 0)
838 return svc_stop_kthreads(serv, pool, nrservs);
839 return 0;
840}
841EXPORT_SYMBOL_GPL(svc_set_num_threads_sync);
842
843
844
845
846
847
848
849
850
851void svc_rqst_replace_page(struct svc_rqst *rqstp, struct page *page)
852{
853 if (*rqstp->rq_next_page) {
854 if (!pagevec_space(&rqstp->rq_pvec))
855 __pagevec_release(&rqstp->rq_pvec);
856 pagevec_add(&rqstp->rq_pvec, *rqstp->rq_next_page);
857 }
858
859 get_page(page);
860 *(rqstp->rq_next_page++) = page;
861}
862EXPORT_SYMBOL_GPL(svc_rqst_replace_page);
863
864
865
866
867
868void
869svc_rqst_free(struct svc_rqst *rqstp)
870{
871 svc_release_buffer(rqstp);
872 if (rqstp->rq_scratch_page)
873 put_page(rqstp->rq_scratch_page);
874 kfree(rqstp->rq_resp);
875 kfree(rqstp->rq_argp);
876 kfree(rqstp->rq_auth_data);
877 kfree_rcu(rqstp, rq_rcu_head);
878}
879EXPORT_SYMBOL_GPL(svc_rqst_free);
880
881void
882svc_exit_thread(struct svc_rqst *rqstp)
883{
884 struct svc_serv *serv = rqstp->rq_server;
885 struct svc_pool *pool = rqstp->rq_pool;
886
887 spin_lock_bh(&pool->sp_lock);
888 pool->sp_nrthreads--;
889 if (!test_and_set_bit(RQ_VICTIM, &rqstp->rq_flags))
890 list_del_rcu(&rqstp->rq_all);
891 spin_unlock_bh(&pool->sp_lock);
892
893 svc_rqst_free(rqstp);
894
895
896 if (serv)
897 svc_destroy(serv);
898}
899EXPORT_SYMBOL_GPL(svc_exit_thread);
900
901
902
903
904
905
906
907
908
909
910
911static int __svc_rpcb_register4(struct net *net, const u32 program,
912 const u32 version,
913 const unsigned short protocol,
914 const unsigned short port)
915{
916 const struct sockaddr_in sin = {
917 .sin_family = AF_INET,
918 .sin_addr.s_addr = htonl(INADDR_ANY),
919 .sin_port = htons(port),
920 };
921 const char *netid;
922 int error;
923
924 switch (protocol) {
925 case IPPROTO_UDP:
926 netid = RPCBIND_NETID_UDP;
927 break;
928 case IPPROTO_TCP:
929 netid = RPCBIND_NETID_TCP;
930 break;
931 default:
932 return -ENOPROTOOPT;
933 }
934
935 error = rpcb_v4_register(net, program, version,
936 (const struct sockaddr *)&sin, netid);
937
938
939
940
941
942 if (error == -EPROTONOSUPPORT)
943 error = rpcb_register(net, program, version, protocol, port);
944
945 return error;
946}
947
948#if IS_ENABLED(CONFIG_IPV6)
949
950
951
952
953
954
955
956
957
958
959static int __svc_rpcb_register6(struct net *net, const u32 program,
960 const u32 version,
961 const unsigned short protocol,
962 const unsigned short port)
963{
964 const struct sockaddr_in6 sin6 = {
965 .sin6_family = AF_INET6,
966 .sin6_addr = IN6ADDR_ANY_INIT,
967 .sin6_port = htons(port),
968 };
969 const char *netid;
970 int error;
971
972 switch (protocol) {
973 case IPPROTO_UDP:
974 netid = RPCBIND_NETID_UDP6;
975 break;
976 case IPPROTO_TCP:
977 netid = RPCBIND_NETID_TCP6;
978 break;
979 default:
980 return -ENOPROTOOPT;
981 }
982
983 error = rpcb_v4_register(net, program, version,
984 (const struct sockaddr *)&sin6, netid);
985
986
987
988
989
990 if (error == -EPROTONOSUPPORT)
991 error = -EAFNOSUPPORT;
992
993 return error;
994}
995#endif
996
997
998
999
1000
1001
1002
1003static int __svc_register(struct net *net, const char *progname,
1004 const u32 program, const u32 version,
1005 const int family,
1006 const unsigned short protocol,
1007 const unsigned short port)
1008{
1009 int error = -EAFNOSUPPORT;
1010
1011 switch (family) {
1012 case PF_INET:
1013 error = __svc_rpcb_register4(net, program, version,
1014 protocol, port);
1015 break;
1016#if IS_ENABLED(CONFIG_IPV6)
1017 case PF_INET6:
1018 error = __svc_rpcb_register6(net, program, version,
1019 protocol, port);
1020#endif
1021 }
1022
1023 trace_svc_register(progname, version, protocol, port, family, error);
1024 return error;
1025}
1026
1027int svc_rpcbind_set_version(struct net *net,
1028 const struct svc_program *progp,
1029 u32 version, int family,
1030 unsigned short proto,
1031 unsigned short port)
1032{
1033 return __svc_register(net, progp->pg_name, progp->pg_prog,
1034 version, family, proto, port);
1035
1036}
1037EXPORT_SYMBOL_GPL(svc_rpcbind_set_version);
1038
1039int svc_generic_rpcbind_set(struct net *net,
1040 const struct svc_program *progp,
1041 u32 version, int family,
1042 unsigned short proto,
1043 unsigned short port)
1044{
1045 const struct svc_version *vers = progp->pg_vers[version];
1046 int error;
1047
1048 if (vers == NULL)
1049 return 0;
1050
1051 if (vers->vs_hidden) {
1052 trace_svc_noregister(progp->pg_name, version, proto,
1053 port, family, 0);
1054 return 0;
1055 }
1056
1057
1058
1059
1060
1061 if (vers->vs_need_cong_ctrl && proto == IPPROTO_UDP)
1062 return 0;
1063
1064 error = svc_rpcbind_set_version(net, progp, version,
1065 family, proto, port);
1066
1067 return (vers->vs_rpcb_optnl) ? 0 : error;
1068}
1069EXPORT_SYMBOL_GPL(svc_generic_rpcbind_set);
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081int svc_register(const struct svc_serv *serv, struct net *net,
1082 const int family, const unsigned short proto,
1083 const unsigned short port)
1084{
1085 struct svc_program *progp;
1086 unsigned int i;
1087 int error = 0;
1088
1089 WARN_ON_ONCE(proto == 0 && port == 0);
1090 if (proto == 0 && port == 0)
1091 return -EINVAL;
1092
1093 for (progp = serv->sv_program; progp; progp = progp->pg_next) {
1094 for (i = 0; i < progp->pg_nvers; i++) {
1095
1096 error = progp->pg_rpcbind_set(net, progp, i,
1097 family, proto, port);
1098 if (error < 0) {
1099 printk(KERN_WARNING "svc: failed to register "
1100 "%sv%u RPC service (errno %d).\n",
1101 progp->pg_name, i, -error);
1102 break;
1103 }
1104 }
1105 }
1106
1107 return error;
1108}
1109
1110
1111
1112
1113
1114
1115
1116
1117static void __svc_unregister(struct net *net, const u32 program, const u32 version,
1118 const char *progname)
1119{
1120 int error;
1121
1122 error = rpcb_v4_register(net, program, version, NULL, "");
1123
1124
1125
1126
1127
1128 if (error == -EPROTONOSUPPORT)
1129 error = rpcb_register(net, program, version, 0, 0);
1130
1131 trace_svc_unregister(progname, version, error);
1132}
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142static void svc_unregister(const struct svc_serv *serv, struct net *net)
1143{
1144 struct svc_program *progp;
1145 unsigned long flags;
1146 unsigned int i;
1147
1148 clear_thread_flag(TIF_SIGPENDING);
1149
1150 for (progp = serv->sv_program; progp; progp = progp->pg_next) {
1151 for (i = 0; i < progp->pg_nvers; i++) {
1152 if (progp->pg_vers[i] == NULL)
1153 continue;
1154 if (progp->pg_vers[i]->vs_hidden)
1155 continue;
1156 __svc_unregister(net, progp->pg_prog, i, progp->pg_name);
1157 }
1158 }
1159
1160 spin_lock_irqsave(¤t->sighand->siglock, flags);
1161 recalc_sigpending();
1162 spin_unlock_irqrestore(¤t->sighand->siglock, flags);
1163}
1164
1165
1166
1167
1168#if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
1169static __printf(2, 3)
1170void svc_printk(struct svc_rqst *rqstp, const char *fmt, ...)
1171{
1172 struct va_format vaf;
1173 va_list args;
1174 char buf[RPC_MAX_ADDRBUFLEN];
1175
1176 va_start(args, fmt);
1177
1178 vaf.fmt = fmt;
1179 vaf.va = &args;
1180
1181 dprintk("svc: %s: %pV", svc_print_addr(rqstp, buf, sizeof(buf)), &vaf);
1182
1183 va_end(args);
1184}
1185#else
1186static __printf(2,3) void svc_printk(struct svc_rqst *rqstp, const char *fmt, ...) {}
1187#endif
1188
1189__be32
1190svc_generic_init_request(struct svc_rqst *rqstp,
1191 const struct svc_program *progp,
1192 struct svc_process_info *ret)
1193{
1194 const struct svc_version *versp = NULL;
1195 const struct svc_procedure *procp = NULL;
1196
1197 if (rqstp->rq_vers >= progp->pg_nvers )
1198 goto err_bad_vers;
1199 versp = progp->pg_vers[rqstp->rq_vers];
1200 if (!versp)
1201 goto err_bad_vers;
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214 if (versp->vs_need_cong_ctrl && rqstp->rq_xprt &&
1215 !test_bit(XPT_CONG_CTRL, &rqstp->rq_xprt->xpt_flags))
1216 goto err_bad_vers;
1217
1218 if (rqstp->rq_proc >= versp->vs_nproc)
1219 goto err_bad_proc;
1220 rqstp->rq_procinfo = procp = &versp->vs_proc[rqstp->rq_proc];
1221 if (!procp)
1222 goto err_bad_proc;
1223
1224
1225 memset(rqstp->rq_argp, 0, procp->pc_argsize);
1226 memset(rqstp->rq_resp, 0, procp->pc_ressize);
1227
1228
1229 versp->vs_count[rqstp->rq_proc]++;
1230
1231 ret->dispatch = versp->vs_dispatch;
1232 return rpc_success;
1233err_bad_vers:
1234 ret->mismatch.lovers = progp->pg_lovers;
1235 ret->mismatch.hivers = progp->pg_hivers;
1236 return rpc_prog_mismatch;
1237err_bad_proc:
1238 return rpc_proc_unavail;
1239}
1240EXPORT_SYMBOL_GPL(svc_generic_init_request);
1241
1242
1243
1244
1245static int
1246svc_process_common(struct svc_rqst *rqstp, struct kvec *argv, struct kvec *resv)
1247{
1248 struct svc_program *progp;
1249 const struct svc_procedure *procp = NULL;
1250 struct svc_serv *serv = rqstp->rq_server;
1251 struct svc_process_info process;
1252 __be32 *statp;
1253 u32 prog, vers;
1254 __be32 rpc_stat;
1255 int auth_res, rc;
1256 __be32 *reply_statp;
1257
1258 rpc_stat = rpc_success;
1259
1260 if (argv->iov_len < 6*4)
1261 goto err_short_len;
1262
1263
1264 set_bit(RQ_SPLICE_OK, &rqstp->rq_flags);
1265
1266 set_bit(RQ_USEDEFERRAL, &rqstp->rq_flags);
1267 clear_bit(RQ_DROPME, &rqstp->rq_flags);
1268
1269 svc_putu32(resv, rqstp->rq_xid);
1270
1271 vers = svc_getnl(argv);
1272
1273
1274 svc_putnl(resv, 1);
1275
1276 if (vers != 2)
1277 goto err_bad_rpc;
1278
1279
1280 reply_statp = resv->iov_base + resv->iov_len;
1281
1282 svc_putnl(resv, 0);
1283
1284 rqstp->rq_prog = prog = svc_getnl(argv);
1285 rqstp->rq_vers = svc_getnl(argv);
1286 rqstp->rq_proc = svc_getnl(argv);
1287
1288 for (progp = serv->sv_program; progp; progp = progp->pg_next)
1289 if (prog == progp->pg_prog)
1290 break;
1291
1292
1293
1294
1295
1296
1297 auth_res = svc_authenticate(rqstp);
1298
1299 if (auth_res == SVC_OK && progp)
1300 auth_res = progp->pg_authenticate(rqstp);
1301 if (auth_res != SVC_OK)
1302 trace_svc_authenticate(rqstp, auth_res);
1303 switch (auth_res) {
1304 case SVC_OK:
1305 break;
1306 case SVC_GARBAGE:
1307 goto err_garbage;
1308 case SVC_SYSERR:
1309 rpc_stat = rpc_system_err;
1310 goto err_bad;
1311 case SVC_DENIED:
1312 goto err_bad_auth;
1313 case SVC_CLOSE:
1314 goto close;
1315 case SVC_DROP:
1316 goto dropit;
1317 case SVC_COMPLETE:
1318 goto sendit;
1319 }
1320
1321 if (progp == NULL)
1322 goto err_bad_prog;
1323
1324 rpc_stat = progp->pg_init_request(rqstp, progp, &process);
1325 switch (rpc_stat) {
1326 case rpc_success:
1327 break;
1328 case rpc_prog_unavail:
1329 goto err_bad_prog;
1330 case rpc_prog_mismatch:
1331 goto err_bad_vers;
1332 case rpc_proc_unavail:
1333 goto err_bad_proc;
1334 }
1335
1336 procp = rqstp->rq_procinfo;
1337
1338 if (!procp || !procp->pc_func)
1339 goto err_bad_proc;
1340
1341
1342 serv->sv_stats->rpccnt++;
1343 trace_svc_process(rqstp, progp->pg_name);
1344
1345
1346 statp = resv->iov_base +resv->iov_len;
1347 svc_putnl(resv, RPC_SUCCESS);
1348
1349
1350
1351
1352 if (procp->pc_xdrressize)
1353 svc_reserve_auth(rqstp, procp->pc_xdrressize<<2);
1354
1355
1356 rc = process.dispatch(rqstp, statp);
1357 if (procp->pc_release)
1358 procp->pc_release(rqstp);
1359 if (!rc)
1360 goto dropit;
1361 if (rqstp->rq_auth_stat != rpc_auth_ok)
1362 goto err_bad_auth;
1363
1364
1365 if (*statp != rpc_success)
1366 resv->iov_len = ((void*)statp) - resv->iov_base + 4;
1367
1368 if (procp->pc_encode == NULL)
1369 goto dropit;
1370
1371 sendit:
1372 if (svc_authorise(rqstp))
1373 goto close_xprt;
1374 return 1;
1375
1376 dropit:
1377 svc_authorise(rqstp);
1378 dprintk("svc: svc_process dropit\n");
1379 return 0;
1380
1381 close:
1382 svc_authorise(rqstp);
1383close_xprt:
1384 if (rqstp->rq_xprt && test_bit(XPT_TEMP, &rqstp->rq_xprt->xpt_flags))
1385 svc_close_xprt(rqstp->rq_xprt);
1386 dprintk("svc: svc_process close\n");
1387 return 0;
1388
1389err_short_len:
1390 svc_printk(rqstp, "short len %zd, dropping request\n",
1391 argv->iov_len);
1392 goto close_xprt;
1393
1394err_bad_rpc:
1395 serv->sv_stats->rpcbadfmt++;
1396 svc_putnl(resv, 1);
1397 svc_putnl(resv, 0);
1398 svc_putnl(resv, 2);
1399 svc_putnl(resv, 2);
1400 goto sendit;
1401
1402err_bad_auth:
1403 dprintk("svc: authentication failed (%d)\n",
1404 be32_to_cpu(rqstp->rq_auth_stat));
1405 serv->sv_stats->rpcbadauth++;
1406
1407 xdr_ressize_check(rqstp, reply_statp);
1408 svc_putnl(resv, 1);
1409 svc_putnl(resv, 1);
1410 svc_putu32(resv, rqstp->rq_auth_stat);
1411 goto sendit;
1412
1413err_bad_prog:
1414 dprintk("svc: unknown program %d\n", prog);
1415 serv->sv_stats->rpcbadfmt++;
1416 svc_putnl(resv, RPC_PROG_UNAVAIL);
1417 goto sendit;
1418
1419err_bad_vers:
1420 svc_printk(rqstp, "unknown version (%d for prog %d, %s)\n",
1421 rqstp->rq_vers, rqstp->rq_prog, progp->pg_name);
1422
1423 serv->sv_stats->rpcbadfmt++;
1424 svc_putnl(resv, RPC_PROG_MISMATCH);
1425 svc_putnl(resv, process.mismatch.lovers);
1426 svc_putnl(resv, process.mismatch.hivers);
1427 goto sendit;
1428
1429err_bad_proc:
1430 svc_printk(rqstp, "unknown procedure (%d)\n", rqstp->rq_proc);
1431
1432 serv->sv_stats->rpcbadfmt++;
1433 svc_putnl(resv, RPC_PROC_UNAVAIL);
1434 goto sendit;
1435
1436err_garbage:
1437 svc_printk(rqstp, "failed to decode args\n");
1438
1439 rpc_stat = rpc_garbage_args;
1440err_bad:
1441 serv->sv_stats->rpcbadfmt++;
1442 svc_putnl(resv, ntohl(rpc_stat));
1443 goto sendit;
1444}
1445
1446
1447
1448
1449int
1450svc_process(struct svc_rqst *rqstp)
1451{
1452 struct kvec *argv = &rqstp->rq_arg.head[0];
1453 struct kvec *resv = &rqstp->rq_res.head[0];
1454 struct svc_serv *serv = rqstp->rq_server;
1455 u32 dir;
1456
1457#if IS_ENABLED(CONFIG_FAIL_SUNRPC)
1458 if (!fail_sunrpc.ignore_server_disconnect &&
1459 should_fail(&fail_sunrpc.attr, 1))
1460 svc_xprt_deferred_close(rqstp->rq_xprt);
1461#endif
1462
1463
1464
1465
1466
1467 rqstp->rq_next_page = &rqstp->rq_respages[1];
1468 resv->iov_base = page_address(rqstp->rq_respages[0]);
1469 resv->iov_len = 0;
1470 rqstp->rq_res.pages = rqstp->rq_respages + 1;
1471 rqstp->rq_res.len = 0;
1472 rqstp->rq_res.page_base = 0;
1473 rqstp->rq_res.page_len = 0;
1474 rqstp->rq_res.buflen = PAGE_SIZE;
1475 rqstp->rq_res.tail[0].iov_base = NULL;
1476 rqstp->rq_res.tail[0].iov_len = 0;
1477
1478 dir = svc_getnl(argv);
1479 if (dir != 0) {
1480
1481 svc_printk(rqstp, "bad direction %d, dropping request\n", dir);
1482 serv->sv_stats->rpcbadfmt++;
1483 goto out_drop;
1484 }
1485
1486
1487 if (likely(svc_process_common(rqstp, argv, resv)))
1488 return svc_send(rqstp);
1489
1490out_drop:
1491 svc_drop(rqstp);
1492 return 0;
1493}
1494EXPORT_SYMBOL_GPL(svc_process);
1495
1496#if defined(CONFIG_SUNRPC_BACKCHANNEL)
1497
1498
1499
1500
1501int
1502bc_svc_process(struct svc_serv *serv, struct rpc_rqst *req,
1503 struct svc_rqst *rqstp)
1504{
1505 struct kvec *argv = &rqstp->rq_arg.head[0];
1506 struct kvec *resv = &rqstp->rq_res.head[0];
1507 struct rpc_task *task;
1508 int proc_error;
1509 int error;
1510
1511 dprintk("svc: %s(%p)\n", __func__, req);
1512
1513
1514 rqstp->rq_xid = req->rq_xid;
1515 rqstp->rq_prot = req->rq_xprt->prot;
1516 rqstp->rq_server = serv;
1517 rqstp->rq_bc_net = req->rq_xprt->xprt_net;
1518
1519 rqstp->rq_addrlen = sizeof(req->rq_xprt->addr);
1520 memcpy(&rqstp->rq_addr, &req->rq_xprt->addr, rqstp->rq_addrlen);
1521 memcpy(&rqstp->rq_arg, &req->rq_rcv_buf, sizeof(rqstp->rq_arg));
1522 memcpy(&rqstp->rq_res, &req->rq_snd_buf, sizeof(rqstp->rq_res));
1523
1524
1525 rqstp->rq_arg.len = req->rq_private_buf.len;
1526 if (rqstp->rq_arg.len <= rqstp->rq_arg.head[0].iov_len) {
1527 rqstp->rq_arg.head[0].iov_len = rqstp->rq_arg.len;
1528 rqstp->rq_arg.page_len = 0;
1529 } else if (rqstp->rq_arg.len <= rqstp->rq_arg.head[0].iov_len +
1530 rqstp->rq_arg.page_len)
1531 rqstp->rq_arg.page_len = rqstp->rq_arg.len -
1532 rqstp->rq_arg.head[0].iov_len;
1533 else
1534 rqstp->rq_arg.len = rqstp->rq_arg.head[0].iov_len +
1535 rqstp->rq_arg.page_len;
1536
1537
1538 resv->iov_len = 0;
1539
1540
1541
1542
1543
1544 svc_getu32(argv);
1545 svc_getnl(argv);
1546
1547
1548 proc_error = svc_process_common(rqstp, argv, resv);
1549
1550 atomic_dec(&req->rq_xprt->bc_slot_count);
1551 if (!proc_error) {
1552
1553 xprt_free_bc_request(req);
1554 error = -EINVAL;
1555 goto out;
1556 }
1557
1558 memcpy(&req->rq_snd_buf, &rqstp->rq_res, sizeof(req->rq_snd_buf));
1559 task = rpc_run_bc_task(req);
1560 if (IS_ERR(task)) {
1561 error = PTR_ERR(task);
1562 goto out;
1563 }
1564
1565 WARN_ON_ONCE(atomic_read(&task->tk_count) != 1);
1566 error = task->tk_status;
1567 rpc_put_task(task);
1568
1569out:
1570 dprintk("svc: %s(), error=%d\n", __func__, error);
1571 return error;
1572}
1573EXPORT_SYMBOL_GPL(bc_svc_process);
1574#endif
1575
1576
1577
1578
1579u32 svc_max_payload(const struct svc_rqst *rqstp)
1580{
1581 u32 max = rqstp->rq_xprt->xpt_class->xcl_max_payload;
1582
1583 if (rqstp->rq_server->sv_max_payload < max)
1584 max = rqstp->rq_server->sv_max_payload;
1585 return max;
1586}
1587EXPORT_SYMBOL_GPL(svc_max_payload);
1588
1589
1590
1591
1592
1593
1594
1595
1596const char *svc_proc_name(const struct svc_rqst *rqstp)
1597{
1598 if (rqstp && rqstp->rq_procinfo)
1599 return rqstp->rq_procinfo->pc_name;
1600 return "unknown";
1601}
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613int svc_encode_result_payload(struct svc_rqst *rqstp, unsigned int offset,
1614 unsigned int length)
1615{
1616 return rqstp->rq_xprt->xpt_ops->xpo_result_payload(rqstp, offset,
1617 length);
1618}
1619EXPORT_SYMBOL_GPL(svc_encode_result_payload);
1620
1621
1622
1623
1624
1625
1626
1627
1628unsigned int svc_fill_write_vector(struct svc_rqst *rqstp,
1629 struct xdr_buf *payload)
1630{
1631 struct page **pages = payload->pages;
1632 struct kvec *first = payload->head;
1633 struct kvec *vec = rqstp->rq_vec;
1634 size_t total = payload->len;
1635 unsigned int i;
1636
1637
1638
1639
1640 i = 0;
1641 if (first->iov_len) {
1642 vec[i].iov_base = first->iov_base;
1643 vec[i].iov_len = min_t(size_t, total, first->iov_len);
1644 total -= vec[i].iov_len;
1645 ++i;
1646 }
1647
1648 while (total) {
1649 vec[i].iov_base = page_address(*pages);
1650 vec[i].iov_len = min_t(size_t, total, PAGE_SIZE);
1651 total -= vec[i].iov_len;
1652 ++i;
1653 ++pages;
1654 }
1655
1656 WARN_ON_ONCE(i > ARRAY_SIZE(rqstp->rq_vec));
1657 return i;
1658}
1659EXPORT_SYMBOL_GPL(svc_fill_write_vector);
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672char *svc_fill_symlink_pathname(struct svc_rqst *rqstp, struct kvec *first,
1673 void *p, size_t total)
1674{
1675 size_t len, remaining;
1676 char *result, *dst;
1677
1678 result = kmalloc(total + 1, GFP_KERNEL);
1679 if (!result)
1680 return ERR_PTR(-ESERVERFAULT);
1681
1682 dst = result;
1683 remaining = total;
1684
1685 len = min_t(size_t, total, first->iov_len);
1686 if (len) {
1687 memcpy(dst, first->iov_base, len);
1688 dst += len;
1689 remaining -= len;
1690 }
1691
1692 if (remaining) {
1693 len = min_t(size_t, remaining, PAGE_SIZE);
1694 memcpy(dst, p, len);
1695 dst += len;
1696 }
1697
1698 *dst = '\0';
1699
1700
1701
1702
1703 if (strlen(result) != total) {
1704 kfree(result);
1705 return ERR_PTR(-EINVAL);
1706 }
1707 return result;
1708}
1709EXPORT_SYMBOL_GPL(svc_fill_symlink_pathname);
1710