1
2
3
4
5
6
7
8
9
10
11
12#include <linux/module.h>
13
14#include <linux/sched.h>
15#include <linux/interrupt.h>
16#include <linux/slab.h>
17#include <linux/mempool.h>
18#include <linux/smp.h>
19#include <linux/spinlock.h>
20#include <linux/mutex.h>
21
22#include <linux/sunrpc/clnt.h>
23
24#include "sunrpc.h"
25
26#ifdef RPC_DEBUG
27#define RPCDBG_FACILITY RPCDBG_SCHED
28#endif
29
30
31
32
33#define RPC_BUFFER_MAXSIZE (2048)
34#define RPC_BUFFER_POOLSIZE (8)
35#define RPC_TASK_POOLSIZE (8)
36static struct kmem_cache *rpc_task_slabp __read_mostly;
37static struct kmem_cache *rpc_buffer_slabp __read_mostly;
38static mempool_t *rpc_task_mempool __read_mostly;
39static mempool_t *rpc_buffer_mempool __read_mostly;
40
41static void rpc_async_schedule(struct work_struct *);
42static void rpc_release_task(struct rpc_task *task);
43static void __rpc_queue_timer_fn(unsigned long ptr);
44
45
46
47
48static struct rpc_wait_queue delay_queue;
49
50
51
52
53struct workqueue_struct *rpciod_workqueue;
54
55
56
57
58
59
60static void
61__rpc_disable_timer(struct rpc_wait_queue *queue, struct rpc_task *task)
62{
63 if (task->tk_timeout == 0)
64 return;
65 dprintk("RPC: %5u disabling timer\n", task->tk_pid);
66 task->tk_timeout = 0;
67 list_del(&task->u.tk_wait.timer_list);
68 if (list_empty(&queue->timer_list.list))
69 del_timer(&queue->timer_list.timer);
70}
71
72static void
73rpc_set_queue_timer(struct rpc_wait_queue *queue, unsigned long expires)
74{
75 queue->timer_list.expires = expires;
76 mod_timer(&queue->timer_list.timer, expires);
77}
78
79
80
81
82static void
83__rpc_add_timer(struct rpc_wait_queue *queue, struct rpc_task *task)
84{
85 if (!task->tk_timeout)
86 return;
87
88 dprintk("RPC: %5u setting alarm for %lu ms\n",
89 task->tk_pid, task->tk_timeout * 1000 / HZ);
90
91 task->u.tk_wait.expires = jiffies + task->tk_timeout;
92 if (list_empty(&queue->timer_list.list) || time_before(task->u.tk_wait.expires, queue->timer_list.expires))
93 rpc_set_queue_timer(queue, task->u.tk_wait.expires);
94 list_add(&task->u.tk_wait.timer_list, &queue->timer_list.list);
95}
96
97
98
99
100static void __rpc_add_wait_queue_priority(struct rpc_wait_queue *queue, struct rpc_task *task)
101{
102 struct list_head *q;
103 struct rpc_task *t;
104
105 INIT_LIST_HEAD(&task->u.tk_wait.links);
106 q = &queue->tasks[task->tk_priority];
107 if (unlikely(task->tk_priority > queue->maxpriority))
108 q = &queue->tasks[queue->maxpriority];
109 list_for_each_entry(t, q, u.tk_wait.list) {
110 if (t->tk_owner == task->tk_owner) {
111 list_add_tail(&task->u.tk_wait.list, &t->u.tk_wait.links);
112 return;
113 }
114 }
115 list_add_tail(&task->u.tk_wait.list, q);
116}
117
118
119
120
121
122
123
124
125
126static void __rpc_add_wait_queue(struct rpc_wait_queue *queue, struct rpc_task *task)
127{
128 BUG_ON (RPC_IS_QUEUED(task));
129
130 if (RPC_IS_PRIORITY(queue))
131 __rpc_add_wait_queue_priority(queue, task);
132 else if (RPC_IS_SWAPPER(task))
133 list_add(&task->u.tk_wait.list, &queue->tasks[0]);
134 else
135 list_add_tail(&task->u.tk_wait.list, &queue->tasks[0]);
136 task->tk_waitqueue = queue;
137 queue->qlen++;
138 rpc_set_queued(task);
139
140 dprintk("RPC: %5u added to queue %p \"%s\"\n",
141 task->tk_pid, queue, rpc_qname(queue));
142}
143
144
145
146
147static void __rpc_remove_wait_queue_priority(struct rpc_task *task)
148{
149 struct rpc_task *t;
150
151 if (!list_empty(&task->u.tk_wait.links)) {
152 t = list_entry(task->u.tk_wait.links.next, struct rpc_task, u.tk_wait.list);
153 list_move(&t->u.tk_wait.list, &task->u.tk_wait.list);
154 list_splice_init(&task->u.tk_wait.links, &t->u.tk_wait.links);
155 }
156}
157
158
159
160
161
162static void __rpc_remove_wait_queue(struct rpc_wait_queue *queue, struct rpc_task *task)
163{
164 __rpc_disable_timer(queue, task);
165 if (RPC_IS_PRIORITY(queue))
166 __rpc_remove_wait_queue_priority(task);
167 list_del(&task->u.tk_wait.list);
168 queue->qlen--;
169 dprintk("RPC: %5u removed from queue %p \"%s\"\n",
170 task->tk_pid, queue, rpc_qname(queue));
171}
172
173static inline void rpc_set_waitqueue_priority(struct rpc_wait_queue *queue, int priority)
174{
175 queue->priority = priority;
176 queue->count = 1 << (priority * 2);
177}
178
179static inline void rpc_set_waitqueue_owner(struct rpc_wait_queue *queue, pid_t pid)
180{
181 queue->owner = pid;
182 queue->nr = RPC_BATCH_COUNT;
183}
184
185static inline void rpc_reset_waitqueue_priority(struct rpc_wait_queue *queue)
186{
187 rpc_set_waitqueue_priority(queue, queue->maxpriority);
188 rpc_set_waitqueue_owner(queue, 0);
189}
190
191static void __rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname, unsigned char nr_queues)
192{
193 int i;
194
195 spin_lock_init(&queue->lock);
196 for (i = 0; i < ARRAY_SIZE(queue->tasks); i++)
197 INIT_LIST_HEAD(&queue->tasks[i]);
198 queue->maxpriority = nr_queues - 1;
199 rpc_reset_waitqueue_priority(queue);
200 queue->qlen = 0;
201 setup_timer(&queue->timer_list.timer, __rpc_queue_timer_fn, (unsigned long)queue);
202 INIT_LIST_HEAD(&queue->timer_list.list);
203#ifdef RPC_DEBUG
204 queue->name = qname;
205#endif
206}
207
208void rpc_init_priority_wait_queue(struct rpc_wait_queue *queue, const char *qname)
209{
210 __rpc_init_priority_wait_queue(queue, qname, RPC_NR_PRIORITY);
211}
212EXPORT_SYMBOL_GPL(rpc_init_priority_wait_queue);
213
214void rpc_init_wait_queue(struct rpc_wait_queue *queue, const char *qname)
215{
216 __rpc_init_priority_wait_queue(queue, qname, 1);
217}
218EXPORT_SYMBOL_GPL(rpc_init_wait_queue);
219
220void rpc_destroy_wait_queue(struct rpc_wait_queue *queue)
221{
222 del_timer_sync(&queue->timer_list.timer);
223}
224EXPORT_SYMBOL_GPL(rpc_destroy_wait_queue);
225
226static int rpc_wait_bit_killable(void *word)
227{
228 if (fatal_signal_pending(current))
229 return -ERESTARTSYS;
230 schedule();
231 return 0;
232}
233
234#ifdef RPC_DEBUG
235static void rpc_task_set_debuginfo(struct rpc_task *task)
236{
237 static atomic_t rpc_pid;
238
239 task->tk_pid = atomic_inc_return(&rpc_pid);
240}
241#else
242static inline void rpc_task_set_debuginfo(struct rpc_task *task)
243{
244}
245#endif
246
247static void rpc_set_active(struct rpc_task *task)
248{
249 rpc_task_set_debuginfo(task);
250 set_bit(RPC_TASK_ACTIVE, &task->tk_runstate);
251}
252
253
254
255
256
257static int rpc_complete_task(struct rpc_task *task)
258{
259 void *m = &task->tk_runstate;
260 wait_queue_head_t *wq = bit_waitqueue(m, RPC_TASK_ACTIVE);
261 struct wait_bit_key k = __WAIT_BIT_KEY_INITIALIZER(m, RPC_TASK_ACTIVE);
262 unsigned long flags;
263 int ret;
264
265 spin_lock_irqsave(&wq->lock, flags);
266 clear_bit(RPC_TASK_ACTIVE, &task->tk_runstate);
267 ret = atomic_dec_and_test(&task->tk_count);
268 if (waitqueue_active(wq))
269 __wake_up_locked_key(wq, TASK_NORMAL, &k);
270 spin_unlock_irqrestore(&wq->lock, flags);
271 return ret;
272}
273
274
275
276
277
278
279
280
281int __rpc_wait_for_completion_task(struct rpc_task *task, int (*action)(void *))
282{
283 if (action == NULL)
284 action = rpc_wait_bit_killable;
285 return out_of_line_wait_on_bit(&task->tk_runstate, RPC_TASK_ACTIVE,
286 action, TASK_KILLABLE);
287}
288EXPORT_SYMBOL_GPL(__rpc_wait_for_completion_task);
289
290
291
292
293
294
295
296static void rpc_make_runnable(struct rpc_task *task)
297{
298 rpc_clear_queued(task);
299 if (rpc_test_and_set_running(task))
300 return;
301 if (RPC_IS_ASYNC(task)) {
302 int status;
303
304 INIT_WORK(&task->u.tk_work, rpc_async_schedule);
305 status = queue_work(rpciod_workqueue, &task->u.tk_work);
306 if (status < 0) {
307 printk(KERN_WARNING "RPC: failed to add task to queue: error: %d!\n", status);
308 task->tk_status = status;
309 return;
310 }
311 } else
312 wake_up_bit(&task->tk_runstate, RPC_TASK_QUEUED);
313}
314
315
316
317
318
319
320
321static void __rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task,
322 rpc_action action)
323{
324 dprintk("RPC: %5u sleep_on(queue \"%s\" time %lu)\n",
325 task->tk_pid, rpc_qname(q), jiffies);
326
327 __rpc_add_wait_queue(q, task);
328
329 BUG_ON(task->tk_callback != NULL);
330 task->tk_callback = action;
331 __rpc_add_timer(q, task);
332}
333
334void rpc_sleep_on(struct rpc_wait_queue *q, struct rpc_task *task,
335 rpc_action action)
336{
337
338 BUG_ON(!RPC_IS_ACTIVATED(task));
339
340
341
342
343 spin_lock_bh(&q->lock);
344 __rpc_sleep_on(q, task, action);
345 spin_unlock_bh(&q->lock);
346}
347EXPORT_SYMBOL_GPL(rpc_sleep_on);
348
349
350
351
352
353
354
355
356static void __rpc_do_wake_up_task(struct rpc_wait_queue *queue, struct rpc_task *task)
357{
358 dprintk("RPC: %5u __rpc_wake_up_task (now %lu)\n",
359 task->tk_pid, jiffies);
360
361
362 if (!RPC_IS_ACTIVATED(task)) {
363 printk(KERN_ERR "RPC: Inactive task (%p) being woken up!\n", task);
364 return;
365 }
366
367 __rpc_remove_wait_queue(queue, task);
368
369 rpc_make_runnable(task);
370
371 dprintk("RPC: __rpc_wake_up_task done\n");
372}
373
374
375
376
377static void rpc_wake_up_task_queue_locked(struct rpc_wait_queue *queue, struct rpc_task *task)
378{
379 if (RPC_IS_QUEUED(task) && task->tk_waitqueue == queue)
380 __rpc_do_wake_up_task(queue, task);
381}
382
383
384
385
386int rpc_queue_empty(struct rpc_wait_queue *queue)
387{
388 int res;
389
390 spin_lock_bh(&queue->lock);
391 res = queue->qlen;
392 spin_unlock_bh(&queue->lock);
393 return res == 0;
394}
395EXPORT_SYMBOL_GPL(rpc_queue_empty);
396
397
398
399
400void rpc_wake_up_queued_task(struct rpc_wait_queue *queue, struct rpc_task *task)
401{
402 spin_lock_bh(&queue->lock);
403 rpc_wake_up_task_queue_locked(queue, task);
404 spin_unlock_bh(&queue->lock);
405}
406EXPORT_SYMBOL_GPL(rpc_wake_up_queued_task);
407
408
409
410
411static struct rpc_task * __rpc_wake_up_next_priority(struct rpc_wait_queue *queue)
412{
413 struct list_head *q;
414 struct rpc_task *task;
415
416
417
418
419 q = &queue->tasks[queue->priority];
420 if (!list_empty(q)) {
421 task = list_entry(q->next, struct rpc_task, u.tk_wait.list);
422 if (queue->owner == task->tk_owner) {
423 if (--queue->nr)
424 goto out;
425 list_move_tail(&task->u.tk_wait.list, q);
426 }
427
428
429
430 if (--queue->count)
431 goto new_owner;
432 }
433
434
435
436
437 do {
438 if (q == &queue->tasks[0])
439 q = &queue->tasks[queue->maxpriority];
440 else
441 q = q - 1;
442 if (!list_empty(q)) {
443 task = list_entry(q->next, struct rpc_task, u.tk_wait.list);
444 goto new_queue;
445 }
446 } while (q != &queue->tasks[queue->priority]);
447
448 rpc_reset_waitqueue_priority(queue);
449 return NULL;
450
451new_queue:
452 rpc_set_waitqueue_priority(queue, (unsigned int)(q - &queue->tasks[0]));
453new_owner:
454 rpc_set_waitqueue_owner(queue, task->tk_owner);
455out:
456 rpc_wake_up_task_queue_locked(queue, task);
457 return task;
458}
459
460
461
462
463struct rpc_task * rpc_wake_up_next(struct rpc_wait_queue *queue)
464{
465 struct rpc_task *task = NULL;
466
467 dprintk("RPC: wake_up_next(%p \"%s\")\n",
468 queue, rpc_qname(queue));
469 spin_lock_bh(&queue->lock);
470 if (RPC_IS_PRIORITY(queue))
471 task = __rpc_wake_up_next_priority(queue);
472 else {
473 task_for_first(task, &queue->tasks[0])
474 rpc_wake_up_task_queue_locked(queue, task);
475 }
476 spin_unlock_bh(&queue->lock);
477
478 return task;
479}
480EXPORT_SYMBOL_GPL(rpc_wake_up_next);
481
482
483
484
485
486
487
488void rpc_wake_up(struct rpc_wait_queue *queue)
489{
490 struct rpc_task *task, *next;
491 struct list_head *head;
492
493 spin_lock_bh(&queue->lock);
494 head = &queue->tasks[queue->maxpriority];
495 for (;;) {
496 list_for_each_entry_safe(task, next, head, u.tk_wait.list)
497 rpc_wake_up_task_queue_locked(queue, task);
498 if (head == &queue->tasks[0])
499 break;
500 head--;
501 }
502 spin_unlock_bh(&queue->lock);
503}
504EXPORT_SYMBOL_GPL(rpc_wake_up);
505
506
507
508
509
510
511
512
513void rpc_wake_up_status(struct rpc_wait_queue *queue, int status)
514{
515 struct rpc_task *task, *next;
516 struct list_head *head;
517
518 spin_lock_bh(&queue->lock);
519 head = &queue->tasks[queue->maxpriority];
520 for (;;) {
521 list_for_each_entry_safe(task, next, head, u.tk_wait.list) {
522 task->tk_status = status;
523 rpc_wake_up_task_queue_locked(queue, task);
524 }
525 if (head == &queue->tasks[0])
526 break;
527 head--;
528 }
529 spin_unlock_bh(&queue->lock);
530}
531EXPORT_SYMBOL_GPL(rpc_wake_up_status);
532
533static void __rpc_queue_timer_fn(unsigned long ptr)
534{
535 struct rpc_wait_queue *queue = (struct rpc_wait_queue *)ptr;
536 struct rpc_task *task, *n;
537 unsigned long expires, now, timeo;
538
539 spin_lock(&queue->lock);
540 expires = now = jiffies;
541 list_for_each_entry_safe(task, n, &queue->timer_list.list, u.tk_wait.timer_list) {
542 timeo = task->u.tk_wait.expires;
543 if (time_after_eq(now, timeo)) {
544 dprintk("RPC: %5u timeout\n", task->tk_pid);
545 task->tk_status = -ETIMEDOUT;
546 rpc_wake_up_task_queue_locked(queue, task);
547 continue;
548 }
549 if (expires == now || time_after(expires, timeo))
550 expires = timeo;
551 }
552 if (!list_empty(&queue->timer_list.list))
553 rpc_set_queue_timer(queue, expires);
554 spin_unlock(&queue->lock);
555}
556
557static void __rpc_atrun(struct rpc_task *task)
558{
559 task->tk_status = 0;
560}
561
562
563
564
565void rpc_delay(struct rpc_task *task, unsigned long delay)
566{
567 task->tk_timeout = delay;
568 rpc_sleep_on(&delay_queue, task, __rpc_atrun);
569}
570EXPORT_SYMBOL_GPL(rpc_delay);
571
572
573
574
575void rpc_prepare_task(struct rpc_task *task)
576{
577 task->tk_ops->rpc_call_prepare(task, task->tk_calldata);
578}
579
580
581
582
583void rpc_exit_task(struct rpc_task *task)
584{
585 task->tk_action = NULL;
586 if (task->tk_ops->rpc_call_done != NULL) {
587 task->tk_ops->rpc_call_done(task, task->tk_calldata);
588 if (task->tk_action != NULL) {
589 WARN_ON(RPC_ASSASSINATED(task));
590
591 xprt_release(task);
592 }
593 }
594}
595
596void rpc_exit(struct rpc_task *task, int status)
597{
598 task->tk_status = status;
599 task->tk_action = rpc_exit_task;
600 if (RPC_IS_QUEUED(task))
601 rpc_wake_up_queued_task(task->tk_waitqueue, task);
602}
603EXPORT_SYMBOL_GPL(rpc_exit);
604
605void rpc_release_calldata(const struct rpc_call_ops *ops, void *calldata)
606{
607 if (ops->rpc_release != NULL)
608 ops->rpc_release(calldata);
609}
610
611
612
613
614static void __rpc_execute(struct rpc_task *task)
615{
616 struct rpc_wait_queue *queue;
617 int task_is_async = RPC_IS_ASYNC(task);
618 int status = 0;
619
620 dprintk("RPC: %5u __rpc_execute flags=0x%x\n",
621 task->tk_pid, task->tk_flags);
622
623 BUG_ON(RPC_IS_QUEUED(task));
624
625 for (;;) {
626
627
628
629
630 if (task->tk_callback) {
631 void (*save_callback)(struct rpc_task *);
632
633
634
635
636
637 save_callback = task->tk_callback;
638 task->tk_callback = NULL;
639 save_callback(task);
640 }
641
642
643
644
645
646
647 if (!RPC_IS_QUEUED(task)) {
648 if (task->tk_action == NULL)
649 break;
650 task->tk_action(task);
651 }
652
653
654
655
656 if (!RPC_IS_QUEUED(task))
657 continue;
658
659
660
661
662
663
664
665
666
667 queue = task->tk_waitqueue;
668 spin_lock_bh(&queue->lock);
669 if (!RPC_IS_QUEUED(task)) {
670 spin_unlock_bh(&queue->lock);
671 continue;
672 }
673 rpc_clear_running(task);
674 spin_unlock_bh(&queue->lock);
675 if (task_is_async)
676 return;
677
678
679 dprintk("RPC: %5u sync task going to sleep\n", task->tk_pid);
680 status = out_of_line_wait_on_bit(&task->tk_runstate,
681 RPC_TASK_QUEUED, rpc_wait_bit_killable,
682 TASK_KILLABLE);
683 if (status == -ERESTARTSYS) {
684
685
686
687
688
689
690 dprintk("RPC: %5u got signal\n", task->tk_pid);
691 task->tk_flags |= RPC_TASK_KILLED;
692 rpc_exit(task, -ERESTARTSYS);
693 }
694 rpc_set_running(task);
695 dprintk("RPC: %5u sync task resuming\n", task->tk_pid);
696 }
697
698 dprintk("RPC: %5u return %d, status %d\n", task->tk_pid, status,
699 task->tk_status);
700
701 rpc_release_task(task);
702}
703
704
705
706
707
708
709
710
711
712
713void rpc_execute(struct rpc_task *task)
714{
715 rpc_set_active(task);
716 rpc_make_runnable(task);
717 if (!RPC_IS_ASYNC(task))
718 __rpc_execute(task);
719}
720
721static void rpc_async_schedule(struct work_struct *work)
722{
723 __rpc_execute(container_of(work, struct rpc_task, u.tk_work));
724}
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742void *rpc_malloc(struct rpc_task *task, size_t size)
743{
744 struct rpc_buffer *buf;
745 gfp_t gfp = RPC_IS_SWAPPER(task) ? GFP_ATOMIC : GFP_NOWAIT;
746
747 size += sizeof(struct rpc_buffer);
748 if (size <= RPC_BUFFER_MAXSIZE)
749 buf = mempool_alloc(rpc_buffer_mempool, gfp);
750 else
751 buf = kmalloc(size, gfp);
752
753 if (!buf)
754 return NULL;
755
756 buf->len = size;
757 dprintk("RPC: %5u allocated buffer of size %zu at %p\n",
758 task->tk_pid, size, buf);
759 return &buf->data;
760}
761EXPORT_SYMBOL_GPL(rpc_malloc);
762
763
764
765
766
767
768void rpc_free(void *buffer)
769{
770 size_t size;
771 struct rpc_buffer *buf;
772
773 if (!buffer)
774 return;
775
776 buf = container_of(buffer, struct rpc_buffer, data);
777 size = buf->len;
778
779 dprintk("RPC: freeing buffer of size %zu at %p\n",
780 size, buf);
781
782 if (size <= RPC_BUFFER_MAXSIZE)
783 mempool_free(buf, rpc_buffer_mempool);
784 else
785 kfree(buf);
786}
787EXPORT_SYMBOL_GPL(rpc_free);
788
789
790
791
792static void rpc_init_task(struct rpc_task *task, const struct rpc_task_setup *task_setup_data)
793{
794 memset(task, 0, sizeof(*task));
795 atomic_set(&task->tk_count, 1);
796 task->tk_flags = task_setup_data->flags;
797 task->tk_ops = task_setup_data->callback_ops;
798 task->tk_calldata = task_setup_data->callback_data;
799 INIT_LIST_HEAD(&task->tk_task);
800
801
802 task->tk_garb_retry = 2;
803 task->tk_cred_retry = 2;
804
805 task->tk_priority = task_setup_data->priority - RPC_PRIORITY_LOW;
806 task->tk_owner = current->tgid;
807
808
809 task->tk_workqueue = task_setup_data->workqueue;
810
811 if (task->tk_ops->rpc_call_prepare != NULL)
812 task->tk_action = rpc_prepare_task;
813
814
815 task->tk_start = ktime_get();
816
817 dprintk("RPC: new task initialized, procpid %u\n",
818 task_pid_nr(current));
819}
820
821static struct rpc_task *
822rpc_alloc_task(void)
823{
824 return (struct rpc_task *)mempool_alloc(rpc_task_mempool, GFP_NOFS);
825}
826
827
828
829
830struct rpc_task *rpc_new_task(const struct rpc_task_setup *setup_data)
831{
832 struct rpc_task *task = setup_data->task;
833 unsigned short flags = 0;
834
835 if (task == NULL) {
836 task = rpc_alloc_task();
837 if (task == NULL) {
838 rpc_release_calldata(setup_data->callback_ops,
839 setup_data->callback_data);
840 return ERR_PTR(-ENOMEM);
841 }
842 flags = RPC_TASK_DYNAMIC;
843 }
844
845 rpc_init_task(task, setup_data);
846 if (task->tk_status < 0) {
847 int err = task->tk_status;
848 rpc_put_task(task);
849 return ERR_PTR(err);
850 }
851
852 task->tk_flags |= flags;
853 dprintk("RPC: allocated task %p\n", task);
854 return task;
855}
856
857static void rpc_free_task(struct rpc_task *task)
858{
859 const struct rpc_call_ops *tk_ops = task->tk_ops;
860 void *calldata = task->tk_calldata;
861
862 if (task->tk_flags & RPC_TASK_DYNAMIC) {
863 dprintk("RPC: %5u freeing task\n", task->tk_pid);
864 mempool_free(task, rpc_task_mempool);
865 }
866 rpc_release_calldata(tk_ops, calldata);
867}
868
869static void rpc_async_release(struct work_struct *work)
870{
871 rpc_free_task(container_of(work, struct rpc_task, u.tk_work));
872}
873
874static void rpc_release_resources_task(struct rpc_task *task)
875{
876 if (task->tk_rqstp)
877 xprt_release(task);
878 if (task->tk_msg.rpc_cred)
879 put_rpccred(task->tk_msg.rpc_cred);
880 rpc_task_release_client(task);
881}
882
883static void rpc_final_put_task(struct rpc_task *task,
884 struct workqueue_struct *q)
885{
886 if (q != NULL) {
887 INIT_WORK(&task->u.tk_work, rpc_async_release);
888 queue_work(q, &task->u.tk_work);
889 } else
890 rpc_free_task(task);
891}
892
893static void rpc_do_put_task(struct rpc_task *task, struct workqueue_struct *q)
894{
895 if (atomic_dec_and_test(&task->tk_count)) {
896 rpc_release_resources_task(task);
897 rpc_final_put_task(task, q);
898 }
899}
900
901void rpc_put_task(struct rpc_task *task)
902{
903 rpc_do_put_task(task, NULL);
904}
905EXPORT_SYMBOL_GPL(rpc_put_task);
906
907void rpc_put_task_async(struct rpc_task *task)
908{
909 rpc_do_put_task(task, task->tk_workqueue);
910}
911EXPORT_SYMBOL_GPL(rpc_put_task_async);
912
913static void rpc_release_task(struct rpc_task *task)
914{
915 dprintk("RPC: %5u release task\n", task->tk_pid);
916
917 BUG_ON (RPC_IS_QUEUED(task));
918
919 rpc_release_resources_task(task);
920
921
922
923
924
925
926 if (atomic_read(&task->tk_count) != 1 + !RPC_IS_ASYNC(task)) {
927
928 if (!rpc_complete_task(task))
929 return;
930 } else {
931 if (!atomic_dec_and_test(&task->tk_count))
932 return;
933 }
934 rpc_final_put_task(task, task->tk_workqueue);
935}
936
937int rpciod_up(void)
938{
939 return try_module_get(THIS_MODULE) ? 0 : -EINVAL;
940}
941
942void rpciod_down(void)
943{
944 module_put(THIS_MODULE);
945}
946
947
948
949
950static int rpciod_start(void)
951{
952 struct workqueue_struct *wq;
953
954
955
956
957 dprintk("RPC: creating workqueue rpciod\n");
958 wq = alloc_workqueue("rpciod", WQ_RESCUER, 0);
959 rpciod_workqueue = wq;
960 return rpciod_workqueue != NULL;
961}
962
963static void rpciod_stop(void)
964{
965 struct workqueue_struct *wq = NULL;
966
967 if (rpciod_workqueue == NULL)
968 return;
969 dprintk("RPC: destroying workqueue rpciod\n");
970
971 wq = rpciod_workqueue;
972 rpciod_workqueue = NULL;
973 destroy_workqueue(wq);
974}
975
976void
977rpc_destroy_mempool(void)
978{
979 rpciod_stop();
980 if (rpc_buffer_mempool)
981 mempool_destroy(rpc_buffer_mempool);
982 if (rpc_task_mempool)
983 mempool_destroy(rpc_task_mempool);
984 if (rpc_task_slabp)
985 kmem_cache_destroy(rpc_task_slabp);
986 if (rpc_buffer_slabp)
987 kmem_cache_destroy(rpc_buffer_slabp);
988 rpc_destroy_wait_queue(&delay_queue);
989}
990
991int
992rpc_init_mempool(void)
993{
994
995
996
997
998 rpc_init_wait_queue(&delay_queue, "delayq");
999 if (!rpciod_start())
1000 goto err_nomem;
1001
1002 rpc_task_slabp = kmem_cache_create("rpc_tasks",
1003 sizeof(struct rpc_task),
1004 0, SLAB_HWCACHE_ALIGN,
1005 NULL);
1006 if (!rpc_task_slabp)
1007 goto err_nomem;
1008 rpc_buffer_slabp = kmem_cache_create("rpc_buffers",
1009 RPC_BUFFER_MAXSIZE,
1010 0, SLAB_HWCACHE_ALIGN,
1011 NULL);
1012 if (!rpc_buffer_slabp)
1013 goto err_nomem;
1014 rpc_task_mempool = mempool_create_slab_pool(RPC_TASK_POOLSIZE,
1015 rpc_task_slabp);
1016 if (!rpc_task_mempool)
1017 goto err_nomem;
1018 rpc_buffer_mempool = mempool_create_slab_pool(RPC_BUFFER_POOLSIZE,
1019 rpc_buffer_slabp);
1020 if (!rpc_buffer_mempool)
1021 goto err_nomem;
1022 return 0;
1023err_nomem:
1024 rpc_destroy_mempool();
1025 return -ENOMEM;
1026}
1027