1
2
3
4
5
6
7
8
9
10
11
12#include <linux/slab.h>
13#include <linux/file.h>
14#include <linux/sched.h>
15#include <linux/sunrpc/clnt.h>
16#include <linux/nfs.h>
17#include <linux/nfs3.h>
18#include <linux/nfs4.h>
19#include <linux/nfs_page.h>
20#include <linux/nfs_fs.h>
21#include <linux/nfs_mount.h>
22#include <linux/export.h>
23
24#include "internal.h"
25#include "pnfs.h"
26
27#define NFSDBG_FACILITY NFSDBG_PAGECACHE
28
29static struct kmem_cache *nfs_page_cachep;
30static const struct rpc_call_ops nfs_pgio_common_ops;
31
32static bool nfs_pgarray_set(struct nfs_page_array *p, unsigned int pagecount)
33{
34 p->npages = pagecount;
35 if (pagecount <= ARRAY_SIZE(p->page_array))
36 p->pagevec = p->page_array;
37 else {
38 p->pagevec = kcalloc(pagecount, sizeof(struct page *), GFP_KERNEL);
39 if (!p->pagevec)
40 p->npages = 0;
41 }
42 return p->pagevec != NULL;
43}
44
45struct nfs_pgio_mirror *
46nfs_pgio_current_mirror(struct nfs_pageio_descriptor *desc)
47{
48 return nfs_pgio_has_mirroring(desc) ?
49 &desc->pg_mirrors[desc->pg_mirror_idx] :
50 &desc->pg_mirrors[0];
51}
52EXPORT_SYMBOL_GPL(nfs_pgio_current_mirror);
53
54void nfs_pgheader_init(struct nfs_pageio_descriptor *desc,
55 struct nfs_pgio_header *hdr,
56 void (*release)(struct nfs_pgio_header *hdr))
57{
58 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
59
60
61 hdr->req = nfs_list_entry(mirror->pg_list.next);
62 hdr->inode = desc->pg_inode;
63 hdr->cred = hdr->req->wb_context->cred;
64 hdr->io_start = req_offset(hdr->req);
65 hdr->good_bytes = mirror->pg_count;
66 hdr->dreq = desc->pg_dreq;
67 hdr->layout_private = desc->pg_layout_private;
68 hdr->release = release;
69 hdr->completion_ops = desc->pg_completion_ops;
70 if (hdr->completion_ops->init_hdr)
71 hdr->completion_ops->init_hdr(hdr);
72
73 hdr->pgio_mirror_idx = desc->pg_mirror_idx;
74}
75EXPORT_SYMBOL_GPL(nfs_pgheader_init);
76
77void nfs_set_pgio_error(struct nfs_pgio_header *hdr, int error, loff_t pos)
78{
79 spin_lock(&hdr->lock);
80 if (!test_and_set_bit(NFS_IOHDR_ERROR, &hdr->flags)
81 || pos < hdr->io_start + hdr->good_bytes) {
82 clear_bit(NFS_IOHDR_EOF, &hdr->flags);
83 hdr->good_bytes = pos - hdr->io_start;
84 hdr->error = error;
85 }
86 spin_unlock(&hdr->lock);
87}
88
89static inline struct nfs_page *
90nfs_page_alloc(void)
91{
92 struct nfs_page *p = kmem_cache_zalloc(nfs_page_cachep, GFP_NOIO);
93 if (p)
94 INIT_LIST_HEAD(&p->wb_list);
95 return p;
96}
97
98static inline void
99nfs_page_free(struct nfs_page *p)
100{
101 kmem_cache_free(nfs_page_cachep, p);
102}
103
104
105
106
107
108
109
110
111int
112nfs_iocounter_wait(struct nfs_lock_context *l_ctx)
113{
114 return wait_on_atomic_t(&l_ctx->io_count, nfs_wait_atomic_killable,
115 TASK_KILLABLE);
116}
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132int
133nfs_page_group_lock(struct nfs_page *req, bool nonblock)
134{
135 struct nfs_page *head = req->wb_head;
136
137 WARN_ON_ONCE(head != head->wb_head);
138
139 if (!test_and_set_bit(PG_HEADLOCK, &head->wb_flags))
140 return 0;
141
142 if (!nonblock)
143 return wait_on_bit_lock(&head->wb_flags, PG_HEADLOCK,
144 TASK_UNINTERRUPTIBLE);
145
146 return -EAGAIN;
147}
148
149
150
151
152
153
154
155void
156nfs_page_group_lock_wait(struct nfs_page *req)
157{
158 struct nfs_page *head = req->wb_head;
159
160 WARN_ON_ONCE(head != head->wb_head);
161
162 wait_on_bit(&head->wb_flags, PG_HEADLOCK,
163 TASK_UNINTERRUPTIBLE);
164}
165
166
167
168
169
170void
171nfs_page_group_unlock(struct nfs_page *req)
172{
173 struct nfs_page *head = req->wb_head;
174
175 WARN_ON_ONCE(head != head->wb_head);
176
177 smp_mb__before_atomic();
178 clear_bit(PG_HEADLOCK, &head->wb_flags);
179 smp_mb__after_atomic();
180 wake_up_bit(&head->wb_flags, PG_HEADLOCK);
181}
182
183
184
185
186
187
188static bool
189nfs_page_group_sync_on_bit_locked(struct nfs_page *req, unsigned int bit)
190{
191 struct nfs_page *head = req->wb_head;
192 struct nfs_page *tmp;
193
194 WARN_ON_ONCE(!test_bit(PG_HEADLOCK, &head->wb_flags));
195 WARN_ON_ONCE(test_and_set_bit(bit, &req->wb_flags));
196
197 tmp = req->wb_this_page;
198 while (tmp != req) {
199 if (!test_bit(bit, &tmp->wb_flags))
200 return false;
201 tmp = tmp->wb_this_page;
202 }
203
204
205 tmp = req;
206 do {
207 clear_bit(bit, &tmp->wb_flags);
208 tmp = tmp->wb_this_page;
209 } while (tmp != req);
210
211 return true;
212}
213
214
215
216
217
218
219
220bool nfs_page_group_sync_on_bit(struct nfs_page *req, unsigned int bit)
221{
222 bool ret;
223
224 nfs_page_group_lock(req, false);
225 ret = nfs_page_group_sync_on_bit_locked(req, bit);
226 nfs_page_group_unlock(req);
227
228 return ret;
229}
230
231
232
233
234
235
236
237static inline void
238nfs_page_group_init(struct nfs_page *req, struct nfs_page *prev)
239{
240 struct inode *inode;
241 WARN_ON_ONCE(prev == req);
242
243 if (!prev) {
244
245 req->wb_head = req;
246 req->wb_this_page = req;
247 } else {
248
249 WARN_ON_ONCE(prev->wb_this_page != prev->wb_head);
250 WARN_ON_ONCE(!test_bit(PG_HEADLOCK, &prev->wb_head->wb_flags));
251 req->wb_head = prev->wb_head;
252 req->wb_this_page = prev->wb_this_page;
253 prev->wb_this_page = req;
254
255
256
257 kref_get(&req->wb_head->wb_kref);
258
259
260
261
262 if (test_bit(PG_INODE_REF, &prev->wb_head->wb_flags)) {
263 inode = page_file_mapping(req->wb_page)->host;
264 set_bit(PG_INODE_REF, &req->wb_flags);
265 kref_get(&req->wb_kref);
266 spin_lock(&inode->i_lock);
267 NFS_I(inode)->nrequests++;
268 spin_unlock(&inode->i_lock);
269 }
270 }
271}
272
273
274
275
276
277
278
279
280static void
281nfs_page_group_destroy(struct kref *kref)
282{
283 struct nfs_page *req = container_of(kref, struct nfs_page, wb_kref);
284 struct nfs_page *tmp, *next;
285
286
287 if (req->wb_head != req)
288 nfs_release_request(req->wb_head);
289
290 if (!nfs_page_group_sync_on_bit(req, PG_TEARDOWN))
291 return;
292
293 tmp = req;
294 do {
295 next = tmp->wb_this_page;
296
297 tmp->wb_this_page = tmp;
298 tmp->wb_head = tmp;
299 nfs_free_request(tmp);
300 tmp = next;
301 } while (tmp != req);
302}
303
304
305
306
307
308
309
310
311
312
313
314
315
316struct nfs_page *
317nfs_create_request(struct nfs_open_context *ctx, struct page *page,
318 struct nfs_page *last, unsigned int offset,
319 unsigned int count)
320{
321 struct nfs_page *req;
322 struct nfs_lock_context *l_ctx;
323
324 if (test_bit(NFS_CONTEXT_BAD, &ctx->flags))
325 return ERR_PTR(-EBADF);
326
327 req = nfs_page_alloc();
328 if (req == NULL)
329 return ERR_PTR(-ENOMEM);
330
331
332 l_ctx = nfs_get_lock_context(ctx);
333 if (IS_ERR(l_ctx)) {
334 nfs_page_free(req);
335 return ERR_CAST(l_ctx);
336 }
337 req->wb_lock_context = l_ctx;
338 atomic_inc(&l_ctx->io_count);
339
340
341
342
343 req->wb_page = page;
344 if (page) {
345 req->wb_index = page_file_index(page);
346 get_page(page);
347 }
348 req->wb_offset = offset;
349 req->wb_pgbase = offset;
350 req->wb_bytes = count;
351 req->wb_context = get_nfs_open_context(ctx);
352 kref_init(&req->wb_kref);
353 nfs_page_group_init(req, last);
354 return req;
355}
356
357
358
359
360
361void nfs_unlock_request(struct nfs_page *req)
362{
363 if (!NFS_WBACK_BUSY(req)) {
364 printk(KERN_ERR "NFS: Invalid unlock attempted\n");
365 BUG();
366 }
367 smp_mb__before_atomic();
368 clear_bit(PG_BUSY, &req->wb_flags);
369 smp_mb__after_atomic();
370 wake_up_bit(&req->wb_flags, PG_BUSY);
371}
372
373
374
375
376
377void nfs_unlock_and_release_request(struct nfs_page *req)
378{
379 nfs_unlock_request(req);
380 nfs_release_request(req);
381}
382
383
384
385
386
387
388
389
390static void nfs_clear_request(struct nfs_page *req)
391{
392 struct page *page = req->wb_page;
393 struct nfs_open_context *ctx = req->wb_context;
394 struct nfs_lock_context *l_ctx = req->wb_lock_context;
395
396 if (page != NULL) {
397 put_page(page);
398 req->wb_page = NULL;
399 }
400 if (l_ctx != NULL) {
401 if (atomic_dec_and_test(&l_ctx->io_count))
402 wake_up_atomic_t(&l_ctx->io_count);
403 nfs_put_lock_context(l_ctx);
404 req->wb_lock_context = NULL;
405 }
406 if (ctx != NULL) {
407 put_nfs_open_context(ctx);
408 req->wb_context = NULL;
409 }
410}
411
412
413
414
415
416
417
418void nfs_free_request(struct nfs_page *req)
419{
420 WARN_ON_ONCE(req->wb_this_page != req);
421
422
423 WARN_ON_ONCE(test_bit(PG_TEARDOWN, &req->wb_flags));
424 WARN_ON_ONCE(test_bit(PG_UNLOCKPAGE, &req->wb_flags));
425 WARN_ON_ONCE(test_bit(PG_UPTODATE, &req->wb_flags));
426 WARN_ON_ONCE(test_bit(PG_WB_END, &req->wb_flags));
427 WARN_ON_ONCE(test_bit(PG_REMOVE, &req->wb_flags));
428
429
430 nfs_clear_request(req);
431 nfs_page_free(req);
432}
433
434void nfs_release_request(struct nfs_page *req)
435{
436 kref_put(&req->wb_kref, nfs_page_group_destroy);
437}
438
439
440
441
442
443
444
445
446int
447nfs_wait_on_request(struct nfs_page *req)
448{
449 return wait_on_bit_io(&req->wb_flags, PG_BUSY,
450 TASK_UNINTERRUPTIBLE);
451}
452
453
454
455
456
457
458
459
460
461
462size_t nfs_generic_pg_test(struct nfs_pageio_descriptor *desc,
463 struct nfs_page *prev, struct nfs_page *req)
464{
465 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
466
467
468 if (mirror->pg_count > mirror->pg_bsize) {
469
470 WARN_ON_ONCE(1);
471 return 0;
472 }
473
474
475
476
477
478 if (((mirror->pg_count + req->wb_bytes) >> PAGE_SHIFT) *
479 sizeof(struct page *) > PAGE_SIZE)
480 return 0;
481
482 return min(mirror->pg_bsize - mirror->pg_count, (size_t)req->wb_bytes);
483}
484EXPORT_SYMBOL_GPL(nfs_generic_pg_test);
485
486struct nfs_pgio_header *nfs_pgio_header_alloc(const struct nfs_rw_ops *ops)
487{
488 struct nfs_pgio_header *hdr = ops->rw_alloc_header();
489
490 if (hdr) {
491 INIT_LIST_HEAD(&hdr->pages);
492 spin_lock_init(&hdr->lock);
493 hdr->rw_ops = ops;
494 }
495 return hdr;
496}
497EXPORT_SYMBOL_GPL(nfs_pgio_header_alloc);
498
499
500
501
502
503void nfs_pgio_header_free(struct nfs_pgio_header *hdr)
504{
505 hdr->rw_ops->rw_free_header(hdr);
506}
507EXPORT_SYMBOL_GPL(nfs_pgio_header_free);
508
509
510
511
512
513
514
515
516
517void nfs_pgio_data_destroy(struct nfs_pgio_header *hdr)
518{
519 if (hdr->args.context)
520 put_nfs_open_context(hdr->args.context);
521 if (hdr->page_array.pagevec != hdr->page_array.page_array)
522 kfree(hdr->page_array.pagevec);
523}
524EXPORT_SYMBOL_GPL(nfs_pgio_data_destroy);
525
526
527
528
529
530
531
532
533
534static void nfs_pgio_rpcsetup(struct nfs_pgio_header *hdr,
535 unsigned int count, unsigned int offset,
536 int how, struct nfs_commit_info *cinfo)
537{
538 struct nfs_page *req = hdr->req;
539
540
541
542
543 hdr->args.fh = NFS_FH(hdr->inode);
544 hdr->args.offset = req_offset(req) + offset;
545
546 hdr->mds_offset = hdr->args.offset;
547 hdr->args.pgbase = req->wb_pgbase + offset;
548 hdr->args.pages = hdr->page_array.pagevec;
549 hdr->args.count = count;
550 hdr->args.context = get_nfs_open_context(req->wb_context);
551 hdr->args.lock_context = req->wb_lock_context;
552 hdr->args.stable = NFS_UNSTABLE;
553 switch (how & (FLUSH_STABLE | FLUSH_COND_STABLE)) {
554 case 0:
555 break;
556 case FLUSH_COND_STABLE:
557 if (nfs_reqs_to_commit(cinfo))
558 break;
559 default:
560 hdr->args.stable = NFS_FILE_SYNC;
561 }
562
563 hdr->res.fattr = &hdr->fattr;
564 hdr->res.count = count;
565 hdr->res.eof = 0;
566 hdr->res.verf = &hdr->verf;
567 nfs_fattr_init(&hdr->fattr);
568}
569
570
571
572
573
574
575static void nfs_pgio_prepare(struct rpc_task *task, void *calldata)
576{
577 struct nfs_pgio_header *hdr = calldata;
578 int err;
579 err = NFS_PROTO(hdr->inode)->pgio_rpc_prepare(task, hdr);
580 if (err)
581 rpc_exit(task, err);
582}
583
584int nfs_initiate_pgio(struct rpc_clnt *clnt, struct nfs_pgio_header *hdr,
585 struct rpc_cred *cred, const struct nfs_rpc_ops *rpc_ops,
586 const struct rpc_call_ops *call_ops, int how, int flags)
587{
588 struct rpc_task *task;
589 struct rpc_message msg = {
590 .rpc_argp = &hdr->args,
591 .rpc_resp = &hdr->res,
592 .rpc_cred = cred,
593 };
594 struct rpc_task_setup task_setup_data = {
595 .rpc_client = clnt,
596 .task = &hdr->task,
597 .rpc_message = &msg,
598 .callback_ops = call_ops,
599 .callback_data = hdr,
600 .workqueue = nfsiod_workqueue,
601 .flags = RPC_TASK_ASYNC | flags,
602 };
603 int ret = 0;
604
605 hdr->rw_ops->rw_initiate(hdr, &msg, rpc_ops, &task_setup_data, how);
606
607 dprintk("NFS: initiated pgio call "
608 "(req %s/%llu, %u bytes @ offset %llu)\n",
609 hdr->inode->i_sb->s_id,
610 (unsigned long long)NFS_FILEID(hdr->inode),
611 hdr->args.count,
612 (unsigned long long)hdr->args.offset);
613
614 task = rpc_run_task(&task_setup_data);
615 if (IS_ERR(task)) {
616 ret = PTR_ERR(task);
617 goto out;
618 }
619 if (how & FLUSH_SYNC) {
620 ret = rpc_wait_for_completion_task(task);
621 if (ret == 0)
622 ret = task->tk_status;
623 }
624 rpc_put_task(task);
625out:
626 return ret;
627}
628EXPORT_SYMBOL_GPL(nfs_initiate_pgio);
629
630
631
632
633
634
635static void nfs_pgio_error(struct nfs_pgio_header *hdr)
636{
637 set_bit(NFS_IOHDR_REDO, &hdr->flags);
638 nfs_pgio_data_destroy(hdr);
639 hdr->completion_ops->completion(hdr);
640}
641
642
643
644
645
646static void nfs_pgio_release(void *calldata)
647{
648 struct nfs_pgio_header *hdr = calldata;
649 nfs_pgio_data_destroy(hdr);
650 hdr->completion_ops->completion(hdr);
651}
652
653static void nfs_pageio_mirror_init(struct nfs_pgio_mirror *mirror,
654 unsigned int bsize)
655{
656 INIT_LIST_HEAD(&mirror->pg_list);
657 mirror->pg_bytes_written = 0;
658 mirror->pg_count = 0;
659 mirror->pg_bsize = bsize;
660 mirror->pg_base = 0;
661 mirror->pg_recoalesce = 0;
662}
663
664
665
666
667
668
669
670
671
672
673
674void nfs_pageio_init(struct nfs_pageio_descriptor *desc,
675 struct inode *inode,
676 const struct nfs_pageio_ops *pg_ops,
677 const struct nfs_pgio_completion_ops *compl_ops,
678 const struct nfs_rw_ops *rw_ops,
679 size_t bsize,
680 int io_flags)
681{
682 struct nfs_pgio_mirror *new;
683 int i;
684
685 desc->pg_moreio = 0;
686 desc->pg_inode = inode;
687 desc->pg_ops = pg_ops;
688 desc->pg_completion_ops = compl_ops;
689 desc->pg_rw_ops = rw_ops;
690 desc->pg_ioflags = io_flags;
691 desc->pg_error = 0;
692 desc->pg_lseg = NULL;
693 desc->pg_dreq = NULL;
694 desc->pg_layout_private = NULL;
695 desc->pg_bsize = bsize;
696
697 desc->pg_mirror_count = 1;
698 desc->pg_mirror_idx = 0;
699
700 if (pg_ops->pg_get_mirror_count) {
701
702
703 new = kcalloc(NFS_PAGEIO_DESCRIPTOR_MIRROR_MAX,
704 sizeof(struct nfs_pgio_mirror), GFP_KERNEL);
705 desc->pg_mirrors_dynamic = new;
706 desc->pg_mirrors = new;
707
708 for (i = 0; i < NFS_PAGEIO_DESCRIPTOR_MIRROR_MAX; i++)
709 nfs_pageio_mirror_init(&desc->pg_mirrors[i], bsize);
710 } else {
711 desc->pg_mirrors_dynamic = NULL;
712 desc->pg_mirrors = desc->pg_mirrors_static;
713 nfs_pageio_mirror_init(&desc->pg_mirrors[0], bsize);
714 }
715}
716EXPORT_SYMBOL_GPL(nfs_pageio_init);
717
718
719
720
721
722
723static void nfs_pgio_result(struct rpc_task *task, void *calldata)
724{
725 struct nfs_pgio_header *hdr = calldata;
726 struct inode *inode = hdr->inode;
727
728 dprintk("NFS: %s: %5u, (status %d)\n", __func__,
729 task->tk_pid, task->tk_status);
730
731 if (hdr->rw_ops->rw_done(task, hdr, inode) != 0)
732 return;
733 if (task->tk_status < 0)
734 nfs_set_pgio_error(hdr, task->tk_status, hdr->args.offset);
735 else
736 hdr->rw_ops->rw_result(task, hdr);
737}
738
739
740
741
742
743
744
745
746
747int nfs_generic_pgio(struct nfs_pageio_descriptor *desc,
748 struct nfs_pgio_header *hdr)
749{
750 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
751
752 struct nfs_page *req;
753 struct page **pages,
754 *last_page;
755 struct list_head *head = &mirror->pg_list;
756 struct nfs_commit_info cinfo;
757 unsigned int pagecount, pageused;
758
759 pagecount = nfs_page_array_len(mirror->pg_base, mirror->pg_count);
760 if (!nfs_pgarray_set(&hdr->page_array, pagecount)) {
761 nfs_pgio_error(hdr);
762 desc->pg_error = -ENOMEM;
763 return desc->pg_error;
764 }
765
766 nfs_init_cinfo(&cinfo, desc->pg_inode, desc->pg_dreq);
767 pages = hdr->page_array.pagevec;
768 last_page = NULL;
769 pageused = 0;
770 while (!list_empty(head)) {
771 req = nfs_list_entry(head->next);
772 nfs_list_remove_request(req);
773 nfs_list_add_request(req, &hdr->pages);
774
775 if (!last_page || last_page != req->wb_page) {
776 pageused++;
777 if (pageused > pagecount)
778 break;
779 *pages++ = last_page = req->wb_page;
780 }
781 }
782 if (WARN_ON_ONCE(pageused != pagecount)) {
783 nfs_pgio_error(hdr);
784 desc->pg_error = -EINVAL;
785 return desc->pg_error;
786 }
787
788 if ((desc->pg_ioflags & FLUSH_COND_STABLE) &&
789 (desc->pg_moreio || nfs_reqs_to_commit(&cinfo)))
790 desc->pg_ioflags &= ~FLUSH_COND_STABLE;
791
792
793 nfs_pgio_rpcsetup(hdr, mirror->pg_count, 0, desc->pg_ioflags, &cinfo);
794 desc->pg_rpc_callops = &nfs_pgio_common_ops;
795 return 0;
796}
797EXPORT_SYMBOL_GPL(nfs_generic_pgio);
798
799static int nfs_generic_pg_pgios(struct nfs_pageio_descriptor *desc)
800{
801 struct nfs_pgio_header *hdr;
802 int ret;
803
804 hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);
805 if (!hdr) {
806 desc->pg_error = -ENOMEM;
807 return desc->pg_error;
808 }
809 nfs_pgheader_init(desc, hdr, nfs_pgio_header_free);
810 ret = nfs_generic_pgio(desc, hdr);
811 if (ret == 0)
812 ret = nfs_initiate_pgio(NFS_CLIENT(hdr->inode),
813 hdr,
814 hdr->cred,
815 NFS_PROTO(hdr->inode),
816 desc->pg_rpc_callops,
817 desc->pg_ioflags, 0);
818 return ret;
819}
820
821
822
823
824
825static int nfs_pageio_setup_mirroring(struct nfs_pageio_descriptor *pgio,
826 struct nfs_page *req)
827{
828 int mirror_count = 1;
829
830 if (!pgio->pg_ops->pg_get_mirror_count)
831 return 0;
832
833 mirror_count = pgio->pg_ops->pg_get_mirror_count(pgio, req);
834
835 if (pgio->pg_error < 0)
836 return pgio->pg_error;
837
838 if (!mirror_count || mirror_count > NFS_PAGEIO_DESCRIPTOR_MIRROR_MAX)
839 return -EINVAL;
840
841 if (WARN_ON_ONCE(!pgio->pg_mirrors_dynamic))
842 return -EINVAL;
843
844 pgio->pg_mirror_count = mirror_count;
845
846 return 0;
847}
848
849
850
851
852void nfs_pageio_stop_mirroring(struct nfs_pageio_descriptor *pgio)
853{
854 pgio->pg_mirror_count = 1;
855 pgio->pg_mirror_idx = 0;
856}
857
858static void nfs_pageio_cleanup_mirroring(struct nfs_pageio_descriptor *pgio)
859{
860 pgio->pg_mirror_count = 1;
861 pgio->pg_mirror_idx = 0;
862 pgio->pg_mirrors = pgio->pg_mirrors_static;
863 kfree(pgio->pg_mirrors_dynamic);
864 pgio->pg_mirrors_dynamic = NULL;
865}
866
867static bool nfs_match_lock_context(const struct nfs_lock_context *l1,
868 const struct nfs_lock_context *l2)
869{
870 return l1->lockowner.l_owner == l2->lockowner.l_owner
871 && l1->lockowner.l_pid == l2->lockowner.l_pid;
872}
873
874
875
876
877
878
879
880
881
882
883
884
885static bool nfs_can_coalesce_requests(struct nfs_page *prev,
886 struct nfs_page *req,
887 struct nfs_pageio_descriptor *pgio)
888{
889 size_t size;
890 struct file_lock_context *flctx;
891
892 if (prev) {
893 if (!nfs_match_open_context(req->wb_context, prev->wb_context))
894 return false;
895 flctx = d_inode(req->wb_context->dentry)->i_flctx;
896 if (flctx != NULL &&
897 !(list_empty_careful(&flctx->flc_posix) &&
898 list_empty_careful(&flctx->flc_flock)) &&
899 !nfs_match_lock_context(req->wb_lock_context,
900 prev->wb_lock_context))
901 return false;
902 if (req_offset(req) != req_offset(prev) + prev->wb_bytes)
903 return false;
904 if (req->wb_page == prev->wb_page) {
905 if (req->wb_pgbase != prev->wb_pgbase + prev->wb_bytes)
906 return false;
907 } else {
908 if (req->wb_pgbase != 0 ||
909 prev->wb_pgbase + prev->wb_bytes != PAGE_SIZE)
910 return false;
911 }
912 }
913 size = pgio->pg_ops->pg_test(pgio, prev, req);
914 WARN_ON_ONCE(size > req->wb_bytes);
915 if (size && size < req->wb_bytes)
916 req->wb_bytes = size;
917 return size > 0;
918}
919
920
921
922
923
924
925
926
927
928static int nfs_pageio_do_add_request(struct nfs_pageio_descriptor *desc,
929 struct nfs_page *req)
930{
931 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
932
933 struct nfs_page *prev = NULL;
934
935 if (mirror->pg_count != 0) {
936 prev = nfs_list_entry(mirror->pg_list.prev);
937 } else {
938 if (desc->pg_ops->pg_init)
939 desc->pg_ops->pg_init(desc, req);
940 if (desc->pg_error < 0)
941 return 0;
942 mirror->pg_base = req->wb_pgbase;
943 }
944 if (!nfs_can_coalesce_requests(prev, req, desc))
945 return 0;
946 nfs_list_remove_request(req);
947 nfs_list_add_request(req, &mirror->pg_list);
948 mirror->pg_count += req->wb_bytes;
949 return 1;
950}
951
952
953
954
955static void nfs_pageio_doio(struct nfs_pageio_descriptor *desc)
956{
957 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
958
959
960 if (!list_empty(&mirror->pg_list)) {
961 int error = desc->pg_ops->pg_doio(desc);
962 if (error < 0)
963 desc->pg_error = error;
964 else
965 mirror->pg_bytes_written += mirror->pg_count;
966 }
967 if (list_empty(&mirror->pg_list)) {
968 mirror->pg_count = 0;
969 mirror->pg_base = 0;
970 }
971}
972
973
974
975
976
977
978
979
980
981
982
983
984static int __nfs_pageio_add_request(struct nfs_pageio_descriptor *desc,
985 struct nfs_page *req)
986{
987 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
988
989 struct nfs_page *subreq;
990 unsigned int bytes_left = 0;
991 unsigned int offset, pgbase;
992
993 nfs_page_group_lock(req, false);
994
995 subreq = req;
996 bytes_left = subreq->wb_bytes;
997 offset = subreq->wb_offset;
998 pgbase = subreq->wb_pgbase;
999
1000 do {
1001 if (!nfs_pageio_do_add_request(desc, subreq)) {
1002
1003 WARN_ON_ONCE(subreq->wb_bytes != bytes_left);
1004 WARN_ON_ONCE(subreq->wb_offset != offset);
1005 WARN_ON_ONCE(subreq->wb_pgbase != pgbase);
1006
1007 nfs_page_group_unlock(req);
1008 desc->pg_moreio = 1;
1009 nfs_pageio_doio(desc);
1010 if (desc->pg_error < 0)
1011 return 0;
1012 if (mirror->pg_recoalesce)
1013 return 0;
1014
1015 nfs_page_group_lock(req, false);
1016 continue;
1017 }
1018
1019
1020 WARN_ON_ONCE(subreq->wb_bytes + subreq->wb_pgbase > PAGE_SIZE);
1021 WARN_ON_ONCE(subreq->wb_bytes > bytes_left);
1022 WARN_ON_ONCE(subreq->wb_bytes == 0);
1023
1024 bytes_left -= subreq->wb_bytes;
1025 offset += subreq->wb_bytes;
1026 pgbase += subreq->wb_bytes;
1027
1028 if (bytes_left) {
1029 subreq = nfs_create_request(req->wb_context,
1030 req->wb_page,
1031 subreq, pgbase, bytes_left);
1032 if (IS_ERR(subreq))
1033 goto err_ptr;
1034 nfs_lock_request(subreq);
1035 subreq->wb_offset = offset;
1036 subreq->wb_index = req->wb_index;
1037 }
1038 } while (bytes_left > 0);
1039
1040 nfs_page_group_unlock(req);
1041 return 1;
1042err_ptr:
1043 desc->pg_error = PTR_ERR(subreq);
1044 nfs_page_group_unlock(req);
1045 return 0;
1046}
1047
1048static int nfs_do_recoalesce(struct nfs_pageio_descriptor *desc)
1049{
1050 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);
1051 LIST_HEAD(head);
1052
1053 do {
1054 list_splice_init(&mirror->pg_list, &head);
1055 mirror->pg_bytes_written -= mirror->pg_count;
1056 mirror->pg_count = 0;
1057 mirror->pg_base = 0;
1058 mirror->pg_recoalesce = 0;
1059
1060 while (!list_empty(&head)) {
1061 struct nfs_page *req;
1062
1063 req = list_first_entry(&head, struct nfs_page, wb_list);
1064 nfs_list_remove_request(req);
1065 if (__nfs_pageio_add_request(desc, req))
1066 continue;
1067 if (desc->pg_error < 0) {
1068 list_splice_tail(&head, &mirror->pg_list);
1069 mirror->pg_recoalesce = 1;
1070 return 0;
1071 }
1072 break;
1073 }
1074 } while (mirror->pg_recoalesce);
1075 return 1;
1076}
1077
1078static int nfs_pageio_add_request_mirror(struct nfs_pageio_descriptor *desc,
1079 struct nfs_page *req)
1080{
1081 int ret;
1082
1083 do {
1084 ret = __nfs_pageio_add_request(desc, req);
1085 if (ret)
1086 break;
1087 if (desc->pg_error < 0)
1088 break;
1089 ret = nfs_do_recoalesce(desc);
1090 } while (ret);
1091
1092 return ret;
1093}
1094
1095int nfs_pageio_add_request(struct nfs_pageio_descriptor *desc,
1096 struct nfs_page *req)
1097{
1098 u32 midx;
1099 unsigned int pgbase, offset, bytes;
1100 struct nfs_page *dupreq, *lastreq;
1101
1102 pgbase = req->wb_pgbase;
1103 offset = req->wb_offset;
1104 bytes = req->wb_bytes;
1105
1106 nfs_pageio_setup_mirroring(desc, req);
1107 if (desc->pg_error < 0)
1108 goto out_failed;
1109
1110 for (midx = 0; midx < desc->pg_mirror_count; midx++) {
1111 if (midx) {
1112 nfs_page_group_lock(req, false);
1113
1114
1115 for (lastreq = req->wb_head;
1116 lastreq->wb_this_page != req->wb_head;
1117 lastreq = lastreq->wb_this_page)
1118 ;
1119
1120 dupreq = nfs_create_request(req->wb_context,
1121 req->wb_page, lastreq, pgbase, bytes);
1122
1123 if (IS_ERR(dupreq)) {
1124 nfs_page_group_unlock(req);
1125 desc->pg_error = PTR_ERR(dupreq);
1126 goto out_failed;
1127 }
1128
1129 nfs_lock_request(dupreq);
1130 nfs_page_group_unlock(req);
1131 dupreq->wb_offset = offset;
1132 dupreq->wb_index = req->wb_index;
1133 } else
1134 dupreq = req;
1135
1136 if (nfs_pgio_has_mirroring(desc))
1137 desc->pg_mirror_idx = midx;
1138 if (!nfs_pageio_add_request_mirror(desc, dupreq))
1139 goto out_failed;
1140 }
1141
1142 return 1;
1143
1144out_failed:
1145
1146
1147
1148
1149 if (desc->pg_error) {
1150 struct nfs_pgio_mirror *mirror;
1151 void (*func)(struct list_head *);
1152
1153
1154 if (nfs_error_is_fatal(desc->pg_error))
1155 mapping_set_error(desc->pg_inode->i_mapping,
1156 desc->pg_error);
1157
1158 func = desc->pg_completion_ops->error_cleanup;
1159 for (midx = 0; midx < desc->pg_mirror_count; midx++) {
1160 mirror = &desc->pg_mirrors[midx];
1161 func(&mirror->pg_list);
1162 }
1163 }
1164 return 0;
1165}
1166
1167
1168
1169
1170
1171
1172
1173static void nfs_pageio_complete_mirror(struct nfs_pageio_descriptor *desc,
1174 u32 mirror_idx)
1175{
1176 struct nfs_pgio_mirror *mirror = &desc->pg_mirrors[mirror_idx];
1177 u32 restore_idx = desc->pg_mirror_idx;
1178
1179 if (nfs_pgio_has_mirroring(desc))
1180 desc->pg_mirror_idx = mirror_idx;
1181 for (;;) {
1182 nfs_pageio_doio(desc);
1183 if (!mirror->pg_recoalesce)
1184 break;
1185 if (!nfs_do_recoalesce(desc))
1186 break;
1187 }
1188 desc->pg_mirror_idx = restore_idx;
1189}
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201int nfs_pageio_resend(struct nfs_pageio_descriptor *desc,
1202 struct nfs_pgio_header *hdr)
1203{
1204 LIST_HEAD(failed);
1205
1206 desc->pg_dreq = hdr->dreq;
1207 while (!list_empty(&hdr->pages)) {
1208 struct nfs_page *req = nfs_list_entry(hdr->pages.next);
1209
1210 nfs_list_remove_request(req);
1211 if (!nfs_pageio_add_request(desc, req))
1212 nfs_list_add_request(req, &failed);
1213 }
1214 nfs_pageio_complete(desc);
1215 if (!list_empty(&failed)) {
1216 list_move(&failed, &hdr->pages);
1217 return desc->pg_error < 0 ? desc->pg_error : -EIO;
1218 }
1219 return 0;
1220}
1221EXPORT_SYMBOL_GPL(nfs_pageio_resend);
1222
1223
1224
1225
1226
1227void nfs_pageio_complete(struct nfs_pageio_descriptor *desc)
1228{
1229 u32 midx;
1230
1231 for (midx = 0; midx < desc->pg_mirror_count; midx++)
1232 nfs_pageio_complete_mirror(desc, midx);
1233
1234 if (desc->pg_ops->pg_cleanup)
1235 desc->pg_ops->pg_cleanup(desc);
1236 nfs_pageio_cleanup_mirroring(desc);
1237}
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250void nfs_pageio_cond_complete(struct nfs_pageio_descriptor *desc, pgoff_t index)
1251{
1252 struct nfs_pgio_mirror *mirror;
1253 struct nfs_page *prev;
1254 u32 midx;
1255
1256 for (midx = 0; midx < desc->pg_mirror_count; midx++) {
1257 mirror = &desc->pg_mirrors[midx];
1258 if (!list_empty(&mirror->pg_list)) {
1259 prev = nfs_list_entry(mirror->pg_list.prev);
1260 if (index != prev->wb_index + 1)
1261 nfs_pageio_complete_mirror(desc, midx);
1262 }
1263 }
1264}
1265
1266int __init nfs_init_nfspagecache(void)
1267{
1268 nfs_page_cachep = kmem_cache_create("nfs_page",
1269 sizeof(struct nfs_page),
1270 0, SLAB_HWCACHE_ALIGN,
1271 NULL);
1272 if (nfs_page_cachep == NULL)
1273 return -ENOMEM;
1274
1275 return 0;
1276}
1277
1278void nfs_destroy_nfspagecache(void)
1279{
1280 kmem_cache_destroy(nfs_page_cachep);
1281}
1282
1283static const struct rpc_call_ops nfs_pgio_common_ops = {
1284 .rpc_call_prepare = nfs_pgio_prepare,
1285 .rpc_call_done = nfs_pgio_result,
1286 .rpc_release = nfs_pgio_release,
1287};
1288
1289const struct nfs_pageio_ops nfs_pgio_rw_ops = {
1290 .pg_test = nfs_generic_pg_test,
1291 .pg_doio = nfs_generic_pg_pgios,
1292};
1293