1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41#include <linux/errno.h>
42#include <linux/sched.h>
43#include <linux/kernel.h>
44#include <linux/file.h>
45#include <linux/pagemap.h>
46#include <linux/kref.h>
47#include <linux/slab.h>
48#include <linux/task_io_accounting_ops.h>
49#include <linux/module.h>
50
51#include <linux/nfs_fs.h>
52#include <linux/nfs_page.h>
53#include <linux/sunrpc/clnt.h>
54
55#include <linux/uaccess.h>
56#include <linux/atomic.h>
57
58#include "internal.h"
59#include "iostat.h"
60#include "pnfs.h"
61
62#define NFSDBG_FACILITY NFSDBG_VFS
63
64static struct kmem_cache *nfs_direct_cachep;
65
66
67
68
69struct nfs_direct_mirror {
70 ssize_t count;
71};
72
73struct nfs_direct_req {
74 struct kref kref;
75
76
77 struct nfs_open_context *ctx;
78 struct nfs_lock_context *l_ctx;
79 struct kiocb * iocb;
80 struct inode * inode;
81
82
83 atomic_t io_count;
84 spinlock_t lock;
85
86 struct nfs_direct_mirror mirrors[NFS_PAGEIO_DESCRIPTOR_MIRROR_MAX];
87 int mirror_count;
88
89 ssize_t count,
90 max_count,
91 bytes_left,
92 io_start,
93 error;
94 struct completion completion;
95
96
97 struct nfs_mds_commit_info mds_cinfo;
98 struct pnfs_ds_commit_info ds_cinfo;
99 struct work_struct work;
100 int flags;
101#define NFS_ODIRECT_DO_COMMIT (1)
102#define NFS_ODIRECT_RESCHED_WRITES (2)
103 struct nfs_writeverf verf;
104};
105
106static const struct nfs_pgio_completion_ops nfs_direct_write_completion_ops;
107static const struct nfs_commit_completion_ops nfs_direct_commit_completion_ops;
108static void nfs_direct_write_complete(struct nfs_direct_req *dreq);
109static void nfs_direct_write_schedule_work(struct work_struct *work);
110
111static inline void get_dreq(struct nfs_direct_req *dreq)
112{
113 atomic_inc(&dreq->io_count);
114}
115
116static inline int put_dreq(struct nfs_direct_req *dreq)
117{
118 return atomic_dec_and_test(&dreq->io_count);
119}
120
121static void
122nfs_direct_good_bytes(struct nfs_direct_req *dreq, struct nfs_pgio_header *hdr)
123{
124 int i;
125 ssize_t count;
126
127 WARN_ON_ONCE(dreq->count >= dreq->max_count);
128
129 if (dreq->mirror_count == 1) {
130 dreq->mirrors[hdr->pgio_mirror_idx].count += hdr->good_bytes;
131 dreq->count += hdr->good_bytes;
132 } else {
133
134 count = dreq->mirrors[hdr->pgio_mirror_idx].count;
135 if (count + dreq->io_start < hdr->io_start + hdr->good_bytes) {
136 count = hdr->io_start + hdr->good_bytes - dreq->io_start;
137 dreq->mirrors[hdr->pgio_mirror_idx].count = count;
138 }
139
140
141 count = dreq->mirrors[0].count;
142
143 for (i = 1; i < dreq->mirror_count; i++)
144 count = min(count, dreq->mirrors[i].count);
145
146 dreq->count = count;
147 }
148}
149
150
151
152
153
154
155
156
157
158static struct nfs_writeverf *
159nfs_direct_select_verf(struct nfs_direct_req *dreq,
160 struct nfs_client *ds_clp,
161 int commit_idx)
162{
163 struct nfs_writeverf *verfp = &dreq->verf;
164
165#ifdef CONFIG_NFS_V4_1
166
167
168
169
170 if (ds_clp && dreq->ds_cinfo.nbuckets > 0) {
171 if (commit_idx >= 0 && commit_idx < dreq->ds_cinfo.nbuckets)
172 verfp = &dreq->ds_cinfo.buckets[commit_idx].direct_verf;
173 else
174 WARN_ON_ONCE(1);
175 }
176#endif
177 return verfp;
178}
179
180
181
182
183
184
185
186
187
188static void nfs_direct_set_hdr_verf(struct nfs_direct_req *dreq,
189 struct nfs_pgio_header *hdr)
190{
191 struct nfs_writeverf *verfp;
192
193 verfp = nfs_direct_select_verf(dreq, hdr->ds_clp, hdr->ds_commit_idx);
194 WARN_ON_ONCE(verfp->committed >= 0);
195 memcpy(verfp, &hdr->verf, sizeof(struct nfs_writeverf));
196 WARN_ON_ONCE(verfp->committed < 0);
197}
198
199static int nfs_direct_cmp_verf(const struct nfs_writeverf *v1,
200 const struct nfs_writeverf *v2)
201{
202 return nfs_write_verifier_cmp(&v1->verifier, &v2->verifier);
203}
204
205
206
207
208
209
210
211
212
213
214static int nfs_direct_set_or_cmp_hdr_verf(struct nfs_direct_req *dreq,
215 struct nfs_pgio_header *hdr)
216{
217 struct nfs_writeverf *verfp;
218
219 verfp = nfs_direct_select_verf(dreq, hdr->ds_clp, hdr->ds_commit_idx);
220 if (verfp->committed < 0) {
221 nfs_direct_set_hdr_verf(dreq, hdr);
222 return 0;
223 }
224 return nfs_direct_cmp_verf(verfp, &hdr->verf);
225}
226
227
228
229
230
231
232
233
234
235static int nfs_direct_cmp_commit_data_verf(struct nfs_direct_req *dreq,
236 struct nfs_commit_data *data)
237{
238 struct nfs_writeverf *verfp;
239
240 verfp = nfs_direct_select_verf(dreq, data->ds_clp,
241 data->ds_commit_index);
242
243
244 if (verfp->committed < 0)
245 return 1;
246
247 return nfs_direct_cmp_verf(verfp, &data->verf);
248}
249
250
251
252
253
254
255
256
257
258
259
260ssize_t nfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
261{
262 struct inode *inode = iocb->ki_filp->f_mapping->host;
263
264
265 if (!IS_SWAPFILE(inode))
266 return 0;
267
268 VM_BUG_ON(iov_iter_count(iter) != PAGE_SIZE);
269
270 if (iov_iter_rw(iter) == READ)
271 return nfs_file_direct_read(iocb, iter);
272 return nfs_file_direct_write(iocb, iter);
273}
274
275static void nfs_direct_release_pages(struct page **pages, unsigned int npages)
276{
277 unsigned int i;
278 for (i = 0; i < npages; i++)
279 put_page(pages[i]);
280}
281
282void nfs_init_cinfo_from_dreq(struct nfs_commit_info *cinfo,
283 struct nfs_direct_req *dreq)
284{
285 cinfo->inode = dreq->inode;
286 cinfo->mds = &dreq->mds_cinfo;
287 cinfo->ds = &dreq->ds_cinfo;
288 cinfo->dreq = dreq;
289 cinfo->completion_ops = &nfs_direct_commit_completion_ops;
290}
291
292static inline void nfs_direct_setup_mirroring(struct nfs_direct_req *dreq,
293 struct nfs_pageio_descriptor *pgio,
294 struct nfs_page *req)
295{
296 int mirror_count = 1;
297
298 if (pgio->pg_ops->pg_get_mirror_count)
299 mirror_count = pgio->pg_ops->pg_get_mirror_count(pgio, req);
300
301 dreq->mirror_count = mirror_count;
302}
303
304static inline struct nfs_direct_req *nfs_direct_req_alloc(void)
305{
306 struct nfs_direct_req *dreq;
307
308 dreq = kmem_cache_zalloc(nfs_direct_cachep, GFP_KERNEL);
309 if (!dreq)
310 return NULL;
311
312 kref_init(&dreq->kref);
313 kref_get(&dreq->kref);
314 init_completion(&dreq->completion);
315 INIT_LIST_HEAD(&dreq->mds_cinfo.list);
316 dreq->verf.committed = NFS_INVALID_STABLE_HOW;
317 INIT_WORK(&dreq->work, nfs_direct_write_schedule_work);
318 dreq->mirror_count = 1;
319 spin_lock_init(&dreq->lock);
320
321 return dreq;
322}
323
324static void nfs_direct_req_free(struct kref *kref)
325{
326 struct nfs_direct_req *dreq = container_of(kref, struct nfs_direct_req, kref);
327
328 nfs_free_pnfs_ds_cinfo(&dreq->ds_cinfo);
329 if (dreq->l_ctx != NULL)
330 nfs_put_lock_context(dreq->l_ctx);
331 if (dreq->ctx != NULL)
332 put_nfs_open_context(dreq->ctx);
333 kmem_cache_free(nfs_direct_cachep, dreq);
334}
335
336static void nfs_direct_req_release(struct nfs_direct_req *dreq)
337{
338 kref_put(&dreq->kref, nfs_direct_req_free);
339}
340
341ssize_t nfs_dreq_bytes_left(struct nfs_direct_req *dreq)
342{
343 return dreq->bytes_left;
344}
345EXPORT_SYMBOL_GPL(nfs_dreq_bytes_left);
346
347
348
349
350static ssize_t nfs_direct_wait(struct nfs_direct_req *dreq)
351{
352 ssize_t result = -EIOCBQUEUED;
353
354
355 if (dreq->iocb)
356 goto out;
357
358 result = wait_for_completion_killable(&dreq->completion);
359
360 if (!result) {
361 result = dreq->count;
362 WARN_ON_ONCE(dreq->count < 0);
363 }
364 if (!result)
365 result = dreq->error;
366
367out:
368 return (ssize_t) result;
369}
370
371
372
373
374
375static void nfs_direct_complete(struct nfs_direct_req *dreq)
376{
377 struct inode *inode = dreq->inode;
378
379 inode_dio_end(inode);
380
381 if (dreq->iocb) {
382 long res = (long) dreq->error;
383 if (dreq->count != 0) {
384 res = (long) dreq->count;
385 WARN_ON_ONCE(dreq->count < 0);
386 }
387 dreq->iocb->ki_complete(dreq->iocb, res, 0);
388 }
389
390 complete(&dreq->completion);
391
392 nfs_direct_req_release(dreq);
393}
394
395static void nfs_direct_read_completion(struct nfs_pgio_header *hdr)
396{
397 unsigned long bytes = 0;
398 struct nfs_direct_req *dreq = hdr->dreq;
399
400 if (test_bit(NFS_IOHDR_REDO, &hdr->flags))
401 goto out_put;
402
403 spin_lock(&dreq->lock);
404 if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) && (hdr->good_bytes == 0))
405 dreq->error = hdr->error;
406 else
407 nfs_direct_good_bytes(dreq, hdr);
408
409 spin_unlock(&dreq->lock);
410
411 while (!list_empty(&hdr->pages)) {
412 struct nfs_page *req = nfs_list_entry(hdr->pages.next);
413 struct page *page = req->wb_page;
414
415 if (!PageCompound(page) && bytes < hdr->good_bytes)
416 set_page_dirty(page);
417 bytes += req->wb_bytes;
418 nfs_list_remove_request(req);
419 nfs_release_request(req);
420 }
421out_put:
422 if (put_dreq(dreq))
423 nfs_direct_complete(dreq);
424 hdr->release(hdr);
425}
426
427static void nfs_read_sync_pgio_error(struct list_head *head)
428{
429 struct nfs_page *req;
430
431 while (!list_empty(head)) {
432 req = nfs_list_entry(head->next);
433 nfs_list_remove_request(req);
434 nfs_release_request(req);
435 }
436}
437
438static void nfs_direct_pgio_init(struct nfs_pgio_header *hdr)
439{
440 get_dreq(hdr->dreq);
441}
442
443static const struct nfs_pgio_completion_ops nfs_direct_read_completion_ops = {
444 .error_cleanup = nfs_read_sync_pgio_error,
445 .init_hdr = nfs_direct_pgio_init,
446 .completion = nfs_direct_read_completion,
447};
448
449
450
451
452
453
454
455
456
457static ssize_t nfs_direct_read_schedule_iovec(struct nfs_direct_req *dreq,
458 struct iov_iter *iter,
459 loff_t pos)
460{
461 struct nfs_pageio_descriptor desc;
462 struct inode *inode = dreq->inode;
463 ssize_t result = -EINVAL;
464 size_t requested_bytes = 0;
465 size_t rsize = max_t(size_t, NFS_SERVER(inode)->rsize, PAGE_SIZE);
466
467 nfs_pageio_init_read(&desc, dreq->inode, false,
468 &nfs_direct_read_completion_ops);
469 get_dreq(dreq);
470 desc.pg_dreq = dreq;
471 inode_dio_begin(inode);
472
473 while (iov_iter_count(iter)) {
474 struct page **pagevec;
475 size_t bytes;
476 size_t pgbase;
477 unsigned npages, i;
478
479 result = iov_iter_get_pages_alloc(iter, &pagevec,
480 rsize, &pgbase);
481 if (result < 0)
482 break;
483
484 bytes = result;
485 iov_iter_advance(iter, bytes);
486 npages = (result + pgbase + PAGE_SIZE - 1) / PAGE_SIZE;
487 for (i = 0; i < npages; i++) {
488 struct nfs_page *req;
489 unsigned int req_len = min_t(size_t, bytes, PAGE_SIZE - pgbase);
490
491 req = nfs_create_request(dreq->ctx, pagevec[i], NULL,
492 pgbase, req_len);
493 if (IS_ERR(req)) {
494 result = PTR_ERR(req);
495 break;
496 }
497 req->wb_index = pos >> PAGE_SHIFT;
498 req->wb_offset = pos & ~PAGE_MASK;
499 if (!nfs_pageio_add_request(&desc, req)) {
500 result = desc.pg_error;
501 nfs_release_request(req);
502 break;
503 }
504 pgbase = 0;
505 bytes -= req_len;
506 requested_bytes += req_len;
507 pos += req_len;
508 dreq->bytes_left -= req_len;
509 }
510 nfs_direct_release_pages(pagevec, npages);
511 kvfree(pagevec);
512 if (result < 0)
513 break;
514 }
515
516 nfs_pageio_complete(&desc);
517
518
519
520
521
522 if (requested_bytes == 0) {
523 inode_dio_end(inode);
524 nfs_direct_req_release(dreq);
525 return result < 0 ? result : -EIO;
526 }
527
528 if (put_dreq(dreq))
529 nfs_direct_complete(dreq);
530 return requested_bytes;
531}
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552ssize_t nfs_file_direct_read(struct kiocb *iocb, struct iov_iter *iter)
553{
554 struct file *file = iocb->ki_filp;
555 struct address_space *mapping = file->f_mapping;
556 struct inode *inode = mapping->host;
557 struct nfs_direct_req *dreq;
558 struct nfs_lock_context *l_ctx;
559 ssize_t result = -EINVAL, requested;
560 size_t count = iov_iter_count(iter);
561 nfs_add_stats(mapping->host, NFSIOS_DIRECTREADBYTES, count);
562
563 dfprintk(FILE, "NFS: direct read(%pD2, %zd@%Ld)\n",
564 file, count, (long long) iocb->ki_pos);
565
566 result = 0;
567 if (!count)
568 goto out;
569
570 task_io_account_read(count);
571
572 result = -ENOMEM;
573 dreq = nfs_direct_req_alloc();
574 if (dreq == NULL)
575 goto out;
576
577 dreq->inode = inode;
578 dreq->bytes_left = dreq->max_count = count;
579 dreq->io_start = iocb->ki_pos;
580 dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));
581 l_ctx = nfs_get_lock_context(dreq->ctx);
582 if (IS_ERR(l_ctx)) {
583 result = PTR_ERR(l_ctx);
584 goto out_release;
585 }
586 dreq->l_ctx = l_ctx;
587 if (!is_sync_kiocb(iocb))
588 dreq->iocb = iocb;
589
590 nfs_start_io_direct(inode);
591
592 NFS_I(inode)->read_io += count;
593 requested = nfs_direct_read_schedule_iovec(dreq, iter, iocb->ki_pos);
594
595 nfs_end_io_direct(inode);
596
597 if (requested > 0) {
598 result = nfs_direct_wait(dreq);
599 if (result > 0) {
600 requested -= result;
601 iocb->ki_pos += result;
602 }
603 iov_iter_revert(iter, requested);
604 } else {
605 result = requested;
606 }
607
608out_release:
609 nfs_direct_req_release(dreq);
610out:
611 return result;
612}
613
614static void
615nfs_direct_write_scan_commit_list(struct inode *inode,
616 struct list_head *list,
617 struct nfs_commit_info *cinfo)
618{
619 mutex_lock(&NFS_I(cinfo->inode)->commit_mutex);
620#ifdef CONFIG_NFS_V4_1
621 if (cinfo->ds != NULL && cinfo->ds->nwritten != 0)
622 NFS_SERVER(inode)->pnfs_curr_ld->recover_commit_reqs(list, cinfo);
623#endif
624 nfs_scan_commit_list(&cinfo->mds->list, list, cinfo, 0);
625 mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex);
626}
627
628static void nfs_direct_write_reschedule(struct nfs_direct_req *dreq)
629{
630 struct nfs_pageio_descriptor desc;
631 struct nfs_page *req, *tmp;
632 LIST_HEAD(reqs);
633 struct nfs_commit_info cinfo;
634 LIST_HEAD(failed);
635 int i;
636
637 nfs_init_cinfo_from_dreq(&cinfo, dreq);
638 nfs_direct_write_scan_commit_list(dreq->inode, &reqs, &cinfo);
639
640 dreq->count = 0;
641 dreq->verf.committed = NFS_INVALID_STABLE_HOW;
642 nfs_clear_pnfs_ds_commit_verifiers(&dreq->ds_cinfo);
643 for (i = 0; i < dreq->mirror_count; i++)
644 dreq->mirrors[i].count = 0;
645 get_dreq(dreq);
646
647 nfs_pageio_init_write(&desc, dreq->inode, FLUSH_STABLE, false,
648 &nfs_direct_write_completion_ops);
649 desc.pg_dreq = dreq;
650
651 req = nfs_list_entry(reqs.next);
652 nfs_direct_setup_mirroring(dreq, &desc, req);
653 if (desc.pg_error < 0) {
654 list_splice_init(&reqs, &failed);
655 goto out_failed;
656 }
657
658 list_for_each_entry_safe(req, tmp, &reqs, wb_list) {
659 if (!nfs_pageio_add_request(&desc, req)) {
660 nfs_list_remove_request(req);
661 nfs_list_add_request(req, &failed);
662 spin_lock(&cinfo.inode->i_lock);
663 dreq->flags = 0;
664 if (desc.pg_error < 0)
665 dreq->error = desc.pg_error;
666 else
667 dreq->error = -EIO;
668 spin_unlock(&cinfo.inode->i_lock);
669 }
670 nfs_release_request(req);
671 }
672 nfs_pageio_complete(&desc);
673
674out_failed:
675 while (!list_empty(&failed)) {
676 req = nfs_list_entry(failed.next);
677 nfs_list_remove_request(req);
678 nfs_unlock_and_release_request(req);
679 }
680
681 if (put_dreq(dreq))
682 nfs_direct_write_complete(dreq);
683}
684
685static void nfs_direct_commit_complete(struct nfs_commit_data *data)
686{
687 struct nfs_direct_req *dreq = data->dreq;
688 struct nfs_commit_info cinfo;
689 struct nfs_page *req;
690 int status = data->task.tk_status;
691
692 nfs_init_cinfo_from_dreq(&cinfo, dreq);
693 if (status < 0 || nfs_direct_cmp_commit_data_verf(dreq, data))
694 dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
695
696 while (!list_empty(&data->pages)) {
697 req = nfs_list_entry(data->pages.next);
698 nfs_list_remove_request(req);
699 if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES) {
700
701 nfs_mark_request_commit(req, NULL, &cinfo, 0);
702 } else
703 nfs_release_request(req);
704 nfs_unlock_and_release_request(req);
705 }
706
707 if (atomic_dec_and_test(&cinfo.mds->rpcs_out))
708 nfs_direct_write_complete(dreq);
709}
710
711static void nfs_direct_resched_write(struct nfs_commit_info *cinfo,
712 struct nfs_page *req)
713{
714 struct nfs_direct_req *dreq = cinfo->dreq;
715
716 spin_lock(&dreq->lock);
717 dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
718 spin_unlock(&dreq->lock);
719 nfs_mark_request_commit(req, NULL, cinfo, 0);
720}
721
722static const struct nfs_commit_completion_ops nfs_direct_commit_completion_ops = {
723 .completion = nfs_direct_commit_complete,
724 .resched_write = nfs_direct_resched_write,
725};
726
727static void nfs_direct_commit_schedule(struct nfs_direct_req *dreq)
728{
729 int res;
730 struct nfs_commit_info cinfo;
731 LIST_HEAD(mds_list);
732
733 nfs_init_cinfo_from_dreq(&cinfo, dreq);
734 nfs_scan_commit(dreq->inode, &mds_list, &cinfo);
735 res = nfs_generic_commit_list(dreq->inode, &mds_list, 0, &cinfo);
736 if (res < 0)
737 nfs_direct_write_reschedule(dreq);
738}
739
740static void nfs_direct_write_schedule_work(struct work_struct *work)
741{
742 struct nfs_direct_req *dreq = container_of(work, struct nfs_direct_req, work);
743 int flags = dreq->flags;
744
745 dreq->flags = 0;
746 switch (flags) {
747 case NFS_ODIRECT_DO_COMMIT:
748 nfs_direct_commit_schedule(dreq);
749 break;
750 case NFS_ODIRECT_RESCHED_WRITES:
751 nfs_direct_write_reschedule(dreq);
752 break;
753 default:
754 nfs_zap_mapping(dreq->inode, dreq->inode->i_mapping);
755 nfs_direct_complete(dreq);
756 }
757}
758
759static void nfs_direct_write_complete(struct nfs_direct_req *dreq)
760{
761 schedule_work(&dreq->work);
762}
763
764static void nfs_direct_write_completion(struct nfs_pgio_header *hdr)
765{
766 struct nfs_direct_req *dreq = hdr->dreq;
767 struct nfs_commit_info cinfo;
768 bool request_commit = false;
769 struct nfs_page *req = nfs_list_entry(hdr->pages.next);
770
771 if (test_bit(NFS_IOHDR_REDO, &hdr->flags))
772 goto out_put;
773
774 nfs_init_cinfo_from_dreq(&cinfo, dreq);
775
776 spin_lock(&dreq->lock);
777
778 if (test_bit(NFS_IOHDR_ERROR, &hdr->flags)) {
779 dreq->flags = 0;
780 dreq->error = hdr->error;
781 }
782 if (dreq->error == 0) {
783 nfs_direct_good_bytes(dreq, hdr);
784 if (nfs_write_need_commit(hdr)) {
785 if (dreq->flags == NFS_ODIRECT_RESCHED_WRITES)
786 request_commit = true;
787 else if (dreq->flags == 0) {
788 nfs_direct_set_hdr_verf(dreq, hdr);
789 request_commit = true;
790 dreq->flags = NFS_ODIRECT_DO_COMMIT;
791 } else if (dreq->flags == NFS_ODIRECT_DO_COMMIT) {
792 request_commit = true;
793 if (nfs_direct_set_or_cmp_hdr_verf(dreq, hdr))
794 dreq->flags =
795 NFS_ODIRECT_RESCHED_WRITES;
796 }
797 }
798 }
799 spin_unlock(&dreq->lock);
800
801 while (!list_empty(&hdr->pages)) {
802
803 req = nfs_list_entry(hdr->pages.next);
804 nfs_list_remove_request(req);
805 if (request_commit) {
806 kref_get(&req->wb_kref);
807 nfs_mark_request_commit(req, hdr->lseg, &cinfo,
808 hdr->ds_commit_idx);
809 }
810 nfs_unlock_and_release_request(req);
811 }
812
813out_put:
814 if (put_dreq(dreq))
815 nfs_direct_write_complete(dreq);
816 hdr->release(hdr);
817}
818
819static void nfs_write_sync_pgio_error(struct list_head *head)
820{
821 struct nfs_page *req;
822
823 while (!list_empty(head)) {
824 req = nfs_list_entry(head->next);
825 nfs_list_remove_request(req);
826 nfs_unlock_and_release_request(req);
827 }
828}
829
830static void nfs_direct_write_reschedule_io(struct nfs_pgio_header *hdr)
831{
832 struct nfs_direct_req *dreq = hdr->dreq;
833
834 spin_lock(&dreq->lock);
835 if (dreq->error == 0) {
836 dreq->flags = NFS_ODIRECT_RESCHED_WRITES;
837
838 hdr->verf.committed = NFS_UNSTABLE;
839 hdr->good_bytes = hdr->args.count;
840 }
841 spin_unlock(&dreq->lock);
842}
843
844static const struct nfs_pgio_completion_ops nfs_direct_write_completion_ops = {
845 .error_cleanup = nfs_write_sync_pgio_error,
846 .init_hdr = nfs_direct_pgio_init,
847 .completion = nfs_direct_write_completion,
848 .reschedule_io = nfs_direct_write_reschedule_io,
849};
850
851
852
853
854
855
856
857
858
859
860
861
862
863static ssize_t nfs_direct_write_schedule_iovec(struct nfs_direct_req *dreq,
864 struct iov_iter *iter,
865 loff_t pos)
866{
867 struct nfs_pageio_descriptor desc;
868 struct inode *inode = dreq->inode;
869 ssize_t result = 0;
870 size_t requested_bytes = 0;
871 size_t wsize = max_t(size_t, NFS_SERVER(inode)->wsize, PAGE_SIZE);
872
873 nfs_pageio_init_write(&desc, inode, FLUSH_COND_STABLE, false,
874 &nfs_direct_write_completion_ops);
875 desc.pg_dreq = dreq;
876 get_dreq(dreq);
877 inode_dio_begin(inode);
878
879 NFS_I(inode)->write_io += iov_iter_count(iter);
880 while (iov_iter_count(iter)) {
881 struct page **pagevec;
882 size_t bytes;
883 size_t pgbase;
884 unsigned npages, i;
885
886 result = iov_iter_get_pages_alloc(iter, &pagevec,
887 wsize, &pgbase);
888 if (result < 0)
889 break;
890
891 bytes = result;
892 iov_iter_advance(iter, bytes);
893 npages = (result + pgbase + PAGE_SIZE - 1) / PAGE_SIZE;
894 for (i = 0; i < npages; i++) {
895 struct nfs_page *req;
896 unsigned int req_len = min_t(size_t, bytes, PAGE_SIZE - pgbase);
897
898 req = nfs_create_request(dreq->ctx, pagevec[i], NULL,
899 pgbase, req_len);
900 if (IS_ERR(req)) {
901 result = PTR_ERR(req);
902 break;
903 }
904
905 nfs_direct_setup_mirroring(dreq, &desc, req);
906 if (desc.pg_error < 0) {
907 nfs_free_request(req);
908 result = desc.pg_error;
909 break;
910 }
911
912 nfs_lock_request(req);
913 req->wb_index = pos >> PAGE_SHIFT;
914 req->wb_offset = pos & ~PAGE_MASK;
915 if (!nfs_pageio_add_request(&desc, req)) {
916 result = desc.pg_error;
917 nfs_unlock_and_release_request(req);
918 break;
919 }
920 pgbase = 0;
921 bytes -= req_len;
922 requested_bytes += req_len;
923 pos += req_len;
924 dreq->bytes_left -= req_len;
925 }
926 nfs_direct_release_pages(pagevec, npages);
927 kvfree(pagevec);
928 if (result < 0)
929 break;
930 }
931 nfs_pageio_complete(&desc);
932
933
934
935
936
937 if (requested_bytes == 0) {
938 inode_dio_end(inode);
939 nfs_direct_req_release(dreq);
940 return result < 0 ? result : -EIO;
941 }
942
943 if (put_dreq(dreq))
944 nfs_direct_write_complete(dreq);
945 return requested_bytes;
946}
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968ssize_t nfs_file_direct_write(struct kiocb *iocb, struct iov_iter *iter)
969{
970 ssize_t result = -EINVAL, requested;
971 size_t count;
972 struct file *file = iocb->ki_filp;
973 struct address_space *mapping = file->f_mapping;
974 struct inode *inode = mapping->host;
975 struct nfs_direct_req *dreq;
976 struct nfs_lock_context *l_ctx;
977 loff_t pos, end;
978
979 dfprintk(FILE, "NFS: direct write(%pD2, %zd@%Ld)\n",
980 file, iov_iter_count(iter), (long long) iocb->ki_pos);
981
982 result = generic_write_checks(iocb, iter);
983 if (result <= 0)
984 return result;
985 count = result;
986 nfs_add_stats(mapping->host, NFSIOS_DIRECTWRITTENBYTES, count);
987
988 pos = iocb->ki_pos;
989 end = (pos + iov_iter_count(iter) - 1) >> PAGE_SHIFT;
990
991 task_io_account_write(count);
992
993 result = -ENOMEM;
994 dreq = nfs_direct_req_alloc();
995 if (!dreq)
996 goto out;
997
998 dreq->inode = inode;
999 dreq->bytes_left = dreq->max_count = count;
1000 dreq->io_start = pos;
1001 dreq->ctx = get_nfs_open_context(nfs_file_open_context(iocb->ki_filp));
1002 l_ctx = nfs_get_lock_context(dreq->ctx);
1003 if (IS_ERR(l_ctx)) {
1004 result = PTR_ERR(l_ctx);
1005 goto out_release;
1006 }
1007 dreq->l_ctx = l_ctx;
1008 if (!is_sync_kiocb(iocb))
1009 dreq->iocb = iocb;
1010
1011 nfs_start_io_direct(inode);
1012
1013 requested = nfs_direct_write_schedule_iovec(dreq, iter, pos);
1014
1015 if (mapping->nrpages) {
1016 invalidate_inode_pages2_range(mapping,
1017 pos >> PAGE_SHIFT, end);
1018 }
1019
1020 nfs_end_io_direct(inode);
1021
1022 if (requested > 0) {
1023 result = nfs_direct_wait(dreq);
1024 if (result > 0) {
1025 requested -= result;
1026 iocb->ki_pos = pos + result;
1027
1028 generic_write_sync(iocb, result);
1029 }
1030 iov_iter_revert(iter, requested);
1031 } else {
1032 result = requested;
1033 }
1034out_release:
1035 nfs_direct_req_release(dreq);
1036out:
1037 return result;
1038}
1039
1040
1041
1042
1043
1044int __init nfs_init_directcache(void)
1045{
1046 nfs_direct_cachep = kmem_cache_create("nfs_direct_cache",
1047 sizeof(struct nfs_direct_req),
1048 0, (SLAB_RECLAIM_ACCOUNT|
1049 SLAB_MEM_SPREAD),
1050 NULL);
1051 if (nfs_direct_cachep == NULL)
1052 return -ENOMEM;
1053
1054 return 0;
1055}
1056
1057
1058
1059
1060
1061void nfs_destroy_directcache(void)
1062{
1063 kmem_cache_destroy(nfs_direct_cachep);
1064}
1065