1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20#include <linux/fs.h>
21#include <linux/file.h>
22#include <linux/pagemap.h>
23#include <linux/splice.h>
24#include <linux/memcontrol.h>
25#include <linux/mm_inline.h>
26#include <linux/swap.h>
27#include <linux/writeback.h>
28#include <linux/export.h>
29#include <linux/syscalls.h>
30#include <linux/uio.h>
31#include <linux/security.h>
32#include <linux/gfp.h>
33#include <linux/socket.h>
34#include <linux/compat.h>
35#include "internal.h"
36
37
38
39
40
41
42
43static int page_cache_pipe_buf_steal(struct pipe_inode_info *pipe,
44 struct pipe_buffer *buf)
45{
46 struct page *page = buf->page;
47 struct address_space *mapping;
48
49 lock_page(page);
50
51 mapping = page_mapping(page);
52 if (mapping) {
53 WARN_ON(!PageUptodate(page));
54
55
56
57
58
59
60
61
62
63 wait_on_page_writeback(page);
64
65 if (page_has_private(page) &&
66 !try_to_release_page(page, GFP_KERNEL))
67 goto out_unlock;
68
69
70
71
72
73 if (remove_mapping(mapping, page)) {
74 buf->flags |= PIPE_BUF_FLAG_LRU;
75 return 0;
76 }
77 }
78
79
80
81
82
83out_unlock:
84 unlock_page(page);
85 return 1;
86}
87
88static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe,
89 struct pipe_buffer *buf)
90{
91 put_page(buf->page);
92 buf->flags &= ~PIPE_BUF_FLAG_LRU;
93}
94
95
96
97
98
99static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe,
100 struct pipe_buffer *buf)
101{
102 struct page *page = buf->page;
103 int err;
104
105 if (!PageUptodate(page)) {
106 lock_page(page);
107
108
109
110
111
112 if (!page->mapping) {
113 err = -ENODATA;
114 goto error;
115 }
116
117
118
119
120 if (!PageUptodate(page)) {
121 err = -EIO;
122 goto error;
123 }
124
125
126
127
128 unlock_page(page);
129 }
130
131 return 0;
132error:
133 unlock_page(page);
134 return err;
135}
136
137const struct pipe_buf_operations page_cache_pipe_buf_ops = {
138 .can_merge = 0,
139 .confirm = page_cache_pipe_buf_confirm,
140 .release = page_cache_pipe_buf_release,
141 .steal = page_cache_pipe_buf_steal,
142 .get = generic_pipe_buf_get,
143};
144
145static int user_page_pipe_buf_steal(struct pipe_inode_info *pipe,
146 struct pipe_buffer *buf)
147{
148 if (!(buf->flags & PIPE_BUF_FLAG_GIFT))
149 return 1;
150
151 buf->flags |= PIPE_BUF_FLAG_LRU;
152 return generic_pipe_buf_steal(pipe, buf);
153}
154
155static const struct pipe_buf_operations user_page_pipe_buf_ops = {
156 .can_merge = 0,
157 .confirm = generic_pipe_buf_confirm,
158 .release = page_cache_pipe_buf_release,
159 .steal = user_page_pipe_buf_steal,
160 .get = generic_pipe_buf_get,
161};
162
163static void wakeup_pipe_readers(struct pipe_inode_info *pipe)
164{
165 smp_mb();
166 if (waitqueue_active(&pipe->wait))
167 wake_up_interruptible(&pipe->wait);
168 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
169}
170
171
172
173
174
175
176
177
178
179
180
181
182ssize_t splice_to_pipe(struct pipe_inode_info *pipe,
183 struct splice_pipe_desc *spd)
184{
185 unsigned int spd_pages = spd->nr_pages;
186 int ret, do_wakeup, page_nr;
187
188 if (!spd_pages)
189 return 0;
190
191 ret = 0;
192 do_wakeup = 0;
193 page_nr = 0;
194
195 pipe_lock(pipe);
196
197 for (;;) {
198 if (!pipe->readers) {
199 send_sig(SIGPIPE, current, 0);
200 if (!ret)
201 ret = -EPIPE;
202 break;
203 }
204
205 if (pipe->nrbufs < pipe->buffers) {
206 int newbuf = (pipe->curbuf + pipe->nrbufs) & (pipe->buffers - 1);
207 struct pipe_buffer *buf = pipe->bufs + newbuf;
208
209 buf->page = spd->pages[page_nr];
210 buf->offset = spd->partial[page_nr].offset;
211 buf->len = spd->partial[page_nr].len;
212 buf->private = spd->partial[page_nr].private;
213 buf->ops = spd->ops;
214 if (spd->flags & SPLICE_F_GIFT)
215 buf->flags |= PIPE_BUF_FLAG_GIFT;
216
217 pipe->nrbufs++;
218 page_nr++;
219 ret += buf->len;
220
221 if (pipe->files)
222 do_wakeup = 1;
223
224 if (!--spd->nr_pages)
225 break;
226 if (pipe->nrbufs < pipe->buffers)
227 continue;
228
229 break;
230 }
231
232 if (spd->flags & SPLICE_F_NONBLOCK) {
233 if (!ret)
234 ret = -EAGAIN;
235 break;
236 }
237
238 if (signal_pending(current)) {
239 if (!ret)
240 ret = -ERESTARTSYS;
241 break;
242 }
243
244 if (do_wakeup) {
245 smp_mb();
246 if (waitqueue_active(&pipe->wait))
247 wake_up_interruptible_sync(&pipe->wait);
248 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
249 do_wakeup = 0;
250 }
251
252 pipe->waiting_writers++;
253 pipe_wait(pipe);
254 pipe->waiting_writers--;
255 }
256
257 pipe_unlock(pipe);
258
259 if (do_wakeup)
260 wakeup_pipe_readers(pipe);
261
262 while (page_nr < spd_pages)
263 spd->spd_release(spd, page_nr++);
264
265 return ret;
266}
267EXPORT_SYMBOL_GPL(splice_to_pipe);
268
269void spd_release_page(struct splice_pipe_desc *spd, unsigned int i)
270{
271 put_page(spd->pages[i]);
272}
273
274
275
276
277
278int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd)
279{
280 unsigned int buffers = ACCESS_ONCE(pipe->buffers);
281
282 spd->nr_pages_max = buffers;
283 if (buffers <= PIPE_DEF_BUFFERS)
284 return 0;
285
286 spd->pages = kmalloc(buffers * sizeof(struct page *), GFP_KERNEL);
287 spd->partial = kmalloc(buffers * sizeof(struct partial_page), GFP_KERNEL);
288
289 if (spd->pages && spd->partial)
290 return 0;
291
292 kfree(spd->pages);
293 kfree(spd->partial);
294 return -ENOMEM;
295}
296
297void splice_shrink_spd(struct splice_pipe_desc *spd)
298{
299 if (spd->nr_pages_max <= PIPE_DEF_BUFFERS)
300 return;
301
302 kfree(spd->pages);
303 kfree(spd->partial);
304}
305
306static int
307__generic_file_splice_read(struct file *in, loff_t *ppos,
308 struct pipe_inode_info *pipe, size_t len,
309 unsigned int flags)
310{
311 struct address_space *mapping = in->f_mapping;
312 unsigned int loff, nr_pages, req_pages;
313 struct page *pages[PIPE_DEF_BUFFERS];
314 struct partial_page partial[PIPE_DEF_BUFFERS];
315 struct page *page;
316 pgoff_t index, end_index;
317 loff_t isize;
318 int error, page_nr;
319 struct splice_pipe_desc spd = {
320 .pages = pages,
321 .partial = partial,
322 .nr_pages_max = PIPE_DEF_BUFFERS,
323 .flags = flags,
324 .ops = &page_cache_pipe_buf_ops,
325 .spd_release = spd_release_page,
326 };
327
328 if (splice_grow_spd(pipe, &spd))
329 return -ENOMEM;
330
331 index = *ppos >> PAGE_SHIFT;
332 loff = *ppos & ~PAGE_MASK;
333 req_pages = (len + loff + PAGE_SIZE - 1) >> PAGE_SHIFT;
334 nr_pages = min(req_pages, spd.nr_pages_max);
335
336
337
338
339 spd.nr_pages = find_get_pages_contig(mapping, index, nr_pages, spd.pages);
340 index += spd.nr_pages;
341
342
343
344
345
346 if (spd.nr_pages < nr_pages)
347 page_cache_sync_readahead(mapping, &in->f_ra, in,
348 index, req_pages - spd.nr_pages);
349
350 error = 0;
351 while (spd.nr_pages < nr_pages) {
352
353
354
355
356 page = find_get_page(mapping, index);
357 if (!page) {
358
359
360
361 page = page_cache_alloc_cold(mapping);
362 if (!page)
363 break;
364
365 error = add_to_page_cache_lru(page, mapping, index,
366 mapping_gfp_constraint(mapping, GFP_KERNEL));
367 if (unlikely(error)) {
368 put_page(page);
369 if (error == -EEXIST)
370 continue;
371 break;
372 }
373
374
375
376
377 unlock_page(page);
378 }
379
380 spd.pages[spd.nr_pages++] = page;
381 index++;
382 }
383
384
385
386
387
388 index = *ppos >> PAGE_SHIFT;
389 nr_pages = spd.nr_pages;
390 spd.nr_pages = 0;
391 for (page_nr = 0; page_nr < nr_pages; page_nr++) {
392 unsigned int this_len;
393
394 if (!len)
395 break;
396
397
398
399
400 this_len = min_t(unsigned long, len, PAGE_SIZE - loff);
401 page = spd.pages[page_nr];
402
403 if (PageReadahead(page))
404 page_cache_async_readahead(mapping, &in->f_ra, in,
405 page, index, req_pages - page_nr);
406
407
408
409
410 if (!PageUptodate(page)) {
411 lock_page(page);
412
413
414
415
416
417
418
419 if (!page->mapping) {
420 unlock_page(page);
421retry_lookup:
422 page = find_or_create_page(mapping, index,
423 mapping_gfp_mask(mapping));
424
425 if (!page) {
426 error = -ENOMEM;
427 break;
428 }
429 put_page(spd.pages[page_nr]);
430 spd.pages[page_nr] = page;
431 }
432
433
434
435 if (PageUptodate(page)) {
436 unlock_page(page);
437 goto fill_it;
438 }
439
440
441
442
443 error = mapping->a_ops->readpage(in, page);
444 if (unlikely(error)) {
445
446
447
448 if (error == AOP_TRUNCATED_PAGE)
449 goto retry_lookup;
450
451 break;
452 }
453 }
454fill_it:
455
456
457
458 isize = i_size_read(mapping->host);
459 end_index = (isize - 1) >> PAGE_SHIFT;
460 if (unlikely(!isize || index > end_index))
461 break;
462
463
464
465
466
467 if (end_index == index) {
468 unsigned int plen;
469
470
471
472
473 plen = ((isize - 1) & ~PAGE_MASK) + 1;
474 if (plen <= loff)
475 break;
476
477
478
479
480 this_len = min(this_len, plen - loff);
481 len = this_len;
482 }
483
484 spd.partial[page_nr].offset = loff;
485 spd.partial[page_nr].len = this_len;
486 len -= this_len;
487 loff = 0;
488 spd.nr_pages++;
489 index++;
490 }
491
492
493
494
495
496 while (page_nr < nr_pages)
497 put_page(spd.pages[page_nr++]);
498 in->f_ra.prev_pos = (loff_t)index << PAGE_SHIFT;
499
500 if (spd.nr_pages)
501 error = splice_to_pipe(pipe, &spd);
502
503 splice_shrink_spd(&spd);
504 return error;
505}
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521ssize_t generic_file_splice_read(struct file *in, loff_t *ppos,
522 struct pipe_inode_info *pipe, size_t len,
523 unsigned int flags)
524{
525 loff_t isize, left;
526 int ret;
527
528 if (IS_DAX(in->f_mapping->host))
529 return default_file_splice_read(in, ppos, pipe, len, flags);
530
531 isize = i_size_read(in->f_mapping->host);
532 if (unlikely(*ppos >= isize))
533 return 0;
534
535 left = isize - *ppos;
536 if (unlikely(left < len))
537 len = left;
538
539 ret = __generic_file_splice_read(in, ppos, pipe, len, flags);
540 if (ret > 0) {
541 *ppos += ret;
542 file_accessed(in);
543 }
544
545 return ret;
546}
547EXPORT_SYMBOL(generic_file_splice_read);
548
549static const struct pipe_buf_operations default_pipe_buf_ops = {
550 .can_merge = 0,
551 .confirm = generic_pipe_buf_confirm,
552 .release = generic_pipe_buf_release,
553 .steal = generic_pipe_buf_steal,
554 .get = generic_pipe_buf_get,
555};
556
557static int generic_pipe_buf_nosteal(struct pipe_inode_info *pipe,
558 struct pipe_buffer *buf)
559{
560 return 1;
561}
562
563
564const struct pipe_buf_operations nosteal_pipe_buf_ops = {
565 .can_merge = 0,
566 .confirm = generic_pipe_buf_confirm,
567 .release = generic_pipe_buf_release,
568 .steal = generic_pipe_buf_nosteal,
569 .get = generic_pipe_buf_get,
570};
571EXPORT_SYMBOL(nosteal_pipe_buf_ops);
572
573static ssize_t kernel_readv(struct file *file, const struct iovec *vec,
574 unsigned long vlen, loff_t offset)
575{
576 mm_segment_t old_fs;
577 loff_t pos = offset;
578 ssize_t res;
579
580 old_fs = get_fs();
581 set_fs(get_ds());
582
583 res = vfs_readv(file, (const struct iovec __user *)vec, vlen, &pos, 0);
584 set_fs(old_fs);
585
586 return res;
587}
588
589ssize_t kernel_write(struct file *file, const char *buf, size_t count,
590 loff_t pos)
591{
592 mm_segment_t old_fs;
593 ssize_t res;
594
595 old_fs = get_fs();
596 set_fs(get_ds());
597
598 res = vfs_write(file, (__force const char __user *)buf, count, &pos);
599 set_fs(old_fs);
600
601 return res;
602}
603EXPORT_SYMBOL(kernel_write);
604
605ssize_t default_file_splice_read(struct file *in, loff_t *ppos,
606 struct pipe_inode_info *pipe, size_t len,
607 unsigned int flags)
608{
609 unsigned int nr_pages;
610 unsigned int nr_freed;
611 size_t offset;
612 struct page *pages[PIPE_DEF_BUFFERS];
613 struct partial_page partial[PIPE_DEF_BUFFERS];
614 struct iovec *vec, __vec[PIPE_DEF_BUFFERS];
615 ssize_t res;
616 size_t this_len;
617 int error;
618 int i;
619 struct splice_pipe_desc spd = {
620 .pages = pages,
621 .partial = partial,
622 .nr_pages_max = PIPE_DEF_BUFFERS,
623 .flags = flags,
624 .ops = &default_pipe_buf_ops,
625 .spd_release = spd_release_page,
626 };
627
628 if (splice_grow_spd(pipe, &spd))
629 return -ENOMEM;
630
631 res = -ENOMEM;
632 vec = __vec;
633 if (spd.nr_pages_max > PIPE_DEF_BUFFERS) {
634 vec = kmalloc(spd.nr_pages_max * sizeof(struct iovec), GFP_KERNEL);
635 if (!vec)
636 goto shrink_ret;
637 }
638
639 offset = *ppos & ~PAGE_MASK;
640 nr_pages = (len + offset + PAGE_SIZE - 1) >> PAGE_SHIFT;
641
642 for (i = 0; i < nr_pages && i < spd.nr_pages_max && len; i++) {
643 struct page *page;
644
645 page = alloc_page(GFP_USER);
646 error = -ENOMEM;
647 if (!page)
648 goto err;
649
650 this_len = min_t(size_t, len, PAGE_SIZE - offset);
651 vec[i].iov_base = (void __user *) page_address(page);
652 vec[i].iov_len = this_len;
653 spd.pages[i] = page;
654 spd.nr_pages++;
655 len -= this_len;
656 offset = 0;
657 }
658
659 res = kernel_readv(in, vec, spd.nr_pages, *ppos);
660 if (res < 0) {
661 error = res;
662 goto err;
663 }
664
665 error = 0;
666 if (!res)
667 goto err;
668
669 nr_freed = 0;
670 for (i = 0; i < spd.nr_pages; i++) {
671 this_len = min_t(size_t, vec[i].iov_len, res);
672 spd.partial[i].offset = 0;
673 spd.partial[i].len = this_len;
674 if (!this_len) {
675 __free_page(spd.pages[i]);
676 spd.pages[i] = NULL;
677 nr_freed++;
678 }
679 res -= this_len;
680 }
681 spd.nr_pages -= nr_freed;
682
683 res = splice_to_pipe(pipe, &spd);
684 if (res > 0)
685 *ppos += res;
686
687shrink_ret:
688 if (vec != __vec)
689 kfree(vec);
690 splice_shrink_spd(&spd);
691 return res;
692
693err:
694 for (i = 0; i < spd.nr_pages; i++)
695 __free_page(spd.pages[i]);
696
697 res = error;
698 goto shrink_ret;
699}
700EXPORT_SYMBOL(default_file_splice_read);
701
702
703
704
705
706static int pipe_to_sendpage(struct pipe_inode_info *pipe,
707 struct pipe_buffer *buf, struct splice_desc *sd)
708{
709 struct file *file = sd->u.file;
710 loff_t pos = sd->pos;
711 int more;
712
713 if (!likely(file->f_op->sendpage))
714 return -EINVAL;
715
716 more = (sd->flags & SPLICE_F_MORE) ? MSG_MORE : 0;
717
718 if (sd->len < sd->total_len && pipe->nrbufs > 1)
719 more |= MSG_SENDPAGE_NOTLAST;
720
721 return file->f_op->sendpage(file, buf->page, buf->offset,
722 sd->len, &pos, more);
723}
724
725static void wakeup_pipe_writers(struct pipe_inode_info *pipe)
726{
727 smp_mb();
728 if (waitqueue_active(&pipe->wait))
729 wake_up_interruptible(&pipe->wait);
730 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
731}
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd,
754 splice_actor *actor)
755{
756 int ret;
757
758 while (pipe->nrbufs) {
759 struct pipe_buffer *buf = pipe->bufs + pipe->curbuf;
760 const struct pipe_buf_operations *ops = buf->ops;
761
762 sd->len = buf->len;
763 if (sd->len > sd->total_len)
764 sd->len = sd->total_len;
765
766 ret = buf->ops->confirm(pipe, buf);
767 if (unlikely(ret)) {
768 if (ret == -ENODATA)
769 ret = 0;
770 return ret;
771 }
772
773 ret = actor(pipe, buf, sd);
774 if (ret <= 0)
775 return ret;
776
777 buf->offset += ret;
778 buf->len -= ret;
779
780 sd->num_spliced += ret;
781 sd->len -= ret;
782 sd->pos += ret;
783 sd->total_len -= ret;
784
785 if (!buf->len) {
786 buf->ops = NULL;
787 ops->release(pipe, buf);
788 pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1);
789 pipe->nrbufs--;
790 if (pipe->files)
791 sd->need_wakeup = true;
792 }
793
794 if (!sd->total_len)
795 return 0;
796 }
797
798 return 1;
799}
800
801
802
803
804
805
806
807
808
809
810
811static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd)
812{
813
814
815
816
817 if (signal_pending(current))
818 return -ERESTARTSYS;
819
820 while (!pipe->nrbufs) {
821 if (!pipe->writers)
822 return 0;
823
824 if (!pipe->waiting_writers && sd->num_spliced)
825 return 0;
826
827 if (sd->flags & SPLICE_F_NONBLOCK)
828 return -EAGAIN;
829
830 if (signal_pending(current))
831 return -ERESTARTSYS;
832
833 if (sd->need_wakeup) {
834 wakeup_pipe_writers(pipe);
835 sd->need_wakeup = false;
836 }
837
838 pipe_wait(pipe);
839 }
840
841 return 1;
842}
843
844
845
846
847
848
849
850
851
852
853static void splice_from_pipe_begin(struct splice_desc *sd)
854{
855 sd->num_spliced = 0;
856 sd->need_wakeup = false;
857}
858
859
860
861
862
863
864
865
866
867
868
869static void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd)
870{
871 if (sd->need_wakeup)
872 wakeup_pipe_writers(pipe);
873}
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd,
889 splice_actor *actor)
890{
891 int ret;
892
893 splice_from_pipe_begin(sd);
894 do {
895 cond_resched();
896 ret = splice_from_pipe_next(pipe, sd);
897 if (ret > 0)
898 ret = splice_from_pipe_feed(pipe, sd, actor);
899 } while (ret > 0);
900 splice_from_pipe_end(pipe, sd);
901
902 return sd->num_spliced ? sd->num_spliced : ret;
903}
904EXPORT_SYMBOL(__splice_from_pipe);
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out,
921 loff_t *ppos, size_t len, unsigned int flags,
922 splice_actor *actor)
923{
924 ssize_t ret;
925 struct splice_desc sd = {
926 .total_len = len,
927 .flags = flags,
928 .pos = *ppos,
929 .u.file = out,
930 };
931
932 pipe_lock(pipe);
933 ret = __splice_from_pipe(pipe, &sd, actor);
934 pipe_unlock(pipe);
935
936 return ret;
937}
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953ssize_t
954iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out,
955 loff_t *ppos, size_t len, unsigned int flags)
956{
957 struct splice_desc sd = {
958 .total_len = len,
959 .flags = flags,
960 .pos = *ppos,
961 .u.file = out,
962 };
963 int nbufs = pipe->buffers;
964 struct bio_vec *array = kcalloc(nbufs, sizeof(struct bio_vec),
965 GFP_KERNEL);
966 ssize_t ret;
967
968 if (unlikely(!array))
969 return -ENOMEM;
970
971 pipe_lock(pipe);
972
973 splice_from_pipe_begin(&sd);
974 while (sd.total_len) {
975 struct iov_iter from;
976 size_t left;
977 int n, idx;
978
979 ret = splice_from_pipe_next(pipe, &sd);
980 if (ret <= 0)
981 break;
982
983 if (unlikely(nbufs < pipe->buffers)) {
984 kfree(array);
985 nbufs = pipe->buffers;
986 array = kcalloc(nbufs, sizeof(struct bio_vec),
987 GFP_KERNEL);
988 if (!array) {
989 ret = -ENOMEM;
990 break;
991 }
992 }
993
994
995 left = sd.total_len;
996 for (n = 0, idx = pipe->curbuf; left && n < pipe->nrbufs; n++, idx++) {
997 struct pipe_buffer *buf = pipe->bufs + idx;
998 size_t this_len = buf->len;
999
1000 if (this_len > left)
1001 this_len = left;
1002
1003 if (idx == pipe->buffers - 1)
1004 idx = -1;
1005
1006 ret = buf->ops->confirm(pipe, buf);
1007 if (unlikely(ret)) {
1008 if (ret == -ENODATA)
1009 ret = 0;
1010 goto done;
1011 }
1012
1013 array[n].bv_page = buf->page;
1014 array[n].bv_len = this_len;
1015 array[n].bv_offset = buf->offset;
1016 left -= this_len;
1017 }
1018
1019 iov_iter_bvec(&from, ITER_BVEC | WRITE, array, n,
1020 sd.total_len - left);
1021 ret = vfs_iter_write(out, &from, &sd.pos);
1022 if (ret <= 0)
1023 break;
1024
1025 sd.num_spliced += ret;
1026 sd.total_len -= ret;
1027 *ppos = sd.pos;
1028
1029
1030 while (ret) {
1031 struct pipe_buffer *buf = pipe->bufs + pipe->curbuf;
1032 if (ret >= buf->len) {
1033 const struct pipe_buf_operations *ops = buf->ops;
1034 ret -= buf->len;
1035 buf->len = 0;
1036 buf->ops = NULL;
1037 ops->release(pipe, buf);
1038 pipe->curbuf = (pipe->curbuf + 1) & (pipe->buffers - 1);
1039 pipe->nrbufs--;
1040 if (pipe->files)
1041 sd.need_wakeup = true;
1042 } else {
1043 buf->offset += ret;
1044 buf->len -= ret;
1045 ret = 0;
1046 }
1047 }
1048 }
1049done:
1050 kfree(array);
1051 splice_from_pipe_end(pipe, &sd);
1052
1053 pipe_unlock(pipe);
1054
1055 if (sd.num_spliced)
1056 ret = sd.num_spliced;
1057
1058 return ret;
1059}
1060
1061EXPORT_SYMBOL(iter_file_splice_write);
1062
1063static int write_pipe_buf(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
1064 struct splice_desc *sd)
1065{
1066 int ret;
1067 void *data;
1068 loff_t tmp = sd->pos;
1069
1070 data = kmap(buf->page);
1071 ret = __kernel_write(sd->u.file, data + buf->offset, sd->len, &tmp);
1072 kunmap(buf->page);
1073
1074 return ret;
1075}
1076
1077static ssize_t default_file_splice_write(struct pipe_inode_info *pipe,
1078 struct file *out, loff_t *ppos,
1079 size_t len, unsigned int flags)
1080{
1081 ssize_t ret;
1082
1083 ret = splice_from_pipe(pipe, out, ppos, len, flags, write_pipe_buf);
1084 if (ret > 0)
1085 *ppos += ret;
1086
1087 return ret;
1088}
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out,
1104 loff_t *ppos, size_t len, unsigned int flags)
1105{
1106 return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage);
1107}
1108
1109EXPORT_SYMBOL(generic_splice_sendpage);
1110
1111
1112
1113
1114static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,
1115 loff_t *ppos, size_t len, unsigned int flags)
1116{
1117 ssize_t (*splice_write)(struct pipe_inode_info *, struct file *,
1118 loff_t *, size_t, unsigned int);
1119
1120 if (out->f_op->splice_write)
1121 splice_write = out->f_op->splice_write;
1122 else
1123 splice_write = default_file_splice_write;
1124
1125 return splice_write(pipe, out, ppos, len, flags);
1126}
1127
1128
1129
1130
1131static long do_splice_to(struct file *in, loff_t *ppos,
1132 struct pipe_inode_info *pipe, size_t len,
1133 unsigned int flags)
1134{
1135 ssize_t (*splice_read)(struct file *, loff_t *,
1136 struct pipe_inode_info *, size_t, unsigned int);
1137 int ret;
1138
1139 if (unlikely(!(in->f_mode & FMODE_READ)))
1140 return -EBADF;
1141
1142 ret = rw_verify_area(READ, in, ppos, len);
1143 if (unlikely(ret < 0))
1144 return ret;
1145
1146 if (unlikely(len > MAX_RW_COUNT))
1147 len = MAX_RW_COUNT;
1148
1149 if (in->f_op->splice_read)
1150 splice_read = in->f_op->splice_read;
1151 else
1152 splice_read = default_file_splice_read;
1153
1154 return splice_read(in, ppos, pipe, len, flags);
1155}
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd,
1171 splice_direct_actor *actor)
1172{
1173 struct pipe_inode_info *pipe;
1174 long ret, bytes;
1175 umode_t i_mode;
1176 size_t len;
1177 int i, flags, more;
1178
1179
1180
1181
1182
1183
1184 i_mode = file_inode(in)->i_mode;
1185 if (unlikely(!S_ISREG(i_mode) && !S_ISBLK(i_mode)))
1186 return -EINVAL;
1187
1188
1189
1190
1191
1192 pipe = current->splice_pipe;
1193 if (unlikely(!pipe)) {
1194 pipe = alloc_pipe_info();
1195 if (!pipe)
1196 return -ENOMEM;
1197
1198
1199
1200
1201
1202
1203 pipe->readers = 1;
1204
1205 current->splice_pipe = pipe;
1206 }
1207
1208
1209
1210
1211 ret = 0;
1212 bytes = 0;
1213 len = sd->total_len;
1214 flags = sd->flags;
1215
1216
1217
1218
1219 sd->flags &= ~SPLICE_F_NONBLOCK;
1220 more = sd->flags & SPLICE_F_MORE;
1221
1222 while (len) {
1223 size_t read_len;
1224 loff_t pos = sd->pos, prev_pos = pos;
1225
1226 ret = do_splice_to(in, &pos, pipe, len, flags);
1227 if (unlikely(ret <= 0))
1228 goto out_release;
1229
1230 read_len = ret;
1231 sd->total_len = read_len;
1232
1233
1234
1235
1236
1237
1238 if (read_len < len)
1239 sd->flags |= SPLICE_F_MORE;
1240 else if (!more)
1241 sd->flags &= ~SPLICE_F_MORE;
1242
1243
1244
1245
1246
1247 ret = actor(pipe, sd);
1248 if (unlikely(ret <= 0)) {
1249 sd->pos = prev_pos;
1250 goto out_release;
1251 }
1252
1253 bytes += ret;
1254 len -= ret;
1255 sd->pos = pos;
1256
1257 if (ret < read_len) {
1258 sd->pos = prev_pos + ret;
1259 goto out_release;
1260 }
1261 }
1262
1263done:
1264 pipe->nrbufs = pipe->curbuf = 0;
1265 file_accessed(in);
1266 return bytes;
1267
1268out_release:
1269
1270
1271
1272
1273 for (i = 0; i < pipe->buffers; i++) {
1274 struct pipe_buffer *buf = pipe->bufs + i;
1275
1276 if (buf->ops) {
1277 buf->ops->release(pipe, buf);
1278 buf->ops = NULL;
1279 }
1280 }
1281
1282 if (!bytes)
1283 bytes = ret;
1284
1285 goto done;
1286}
1287EXPORT_SYMBOL(splice_direct_to_actor);
1288
1289static int direct_splice_actor(struct pipe_inode_info *pipe,
1290 struct splice_desc *sd)
1291{
1292 struct file *file = sd->u.file;
1293
1294 return do_splice_from(pipe, file, sd->opos, sd->total_len,
1295 sd->flags);
1296}
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314long do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
1315 loff_t *opos, size_t len, unsigned int flags)
1316{
1317 struct splice_desc sd = {
1318 .len = len,
1319 .total_len = len,
1320 .flags = flags,
1321 .pos = *ppos,
1322 .u.file = out,
1323 .opos = opos,
1324 };
1325 long ret;
1326
1327 if (unlikely(!(out->f_mode & FMODE_WRITE)))
1328 return -EBADF;
1329
1330 if (unlikely(out->f_flags & O_APPEND))
1331 return -EINVAL;
1332
1333 ret = rw_verify_area(WRITE, out, opos, len);
1334 if (unlikely(ret < 0))
1335 return ret;
1336
1337 ret = splice_direct_to_actor(in, &sd, direct_splice_actor);
1338 if (ret > 0)
1339 *ppos = sd.pos;
1340
1341 return ret;
1342}
1343EXPORT_SYMBOL(do_splice_direct);
1344
1345static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
1346 struct pipe_inode_info *opipe,
1347 size_t len, unsigned int flags);
1348
1349
1350
1351
1352static long do_splice(struct file *in, loff_t __user *off_in,
1353 struct file *out, loff_t __user *off_out,
1354 size_t len, unsigned int flags)
1355{
1356 struct pipe_inode_info *ipipe;
1357 struct pipe_inode_info *opipe;
1358 loff_t offset;
1359 long ret;
1360
1361 ipipe = get_pipe_info(in);
1362 opipe = get_pipe_info(out);
1363
1364 if (ipipe && opipe) {
1365 if (off_in || off_out)
1366 return -ESPIPE;
1367
1368 if (!(in->f_mode & FMODE_READ))
1369 return -EBADF;
1370
1371 if (!(out->f_mode & FMODE_WRITE))
1372 return -EBADF;
1373
1374
1375 if (ipipe == opipe)
1376 return -EINVAL;
1377
1378 return splice_pipe_to_pipe(ipipe, opipe, len, flags);
1379 }
1380
1381 if (ipipe) {
1382 if (off_in)
1383 return -ESPIPE;
1384 if (off_out) {
1385 if (!(out->f_mode & FMODE_PWRITE))
1386 return -EINVAL;
1387 if (copy_from_user(&offset, off_out, sizeof(loff_t)))
1388 return -EFAULT;
1389 } else {
1390 offset = out->f_pos;
1391 }
1392
1393 if (unlikely(!(out->f_mode & FMODE_WRITE)))
1394 return -EBADF;
1395
1396 if (unlikely(out->f_flags & O_APPEND))
1397 return -EINVAL;
1398
1399 ret = rw_verify_area(WRITE, out, &offset, len);
1400 if (unlikely(ret < 0))
1401 return ret;
1402
1403 file_start_write(out);
1404 ret = do_splice_from(ipipe, out, &offset, len, flags);
1405 file_end_write(out);
1406
1407 if (!off_out)
1408 out->f_pos = offset;
1409 else if (copy_to_user(off_out, &offset, sizeof(loff_t)))
1410 ret = -EFAULT;
1411
1412 return ret;
1413 }
1414
1415 if (opipe) {
1416 if (off_out)
1417 return -ESPIPE;
1418 if (off_in) {
1419 if (!(in->f_mode & FMODE_PREAD))
1420 return -EINVAL;
1421 if (copy_from_user(&offset, off_in, sizeof(loff_t)))
1422 return -EFAULT;
1423 } else {
1424 offset = in->f_pos;
1425 }
1426
1427 ret = do_splice_to(in, &offset, opipe, len, flags);
1428
1429 if (!off_in)
1430 in->f_pos = offset;
1431 else if (copy_to_user(off_in, &offset, sizeof(loff_t)))
1432 ret = -EFAULT;
1433
1434 return ret;
1435 }
1436
1437 return -EINVAL;
1438}
1439
1440
1441
1442
1443
1444
1445
1446
1447static int get_iovec_page_array(const struct iovec __user *iov,
1448 unsigned int nr_vecs, struct page **pages,
1449 struct partial_page *partial, bool aligned,
1450 unsigned int pipe_buffers)
1451{
1452 int buffers = 0, error = 0;
1453
1454 while (nr_vecs) {
1455 unsigned long off, npages;
1456 struct iovec entry;
1457 void __user *base;
1458 size_t len;
1459 int i;
1460
1461 error = -EFAULT;
1462 if (copy_from_user(&entry, iov, sizeof(entry)))
1463 break;
1464
1465 base = entry.iov_base;
1466 len = entry.iov_len;
1467
1468
1469
1470
1471 error = 0;
1472 if (unlikely(!len))
1473 break;
1474 error = -EFAULT;
1475 if (!access_ok(VERIFY_READ, base, len))
1476 break;
1477
1478
1479
1480
1481
1482 off = (unsigned long) base & ~PAGE_MASK;
1483
1484
1485
1486
1487
1488 error = -EINVAL;
1489 if (aligned && (off || len & ~PAGE_MASK))
1490 break;
1491
1492 npages = (off + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
1493 if (npages > pipe_buffers - buffers)
1494 npages = pipe_buffers - buffers;
1495
1496 error = get_user_pages_fast((unsigned long)base, npages,
1497 0, &pages[buffers]);
1498
1499 if (unlikely(error <= 0))
1500 break;
1501
1502
1503
1504
1505 for (i = 0; i < error; i++) {
1506 const int plen = min_t(size_t, len, PAGE_SIZE - off);
1507
1508 partial[buffers].offset = off;
1509 partial[buffers].len = plen;
1510
1511 off = 0;
1512 len -= plen;
1513 buffers++;
1514 }
1515
1516
1517
1518
1519
1520
1521 if (len)
1522 break;
1523
1524
1525
1526
1527
1528
1529 if (error < npages || buffers == pipe_buffers)
1530 break;
1531
1532 nr_vecs--;
1533 iov++;
1534 }
1535
1536 if (buffers)
1537 return buffers;
1538
1539 return error;
1540}
1541
1542static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
1543 struct splice_desc *sd)
1544{
1545 int n = copy_page_to_iter(buf->page, buf->offset, sd->len, sd->u.data);
1546 return n == sd->len ? n : -EFAULT;
1547}
1548
1549
1550
1551
1552
1553static long vmsplice_to_user(struct file *file, const struct iovec __user *uiov,
1554 unsigned long nr_segs, unsigned int flags)
1555{
1556 struct pipe_inode_info *pipe;
1557 struct splice_desc sd;
1558 long ret;
1559 struct iovec iovstack[UIO_FASTIOV];
1560 struct iovec *iov = iovstack;
1561 struct iov_iter iter;
1562
1563 pipe = get_pipe_info(file);
1564 if (!pipe)
1565 return -EBADF;
1566
1567 ret = import_iovec(READ, uiov, nr_segs,
1568 ARRAY_SIZE(iovstack), &iov, &iter);
1569 if (ret < 0)
1570 return ret;
1571
1572 sd.total_len = iov_iter_count(&iter);
1573 sd.len = 0;
1574 sd.flags = flags;
1575 sd.u.data = &iter;
1576 sd.pos = 0;
1577
1578 if (sd.total_len) {
1579 pipe_lock(pipe);
1580 ret = __splice_from_pipe(pipe, &sd, pipe_to_user);
1581 pipe_unlock(pipe);
1582 }
1583
1584 kfree(iov);
1585 return ret;
1586}
1587
1588
1589
1590
1591
1592
1593static long vmsplice_to_pipe(struct file *file, const struct iovec __user *iov,
1594 unsigned long nr_segs, unsigned int flags)
1595{
1596 struct pipe_inode_info *pipe;
1597 struct page *pages[PIPE_DEF_BUFFERS];
1598 struct partial_page partial[PIPE_DEF_BUFFERS];
1599 struct splice_pipe_desc spd = {
1600 .pages = pages,
1601 .partial = partial,
1602 .nr_pages_max = PIPE_DEF_BUFFERS,
1603 .flags = flags,
1604 .ops = &user_page_pipe_buf_ops,
1605 .spd_release = spd_release_page,
1606 };
1607 long ret;
1608
1609 pipe = get_pipe_info(file);
1610 if (!pipe)
1611 return -EBADF;
1612
1613 if (splice_grow_spd(pipe, &spd))
1614 return -ENOMEM;
1615
1616 spd.nr_pages = get_iovec_page_array(iov, nr_segs, spd.pages,
1617 spd.partial, false,
1618 spd.nr_pages_max);
1619 if (spd.nr_pages <= 0)
1620 ret = spd.nr_pages;
1621 else
1622 ret = splice_to_pipe(pipe, &spd);
1623
1624 splice_shrink_spd(&spd);
1625 return ret;
1626}
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644SYSCALL_DEFINE4(vmsplice, int, fd, const struct iovec __user *, iov,
1645 unsigned long, nr_segs, unsigned int, flags)
1646{
1647 struct fd f;
1648 long error;
1649
1650 if (unlikely(nr_segs > UIO_MAXIOV))
1651 return -EINVAL;
1652 else if (unlikely(!nr_segs))
1653 return 0;
1654
1655 error = -EBADF;
1656 f = fdget(fd);
1657 if (f.file) {
1658 if (f.file->f_mode & FMODE_WRITE)
1659 error = vmsplice_to_pipe(f.file, iov, nr_segs, flags);
1660 else if (f.file->f_mode & FMODE_READ)
1661 error = vmsplice_to_user(f.file, iov, nr_segs, flags);
1662
1663 fdput(f);
1664 }
1665
1666 return error;
1667}
1668
1669#ifdef CONFIG_COMPAT
1670COMPAT_SYSCALL_DEFINE4(vmsplice, int, fd, const struct compat_iovec __user *, iov32,
1671 unsigned int, nr_segs, unsigned int, flags)
1672{
1673 unsigned i;
1674 struct iovec __user *iov;
1675 if (nr_segs > UIO_MAXIOV)
1676 return -EINVAL;
1677 iov = compat_alloc_user_space(nr_segs * sizeof(struct iovec));
1678 for (i = 0; i < nr_segs; i++) {
1679 struct compat_iovec v;
1680 if (get_user(v.iov_base, &iov32[i].iov_base) ||
1681 get_user(v.iov_len, &iov32[i].iov_len) ||
1682 put_user(compat_ptr(v.iov_base), &iov[i].iov_base) ||
1683 put_user(v.iov_len, &iov[i].iov_len))
1684 return -EFAULT;
1685 }
1686 return sys_vmsplice(fd, iov, nr_segs, flags);
1687}
1688#endif
1689
1690SYSCALL_DEFINE6(splice, int, fd_in, loff_t __user *, off_in,
1691 int, fd_out, loff_t __user *, off_out,
1692 size_t, len, unsigned int, flags)
1693{
1694 struct fd in, out;
1695 long error;
1696
1697 if (unlikely(!len))
1698 return 0;
1699
1700 error = -EBADF;
1701 in = fdget(fd_in);
1702 if (in.file) {
1703 if (in.file->f_mode & FMODE_READ) {
1704 out = fdget(fd_out);
1705 if (out.file) {
1706 if (out.file->f_mode & FMODE_WRITE)
1707 error = do_splice(in.file, off_in,
1708 out.file, off_out,
1709 len, flags);
1710 fdput(out);
1711 }
1712 }
1713 fdput(in);
1714 }
1715 return error;
1716}
1717
1718
1719
1720
1721
1722static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
1723{
1724 int ret;
1725
1726
1727
1728
1729
1730 if (pipe->nrbufs)
1731 return 0;
1732
1733 ret = 0;
1734 pipe_lock(pipe);
1735
1736 while (!pipe->nrbufs) {
1737 if (signal_pending(current)) {
1738 ret = -ERESTARTSYS;
1739 break;
1740 }
1741 if (!pipe->writers)
1742 break;
1743 if (!pipe->waiting_writers) {
1744 if (flags & SPLICE_F_NONBLOCK) {
1745 ret = -EAGAIN;
1746 break;
1747 }
1748 }
1749 pipe_wait(pipe);
1750 }
1751
1752 pipe_unlock(pipe);
1753 return ret;
1754}
1755
1756
1757
1758
1759
1760static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
1761{
1762 int ret;
1763
1764
1765
1766
1767
1768 if (pipe->nrbufs < pipe->buffers)
1769 return 0;
1770
1771 ret = 0;
1772 pipe_lock(pipe);
1773
1774 while (pipe->nrbufs >= pipe->buffers) {
1775 if (!pipe->readers) {
1776 send_sig(SIGPIPE, current, 0);
1777 ret = -EPIPE;
1778 break;
1779 }
1780 if (flags & SPLICE_F_NONBLOCK) {
1781 ret = -EAGAIN;
1782 break;
1783 }
1784 if (signal_pending(current)) {
1785 ret = -ERESTARTSYS;
1786 break;
1787 }
1788 pipe->waiting_writers++;
1789 pipe_wait(pipe);
1790 pipe->waiting_writers--;
1791 }
1792
1793 pipe_unlock(pipe);
1794 return ret;
1795}
1796
1797
1798
1799
1800static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
1801 struct pipe_inode_info *opipe,
1802 size_t len, unsigned int flags)
1803{
1804 struct pipe_buffer *ibuf, *obuf;
1805 int ret = 0, nbuf;
1806 bool input_wakeup = false;
1807
1808
1809retry:
1810 ret = ipipe_prep(ipipe, flags);
1811 if (ret)
1812 return ret;
1813
1814 ret = opipe_prep(opipe, flags);
1815 if (ret)
1816 return ret;
1817
1818
1819
1820
1821
1822
1823 pipe_double_lock(ipipe, opipe);
1824
1825 do {
1826 if (!opipe->readers) {
1827 send_sig(SIGPIPE, current, 0);
1828 if (!ret)
1829 ret = -EPIPE;
1830 break;
1831 }
1832
1833 if (!ipipe->nrbufs && !ipipe->writers)
1834 break;
1835
1836
1837
1838
1839
1840 if (!ipipe->nrbufs || opipe->nrbufs >= opipe->buffers) {
1841
1842 if (ret)
1843 break;
1844
1845 if (flags & SPLICE_F_NONBLOCK) {
1846 ret = -EAGAIN;
1847 break;
1848 }
1849
1850
1851
1852
1853
1854
1855 pipe_unlock(ipipe);
1856 pipe_unlock(opipe);
1857 goto retry;
1858 }
1859
1860 ibuf = ipipe->bufs + ipipe->curbuf;
1861 nbuf = (opipe->curbuf + opipe->nrbufs) & (opipe->buffers - 1);
1862 obuf = opipe->bufs + nbuf;
1863
1864 if (len >= ibuf->len) {
1865
1866
1867
1868 *obuf = *ibuf;
1869 ibuf->ops = NULL;
1870 opipe->nrbufs++;
1871 ipipe->curbuf = (ipipe->curbuf + 1) & (ipipe->buffers - 1);
1872 ipipe->nrbufs--;
1873 input_wakeup = true;
1874 } else {
1875
1876
1877
1878
1879 ibuf->ops->get(ipipe, ibuf);
1880 *obuf = *ibuf;
1881
1882
1883
1884
1885
1886 obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
1887
1888 obuf->len = len;
1889 opipe->nrbufs++;
1890 ibuf->offset += obuf->len;
1891 ibuf->len -= obuf->len;
1892 }
1893 ret += obuf->len;
1894 len -= obuf->len;
1895 } while (len);
1896
1897 pipe_unlock(ipipe);
1898 pipe_unlock(opipe);
1899
1900
1901
1902
1903 if (ret > 0)
1904 wakeup_pipe_readers(opipe);
1905
1906 if (input_wakeup)
1907 wakeup_pipe_writers(ipipe);
1908
1909 return ret;
1910}
1911
1912
1913
1914
1915static int link_pipe(struct pipe_inode_info *ipipe,
1916 struct pipe_inode_info *opipe,
1917 size_t len, unsigned int flags)
1918{
1919 struct pipe_buffer *ibuf, *obuf;
1920 int ret = 0, i = 0, nbuf;
1921
1922
1923
1924
1925
1926
1927 pipe_double_lock(ipipe, opipe);
1928
1929 do {
1930 if (!opipe->readers) {
1931 send_sig(SIGPIPE, current, 0);
1932 if (!ret)
1933 ret = -EPIPE;
1934 break;
1935 }
1936
1937
1938
1939
1940
1941 if (i >= ipipe->nrbufs || opipe->nrbufs >= opipe->buffers)
1942 break;
1943
1944 ibuf = ipipe->bufs + ((ipipe->curbuf + i) & (ipipe->buffers-1));
1945 nbuf = (opipe->curbuf + opipe->nrbufs) & (opipe->buffers - 1);
1946
1947
1948
1949
1950
1951 ibuf->ops->get(ipipe, ibuf);
1952
1953 obuf = opipe->bufs + nbuf;
1954 *obuf = *ibuf;
1955
1956
1957
1958
1959
1960 obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
1961
1962 if (obuf->len > len)
1963 obuf->len = len;
1964
1965 opipe->nrbufs++;
1966 ret += obuf->len;
1967 len -= obuf->len;
1968 i++;
1969 } while (len);
1970
1971
1972
1973
1974
1975 if (!ret && ipipe->waiting_writers && (flags & SPLICE_F_NONBLOCK))
1976 ret = -EAGAIN;
1977
1978 pipe_unlock(ipipe);
1979 pipe_unlock(opipe);
1980
1981
1982
1983
1984 if (ret > 0)
1985 wakeup_pipe_readers(opipe);
1986
1987 return ret;
1988}
1989
1990
1991
1992
1993
1994
1995
1996static long do_tee(struct file *in, struct file *out, size_t len,
1997 unsigned int flags)
1998{
1999 struct pipe_inode_info *ipipe = get_pipe_info(in);
2000 struct pipe_inode_info *opipe = get_pipe_info(out);
2001 int ret = -EINVAL;
2002
2003
2004
2005
2006
2007 if (ipipe && opipe && ipipe != opipe) {
2008
2009
2010
2011
2012 ret = ipipe_prep(ipipe, flags);
2013 if (!ret) {
2014 ret = opipe_prep(opipe, flags);
2015 if (!ret)
2016 ret = link_pipe(ipipe, opipe, len, flags);
2017 }
2018 }
2019
2020 return ret;
2021}
2022
2023SYSCALL_DEFINE4(tee, int, fdin, int, fdout, size_t, len, unsigned int, flags)
2024{
2025 struct fd in;
2026 int error;
2027
2028 if (unlikely(!len))
2029 return 0;
2030
2031 error = -EBADF;
2032 in = fdget(fdin);
2033 if (in.file) {
2034 if (in.file->f_mode & FMODE_READ) {
2035 struct fd out = fdget(fdout);
2036 if (out.file) {
2037 if (out.file->f_mode & FMODE_WRITE)
2038 error = do_tee(in.file, out.file,
2039 len, flags);
2040 fdput(out);
2041 }
2042 }
2043 fdput(in);
2044 }
2045
2046 return error;
2047}
2048