1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21#include <linux/bvec.h>
22#include <linux/fs.h>
23#include <linux/file.h>
24#include <linux/pagemap.h>
25#include <linux/splice.h>
26#include <linux/memcontrol.h>
27#include <linux/mm_inline.h>
28#include <linux/swap.h>
29#include <linux/writeback.h>
30#include <linux/export.h>
31#include <linux/syscalls.h>
32#include <linux/uio.h>
33#include <linux/security.h>
34#include <linux/gfp.h>
35#include <linux/socket.h>
36#include <linux/compat.h>
37#include <linux/sched/signal.h>
38
39#include "internal.h"
40
41
42
43
44
45
46
47static bool page_cache_pipe_buf_try_steal(struct pipe_inode_info *pipe,
48 struct pipe_buffer *buf)
49{
50 struct page *page = buf->page;
51 struct address_space *mapping;
52
53 lock_page(page);
54
55 mapping = page_mapping(page);
56 if (mapping) {
57 WARN_ON(!PageUptodate(page));
58
59
60
61
62
63
64
65
66
67 wait_on_page_writeback(page);
68
69 if (page_has_private(page) &&
70 !try_to_release_page(page, GFP_KERNEL))
71 goto out_unlock;
72
73
74
75
76
77 if (remove_mapping(mapping, page)) {
78 buf->flags |= PIPE_BUF_FLAG_LRU;
79 return true;
80 }
81 }
82
83
84
85
86
87out_unlock:
88 unlock_page(page);
89 return false;
90}
91
92static void page_cache_pipe_buf_release(struct pipe_inode_info *pipe,
93 struct pipe_buffer *buf)
94{
95 put_page(buf->page);
96 buf->flags &= ~PIPE_BUF_FLAG_LRU;
97}
98
99
100
101
102
103static int page_cache_pipe_buf_confirm(struct pipe_inode_info *pipe,
104 struct pipe_buffer *buf)
105{
106 struct page *page = buf->page;
107 int err;
108
109 if (!PageUptodate(page)) {
110 lock_page(page);
111
112
113
114
115
116 if (!page->mapping) {
117 err = -ENODATA;
118 goto error;
119 }
120
121
122
123
124 if (!PageUptodate(page)) {
125 err = -EIO;
126 goto error;
127 }
128
129
130
131
132 unlock_page(page);
133 }
134
135 return 0;
136error:
137 unlock_page(page);
138 return err;
139}
140
141const struct pipe_buf_operations page_cache_pipe_buf_ops = {
142 .confirm = page_cache_pipe_buf_confirm,
143 .release = page_cache_pipe_buf_release,
144 .try_steal = page_cache_pipe_buf_try_steal,
145 .get = generic_pipe_buf_get,
146};
147
148static bool user_page_pipe_buf_try_steal(struct pipe_inode_info *pipe,
149 struct pipe_buffer *buf)
150{
151 if (!(buf->flags & PIPE_BUF_FLAG_GIFT))
152 return false;
153
154 buf->flags |= PIPE_BUF_FLAG_LRU;
155 return generic_pipe_buf_try_steal(pipe, buf);
156}
157
158static const struct pipe_buf_operations user_page_pipe_buf_ops = {
159 .release = page_cache_pipe_buf_release,
160 .try_steal = user_page_pipe_buf_try_steal,
161 .get = generic_pipe_buf_get,
162};
163
164static void wakeup_pipe_readers(struct pipe_inode_info *pipe)
165{
166 smp_mb();
167 if (waitqueue_active(&pipe->rd_wait))
168 wake_up_interruptible(&pipe->rd_wait);
169 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN);
170}
171
172
173
174
175
176
177
178
179
180
181
182
183ssize_t splice_to_pipe(struct pipe_inode_info *pipe,
184 struct splice_pipe_desc *spd)
185{
186 unsigned int spd_pages = spd->nr_pages;
187 unsigned int tail = pipe->tail;
188 unsigned int head = pipe->head;
189 unsigned int mask = pipe->ring_size - 1;
190 int ret = 0, page_nr = 0;
191
192 if (!spd_pages)
193 return 0;
194
195 if (unlikely(!pipe->readers)) {
196 send_sig(SIGPIPE, current, 0);
197 ret = -EPIPE;
198 goto out;
199 }
200
201 while (!pipe_full(head, tail, pipe->max_usage)) {
202 struct pipe_buffer *buf = &pipe->bufs[head & mask];
203
204 buf->page = spd->pages[page_nr];
205 buf->offset = spd->partial[page_nr].offset;
206 buf->len = spd->partial[page_nr].len;
207 buf->private = spd->partial[page_nr].private;
208 buf->ops = spd->ops;
209 buf->flags = 0;
210
211 head++;
212 pipe->head = head;
213 page_nr++;
214 ret += buf->len;
215
216 if (!--spd->nr_pages)
217 break;
218 }
219
220 if (!ret)
221 ret = -EAGAIN;
222
223out:
224 while (page_nr < spd_pages)
225 spd->spd_release(spd, page_nr++);
226
227 return ret;
228}
229EXPORT_SYMBOL_GPL(splice_to_pipe);
230
231ssize_t add_to_pipe(struct pipe_inode_info *pipe, struct pipe_buffer *buf)
232{
233 unsigned int head = pipe->head;
234 unsigned int tail = pipe->tail;
235 unsigned int mask = pipe->ring_size - 1;
236 int ret;
237
238 if (unlikely(!pipe->readers)) {
239 send_sig(SIGPIPE, current, 0);
240 ret = -EPIPE;
241 } else if (pipe_full(head, tail, pipe->max_usage)) {
242 ret = -EAGAIN;
243 } else {
244 pipe->bufs[head & mask] = *buf;
245 pipe->head = head + 1;
246 return buf->len;
247 }
248 pipe_buf_release(pipe, buf);
249 return ret;
250}
251EXPORT_SYMBOL(add_to_pipe);
252
253
254
255
256
257int splice_grow_spd(const struct pipe_inode_info *pipe, struct splice_pipe_desc *spd)
258{
259 unsigned int max_usage = READ_ONCE(pipe->max_usage);
260
261 spd->nr_pages_max = max_usage;
262 if (max_usage <= PIPE_DEF_BUFFERS)
263 return 0;
264
265 spd->pages = kmalloc_array(max_usage, sizeof(struct page *), GFP_KERNEL);
266 spd->partial = kmalloc_array(max_usage, sizeof(struct partial_page),
267 GFP_KERNEL);
268
269 if (spd->pages && spd->partial)
270 return 0;
271
272 kfree(spd->pages);
273 kfree(spd->partial);
274 return -ENOMEM;
275}
276
277void splice_shrink_spd(struct splice_pipe_desc *spd)
278{
279 if (spd->nr_pages_max <= PIPE_DEF_BUFFERS)
280 return;
281
282 kfree(spd->pages);
283 kfree(spd->partial);
284}
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299ssize_t generic_file_splice_read(struct file *in, loff_t *ppos,
300 struct pipe_inode_info *pipe, size_t len,
301 unsigned int flags)
302{
303 struct iov_iter to;
304 struct kiocb kiocb;
305 unsigned int i_head;
306 int ret;
307
308 iov_iter_pipe(&to, READ, pipe, len);
309 i_head = to.head;
310 init_sync_kiocb(&kiocb, in);
311 kiocb.ki_pos = *ppos;
312 ret = call_read_iter(in, &kiocb, &to);
313 if (ret > 0) {
314 *ppos = kiocb.ki_pos;
315 file_accessed(in);
316 } else if (ret < 0) {
317 to.head = i_head;
318 to.iov_offset = 0;
319 iov_iter_advance(&to, 0);
320
321
322
323
324 if (ret == -EFAULT)
325 ret = -EAGAIN;
326 }
327
328 return ret;
329}
330EXPORT_SYMBOL(generic_file_splice_read);
331
332const struct pipe_buf_operations default_pipe_buf_ops = {
333 .release = generic_pipe_buf_release,
334 .try_steal = generic_pipe_buf_try_steal,
335 .get = generic_pipe_buf_get,
336};
337
338
339const struct pipe_buf_operations nosteal_pipe_buf_ops = {
340 .release = generic_pipe_buf_release,
341 .get = generic_pipe_buf_get,
342};
343EXPORT_SYMBOL(nosteal_pipe_buf_ops);
344
345static ssize_t kernel_readv(struct file *file, const struct kvec *vec,
346 unsigned long vlen, loff_t offset)
347{
348 mm_segment_t old_fs;
349 loff_t pos = offset;
350 ssize_t res;
351
352 old_fs = get_fs();
353 set_fs(KERNEL_DS);
354
355 res = vfs_readv(file, (const struct iovec __user *)vec, vlen, &pos, 0);
356 set_fs(old_fs);
357
358 return res;
359}
360
361static ssize_t default_file_splice_read(struct file *in, loff_t *ppos,
362 struct pipe_inode_info *pipe, size_t len,
363 unsigned int flags)
364{
365 struct kvec *vec, __vec[PIPE_DEF_BUFFERS];
366 struct iov_iter to;
367 struct page **pages;
368 unsigned int nr_pages;
369 unsigned int mask;
370 size_t offset, base, copied = 0;
371 ssize_t res;
372 int i;
373
374 if (pipe_full(pipe->head, pipe->tail, pipe->max_usage))
375 return -EAGAIN;
376
377
378
379
380
381 offset = *ppos & ~PAGE_MASK;
382
383 iov_iter_pipe(&to, READ, pipe, len + offset);
384
385 res = iov_iter_get_pages_alloc(&to, &pages, len + offset, &base);
386 if (res <= 0)
387 return -ENOMEM;
388
389 nr_pages = DIV_ROUND_UP(res + base, PAGE_SIZE);
390
391 vec = __vec;
392 if (nr_pages > PIPE_DEF_BUFFERS) {
393 vec = kmalloc_array(nr_pages, sizeof(struct kvec), GFP_KERNEL);
394 if (unlikely(!vec)) {
395 res = -ENOMEM;
396 goto out;
397 }
398 }
399
400 mask = pipe->ring_size - 1;
401 pipe->bufs[to.head & mask].offset = offset;
402 pipe->bufs[to.head & mask].len -= offset;
403
404 for (i = 0; i < nr_pages; i++) {
405 size_t this_len = min_t(size_t, len, PAGE_SIZE - offset);
406 vec[i].iov_base = page_address(pages[i]) + offset;
407 vec[i].iov_len = this_len;
408 len -= this_len;
409 offset = 0;
410 }
411
412 res = kernel_readv(in, vec, nr_pages, *ppos);
413 if (res > 0) {
414 copied = res;
415 *ppos += res;
416 }
417
418 if (vec != __vec)
419 kfree(vec);
420out:
421 for (i = 0; i < nr_pages; i++)
422 put_page(pages[i]);
423 kvfree(pages);
424 iov_iter_advance(&to, copied);
425 return res;
426}
427
428
429
430
431
432static int pipe_to_sendpage(struct pipe_inode_info *pipe,
433 struct pipe_buffer *buf, struct splice_desc *sd)
434{
435 struct file *file = sd->u.file;
436 loff_t pos = sd->pos;
437 int more;
438
439 if (!likely(file->f_op->sendpage))
440 return -EINVAL;
441
442 more = (sd->flags & SPLICE_F_MORE) ? MSG_MORE : 0;
443
444 if (sd->len < sd->total_len &&
445 pipe_occupancy(pipe->head, pipe->tail) > 1)
446 more |= MSG_SENDPAGE_NOTLAST;
447
448 return file->f_op->sendpage(file, buf->page, buf->offset,
449 sd->len, &pos, more);
450}
451
452static void wakeup_pipe_writers(struct pipe_inode_info *pipe)
453{
454 smp_mb();
455 if (waitqueue_active(&pipe->wr_wait))
456 wake_up_interruptible(&pipe->wr_wait);
457 kill_fasync(&pipe->fasync_writers, SIGIO, POLL_OUT);
458}
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480static int splice_from_pipe_feed(struct pipe_inode_info *pipe, struct splice_desc *sd,
481 splice_actor *actor)
482{
483 unsigned int head = pipe->head;
484 unsigned int tail = pipe->tail;
485 unsigned int mask = pipe->ring_size - 1;
486 int ret;
487
488 while (!pipe_empty(head, tail)) {
489 struct pipe_buffer *buf = &pipe->bufs[tail & mask];
490
491 sd->len = buf->len;
492 if (sd->len > sd->total_len)
493 sd->len = sd->total_len;
494
495 ret = pipe_buf_confirm(pipe, buf);
496 if (unlikely(ret)) {
497 if (ret == -ENODATA)
498 ret = 0;
499 return ret;
500 }
501
502 ret = actor(pipe, buf, sd);
503 if (ret <= 0)
504 return ret;
505
506 buf->offset += ret;
507 buf->len -= ret;
508
509 sd->num_spliced += ret;
510 sd->len -= ret;
511 sd->pos += ret;
512 sd->total_len -= ret;
513
514 if (!buf->len) {
515 pipe_buf_release(pipe, buf);
516 tail++;
517 pipe->tail = tail;
518 if (pipe->files)
519 sd->need_wakeup = true;
520 }
521
522 if (!sd->total_len)
523 return 0;
524 }
525
526 return 1;
527}
528
529
530static inline bool eat_empty_buffer(struct pipe_inode_info *pipe)
531{
532 unsigned int tail = pipe->tail;
533 unsigned int mask = pipe->ring_size - 1;
534 struct pipe_buffer *buf = &pipe->bufs[tail & mask];
535
536 if (unlikely(!buf->len)) {
537 pipe_buf_release(pipe, buf);
538 pipe->tail = tail+1;
539 return true;
540 }
541
542 return false;
543}
544
545
546
547
548
549
550
551
552
553
554
555static int splice_from_pipe_next(struct pipe_inode_info *pipe, struct splice_desc *sd)
556{
557
558
559
560
561 if (signal_pending(current))
562 return -ERESTARTSYS;
563
564repeat:
565 while (pipe_empty(pipe->head, pipe->tail)) {
566 if (!pipe->writers)
567 return 0;
568
569 if (sd->num_spliced)
570 return 0;
571
572 if (sd->flags & SPLICE_F_NONBLOCK)
573 return -EAGAIN;
574
575 if (signal_pending(current))
576 return -ERESTARTSYS;
577
578 if (sd->need_wakeup) {
579 wakeup_pipe_writers(pipe);
580 sd->need_wakeup = false;
581 }
582
583 pipe_wait_readable(pipe);
584 }
585
586 if (eat_empty_buffer(pipe))
587 goto repeat;
588
589 return 1;
590}
591
592
593
594
595
596
597
598
599
600
601static void splice_from_pipe_begin(struct splice_desc *sd)
602{
603 sd->num_spliced = 0;
604 sd->need_wakeup = false;
605}
606
607
608
609
610
611
612
613
614
615
616
617static void splice_from_pipe_end(struct pipe_inode_info *pipe, struct splice_desc *sd)
618{
619 if (sd->need_wakeup)
620 wakeup_pipe_writers(pipe);
621}
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636ssize_t __splice_from_pipe(struct pipe_inode_info *pipe, struct splice_desc *sd,
637 splice_actor *actor)
638{
639 int ret;
640
641 splice_from_pipe_begin(sd);
642 do {
643 cond_resched();
644 ret = splice_from_pipe_next(pipe, sd);
645 if (ret > 0)
646 ret = splice_from_pipe_feed(pipe, sd, actor);
647 } while (ret > 0);
648 splice_from_pipe_end(pipe, sd);
649
650 return sd->num_spliced ? sd->num_spliced : ret;
651}
652EXPORT_SYMBOL(__splice_from_pipe);
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668ssize_t splice_from_pipe(struct pipe_inode_info *pipe, struct file *out,
669 loff_t *ppos, size_t len, unsigned int flags,
670 splice_actor *actor)
671{
672 ssize_t ret;
673 struct splice_desc sd = {
674 .total_len = len,
675 .flags = flags,
676 .pos = *ppos,
677 .u.file = out,
678 };
679
680 pipe_lock(pipe);
681 ret = __splice_from_pipe(pipe, &sd, actor);
682 pipe_unlock(pipe);
683
684 return ret;
685}
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701ssize_t
702iter_file_splice_write(struct pipe_inode_info *pipe, struct file *out,
703 loff_t *ppos, size_t len, unsigned int flags)
704{
705 struct splice_desc sd = {
706 .total_len = len,
707 .flags = flags,
708 .pos = *ppos,
709 .u.file = out,
710 };
711 int nbufs = pipe->max_usage;
712 struct bio_vec *array = kcalloc(nbufs, sizeof(struct bio_vec),
713 GFP_KERNEL);
714 ssize_t ret;
715
716 if (unlikely(!array))
717 return -ENOMEM;
718
719 pipe_lock(pipe);
720
721 splice_from_pipe_begin(&sd);
722 while (sd.total_len) {
723 struct iov_iter from;
724 unsigned int head, tail, mask;
725 size_t left;
726 int n;
727
728 ret = splice_from_pipe_next(pipe, &sd);
729 if (ret <= 0)
730 break;
731
732 if (unlikely(nbufs < pipe->max_usage)) {
733 kfree(array);
734 nbufs = pipe->max_usage;
735 array = kcalloc(nbufs, sizeof(struct bio_vec),
736 GFP_KERNEL);
737 if (!array) {
738 ret = -ENOMEM;
739 break;
740 }
741 }
742
743 head = pipe->head;
744 tail = pipe->tail;
745 mask = pipe->ring_size - 1;
746
747
748 left = sd.total_len;
749 for (n = 0; !pipe_empty(head, tail) && left && n < nbufs; tail++, n++) {
750 struct pipe_buffer *buf = &pipe->bufs[tail & mask];
751 size_t this_len = buf->len;
752
753 if (this_len > left)
754 this_len = left;
755
756 ret = pipe_buf_confirm(pipe, buf);
757 if (unlikely(ret)) {
758 if (ret == -ENODATA)
759 ret = 0;
760 goto done;
761 }
762
763 array[n].bv_page = buf->page;
764 array[n].bv_len = this_len;
765 array[n].bv_offset = buf->offset;
766 left -= this_len;
767 }
768
769 iov_iter_bvec(&from, WRITE, array, n, sd.total_len - left);
770 ret = vfs_iter_write(out, &from, &sd.pos, 0);
771 if (ret <= 0)
772 break;
773
774 sd.num_spliced += ret;
775 sd.total_len -= ret;
776 *ppos = sd.pos;
777
778
779 tail = pipe->tail;
780 while (ret) {
781 struct pipe_buffer *buf = &pipe->bufs[tail & mask];
782 if (ret >= buf->len) {
783 ret -= buf->len;
784 buf->len = 0;
785 pipe_buf_release(pipe, buf);
786 tail++;
787 pipe->tail = tail;
788 if (pipe->files)
789 sd.need_wakeup = true;
790 } else {
791 buf->offset += ret;
792 buf->len -= ret;
793 ret = 0;
794 }
795 }
796 }
797done:
798 kfree(array);
799 splice_from_pipe_end(pipe, &sd);
800
801 pipe_unlock(pipe);
802
803 if (sd.num_spliced)
804 ret = sd.num_spliced;
805
806 return ret;
807}
808
809EXPORT_SYMBOL(iter_file_splice_write);
810
811static int write_pipe_buf(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
812 struct splice_desc *sd)
813{
814 int ret;
815 void *data;
816 loff_t tmp = sd->pos;
817
818 data = kmap(buf->page);
819 ret = __kernel_write(sd->u.file, data + buf->offset, sd->len, &tmp);
820 kunmap(buf->page);
821
822 return ret;
823}
824
825static ssize_t default_file_splice_write(struct pipe_inode_info *pipe,
826 struct file *out, loff_t *ppos,
827 size_t len, unsigned int flags)
828{
829 ssize_t ret;
830
831 ret = splice_from_pipe(pipe, out, ppos, len, flags, write_pipe_buf);
832 if (ret > 0)
833 *ppos += ret;
834
835 return ret;
836}
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851ssize_t generic_splice_sendpage(struct pipe_inode_info *pipe, struct file *out,
852 loff_t *ppos, size_t len, unsigned int flags)
853{
854 return splice_from_pipe(pipe, out, ppos, len, flags, pipe_to_sendpage);
855}
856
857EXPORT_SYMBOL(generic_splice_sendpage);
858
859
860
861
862static long do_splice_from(struct pipe_inode_info *pipe, struct file *out,
863 loff_t *ppos, size_t len, unsigned int flags)
864{
865 if (out->f_op->splice_write)
866 return out->f_op->splice_write(pipe, out, ppos, len, flags);
867 return default_file_splice_write(pipe, out, ppos, len, flags);
868}
869
870
871
872
873static long do_splice_to(struct file *in, loff_t *ppos,
874 struct pipe_inode_info *pipe, size_t len,
875 unsigned int flags)
876{
877 int ret;
878
879 if (unlikely(!(in->f_mode & FMODE_READ)))
880 return -EBADF;
881
882 ret = rw_verify_area(READ, in, ppos, len);
883 if (unlikely(ret < 0))
884 return ret;
885
886 if (unlikely(len > MAX_RW_COUNT))
887 len = MAX_RW_COUNT;
888
889 if (in->f_op->splice_read)
890 return in->f_op->splice_read(in, ppos, pipe, len, flags);
891 return default_file_splice_read(in, ppos, pipe, len, flags);
892}
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907ssize_t splice_direct_to_actor(struct file *in, struct splice_desc *sd,
908 splice_direct_actor *actor)
909{
910 struct pipe_inode_info *pipe;
911 long ret, bytes;
912 umode_t i_mode;
913 size_t len;
914 int i, flags, more;
915
916
917
918
919
920
921 i_mode = file_inode(in)->i_mode;
922 if (unlikely(!S_ISREG(i_mode) && !S_ISBLK(i_mode)))
923 return -EINVAL;
924
925
926
927
928
929 pipe = current->splice_pipe;
930 if (unlikely(!pipe)) {
931 pipe = alloc_pipe_info();
932 if (!pipe)
933 return -ENOMEM;
934
935
936
937
938
939
940 pipe->readers = 1;
941
942 current->splice_pipe = pipe;
943 }
944
945
946
947
948 ret = 0;
949 bytes = 0;
950 len = sd->total_len;
951 flags = sd->flags;
952
953
954
955
956 sd->flags &= ~SPLICE_F_NONBLOCK;
957 more = sd->flags & SPLICE_F_MORE;
958
959 WARN_ON_ONCE(!pipe_empty(pipe->head, pipe->tail));
960
961 while (len) {
962 unsigned int p_space;
963 size_t read_len;
964 loff_t pos = sd->pos, prev_pos = pos;
965
966
967 p_space = pipe->max_usage -
968 pipe_occupancy(pipe->head, pipe->tail);
969 read_len = min_t(size_t, len, p_space << PAGE_SHIFT);
970 ret = do_splice_to(in, &pos, pipe, read_len, flags);
971 if (unlikely(ret <= 0))
972 goto out_release;
973
974 read_len = ret;
975 sd->total_len = read_len;
976
977
978
979
980
981
982 if (read_len < len)
983 sd->flags |= SPLICE_F_MORE;
984 else if (!more)
985 sd->flags &= ~SPLICE_F_MORE;
986
987
988
989
990
991 ret = actor(pipe, sd);
992 if (unlikely(ret <= 0)) {
993 sd->pos = prev_pos;
994 goto out_release;
995 }
996
997 bytes += ret;
998 len -= ret;
999 sd->pos = pos;
1000
1001 if (ret < read_len) {
1002 sd->pos = prev_pos + ret;
1003 goto out_release;
1004 }
1005 }
1006
1007done:
1008 pipe->tail = pipe->head = 0;
1009 file_accessed(in);
1010 return bytes;
1011
1012out_release:
1013
1014
1015
1016
1017 for (i = 0; i < pipe->ring_size; i++) {
1018 struct pipe_buffer *buf = &pipe->bufs[i];
1019
1020 if (buf->ops)
1021 pipe_buf_release(pipe, buf);
1022 }
1023
1024 if (!bytes)
1025 bytes = ret;
1026
1027 goto done;
1028}
1029EXPORT_SYMBOL(splice_direct_to_actor);
1030
1031static int direct_splice_actor(struct pipe_inode_info *pipe,
1032 struct splice_desc *sd)
1033{
1034 struct file *file = sd->u.file;
1035
1036 return do_splice_from(pipe, file, sd->opos, sd->total_len,
1037 sd->flags);
1038}
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056long do_splice_direct(struct file *in, loff_t *ppos, struct file *out,
1057 loff_t *opos, size_t len, unsigned int flags)
1058{
1059 struct splice_desc sd = {
1060 .len = len,
1061 .total_len = len,
1062 .flags = flags,
1063 .pos = *ppos,
1064 .u.file = out,
1065 .opos = opos,
1066 };
1067 long ret;
1068
1069 if (unlikely(!(out->f_mode & FMODE_WRITE)))
1070 return -EBADF;
1071
1072 if (unlikely(out->f_flags & O_APPEND))
1073 return -EINVAL;
1074
1075 ret = rw_verify_area(WRITE, out, opos, len);
1076 if (unlikely(ret < 0))
1077 return ret;
1078
1079 ret = splice_direct_to_actor(in, &sd, direct_splice_actor);
1080 if (ret > 0)
1081 *ppos = sd.pos;
1082
1083 return ret;
1084}
1085EXPORT_SYMBOL(do_splice_direct);
1086
1087static int wait_for_space(struct pipe_inode_info *pipe, unsigned flags)
1088{
1089 for (;;) {
1090 if (unlikely(!pipe->readers)) {
1091 send_sig(SIGPIPE, current, 0);
1092 return -EPIPE;
1093 }
1094 if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage))
1095 return 0;
1096 if (flags & SPLICE_F_NONBLOCK)
1097 return -EAGAIN;
1098 if (signal_pending(current))
1099 return -ERESTARTSYS;
1100 pipe_wait_writable(pipe);
1101 }
1102}
1103
1104static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
1105 struct pipe_inode_info *opipe,
1106 size_t len, unsigned int flags);
1107
1108
1109
1110
1111long do_splice(struct file *in, loff_t __user *off_in,
1112 struct file *out, loff_t __user *off_out,
1113 size_t len, unsigned int flags)
1114{
1115 struct pipe_inode_info *ipipe;
1116 struct pipe_inode_info *opipe;
1117 loff_t offset;
1118 long ret;
1119
1120 if (unlikely(!(in->f_mode & FMODE_READ) ||
1121 !(out->f_mode & FMODE_WRITE)))
1122 return -EBADF;
1123
1124 ipipe = get_pipe_info(in, true);
1125 opipe = get_pipe_info(out, true);
1126
1127 if (ipipe && opipe) {
1128 if (off_in || off_out)
1129 return -ESPIPE;
1130
1131
1132 if (ipipe == opipe)
1133 return -EINVAL;
1134
1135 if ((in->f_flags | out->f_flags) & O_NONBLOCK)
1136 flags |= SPLICE_F_NONBLOCK;
1137
1138 return splice_pipe_to_pipe(ipipe, opipe, len, flags);
1139 }
1140
1141 if (ipipe) {
1142 if (off_in)
1143 return -ESPIPE;
1144 if (off_out) {
1145 if (!(out->f_mode & FMODE_PWRITE))
1146 return -EINVAL;
1147 if (copy_from_user(&offset, off_out, sizeof(loff_t)))
1148 return -EFAULT;
1149 } else {
1150 offset = out->f_pos;
1151 }
1152
1153 if (unlikely(out->f_flags & O_APPEND))
1154 return -EINVAL;
1155
1156 ret = rw_verify_area(WRITE, out, &offset, len);
1157 if (unlikely(ret < 0))
1158 return ret;
1159
1160 if (in->f_flags & O_NONBLOCK)
1161 flags |= SPLICE_F_NONBLOCK;
1162
1163 file_start_write(out);
1164 ret = do_splice_from(ipipe, out, &offset, len, flags);
1165 file_end_write(out);
1166
1167 if (!off_out)
1168 out->f_pos = offset;
1169 else if (copy_to_user(off_out, &offset, sizeof(loff_t)))
1170 ret = -EFAULT;
1171
1172 return ret;
1173 }
1174
1175 if (opipe) {
1176 if (off_out)
1177 return -ESPIPE;
1178 if (off_in) {
1179 if (!(in->f_mode & FMODE_PREAD))
1180 return -EINVAL;
1181 if (copy_from_user(&offset, off_in, sizeof(loff_t)))
1182 return -EFAULT;
1183 } else {
1184 offset = in->f_pos;
1185 }
1186
1187 if (out->f_flags & O_NONBLOCK)
1188 flags |= SPLICE_F_NONBLOCK;
1189
1190 pipe_lock(opipe);
1191 ret = wait_for_space(opipe, flags);
1192 if (!ret) {
1193 unsigned int p_space;
1194
1195
1196 p_space = opipe->max_usage - pipe_occupancy(opipe->head, opipe->tail);
1197 len = min_t(size_t, len, p_space << PAGE_SHIFT);
1198
1199 ret = do_splice_to(in, &offset, opipe, len, flags);
1200 }
1201 pipe_unlock(opipe);
1202 if (ret > 0)
1203 wakeup_pipe_readers(opipe);
1204 if (!off_in)
1205 in->f_pos = offset;
1206 else if (copy_to_user(off_in, &offset, sizeof(loff_t)))
1207 ret = -EFAULT;
1208
1209 return ret;
1210 }
1211
1212 return -EINVAL;
1213}
1214
1215static int iter_to_pipe(struct iov_iter *from,
1216 struct pipe_inode_info *pipe,
1217 unsigned flags)
1218{
1219 struct pipe_buffer buf = {
1220 .ops = &user_page_pipe_buf_ops,
1221 .flags = flags
1222 };
1223 size_t total = 0;
1224 int ret = 0;
1225 bool failed = false;
1226
1227 while (iov_iter_count(from) && !failed) {
1228 struct page *pages[16];
1229 ssize_t copied;
1230 size_t start;
1231 int n;
1232
1233 copied = iov_iter_get_pages(from, pages, ~0UL, 16, &start);
1234 if (copied <= 0) {
1235 ret = copied;
1236 break;
1237 }
1238
1239 for (n = 0; copied; n++, start = 0) {
1240 int size = min_t(int, copied, PAGE_SIZE - start);
1241 if (!failed) {
1242 buf.page = pages[n];
1243 buf.offset = start;
1244 buf.len = size;
1245 ret = add_to_pipe(pipe, &buf);
1246 if (unlikely(ret < 0)) {
1247 failed = true;
1248 } else {
1249 iov_iter_advance(from, ret);
1250 total += ret;
1251 }
1252 } else {
1253 put_page(pages[n]);
1254 }
1255 copied -= size;
1256 }
1257 }
1258 return total ? total : ret;
1259}
1260
1261static int pipe_to_user(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
1262 struct splice_desc *sd)
1263{
1264 int n = copy_page_to_iter(buf->page, buf->offset, sd->len, sd->u.data);
1265 return n == sd->len ? n : -EFAULT;
1266}
1267
1268
1269
1270
1271
1272static long vmsplice_to_user(struct file *file, struct iov_iter *iter,
1273 unsigned int flags)
1274{
1275 struct pipe_inode_info *pipe = get_pipe_info(file, true);
1276 struct splice_desc sd = {
1277 .total_len = iov_iter_count(iter),
1278 .flags = flags,
1279 .u.data = iter
1280 };
1281 long ret = 0;
1282
1283 if (!pipe)
1284 return -EBADF;
1285
1286 if (sd.total_len) {
1287 pipe_lock(pipe);
1288 ret = __splice_from_pipe(pipe, &sd, pipe_to_user);
1289 pipe_unlock(pipe);
1290 }
1291
1292 return ret;
1293}
1294
1295
1296
1297
1298
1299
1300static long vmsplice_to_pipe(struct file *file, struct iov_iter *iter,
1301 unsigned int flags)
1302{
1303 struct pipe_inode_info *pipe;
1304 long ret = 0;
1305 unsigned buf_flag = 0;
1306
1307 if (flags & SPLICE_F_GIFT)
1308 buf_flag = PIPE_BUF_FLAG_GIFT;
1309
1310 pipe = get_pipe_info(file, true);
1311 if (!pipe)
1312 return -EBADF;
1313
1314 pipe_lock(pipe);
1315 ret = wait_for_space(pipe, flags);
1316 if (!ret)
1317 ret = iter_to_pipe(iter, pipe, buf_flag);
1318 pipe_unlock(pipe);
1319 if (ret > 0)
1320 wakeup_pipe_readers(pipe);
1321 return ret;
1322}
1323
1324static int vmsplice_type(struct fd f, int *type)
1325{
1326 if (!f.file)
1327 return -EBADF;
1328 if (f.file->f_mode & FMODE_WRITE) {
1329 *type = WRITE;
1330 } else if (f.file->f_mode & FMODE_READ) {
1331 *type = READ;
1332 } else {
1333 fdput(f);
1334 return -EBADF;
1335 }
1336 return 0;
1337}
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355static long do_vmsplice(struct file *f, struct iov_iter *iter, unsigned int flags)
1356{
1357 if (unlikely(flags & ~SPLICE_F_ALL))
1358 return -EINVAL;
1359
1360 if (!iov_iter_count(iter))
1361 return 0;
1362
1363 if (iov_iter_rw(iter) == WRITE)
1364 return vmsplice_to_pipe(f, iter, flags);
1365 else
1366 return vmsplice_to_user(f, iter, flags);
1367}
1368
1369SYSCALL_DEFINE4(vmsplice, int, fd, const struct iovec __user *, uiov,
1370 unsigned long, nr_segs, unsigned int, flags)
1371{
1372 struct iovec iovstack[UIO_FASTIOV];
1373 struct iovec *iov = iovstack;
1374 struct iov_iter iter;
1375 ssize_t error;
1376 struct fd f;
1377 int type;
1378
1379 f = fdget(fd);
1380 error = vmsplice_type(f, &type);
1381 if (error)
1382 return error;
1383
1384 error = import_iovec(type, uiov, nr_segs,
1385 ARRAY_SIZE(iovstack), &iov, &iter);
1386 if (error >= 0) {
1387 error = do_vmsplice(f.file, &iter, flags);
1388 kfree(iov);
1389 }
1390 fdput(f);
1391 return error;
1392}
1393
1394#ifdef CONFIG_COMPAT
1395COMPAT_SYSCALL_DEFINE4(vmsplice, int, fd, const struct compat_iovec __user *, iov32,
1396 unsigned int, nr_segs, unsigned int, flags)
1397{
1398 struct iovec iovstack[UIO_FASTIOV];
1399 struct iovec *iov = iovstack;
1400 struct iov_iter iter;
1401 ssize_t error;
1402 struct fd f;
1403 int type;
1404
1405 f = fdget(fd);
1406 error = vmsplice_type(f, &type);
1407 if (error)
1408 return error;
1409
1410 error = compat_import_iovec(type, iov32, nr_segs,
1411 ARRAY_SIZE(iovstack), &iov, &iter);
1412 if (error >= 0) {
1413 error = do_vmsplice(f.file, &iter, flags);
1414 kfree(iov);
1415 }
1416 fdput(f);
1417 return error;
1418}
1419#endif
1420
1421SYSCALL_DEFINE6(splice, int, fd_in, loff_t __user *, off_in,
1422 int, fd_out, loff_t __user *, off_out,
1423 size_t, len, unsigned int, flags)
1424{
1425 struct fd in, out;
1426 long error;
1427
1428 if (unlikely(!len))
1429 return 0;
1430
1431 if (unlikely(flags & ~SPLICE_F_ALL))
1432 return -EINVAL;
1433
1434 error = -EBADF;
1435 in = fdget(fd_in);
1436 if (in.file) {
1437 out = fdget(fd_out);
1438 if (out.file) {
1439 error = do_splice(in.file, off_in, out.file, off_out,
1440 len, flags);
1441 fdput(out);
1442 }
1443 fdput(in);
1444 }
1445 return error;
1446}
1447
1448
1449
1450
1451
1452static int ipipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
1453{
1454 int ret;
1455
1456
1457
1458
1459
1460 if (!pipe_empty(pipe->head, pipe->tail))
1461 return 0;
1462
1463 ret = 0;
1464 pipe_lock(pipe);
1465
1466 while (pipe_empty(pipe->head, pipe->tail)) {
1467 if (signal_pending(current)) {
1468 ret = -ERESTARTSYS;
1469 break;
1470 }
1471 if (!pipe->writers)
1472 break;
1473 if (flags & SPLICE_F_NONBLOCK) {
1474 ret = -EAGAIN;
1475 break;
1476 }
1477 pipe_wait_readable(pipe);
1478 }
1479
1480 pipe_unlock(pipe);
1481 return ret;
1482}
1483
1484
1485
1486
1487
1488static int opipe_prep(struct pipe_inode_info *pipe, unsigned int flags)
1489{
1490 int ret;
1491
1492
1493
1494
1495
1496 if (!pipe_full(pipe->head, pipe->tail, pipe->max_usage))
1497 return 0;
1498
1499 ret = 0;
1500 pipe_lock(pipe);
1501
1502 while (pipe_full(pipe->head, pipe->tail, pipe->max_usage)) {
1503 if (!pipe->readers) {
1504 send_sig(SIGPIPE, current, 0);
1505 ret = -EPIPE;
1506 break;
1507 }
1508 if (flags & SPLICE_F_NONBLOCK) {
1509 ret = -EAGAIN;
1510 break;
1511 }
1512 if (signal_pending(current)) {
1513 ret = -ERESTARTSYS;
1514 break;
1515 }
1516 pipe_wait_writable(pipe);
1517 }
1518
1519 pipe_unlock(pipe);
1520 return ret;
1521}
1522
1523
1524
1525
1526static int splice_pipe_to_pipe(struct pipe_inode_info *ipipe,
1527 struct pipe_inode_info *opipe,
1528 size_t len, unsigned int flags)
1529{
1530 struct pipe_buffer *ibuf, *obuf;
1531 unsigned int i_head, o_head;
1532 unsigned int i_tail, o_tail;
1533 unsigned int i_mask, o_mask;
1534 int ret = 0;
1535 bool input_wakeup = false;
1536
1537
1538retry:
1539 ret = ipipe_prep(ipipe, flags);
1540 if (ret)
1541 return ret;
1542
1543 ret = opipe_prep(opipe, flags);
1544 if (ret)
1545 return ret;
1546
1547
1548
1549
1550
1551
1552 pipe_double_lock(ipipe, opipe);
1553
1554 i_tail = ipipe->tail;
1555 i_mask = ipipe->ring_size - 1;
1556 o_head = opipe->head;
1557 o_mask = opipe->ring_size - 1;
1558
1559 do {
1560 size_t o_len;
1561
1562 if (!opipe->readers) {
1563 send_sig(SIGPIPE, current, 0);
1564 if (!ret)
1565 ret = -EPIPE;
1566 break;
1567 }
1568
1569 i_head = ipipe->head;
1570 o_tail = opipe->tail;
1571
1572 if (pipe_empty(i_head, i_tail) && !ipipe->writers)
1573 break;
1574
1575
1576
1577
1578
1579 if (pipe_empty(i_head, i_tail) ||
1580 pipe_full(o_head, o_tail, opipe->max_usage)) {
1581
1582 if (ret)
1583 break;
1584
1585 if (flags & SPLICE_F_NONBLOCK) {
1586 ret = -EAGAIN;
1587 break;
1588 }
1589
1590
1591
1592
1593
1594
1595 pipe_unlock(ipipe);
1596 pipe_unlock(opipe);
1597 goto retry;
1598 }
1599
1600 ibuf = &ipipe->bufs[i_tail & i_mask];
1601 obuf = &opipe->bufs[o_head & o_mask];
1602
1603 if (len >= ibuf->len) {
1604
1605
1606
1607 *obuf = *ibuf;
1608 ibuf->ops = NULL;
1609 i_tail++;
1610 ipipe->tail = i_tail;
1611 input_wakeup = true;
1612 o_len = obuf->len;
1613 o_head++;
1614 opipe->head = o_head;
1615 } else {
1616
1617
1618
1619
1620 if (!pipe_buf_get(ipipe, ibuf)) {
1621 if (ret == 0)
1622 ret = -EFAULT;
1623 break;
1624 }
1625 *obuf = *ibuf;
1626
1627
1628
1629
1630
1631 obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
1632 obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE;
1633
1634 obuf->len = len;
1635 ibuf->offset += len;
1636 ibuf->len -= len;
1637 o_len = len;
1638 o_head++;
1639 opipe->head = o_head;
1640 }
1641 ret += o_len;
1642 len -= o_len;
1643 } while (len);
1644
1645 pipe_unlock(ipipe);
1646 pipe_unlock(opipe);
1647
1648
1649
1650
1651 if (ret > 0)
1652 wakeup_pipe_readers(opipe);
1653
1654 if (input_wakeup)
1655 wakeup_pipe_writers(ipipe);
1656
1657 return ret;
1658}
1659
1660
1661
1662
1663static int link_pipe(struct pipe_inode_info *ipipe,
1664 struct pipe_inode_info *opipe,
1665 size_t len, unsigned int flags)
1666{
1667 struct pipe_buffer *ibuf, *obuf;
1668 unsigned int i_head, o_head;
1669 unsigned int i_tail, o_tail;
1670 unsigned int i_mask, o_mask;
1671 int ret = 0;
1672
1673
1674
1675
1676
1677
1678 pipe_double_lock(ipipe, opipe);
1679
1680 i_tail = ipipe->tail;
1681 i_mask = ipipe->ring_size - 1;
1682 o_head = opipe->head;
1683 o_mask = opipe->ring_size - 1;
1684
1685 do {
1686 if (!opipe->readers) {
1687 send_sig(SIGPIPE, current, 0);
1688 if (!ret)
1689 ret = -EPIPE;
1690 break;
1691 }
1692
1693 i_head = ipipe->head;
1694 o_tail = opipe->tail;
1695
1696
1697
1698
1699
1700 if (pipe_empty(i_head, i_tail) ||
1701 pipe_full(o_head, o_tail, opipe->max_usage))
1702 break;
1703
1704 ibuf = &ipipe->bufs[i_tail & i_mask];
1705 obuf = &opipe->bufs[o_head & o_mask];
1706
1707
1708
1709
1710
1711 if (!pipe_buf_get(ipipe, ibuf)) {
1712 if (ret == 0)
1713 ret = -EFAULT;
1714 break;
1715 }
1716
1717 *obuf = *ibuf;
1718
1719
1720
1721
1722
1723 obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
1724 obuf->flags &= ~PIPE_BUF_FLAG_CAN_MERGE;
1725
1726 if (obuf->len > len)
1727 obuf->len = len;
1728 ret += obuf->len;
1729 len -= obuf->len;
1730
1731 o_head++;
1732 opipe->head = o_head;
1733 i_tail++;
1734 } while (len);
1735
1736 pipe_unlock(ipipe);
1737 pipe_unlock(opipe);
1738
1739
1740
1741
1742 if (ret > 0)
1743 wakeup_pipe_readers(opipe);
1744
1745 return ret;
1746}
1747
1748
1749
1750
1751
1752
1753
1754long do_tee(struct file *in, struct file *out, size_t len, unsigned int flags)
1755{
1756 struct pipe_inode_info *ipipe = get_pipe_info(in, true);
1757 struct pipe_inode_info *opipe = get_pipe_info(out, true);
1758 int ret = -EINVAL;
1759
1760 if (unlikely(!(in->f_mode & FMODE_READ) ||
1761 !(out->f_mode & FMODE_WRITE)))
1762 return -EBADF;
1763
1764
1765
1766
1767
1768 if (ipipe && opipe && ipipe != opipe) {
1769 if ((in->f_flags | out->f_flags) & O_NONBLOCK)
1770 flags |= SPLICE_F_NONBLOCK;
1771
1772
1773
1774
1775
1776 ret = ipipe_prep(ipipe, flags);
1777 if (!ret) {
1778 ret = opipe_prep(opipe, flags);
1779 if (!ret)
1780 ret = link_pipe(ipipe, opipe, len, flags);
1781 }
1782 }
1783
1784 return ret;
1785}
1786
1787SYSCALL_DEFINE4(tee, int, fdin, int, fdout, size_t, len, unsigned int, flags)
1788{
1789 struct fd in, out;
1790 int error;
1791
1792 if (unlikely(flags & ~SPLICE_F_ALL))
1793 return -EINVAL;
1794
1795 if (unlikely(!len))
1796 return 0;
1797
1798 error = -EBADF;
1799 in = fdget(fdin);
1800 if (in.file) {
1801 out = fdget(fdout);
1802 if (out.file) {
1803 error = do_tee(in.file, out.file, len, flags);
1804 fdput(out);
1805 }
1806 fdput(in);
1807 }
1808
1809 return error;
1810}
1811