1
2
3
4
5
6
7
8
9#ifndef __LINUX_UIO_H
10#define __LINUX_UIO_H
11
12#include <linux/kernel.h>
13#include <linux/thread_info.h>
14#include <uapi/linux/uio.h>
15
16struct page;
17struct pipe_inode_info;
18
19struct kvec {
20 void *iov_base;
21 size_t iov_len;
22};
23
24enum {
25 ITER_IOVEC = 0,
26 ITER_KVEC = 2,
27 ITER_BVEC = 4,
28 ITER_PIPE = 8,
29};
30
31struct iov_iter {
32 int type;
33 size_t iov_offset;
34 size_t count;
35 union {
36 const struct iovec *iov;
37 const struct kvec *kvec;
38 const struct bio_vec *bvec;
39 struct pipe_inode_info *pipe;
40 };
41 union {
42 unsigned long nr_segs;
43 struct {
44 int idx;
45 int start_idx;
46 };
47 };
48};
49
50
51
52
53
54
55
56
57static inline size_t iov_length(const struct iovec *iov, unsigned long nr_segs)
58{
59 unsigned long seg;
60 size_t ret = 0;
61
62 for (seg = 0; seg < nr_segs; seg++)
63 ret += iov[seg].iov_len;
64 return ret;
65}
66
67static inline struct iovec iov_iter_iovec(const struct iov_iter *iter)
68{
69 return (struct iovec) {
70 .iov_base = iter->iov->iov_base + iter->iov_offset,
71 .iov_len = min(iter->count,
72 iter->iov->iov_len - iter->iov_offset),
73 };
74}
75
76#define iov_for_each(iov, iter, start) \
77 if (!((start).type & (ITER_BVEC | ITER_PIPE))) \
78 for (iter = (start); \
79 (iter).count && \
80 ((iov = iov_iter_iovec(&(iter))), 1); \
81 iov_iter_advance(&(iter), (iov).iov_len))
82
83size_t iov_iter_copy_from_user_atomic(struct page *page,
84 struct iov_iter *i, unsigned long offset, size_t bytes);
85void iov_iter_advance(struct iov_iter *i, size_t bytes);
86void iov_iter_revert(struct iov_iter *i, size_t bytes);
87int iov_iter_fault_in_readable(struct iov_iter *i, size_t bytes);
88size_t iov_iter_single_seg_count(const struct iov_iter *i);
89size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
90 struct iov_iter *i);
91size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
92 struct iov_iter *i);
93
94size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i);
95size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i);
96bool _copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i);
97size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i);
98bool _copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i);
99
100static __always_inline __must_check
101size_t copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
102{
103 if (unlikely(!check_copy_size(addr, bytes, true)))
104 return 0;
105 else
106 return _copy_to_iter(addr, bytes, i);
107}
108
109static __always_inline __must_check
110size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
111{
112 if (unlikely(!check_copy_size(addr, bytes, false)))
113 return 0;
114 else
115 return _copy_from_iter(addr, bytes, i);
116}
117
118static __always_inline __must_check
119bool copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i)
120{
121 if (unlikely(!check_copy_size(addr, bytes, false)))
122 return false;
123 else
124 return _copy_from_iter_full(addr, bytes, i);
125}
126
127static __always_inline __must_check
128size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
129{
130 if (unlikely(!check_copy_size(addr, bytes, false)))
131 return 0;
132 else
133 return _copy_from_iter_nocache(addr, bytes, i);
134}
135
136static __always_inline __must_check
137bool copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i)
138{
139 if (unlikely(!check_copy_size(addr, bytes, false)))
140 return false;
141 else
142 return _copy_from_iter_full_nocache(addr, bytes, i);
143}
144
145#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
146
147
148
149
150
151
152size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i);
153#else
154#define _copy_from_iter_flushcache _copy_from_iter_nocache
155#endif
156
157#ifdef CONFIG_ARCH_HAS_UACCESS_MCSAFE
158size_t _copy_to_iter_mcsafe(const void *addr, size_t bytes, struct iov_iter *i);
159#else
160#define _copy_to_iter_mcsafe _copy_to_iter
161#endif
162
163static __always_inline __must_check
164size_t copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i)
165{
166 if (unlikely(!check_copy_size(addr, bytes, false)))
167 return 0;
168 else
169 return _copy_from_iter_flushcache(addr, bytes, i);
170}
171
172static __always_inline __must_check
173size_t copy_to_iter_mcsafe(void *addr, size_t bytes, struct iov_iter *i)
174{
175 if (unlikely(!check_copy_size(addr, bytes, true)))
176 return 0;
177 else
178 return _copy_to_iter_mcsafe(addr, bytes, i);
179}
180
181size_t iov_iter_zero(size_t bytes, struct iov_iter *);
182unsigned long iov_iter_alignment(const struct iov_iter *i);
183unsigned long iov_iter_gap_alignment(const struct iov_iter *i);
184void iov_iter_init(struct iov_iter *i, int direction, const struct iovec *iov,
185 unsigned long nr_segs, size_t count);
186void iov_iter_kvec(struct iov_iter *i, int direction, const struct kvec *kvec,
187 unsigned long nr_segs, size_t count);
188void iov_iter_bvec(struct iov_iter *i, int direction, const struct bio_vec *bvec,
189 unsigned long nr_segs, size_t count);
190void iov_iter_pipe(struct iov_iter *i, int direction, struct pipe_inode_info *pipe,
191 size_t count);
192ssize_t iov_iter_get_pages(struct iov_iter *i, struct page **pages,
193 size_t maxsize, unsigned maxpages, size_t *start);
194ssize_t iov_iter_get_pages_alloc(struct iov_iter *i, struct page ***pages,
195 size_t maxsize, size_t *start);
196int iov_iter_npages(const struct iov_iter *i, int maxpages);
197
198const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags);
199
200static inline size_t iov_iter_count(const struct iov_iter *i)
201{
202 return i->count;
203}
204
205static inline bool iter_is_iovec(const struct iov_iter *i)
206{
207 return !(i->type & (ITER_BVEC | ITER_KVEC | ITER_PIPE));
208}
209
210
211
212
213
214
215
216#define iov_iter_rw(i) ((0 ? (struct iov_iter *)0 : (i))->type & (READ | WRITE))
217
218
219
220
221
222
223
224static inline void iov_iter_truncate(struct iov_iter *i, u64 count)
225{
226
227
228
229
230
231
232 if (i->count > count)
233 i->count = count;
234}
235
236
237
238
239
240static inline void iov_iter_reexpand(struct iov_iter *i, size_t count)
241{
242 i->count = count;
243}
244size_t csum_and_copy_to_iter(const void *addr, size_t bytes, __wsum *csum, struct iov_iter *i);
245size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum, struct iov_iter *i);
246bool csum_and_copy_from_iter_full(void *addr, size_t bytes, __wsum *csum, struct iov_iter *i);
247
248int import_iovec(int type, const struct iovec __user * uvector,
249 unsigned nr_segs, unsigned fast_segs,
250 struct iovec **iov, struct iov_iter *i);
251
252#ifdef CONFIG_COMPAT
253struct compat_iovec;
254int compat_import_iovec(int type, const struct compat_iovec __user * uvector,
255 unsigned nr_segs, unsigned fast_segs,
256 struct iovec **iov, struct iov_iter *i);
257#endif
258
259int import_single_range(int type, void __user *buf, size_t len,
260 struct iovec *iov, struct iov_iter *i);
261
262int iov_iter_for_each_range(struct iov_iter *i, size_t bytes,
263 int (*f)(struct kvec *vec, void *context),
264 void *context);
265
266#endif
267