1
2
3
4
5#ifndef __LINUX_UIO_H
6#define __LINUX_UIO_H
7
8#include <linux/kernel.h>
9#include <linux/thread_info.h>
10#include <linux/mm_types.h>
11#include <uapi/linux/uio.h>
12
13struct page;
14struct pipe_inode_info;
15
16struct kvec {
17 void *iov_base;
18 size_t iov_len;
19};
20
21enum iter_type {
22
23 ITER_IOVEC,
24 ITER_KVEC,
25 ITER_BVEC,
26 ITER_PIPE,
27 ITER_XARRAY,
28 ITER_DISCARD,
29};
30
31struct iov_iter_state {
32 size_t iov_offset;
33 size_t count;
34 unsigned long nr_segs;
35};
36
37struct iov_iter {
38 u8 iter_type;
39 bool nofault;
40 bool data_source;
41 size_t iov_offset;
42 size_t count;
43 union {
44 const struct iovec *iov;
45 const struct kvec *kvec;
46 const struct bio_vec *bvec;
47 struct xarray *xarray;
48 struct pipe_inode_info *pipe;
49 };
50 union {
51 unsigned long nr_segs;
52 struct {
53 unsigned int head;
54 unsigned int start_head;
55 };
56 loff_t xarray_start;
57 };
58};
59
60static inline enum iter_type iov_iter_type(const struct iov_iter *i)
61{
62 return i->iter_type;
63}
64
65static inline void iov_iter_save_state(struct iov_iter *iter,
66 struct iov_iter_state *state)
67{
68 state->iov_offset = iter->iov_offset;
69 state->count = iter->count;
70 state->nr_segs = iter->nr_segs;
71}
72
73static inline bool iter_is_iovec(const struct iov_iter *i)
74{
75 return iov_iter_type(i) == ITER_IOVEC;
76}
77
78static inline bool iov_iter_is_kvec(const struct iov_iter *i)
79{
80 return iov_iter_type(i) == ITER_KVEC;
81}
82
83static inline bool iov_iter_is_bvec(const struct iov_iter *i)
84{
85 return iov_iter_type(i) == ITER_BVEC;
86}
87
88static inline bool iov_iter_is_pipe(const struct iov_iter *i)
89{
90 return iov_iter_type(i) == ITER_PIPE;
91}
92
93static inline bool iov_iter_is_discard(const struct iov_iter *i)
94{
95 return iov_iter_type(i) == ITER_DISCARD;
96}
97
98static inline bool iov_iter_is_xarray(const struct iov_iter *i)
99{
100 return iov_iter_type(i) == ITER_XARRAY;
101}
102
103static inline unsigned char iov_iter_rw(const struct iov_iter *i)
104{
105 return i->data_source ? WRITE : READ;
106}
107
108
109
110
111
112
113
114
115static inline size_t iov_length(const struct iovec *iov, unsigned long nr_segs)
116{
117 unsigned long seg;
118 size_t ret = 0;
119
120 for (seg = 0; seg < nr_segs; seg++)
121 ret += iov[seg].iov_len;
122 return ret;
123}
124
125static inline struct iovec iov_iter_iovec(const struct iov_iter *iter)
126{
127 return (struct iovec) {
128 .iov_base = iter->iov->iov_base + iter->iov_offset,
129 .iov_len = min(iter->count,
130 iter->iov->iov_len - iter->iov_offset),
131 };
132}
133
134size_t copy_page_from_iter_atomic(struct page *page, unsigned offset,
135 size_t bytes, struct iov_iter *i);
136void iov_iter_advance(struct iov_iter *i, size_t bytes);
137void iov_iter_revert(struct iov_iter *i, size_t bytes);
138size_t fault_in_iov_iter_readable(const struct iov_iter *i, size_t bytes);
139size_t fault_in_iov_iter_writeable(const struct iov_iter *i, size_t bytes);
140size_t iov_iter_single_seg_count(const struct iov_iter *i);
141size_t copy_page_to_iter(struct page *page, size_t offset, size_t bytes,
142 struct iov_iter *i);
143size_t copy_page_from_iter(struct page *page, size_t offset, size_t bytes,
144 struct iov_iter *i);
145
146size_t _copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i);
147size_t _copy_from_iter(void *addr, size_t bytes, struct iov_iter *i);
148size_t _copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i);
149
150static inline size_t copy_folio_to_iter(struct folio *folio, size_t offset,
151 size_t bytes, struct iov_iter *i)
152{
153 return copy_page_to_iter(&folio->page, offset, bytes, i);
154}
155
156static __always_inline __must_check
157size_t copy_to_iter(const void *addr, size_t bytes, struct iov_iter *i)
158{
159 if (unlikely(!check_copy_size(addr, bytes, true)))
160 return 0;
161 else
162 return _copy_to_iter(addr, bytes, i);
163}
164
165static __always_inline __must_check
166size_t copy_from_iter(void *addr, size_t bytes, struct iov_iter *i)
167{
168 if (unlikely(!check_copy_size(addr, bytes, false)))
169 return 0;
170 else
171 return _copy_from_iter(addr, bytes, i);
172}
173
174static __always_inline __must_check
175bool copy_from_iter_full(void *addr, size_t bytes, struct iov_iter *i)
176{
177 size_t copied = copy_from_iter(addr, bytes, i);
178 if (likely(copied == bytes))
179 return true;
180 iov_iter_revert(i, copied);
181 return false;
182}
183
184static __always_inline __must_check
185size_t copy_from_iter_nocache(void *addr, size_t bytes, struct iov_iter *i)
186{
187 if (unlikely(!check_copy_size(addr, bytes, false)))
188 return 0;
189 else
190 return _copy_from_iter_nocache(addr, bytes, i);
191}
192
193static __always_inline __must_check
194bool copy_from_iter_full_nocache(void *addr, size_t bytes, struct iov_iter *i)
195{
196 size_t copied = copy_from_iter_nocache(addr, bytes, i);
197 if (likely(copied == bytes))
198 return true;
199 iov_iter_revert(i, copied);
200 return false;
201}
202
203#ifdef CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE
204
205
206
207
208
209
210size_t _copy_from_iter_flushcache(void *addr, size_t bytes, struct iov_iter *i);
211#else
212#define _copy_from_iter_flushcache _copy_from_iter_nocache
213#endif
214
215#ifdef CONFIG_ARCH_HAS_COPY_MC
216size_t _copy_mc_to_iter(const void *addr, size_t bytes, struct iov_iter *i);
217#else
218#define _copy_mc_to_iter _copy_to_iter
219#endif
220
221size_t iov_iter_zero(size_t bytes, struct iov_iter *);
222unsigned long iov_iter_alignment(const struct iov_iter *i);
223unsigned long iov_iter_gap_alignment(const struct iov_iter *i);
224void iov_iter_init(struct iov_iter *i, unsigned int direction, const struct iovec *iov,
225 unsigned long nr_segs, size_t count);
226void iov_iter_kvec(struct iov_iter *i, unsigned int direction, const struct kvec *kvec,
227 unsigned long nr_segs, size_t count);
228void iov_iter_bvec(struct iov_iter *i, unsigned int direction, const struct bio_vec *bvec,
229 unsigned long nr_segs, size_t count);
230void iov_iter_pipe(struct iov_iter *i, unsigned int direction, struct pipe_inode_info *pipe,
231 size_t count);
232void iov_iter_discard(struct iov_iter *i, unsigned int direction, size_t count);
233void iov_iter_xarray(struct iov_iter *i, unsigned int direction, struct xarray *xarray,
234 loff_t start, size_t count);
235ssize_t iov_iter_get_pages(struct iov_iter *i, struct page **pages,
236 size_t maxsize, unsigned maxpages, size_t *start);
237ssize_t iov_iter_get_pages_alloc(struct iov_iter *i, struct page ***pages,
238 size_t maxsize, size_t *start);
239int iov_iter_npages(const struct iov_iter *i, int maxpages);
240void iov_iter_restore(struct iov_iter *i, struct iov_iter_state *state);
241
242const void *dup_iter(struct iov_iter *new, struct iov_iter *old, gfp_t flags);
243
244static inline size_t iov_iter_count(const struct iov_iter *i)
245{
246 return i->count;
247}
248
249
250
251
252
253
254
255static inline void iov_iter_truncate(struct iov_iter *i, u64 count)
256{
257
258
259
260
261
262
263 if (i->count > count)
264 i->count = count;
265}
266
267
268
269
270
271static inline void iov_iter_reexpand(struct iov_iter *i, size_t count)
272{
273 i->count = count;
274}
275
276static inline int
277iov_iter_npages_cap(struct iov_iter *i, int maxpages, size_t max_bytes)
278{
279 size_t shorted = 0;
280 int npages;
281
282 if (iov_iter_count(i) > max_bytes) {
283 shorted = iov_iter_count(i) - max_bytes;
284 iov_iter_truncate(i, max_bytes);
285 }
286 npages = iov_iter_npages(i, INT_MAX);
287 if (shorted)
288 iov_iter_reexpand(i, iov_iter_count(i) + shorted);
289
290 return npages;
291}
292
293struct csum_state {
294 __wsum csum;
295 size_t off;
296};
297
298size_t csum_and_copy_to_iter(const void *addr, size_t bytes, void *csstate, struct iov_iter *i);
299size_t csum_and_copy_from_iter(void *addr, size_t bytes, __wsum *csum, struct iov_iter *i);
300
301static __always_inline __must_check
302bool csum_and_copy_from_iter_full(void *addr, size_t bytes,
303 __wsum *csum, struct iov_iter *i)
304{
305 size_t copied = csum_and_copy_from_iter(addr, bytes, csum, i);
306 if (likely(copied == bytes))
307 return true;
308 iov_iter_revert(i, copied);
309 return false;
310}
311size_t hash_and_copy_to_iter(const void *addr, size_t bytes, void *hashp,
312 struct iov_iter *i);
313
314struct iovec *iovec_from_user(const struct iovec __user *uvector,
315 unsigned long nr_segs, unsigned long fast_segs,
316 struct iovec *fast_iov, bool compat);
317ssize_t import_iovec(int type, const struct iovec __user *uvec,
318 unsigned nr_segs, unsigned fast_segs, struct iovec **iovp,
319 struct iov_iter *i);
320ssize_t __import_iovec(int type, const struct iovec __user *uvec,
321 unsigned nr_segs, unsigned fast_segs, struct iovec **iovp,
322 struct iov_iter *i, bool compat);
323int import_single_range(int type, void __user *buf, size_t len,
324 struct iovec *iov, struct iov_iter *i);
325
326#endif
327