1
2
3
4
5
6#ifndef __LINUX_BLK_TYPES_H
7#define __LINUX_BLK_TYPES_H
8
9#include <linux/types.h>
10#include <linux/bvec.h>
11#include <linux/ktime.h>
12
13struct bio_set;
14struct bio;
15struct bio_integrity_payload;
16struct page;
17struct block_device;
18struct io_context;
19struct cgroup_subsys_state;
20typedef void (bio_end_io_t) (struct bio *);
21
22
23
24
25
26#if defined(CONFIG_ALPHA) && !defined(__alpha_bwx__)
27typedef u32 __bitwise blk_status_t;
28#else
29typedef u8 __bitwise blk_status_t;
30#endif
31#define BLK_STS_OK 0
32#define BLK_STS_NOTSUPP ((__force blk_status_t)1)
33#define BLK_STS_TIMEOUT ((__force blk_status_t)2)
34#define BLK_STS_NOSPC ((__force blk_status_t)3)
35#define BLK_STS_TRANSPORT ((__force blk_status_t)4)
36#define BLK_STS_TARGET ((__force blk_status_t)5)
37#define BLK_STS_NEXUS ((__force blk_status_t)6)
38#define BLK_STS_MEDIUM ((__force blk_status_t)7)
39#define BLK_STS_PROTECTION ((__force blk_status_t)8)
40#define BLK_STS_RESOURCE ((__force blk_status_t)9)
41#define BLK_STS_IOERR ((__force blk_status_t)10)
42
43
44#define BLK_STS_DM_REQUEUE ((__force blk_status_t)11)
45
46#define BLK_STS_AGAIN ((__force blk_status_t)12)
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64#define BLK_STS_DEV_RESOURCE ((__force blk_status_t)13)
65
66
67
68
69
70
71
72
73
74
75
76
77
78static inline bool blk_path_error(blk_status_t error)
79{
80 switch (error) {
81 case BLK_STS_NOTSUPP:
82 case BLK_STS_NOSPC:
83 case BLK_STS_TARGET:
84 case BLK_STS_NEXUS:
85 case BLK_STS_MEDIUM:
86 case BLK_STS_PROTECTION:
87 return false;
88 }
89
90
91 return true;
92}
93
94
95
96
97
98
99
100#define BIO_ISSUE_RES_BITS 1
101#define BIO_ISSUE_SIZE_BITS 12
102#define BIO_ISSUE_RES_SHIFT (64 - BIO_ISSUE_RES_BITS)
103#define BIO_ISSUE_SIZE_SHIFT (BIO_ISSUE_RES_SHIFT - BIO_ISSUE_SIZE_BITS)
104#define BIO_ISSUE_TIME_MASK ((1ULL << BIO_ISSUE_SIZE_SHIFT) - 1)
105#define BIO_ISSUE_SIZE_MASK \
106 (((1ULL << BIO_ISSUE_SIZE_BITS) - 1) << BIO_ISSUE_SIZE_SHIFT)
107#define BIO_ISSUE_RES_MASK (~((1ULL << BIO_ISSUE_RES_SHIFT) - 1))
108
109
110#define BIO_ISSUE_THROTL_SKIP_LATENCY (1ULL << 63)
111
112struct bio_issue {
113 u64 value;
114};
115
116static inline u64 __bio_issue_time(u64 time)
117{
118 return time & BIO_ISSUE_TIME_MASK;
119}
120
121static inline u64 bio_issue_time(struct bio_issue *issue)
122{
123 return __bio_issue_time(issue->value);
124}
125
126static inline sector_t bio_issue_size(struct bio_issue *issue)
127{
128 return ((issue->value & BIO_ISSUE_SIZE_MASK) >> BIO_ISSUE_SIZE_SHIFT);
129}
130
131static inline void bio_issue_init(struct bio_issue *issue,
132 sector_t size)
133{
134 size &= (1ULL << BIO_ISSUE_SIZE_BITS) - 1;
135 issue->value = ((issue->value & BIO_ISSUE_RES_MASK) |
136 (ktime_get_ns() & BIO_ISSUE_TIME_MASK) |
137 ((u64)size << BIO_ISSUE_SIZE_SHIFT));
138}
139
140
141
142
143
144struct bio {
145 struct bio *bi_next;
146 struct gendisk *bi_disk;
147 unsigned int bi_opf;
148
149
150
151 unsigned short bi_flags;
152 unsigned short bi_ioprio;
153 unsigned short bi_write_hint;
154 blk_status_t bi_status;
155 u8 bi_partno;
156
157 struct bvec_iter bi_iter;
158
159 atomic_t __bi_remaining;
160 bio_end_io_t *bi_end_io;
161
162 void *bi_private;
163#ifdef CONFIG_BLK_CGROUP
164
165
166
167
168
169
170 struct blkcg_gq *bi_blkg;
171 struct bio_issue bi_issue;
172#endif
173 union {
174#if defined(CONFIG_BLK_DEV_INTEGRITY)
175 struct bio_integrity_payload *bi_integrity;
176#endif
177 };
178
179 unsigned short bi_vcnt;
180
181
182
183
184
185 unsigned short bi_max_vecs;
186
187 atomic_t __bi_cnt;
188
189 struct bio_vec *bi_io_vec;
190
191 struct bio_set *bi_pool;
192
193
194
195
196
197
198 struct bio_vec bi_inline_vecs[0];
199};
200
201#define BIO_RESET_BYTES offsetof(struct bio, bi_max_vecs)
202
203
204
205
206enum {
207 BIO_NO_PAGE_REF,
208 BIO_CLONED,
209 BIO_BOUNCED,
210 BIO_USER_MAPPED,
211 BIO_NULL_MAPPED,
212 BIO_QUIET,
213 BIO_CHAIN,
214 BIO_REFFED,
215 BIO_THROTTLED,
216
217 BIO_TRACE_COMPLETION,
218
219 BIO_QUEUE_ENTERED,
220 BIO_TRACKED,
221 BIO_FLAG_LAST
222};
223
224
225
226
227
228
229
230#define BVEC_POOL_NR 6
231#define BVEC_POOL_MAX (BVEC_POOL_NR - 1)
232
233
234
235
236
237
238#define BVEC_POOL_BITS (3)
239#define BVEC_POOL_OFFSET (16 - BVEC_POOL_BITS)
240#define BVEC_POOL_IDX(bio) ((bio)->bi_flags >> BVEC_POOL_OFFSET)
241#if (1<< BVEC_POOL_BITS) < (BVEC_POOL_NR+1)
242# error "BVEC_POOL_BITS is too small"
243#endif
244
245
246
247
248
249#define BIO_RESET_BITS BVEC_POOL_OFFSET
250
251typedef __u32 __bitwise blk_mq_req_flags_t;
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266#define REQ_OP_BITS 8
267#define REQ_OP_MASK ((1 << REQ_OP_BITS) - 1)
268#define REQ_FLAG_BITS 24
269
270enum req_opf {
271
272 REQ_OP_READ = 0,
273
274 REQ_OP_WRITE = 1,
275
276 REQ_OP_FLUSH = 2,
277
278 REQ_OP_DISCARD = 3,
279
280 REQ_OP_SECURE_ERASE = 5,
281
282 REQ_OP_ZONE_RESET = 6,
283
284 REQ_OP_WRITE_SAME = 7,
285
286 REQ_OP_WRITE_ZEROES = 9,
287
288
289 REQ_OP_SCSI_IN = 32,
290 REQ_OP_SCSI_OUT = 33,
291
292 REQ_OP_DRV_IN = 34,
293 REQ_OP_DRV_OUT = 35,
294
295 REQ_OP_LAST,
296};
297
298enum req_flag_bits {
299 __REQ_FAILFAST_DEV =
300 REQ_OP_BITS,
301 __REQ_FAILFAST_TRANSPORT,
302 __REQ_FAILFAST_DRIVER,
303 __REQ_SYNC,
304 __REQ_META,
305 __REQ_PRIO,
306 __REQ_NOMERGE,
307 __REQ_IDLE,
308 __REQ_INTEGRITY,
309 __REQ_FUA,
310 __REQ_PREFLUSH,
311 __REQ_RAHEAD,
312 __REQ_BACKGROUND,
313 __REQ_NOWAIT,
314
315
316
317
318
319
320
321 __REQ_CGROUP_PUNT,
322
323
324 __REQ_NOUNMAP,
325
326 __REQ_HIPRI,
327
328
329 __REQ_DRV,
330 __REQ_SWAP,
331 __REQ_NR_BITS,
332};
333
334#define REQ_FAILFAST_DEV (1ULL << __REQ_FAILFAST_DEV)
335#define REQ_FAILFAST_TRANSPORT (1ULL << __REQ_FAILFAST_TRANSPORT)
336#define REQ_FAILFAST_DRIVER (1ULL << __REQ_FAILFAST_DRIVER)
337#define REQ_SYNC (1ULL << __REQ_SYNC)
338#define REQ_META (1ULL << __REQ_META)
339#define REQ_PRIO (1ULL << __REQ_PRIO)
340#define REQ_NOMERGE (1ULL << __REQ_NOMERGE)
341#define REQ_IDLE (1ULL << __REQ_IDLE)
342#define REQ_INTEGRITY (1ULL << __REQ_INTEGRITY)
343#define REQ_FUA (1ULL << __REQ_FUA)
344#define REQ_PREFLUSH (1ULL << __REQ_PREFLUSH)
345#define REQ_RAHEAD (1ULL << __REQ_RAHEAD)
346#define REQ_BACKGROUND (1ULL << __REQ_BACKGROUND)
347#define REQ_NOWAIT (1ULL << __REQ_NOWAIT)
348#define REQ_CGROUP_PUNT (1ULL << __REQ_CGROUP_PUNT)
349
350#define REQ_NOUNMAP (1ULL << __REQ_NOUNMAP)
351#define REQ_HIPRI (1ULL << __REQ_HIPRI)
352
353#define REQ_DRV (1ULL << __REQ_DRV)
354#define REQ_SWAP (1ULL << __REQ_SWAP)
355
356#define REQ_FAILFAST_MASK \
357 (REQ_FAILFAST_DEV | REQ_FAILFAST_TRANSPORT | REQ_FAILFAST_DRIVER)
358
359#define REQ_NOMERGE_FLAGS \
360 (REQ_NOMERGE | REQ_PREFLUSH | REQ_FUA)
361
362enum stat_group {
363 STAT_READ,
364 STAT_WRITE,
365 STAT_DISCARD,
366
367 NR_STAT_GROUPS
368};
369
370#define bio_op(bio) \
371 ((bio)->bi_opf & REQ_OP_MASK)
372#define req_op(req) \
373 ((req)->cmd_flags & REQ_OP_MASK)
374
375
376static inline void bio_set_op_attrs(struct bio *bio, unsigned op,
377 unsigned op_flags)
378{
379 bio->bi_opf = op | op_flags;
380}
381
382static inline bool op_is_write(unsigned int op)
383{
384 return (op & 1);
385}
386
387
388
389
390
391static inline bool op_is_flush(unsigned int op)
392{
393 return op & (REQ_FUA | REQ_PREFLUSH);
394}
395
396
397
398
399
400
401static inline bool op_is_sync(unsigned int op)
402{
403 return (op & REQ_OP_MASK) == REQ_OP_READ ||
404 (op & (REQ_SYNC | REQ_FUA | REQ_PREFLUSH));
405}
406
407static inline bool op_is_discard(unsigned int op)
408{
409 return (op & REQ_OP_MASK) == REQ_OP_DISCARD;
410}
411
412static inline int op_stat_group(unsigned int op)
413{
414 if (op_is_discard(op))
415 return STAT_DISCARD;
416 return op_is_write(op);
417}
418
419typedef unsigned int blk_qc_t;
420#define BLK_QC_T_NONE -1U
421#define BLK_QC_T_SHIFT 16
422#define BLK_QC_T_INTERNAL (1U << 31)
423
424static inline bool blk_qc_t_valid(blk_qc_t cookie)
425{
426 return cookie != BLK_QC_T_NONE;
427}
428
429static inline unsigned int blk_qc_t_to_queue_num(blk_qc_t cookie)
430{
431 return (cookie & ~BLK_QC_T_INTERNAL) >> BLK_QC_T_SHIFT;
432}
433
434static inline unsigned int blk_qc_t_to_tag(blk_qc_t cookie)
435{
436 return cookie & ((1u << BLK_QC_T_SHIFT) - 1);
437}
438
439static inline bool blk_qc_t_is_internal(blk_qc_t cookie)
440{
441 return (cookie & BLK_QC_T_INTERNAL) != 0;
442}
443
444struct blk_rq_stat {
445 u64 mean;
446 u64 min;
447 u64 max;
448 u32 nr_samples;
449 u64 batch;
450};
451
452#endif
453