1
2
3
4#include <linux/kernel.h>
5#include <linux/module.h>
6#include <linux/bio.h>
7#include <linux/blkdev.h>
8#include <linux/scatterlist.h>
9
10#include "blk.h"
11
12struct bio_batch {
13 atomic_t done;
14 int error;
15 struct completion *wait;
16};
17
18static void bio_batch_end_io(struct bio *bio)
19{
20 struct bio_batch *bb = bio->bi_private;
21
22 if (bio->bi_error && bio->bi_error != -EOPNOTSUPP)
23 bb->error = bio->bi_error;
24 if (atomic_dec_and_test(&bb->done))
25 complete(bb->wait);
26 bio_put(bio);
27}
28
29
30
31
32
33
34
35
36
37
38
39
40int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
41 sector_t nr_sects, gfp_t gfp_mask, unsigned long flags)
42{
43 DECLARE_COMPLETION_ONSTACK(wait);
44 struct request_queue *q = bdev_get_queue(bdev);
45 int type = REQ_WRITE | REQ_DISCARD;
46 unsigned int granularity;
47 int alignment;
48 struct bio_batch bb;
49 struct bio *bio;
50 int ret = 0;
51 struct blk_plug plug;
52
53 if (!q)
54 return -ENXIO;
55
56 if (!blk_queue_discard(q))
57 return -EOPNOTSUPP;
58
59
60 granularity = max(q->limits.discard_granularity >> 9, 1U);
61 alignment = (bdev_discard_alignment(bdev) >> 9) % granularity;
62
63 if (flags & BLKDEV_DISCARD_SECURE) {
64 if (!blk_queue_secdiscard(q))
65 return -EOPNOTSUPP;
66 type |= REQ_SECURE;
67 }
68
69 atomic_set(&bb.done, 1);
70 bb.error = 0;
71 bb.wait = &wait;
72
73 blk_start_plug(&plug);
74 while (nr_sects) {
75 unsigned int req_sects;
76 sector_t end_sect, tmp;
77
78 bio = bio_alloc(gfp_mask, 1);
79 if (!bio) {
80 ret = -ENOMEM;
81 break;
82 }
83
84
85 req_sects = min_t(sector_t, nr_sects, UINT_MAX >> 9);
86
87
88
89
90
91 end_sect = sector + req_sects;
92 tmp = end_sect;
93 if (req_sects < nr_sects &&
94 sector_div(tmp, granularity) != alignment) {
95 end_sect = end_sect - alignment;
96 sector_div(end_sect, granularity);
97 end_sect = end_sect * granularity + alignment;
98 req_sects = end_sect - sector;
99 }
100
101 bio->bi_iter.bi_sector = sector;
102 bio->bi_end_io = bio_batch_end_io;
103 bio->bi_bdev = bdev;
104 bio->bi_private = &bb;
105
106 bio->bi_iter.bi_size = req_sects << 9;
107 nr_sects -= req_sects;
108 sector = end_sect;
109
110 atomic_inc(&bb.done);
111 submit_bio(type, bio);
112
113
114
115
116
117
118
119 cond_resched();
120 }
121 blk_finish_plug(&plug);
122
123
124 if (!atomic_dec_and_test(&bb.done))
125 wait_for_completion_io(&wait);
126
127 if (bb.error)
128 return bb.error;
129 return ret;
130}
131EXPORT_SYMBOL(blkdev_issue_discard);
132
133
134
135
136
137
138
139
140
141
142
143
144int blkdev_issue_write_same(struct block_device *bdev, sector_t sector,
145 sector_t nr_sects, gfp_t gfp_mask,
146 struct page *page)
147{
148 DECLARE_COMPLETION_ONSTACK(wait);
149 struct request_queue *q = bdev_get_queue(bdev);
150 unsigned int max_write_same_sectors;
151 struct bio_batch bb;
152 struct bio *bio;
153 int ret = 0;
154
155 if (!q)
156 return -ENXIO;
157
158
159 max_write_same_sectors = UINT_MAX >> 9;
160
161 atomic_set(&bb.done, 1);
162 bb.error = 0;
163 bb.wait = &wait;
164
165 while (nr_sects) {
166 bio = bio_alloc(gfp_mask, 1);
167 if (!bio) {
168 ret = -ENOMEM;
169 break;
170 }
171
172 bio->bi_iter.bi_sector = sector;
173 bio->bi_end_io = bio_batch_end_io;
174 bio->bi_bdev = bdev;
175 bio->bi_private = &bb;
176 bio->bi_vcnt = 1;
177 bio->bi_io_vec->bv_page = page;
178 bio->bi_io_vec->bv_offset = 0;
179 bio->bi_io_vec->bv_len = bdev_logical_block_size(bdev);
180
181 if (nr_sects > max_write_same_sectors) {
182 bio->bi_iter.bi_size = max_write_same_sectors << 9;
183 nr_sects -= max_write_same_sectors;
184 sector += max_write_same_sectors;
185 } else {
186 bio->bi_iter.bi_size = nr_sects << 9;
187 nr_sects = 0;
188 }
189
190 atomic_inc(&bb.done);
191 submit_bio(REQ_WRITE | REQ_WRITE_SAME, bio);
192 }
193
194
195 if (!atomic_dec_and_test(&bb.done))
196 wait_for_completion_io(&wait);
197
198 if (bb.error)
199 return bb.error;
200 return ret;
201}
202EXPORT_SYMBOL(blkdev_issue_write_same);
203
204
205
206
207
208
209
210
211
212
213
214
215static int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
216 sector_t nr_sects, gfp_t gfp_mask)
217{
218 int ret;
219 struct bio *bio;
220 struct bio_batch bb;
221 unsigned int sz;
222 DECLARE_COMPLETION_ONSTACK(wait);
223
224 atomic_set(&bb.done, 1);
225 bb.error = 0;
226 bb.wait = &wait;
227
228 ret = 0;
229 while (nr_sects != 0) {
230 bio = bio_alloc(gfp_mask,
231 min(nr_sects, (sector_t)BIO_MAX_PAGES));
232 if (!bio) {
233 ret = -ENOMEM;
234 break;
235 }
236
237 bio->bi_iter.bi_sector = sector;
238 bio->bi_bdev = bdev;
239 bio->bi_end_io = bio_batch_end_io;
240 bio->bi_private = &bb;
241
242 while (nr_sects != 0) {
243 sz = min((sector_t) PAGE_SIZE >> 9 , nr_sects);
244 ret = bio_add_page(bio, ZERO_PAGE(0), sz << 9, 0);
245 nr_sects -= ret >> 9;
246 sector += ret >> 9;
247 if (ret < (sz << 9))
248 break;
249 }
250 ret = 0;
251 atomic_inc(&bb.done);
252 submit_bio(WRITE, bio);
253 }
254
255
256 if (!atomic_dec_and_test(&bb.done))
257 wait_for_completion_io(&wait);
258
259 if (bb.error)
260 return bb.error;
261 return ret;
262}
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
286 sector_t nr_sects, gfp_t gfp_mask, bool discard)
287{
288 struct request_queue *q = bdev_get_queue(bdev);
289
290 if (discard && blk_queue_discard(q) && q->limits.discard_zeroes_data &&
291 blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, 0) == 0)
292 return 0;
293
294 if (bdev_write_same(bdev) &&
295 blkdev_issue_write_same(bdev, sector, nr_sects, gfp_mask,
296 ZERO_PAGE(0)) == 0)
297 return 0;
298
299 return __blkdev_issue_zeroout(bdev, sector, nr_sects, gfp_mask);
300}
301EXPORT_SYMBOL(blkdev_issue_zeroout);
302