1
2
3
4
5
6
7
8
9#include <linux/blkdev.h>
10#include <linux/mempool.h>
11#include <linux/export.h>
12#include <linux/bio.h>
13#include <linux/workqueue.h>
14#include <linux/slab.h>
15#include "blk.h"
16
17static struct kmem_cache *bip_slab;
18static struct workqueue_struct *kintegrityd_wq;
19
20void blk_flush_integrity(void)
21{
22 flush_workqueue(kintegrityd_wq);
23}
24
25static void __bio_integrity_free(struct bio_set *bs,
26 struct bio_integrity_payload *bip)
27{
28 if (bs && mempool_initialized(&bs->bio_integrity_pool)) {
29 if (bip->bip_vec)
30 bvec_free(&bs->bvec_integrity_pool, bip->bip_vec,
31 bip->bip_max_vcnt);
32 mempool_free(bip, &bs->bio_integrity_pool);
33 } else {
34 kfree(bip);
35 }
36}
37
38
39
40
41
42
43
44
45
46
47
48struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio,
49 gfp_t gfp_mask,
50 unsigned int nr_vecs)
51{
52 struct bio_integrity_payload *bip;
53 struct bio_set *bs = bio->bi_pool;
54 unsigned inline_vecs;
55
56 if (WARN_ON_ONCE(bio_has_crypt_ctx(bio)))
57 return ERR_PTR(-EOPNOTSUPP);
58
59 if (!bs || !mempool_initialized(&bs->bio_integrity_pool)) {
60 bip = kmalloc(struct_size(bip, bip_inline_vecs, nr_vecs), gfp_mask);
61 inline_vecs = nr_vecs;
62 } else {
63 bip = mempool_alloc(&bs->bio_integrity_pool, gfp_mask);
64 inline_vecs = BIO_INLINE_VECS;
65 }
66
67 if (unlikely(!bip))
68 return ERR_PTR(-ENOMEM);
69
70 memset(bip, 0, sizeof(*bip));
71
72 if (nr_vecs > inline_vecs) {
73 bip->bip_max_vcnt = nr_vecs;
74 bip->bip_vec = bvec_alloc(&bs->bvec_integrity_pool,
75 &bip->bip_max_vcnt, gfp_mask);
76 if (!bip->bip_vec)
77 goto err;
78 } else {
79 bip->bip_vec = bip->bip_inline_vecs;
80 bip->bip_max_vcnt = inline_vecs;
81 }
82
83 bip->bip_bio = bio;
84 bio->bi_integrity = bip;
85 bio->bi_opf |= REQ_INTEGRITY;
86
87 return bip;
88err:
89 __bio_integrity_free(bs, bip);
90 return ERR_PTR(-ENOMEM);
91}
92EXPORT_SYMBOL(bio_integrity_alloc);
93
94
95
96
97
98
99
100
101void bio_integrity_free(struct bio *bio)
102{
103 struct bio_integrity_payload *bip = bio_integrity(bio);
104 struct bio_set *bs = bio->bi_pool;
105
106 if (bip->bip_flags & BIP_BLOCK_INTEGRITY)
107 kfree(page_address(bip->bip_vec->bv_page) +
108 bip->bip_vec->bv_offset);
109
110 __bio_integrity_free(bs, bip);
111 bio->bi_integrity = NULL;
112 bio->bi_opf &= ~REQ_INTEGRITY;
113}
114
115
116
117
118
119
120
121
122
123
124int bio_integrity_add_page(struct bio *bio, struct page *page,
125 unsigned int len, unsigned int offset)
126{
127 struct bio_integrity_payload *bip = bio_integrity(bio);
128 struct bio_vec *iv;
129
130 if (bip->bip_vcnt >= bip->bip_max_vcnt) {
131 printk(KERN_ERR "%s: bip_vec full\n", __func__);
132 return 0;
133 }
134
135 iv = bip->bip_vec + bip->bip_vcnt;
136
137 if (bip->bip_vcnt &&
138 bvec_gap_to_prev(bio->bi_bdev->bd_disk->queue,
139 &bip->bip_vec[bip->bip_vcnt - 1], offset))
140 return 0;
141
142 iv->bv_page = page;
143 iv->bv_len = len;
144 iv->bv_offset = offset;
145 bip->bip_vcnt++;
146
147 return len;
148}
149EXPORT_SYMBOL(bio_integrity_add_page);
150
151
152
153
154
155
156
157static blk_status_t bio_integrity_process(struct bio *bio,
158 struct bvec_iter *proc_iter, integrity_processing_fn *proc_fn)
159{
160 struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
161 struct blk_integrity_iter iter;
162 struct bvec_iter bviter;
163 struct bio_vec bv;
164 struct bio_integrity_payload *bip = bio_integrity(bio);
165 blk_status_t ret = BLK_STS_OK;
166 void *prot_buf = page_address(bip->bip_vec->bv_page) +
167 bip->bip_vec->bv_offset;
168
169 iter.disk_name = bio->bi_bdev->bd_disk->disk_name;
170 iter.interval = 1 << bi->interval_exp;
171 iter.seed = proc_iter->bi_sector;
172 iter.prot_buf = prot_buf;
173
174 __bio_for_each_segment(bv, bio, bviter, *proc_iter) {
175 void *kaddr = kmap_atomic(bv.bv_page);
176
177 iter.data_buf = kaddr + bv.bv_offset;
178 iter.data_size = bv.bv_len;
179
180 ret = proc_fn(&iter);
181 if (ret) {
182 kunmap_atomic(kaddr);
183 return ret;
184 }
185
186 kunmap_atomic(kaddr);
187 }
188 return ret;
189}
190
191
192
193
194
195
196
197
198
199
200
201
202
203bool bio_integrity_prep(struct bio *bio)
204{
205 struct bio_integrity_payload *bip;
206 struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
207 void *buf;
208 unsigned long start, end;
209 unsigned int len, nr_pages;
210 unsigned int bytes, offset, i;
211 unsigned int intervals;
212 blk_status_t status;
213
214 if (!bi)
215 return true;
216
217 if (bio_op(bio) != REQ_OP_READ && bio_op(bio) != REQ_OP_WRITE)
218 return true;
219
220 if (!bio_sectors(bio))
221 return true;
222
223
224 if (bio_integrity(bio))
225 return true;
226
227 if (bio_data_dir(bio) == READ) {
228 if (!bi->profile->verify_fn ||
229 !(bi->flags & BLK_INTEGRITY_VERIFY))
230 return true;
231 } else {
232 if (!bi->profile->generate_fn ||
233 !(bi->flags & BLK_INTEGRITY_GENERATE))
234 return true;
235 }
236 intervals = bio_integrity_intervals(bi, bio_sectors(bio));
237
238
239 len = intervals * bi->tuple_size;
240 buf = kmalloc(len, GFP_NOIO);
241 status = BLK_STS_RESOURCE;
242 if (unlikely(buf == NULL)) {
243 printk(KERN_ERR "could not allocate integrity buffer\n");
244 goto err_end_io;
245 }
246
247 end = (((unsigned long) buf) + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
248 start = ((unsigned long) buf) >> PAGE_SHIFT;
249 nr_pages = end - start;
250
251
252 bip = bio_integrity_alloc(bio, GFP_NOIO, nr_pages);
253 if (IS_ERR(bip)) {
254 printk(KERN_ERR "could not allocate data integrity bioset\n");
255 kfree(buf);
256 status = BLK_STS_RESOURCE;
257 goto err_end_io;
258 }
259
260 bip->bip_flags |= BIP_BLOCK_INTEGRITY;
261 bip->bip_iter.bi_size = len;
262 bip_set_seed(bip, bio->bi_iter.bi_sector);
263
264 if (bi->flags & BLK_INTEGRITY_IP_CHECKSUM)
265 bip->bip_flags |= BIP_IP_CHECKSUM;
266
267
268 offset = offset_in_page(buf);
269 for (i = 0 ; i < nr_pages ; i++) {
270 int ret;
271 bytes = PAGE_SIZE - offset;
272
273 if (len <= 0)
274 break;
275
276 if (bytes > len)
277 bytes = len;
278
279 ret = bio_integrity_add_page(bio, virt_to_page(buf),
280 bytes, offset);
281
282 if (ret == 0) {
283 printk(KERN_ERR "could not attach integrity payload\n");
284 status = BLK_STS_RESOURCE;
285 goto err_end_io;
286 }
287
288 if (ret < bytes)
289 break;
290
291 buf += bytes;
292 len -= bytes;
293 offset = 0;
294 }
295
296
297 if (bio_data_dir(bio) == WRITE) {
298 bio_integrity_process(bio, &bio->bi_iter,
299 bi->profile->generate_fn);
300 } else {
301 bip->bio_iter = bio->bi_iter;
302 }
303 return true;
304
305err_end_io:
306 bio->bi_status = status;
307 bio_endio(bio);
308 return false;
309
310}
311EXPORT_SYMBOL(bio_integrity_prep);
312
313
314
315
316
317
318
319
320
321static void bio_integrity_verify_fn(struct work_struct *work)
322{
323 struct bio_integrity_payload *bip =
324 container_of(work, struct bio_integrity_payload, bip_work);
325 struct bio *bio = bip->bip_bio;
326 struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
327
328
329
330
331
332
333 bio->bi_status = bio_integrity_process(bio, &bip->bio_iter,
334 bi->profile->verify_fn);
335 bio_integrity_free(bio);
336 bio_endio(bio);
337}
338
339
340
341
342
343
344
345
346
347
348
349
350bool __bio_integrity_endio(struct bio *bio)
351{
352 struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
353 struct bio_integrity_payload *bip = bio_integrity(bio);
354
355 if (bio_op(bio) == REQ_OP_READ && !bio->bi_status &&
356 (bip->bip_flags & BIP_BLOCK_INTEGRITY) && bi->profile->verify_fn) {
357 INIT_WORK(&bip->bip_work, bio_integrity_verify_fn);
358 queue_work(kintegrityd_wq, &bip->bip_work);
359 return false;
360 }
361
362 bio_integrity_free(bio);
363 return true;
364}
365
366
367
368
369
370
371
372
373
374
375void bio_integrity_advance(struct bio *bio, unsigned int bytes_done)
376{
377 struct bio_integrity_payload *bip = bio_integrity(bio);
378 struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
379 unsigned bytes = bio_integrity_bytes(bi, bytes_done >> 9);
380
381 bip->bip_iter.bi_sector += bytes_done >> 9;
382 bvec_iter_advance(bip->bip_vec, &bip->bip_iter, bytes);
383}
384
385
386
387
388
389
390
391void bio_integrity_trim(struct bio *bio)
392{
393 struct bio_integrity_payload *bip = bio_integrity(bio);
394 struct blk_integrity *bi = blk_get_integrity(bio->bi_bdev->bd_disk);
395
396 bip->bip_iter.bi_size = bio_integrity_bytes(bi, bio_sectors(bio));
397}
398EXPORT_SYMBOL(bio_integrity_trim);
399
400
401
402
403
404
405
406
407
408int bio_integrity_clone(struct bio *bio, struct bio *bio_src,
409 gfp_t gfp_mask)
410{
411 struct bio_integrity_payload *bip_src = bio_integrity(bio_src);
412 struct bio_integrity_payload *bip;
413
414 BUG_ON(bip_src == NULL);
415
416 bip = bio_integrity_alloc(bio, gfp_mask, bip_src->bip_vcnt);
417 if (IS_ERR(bip))
418 return PTR_ERR(bip);
419
420 memcpy(bip->bip_vec, bip_src->bip_vec,
421 bip_src->bip_vcnt * sizeof(struct bio_vec));
422
423 bip->bip_vcnt = bip_src->bip_vcnt;
424 bip->bip_iter = bip_src->bip_iter;
425
426 return 0;
427}
428EXPORT_SYMBOL(bio_integrity_clone);
429
430int bioset_integrity_create(struct bio_set *bs, int pool_size)
431{
432 if (mempool_initialized(&bs->bio_integrity_pool))
433 return 0;
434
435 if (mempool_init_slab_pool(&bs->bio_integrity_pool,
436 pool_size, bip_slab))
437 return -1;
438
439 if (biovec_init_pool(&bs->bvec_integrity_pool, pool_size)) {
440 mempool_exit(&bs->bio_integrity_pool);
441 return -1;
442 }
443
444 return 0;
445}
446EXPORT_SYMBOL(bioset_integrity_create);
447
448void bioset_integrity_free(struct bio_set *bs)
449{
450 mempool_exit(&bs->bio_integrity_pool);
451 mempool_exit(&bs->bvec_integrity_pool);
452}
453
454void __init bio_integrity_init(void)
455{
456
457
458
459
460 kintegrityd_wq = alloc_workqueue("kintegrityd", WQ_MEM_RECLAIM |
461 WQ_HIGHPRI | WQ_CPU_INTENSIVE, 1);
462 if (!kintegrityd_wq)
463 panic("Failed to create kintegrityd\n");
464
465 bip_slab = kmem_cache_create("bio_integrity_payload",
466 sizeof(struct bio_integrity_payload) +
467 sizeof(struct bio_vec) * BIO_INLINE_VECS,
468 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
469}
470