1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62#include <linux/kernel.h>
63#include <linux/device.h>
64#include <linux/module.h>
65#include <linux/fs.h>
66#include <scsi/osd_initiator.h>
67#include <scsi/osd_attributes.h>
68#include <scsi/osd_sec.h>
69#include <scsi/scsi_device.h>
70
71#define DRV_NAME "osdblk"
72#define PFX DRV_NAME ": "
73
74
75#ifdef _OSDBLK_DEBUG
76#define OSDBLK_DEBUG(fmt, a...) \
77 printk(KERN_NOTICE "osdblk @%s:%d: " fmt, __func__, __LINE__, ##a)
78#else
79#define OSDBLK_DEBUG(fmt, a...) \
80 do { if (0) printk(fmt, ##a); } while (0)
81#endif
82
83MODULE_AUTHOR("Jeff Garzik <jeff@garzik.org>");
84MODULE_DESCRIPTION("block device inside an OSD object osdblk.ko");
85MODULE_LICENSE("GPL");
86
87struct osdblk_device;
88
89enum {
90 OSDBLK_MINORS_PER_MAJOR = 256,
91 OSDBLK_MAX_REQ = 32,
92 OSDBLK_OP_TIMEOUT = 4 * 60,
93};
94
95struct osdblk_request {
96 struct request *rq;
97 struct bio *bio;
98 struct osdblk_device *osdev;
99};
100
101struct osdblk_device {
102 int id;
103
104 int major;
105 struct gendisk *disk;
106 struct request_queue *q;
107
108 struct osd_dev *osd;
109
110 char name[32];
111
112 spinlock_t lock;
113
114 struct osd_obj_id obj;
115 uint8_t obj_cred[OSD_CAP_LEN];
116
117 struct osdblk_request req[OSDBLK_MAX_REQ];
118
119 struct list_head node;
120
121 char osd_path[0];
122};
123
124static struct class *class_osdblk;
125static DEFINE_MUTEX(ctl_mutex);
126static LIST_HEAD(osdblkdev_list);
127
128static const struct block_device_operations osdblk_bd_ops = {
129 .owner = THIS_MODULE,
130};
131
132static const struct osd_attr g_attr_logical_length = ATTR_DEF(
133 OSD_APAGE_OBJECT_INFORMATION, OSD_ATTR_OI_LOGICAL_LENGTH, 8);
134
135static void osdblk_make_credential(u8 cred_a[OSD_CAP_LEN],
136 const struct osd_obj_id *obj)
137{
138 osd_sec_init_nosec_doall_caps(cred_a, obj, false, true);
139}
140
141
142
143
144
145static int osd_sync_op(struct osd_request *or, int timeout, uint8_t *credential)
146{
147 int ret;
148
149 or->timeout = timeout;
150 ret = osd_finalize_request(or, 0, credential, NULL);
151 if (ret)
152 return ret;
153
154 ret = osd_execute_request(or);
155
156
157 return ret;
158}
159
160
161
162
163static int osd_async_op(struct osd_request *or, osd_req_done_fn *async_done,
164 void *caller_context, u8 *cred)
165{
166 int ret;
167
168 ret = osd_finalize_request(or, 0, cred, NULL);
169 if (ret)
170 return ret;
171
172 ret = osd_execute_request_async(or, async_done, caller_context);
173
174 return ret;
175}
176
177
178static int extract_attr_from_req(struct osd_request *or, struct osd_attr *attr)
179{
180 struct osd_attr cur_attr = {.attr_page = 0};
181 void *iter = NULL;
182 int nelem;
183
184 do {
185 nelem = 1;
186 osd_req_decode_get_attr_list(or, &cur_attr, &nelem, &iter);
187 if ((cur_attr.attr_page == attr->attr_page) &&
188 (cur_attr.attr_id == attr->attr_id)) {
189 attr->len = cur_attr.len;
190 attr->val_ptr = cur_attr.val_ptr;
191 return 0;
192 }
193 } while (iter);
194
195 return -EIO;
196}
197
198static int osdblk_get_obj_size(struct osdblk_device *osdev, u64 *size_out)
199{
200 struct osd_request *or;
201 struct osd_attr attr;
202 int ret;
203
204
205 or = osd_start_request(osdev->osd, GFP_KERNEL);
206 if (!or)
207 return -ENOMEM;
208
209
210 osd_req_get_attributes(or, &osdev->obj);
211
212 osd_req_add_get_attr_list(or, &g_attr_logical_length, 1);
213
214
215 ret = osd_sync_op(or, OSDBLK_OP_TIMEOUT, osdev->obj_cred);
216 if (ret)
217 goto out;
218
219
220 attr = g_attr_logical_length;
221 ret = extract_attr_from_req(or, &attr);
222 if (ret)
223 goto out;
224
225 *size_out = get_unaligned_be64(attr.val_ptr);
226
227out:
228 osd_end_request(or);
229 return ret;
230
231}
232
233static void osdblk_osd_complete(struct osd_request *or, void *private)
234{
235 struct osdblk_request *orq = private;
236 struct osd_sense_info osi;
237 int ret = osd_req_decode_sense(or, &osi);
238
239 if (ret) {
240 ret = -EIO;
241 OSDBLK_DEBUG("osdblk_osd_complete with err=%d\n", ret);
242 }
243
244
245 osd_end_request(or);
246
247
248 __blk_end_request_all(orq->rq, ret);
249}
250
251static void bio_chain_put(struct bio *chain)
252{
253 struct bio *tmp;
254
255 while (chain) {
256 tmp = chain;
257 chain = chain->bi_next;
258
259 bio_put(tmp);
260 }
261}
262
263static struct bio *bio_chain_clone(struct bio *old_chain, gfp_t gfpmask)
264{
265 struct bio *tmp, *new_chain = NULL, *tail = NULL;
266
267 while (old_chain) {
268 tmp = bio_kmalloc(gfpmask, old_chain->bi_max_vecs);
269 if (!tmp)
270 goto err_out;
271
272 __bio_clone(tmp, old_chain);
273 tmp->bi_bdev = NULL;
274 gfpmask &= ~__GFP_WAIT;
275 tmp->bi_next = NULL;
276
277 if (!new_chain)
278 new_chain = tail = tmp;
279 else {
280 tail->bi_next = tmp;
281 tail = tmp;
282 }
283
284 old_chain = old_chain->bi_next;
285 }
286
287 return new_chain;
288
289err_out:
290 OSDBLK_DEBUG("bio_chain_clone with err\n");
291 bio_chain_put(new_chain);
292 return NULL;
293}
294
295static void osdblk_rq_fn(struct request_queue *q)
296{
297 struct osdblk_device *osdev = q->queuedata;
298
299 while (1) {
300 struct request *rq;
301 struct osdblk_request *orq;
302 struct osd_request *or;
303 struct bio *bio;
304 bool do_write, do_flush;
305
306
307 rq = blk_fetch_request(q);
308 if (!rq)
309 break;
310
311
312 if (!blk_fs_request(rq) && !blk_barrier_rq(rq)) {
313 blk_end_request_all(rq, 0);
314 continue;
315 }
316
317
318
319
320
321
322
323
324 do_flush = (rq->special == (void *) 0xdeadbeefUL);
325 do_write = (rq_data_dir(rq) == WRITE);
326
327 if (!do_flush) {
328
329 bio = bio_chain_clone(rq->bio, GFP_ATOMIC);
330 if (!bio)
331 break;
332 } else
333 bio = NULL;
334
335
336 or = osd_start_request(osdev->osd, GFP_ATOMIC);
337 if (!or) {
338 bio_chain_put(bio);
339 OSDBLK_DEBUG("osd_start_request with err\n");
340 break;
341 }
342
343 orq = &osdev->req[rq->tag];
344 orq->rq = rq;
345 orq->bio = bio;
346 orq->osdev = osdev;
347
348
349 if (do_flush)
350 osd_req_flush_object(or, &osdev->obj,
351 OSD_CDB_FLUSH_ALL, 0, 0);
352 else if (do_write)
353 osd_req_write(or, &osdev->obj, blk_rq_pos(rq) * 512ULL,
354 bio, blk_rq_bytes(rq));
355 else
356 osd_req_read(or, &osdev->obj, blk_rq_pos(rq) * 512ULL,
357 bio, blk_rq_bytes(rq));
358
359 OSDBLK_DEBUG("%s 0x%x bytes at 0x%llx\n",
360 do_flush ? "flush" : do_write ?
361 "write" : "read", blk_rq_bytes(rq),
362 blk_rq_pos(rq) * 512ULL);
363
364
365 if (osd_async_op(or, osdblk_osd_complete, orq,
366 osdev->obj_cred)) {
367 osd_end_request(or);
368 blk_requeue_request(q, rq);
369 bio_chain_put(bio);
370 OSDBLK_DEBUG("osd_execute_request_async with err\n");
371 break;
372 }
373
374
375
376
377 rq->special = NULL;
378 }
379}
380
381static void osdblk_prepare_flush(struct request_queue *q, struct request *rq)
382{
383
384
385
386 rq->special = (void *) 0xdeadbeefUL;
387}
388
389static void osdblk_free_disk(struct osdblk_device *osdev)
390{
391 struct gendisk *disk = osdev->disk;
392
393 if (!disk)
394 return;
395
396 if (disk->flags & GENHD_FL_UP)
397 del_gendisk(disk);
398 if (disk->queue)
399 blk_cleanup_queue(disk->queue);
400 put_disk(disk);
401}
402
403static int osdblk_init_disk(struct osdblk_device *osdev)
404{
405 struct gendisk *disk;
406 struct request_queue *q;
407 int rc;
408 u64 obj_size = 0;
409
410
411 rc = osdblk_get_obj_size(osdev, &obj_size);
412 if (rc)
413 return rc;
414
415
416 disk = alloc_disk(OSDBLK_MINORS_PER_MAJOR);
417 if (!disk)
418 return -ENOMEM;
419
420 sprintf(disk->disk_name, DRV_NAME "%d", osdev->id);
421 disk->major = osdev->major;
422 disk->first_minor = 0;
423 disk->fops = &osdblk_bd_ops;
424 disk->private_data = osdev;
425
426
427 q = blk_init_queue(osdblk_rq_fn, &osdev->lock);
428 if (!q) {
429 put_disk(disk);
430 return -ENOMEM;
431 }
432
433
434 rc = blk_queue_init_tags(q, OSDBLK_MAX_REQ, NULL);
435 if (rc) {
436 blk_cleanup_queue(q);
437 put_disk(disk);
438 return rc;
439 }
440
441
442
443
444
445 blk_queue_stack_limits(q, osd_request_queue(osdev->osd));
446
447 blk_queue_prep_rq(q, blk_queue_start_tag);
448 blk_queue_ordered(q, QUEUE_ORDERED_DRAIN_FLUSH, osdblk_prepare_flush);
449
450 disk->queue = q;
451
452 q->queuedata = osdev;
453
454 osdev->disk = disk;
455 osdev->q = q;
456
457
458 set_capacity(disk, obj_size / 512ULL);
459 add_disk(disk);
460
461 printk(KERN_INFO "%s: Added of size 0x%llx\n",
462 disk->disk_name, (unsigned long long)obj_size);
463
464 return 0;
465}
466
467
468
469
470
471
472
473
474static void class_osdblk_release(struct class *cls)
475{
476 kfree(cls);
477}
478
479static ssize_t class_osdblk_list(struct class *c, char *data)
480{
481 int n = 0;
482 struct list_head *tmp;
483
484 mutex_lock_nested(&ctl_mutex, SINGLE_DEPTH_NESTING);
485
486 list_for_each(tmp, &osdblkdev_list) {
487 struct osdblk_device *osdev;
488
489 osdev = list_entry(tmp, struct osdblk_device, node);
490
491 n += sprintf(data+n, "%d %d %llu %llu %s\n",
492 osdev->id,
493 osdev->major,
494 osdev->obj.partition,
495 osdev->obj.id,
496 osdev->osd_path);
497 }
498
499 mutex_unlock(&ctl_mutex);
500 return n;
501}
502
503static ssize_t class_osdblk_add(struct class *c, const char *buf, size_t count)
504{
505 struct osdblk_device *osdev;
506 ssize_t rc;
507 int irc, new_id = 0;
508 struct list_head *tmp;
509
510 if (!try_module_get(THIS_MODULE))
511 return -ENODEV;
512
513
514 osdev = kzalloc(sizeof(*osdev) + strlen(buf) + 1, GFP_KERNEL);
515 if (!osdev) {
516 rc = -ENOMEM;
517 goto err_out_mod;
518 }
519
520
521 spin_lock_init(&osdev->lock);
522 INIT_LIST_HEAD(&osdev->node);
523
524
525
526 mutex_lock_nested(&ctl_mutex, SINGLE_DEPTH_NESTING);
527
528 list_for_each(tmp, &osdblkdev_list) {
529 struct osdblk_device *osdev;
530
531 osdev = list_entry(tmp, struct osdblk_device, node);
532 if (osdev->id > new_id)
533 new_id = osdev->id + 1;
534 }
535
536 osdev->id = new_id;
537
538
539 list_add_tail(&osdev->node, &osdblkdev_list);
540
541 mutex_unlock(&ctl_mutex);
542
543
544 if (sscanf(buf, "%llu %llu %s", &osdev->obj.partition, &osdev->obj.id,
545 osdev->osd_path) != 3) {
546 rc = -EINVAL;
547 goto err_out_slot;
548 }
549
550
551 sprintf(osdev->name, DRV_NAME "%d", osdev->id);
552
553
554 osdev->osd = osduld_path_lookup(osdev->osd_path);
555 if (IS_ERR(osdev->osd)) {
556 rc = PTR_ERR(osdev->osd);
557 goto err_out_slot;
558 }
559
560
561 osdblk_make_credential(osdev->obj_cred, &osdev->obj);
562
563
564 irc = register_blkdev(0, osdev->name);
565 if (irc < 0) {
566 rc = irc;
567 goto err_out_osd;
568 }
569
570 osdev->major = irc;
571
572
573 rc = osdblk_init_disk(osdev);
574 if (rc)
575 goto err_out_blkdev;
576
577 return count;
578
579err_out_blkdev:
580 unregister_blkdev(osdev->major, osdev->name);
581err_out_osd:
582 osduld_put_device(osdev->osd);
583err_out_slot:
584 mutex_lock_nested(&ctl_mutex, SINGLE_DEPTH_NESTING);
585 list_del_init(&osdev->node);
586 mutex_unlock(&ctl_mutex);
587
588 kfree(osdev);
589err_out_mod:
590 OSDBLK_DEBUG("Error adding device %s\n", buf);
591 module_put(THIS_MODULE);
592 return rc;
593}
594
595static ssize_t class_osdblk_remove(struct class *c, const char *buf,
596 size_t count)
597{
598 struct osdblk_device *osdev = NULL;
599 int target_id, rc;
600 unsigned long ul;
601 struct list_head *tmp;
602
603 rc = strict_strtoul(buf, 10, &ul);
604 if (rc)
605 return rc;
606
607
608 target_id = (int) ul;
609 if (target_id != ul)
610 return -EINVAL;
611
612
613 mutex_lock_nested(&ctl_mutex, SINGLE_DEPTH_NESTING);
614
615 list_for_each(tmp, &osdblkdev_list) {
616 osdev = list_entry(tmp, struct osdblk_device, node);
617 if (osdev->id == target_id) {
618 list_del_init(&osdev->node);
619 break;
620 }
621 osdev = NULL;
622 }
623
624 mutex_unlock(&ctl_mutex);
625
626 if (!osdev)
627 return -ENOENT;
628
629
630 osdblk_free_disk(osdev);
631 unregister_blkdev(osdev->major, osdev->name);
632 osduld_put_device(osdev->osd);
633 kfree(osdev);
634
635
636 module_put(THIS_MODULE);
637
638 return count;
639}
640
641static struct class_attribute class_osdblk_attrs[] = {
642 __ATTR(add, 0200, NULL, class_osdblk_add),
643 __ATTR(remove, 0200, NULL, class_osdblk_remove),
644 __ATTR(list, 0444, class_osdblk_list, NULL),
645 __ATTR_NULL
646};
647
648static int osdblk_sysfs_init(void)
649{
650 int ret = 0;
651
652
653
654
655
656 class_osdblk = kzalloc(sizeof(*class_osdblk), GFP_KERNEL);
657 if (!class_osdblk)
658 return -ENOMEM;
659
660 class_osdblk->name = DRV_NAME;
661 class_osdblk->owner = THIS_MODULE;
662 class_osdblk->class_release = class_osdblk_release;
663 class_osdblk->class_attrs = class_osdblk_attrs;
664
665 ret = class_register(class_osdblk);
666 if (ret) {
667 kfree(class_osdblk);
668 class_osdblk = NULL;
669 printk(PFX "failed to create class osdblk\n");
670 return ret;
671 }
672
673 return 0;
674}
675
676static void osdblk_sysfs_cleanup(void)
677{
678 if (class_osdblk)
679 class_destroy(class_osdblk);
680 class_osdblk = NULL;
681}
682
683static int __init osdblk_init(void)
684{
685 int rc;
686
687 rc = osdblk_sysfs_init();
688 if (rc)
689 return rc;
690
691 return 0;
692}
693
694static void __exit osdblk_exit(void)
695{
696 osdblk_sysfs_cleanup();
697}
698
699module_init(osdblk_init);
700module_exit(osdblk_exit);
701
702