1
2#ifndef _BCACHE_H
3#define _BCACHE_H
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179#define pr_fmt(fmt) "bcache: %s() " fmt "\n", __func__
180
181#include <linux/bcache.h>
182#include <linux/bio.h>
183#include <linux/kobject.h>
184#include <linux/list.h>
185#include <linux/mutex.h>
186#include <linux/rbtree.h>
187#include <linux/rwsem.h>
188#include <linux/refcount.h>
189#include <linux/types.h>
190#include <linux/workqueue.h>
191#include <linux/kthread.h>
192
193#include "bset.h"
194#include "util.h"
195#include "closure.h"
196
197struct bucket {
198 atomic_t pin;
199 uint16_t prio;
200 uint8_t gen;
201 uint8_t last_gc;
202 uint16_t gc_mark;
203};
204
205
206
207
208
209
210BITMASK(GC_MARK, struct bucket, gc_mark, 0, 2);
211#define GC_MARK_RECLAIMABLE 1
212#define GC_MARK_DIRTY 2
213#define GC_MARK_METADATA 3
214#define GC_SECTORS_USED_SIZE 13
215#define MAX_GC_SECTORS_USED (~(~0ULL << GC_SECTORS_USED_SIZE))
216BITMASK(GC_SECTORS_USED, struct bucket, gc_mark, 2, GC_SECTORS_USED_SIZE);
217BITMASK(GC_MOVE, struct bucket, gc_mark, 15, 1);
218
219#include "journal.h"
220#include "stats.h"
221struct search;
222struct btree;
223struct keybuf;
224
225struct keybuf_key {
226 struct rb_node node;
227 BKEY_PADDED(key);
228 void *private;
229};
230
231struct keybuf {
232 struct bkey last_scanned;
233 spinlock_t lock;
234
235
236
237
238
239
240 struct bkey start;
241 struct bkey end;
242
243 struct rb_root keys;
244
245#define KEYBUF_NR 500
246 DECLARE_ARRAY_ALLOCATOR(struct keybuf_key, freelist, KEYBUF_NR);
247};
248
249struct bcache_device {
250 struct closure cl;
251
252 struct kobject kobj;
253
254 struct cache_set *c;
255 unsigned int id;
256#define BCACHEDEVNAME_SIZE 12
257 char name[BCACHEDEVNAME_SIZE];
258
259 struct gendisk *disk;
260
261 unsigned long flags;
262#define BCACHE_DEV_CLOSING 0
263#define BCACHE_DEV_DETACHING 1
264#define BCACHE_DEV_UNLINK_DONE 2
265#define BCACHE_DEV_WB_RUNNING 3
266#define BCACHE_DEV_RATE_DW_RUNNING 4
267 unsigned int nr_stripes;
268 unsigned int stripe_size;
269 atomic_t *stripe_sectors_dirty;
270 unsigned long *full_dirty_stripes;
271
272 struct bio_set bio_split;
273
274 unsigned int data_csum:1;
275
276 int (*cache_miss)(struct btree *b, struct search *s,
277 struct bio *bio, unsigned int sectors);
278 int (*ioctl)(struct bcache_device *d, fmode_t mode,
279 unsigned int cmd, unsigned long arg);
280};
281
282struct io {
283
284 struct hlist_node hash;
285 struct list_head lru;
286
287 unsigned long jiffies;
288 unsigned int sequential;
289 sector_t last;
290};
291
292enum stop_on_failure {
293 BCH_CACHED_DEV_STOP_AUTO = 0,
294 BCH_CACHED_DEV_STOP_ALWAYS,
295 BCH_CACHED_DEV_STOP_MODE_MAX,
296};
297
298struct cached_dev {
299 struct list_head list;
300 struct bcache_device disk;
301 struct block_device *bdev;
302
303 struct cache_sb sb;
304 struct cache_sb_disk *sb_disk;
305 struct bio sb_bio;
306 struct bio_vec sb_bv[1];
307 struct closure sb_write;
308 struct semaphore sb_write_mutex;
309
310
311 refcount_t count;
312 struct work_struct detach;
313
314
315
316
317
318 atomic_t running;
319
320
321
322
323
324 struct rw_semaphore writeback_lock;
325
326
327
328
329
330
331 atomic_t has_dirty;
332
333#define BCH_CACHE_READA_ALL 0
334#define BCH_CACHE_READA_META_ONLY 1
335 unsigned int cache_readahead_policy;
336 struct bch_ratelimit writeback_rate;
337 struct delayed_work writeback_rate_update;
338
339
340 struct semaphore in_flight;
341 struct task_struct *writeback_thread;
342 struct workqueue_struct *writeback_write_wq;
343
344 struct keybuf writeback_keys;
345
346 struct task_struct *status_update_thread;
347
348
349
350
351
352 struct closure_waitlist writeback_ordering_wait;
353 atomic_t writeback_sequence_next;
354
355
356#define RECENT_IO_BITS 7
357#define RECENT_IO (1 << RECENT_IO_BITS)
358 struct io io[RECENT_IO];
359 struct hlist_head io_hash[RECENT_IO + 1];
360 struct list_head io_lru;
361 spinlock_t io_lock;
362
363 struct cache_accounting accounting;
364
365
366 unsigned int sequential_cutoff;
367 unsigned int readahead;
368
369 unsigned int io_disable:1;
370 unsigned int verify:1;
371 unsigned int bypass_torture_test:1;
372
373 unsigned int partial_stripes_expensive:1;
374 unsigned int writeback_metadata:1;
375 unsigned int writeback_running:1;
376 unsigned char writeback_percent;
377 unsigned int writeback_delay;
378
379 uint64_t writeback_rate_target;
380 int64_t writeback_rate_proportional;
381 int64_t writeback_rate_integral;
382 int64_t writeback_rate_integral_scaled;
383 int32_t writeback_rate_change;
384
385 unsigned int writeback_rate_update_seconds;
386 unsigned int writeback_rate_i_term_inverse;
387 unsigned int writeback_rate_p_term_inverse;
388 unsigned int writeback_rate_minimum;
389
390 enum stop_on_failure stop_when_cache_set_failed;
391#define DEFAULT_CACHED_DEV_ERROR_LIMIT 64
392 atomic_t io_errors;
393 unsigned int error_limit;
394 unsigned int offline_seconds;
395
396 char backing_dev_name[BDEVNAME_SIZE];
397};
398
399enum alloc_reserve {
400 RESERVE_BTREE,
401 RESERVE_PRIO,
402 RESERVE_MOVINGGC,
403 RESERVE_NONE,
404 RESERVE_NR,
405};
406
407struct cache {
408 struct cache_set *set;
409 struct cache_sb sb;
410 struct cache_sb_disk *sb_disk;
411 struct bio sb_bio;
412 struct bio_vec sb_bv[1];
413
414 struct kobject kobj;
415 struct block_device *bdev;
416
417 struct task_struct *alloc_thread;
418
419 struct closure prio;
420 struct prio_set *disk_buckets;
421
422
423
424
425
426
427
428
429 uint64_t *prio_buckets;
430 uint64_t *prio_last_buckets;
431
432
433
434
435
436
437
438
439
440
441 DECLARE_FIFO(long, free)[RESERVE_NR];
442 DECLARE_FIFO(long, free_inc);
443
444 size_t fifo_last_bucket;
445
446
447 struct bucket *buckets;
448
449 DECLARE_HEAP(struct bucket *, heap);
450
451
452
453
454
455
456 unsigned int invalidate_needs_gc;
457
458 bool discard;
459
460 struct journal_device journal;
461
462
463#define IO_ERROR_SHIFT 20
464 atomic_t io_errors;
465 atomic_t io_count;
466
467 atomic_long_t meta_sectors_written;
468 atomic_long_t btree_sectors_written;
469 atomic_long_t sectors_written;
470
471 char cache_dev_name[BDEVNAME_SIZE];
472};
473
474struct gc_stat {
475 size_t nodes;
476 size_t nodes_pre;
477 size_t key_bytes;
478
479 size_t nkeys;
480 uint64_t data;
481 unsigned int in_use;
482};
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502#define CACHE_SET_UNREGISTERING 0
503#define CACHE_SET_STOPPING 1
504#define CACHE_SET_RUNNING 2
505#define CACHE_SET_IO_DISABLE 3
506
507struct cache_set {
508 struct closure cl;
509
510 struct list_head list;
511 struct kobject kobj;
512 struct kobject internal;
513 struct dentry *debug;
514 struct cache_accounting accounting;
515
516 unsigned long flags;
517 atomic_t idle_counter;
518 atomic_t at_max_writeback_rate;
519
520 struct cache_sb sb;
521
522 struct cache *cache[MAX_CACHES_PER_SET];
523 struct cache *cache_by_alloc[MAX_CACHES_PER_SET];
524 int caches_loaded;
525
526 struct bcache_device **devices;
527 unsigned int devices_max_used;
528 atomic_t attached_dev_nr;
529 struct list_head cached_devs;
530 uint64_t cached_dev_sectors;
531 atomic_long_t flash_dev_dirty_sectors;
532 struct closure caching;
533
534 struct closure sb_write;
535 struct semaphore sb_write_mutex;
536
537 mempool_t search;
538 mempool_t bio_meta;
539 struct bio_set bio_split;
540
541
542 struct shrinker shrink;
543
544
545 struct mutex bucket_lock;
546
547
548 unsigned short bucket_bits;
549
550
551 unsigned short block_bits;
552
553
554
555
556
557 unsigned int btree_pages;
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575 struct list_head btree_cache;
576 struct list_head btree_cache_freeable;
577 struct list_head btree_cache_freed;
578
579
580 unsigned int btree_cache_used;
581
582
583
584
585
586
587
588 wait_queue_head_t btree_cache_wait;
589 struct task_struct *btree_cache_alloc_lock;
590 spinlock_t btree_cannibalize_lock;
591
592
593
594
595
596
597
598
599
600
601
602 atomic_t prio_blocked;
603 wait_queue_head_t bucket_wait;
604
605
606
607
608
609 atomic_t rescale;
610
611
612
613 atomic_t search_inflight;
614
615
616
617
618
619
620 uint16_t min_prio;
621
622
623
624
625
626 uint8_t need_gc;
627 struct gc_stat gc_stats;
628 size_t nbuckets;
629 size_t avail_nbuckets;
630
631 struct task_struct *gc_thread;
632
633 struct bkey gc_done;
634
635
636
637
638
639
640
641
642
643
644
645#define BCH_ENABLE_AUTO_GC 1
646#define BCH_DO_AUTO_GC 2
647 uint8_t gc_after_writeback;
648
649
650
651
652
653 int gc_mark_valid;
654
655
656 atomic_t sectors_to_gc;
657 wait_queue_head_t gc_wait;
658
659 struct keybuf moving_gc_keys;
660
661 struct semaphore moving_in_flight;
662
663 struct workqueue_struct *moving_gc_wq;
664
665 struct btree *root;
666
667#ifdef CONFIG_BCACHE_DEBUG
668 struct btree *verify_data;
669 struct bset *verify_ondisk;
670 struct mutex verify_lock;
671#endif
672
673 unsigned int nr_uuids;
674 struct uuid_entry *uuids;
675 BKEY_PADDED(uuid_bucket);
676 struct closure uuid_write;
677 struct semaphore uuid_write_mutex;
678
679
680
681
682
683
684
685
686
687 mempool_t fill_iter;
688
689 struct bset_sort_state sort;
690
691
692 struct list_head data_buckets;
693 spinlock_t data_bucket_lock;
694
695 struct journal journal;
696
697#define CONGESTED_MAX 1024
698 unsigned int congested_last_us;
699 atomic_t congested;
700
701
702 unsigned int congested_read_threshold_us;
703 unsigned int congested_write_threshold_us;
704
705 struct time_stats btree_gc_time;
706 struct time_stats btree_split_time;
707 struct time_stats btree_read_time;
708
709 atomic_long_t cache_read_races;
710 atomic_long_t writeback_keys_done;
711 atomic_long_t writeback_keys_failed;
712
713 atomic_long_t reclaim;
714 atomic_long_t reclaimed_journal_buckets;
715 atomic_long_t flush_write;
716
717 enum {
718 ON_ERROR_UNREGISTER,
719 ON_ERROR_PANIC,
720 } on_error;
721#define DEFAULT_IO_ERROR_LIMIT 8
722 unsigned int error_limit;
723 unsigned int error_decay;
724
725 unsigned short journal_delay_ms;
726 bool expensive_debug_checks;
727 unsigned int verify:1;
728 unsigned int key_merging_disabled:1;
729 unsigned int gc_always_rewrite:1;
730 unsigned int shrinker_disabled:1;
731 unsigned int copy_gc_enabled:1;
732 unsigned int idle_max_writeback_rate_enabled:1;
733
734#define BUCKET_HASH_BITS 12
735 struct hlist_head bucket_hash[1 << BUCKET_HASH_BITS];
736};
737
738struct bbio {
739 unsigned int submit_time_us;
740 union {
741 struct bkey key;
742 uint64_t _pad[3];
743
744
745
746
747 };
748 struct bio bio;
749};
750
751#define BTREE_PRIO USHRT_MAX
752#define INITIAL_PRIO 32768U
753
754#define btree_bytes(c) ((c)->btree_pages * PAGE_SIZE)
755#define btree_blocks(b) \
756 ((unsigned int) (KEY_SIZE(&b->key) >> (b)->c->block_bits))
757
758#define btree_default_blocks(c) \
759 ((unsigned int) ((PAGE_SECTORS * (c)->btree_pages) >> (c)->block_bits))
760
761#define bucket_pages(c) ((c)->sb.bucket_size / PAGE_SECTORS)
762#define bucket_bytes(c) ((c)->sb.bucket_size << 9)
763#define block_bytes(c) ((c)->sb.block_size << 9)
764
765#define prios_per_bucket(c) \
766 ((bucket_bytes(c) - sizeof(struct prio_set)) / \
767 sizeof(struct bucket_disk))
768#define prio_buckets(c) \
769 DIV_ROUND_UP((size_t) (c)->sb.nbuckets, prios_per_bucket(c))
770
771static inline size_t sector_to_bucket(struct cache_set *c, sector_t s)
772{
773 return s >> c->bucket_bits;
774}
775
776static inline sector_t bucket_to_sector(struct cache_set *c, size_t b)
777{
778 return ((sector_t) b) << c->bucket_bits;
779}
780
781static inline sector_t bucket_remainder(struct cache_set *c, sector_t s)
782{
783 return s & (c->sb.bucket_size - 1);
784}
785
786static inline struct cache *PTR_CACHE(struct cache_set *c,
787 const struct bkey *k,
788 unsigned int ptr)
789{
790 return c->cache[PTR_DEV(k, ptr)];
791}
792
793static inline size_t PTR_BUCKET_NR(struct cache_set *c,
794 const struct bkey *k,
795 unsigned int ptr)
796{
797 return sector_to_bucket(c, PTR_OFFSET(k, ptr));
798}
799
800static inline struct bucket *PTR_BUCKET(struct cache_set *c,
801 const struct bkey *k,
802 unsigned int ptr)
803{
804 return PTR_CACHE(c, k, ptr)->buckets + PTR_BUCKET_NR(c, k, ptr);
805}
806
807static inline uint8_t gen_after(uint8_t a, uint8_t b)
808{
809 uint8_t r = a - b;
810
811 return r > 128U ? 0 : r;
812}
813
814static inline uint8_t ptr_stale(struct cache_set *c, const struct bkey *k,
815 unsigned int i)
816{
817 return gen_after(PTR_BUCKET(c, k, i)->gen, PTR_GEN(k, i));
818}
819
820static inline bool ptr_available(struct cache_set *c, const struct bkey *k,
821 unsigned int i)
822{
823 return (PTR_DEV(k, i) < MAX_CACHES_PER_SET) && PTR_CACHE(c, k, i);
824}
825
826
827
828
829
830
831
832#define csum_set(i) \
833 bch_crc64(((void *) (i)) + sizeof(uint64_t), \
834 ((void *) bset_bkey_last(i)) - \
835 (((void *) (i)) + sizeof(uint64_t)))
836
837
838
839#define btree_bug(b, ...) \
840do { \
841 if (bch_cache_set_error((b)->c, __VA_ARGS__)) \
842 dump_stack(); \
843} while (0)
844
845#define cache_bug(c, ...) \
846do { \
847 if (bch_cache_set_error(c, __VA_ARGS__)) \
848 dump_stack(); \
849} while (0)
850
851#define btree_bug_on(cond, b, ...) \
852do { \
853 if (cond) \
854 btree_bug(b, __VA_ARGS__); \
855} while (0)
856
857#define cache_bug_on(cond, c, ...) \
858do { \
859 if (cond) \
860 cache_bug(c, __VA_ARGS__); \
861} while (0)
862
863#define cache_set_err_on(cond, c, ...) \
864do { \
865 if (cond) \
866 bch_cache_set_error(c, __VA_ARGS__); \
867} while (0)
868
869
870
871#define for_each_cache(ca, cs, iter) \
872 for (iter = 0; ca = cs->cache[iter], iter < (cs)->sb.nr_in_set; iter++)
873
874#define for_each_bucket(b, ca) \
875 for (b = (ca)->buckets + (ca)->sb.first_bucket; \
876 b < (ca)->buckets + (ca)->sb.nbuckets; b++)
877
878static inline void cached_dev_put(struct cached_dev *dc)
879{
880 if (refcount_dec_and_test(&dc->count))
881 schedule_work(&dc->detach);
882}
883
884static inline bool cached_dev_get(struct cached_dev *dc)
885{
886 if (!refcount_inc_not_zero(&dc->count))
887 return false;
888
889
890 smp_mb__after_atomic();
891 return true;
892}
893
894
895
896
897
898
899static inline uint8_t bucket_gc_gen(struct bucket *b)
900{
901 return b->gen - b->last_gc;
902}
903
904#define BUCKET_GC_GEN_MAX 96U
905
906#define kobj_attribute_write(n, fn) \
907 static struct kobj_attribute ksysfs_##n = __ATTR(n, 0200, NULL, fn)
908
909#define kobj_attribute_rw(n, show, store) \
910 static struct kobj_attribute ksysfs_##n = \
911 __ATTR(n, 0600, show, store)
912
913static inline void wake_up_allocators(struct cache_set *c)
914{
915 struct cache *ca;
916 unsigned int i;
917
918 for_each_cache(ca, c, i)
919 wake_up_process(ca->alloc_thread);
920}
921
922static inline void closure_bio_submit(struct cache_set *c,
923 struct bio *bio,
924 struct closure *cl)
925{
926 closure_get(cl);
927 if (unlikely(test_bit(CACHE_SET_IO_DISABLE, &c->flags))) {
928 bio->bi_status = BLK_STS_IOERR;
929 bio_endio(bio);
930 return;
931 }
932 generic_make_request(bio);
933}
934
935
936
937
938
939
940
941static inline void wait_for_kthread_stop(void)
942{
943 while (!kthread_should_stop()) {
944 set_current_state(TASK_INTERRUPTIBLE);
945 schedule();
946 }
947}
948
949
950
951void bch_count_backing_io_errors(struct cached_dev *dc, struct bio *bio);
952void bch_count_io_errors(struct cache *ca, blk_status_t error,
953 int is_read, const char *m);
954void bch_bbio_count_io_errors(struct cache_set *c, struct bio *bio,
955 blk_status_t error, const char *m);
956void bch_bbio_endio(struct cache_set *c, struct bio *bio,
957 blk_status_t error, const char *m);
958void bch_bbio_free(struct bio *bio, struct cache_set *c);
959struct bio *bch_bbio_alloc(struct cache_set *c);
960
961void __bch_submit_bbio(struct bio *bio, struct cache_set *c);
962void bch_submit_bbio(struct bio *bio, struct cache_set *c,
963 struct bkey *k, unsigned int ptr);
964
965uint8_t bch_inc_gen(struct cache *ca, struct bucket *b);
966void bch_rescale_priorities(struct cache_set *c, int sectors);
967
968bool bch_can_invalidate_bucket(struct cache *ca, struct bucket *b);
969void __bch_invalidate_one_bucket(struct cache *ca, struct bucket *b);
970
971void __bch_bucket_free(struct cache *ca, struct bucket *b);
972void bch_bucket_free(struct cache_set *c, struct bkey *k);
973
974long bch_bucket_alloc(struct cache *ca, unsigned int reserve, bool wait);
975int __bch_bucket_alloc_set(struct cache_set *c, unsigned int reserve,
976 struct bkey *k, int n, bool wait);
977int bch_bucket_alloc_set(struct cache_set *c, unsigned int reserve,
978 struct bkey *k, int n, bool wait);
979bool bch_alloc_sectors(struct cache_set *c, struct bkey *k,
980 unsigned int sectors, unsigned int write_point,
981 unsigned int write_prio, bool wait);
982bool bch_cached_dev_error(struct cached_dev *dc);
983
984__printf(2, 3)
985bool bch_cache_set_error(struct cache_set *c, const char *fmt, ...);
986
987int bch_prio_write(struct cache *ca, bool wait);
988void bch_write_bdev_super(struct cached_dev *dc, struct closure *parent);
989
990extern struct workqueue_struct *bcache_wq;
991extern struct workqueue_struct *bch_journal_wq;
992extern struct mutex bch_register_lock;
993extern struct list_head bch_cache_sets;
994
995extern struct kobj_type bch_cached_dev_ktype;
996extern struct kobj_type bch_flash_dev_ktype;
997extern struct kobj_type bch_cache_set_ktype;
998extern struct kobj_type bch_cache_set_internal_ktype;
999extern struct kobj_type bch_cache_ktype;
1000
1001void bch_cached_dev_release(struct kobject *kobj);
1002void bch_flash_dev_release(struct kobject *kobj);
1003void bch_cache_set_release(struct kobject *kobj);
1004void bch_cache_release(struct kobject *kobj);
1005
1006int bch_uuid_write(struct cache_set *c);
1007void bcache_write_super(struct cache_set *c);
1008
1009int bch_flash_dev_create(struct cache_set *c, uint64_t size);
1010
1011int bch_cached_dev_attach(struct cached_dev *dc, struct cache_set *c,
1012 uint8_t *set_uuid);
1013void bch_cached_dev_detach(struct cached_dev *dc);
1014int bch_cached_dev_run(struct cached_dev *dc);
1015void bcache_device_stop(struct bcache_device *d);
1016
1017void bch_cache_set_unregister(struct cache_set *c);
1018void bch_cache_set_stop(struct cache_set *c);
1019
1020struct cache_set *bch_cache_set_alloc(struct cache_sb *sb);
1021void bch_btree_cache_free(struct cache_set *c);
1022int bch_btree_cache_alloc(struct cache_set *c);
1023void bch_moving_init_cache_set(struct cache_set *c);
1024int bch_open_buckets_alloc(struct cache_set *c);
1025void bch_open_buckets_free(struct cache_set *c);
1026
1027int bch_cache_allocator_start(struct cache *ca);
1028
1029void bch_debug_exit(void);
1030void bch_debug_init(void);
1031void bch_request_exit(void);
1032int bch_request_init(void);
1033
1034#endif
1035