1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20#include <linux/circ_buf.h>
21
22#include "pblk.h"
23
24static DECLARE_RWSEM(pblk_rb_lock);
25
26static void pblk_rb_data_free(struct pblk_rb *rb)
27{
28 struct pblk_rb_pages *p, *t;
29
30 down_write(&pblk_rb_lock);
31 list_for_each_entry_safe(p, t, &rb->pages, list) {
32 free_pages((unsigned long)page_address(p->pages), p->order);
33 list_del(&p->list);
34 kfree(p);
35 }
36 up_write(&pblk_rb_lock);
37}
38
39void pblk_rb_free(struct pblk_rb *rb)
40{
41 pblk_rb_data_free(rb);
42 vfree(rb->entries);
43}
44
45
46
47
48static unsigned int pblk_rb_calculate_size(unsigned int nr_entries)
49{
50
51 return (1 << max(get_count_order(nr_entries), 7));
52}
53
54
55
56
57
58
59int pblk_rb_init(struct pblk_rb *rb, unsigned int size, unsigned int threshold,
60 unsigned int seg_size)
61{
62 struct pblk *pblk = container_of(rb, struct pblk, rwb);
63 struct pblk_rb_entry *entries;
64 unsigned int init_entry = 0;
65 unsigned int max_order = MAX_ORDER - 1;
66 unsigned int power_size, power_seg_sz;
67 unsigned int alloc_order, order, iter;
68 unsigned int nr_entries;
69
70 nr_entries = pblk_rb_calculate_size(size);
71 entries = vzalloc(array_size(nr_entries, sizeof(struct pblk_rb_entry)));
72 if (!entries)
73 return -ENOMEM;
74
75 power_size = get_count_order(size);
76 power_seg_sz = get_count_order(seg_size);
77
78 down_write(&pblk_rb_lock);
79 rb->entries = entries;
80 rb->seg_size = (1 << power_seg_sz);
81 rb->nr_entries = (1 << power_size);
82 rb->mem = rb->subm = rb->sync = rb->l2p_update = 0;
83 rb->back_thres = threshold;
84 rb->flush_point = EMPTY_ENTRY;
85
86 spin_lock_init(&rb->w_lock);
87 spin_lock_init(&rb->s_lock);
88
89 INIT_LIST_HEAD(&rb->pages);
90
91 alloc_order = power_size;
92 if (alloc_order >= max_order) {
93 order = max_order;
94 iter = (1 << (alloc_order - max_order));
95 } else {
96 order = alloc_order;
97 iter = 1;
98 }
99
100 do {
101 struct pblk_rb_entry *entry;
102 struct pblk_rb_pages *page_set;
103 void *kaddr;
104 unsigned long set_size;
105 int i;
106
107 page_set = kmalloc(sizeof(struct pblk_rb_pages), GFP_KERNEL);
108 if (!page_set) {
109 up_write(&pblk_rb_lock);
110 vfree(entries);
111 return -ENOMEM;
112 }
113
114 page_set->order = order;
115 page_set->pages = alloc_pages(GFP_KERNEL, order);
116 if (!page_set->pages) {
117 kfree(page_set);
118 pblk_rb_data_free(rb);
119 up_write(&pblk_rb_lock);
120 vfree(entries);
121 return -ENOMEM;
122 }
123 kaddr = page_address(page_set->pages);
124
125 entry = &rb->entries[init_entry];
126 entry->data = kaddr;
127 entry->cacheline = pblk_cacheline_to_addr(init_entry++);
128 entry->w_ctx.flags = PBLK_WRITABLE_ENTRY;
129
130 set_size = (1 << order);
131 for (i = 1; i < set_size; i++) {
132 entry = &rb->entries[init_entry];
133 entry->cacheline = pblk_cacheline_to_addr(init_entry++);
134 entry->data = kaddr + (i * rb->seg_size);
135 entry->w_ctx.flags = PBLK_WRITABLE_ENTRY;
136 bio_list_init(&entry->w_ctx.bios);
137 }
138
139 list_add_tail(&page_set->list, &rb->pages);
140 iter--;
141 } while (iter > 0);
142 up_write(&pblk_rb_lock);
143
144#ifdef CONFIG_NVM_PBLK_DEBUG
145 atomic_set(&rb->inflight_flush_point, 0);
146#endif
147
148
149
150
151
152 pblk_rl_init(&pblk->rl, rb->nr_entries);
153
154 return 0;
155}
156
157static void clean_wctx(struct pblk_w_ctx *w_ctx)
158{
159 int flags;
160
161 flags = READ_ONCE(w_ctx->flags);
162 WARN_ONCE(!(flags & PBLK_SUBMITTED_ENTRY),
163 "pblk: overwriting unsubmitted data\n");
164
165
166 smp_store_release(&w_ctx->flags, PBLK_WRITABLE_ENTRY);
167 pblk_ppa_set_empty(&w_ctx->ppa);
168 w_ctx->lba = ADDR_EMPTY;
169}
170
171#define pblk_rb_ring_count(head, tail, size) CIRC_CNT(head, tail, size)
172#define pblk_rb_ring_space(rb, head, tail, size) \
173 (CIRC_SPACE(head, tail, size))
174
175
176
177
178
179static unsigned int pblk_rb_space(struct pblk_rb *rb)
180{
181 unsigned int mem = READ_ONCE(rb->mem);
182 unsigned int sync = READ_ONCE(rb->sync);
183
184 return pblk_rb_ring_space(rb, mem, sync, rb->nr_entries);
185}
186
187unsigned int pblk_rb_ptr_wrap(struct pblk_rb *rb, unsigned int p,
188 unsigned int nr_entries)
189{
190 return (p + nr_entries) & (rb->nr_entries - 1);
191}
192
193
194
195
196
197unsigned int pblk_rb_read_count(struct pblk_rb *rb)
198{
199 unsigned int mem = READ_ONCE(rb->mem);
200 unsigned int subm = READ_ONCE(rb->subm);
201
202 return pblk_rb_ring_count(mem, subm, rb->nr_entries);
203}
204
205unsigned int pblk_rb_sync_count(struct pblk_rb *rb)
206{
207 unsigned int mem = READ_ONCE(rb->mem);
208 unsigned int sync = READ_ONCE(rb->sync);
209
210 return pblk_rb_ring_count(mem, sync, rb->nr_entries);
211}
212
213unsigned int pblk_rb_read_commit(struct pblk_rb *rb, unsigned int nr_entries)
214{
215 unsigned int subm;
216
217 subm = READ_ONCE(rb->subm);
218
219 smp_store_release(&rb->subm, pblk_rb_ptr_wrap(rb, subm, nr_entries));
220
221 return subm;
222}
223
224static int __pblk_rb_update_l2p(struct pblk_rb *rb, unsigned int to_update)
225{
226 struct pblk *pblk = container_of(rb, struct pblk, rwb);
227 struct pblk_line *line;
228 struct pblk_rb_entry *entry;
229 struct pblk_w_ctx *w_ctx;
230 unsigned int user_io = 0, gc_io = 0;
231 unsigned int i;
232 int flags;
233
234 for (i = 0; i < to_update; i++) {
235 entry = &rb->entries[rb->l2p_update];
236 w_ctx = &entry->w_ctx;
237
238 flags = READ_ONCE(entry->w_ctx.flags);
239 if (flags & PBLK_IOTYPE_USER)
240 user_io++;
241 else if (flags & PBLK_IOTYPE_GC)
242 gc_io++;
243 else
244 WARN(1, "pblk: unknown IO type\n");
245
246 pblk_update_map_dev(pblk, w_ctx->lba, w_ctx->ppa,
247 entry->cacheline);
248
249 line = pblk_ppa_to_line(pblk, w_ctx->ppa);
250 kref_put(&line->ref, pblk_line_put);
251 clean_wctx(w_ctx);
252 rb->l2p_update = pblk_rb_ptr_wrap(rb, rb->l2p_update, 1);
253 }
254
255 pblk_rl_out(&pblk->rl, user_io, gc_io);
256
257 return 0;
258}
259
260
261
262
263
264
265static int pblk_rb_update_l2p(struct pblk_rb *rb, unsigned int nr_entries,
266 unsigned int mem, unsigned int sync)
267{
268 unsigned int space, count;
269 int ret = 0;
270
271 lockdep_assert_held(&rb->w_lock);
272
273
274 space = pblk_rb_ring_space(rb, mem, rb->l2p_update, rb->nr_entries);
275 if (space > nr_entries)
276 goto out;
277
278 count = nr_entries - space;
279
280 ret = __pblk_rb_update_l2p(rb, count);
281
282out:
283 return ret;
284}
285
286
287
288
289
290
291void pblk_rb_sync_l2p(struct pblk_rb *rb)
292{
293 unsigned int sync;
294 unsigned int to_update;
295
296 spin_lock(&rb->w_lock);
297
298
299 sync = smp_load_acquire(&rb->sync);
300
301 to_update = pblk_rb_ring_count(sync, rb->l2p_update, rb->nr_entries);
302 __pblk_rb_update_l2p(rb, to_update);
303
304 spin_unlock(&rb->w_lock);
305}
306
307
308
309
310
311
312
313static void __pblk_rb_write_entry(struct pblk_rb *rb, void *data,
314 struct pblk_w_ctx w_ctx,
315 struct pblk_rb_entry *entry)
316{
317 memcpy(entry->data, data, rb->seg_size);
318
319 entry->w_ctx.lba = w_ctx.lba;
320 entry->w_ctx.ppa = w_ctx.ppa;
321}
322
323void pblk_rb_write_entry_user(struct pblk_rb *rb, void *data,
324 struct pblk_w_ctx w_ctx, unsigned int ring_pos)
325{
326 struct pblk *pblk = container_of(rb, struct pblk, rwb);
327 struct pblk_rb_entry *entry;
328 int flags;
329
330 entry = &rb->entries[ring_pos];
331 flags = READ_ONCE(entry->w_ctx.flags);
332#ifdef CONFIG_NVM_PBLK_DEBUG
333
334 BUG_ON(!(flags & PBLK_WRITABLE_ENTRY));
335#endif
336
337 __pblk_rb_write_entry(rb, data, w_ctx, entry);
338
339 pblk_update_map_cache(pblk, w_ctx.lba, entry->cacheline);
340 flags = w_ctx.flags | PBLK_WRITTEN_DATA;
341
342
343 smp_store_release(&entry->w_ctx.flags, flags);
344}
345
346void pblk_rb_write_entry_gc(struct pblk_rb *rb, void *data,
347 struct pblk_w_ctx w_ctx, struct pblk_line *line,
348 u64 paddr, unsigned int ring_pos)
349{
350 struct pblk *pblk = container_of(rb, struct pblk, rwb);
351 struct pblk_rb_entry *entry;
352 int flags;
353
354 entry = &rb->entries[ring_pos];
355 flags = READ_ONCE(entry->w_ctx.flags);
356#ifdef CONFIG_NVM_PBLK_DEBUG
357
358 BUG_ON(!(flags & PBLK_WRITABLE_ENTRY));
359#endif
360
361 __pblk_rb_write_entry(rb, data, w_ctx, entry);
362
363 if (!pblk_update_map_gc(pblk, w_ctx.lba, entry->cacheline, line, paddr))
364 entry->w_ctx.lba = ADDR_EMPTY;
365
366 flags = w_ctx.flags | PBLK_WRITTEN_DATA;
367
368
369 smp_store_release(&entry->w_ctx.flags, flags);
370}
371
372static int pblk_rb_flush_point_set(struct pblk_rb *rb, struct bio *bio,
373 unsigned int pos)
374{
375 struct pblk_rb_entry *entry;
376 unsigned int sync, flush_point;
377
378 pblk_rb_sync_init(rb, NULL);
379 sync = READ_ONCE(rb->sync);
380
381 if (pos == sync) {
382 pblk_rb_sync_end(rb, NULL);
383 return 0;
384 }
385
386#ifdef CONFIG_NVM_PBLK_DEBUG
387 atomic_inc(&rb->inflight_flush_point);
388#endif
389
390 flush_point = (pos == 0) ? (rb->nr_entries - 1) : (pos - 1);
391 entry = &rb->entries[flush_point];
392
393
394 smp_store_release(&rb->flush_point, flush_point);
395
396 if (bio)
397 bio_list_add(&entry->w_ctx.bios, bio);
398
399 pblk_rb_sync_end(rb, NULL);
400
401 return bio ? 1 : 0;
402}
403
404static int __pblk_rb_may_write(struct pblk_rb *rb, unsigned int nr_entries,
405 unsigned int *pos)
406{
407 unsigned int mem;
408 unsigned int sync;
409 unsigned int threshold;
410
411 sync = READ_ONCE(rb->sync);
412 mem = READ_ONCE(rb->mem);
413
414 threshold = nr_entries + rb->back_thres;
415
416 if (pblk_rb_ring_space(rb, mem, sync, rb->nr_entries) < threshold)
417 return 0;
418
419 if (pblk_rb_update_l2p(rb, nr_entries, mem, sync))
420 return 0;
421
422 *pos = mem;
423
424 return 1;
425}
426
427static int pblk_rb_may_write(struct pblk_rb *rb, unsigned int nr_entries,
428 unsigned int *pos)
429{
430 if (!__pblk_rb_may_write(rb, nr_entries, pos))
431 return 0;
432
433
434 smp_store_release(&rb->mem, pblk_rb_ptr_wrap(rb, *pos, nr_entries));
435 return 1;
436}
437
438void pblk_rb_flush(struct pblk_rb *rb)
439{
440 struct pblk *pblk = container_of(rb, struct pblk, rwb);
441 unsigned int mem = READ_ONCE(rb->mem);
442
443 if (pblk_rb_flush_point_set(rb, NULL, mem))
444 return;
445
446 pblk_write_kick(pblk);
447}
448
449static int pblk_rb_may_write_flush(struct pblk_rb *rb, unsigned int nr_entries,
450 unsigned int *pos, struct bio *bio,
451 int *io_ret)
452{
453 unsigned int mem;
454
455 if (!__pblk_rb_may_write(rb, nr_entries, pos))
456 return 0;
457
458 mem = pblk_rb_ptr_wrap(rb, *pos, nr_entries);
459 *io_ret = NVM_IO_DONE;
460
461 if (bio->bi_opf & REQ_PREFLUSH) {
462 struct pblk *pblk = container_of(rb, struct pblk, rwb);
463
464 atomic64_inc(&pblk->nr_flush);
465 if (pblk_rb_flush_point_set(&pblk->rwb, bio, mem))
466 *io_ret = NVM_IO_OK;
467 }
468
469
470 smp_store_release(&rb->mem, mem);
471
472 return 1;
473}
474
475
476
477
478
479
480int pblk_rb_may_write_user(struct pblk_rb *rb, struct bio *bio,
481 unsigned int nr_entries, unsigned int *pos)
482{
483 struct pblk *pblk = container_of(rb, struct pblk, rwb);
484 int io_ret;
485
486 spin_lock(&rb->w_lock);
487 io_ret = pblk_rl_user_may_insert(&pblk->rl, nr_entries);
488 if (io_ret) {
489 spin_unlock(&rb->w_lock);
490 return io_ret;
491 }
492
493 if (!pblk_rb_may_write_flush(rb, nr_entries, pos, bio, &io_ret)) {
494 spin_unlock(&rb->w_lock);
495 return NVM_IO_REQUEUE;
496 }
497
498 pblk_rl_user_in(&pblk->rl, nr_entries);
499 spin_unlock(&rb->w_lock);
500
501 return io_ret;
502}
503
504
505
506
507int pblk_rb_may_write_gc(struct pblk_rb *rb, unsigned int nr_entries,
508 unsigned int *pos)
509{
510 struct pblk *pblk = container_of(rb, struct pblk, rwb);
511
512 spin_lock(&rb->w_lock);
513 if (!pblk_rl_gc_may_insert(&pblk->rl, nr_entries)) {
514 spin_unlock(&rb->w_lock);
515 return 0;
516 }
517
518 if (!pblk_rb_may_write(rb, nr_entries, pos)) {
519 spin_unlock(&rb->w_lock);
520 return 0;
521 }
522
523 pblk_rl_gc_in(&pblk->rl, nr_entries);
524 spin_unlock(&rb->w_lock);
525
526 return 1;
527}
528
529
530
531
532
533
534
535
536unsigned int pblk_rb_read_to_bio(struct pblk_rb *rb, struct nvm_rq *rqd,
537 unsigned int pos, unsigned int nr_entries,
538 unsigned int count)
539{
540 struct pblk *pblk = container_of(rb, struct pblk, rwb);
541 struct request_queue *q = pblk->dev->q;
542 struct pblk_c_ctx *c_ctx = nvm_rq_to_pdu(rqd);
543 struct bio *bio = rqd->bio;
544 struct pblk_rb_entry *entry;
545 struct page *page;
546 unsigned int pad = 0, to_read = nr_entries;
547 unsigned int i;
548 int flags;
549
550 if (count < nr_entries) {
551 pad = nr_entries - count;
552 to_read = count;
553 }
554
555 c_ctx->sentry = pos;
556 c_ctx->nr_valid = to_read;
557 c_ctx->nr_padded = pad;
558
559 for (i = 0; i < to_read; i++) {
560 entry = &rb->entries[pos];
561
562
563
564
565try:
566 flags = READ_ONCE(entry->w_ctx.flags);
567 if (!(flags & PBLK_WRITTEN_DATA)) {
568 io_schedule();
569 goto try;
570 }
571
572 page = virt_to_page(entry->data);
573 if (!page) {
574 pblk_err(pblk, "could not allocate write bio page\n");
575 flags &= ~PBLK_WRITTEN_DATA;
576 flags |= PBLK_SUBMITTED_ENTRY;
577
578 smp_store_release(&entry->w_ctx.flags, flags);
579 return NVM_IO_ERR;
580 }
581
582 if (bio_add_pc_page(q, bio, page, rb->seg_size, 0) !=
583 rb->seg_size) {
584 pblk_err(pblk, "could not add page to write bio\n");
585 flags &= ~PBLK_WRITTEN_DATA;
586 flags |= PBLK_SUBMITTED_ENTRY;
587
588 smp_store_release(&entry->w_ctx.flags, flags);
589 return NVM_IO_ERR;
590 }
591
592 flags &= ~PBLK_WRITTEN_DATA;
593 flags |= PBLK_SUBMITTED_ENTRY;
594
595
596 smp_store_release(&entry->w_ctx.flags, flags);
597
598 pos = pblk_rb_ptr_wrap(rb, pos, 1);
599 }
600
601 if (pad) {
602 if (pblk_bio_add_pages(pblk, bio, GFP_KERNEL, pad)) {
603 pblk_err(pblk, "could not pad page in write bio\n");
604 return NVM_IO_ERR;
605 }
606
607 if (pad < pblk->min_write_pgs)
608 atomic64_inc(&pblk->pad_dist[pad - 1]);
609 else
610 pblk_warn(pblk, "padding more than min. sectors\n");
611
612 atomic64_add(pad, &pblk->pad_wa);
613 }
614
615#ifdef CONFIG_NVM_PBLK_DEBUG
616 atomic_long_add(pad, &pblk->padded_writes);
617#endif
618
619 return NVM_IO_OK;
620}
621
622
623
624
625
626
627int pblk_rb_copy_to_bio(struct pblk_rb *rb, struct bio *bio, sector_t lba,
628 struct ppa_addr ppa, int bio_iter, bool advanced_bio)
629{
630 struct pblk *pblk = container_of(rb, struct pblk, rwb);
631 struct pblk_rb_entry *entry;
632 struct pblk_w_ctx *w_ctx;
633 struct ppa_addr l2p_ppa;
634 u64 pos = pblk_addr_to_cacheline(ppa);
635 void *data;
636 int flags;
637 int ret = 1;
638
639
640#ifdef CONFIG_NVM_PBLK_DEBUG
641
642 BUG_ON(pos >= rb->nr_entries);
643#endif
644 entry = &rb->entries[pos];
645 w_ctx = &entry->w_ctx;
646 flags = READ_ONCE(w_ctx->flags);
647
648 spin_lock(&rb->w_lock);
649 spin_lock(&pblk->trans_lock);
650 l2p_ppa = pblk_trans_map_get(pblk, lba);
651 spin_unlock(&pblk->trans_lock);
652
653
654 if (!pblk_ppa_comp(l2p_ppa, ppa) || w_ctx->lba != lba ||
655 flags & PBLK_WRITABLE_ENTRY) {
656 ret = 0;
657 goto out;
658 }
659
660
661
662
663
664
665 if (unlikely(!advanced_bio))
666 bio_advance(bio, bio_iter * PBLK_EXPOSED_PAGE_SIZE);
667
668 data = bio_data(bio);
669 memcpy(data, entry->data, rb->seg_size);
670
671out:
672 spin_unlock(&rb->w_lock);
673 return ret;
674}
675
676struct pblk_w_ctx *pblk_rb_w_ctx(struct pblk_rb *rb, unsigned int pos)
677{
678 unsigned int entry = pblk_rb_ptr_wrap(rb, pos, 0);
679
680 return &rb->entries[entry].w_ctx;
681}
682
683unsigned int pblk_rb_sync_init(struct pblk_rb *rb, unsigned long *flags)
684 __acquires(&rb->s_lock)
685{
686 if (flags)
687 spin_lock_irqsave(&rb->s_lock, *flags);
688 else
689 spin_lock_irq(&rb->s_lock);
690
691 return rb->sync;
692}
693
694void pblk_rb_sync_end(struct pblk_rb *rb, unsigned long *flags)
695 __releases(&rb->s_lock)
696{
697 lockdep_assert_held(&rb->s_lock);
698
699 if (flags)
700 spin_unlock_irqrestore(&rb->s_lock, *flags);
701 else
702 spin_unlock_irq(&rb->s_lock);
703}
704
705unsigned int pblk_rb_sync_advance(struct pblk_rb *rb, unsigned int nr_entries)
706{
707 unsigned int sync, flush_point;
708 lockdep_assert_held(&rb->s_lock);
709
710 sync = READ_ONCE(rb->sync);
711 flush_point = READ_ONCE(rb->flush_point);
712
713 if (flush_point != EMPTY_ENTRY) {
714 unsigned int secs_to_flush;
715
716 secs_to_flush = pblk_rb_ring_count(flush_point, sync,
717 rb->nr_entries);
718 if (secs_to_flush < nr_entries) {
719
720 smp_store_release(&rb->flush_point, EMPTY_ENTRY);
721 }
722 }
723
724 sync = pblk_rb_ptr_wrap(rb, sync, nr_entries);
725
726
727 smp_store_release(&rb->sync, sync);
728
729 return sync;
730}
731
732
733unsigned int pblk_rb_flush_point_count(struct pblk_rb *rb)
734{
735 unsigned int subm, sync, flush_point;
736 unsigned int submitted, to_flush;
737
738
739 flush_point = smp_load_acquire(&rb->flush_point);
740 if (flush_point == EMPTY_ENTRY)
741 return 0;
742
743
744 sync = smp_load_acquire(&rb->sync);
745
746 subm = READ_ONCE(rb->subm);
747 submitted = pblk_rb_ring_count(subm, sync, rb->nr_entries);
748
749
750 to_flush = pblk_rb_ring_count(flush_point, sync, rb->nr_entries) + 1;
751
752 return (submitted < to_flush) ? (to_flush - submitted) : 0;
753}
754
755int pblk_rb_tear_down_check(struct pblk_rb *rb)
756{
757 struct pblk_rb_entry *entry;
758 int i;
759 int ret = 0;
760
761 spin_lock(&rb->w_lock);
762 spin_lock_irq(&rb->s_lock);
763
764 if ((rb->mem == rb->subm) && (rb->subm == rb->sync) &&
765 (rb->sync == rb->l2p_update) &&
766 (rb->flush_point == EMPTY_ENTRY)) {
767 goto out;
768 }
769
770 if (!rb->entries) {
771 ret = 1;
772 goto out;
773 }
774
775 for (i = 0; i < rb->nr_entries; i++) {
776 entry = &rb->entries[i];
777
778 if (!entry->data) {
779 ret = 1;
780 goto out;
781 }
782 }
783
784out:
785 spin_unlock(&rb->w_lock);
786 spin_unlock_irq(&rb->s_lock);
787
788 return ret;
789}
790
791unsigned int pblk_rb_wrap_pos(struct pblk_rb *rb, unsigned int pos)
792{
793 return (pos & (rb->nr_entries - 1));
794}
795
796int pblk_rb_pos_oob(struct pblk_rb *rb, u64 pos)
797{
798 return (pos >= rb->nr_entries);
799}
800
801ssize_t pblk_rb_sysfs(struct pblk_rb *rb, char *buf)
802{
803 struct pblk *pblk = container_of(rb, struct pblk, rwb);
804 struct pblk_c_ctx *c;
805 ssize_t offset;
806 int queued_entries = 0;
807
808 spin_lock_irq(&rb->s_lock);
809 list_for_each_entry(c, &pblk->compl_list, list)
810 queued_entries++;
811 spin_unlock_irq(&rb->s_lock);
812
813 if (rb->flush_point != EMPTY_ENTRY)
814 offset = scnprintf(buf, PAGE_SIZE,
815 "%u\t%u\t%u\t%u\t%u\t%u\t%u - %u/%u/%u - %d\n",
816 rb->nr_entries,
817 rb->mem,
818 rb->subm,
819 rb->sync,
820 rb->l2p_update,
821#ifdef CONFIG_NVM_PBLK_DEBUG
822 atomic_read(&rb->inflight_flush_point),
823#else
824 0,
825#endif
826 rb->flush_point,
827 pblk_rb_read_count(rb),
828 pblk_rb_space(rb),
829 pblk_rb_flush_point_count(rb),
830 queued_entries);
831 else
832 offset = scnprintf(buf, PAGE_SIZE,
833 "%u\t%u\t%u\t%u\t%u\t%u\tNULL - %u/%u/%u - %d\n",
834 rb->nr_entries,
835 rb->mem,
836 rb->subm,
837 rb->sync,
838 rb->l2p_update,
839#ifdef CONFIG_NVM_PBLK_DEBUG
840 atomic_read(&rb->inflight_flush_point),
841#else
842 0,
843#endif
844 pblk_rb_read_count(rb),
845 pblk_rb_space(rb),
846 pblk_rb_flush_point_count(rb),
847 queued_entries);
848
849 return offset;
850}
851