1
2
3
4
5
6
7
8#ifndef _LINUX_KFIFO_H
9#define _LINUX_KFIFO_H
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39#include <linux/kernel.h>
40#include <linux/spinlock.h>
41#include <linux/stddef.h>
42#include <linux/scatterlist.h>
43
44struct __kfifo {
45 unsigned int in;
46 unsigned int out;
47 unsigned int mask;
48 unsigned int esize;
49 void *data;
50};
51
52#define __STRUCT_KFIFO_COMMON(datatype, recsize, ptrtype) \
53 union { \
54 struct __kfifo kfifo; \
55 datatype *type; \
56 const datatype *const_type; \
57 char (*rectype)[recsize]; \
58 ptrtype *ptr; \
59 ptrtype const *ptr_const; \
60 }
61
62#define __STRUCT_KFIFO(type, size, recsize, ptrtype) \
63{ \
64 __STRUCT_KFIFO_COMMON(type, recsize, ptrtype); \
65 type buf[((size < 2) || (size & (size - 1))) ? -1 : size]; \
66}
67
68#define STRUCT_KFIFO(type, size) \
69 struct __STRUCT_KFIFO(type, size, 0, type)
70
71#define __STRUCT_KFIFO_PTR(type, recsize, ptrtype) \
72{ \
73 __STRUCT_KFIFO_COMMON(type, recsize, ptrtype); \
74 type buf[0]; \
75}
76
77#define STRUCT_KFIFO_PTR(type) \
78 struct __STRUCT_KFIFO_PTR(type, 0, type)
79
80
81
82
83struct kfifo __STRUCT_KFIFO_PTR(unsigned char, 0, void);
84
85#define STRUCT_KFIFO_REC_1(size) \
86 struct __STRUCT_KFIFO(unsigned char, size, 1, void)
87
88#define STRUCT_KFIFO_REC_2(size) \
89 struct __STRUCT_KFIFO(unsigned char, size, 2, void)
90
91
92
93
94struct kfifo_rec_ptr_1 __STRUCT_KFIFO_PTR(unsigned char, 1, void);
95struct kfifo_rec_ptr_2 __STRUCT_KFIFO_PTR(unsigned char, 2, void);
96
97
98
99
100
101
102#define __is_kfifo_ptr(fifo) \
103 (sizeof(*fifo) == sizeof(STRUCT_KFIFO_PTR(typeof(*(fifo)->type))))
104
105
106
107
108
109
110#define DECLARE_KFIFO_PTR(fifo, type) STRUCT_KFIFO_PTR(type) fifo
111
112
113
114
115
116
117
118#define DECLARE_KFIFO(fifo, type, size) STRUCT_KFIFO(type, size) fifo
119
120
121
122
123
124#define INIT_KFIFO(fifo) \
125(void)({ \
126 typeof(&(fifo)) __tmp = &(fifo); \
127 struct __kfifo *__kfifo = &__tmp->kfifo; \
128 __kfifo->in = 0; \
129 __kfifo->out = 0; \
130 __kfifo->mask = __is_kfifo_ptr(__tmp) ? 0 : ARRAY_SIZE(__tmp->buf) - 1;\
131 __kfifo->esize = sizeof(*__tmp->buf); \
132 __kfifo->data = __is_kfifo_ptr(__tmp) ? NULL : __tmp->buf; \
133})
134
135
136
137
138
139
140
141
142
143#define DEFINE_KFIFO(fifo, type, size) \
144 DECLARE_KFIFO(fifo, type, size) = \
145 (typeof(fifo)) { \
146 { \
147 { \
148 .in = 0, \
149 .out = 0, \
150 .mask = __is_kfifo_ptr(&(fifo)) ? \
151 0 : \
152 ARRAY_SIZE((fifo).buf) - 1, \
153 .esize = sizeof(*(fifo).buf), \
154 .data = __is_kfifo_ptr(&(fifo)) ? \
155 NULL : \
156 (fifo).buf, \
157 } \
158 } \
159 }
160
161
162static inline unsigned int __must_check
163__kfifo_uint_must_check_helper(unsigned int val)
164{
165 return val;
166}
167
168static inline int __must_check
169__kfifo_int_must_check_helper(int val)
170{
171 return val;
172}
173
174
175
176
177
178
179
180
181#define kfifo_initialized(fifo) ((fifo)->kfifo.mask)
182
183
184
185
186
187#define kfifo_esize(fifo) ((fifo)->kfifo.esize)
188
189
190
191
192
193#define kfifo_recsize(fifo) (sizeof(*(fifo)->rectype))
194
195
196
197
198
199#define kfifo_size(fifo) ((fifo)->kfifo.mask + 1)
200
201
202
203
204
205
206
207
208
209#define kfifo_reset(fifo) \
210(void)({ \
211 typeof((fifo) + 1) __tmp = (fifo); \
212 __tmp->kfifo.in = __tmp->kfifo.out = 0; \
213})
214
215
216
217
218
219
220
221
222
223#define kfifo_reset_out(fifo) \
224(void)({ \
225 typeof((fifo) + 1) __tmp = (fifo); \
226 __tmp->kfifo.out = __tmp->kfifo.in; \
227})
228
229
230
231
232
233#define kfifo_len(fifo) \
234({ \
235 typeof((fifo) + 1) __tmpl = (fifo); \
236 __tmpl->kfifo.in - __tmpl->kfifo.out; \
237})
238
239
240
241
242
243#define kfifo_is_empty(fifo) \
244({ \
245 typeof((fifo) + 1) __tmpq = (fifo); \
246 __tmpq->kfifo.in == __tmpq->kfifo.out; \
247})
248
249
250
251
252
253
254
255#define kfifo_is_empty_spinlocked(fifo, lock) \
256({ \
257 unsigned long __flags; \
258 bool __ret; \
259 spin_lock_irqsave(lock, __flags); \
260 __ret = kfifo_is_empty(fifo); \
261 spin_unlock_irqrestore(lock, __flags); \
262 __ret; \
263})
264
265
266
267
268
269
270
271#define kfifo_is_empty_spinlocked_noirqsave(fifo, lock) \
272({ \
273 bool __ret; \
274 spin_lock(lock); \
275 __ret = kfifo_is_empty(fifo); \
276 spin_unlock(lock); \
277 __ret; \
278})
279
280
281
282
283
284#define kfifo_is_full(fifo) \
285({ \
286 typeof((fifo) + 1) __tmpq = (fifo); \
287 kfifo_len(__tmpq) > __tmpq->kfifo.mask; \
288})
289
290
291
292
293
294#define kfifo_avail(fifo) \
295__kfifo_uint_must_check_helper( \
296({ \
297 typeof((fifo) + 1) __tmpq = (fifo); \
298 const size_t __recsize = sizeof(*__tmpq->rectype); \
299 unsigned int __avail = kfifo_size(__tmpq) - kfifo_len(__tmpq); \
300 (__recsize) ? ((__avail <= __recsize) ? 0 : \
301 __kfifo_max_r(__avail - __recsize, __recsize)) : \
302 __avail; \
303}) \
304)
305
306
307
308
309
310#define kfifo_skip(fifo) \
311(void)({ \
312 typeof((fifo) + 1) __tmp = (fifo); \
313 const size_t __recsize = sizeof(*__tmp->rectype); \
314 struct __kfifo *__kfifo = &__tmp->kfifo; \
315 if (__recsize) \
316 __kfifo_skip_r(__kfifo, __recsize); \
317 else \
318 __kfifo->out++; \
319})
320
321
322
323
324
325
326
327#define kfifo_peek_len(fifo) \
328__kfifo_uint_must_check_helper( \
329({ \
330 typeof((fifo) + 1) __tmp = (fifo); \
331 const size_t __recsize = sizeof(*__tmp->rectype); \
332 struct __kfifo *__kfifo = &__tmp->kfifo; \
333 (!__recsize) ? kfifo_len(__tmp) * sizeof(*__tmp->type) : \
334 __kfifo_len_r(__kfifo, __recsize); \
335}) \
336)
337
338
339
340
341
342
343
344
345
346
347
348
349
350#define kfifo_alloc(fifo, size, gfp_mask) \
351__kfifo_int_must_check_helper( \
352({ \
353 typeof((fifo) + 1) __tmp = (fifo); \
354 struct __kfifo *__kfifo = &__tmp->kfifo; \
355 __is_kfifo_ptr(__tmp) ? \
356 __kfifo_alloc(__kfifo, size, sizeof(*__tmp->type), gfp_mask) : \
357 -EINVAL; \
358}) \
359)
360
361
362
363
364
365#define kfifo_free(fifo) \
366({ \
367 typeof((fifo) + 1) __tmp = (fifo); \
368 struct __kfifo *__kfifo = &__tmp->kfifo; \
369 if (__is_kfifo_ptr(__tmp)) \
370 __kfifo_free(__kfifo); \
371})
372
373
374
375
376
377
378
379
380
381
382
383
384#define kfifo_init(fifo, buffer, size) \
385({ \
386 typeof((fifo) + 1) __tmp = (fifo); \
387 struct __kfifo *__kfifo = &__tmp->kfifo; \
388 __is_kfifo_ptr(__tmp) ? \
389 __kfifo_init(__kfifo, buffer, size, sizeof(*__tmp->type)) : \
390 -EINVAL; \
391})
392
393
394
395
396
397
398
399
400
401
402
403
404
405#define kfifo_put(fifo, val) \
406({ \
407 typeof((fifo) + 1) __tmp = (fifo); \
408 typeof(*__tmp->const_type) __val = (val); \
409 unsigned int __ret; \
410 size_t __recsize = sizeof(*__tmp->rectype); \
411 struct __kfifo *__kfifo = &__tmp->kfifo; \
412 if (__recsize) \
413 __ret = __kfifo_in_r(__kfifo, &__val, sizeof(__val), \
414 __recsize); \
415 else { \
416 __ret = !kfifo_is_full(__tmp); \
417 if (__ret) { \
418 (__is_kfifo_ptr(__tmp) ? \
419 ((typeof(__tmp->type))__kfifo->data) : \
420 (__tmp->buf) \
421 )[__kfifo->in & __tmp->kfifo.mask] = \
422 *(typeof(__tmp->type))&__val; \
423 smp_wmb(); \
424 __kfifo->in++; \
425 } \
426 } \
427 __ret; \
428})
429
430
431
432
433
434
435
436
437
438
439
440
441
442#define kfifo_get(fifo, val) \
443__kfifo_uint_must_check_helper( \
444({ \
445 typeof((fifo) + 1) __tmp = (fifo); \
446 typeof(__tmp->ptr) __val = (val); \
447 unsigned int __ret; \
448 const size_t __recsize = sizeof(*__tmp->rectype); \
449 struct __kfifo *__kfifo = &__tmp->kfifo; \
450 if (__recsize) \
451 __ret = __kfifo_out_r(__kfifo, __val, sizeof(*__val), \
452 __recsize); \
453 else { \
454 __ret = !kfifo_is_empty(__tmp); \
455 if (__ret) { \
456 *(typeof(__tmp->type))__val = \
457 (__is_kfifo_ptr(__tmp) ? \
458 ((typeof(__tmp->type))__kfifo->data) : \
459 (__tmp->buf) \
460 )[__kfifo->out & __tmp->kfifo.mask]; \
461 smp_wmb(); \
462 __kfifo->out++; \
463 } \
464 } \
465 __ret; \
466}) \
467)
468
469
470
471
472
473
474
475
476
477
478
479
480
481#define kfifo_peek(fifo, val) \
482__kfifo_uint_must_check_helper( \
483({ \
484 typeof((fifo) + 1) __tmp = (fifo); \
485 typeof(__tmp->ptr) __val = (val); \
486 unsigned int __ret; \
487 const size_t __recsize = sizeof(*__tmp->rectype); \
488 struct __kfifo *__kfifo = &__tmp->kfifo; \
489 if (__recsize) \
490 __ret = __kfifo_out_peek_r(__kfifo, __val, sizeof(*__val), \
491 __recsize); \
492 else { \
493 __ret = !kfifo_is_empty(__tmp); \
494 if (__ret) { \
495 *(typeof(__tmp->type))__val = \
496 (__is_kfifo_ptr(__tmp) ? \
497 ((typeof(__tmp->type))__kfifo->data) : \
498 (__tmp->buf) \
499 )[__kfifo->out & __tmp->kfifo.mask]; \
500 smp_wmb(); \
501 } \
502 } \
503 __ret; \
504}) \
505)
506
507
508
509
510
511
512
513
514
515
516
517
518
519#define kfifo_in(fifo, buf, n) \
520({ \
521 typeof((fifo) + 1) __tmp = (fifo); \
522 typeof(__tmp->ptr_const) __buf = (buf); \
523 unsigned long __n = (n); \
524 const size_t __recsize = sizeof(*__tmp->rectype); \
525 struct __kfifo *__kfifo = &__tmp->kfifo; \
526 (__recsize) ?\
527 __kfifo_in_r(__kfifo, __buf, __n, __recsize) : \
528 __kfifo_in(__kfifo, __buf, __n); \
529})
530
531
532
533
534
535
536
537
538
539
540
541#define kfifo_in_spinlocked(fifo, buf, n, lock) \
542({ \
543 unsigned long __flags; \
544 unsigned int __ret; \
545 spin_lock_irqsave(lock, __flags); \
546 __ret = kfifo_in(fifo, buf, n); \
547 spin_unlock_irqrestore(lock, __flags); \
548 __ret; \
549})
550
551
552
553
554
555
556
557
558
559
560
561
562#define kfifo_in_spinlocked_noirqsave(fifo, buf, n, lock) \
563({ \
564 unsigned int __ret; \
565 spin_lock(lock); \
566 __ret = kfifo_in(fifo, buf, n); \
567 spin_unlock(lock); \
568 __ret; \
569})
570
571
572#define kfifo_in_locked(fifo, buf, n, lock) \
573 kfifo_in_spinlocked(fifo, buf, n, lock)
574
575
576
577
578
579
580
581
582
583
584
585
586
587#define kfifo_out(fifo, buf, n) \
588__kfifo_uint_must_check_helper( \
589({ \
590 typeof((fifo) + 1) __tmp = (fifo); \
591 typeof(__tmp->ptr) __buf = (buf); \
592 unsigned long __n = (n); \
593 const size_t __recsize = sizeof(*__tmp->rectype); \
594 struct __kfifo *__kfifo = &__tmp->kfifo; \
595 (__recsize) ?\
596 __kfifo_out_r(__kfifo, __buf, __n, __recsize) : \
597 __kfifo_out(__kfifo, __buf, __n); \
598}) \
599)
600
601
602
603
604
605
606
607
608
609
610
611#define kfifo_out_spinlocked(fifo, buf, n, lock) \
612__kfifo_uint_must_check_helper( \
613({ \
614 unsigned long __flags; \
615 unsigned int __ret; \
616 spin_lock_irqsave(lock, __flags); \
617 __ret = kfifo_out(fifo, buf, n); \
618 spin_unlock_irqrestore(lock, __flags); \
619 __ret; \
620}) \
621)
622
623
624
625
626
627
628
629
630
631
632
633
634#define kfifo_out_spinlocked_noirqsave(fifo, buf, n, lock) \
635__kfifo_uint_must_check_helper( \
636({ \
637 unsigned int __ret; \
638 spin_lock(lock); \
639 __ret = kfifo_out(fifo, buf, n); \
640 spin_unlock(lock); \
641 __ret; \
642}) \
643)
644
645
646#define kfifo_out_locked(fifo, buf, n, lock) \
647 kfifo_out_spinlocked(fifo, buf, n, lock)
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662#define kfifo_from_user(fifo, from, len, copied) \
663__kfifo_uint_must_check_helper( \
664({ \
665 typeof((fifo) + 1) __tmp = (fifo); \
666 const void __user *__from = (from); \
667 unsigned int __len = (len); \
668 unsigned int *__copied = (copied); \
669 const size_t __recsize = sizeof(*__tmp->rectype); \
670 struct __kfifo *__kfifo = &__tmp->kfifo; \
671 (__recsize) ? \
672 __kfifo_from_user_r(__kfifo, __from, __len, __copied, __recsize) : \
673 __kfifo_from_user(__kfifo, __from, __len, __copied); \
674}) \
675)
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690#define kfifo_to_user(fifo, to, len, copied) \
691__kfifo_uint_must_check_helper( \
692({ \
693 typeof((fifo) + 1) __tmp = (fifo); \
694 void __user *__to = (to); \
695 unsigned int __len = (len); \
696 unsigned int *__copied = (copied); \
697 const size_t __recsize = sizeof(*__tmp->rectype); \
698 struct __kfifo *__kfifo = &__tmp->kfifo; \
699 (__recsize) ? \
700 __kfifo_to_user_r(__kfifo, __to, __len, __copied, __recsize) : \
701 __kfifo_to_user(__kfifo, __to, __len, __copied); \
702}) \
703)
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718#define kfifo_dma_in_prepare(fifo, sgl, nents, len) \
719({ \
720 typeof((fifo) + 1) __tmp = (fifo); \
721 struct scatterlist *__sgl = (sgl); \
722 int __nents = (nents); \
723 unsigned int __len = (len); \
724 const size_t __recsize = sizeof(*__tmp->rectype); \
725 struct __kfifo *__kfifo = &__tmp->kfifo; \
726 (__recsize) ? \
727 __kfifo_dma_in_prepare_r(__kfifo, __sgl, __nents, __len, __recsize) : \
728 __kfifo_dma_in_prepare(__kfifo, __sgl, __nents, __len); \
729})
730
731
732
733
734
735
736
737
738
739
740
741
742#define kfifo_dma_in_finish(fifo, len) \
743(void)({ \
744 typeof((fifo) + 1) __tmp = (fifo); \
745 unsigned int __len = (len); \
746 const size_t __recsize = sizeof(*__tmp->rectype); \
747 struct __kfifo *__kfifo = &__tmp->kfifo; \
748 if (__recsize) \
749 __kfifo_dma_in_finish_r(__kfifo, __len, __recsize); \
750 else \
751 __kfifo->in += __len / sizeof(*__tmp->type); \
752})
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769#define kfifo_dma_out_prepare(fifo, sgl, nents, len) \
770({ \
771 typeof((fifo) + 1) __tmp = (fifo); \
772 struct scatterlist *__sgl = (sgl); \
773 int __nents = (nents); \
774 unsigned int __len = (len); \
775 const size_t __recsize = sizeof(*__tmp->rectype); \
776 struct __kfifo *__kfifo = &__tmp->kfifo; \
777 (__recsize) ? \
778 __kfifo_dma_out_prepare_r(__kfifo, __sgl, __nents, __len, __recsize) : \
779 __kfifo_dma_out_prepare(__kfifo, __sgl, __nents, __len); \
780})
781
782
783
784
785
786
787
788
789
790
791
792
793#define kfifo_dma_out_finish(fifo, len) \
794(void)({ \
795 typeof((fifo) + 1) __tmp = (fifo); \
796 unsigned int __len = (len); \
797 const size_t __recsize = sizeof(*__tmp->rectype); \
798 struct __kfifo *__kfifo = &__tmp->kfifo; \
799 if (__recsize) \
800 __kfifo_dma_out_finish_r(__kfifo, __recsize); \
801 else \
802 __kfifo->out += __len / sizeof(*__tmp->type); \
803})
804
805
806
807
808
809
810
811
812
813
814
815
816
817#define kfifo_out_peek(fifo, buf, n) \
818__kfifo_uint_must_check_helper( \
819({ \
820 typeof((fifo) + 1) __tmp = (fifo); \
821 typeof(__tmp->ptr) __buf = (buf); \
822 unsigned long __n = (n); \
823 const size_t __recsize = sizeof(*__tmp->rectype); \
824 struct __kfifo *__kfifo = &__tmp->kfifo; \
825 (__recsize) ? \
826 __kfifo_out_peek_r(__kfifo, __buf, __n, __recsize) : \
827 __kfifo_out_peek(__kfifo, __buf, __n); \
828}) \
829)
830
831extern int __kfifo_alloc(struct __kfifo *fifo, unsigned int size,
832 size_t esize, gfp_t gfp_mask);
833
834extern void __kfifo_free(struct __kfifo *fifo);
835
836extern int __kfifo_init(struct __kfifo *fifo, void *buffer,
837 unsigned int size, size_t esize);
838
839extern unsigned int __kfifo_in(struct __kfifo *fifo,
840 const void *buf, unsigned int len);
841
842extern unsigned int __kfifo_out(struct __kfifo *fifo,
843 void *buf, unsigned int len);
844
845extern int __kfifo_from_user(struct __kfifo *fifo,
846 const void __user *from, unsigned long len, unsigned int *copied);
847
848extern int __kfifo_to_user(struct __kfifo *fifo,
849 void __user *to, unsigned long len, unsigned int *copied);
850
851extern unsigned int __kfifo_dma_in_prepare(struct __kfifo *fifo,
852 struct scatterlist *sgl, int nents, unsigned int len);
853
854extern unsigned int __kfifo_dma_out_prepare(struct __kfifo *fifo,
855 struct scatterlist *sgl, int nents, unsigned int len);
856
857extern unsigned int __kfifo_out_peek(struct __kfifo *fifo,
858 void *buf, unsigned int len);
859
860extern unsigned int __kfifo_in_r(struct __kfifo *fifo,
861 const void *buf, unsigned int len, size_t recsize);
862
863extern unsigned int __kfifo_out_r(struct __kfifo *fifo,
864 void *buf, unsigned int len, size_t recsize);
865
866extern int __kfifo_from_user_r(struct __kfifo *fifo,
867 const void __user *from, unsigned long len, unsigned int *copied,
868 size_t recsize);
869
870extern int __kfifo_to_user_r(struct __kfifo *fifo, void __user *to,
871 unsigned long len, unsigned int *copied, size_t recsize);
872
873extern unsigned int __kfifo_dma_in_prepare_r(struct __kfifo *fifo,
874 struct scatterlist *sgl, int nents, unsigned int len, size_t recsize);
875
876extern void __kfifo_dma_in_finish_r(struct __kfifo *fifo,
877 unsigned int len, size_t recsize);
878
879extern unsigned int __kfifo_dma_out_prepare_r(struct __kfifo *fifo,
880 struct scatterlist *sgl, int nents, unsigned int len, size_t recsize);
881
882extern void __kfifo_dma_out_finish_r(struct __kfifo *fifo, size_t recsize);
883
884extern unsigned int __kfifo_len_r(struct __kfifo *fifo, size_t recsize);
885
886extern void __kfifo_skip_r(struct __kfifo *fifo, size_t recsize);
887
888extern unsigned int __kfifo_out_peek_r(struct __kfifo *fifo,
889 void *buf, unsigned int len, size_t recsize);
890
891extern unsigned int __kfifo_max_r(unsigned int len, size_t recsize);
892
893#endif
894