1
2
3
4
5
6#include <linux/trace_events.h>
7#include <linux/ring_buffer.h>
8#include <linux/trace_clock.h>
9#include <linux/sched/clock.h>
10#include <linux/trace_seq.h>
11#include <linux/spinlock.h>
12#include <linux/irq_work.h>
13#include <linux/uaccess.h>
14#include <linux/hardirq.h>
15#include <linux/kthread.h>
16#include <linux/module.h>
17#include <linux/percpu.h>
18#include <linux/mutex.h>
19#include <linux/delay.h>
20#include <linux/slab.h>
21#include <linux/init.h>
22#include <linux/hash.h>
23#include <linux/list.h>
24#include <linux/cpu.h>
25
26#include <asm/local.h>
27
28static void update_pages_handler(struct work_struct *work);
29
30
31
32
33int ring_buffer_print_entry_header(struct trace_seq *s)
34{
35 trace_seq_puts(s, "# compressed entry header\n");
36 trace_seq_puts(s, "\ttype_len : 5 bits\n");
37 trace_seq_puts(s, "\ttime_delta : 27 bits\n");
38 trace_seq_puts(s, "\tarray : 32 bits\n");
39 trace_seq_putc(s, '\n');
40 trace_seq_printf(s, "\tpadding : type == %d\n",
41 RINGBUF_TYPE_PADDING);
42 trace_seq_printf(s, "\ttime_extend : type == %d\n",
43 RINGBUF_TYPE_TIME_EXTEND);
44 trace_seq_printf(s, "\tdata max type_len == %d\n",
45 RINGBUF_TYPE_DATA_TYPE_LEN_MAX);
46
47 return !trace_seq_has_overflowed(s);
48}
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119#define RB_BUFFER_OFF (1 << 20)
120
121#define BUF_PAGE_HDR_SIZE offsetof(struct buffer_data_page, data)
122
123#define RB_EVNT_HDR_SIZE (offsetof(struct ring_buffer_event, array))
124#define RB_ALIGNMENT 4U
125#define RB_MAX_SMALL_DATA (RB_ALIGNMENT * RINGBUF_TYPE_DATA_TYPE_LEN_MAX)
126#define RB_EVNT_MIN_SIZE 8U
127
128#ifndef CONFIG_HAVE_64BIT_ALIGNED_ACCESS
129# define RB_FORCE_8BYTE_ALIGNMENT 0
130# define RB_ARCH_ALIGNMENT RB_ALIGNMENT
131#else
132# define RB_FORCE_8BYTE_ALIGNMENT 1
133# define RB_ARCH_ALIGNMENT 8U
134#endif
135
136#define RB_ALIGN_DATA __aligned(RB_ARCH_ALIGNMENT)
137
138
139#define RINGBUF_TYPE_DATA 0 ... RINGBUF_TYPE_DATA_TYPE_LEN_MAX
140
141enum {
142 RB_LEN_TIME_EXTEND = 8,
143 RB_LEN_TIME_STAMP = 16,
144};
145
146#define skip_time_extend(event) \
147 ((struct ring_buffer_event *)((char *)event + RB_LEN_TIME_EXTEND))
148
149static inline int rb_null_event(struct ring_buffer_event *event)
150{
151 return event->type_len == RINGBUF_TYPE_PADDING && !event->time_delta;
152}
153
154static void rb_event_set_padding(struct ring_buffer_event *event)
155{
156
157 event->type_len = RINGBUF_TYPE_PADDING;
158 event->time_delta = 0;
159}
160
161static unsigned
162rb_event_data_length(struct ring_buffer_event *event)
163{
164 unsigned length;
165
166 if (event->type_len)
167 length = event->type_len * RB_ALIGNMENT;
168 else
169 length = event->array[0];
170 return length + RB_EVNT_HDR_SIZE;
171}
172
173
174
175
176
177
178static inline unsigned
179rb_event_length(struct ring_buffer_event *event)
180{
181 switch (event->type_len) {
182 case RINGBUF_TYPE_PADDING:
183 if (rb_null_event(event))
184
185 return -1;
186 return event->array[0] + RB_EVNT_HDR_SIZE;
187
188 case RINGBUF_TYPE_TIME_EXTEND:
189 return RB_LEN_TIME_EXTEND;
190
191 case RINGBUF_TYPE_TIME_STAMP:
192 return RB_LEN_TIME_STAMP;
193
194 case RINGBUF_TYPE_DATA:
195 return rb_event_data_length(event);
196 default:
197 BUG();
198 }
199
200 return 0;
201}
202
203
204
205
206
207static inline unsigned
208rb_event_ts_length(struct ring_buffer_event *event)
209{
210 unsigned len = 0;
211
212 if (event->type_len == RINGBUF_TYPE_TIME_EXTEND) {
213
214 len = RB_LEN_TIME_EXTEND;
215 event = skip_time_extend(event);
216 }
217 return len + rb_event_length(event);
218}
219
220
221
222
223
224
225
226
227
228
229
230unsigned ring_buffer_event_length(struct ring_buffer_event *event)
231{
232 unsigned length;
233
234 if (event->type_len == RINGBUF_TYPE_TIME_EXTEND)
235 event = skip_time_extend(event);
236
237 length = rb_event_length(event);
238 if (event->type_len > RINGBUF_TYPE_DATA_TYPE_LEN_MAX)
239 return length;
240 length -= RB_EVNT_HDR_SIZE;
241 if (length > RB_MAX_SMALL_DATA + sizeof(event->array[0]))
242 length -= sizeof(event->array[0]);
243 return length;
244}
245EXPORT_SYMBOL_GPL(ring_buffer_event_length);
246
247
248static __always_inline void *
249rb_event_data(struct ring_buffer_event *event)
250{
251 if (event->type_len == RINGBUF_TYPE_TIME_EXTEND)
252 event = skip_time_extend(event);
253 BUG_ON(event->type_len > RINGBUF_TYPE_DATA_TYPE_LEN_MAX);
254
255 if (event->type_len)
256 return (void *)&event->array[0];
257
258 return (void *)&event->array[1];
259}
260
261
262
263
264
265void *ring_buffer_event_data(struct ring_buffer_event *event)
266{
267 return rb_event_data(event);
268}
269EXPORT_SYMBOL_GPL(ring_buffer_event_data);
270
271#define for_each_buffer_cpu(buffer, cpu) \
272 for_each_cpu(cpu, buffer->cpumask)
273
274#define TS_SHIFT 27
275#define TS_MASK ((1ULL << TS_SHIFT) - 1)
276#define TS_DELTA_TEST (~TS_MASK)
277
278
279#define RB_MISSED_EVENTS (1 << 31)
280
281#define RB_MISSED_STORED (1 << 30)
282
283#define RB_MISSED_FLAGS (RB_MISSED_EVENTS|RB_MISSED_STORED)
284
285struct buffer_data_page {
286 u64 time_stamp;
287 local_t commit;
288 unsigned char data[] RB_ALIGN_DATA;
289};
290
291
292
293
294
295
296
297
298
299struct buffer_page {
300 struct list_head list;
301 local_t write;
302 unsigned read;
303 local_t entries;
304 unsigned long real_end;
305 struct buffer_data_page *page;
306};
307
308
309
310
311
312
313
314
315
316
317
318
319
320#define RB_WRITE_MASK 0xfffff
321#define RB_WRITE_INTCNT (1 << 20)
322
323static void rb_init_page(struct buffer_data_page *bpage)
324{
325 local_set(&bpage->commit, 0);
326}
327
328
329
330
331
332
333
334size_t ring_buffer_page_len(void *page)
335{
336 struct buffer_data_page *bpage = page;
337
338 return (local_read(&bpage->commit) & ~RB_MISSED_FLAGS)
339 + BUF_PAGE_HDR_SIZE;
340}
341
342
343
344
345
346static void free_buffer_page(struct buffer_page *bpage)
347{
348 free_page((unsigned long)bpage->page);
349 kfree(bpage);
350}
351
352
353
354
355static inline int test_time_stamp(u64 delta)
356{
357 if (delta & TS_DELTA_TEST)
358 return 1;
359 return 0;
360}
361
362#define BUF_PAGE_SIZE (PAGE_SIZE - BUF_PAGE_HDR_SIZE)
363
364
365#define BUF_MAX_DATA_SIZE (BUF_PAGE_SIZE - (sizeof(u32) * 2))
366
367int ring_buffer_print_page_header(struct trace_seq *s)
368{
369 struct buffer_data_page field;
370
371 trace_seq_printf(s, "\tfield: u64 timestamp;\t"
372 "offset:0;\tsize:%u;\tsigned:%u;\n",
373 (unsigned int)sizeof(field.time_stamp),
374 (unsigned int)is_signed_type(u64));
375
376 trace_seq_printf(s, "\tfield: local_t commit;\t"
377 "offset:%u;\tsize:%u;\tsigned:%u;\n",
378 (unsigned int)offsetof(typeof(field), commit),
379 (unsigned int)sizeof(field.commit),
380 (unsigned int)is_signed_type(long));
381
382 trace_seq_printf(s, "\tfield: int overwrite;\t"
383 "offset:%u;\tsize:%u;\tsigned:%u;\n",
384 (unsigned int)offsetof(typeof(field), commit),
385 1,
386 (unsigned int)is_signed_type(long));
387
388 trace_seq_printf(s, "\tfield: char data;\t"
389 "offset:%u;\tsize:%u;\tsigned:%u;\n",
390 (unsigned int)offsetof(typeof(field), data),
391 (unsigned int)BUF_PAGE_SIZE,
392 (unsigned int)is_signed_type(char));
393
394 return !trace_seq_has_overflowed(s);
395}
396
397struct rb_irq_work {
398 struct irq_work work;
399 wait_queue_head_t waiters;
400 wait_queue_head_t full_waiters;
401 bool waiters_pending;
402 bool full_waiters_pending;
403 bool wakeup_full;
404};
405
406
407
408
409struct rb_event_info {
410 u64 ts;
411 u64 delta;
412 unsigned long length;
413 struct buffer_page *tail_page;
414 int add_timestamp;
415};
416
417
418
419
420
421
422
423
424
425
426enum {
427 RB_CTX_NMI,
428 RB_CTX_IRQ,
429 RB_CTX_SOFTIRQ,
430 RB_CTX_NORMAL,
431 RB_CTX_MAX
432};
433
434
435
436
437struct ring_buffer_per_cpu {
438 int cpu;
439 atomic_t record_disabled;
440 struct ring_buffer *buffer;
441 raw_spinlock_t reader_lock;
442 arch_spinlock_t lock;
443 struct lock_class_key lock_key;
444 struct buffer_data_page *free_page;
445 unsigned long nr_pages;
446 unsigned int current_context;
447 struct list_head *pages;
448 struct buffer_page *head_page;
449 struct buffer_page *tail_page;
450 struct buffer_page *commit_page;
451 struct buffer_page *reader_page;
452 unsigned long lost_events;
453 unsigned long last_overrun;
454 local_t entries_bytes;
455 local_t entries;
456 local_t overrun;
457 local_t commit_overrun;
458 local_t dropped_events;
459 local_t committing;
460 local_t commits;
461 unsigned long read;
462 unsigned long read_bytes;
463 u64 write_stamp;
464 u64 read_stamp;
465
466 long nr_pages_to_update;
467 struct list_head new_pages;
468 struct work_struct update_pages_work;
469 struct completion update_done;
470
471 struct rb_irq_work irq_work;
472};
473
474struct ring_buffer {
475 unsigned flags;
476 int cpus;
477 atomic_t record_disabled;
478 atomic_t resize_disabled;
479 cpumask_var_t cpumask;
480
481 struct lock_class_key *reader_lock_key;
482
483 struct mutex mutex;
484
485 struct ring_buffer_per_cpu **buffers;
486
487 struct hlist_node node;
488 u64 (*clock)(void);
489
490 struct rb_irq_work irq_work;
491};
492
493struct ring_buffer_iter {
494 struct ring_buffer_per_cpu *cpu_buffer;
495 unsigned long head;
496 struct buffer_page *head_page;
497 struct buffer_page *cache_reader_page;
498 unsigned long cache_read;
499 u64 read_stamp;
500};
501
502
503
504
505
506
507
508static void rb_wake_up_waiters(struct irq_work *work)
509{
510 struct rb_irq_work *rbwork = container_of(work, struct rb_irq_work, work);
511
512 wake_up_all(&rbwork->waiters);
513 if (rbwork->wakeup_full) {
514 rbwork->wakeup_full = false;
515 wake_up_all(&rbwork->full_waiters);
516 }
517}
518
519
520
521
522
523
524
525
526
527
528
529int ring_buffer_wait(struct ring_buffer *buffer, int cpu, bool full)
530{
531 struct ring_buffer_per_cpu *uninitialized_var(cpu_buffer);
532 DEFINE_WAIT(wait);
533 struct rb_irq_work *work;
534 int ret = 0;
535
536
537
538
539
540
541 if (cpu == RING_BUFFER_ALL_CPUS) {
542 work = &buffer->irq_work;
543
544 full = false;
545 } else {
546 if (!cpumask_test_cpu(cpu, buffer->cpumask))
547 return -ENODEV;
548 cpu_buffer = buffer->buffers[cpu];
549 work = &cpu_buffer->irq_work;
550 }
551
552
553 while (true) {
554 if (full)
555 prepare_to_wait(&work->full_waiters, &wait, TASK_INTERRUPTIBLE);
556 else
557 prepare_to_wait(&work->waiters, &wait, TASK_INTERRUPTIBLE);
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579 if (full)
580 work->full_waiters_pending = true;
581 else
582 work->waiters_pending = true;
583
584 if (signal_pending(current)) {
585 ret = -EINTR;
586 break;
587 }
588
589 if (cpu == RING_BUFFER_ALL_CPUS && !ring_buffer_empty(buffer))
590 break;
591
592 if (cpu != RING_BUFFER_ALL_CPUS &&
593 !ring_buffer_empty_cpu(buffer, cpu)) {
594 unsigned long flags;
595 bool pagebusy;
596
597 if (!full)
598 break;
599
600 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
601 pagebusy = cpu_buffer->reader_page == cpu_buffer->commit_page;
602 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
603
604 if (!pagebusy)
605 break;
606 }
607
608 schedule();
609 }
610
611 if (full)
612 finish_wait(&work->full_waiters, &wait);
613 else
614 finish_wait(&work->waiters, &wait);
615
616 return ret;
617}
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633int ring_buffer_poll_wait(struct ring_buffer *buffer, int cpu,
634 struct file *filp, poll_table *poll_table)
635{
636 struct ring_buffer_per_cpu *cpu_buffer;
637 struct rb_irq_work *work;
638
639 if (cpu == RING_BUFFER_ALL_CPUS)
640 work = &buffer->irq_work;
641 else {
642 if (!cpumask_test_cpu(cpu, buffer->cpumask))
643 return -EINVAL;
644
645 cpu_buffer = buffer->buffers[cpu];
646 work = &cpu_buffer->irq_work;
647 }
648
649 poll_wait(filp, &work->waiters, poll_table);
650 work->waiters_pending = true;
651
652
653
654
655
656
657
658
659
660
661
662
663
664 smp_mb();
665
666 if ((cpu == RING_BUFFER_ALL_CPUS && !ring_buffer_empty(buffer)) ||
667 (cpu != RING_BUFFER_ALL_CPUS && !ring_buffer_empty_cpu(buffer, cpu)))
668 return POLLIN | POLLRDNORM;
669 return 0;
670}
671
672
673#define RB_WARN_ON(b, cond) \
674 ({ \
675 int _____ret = unlikely(cond); \
676 if (_____ret) { \
677 if (__same_type(*(b), struct ring_buffer_per_cpu)) { \
678 struct ring_buffer_per_cpu *__b = \
679 (void *)b; \
680 atomic_inc(&__b->buffer->record_disabled); \
681 } else \
682 atomic_inc(&b->record_disabled); \
683 WARN_ON(1); \
684 } \
685 _____ret; \
686 })
687
688
689#define DEBUG_SHIFT 0
690
691static inline u64 rb_time_stamp(struct ring_buffer *buffer)
692{
693
694 return buffer->clock() << DEBUG_SHIFT;
695}
696
697u64 ring_buffer_time_stamp(struct ring_buffer *buffer, int cpu)
698{
699 u64 time;
700
701 preempt_disable_notrace();
702 time = rb_time_stamp(buffer);
703 preempt_enable_no_resched_notrace();
704
705 return time;
706}
707EXPORT_SYMBOL_GPL(ring_buffer_time_stamp);
708
709void ring_buffer_normalize_time_stamp(struct ring_buffer *buffer,
710 int cpu, u64 *ts)
711{
712
713 *ts >>= DEBUG_SHIFT;
714}
715EXPORT_SYMBOL_GPL(ring_buffer_normalize_time_stamp);
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786#define RB_PAGE_NORMAL 0UL
787#define RB_PAGE_HEAD 1UL
788#define RB_PAGE_UPDATE 2UL
789
790
791#define RB_FLAG_MASK 3UL
792
793
794#define RB_PAGE_MOVED 4UL
795
796
797
798
799static struct list_head *rb_list_head(struct list_head *list)
800{
801 unsigned long val = (unsigned long)list;
802
803 return (struct list_head *)(val & ~RB_FLAG_MASK);
804}
805
806
807
808
809
810
811
812
813
814static inline int
815rb_is_head_page(struct ring_buffer_per_cpu *cpu_buffer,
816 struct buffer_page *page, struct list_head *list)
817{
818 unsigned long val;
819
820 val = (unsigned long)list->next;
821
822 if ((val & ~RB_FLAG_MASK) != (unsigned long)&page->list)
823 return RB_PAGE_MOVED;
824
825 return val & RB_FLAG_MASK;
826}
827
828
829
830
831
832
833
834
835static bool rb_is_reader_page(struct buffer_page *page)
836{
837 struct list_head *list = page->list.prev;
838
839 return rb_list_head(list->next) != &page->list;
840}
841
842
843
844
845static void rb_set_list_to_head(struct ring_buffer_per_cpu *cpu_buffer,
846 struct list_head *list)
847{
848 unsigned long *ptr;
849
850 ptr = (unsigned long *)&list->next;
851 *ptr |= RB_PAGE_HEAD;
852 *ptr &= ~RB_PAGE_UPDATE;
853}
854
855
856
857
858static void rb_head_page_activate(struct ring_buffer_per_cpu *cpu_buffer)
859{
860 struct buffer_page *head;
861
862 head = cpu_buffer->head_page;
863 if (!head)
864 return;
865
866
867
868
869 rb_set_list_to_head(cpu_buffer, head->list.prev);
870}
871
872static void rb_list_head_clear(struct list_head *list)
873{
874 unsigned long *ptr = (unsigned long *)&list->next;
875
876 *ptr &= ~RB_FLAG_MASK;
877}
878
879
880
881
882static void
883rb_head_page_deactivate(struct ring_buffer_per_cpu *cpu_buffer)
884{
885 struct list_head *hd;
886
887
888 rb_list_head_clear(cpu_buffer->pages);
889
890 list_for_each(hd, cpu_buffer->pages)
891 rb_list_head_clear(hd);
892}
893
894static int rb_head_page_set(struct ring_buffer_per_cpu *cpu_buffer,
895 struct buffer_page *head,
896 struct buffer_page *prev,
897 int old_flag, int new_flag)
898{
899 struct list_head *list;
900 unsigned long val = (unsigned long)&head->list;
901 unsigned long ret;
902
903 list = &prev->list;
904
905 val &= ~RB_FLAG_MASK;
906
907 ret = cmpxchg((unsigned long *)&list->next,
908 val | old_flag, val | new_flag);
909
910
911 if ((ret & ~RB_FLAG_MASK) != val)
912 return RB_PAGE_MOVED;
913
914 return ret & RB_FLAG_MASK;
915}
916
917static int rb_head_page_set_update(struct ring_buffer_per_cpu *cpu_buffer,
918 struct buffer_page *head,
919 struct buffer_page *prev,
920 int old_flag)
921{
922 return rb_head_page_set(cpu_buffer, head, prev,
923 old_flag, RB_PAGE_UPDATE);
924}
925
926static int rb_head_page_set_head(struct ring_buffer_per_cpu *cpu_buffer,
927 struct buffer_page *head,
928 struct buffer_page *prev,
929 int old_flag)
930{
931 return rb_head_page_set(cpu_buffer, head, prev,
932 old_flag, RB_PAGE_HEAD);
933}
934
935static int rb_head_page_set_normal(struct ring_buffer_per_cpu *cpu_buffer,
936 struct buffer_page *head,
937 struct buffer_page *prev,
938 int old_flag)
939{
940 return rb_head_page_set(cpu_buffer, head, prev,
941 old_flag, RB_PAGE_NORMAL);
942}
943
944static inline void rb_inc_page(struct ring_buffer_per_cpu *cpu_buffer,
945 struct buffer_page **bpage)
946{
947 struct list_head *p = rb_list_head((*bpage)->list.next);
948
949 *bpage = list_entry(p, struct buffer_page, list);
950}
951
952static struct buffer_page *
953rb_set_head_page(struct ring_buffer_per_cpu *cpu_buffer)
954{
955 struct buffer_page *head;
956 struct buffer_page *page;
957 struct list_head *list;
958 int i;
959
960 if (RB_WARN_ON(cpu_buffer, !cpu_buffer->head_page))
961 return NULL;
962
963
964 list = cpu_buffer->pages;
965 if (RB_WARN_ON(cpu_buffer, rb_list_head(list->prev->next) != list))
966 return NULL;
967
968 page = head = cpu_buffer->head_page;
969
970
971
972
973
974
975 for (i = 0; i < 3; i++) {
976 do {
977 if (rb_is_head_page(cpu_buffer, page, page->list.prev)) {
978 cpu_buffer->head_page = page;
979 return page;
980 }
981 rb_inc_page(cpu_buffer, &page);
982 } while (page != head);
983 }
984
985 RB_WARN_ON(cpu_buffer, 1);
986
987 return NULL;
988}
989
990static int rb_head_page_replace(struct buffer_page *old,
991 struct buffer_page *new)
992{
993 unsigned long *ptr = (unsigned long *)&old->list.prev->next;
994 unsigned long val;
995 unsigned long ret;
996
997 val = *ptr & ~RB_FLAG_MASK;
998 val |= RB_PAGE_HEAD;
999
1000 ret = cmpxchg(ptr, val, (unsigned long)&new->list);
1001
1002 return ret == val;
1003}
1004
1005
1006
1007
1008static void rb_tail_page_update(struct ring_buffer_per_cpu *cpu_buffer,
1009 struct buffer_page *tail_page,
1010 struct buffer_page *next_page)
1011{
1012 unsigned long old_entries;
1013 unsigned long old_write;
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024 old_write = local_add_return(RB_WRITE_INTCNT, &next_page->write);
1025 old_entries = local_add_return(RB_WRITE_INTCNT, &next_page->entries);
1026
1027
1028
1029
1030
1031 barrier();
1032
1033
1034
1035
1036
1037
1038 if (tail_page == READ_ONCE(cpu_buffer->tail_page)) {
1039
1040 unsigned long val = old_write & ~RB_WRITE_MASK;
1041 unsigned long eval = old_entries & ~RB_WRITE_MASK;
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053 (void)local_cmpxchg(&next_page->write, old_write, val);
1054 (void)local_cmpxchg(&next_page->entries, old_entries, eval);
1055
1056
1057
1058
1059
1060
1061 local_set(&next_page->page->commit, 0);
1062
1063
1064 (void)cmpxchg(&cpu_buffer->tail_page, tail_page, next_page);
1065 }
1066}
1067
1068static int rb_check_bpage(struct ring_buffer_per_cpu *cpu_buffer,
1069 struct buffer_page *bpage)
1070{
1071 unsigned long val = (unsigned long)bpage;
1072
1073 if (RB_WARN_ON(cpu_buffer, val & RB_FLAG_MASK))
1074 return 1;
1075
1076 return 0;
1077}
1078
1079
1080
1081
1082static int rb_check_list(struct ring_buffer_per_cpu *cpu_buffer,
1083 struct list_head *list)
1084{
1085 if (RB_WARN_ON(cpu_buffer, rb_list_head(list->prev) != list->prev))
1086 return 1;
1087 if (RB_WARN_ON(cpu_buffer, rb_list_head(list->next) != list->next))
1088 return 1;
1089 return 0;
1090}
1091
1092
1093
1094
1095
1096
1097
1098
1099static int rb_check_pages(struct ring_buffer_per_cpu *cpu_buffer)
1100{
1101 struct list_head *head = cpu_buffer->pages;
1102 struct buffer_page *bpage, *tmp;
1103
1104
1105 if (cpu_buffer->head_page)
1106 rb_set_head_page(cpu_buffer);
1107
1108 rb_head_page_deactivate(cpu_buffer);
1109
1110 if (RB_WARN_ON(cpu_buffer, head->next->prev != head))
1111 return -1;
1112 if (RB_WARN_ON(cpu_buffer, head->prev->next != head))
1113 return -1;
1114
1115 if (rb_check_list(cpu_buffer, head))
1116 return -1;
1117
1118 list_for_each_entry_safe(bpage, tmp, head, list) {
1119 if (RB_WARN_ON(cpu_buffer,
1120 bpage->list.next->prev != &bpage->list))
1121 return -1;
1122 if (RB_WARN_ON(cpu_buffer,
1123 bpage->list.prev->next != &bpage->list))
1124 return -1;
1125 if (rb_check_list(cpu_buffer, &bpage->list))
1126 return -1;
1127 }
1128
1129 rb_head_page_activate(cpu_buffer);
1130
1131 return 0;
1132}
1133
1134static int __rb_allocate_pages(long nr_pages, struct list_head *pages, int cpu)
1135{
1136 struct buffer_page *bpage, *tmp;
1137 long i;
1138
1139 for (i = 0; i < nr_pages; i++) {
1140 struct page *page;
1141
1142
1143
1144
1145
1146 bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),
1147 GFP_KERNEL | __GFP_RETRY_MAYFAIL,
1148 cpu_to_node(cpu));
1149 if (!bpage)
1150 goto free_pages;
1151
1152 list_add(&bpage->list, pages);
1153
1154 page = alloc_pages_node(cpu_to_node(cpu),
1155 GFP_KERNEL | __GFP_RETRY_MAYFAIL, 0);
1156 if (!page)
1157 goto free_pages;
1158 bpage->page = page_address(page);
1159 rb_init_page(bpage->page);
1160 }
1161
1162 return 0;
1163
1164free_pages:
1165 list_for_each_entry_safe(bpage, tmp, pages, list) {
1166 list_del_init(&bpage->list);
1167 free_buffer_page(bpage);
1168 }
1169
1170 return -ENOMEM;
1171}
1172
1173static int rb_allocate_pages(struct ring_buffer_per_cpu *cpu_buffer,
1174 unsigned long nr_pages)
1175{
1176 LIST_HEAD(pages);
1177
1178 WARN_ON(!nr_pages);
1179
1180 if (__rb_allocate_pages(nr_pages, &pages, cpu_buffer->cpu))
1181 return -ENOMEM;
1182
1183
1184
1185
1186
1187
1188 cpu_buffer->pages = pages.next;
1189 list_del(&pages);
1190
1191 cpu_buffer->nr_pages = nr_pages;
1192
1193 rb_check_pages(cpu_buffer);
1194
1195 return 0;
1196}
1197
1198static struct ring_buffer_per_cpu *
1199rb_allocate_cpu_buffer(struct ring_buffer *buffer, long nr_pages, int cpu)
1200{
1201 struct ring_buffer_per_cpu *cpu_buffer;
1202 struct buffer_page *bpage;
1203 struct page *page;
1204 int ret;
1205
1206 cpu_buffer = kzalloc_node(ALIGN(sizeof(*cpu_buffer), cache_line_size()),
1207 GFP_KERNEL, cpu_to_node(cpu));
1208 if (!cpu_buffer)
1209 return NULL;
1210
1211 cpu_buffer->cpu = cpu;
1212 cpu_buffer->buffer = buffer;
1213 raw_spin_lock_init(&cpu_buffer->reader_lock);
1214 lockdep_set_class(&cpu_buffer->reader_lock, buffer->reader_lock_key);
1215 cpu_buffer->lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
1216 INIT_WORK(&cpu_buffer->update_pages_work, update_pages_handler);
1217 init_completion(&cpu_buffer->update_done);
1218 init_irq_work(&cpu_buffer->irq_work.work, rb_wake_up_waiters);
1219 init_waitqueue_head(&cpu_buffer->irq_work.waiters);
1220 init_waitqueue_head(&cpu_buffer->irq_work.full_waiters);
1221
1222 bpage = kzalloc_node(ALIGN(sizeof(*bpage), cache_line_size()),
1223 GFP_KERNEL, cpu_to_node(cpu));
1224 if (!bpage)
1225 goto fail_free_buffer;
1226
1227 rb_check_bpage(cpu_buffer, bpage);
1228
1229 cpu_buffer->reader_page = bpage;
1230 page = alloc_pages_node(cpu_to_node(cpu), GFP_KERNEL, 0);
1231 if (!page)
1232 goto fail_free_reader;
1233 bpage->page = page_address(page);
1234 rb_init_page(bpage->page);
1235
1236 INIT_LIST_HEAD(&cpu_buffer->reader_page->list);
1237 INIT_LIST_HEAD(&cpu_buffer->new_pages);
1238
1239 ret = rb_allocate_pages(cpu_buffer, nr_pages);
1240 if (ret < 0)
1241 goto fail_free_reader;
1242
1243 cpu_buffer->head_page
1244 = list_entry(cpu_buffer->pages, struct buffer_page, list);
1245 cpu_buffer->tail_page = cpu_buffer->commit_page = cpu_buffer->head_page;
1246
1247 rb_head_page_activate(cpu_buffer);
1248
1249 return cpu_buffer;
1250
1251 fail_free_reader:
1252 free_buffer_page(cpu_buffer->reader_page);
1253
1254 fail_free_buffer:
1255 kfree(cpu_buffer);
1256 return NULL;
1257}
1258
1259static void rb_free_cpu_buffer(struct ring_buffer_per_cpu *cpu_buffer)
1260{
1261 struct list_head *head = cpu_buffer->pages;
1262 struct buffer_page *bpage, *tmp;
1263
1264 free_buffer_page(cpu_buffer->reader_page);
1265
1266 rb_head_page_deactivate(cpu_buffer);
1267
1268 if (head) {
1269 list_for_each_entry_safe(bpage, tmp, head, list) {
1270 list_del_init(&bpage->list);
1271 free_buffer_page(bpage);
1272 }
1273 bpage = list_entry(head, struct buffer_page, list);
1274 free_buffer_page(bpage);
1275 }
1276
1277 kfree(cpu_buffer);
1278}
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290struct ring_buffer *__ring_buffer_alloc(unsigned long size, unsigned flags,
1291 struct lock_class_key *key)
1292{
1293 struct ring_buffer *buffer;
1294 long nr_pages;
1295 int bsize;
1296 int cpu;
1297 int ret;
1298
1299
1300 buffer = kzalloc(ALIGN(sizeof(*buffer), cache_line_size()),
1301 GFP_KERNEL);
1302 if (!buffer)
1303 return NULL;
1304
1305 if (!zalloc_cpumask_var(&buffer->cpumask, GFP_KERNEL))
1306 goto fail_free_buffer;
1307
1308 nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
1309 buffer->flags = flags;
1310 buffer->clock = trace_clock_local;
1311 buffer->reader_lock_key = key;
1312
1313 init_irq_work(&buffer->irq_work.work, rb_wake_up_waiters);
1314 init_waitqueue_head(&buffer->irq_work.waiters);
1315
1316
1317 if (nr_pages < 2)
1318 nr_pages = 2;
1319
1320 buffer->cpus = nr_cpu_ids;
1321
1322 bsize = sizeof(void *) * nr_cpu_ids;
1323 buffer->buffers = kzalloc(ALIGN(bsize, cache_line_size()),
1324 GFP_KERNEL);
1325 if (!buffer->buffers)
1326 goto fail_free_cpumask;
1327
1328 cpu = raw_smp_processor_id();
1329 cpumask_set_cpu(cpu, buffer->cpumask);
1330 buffer->buffers[cpu] = rb_allocate_cpu_buffer(buffer, nr_pages, cpu);
1331 if (!buffer->buffers[cpu])
1332 goto fail_free_buffers;
1333
1334 ret = cpuhp_state_add_instance(CPUHP_TRACE_RB_PREPARE, &buffer->node);
1335 if (ret < 0)
1336 goto fail_free_buffers;
1337
1338 mutex_init(&buffer->mutex);
1339
1340 return buffer;
1341
1342 fail_free_buffers:
1343 for_each_buffer_cpu(buffer, cpu) {
1344 if (buffer->buffers[cpu])
1345 rb_free_cpu_buffer(buffer->buffers[cpu]);
1346 }
1347 kfree(buffer->buffers);
1348
1349 fail_free_cpumask:
1350 free_cpumask_var(buffer->cpumask);
1351
1352 fail_free_buffer:
1353 kfree(buffer);
1354 return NULL;
1355}
1356EXPORT_SYMBOL_GPL(__ring_buffer_alloc);
1357
1358
1359
1360
1361
1362void
1363ring_buffer_free(struct ring_buffer *buffer)
1364{
1365 int cpu;
1366
1367 cpuhp_state_remove_instance(CPUHP_TRACE_RB_PREPARE, &buffer->node);
1368
1369 for_each_buffer_cpu(buffer, cpu)
1370 rb_free_cpu_buffer(buffer->buffers[cpu]);
1371
1372 kfree(buffer->buffers);
1373 free_cpumask_var(buffer->cpumask);
1374
1375 kfree(buffer);
1376}
1377EXPORT_SYMBOL_GPL(ring_buffer_free);
1378
1379void ring_buffer_set_clock(struct ring_buffer *buffer,
1380 u64 (*clock)(void))
1381{
1382 buffer->clock = clock;
1383}
1384
1385static void rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer);
1386
1387static inline unsigned long rb_page_entries(struct buffer_page *bpage)
1388{
1389 return local_read(&bpage->entries) & RB_WRITE_MASK;
1390}
1391
1392static inline unsigned long rb_page_write(struct buffer_page *bpage)
1393{
1394 return local_read(&bpage->write) & RB_WRITE_MASK;
1395}
1396
1397static int
1398rb_remove_pages(struct ring_buffer_per_cpu *cpu_buffer, unsigned long nr_pages)
1399{
1400 struct list_head *tail_page, *to_remove, *next_page;
1401 struct buffer_page *to_remove_page, *tmp_iter_page;
1402 struct buffer_page *last_page, *first_page;
1403 unsigned long nr_removed;
1404 unsigned long head_bit;
1405 int page_entries;
1406
1407 head_bit = 0;
1408
1409 raw_spin_lock_irq(&cpu_buffer->reader_lock);
1410 atomic_inc(&cpu_buffer->record_disabled);
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420 tail_page = &cpu_buffer->tail_page->list;
1421
1422
1423
1424
1425
1426 if (cpu_buffer->tail_page == cpu_buffer->reader_page)
1427 tail_page = rb_list_head(tail_page->next);
1428 to_remove = tail_page;
1429
1430
1431 first_page = list_entry(rb_list_head(to_remove->next),
1432 struct buffer_page, list);
1433
1434 for (nr_removed = 0; nr_removed < nr_pages; nr_removed++) {
1435 to_remove = rb_list_head(to_remove)->next;
1436 head_bit |= (unsigned long)to_remove & RB_PAGE_HEAD;
1437 }
1438
1439 next_page = rb_list_head(to_remove)->next;
1440
1441
1442
1443
1444
1445
1446 tail_page->next = (struct list_head *)((unsigned long)next_page |
1447 head_bit);
1448 next_page = rb_list_head(next_page);
1449 next_page->prev = tail_page;
1450
1451
1452 cpu_buffer->pages = next_page;
1453
1454
1455 if (head_bit)
1456 cpu_buffer->head_page = list_entry(next_page,
1457 struct buffer_page, list);
1458
1459
1460
1461
1462
1463 cpu_buffer->read = 0;
1464
1465
1466 atomic_dec(&cpu_buffer->record_disabled);
1467 raw_spin_unlock_irq(&cpu_buffer->reader_lock);
1468
1469 RB_WARN_ON(cpu_buffer, list_empty(cpu_buffer->pages));
1470
1471
1472 last_page = list_entry(rb_list_head(to_remove), struct buffer_page,
1473 list);
1474 tmp_iter_page = first_page;
1475
1476 do {
1477 to_remove_page = tmp_iter_page;
1478 rb_inc_page(cpu_buffer, &tmp_iter_page);
1479
1480
1481 page_entries = rb_page_entries(to_remove_page);
1482 if (page_entries) {
1483
1484
1485
1486
1487
1488
1489 local_add(page_entries, &cpu_buffer->overrun);
1490 local_sub(BUF_PAGE_SIZE, &cpu_buffer->entries_bytes);
1491 }
1492
1493
1494
1495
1496
1497 free_buffer_page(to_remove_page);
1498 nr_removed--;
1499
1500 } while (to_remove_page != last_page);
1501
1502 RB_WARN_ON(cpu_buffer, nr_removed);
1503
1504 return nr_removed == 0;
1505}
1506
1507static int
1508rb_insert_pages(struct ring_buffer_per_cpu *cpu_buffer)
1509{
1510 struct list_head *pages = &cpu_buffer->new_pages;
1511 int retries, success;
1512
1513 raw_spin_lock_irq(&cpu_buffer->reader_lock);
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528 retries = 10;
1529 success = 0;
1530 while (retries--) {
1531 struct list_head *head_page, *prev_page, *r;
1532 struct list_head *last_page, *first_page;
1533 struct list_head *head_page_with_bit;
1534
1535 head_page = &rb_set_head_page(cpu_buffer)->list;
1536 if (!head_page)
1537 break;
1538 prev_page = head_page->prev;
1539
1540 first_page = pages->next;
1541 last_page = pages->prev;
1542
1543 head_page_with_bit = (struct list_head *)
1544 ((unsigned long)head_page | RB_PAGE_HEAD);
1545
1546 last_page->next = head_page_with_bit;
1547 first_page->prev = prev_page;
1548
1549 r = cmpxchg(&prev_page->next, head_page_with_bit, first_page);
1550
1551 if (r == head_page_with_bit) {
1552
1553
1554
1555
1556
1557 head_page->prev = last_page;
1558 success = 1;
1559 break;
1560 }
1561 }
1562
1563 if (success)
1564 INIT_LIST_HEAD(pages);
1565
1566
1567
1568
1569 RB_WARN_ON(cpu_buffer, !success);
1570 raw_spin_unlock_irq(&cpu_buffer->reader_lock);
1571
1572
1573 if (!success) {
1574 struct buffer_page *bpage, *tmp;
1575 list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages,
1576 list) {
1577 list_del_init(&bpage->list);
1578 free_buffer_page(bpage);
1579 }
1580 }
1581 return success;
1582}
1583
1584static void rb_update_pages(struct ring_buffer_per_cpu *cpu_buffer)
1585{
1586 int success;
1587
1588 if (cpu_buffer->nr_pages_to_update > 0)
1589 success = rb_insert_pages(cpu_buffer);
1590 else
1591 success = rb_remove_pages(cpu_buffer,
1592 -cpu_buffer->nr_pages_to_update);
1593
1594 if (success)
1595 cpu_buffer->nr_pages += cpu_buffer->nr_pages_to_update;
1596}
1597
1598static void update_pages_handler(struct work_struct *work)
1599{
1600 struct ring_buffer_per_cpu *cpu_buffer = container_of(work,
1601 struct ring_buffer_per_cpu, update_pages_work);
1602 rb_update_pages(cpu_buffer);
1603 complete(&cpu_buffer->update_done);
1604}
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616int ring_buffer_resize(struct ring_buffer *buffer, unsigned long size,
1617 int cpu_id)
1618{
1619 struct ring_buffer_per_cpu *cpu_buffer;
1620 unsigned long nr_pages;
1621 int cpu, err = 0;
1622
1623
1624
1625
1626 if (!buffer)
1627 return size;
1628
1629
1630 if (cpu_id != RING_BUFFER_ALL_CPUS &&
1631 !cpumask_test_cpu(cpu_id, buffer->cpumask))
1632 return size;
1633
1634 nr_pages = DIV_ROUND_UP(size, BUF_PAGE_SIZE);
1635
1636
1637 if (nr_pages < 2)
1638 nr_pages = 2;
1639
1640 size = nr_pages * BUF_PAGE_SIZE;
1641
1642
1643
1644
1645
1646
1647 if (atomic_read(&buffer->resize_disabled))
1648 return -EBUSY;
1649
1650
1651 mutex_lock(&buffer->mutex);
1652
1653 if (cpu_id == RING_BUFFER_ALL_CPUS) {
1654
1655 for_each_buffer_cpu(buffer, cpu) {
1656 cpu_buffer = buffer->buffers[cpu];
1657
1658 cpu_buffer->nr_pages_to_update = nr_pages -
1659 cpu_buffer->nr_pages;
1660
1661
1662
1663 if (cpu_buffer->nr_pages_to_update <= 0)
1664 continue;
1665
1666
1667
1668
1669 INIT_LIST_HEAD(&cpu_buffer->new_pages);
1670 if (__rb_allocate_pages(cpu_buffer->nr_pages_to_update,
1671 &cpu_buffer->new_pages, cpu)) {
1672
1673 err = -ENOMEM;
1674 goto out_err;
1675 }
1676 }
1677
1678 get_online_cpus();
1679
1680
1681
1682
1683
1684 for_each_buffer_cpu(buffer, cpu) {
1685 cpu_buffer = buffer->buffers[cpu];
1686 if (!cpu_buffer->nr_pages_to_update)
1687 continue;
1688
1689
1690 if (!cpu_online(cpu)) {
1691 rb_update_pages(cpu_buffer);
1692 cpu_buffer->nr_pages_to_update = 0;
1693 } else {
1694 schedule_work_on(cpu,
1695 &cpu_buffer->update_pages_work);
1696 }
1697 }
1698
1699
1700 for_each_buffer_cpu(buffer, cpu) {
1701 cpu_buffer = buffer->buffers[cpu];
1702 if (!cpu_buffer->nr_pages_to_update)
1703 continue;
1704
1705 if (cpu_online(cpu))
1706 wait_for_completion(&cpu_buffer->update_done);
1707 cpu_buffer->nr_pages_to_update = 0;
1708 }
1709
1710 put_online_cpus();
1711 } else {
1712
1713 if (!cpumask_test_cpu(cpu_id, buffer->cpumask))
1714 goto out;
1715
1716 cpu_buffer = buffer->buffers[cpu_id];
1717
1718 if (nr_pages == cpu_buffer->nr_pages)
1719 goto out;
1720
1721 cpu_buffer->nr_pages_to_update = nr_pages -
1722 cpu_buffer->nr_pages;
1723
1724 INIT_LIST_HEAD(&cpu_buffer->new_pages);
1725 if (cpu_buffer->nr_pages_to_update > 0 &&
1726 __rb_allocate_pages(cpu_buffer->nr_pages_to_update,
1727 &cpu_buffer->new_pages, cpu_id)) {
1728 err = -ENOMEM;
1729 goto out_err;
1730 }
1731
1732 get_online_cpus();
1733
1734
1735 if (!cpu_online(cpu_id))
1736 rb_update_pages(cpu_buffer);
1737 else {
1738 schedule_work_on(cpu_id,
1739 &cpu_buffer->update_pages_work);
1740 wait_for_completion(&cpu_buffer->update_done);
1741 }
1742
1743 cpu_buffer->nr_pages_to_update = 0;
1744 put_online_cpus();
1745 }
1746
1747 out:
1748
1749
1750
1751
1752
1753
1754
1755 if (atomic_read(&buffer->record_disabled)) {
1756 atomic_inc(&buffer->record_disabled);
1757
1758
1759
1760
1761
1762
1763 synchronize_sched();
1764 for_each_buffer_cpu(buffer, cpu) {
1765 cpu_buffer = buffer->buffers[cpu];
1766 rb_check_pages(cpu_buffer);
1767 }
1768 atomic_dec(&buffer->record_disabled);
1769 }
1770
1771 mutex_unlock(&buffer->mutex);
1772 return size;
1773
1774 out_err:
1775 for_each_buffer_cpu(buffer, cpu) {
1776 struct buffer_page *bpage, *tmp;
1777
1778 cpu_buffer = buffer->buffers[cpu];
1779 cpu_buffer->nr_pages_to_update = 0;
1780
1781 if (list_empty(&cpu_buffer->new_pages))
1782 continue;
1783
1784 list_for_each_entry_safe(bpage, tmp, &cpu_buffer->new_pages,
1785 list) {
1786 list_del_init(&bpage->list);
1787 free_buffer_page(bpage);
1788 }
1789 }
1790 mutex_unlock(&buffer->mutex);
1791 return err;
1792}
1793EXPORT_SYMBOL_GPL(ring_buffer_resize);
1794
1795void ring_buffer_change_overwrite(struct ring_buffer *buffer, int val)
1796{
1797 mutex_lock(&buffer->mutex);
1798 if (val)
1799 buffer->flags |= RB_FL_OVERWRITE;
1800 else
1801 buffer->flags &= ~RB_FL_OVERWRITE;
1802 mutex_unlock(&buffer->mutex);
1803}
1804EXPORT_SYMBOL_GPL(ring_buffer_change_overwrite);
1805
1806static __always_inline void *__rb_page_index(struct buffer_page *bpage, unsigned index)
1807{
1808 return bpage->page->data + index;
1809}
1810
1811static __always_inline struct ring_buffer_event *
1812rb_reader_event(struct ring_buffer_per_cpu *cpu_buffer)
1813{
1814 return __rb_page_index(cpu_buffer->reader_page,
1815 cpu_buffer->reader_page->read);
1816}
1817
1818static __always_inline struct ring_buffer_event *
1819rb_iter_head_event(struct ring_buffer_iter *iter)
1820{
1821 return __rb_page_index(iter->head_page, iter->head);
1822}
1823
1824static __always_inline unsigned rb_page_commit(struct buffer_page *bpage)
1825{
1826 return local_read(&bpage->page->commit);
1827}
1828
1829
1830static __always_inline unsigned rb_page_size(struct buffer_page *bpage)
1831{
1832 return rb_page_commit(bpage);
1833}
1834
1835static __always_inline unsigned
1836rb_commit_index(struct ring_buffer_per_cpu *cpu_buffer)
1837{
1838 return rb_page_commit(cpu_buffer->commit_page);
1839}
1840
1841static __always_inline unsigned
1842rb_event_index(struct ring_buffer_event *event)
1843{
1844 unsigned long addr = (unsigned long)event;
1845
1846 return (addr & ~PAGE_MASK) - BUF_PAGE_HDR_SIZE;
1847}
1848
1849static void rb_inc_iter(struct ring_buffer_iter *iter)
1850{
1851 struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
1852
1853
1854
1855
1856
1857
1858
1859 if (iter->head_page == cpu_buffer->reader_page)
1860 iter->head_page = rb_set_head_page(cpu_buffer);
1861 else
1862 rb_inc_page(cpu_buffer, &iter->head_page);
1863
1864 iter->read_stamp = iter->head_page->page->time_stamp;
1865 iter->head = 0;
1866}
1867
1868
1869
1870
1871
1872
1873
1874
1875static int
1876rb_handle_head_page(struct ring_buffer_per_cpu *cpu_buffer,
1877 struct buffer_page *tail_page,
1878 struct buffer_page *next_page)
1879{
1880 struct buffer_page *new_head;
1881 int entries;
1882 int type;
1883 int ret;
1884
1885 entries = rb_page_entries(next_page);
1886
1887
1888
1889
1890
1891
1892 type = rb_head_page_set_update(cpu_buffer, next_page, tail_page,
1893 RB_PAGE_HEAD);
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906 switch (type) {
1907 case RB_PAGE_HEAD:
1908
1909
1910
1911
1912
1913 local_add(entries, &cpu_buffer->overrun);
1914 local_sub(BUF_PAGE_SIZE, &cpu_buffer->entries_bytes);
1915
1916
1917
1918
1919
1920
1921
1922 break;
1923
1924 case RB_PAGE_UPDATE:
1925
1926
1927
1928
1929 break;
1930 case RB_PAGE_NORMAL:
1931
1932
1933
1934
1935
1936 return 1;
1937 case RB_PAGE_MOVED:
1938
1939
1940
1941
1942
1943 return 1;
1944 default:
1945 RB_WARN_ON(cpu_buffer, 1);
1946 return -1;
1947 }
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963 new_head = next_page;
1964 rb_inc_page(cpu_buffer, &new_head);
1965
1966 ret = rb_head_page_set_head(cpu_buffer, new_head, next_page,
1967 RB_PAGE_NORMAL);
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977 switch (ret) {
1978 case RB_PAGE_HEAD:
1979 case RB_PAGE_NORMAL:
1980
1981 break;
1982 default:
1983 RB_WARN_ON(cpu_buffer, 1);
1984 return -1;
1985 }
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997 if (ret == RB_PAGE_NORMAL) {
1998 struct buffer_page *buffer_tail_page;
1999
2000 buffer_tail_page = READ_ONCE(cpu_buffer->tail_page);
2001
2002
2003
2004
2005 if (buffer_tail_page != tail_page &&
2006 buffer_tail_page != next_page)
2007 rb_head_page_set_normal(cpu_buffer, new_head,
2008 next_page,
2009 RB_PAGE_HEAD);
2010 }
2011
2012
2013
2014
2015
2016
2017 if (type == RB_PAGE_HEAD) {
2018 ret = rb_head_page_set_normal(cpu_buffer, next_page,
2019 tail_page,
2020 RB_PAGE_UPDATE);
2021 if (RB_WARN_ON(cpu_buffer,
2022 ret != RB_PAGE_UPDATE))
2023 return -1;
2024 }
2025
2026 return 0;
2027}
2028
2029static inline void
2030rb_reset_tail(struct ring_buffer_per_cpu *cpu_buffer,
2031 unsigned long tail, struct rb_event_info *info)
2032{
2033 struct buffer_page *tail_page = info->tail_page;
2034 struct ring_buffer_event *event;
2035 unsigned long length = info->length;
2036
2037
2038
2039
2040
2041 if (tail >= BUF_PAGE_SIZE) {
2042
2043
2044
2045
2046
2047 if (tail == BUF_PAGE_SIZE)
2048 tail_page->real_end = 0;
2049
2050 local_sub(length, &tail_page->write);
2051 return;
2052 }
2053
2054 event = __rb_page_index(tail_page, tail);
2055
2056
2057 local_add(BUF_PAGE_SIZE - tail, &cpu_buffer->entries_bytes);
2058
2059
2060
2061
2062
2063
2064 tail_page->real_end = tail;
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077 if (tail > (BUF_PAGE_SIZE - RB_EVNT_MIN_SIZE)) {
2078
2079
2080
2081 rb_event_set_padding(event);
2082
2083
2084 local_sub(length, &tail_page->write);
2085 return;
2086 }
2087
2088
2089 event->array[0] = (BUF_PAGE_SIZE - tail) - RB_EVNT_HDR_SIZE;
2090 event->type_len = RINGBUF_TYPE_PADDING;
2091
2092 event->time_delta = 1;
2093
2094
2095 length = (tail + length) - BUF_PAGE_SIZE;
2096 local_sub(length, &tail_page->write);
2097}
2098
2099static inline void rb_end_commit(struct ring_buffer_per_cpu *cpu_buffer);
2100
2101
2102
2103
2104static noinline struct ring_buffer_event *
2105rb_move_tail(struct ring_buffer_per_cpu *cpu_buffer,
2106 unsigned long tail, struct rb_event_info *info)
2107{
2108 struct buffer_page *tail_page = info->tail_page;
2109 struct buffer_page *commit_page = cpu_buffer->commit_page;
2110 struct ring_buffer *buffer = cpu_buffer->buffer;
2111 struct buffer_page *next_page;
2112 int ret;
2113
2114 next_page = tail_page;
2115
2116 rb_inc_page(cpu_buffer, &next_page);
2117
2118
2119
2120
2121
2122
2123 if (unlikely(next_page == commit_page)) {
2124 local_inc(&cpu_buffer->commit_overrun);
2125 goto out_reset;
2126 }
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142 if (rb_is_head_page(cpu_buffer, next_page, &tail_page->list)) {
2143
2144
2145
2146
2147
2148 if (!rb_is_reader_page(cpu_buffer->commit_page)) {
2149
2150
2151
2152
2153 if (!(buffer->flags & RB_FL_OVERWRITE)) {
2154 local_inc(&cpu_buffer->dropped_events);
2155 goto out_reset;
2156 }
2157
2158 ret = rb_handle_head_page(cpu_buffer,
2159 tail_page,
2160 next_page);
2161 if (ret < 0)
2162 goto out_reset;
2163 if (ret)
2164 goto out_again;
2165 } else {
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176 if (unlikely((cpu_buffer->commit_page !=
2177 cpu_buffer->tail_page) &&
2178 (cpu_buffer->commit_page ==
2179 cpu_buffer->reader_page))) {
2180 local_inc(&cpu_buffer->commit_overrun);
2181 goto out_reset;
2182 }
2183 }
2184 }
2185
2186 rb_tail_page_update(cpu_buffer, tail_page, next_page);
2187
2188 out_again:
2189
2190 rb_reset_tail(cpu_buffer, tail, info);
2191
2192
2193 rb_end_commit(cpu_buffer);
2194
2195 local_inc(&cpu_buffer->committing);
2196
2197
2198 return ERR_PTR(-EAGAIN);
2199
2200 out_reset:
2201
2202 rb_reset_tail(cpu_buffer, tail, info);
2203
2204 return NULL;
2205}
2206
2207
2208static noinline struct ring_buffer_event *
2209rb_add_time_stamp(struct ring_buffer_event *event, u64 delta)
2210{
2211 event->type_len = RINGBUF_TYPE_TIME_EXTEND;
2212
2213
2214 if (rb_event_index(event)) {
2215 event->time_delta = delta & TS_MASK;
2216 event->array[0] = delta >> TS_SHIFT;
2217 } else {
2218
2219 event->time_delta = 0;
2220 event->array[0] = 0;
2221 }
2222
2223 return skip_time_extend(event);
2224}
2225
2226static inline bool rb_event_is_commit(struct ring_buffer_per_cpu *cpu_buffer,
2227 struct ring_buffer_event *event);
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240static void
2241rb_update_event(struct ring_buffer_per_cpu *cpu_buffer,
2242 struct ring_buffer_event *event,
2243 struct rb_event_info *info)
2244{
2245 unsigned length = info->length;
2246 u64 delta = info->delta;
2247
2248
2249 if (unlikely(!rb_event_is_commit(cpu_buffer, event)))
2250 delta = 0;
2251
2252
2253
2254
2255
2256 if (unlikely(info->add_timestamp)) {
2257 event = rb_add_time_stamp(event, delta);
2258 length -= RB_LEN_TIME_EXTEND;
2259 delta = 0;
2260 }
2261
2262 event->time_delta = delta;
2263 length -= RB_EVNT_HDR_SIZE;
2264 if (length > RB_MAX_SMALL_DATA || RB_FORCE_8BYTE_ALIGNMENT) {
2265 event->type_len = 0;
2266 event->array[0] = length;
2267 } else
2268 event->type_len = DIV_ROUND_UP(length, RB_ALIGNMENT);
2269}
2270
2271static unsigned rb_calculate_event_length(unsigned length)
2272{
2273 struct ring_buffer_event event;
2274
2275
2276 if (!length)
2277 length++;
2278
2279 if (length > RB_MAX_SMALL_DATA || RB_FORCE_8BYTE_ALIGNMENT)
2280 length += sizeof(event.array[0]);
2281
2282 length += RB_EVNT_HDR_SIZE;
2283 length = ALIGN(length, RB_ARCH_ALIGNMENT);
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297 if (length == RB_LEN_TIME_EXTEND + RB_ALIGNMENT)
2298 length += RB_ALIGNMENT;
2299
2300 return length;
2301}
2302
2303#ifndef CONFIG_HAVE_UNSTABLE_SCHED_CLOCK
2304static inline bool sched_clock_stable(void)
2305{
2306 return true;
2307}
2308#endif
2309
2310static inline int
2311rb_try_to_discard(struct ring_buffer_per_cpu *cpu_buffer,
2312 struct ring_buffer_event *event)
2313{
2314 unsigned long new_index, old_index;
2315 struct buffer_page *bpage;
2316 unsigned long index;
2317 unsigned long addr;
2318
2319 new_index = rb_event_index(event);
2320 old_index = new_index + rb_event_ts_length(event);
2321 addr = (unsigned long)event;
2322 addr &= PAGE_MASK;
2323
2324 bpage = READ_ONCE(cpu_buffer->tail_page);
2325
2326 if (bpage->page == (void *)addr && rb_page_write(bpage) == old_index) {
2327 unsigned long write_mask =
2328 local_read(&bpage->write) & ~RB_WRITE_MASK;
2329 unsigned long event_length = rb_event_length(event);
2330
2331
2332
2333
2334
2335
2336 old_index += write_mask;
2337 new_index += write_mask;
2338 index = local_cmpxchg(&bpage->write, old_index, new_index);
2339 if (index == old_index) {
2340
2341 local_sub(event_length, &cpu_buffer->entries_bytes);
2342 return 1;
2343 }
2344 }
2345
2346
2347 return 0;
2348}
2349
2350static void rb_start_commit(struct ring_buffer_per_cpu *cpu_buffer)
2351{
2352 local_inc(&cpu_buffer->committing);
2353 local_inc(&cpu_buffer->commits);
2354}
2355
2356static __always_inline void
2357rb_set_commit_to_write(struct ring_buffer_per_cpu *cpu_buffer)
2358{
2359 unsigned long max_count;
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369 again:
2370 max_count = cpu_buffer->nr_pages * 100;
2371
2372 while (cpu_buffer->commit_page != READ_ONCE(cpu_buffer->tail_page)) {
2373 if (RB_WARN_ON(cpu_buffer, !(--max_count)))
2374 return;
2375 if (RB_WARN_ON(cpu_buffer,
2376 rb_is_reader_page(cpu_buffer->tail_page)))
2377 return;
2378 local_set(&cpu_buffer->commit_page->page->commit,
2379 rb_page_write(cpu_buffer->commit_page));
2380 rb_inc_page(cpu_buffer, &cpu_buffer->commit_page);
2381
2382 if (rb_page_write(cpu_buffer->commit_page))
2383 cpu_buffer->write_stamp =
2384 cpu_buffer->commit_page->page->time_stamp;
2385
2386 barrier();
2387 }
2388 while (rb_commit_index(cpu_buffer) !=
2389 rb_page_write(cpu_buffer->commit_page)) {
2390
2391 local_set(&cpu_buffer->commit_page->page->commit,
2392 rb_page_write(cpu_buffer->commit_page));
2393 RB_WARN_ON(cpu_buffer,
2394 local_read(&cpu_buffer->commit_page->page->commit) &
2395 ~RB_WRITE_MASK);
2396 barrier();
2397 }
2398
2399
2400 barrier();
2401
2402
2403
2404
2405
2406
2407 if (unlikely(cpu_buffer->commit_page != READ_ONCE(cpu_buffer->tail_page)))
2408 goto again;
2409}
2410
2411static __always_inline void rb_end_commit(struct ring_buffer_per_cpu *cpu_buffer)
2412{
2413 unsigned long commits;
2414
2415 if (RB_WARN_ON(cpu_buffer,
2416 !local_read(&cpu_buffer->committing)))
2417 return;
2418
2419 again:
2420 commits = local_read(&cpu_buffer->commits);
2421
2422 barrier();
2423 if (local_read(&cpu_buffer->committing) == 1)
2424 rb_set_commit_to_write(cpu_buffer);
2425
2426 local_dec(&cpu_buffer->committing);
2427
2428
2429 barrier();
2430
2431
2432
2433
2434
2435
2436 if (unlikely(local_read(&cpu_buffer->commits) != commits) &&
2437 !local_read(&cpu_buffer->committing)) {
2438 local_inc(&cpu_buffer->committing);
2439 goto again;
2440 }
2441}
2442
2443static inline void rb_event_discard(struct ring_buffer_event *event)
2444{
2445 if (event->type_len == RINGBUF_TYPE_TIME_EXTEND)
2446 event = skip_time_extend(event);
2447
2448
2449 event->array[0] = rb_event_data_length(event) - RB_EVNT_HDR_SIZE;
2450 event->type_len = RINGBUF_TYPE_PADDING;
2451
2452 if (!event->time_delta)
2453 event->time_delta = 1;
2454}
2455
2456static __always_inline bool
2457rb_event_is_commit(struct ring_buffer_per_cpu *cpu_buffer,
2458 struct ring_buffer_event *event)
2459{
2460 unsigned long addr = (unsigned long)event;
2461 unsigned long index;
2462
2463 index = rb_event_index(event);
2464 addr &= PAGE_MASK;
2465
2466 return cpu_buffer->commit_page->page == (void *)addr &&
2467 rb_commit_index(cpu_buffer) == index;
2468}
2469
2470static __always_inline void
2471rb_update_write_stamp(struct ring_buffer_per_cpu *cpu_buffer,
2472 struct ring_buffer_event *event)
2473{
2474 u64 delta;
2475
2476
2477
2478
2479
2480 if (rb_event_is_commit(cpu_buffer, event)) {
2481
2482
2483
2484
2485 if (!rb_event_index(event))
2486 cpu_buffer->write_stamp =
2487 cpu_buffer->commit_page->page->time_stamp;
2488 else if (event->type_len == RINGBUF_TYPE_TIME_EXTEND) {
2489 delta = event->array[0];
2490 delta <<= TS_SHIFT;
2491 delta += event->time_delta;
2492 cpu_buffer->write_stamp += delta;
2493 } else
2494 cpu_buffer->write_stamp += event->time_delta;
2495 }
2496}
2497
2498static void rb_commit(struct ring_buffer_per_cpu *cpu_buffer,
2499 struct ring_buffer_event *event)
2500{
2501 local_inc(&cpu_buffer->entries);
2502 rb_update_write_stamp(cpu_buffer, event);
2503 rb_end_commit(cpu_buffer);
2504}
2505
2506static __always_inline void
2507rb_wakeups(struct ring_buffer *buffer, struct ring_buffer_per_cpu *cpu_buffer)
2508{
2509 bool pagebusy;
2510
2511 if (buffer->irq_work.waiters_pending) {
2512 buffer->irq_work.waiters_pending = false;
2513
2514 irq_work_queue(&buffer->irq_work.work);
2515 }
2516
2517 if (cpu_buffer->irq_work.waiters_pending) {
2518 cpu_buffer->irq_work.waiters_pending = false;
2519
2520 irq_work_queue(&cpu_buffer->irq_work.work);
2521 }
2522
2523 pagebusy = cpu_buffer->reader_page == cpu_buffer->commit_page;
2524
2525 if (!pagebusy && cpu_buffer->irq_work.full_waiters_pending) {
2526 cpu_buffer->irq_work.wakeup_full = true;
2527 cpu_buffer->irq_work.full_waiters_pending = false;
2528
2529 irq_work_queue(&cpu_buffer->irq_work.work);
2530 }
2531}
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571static __always_inline int
2572trace_recursive_lock(struct ring_buffer_per_cpu *cpu_buffer)
2573{
2574 unsigned int val = cpu_buffer->current_context;
2575 unsigned long pc = preempt_count();
2576 int bit;
2577
2578 if (!(pc & (NMI_MASK | HARDIRQ_MASK | SOFTIRQ_OFFSET)))
2579 bit = RB_CTX_NORMAL;
2580 else
2581 bit = pc & NMI_MASK ? RB_CTX_NMI :
2582 pc & HARDIRQ_MASK ? RB_CTX_IRQ : RB_CTX_SOFTIRQ;
2583
2584 if (unlikely(val & (1 << bit)))
2585 return 1;
2586
2587 val |= (1 << bit);
2588 cpu_buffer->current_context = val;
2589
2590 return 0;
2591}
2592
2593static __always_inline void
2594trace_recursive_unlock(struct ring_buffer_per_cpu *cpu_buffer)
2595{
2596 cpu_buffer->current_context &= cpu_buffer->current_context - 1;
2597}
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608int ring_buffer_unlock_commit(struct ring_buffer *buffer,
2609 struct ring_buffer_event *event)
2610{
2611 struct ring_buffer_per_cpu *cpu_buffer;
2612 int cpu = raw_smp_processor_id();
2613
2614 cpu_buffer = buffer->buffers[cpu];
2615
2616 rb_commit(cpu_buffer, event);
2617
2618 rb_wakeups(buffer, cpu_buffer);
2619
2620 trace_recursive_unlock(cpu_buffer);
2621
2622 preempt_enable_notrace();
2623
2624 return 0;
2625}
2626EXPORT_SYMBOL_GPL(ring_buffer_unlock_commit);
2627
2628static noinline void
2629rb_handle_timestamp(struct ring_buffer_per_cpu *cpu_buffer,
2630 struct rb_event_info *info)
2631{
2632 WARN_ONCE(info->delta > (1ULL << 59),
2633 KERN_WARNING "Delta way too big! %llu ts=%llu write stamp = %llu\n%s",
2634 (unsigned long long)info->delta,
2635 (unsigned long long)info->ts,
2636 (unsigned long long)cpu_buffer->write_stamp,
2637 sched_clock_stable() ? "" :
2638 "If you just came from a suspend/resume,\n"
2639 "please switch to the trace global clock:\n"
2640 " echo global > /sys/kernel/debug/tracing/trace_clock\n");
2641 info->add_timestamp = 1;
2642}
2643
2644static struct ring_buffer_event *
2645__rb_reserve_next(struct ring_buffer_per_cpu *cpu_buffer,
2646 struct rb_event_info *info)
2647{
2648 struct ring_buffer_event *event;
2649 struct buffer_page *tail_page;
2650 unsigned long tail, write;
2651
2652
2653
2654
2655
2656
2657 if (unlikely(info->add_timestamp))
2658 info->length += RB_LEN_TIME_EXTEND;
2659
2660
2661 tail_page = info->tail_page = READ_ONCE(cpu_buffer->tail_page);
2662 write = local_add_return(info->length, &tail_page->write);
2663
2664
2665 write &= RB_WRITE_MASK;
2666 tail = write - info->length;
2667
2668
2669
2670
2671
2672 if (!tail)
2673 info->delta = 0;
2674
2675
2676 if (unlikely(write > BUF_PAGE_SIZE))
2677 return rb_move_tail(cpu_buffer, tail, info);
2678
2679
2680
2681 event = __rb_page_index(tail_page, tail);
2682 rb_update_event(cpu_buffer, event, info);
2683
2684 local_inc(&tail_page->entries);
2685
2686
2687
2688
2689
2690 if (!tail)
2691 tail_page->page->time_stamp = info->ts;
2692
2693
2694 local_add(info->length, &cpu_buffer->entries_bytes);
2695
2696 return event;
2697}
2698
2699static __always_inline struct ring_buffer_event *
2700rb_reserve_next_event(struct ring_buffer *buffer,
2701 struct ring_buffer_per_cpu *cpu_buffer,
2702 unsigned long length)
2703{
2704 struct ring_buffer_event *event;
2705 struct rb_event_info info;
2706 int nr_loops = 0;
2707 u64 diff;
2708
2709 rb_start_commit(cpu_buffer);
2710
2711#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
2712
2713
2714
2715
2716
2717
2718 barrier();
2719 if (unlikely(READ_ONCE(cpu_buffer->buffer) != buffer)) {
2720 local_dec(&cpu_buffer->committing);
2721 local_dec(&cpu_buffer->commits);
2722 return NULL;
2723 }
2724#endif
2725
2726 info.length = rb_calculate_event_length(length);
2727 again:
2728 info.add_timestamp = 0;
2729 info.delta = 0;
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740 if (RB_WARN_ON(cpu_buffer, ++nr_loops > 1000))
2741 goto out_fail;
2742
2743 info.ts = rb_time_stamp(cpu_buffer->buffer);
2744 diff = info.ts - cpu_buffer->write_stamp;
2745
2746
2747 barrier();
2748
2749
2750 if (likely(info.ts >= cpu_buffer->write_stamp)) {
2751 info.delta = diff;
2752 if (unlikely(test_time_stamp(info.delta)))
2753 rb_handle_timestamp(cpu_buffer, &info);
2754 }
2755
2756 event = __rb_reserve_next(cpu_buffer, &info);
2757
2758 if (unlikely(PTR_ERR(event) == -EAGAIN)) {
2759 if (info.add_timestamp)
2760 info.length -= RB_LEN_TIME_EXTEND;
2761 goto again;
2762 }
2763
2764 if (!event)
2765 goto out_fail;
2766
2767 return event;
2768
2769 out_fail:
2770 rb_end_commit(cpu_buffer);
2771 return NULL;
2772}
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789struct ring_buffer_event *
2790ring_buffer_lock_reserve(struct ring_buffer *buffer, unsigned long length)
2791{
2792 struct ring_buffer_per_cpu *cpu_buffer;
2793 struct ring_buffer_event *event;
2794 int cpu;
2795
2796
2797 preempt_disable_notrace();
2798
2799 if (unlikely(atomic_read(&buffer->record_disabled)))
2800 goto out;
2801
2802 cpu = raw_smp_processor_id();
2803
2804 if (unlikely(!cpumask_test_cpu(cpu, buffer->cpumask)))
2805 goto out;
2806
2807 cpu_buffer = buffer->buffers[cpu];
2808
2809 if (unlikely(atomic_read(&cpu_buffer->record_disabled)))
2810 goto out;
2811
2812 if (unlikely(length > BUF_MAX_DATA_SIZE))
2813 goto out;
2814
2815 if (unlikely(trace_recursive_lock(cpu_buffer)))
2816 goto out;
2817
2818 event = rb_reserve_next_event(buffer, cpu_buffer, length);
2819 if (!event)
2820 goto out_unlock;
2821
2822 return event;
2823
2824 out_unlock:
2825 trace_recursive_unlock(cpu_buffer);
2826 out:
2827 preempt_enable_notrace();
2828 return NULL;
2829}
2830EXPORT_SYMBOL_GPL(ring_buffer_lock_reserve);
2831
2832
2833
2834
2835
2836
2837
2838static inline void
2839rb_decrement_entry(struct ring_buffer_per_cpu *cpu_buffer,
2840 struct ring_buffer_event *event)
2841{
2842 unsigned long addr = (unsigned long)event;
2843 struct buffer_page *bpage = cpu_buffer->commit_page;
2844 struct buffer_page *start;
2845
2846 addr &= PAGE_MASK;
2847
2848
2849 if (likely(bpage->page == (void *)addr)) {
2850 local_dec(&bpage->entries);
2851 return;
2852 }
2853
2854
2855
2856
2857
2858 rb_inc_page(cpu_buffer, &bpage);
2859 start = bpage;
2860 do {
2861 if (bpage->page == (void *)addr) {
2862 local_dec(&bpage->entries);
2863 return;
2864 }
2865 rb_inc_page(cpu_buffer, &bpage);
2866 } while (bpage != start);
2867
2868
2869 RB_WARN_ON(cpu_buffer, 1);
2870}
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891void ring_buffer_discard_commit(struct ring_buffer *buffer,
2892 struct ring_buffer_event *event)
2893{
2894 struct ring_buffer_per_cpu *cpu_buffer;
2895 int cpu;
2896
2897
2898 rb_event_discard(event);
2899
2900 cpu = smp_processor_id();
2901 cpu_buffer = buffer->buffers[cpu];
2902
2903
2904
2905
2906
2907
2908 RB_WARN_ON(buffer, !local_read(&cpu_buffer->committing));
2909
2910 rb_decrement_entry(cpu_buffer, event);
2911 if (rb_try_to_discard(cpu_buffer, event))
2912 goto out;
2913
2914
2915
2916
2917
2918 rb_update_write_stamp(cpu_buffer, event);
2919 out:
2920 rb_end_commit(cpu_buffer);
2921
2922 trace_recursive_unlock(cpu_buffer);
2923
2924 preempt_enable_notrace();
2925
2926}
2927EXPORT_SYMBOL_GPL(ring_buffer_discard_commit);
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942int ring_buffer_write(struct ring_buffer *buffer,
2943 unsigned long length,
2944 void *data)
2945{
2946 struct ring_buffer_per_cpu *cpu_buffer;
2947 struct ring_buffer_event *event;
2948 void *body;
2949 int ret = -EBUSY;
2950 int cpu;
2951
2952 preempt_disable_notrace();
2953
2954 if (atomic_read(&buffer->record_disabled))
2955 goto out;
2956
2957 cpu = raw_smp_processor_id();
2958
2959 if (!cpumask_test_cpu(cpu, buffer->cpumask))
2960 goto out;
2961
2962 cpu_buffer = buffer->buffers[cpu];
2963
2964 if (atomic_read(&cpu_buffer->record_disabled))
2965 goto out;
2966
2967 if (length > BUF_MAX_DATA_SIZE)
2968 goto out;
2969
2970 if (unlikely(trace_recursive_lock(cpu_buffer)))
2971 goto out;
2972
2973 event = rb_reserve_next_event(buffer, cpu_buffer, length);
2974 if (!event)
2975 goto out_unlock;
2976
2977 body = rb_event_data(event);
2978
2979 memcpy(body, data, length);
2980
2981 rb_commit(cpu_buffer, event);
2982
2983 rb_wakeups(buffer, cpu_buffer);
2984
2985 ret = 0;
2986
2987 out_unlock:
2988 trace_recursive_unlock(cpu_buffer);
2989
2990 out:
2991 preempt_enable_notrace();
2992
2993 return ret;
2994}
2995EXPORT_SYMBOL_GPL(ring_buffer_write);
2996
2997static bool rb_per_cpu_empty(struct ring_buffer_per_cpu *cpu_buffer)
2998{
2999 struct buffer_page *reader = cpu_buffer->reader_page;
3000 struct buffer_page *head = rb_set_head_page(cpu_buffer);
3001 struct buffer_page *commit = cpu_buffer->commit_page;
3002
3003
3004 if (unlikely(!head))
3005 return true;
3006
3007 return reader->read == rb_page_commit(reader) &&
3008 (commit == reader ||
3009 (commit == head &&
3010 head->read == rb_page_commit(commit)));
3011}
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022void ring_buffer_record_disable(struct ring_buffer *buffer)
3023{
3024 atomic_inc(&buffer->record_disabled);
3025}
3026EXPORT_SYMBOL_GPL(ring_buffer_record_disable);
3027
3028
3029
3030
3031
3032
3033
3034
3035void ring_buffer_record_enable(struct ring_buffer *buffer)
3036{
3037 atomic_dec(&buffer->record_disabled);
3038}
3039EXPORT_SYMBOL_GPL(ring_buffer_record_enable);
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052void ring_buffer_record_off(struct ring_buffer *buffer)
3053{
3054 unsigned int rd;
3055 unsigned int new_rd;
3056
3057 do {
3058 rd = atomic_read(&buffer->record_disabled);
3059 new_rd = rd | RB_BUFFER_OFF;
3060 } while (atomic_cmpxchg(&buffer->record_disabled, rd, new_rd) != rd);
3061}
3062EXPORT_SYMBOL_GPL(ring_buffer_record_off);
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075void ring_buffer_record_on(struct ring_buffer *buffer)
3076{
3077 unsigned int rd;
3078 unsigned int new_rd;
3079
3080 do {
3081 rd = atomic_read(&buffer->record_disabled);
3082 new_rd = rd & ~RB_BUFFER_OFF;
3083 } while (atomic_cmpxchg(&buffer->record_disabled, rd, new_rd) != rd);
3084}
3085EXPORT_SYMBOL_GPL(ring_buffer_record_on);
3086
3087
3088
3089
3090
3091
3092
3093int ring_buffer_record_is_on(struct ring_buffer *buffer)
3094{
3095 return !atomic_read(&buffer->record_disabled);
3096}
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108void ring_buffer_record_disable_cpu(struct ring_buffer *buffer, int cpu)
3109{
3110 struct ring_buffer_per_cpu *cpu_buffer;
3111
3112 if (!cpumask_test_cpu(cpu, buffer->cpumask))
3113 return;
3114
3115 cpu_buffer = buffer->buffers[cpu];
3116 atomic_inc(&cpu_buffer->record_disabled);
3117}
3118EXPORT_SYMBOL_GPL(ring_buffer_record_disable_cpu);
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128void ring_buffer_record_enable_cpu(struct ring_buffer *buffer, int cpu)
3129{
3130 struct ring_buffer_per_cpu *cpu_buffer;
3131
3132 if (!cpumask_test_cpu(cpu, buffer->cpumask))
3133 return;
3134
3135 cpu_buffer = buffer->buffers[cpu];
3136 atomic_dec(&cpu_buffer->record_disabled);
3137}
3138EXPORT_SYMBOL_GPL(ring_buffer_record_enable_cpu);
3139
3140
3141
3142
3143
3144
3145
3146static inline unsigned long
3147rb_num_of_entries(struct ring_buffer_per_cpu *cpu_buffer)
3148{
3149 return local_read(&cpu_buffer->entries) -
3150 (local_read(&cpu_buffer->overrun) + cpu_buffer->read);
3151}
3152
3153
3154
3155
3156
3157
3158u64 ring_buffer_oldest_event_ts(struct ring_buffer *buffer, int cpu)
3159{
3160 unsigned long flags;
3161 struct ring_buffer_per_cpu *cpu_buffer;
3162 struct buffer_page *bpage;
3163 u64 ret = 0;
3164
3165 if (!cpumask_test_cpu(cpu, buffer->cpumask))
3166 return 0;
3167
3168 cpu_buffer = buffer->buffers[cpu];
3169 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
3170
3171
3172
3173
3174 if (cpu_buffer->tail_page == cpu_buffer->reader_page)
3175 bpage = cpu_buffer->reader_page;
3176 else
3177 bpage = rb_set_head_page(cpu_buffer);
3178 if (bpage)
3179 ret = bpage->page->time_stamp;
3180 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
3181
3182 return ret;
3183}
3184EXPORT_SYMBOL_GPL(ring_buffer_oldest_event_ts);
3185
3186
3187
3188
3189
3190
3191unsigned long ring_buffer_bytes_cpu(struct ring_buffer *buffer, int cpu)
3192{
3193 struct ring_buffer_per_cpu *cpu_buffer;
3194 unsigned long ret;
3195
3196 if (!cpumask_test_cpu(cpu, buffer->cpumask))
3197 return 0;
3198
3199 cpu_buffer = buffer->buffers[cpu];
3200 ret = local_read(&cpu_buffer->entries_bytes) - cpu_buffer->read_bytes;
3201
3202 return ret;
3203}
3204EXPORT_SYMBOL_GPL(ring_buffer_bytes_cpu);
3205
3206
3207
3208
3209
3210
3211unsigned long ring_buffer_entries_cpu(struct ring_buffer *buffer, int cpu)
3212{
3213 struct ring_buffer_per_cpu *cpu_buffer;
3214
3215 if (!cpumask_test_cpu(cpu, buffer->cpumask))
3216 return 0;
3217
3218 cpu_buffer = buffer->buffers[cpu];
3219
3220 return rb_num_of_entries(cpu_buffer);
3221}
3222EXPORT_SYMBOL_GPL(ring_buffer_entries_cpu);
3223
3224
3225
3226
3227
3228
3229
3230unsigned long ring_buffer_overrun_cpu(struct ring_buffer *buffer, int cpu)
3231{
3232 struct ring_buffer_per_cpu *cpu_buffer;
3233 unsigned long ret;
3234
3235 if (!cpumask_test_cpu(cpu, buffer->cpumask))
3236 return 0;
3237
3238 cpu_buffer = buffer->buffers[cpu];
3239 ret = local_read(&cpu_buffer->overrun);
3240
3241 return ret;
3242}
3243EXPORT_SYMBOL_GPL(ring_buffer_overrun_cpu);
3244
3245
3246
3247
3248
3249
3250
3251
3252unsigned long
3253ring_buffer_commit_overrun_cpu(struct ring_buffer *buffer, int cpu)
3254{
3255 struct ring_buffer_per_cpu *cpu_buffer;
3256 unsigned long ret;
3257
3258 if (!cpumask_test_cpu(cpu, buffer->cpumask))
3259 return 0;
3260
3261 cpu_buffer = buffer->buffers[cpu];
3262 ret = local_read(&cpu_buffer->commit_overrun);
3263
3264 return ret;
3265}
3266EXPORT_SYMBOL_GPL(ring_buffer_commit_overrun_cpu);
3267
3268
3269
3270
3271
3272
3273
3274unsigned long
3275ring_buffer_dropped_events_cpu(struct ring_buffer *buffer, int cpu)
3276{
3277 struct ring_buffer_per_cpu *cpu_buffer;
3278 unsigned long ret;
3279
3280 if (!cpumask_test_cpu(cpu, buffer->cpumask))
3281 return 0;
3282
3283 cpu_buffer = buffer->buffers[cpu];
3284 ret = local_read(&cpu_buffer->dropped_events);
3285
3286 return ret;
3287}
3288EXPORT_SYMBOL_GPL(ring_buffer_dropped_events_cpu);
3289
3290
3291
3292
3293
3294
3295unsigned long
3296ring_buffer_read_events_cpu(struct ring_buffer *buffer, int cpu)
3297{
3298 struct ring_buffer_per_cpu *cpu_buffer;
3299
3300 if (!cpumask_test_cpu(cpu, buffer->cpumask))
3301 return 0;
3302
3303 cpu_buffer = buffer->buffers[cpu];
3304 return cpu_buffer->read;
3305}
3306EXPORT_SYMBOL_GPL(ring_buffer_read_events_cpu);
3307
3308
3309
3310
3311
3312
3313
3314
3315unsigned long ring_buffer_entries(struct ring_buffer *buffer)
3316{
3317 struct ring_buffer_per_cpu *cpu_buffer;
3318 unsigned long entries = 0;
3319 int cpu;
3320
3321
3322 for_each_buffer_cpu(buffer, cpu) {
3323 cpu_buffer = buffer->buffers[cpu];
3324 entries += rb_num_of_entries(cpu_buffer);
3325 }
3326
3327 return entries;
3328}
3329EXPORT_SYMBOL_GPL(ring_buffer_entries);
3330
3331
3332
3333
3334
3335
3336
3337
3338unsigned long ring_buffer_overruns(struct ring_buffer *buffer)
3339{
3340 struct ring_buffer_per_cpu *cpu_buffer;
3341 unsigned long overruns = 0;
3342 int cpu;
3343
3344
3345 for_each_buffer_cpu(buffer, cpu) {
3346 cpu_buffer = buffer->buffers[cpu];
3347 overruns += local_read(&cpu_buffer->overrun);
3348 }
3349
3350 return overruns;
3351}
3352EXPORT_SYMBOL_GPL(ring_buffer_overruns);
3353
3354static void rb_iter_reset(struct ring_buffer_iter *iter)
3355{
3356 struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
3357
3358
3359 iter->head_page = cpu_buffer->reader_page;
3360 iter->head = cpu_buffer->reader_page->read;
3361
3362 iter->cache_reader_page = iter->head_page;
3363 iter->cache_read = cpu_buffer->read;
3364
3365 if (iter->head)
3366 iter->read_stamp = cpu_buffer->read_stamp;
3367 else
3368 iter->read_stamp = iter->head_page->page->time_stamp;
3369}
3370
3371
3372
3373
3374
3375
3376
3377
3378void ring_buffer_iter_reset(struct ring_buffer_iter *iter)
3379{
3380 struct ring_buffer_per_cpu *cpu_buffer;
3381 unsigned long flags;
3382
3383 if (!iter)
3384 return;
3385
3386 cpu_buffer = iter->cpu_buffer;
3387
3388 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
3389 rb_iter_reset(iter);
3390 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
3391}
3392EXPORT_SYMBOL_GPL(ring_buffer_iter_reset);
3393
3394
3395
3396
3397
3398int ring_buffer_iter_empty(struct ring_buffer_iter *iter)
3399{
3400 struct ring_buffer_per_cpu *cpu_buffer;
3401 struct buffer_page *reader;
3402 struct buffer_page *head_page;
3403 struct buffer_page *commit_page;
3404 unsigned commit;
3405
3406 cpu_buffer = iter->cpu_buffer;
3407
3408
3409 reader = cpu_buffer->reader_page;
3410 head_page = cpu_buffer->head_page;
3411 commit_page = cpu_buffer->commit_page;
3412 commit = rb_page_commit(commit_page);
3413
3414 return ((iter->head_page == commit_page && iter->head == commit) ||
3415 (iter->head_page == reader && commit_page == head_page &&
3416 head_page->read == commit &&
3417 iter->head == rb_page_commit(cpu_buffer->reader_page)));
3418}
3419EXPORT_SYMBOL_GPL(ring_buffer_iter_empty);
3420
3421static void
3422rb_update_read_stamp(struct ring_buffer_per_cpu *cpu_buffer,
3423 struct ring_buffer_event *event)
3424{
3425 u64 delta;
3426
3427 switch (event->type_len) {
3428 case RINGBUF_TYPE_PADDING:
3429 return;
3430
3431 case RINGBUF_TYPE_TIME_EXTEND:
3432 delta = event->array[0];
3433 delta <<= TS_SHIFT;
3434 delta += event->time_delta;
3435 cpu_buffer->read_stamp += delta;
3436 return;
3437
3438 case RINGBUF_TYPE_TIME_STAMP:
3439
3440 return;
3441
3442 case RINGBUF_TYPE_DATA:
3443 cpu_buffer->read_stamp += event->time_delta;
3444 return;
3445
3446 default:
3447 BUG();
3448 }
3449 return;
3450}
3451
3452static void
3453rb_update_iter_read_stamp(struct ring_buffer_iter *iter,
3454 struct ring_buffer_event *event)
3455{
3456 u64 delta;
3457
3458 switch (event->type_len) {
3459 case RINGBUF_TYPE_PADDING:
3460 return;
3461
3462 case RINGBUF_TYPE_TIME_EXTEND:
3463 delta = event->array[0];
3464 delta <<= TS_SHIFT;
3465 delta += event->time_delta;
3466 iter->read_stamp += delta;
3467 return;
3468
3469 case RINGBUF_TYPE_TIME_STAMP:
3470
3471 return;
3472
3473 case RINGBUF_TYPE_DATA:
3474 iter->read_stamp += event->time_delta;
3475 return;
3476
3477 default:
3478 BUG();
3479 }
3480 return;
3481}
3482
3483static struct buffer_page *
3484rb_get_reader_page(struct ring_buffer_per_cpu *cpu_buffer)
3485{
3486 struct buffer_page *reader = NULL;
3487 unsigned long overwrite;
3488 unsigned long flags;
3489 int nr_loops = 0;
3490 int ret;
3491
3492 local_irq_save(flags);
3493 arch_spin_lock(&cpu_buffer->lock);
3494
3495 again:
3496
3497
3498
3499
3500
3501
3502 if (RB_WARN_ON(cpu_buffer, ++nr_loops > 3)) {
3503 reader = NULL;
3504 goto out;
3505 }
3506
3507 reader = cpu_buffer->reader_page;
3508
3509
3510 if (cpu_buffer->reader_page->read < rb_page_size(reader))
3511 goto out;
3512
3513
3514 if (RB_WARN_ON(cpu_buffer,
3515 cpu_buffer->reader_page->read > rb_page_size(reader)))
3516 goto out;
3517
3518
3519 reader = NULL;
3520 if (cpu_buffer->commit_page == cpu_buffer->reader_page)
3521 goto out;
3522
3523
3524 if (rb_num_of_entries(cpu_buffer) == 0)
3525 goto out;
3526
3527
3528
3529
3530 local_set(&cpu_buffer->reader_page->write, 0);
3531 local_set(&cpu_buffer->reader_page->entries, 0);
3532 local_set(&cpu_buffer->reader_page->page->commit, 0);
3533 cpu_buffer->reader_page->real_end = 0;
3534
3535 spin:
3536
3537
3538
3539 reader = rb_set_head_page(cpu_buffer);
3540 if (!reader)
3541 goto out;
3542 cpu_buffer->reader_page->list.next = rb_list_head(reader->list.next);
3543 cpu_buffer->reader_page->list.prev = reader->list.prev;
3544
3545
3546
3547
3548
3549
3550 cpu_buffer->pages = reader->list.prev;
3551
3552
3553 rb_set_list_to_head(cpu_buffer, &cpu_buffer->reader_page->list);
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564 smp_mb();
3565 overwrite = local_read(&(cpu_buffer->overrun));
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578 ret = rb_head_page_replace(reader, cpu_buffer->reader_page);
3579
3580
3581
3582
3583 if (!ret)
3584 goto spin;
3585
3586
3587
3588
3589
3590
3591 rb_list_head(reader->list.next)->prev = &cpu_buffer->reader_page->list;
3592 rb_inc_page(cpu_buffer, &cpu_buffer->head_page);
3593
3594
3595 cpu_buffer->reader_page = reader;
3596 cpu_buffer->reader_page->read = 0;
3597
3598 if (overwrite != cpu_buffer->last_overrun) {
3599 cpu_buffer->lost_events = overwrite - cpu_buffer->last_overrun;
3600 cpu_buffer->last_overrun = overwrite;
3601 }
3602
3603 goto again;
3604
3605 out:
3606
3607 if (reader && reader->read == 0)
3608 cpu_buffer->read_stamp = reader->page->time_stamp;
3609
3610 arch_spin_unlock(&cpu_buffer->lock);
3611 local_irq_restore(flags);
3612
3613 return reader;
3614}
3615
3616static void rb_advance_reader(struct ring_buffer_per_cpu *cpu_buffer)
3617{
3618 struct ring_buffer_event *event;
3619 struct buffer_page *reader;
3620 unsigned length;
3621
3622 reader = rb_get_reader_page(cpu_buffer);
3623
3624
3625 if (RB_WARN_ON(cpu_buffer, !reader))
3626 return;
3627
3628 event = rb_reader_event(cpu_buffer);
3629
3630 if (event->type_len <= RINGBUF_TYPE_DATA_TYPE_LEN_MAX)
3631 cpu_buffer->read++;
3632
3633 rb_update_read_stamp(cpu_buffer, event);
3634
3635 length = rb_event_length(event);
3636 cpu_buffer->reader_page->read += length;
3637}
3638
3639static void rb_advance_iter(struct ring_buffer_iter *iter)
3640{
3641 struct ring_buffer_per_cpu *cpu_buffer;
3642 struct ring_buffer_event *event;
3643 unsigned length;
3644
3645 cpu_buffer = iter->cpu_buffer;
3646
3647
3648
3649
3650 if (iter->head >= rb_page_size(iter->head_page)) {
3651
3652 if (iter->head_page == cpu_buffer->commit_page)
3653 return;
3654 rb_inc_iter(iter);
3655 return;
3656 }
3657
3658 event = rb_iter_head_event(iter);
3659
3660 length = rb_event_length(event);
3661
3662
3663
3664
3665
3666 if (RB_WARN_ON(cpu_buffer,
3667 (iter->head_page == cpu_buffer->commit_page) &&
3668 (iter->head + length > rb_commit_index(cpu_buffer))))
3669 return;
3670
3671 rb_update_iter_read_stamp(iter, event);
3672
3673 iter->head += length;
3674
3675
3676 if ((iter->head >= rb_page_size(iter->head_page)) &&
3677 (iter->head_page != cpu_buffer->commit_page))
3678 rb_inc_iter(iter);
3679}
3680
3681static int rb_lost_events(struct ring_buffer_per_cpu *cpu_buffer)
3682{
3683 return cpu_buffer->lost_events;
3684}
3685
3686static struct ring_buffer_event *
3687rb_buffer_peek(struct ring_buffer_per_cpu *cpu_buffer, u64 *ts,
3688 unsigned long *lost_events)
3689{
3690 struct ring_buffer_event *event;
3691 struct buffer_page *reader;
3692 int nr_loops = 0;
3693
3694 again:
3695
3696
3697
3698
3699
3700
3701 if (RB_WARN_ON(cpu_buffer, ++nr_loops > 2))
3702 return NULL;
3703
3704 reader = rb_get_reader_page(cpu_buffer);
3705 if (!reader)
3706 return NULL;
3707
3708 event = rb_reader_event(cpu_buffer);
3709
3710 switch (event->type_len) {
3711 case RINGBUF_TYPE_PADDING:
3712 if (rb_null_event(event))
3713 RB_WARN_ON(cpu_buffer, 1);
3714
3715
3716
3717
3718
3719
3720
3721
3722 return event;
3723
3724 case RINGBUF_TYPE_TIME_EXTEND:
3725
3726 rb_advance_reader(cpu_buffer);
3727 goto again;
3728
3729 case RINGBUF_TYPE_TIME_STAMP:
3730
3731 rb_advance_reader(cpu_buffer);
3732 goto again;
3733
3734 case RINGBUF_TYPE_DATA:
3735 if (ts) {
3736 *ts = cpu_buffer->read_stamp + event->time_delta;
3737 ring_buffer_normalize_time_stamp(cpu_buffer->buffer,
3738 cpu_buffer->cpu, ts);
3739 }
3740 if (lost_events)
3741 *lost_events = rb_lost_events(cpu_buffer);
3742 return event;
3743
3744 default:
3745 BUG();
3746 }
3747
3748 return NULL;
3749}
3750EXPORT_SYMBOL_GPL(ring_buffer_peek);
3751
3752static struct ring_buffer_event *
3753rb_iter_peek(struct ring_buffer_iter *iter, u64 *ts)
3754{
3755 struct ring_buffer *buffer;
3756 struct ring_buffer_per_cpu *cpu_buffer;
3757 struct ring_buffer_event *event;
3758 int nr_loops = 0;
3759
3760 cpu_buffer = iter->cpu_buffer;
3761 buffer = cpu_buffer->buffer;
3762
3763
3764
3765
3766
3767
3768 if (unlikely(iter->cache_read != cpu_buffer->read ||
3769 iter->cache_reader_page != cpu_buffer->reader_page))
3770 rb_iter_reset(iter);
3771
3772 again:
3773 if (ring_buffer_iter_empty(iter))
3774 return NULL;
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784 if (RB_WARN_ON(cpu_buffer, ++nr_loops > 3))
3785 return NULL;
3786
3787 if (rb_per_cpu_empty(cpu_buffer))
3788 return NULL;
3789
3790 if (iter->head >= rb_page_size(iter->head_page)) {
3791 rb_inc_iter(iter);
3792 goto again;
3793 }
3794
3795 event = rb_iter_head_event(iter);
3796
3797 switch (event->type_len) {
3798 case RINGBUF_TYPE_PADDING:
3799 if (rb_null_event(event)) {
3800 rb_inc_iter(iter);
3801 goto again;
3802 }
3803 rb_advance_iter(iter);
3804 return event;
3805
3806 case RINGBUF_TYPE_TIME_EXTEND:
3807
3808 rb_advance_iter(iter);
3809 goto again;
3810
3811 case RINGBUF_TYPE_TIME_STAMP:
3812
3813 rb_advance_iter(iter);
3814 goto again;
3815
3816 case RINGBUF_TYPE_DATA:
3817 if (ts) {
3818 *ts = iter->read_stamp + event->time_delta;
3819 ring_buffer_normalize_time_stamp(buffer,
3820 cpu_buffer->cpu, ts);
3821 }
3822 return event;
3823
3824 default:
3825 BUG();
3826 }
3827
3828 return NULL;
3829}
3830EXPORT_SYMBOL_GPL(ring_buffer_iter_peek);
3831
3832static inline bool rb_reader_lock(struct ring_buffer_per_cpu *cpu_buffer)
3833{
3834 if (likely(!in_nmi())) {
3835 raw_spin_lock(&cpu_buffer->reader_lock);
3836 return true;
3837 }
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848 if (raw_spin_trylock(&cpu_buffer->reader_lock))
3849 return true;
3850
3851
3852 atomic_inc(&cpu_buffer->record_disabled);
3853 return false;
3854}
3855
3856static inline void
3857rb_reader_unlock(struct ring_buffer_per_cpu *cpu_buffer, bool locked)
3858{
3859 if (likely(locked))
3860 raw_spin_unlock(&cpu_buffer->reader_lock);
3861 return;
3862}
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874struct ring_buffer_event *
3875ring_buffer_peek(struct ring_buffer *buffer, int cpu, u64 *ts,
3876 unsigned long *lost_events)
3877{
3878 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
3879 struct ring_buffer_event *event;
3880 unsigned long flags;
3881 bool dolock;
3882
3883 if (!cpumask_test_cpu(cpu, buffer->cpumask))
3884 return NULL;
3885
3886 again:
3887 local_irq_save(flags);
3888 dolock = rb_reader_lock(cpu_buffer);
3889 event = rb_buffer_peek(cpu_buffer, ts, lost_events);
3890 if (event && event->type_len == RINGBUF_TYPE_PADDING)
3891 rb_advance_reader(cpu_buffer);
3892 rb_reader_unlock(cpu_buffer, dolock);
3893 local_irq_restore(flags);
3894
3895 if (event && event->type_len == RINGBUF_TYPE_PADDING)
3896 goto again;
3897
3898 return event;
3899}
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909struct ring_buffer_event *
3910ring_buffer_iter_peek(struct ring_buffer_iter *iter, u64 *ts)
3911{
3912 struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
3913 struct ring_buffer_event *event;
3914 unsigned long flags;
3915
3916 again:
3917 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
3918 event = rb_iter_peek(iter, ts);
3919 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
3920
3921 if (event && event->type_len == RINGBUF_TYPE_PADDING)
3922 goto again;
3923
3924 return event;
3925}
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938struct ring_buffer_event *
3939ring_buffer_consume(struct ring_buffer *buffer, int cpu, u64 *ts,
3940 unsigned long *lost_events)
3941{
3942 struct ring_buffer_per_cpu *cpu_buffer;
3943 struct ring_buffer_event *event = NULL;
3944 unsigned long flags;
3945 bool dolock;
3946
3947 again:
3948
3949 preempt_disable();
3950
3951 if (!cpumask_test_cpu(cpu, buffer->cpumask))
3952 goto out;
3953
3954 cpu_buffer = buffer->buffers[cpu];
3955 local_irq_save(flags);
3956 dolock = rb_reader_lock(cpu_buffer);
3957
3958 event = rb_buffer_peek(cpu_buffer, ts, lost_events);
3959 if (event) {
3960 cpu_buffer->lost_events = 0;
3961 rb_advance_reader(cpu_buffer);
3962 }
3963
3964 rb_reader_unlock(cpu_buffer, dolock);
3965 local_irq_restore(flags);
3966
3967 out:
3968 preempt_enable();
3969
3970 if (event && event->type_len == RINGBUF_TYPE_PADDING)
3971 goto again;
3972
3973 return event;
3974}
3975EXPORT_SYMBOL_GPL(ring_buffer_consume);
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997struct ring_buffer_iter *
3998ring_buffer_read_prepare(struct ring_buffer *buffer, int cpu)
3999{
4000 struct ring_buffer_per_cpu *cpu_buffer;
4001 struct ring_buffer_iter *iter;
4002
4003 if (!cpumask_test_cpu(cpu, buffer->cpumask))
4004 return NULL;
4005
4006 iter = kmalloc(sizeof(*iter), GFP_KERNEL);
4007 if (!iter)
4008 return NULL;
4009
4010 cpu_buffer = buffer->buffers[cpu];
4011
4012 iter->cpu_buffer = cpu_buffer;
4013
4014 atomic_inc(&buffer->resize_disabled);
4015 atomic_inc(&cpu_buffer->record_disabled);
4016
4017 return iter;
4018}
4019EXPORT_SYMBOL_GPL(ring_buffer_read_prepare);
4020
4021
4022
4023
4024
4025
4026
4027
4028void
4029ring_buffer_read_prepare_sync(void)
4030{
4031 synchronize_sched();
4032}
4033EXPORT_SYMBOL_GPL(ring_buffer_read_prepare_sync);
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046void
4047ring_buffer_read_start(struct ring_buffer_iter *iter)
4048{
4049 struct ring_buffer_per_cpu *cpu_buffer;
4050 unsigned long flags;
4051
4052 if (!iter)
4053 return;
4054
4055 cpu_buffer = iter->cpu_buffer;
4056
4057 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
4058 arch_spin_lock(&cpu_buffer->lock);
4059 rb_iter_reset(iter);
4060 arch_spin_unlock(&cpu_buffer->lock);
4061 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
4062}
4063EXPORT_SYMBOL_GPL(ring_buffer_read_start);
4064
4065
4066
4067
4068
4069
4070
4071
4072void
4073ring_buffer_read_finish(struct ring_buffer_iter *iter)
4074{
4075 struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
4076 unsigned long flags;
4077
4078
4079
4080
4081
4082
4083
4084 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
4085 rb_check_pages(cpu_buffer);
4086 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
4087
4088 atomic_dec(&cpu_buffer->record_disabled);
4089 atomic_dec(&cpu_buffer->buffer->resize_disabled);
4090 kfree(iter);
4091}
4092EXPORT_SYMBOL_GPL(ring_buffer_read_finish);
4093
4094
4095
4096
4097
4098
4099
4100
4101struct ring_buffer_event *
4102ring_buffer_read(struct ring_buffer_iter *iter, u64 *ts)
4103{
4104 struct ring_buffer_event *event;
4105 struct ring_buffer_per_cpu *cpu_buffer = iter->cpu_buffer;
4106 unsigned long flags;
4107
4108 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
4109 again:
4110 event = rb_iter_peek(iter, ts);
4111 if (!event)
4112 goto out;
4113
4114 if (event->type_len == RINGBUF_TYPE_PADDING)
4115 goto again;
4116
4117 rb_advance_iter(iter);
4118 out:
4119 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
4120
4121 return event;
4122}
4123EXPORT_SYMBOL_GPL(ring_buffer_read);
4124
4125
4126
4127
4128
4129unsigned long ring_buffer_size(struct ring_buffer *buffer, int cpu)
4130{
4131
4132
4133
4134
4135
4136
4137 if (!cpumask_test_cpu(cpu, buffer->cpumask))
4138 return 0;
4139
4140 return BUF_PAGE_SIZE * buffer->buffers[cpu]->nr_pages;
4141}
4142EXPORT_SYMBOL_GPL(ring_buffer_size);
4143
4144static void
4145rb_reset_cpu(struct ring_buffer_per_cpu *cpu_buffer)
4146{
4147 rb_head_page_deactivate(cpu_buffer);
4148
4149 cpu_buffer->head_page
4150 = list_entry(cpu_buffer->pages, struct buffer_page, list);
4151 local_set(&cpu_buffer->head_page->write, 0);
4152 local_set(&cpu_buffer->head_page->entries, 0);
4153 local_set(&cpu_buffer->head_page->page->commit, 0);
4154
4155 cpu_buffer->head_page->read = 0;
4156
4157 cpu_buffer->tail_page = cpu_buffer->head_page;
4158 cpu_buffer->commit_page = cpu_buffer->head_page;
4159
4160 INIT_LIST_HEAD(&cpu_buffer->reader_page->list);
4161 INIT_LIST_HEAD(&cpu_buffer->new_pages);
4162 local_set(&cpu_buffer->reader_page->write, 0);
4163 local_set(&cpu_buffer->reader_page->entries, 0);
4164 local_set(&cpu_buffer->reader_page->page->commit, 0);
4165 cpu_buffer->reader_page->read = 0;
4166
4167 local_set(&cpu_buffer->entries_bytes, 0);
4168 local_set(&cpu_buffer->overrun, 0);
4169 local_set(&cpu_buffer->commit_overrun, 0);
4170 local_set(&cpu_buffer->dropped_events, 0);
4171 local_set(&cpu_buffer->entries, 0);
4172 local_set(&cpu_buffer->committing, 0);
4173 local_set(&cpu_buffer->commits, 0);
4174 cpu_buffer->read = 0;
4175 cpu_buffer->read_bytes = 0;
4176
4177 cpu_buffer->write_stamp = 0;
4178 cpu_buffer->read_stamp = 0;
4179
4180 cpu_buffer->lost_events = 0;
4181 cpu_buffer->last_overrun = 0;
4182
4183 rb_head_page_activate(cpu_buffer);
4184}
4185
4186
4187
4188
4189
4190
4191void ring_buffer_reset_cpu(struct ring_buffer *buffer, int cpu)
4192{
4193 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
4194 unsigned long flags;
4195
4196 if (!cpumask_test_cpu(cpu, buffer->cpumask))
4197 return;
4198
4199 atomic_inc(&buffer->resize_disabled);
4200 atomic_inc(&cpu_buffer->record_disabled);
4201
4202
4203 synchronize_sched();
4204
4205 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
4206
4207 if (RB_WARN_ON(cpu_buffer, local_read(&cpu_buffer->committing)))
4208 goto out;
4209
4210 arch_spin_lock(&cpu_buffer->lock);
4211
4212 rb_reset_cpu(cpu_buffer);
4213
4214 arch_spin_unlock(&cpu_buffer->lock);
4215
4216 out:
4217 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
4218
4219 atomic_dec(&cpu_buffer->record_disabled);
4220 atomic_dec(&buffer->resize_disabled);
4221}
4222EXPORT_SYMBOL_GPL(ring_buffer_reset_cpu);
4223
4224
4225
4226
4227
4228void ring_buffer_reset(struct ring_buffer *buffer)
4229{
4230 int cpu;
4231
4232 for_each_buffer_cpu(buffer, cpu)
4233 ring_buffer_reset_cpu(buffer, cpu);
4234}
4235EXPORT_SYMBOL_GPL(ring_buffer_reset);
4236
4237
4238
4239
4240
4241bool ring_buffer_empty(struct ring_buffer *buffer)
4242{
4243 struct ring_buffer_per_cpu *cpu_buffer;
4244 unsigned long flags;
4245 bool dolock;
4246 int cpu;
4247 int ret;
4248
4249
4250 for_each_buffer_cpu(buffer, cpu) {
4251 cpu_buffer = buffer->buffers[cpu];
4252 local_irq_save(flags);
4253 dolock = rb_reader_lock(cpu_buffer);
4254 ret = rb_per_cpu_empty(cpu_buffer);
4255 rb_reader_unlock(cpu_buffer, dolock);
4256 local_irq_restore(flags);
4257
4258 if (!ret)
4259 return false;
4260 }
4261
4262 return true;
4263}
4264EXPORT_SYMBOL_GPL(ring_buffer_empty);
4265
4266
4267
4268
4269
4270
4271bool ring_buffer_empty_cpu(struct ring_buffer *buffer, int cpu)
4272{
4273 struct ring_buffer_per_cpu *cpu_buffer;
4274 unsigned long flags;
4275 bool dolock;
4276 int ret;
4277
4278 if (!cpumask_test_cpu(cpu, buffer->cpumask))
4279 return true;
4280
4281 cpu_buffer = buffer->buffers[cpu];
4282 local_irq_save(flags);
4283 dolock = rb_reader_lock(cpu_buffer);
4284 ret = rb_per_cpu_empty(cpu_buffer);
4285 rb_reader_unlock(cpu_buffer, dolock);
4286 local_irq_restore(flags);
4287
4288 return ret;
4289}
4290EXPORT_SYMBOL_GPL(ring_buffer_empty_cpu);
4291
4292#ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303int ring_buffer_swap_cpu(struct ring_buffer *buffer_a,
4304 struct ring_buffer *buffer_b, int cpu)
4305{
4306 struct ring_buffer_per_cpu *cpu_buffer_a;
4307 struct ring_buffer_per_cpu *cpu_buffer_b;
4308 int ret = -EINVAL;
4309
4310 if (!cpumask_test_cpu(cpu, buffer_a->cpumask) ||
4311 !cpumask_test_cpu(cpu, buffer_b->cpumask))
4312 goto out;
4313
4314 cpu_buffer_a = buffer_a->buffers[cpu];
4315 cpu_buffer_b = buffer_b->buffers[cpu];
4316
4317
4318 if (cpu_buffer_a->nr_pages != cpu_buffer_b->nr_pages)
4319 goto out;
4320
4321 ret = -EAGAIN;
4322
4323 if (atomic_read(&buffer_a->record_disabled))
4324 goto out;
4325
4326 if (atomic_read(&buffer_b->record_disabled))
4327 goto out;
4328
4329 if (atomic_read(&cpu_buffer_a->record_disabled))
4330 goto out;
4331
4332 if (atomic_read(&cpu_buffer_b->record_disabled))
4333 goto out;
4334
4335
4336
4337
4338
4339
4340
4341 atomic_inc(&cpu_buffer_a->record_disabled);
4342 atomic_inc(&cpu_buffer_b->record_disabled);
4343
4344 ret = -EBUSY;
4345 if (local_read(&cpu_buffer_a->committing))
4346 goto out_dec;
4347 if (local_read(&cpu_buffer_b->committing))
4348 goto out_dec;
4349
4350 buffer_a->buffers[cpu] = cpu_buffer_b;
4351 buffer_b->buffers[cpu] = cpu_buffer_a;
4352
4353 cpu_buffer_b->buffer = buffer_a;
4354 cpu_buffer_a->buffer = buffer_b;
4355
4356 ret = 0;
4357
4358out_dec:
4359 atomic_dec(&cpu_buffer_a->record_disabled);
4360 atomic_dec(&cpu_buffer_b->record_disabled);
4361out:
4362 return ret;
4363}
4364EXPORT_SYMBOL_GPL(ring_buffer_swap_cpu);
4365#endif
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383void *ring_buffer_alloc_read_page(struct ring_buffer *buffer, int cpu)
4384{
4385 struct ring_buffer_per_cpu *cpu_buffer;
4386 struct buffer_data_page *bpage = NULL;
4387 unsigned long flags;
4388 struct page *page;
4389
4390 if (!cpumask_test_cpu(cpu, buffer->cpumask))
4391 return ERR_PTR(-ENODEV);
4392
4393 cpu_buffer = buffer->buffers[cpu];
4394 local_irq_save(flags);
4395 arch_spin_lock(&cpu_buffer->lock);
4396
4397 if (cpu_buffer->free_page) {
4398 bpage = cpu_buffer->free_page;
4399 cpu_buffer->free_page = NULL;
4400 }
4401
4402 arch_spin_unlock(&cpu_buffer->lock);
4403 local_irq_restore(flags);
4404
4405 if (bpage)
4406 goto out;
4407
4408 page = alloc_pages_node(cpu_to_node(cpu),
4409 GFP_KERNEL | __GFP_NORETRY, 0);
4410 if (!page)
4411 return ERR_PTR(-ENOMEM);
4412
4413 bpage = page_address(page);
4414
4415 out:
4416 rb_init_page(bpage);
4417
4418 return bpage;
4419}
4420EXPORT_SYMBOL_GPL(ring_buffer_alloc_read_page);
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430void ring_buffer_free_read_page(struct ring_buffer *buffer, int cpu, void *data)
4431{
4432 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
4433 struct buffer_data_page *bpage = data;
4434 struct page *page = virt_to_page(bpage);
4435 unsigned long flags;
4436
4437
4438 if (page_ref_count(page) > 1)
4439 goto out;
4440
4441 local_irq_save(flags);
4442 arch_spin_lock(&cpu_buffer->lock);
4443
4444 if (!cpu_buffer->free_page) {
4445 cpu_buffer->free_page = bpage;
4446 bpage = NULL;
4447 }
4448
4449 arch_spin_unlock(&cpu_buffer->lock);
4450 local_irq_restore(flags);
4451
4452 out:
4453 free_page((unsigned long)bpage);
4454}
4455EXPORT_SYMBOL_GPL(ring_buffer_free_read_page);
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490int ring_buffer_read_page(struct ring_buffer *buffer,
4491 void **data_page, size_t len, int cpu, int full)
4492{
4493 struct ring_buffer_per_cpu *cpu_buffer = buffer->buffers[cpu];
4494 struct ring_buffer_event *event;
4495 struct buffer_data_page *bpage;
4496 struct buffer_page *reader;
4497 unsigned long missed_events;
4498 unsigned long flags;
4499 unsigned int commit;
4500 unsigned int read;
4501 u64 save_timestamp;
4502 int ret = -1;
4503
4504 if (!cpumask_test_cpu(cpu, buffer->cpumask))
4505 goto out;
4506
4507
4508
4509
4510
4511 if (len <= BUF_PAGE_HDR_SIZE)
4512 goto out;
4513
4514 len -= BUF_PAGE_HDR_SIZE;
4515
4516 if (!data_page)
4517 goto out;
4518
4519 bpage = *data_page;
4520 if (!bpage)
4521 goto out;
4522
4523 raw_spin_lock_irqsave(&cpu_buffer->reader_lock, flags);
4524
4525 reader = rb_get_reader_page(cpu_buffer);
4526 if (!reader)
4527 goto out_unlock;
4528
4529 event = rb_reader_event(cpu_buffer);
4530
4531 read = reader->read;
4532 commit = rb_page_commit(reader);
4533
4534
4535 missed_events = cpu_buffer->lost_events;
4536
4537
4538
4539
4540
4541
4542
4543
4544 if (read || (len < (commit - read)) ||
4545 cpu_buffer->reader_page == cpu_buffer->commit_page) {
4546 struct buffer_data_page *rpage = cpu_buffer->reader_page->page;
4547 unsigned int rpos = read;
4548 unsigned int pos = 0;
4549 unsigned int size;
4550
4551 if (full)
4552 goto out_unlock;
4553
4554 if (len > (commit - read))
4555 len = (commit - read);
4556
4557
4558 size = rb_event_ts_length(event);
4559
4560 if (len < size)
4561 goto out_unlock;
4562
4563
4564 save_timestamp = cpu_buffer->read_stamp;
4565
4566
4567 do {
4568
4569
4570
4571
4572
4573
4574 size = rb_event_length(event);
4575 memcpy(bpage->data + pos, rpage->data + rpos, size);
4576
4577 len -= size;
4578
4579 rb_advance_reader(cpu_buffer);
4580 rpos = reader->read;
4581 pos += size;
4582
4583 if (rpos >= commit)
4584 break;
4585
4586 event = rb_reader_event(cpu_buffer);
4587
4588 size = rb_event_ts_length(event);
4589 } while (len >= size);
4590
4591
4592 local_set(&bpage->commit, pos);
4593 bpage->time_stamp = save_timestamp;
4594
4595
4596 read = 0;
4597 } else {
4598
4599 cpu_buffer->read += rb_page_entries(reader);
4600 cpu_buffer->read_bytes += BUF_PAGE_SIZE;
4601
4602
4603 rb_init_page(bpage);
4604 bpage = reader->page;
4605 reader->page = *data_page;
4606 local_set(&reader->write, 0);
4607 local_set(&reader->entries, 0);
4608 reader->read = 0;
4609 *data_page = bpage;
4610
4611
4612
4613
4614
4615
4616 if (reader->real_end)
4617 local_set(&bpage->commit, reader->real_end);
4618 }
4619 ret = read;
4620
4621 cpu_buffer->lost_events = 0;
4622
4623 commit = local_read(&bpage->commit);
4624
4625
4626
4627 if (missed_events) {
4628
4629
4630
4631 if (BUF_PAGE_SIZE - commit >= sizeof(missed_events)) {
4632 memcpy(&bpage->data[commit], &missed_events,
4633 sizeof(missed_events));
4634 local_add(RB_MISSED_STORED, &bpage->commit);
4635 commit += sizeof(missed_events);
4636 }
4637 local_add(RB_MISSED_EVENTS, &bpage->commit);
4638 }
4639
4640
4641
4642
4643 if (commit < BUF_PAGE_SIZE)
4644 memset(&bpage->data[commit], 0, BUF_PAGE_SIZE - commit);
4645
4646 out_unlock:
4647 raw_spin_unlock_irqrestore(&cpu_buffer->reader_lock, flags);
4648
4649 out:
4650 return ret;
4651}
4652EXPORT_SYMBOL_GPL(ring_buffer_read_page);
4653
4654
4655
4656
4657
4658
4659int trace_rb_cpu_prepare(unsigned int cpu, struct hlist_node *node)
4660{
4661 struct ring_buffer *buffer;
4662 long nr_pages_same;
4663 int cpu_i;
4664 unsigned long nr_pages;
4665
4666 buffer = container_of(node, struct ring_buffer, node);
4667 if (cpumask_test_cpu(cpu, buffer->cpumask))
4668 return 0;
4669
4670 nr_pages = 0;
4671 nr_pages_same = 1;
4672
4673 for_each_buffer_cpu(buffer, cpu_i) {
4674
4675 if (nr_pages == 0)
4676 nr_pages = buffer->buffers[cpu_i]->nr_pages;
4677 if (nr_pages != buffer->buffers[cpu_i]->nr_pages) {
4678 nr_pages_same = 0;
4679 break;
4680 }
4681 }
4682
4683 if (!nr_pages_same)
4684 nr_pages = 2;
4685 buffer->buffers[cpu] =
4686 rb_allocate_cpu_buffer(buffer, nr_pages, cpu);
4687 if (!buffer->buffers[cpu]) {
4688 WARN(1, "failed to allocate ring buffer on CPU %u\n",
4689 cpu);
4690 return -ENOMEM;
4691 }
4692 smp_wmb();
4693 cpumask_set_cpu(cpu, buffer->cpumask);
4694 return 0;
4695}
4696
4697#ifdef CONFIG_RING_BUFFER_STARTUP_TEST
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713static struct task_struct *rb_threads[NR_CPUS] __initdata;
4714
4715struct rb_test_data {
4716 struct ring_buffer *buffer;
4717 unsigned long events;
4718 unsigned long bytes_written;
4719 unsigned long bytes_alloc;
4720 unsigned long bytes_dropped;
4721 unsigned long events_nested;
4722 unsigned long bytes_written_nested;
4723 unsigned long bytes_alloc_nested;
4724 unsigned long bytes_dropped_nested;
4725 int min_size_nested;
4726 int max_size_nested;
4727 int max_size;
4728 int min_size;
4729 int cpu;
4730 int cnt;
4731};
4732
4733static struct rb_test_data rb_data[NR_CPUS] __initdata;
4734
4735
4736#define RB_TEST_BUFFER_SIZE 1048576
4737
4738static char rb_string[] __initdata =
4739 "abcdefghijklmnopqrstuvwxyz1234567890!@#$%^&*()?+\\"
4740 "?+|:';\",.<>/?abcdefghijklmnopqrstuvwxyz1234567890"
4741 "!@#$%^&*()?+\\?+|:';\",.<>/?abcdefghijklmnopqrstuv";
4742
4743static bool rb_test_started __initdata;
4744
4745struct rb_item {
4746 int size;
4747 char str[];
4748};
4749
4750static __init int rb_write_something(struct rb_test_data *data, bool nested)
4751{
4752 struct ring_buffer_event *event;
4753 struct rb_item *item;
4754 bool started;
4755 int event_len;
4756 int size;
4757 int len;
4758 int cnt;
4759
4760
4761 cnt = data->cnt + (nested ? 27 : 0);
4762
4763
4764 size = (data->cnt * 68 / 25) % (sizeof(rb_string) - 1);
4765
4766 len = size + sizeof(struct rb_item);
4767
4768 started = rb_test_started;
4769
4770 smp_rmb();
4771
4772 event = ring_buffer_lock_reserve(data->buffer, len);
4773 if (!event) {
4774
4775 if (started) {
4776 if (nested)
4777 data->bytes_dropped += len;
4778 else
4779 data->bytes_dropped_nested += len;
4780 }
4781 return len;
4782 }
4783
4784 event_len = ring_buffer_event_length(event);
4785
4786 if (RB_WARN_ON(data->buffer, event_len < len))
4787 goto out;
4788
4789 item = ring_buffer_event_data(event);
4790 item->size = size;
4791 memcpy(item->str, rb_string, size);
4792
4793 if (nested) {
4794 data->bytes_alloc_nested += event_len;
4795 data->bytes_written_nested += len;
4796 data->events_nested++;
4797 if (!data->min_size_nested || len < data->min_size_nested)
4798 data->min_size_nested = len;
4799 if (len > data->max_size_nested)
4800 data->max_size_nested = len;
4801 } else {
4802 data->bytes_alloc += event_len;
4803 data->bytes_written += len;
4804 data->events++;
4805 if (!data->min_size || len < data->min_size)
4806 data->max_size = len;
4807 if (len > data->max_size)
4808 data->max_size = len;
4809 }
4810
4811 out:
4812 ring_buffer_unlock_commit(data->buffer, event);
4813
4814 return 0;
4815}
4816
4817static __init int rb_test(void *arg)
4818{
4819 struct rb_test_data *data = arg;
4820
4821 while (!kthread_should_stop()) {
4822 rb_write_something(data, false);
4823 data->cnt++;
4824
4825 set_current_state(TASK_INTERRUPTIBLE);
4826
4827 usleep_range(((data->cnt % 3) + 1) * 100, 1000);
4828 }
4829
4830 return 0;
4831}
4832
4833static __init void rb_ipi(void *ignore)
4834{
4835 struct rb_test_data *data;
4836 int cpu = smp_processor_id();
4837
4838 data = &rb_data[cpu];
4839 rb_write_something(data, true);
4840}
4841
4842static __init int rb_hammer_test(void *arg)
4843{
4844 while (!kthread_should_stop()) {
4845
4846
4847 smp_call_function(rb_ipi, NULL, 1);
4848
4849 schedule();
4850 }
4851
4852 return 0;
4853}
4854
4855static __init int test_ringbuffer(void)
4856{
4857 struct task_struct *rb_hammer;
4858 struct ring_buffer *buffer;
4859 int cpu;
4860 int ret = 0;
4861
4862 pr_info("Running ring buffer tests...\n");
4863
4864 buffer = ring_buffer_alloc(RB_TEST_BUFFER_SIZE, RB_FL_OVERWRITE);
4865 if (WARN_ON(!buffer))
4866 return 0;
4867
4868
4869 ring_buffer_record_off(buffer);
4870
4871 for_each_online_cpu(cpu) {
4872 rb_data[cpu].buffer = buffer;
4873 rb_data[cpu].cpu = cpu;
4874 rb_data[cpu].cnt = cpu;
4875 rb_threads[cpu] = kthread_create(rb_test, &rb_data[cpu],
4876 "rbtester/%d", cpu);
4877 if (WARN_ON(IS_ERR(rb_threads[cpu]))) {
4878 pr_cont("FAILED\n");
4879 ret = PTR_ERR(rb_threads[cpu]);
4880 goto out_free;
4881 }
4882
4883 kthread_bind(rb_threads[cpu], cpu);
4884 wake_up_process(rb_threads[cpu]);
4885 }
4886
4887
4888 rb_hammer = kthread_run(rb_hammer_test, NULL, "rbhammer");
4889 if (WARN_ON(IS_ERR(rb_hammer))) {
4890 pr_cont("FAILED\n");
4891 ret = PTR_ERR(rb_hammer);
4892 goto out_free;
4893 }
4894
4895 ring_buffer_record_on(buffer);
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905 smp_wmb();
4906 rb_test_started = true;
4907
4908 set_current_state(TASK_INTERRUPTIBLE);
4909 ;
4910 schedule_timeout(10 * HZ);
4911
4912 kthread_stop(rb_hammer);
4913
4914 out_free:
4915 for_each_online_cpu(cpu) {
4916 if (!rb_threads[cpu])
4917 break;
4918 kthread_stop(rb_threads[cpu]);
4919 }
4920 if (ret) {
4921 ring_buffer_free(buffer);
4922 return ret;
4923 }
4924
4925
4926 pr_info("finished\n");
4927 for_each_online_cpu(cpu) {
4928 struct ring_buffer_event *event;
4929 struct rb_test_data *data = &rb_data[cpu];
4930 struct rb_item *item;
4931 unsigned long total_events;
4932 unsigned long total_dropped;
4933 unsigned long total_written;
4934 unsigned long total_alloc;
4935 unsigned long total_read = 0;
4936 unsigned long total_size = 0;
4937 unsigned long total_len = 0;
4938 unsigned long total_lost = 0;
4939 unsigned long lost;
4940 int big_event_size;
4941 int small_event_size;
4942
4943 ret = -1;
4944
4945 total_events = data->events + data->events_nested;
4946 total_written = data->bytes_written + data->bytes_written_nested;
4947 total_alloc = data->bytes_alloc + data->bytes_alloc_nested;
4948 total_dropped = data->bytes_dropped + data->bytes_dropped_nested;
4949
4950 big_event_size = data->max_size + data->max_size_nested;
4951 small_event_size = data->min_size + data->min_size_nested;
4952
4953 pr_info("CPU %d:\n", cpu);
4954 pr_info(" events: %ld\n", total_events);
4955 pr_info(" dropped bytes: %ld\n", total_dropped);
4956 pr_info(" alloced bytes: %ld\n", total_alloc);
4957 pr_info(" written bytes: %ld\n", total_written);
4958 pr_info(" biggest event: %d\n", big_event_size);
4959 pr_info(" smallest event: %d\n", small_event_size);
4960
4961 if (RB_WARN_ON(buffer, total_dropped))
4962 break;
4963
4964 ret = 0;
4965
4966 while ((event = ring_buffer_consume(buffer, cpu, NULL, &lost))) {
4967 total_lost += lost;
4968 item = ring_buffer_event_data(event);
4969 total_len += ring_buffer_event_length(event);
4970 total_size += item->size + sizeof(struct rb_item);
4971 if (memcmp(&item->str[0], rb_string, item->size) != 0) {
4972 pr_info("FAILED!\n");
4973 pr_info("buffer had: %.*s\n", item->size, item->str);
4974 pr_info("expected: %.*s\n", item->size, rb_string);
4975 RB_WARN_ON(buffer, 1);
4976 ret = -1;
4977 break;
4978 }
4979 total_read++;
4980 }
4981 if (ret)
4982 break;
4983
4984 ret = -1;
4985
4986 pr_info(" read events: %ld\n", total_read);
4987 pr_info(" lost events: %ld\n", total_lost);
4988 pr_info(" total events: %ld\n", total_lost + total_read);
4989 pr_info(" recorded len bytes: %ld\n", total_len);
4990 pr_info(" recorded size bytes: %ld\n", total_size);
4991 if (total_lost)
4992 pr_info(" With dropped events, record len and size may not match\n"
4993 " alloced and written from above\n");
4994 if (!total_lost) {
4995 if (RB_WARN_ON(buffer, total_len != total_alloc ||
4996 total_size != total_written))
4997 break;
4998 }
4999 if (RB_WARN_ON(buffer, total_lost + total_read != total_events))
5000 break;
5001
5002 ret = 0;
5003 }
5004 if (!ret)
5005 pr_info("Ring buffer PASSED!\n");
5006
5007 ring_buffer_free(buffer);
5008 return 0;
5009}
5010
5011late_initcall(test_ringbuffer);
5012#endif
5013