1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19#include "xfs.h"
20#include "xfs_fs.h"
21#include "xfs_format.h"
22#include "xfs_log_format.h"
23#include "xfs_trans_resv.h"
24#include "xfs_mount.h"
25#include "xfs_trans.h"
26#include "xfs_trans_priv.h"
27#include "xfs_trace.h"
28#include "xfs_error.h"
29#include "xfs_log.h"
30
31#ifdef DEBUG
32
33
34
35STATIC void
36xfs_ail_check(
37 struct xfs_ail *ailp,
38 xfs_log_item_t *lip)
39{
40 xfs_log_item_t *prev_lip;
41
42 if (list_empty(&ailp->xa_ail))
43 return;
44
45
46
47
48 ASSERT((lip->li_flags & XFS_LI_IN_AIL) != 0);
49 prev_lip = list_entry(lip->li_ail.prev, xfs_log_item_t, li_ail);
50 if (&prev_lip->li_ail != &ailp->xa_ail)
51 ASSERT(XFS_LSN_CMP(prev_lip->li_lsn, lip->li_lsn) <= 0);
52
53 prev_lip = list_entry(lip->li_ail.next, xfs_log_item_t, li_ail);
54 if (&prev_lip->li_ail != &ailp->xa_ail)
55 ASSERT(XFS_LSN_CMP(prev_lip->li_lsn, lip->li_lsn) >= 0);
56
57
58}
59#else
60#define xfs_ail_check(a,l)
61#endif
62
63
64
65
66
67static xfs_log_item_t *
68xfs_ail_max(
69 struct xfs_ail *ailp)
70{
71 if (list_empty(&ailp->xa_ail))
72 return NULL;
73
74 return list_entry(ailp->xa_ail.prev, xfs_log_item_t, li_ail);
75}
76
77
78
79
80
81static xfs_log_item_t *
82xfs_ail_next(
83 struct xfs_ail *ailp,
84 xfs_log_item_t *lip)
85{
86 if (lip->li_ail.next == &ailp->xa_ail)
87 return NULL;
88
89 return list_first_entry(&lip->li_ail, xfs_log_item_t, li_ail);
90}
91
92
93
94
95
96
97
98
99
100xfs_lsn_t
101xfs_ail_min_lsn(
102 struct xfs_ail *ailp)
103{
104 xfs_lsn_t lsn = 0;
105 xfs_log_item_t *lip;
106
107 spin_lock(&ailp->xa_lock);
108 lip = xfs_ail_min(ailp);
109 if (lip)
110 lsn = lip->li_lsn;
111 spin_unlock(&ailp->xa_lock);
112
113 return lsn;
114}
115
116
117
118
119static xfs_lsn_t
120xfs_ail_max_lsn(
121 struct xfs_ail *ailp)
122{
123 xfs_lsn_t lsn = 0;
124 xfs_log_item_t *lip;
125
126 spin_lock(&ailp->xa_lock);
127 lip = xfs_ail_max(ailp);
128 if (lip)
129 lsn = lip->li_lsn;
130 spin_unlock(&ailp->xa_lock);
131
132 return lsn;
133}
134
135
136
137
138
139
140
141
142STATIC void
143xfs_trans_ail_cursor_init(
144 struct xfs_ail *ailp,
145 struct xfs_ail_cursor *cur)
146{
147 cur->item = NULL;
148 list_add_tail(&cur->list, &ailp->xa_cursors);
149}
150
151
152
153
154
155struct xfs_log_item *
156xfs_trans_ail_cursor_next(
157 struct xfs_ail *ailp,
158 struct xfs_ail_cursor *cur)
159{
160 struct xfs_log_item *lip = cur->item;
161
162 if ((uintptr_t)lip & 1)
163 lip = xfs_ail_min(ailp);
164 if (lip)
165 cur->item = xfs_ail_next(ailp, lip);
166 return lip;
167}
168
169
170
171
172
173void
174xfs_trans_ail_cursor_done(
175 struct xfs_ail_cursor *cur)
176{
177 cur->item = NULL;
178 list_del_init(&cur->list);
179}
180
181
182
183
184
185
186
187
188
189STATIC void
190xfs_trans_ail_cursor_clear(
191 struct xfs_ail *ailp,
192 struct xfs_log_item *lip)
193{
194 struct xfs_ail_cursor *cur;
195
196 list_for_each_entry(cur, &ailp->xa_cursors, list) {
197 if (cur->item == lip)
198 cur->item = (struct xfs_log_item *)
199 ((uintptr_t)cur->item | 1);
200 }
201}
202
203
204
205
206
207
208
209xfs_log_item_t *
210xfs_trans_ail_cursor_first(
211 struct xfs_ail *ailp,
212 struct xfs_ail_cursor *cur,
213 xfs_lsn_t lsn)
214{
215 xfs_log_item_t *lip;
216
217 xfs_trans_ail_cursor_init(ailp, cur);
218
219 if (lsn == 0) {
220 lip = xfs_ail_min(ailp);
221 goto out;
222 }
223
224 list_for_each_entry(lip, &ailp->xa_ail, li_ail) {
225 if (XFS_LSN_CMP(lip->li_lsn, lsn) >= 0)
226 goto out;
227 }
228 return NULL;
229
230out:
231 if (lip)
232 cur->item = xfs_ail_next(ailp, lip);
233 return lip;
234}
235
236static struct xfs_log_item *
237__xfs_trans_ail_cursor_last(
238 struct xfs_ail *ailp,
239 xfs_lsn_t lsn)
240{
241 xfs_log_item_t *lip;
242
243 list_for_each_entry_reverse(lip, &ailp->xa_ail, li_ail) {
244 if (XFS_LSN_CMP(lip->li_lsn, lsn) <= 0)
245 return lip;
246 }
247 return NULL;
248}
249
250
251
252
253
254
255
256struct xfs_log_item *
257xfs_trans_ail_cursor_last(
258 struct xfs_ail *ailp,
259 struct xfs_ail_cursor *cur,
260 xfs_lsn_t lsn)
261{
262 xfs_trans_ail_cursor_init(ailp, cur);
263 cur->item = __xfs_trans_ail_cursor_last(ailp, lsn);
264 return cur->item;
265}
266
267
268
269
270
271
272
273static void
274xfs_ail_splice(
275 struct xfs_ail *ailp,
276 struct xfs_ail_cursor *cur,
277 struct list_head *list,
278 xfs_lsn_t lsn)
279{
280 struct xfs_log_item *lip;
281
282 ASSERT(!list_empty(list));
283
284
285
286
287
288
289 lip = cur ? cur->item : NULL;
290 if (!lip || (uintptr_t)lip & 1)
291 lip = __xfs_trans_ail_cursor_last(ailp, lsn);
292
293
294
295
296
297
298
299
300 if (cur)
301 cur->item = list_entry(list->prev, struct xfs_log_item, li_ail);
302
303
304
305
306
307
308
309 if (lip)
310 list_splice(list, &lip->li_ail);
311 else
312 list_splice(list, &ailp->xa_ail);
313}
314
315
316
317
318static void
319xfs_ail_delete(
320 struct xfs_ail *ailp,
321 xfs_log_item_t *lip)
322{
323 xfs_ail_check(ailp, lip);
324 list_del(&lip->li_ail);
325 xfs_trans_ail_cursor_clear(ailp, lip);
326}
327
328static inline uint
329xfsaild_push_item(
330 struct xfs_ail *ailp,
331 struct xfs_log_item *lip)
332{
333
334
335
336
337 if (XFS_TEST_ERROR(false, ailp->xa_mount, XFS_ERRTAG_LOG_ITEM_PIN))
338 return XFS_ITEM_PINNED;
339
340 return lip->li_ops->iop_push(lip, &ailp->xa_buf_list);
341}
342
343static long
344xfsaild_push(
345 struct xfs_ail *ailp)
346{
347 xfs_mount_t *mp = ailp->xa_mount;
348 struct xfs_ail_cursor cur;
349 xfs_log_item_t *lip;
350 xfs_lsn_t lsn;
351 xfs_lsn_t target;
352 long tout;
353 int stuck = 0;
354 int flushing = 0;
355 int count = 0;
356
357
358
359
360
361
362 if (ailp->xa_log_flush && ailp->xa_last_pushed_lsn == 0 &&
363 (!list_empty_careful(&ailp->xa_buf_list) ||
364 xfs_ail_min_lsn(ailp))) {
365 ailp->xa_log_flush = 0;
366
367 XFS_STATS_INC(mp, xs_push_ail_flush);
368 xfs_log_force(mp, XFS_LOG_SYNC);
369 }
370
371 spin_lock(&ailp->xa_lock);
372
373
374 smp_rmb();
375 target = ailp->xa_target;
376 ailp->xa_target_prev = target;
377
378 lip = xfs_trans_ail_cursor_first(ailp, &cur, ailp->xa_last_pushed_lsn);
379 if (!lip) {
380
381
382
383
384 xfs_trans_ail_cursor_done(&cur);
385 spin_unlock(&ailp->xa_lock);
386 goto out_done;
387 }
388
389 XFS_STATS_INC(mp, xs_push_ail);
390
391 lsn = lip->li_lsn;
392 while ((XFS_LSN_CMP(lip->li_lsn, target) <= 0)) {
393 int lock_result;
394
395
396
397
398
399
400 lock_result = xfsaild_push_item(ailp, lip);
401 switch (lock_result) {
402 case XFS_ITEM_SUCCESS:
403 XFS_STATS_INC(mp, xs_push_ail_success);
404 trace_xfs_ail_push(lip);
405
406 ailp->xa_last_pushed_lsn = lsn;
407 break;
408
409 case XFS_ITEM_FLUSHING:
410
411
412
413
414
415
416
417
418
419
420
421 XFS_STATS_INC(mp, xs_push_ail_flushing);
422 trace_xfs_ail_flushing(lip);
423
424 flushing++;
425 ailp->xa_last_pushed_lsn = lsn;
426 break;
427
428 case XFS_ITEM_PINNED:
429 XFS_STATS_INC(mp, xs_push_ail_pinned);
430 trace_xfs_ail_pinned(lip);
431
432 stuck++;
433 ailp->xa_log_flush++;
434 break;
435 case XFS_ITEM_LOCKED:
436 XFS_STATS_INC(mp, xs_push_ail_locked);
437 trace_xfs_ail_locked(lip);
438
439 stuck++;
440 break;
441 default:
442 ASSERT(0);
443 break;
444 }
445
446 count++;
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461 if (stuck > 100)
462 break;
463
464 lip = xfs_trans_ail_cursor_next(ailp, &cur);
465 if (lip == NULL)
466 break;
467 lsn = lip->li_lsn;
468 }
469 xfs_trans_ail_cursor_done(&cur);
470 spin_unlock(&ailp->xa_lock);
471
472 if (xfs_buf_delwri_submit_nowait(&ailp->xa_buf_list))
473 ailp->xa_log_flush++;
474
475 if (!count || XFS_LSN_CMP(lsn, target) >= 0) {
476out_done:
477
478
479
480
481
482 tout = 50;
483 ailp->xa_last_pushed_lsn = 0;
484 } else if (((stuck + flushing) * 100) / count > 90) {
485
486
487
488
489
490
491
492
493
494
495 tout = 20;
496 ailp->xa_last_pushed_lsn = 0;
497 } else {
498
499
500
501 tout = 10;
502 }
503
504 return tout;
505}
506
507static int
508xfsaild(
509 void *data)
510{
511 struct xfs_ail *ailp = data;
512 long tout = 0;
513
514 current->flags |= PF_MEMALLOC;
515 set_freezable();
516
517 while (!kthread_should_stop()) {
518 if (tout && tout <= 20)
519 __set_current_state(TASK_KILLABLE);
520 else
521 __set_current_state(TASK_INTERRUPTIBLE);
522
523 spin_lock(&ailp->xa_lock);
524
525
526
527
528
529
530
531
532
533
534 smp_rmb();
535 if (!xfs_ail_min(ailp) &&
536 ailp->xa_target == ailp->xa_target_prev) {
537 spin_unlock(&ailp->xa_lock);
538 freezable_schedule();
539 tout = 0;
540 continue;
541 }
542 spin_unlock(&ailp->xa_lock);
543
544 if (tout)
545 freezable_schedule_timeout(msecs_to_jiffies(tout));
546
547 __set_current_state(TASK_RUNNING);
548
549 try_to_freeze();
550
551 tout = xfsaild_push(ailp);
552 }
553
554 return 0;
555}
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571void
572xfs_ail_push(
573 struct xfs_ail *ailp,
574 xfs_lsn_t threshold_lsn)
575{
576 xfs_log_item_t *lip;
577
578 lip = xfs_ail_min(ailp);
579 if (!lip || XFS_FORCED_SHUTDOWN(ailp->xa_mount) ||
580 XFS_LSN_CMP(threshold_lsn, ailp->xa_target) <= 0)
581 return;
582
583
584
585
586
587 smp_wmb();
588 xfs_trans_ail_copy_lsn(ailp, &ailp->xa_target, &threshold_lsn);
589 smp_wmb();
590
591 wake_up_process(ailp->xa_task);
592}
593
594
595
596
597void
598xfs_ail_push_all(
599 struct xfs_ail *ailp)
600{
601 xfs_lsn_t threshold_lsn = xfs_ail_max_lsn(ailp);
602
603 if (threshold_lsn)
604 xfs_ail_push(ailp, threshold_lsn);
605}
606
607
608
609
610void
611xfs_ail_push_all_sync(
612 struct xfs_ail *ailp)
613{
614 struct xfs_log_item *lip;
615 DEFINE_WAIT(wait);
616
617 spin_lock(&ailp->xa_lock);
618 while ((lip = xfs_ail_max(ailp)) != NULL) {
619 prepare_to_wait(&ailp->xa_empty, &wait, TASK_UNINTERRUPTIBLE);
620 ailp->xa_target = lip->li_lsn;
621 wake_up_process(ailp->xa_task);
622 spin_unlock(&ailp->xa_lock);
623 schedule();
624 spin_lock(&ailp->xa_lock);
625 }
626 spin_unlock(&ailp->xa_lock);
627
628 finish_wait(&ailp->xa_empty, &wait);
629}
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653void
654xfs_trans_ail_update_bulk(
655 struct xfs_ail *ailp,
656 struct xfs_ail_cursor *cur,
657 struct xfs_log_item **log_items,
658 int nr_items,
659 xfs_lsn_t lsn) __releases(ailp->xa_lock)
660{
661 xfs_log_item_t *mlip;
662 int mlip_changed = 0;
663 int i;
664 LIST_HEAD(tmp);
665
666 ASSERT(nr_items > 0);
667 mlip = xfs_ail_min(ailp);
668
669 for (i = 0; i < nr_items; i++) {
670 struct xfs_log_item *lip = log_items[i];
671 if (lip->li_flags & XFS_LI_IN_AIL) {
672
673 if (XFS_LSN_CMP(lsn, lip->li_lsn) <= 0)
674 continue;
675
676 trace_xfs_ail_move(lip, lip->li_lsn, lsn);
677 xfs_ail_delete(ailp, lip);
678 if (mlip == lip)
679 mlip_changed = 1;
680 } else {
681 lip->li_flags |= XFS_LI_IN_AIL;
682 trace_xfs_ail_insert(lip, 0, lsn);
683 }
684 lip->li_lsn = lsn;
685 list_add(&lip->li_ail, &tmp);
686 }
687
688 if (!list_empty(&tmp))
689 xfs_ail_splice(ailp, cur, &tmp, lsn);
690
691 if (mlip_changed) {
692 if (!XFS_FORCED_SHUTDOWN(ailp->xa_mount))
693 xlog_assign_tail_lsn_locked(ailp->xa_mount);
694 spin_unlock(&ailp->xa_lock);
695
696 xfs_log_space_wake(ailp->xa_mount);
697 } else {
698 spin_unlock(&ailp->xa_lock);
699 }
700}
701
702bool
703xfs_ail_delete_one(
704 struct xfs_ail *ailp,
705 struct xfs_log_item *lip)
706{
707 struct xfs_log_item *mlip = xfs_ail_min(ailp);
708
709 trace_xfs_ail_delete(lip, mlip->li_lsn, lip->li_lsn);
710 xfs_ail_delete(ailp, lip);
711 xfs_clear_li_failed(lip);
712 lip->li_flags &= ~XFS_LI_IN_AIL;
713 lip->li_lsn = 0;
714
715 return mlip == lip;
716}
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739void
740xfs_trans_ail_delete(
741 struct xfs_ail *ailp,
742 struct xfs_log_item *lip,
743 int shutdown_type) __releases(ailp->xa_lock)
744{
745 struct xfs_mount *mp = ailp->xa_mount;
746 bool mlip_changed;
747
748 if (!(lip->li_flags & XFS_LI_IN_AIL)) {
749 spin_unlock(&ailp->xa_lock);
750 if (!XFS_FORCED_SHUTDOWN(mp)) {
751 xfs_alert_tag(mp, XFS_PTAG_AILDELETE,
752 "%s: attempting to delete a log item that is not in the AIL",
753 __func__);
754 xfs_force_shutdown(mp, shutdown_type);
755 }
756 return;
757 }
758
759 mlip_changed = xfs_ail_delete_one(ailp, lip);
760 if (mlip_changed) {
761 if (!XFS_FORCED_SHUTDOWN(mp))
762 xlog_assign_tail_lsn_locked(mp);
763 if (list_empty(&ailp->xa_ail))
764 wake_up_all(&ailp->xa_empty);
765 }
766
767 spin_unlock(&ailp->xa_lock);
768 if (mlip_changed)
769 xfs_log_space_wake(ailp->xa_mount);
770}
771
772int
773xfs_trans_ail_init(
774 xfs_mount_t *mp)
775{
776 struct xfs_ail *ailp;
777
778 ailp = kmem_zalloc(sizeof(struct xfs_ail), KM_MAYFAIL);
779 if (!ailp)
780 return -ENOMEM;
781
782 ailp->xa_mount = mp;
783 INIT_LIST_HEAD(&ailp->xa_ail);
784 INIT_LIST_HEAD(&ailp->xa_cursors);
785 spin_lock_init(&ailp->xa_lock);
786 INIT_LIST_HEAD(&ailp->xa_buf_list);
787 init_waitqueue_head(&ailp->xa_empty);
788
789 ailp->xa_task = kthread_run(xfsaild, ailp, "xfsaild/%s",
790 ailp->xa_mount->m_fsname);
791 if (IS_ERR(ailp->xa_task))
792 goto out_free_ailp;
793
794 mp->m_ail = ailp;
795 return 0;
796
797out_free_ailp:
798 kmem_free(ailp);
799 return -ENOMEM;
800}
801
802void
803xfs_trans_ail_destroy(
804 xfs_mount_t *mp)
805{
806 struct xfs_ail *ailp = mp->m_ail;
807
808 kthread_stop(ailp->xa_task);
809 kmem_free(ailp);
810}
811