1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19#include "xfs.h"
20#include "xfs_fs.h"
21#include "xfs_format.h"
22#include "xfs_log_format.h"
23#include "xfs_trans_resv.h"
24#include "xfs_mount.h"
25#include "xfs_trans.h"
26#include "xfs_trans_priv.h"
27#include "xfs_trace.h"
28#include "xfs_error.h"
29#include "xfs_log.h"
30
31#ifdef DEBUG
32
33
34
35STATIC void
36xfs_ail_check(
37 struct xfs_ail *ailp,
38 xfs_log_item_t *lip)
39{
40 xfs_log_item_t *prev_lip;
41
42 if (list_empty(&ailp->xa_ail))
43 return;
44
45
46
47
48 ASSERT((lip->li_flags & XFS_LI_IN_AIL) != 0);
49 prev_lip = list_entry(lip->li_ail.prev, xfs_log_item_t, li_ail);
50 if (&prev_lip->li_ail != &ailp->xa_ail)
51 ASSERT(XFS_LSN_CMP(prev_lip->li_lsn, lip->li_lsn) <= 0);
52
53 prev_lip = list_entry(lip->li_ail.next, xfs_log_item_t, li_ail);
54 if (&prev_lip->li_ail != &ailp->xa_ail)
55 ASSERT(XFS_LSN_CMP(prev_lip->li_lsn, lip->li_lsn) >= 0);
56
57
58}
59#else
60#define xfs_ail_check(a,l)
61#endif
62
63
64
65
66
67static xfs_log_item_t *
68xfs_ail_max(
69 struct xfs_ail *ailp)
70{
71 if (list_empty(&ailp->xa_ail))
72 return NULL;
73
74 return list_entry(ailp->xa_ail.prev, xfs_log_item_t, li_ail);
75}
76
77
78
79
80
81static xfs_log_item_t *
82xfs_ail_next(
83 struct xfs_ail *ailp,
84 xfs_log_item_t *lip)
85{
86 if (lip->li_ail.next == &ailp->xa_ail)
87 return NULL;
88
89 return list_first_entry(&lip->li_ail, xfs_log_item_t, li_ail);
90}
91
92
93
94
95
96
97
98
99
100xfs_lsn_t
101xfs_ail_min_lsn(
102 struct xfs_ail *ailp)
103{
104 xfs_lsn_t lsn = 0;
105 xfs_log_item_t *lip;
106
107 spin_lock(&ailp->xa_lock);
108 lip = xfs_ail_min(ailp);
109 if (lip)
110 lsn = lip->li_lsn;
111 spin_unlock(&ailp->xa_lock);
112
113 return lsn;
114}
115
116
117
118
119static xfs_lsn_t
120xfs_ail_max_lsn(
121 struct xfs_ail *ailp)
122{
123 xfs_lsn_t lsn = 0;
124 xfs_log_item_t *lip;
125
126 spin_lock(&ailp->xa_lock);
127 lip = xfs_ail_max(ailp);
128 if (lip)
129 lsn = lip->li_lsn;
130 spin_unlock(&ailp->xa_lock);
131
132 return lsn;
133}
134
135
136
137
138
139
140
141
142STATIC void
143xfs_trans_ail_cursor_init(
144 struct xfs_ail *ailp,
145 struct xfs_ail_cursor *cur)
146{
147 cur->item = NULL;
148 list_add_tail(&cur->list, &ailp->xa_cursors);
149}
150
151
152
153
154
155struct xfs_log_item *
156xfs_trans_ail_cursor_next(
157 struct xfs_ail *ailp,
158 struct xfs_ail_cursor *cur)
159{
160 struct xfs_log_item *lip = cur->item;
161
162 if ((__psint_t)lip & 1)
163 lip = xfs_ail_min(ailp);
164 if (lip)
165 cur->item = xfs_ail_next(ailp, lip);
166 return lip;
167}
168
169
170
171
172
173void
174xfs_trans_ail_cursor_done(
175 struct xfs_ail_cursor *cur)
176{
177 cur->item = NULL;
178 list_del_init(&cur->list);
179}
180
181
182
183
184
185
186
187
188
189STATIC void
190xfs_trans_ail_cursor_clear(
191 struct xfs_ail *ailp,
192 struct xfs_log_item *lip)
193{
194 struct xfs_ail_cursor *cur;
195
196 list_for_each_entry(cur, &ailp->xa_cursors, list) {
197 if (cur->item == lip)
198 cur->item = (struct xfs_log_item *)
199 ((__psint_t)cur->item | 1);
200 }
201}
202
203
204
205
206
207
208
209xfs_log_item_t *
210xfs_trans_ail_cursor_first(
211 struct xfs_ail *ailp,
212 struct xfs_ail_cursor *cur,
213 xfs_lsn_t lsn)
214{
215 xfs_log_item_t *lip;
216
217 xfs_trans_ail_cursor_init(ailp, cur);
218
219 if (lsn == 0) {
220 lip = xfs_ail_min(ailp);
221 goto out;
222 }
223
224 list_for_each_entry(lip, &ailp->xa_ail, li_ail) {
225 if (XFS_LSN_CMP(lip->li_lsn, lsn) >= 0)
226 goto out;
227 }
228 return NULL;
229
230out:
231 if (lip)
232 cur->item = xfs_ail_next(ailp, lip);
233 return lip;
234}
235
236static struct xfs_log_item *
237__xfs_trans_ail_cursor_last(
238 struct xfs_ail *ailp,
239 xfs_lsn_t lsn)
240{
241 xfs_log_item_t *lip;
242
243 list_for_each_entry_reverse(lip, &ailp->xa_ail, li_ail) {
244 if (XFS_LSN_CMP(lip->li_lsn, lsn) <= 0)
245 return lip;
246 }
247 return NULL;
248}
249
250
251
252
253
254
255
256struct xfs_log_item *
257xfs_trans_ail_cursor_last(
258 struct xfs_ail *ailp,
259 struct xfs_ail_cursor *cur,
260 xfs_lsn_t lsn)
261{
262 xfs_trans_ail_cursor_init(ailp, cur);
263 cur->item = __xfs_trans_ail_cursor_last(ailp, lsn);
264 return cur->item;
265}
266
267
268
269
270
271
272
273static void
274xfs_ail_splice(
275 struct xfs_ail *ailp,
276 struct xfs_ail_cursor *cur,
277 struct list_head *list,
278 xfs_lsn_t lsn)
279{
280 struct xfs_log_item *lip;
281
282 ASSERT(!list_empty(list));
283
284
285
286
287
288
289 lip = cur ? cur->item : NULL;
290 if (!lip || (__psint_t) lip & 1)
291 lip = __xfs_trans_ail_cursor_last(ailp, lsn);
292
293
294
295
296
297
298
299
300 if (cur)
301 cur->item = list_entry(list->prev, struct xfs_log_item, li_ail);
302
303
304
305
306
307
308
309 if (lip)
310 list_splice(list, &lip->li_ail);
311 else
312 list_splice(list, &ailp->xa_ail);
313}
314
315
316
317
318static void
319xfs_ail_delete(
320 struct xfs_ail *ailp,
321 xfs_log_item_t *lip)
322{
323 xfs_ail_check(ailp, lip);
324 list_del(&lip->li_ail);
325 xfs_trans_ail_cursor_clear(ailp, lip);
326}
327
328static long
329xfsaild_push(
330 struct xfs_ail *ailp)
331{
332 xfs_mount_t *mp = ailp->xa_mount;
333 struct xfs_ail_cursor cur;
334 xfs_log_item_t *lip;
335 xfs_lsn_t lsn;
336 xfs_lsn_t target;
337 long tout;
338 int stuck = 0;
339 int flushing = 0;
340 int count = 0;
341
342
343
344
345
346
347 if (ailp->xa_log_flush && ailp->xa_last_pushed_lsn == 0 &&
348 (!list_empty_careful(&ailp->xa_buf_list) ||
349 xfs_ail_min_lsn(ailp))) {
350 ailp->xa_log_flush = 0;
351
352 XFS_STATS_INC(xs_push_ail_flush);
353 xfs_log_force(mp, XFS_LOG_SYNC);
354 }
355
356 spin_lock(&ailp->xa_lock);
357
358
359 smp_rmb();
360 target = ailp->xa_target;
361 ailp->xa_target_prev = target;
362
363 lip = xfs_trans_ail_cursor_first(ailp, &cur, ailp->xa_last_pushed_lsn);
364 if (!lip) {
365
366
367
368
369 xfs_trans_ail_cursor_done(&cur);
370 spin_unlock(&ailp->xa_lock);
371 goto out_done;
372 }
373
374 XFS_STATS_INC(xs_push_ail);
375
376 lsn = lip->li_lsn;
377 while ((XFS_LSN_CMP(lip->li_lsn, target) <= 0)) {
378 int lock_result;
379
380
381
382
383
384
385 lock_result = lip->li_ops->iop_push(lip, &ailp->xa_buf_list);
386 switch (lock_result) {
387 case XFS_ITEM_SUCCESS:
388 XFS_STATS_INC(xs_push_ail_success);
389 trace_xfs_ail_push(lip);
390
391 ailp->xa_last_pushed_lsn = lsn;
392 break;
393
394 case XFS_ITEM_FLUSHING:
395
396
397
398
399
400
401
402
403
404
405
406 XFS_STATS_INC(xs_push_ail_flushing);
407 trace_xfs_ail_flushing(lip);
408
409 flushing++;
410 ailp->xa_last_pushed_lsn = lsn;
411 break;
412
413 case XFS_ITEM_PINNED:
414 XFS_STATS_INC(xs_push_ail_pinned);
415 trace_xfs_ail_pinned(lip);
416
417 stuck++;
418 ailp->xa_log_flush++;
419 break;
420 case XFS_ITEM_LOCKED:
421 XFS_STATS_INC(xs_push_ail_locked);
422 trace_xfs_ail_locked(lip);
423
424 stuck++;
425 break;
426 default:
427 ASSERT(0);
428 break;
429 }
430
431 count++;
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446 if (stuck > 100)
447 break;
448
449 lip = xfs_trans_ail_cursor_next(ailp, &cur);
450 if (lip == NULL)
451 break;
452 lsn = lip->li_lsn;
453 }
454 xfs_trans_ail_cursor_done(&cur);
455 spin_unlock(&ailp->xa_lock);
456
457 if (xfs_buf_delwri_submit_nowait(&ailp->xa_buf_list))
458 ailp->xa_log_flush++;
459
460 if (!count || XFS_LSN_CMP(lsn, target) >= 0) {
461out_done:
462
463
464
465
466
467 tout = 50;
468 ailp->xa_last_pushed_lsn = 0;
469 } else if (((stuck + flushing) * 100) / count > 90) {
470
471
472
473
474
475
476
477
478
479
480 tout = 20;
481 ailp->xa_last_pushed_lsn = 0;
482 } else {
483
484
485
486 tout = 10;
487 }
488
489 return tout;
490}
491
492static int
493xfsaild(
494 void *data)
495{
496 struct xfs_ail *ailp = data;
497 long tout = 0;
498
499 current->flags |= PF_MEMALLOC;
500
501 while (!kthread_should_stop()) {
502 if (tout && tout <= 20)
503 __set_current_state(TASK_KILLABLE);
504 else
505 __set_current_state(TASK_INTERRUPTIBLE);
506
507 spin_lock(&ailp->xa_lock);
508
509
510
511
512
513
514
515
516
517
518 smp_rmb();
519 if (!xfs_ail_min(ailp) &&
520 ailp->xa_target == ailp->xa_target_prev) {
521 spin_unlock(&ailp->xa_lock);
522 schedule();
523 tout = 0;
524 continue;
525 }
526 spin_unlock(&ailp->xa_lock);
527
528 if (tout)
529 schedule_timeout(msecs_to_jiffies(tout));
530
531 __set_current_state(TASK_RUNNING);
532
533 try_to_freeze();
534
535 tout = xfsaild_push(ailp);
536 }
537
538 return 0;
539}
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555void
556xfs_ail_push(
557 struct xfs_ail *ailp,
558 xfs_lsn_t threshold_lsn)
559{
560 xfs_log_item_t *lip;
561
562 lip = xfs_ail_min(ailp);
563 if (!lip || XFS_FORCED_SHUTDOWN(ailp->xa_mount) ||
564 XFS_LSN_CMP(threshold_lsn, ailp->xa_target) <= 0)
565 return;
566
567
568
569
570
571 smp_wmb();
572 xfs_trans_ail_copy_lsn(ailp, &ailp->xa_target, &threshold_lsn);
573 smp_wmb();
574
575 wake_up_process(ailp->xa_task);
576}
577
578
579
580
581void
582xfs_ail_push_all(
583 struct xfs_ail *ailp)
584{
585 xfs_lsn_t threshold_lsn = xfs_ail_max_lsn(ailp);
586
587 if (threshold_lsn)
588 xfs_ail_push(ailp, threshold_lsn);
589}
590
591
592
593
594void
595xfs_ail_push_all_sync(
596 struct xfs_ail *ailp)
597{
598 struct xfs_log_item *lip;
599 DEFINE_WAIT(wait);
600
601 spin_lock(&ailp->xa_lock);
602 while ((lip = xfs_ail_max(ailp)) != NULL) {
603 prepare_to_wait(&ailp->xa_empty, &wait, TASK_UNINTERRUPTIBLE);
604 ailp->xa_target = lip->li_lsn;
605 wake_up_process(ailp->xa_task);
606 spin_unlock(&ailp->xa_lock);
607 schedule();
608 spin_lock(&ailp->xa_lock);
609 }
610 spin_unlock(&ailp->xa_lock);
611
612 finish_wait(&ailp->xa_empty, &wait);
613}
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637void
638xfs_trans_ail_update_bulk(
639 struct xfs_ail *ailp,
640 struct xfs_ail_cursor *cur,
641 struct xfs_log_item **log_items,
642 int nr_items,
643 xfs_lsn_t lsn) __releases(ailp->xa_lock)
644{
645 xfs_log_item_t *mlip;
646 int mlip_changed = 0;
647 int i;
648 LIST_HEAD(tmp);
649
650 ASSERT(nr_items > 0);
651 mlip = xfs_ail_min(ailp);
652
653 for (i = 0; i < nr_items; i++) {
654 struct xfs_log_item *lip = log_items[i];
655 if (lip->li_flags & XFS_LI_IN_AIL) {
656
657 if (XFS_LSN_CMP(lsn, lip->li_lsn) <= 0)
658 continue;
659
660 trace_xfs_ail_move(lip, lip->li_lsn, lsn);
661 xfs_ail_delete(ailp, lip);
662 if (mlip == lip)
663 mlip_changed = 1;
664 } else {
665 lip->li_flags |= XFS_LI_IN_AIL;
666 trace_xfs_ail_insert(lip, 0, lsn);
667 }
668 lip->li_lsn = lsn;
669 list_add(&lip->li_ail, &tmp);
670 }
671
672 if (!list_empty(&tmp))
673 xfs_ail_splice(ailp, cur, &tmp, lsn);
674
675 if (mlip_changed) {
676 if (!XFS_FORCED_SHUTDOWN(ailp->xa_mount))
677 xlog_assign_tail_lsn_locked(ailp->xa_mount);
678 spin_unlock(&ailp->xa_lock);
679
680 xfs_log_space_wake(ailp->xa_mount);
681 } else {
682 spin_unlock(&ailp->xa_lock);
683 }
684}
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707void
708xfs_trans_ail_delete_bulk(
709 struct xfs_ail *ailp,
710 struct xfs_log_item **log_items,
711 int nr_items,
712 int shutdown_type) __releases(ailp->xa_lock)
713{
714 xfs_log_item_t *mlip;
715 int mlip_changed = 0;
716 int i;
717
718 mlip = xfs_ail_min(ailp);
719
720 for (i = 0; i < nr_items; i++) {
721 struct xfs_log_item *lip = log_items[i];
722 if (!(lip->li_flags & XFS_LI_IN_AIL)) {
723 struct xfs_mount *mp = ailp->xa_mount;
724
725 spin_unlock(&ailp->xa_lock);
726 if (!XFS_FORCED_SHUTDOWN(mp)) {
727 xfs_alert_tag(mp, XFS_PTAG_AILDELETE,
728 "%s: attempting to delete a log item that is not in the AIL",
729 __func__);
730 xfs_force_shutdown(mp, shutdown_type);
731 }
732 return;
733 }
734
735 trace_xfs_ail_delete(lip, mlip->li_lsn, lip->li_lsn);
736 xfs_ail_delete(ailp, lip);
737 lip->li_flags &= ~XFS_LI_IN_AIL;
738 lip->li_lsn = 0;
739 if (mlip == lip)
740 mlip_changed = 1;
741 }
742
743 if (mlip_changed) {
744 if (!XFS_FORCED_SHUTDOWN(ailp->xa_mount))
745 xlog_assign_tail_lsn_locked(ailp->xa_mount);
746 if (list_empty(&ailp->xa_ail))
747 wake_up_all(&ailp->xa_empty);
748 spin_unlock(&ailp->xa_lock);
749
750 xfs_log_space_wake(ailp->xa_mount);
751 } else {
752 spin_unlock(&ailp->xa_lock);
753 }
754}
755
756int
757xfs_trans_ail_init(
758 xfs_mount_t *mp)
759{
760 struct xfs_ail *ailp;
761
762 ailp = kmem_zalloc(sizeof(struct xfs_ail), KM_MAYFAIL);
763 if (!ailp)
764 return -ENOMEM;
765
766 ailp->xa_mount = mp;
767 INIT_LIST_HEAD(&ailp->xa_ail);
768 INIT_LIST_HEAD(&ailp->xa_cursors);
769 spin_lock_init(&ailp->xa_lock);
770 INIT_LIST_HEAD(&ailp->xa_buf_list);
771 init_waitqueue_head(&ailp->xa_empty);
772
773 ailp->xa_task = kthread_run(xfsaild, ailp, "xfsaild/%s",
774 ailp->xa_mount->m_fsname);
775 if (IS_ERR(ailp->xa_task))
776 goto out_free_ailp;
777
778 mp->m_ail = ailp;
779 return 0;
780
781out_free_ailp:
782 kmem_free(ailp);
783 return -ENOMEM;
784}
785
786void
787xfs_trans_ail_destroy(
788 xfs_mount_t *mp)
789{
790 struct xfs_ail *ailp = mp->m_ail;
791
792 kthread_stop(ailp->xa_task);
793 kmem_free(ailp);
794}
795