1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26#include <linux/module.h>
27
28#include <linux/slab.h>
29#include <linux/drbd.h>
30#include "drbd_int.h"
31#include "drbd_req.h"
32
33
34static bool drbd_may_do_local_read(struct drbd_device *device, sector_t sector, int size);
35
36static struct drbd_request *drbd_req_new(struct drbd_device *device, struct bio *bio_src)
37{
38 struct drbd_request *req;
39
40 req = mempool_alloc(&drbd_request_mempool, GFP_NOIO);
41 if (!req)
42 return NULL;
43 memset(req, 0, sizeof(*req));
44
45 drbd_req_make_private_bio(req, bio_src);
46 req->rq_state = (bio_data_dir(bio_src) == WRITE ? RQ_WRITE : 0)
47 | (bio_op(bio_src) == REQ_OP_WRITE_SAME ? RQ_WSAME : 0)
48 | (bio_op(bio_src) == REQ_OP_WRITE_ZEROES ? RQ_UNMAP : 0)
49 | (bio_op(bio_src) == REQ_OP_DISCARD ? RQ_UNMAP : 0);
50 req->device = device;
51 req->master_bio = bio_src;
52 req->epoch = 0;
53
54 drbd_clear_interval(&req->i);
55 req->i.sector = bio_src->bi_iter.bi_sector;
56 req->i.size = bio_src->bi_iter.bi_size;
57 req->i.local = true;
58 req->i.waiting = false;
59
60 INIT_LIST_HEAD(&req->tl_requests);
61 INIT_LIST_HEAD(&req->w.list);
62 INIT_LIST_HEAD(&req->req_pending_master_completion);
63 INIT_LIST_HEAD(&req->req_pending_local);
64
65
66 atomic_set(&req->completion_ref, 1);
67
68 kref_init(&req->kref);
69 return req;
70}
71
72static void drbd_remove_request_interval(struct rb_root *root,
73 struct drbd_request *req)
74{
75 struct drbd_device *device = req->device;
76 struct drbd_interval *i = &req->i;
77
78 drbd_remove_interval(root, i);
79
80
81 if (i->waiting)
82 wake_up(&device->misc_wait);
83}
84
85void drbd_req_destroy(struct kref *kref)
86{
87 struct drbd_request *req = container_of(kref, struct drbd_request, kref);
88 struct drbd_device *device = req->device;
89 const unsigned s = req->rq_state;
90
91 if ((req->master_bio && !(s & RQ_POSTPONED)) ||
92 atomic_read(&req->completion_ref) ||
93 (s & RQ_LOCAL_PENDING) ||
94 ((s & RQ_NET_MASK) && !(s & RQ_NET_DONE))) {
95 drbd_err(device, "drbd_req_destroy: Logic BUG rq_state = 0x%x, completion_ref = %d\n",
96 s, atomic_read(&req->completion_ref));
97 return;
98 }
99
100
101
102
103
104
105
106
107
108 list_del_init(&req->tl_requests);
109
110
111
112 if (!drbd_interval_empty(&req->i)) {
113 struct rb_root *root;
114
115 if (s & RQ_WRITE)
116 root = &device->write_requests;
117 else
118 root = &device->read_requests;
119 drbd_remove_request_interval(root, req);
120 } else if (s & (RQ_NET_MASK & ~RQ_NET_DONE) && req->i.size != 0)
121 drbd_err(device, "drbd_req_destroy: Logic BUG: interval empty, but: rq_state=0x%x, sect=%llu, size=%u\n",
122 s, (unsigned long long)req->i.sector, req->i.size);
123
124
125
126
127 if (s & RQ_WRITE) {
128
129
130
131
132
133
134
135
136
137
138
139 if ((s & (RQ_POSTPONED|RQ_LOCAL_MASK|RQ_NET_MASK)) != RQ_POSTPONED) {
140 if (!(s & RQ_NET_OK) || !(s & RQ_LOCAL_OK))
141 drbd_set_out_of_sync(device, req->i.sector, req->i.size);
142
143 if ((s & RQ_NET_OK) && (s & RQ_LOCAL_OK) && (s & RQ_NET_SIS))
144 drbd_set_in_sync(device, req->i.sector, req->i.size);
145 }
146
147
148
149
150
151
152
153
154
155
156
157 if (s & RQ_IN_ACT_LOG) {
158 if (get_ldev_if_state(device, D_FAILED)) {
159 drbd_al_complete_io(device, &req->i);
160 put_ldev(device);
161 } else if (__ratelimit(&drbd_ratelimit_state)) {
162 drbd_warn(device, "Should have called drbd_al_complete_io(, %llu, %u), "
163 "but my Disk seems to have failed :(\n",
164 (unsigned long long) req->i.sector, req->i.size);
165 }
166 }
167 }
168
169 mempool_free(req, &drbd_request_mempool);
170}
171
172static void wake_all_senders(struct drbd_connection *connection)
173{
174 wake_up(&connection->sender_work.q_wait);
175}
176
177
178void start_new_tl_epoch(struct drbd_connection *connection)
179{
180
181 if (connection->current_tle_writes == 0)
182 return;
183
184 connection->current_tle_writes = 0;
185 atomic_inc(&connection->current_tle_nr);
186 wake_all_senders(connection);
187}
188
189void complete_master_bio(struct drbd_device *device,
190 struct bio_and_error *m)
191{
192 m->bio->bi_status = errno_to_blk_status(m->error);
193 bio_endio(m->bio);
194 dec_ap_bio(device);
195}
196
197
198
199
200
201
202
203
204static
205void drbd_req_complete(struct drbd_request *req, struct bio_and_error *m)
206{
207 const unsigned s = req->rq_state;
208 struct drbd_device *device = req->device;
209 int error, ok;
210
211
212
213
214
215
216
217
218
219
220 if ((s & RQ_LOCAL_PENDING && !(s & RQ_LOCAL_ABORTED)) ||
221 (s & RQ_NET_QUEUED) || (s & RQ_NET_PENDING) ||
222 (s & RQ_COMPLETION_SUSP)) {
223 drbd_err(device, "drbd_req_complete: Logic BUG rq_state = 0x%x\n", s);
224 return;
225 }
226
227 if (!req->master_bio) {
228 drbd_err(device, "drbd_req_complete: Logic BUG, master_bio == NULL!\n");
229 return;
230 }
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245 ok = (s & RQ_LOCAL_OK) || (s & RQ_NET_OK);
246 error = PTR_ERR(req->private_bio);
247
248
249
250
251
252
253
254
255 if (op_is_write(bio_op(req->master_bio)) &&
256 req->epoch == atomic_read(&first_peer_device(device)->connection->current_tle_nr))
257 start_new_tl_epoch(first_peer_device(device)->connection);
258
259
260 bio_end_io_acct(req->master_bio, req->start_jif);
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276 if (!ok &&
277 bio_op(req->master_bio) == REQ_OP_READ &&
278 !(req->master_bio->bi_opf & REQ_RAHEAD) &&
279 !list_empty(&req->tl_requests))
280 req->rq_state |= RQ_POSTPONED;
281
282 if (!(req->rq_state & RQ_POSTPONED)) {
283 m->error = ok ? 0 : (error ?: -EIO);
284 m->bio = req->master_bio;
285 req->master_bio = NULL;
286
287
288
289
290 req->i.completed = true;
291 }
292
293 if (req->i.waiting)
294 wake_up(&device->misc_wait);
295
296
297
298
299
300 list_del_init(&req->req_pending_master_completion);
301}
302
303
304static void drbd_req_put_completion_ref(struct drbd_request *req, struct bio_and_error *m, int put)
305{
306 struct drbd_device *device = req->device;
307 D_ASSERT(device, m || (req->rq_state & RQ_POSTPONED));
308
309 if (!put)
310 return;
311
312 if (!atomic_sub_and_test(put, &req->completion_ref))
313 return;
314
315 drbd_req_complete(req, m);
316
317
318
319 if (req->rq_state & RQ_LOCAL_ABORTED)
320 return;
321
322 if (req->rq_state & RQ_POSTPONED) {
323
324
325 drbd_restart_request(req);
326 return;
327 }
328
329 kref_put(&req->kref, drbd_req_destroy);
330}
331
332static void set_if_null_req_next(struct drbd_peer_device *peer_device, struct drbd_request *req)
333{
334 struct drbd_connection *connection = peer_device ? peer_device->connection : NULL;
335 if (!connection)
336 return;
337 if (connection->req_next == NULL)
338 connection->req_next = req;
339}
340
341static void advance_conn_req_next(struct drbd_peer_device *peer_device, struct drbd_request *req)
342{
343 struct drbd_connection *connection = peer_device ? peer_device->connection : NULL;
344 if (!connection)
345 return;
346 if (connection->req_next != req)
347 return;
348 list_for_each_entry_continue(req, &connection->transfer_log, tl_requests) {
349 const unsigned s = req->rq_state;
350 if (s & RQ_NET_QUEUED)
351 break;
352 }
353 if (&req->tl_requests == &connection->transfer_log)
354 req = NULL;
355 connection->req_next = req;
356}
357
358static void set_if_null_req_ack_pending(struct drbd_peer_device *peer_device, struct drbd_request *req)
359{
360 struct drbd_connection *connection = peer_device ? peer_device->connection : NULL;
361 if (!connection)
362 return;
363 if (connection->req_ack_pending == NULL)
364 connection->req_ack_pending = req;
365}
366
367static void advance_conn_req_ack_pending(struct drbd_peer_device *peer_device, struct drbd_request *req)
368{
369 struct drbd_connection *connection = peer_device ? peer_device->connection : NULL;
370 if (!connection)
371 return;
372 if (connection->req_ack_pending != req)
373 return;
374 list_for_each_entry_continue(req, &connection->transfer_log, tl_requests) {
375 const unsigned s = req->rq_state;
376 if ((s & RQ_NET_SENT) && (s & RQ_NET_PENDING))
377 break;
378 }
379 if (&req->tl_requests == &connection->transfer_log)
380 req = NULL;
381 connection->req_ack_pending = req;
382}
383
384static void set_if_null_req_not_net_done(struct drbd_peer_device *peer_device, struct drbd_request *req)
385{
386 struct drbd_connection *connection = peer_device ? peer_device->connection : NULL;
387 if (!connection)
388 return;
389 if (connection->req_not_net_done == NULL)
390 connection->req_not_net_done = req;
391}
392
393static void advance_conn_req_not_net_done(struct drbd_peer_device *peer_device, struct drbd_request *req)
394{
395 struct drbd_connection *connection = peer_device ? peer_device->connection : NULL;
396 if (!connection)
397 return;
398 if (connection->req_not_net_done != req)
399 return;
400 list_for_each_entry_continue(req, &connection->transfer_log, tl_requests) {
401 const unsigned s = req->rq_state;
402 if ((s & RQ_NET_SENT) && !(s & RQ_NET_DONE))
403 break;
404 }
405 if (&req->tl_requests == &connection->transfer_log)
406 req = NULL;
407 connection->req_not_net_done = req;
408}
409
410
411
412static void mod_rq_state(struct drbd_request *req, struct bio_and_error *m,
413 int clear, int set)
414{
415 struct drbd_device *device = req->device;
416 struct drbd_peer_device *peer_device = first_peer_device(device);
417 unsigned s = req->rq_state;
418 int c_put = 0;
419
420 if (drbd_suspended(device) && !((s | clear) & RQ_COMPLETION_SUSP))
421 set |= RQ_COMPLETION_SUSP;
422
423
424
425 req->rq_state &= ~clear;
426 req->rq_state |= set;
427
428
429 if (req->rq_state == s)
430 return;
431
432
433
434 kref_get(&req->kref);
435
436 if (!(s & RQ_LOCAL_PENDING) && (set & RQ_LOCAL_PENDING))
437 atomic_inc(&req->completion_ref);
438
439 if (!(s & RQ_NET_PENDING) && (set & RQ_NET_PENDING)) {
440 inc_ap_pending(device);
441 atomic_inc(&req->completion_ref);
442 }
443
444 if (!(s & RQ_NET_QUEUED) && (set & RQ_NET_QUEUED)) {
445 atomic_inc(&req->completion_ref);
446 set_if_null_req_next(peer_device, req);
447 }
448
449 if (!(s & RQ_EXP_BARR_ACK) && (set & RQ_EXP_BARR_ACK))
450 kref_get(&req->kref);
451
452 if (!(s & RQ_NET_SENT) && (set & RQ_NET_SENT)) {
453
454 if (!(s & RQ_NET_DONE)) {
455 atomic_add(req->i.size >> 9, &device->ap_in_flight);
456 set_if_null_req_not_net_done(peer_device, req);
457 }
458 if (req->rq_state & RQ_NET_PENDING)
459 set_if_null_req_ack_pending(peer_device, req);
460 }
461
462 if (!(s & RQ_COMPLETION_SUSP) && (set & RQ_COMPLETION_SUSP))
463 atomic_inc(&req->completion_ref);
464
465
466
467 if ((s & RQ_COMPLETION_SUSP) && (clear & RQ_COMPLETION_SUSP))
468 ++c_put;
469
470 if (!(s & RQ_LOCAL_ABORTED) && (set & RQ_LOCAL_ABORTED)) {
471 D_ASSERT(device, req->rq_state & RQ_LOCAL_PENDING);
472 ++c_put;
473 }
474
475 if ((s & RQ_LOCAL_PENDING) && (clear & RQ_LOCAL_PENDING)) {
476 if (req->rq_state & RQ_LOCAL_ABORTED)
477 kref_put(&req->kref, drbd_req_destroy);
478 else
479 ++c_put;
480 list_del_init(&req->req_pending_local);
481 }
482
483 if ((s & RQ_NET_PENDING) && (clear & RQ_NET_PENDING)) {
484 dec_ap_pending(device);
485 ++c_put;
486 req->acked_jif = jiffies;
487 advance_conn_req_ack_pending(peer_device, req);
488 }
489
490 if ((s & RQ_NET_QUEUED) && (clear & RQ_NET_QUEUED)) {
491 ++c_put;
492 advance_conn_req_next(peer_device, req);
493 }
494
495 if (!(s & RQ_NET_DONE) && (set & RQ_NET_DONE)) {
496 if (s & RQ_NET_SENT)
497 atomic_sub(req->i.size >> 9, &device->ap_in_flight);
498 if (s & RQ_EXP_BARR_ACK)
499 kref_put(&req->kref, drbd_req_destroy);
500 req->net_done_jif = jiffies;
501
502
503
504
505 advance_conn_req_next(peer_device, req);
506 advance_conn_req_ack_pending(peer_device, req);
507 advance_conn_req_not_net_done(peer_device, req);
508 }
509
510
511
512
513 if (req->i.waiting)
514 wake_up(&device->misc_wait);
515
516 drbd_req_put_completion_ref(req, m, c_put);
517 kref_put(&req->kref, drbd_req_destroy);
518}
519
520static void drbd_report_io_error(struct drbd_device *device, struct drbd_request *req)
521{
522 char b[BDEVNAME_SIZE];
523
524 if (!__ratelimit(&drbd_ratelimit_state))
525 return;
526
527 drbd_warn(device, "local %s IO error sector %llu+%u on %s\n",
528 (req->rq_state & RQ_WRITE) ? "WRITE" : "READ",
529 (unsigned long long)req->i.sector,
530 req->i.size >> 9,
531 bdevname(device->ldev->backing_bdev, b));
532}
533
534
535
536
537
538
539
540static inline bool is_pending_write_protocol_A(struct drbd_request *req)
541{
542 return (req->rq_state &
543 (RQ_WRITE|RQ_NET_PENDING|RQ_EXP_WRITE_ACK|RQ_EXP_RECEIVE_ACK))
544 == (RQ_WRITE|RQ_NET_PENDING);
545}
546
547
548
549
550
551
552
553
554
555
556
557
558
559int __req_mod(struct drbd_request *req, enum drbd_req_event what,
560 struct bio_and_error *m)
561{
562 struct drbd_device *const device = req->device;
563 struct drbd_peer_device *const peer_device = first_peer_device(device);
564 struct drbd_connection *const connection = peer_device ? peer_device->connection : NULL;
565 struct net_conf *nc;
566 int p, rv = 0;
567
568 if (m)
569 m->bio = NULL;
570
571 switch (what) {
572 default:
573 drbd_err(device, "LOGIC BUG in %s:%u\n", __FILE__ , __LINE__);
574 break;
575
576
577
578
579
580
581
582 case TO_BE_SENT:
583
584
585 D_ASSERT(device, !(req->rq_state & RQ_NET_MASK));
586 rcu_read_lock();
587 nc = rcu_dereference(connection->net_conf);
588 p = nc->wire_protocol;
589 rcu_read_unlock();
590 req->rq_state |=
591 p == DRBD_PROT_C ? RQ_EXP_WRITE_ACK :
592 p == DRBD_PROT_B ? RQ_EXP_RECEIVE_ACK : 0;
593 mod_rq_state(req, m, 0, RQ_NET_PENDING);
594 break;
595
596 case TO_BE_SUBMITTED:
597
598 D_ASSERT(device, !(req->rq_state & RQ_LOCAL_MASK));
599 mod_rq_state(req, m, 0, RQ_LOCAL_PENDING);
600 break;
601
602 case COMPLETED_OK:
603 if (req->rq_state & RQ_WRITE)
604 device->writ_cnt += req->i.size >> 9;
605 else
606 device->read_cnt += req->i.size >> 9;
607
608 mod_rq_state(req, m, RQ_LOCAL_PENDING,
609 RQ_LOCAL_COMPLETED|RQ_LOCAL_OK);
610 break;
611
612 case ABORT_DISK_IO:
613 mod_rq_state(req, m, 0, RQ_LOCAL_ABORTED);
614 break;
615
616 case WRITE_COMPLETED_WITH_ERROR:
617 drbd_report_io_error(device, req);
618 __drbd_chk_io_error(device, DRBD_WRITE_ERROR);
619 mod_rq_state(req, m, RQ_LOCAL_PENDING, RQ_LOCAL_COMPLETED);
620 break;
621
622 case READ_COMPLETED_WITH_ERROR:
623 drbd_set_out_of_sync(device, req->i.sector, req->i.size);
624 drbd_report_io_error(device, req);
625 __drbd_chk_io_error(device, DRBD_READ_ERROR);
626
627 case READ_AHEAD_COMPLETED_WITH_ERROR:
628
629 mod_rq_state(req, m, RQ_LOCAL_PENDING, RQ_LOCAL_COMPLETED);
630 break;
631
632 case DISCARD_COMPLETED_NOTSUPP:
633 case DISCARD_COMPLETED_WITH_ERROR:
634
635
636 mod_rq_state(req, m, RQ_LOCAL_PENDING, RQ_LOCAL_COMPLETED);
637 break;
638
639 case QUEUE_FOR_NET_READ:
640
641
642
643
644
645
646
647
648
649
650 D_ASSERT(device, drbd_interval_empty(&req->i));
651 drbd_insert_interval(&device->read_requests, &req->i);
652
653 set_bit(UNPLUG_REMOTE, &device->flags);
654
655 D_ASSERT(device, req->rq_state & RQ_NET_PENDING);
656 D_ASSERT(device, (req->rq_state & RQ_LOCAL_MASK) == 0);
657 mod_rq_state(req, m, 0, RQ_NET_QUEUED);
658 req->w.cb = w_send_read_req;
659 drbd_queue_work(&connection->sender_work,
660 &req->w);
661 break;
662
663 case QUEUE_FOR_NET_WRITE:
664
665
666
667
668
669 D_ASSERT(device, drbd_interval_empty(&req->i));
670 drbd_insert_interval(&device->write_requests, &req->i);
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689 set_bit(UNPLUG_REMOTE, &device->flags);
690
691
692 D_ASSERT(device, req->rq_state & RQ_NET_PENDING);
693 mod_rq_state(req, m, 0, RQ_NET_QUEUED|RQ_EXP_BARR_ACK);
694 req->w.cb = w_send_dblock;
695 drbd_queue_work(&connection->sender_work,
696 &req->w);
697
698
699 rcu_read_lock();
700 nc = rcu_dereference(connection->net_conf);
701 p = nc->max_epoch_size;
702 rcu_read_unlock();
703 if (connection->current_tle_writes >= p)
704 start_new_tl_epoch(connection);
705
706 break;
707
708 case QUEUE_FOR_SEND_OOS:
709 mod_rq_state(req, m, 0, RQ_NET_QUEUED);
710 req->w.cb = w_send_out_of_sync;
711 drbd_queue_work(&connection->sender_work,
712 &req->w);
713 break;
714
715 case READ_RETRY_REMOTE_CANCELED:
716 case SEND_CANCELED:
717 case SEND_FAILED:
718
719
720 mod_rq_state(req, m, RQ_NET_QUEUED, 0);
721 break;
722
723 case HANDED_OVER_TO_NETWORK:
724
725 if (is_pending_write_protocol_A(req))
726
727
728 mod_rq_state(req, m, RQ_NET_QUEUED|RQ_NET_PENDING,
729 RQ_NET_SENT|RQ_NET_OK);
730 else
731 mod_rq_state(req, m, RQ_NET_QUEUED, RQ_NET_SENT);
732
733
734
735 break;
736
737 case OOS_HANDED_TO_NETWORK:
738
739
740 mod_rq_state(req, m, RQ_NET_QUEUED, RQ_NET_DONE);
741 break;
742
743 case CONNECTION_LOST_WHILE_PENDING:
744
745 mod_rq_state(req, m,
746 RQ_NET_OK|RQ_NET_PENDING|RQ_COMPLETION_SUSP,
747 RQ_NET_DONE);
748 break;
749
750 case CONFLICT_RESOLVED:
751
752
753
754
755
756
757
758 D_ASSERT(device, req->rq_state & RQ_NET_PENDING);
759 D_ASSERT(device, req->rq_state & RQ_EXP_WRITE_ACK);
760 mod_rq_state(req, m, RQ_NET_PENDING, RQ_NET_DONE|RQ_NET_OK);
761 break;
762
763 case WRITE_ACKED_BY_PEER_AND_SIS:
764 req->rq_state |= RQ_NET_SIS;
765 case WRITE_ACKED_BY_PEER:
766
767
768
769
770
771
772
773 goto ack_common;
774 case RECV_ACKED_BY_PEER:
775 D_ASSERT(device, req->rq_state & RQ_EXP_RECEIVE_ACK);
776
777
778
779 ack_common:
780 mod_rq_state(req, m, RQ_NET_PENDING, RQ_NET_OK);
781 break;
782
783 case POSTPONE_WRITE:
784 D_ASSERT(device, req->rq_state & RQ_EXP_WRITE_ACK);
785
786
787
788
789 D_ASSERT(device, req->rq_state & RQ_NET_PENDING);
790 req->rq_state |= RQ_POSTPONED;
791 if (req->i.waiting)
792 wake_up(&device->misc_wait);
793
794
795
796 break;
797
798 case NEG_ACKED:
799 mod_rq_state(req, m, RQ_NET_OK|RQ_NET_PENDING, 0);
800 break;
801
802 case FAIL_FROZEN_DISK_IO:
803 if (!(req->rq_state & RQ_LOCAL_COMPLETED))
804 break;
805 mod_rq_state(req, m, RQ_COMPLETION_SUSP, 0);
806 break;
807
808 case RESTART_FROZEN_DISK_IO:
809 if (!(req->rq_state & RQ_LOCAL_COMPLETED))
810 break;
811
812 mod_rq_state(req, m,
813 RQ_COMPLETION_SUSP|RQ_LOCAL_COMPLETED,
814 RQ_LOCAL_PENDING);
815
816 rv = MR_READ;
817 if (bio_data_dir(req->master_bio) == WRITE)
818 rv = MR_WRITE;
819
820 get_ldev(device);
821 req->w.cb = w_restart_disk_io;
822 drbd_queue_work(&connection->sender_work,
823 &req->w);
824 break;
825
826 case RESEND:
827
828 if (!(req->rq_state & RQ_WRITE) && !req->w.cb) {
829 mod_rq_state(req, m, RQ_COMPLETION_SUSP, 0);
830 break;
831 }
832
833
834
835
836
837
838 if (!(req->rq_state & RQ_NET_OK)) {
839
840
841
842 mod_rq_state(req, m, RQ_COMPLETION_SUSP, RQ_NET_QUEUED|RQ_NET_PENDING);
843 if (req->w.cb) {
844
845 drbd_queue_work(&connection->sender_work,
846 &req->w);
847 rv = req->rq_state & RQ_WRITE ? MR_WRITE : MR_READ;
848 }
849 break;
850 }
851
852
853 case BARRIER_ACKED:
854
855 if (!(req->rq_state & RQ_WRITE))
856 break;
857
858 if (req->rq_state & RQ_NET_PENDING) {
859
860
861
862 drbd_err(device, "FIXME (BARRIER_ACKED but pending)\n");
863 }
864
865
866
867
868 mod_rq_state(req, m, RQ_COMPLETION_SUSP,
869 (req->rq_state & RQ_NET_MASK) ? RQ_NET_DONE : 0);
870 break;
871
872 case DATA_RECEIVED:
873 D_ASSERT(device, req->rq_state & RQ_NET_PENDING);
874 mod_rq_state(req, m, RQ_NET_PENDING, RQ_NET_OK|RQ_NET_DONE);
875 break;
876
877 case QUEUE_AS_DRBD_BARRIER:
878 start_new_tl_epoch(connection);
879 mod_rq_state(req, m, 0, RQ_NET_OK|RQ_NET_DONE);
880 break;
881 }
882
883 return rv;
884}
885
886
887
888
889
890
891
892
893static bool drbd_may_do_local_read(struct drbd_device *device, sector_t sector, int size)
894{
895 unsigned long sbnr, ebnr;
896 sector_t esector, nr_sectors;
897
898 if (device->state.disk == D_UP_TO_DATE)
899 return true;
900 if (device->state.disk != D_INCONSISTENT)
901 return false;
902 esector = sector + (size >> 9) - 1;
903 nr_sectors = drbd_get_capacity(device->this_bdev);
904 D_ASSERT(device, sector < nr_sectors);
905 D_ASSERT(device, esector < nr_sectors);
906
907 sbnr = BM_SECT_TO_BIT(sector);
908 ebnr = BM_SECT_TO_BIT(esector);
909
910 return drbd_bm_count_bits(device, sbnr, ebnr) == 0;
911}
912
913static bool remote_due_to_read_balancing(struct drbd_device *device, sector_t sector,
914 enum drbd_read_balancing rbm)
915{
916 struct backing_dev_info *bdi;
917 int stripe_shift;
918
919 switch (rbm) {
920 case RB_CONGESTED_REMOTE:
921 bdi = device->ldev->backing_bdev->bd_disk->queue->backing_dev_info;
922 return bdi_read_congested(bdi);
923 case RB_LEAST_PENDING:
924 return atomic_read(&device->local_cnt) >
925 atomic_read(&device->ap_pending_cnt) + atomic_read(&device->rs_pending_cnt);
926 case RB_32K_STRIPING:
927 case RB_64K_STRIPING:
928 case RB_128K_STRIPING:
929 case RB_256K_STRIPING:
930 case RB_512K_STRIPING:
931 case RB_1M_STRIPING:
932 stripe_shift = (rbm - RB_32K_STRIPING + 15);
933 return (sector >> (stripe_shift - 9)) & 1;
934 case RB_ROUND_ROBIN:
935 return test_and_change_bit(READ_BALANCE_RR, &device->flags);
936 case RB_PREFER_REMOTE:
937 return true;
938 case RB_PREFER_LOCAL:
939 default:
940 return false;
941 }
942}
943
944
945
946
947
948
949
950
951
952
953static void complete_conflicting_writes(struct drbd_request *req)
954{
955 DEFINE_WAIT(wait);
956 struct drbd_device *device = req->device;
957 struct drbd_interval *i;
958 sector_t sector = req->i.sector;
959 int size = req->i.size;
960
961 for (;;) {
962 drbd_for_each_overlap(i, &device->write_requests, sector, size) {
963
964 if (i->completed)
965 continue;
966
967
968 break;
969 }
970 if (!i)
971 break;
972
973
974 prepare_to_wait(&device->misc_wait, &wait, TASK_UNINTERRUPTIBLE);
975 i->waiting = true;
976 spin_unlock_irq(&device->resource->req_lock);
977 schedule();
978 spin_lock_irq(&device->resource->req_lock);
979 }
980 finish_wait(&device->misc_wait, &wait);
981}
982
983
984static void maybe_pull_ahead(struct drbd_device *device)
985{
986 struct drbd_connection *connection = first_peer_device(device)->connection;
987 struct net_conf *nc;
988 bool congested = false;
989 enum drbd_on_congestion on_congestion;
990
991 rcu_read_lock();
992 nc = rcu_dereference(connection->net_conf);
993 on_congestion = nc ? nc->on_congestion : OC_BLOCK;
994 rcu_read_unlock();
995 if (on_congestion == OC_BLOCK ||
996 connection->agreed_pro_version < 96)
997 return;
998
999 if (on_congestion == OC_PULL_AHEAD && device->state.conn == C_AHEAD)
1000 return;
1001
1002
1003
1004
1005
1006 if (!get_ldev_if_state(device, D_UP_TO_DATE))
1007 return;
1008
1009 if (nc->cong_fill &&
1010 atomic_read(&device->ap_in_flight) >= nc->cong_fill) {
1011 drbd_info(device, "Congestion-fill threshold reached\n");
1012 congested = true;
1013 }
1014
1015 if (device->act_log->used >= nc->cong_extents) {
1016 drbd_info(device, "Congestion-extents threshold reached\n");
1017 congested = true;
1018 }
1019
1020 if (congested) {
1021
1022 start_new_tl_epoch(first_peer_device(device)->connection);
1023
1024 if (on_congestion == OC_PULL_AHEAD)
1025 _drbd_set_state(_NS(device, conn, C_AHEAD), 0, NULL);
1026 else
1027 _drbd_set_state(_NS(device, conn, C_DISCONNECTING), 0, NULL);
1028 }
1029 put_ldev(device);
1030}
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041static bool do_remote_read(struct drbd_request *req)
1042{
1043 struct drbd_device *device = req->device;
1044 enum drbd_read_balancing rbm;
1045
1046 if (req->private_bio) {
1047 if (!drbd_may_do_local_read(device,
1048 req->i.sector, req->i.size)) {
1049 bio_put(req->private_bio);
1050 req->private_bio = NULL;
1051 put_ldev(device);
1052 }
1053 }
1054
1055 if (device->state.pdsk != D_UP_TO_DATE)
1056 return false;
1057
1058 if (req->private_bio == NULL)
1059 return true;
1060
1061
1062
1063
1064 rcu_read_lock();
1065 rbm = rcu_dereference(device->ldev->disk_conf)->read_balancing;
1066 rcu_read_unlock();
1067
1068 if (rbm == RB_PREFER_LOCAL && req->private_bio)
1069 return false;
1070
1071 if (remote_due_to_read_balancing(device, req->i.sector, rbm)) {
1072 if (req->private_bio) {
1073 bio_put(req->private_bio);
1074 req->private_bio = NULL;
1075 put_ldev(device);
1076 }
1077 return true;
1078 }
1079
1080 return false;
1081}
1082
1083bool drbd_should_do_remote(union drbd_dev_state s)
1084{
1085 return s.pdsk == D_UP_TO_DATE ||
1086 (s.pdsk >= D_INCONSISTENT &&
1087 s.conn >= C_WF_BITMAP_T &&
1088 s.conn < C_AHEAD);
1089
1090
1091
1092}
1093
1094static bool drbd_should_send_out_of_sync(union drbd_dev_state s)
1095{
1096 return s.conn == C_AHEAD || s.conn == C_WF_BITMAP_S;
1097
1098
1099}
1100
1101
1102
1103
1104static int drbd_process_write_request(struct drbd_request *req)
1105{
1106 struct drbd_device *device = req->device;
1107 int remote, send_oos;
1108
1109 remote = drbd_should_do_remote(device->state);
1110 send_oos = drbd_should_send_out_of_sync(device->state);
1111
1112
1113
1114
1115
1116
1117
1118 if (unlikely(req->i.size == 0)) {
1119
1120 D_ASSERT(device, req->master_bio->bi_opf & REQ_PREFLUSH);
1121 if (remote)
1122 _req_mod(req, QUEUE_AS_DRBD_BARRIER);
1123 return remote;
1124 }
1125
1126 if (!remote && !send_oos)
1127 return 0;
1128
1129 D_ASSERT(device, !(remote && send_oos));
1130
1131 if (remote) {
1132 _req_mod(req, TO_BE_SENT);
1133 _req_mod(req, QUEUE_FOR_NET_WRITE);
1134 } else if (drbd_set_out_of_sync(device, req->i.sector, req->i.size))
1135 _req_mod(req, QUEUE_FOR_SEND_OOS);
1136
1137 return remote;
1138}
1139
1140static void drbd_process_discard_req(struct drbd_request *req)
1141{
1142 struct block_device *bdev = req->device->ldev->backing_bdev;
1143
1144 if (blkdev_issue_zeroout(bdev, req->i.sector, req->i.size >> 9,
1145 GFP_NOIO, 0))
1146 req->private_bio->bi_status = BLK_STS_IOERR;
1147 bio_endio(req->private_bio);
1148}
1149
1150static void
1151drbd_submit_req_private_bio(struct drbd_request *req)
1152{
1153 struct drbd_device *device = req->device;
1154 struct bio *bio = req->private_bio;
1155 unsigned int type;
1156
1157 if (bio_op(bio) != REQ_OP_READ)
1158 type = DRBD_FAULT_DT_WR;
1159 else if (bio->bi_opf & REQ_RAHEAD)
1160 type = DRBD_FAULT_DT_RA;
1161 else
1162 type = DRBD_FAULT_DT_RD;
1163
1164 bio_set_dev(bio, device->ldev->backing_bdev);
1165
1166
1167
1168
1169
1170
1171 if (get_ldev(device)) {
1172 if (drbd_insert_fault(device, type))
1173 bio_io_error(bio);
1174 else if (bio_op(bio) == REQ_OP_WRITE_ZEROES ||
1175 bio_op(bio) == REQ_OP_DISCARD)
1176 drbd_process_discard_req(req);
1177 else
1178 generic_make_request(bio);
1179 put_ldev(device);
1180 } else
1181 bio_io_error(bio);
1182}
1183
1184static void drbd_queue_write(struct drbd_device *device, struct drbd_request *req)
1185{
1186 spin_lock_irq(&device->resource->req_lock);
1187 list_add_tail(&req->tl_requests, &device->submit.writes);
1188 list_add_tail(&req->req_pending_master_completion,
1189 &device->pending_master_completion[1 ]);
1190 spin_unlock_irq(&device->resource->req_lock);
1191 queue_work(device->submit.wq, &device->submit.worker);
1192
1193 wake_up(&device->al_wait);
1194}
1195
1196
1197
1198
1199
1200
1201static struct drbd_request *
1202drbd_request_prepare(struct drbd_device *device, struct bio *bio, unsigned long start_jif)
1203{
1204 const int rw = bio_data_dir(bio);
1205 struct drbd_request *req;
1206
1207
1208 req = drbd_req_new(device, bio);
1209 if (!req) {
1210 dec_ap_bio(device);
1211
1212
1213 drbd_err(device, "could not kmalloc() req\n");
1214 bio->bi_status = BLK_STS_RESOURCE;
1215 bio_endio(bio);
1216 return ERR_PTR(-ENOMEM);
1217 }
1218
1219
1220 req->start_jif = bio_start_io_acct(req->master_bio);
1221
1222 if (!get_ldev(device)) {
1223 bio_put(req->private_bio);
1224 req->private_bio = NULL;
1225 }
1226
1227
1228 if (bio_op(bio) == REQ_OP_WRITE_ZEROES ||
1229 bio_op(bio) == REQ_OP_DISCARD)
1230 goto queue_for_submitter_thread;
1231
1232 if (rw == WRITE && req->private_bio && req->i.size
1233 && !test_bit(AL_SUSPENDED, &device->flags)) {
1234 if (!drbd_al_begin_io_fastpath(device, &req->i))
1235 goto queue_for_submitter_thread;
1236 req->rq_state |= RQ_IN_ACT_LOG;
1237 req->in_actlog_jif = jiffies;
1238 }
1239 return req;
1240
1241 queue_for_submitter_thread:
1242 atomic_inc(&device->ap_actlog_cnt);
1243 drbd_queue_write(device, req);
1244 return NULL;
1245}
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257static bool may_do_writes(struct drbd_device *device)
1258{
1259 const union drbd_dev_state s = device->state;
1260 return s.disk == D_UP_TO_DATE || s.pdsk == D_UP_TO_DATE;
1261}
1262
1263struct drbd_plug_cb {
1264 struct blk_plug_cb cb;
1265 struct drbd_request *most_recent_req;
1266
1267};
1268
1269static void drbd_unplug(struct blk_plug_cb *cb, bool from_schedule)
1270{
1271 struct drbd_plug_cb *plug = container_of(cb, struct drbd_plug_cb, cb);
1272 struct drbd_resource *resource = plug->cb.data;
1273 struct drbd_request *req = plug->most_recent_req;
1274
1275 kfree(cb);
1276 if (!req)
1277 return;
1278
1279 spin_lock_irq(&resource->req_lock);
1280
1281
1282 req->rq_state |= RQ_UNPLUG;
1283
1284 drbd_queue_unplug(req->device);
1285 kref_put(&req->kref, drbd_req_destroy);
1286 spin_unlock_irq(&resource->req_lock);
1287}
1288
1289static struct drbd_plug_cb* drbd_check_plugged(struct drbd_resource *resource)
1290{
1291
1292
1293 struct drbd_plug_cb *plug;
1294 struct blk_plug_cb *cb = blk_check_plugged(drbd_unplug, resource, sizeof(*plug));
1295
1296 if (cb)
1297 plug = container_of(cb, struct drbd_plug_cb, cb);
1298 else
1299 plug = NULL;
1300 return plug;
1301}
1302
1303static void drbd_update_plug(struct drbd_plug_cb *plug, struct drbd_request *req)
1304{
1305 struct drbd_request *tmp = plug->most_recent_req;
1306
1307
1308 kref_get(&req->kref);
1309 plug->most_recent_req = req;
1310 if (tmp)
1311 kref_put(&tmp->kref, drbd_req_destroy);
1312}
1313
1314static void drbd_send_and_submit(struct drbd_device *device, struct drbd_request *req)
1315{
1316 struct drbd_resource *resource = device->resource;
1317 const int rw = bio_data_dir(req->master_bio);
1318 struct bio_and_error m = { NULL, };
1319 bool no_remote = false;
1320 bool submit_private_bio = false;
1321
1322 spin_lock_irq(&resource->req_lock);
1323 if (rw == WRITE) {
1324
1325
1326
1327 complete_conflicting_writes(req);
1328
1329
1330
1331
1332 maybe_pull_ahead(device);
1333 }
1334
1335
1336 if (drbd_suspended(device)) {
1337
1338 req->rq_state |= RQ_POSTPONED;
1339 if (req->private_bio) {
1340 bio_put(req->private_bio);
1341 req->private_bio = NULL;
1342 put_ldev(device);
1343 }
1344 goto out;
1345 }
1346
1347
1348
1349
1350 if (rw != WRITE) {
1351 if (!do_remote_read(req) && !req->private_bio)
1352 goto nodata;
1353 }
1354
1355
1356 req->epoch = atomic_read(&first_peer_device(device)->connection->current_tle_nr);
1357
1358
1359
1360 if (likely(req->i.size!=0)) {
1361 if (rw == WRITE)
1362 first_peer_device(device)->connection->current_tle_writes++;
1363
1364 list_add_tail(&req->tl_requests, &first_peer_device(device)->connection->transfer_log);
1365 }
1366
1367 if (rw == WRITE) {
1368 if (req->private_bio && !may_do_writes(device)) {
1369 bio_put(req->private_bio);
1370 req->private_bio = NULL;
1371 put_ldev(device);
1372 goto nodata;
1373 }
1374 if (!drbd_process_write_request(req))
1375 no_remote = true;
1376 } else {
1377
1378
1379 if (req->private_bio == NULL) {
1380 _req_mod(req, TO_BE_SENT);
1381 _req_mod(req, QUEUE_FOR_NET_READ);
1382 } else
1383 no_remote = true;
1384 }
1385
1386 if (no_remote == false) {
1387 struct drbd_plug_cb *plug = drbd_check_plugged(resource);
1388 if (plug)
1389 drbd_update_plug(plug, req);
1390 }
1391
1392
1393
1394 if (list_empty(&req->req_pending_master_completion))
1395 list_add_tail(&req->req_pending_master_completion,
1396 &device->pending_master_completion[rw == WRITE]);
1397 if (req->private_bio) {
1398
1399 req->pre_submit_jif = jiffies;
1400 list_add_tail(&req->req_pending_local,
1401 &device->pending_completion[rw == WRITE]);
1402 _req_mod(req, TO_BE_SUBMITTED);
1403
1404 submit_private_bio = true;
1405 } else if (no_remote) {
1406nodata:
1407 if (__ratelimit(&drbd_ratelimit_state))
1408 drbd_err(device, "IO ERROR: neither local nor remote data, sector %llu+%u\n",
1409 (unsigned long long)req->i.sector, req->i.size >> 9);
1410
1411
1412 }
1413
1414out:
1415 drbd_req_put_completion_ref(req, &m, 1);
1416 spin_unlock_irq(&resource->req_lock);
1417
1418
1419
1420
1421
1422
1423
1424 if (submit_private_bio)
1425 drbd_submit_req_private_bio(req);
1426 if (m.bio)
1427 complete_master_bio(device, &m);
1428}
1429
1430void __drbd_make_request(struct drbd_device *device, struct bio *bio, unsigned long start_jif)
1431{
1432 struct drbd_request *req = drbd_request_prepare(device, bio, start_jif);
1433 if (IS_ERR_OR_NULL(req))
1434 return;
1435 drbd_send_and_submit(device, req);
1436}
1437
1438static void submit_fast_path(struct drbd_device *device, struct list_head *incoming)
1439{
1440 struct blk_plug plug;
1441 struct drbd_request *req, *tmp;
1442
1443 blk_start_plug(&plug);
1444 list_for_each_entry_safe(req, tmp, incoming, tl_requests) {
1445 const int rw = bio_data_dir(req->master_bio);
1446
1447 if (rw == WRITE
1448 && req->private_bio && req->i.size
1449 && !test_bit(AL_SUSPENDED, &device->flags)) {
1450 if (!drbd_al_begin_io_fastpath(device, &req->i))
1451 continue;
1452
1453 req->rq_state |= RQ_IN_ACT_LOG;
1454 req->in_actlog_jif = jiffies;
1455 atomic_dec(&device->ap_actlog_cnt);
1456 }
1457
1458 list_del_init(&req->tl_requests);
1459 drbd_send_and_submit(device, req);
1460 }
1461 blk_finish_plug(&plug);
1462}
1463
1464static bool prepare_al_transaction_nonblock(struct drbd_device *device,
1465 struct list_head *incoming,
1466 struct list_head *pending,
1467 struct list_head *later)
1468{
1469 struct drbd_request *req;
1470 int wake = 0;
1471 int err;
1472
1473 spin_lock_irq(&device->al_lock);
1474 while ((req = list_first_entry_or_null(incoming, struct drbd_request, tl_requests))) {
1475 err = drbd_al_begin_io_nonblock(device, &req->i);
1476 if (err == -ENOBUFS)
1477 break;
1478 if (err == -EBUSY)
1479 wake = 1;
1480 if (err)
1481 list_move_tail(&req->tl_requests, later);
1482 else
1483 list_move_tail(&req->tl_requests, pending);
1484 }
1485 spin_unlock_irq(&device->al_lock);
1486 if (wake)
1487 wake_up(&device->al_wait);
1488 return !list_empty(pending);
1489}
1490
1491static void send_and_submit_pending(struct drbd_device *device, struct list_head *pending)
1492{
1493 struct blk_plug plug;
1494 struct drbd_request *req;
1495
1496 blk_start_plug(&plug);
1497 while ((req = list_first_entry_or_null(pending, struct drbd_request, tl_requests))) {
1498 req->rq_state |= RQ_IN_ACT_LOG;
1499 req->in_actlog_jif = jiffies;
1500 atomic_dec(&device->ap_actlog_cnt);
1501 list_del_init(&req->tl_requests);
1502 drbd_send_and_submit(device, req);
1503 }
1504 blk_finish_plug(&plug);
1505}
1506
1507void do_submit(struct work_struct *ws)
1508{
1509 struct drbd_device *device = container_of(ws, struct drbd_device, submit.worker);
1510 LIST_HEAD(incoming);
1511 LIST_HEAD(pending);
1512 LIST_HEAD(busy);
1513
1514
1515 spin_lock_irq(&device->resource->req_lock);
1516 list_splice_tail_init(&device->submit.writes, &incoming);
1517 spin_unlock_irq(&device->resource->req_lock);
1518
1519 for (;;) {
1520 DEFINE_WAIT(wait);
1521
1522
1523 list_splice_init(&busy, &incoming);
1524 submit_fast_path(device, &incoming);
1525 if (list_empty(&incoming))
1526 break;
1527
1528 for (;;) {
1529 prepare_to_wait(&device->al_wait, &wait, TASK_UNINTERRUPTIBLE);
1530
1531 list_splice_init(&busy, &incoming);
1532 prepare_al_transaction_nonblock(device, &incoming, &pending, &busy);
1533 if (!list_empty(&pending))
1534 break;
1535
1536 schedule();
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549 if (!list_empty(&incoming))
1550 continue;
1551
1552
1553
1554
1555 spin_lock_irq(&device->resource->req_lock);
1556 list_splice_tail_init(&device->submit.writes, &incoming);
1557 spin_unlock_irq(&device->resource->req_lock);
1558 }
1559 finish_wait(&device->al_wait, &wait);
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577 while (list_empty(&incoming)) {
1578 LIST_HEAD(more_pending);
1579 LIST_HEAD(more_incoming);
1580 bool made_progress;
1581
1582
1583
1584 if (list_empty(&device->submit.writes))
1585 break;
1586
1587 spin_lock_irq(&device->resource->req_lock);
1588 list_splice_tail_init(&device->submit.writes, &more_incoming);
1589 spin_unlock_irq(&device->resource->req_lock);
1590
1591 if (list_empty(&more_incoming))
1592 break;
1593
1594 made_progress = prepare_al_transaction_nonblock(device, &more_incoming, &more_pending, &busy);
1595
1596 list_splice_tail_init(&more_pending, &pending);
1597 list_splice_tail_init(&more_incoming, &incoming);
1598 if (!made_progress)
1599 break;
1600 }
1601
1602 drbd_al_begin_io_commit(device);
1603 send_and_submit_pending(device, &pending);
1604 }
1605}
1606
1607blk_qc_t drbd_make_request(struct request_queue *q, struct bio *bio)
1608{
1609 struct drbd_device *device = (struct drbd_device *) q->queuedata;
1610 unsigned long start_jif;
1611
1612 blk_queue_split(q, &bio);
1613
1614 start_jif = jiffies;
1615
1616
1617
1618
1619 D_ASSERT(device, IS_ALIGNED(bio->bi_iter.bi_size, 512));
1620
1621 inc_ap_bio(device);
1622 __drbd_make_request(device, bio, start_jif);
1623 return BLK_QC_T_NONE;
1624}
1625
1626static bool net_timeout_reached(struct drbd_request *net_req,
1627 struct drbd_connection *connection,
1628 unsigned long now, unsigned long ent,
1629 unsigned int ko_count, unsigned int timeout)
1630{
1631 struct drbd_device *device = net_req->device;
1632
1633 if (!time_after(now, net_req->pre_send_jif + ent))
1634 return false;
1635
1636 if (time_in_range(now, connection->last_reconnect_jif, connection->last_reconnect_jif + ent))
1637 return false;
1638
1639 if (net_req->rq_state & RQ_NET_PENDING) {
1640 drbd_warn(device, "Remote failed to finish a request within %ums > ko-count (%u) * timeout (%u * 0.1s)\n",
1641 jiffies_to_msecs(now - net_req->pre_send_jif), ko_count, timeout);
1642 return true;
1643 }
1644
1645
1646
1647
1648
1649 if (net_req->epoch == connection->send.current_epoch_nr) {
1650 drbd_warn(device,
1651 "We did not send a P_BARRIER for %ums > ko-count (%u) * timeout (%u * 0.1s); drbd kernel thread blocked?\n",
1652 jiffies_to_msecs(now - net_req->pre_send_jif), ko_count, timeout);
1653 return false;
1654 }
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672 if (time_after(now, connection->send.last_sent_barrier_jif + ent)) {
1673 drbd_warn(device, "Remote failed to answer a P_BARRIER (sent at %lu jif; now=%lu jif) within %ums > ko-count (%u) * timeout (%u * 0.1s)\n",
1674 connection->send.last_sent_barrier_jif, now,
1675 jiffies_to_msecs(now - connection->send.last_sent_barrier_jif), ko_count, timeout);
1676 return true;
1677 }
1678 return false;
1679}
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698void request_timer_fn(struct timer_list *t)
1699{
1700 struct drbd_device *device = from_timer(device, t, request_timer);
1701 struct drbd_connection *connection = first_peer_device(device)->connection;
1702 struct drbd_request *req_read, *req_write, *req_peer;
1703 struct net_conf *nc;
1704 unsigned long oldest_submit_jif;
1705 unsigned long ent = 0, dt = 0, et, nt;
1706 unsigned long now;
1707 unsigned int ko_count = 0, timeout = 0;
1708
1709 rcu_read_lock();
1710 nc = rcu_dereference(connection->net_conf);
1711 if (nc && device->state.conn >= C_WF_REPORT_PARAMS) {
1712 ko_count = nc->ko_count;
1713 timeout = nc->timeout;
1714 }
1715
1716 if (get_ldev(device)) {
1717 dt = rcu_dereference(device->ldev->disk_conf)->disk_timeout * HZ / 10;
1718 put_ldev(device);
1719 }
1720 rcu_read_unlock();
1721
1722
1723 ent = timeout * HZ/10 * ko_count;
1724 et = min_not_zero(dt, ent);
1725
1726 if (!et)
1727 return;
1728
1729 now = jiffies;
1730 nt = now + et;
1731
1732 spin_lock_irq(&device->resource->req_lock);
1733 req_read = list_first_entry_or_null(&device->pending_completion[0], struct drbd_request, req_pending_local);
1734 req_write = list_first_entry_or_null(&device->pending_completion[1], struct drbd_request, req_pending_local);
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744 req_peer = connection->req_ack_pending;
1745
1746
1747
1748
1749 if (!req_peer)
1750 req_peer = connection->req_not_net_done;
1751
1752
1753 if (req_peer && req_peer->device != device)
1754 req_peer = NULL;
1755
1756
1757 if (req_peer == NULL && req_write == NULL && req_read == NULL)
1758 goto out;
1759
1760 oldest_submit_jif =
1761 (req_write && req_read)
1762 ? ( time_before(req_write->pre_submit_jif, req_read->pre_submit_jif)
1763 ? req_write->pre_submit_jif : req_read->pre_submit_jif )
1764 : req_write ? req_write->pre_submit_jif
1765 : req_read ? req_read->pre_submit_jif : now;
1766
1767 if (ent && req_peer && net_timeout_reached(req_peer, connection, now, ent, ko_count, timeout))
1768 _conn_request_state(connection, NS(conn, C_TIMEOUT), CS_VERBOSE | CS_HARD);
1769
1770 if (dt && oldest_submit_jif != now &&
1771 time_after(now, oldest_submit_jif + dt) &&
1772 !time_in_range(now, device->last_reattach_jif, device->last_reattach_jif + dt)) {
1773 drbd_warn(device, "Local backing device failed to meet the disk-timeout\n");
1774 __drbd_chk_io_error(device, DRBD_FORCE_DETACH);
1775 }
1776
1777
1778
1779 ent = (ent && req_peer && time_before(now, req_peer->pre_send_jif + ent))
1780 ? req_peer->pre_send_jif + ent : now + et;
1781 dt = (dt && oldest_submit_jif != now && time_before(now, oldest_submit_jif + dt))
1782 ? oldest_submit_jif + dt : now + et;
1783 nt = time_before(ent, dt) ? ent : dt;
1784out:
1785 spin_unlock_irq(&device->resource->req_lock);
1786 mod_timer(&device->request_timer, nt);
1787}
1788