Lines Matching +full:disk +full:- +full:activity
1 // SPDX-License-Identifier: GPL-2.0-only
7 Copyright (C) 2001-2008, LINBIT Information Technologies GmbH.
8 Copyright (C) 1999-2008, Philipp Reisner <philipp.reisner@linbit.com>.
9 Copyright (C) 2002-2008, Lars Ellenberg <lars.ellenberg@linbit.com>.
33 req->rq_state = (bio_data_dir(bio_src) == WRITE ? RQ_WRITE : 0) in drbd_req_new()
36 req->device = device; in drbd_req_new()
37 req->master_bio = bio_src; in drbd_req_new()
38 req->epoch = 0; in drbd_req_new()
40 drbd_clear_interval(&req->i); in drbd_req_new()
41 req->i.sector = bio_src->bi_iter.bi_sector; in drbd_req_new()
42 req->i.size = bio_src->bi_iter.bi_size; in drbd_req_new()
43 req->i.local = true; in drbd_req_new()
44 req->i.waiting = false; in drbd_req_new()
46 INIT_LIST_HEAD(&req->tl_requests); in drbd_req_new()
47 INIT_LIST_HEAD(&req->w.list); in drbd_req_new()
48 INIT_LIST_HEAD(&req->req_pending_master_completion); in drbd_req_new()
49 INIT_LIST_HEAD(&req->req_pending_local); in drbd_req_new()
52 atomic_set(&req->completion_ref, 1); in drbd_req_new()
54 kref_init(&req->kref); in drbd_req_new()
61 struct drbd_device *device = req->device; in drbd_remove_request_interval()
62 struct drbd_interval *i = &req->i; in drbd_remove_request_interval()
67 if (i->waiting) in drbd_remove_request_interval()
68 wake_up(&device->misc_wait); in drbd_remove_request_interval()
74 struct drbd_device *device = req->device; in drbd_req_destroy()
75 const unsigned s = req->rq_state; in drbd_req_destroy()
77 if ((req->master_bio && !(s & RQ_POSTPONED)) || in drbd_req_destroy()
78 atomic_read(&req->completion_ref) || in drbd_req_destroy()
82 s, atomic_read(&req->completion_ref)); in drbd_req_destroy()
88 * req_lock, and req->tl_requests will typicaly be on ->transfer_log, in drbd_req_destroy()
92 * still allowed to unconditionally list_del(&req->tl_requests), in drbd_req_destroy()
93 * because it will be on a local on-stack list only. */ in drbd_req_destroy()
94 list_del_init(&req->tl_requests); in drbd_req_destroy()
98 if (!drbd_interval_empty(&req->i)) { in drbd_req_destroy()
102 root = &device->write_requests; in drbd_req_destroy()
104 root = &device->read_requests; in drbd_req_destroy()
106 } else if (s & (RQ_NET_MASK & ~RQ_NET_DONE) && req->i.size != 0) in drbd_req_destroy()
108 s, (unsigned long long)req->i.sector, req->i.size); in drbd_req_destroy()
111 * bit(s) out-of-sync first. If it had a local part, we need to in drbd_req_destroy()
112 * release the reference to the activity log. */ in drbd_req_destroy()
114 /* Set out-of-sync unless both OK flags are set in drbd_req_destroy()
116 * Other places where we set out-of-sync: in drbd_req_destroy()
117 * READ with local io-error */ in drbd_req_destroy()
128 drbd_set_out_of_sync(peer_device, req->i.sector, req->i.size); in drbd_req_destroy()
131 drbd_set_in_sync(peer_device, req->i.sector, req->i.size); in drbd_req_destroy()
146 drbd_al_complete_io(device, &req->i); in drbd_req_destroy()
150 "but my Disk seems to have failed :(\n", in drbd_req_destroy()
151 (unsigned long long) req->i.sector, req->i.size); in drbd_req_destroy()
161 wake_up(&connection->sender_work.q_wait); in wake_all_senders()
164 /* must hold resource->req_lock */
168 if (connection->current_tle_writes == 0) in start_new_tl_epoch()
171 connection->current_tle_writes = 0; in start_new_tl_epoch()
172 atomic_inc(&connection->current_tle_nr); in start_new_tl_epoch()
179 if (unlikely(m->error)) in complete_master_bio()
180 m->bio->bi_status = errno_to_blk_status(m->error); in complete_master_bio()
181 bio_endio(m->bio); in complete_master_bio()
187 * Set m->bio to the master bio, if it is fit to be completed,
190 * If m->bio is set, the error status to be returned is placed in m->error.
195 const unsigned s = req->rq_state; in drbd_req_complete()
196 struct drbd_device *device = req->device; in drbd_req_complete()
215 if (!req->master_bio) { in drbd_req_complete()
234 error = PTR_ERR(req->private_bio); in drbd_req_complete()
243 if (op_is_write(bio_op(req->master_bio)) && in drbd_req_complete()
244 req->epoch == atomic_read(&first_peer_device(device)->connection->current_tle_nr)) in drbd_req_complete()
245 start_new_tl_epoch(first_peer_device(device)->connection); in drbd_req_complete()
247 /* Update disk stats */ in drbd_req_complete()
248 bio_end_io_acct(req->master_bio, req->start_jif); in drbd_req_complete()
252 * so it will re-enter __drbd_make_request(), in drbd_req_complete()
253 * and be re-assigned to a suitable local or remote path, in drbd_req_complete()
260 * read-ahead may fail, and will not be retried. in drbd_req_complete()
265 bio_op(req->master_bio) == REQ_OP_READ && in drbd_req_complete()
266 !(req->master_bio->bi_opf & REQ_RAHEAD) && in drbd_req_complete()
267 !list_empty(&req->tl_requests)) in drbd_req_complete()
268 req->rq_state |= RQ_POSTPONED; in drbd_req_complete()
270 if (!(req->rq_state & RQ_POSTPONED)) { in drbd_req_complete()
271 m->error = ok ? 0 : (error ?: -EIO); in drbd_req_complete()
272 m->bio = req->master_bio; in drbd_req_complete()
273 req->master_bio = NULL; in drbd_req_complete()
275 * write-acks in protocol != C during resync. in drbd_req_complete()
277 * conflict in a multi-primary setup. */ in drbd_req_complete()
278 req->i.completed = true; in drbd_req_complete()
281 if (req->i.waiting) in drbd_req_complete()
282 wake_up(&device->misc_wait); in drbd_req_complete()
288 list_del_init(&req->req_pending_master_completion); in drbd_req_complete()
291 /* still holds resource->req_lock */
294 struct drbd_device *device = req->device; in drbd_req_put_completion_ref()
295 D_ASSERT(device, m || (req->rq_state & RQ_POSTPONED)); in drbd_req_put_completion_ref()
300 if (!atomic_sub_and_test(put, &req->completion_ref)) in drbd_req_put_completion_ref()
307 if (req->rq_state & RQ_LOCAL_ABORTED) in drbd_req_put_completion_ref()
310 if (req->rq_state & RQ_POSTPONED) { in drbd_req_put_completion_ref()
317 kref_put(&req->kref, drbd_req_destroy); in drbd_req_put_completion_ref()
322 struct drbd_connection *connection = peer_device ? peer_device->connection : NULL; in set_if_null_req_next()
325 if (connection->req_next == NULL) in set_if_null_req_next()
326 connection->req_next = req; in set_if_null_req_next()
331 struct drbd_connection *connection = peer_device ? peer_device->connection : NULL; in advance_conn_req_next()
335 if (connection->req_next != req) in advance_conn_req_next()
339 list_for_each_entry_continue(iter, &connection->transfer_log, tl_requests) { in advance_conn_req_next()
340 const unsigned int s = iter->rq_state; in advance_conn_req_next()
347 connection->req_next = req; in advance_conn_req_next()
352 struct drbd_connection *connection = peer_device ? peer_device->connection : NULL; in set_if_null_req_ack_pending()
355 if (connection->req_ack_pending == NULL) in set_if_null_req_ack_pending()
356 connection->req_ack_pending = req; in set_if_null_req_ack_pending()
361 struct drbd_connection *connection = peer_device ? peer_device->connection : NULL; in advance_conn_req_ack_pending()
365 if (connection->req_ack_pending != req) in advance_conn_req_ack_pending()
369 list_for_each_entry_continue(iter, &connection->transfer_log, tl_requests) { in advance_conn_req_ack_pending()
370 const unsigned int s = iter->rq_state; in advance_conn_req_ack_pending()
377 connection->req_ack_pending = req; in advance_conn_req_ack_pending()
382 struct drbd_connection *connection = peer_device ? peer_device->connection : NULL; in set_if_null_req_not_net_done()
385 if (connection->req_not_net_done == NULL) in set_if_null_req_not_net_done()
386 connection->req_not_net_done = req; in set_if_null_req_not_net_done()
391 struct drbd_connection *connection = peer_device ? peer_device->connection : NULL; in advance_conn_req_not_net_done()
395 if (connection->req_not_net_done != req) in advance_conn_req_not_net_done()
399 list_for_each_entry_continue(iter, &connection->transfer_log, tl_requests) { in advance_conn_req_not_net_done()
400 const unsigned int s = iter->rq_state; in advance_conn_req_not_net_done()
407 connection->req_not_net_done = req; in advance_conn_req_not_net_done()
411 * req->completion_ref and req->kref. */
415 struct drbd_device *device = req->device; in mod_rq_state()
417 unsigned s = req->rq_state; in mod_rq_state()
425 req->rq_state &= ~clear; in mod_rq_state()
426 req->rq_state |= set; in mod_rq_state()
429 if (req->rq_state == s) in mod_rq_state()
434 kref_get(&req->kref); in mod_rq_state()
437 atomic_inc(&req->completion_ref); in mod_rq_state()
441 atomic_inc(&req->completion_ref); in mod_rq_state()
445 atomic_inc(&req->completion_ref); in mod_rq_state()
450 kref_get(&req->kref); /* wait for the DONE */ in mod_rq_state()
455 atomic_add(req->i.size >> 9, &device->ap_in_flight); in mod_rq_state()
458 if (req->rq_state & RQ_NET_PENDING) in mod_rq_state()
463 atomic_inc(&req->completion_ref); in mod_rq_state()
471 D_ASSERT(device, req->rq_state & RQ_LOCAL_PENDING); in mod_rq_state()
476 if (req->rq_state & RQ_LOCAL_ABORTED) in mod_rq_state()
477 kref_put(&req->kref, drbd_req_destroy); in mod_rq_state()
480 list_del_init(&req->req_pending_local); in mod_rq_state()
486 req->acked_jif = jiffies; in mod_rq_state()
497 atomic_sub(req->i.size >> 9, &device->ap_in_flight); in mod_rq_state()
499 kref_put(&req->kref, drbd_req_destroy); in mod_rq_state()
500 req->net_done_jif = jiffies; in mod_rq_state()
513 if (req->i.waiting) in mod_rq_state()
514 wake_up(&device->misc_wait); in mod_rq_state()
517 kref_put(&req->kref, drbd_req_destroy); in mod_rq_state()
526 (req->rq_state & RQ_WRITE) ? "WRITE" : "READ", in drbd_report_io_error()
527 (unsigned long long)req->i.sector, in drbd_report_io_error()
528 req->i.size >> 9, in drbd_report_io_error()
529 device->ldev->backing_bdev); in drbd_report_io_error()
535 * --> If so, clear PENDING and set NET_OK below.
536 * If it is a protocol A write, but not RQ_PENDING anymore, neg-ack was faster
540 return (req->rq_state & in is_pending_write_protocol_A()
558 * peer_device == NULL means local disk
564 struct drbd_device *const device = req->device; in __req_mod()
565 struct drbd_connection *const connection = peer_device ? peer_device->connection : NULL; in __req_mod()
570 m->bio = NULL; in __req_mod()
586 D_ASSERT(device, !(req->rq_state & RQ_NET_MASK)); in __req_mod()
588 nc = rcu_dereference(connection->net_conf); in __req_mod()
589 p = nc->wire_protocol; in __req_mod()
591 req->rq_state |= in __req_mod()
599 D_ASSERT(device, !(req->rq_state & RQ_LOCAL_MASK)); in __req_mod()
604 if (req->rq_state & RQ_WRITE) in __req_mod()
605 device->writ_cnt += req->i.size >> 9; in __req_mod()
607 device->read_cnt += req->i.size >> 9; in __req_mod()
625 req->i.sector, req->i.size); in __req_mod()
630 /* it is legal to fail read-ahead, no __drbd_chk_io_error in that case. */ in __req_mod()
636 /* I'd rather not detach from local disk just because it in __req_mod()
643 * no local disk, in __req_mod()
645 * or just got an io-error. */ in __req_mod()
647 * or from bio_endio during read io-error recovery */ in __req_mod()
652 D_ASSERT(device, drbd_interval_empty(&req->i)); in __req_mod()
653 drbd_insert_interval(&device->read_requests, &req->i); in __req_mod()
655 set_bit(UNPLUG_REMOTE, &device->flags); in __req_mod()
657 D_ASSERT(device, req->rq_state & RQ_NET_PENDING); in __req_mod()
658 D_ASSERT(device, (req->rq_state & RQ_LOCAL_MASK) == 0); in __req_mod()
660 req->w.cb = w_send_read_req; in __req_mod()
661 drbd_queue_work(&connection->sender_work, in __req_mod()
662 &req->w); in __req_mod()
671 D_ASSERT(device, drbd_interval_empty(&req->i)); in __req_mod()
672 drbd_insert_interval(&device->write_requests, &req->i); in __req_mod()
689 * io-scheduler timeout to expire, increasing maximum latency, in __req_mod()
691 set_bit(UNPLUG_REMOTE, &device->flags); in __req_mod()
694 D_ASSERT(device, req->rq_state & RQ_NET_PENDING); in __req_mod()
696 req->w.cb = w_send_dblock; in __req_mod()
697 drbd_queue_work(&connection->sender_work, in __req_mod()
698 &req->w); in __req_mod()
702 nc = rcu_dereference(connection->net_conf); in __req_mod()
703 p = nc->max_epoch_size; in __req_mod()
705 if (connection->current_tle_writes >= p) in __req_mod()
712 req->w.cb = w_send_out_of_sync; in __req_mod()
713 drbd_queue_work(&connection->sender_work, in __req_mod()
714 &req->w); in __req_mod()
755 * node crashes are covered by the activity log. in __req_mod()
760 D_ASSERT(device, req->rq_state & RQ_NET_PENDING); in __req_mod()
761 D_ASSERT(device, req->rq_state & RQ_EXP_WRITE_ACK); in __req_mod()
766 req->rq_state |= RQ_NET_SIS; in __req_mod()
775 * for volatile write-back caches on lower level devices. */ in __req_mod()
778 D_ASSERT(device, req->rq_state & RQ_EXP_RECEIVE_ACK); in __req_mod()
787 D_ASSERT(device, req->rq_state & RQ_EXP_WRITE_ACK); in __req_mod()
792 D_ASSERT(device, req->rq_state & RQ_NET_PENDING); in __req_mod()
793 req->rq_state |= RQ_POSTPONED; in __req_mod()
794 if (req->i.waiting) in __req_mod()
795 wake_up(&device->misc_wait); in __req_mod()
806 if (!(req->rq_state & RQ_LOCAL_COMPLETED)) in __req_mod()
812 if (!(req->rq_state & RQ_LOCAL_COMPLETED)) in __req_mod()
820 if (bio_data_dir(req->master_bio) == WRITE) in __req_mod()
824 req->w.cb = w_restart_disk_io; in __req_mod()
825 drbd_queue_work(&connection->sender_work, in __req_mod()
826 &req->w); in __req_mod()
831 if (!(req->rq_state & RQ_WRITE) && !req->w.cb) { in __req_mod()
841 if (!(req->rq_state & RQ_NET_OK)) { in __req_mod()
842 /* FIXME could this possibly be a req->dw.cb == w_send_out_of_sync? in __req_mod()
846 if (req->w.cb) { in __req_mod()
848 drbd_queue_work(&connection->sender_work, in __req_mod()
849 &req->w); in __req_mod()
850 rv = req->rq_state & RQ_WRITE ? MR_WRITE : MR_READ; in __req_mod()
858 if (!(req->rq_state & RQ_WRITE)) in __req_mod()
861 if (req->rq_state & RQ_NET_PENDING) { in __req_mod()
872 (req->rq_state & RQ_NET_MASK) ? RQ_NET_DONE : 0); in __req_mod()
876 D_ASSERT(device, req->rq_state & RQ_NET_PENDING); in __req_mod()
890 * - we are consistent (of course),
891 * - or we are generally inconsistent,
901 if (device->state.disk == D_UP_TO_DATE) in drbd_may_do_local_read()
903 if (device->state.disk != D_INCONSISTENT) in drbd_may_do_local_read()
905 esector = sector + (size >> 9) - 1; in drbd_may_do_local_read()
906 nr_sectors = get_capacity(device->vdisk); in drbd_may_do_local_read()
925 return atomic_read(&device->local_cnt) > in remote_due_to_read_balancing()
926 atomic_read(&device->ap_pending_cnt) + atomic_read(&device->rs_pending_cnt); in remote_due_to_read_balancing()
933 stripe_shift = (rbm - RB_32K_STRIPING + 15); in remote_due_to_read_balancing()
934 return (sector >> (stripe_shift - 9)) & 1; in remote_due_to_read_balancing()
936 return test_and_change_bit(READ_BALANCE_RR, &device->flags); in remote_due_to_read_balancing()
946 * complete_conflicting_writes - wait for any conflicting write requests
957 struct drbd_device *device = req->device; in complete_conflicting_writes()
959 sector_t sector = req->i.sector; in complete_conflicting_writes()
960 int size = req->i.size; in complete_conflicting_writes()
963 drbd_for_each_overlap(i, &device->write_requests, sector, size) { in complete_conflicting_writes()
965 if (i->completed) in complete_conflicting_writes()
974 /* Indicate to wake up device->misc_wait on progress. */ in complete_conflicting_writes()
975 prepare_to_wait(&device->misc_wait, &wait, TASK_UNINTERRUPTIBLE); in complete_conflicting_writes()
976 i->waiting = true; in complete_conflicting_writes()
977 spin_unlock_irq(&device->resource->req_lock); in complete_conflicting_writes()
979 spin_lock_irq(&device->resource->req_lock); in complete_conflicting_writes()
981 finish_wait(&device->misc_wait, &wait); in complete_conflicting_writes()
987 struct drbd_connection *connection = first_peer_device(device)->connection; in maybe_pull_ahead()
993 nc = rcu_dereference(connection->net_conf); in maybe_pull_ahead()
994 on_congestion = nc ? nc->on_congestion : OC_BLOCK; in maybe_pull_ahead()
997 connection->agreed_pro_version < 96) in maybe_pull_ahead()
1000 if (on_congestion == OC_PULL_AHEAD && device->state.conn == C_AHEAD) in maybe_pull_ahead()
1005 * sure device->act_log is there. in maybe_pull_ahead()
1010 if (nc->cong_fill && in maybe_pull_ahead()
1011 atomic_read(&device->ap_in_flight) >= nc->cong_fill) { in maybe_pull_ahead()
1012 drbd_info(device, "Congestion-fill threshold reached\n"); in maybe_pull_ahead()
1016 if (device->act_log->used >= nc->cong_extents) { in maybe_pull_ahead()
1017 drbd_info(device, "Congestion-extents threshold reached\n"); in maybe_pull_ahead()
1022 /* start a new epoch for non-mirrored writes */ in maybe_pull_ahead()
1023 start_new_tl_epoch(first_peer_device(device)->connection); in maybe_pull_ahead()
1027 else /*nc->on_congestion == OC_DISCONNECT */ in maybe_pull_ahead()
1033 /* If this returns false, and req->private_bio is still set,
1036 * If it returns false, but req->private_bio is not set,
1039 * Otherwise, this destroys req->private_bio, if any,
1044 struct drbd_device *device = req->device; in do_remote_read()
1047 if (req->private_bio) { in do_remote_read()
1049 req->i.sector, req->i.size)) { in do_remote_read()
1050 bio_put(req->private_bio); in do_remote_read()
1051 req->private_bio = NULL; in do_remote_read()
1056 if (device->state.pdsk != D_UP_TO_DATE) in do_remote_read()
1059 if (req->private_bio == NULL) in do_remote_read()
1066 rbm = rcu_dereference(device->ldev->disk_conf)->read_balancing; in do_remote_read()
1069 if (rbm == RB_PREFER_LOCAL && req->private_bio) in do_remote_read()
1072 if (remote_due_to_read_balancing(device, req->i.sector, rbm)) { in do_remote_read()
1073 if (req->private_bio) { in do_remote_read()
1074 bio_put(req->private_bio); in do_remote_read()
1075 req->private_bio = NULL; in do_remote_read()
1107 struct drbd_device *device = req->device; in drbd_process_write_request()
1111 remote = drbd_should_do_remote(device->state); in drbd_process_write_request()
1112 send_oos = drbd_should_send_out_of_sync(device->state); in drbd_process_write_request()
1120 if (unlikely(req->i.size == 0)) { in drbd_process_write_request()
1122 D_ASSERT(device, req->master_bio->bi_opf & REQ_PREFLUSH); in drbd_process_write_request()
1136 } else if (drbd_set_out_of_sync(peer_device, req->i.sector, req->i.size)) in drbd_process_write_request()
1144 int err = drbd_issue_discard_or_zero_out(req->device, in drbd_process_discard_or_zeroes_req()
1145 req->i.sector, req->i.size >> 9, flags); in drbd_process_discard_or_zeroes_req()
1147 req->private_bio->bi_status = BLK_STS_IOERR; in drbd_process_discard_or_zeroes_req()
1148 bio_endio(req->private_bio); in drbd_process_discard_or_zeroes_req()
1154 struct drbd_device *device = req->device; in drbd_submit_req_private_bio()
1155 struct bio *bio = req->private_bio; in drbd_submit_req_private_bio()
1160 else if (bio->bi_opf & REQ_RAHEAD) in drbd_submit_req_private_bio()
1166 * ->ldev member. Double check, and short-circuit to endio. in drbd_submit_req_private_bio()
1167 * In case the last activity log transaction failed to get on in drbd_submit_req_private_bio()
1175 ((bio->bi_opf & REQ_NOUNMAP) ? 0 : EE_TRIM)); in drbd_submit_req_private_bio()
1187 spin_lock_irq(&device->resource->req_lock); in drbd_queue_write()
1188 list_add_tail(&req->tl_requests, &device->submit.writes); in drbd_queue_write()
1189 list_add_tail(&req->req_pending_master_completion, in drbd_queue_write()
1190 &device->pending_master_completion[1 /* WRITE */]); in drbd_queue_write()
1191 spin_unlock_irq(&device->resource->req_lock); in drbd_queue_write()
1192 queue_work(device->submit.wq, &device->submit.worker); in drbd_queue_write()
1194 wake_up(&device->al_wait); in drbd_queue_write()
1200 * Returns ERR_PTR(-ENOMEM) if we cannot allocate a drbd_request.
1215 bio->bi_status = BLK_STS_RESOURCE; in drbd_request_prepare()
1217 return ERR_PTR(-ENOMEM); in drbd_request_prepare()
1220 /* Update disk stats */ in drbd_request_prepare()
1221 req->start_jif = bio_start_io_acct(req->master_bio); in drbd_request_prepare()
1224 req->private_bio = bio_alloc_clone(device->ldev->backing_bdev, in drbd_request_prepare()
1227 req->private_bio->bi_private = req; in drbd_request_prepare()
1228 req->private_bio->bi_end_io = drbd_request_endio; in drbd_request_prepare()
1236 if (rw == WRITE && req->private_bio && req->i.size in drbd_request_prepare()
1237 && !test_bit(AL_SUSPENDED, &device->flags)) { in drbd_request_prepare()
1238 if (!drbd_al_begin_io_fastpath(device, &req->i)) in drbd_request_prepare()
1240 req->rq_state |= RQ_IN_ACT_LOG; in drbd_request_prepare()
1241 req->in_actlog_jif = jiffies; in drbd_request_prepare()
1246 atomic_inc(&device->ap_actlog_cnt); in drbd_request_prepare()
1263 const union drbd_dev_state s = device->state; in may_do_writes()
1264 return s.disk == D_UP_TO_DATE || s.pdsk == D_UP_TO_DATE; in may_do_writes()
1276 struct drbd_resource *resource = plug->cb.data; in drbd_unplug()
1277 struct drbd_request *req = plug->most_recent_req; in drbd_unplug()
1283 spin_lock_irq(&resource->req_lock); in drbd_unplug()
1286 req->rq_state |= RQ_UNPLUG; in drbd_unplug()
1288 drbd_queue_unplug(req->device); in drbd_unplug()
1289 kref_put(&req->kref, drbd_req_destroy); in drbd_unplug()
1290 spin_unlock_irq(&resource->req_lock); in drbd_unplug()
1309 struct drbd_request *tmp = plug->most_recent_req; in drbd_update_plug()
1312 kref_get(&req->kref); in drbd_update_plug()
1313 plug->most_recent_req = req; in drbd_update_plug()
1315 kref_put(&tmp->kref, drbd_req_destroy); in drbd_update_plug()
1320 struct drbd_resource *resource = device->resource; in drbd_send_and_submit()
1322 const int rw = bio_data_dir(req->master_bio); in drbd_send_and_submit()
1327 spin_lock_irq(&resource->req_lock); in drbd_send_and_submit()
1330 * but will re-aquire it before it returns here. in drbd_send_and_submit()
1343 req->rq_state |= RQ_POSTPONED; in drbd_send_and_submit()
1344 if (req->private_bio) { in drbd_send_and_submit()
1345 bio_put(req->private_bio); in drbd_send_and_submit()
1346 req->private_bio = NULL; in drbd_send_and_submit()
1356 if (!do_remote_read(req) && !req->private_bio) in drbd_send_and_submit()
1361 req->epoch = atomic_read(&first_peer_device(device)->connection->current_tle_nr); in drbd_send_and_submit()
1365 if (likely(req->i.size!=0)) { in drbd_send_and_submit()
1367 first_peer_device(device)->connection->current_tle_writes++; in drbd_send_and_submit()
1369 list_add_tail(&req->tl_requests, &first_peer_device(device)->connection->transfer_log); in drbd_send_and_submit()
1373 if (req->private_bio && !may_do_writes(device)) { in drbd_send_and_submit()
1374 bio_put(req->private_bio); in drbd_send_and_submit()
1375 req->private_bio = NULL; in drbd_send_and_submit()
1384 if (req->private_bio == NULL) { in drbd_send_and_submit()
1399 if (list_empty(&req->req_pending_master_completion)) in drbd_send_and_submit()
1400 list_add_tail(&req->req_pending_master_completion, in drbd_send_and_submit()
1401 &device->pending_master_completion[rw == WRITE]); in drbd_send_and_submit()
1402 if (req->private_bio) { in drbd_send_and_submit()
1404 req->pre_submit_jif = jiffies; in drbd_send_and_submit()
1405 list_add_tail(&req->req_pending_local, in drbd_send_and_submit()
1406 &device->pending_completion[rw == WRITE]); in drbd_send_and_submit()
1414 (unsigned long long)req->i.sector, req->i.size >> 9); in drbd_send_and_submit()
1421 spin_unlock_irq(&resource->req_lock); in drbd_send_and_submit()
1428 * That's why we cannot check on req->private_bio. */ in drbd_send_and_submit()
1450 const int rw = bio_data_dir(req->master_bio); in submit_fast_path()
1453 && req->private_bio && req->i.size in submit_fast_path()
1454 && !test_bit(AL_SUSPENDED, &device->flags)) { in submit_fast_path()
1455 if (!drbd_al_begin_io_fastpath(device, &req->i)) in submit_fast_path()
1458 req->rq_state |= RQ_IN_ACT_LOG; in submit_fast_path()
1459 req->in_actlog_jif = jiffies; in submit_fast_path()
1460 atomic_dec(&device->ap_actlog_cnt); in submit_fast_path()
1463 list_del_init(&req->tl_requests); in submit_fast_path()
1478 spin_lock_irq(&device->al_lock); in prepare_al_transaction_nonblock()
1480 err = drbd_al_begin_io_nonblock(device, &req->i); in prepare_al_transaction_nonblock()
1481 if (err == -ENOBUFS) in prepare_al_transaction_nonblock()
1483 if (err == -EBUSY) in prepare_al_transaction_nonblock()
1486 list_move_tail(&req->tl_requests, later); in prepare_al_transaction_nonblock()
1488 list_move_tail(&req->tl_requests, pending); in prepare_al_transaction_nonblock()
1490 spin_unlock_irq(&device->al_lock); in prepare_al_transaction_nonblock()
1492 wake_up(&device->al_wait); in prepare_al_transaction_nonblock()
1503 req->rq_state |= RQ_IN_ACT_LOG; in send_and_submit_pending()
1504 req->in_actlog_jif = jiffies; in send_and_submit_pending()
1505 atomic_dec(&device->ap_actlog_cnt); in send_and_submit_pending()
1506 list_del_init(&req->tl_requests); in send_and_submit_pending()
1516 LIST_HEAD(pending); /* to be submitted after next AL-transaction commit */ in do_submit()
1520 spin_lock_irq(&device->resource->req_lock); in do_submit()
1521 list_splice_tail_init(&device->submit.writes, &incoming); in do_submit()
1522 spin_unlock_irq(&device->resource->req_lock); in do_submit()
1527 /* move used-to-be-busy back to front of incoming */ in do_submit()
1534 prepare_to_wait(&device->al_wait, &wait, TASK_UNINTERRUPTIBLE); in do_submit()
1543 /* If all currently "hot" activity log extents are kept busy by in do_submit()
1547 * enough update slots available, and the activity log in do_submit()
1560 spin_lock_irq(&device->resource->req_lock); in do_submit()
1561 list_splice_tail_init(&device->submit.writes, &incoming); in do_submit()
1562 spin_unlock_irq(&device->resource->req_lock); in do_submit()
1564 finish_wait(&device->al_wait, &wait); in do_submit()
1576 * Be strictly non-blocking here, in do_submit()
1589 if (list_empty(&device->submit.writes)) in do_submit()
1592 spin_lock_irq(&device->resource->req_lock); in do_submit()
1593 list_splice_tail_init(&device->submit.writes, &more_incoming); in do_submit()
1594 spin_unlock_irq(&device->resource->req_lock); in do_submit()
1614 struct drbd_device *device = bio->bi_bdev->bd_disk->private_data; in drbd_submit_bio()
1623 D_ASSERT(device, IS_ALIGNED(bio->bi_iter.bi_size, 512)); in drbd_submit_bio()
1634 struct drbd_device *device = net_req->device; in net_timeout_reached()
1636 if (!time_after(now, net_req->pre_send_jif + ent)) in net_timeout_reached()
1639 if (time_in_range(now, connection->last_reconnect_jif, connection->last_reconnect_jif + ent)) in net_timeout_reached()
1642 if (net_req->rq_state & RQ_NET_PENDING) { in net_timeout_reached()
1643 …drbd_warn(device, "Remote failed to finish a request within %ums > ko-count (%u) * timeout (%u * 0… in net_timeout_reached()
1644 jiffies_to_msecs(now - net_req->pre_send_jif), ko_count, timeout); in net_timeout_reached()
1652 if (net_req->epoch == connection->send.current_epoch_nr) { in net_timeout_reached()
1654 …"We did not send a P_BARRIER for %ums > ko-count (%u) * timeout (%u * 0.1s); drbd kernel thread bl… in net_timeout_reached()
1655 jiffies_to_msecs(now - net_req->pre_send_jif), ko_count, timeout); in net_timeout_reached()
1675 if (time_after(now, connection->send.last_sent_barrier_jif + ent)) { in net_timeout_reached()
1676 …failed to answer a P_BARRIER (sent at %lu jif; now=%lu jif) within %ums > ko-count (%u) * timeout … in net_timeout_reached()
1677 connection->send.last_sent_barrier_jif, now, in net_timeout_reached()
1678 jiffies_to_msecs(now - connection->send.last_sent_barrier_jif), ko_count, timeout); in net_timeout_reached()
1685 * - we have some effective timeout from the configuration,
1687 * - the oldest request is waiting for a response from the network
1688 * resp. the local disk,
1689 * - the oldest request is in fact older than the effective timeout,
1690 * - the connection was established (resp. disk was attached)
1696 * Side effect: once per 32bit wrap-around interval, which means every
1705 struct drbd_connection *connection = first_peer_device(device)->connection; in request_timer_fn()
1714 nc = rcu_dereference(connection->net_conf); in request_timer_fn()
1715 if (nc && device->state.conn >= C_WF_REPORT_PARAMS) { in request_timer_fn()
1716 ko_count = nc->ko_count; in request_timer_fn()
1717 timeout = nc->timeout; in request_timer_fn()
1720 if (get_ldev(device)) { /* implicit state.disk >= D_INCONSISTENT */ in request_timer_fn()
1721 dt = rcu_dereference(device->ldev->disk_conf)->disk_timeout * HZ / 10; in request_timer_fn()
1736 spin_lock_irq(&device->resource->req_lock); in request_timer_fn()
1737 …req_read = list_first_entry_or_null(&device->pending_completion[0], struct drbd_request, req_pendi… in request_timer_fn()
1738 …req_write = list_first_entry_or_null(&device->pending_completion[1], struct drbd_request, req_pend… in request_timer_fn()
1742 * socket send timeout, requesting a ping, and bumping ko-count in in request_timer_fn()
1748 req_peer = connection->req_ack_pending; in request_timer_fn()
1754 req_peer = connection->req_not_net_done; in request_timer_fn()
1757 if (req_peer && req_peer->device != device) in request_timer_fn()
1766 ? ( time_before(req_write->pre_submit_jif, req_read->pre_submit_jif) in request_timer_fn()
1767 ? req_write->pre_submit_jif : req_read->pre_submit_jif ) in request_timer_fn()
1768 : req_write ? req_write->pre_submit_jif in request_timer_fn()
1769 : req_read ? req_read->pre_submit_jif : now; in request_timer_fn()
1776 !time_in_range(now, device->last_reattach_jif, device->last_reattach_jif + dt)) { in request_timer_fn()
1777 drbd_warn(device, "Local backing device failed to meet the disk-timeout\n"); in request_timer_fn()
1782 * Fallback to now + min(effective network timeout, disk timeout). */ in request_timer_fn()
1783 ent = (ent && req_peer && time_before(now, req_peer->pre_send_jif + ent)) in request_timer_fn()
1784 ? req_peer->pre_send_jif + ent : now + et; in request_timer_fn()
1789 spin_unlock_irq(&device->resource->req_lock); in request_timer_fn()
1790 mod_timer(&device->request_timer, nt); in request_timer_fn()