Lines Matching +full:dout +full:- +full:default
1 // SPDX-License-Identifier: GPL-2.0
71 WARN_ON(!rwsem_is_locked(&osdc->lock)); in verify_osdc_locked()
75 WARN_ON(!rwsem_is_wrlocked(&osdc->lock)); in verify_osdc_wrlocked()
79 struct ceph_osd_client *osdc = osd->o_osdc; in verify_osd_locked()
81 WARN_ON(!(mutex_is_locked(&osd->lock) && in verify_osd_locked()
82 rwsem_is_locked(&osdc->lock)) && in verify_osd_locked()
83 !rwsem_is_wrlocked(&osdc->lock)); in verify_osd_locked()
87 WARN_ON(!mutex_is_locked(&lreq->lock)); in verify_lreq_locked()
115 dout(" skipping last %llu, final file extent %llu~%llu\n", in calc_layout()
116 orig_len - *plen, off, *plen); in calc_layout()
119 dout("calc_layout objnum=%llx %llu~%llu\n", *objnum, *objoff, *objlen); in calc_layout()
126 osd_data->type = CEPH_OSD_DATA_TYPE_NONE; in ceph_osd_data_init()
136 osd_data->type = CEPH_OSD_DATA_TYPE_PAGES; in ceph_osd_data_pages_init()
137 osd_data->pages = pages; in ceph_osd_data_pages_init()
138 osd_data->length = length; in ceph_osd_data_pages_init()
139 osd_data->alignment = alignment; in ceph_osd_data_pages_init()
140 osd_data->pages_from_pool = pages_from_pool; in ceph_osd_data_pages_init()
141 osd_data->own_pages = own_pages; in ceph_osd_data_pages_init()
150 osd_data->type = CEPH_OSD_DATA_TYPE_PAGELIST; in ceph_osd_data_pagelist_init()
151 osd_data->pagelist = pagelist; in ceph_osd_data_pagelist_init()
159 osd_data->type = CEPH_OSD_DATA_TYPE_BIO; in ceph_osd_data_bio_init()
160 osd_data->bio_pos = *bio_pos; in ceph_osd_data_bio_init()
161 osd_data->bio_length = bio_length; in ceph_osd_data_bio_init()
169 osd_data->type = CEPH_OSD_DATA_TYPE_BVECS; in ceph_osd_data_bvecs_init()
170 osd_data->bvec_pos = *bvec_pos; in ceph_osd_data_bvecs_init()
171 osd_data->num_bvecs = num_bvecs; in ceph_osd_data_bvecs_init()
177 osd_data->type = CEPH_OSD_DATA_TYPE_ITER; in ceph_osd_iter_init()
178 osd_data->iter = *iter; in ceph_osd_iter_init()
184 BUG_ON(which >= osd_req->r_num_ops); in osd_req_op_raw_data_in()
186 return &osd_req->r_ops[which].raw_data_in; in osd_req_op_raw_data_in()
275 * osd_req_op_extent_osd_iter - Set up an operation with an iterator buffer
308 osd_req->r_ops[which].cls.indata_len += pagelist->length; in osd_req_op_cls_request_data_pagelist()
309 osd_req->r_ops[which].indata_len += pagelist->length; in osd_req_op_cls_request_data_pagelist()
322 osd_req->r_ops[which].cls.indata_len += length; in osd_req_op_cls_request_data_pages()
323 osd_req->r_ops[which].indata_len += length; in osd_req_op_cls_request_data_pages()
340 osd_req->r_ops[which].cls.indata_len += bytes; in osd_req_op_cls_request_data_bvecs()
341 osd_req->r_ops[which].indata_len += bytes; in osd_req_op_cls_request_data_bvecs()
359 switch (osd_data->type) { in ceph_osd_data_length()
363 return osd_data->length; in ceph_osd_data_length()
365 return (u64)osd_data->pagelist->length; in ceph_osd_data_length()
368 return (u64)osd_data->bio_length; in ceph_osd_data_length()
371 return osd_data->bvec_pos.iter.bi_size; in ceph_osd_data_length()
373 return iov_iter_count(&osd_data->iter); in ceph_osd_data_length()
374 default: in ceph_osd_data_length()
375 WARN(true, "unrecognized data type %d\n", (int)osd_data->type); in ceph_osd_data_length()
382 if (osd_data->type == CEPH_OSD_DATA_TYPE_PAGES && osd_data->own_pages) { in ceph_osd_data_release()
385 num_pages = calc_pages_for((u64)osd_data->alignment, in ceph_osd_data_release()
386 (u64)osd_data->length); in ceph_osd_data_release()
387 ceph_release_page_vector(osd_data->pages, num_pages); in ceph_osd_data_release()
388 } else if (osd_data->type == CEPH_OSD_DATA_TYPE_PAGELIST) { in ceph_osd_data_release()
389 ceph_pagelist_release(osd_data->pagelist); in ceph_osd_data_release()
399 BUG_ON(which >= osd_req->r_num_ops); in osd_req_op_data_release()
400 op = &osd_req->r_ops[which]; in osd_req_op_data_release()
402 switch (op->op) { in osd_req_op_data_release()
407 kfree(op->extent.sparse_ext); in osd_req_op_data_release()
408 ceph_osd_data_release(&op->extent.osd_data); in osd_req_op_data_release()
411 ceph_osd_data_release(&op->cls.request_info); in osd_req_op_data_release()
412 ceph_osd_data_release(&op->cls.request_data); in osd_req_op_data_release()
413 ceph_osd_data_release(&op->cls.response_data); in osd_req_op_data_release()
417 ceph_osd_data_release(&op->xattr.osd_data); in osd_req_op_data_release()
420 ceph_osd_data_release(&op->raw_data_in); in osd_req_op_data_release()
423 ceph_osd_data_release(&op->notify_ack.request_data); in osd_req_op_data_release()
426 ceph_osd_data_release(&op->notify.request_data); in osd_req_op_data_release()
427 ceph_osd_data_release(&op->notify.response_data); in osd_req_op_data_release()
430 ceph_osd_data_release(&op->list_watchers.response_data); in osd_req_op_data_release()
433 ceph_osd_data_release(&op->copy_from.osd_data); in osd_req_op_data_release()
435 default: in osd_req_op_data_release()
441 * Assumes @t is zero-initialized.
445 ceph_oid_init(&t->base_oid); in target_init()
446 ceph_oloc_init(&t->base_oloc); in target_init()
447 ceph_oid_init(&t->target_oid); in target_init()
448 ceph_oloc_init(&t->target_oloc); in target_init()
450 ceph_osds_init(&t->acting); in target_init()
451 ceph_osds_init(&t->up); in target_init()
452 t->size = -1; in target_init()
453 t->min_size = -1; in target_init()
455 t->osd = CEPH_HOMELESS_OSD; in target_init()
461 ceph_oid_copy(&dest->base_oid, &src->base_oid); in target_copy()
462 ceph_oloc_copy(&dest->base_oloc, &src->base_oloc); in target_copy()
463 ceph_oid_copy(&dest->target_oid, &src->target_oid); in target_copy()
464 ceph_oloc_copy(&dest->target_oloc, &src->target_oloc); in target_copy()
466 dest->pgid = src->pgid; /* struct */ in target_copy()
467 dest->spgid = src->spgid; /* struct */ in target_copy()
468 dest->pg_num = src->pg_num; in target_copy()
469 dest->pg_num_mask = src->pg_num_mask; in target_copy()
470 ceph_osds_copy(&dest->acting, &src->acting); in target_copy()
471 ceph_osds_copy(&dest->up, &src->up); in target_copy()
472 dest->size = src->size; in target_copy()
473 dest->min_size = src->min_size; in target_copy()
474 dest->sort_bitwise = src->sort_bitwise; in target_copy()
475 dest->recovery_deletes = src->recovery_deletes; in target_copy()
477 dest->flags = src->flags; in target_copy()
478 dest->used_replica = src->used_replica; in target_copy()
479 dest->paused = src->paused; in target_copy()
481 dest->epoch = src->epoch; in target_copy()
482 dest->last_force_resend = src->last_force_resend; in target_copy()
484 dest->osd = src->osd; in target_copy()
489 ceph_oid_destroy(&t->base_oid); in target_destroy()
490 ceph_oloc_destroy(&t->base_oloc); in target_destroy()
491 ceph_oid_destroy(&t->target_oid); in target_destroy()
492 ceph_oloc_destroy(&t->target_oloc); in target_destroy()
500 WARN_ON(!RB_EMPTY_NODE(&req->r_node)); in request_release_checks()
501 WARN_ON(!RB_EMPTY_NODE(&req->r_mc_node)); in request_release_checks()
502 WARN_ON(!list_empty(&req->r_private_item)); in request_release_checks()
503 WARN_ON(req->r_osd); in request_release_checks()
512 dout("%s %p (r_request %p r_reply %p)\n", __func__, req, in ceph_osdc_release_request()
513 req->r_request, req->r_reply); in ceph_osdc_release_request()
516 if (req->r_request) in ceph_osdc_release_request()
517 ceph_msg_put(req->r_request); in ceph_osdc_release_request()
518 if (req->r_reply) in ceph_osdc_release_request()
519 ceph_msg_put(req->r_reply); in ceph_osdc_release_request()
521 for (which = 0; which < req->r_num_ops; which++) in ceph_osdc_release_request()
524 target_destroy(&req->r_t); in ceph_osdc_release_request()
525 ceph_put_snap_context(req->r_snapc); in ceph_osdc_release_request()
527 if (req->r_mempool) in ceph_osdc_release_request()
528 mempool_free(req, req->r_osdc->req_mempool); in ceph_osdc_release_request()
529 else if (req->r_num_ops <= CEPH_OSD_SLAB_OPS) in ceph_osdc_release_request()
537 dout("%s %p (was %d)\n", __func__, req, in ceph_osdc_get_request()
538 kref_read(&req->r_kref)); in ceph_osdc_get_request()
539 kref_get(&req->r_kref); in ceph_osdc_get_request()
546 dout("%s %p (was %d)\n", __func__, req, in ceph_osdc_put_request()
547 kref_read(&req->r_kref)); in ceph_osdc_put_request()
548 kref_put(&req->r_kref, ceph_osdc_release_request); in ceph_osdc_put_request()
558 kref_init(&req->r_kref); in request_init()
559 init_completion(&req->r_completion); in request_init()
560 RB_CLEAR_NODE(&req->r_node); in request_init()
561 RB_CLEAR_NODE(&req->r_mc_node); in request_init()
562 INIT_LIST_HEAD(&req->r_private_item); in request_init()
564 target_init(&req->r_t); in request_init()
577 req = mempool_alloc(osdc->req_mempool, gfp_flags); in ceph_osdc_alloc_request()
588 req->r_osdc = osdc; in ceph_osdc_alloc_request()
589 req->r_mempool = use_mempool; in ceph_osdc_alloc_request()
590 req->r_num_ops = num_ops; in ceph_osdc_alloc_request()
591 req->r_snapid = CEPH_NOSNAP; in ceph_osdc_alloc_request()
592 req->r_snapc = ceph_get_snap_context(snapc); in ceph_osdc_alloc_request()
594 dout("%s req %p\n", __func__, req); in ceph_osdc_alloc_request()
601 return 8 + 4 + 4 + 4 + (oloc->pool_ns ? oloc->pool_ns->len : 0); in ceph_oloc_encoding_size()
608 struct ceph_osd_client *osdc = req->r_osdc; in __ceph_osdc_alloc_messages()
612 WARN_ON(req->r_request || req->r_reply); in __ceph_osdc_alloc_messages()
613 WARN_ON(ceph_oid_empty(&req->r_base_oid)); in __ceph_osdc_alloc_messages()
614 WARN_ON(ceph_oloc_empty(&req->r_base_oloc)); in __ceph_osdc_alloc_messages()
625 ceph_oloc_encoding_size(&req->r_base_oloc); /* oloc */ in __ceph_osdc_alloc_messages()
626 msg_size += 4 + req->r_base_oid.name_len; /* oid */ in __ceph_osdc_alloc_messages()
627 msg_size += 2 + req->r_num_ops * sizeof(struct ceph_osd_op); in __ceph_osdc_alloc_messages()
630 msg_size += 4 + 8 * (req->r_snapc ? req->r_snapc->num_snaps : 0); in __ceph_osdc_alloc_messages()
633 if (req->r_mempool) in __ceph_osdc_alloc_messages()
634 msg = ceph_msgpool_get(&osdc->msgpool_op, msg_size, in __ceph_osdc_alloc_messages()
640 return -ENOMEM; in __ceph_osdc_alloc_messages()
642 memset(msg->front.iov_base, 0, msg->front.iov_len); in __ceph_osdc_alloc_messages()
643 req->r_request = msg; in __ceph_osdc_alloc_messages()
647 msg_size += req->r_base_oid.name_len; in __ceph_osdc_alloc_messages()
648 msg_size += req->r_num_ops * sizeof(struct ceph_osd_op); in __ceph_osdc_alloc_messages()
650 if (req->r_mempool) in __ceph_osdc_alloc_messages()
651 msg = ceph_msgpool_get(&osdc->msgpool_op_reply, msg_size, in __ceph_osdc_alloc_messages()
657 return -ENOMEM; in __ceph_osdc_alloc_messages()
659 req->r_reply = msg; in __ceph_osdc_alloc_messages()
670 default: in osd_req_opcode_valid()
684 for (op = req->r_ops; op != &req->r_ops[req->r_num_ops]; op++) { in get_num_data_items()
685 switch (op->op) { in get_num_data_items()
714 default: in get_num_data_items()
715 WARN_ON(!osd_req_opcode_valid(op->op)); in get_num_data_items()
746 BUG_ON(which >= osd_req->r_num_ops); in osd_req_op_init()
749 op = &osd_req->r_ops[which]; in osd_req_op_init()
751 op->op = opcode; in osd_req_op_init()
752 op->flags = flags; in osd_req_op_init()
771 op->extent.offset = offset; in osd_req_op_extent_init()
772 op->extent.length = length; in osd_req_op_extent_init()
773 op->extent.truncate_size = truncate_size; in osd_req_op_extent_init()
774 op->extent.truncate_seq = truncate_seq; in osd_req_op_extent_init()
778 op->indata_len = payload_len; in osd_req_op_extent_init()
788 BUG_ON(which >= osd_req->r_num_ops); in osd_req_op_extent_update()
789 op = &osd_req->r_ops[which]; in osd_req_op_extent_update()
790 previous = op->extent.length; in osd_req_op_extent_update()
796 op->extent.length = length; in osd_req_op_extent_update()
797 if (op->op == CEPH_OSD_OP_WRITE || op->op == CEPH_OSD_OP_WRITEFULL) in osd_req_op_extent_update()
798 op->indata_len -= previous - length; in osd_req_op_extent_update()
807 BUG_ON(which + 1 >= osd_req->r_num_ops); in osd_req_op_extent_dup_last()
809 prev_op = &osd_req->r_ops[which]; in osd_req_op_extent_dup_last()
810 op = osd_req_op_init(osd_req, which + 1, prev_op->op, prev_op->flags); in osd_req_op_extent_dup_last()
812 op->indata_len = prev_op->indata_len; in osd_req_op_extent_dup_last()
813 op->outdata_len = prev_op->outdata_len; in osd_req_op_extent_dup_last()
814 op->extent = prev_op->extent; in osd_req_op_extent_dup_last()
816 op->extent.offset += offset_inc; in osd_req_op_extent_dup_last()
817 op->extent.length -= offset_inc; in osd_req_op_extent_dup_last()
819 if (op->op == CEPH_OSD_OP_WRITE || op->op == CEPH_OSD_OP_WRITEFULL) in osd_req_op_extent_dup_last()
820 op->indata_len -= offset_inc; in osd_req_op_extent_dup_last()
837 return -ENOMEM; in osd_req_op_cls_init()
839 op->cls.class_name = class; in osd_req_op_cls_init()
842 op->cls.class_len = size; in osd_req_op_cls_init()
848 op->cls.method_name = method; in osd_req_op_cls_init()
851 op->cls.method_len = size; in osd_req_op_cls_init()
858 op->indata_len = payload_len; in osd_req_op_cls_init()
881 return -ENOMEM; in osd_req_op_xattr_init()
884 op->xattr.name_len = payload_len; in osd_req_op_xattr_init()
889 op->xattr.value_len = size; in osd_req_op_xattr_init()
895 op->xattr.cmp_op = cmp_op; in osd_req_op_xattr_init()
896 op->xattr.cmp_mode = cmp_mode; in osd_req_op_xattr_init()
898 ceph_osd_data_pagelist_init(&op->xattr.osd_data, pagelist); in osd_req_op_xattr_init()
899 op->indata_len = payload_len; in osd_req_op_xattr_init()
917 op->watch.cookie = cookie; in osd_req_op_watch_init()
918 op->watch.op = watch_opcode; in osd_req_op_watch_init()
919 op->watch.gen = gen; in osd_req_op_watch_init()
932 op->notify.cookie = cookie; in osd_req_op_notify_init()
934 ceph_osd_data_pagelist_init(&op->notify.request_data, request_pl); in osd_req_op_notify_init()
935 op->indata_len = request_pl->length; in osd_req_op_notify_init()
950 op->alloc_hint.expected_object_size = expected_object_size; in osd_req_op_alloc_hint_init()
951 op->alloc_hint.expected_write_size = expected_write_size; in osd_req_op_alloc_hint_init()
952 op->alloc_hint.flags = flags; in osd_req_op_alloc_hint_init()
956 * not worth a feature bit. Set FAILOK per-op flag to make in osd_req_op_alloc_hint_init()
959 op->flags |= CEPH_OSD_OP_FLAG_FAILOK; in osd_req_op_alloc_hint_init()
968 if (osd_data->type == CEPH_OSD_DATA_TYPE_PAGES) { in ceph_osdc_msg_data_add()
971 ceph_msg_data_add_pages(msg, osd_data->pages, in ceph_osdc_msg_data_add()
972 length, osd_data->alignment, false); in ceph_osdc_msg_data_add()
973 } else if (osd_data->type == CEPH_OSD_DATA_TYPE_PAGELIST) { in ceph_osdc_msg_data_add()
975 ceph_msg_data_add_pagelist(msg, osd_data->pagelist); in ceph_osdc_msg_data_add()
977 } else if (osd_data->type == CEPH_OSD_DATA_TYPE_BIO) { in ceph_osdc_msg_data_add()
978 ceph_msg_data_add_bio(msg, &osd_data->bio_pos, length); in ceph_osdc_msg_data_add()
980 } else if (osd_data->type == CEPH_OSD_DATA_TYPE_BVECS) { in ceph_osdc_msg_data_add()
981 ceph_msg_data_add_bvecs(msg, &osd_data->bvec_pos); in ceph_osdc_msg_data_add()
982 } else if (osd_data->type == CEPH_OSD_DATA_TYPE_ITER) { in ceph_osdc_msg_data_add()
983 ceph_msg_data_add_iter(msg, &osd_data->iter); in ceph_osdc_msg_data_add()
985 BUG_ON(osd_data->type != CEPH_OSD_DATA_TYPE_NONE); in ceph_osdc_msg_data_add()
992 switch (src->op) { in osd_req_encode_op()
1001 dst->extent.offset = cpu_to_le64(src->extent.offset); in osd_req_encode_op()
1002 dst->extent.length = cpu_to_le64(src->extent.length); in osd_req_encode_op()
1003 dst->extent.truncate_size = in osd_req_encode_op()
1004 cpu_to_le64(src->extent.truncate_size); in osd_req_encode_op()
1005 dst->extent.truncate_seq = in osd_req_encode_op()
1006 cpu_to_le32(src->extent.truncate_seq); in osd_req_encode_op()
1009 dst->cls.class_len = src->cls.class_len; in osd_req_encode_op()
1010 dst->cls.method_len = src->cls.method_len; in osd_req_encode_op()
1011 dst->cls.indata_len = cpu_to_le32(src->cls.indata_len); in osd_req_encode_op()
1014 dst->watch.cookie = cpu_to_le64(src->watch.cookie); in osd_req_encode_op()
1015 dst->watch.ver = cpu_to_le64(0); in osd_req_encode_op()
1016 dst->watch.op = src->watch.op; in osd_req_encode_op()
1017 dst->watch.gen = cpu_to_le32(src->watch.gen); in osd_req_encode_op()
1022 dst->notify.cookie = cpu_to_le64(src->notify.cookie); in osd_req_encode_op()
1027 dst->alloc_hint.expected_object_size = in osd_req_encode_op()
1028 cpu_to_le64(src->alloc_hint.expected_object_size); in osd_req_encode_op()
1029 dst->alloc_hint.expected_write_size = in osd_req_encode_op()
1030 cpu_to_le64(src->alloc_hint.expected_write_size); in osd_req_encode_op()
1031 dst->alloc_hint.flags = cpu_to_le32(src->alloc_hint.flags); in osd_req_encode_op()
1035 dst->xattr.name_len = cpu_to_le32(src->xattr.name_len); in osd_req_encode_op()
1036 dst->xattr.value_len = cpu_to_le32(src->xattr.value_len); in osd_req_encode_op()
1037 dst->xattr.cmp_op = src->xattr.cmp_op; in osd_req_encode_op()
1038 dst->xattr.cmp_mode = src->xattr.cmp_mode; in osd_req_encode_op()
1044 dst->copy_from.snapid = cpu_to_le64(src->copy_from.snapid); in osd_req_encode_op()
1045 dst->copy_from.src_version = in osd_req_encode_op()
1046 cpu_to_le64(src->copy_from.src_version); in osd_req_encode_op()
1047 dst->copy_from.flags = src->copy_from.flags; in osd_req_encode_op()
1048 dst->copy_from.src_fadvise_flags = in osd_req_encode_op()
1049 cpu_to_le32(src->copy_from.src_fadvise_flags); in osd_req_encode_op()
1052 dst->assert_ver.unused = cpu_to_le64(0); in osd_req_encode_op()
1053 dst->assert_ver.ver = cpu_to_le64(src->assert_ver.ver); in osd_req_encode_op()
1055 default: in osd_req_encode_op()
1057 ceph_osd_op_name(src->op)); in osd_req_encode_op()
1063 dst->op = cpu_to_le16(src->op); in osd_req_encode_op()
1064 dst->flags = cpu_to_le32(src->flags); in osd_req_encode_op()
1065 dst->payload_len = cpu_to_le32(src->indata_len); in osd_req_encode_op()
1067 return src->indata_len; in osd_req_encode_op()
1103 r = -ENOMEM; in ceph_osdc_new_request()
1115 u32 object_size = layout->object_size; in ceph_osdc_new_request()
1116 u32 object_base = off - objoff; in ceph_osdc_new_request()
1117 if (!(truncate_seq == 1 && truncate_size == -1ULL)) { in ceph_osdc_new_request()
1121 truncate_size -= object_base; in ceph_osdc_new_request()
1130 req->r_base_oloc.pool = layout->pool_id; in ceph_osdc_new_request()
1131 req->r_base_oloc.pool_ns = ceph_try_get_string(layout->pool_ns); in ceph_osdc_new_request()
1132 ceph_oid_printf(&req->r_base_oid, "%llx.%08llx", vino.ino, objnum); in ceph_osdc_new_request()
1133 req->r_flags = flags | osdc->client->options->read_from_replica; in ceph_osdc_new_request()
1135 req->r_snapid = vino.snap; in ceph_osdc_new_request()
1137 req->r_data_offset = off; in ceph_osdc_new_request()
1143 * If this is a multi-op write request, assume that we'll need in ceph_osdc_new_request()
1144 * request ops. If it's a multi-op read then assume we'll need in ceph_osdc_new_request()
1145 * reply ops. Anything else and call it -EINVAL. in ceph_osdc_new_request()
1154 r = -EINVAL; in ceph_osdc_new_request()
1176 op->extent.sparse_ext_cnt = cnt; in __ceph_alloc_sparse_ext_map()
1177 op->extent.sparse_ext = kmalloc_array(cnt, in __ceph_alloc_sparse_ext_map()
1178 sizeof(*op->extent.sparse_ext), in __ceph_alloc_sparse_ext_map()
1180 if (!op->extent.sparse_ext) in __ceph_alloc_sparse_ext_map()
1181 return -ENOMEM; in __ceph_alloc_sparse_ext_map()
1187 * We keep osd requests in an rbtree, sorted by ->r_tid.
1201 for (n = rb_first(&osdc->osds); n; n = rb_next(n)) { in DEFINE_RB_FUNCS()
1204 for (p = rb_first(&osd->o_requests); p; ) { in DEFINE_RB_FUNCS()
1214 for (p = rb_first(&osdc->homeless_osd.o_requests); p; ) { in DEFINE_RB_FUNCS()
1226 return osd->o_osd == CEPH_HOMELESS_OSD; in osd_homeless()
1231 verify_osdc_locked(osd->o_osdc); in osd_registered()
1233 return !RB_EMPTY_NODE(&osd->o_node); in osd_registered()
1237 * Assumes @osd is zero-initialized.
1241 refcount_set(&osd->o_ref, 1); in osd_init()
1242 RB_CLEAR_NODE(&osd->o_node); in osd_init()
1243 spin_lock_init(&osd->o_requests_lock); in osd_init()
1244 osd->o_requests = RB_ROOT; in osd_init()
1245 osd->o_linger_requests = RB_ROOT; in osd_init()
1246 osd->o_backoff_mappings = RB_ROOT; in osd_init()
1247 osd->o_backoffs_by_id = RB_ROOT; in osd_init()
1248 INIT_LIST_HEAD(&osd->o_osd_lru); in osd_init()
1249 INIT_LIST_HEAD(&osd->o_keepalive_item); in osd_init()
1250 osd->o_incarnation = 1; in osd_init()
1251 mutex_init(&osd->lock); in osd_init()
1256 kfree(sr->sr_extent); in ceph_init_sparse_read()
1258 sr->sr_state = CEPH_SPARSE_READ_HDR; in ceph_init_sparse_read()
1263 WARN_ON(!RB_EMPTY_NODE(&osd->o_node)); in osd_cleanup()
1264 WARN_ON(!RB_EMPTY_ROOT(&osd->o_requests)); in osd_cleanup()
1265 WARN_ON(!RB_EMPTY_ROOT(&osd->o_linger_requests)); in osd_cleanup()
1266 WARN_ON(!RB_EMPTY_ROOT(&osd->o_backoff_mappings)); in osd_cleanup()
1267 WARN_ON(!RB_EMPTY_ROOT(&osd->o_backoffs_by_id)); in osd_cleanup()
1268 WARN_ON(!list_empty(&osd->o_osd_lru)); in osd_cleanup()
1269 WARN_ON(!list_empty(&osd->o_keepalive_item)); in osd_cleanup()
1271 ceph_init_sparse_read(&osd->o_sparse_read); in osd_cleanup()
1273 if (osd->o_auth.authorizer) { in osd_cleanup()
1275 ceph_auth_destroy_authorizer(osd->o_auth.authorizer); in osd_cleanup()
1290 osd->o_osdc = osdc; in create_osd()
1291 osd->o_osd = onum; in create_osd()
1292 osd->o_sparse_op_idx = -1; in create_osd()
1294 ceph_init_sparse_read(&osd->o_sparse_read); in create_osd()
1296 ceph_con_init(&osd->o_con, osd, &osd_con_ops, &osdc->client->msgr); in create_osd()
1303 if (refcount_inc_not_zero(&osd->o_ref)) { in get_osd()
1304 dout("get_osd %p %d -> %d\n", osd, refcount_read(&osd->o_ref)-1, in get_osd()
1305 refcount_read(&osd->o_ref)); in get_osd()
1308 dout("get_osd %p FAIL\n", osd); in get_osd()
1315 dout("put_osd %p %d -> %d\n", osd, refcount_read(&osd->o_ref), in put_osd()
1316 refcount_read(&osd->o_ref) - 1); in put_osd()
1317 if (refcount_dec_and_test(&osd->o_ref)) { in put_osd()
1327 struct ceph_osd_client *osdc = osd->o_osdc; in DEFINE_RB_FUNCS()
1329 dout("%s osd %p osd%d\n", __func__, osd, osd->o_osd); in DEFINE_RB_FUNCS()
1330 BUG_ON(!list_empty(&osd->o_osd_lru)); in DEFINE_RB_FUNCS()
1332 spin_lock(&osdc->osd_lru_lock); in DEFINE_RB_FUNCS()
1333 list_add_tail(&osd->o_osd_lru, &osdc->osd_lru); in DEFINE_RB_FUNCS()
1334 spin_unlock(&osdc->osd_lru_lock); in DEFINE_RB_FUNCS()
1336 osd->lru_ttl = jiffies + osdc->client->options->osd_idle_ttl; in DEFINE_RB_FUNCS()
1341 if (RB_EMPTY_ROOT(&osd->o_requests) && in maybe_move_osd_to_lru()
1342 RB_EMPTY_ROOT(&osd->o_linger_requests)) in maybe_move_osd_to_lru()
1348 struct ceph_osd_client *osdc = osd->o_osdc; in __remove_osd_from_lru()
1350 dout("%s osd %p osd%d\n", __func__, osd, osd->o_osd); in __remove_osd_from_lru()
1352 spin_lock(&osdc->osd_lru_lock); in __remove_osd_from_lru()
1353 if (!list_empty(&osd->o_osd_lru)) in __remove_osd_from_lru()
1354 list_del_init(&osd->o_osd_lru); in __remove_osd_from_lru()
1355 spin_unlock(&osdc->osd_lru_lock); in __remove_osd_from_lru()
1364 struct ceph_osd_client *osdc = osd->o_osdc; in close_osd()
1368 dout("%s osd %p osd%d\n", __func__, osd, osd->o_osd); in close_osd()
1370 ceph_con_close(&osd->o_con); in close_osd()
1372 for (n = rb_first(&osd->o_requests); n; ) { in close_osd()
1378 dout(" reassigning req %p tid %llu\n", req, req->r_tid); in close_osd()
1380 link_request(&osdc->homeless_osd, req); in close_osd()
1382 for (n = rb_first(&osd->o_linger_requests); n; ) { in close_osd()
1388 dout(" reassigning lreq %p linger_id %llu\n", lreq, in close_osd()
1389 lreq->linger_id); in close_osd()
1391 link_linger(&osdc->homeless_osd, lreq); in close_osd()
1396 erase_osd(&osdc->osds, osd); in close_osd()
1407 dout("%s osd %p osd%d\n", __func__, osd, osd->o_osd); in reopen_osd()
1409 if (RB_EMPTY_ROOT(&osd->o_requests) && in reopen_osd()
1410 RB_EMPTY_ROOT(&osd->o_linger_requests)) { in reopen_osd()
1412 return -ENODEV; in reopen_osd()
1415 peer_addr = &osd->o_osdc->osdmap->osd_addr[osd->o_osd]; in reopen_osd()
1416 if (!memcmp(peer_addr, &osd->o_con.peer_addr, sizeof (*peer_addr)) && in reopen_osd()
1417 !ceph_con_opened(&osd->o_con)) { in reopen_osd()
1420 dout("osd addr hasn't changed and connection never opened, " in reopen_osd()
1423 for (n = rb_first(&osd->o_requests); n; n = rb_next(n)) { in reopen_osd()
1426 req->r_stamp = jiffies; in reopen_osd()
1429 return -EAGAIN; in reopen_osd()
1432 ceph_con_close(&osd->o_con); in reopen_osd()
1433 ceph_con_open(&osd->o_con, CEPH_ENTITY_TYPE_OSD, osd->o_osd, peer_addr); in reopen_osd()
1434 osd->o_incarnation++; in reopen_osd()
1450 osd = lookup_osd(&osdc->osds, o); in lookup_create_osd()
1452 osd = &osdc->homeless_osd; in lookup_create_osd()
1455 return ERR_PTR(-EAGAIN); in lookup_create_osd()
1458 insert_osd(&osdc->osds, osd); in lookup_create_osd()
1459 ceph_con_open(&osd->o_con, CEPH_ENTITY_TYPE_OSD, osd->o_osd, in lookup_create_osd()
1460 &osdc->osdmap->osd_addr[osd->o_osd]); in lookup_create_osd()
1463 dout("%s osdc %p osd%d -> osd %p\n", __func__, osdc, o, osd); in lookup_create_osd()
1468 * Create request <-> OSD session relation.
1475 WARN_ON(!req->r_tid || req->r_osd); in link_request()
1476 dout("%s osd %p osd%d req %p tid %llu\n", __func__, osd, osd->o_osd, in link_request()
1477 req, req->r_tid); in link_request()
1482 atomic_inc(&osd->o_osdc->num_homeless); in link_request()
1485 spin_lock(&osd->o_requests_lock); in link_request()
1486 insert_request(&osd->o_requests, req); in link_request()
1487 spin_unlock(&osd->o_requests_lock); in link_request()
1488 req->r_osd = osd; in link_request()
1494 WARN_ON(req->r_osd != osd); in unlink_request()
1495 dout("%s osd %p osd%d req %p tid %llu\n", __func__, osd, osd->o_osd, in unlink_request()
1496 req, req->r_tid); in unlink_request()
1498 req->r_osd = NULL; in unlink_request()
1499 spin_lock(&osd->o_requests_lock); in unlink_request()
1500 erase_request(&osd->o_requests, req); in unlink_request()
1501 spin_unlock(&osd->o_requests_lock); in unlink_request()
1507 atomic_dec(&osd->o_osdc->num_homeless); in unlink_request()
1512 return pi->flags & CEPH_POOL_FLAG_FULL; in __pool_full()
1519 for (n = rb_first(&osdc->osdmap->pg_pools); n; n = rb_next(n)) { in have_pool_full()
1534 pi = ceph_pg_pool_by_id(osdc->osdmap, pool_id); in pool_full()
1554 WARN_ON(pi->id != t->target_oloc.pool); in target_should_be_paused()
1555 return ((t->flags & CEPH_OSD_FLAG_READ) && pauserd) || in target_should_be_paused()
1556 ((t->flags & CEPH_OSD_FLAG_WRITE) && pausewr) || in target_should_be_paused()
1557 (osdc->osdmap->epoch < osdc->epoch_barrier); in target_should_be_paused()
1562 int i = get_random_u32_below(acting->size); in pick_random_replica()
1564 dout("%s picked osd%d, primary osd%d\n", __func__, in pick_random_replica()
1565 acting->osds[i], acting->primary); in pick_random_replica()
1577 struct ceph_options *opt = osdc->client->options; in pick_closest_replica()
1582 locality = ceph_get_crush_locality(osdc->osdmap, in pick_closest_replica()
1583 acting->osds[i], in pick_closest_replica()
1584 &opt->crush_locs); in pick_closest_replica()
1592 } while (++i < acting->size); in pick_closest_replica()
1594 dout("%s picked osd%d with locality %d, primary osd%d\n", __func__, in pick_closest_replica()
1595 acting->osds[best_i], best_locality, acting->primary); in pick_closest_replica()
1612 bool is_read = t->flags & CEPH_OSD_FLAG_READ; in calc_target()
1613 bool is_write = t->flags & CEPH_OSD_FLAG_WRITE; in calc_target()
1623 t->epoch = osdc->osdmap->epoch; in calc_target()
1624 pi = ceph_pg_pool_by_id(osdc->osdmap, t->base_oloc.pool); in calc_target()
1626 t->osd = CEPH_HOMELESS_OSD; in calc_target()
1631 if (osdc->osdmap->epoch == pi->last_force_request_resend) { in calc_target()
1632 if (t->last_force_resend < pi->last_force_request_resend) { in calc_target()
1633 t->last_force_resend = pi->last_force_request_resend; in calc_target()
1635 } else if (t->last_force_resend == 0) { in calc_target()
1641 ceph_oid_copy(&t->target_oid, &t->base_oid); in calc_target()
1642 ceph_oloc_copy(&t->target_oloc, &t->base_oloc); in calc_target()
1643 if ((t->flags & CEPH_OSD_FLAG_IGNORE_OVERLAY) == 0) { in calc_target()
1644 if (is_read && pi->read_tier >= 0) in calc_target()
1645 t->target_oloc.pool = pi->read_tier; in calc_target()
1646 if (is_write && pi->write_tier >= 0) in calc_target()
1647 t->target_oloc.pool = pi->write_tier; in calc_target()
1649 pi = ceph_pg_pool_by_id(osdc->osdmap, t->target_oloc.pool); in calc_target()
1651 t->osd = CEPH_HOMELESS_OSD; in calc_target()
1657 __ceph_object_locator_to_pg(pi, &t->target_oid, &t->target_oloc, &pgid); in calc_target()
1659 last_pgid.seed = ceph_stable_mod(pgid.seed, t->pg_num, t->pg_num_mask); in calc_target()
1661 ceph_pg_to_up_acting_osds(osdc->osdmap, pi, &pgid, &up, &acting); in calc_target()
1663 ceph_is_new_interval(&t->acting, in calc_target()
1665 &t->up, in calc_target()
1667 t->size, in calc_target()
1668 pi->size, in calc_target()
1669 t->min_size, in calc_target()
1670 pi->min_size, in calc_target()
1671 t->pg_num, in calc_target()
1672 pi->pg_num, in calc_target()
1673 t->sort_bitwise, in calc_target()
1675 t->recovery_deletes, in calc_target()
1680 if (t->paused && !target_should_be_paused(osdc, t, pi)) { in calc_target()
1681 t->paused = false; in calc_target()
1684 legacy_change = ceph_pg_compare(&t->pgid, &pgid) || in calc_target()
1685 ceph_osds_changed(&t->acting, &acting, in calc_target()
1686 t->used_replica || any_change); in calc_target()
1687 if (t->pg_num) in calc_target()
1688 split = ceph_pg_is_split(&last_pgid, t->pg_num, pi->pg_num); in calc_target()
1691 t->pgid = pgid; /* struct */ in calc_target()
1692 ceph_pg_to_primary_shard(osdc->osdmap, pi, &pgid, &t->spgid); in calc_target()
1693 ceph_osds_copy(&t->acting, &acting); in calc_target()
1694 ceph_osds_copy(&t->up, &up); in calc_target()
1695 t->size = pi->size; in calc_target()
1696 t->min_size = pi->min_size; in calc_target()
1697 t->pg_num = pi->pg_num; in calc_target()
1698 t->pg_num_mask = pi->pg_num_mask; in calc_target()
1699 t->sort_bitwise = sort_bitwise; in calc_target()
1700 t->recovery_deletes = recovery_deletes; in calc_target()
1702 if ((t->flags & (CEPH_OSD_FLAG_BALANCE_READS | in calc_target()
1704 !is_write && pi->type == CEPH_POOL_TYPE_REP && in calc_target()
1709 if (t->flags & CEPH_OSD_FLAG_BALANCE_READS) { in calc_target()
1714 t->osd = acting.osds[pos]; in calc_target()
1715 t->used_replica = pos > 0; in calc_target()
1717 t->osd = acting.primary; in calc_target()
1718 t->used_replica = false; in calc_target()
1728 dout("%s t %p -> %d%d%d%d ct_res %d osd%d\n", __func__, t, unpaused, in calc_target()
1729 legacy_change, force_resend, split, ct_res, t->osd); in calc_target()
1741 RB_CLEAR_NODE(&spg->node); in alloc_spg_mapping()
1742 spg->backoffs = RB_ROOT; in alloc_spg_mapping()
1748 WARN_ON(!RB_EMPTY_NODE(&spg->node)); in free_spg_mapping()
1749 WARN_ON(!RB_EMPTY_ROOT(&spg->backoffs)); in free_spg_mapping()
1756 * ceph_pg_mapping. Used to track OSD backoffs -- a backoff [range] is
1765 return hoid->is_max ? 0x100000000ull : hoid->hash_reverse_bits; in DEFINE_RB_FUNCS2()
1771 if (hoid->key_len) { in hoid_get_effective_key()
1772 *pkey = hoid->key; in hoid_get_effective_key()
1773 *pkey_len = hoid->key_len; in hoid_get_effective_key()
1775 *pkey = hoid->oid; in hoid_get_effective_key()
1776 *pkey_len = hoid->oid_len; in hoid_get_effective_key()
1788 ret = -1; in compare_names()
1802 if (lhs->is_max < rhs->is_max) in hoid_compare()
1803 return -1; in hoid_compare()
1804 if (lhs->is_max > rhs->is_max) in hoid_compare()
1807 if (lhs->pool < rhs->pool) in hoid_compare()
1808 return -1; in hoid_compare()
1809 if (lhs->pool > rhs->pool) in hoid_compare()
1813 return -1; in hoid_compare()
1817 ret = compare_names(lhs->nspace, lhs->nspace_len, in hoid_compare()
1818 rhs->nspace, rhs->nspace_len); in hoid_compare()
1829 ret = compare_names(lhs->oid, lhs->oid_len, rhs->oid, rhs->oid_len); in hoid_compare()
1833 if (lhs->snapid < rhs->snapid) in hoid_compare()
1834 return -1; in hoid_compare()
1835 if (lhs->snapid > rhs->snapid) in hoid_compare()
1842 * For decoding ->begin and ->end of MOSDBackoff only -- no MIN/MAX
1845 * Assumes @hoid is zero-initialized.
1863 hoid->key = ceph_extract_encoded_string(p, end, &hoid->key_len, in decode_hoid()
1865 if (IS_ERR(hoid->key)) { in decode_hoid()
1866 ret = PTR_ERR(hoid->key); in decode_hoid()
1867 hoid->key = NULL; in decode_hoid()
1871 hoid->oid = ceph_extract_encoded_string(p, end, &hoid->oid_len, in decode_hoid()
1873 if (IS_ERR(hoid->oid)) { in decode_hoid()
1874 ret = PTR_ERR(hoid->oid); in decode_hoid()
1875 hoid->oid = NULL; in decode_hoid()
1879 ceph_decode_64_safe(p, end, hoid->snapid, e_inval); in decode_hoid()
1880 ceph_decode_32_safe(p, end, hoid->hash, e_inval); in decode_hoid()
1881 ceph_decode_8_safe(p, end, hoid->is_max, e_inval); in decode_hoid()
1883 hoid->nspace = ceph_extract_encoded_string(p, end, &hoid->nspace_len, in decode_hoid()
1885 if (IS_ERR(hoid->nspace)) { in decode_hoid()
1886 ret = PTR_ERR(hoid->nspace); in decode_hoid()
1887 hoid->nspace = NULL; in decode_hoid()
1891 ceph_decode_64_safe(p, end, hoid->pool, e_inval); in decode_hoid()
1897 return -EINVAL; in decode_hoid()
1903 4 + hoid->key_len + 4 + hoid->oid_len + 4 + hoid->nspace_len; in hoid_encoding_size()
1909 ceph_encode_string(p, end, hoid->key, hoid->key_len); in encode_hoid()
1910 ceph_encode_string(p, end, hoid->oid, hoid->oid_len); in encode_hoid()
1911 ceph_encode_64(p, hoid->snapid); in encode_hoid()
1912 ceph_encode_32(p, hoid->hash); in encode_hoid()
1913 ceph_encode_8(p, hoid->is_max); in encode_hoid()
1914 ceph_encode_string(p, end, hoid->nspace, hoid->nspace_len); in encode_hoid()
1915 ceph_encode_64(p, hoid->pool); in encode_hoid()
1921 kfree(hoid->key); in free_hoid()
1922 kfree(hoid->oid); in free_hoid()
1923 kfree(hoid->nspace); in free_hoid()
1936 RB_CLEAR_NODE(&backoff->spg_node); in alloc_backoff()
1937 RB_CLEAR_NODE(&backoff->id_node); in alloc_backoff()
1943 WARN_ON(!RB_EMPTY_NODE(&backoff->spg_node)); in free_backoff()
1944 WARN_ON(!RB_EMPTY_NODE(&backoff->id_node)); in free_backoff()
1946 free_hoid(backoff->begin); in free_backoff()
1947 free_hoid(backoff->end); in free_backoff()
1952 * Within a specific spgid, backoffs are managed by ->begin hoid.
1960 struct rb_node *n = root->rb_node; in lookup_containing_backoff()
1967 cmp = hoid_compare(hoid, cur->begin); in lookup_containing_backoff()
1969 n = n->rb_left; in lookup_containing_backoff()
1971 if (hoid_compare(hoid, cur->end) < 0) in lookup_containing_backoff()
1974 n = n->rb_right; in lookup_containing_backoff()
1990 while (!RB_EMPTY_ROOT(&osd->o_backoff_mappings)) { in DEFINE_RB_FUNCS()
1992 rb_entry(rb_first(&osd->o_backoff_mappings), in DEFINE_RB_FUNCS()
1995 while (!RB_EMPTY_ROOT(&spg->backoffs)) { in DEFINE_RB_FUNCS()
1997 rb_entry(rb_first(&spg->backoffs), in DEFINE_RB_FUNCS()
2000 erase_backoff(&spg->backoffs, backoff); in DEFINE_RB_FUNCS()
2001 erase_backoff_by_id(&osd->o_backoffs_by_id, backoff); in DEFINE_RB_FUNCS()
2004 erase_spg_mapping(&osd->o_backoff_mappings, spg); in DEFINE_RB_FUNCS()
2010 * Set up a temporary, non-owning view into @t.
2015 hoid->key = NULL; in hoid_fill_from_target()
2016 hoid->key_len = 0; in hoid_fill_from_target()
2017 hoid->oid = t->target_oid.name; in hoid_fill_from_target()
2018 hoid->oid_len = t->target_oid.name_len; in hoid_fill_from_target()
2019 hoid->snapid = CEPH_NOSNAP; in hoid_fill_from_target()
2020 hoid->hash = t->pgid.seed; in hoid_fill_from_target()
2021 hoid->is_max = false; in hoid_fill_from_target()
2022 if (t->target_oloc.pool_ns) { in hoid_fill_from_target()
2023 hoid->nspace = t->target_oloc.pool_ns->str; in hoid_fill_from_target()
2024 hoid->nspace_len = t->target_oloc.pool_ns->len; in hoid_fill_from_target()
2026 hoid->nspace = NULL; in hoid_fill_from_target()
2027 hoid->nspace_len = 0; in hoid_fill_from_target()
2029 hoid->pool = t->target_oloc.pool; in hoid_fill_from_target()
2035 struct ceph_osd *osd = req->r_osd; in should_plug_request()
2040 spg = lookup_spg_mapping(&osd->o_backoff_mappings, &req->r_t.spgid); in should_plug_request()
2044 hoid_fill_from_target(&hoid, &req->r_t); in should_plug_request()
2045 backoff = lookup_containing_backoff(&spg->backoffs, &hoid); in should_plug_request()
2049 dout("%s req %p tid %llu backoff osd%d spgid %llu.%xs%d id %llu\n", in should_plug_request()
2050 __func__, req, req->r_tid, osd->o_osd, backoff->spgid.pgid.pool, in should_plug_request()
2051 backoff->spgid.pgid.seed, backoff->spgid.shard, backoff->id); in should_plug_request()
2060 struct ceph_msg *request_msg = req->r_request; in setup_request_data()
2061 struct ceph_msg *reply_msg = req->r_reply; in setup_request_data()
2064 if (req->r_request->num_data_items || req->r_reply->num_data_items) in setup_request_data()
2067 WARN_ON(request_msg->data_length || reply_msg->data_length); in setup_request_data()
2068 for (op = req->r_ops; op != &req->r_ops[req->r_num_ops]; op++) { in setup_request_data()
2069 switch (op->op) { in setup_request_data()
2073 WARN_ON(op->indata_len != op->extent.length); in setup_request_data()
2075 &op->extent.osd_data); in setup_request_data()
2079 WARN_ON(op->indata_len != op->xattr.name_len + in setup_request_data()
2080 op->xattr.value_len); in setup_request_data()
2082 &op->xattr.osd_data); in setup_request_data()
2086 &op->notify_ack.request_data); in setup_request_data()
2090 &op->copy_from.osd_data); in setup_request_data()
2096 &op->raw_data_in); in setup_request_data()
2101 &op->extent.osd_data); in setup_request_data()
2105 &op->list_watchers.response_data); in setup_request_data()
2110 WARN_ON(op->indata_len != op->cls.class_len + in setup_request_data()
2111 op->cls.method_len + in setup_request_data()
2112 op->cls.indata_len); in setup_request_data()
2114 &op->cls.request_info); in setup_request_data()
2117 &op->cls.request_data); in setup_request_data()
2120 &op->cls.response_data); in setup_request_data()
2124 &op->notify.request_data); in setup_request_data()
2126 &op->notify.response_data); in setup_request_data()
2135 ceph_encode_64(p, pgid->pool); in encode_pgid()
2136 ceph_encode_32(p, pgid->seed); in encode_pgid()
2137 ceph_encode_32(p, -1); /* preferred */ in encode_pgid()
2143 encode_pgid(p, &spgid->pgid); in encode_spgid()
2144 ceph_encode_8(p, spgid->shard); in encode_spgid()
2151 ceph_encode_64(p, oloc->pool); in encode_oloc()
2152 ceph_encode_32(p, -1); /* preferred */ in encode_oloc()
2154 if (oloc->pool_ns) in encode_oloc()
2155 ceph_encode_string(p, end, oloc->pool_ns->str, in encode_oloc()
2156 oloc->pool_ns->len); in encode_oloc()
2164 void *p = msg->front.iov_base; in encode_request_partial()
2165 void *const end = p + msg->front_alloc_len; in encode_request_partial()
2169 if (req->r_flags & CEPH_OSD_FLAG_WRITE) { in encode_request_partial()
2171 WARN_ON(req->r_snapid != CEPH_NOSNAP); in encode_request_partial()
2173 WARN_ON(req->r_mtime.tv_sec || req->r_mtime.tv_nsec || in encode_request_partial()
2174 req->r_data_offset || req->r_snapc); in encode_request_partial()
2179 encode_spgid(&p, &req->r_t.spgid); /* actual spg */ in encode_request_partial()
2180 ceph_encode_32(&p, req->r_t.pgid.seed); /* raw hash */ in encode_request_partial()
2181 ceph_encode_32(&p, req->r_osdc->osdmap->epoch); in encode_request_partial()
2182 ceph_encode_32(&p, req->r_flags); in encode_request_partial()
2194 ceph_encode_timespec64(p, &req->r_mtime); in encode_request_partial()
2197 encode_oloc(&p, end, &req->r_t.target_oloc); in encode_request_partial()
2198 ceph_encode_string(&p, end, req->r_t.target_oid.name, in encode_request_partial()
2199 req->r_t.target_oid.name_len); in encode_request_partial()
2202 ceph_encode_16(&p, req->r_num_ops); in encode_request_partial()
2203 for (i = 0; i < req->r_num_ops; i++) { in encode_request_partial()
2204 data_len += osd_req_encode_op(p, &req->r_ops[i]); in encode_request_partial()
2208 ceph_encode_64(&p, req->r_snapid); /* snapid */ in encode_request_partial()
2209 if (req->r_snapc) { in encode_request_partial()
2210 ceph_encode_64(&p, req->r_snapc->seq); in encode_request_partial()
2211 ceph_encode_32(&p, req->r_snapc->num_snaps); in encode_request_partial()
2212 for (i = 0; i < req->r_snapc->num_snaps; i++) in encode_request_partial()
2213 ceph_encode_64(&p, req->r_snapc->snaps[i]); in encode_request_partial()
2219 ceph_encode_32(&p, req->r_attempts); /* retry_attempt */ in encode_request_partial()
2220 BUG_ON(p > end - 8); /* space for features */ in encode_request_partial()
2222 msg->hdr.version = cpu_to_le16(8); /* MOSDOp v8 */ in encode_request_partial()
2224 msg->front.iov_len = p - msg->front.iov_base; in encode_request_partial()
2225 msg->hdr.front_len = cpu_to_le32(msg->front.iov_len); in encode_request_partial()
2226 msg->hdr.data_len = cpu_to_le32(data_len); in encode_request_partial()
2230 * need to re-copy it before writing it to disk (direct I/O). in encode_request_partial()
2232 msg->hdr.data_off = cpu_to_le16(req->r_data_offset); in encode_request_partial()
2234 dout("%s req %p msg %p oid %s oid_len %d\n", __func__, req, msg, in encode_request_partial()
2235 req->r_t.target_oid.name, req->r_t.target_oid.name_len); in encode_request_partial()
2240 void *p = msg->front.iov_base; in encode_request_finish()
2241 void *const partial_end = p + msg->front.iov_len; in encode_request_finish()
2242 void *const end = p + msg->front_alloc_len; in encode_request_finish()
2244 if (CEPH_HAVE_FEATURE(msg->con->peer_features, RESEND_ON_SPLIT)) { in encode_request_finish()
2245 /* luminous OSD -- encode features and be done */ in encode_request_finish()
2247 ceph_encode_64(&p, msg->con->peer_features); in encode_request_finish()
2267 * Pre-luminous OSD -- reencode v8 into v4 using @head in encode_request_finish()
2281 oloc_len = p - oloc; in encode_request_finish()
2286 oid_len = p - oid; in encode_request_finish()
2289 tail_len = partial_end - p; in encode_request_finish()
2291 p = msg->front.iov_base; in encode_request_finish()
2312 /* tail -- ops, snapid, snapc, retry_attempt */ in encode_request_finish()
2317 msg->hdr.version = cpu_to_le16(4); /* MOSDOp v4 */ in encode_request_finish()
2321 msg->front.iov_len = p - msg->front.iov_base; in encode_request_finish()
2322 msg->hdr.front_len = cpu_to_le32(msg->front.iov_len); in encode_request_finish()
2324 dout("%s msg %p tid %llu %u+%u+%u v%d\n", __func__, msg, in encode_request_finish()
2325 le64_to_cpu(msg->hdr.tid), le32_to_cpu(msg->hdr.front_len), in encode_request_finish()
2326 le32_to_cpu(msg->hdr.middle_len), le32_to_cpu(msg->hdr.data_len), in encode_request_finish()
2327 le16_to_cpu(msg->hdr.version)); in encode_request_finish()
2335 struct ceph_osd *osd = req->r_osd; in send_request()
2338 WARN_ON(osd->o_osd != req->r_t.osd); in send_request()
2348 if (req->r_sent) in send_request()
2349 ceph_msg_revoke(req->r_request); in send_request()
2351 req->r_flags |= CEPH_OSD_FLAG_KNOWN_REDIR; in send_request()
2352 if (req->r_attempts) in send_request()
2353 req->r_flags |= CEPH_OSD_FLAG_RETRY; in send_request()
2355 WARN_ON(req->r_flags & CEPH_OSD_FLAG_RETRY); in send_request()
2357 encode_request_partial(req, req->r_request); in send_request()
2359 dout("%s req %p tid %llu to pgid %llu.%x spgid %llu.%xs%d osd%d e%u flags 0x%x attempt %d\n", in send_request()
2360 __func__, req, req->r_tid, req->r_t.pgid.pool, req->r_t.pgid.seed, in send_request()
2361 req->r_t.spgid.pgid.pool, req->r_t.spgid.pgid.seed, in send_request()
2362 req->r_t.spgid.shard, osd->o_osd, req->r_t.epoch, req->r_flags, in send_request()
2363 req->r_attempts); in send_request()
2365 req->r_t.paused = false; in send_request()
2366 req->r_stamp = jiffies; in send_request()
2367 req->r_attempts++; in send_request()
2369 req->r_sent = osd->o_incarnation; in send_request()
2370 req->r_request->hdr.tid = cpu_to_le64(req->r_tid); in send_request()
2371 ceph_con_send(&osd->o_con, ceph_msg_get(req->r_request)); in send_request()
2379 WARN_ON(!osdc->osdmap->epoch); in maybe_request_map()
2384 dout("%s osdc %p continuous\n", __func__, osdc); in maybe_request_map()
2387 dout("%s osdc %p onetime\n", __func__, osdc); in maybe_request_map()
2390 if (ceph_monc_want_map(&osdc->client->monc, CEPH_SUB_OSDMAP, in maybe_request_map()
2391 osdc->osdmap->epoch + 1, continuous)) in maybe_request_map()
2392 ceph_monc_renew_subs(&osdc->client->monc); in maybe_request_map()
2400 struct ceph_osd_client *osdc = req->r_osdc; in __submit_request()
2407 WARN_ON(req->r_tid); in __submit_request()
2408 dout("%s req %p wrlocked %d\n", __func__, req, wrlocked); in __submit_request()
2411 ct_res = calc_target(osdc, &req->r_t, false); in __submit_request()
2415 osd = lookup_create_osd(osdc, req->r_t.osd, wrlocked); in __submit_request()
2417 WARN_ON(PTR_ERR(osd) != -EAGAIN || wrlocked); in __submit_request()
2421 if (osdc->abort_err) { in __submit_request()
2422 dout("req %p abort_err %d\n", req, osdc->abort_err); in __submit_request()
2423 err = osdc->abort_err; in __submit_request()
2424 } else if (osdc->osdmap->epoch < osdc->epoch_barrier) { in __submit_request()
2425 dout("req %p epoch %u barrier %u\n", req, osdc->osdmap->epoch, in __submit_request()
2426 osdc->epoch_barrier); in __submit_request()
2427 req->r_t.paused = true; in __submit_request()
2429 } else if ((req->r_flags & CEPH_OSD_FLAG_WRITE) && in __submit_request()
2431 dout("req %p pausewr\n", req); in __submit_request()
2432 req->r_t.paused = true; in __submit_request()
2434 } else if ((req->r_flags & CEPH_OSD_FLAG_READ) && in __submit_request()
2436 dout("req %p pauserd\n", req); in __submit_request()
2437 req->r_t.paused = true; in __submit_request()
2439 } else if ((req->r_flags & CEPH_OSD_FLAG_WRITE) && in __submit_request()
2440 !(req->r_flags & (CEPH_OSD_FLAG_FULL_TRY | in __submit_request()
2443 pool_full(osdc, req->r_t.base_oloc.pool))) { in __submit_request()
2444 dout("req %p full/pool_full\n", req); in __submit_request()
2445 if (ceph_test_opt(osdc->client, ABORT_ON_FULL)) { in __submit_request()
2446 err = -ENOSPC; in __submit_request()
2452 req->r_t.base_oloc.pool); in __submit_request()
2453 req->r_t.paused = true; in __submit_request()
2462 mutex_lock(&osd->lock); in __submit_request()
2468 req->r_tid = atomic64_inc_return(&osdc->last_tid); in __submit_request()
2474 mutex_unlock(&osd->lock); in __submit_request()
2480 downgrade_write(&osdc->lock); in __submit_request()
2484 up_read(&osdc->lock); in __submit_request()
2485 down_write(&osdc->lock); in __submit_request()
2493 WARN_ON(req->r_flags & (CEPH_OSD_FLAG_ACK | CEPH_OSD_FLAG_ONDISK)); in account_request()
2494 WARN_ON(!(req->r_flags & (CEPH_OSD_FLAG_READ | CEPH_OSD_FLAG_WRITE))); in account_request()
2496 req->r_flags |= CEPH_OSD_FLAG_ONDISK; in account_request()
2497 atomic_inc(&req->r_osdc->num_requests); in account_request()
2499 req->r_start_stamp = jiffies; in account_request()
2500 req->r_start_latency = ktime_get(); in account_request()
2512 struct ceph_osd_client *osdc = req->r_osdc; in finish_request()
2514 WARN_ON(lookup_request_mc(&osdc->map_checks, req->r_tid)); in finish_request()
2515 dout("%s req %p tid %llu\n", __func__, req, req->r_tid); in finish_request()
2517 req->r_end_latency = ktime_get(); in finish_request()
2519 if (req->r_osd) { in finish_request()
2520 ceph_init_sparse_read(&req->r_osd->o_sparse_read); in finish_request()
2521 unlink_request(req->r_osd, req); in finish_request()
2523 atomic_dec(&osdc->num_requests); in finish_request()
2531 ceph_msg_revoke(req->r_request); in finish_request()
2532 ceph_msg_revoke_incoming(req->r_reply); in finish_request()
2537 dout("%s req %p tid %llu cb %ps result %d\n", __func__, req, in __complete_request()
2538 req->r_tid, req->r_callback, req->r_result); in __complete_request()
2540 if (req->r_callback) in __complete_request()
2541 req->r_callback(req); in __complete_request()
2542 complete_all(&req->r_completion); in __complete_request()
2555 * This is open-coded in handle_reply().
2559 dout("%s req %p tid %llu err %d\n", __func__, req, req->r_tid, err); in complete_request()
2561 req->r_result = err; in complete_request()
2564 INIT_WORK(&req->r_complete_work, complete_request_workfn); in complete_request()
2565 queue_work(req->r_osdc->completion_wq, &req->r_complete_work); in complete_request()
2570 struct ceph_osd_client *osdc = req->r_osdc; in cancel_map_check()
2575 lookup_req = lookup_request_mc(&osdc->map_checks, req->r_tid); in cancel_map_check()
2580 erase_request_mc(&osdc->map_checks, req); in cancel_map_check()
2586 dout("%s req %p tid %llu\n", __func__, req, req->r_tid); in cancel_request()
2590 complete_all(&req->r_completion); in cancel_request()
2596 dout("%s req %p tid %llu err %d\n", __func__, req, req->r_tid, err); in abort_request()
2611 * Abort all in-flight requests with @err and arrange for all future
2616 dout("%s osdc %p err %d\n", __func__, osdc, err); in ceph_osdc_abort_requests()
2617 down_write(&osdc->lock); in ceph_osdc_abort_requests()
2619 osdc->abort_err = err; in ceph_osdc_abort_requests()
2620 up_write(&osdc->lock); in ceph_osdc_abort_requests()
2626 down_write(&osdc->lock); in ceph_osdc_clear_abort_err()
2627 osdc->abort_err = 0; in ceph_osdc_clear_abort_err()
2628 up_write(&osdc->lock); in ceph_osdc_clear_abort_err()
2634 if (likely(eb > osdc->epoch_barrier)) { in update_epoch_barrier()
2635 dout("updating epoch_barrier from %u to %u\n", in update_epoch_barrier()
2636 osdc->epoch_barrier, eb); in update_epoch_barrier()
2637 osdc->epoch_barrier = eb; in update_epoch_barrier()
2639 if (eb > osdc->osdmap->epoch) in update_epoch_barrier()
2646 down_read(&osdc->lock); in ceph_osdc_update_epoch_barrier()
2647 if (unlikely(eb > osdc->epoch_barrier)) { in ceph_osdc_update_epoch_barrier()
2648 up_read(&osdc->lock); in ceph_osdc_update_epoch_barrier()
2649 down_write(&osdc->lock); in ceph_osdc_update_epoch_barrier()
2651 up_write(&osdc->lock); in ceph_osdc_update_epoch_barrier()
2653 up_read(&osdc->lock); in ceph_osdc_update_epoch_barrier()
2666 struct ceph_osd_client *osdc = req->r_osdc; in abort_on_full_fn()
2669 if ((req->r_flags & CEPH_OSD_FLAG_WRITE) && in abort_on_full_fn()
2671 pool_full(osdc, req->r_t.base_oloc.pool))) { in abort_on_full_fn()
2673 update_epoch_barrier(osdc, osdc->osdmap->epoch); in abort_on_full_fn()
2676 abort_request(req, -ENOSPC); in abort_on_full_fn()
2685 * osdc->epoch_barrier to the latest map epoch that we've seen if any were
2692 if (ceph_test_opt(osdc->client, ABORT_ON_FULL) && in ceph_osdc_abort_on_full()
2699 struct ceph_osd_client *osdc = req->r_osdc; in check_pool_dne()
2700 struct ceph_osdmap *map = osdc->osdmap; in check_pool_dne()
2703 WARN_ON(!map->epoch); in check_pool_dne()
2705 if (req->r_attempts) { in check_pool_dne()
2711 req->r_map_dne_bound = map->epoch; in check_pool_dne()
2712 dout("%s req %p tid %llu pool disappeared\n", __func__, req, in check_pool_dne()
2713 req->r_tid); in check_pool_dne()
2715 dout("%s req %p tid %llu map_dne_bound %u have %u\n", __func__, in check_pool_dne()
2716 req, req->r_tid, req->r_map_dne_bound, map->epoch); in check_pool_dne()
2719 if (req->r_map_dne_bound) { in check_pool_dne()
2720 if (map->epoch >= req->r_map_dne_bound) { in check_pool_dne()
2723 req->r_tid); in check_pool_dne()
2724 complete_request(req, -ENOENT); in check_pool_dne()
2733 struct ceph_osd_client *osdc = &greq->monc->client->osdc; in map_check_cb()
2735 u64 tid = greq->private_data; in map_check_cb()
2737 WARN_ON(greq->result || !greq->u.newest); in map_check_cb()
2739 down_write(&osdc->lock); in map_check_cb()
2740 req = lookup_request_mc(&osdc->map_checks, tid); in map_check_cb()
2742 dout("%s tid %llu dne\n", __func__, tid); in map_check_cb()
2746 dout("%s req %p tid %llu map_dne_bound %u newest %llu\n", __func__, in map_check_cb()
2747 req, req->r_tid, req->r_map_dne_bound, greq->u.newest); in map_check_cb()
2748 if (!req->r_map_dne_bound) in map_check_cb()
2749 req->r_map_dne_bound = greq->u.newest; in map_check_cb()
2750 erase_request_mc(&osdc->map_checks, req); in map_check_cb()
2755 up_write(&osdc->lock); in map_check_cb()
2760 struct ceph_osd_client *osdc = req->r_osdc; in send_map_check()
2766 lookup_req = lookup_request_mc(&osdc->map_checks, req->r_tid); in send_map_check()
2773 insert_request_mc(&osdc->map_checks, req); in send_map_check()
2774 ret = ceph_monc_get_version_async(&osdc->client->monc, "osdmap", in send_map_check()
2775 map_check_cb, req->r_tid); in send_map_check()
2787 dout("%s lreq %p reg_req %p ping_req %p\n", __func__, lreq, in linger_release()
2788 lreq->reg_req, lreq->ping_req); in linger_release()
2789 WARN_ON(!RB_EMPTY_NODE(&lreq->node)); in linger_release()
2790 WARN_ON(!RB_EMPTY_NODE(&lreq->osdc_node)); in linger_release()
2791 WARN_ON(!RB_EMPTY_NODE(&lreq->mc_node)); in linger_release()
2792 WARN_ON(!list_empty(&lreq->scan_item)); in linger_release()
2793 WARN_ON(!list_empty(&lreq->pending_lworks)); in linger_release()
2794 WARN_ON(lreq->osd); in linger_release()
2796 if (lreq->request_pl) in linger_release()
2797 ceph_pagelist_release(lreq->request_pl); in linger_release()
2798 if (lreq->notify_id_pages) in linger_release()
2799 ceph_release_page_vector(lreq->notify_id_pages, 1); in linger_release()
2801 ceph_osdc_put_request(lreq->reg_req); in linger_release()
2802 ceph_osdc_put_request(lreq->ping_req); in linger_release()
2803 target_destroy(&lreq->t); in linger_release()
2810 kref_put(&lreq->kref, linger_release); in linger_put()
2816 kref_get(&lreq->kref); in linger_get()
2829 kref_init(&lreq->kref); in linger_alloc()
2830 mutex_init(&lreq->lock); in linger_alloc()
2831 RB_CLEAR_NODE(&lreq->node); in linger_alloc()
2832 RB_CLEAR_NODE(&lreq->osdc_node); in linger_alloc()
2833 RB_CLEAR_NODE(&lreq->mc_node); in linger_alloc()
2834 INIT_LIST_HEAD(&lreq->scan_item); in linger_alloc()
2835 INIT_LIST_HEAD(&lreq->pending_lworks); in linger_alloc()
2836 init_completion(&lreq->reg_commit_wait); in linger_alloc()
2837 init_completion(&lreq->notify_finish_wait); in linger_alloc()
2839 lreq->osdc = osdc; in linger_alloc()
2840 target_init(&lreq->t); in linger_alloc()
2842 dout("%s lreq %p\n", __func__, lreq); in linger_alloc()
2851 * Create linger request <-> OSD session relation. in DEFINE_RB_INSDEL_FUNCS()
2859 WARN_ON(!lreq->linger_id || lreq->osd); in DEFINE_RB_INSDEL_FUNCS()
2860 dout("%s osd %p osd%d lreq %p linger_id %llu\n", __func__, osd, in DEFINE_RB_INSDEL_FUNCS()
2861 osd->o_osd, lreq, lreq->linger_id); in DEFINE_RB_INSDEL_FUNCS()
2866 atomic_inc(&osd->o_osdc->num_homeless); in DEFINE_RB_INSDEL_FUNCS()
2869 insert_linger(&osd->o_linger_requests, lreq); in DEFINE_RB_INSDEL_FUNCS()
2870 lreq->osd = osd; in DEFINE_RB_INSDEL_FUNCS()
2877 WARN_ON(lreq->osd != osd); in unlink_linger()
2878 dout("%s osd %p osd%d lreq %p linger_id %llu\n", __func__, osd, in unlink_linger()
2879 osd->o_osd, lreq, lreq->linger_id); in unlink_linger()
2881 lreq->osd = NULL; in unlink_linger()
2882 erase_linger(&osd->o_linger_requests, lreq); in unlink_linger()
2888 atomic_dec(&osd->o_osdc->num_homeless); in unlink_linger()
2893 verify_osdc_locked(lreq->osdc); in __linger_registered()
2895 return !RB_EMPTY_NODE(&lreq->osdc_node); in __linger_registered()
2900 struct ceph_osd_client *osdc = lreq->osdc; in linger_registered()
2903 down_read(&osdc->lock); in linger_registered()
2905 up_read(&osdc->lock); in linger_registered()
2912 struct ceph_osd_client *osdc = lreq->osdc; in linger_register()
2915 WARN_ON(lreq->linger_id); in linger_register()
2918 lreq->linger_id = ++osdc->last_linger_id; in linger_register()
2919 insert_linger_osdc(&osdc->linger_requests, lreq); in linger_register()
2924 struct ceph_osd_client *osdc = lreq->osdc; in linger_unregister()
2928 erase_linger_osdc(&osdc->linger_requests, lreq); in linger_unregister()
2934 struct ceph_osd_linger_request *lreq = req->r_priv; in cancel_linger_request()
2936 WARN_ON(!req->r_linger); in cancel_linger_request()
2971 INIT_WORK(&lwork->work, workfn); in lwork_alloc()
2972 INIT_LIST_HEAD(&lwork->pending_item); in lwork_alloc()
2973 lwork->lreq = linger_get(lreq); in lwork_alloc()
2980 struct ceph_osd_linger_request *lreq = lwork->lreq; in lwork_free()
2982 mutex_lock(&lreq->lock); in lwork_free()
2983 list_del(&lwork->pending_item); in lwork_free()
2984 mutex_unlock(&lreq->lock); in lwork_free()
2992 struct ceph_osd_linger_request *lreq = lwork->lreq; in lwork_queue()
2993 struct ceph_osd_client *osdc = lreq->osdc; in lwork_queue()
2996 WARN_ON(!list_empty(&lwork->pending_item)); in lwork_queue()
2998 lwork->queued_stamp = jiffies; in lwork_queue()
2999 list_add_tail(&lwork->pending_item, &lreq->pending_lworks); in lwork_queue()
3000 queue_work(osdc->notify_wq, &lwork->work); in lwork_queue()
3006 struct ceph_osd_linger_request *lreq = lwork->lreq; in do_watch_notify()
3009 dout("%s lreq %p not registered\n", __func__, lreq); in do_watch_notify()
3013 WARN_ON(!lreq->is_watch); in do_watch_notify()
3014 dout("%s lreq %p notify_id %llu notifier_id %llu payload_len %zu\n", in do_watch_notify()
3015 __func__, lreq, lwork->notify.notify_id, lwork->notify.notifier_id, in do_watch_notify()
3016 lwork->notify.payload_len); in do_watch_notify()
3017 lreq->wcb(lreq->data, lwork->notify.notify_id, lreq->linger_id, in do_watch_notify()
3018 lwork->notify.notifier_id, lwork->notify.payload, in do_watch_notify()
3019 lwork->notify.payload_len); in do_watch_notify()
3022 ceph_msg_put(lwork->notify.msg); in do_watch_notify()
3029 struct ceph_osd_linger_request *lreq = lwork->lreq; in do_watch_error()
3032 dout("%s lreq %p not registered\n", __func__, lreq); in do_watch_error()
3036 dout("%s lreq %p err %d\n", __func__, lreq, lwork->error.err); in do_watch_error()
3037 lreq->errcb(lreq->data, lreq->linger_id, lwork->error.err); in do_watch_error()
3049 pr_err("failed to allocate error-lwork\n"); in queue_watch_error()
3053 lwork->error.err = lreq->last_error; in queue_watch_error()
3060 if (!completion_done(&lreq->reg_commit_wait)) { in linger_reg_commit_complete()
3061 lreq->reg_commit_error = (result <= 0 ? result : 0); in linger_reg_commit_complete()
3062 complete_all(&lreq->reg_commit_wait); in linger_reg_commit_complete()
3068 struct ceph_osd_linger_request *lreq = req->r_priv; in linger_commit_cb()
3070 mutex_lock(&lreq->lock); in linger_commit_cb()
3071 if (req != lreq->reg_req) { in linger_commit_cb()
3072 dout("%s lreq %p linger_id %llu unknown req (%p != %p)\n", in linger_commit_cb()
3073 __func__, lreq, lreq->linger_id, req, lreq->reg_req); in linger_commit_cb()
3077 dout("%s lreq %p linger_id %llu result %d\n", __func__, lreq, in linger_commit_cb()
3078 lreq->linger_id, req->r_result); in linger_commit_cb()
3079 linger_reg_commit_complete(lreq, req->r_result); in linger_commit_cb()
3080 lreq->committed = true; in linger_commit_cb()
3082 if (!lreq->is_watch) { in linger_commit_cb()
3085 void *p = page_address(osd_data->pages[0]); in linger_commit_cb()
3087 WARN_ON(req->r_ops[0].op != CEPH_OSD_OP_NOTIFY || in linger_commit_cb()
3088 osd_data->type != CEPH_OSD_DATA_TYPE_PAGES); in linger_commit_cb()
3091 if (req->r_ops[0].outdata_len >= sizeof(u64)) { in linger_commit_cb()
3092 lreq->notify_id = ceph_decode_64(&p); in linger_commit_cb()
3093 dout("lreq %p notify_id %llu\n", lreq, in linger_commit_cb()
3094 lreq->notify_id); in linger_commit_cb()
3096 dout("lreq %p no notify_id\n", lreq); in linger_commit_cb()
3101 mutex_unlock(&lreq->lock); in linger_commit_cb()
3108 * Translate ENOENT -> ENOTCONN so that a delete->disconnection in normalize_watch_error()
3112 if (err == -ENOENT) in normalize_watch_error()
3113 err = -ENOTCONN; in normalize_watch_error()
3120 struct ceph_osd_linger_request *lreq = req->r_priv; in linger_reconnect_cb()
3122 mutex_lock(&lreq->lock); in linger_reconnect_cb()
3123 if (req != lreq->reg_req) { in linger_reconnect_cb()
3124 dout("%s lreq %p linger_id %llu unknown req (%p != %p)\n", in linger_reconnect_cb()
3125 __func__, lreq, lreq->linger_id, req, lreq->reg_req); in linger_reconnect_cb()
3129 dout("%s lreq %p linger_id %llu result %d last_error %d\n", __func__, in linger_reconnect_cb()
3130 lreq, lreq->linger_id, req->r_result, lreq->last_error); in linger_reconnect_cb()
3131 if (req->r_result < 0) { in linger_reconnect_cb()
3132 if (!lreq->last_error) { in linger_reconnect_cb()
3133 lreq->last_error = normalize_watch_error(req->r_result); in linger_reconnect_cb()
3139 mutex_unlock(&lreq->lock); in linger_reconnect_cb()
3145 struct ceph_osd_client *osdc = lreq->osdc; in send_linger()
3150 mutex_lock(&lreq->lock); in send_linger()
3151 dout("%s lreq %p linger_id %llu\n", __func__, lreq, lreq->linger_id); in send_linger()
3153 if (lreq->reg_req) { in send_linger()
3154 if (lreq->reg_req->r_osd) in send_linger()
3155 cancel_linger_request(lreq->reg_req); in send_linger()
3156 ceph_osdc_put_request(lreq->reg_req); in send_linger()
3162 target_copy(&req->r_t, &lreq->t); in send_linger()
3163 req->r_mtime = lreq->mtime; in send_linger()
3165 if (lreq->is_watch && lreq->committed) { in send_linger()
3167 lreq->linger_id, ++lreq->register_gen); in send_linger()
3168 dout("lreq %p reconnect register_gen %u\n", lreq, in send_linger()
3169 req->r_ops[0].watch.gen); in send_linger()
3170 req->r_callback = linger_reconnect_cb; in send_linger()
3172 if (lreq->is_watch) { in send_linger()
3174 lreq->linger_id, 0); in send_linger()
3176 lreq->notify_id = 0; in send_linger()
3178 refcount_inc(&lreq->request_pl->refcnt); in send_linger()
3179 osd_req_op_notify_init(req, 0, lreq->linger_id, in send_linger()
3180 lreq->request_pl); in send_linger()
3183 lreq->notify_id_pages, PAGE_SIZE, 0, false, false); in send_linger()
3185 dout("lreq %p register\n", lreq); in send_linger()
3186 req->r_callback = linger_commit_cb; in send_linger()
3192 req->r_priv = linger_get(lreq); in send_linger()
3193 req->r_linger = true; in send_linger()
3194 lreq->reg_req = req; in send_linger()
3195 mutex_unlock(&lreq->lock); in send_linger()
3202 struct ceph_osd_linger_request *lreq = req->r_priv; in linger_ping_cb()
3204 mutex_lock(&lreq->lock); in linger_ping_cb()
3205 if (req != lreq->ping_req) { in linger_ping_cb()
3206 dout("%s lreq %p linger_id %llu unknown req (%p != %p)\n", in linger_ping_cb()
3207 __func__, lreq, lreq->linger_id, req, lreq->ping_req); in linger_ping_cb()
3211 dout("%s lreq %p linger_id %llu result %d ping_sent %lu last_error %d\n", in linger_ping_cb()
3212 __func__, lreq, lreq->linger_id, req->r_result, lreq->ping_sent, in linger_ping_cb()
3213 lreq->last_error); in linger_ping_cb()
3214 if (lreq->register_gen == req->r_ops[0].watch.gen) { in linger_ping_cb()
3215 if (!req->r_result) { in linger_ping_cb()
3216 lreq->watch_valid_thru = lreq->ping_sent; in linger_ping_cb()
3217 } else if (!lreq->last_error) { in linger_ping_cb()
3218 lreq->last_error = normalize_watch_error(req->r_result); in linger_ping_cb()
3222 dout("lreq %p register_gen %u ignoring old pong %u\n", lreq, in linger_ping_cb()
3223 lreq->register_gen, req->r_ops[0].watch.gen); in linger_ping_cb()
3227 mutex_unlock(&lreq->lock); in linger_ping_cb()
3233 struct ceph_osd_client *osdc = lreq->osdc; in send_linger_ping()
3238 dout("%s PAUSERD\n", __func__); in send_linger_ping()
3242 lreq->ping_sent = jiffies; in send_linger_ping()
3243 dout("%s lreq %p linger_id %llu ping_sent %lu register_gen %u\n", in send_linger_ping()
3244 __func__, lreq, lreq->linger_id, lreq->ping_sent, in send_linger_ping()
3245 lreq->register_gen); in send_linger_ping()
3247 if (lreq->ping_req) { in send_linger_ping()
3248 if (lreq->ping_req->r_osd) in send_linger_ping()
3249 cancel_linger_request(lreq->ping_req); in send_linger_ping()
3250 ceph_osdc_put_request(lreq->ping_req); in send_linger_ping()
3256 target_copy(&req->r_t, &lreq->t); in send_linger_ping()
3257 osd_req_op_watch_init(req, 0, CEPH_OSD_WATCH_OP_PING, lreq->linger_id, in send_linger_ping()
3258 lreq->register_gen); in send_linger_ping()
3259 req->r_callback = linger_ping_cb; in send_linger_ping()
3264 req->r_priv = linger_get(lreq); in send_linger_ping()
3265 req->r_linger = true; in send_linger_ping()
3266 lreq->ping_req = req; in send_linger_ping()
3270 req->r_tid = atomic64_inc_return(&osdc->last_tid); in send_linger_ping()
3271 link_request(lreq->osd, req); in send_linger_ping()
3277 struct ceph_osd_client *osdc = lreq->osdc; in linger_submit()
3280 down_write(&osdc->lock); in linger_submit()
3283 calc_target(osdc, &lreq->t, false); in linger_submit()
3284 osd = lookup_create_osd(osdc, lreq->t.osd, true); in linger_submit()
3288 up_write(&osdc->lock); in linger_submit()
3293 struct ceph_osd_client *osdc = lreq->osdc; in cancel_linger_map_check()
3298 lookup_lreq = lookup_linger_mc(&osdc->linger_map_checks, in cancel_linger_map_check()
3299 lreq->linger_id); in cancel_linger_map_check()
3304 erase_linger_mc(&osdc->linger_map_checks, lreq); in cancel_linger_map_check()
3313 if (lreq->ping_req && lreq->ping_req->r_osd) in __linger_cancel()
3314 cancel_linger_request(lreq->ping_req); in __linger_cancel()
3315 if (lreq->reg_req && lreq->reg_req->r_osd) in __linger_cancel()
3316 cancel_linger_request(lreq->reg_req); in __linger_cancel()
3318 unlink_linger(lreq->osd, lreq); in __linger_cancel()
3324 struct ceph_osd_client *osdc = lreq->osdc; in linger_cancel()
3326 down_write(&osdc->lock); in linger_cancel()
3329 up_write(&osdc->lock); in linger_cancel()
3336 struct ceph_osd_client *osdc = lreq->osdc; in check_linger_pool_dne()
3337 struct ceph_osdmap *map = osdc->osdmap; in check_linger_pool_dne()
3340 WARN_ON(!map->epoch); in check_linger_pool_dne()
3342 if (lreq->register_gen) { in check_linger_pool_dne()
3343 lreq->map_dne_bound = map->epoch; in check_linger_pool_dne()
3344 dout("%s lreq %p linger_id %llu pool disappeared\n", __func__, in check_linger_pool_dne()
3345 lreq, lreq->linger_id); in check_linger_pool_dne()
3347 dout("%s lreq %p linger_id %llu map_dne_bound %u have %u\n", in check_linger_pool_dne()
3348 __func__, lreq, lreq->linger_id, lreq->map_dne_bound, in check_linger_pool_dne()
3349 map->epoch); in check_linger_pool_dne()
3352 if (lreq->map_dne_bound) { in check_linger_pool_dne()
3353 if (map->epoch >= lreq->map_dne_bound) { in check_linger_pool_dne()
3356 lreq->linger_id); in check_linger_pool_dne()
3357 linger_reg_commit_complete(lreq, -ENOENT); in check_linger_pool_dne()
3367 struct ceph_osd_client *osdc = &greq->monc->client->osdc; in linger_map_check_cb()
3369 u64 linger_id = greq->private_data; in linger_map_check_cb()
3371 WARN_ON(greq->result || !greq->u.newest); in linger_map_check_cb()
3373 down_write(&osdc->lock); in linger_map_check_cb()
3374 lreq = lookup_linger_mc(&osdc->linger_map_checks, linger_id); in linger_map_check_cb()
3376 dout("%s linger_id %llu dne\n", __func__, linger_id); in linger_map_check_cb()
3380 dout("%s lreq %p linger_id %llu map_dne_bound %u newest %llu\n", in linger_map_check_cb()
3381 __func__, lreq, lreq->linger_id, lreq->map_dne_bound, in linger_map_check_cb()
3382 greq->u.newest); in linger_map_check_cb()
3383 if (!lreq->map_dne_bound) in linger_map_check_cb()
3384 lreq->map_dne_bound = greq->u.newest; in linger_map_check_cb()
3385 erase_linger_mc(&osdc->linger_map_checks, lreq); in linger_map_check_cb()
3390 up_write(&osdc->lock); in linger_map_check_cb()
3395 struct ceph_osd_client *osdc = lreq->osdc; in send_linger_map_check()
3401 lookup_lreq = lookup_linger_mc(&osdc->linger_map_checks, in send_linger_map_check()
3402 lreq->linger_id); in send_linger_map_check()
3409 insert_linger_mc(&osdc->linger_map_checks, lreq); in send_linger_map_check()
3410 ret = ceph_monc_get_version_async(&osdc->client->monc, "osdmap", in send_linger_map_check()
3411 linger_map_check_cb, lreq->linger_id); in send_linger_map_check()
3419 dout("%s lreq %p linger_id %llu\n", __func__, lreq, lreq->linger_id); in linger_reg_commit_wait()
3420 ret = wait_for_completion_killable(&lreq->reg_commit_wait); in linger_reg_commit_wait()
3421 return ret ?: lreq->reg_commit_error; in linger_reg_commit_wait()
3429 dout("%s lreq %p linger_id %llu\n", __func__, lreq, lreq->linger_id); in linger_notify_finish_wait()
3430 left = wait_for_completion_killable_timeout(&lreq->notify_finish_wait, in linger_notify_finish_wait()
3433 left = left ?: -ETIMEDOUT; in linger_notify_finish_wait()
3435 left = lreq->notify_finish_error; /* completed */ in linger_notify_finish_wait()
3450 struct ceph_options *opts = osdc->client->options; in handle_timeout()
3451 unsigned long cutoff = jiffies - opts->osd_keepalive_timeout; in handle_timeout()
3452 unsigned long expiry_cutoff = jiffies - opts->osd_request_timeout; in handle_timeout()
3456 dout("%s osdc %p\n", __func__, osdc); in handle_timeout()
3457 down_write(&osdc->lock); in handle_timeout()
3464 for (n = rb_first(&osdc->osds); n; n = rb_next(n)) { in handle_timeout()
3468 for (p = rb_first(&osd->o_requests); p; ) { in handle_timeout()
3474 if (time_before(req->r_stamp, cutoff)) { in handle_timeout()
3475 dout(" req %p tid %llu on osd%d is laggy\n", in handle_timeout()
3476 req, req->r_tid, osd->o_osd); in handle_timeout()
3479 if (opts->osd_request_timeout && in handle_timeout()
3480 time_before(req->r_start_stamp, expiry_cutoff)) { in handle_timeout()
3482 req->r_tid, osd->o_osd); in handle_timeout()
3483 abort_request(req, -ETIMEDOUT); in handle_timeout()
3486 for (p = rb_first(&osd->o_linger_requests); p; p = rb_next(p)) { in handle_timeout()
3490 dout(" lreq %p linger_id %llu is served by osd%d\n", in handle_timeout()
3491 lreq, lreq->linger_id, osd->o_osd); in handle_timeout()
3494 mutex_lock(&lreq->lock); in handle_timeout()
3495 if (lreq->is_watch && lreq->committed && !lreq->last_error) in handle_timeout()
3497 mutex_unlock(&lreq->lock); in handle_timeout()
3501 list_move_tail(&osd->o_keepalive_item, &slow_osds); in handle_timeout()
3504 if (opts->osd_request_timeout) { in handle_timeout()
3505 for (p = rb_first(&osdc->homeless_osd.o_requests); p; ) { in handle_timeout()
3511 if (time_before(req->r_start_stamp, expiry_cutoff)) { in handle_timeout()
3513 req->r_tid, osdc->homeless_osd.o_osd); in handle_timeout()
3514 abort_request(req, -ETIMEDOUT); in handle_timeout()
3519 if (atomic_read(&osdc->num_homeless) || !list_empty(&slow_osds)) in handle_timeout()
3526 list_del_init(&osd->o_keepalive_item); in handle_timeout()
3527 ceph_con_keepalive(&osd->o_con); in handle_timeout()
3530 up_write(&osdc->lock); in handle_timeout()
3531 schedule_delayed_work(&osdc->timeout_work, in handle_timeout()
3532 osdc->client->options->osd_keepalive_timeout); in handle_timeout()
3540 unsigned long delay = osdc->client->options->osd_idle_ttl / 4; in handle_osds_timeout()
3543 dout("%s osdc %p\n", __func__, osdc); in handle_osds_timeout()
3544 down_write(&osdc->lock); in handle_osds_timeout()
3545 list_for_each_entry_safe(osd, nosd, &osdc->osd_lru, o_osd_lru) { in handle_osds_timeout()
3546 if (time_before(jiffies, osd->lru_ttl)) in handle_osds_timeout()
3549 WARN_ON(!RB_EMPTY_ROOT(&osd->o_requests)); in handle_osds_timeout()
3550 WARN_ON(!RB_EMPTY_ROOT(&osd->o_linger_requests)); in handle_osds_timeout()
3554 up_write(&osdc->lock); in handle_osds_timeout()
3555 schedule_delayed_work(&osdc->osds_timeout_work, in handle_osds_timeout()
3584 oloc->pool = ceph_decode_64(p); in ceph_oloc_decode()
3599 if (!oloc->pool_ns || in ceph_oloc_decode()
3600 ceph_compare_string(oloc->pool_ns, *p, len)) in ceph_oloc_decode()
3604 if (oloc->pool_ns) in ceph_oloc_decode()
3616 if (hash != -1) { in ceph_oloc_decode()
3628 ret = -EINVAL; in ceph_oloc_decode()
3652 ret = ceph_oloc_decode(p, end, &redir->oloc); in ceph_redirect_decode()
3668 ret = -EINVAL; in ceph_redirect_decode()
3688 void *p = msg->front.iov_base; in decode_MOSDOpReply()
3689 void *const end = p + msg->front.iov_len; in decode_MOSDOpReply()
3690 u16 version = le16_to_cpu(msg->hdr.version); in decode_MOSDOpReply()
3701 ret = ceph_decode_pgid(&p, end, &m->pgid); in decode_MOSDOpReply()
3705 ceph_decode_64_safe(&p, end, m->flags, e_inval); in decode_MOSDOpReply()
3706 ceph_decode_32_safe(&p, end, m->result, e_inval); in decode_MOSDOpReply()
3710 ceph_decode_32_safe(&p, end, m->epoch, e_inval); in decode_MOSDOpReply()
3712 ceph_decode_32_safe(&p, end, m->num_ops, e_inval); in decode_MOSDOpReply()
3713 if (m->num_ops > ARRAY_SIZE(m->outdata_len)) in decode_MOSDOpReply()
3716 ceph_decode_need(&p, end, m->num_ops * sizeof(struct ceph_osd_op), in decode_MOSDOpReply()
3718 for (i = 0; i < m->num_ops; i++) { in decode_MOSDOpReply()
3721 m->outdata_len[i] = le32_to_cpu(op->payload_len); in decode_MOSDOpReply()
3725 ceph_decode_32_safe(&p, end, m->retry_attempt, e_inval); in decode_MOSDOpReply()
3726 for (i = 0; i < m->num_ops; i++) in decode_MOSDOpReply()
3727 ceph_decode_32_safe(&p, end, m->rval[i], e_inval); in decode_MOSDOpReply()
3730 ceph_decode_need(&p, end, sizeof(m->replay_version), e_inval); in decode_MOSDOpReply()
3731 memcpy(&m->replay_version, p, sizeof(m->replay_version)); in decode_MOSDOpReply()
3732 p += sizeof(m->replay_version); in decode_MOSDOpReply()
3733 ceph_decode_64_safe(&p, end, m->user_version, e_inval); in decode_MOSDOpReply()
3735 m->replay_version = bad_replay_version; /* struct */ in decode_MOSDOpReply()
3736 m->user_version = le64_to_cpu(m->replay_version.version); in decode_MOSDOpReply()
3749 ret = ceph_redirect_decode(&p, end, &m->redirect); in decode_MOSDOpReply()
3753 ceph_oloc_init(&m->redirect.oloc); in decode_MOSDOpReply()
3759 return -EINVAL; in decode_MOSDOpReply()
3763 * Handle MOSDOpReply. Set ->r_result and call the callback if it is
3768 struct ceph_osd_client *osdc = osd->o_osdc; in handle_reply()
3771 u64 tid = le64_to_cpu(msg->hdr.tid); in handle_reply()
3776 dout("%s msg %p tid %llu\n", __func__, msg, tid); in handle_reply()
3778 down_read(&osdc->lock); in handle_reply()
3780 dout("%s osd%d unknown\n", __func__, osd->o_osd); in handle_reply()
3783 WARN_ON(osd->o_osd != le64_to_cpu(msg->hdr.src.num)); in handle_reply()
3785 mutex_lock(&osd->lock); in handle_reply()
3786 req = lookup_request(&osd->o_requests, tid); in handle_reply()
3788 dout("%s osd%d tid %llu unknown\n", __func__, osd->o_osd, tid); in handle_reply()
3792 m.redirect.oloc.pool_ns = req->r_t.target_oloc.pool_ns; in handle_reply()
3797 req->r_tid, ret); in handle_reply()
3801 dout("%s req %p tid %llu flags 0x%llx pgid %llu.%x epoch %u attempt %d v %u'%llu uv %llu\n", in handle_reply()
3802 __func__, req, req->r_tid, m.flags, m.pgid.pool, m.pgid.seed, in handle_reply()
3807 if (m.retry_attempt != req->r_attempts - 1) { in handle_reply()
3808 dout("req %p tid %llu retry_attempt %d != %d, ignoring\n", in handle_reply()
3809 req, req->r_tid, m.retry_attempt, in handle_reply()
3810 req->r_attempts - 1); in handle_reply()
3818 dout("req %p tid %llu redirect pool %lld\n", req, req->r_tid, in handle_reply()
3821 mutex_unlock(&osd->lock); in handle_reply()
3824 * Not ceph_oloc_copy() - changing pool_ns is not in handle_reply()
3827 req->r_t.target_oloc.pool = m.redirect.oloc.pool; in handle_reply()
3828 req->r_flags |= CEPH_OSD_FLAG_REDIRECTED | in handle_reply()
3831 req->r_tid = 0; in handle_reply()
3836 if (m.result == -EAGAIN) { in handle_reply()
3837 dout("req %p tid %llu EAGAIN\n", req, req->r_tid); in handle_reply()
3839 mutex_unlock(&osd->lock); in handle_reply()
3846 req->r_t.pgid.pool = 0; in handle_reply()
3847 req->r_t.pgid.seed = 0; in handle_reply()
3848 WARN_ON(!req->r_t.used_replica); in handle_reply()
3849 req->r_flags &= ~(CEPH_OSD_FLAG_BALANCE_READS | in handle_reply()
3851 req->r_tid = 0; in handle_reply()
3856 if (m.num_ops != req->r_num_ops) { in handle_reply()
3858 req->r_num_ops, req->r_tid); in handle_reply()
3861 for (i = 0; i < req->r_num_ops; i++) { in handle_reply()
3862 dout(" req %p tid %llu op %d rval %d len %u\n", req, in handle_reply()
3863 req->r_tid, i, m.rval[i], m.outdata_len[i]); in handle_reply()
3864 req->r_ops[i].rval = m.rval[i]; in handle_reply()
3865 req->r_ops[i].outdata_len = m.outdata_len[i]; in handle_reply()
3868 if (data_len != le32_to_cpu(msg->hdr.data_len)) { in handle_reply()
3870 le32_to_cpu(msg->hdr.data_len), req->r_tid); in handle_reply()
3873 dout("%s req %p tid %llu result %d data_len %u\n", __func__, in handle_reply()
3874 req, req->r_tid, m.result, data_len); in handle_reply()
3881 req->r_version = m.user_version; in handle_reply()
3882 req->r_result = m.result ?: data_len; in handle_reply()
3884 mutex_unlock(&osd->lock); in handle_reply()
3885 up_read(&osdc->lock); in handle_reply()
3891 complete_request(req, -EIO); in handle_reply()
3893 mutex_unlock(&osd->lock); in handle_reply()
3895 up_read(&osdc->lock); in handle_reply()
3902 for (n = rb_first(&osdc->osdmap->pg_pools); n; n = rb_next(n)) { in set_pool_was_full()
3906 pi->was_full = __pool_full(pi); in set_pool_was_full()
3914 pi = ceph_pg_pool_by_id(osdc->osdmap, pool_id); in pool_cleared_full()
3918 return pi->was_full && !__pool_full(pi); in pool_cleared_full()
3924 struct ceph_osd_client *osdc = lreq->osdc; in recalc_linger_target()
3927 ct_res = calc_target(osdc, &lreq->t, true); in recalc_linger_target()
3931 osd = lookup_create_osd(osdc, lreq->t.osd, true); in recalc_linger_target()
3932 if (osd != lreq->osd) { in recalc_linger_target()
3933 unlink_linger(lreq->osd, lreq); in recalc_linger_target()
3951 struct ceph_osd_client *osdc = osd->o_osdc; in scan_requests()
3955 for (n = rb_first(&osd->o_linger_requests); n; ) { in scan_requests()
3962 dout("%s lreq %p linger_id %llu\n", __func__, lreq, in scan_requests()
3963 lreq->linger_id); in scan_requests()
3969 pool_cleared_full(osdc, lreq->t.base_oloc.pool)); in scan_requests()
3981 if (list_empty(&lreq->scan_item)) in scan_requests()
3982 list_add_tail(&lreq->scan_item, need_resend_linger); in scan_requests()
3985 list_del_init(&lreq->scan_item); in scan_requests()
3991 for (n = rb_first(&osd->o_requests); n; ) { in scan_requests()
3998 dout("%s req %p tid %llu\n", __func__, req, req->r_tid); in scan_requests()
3999 ct_res = calc_target(osdc, &req->r_t, false); in scan_requests()
4004 pool_cleared_full(osdc, req->r_t.base_oloc.pool)); in scan_requests()
4006 (!(req->r_flags & CEPH_OSD_FLAG_WRITE) || in scan_requests()
4038 ceph_msgr2(osdc->client), in handle_one_map()
4039 osdc->osdmap); in handle_one_map()
4041 newmap = ceph_osdmap_decode(&p, end, ceph_msgr2(osdc->client)); in handle_one_map()
4045 if (newmap != osdc->osdmap) { in handle_one_map()
4047 * Preserve ->was_full before destroying the old map. in handle_one_map()
4048 * For pools that weren't in the old map, ->was_full in handle_one_map()
4051 for (n = rb_first(&newmap->pg_pools); n; n = rb_next(n)) { in handle_one_map()
4056 old_pi = ceph_pg_pool_by_id(osdc->osdmap, pi->id); in handle_one_map()
4058 pi->was_full = old_pi->was_full; in handle_one_map()
4060 WARN_ON(pi->was_full); in handle_one_map()
4063 if (osdc->osdmap->epoch && in handle_one_map()
4064 osdc->osdmap->epoch + 1 < newmap->epoch) { in handle_one_map()
4069 ceph_osdmap_destroy(osdc->osdmap); in handle_one_map()
4070 osdc->osdmap = newmap; in handle_one_map()
4074 scan_requests(&osdc->homeless_osd, skipped_map, was_full, true, in handle_one_map()
4077 for (n = rb_first(&osdc->osds); n; ) { in handle_one_map()
4084 if (!ceph_osd_is_up(osdc->osdmap, osd->o_osd) || in handle_one_map()
4085 memcmp(&osd->o_con.peer_addr, in handle_one_map()
4086 ceph_osd_addr(osdc->osdmap, osd->o_osd), in handle_one_map()
4109 if (req->r_t.epoch < osdc->osdmap->epoch) { in kick_requests()
4110 ct_res = calc_target(osdc, &req->r_t, false); in kick_requests()
4126 osd = lookup_create_osd(osdc, req->r_t.osd, true); in kick_requests()
4128 if (!req->r_linger) { in kick_requests()
4129 if (!osd_homeless(osd) && !req->r_t.paused) in kick_requests()
4137 if (!osd_homeless(lreq->osd)) in kick_requests()
4140 list_del_init(&lreq->scan_item); in kick_requests()
4153 void *p = msg->front.iov_base; in ceph_osdc_handle_map()
4154 void *const end = p + msg->front.iov_len; in ceph_osdc_handle_map()
4165 dout("%s have %u\n", __func__, osdc->osdmap->epoch); in ceph_osdc_handle_map()
4166 down_write(&osdc->lock); in ceph_osdc_handle_map()
4171 if (ceph_check_fsid(osdc->client, &fsid) < 0) in ceph_osdc_handle_map()
4181 dout(" %d inc maps\n", nr_maps); in ceph_osdc_handle_map()
4187 if (osdc->osdmap->epoch && in ceph_osdc_handle_map()
4188 osdc->osdmap->epoch + 1 == epoch) { in ceph_osdc_handle_map()
4189 dout("applying incremental map %u len %d\n", in ceph_osdc_handle_map()
4197 dout("ignoring incremental map %u len %d\n", in ceph_osdc_handle_map()
4201 nr_maps--; in ceph_osdc_handle_map()
4208 dout(" %d full maps\n", nr_maps); in ceph_osdc_handle_map()
4215 dout("skipping non-latest full map %u len %d\n", in ceph_osdc_handle_map()
4217 } else if (osdc->osdmap->epoch >= epoch) { in ceph_osdc_handle_map()
4218 dout("skipping full map %u len %d, " in ceph_osdc_handle_map()
4220 osdc->osdmap->epoch); in ceph_osdc_handle_map()
4222 dout("taking full map %u len %d\n", epoch, maplen); in ceph_osdc_handle_map()
4229 nr_maps--; in ceph_osdc_handle_map()
4243 osdc->osdmap->epoch < osdc->epoch_barrier) in ceph_osdc_handle_map()
4249 ceph_monc_got_map(&osdc->client->monc, CEPH_SUB_OSDMAP, in ceph_osdc_handle_map()
4250 osdc->osdmap->epoch); in ceph_osdc_handle_map()
4251 up_write(&osdc->lock); in ceph_osdc_handle_map()
4252 wake_up_all(&osdc->client->auth_wq); in ceph_osdc_handle_map()
4258 up_write(&osdc->lock); in ceph_osdc_handle_map()
4270 for (n = rb_first(&osd->o_requests); n; ) { in kick_osd_requests()
4276 if (!req->r_linger) { in kick_osd_requests()
4277 if (!req->r_t.paused) in kick_osd_requests()
4283 for (n = rb_first(&osd->o_linger_requests); n; n = rb_next(n)) { in kick_osd_requests()
4296 struct ceph_osd *osd = con->private; in osd_fault()
4297 struct ceph_osd_client *osdc = osd->o_osdc; in osd_fault()
4299 dout("%s osd %p osd%d\n", __func__, osd, osd->o_osd); in osd_fault()
4301 down_write(&osdc->lock); in osd_fault()
4303 dout("%s osd%d unknown\n", __func__, osd->o_osd); in osd_fault()
4312 up_write(&osdc->lock); in osd_fault()
4326 void *p = msg->front.iov_base; in decode_MOSDBackoff()
4327 void *const end = p + msg->front.iov_len; in decode_MOSDBackoff()
4336 ret = ceph_decode_pgid(&p, end, &m->spgid.pgid); in decode_MOSDBackoff()
4340 ceph_decode_8_safe(&p, end, m->spgid.shard, e_inval); in decode_MOSDBackoff()
4341 ceph_decode_32_safe(&p, end, m->map_epoch, e_inval); in decode_MOSDBackoff()
4342 ceph_decode_8_safe(&p, end, m->op, e_inval); in decode_MOSDBackoff()
4343 ceph_decode_64_safe(&p, end, m->id, e_inval); in decode_MOSDBackoff()
4345 m->begin = kzalloc(sizeof(*m->begin), GFP_NOIO); in decode_MOSDBackoff()
4346 if (!m->begin) in decode_MOSDBackoff()
4347 return -ENOMEM; in decode_MOSDBackoff()
4349 ret = decode_hoid(&p, end, m->begin); in decode_MOSDBackoff()
4351 free_hoid(m->begin); in decode_MOSDBackoff()
4355 m->end = kzalloc(sizeof(*m->end), GFP_NOIO); in decode_MOSDBackoff()
4356 if (!m->end) { in decode_MOSDBackoff()
4357 free_hoid(m->begin); in decode_MOSDBackoff()
4358 return -ENOMEM; in decode_MOSDBackoff()
4361 ret = decode_hoid(&p, end, m->end); in decode_MOSDBackoff()
4363 free_hoid(m->begin); in decode_MOSDBackoff()
4364 free_hoid(m->end); in decode_MOSDBackoff()
4371 return -EINVAL; in decode_MOSDBackoff()
4386 hoid_encoding_size(backoff->begin); in create_backoff_message()
4388 hoid_encoding_size(backoff->end); in create_backoff_message()
4394 p = msg->front.iov_base; in create_backoff_message()
4395 end = p + msg->front_alloc_len; in create_backoff_message()
4397 encode_spgid(&p, &backoff->spgid); in create_backoff_message()
4400 ceph_encode_64(&p, backoff->id); in create_backoff_message()
4401 encode_hoid(&p, end, backoff->begin); in create_backoff_message()
4402 encode_hoid(&p, end, backoff->end); in create_backoff_message()
4405 msg->front.iov_len = p - msg->front.iov_base; in create_backoff_message()
4406 msg->hdr.version = cpu_to_le16(1); /* MOSDBackoff v1 */ in create_backoff_message()
4407 msg->hdr.front_len = cpu_to_le32(msg->front.iov_len); in create_backoff_message()
4418 dout("%s osd%d spgid %llu.%xs%d id %llu\n", __func__, osd->o_osd, in handle_backoff_block()
4419 m->spgid.pgid.pool, m->spgid.pgid.seed, m->spgid.shard, m->id); in handle_backoff_block()
4421 spg = lookup_spg_mapping(&osd->o_backoff_mappings, &m->spgid); in handle_backoff_block()
4428 spg->spgid = m->spgid; /* struct */ in handle_backoff_block()
4429 insert_spg_mapping(&osd->o_backoff_mappings, spg); in handle_backoff_block()
4437 backoff->spgid = m->spgid; /* struct */ in handle_backoff_block()
4438 backoff->id = m->id; in handle_backoff_block()
4439 backoff->begin = m->begin; in handle_backoff_block()
4440 m->begin = NULL; /* backoff now owns this */ in handle_backoff_block()
4441 backoff->end = m->end; in handle_backoff_block()
4442 m->end = NULL; /* ditto */ in handle_backoff_block()
4444 insert_backoff(&spg->backoffs, backoff); in handle_backoff_block()
4445 insert_backoff_by_id(&osd->o_backoffs_by_id, backoff); in handle_backoff_block()
4451 msg = create_backoff_message(backoff, m->map_epoch); in handle_backoff_block()
4456 ceph_con_send(&osd->o_con, msg); in handle_backoff_block()
4478 dout("%s osd%d spgid %llu.%xs%d id %llu\n", __func__, osd->o_osd, in handle_backoff_unblock()
4479 m->spgid.pgid.pool, m->spgid.pgid.seed, m->spgid.shard, m->id); in handle_backoff_unblock()
4481 backoff = lookup_backoff_by_id(&osd->o_backoffs_by_id, m->id); in handle_backoff_unblock()
4484 __func__, osd->o_osd, m->spgid.pgid.pool, in handle_backoff_unblock()
4485 m->spgid.pgid.seed, m->spgid.shard, m->id); in handle_backoff_unblock()
4489 if (hoid_compare(backoff->begin, m->begin) && in handle_backoff_unblock()
4490 hoid_compare(backoff->end, m->end)) { in handle_backoff_unblock()
4492 __func__, osd->o_osd, m->spgid.pgid.pool, in handle_backoff_unblock()
4493 m->spgid.pgid.seed, m->spgid.shard, m->id); in handle_backoff_unblock()
4497 spg = lookup_spg_mapping(&osd->o_backoff_mappings, &backoff->spgid); in handle_backoff_unblock()
4500 erase_backoff(&spg->backoffs, backoff); in handle_backoff_unblock()
4501 erase_backoff_by_id(&osd->o_backoffs_by_id, backoff); in handle_backoff_unblock()
4504 if (RB_EMPTY_ROOT(&spg->backoffs)) { in handle_backoff_unblock()
4505 erase_spg_mapping(&osd->o_backoff_mappings, spg); in handle_backoff_unblock()
4509 for (n = rb_first(&osd->o_requests); n; n = rb_next(n)) { in handle_backoff_unblock()
4513 if (!ceph_spg_compare(&req->r_t.spgid, &m->spgid)) { in handle_backoff_unblock()
4515 * Match against @m, not @backoff -- the PG may in handle_backoff_unblock()
4518 if (target_contained_by(&req->r_t, m->begin, m->end)) { in handle_backoff_unblock()
4531 struct ceph_osd_client *osdc = osd->o_osdc; in handle_backoff()
4535 down_read(&osdc->lock); in handle_backoff()
4537 dout("%s osd%d unknown\n", __func__, osd->o_osd); in handle_backoff()
4538 up_read(&osdc->lock); in handle_backoff()
4541 WARN_ON(osd->o_osd != le64_to_cpu(msg->hdr.src.num)); in handle_backoff()
4543 mutex_lock(&osd->lock); in handle_backoff()
4558 default: in handle_backoff()
4559 pr_err("%s osd%d unknown op %d\n", __func__, osd->o_osd, m.op); in handle_backoff()
4566 mutex_unlock(&osd->lock); in handle_backoff()
4567 up_read(&osdc->lock); in handle_backoff()
4576 void *p = msg->front.iov_base; in handle_watch_notify()
4577 void *const end = p + msg->front.iov_len; in handle_watch_notify()
4600 if (le16_to_cpu(msg->hdr.version) >= 2) in handle_watch_notify()
4603 if (le16_to_cpu(msg->hdr.version) >= 3) in handle_watch_notify()
4606 down_read(&osdc->lock); in handle_watch_notify()
4607 lreq = lookup_linger_osdc(&osdc->linger_requests, cookie); in handle_watch_notify()
4609 dout("%s opcode %d cookie %llu dne\n", __func__, opcode, in handle_watch_notify()
4614 mutex_lock(&lreq->lock); in handle_watch_notify()
4615 dout("%s opcode %d cookie %llu lreq %p is_watch %d\n", __func__, in handle_watch_notify()
4616 opcode, cookie, lreq, lreq->is_watch); in handle_watch_notify()
4618 if (!lreq->last_error) { in handle_watch_notify()
4619 lreq->last_error = -ENOTCONN; in handle_watch_notify()
4622 } else if (!lreq->is_watch) { in handle_watch_notify()
4624 if (lreq->notify_id && lreq->notify_id != notify_id) { in handle_watch_notify()
4625 dout("lreq %p notify_id %llu != %llu, ignoring\n", lreq, in handle_watch_notify()
4626 lreq->notify_id, notify_id); in handle_watch_notify()
4627 } else if (!completion_done(&lreq->notify_finish_wait)) { in handle_watch_notify()
4629 msg->num_data_items ? &msg->data[0] : NULL; in handle_watch_notify()
4632 if (lreq->preply_pages) { in handle_watch_notify()
4633 WARN_ON(data->type != in handle_watch_notify()
4635 *lreq->preply_pages = data->pages; in handle_watch_notify()
4636 *lreq->preply_len = data->length; in handle_watch_notify()
4637 data->own_pages = false; in handle_watch_notify()
4640 lreq->notify_finish_error = return_code; in handle_watch_notify()
4641 complete_all(&lreq->notify_finish_wait); in handle_watch_notify()
4647 pr_err("failed to allocate notify-lwork\n"); in handle_watch_notify()
4651 lwork->notify.notify_id = notify_id; in handle_watch_notify()
4652 lwork->notify.notifier_id = notifier_id; in handle_watch_notify()
4653 lwork->notify.payload = payload; in handle_watch_notify()
4654 lwork->notify.payload_len = payload_len; in handle_watch_notify()
4655 lwork->notify.msg = ceph_msg_get(msg); in handle_watch_notify()
4660 mutex_unlock(&lreq->lock); in handle_watch_notify()
4662 up_read(&osdc->lock); in handle_watch_notify()
4675 down_read(&osdc->lock); in ceph_osdc_start_request()
4677 up_read(&osdc->lock); in ceph_osdc_start_request()
4692 struct ceph_osd_client *osdc = req->r_osdc; in ceph_osdc_cancel_request()
4694 down_write(&osdc->lock); in ceph_osdc_cancel_request()
4695 if (req->r_osd) in ceph_osdc_cancel_request()
4697 up_write(&osdc->lock); in ceph_osdc_cancel_request()
4709 dout("%s req %p tid %llu\n", __func__, req, req->r_tid); in wait_request_timeout()
4710 left = wait_for_completion_killable_timeout(&req->r_completion, in wait_request_timeout()
4713 left = left ?: -ETIMEDOUT; in wait_request_timeout()
4716 left = req->r_result; /* completed */ in wait_request_timeout()
4733 * sync - wait for all in-flight requests to flush. avoid starvation.
4738 u64 last_tid = atomic64_read(&osdc->last_tid); in ceph_osdc_sync()
4741 down_read(&osdc->lock); in ceph_osdc_sync()
4742 for (n = rb_first(&osdc->osds); n; n = rb_next(n)) { in ceph_osdc_sync()
4745 mutex_lock(&osd->lock); in ceph_osdc_sync()
4746 for (p = rb_first(&osd->o_requests); p; p = rb_next(p)) { in ceph_osdc_sync()
4750 if (req->r_tid > last_tid) in ceph_osdc_sync()
4753 if (!(req->r_flags & CEPH_OSD_FLAG_WRITE)) in ceph_osdc_sync()
4757 mutex_unlock(&osd->lock); in ceph_osdc_sync()
4758 up_read(&osdc->lock); in ceph_osdc_sync()
4759 dout("%s waiting on req %p tid %llu last_tid %llu\n", in ceph_osdc_sync()
4760 __func__, req, req->r_tid, last_tid); in ceph_osdc_sync()
4761 wait_for_completion(&req->r_completion); in ceph_osdc_sync()
4766 mutex_unlock(&osd->lock); in ceph_osdc_sync()
4769 up_read(&osdc->lock); in ceph_osdc_sync()
4770 dout("%s done last_tid %llu\n", __func__, last_tid); in ceph_osdc_sync()
4790 return ERR_PTR(-ENOMEM); in ceph_osdc_watch()
4792 lreq->is_watch = true; in ceph_osdc_watch()
4793 lreq->wcb = wcb; in ceph_osdc_watch()
4794 lreq->errcb = errcb; in ceph_osdc_watch()
4795 lreq->data = data; in ceph_osdc_watch()
4796 lreq->watch_valid_thru = jiffies; in ceph_osdc_watch()
4798 ceph_oid_copy(&lreq->t.base_oid, oid); in ceph_osdc_watch()
4799 ceph_oloc_copy(&lreq->t.base_oloc, oloc); in ceph_osdc_watch()
4800 lreq->t.flags = CEPH_OSD_FLAG_WRITE; in ceph_osdc_watch()
4801 ktime_get_real_ts64(&lreq->mtime); in ceph_osdc_watch()
4828 struct ceph_options *opts = osdc->client->options; in ceph_osdc_unwatch()
4834 return -ENOMEM; in ceph_osdc_unwatch()
4836 ceph_oid_copy(&req->r_base_oid, &lreq->t.base_oid); in ceph_osdc_unwatch()
4837 ceph_oloc_copy(&req->r_base_oloc, &lreq->t.base_oloc); in ceph_osdc_unwatch()
4838 req->r_flags = CEPH_OSD_FLAG_WRITE; in ceph_osdc_unwatch()
4839 ktime_get_real_ts64(&req->r_mtime); in ceph_osdc_unwatch()
4841 lreq->linger_id, 0); in ceph_osdc_unwatch()
4850 ret = wait_request_timeout(req, opts->mount_timeout); in ceph_osdc_unwatch()
4870 return -ENOMEM; in osd_req_op_notify_ack_init()
4882 return -ENOMEM; in osd_req_op_notify_ack_init()
4885 ceph_osd_data_pagelist_init(&op->notify_ack.request_data, pl); in osd_req_op_notify_ack_init()
4886 op->indata_len = pl->length; in osd_req_op_notify_ack_init()
4903 return -ENOMEM; in ceph_osdc_notify_ack()
4905 ceph_oid_copy(&req->r_base_oid, oid); in ceph_osdc_notify_ack()
4906 ceph_oloc_copy(&req->r_base_oloc, oloc); in ceph_osdc_notify_ack()
4907 req->r_flags = CEPH_OSD_FLAG_READ; in ceph_osdc_notify_ack()
4955 return -ENOMEM; in ceph_osdc_notify()
4957 lreq->request_pl = ceph_pagelist_alloc(GFP_NOIO); in ceph_osdc_notify()
4958 if (!lreq->request_pl) { in ceph_osdc_notify()
4959 ret = -ENOMEM; in ceph_osdc_notify()
4963 ret = ceph_pagelist_encode_32(lreq->request_pl, 1); /* prot_ver */ in ceph_osdc_notify()
4964 ret |= ceph_pagelist_encode_32(lreq->request_pl, timeout); in ceph_osdc_notify()
4965 ret |= ceph_pagelist_encode_32(lreq->request_pl, payload_len); in ceph_osdc_notify()
4966 ret |= ceph_pagelist_append(lreq->request_pl, payload, payload_len); in ceph_osdc_notify()
4968 ret = -ENOMEM; in ceph_osdc_notify()
4973 lreq->notify_id_pages = ceph_alloc_page_vector(1, GFP_NOIO); in ceph_osdc_notify()
4974 if (IS_ERR(lreq->notify_id_pages)) { in ceph_osdc_notify()
4975 ret = PTR_ERR(lreq->notify_id_pages); in ceph_osdc_notify()
4976 lreq->notify_id_pages = NULL; in ceph_osdc_notify()
4980 lreq->preply_pages = preply_pages; in ceph_osdc_notify()
4981 lreq->preply_len = preply_len; in ceph_osdc_notify()
4983 ceph_oid_copy(&lreq->t.base_oid, oid); in ceph_osdc_notify()
4984 ceph_oloc_copy(&lreq->t.base_oloc, oloc); in ceph_osdc_notify()
4985 lreq->t.flags = CEPH_OSD_FLAG_READ; in ceph_osdc_notify()
4993 dout("lreq %p failed to initiate notify %d\n", lreq, ret); in ceph_osdc_notify()
5013 down_read(&osdc->lock); in ceph_osdc_watch_check()
5014 mutex_lock(&lreq->lock); in ceph_osdc_watch_check()
5015 stamp = lreq->watch_valid_thru; in ceph_osdc_watch_check()
5016 if (!list_empty(&lreq->pending_lworks)) { in ceph_osdc_watch_check()
5018 list_first_entry(&lreq->pending_lworks, in ceph_osdc_watch_check()
5022 if (time_before(lwork->queued_stamp, stamp)) in ceph_osdc_watch_check()
5023 stamp = lwork->queued_stamp; in ceph_osdc_watch_check()
5025 age = jiffies - stamp; in ceph_osdc_watch_check()
5026 dout("%s lreq %p linger_id %llu age %lu last_error %d\n", __func__, in ceph_osdc_watch_check()
5027 lreq, lreq->linger_id, age, lreq->last_error); in ceph_osdc_watch_check()
5029 ret = lreq->last_error ?: 1 + jiffies_to_msecs(age); in ceph_osdc_watch_check()
5031 mutex_unlock(&lreq->lock); in ceph_osdc_watch_check()
5032 up_read(&osdc->lock); in ceph_osdc_watch_check()
5047 ret = -EINVAL; in decode_watcher()
5048 ceph_decode_copy_safe(p, end, &item->name, sizeof(item->name), bad); in decode_watcher()
5049 ceph_decode_64_safe(p, end, item->cookie, bad); in decode_watcher()
5053 ret = ceph_decode_entity_addr(p, end, &item->addr); in decode_watcher()
5060 dout("%s %s%llu cookie %llu addr %s\n", __func__, in decode_watcher()
5061 ENTITY_NAME(item->name), item->cookie, in decode_watcher()
5062 ceph_pr_addr(&item->addr)); in decode_watcher()
5084 return -ENOMEM; in decode_watchers()
5114 return -ENOMEM; in ceph_osdc_list_watchers()
5116 ceph_oid_copy(&req->r_base_oid, oid); in ceph_osdc_list_watchers()
5117 ceph_oloc_copy(&req->r_base_oloc, oloc); in ceph_osdc_list_watchers()
5118 req->r_flags = CEPH_OSD_FLAG_READ; in ceph_osdc_list_watchers()
5139 void *const end = p + req->r_ops[0].outdata_len; in ceph_osdc_list_watchers()
5151 * Call all pending notify callbacks - for use after a watch is
5156 dout("%s osdc %p\n", __func__, osdc); in ceph_osdc_flush_notifies()
5157 flush_workqueue(osdc->notify_wq); in ceph_osdc_flush_notifies()
5163 down_read(&osdc->lock); in ceph_osdc_maybe_request_map()
5165 up_read(&osdc->lock); in ceph_osdc_maybe_request_map()
5187 return -E2BIG; in ceph_osdc_call()
5191 return -ENOMEM; in ceph_osdc_call()
5193 ceph_oid_copy(&req->r_base_oid, oid); in ceph_osdc_call()
5194 ceph_oloc_copy(&req->r_base_oloc, oloc); in ceph_osdc_call()
5195 req->r_flags = flags; in ceph_osdc_call()
5215 ret = req->r_ops[0].rval; in ceph_osdc_call()
5217 *resp_len = req->r_ops[0].outdata_len; in ceph_osdc_call()
5233 down_write(&osdc->lock); in ceph_osdc_reopen_osds()
5234 for (n = rb_first(&osdc->osds); n; ) { in ceph_osdc_reopen_osds()
5241 up_write(&osdc->lock); in ceph_osdc_reopen_osds()
5251 dout("init\n"); in ceph_osdc_init()
5252 osdc->client = client; in ceph_osdc_init()
5253 init_rwsem(&osdc->lock); in ceph_osdc_init()
5254 osdc->osds = RB_ROOT; in ceph_osdc_init()
5255 INIT_LIST_HEAD(&osdc->osd_lru); in ceph_osdc_init()
5256 spin_lock_init(&osdc->osd_lru_lock); in ceph_osdc_init()
5257 osd_init(&osdc->homeless_osd); in ceph_osdc_init()
5258 osdc->homeless_osd.o_osdc = osdc; in ceph_osdc_init()
5259 osdc->homeless_osd.o_osd = CEPH_HOMELESS_OSD; in ceph_osdc_init()
5260 osdc->last_linger_id = CEPH_LINGER_ID_START; in ceph_osdc_init()
5261 osdc->linger_requests = RB_ROOT; in ceph_osdc_init()
5262 osdc->map_checks = RB_ROOT; in ceph_osdc_init()
5263 osdc->linger_map_checks = RB_ROOT; in ceph_osdc_init()
5264 INIT_DELAYED_WORK(&osdc->timeout_work, handle_timeout); in ceph_osdc_init()
5265 INIT_DELAYED_WORK(&osdc->osds_timeout_work, handle_osds_timeout); in ceph_osdc_init()
5267 err = -ENOMEM; in ceph_osdc_init()
5268 osdc->osdmap = ceph_osdmap_alloc(); in ceph_osdc_init()
5269 if (!osdc->osdmap) in ceph_osdc_init()
5272 osdc->req_mempool = mempool_create_slab_pool(10, in ceph_osdc_init()
5274 if (!osdc->req_mempool) in ceph_osdc_init()
5277 err = ceph_msgpool_init(&osdc->msgpool_op, CEPH_MSG_OSD_OP, in ceph_osdc_init()
5281 err = ceph_msgpool_init(&osdc->msgpool_op_reply, CEPH_MSG_OSD_OPREPLY, in ceph_osdc_init()
5287 err = -ENOMEM; in ceph_osdc_init()
5288 osdc->notify_wq = create_singlethread_workqueue("ceph-watch-notify"); in ceph_osdc_init()
5289 if (!osdc->notify_wq) in ceph_osdc_init()
5292 osdc->completion_wq = create_singlethread_workqueue("ceph-completion"); in ceph_osdc_init()
5293 if (!osdc->completion_wq) in ceph_osdc_init()
5296 schedule_delayed_work(&osdc->timeout_work, in ceph_osdc_init()
5297 osdc->client->options->osd_keepalive_timeout); in ceph_osdc_init()
5298 schedule_delayed_work(&osdc->osds_timeout_work, in ceph_osdc_init()
5299 round_jiffies_relative(osdc->client->options->osd_idle_ttl)); in ceph_osdc_init()
5304 destroy_workqueue(osdc->notify_wq); in ceph_osdc_init()
5306 ceph_msgpool_destroy(&osdc->msgpool_op_reply); in ceph_osdc_init()
5308 ceph_msgpool_destroy(&osdc->msgpool_op); in ceph_osdc_init()
5310 mempool_destroy(osdc->req_mempool); in ceph_osdc_init()
5312 ceph_osdmap_destroy(osdc->osdmap); in ceph_osdc_init()
5319 destroy_workqueue(osdc->completion_wq); in ceph_osdc_stop()
5320 destroy_workqueue(osdc->notify_wq); in ceph_osdc_stop()
5321 cancel_delayed_work_sync(&osdc->timeout_work); in ceph_osdc_stop()
5322 cancel_delayed_work_sync(&osdc->osds_timeout_work); in ceph_osdc_stop()
5324 down_write(&osdc->lock); in ceph_osdc_stop()
5325 while (!RB_EMPTY_ROOT(&osdc->osds)) { in ceph_osdc_stop()
5326 struct ceph_osd *osd = rb_entry(rb_first(&osdc->osds), in ceph_osdc_stop()
5330 up_write(&osdc->lock); in ceph_osdc_stop()
5331 WARN_ON(refcount_read(&osdc->homeless_osd.o_ref) != 1); in ceph_osdc_stop()
5332 osd_cleanup(&osdc->homeless_osd); in ceph_osdc_stop()
5334 WARN_ON(!list_empty(&osdc->osd_lru)); in ceph_osdc_stop()
5335 WARN_ON(!RB_EMPTY_ROOT(&osdc->linger_requests)); in ceph_osdc_stop()
5336 WARN_ON(!RB_EMPTY_ROOT(&osdc->map_checks)); in ceph_osdc_stop()
5337 WARN_ON(!RB_EMPTY_ROOT(&osdc->linger_map_checks)); in ceph_osdc_stop()
5338 WARN_ON(atomic_read(&osdc->num_requests)); in ceph_osdc_stop()
5339 WARN_ON(atomic_read(&osdc->num_homeless)); in ceph_osdc_stop()
5341 ceph_osdmap_destroy(osdc->osdmap); in ceph_osdc_stop()
5342 mempool_destroy(osdc->req_mempool); in ceph_osdc_stop()
5343 ceph_msgpool_destroy(&osdc->msgpool_op); in ceph_osdc_stop()
5344 ceph_msgpool_destroy(&osdc->msgpool_op_reply); in ceph_osdc_stop()
5366 op->copy_from.snapid = src_snapid; in osd_req_op_copy_from_init()
5367 op->copy_from.src_version = src_version; in osd_req_op_copy_from_init()
5368 op->copy_from.flags = copy_from_flags; in osd_req_op_copy_from_init()
5369 op->copy_from.src_fadvise_flags = src_fadvise_flags; in osd_req_op_copy_from_init()
5373 ceph_encode_string(&p, end, src_oid->name, src_oid->name_len); in osd_req_op_copy_from_init()
5377 op->indata_len = PAGE_SIZE - (end - p); in osd_req_op_copy_from_init()
5379 ceph_osd_data_pages_init(&op->copy_from.osd_data, pages, in osd_req_op_copy_from_init()
5380 op->indata_len, 0, false, true); in osd_req_op_copy_from_init()
5394 return ceph_osd_request_cache ? 0 : -ENOMEM; in ceph_osdc_setup()
5409 struct ceph_osd *osd = con->private; in osd_dispatch()
5410 struct ceph_osd_client *osdc = osd->o_osdc; in osd_dispatch()
5411 int type = le16_to_cpu(msg->hdr.type); in osd_dispatch()
5427 default: in osd_dispatch()
5440 if (req->r_flags & CEPH_OSD_FLAG_READ) { in sparse_data_requested()
5443 for (i = 0; i < req->r_num_ops; ++i) { in sparse_data_requested()
5444 struct ceph_osd_req_op *op = &req->r_ops[i]; in sparse_data_requested()
5446 if (op->op == CEPH_OSD_OP_SPARSE_READ) in sparse_data_requested()
5447 len += op->extent.length; in sparse_data_requested()
5456 * message at the moment - for now, just skip the message.
5462 struct ceph_osd *osd = con->private; in get_reply()
5463 struct ceph_osd_client *osdc = osd->o_osdc; in get_reply()
5466 int front_len = le32_to_cpu(hdr->front_len); in get_reply()
5467 int data_len = le32_to_cpu(hdr->data_len); in get_reply()
5468 u64 tid = le64_to_cpu(hdr->tid); in get_reply()
5471 down_read(&osdc->lock); in get_reply()
5473 dout("%s osd%d unknown, skipping\n", __func__, osd->o_osd); in get_reply()
5477 WARN_ON(osd->o_osd != le64_to_cpu(hdr->src.num)); in get_reply()
5479 mutex_lock(&osd->lock); in get_reply()
5480 req = lookup_request(&osd->o_requests, tid); in get_reply()
5482 dout("%s osd%d tid %llu unknown, skipping\n", __func__, in get_reply()
5483 osd->o_osd, tid); in get_reply()
5488 ceph_msg_revoke_incoming(req->r_reply); in get_reply()
5490 if (front_len > req->r_reply->front_alloc_len) { in get_reply()
5492 __func__, osd->o_osd, req->r_tid, front_len, in get_reply()
5493 req->r_reply->front_alloc_len); in get_reply()
5498 ceph_msg_put(req->r_reply); in get_reply()
5499 req->r_reply = m; in get_reply()
5503 if (!srlen && data_len > req->r_reply->data_length) { in get_reply()
5505 __func__, osd->o_osd, req->r_tid, data_len, in get_reply()
5506 req->r_reply->data_length); in get_reply()
5512 m = ceph_msg_get(req->r_reply); in get_reply()
5513 m->sparse_read_total = srlen; in get_reply()
5515 dout("get_reply tid %lld %p\n", tid, m); in get_reply()
5518 mutex_unlock(&osd->lock); in get_reply()
5520 up_read(&osdc->lock); in get_reply()
5527 int type = le16_to_cpu(hdr->type); in alloc_msg_with_page_vector()
5528 u32 front_len = le32_to_cpu(hdr->front_len); in alloc_msg_with_page_vector()
5529 u32 data_len = le32_to_cpu(hdr->data_len); in alloc_msg_with_page_vector()
5555 struct ceph_osd *osd = con->private; in osd_alloc_msg()
5556 int type = le16_to_cpu(hdr->type); in osd_alloc_msg()
5566 default: in osd_alloc_msg()
5568 osd->o_osd, type); in osd_alloc_msg()
5579 struct ceph_osd *osd = con->private; in osd_get_con()
5587 struct ceph_osd *osd = con->private; in osd_put_con()
5602 struct ceph_osd *o = con->private; in osd_get_authorizer()
5603 struct ceph_osd_client *osdc = o->o_osdc; in osd_get_authorizer()
5604 struct ceph_auth_client *ac = osdc->client->monc.auth; in osd_get_authorizer()
5605 struct ceph_auth_handshake *auth = &o->o_auth; in osd_get_authorizer()
5619 struct ceph_osd *o = con->private; in osd_add_authorizer_challenge()
5620 struct ceph_osd_client *osdc = o->o_osdc; in osd_add_authorizer_challenge()
5621 struct ceph_auth_client *ac = osdc->client->monc.auth; in osd_add_authorizer_challenge()
5623 return ceph_auth_add_authorizer_challenge(ac, o->o_auth.authorizer, in osd_add_authorizer_challenge()
5629 struct ceph_osd *o = con->private; in osd_verify_authorizer_reply()
5630 struct ceph_osd_client *osdc = o->o_osdc; in osd_verify_authorizer_reply()
5631 struct ceph_auth_client *ac = osdc->client->monc.auth; in osd_verify_authorizer_reply()
5632 struct ceph_auth_handshake *auth = &o->o_auth; in osd_verify_authorizer_reply()
5634 return ceph_auth_verify_authorizer_reply(ac, auth->authorizer, in osd_verify_authorizer_reply()
5635 auth->authorizer_reply_buf, auth->authorizer_reply_buf_len, in osd_verify_authorizer_reply()
5641 struct ceph_osd *o = con->private; in osd_invalidate_authorizer()
5642 struct ceph_osd_client *osdc = o->o_osdc; in osd_invalidate_authorizer()
5643 struct ceph_auth_client *ac = osdc->client->monc.auth; in osd_invalidate_authorizer()
5646 return ceph_monc_validate_auth(&osdc->client->monc); in osd_invalidate_authorizer()
5653 struct ceph_osd *o = con->private; in osd_get_auth_request()
5654 struct ceph_auth_client *ac = o->o_osdc->client->monc.auth; in osd_get_auth_request()
5655 struct ceph_auth_handshake *auth = &o->o_auth; in osd_get_auth_request()
5663 *authorizer = auth->authorizer_buf; in osd_get_auth_request()
5664 *authorizer_len = auth->authorizer_buf_len; in osd_get_auth_request()
5673 struct ceph_osd *o = con->private; in osd_handle_auth_reply_more()
5674 struct ceph_auth_client *ac = o->o_osdc->client->monc.auth; in osd_handle_auth_reply_more()
5675 struct ceph_auth_handshake *auth = &o->o_auth; in osd_handle_auth_reply_more()
5683 *authorizer = auth->authorizer_buf; in osd_handle_auth_reply_more()
5684 *authorizer_len = auth->authorizer_buf_len; in osd_handle_auth_reply_more()
5693 struct ceph_osd *o = con->private; in osd_handle_auth_done()
5694 struct ceph_auth_client *ac = o->o_osdc->client->monc.auth; in osd_handle_auth_done()
5695 struct ceph_auth_handshake *auth = &o->o_auth; in osd_handle_auth_done()
5707 struct ceph_osd *o = con->private; in osd_handle_auth_bad_method()
5708 struct ceph_mon_client *monc = &o->o_osdc->client->monc; in osd_handle_auth_bad_method()
5711 if (ceph_auth_handle_bad_authorizer(monc->auth, CEPH_ENTITY_TYPE_OSD, in osd_handle_auth_bad_method()
5720 return -EACCES; in osd_handle_auth_bad_method()
5725 int type = le16_to_cpu(msg->hdr.type); in osd_reencode_message()
5733 struct ceph_osd *o = msg->con->private; in osd_sign_message()
5734 struct ceph_auth_handshake *auth = &o->o_auth; in osd_sign_message()
5741 struct ceph_osd *o = msg->con->private; in osd_check_message_signature()
5742 struct ceph_auth_handshake *auth = &o->o_auth; in osd_check_message_signature()
5759 len -= plen; in advance_cursor()
5767 struct ceph_osd *o = con->private; in prep_next_sparse_read()
5768 struct ceph_sparse_read *sr = &o->o_sparse_read; in prep_next_sparse_read()
5772 spin_lock(&o->o_requests_lock); in prep_next_sparse_read()
5773 req = lookup_request(&o->o_requests, le64_to_cpu(con->in_msg->hdr.tid)); in prep_next_sparse_read()
5775 spin_unlock(&o->o_requests_lock); in prep_next_sparse_read()
5776 return -EBADR; in prep_next_sparse_read()
5779 if (o->o_sparse_op_idx < 0) { in prep_next_sparse_read()
5780 dout("%s: [%d] starting new sparse read req\n", in prep_next_sparse_read()
5781 __func__, o->o_osd); in prep_next_sparse_read()
5785 op = &req->r_ops[o->o_sparse_op_idx]; in prep_next_sparse_read()
5787 WARN_ON_ONCE(op->extent.sparse_ext); in prep_next_sparse_read()
5790 op->extent.sparse_ext = sr->sr_extent; in prep_next_sparse_read()
5791 sr->sr_extent = NULL; in prep_next_sparse_read()
5792 op->extent.sparse_ext_cnt = sr->sr_count; in prep_next_sparse_read()
5793 sr->sr_ext_len = 0; in prep_next_sparse_read()
5794 dout("%s: [%d] completed extent array len %d cursor->resid %zd\n", in prep_next_sparse_read()
5795 __func__, o->o_osd, op->extent.sparse_ext_cnt, cursor->resid); in prep_next_sparse_read()
5798 if (end < sr->sr_req_len) in prep_next_sparse_read()
5799 advance_cursor(cursor, sr->sr_req_len - end, false); in prep_next_sparse_read()
5805 while (++o->o_sparse_op_idx < req->r_num_ops) { in prep_next_sparse_read()
5806 op = &req->r_ops[o->o_sparse_op_idx]; in prep_next_sparse_read()
5807 if (op->op == CEPH_OSD_OP_SPARSE_READ) in prep_next_sparse_read()
5812 spin_unlock(&o->o_requests_lock); in prep_next_sparse_read()
5813 o->o_sparse_op_idx = -1; in prep_next_sparse_read()
5816 sr->sr_req_off = op->extent.offset; in prep_next_sparse_read()
5817 sr->sr_req_len = op->extent.length; in prep_next_sparse_read()
5818 sr->sr_pos = sr->sr_req_off; in prep_next_sparse_read()
5819 dout("%s: [%d] new sparse read op at idx %d 0x%llx~0x%llx\n", __func__, in prep_next_sparse_read()
5820 o->o_osd, o->o_sparse_op_idx, sr->sr_req_off, sr->sr_req_len); in prep_next_sparse_read()
5823 sr->sr_ext_len = op->extent.sparse_ext_cnt; in prep_next_sparse_read()
5824 op->extent.sparse_ext_cnt = 0; in prep_next_sparse_read()
5825 sr->sr_extent = op->extent.sparse_ext; in prep_next_sparse_read()
5826 op->extent.sparse_ext = NULL; in prep_next_sparse_read()
5828 spin_unlock(&o->o_requests_lock); in prep_next_sparse_read()
5837 for (i = 0; i < sr->sr_count; i++) { in convert_extent_map()
5838 struct ceph_sparse_extent *ext = &sr->sr_extent[i]; in convert_extent_map()
5840 ext->off = le64_to_cpu((__force __le64)ext->off); in convert_extent_map()
5841 ext->len = le64_to_cpu((__force __le64)ext->len); in convert_extent_map()
5854 struct ceph_osd *o = con->private; in osd_sparse_read()
5855 struct ceph_sparse_read *sr = &o->o_sparse_read; in osd_sparse_read()
5856 u32 count = sr->sr_count; in osd_sparse_read()
5860 switch (sr->sr_state) { in osd_sparse_read()
5868 ret = sizeof(sr->sr_count); in osd_sparse_read()
5869 *pbuf = (char *)&sr->sr_count; in osd_sparse_read()
5870 sr->sr_state = CEPH_SPARSE_READ_EXTENTS; in osd_sparse_read()
5873 /* Convert sr_count to host-endian */ in osd_sparse_read()
5874 count = le32_to_cpu((__force __le32)sr->sr_count); in osd_sparse_read()
5875 sr->sr_count = count; in osd_sparse_read()
5876 dout("[%d] got %u extents\n", o->o_osd, count); in osd_sparse_read()
5879 if (!sr->sr_extent || count > sr->sr_ext_len) { in osd_sparse_read()
5881 kfree(sr->sr_extent); in osd_sparse_read()
5882 sr->sr_extent = kmalloc_array(count, in osd_sparse_read()
5883 sizeof(*sr->sr_extent), in osd_sparse_read()
5885 if (!sr->sr_extent) { in osd_sparse_read()
5888 return -ENOMEM; in osd_sparse_read()
5890 sr->sr_ext_len = count; in osd_sparse_read()
5892 ret = count * sizeof(*sr->sr_extent); in osd_sparse_read()
5893 *pbuf = (char *)sr->sr_extent; in osd_sparse_read()
5894 sr->sr_state = CEPH_SPARSE_READ_DATA_LEN; in osd_sparse_read()
5901 ret = sizeof(sr->sr_datalen); in osd_sparse_read()
5902 *pbuf = (char *)&sr->sr_datalen; in osd_sparse_read()
5903 sr->sr_state = CEPH_SPARSE_READ_DATA_PRE; in osd_sparse_read()
5906 /* Convert sr_datalen to host-endian */ in osd_sparse_read()
5907 sr->sr_datalen = le32_to_cpu((__force __le32)sr->sr_datalen); in osd_sparse_read()
5909 len += sr->sr_extent[i].len; in osd_sparse_read()
5910 if (sr->sr_datalen != len) { in osd_sparse_read()
5912 sr->sr_datalen, len); in osd_sparse_read()
5913 return -EREMOTEIO; in osd_sparse_read()
5915 sr->sr_state = CEPH_SPARSE_READ_DATA; in osd_sparse_read()
5918 if (sr->sr_index >= count) { in osd_sparse_read()
5919 sr->sr_state = CEPH_SPARSE_READ_HDR; in osd_sparse_read()
5923 eoff = sr->sr_extent[sr->sr_index].off; in osd_sparse_read()
5924 elen = sr->sr_extent[sr->sr_index].len; in osd_sparse_read()
5926 dout("[%d] ext %d off 0x%llx len 0x%llx\n", in osd_sparse_read()
5927 o->o_osd, sr->sr_index, eoff, elen); in osd_sparse_read()
5930 dout("Sparse read extent length too long (0x%llx)\n", in osd_sparse_read()
5932 return -EREMOTEIO; in osd_sparse_read()
5936 if (sr->sr_pos < eoff) in osd_sparse_read()
5937 advance_cursor(cursor, eoff - sr->sr_pos, true); in osd_sparse_read()
5940 sr->sr_pos = eoff + elen; in osd_sparse_read()
5943 cursor->sr_resid = elen; in osd_sparse_read()
5948 ++sr->sr_index; in osd_sparse_read()