Lines Matching +full:ext +full:- +full:gen
1 // SPDX-License-Identifier: GPL-2.0
71 WARN_ON(!rwsem_is_locked(&osdc->lock)); in verify_osdc_locked()
75 WARN_ON(!rwsem_is_wrlocked(&osdc->lock)); in verify_osdc_wrlocked()
79 struct ceph_osd_client *osdc = osd->o_osdc; in verify_osd_locked()
81 WARN_ON(!(mutex_is_locked(&osd->lock) && in verify_osd_locked()
82 rwsem_is_locked(&osdc->lock)) && in verify_osd_locked()
83 !rwsem_is_wrlocked(&osdc->lock)); in verify_osd_locked()
87 WARN_ON(!mutex_is_locked(&lreq->lock)); in verify_lreq_locked()
116 orig_len - *plen, off, *plen); in calc_layout()
126 osd_data->type = CEPH_OSD_DATA_TYPE_NONE; in ceph_osd_data_init()
136 osd_data->type = CEPH_OSD_DATA_TYPE_PAGES; in ceph_osd_data_pages_init()
137 osd_data->pages = pages; in ceph_osd_data_pages_init()
138 osd_data->length = length; in ceph_osd_data_pages_init()
139 osd_data->alignment = alignment; in ceph_osd_data_pages_init()
140 osd_data->pages_from_pool = pages_from_pool; in ceph_osd_data_pages_init()
141 osd_data->own_pages = own_pages; in ceph_osd_data_pages_init()
150 osd_data->type = CEPH_OSD_DATA_TYPE_PAGELIST; in ceph_osd_data_pagelist_init()
151 osd_data->pagelist = pagelist; in ceph_osd_data_pagelist_init()
159 osd_data->type = CEPH_OSD_DATA_TYPE_BIO; in ceph_osd_data_bio_init()
160 osd_data->bio_pos = *bio_pos; in ceph_osd_data_bio_init()
161 osd_data->bio_length = bio_length; in ceph_osd_data_bio_init()
169 osd_data->type = CEPH_OSD_DATA_TYPE_BVECS; in ceph_osd_data_bvecs_init()
170 osd_data->bvec_pos = *bvec_pos; in ceph_osd_data_bvecs_init()
171 osd_data->num_bvecs = num_bvecs; in ceph_osd_data_bvecs_init()
177 osd_data->type = CEPH_OSD_DATA_TYPE_ITER; in ceph_osd_iter_init()
178 osd_data->iter = *iter; in ceph_osd_iter_init()
184 BUG_ON(which >= osd_req->r_num_ops); in osd_req_op_raw_data_in()
186 return &osd_req->r_ops[which].raw_data_in; in osd_req_op_raw_data_in()
275 * osd_req_op_extent_osd_iter - Set up an operation with an iterator buffer
308 osd_req->r_ops[which].cls.indata_len += pagelist->length; in osd_req_op_cls_request_data_pagelist()
309 osd_req->r_ops[which].indata_len += pagelist->length; in osd_req_op_cls_request_data_pagelist()
322 osd_req->r_ops[which].cls.indata_len += length; in osd_req_op_cls_request_data_pages()
323 osd_req->r_ops[which].indata_len += length; in osd_req_op_cls_request_data_pages()
340 osd_req->r_ops[which].cls.indata_len += bytes; in osd_req_op_cls_request_data_bvecs()
341 osd_req->r_ops[which].indata_len += bytes; in osd_req_op_cls_request_data_bvecs()
359 switch (osd_data->type) { in ceph_osd_data_length()
363 return osd_data->length; in ceph_osd_data_length()
365 return (u64)osd_data->pagelist->length; in ceph_osd_data_length()
368 return (u64)osd_data->bio_length; in ceph_osd_data_length()
371 return osd_data->bvec_pos.iter.bi_size; in ceph_osd_data_length()
373 return iov_iter_count(&osd_data->iter); in ceph_osd_data_length()
375 WARN(true, "unrecognized data type %d\n", (int)osd_data->type); in ceph_osd_data_length()
382 if (osd_data->type == CEPH_OSD_DATA_TYPE_PAGES && osd_data->own_pages) { in ceph_osd_data_release()
385 num_pages = calc_pages_for((u64)osd_data->alignment, in ceph_osd_data_release()
386 (u64)osd_data->length); in ceph_osd_data_release()
387 ceph_release_page_vector(osd_data->pages, num_pages); in ceph_osd_data_release()
388 } else if (osd_data->type == CEPH_OSD_DATA_TYPE_PAGELIST) { in ceph_osd_data_release()
389 ceph_pagelist_release(osd_data->pagelist); in ceph_osd_data_release()
399 BUG_ON(which >= osd_req->r_num_ops); in osd_req_op_data_release()
400 op = &osd_req->r_ops[which]; in osd_req_op_data_release()
402 switch (op->op) { in osd_req_op_data_release()
407 kfree(op->extent.sparse_ext); in osd_req_op_data_release()
408 ceph_osd_data_release(&op->extent.osd_data); in osd_req_op_data_release()
411 ceph_osd_data_release(&op->cls.request_info); in osd_req_op_data_release()
412 ceph_osd_data_release(&op->cls.request_data); in osd_req_op_data_release()
413 ceph_osd_data_release(&op->cls.response_data); in osd_req_op_data_release()
417 ceph_osd_data_release(&op->xattr.osd_data); in osd_req_op_data_release()
420 ceph_osd_data_release(&op->raw_data_in); in osd_req_op_data_release()
423 ceph_osd_data_release(&op->notify_ack.request_data); in osd_req_op_data_release()
426 ceph_osd_data_release(&op->notify.request_data); in osd_req_op_data_release()
427 ceph_osd_data_release(&op->notify.response_data); in osd_req_op_data_release()
430 ceph_osd_data_release(&op->list_watchers.response_data); in osd_req_op_data_release()
433 ceph_osd_data_release(&op->copy_from.osd_data); in osd_req_op_data_release()
441 * Assumes @t is zero-initialized.
445 ceph_oid_init(&t->base_oid); in target_init()
446 ceph_oloc_init(&t->base_oloc); in target_init()
447 ceph_oid_init(&t->target_oid); in target_init()
448 ceph_oloc_init(&t->target_oloc); in target_init()
450 ceph_osds_init(&t->acting); in target_init()
451 ceph_osds_init(&t->up); in target_init()
452 t->size = -1; in target_init()
453 t->min_size = -1; in target_init()
455 t->osd = CEPH_HOMELESS_OSD; in target_init()
461 ceph_oid_copy(&dest->base_oid, &src->base_oid); in target_copy()
462 ceph_oloc_copy(&dest->base_oloc, &src->base_oloc); in target_copy()
463 ceph_oid_copy(&dest->target_oid, &src->target_oid); in target_copy()
464 ceph_oloc_copy(&dest->target_oloc, &src->target_oloc); in target_copy()
466 dest->pgid = src->pgid; /* struct */ in target_copy()
467 dest->spgid = src->spgid; /* struct */ in target_copy()
468 dest->pg_num = src->pg_num; in target_copy()
469 dest->pg_num_mask = src->pg_num_mask; in target_copy()
470 ceph_osds_copy(&dest->acting, &src->acting); in target_copy()
471 ceph_osds_copy(&dest->up, &src->up); in target_copy()
472 dest->size = src->size; in target_copy()
473 dest->min_size = src->min_size; in target_copy()
474 dest->sort_bitwise = src->sort_bitwise; in target_copy()
475 dest->recovery_deletes = src->recovery_deletes; in target_copy()
477 dest->flags = src->flags; in target_copy()
478 dest->used_replica = src->used_replica; in target_copy()
479 dest->paused = src->paused; in target_copy()
481 dest->epoch = src->epoch; in target_copy()
482 dest->last_force_resend = src->last_force_resend; in target_copy()
484 dest->osd = src->osd; in target_copy()
489 ceph_oid_destroy(&t->base_oid); in target_destroy()
490 ceph_oloc_destroy(&t->base_oloc); in target_destroy()
491 ceph_oid_destroy(&t->target_oid); in target_destroy()
492 ceph_oloc_destroy(&t->target_oloc); in target_destroy()
500 WARN_ON(!RB_EMPTY_NODE(&req->r_node)); in request_release_checks()
501 WARN_ON(!RB_EMPTY_NODE(&req->r_mc_node)); in request_release_checks()
502 WARN_ON(!list_empty(&req->r_private_item)); in request_release_checks()
503 WARN_ON(req->r_osd); in request_release_checks()
513 req->r_request, req->r_reply); in ceph_osdc_release_request()
516 if (req->r_request) in ceph_osdc_release_request()
517 ceph_msg_put(req->r_request); in ceph_osdc_release_request()
518 if (req->r_reply) in ceph_osdc_release_request()
519 ceph_msg_put(req->r_reply); in ceph_osdc_release_request()
521 for (which = 0; which < req->r_num_ops; which++) in ceph_osdc_release_request()
524 target_destroy(&req->r_t); in ceph_osdc_release_request()
525 ceph_put_snap_context(req->r_snapc); in ceph_osdc_release_request()
527 if (req->r_mempool) in ceph_osdc_release_request()
528 mempool_free(req, req->r_osdc->req_mempool); in ceph_osdc_release_request()
529 else if (req->r_num_ops <= CEPH_OSD_SLAB_OPS) in ceph_osdc_release_request()
538 kref_read(&req->r_kref)); in ceph_osdc_get_request()
539 kref_get(&req->r_kref); in ceph_osdc_get_request()
547 kref_read(&req->r_kref)); in ceph_osdc_put_request()
548 kref_put(&req->r_kref, ceph_osdc_release_request); in ceph_osdc_put_request()
558 kref_init(&req->r_kref); in request_init()
559 init_completion(&req->r_completion); in request_init()
560 RB_CLEAR_NODE(&req->r_node); in request_init()
561 RB_CLEAR_NODE(&req->r_mc_node); in request_init()
562 INIT_LIST_HEAD(&req->r_private_item); in request_init()
564 target_init(&req->r_t); in request_init()
577 req = mempool_alloc(osdc->req_mempool, gfp_flags); in ceph_osdc_alloc_request()
588 req->r_osdc = osdc; in ceph_osdc_alloc_request()
589 req->r_mempool = use_mempool; in ceph_osdc_alloc_request()
590 req->r_num_ops = num_ops; in ceph_osdc_alloc_request()
591 req->r_snapid = CEPH_NOSNAP; in ceph_osdc_alloc_request()
592 req->r_snapc = ceph_get_snap_context(snapc); in ceph_osdc_alloc_request()
601 return 8 + 4 + 4 + 4 + (oloc->pool_ns ? oloc->pool_ns->len : 0); in ceph_oloc_encoding_size()
608 struct ceph_osd_client *osdc = req->r_osdc; in __ceph_osdc_alloc_messages()
612 WARN_ON(req->r_request || req->r_reply); in __ceph_osdc_alloc_messages()
613 WARN_ON(ceph_oid_empty(&req->r_base_oid)); in __ceph_osdc_alloc_messages()
614 WARN_ON(ceph_oloc_empty(&req->r_base_oloc)); in __ceph_osdc_alloc_messages()
625 ceph_oloc_encoding_size(&req->r_base_oloc); /* oloc */ in __ceph_osdc_alloc_messages()
626 msg_size += 4 + req->r_base_oid.name_len; /* oid */ in __ceph_osdc_alloc_messages()
627 msg_size += 2 + req->r_num_ops * sizeof(struct ceph_osd_op); in __ceph_osdc_alloc_messages()
630 msg_size += 4 + 8 * (req->r_snapc ? req->r_snapc->num_snaps : 0); in __ceph_osdc_alloc_messages()
633 if (req->r_mempool) in __ceph_osdc_alloc_messages()
634 msg = ceph_msgpool_get(&osdc->msgpool_op, msg_size, in __ceph_osdc_alloc_messages()
640 return -ENOMEM; in __ceph_osdc_alloc_messages()
642 memset(msg->front.iov_base, 0, msg->front.iov_len); in __ceph_osdc_alloc_messages()
643 req->r_request = msg; in __ceph_osdc_alloc_messages()
647 msg_size += req->r_base_oid.name_len; in __ceph_osdc_alloc_messages()
648 msg_size += req->r_num_ops * sizeof(struct ceph_osd_op); in __ceph_osdc_alloc_messages()
650 if (req->r_mempool) in __ceph_osdc_alloc_messages()
651 msg = ceph_msgpool_get(&osdc->msgpool_op_reply, msg_size, in __ceph_osdc_alloc_messages()
657 return -ENOMEM; in __ceph_osdc_alloc_messages()
659 req->r_reply = msg; in __ceph_osdc_alloc_messages()
684 for (op = req->r_ops; op != &req->r_ops[req->r_num_ops]; op++) { in get_num_data_items()
685 switch (op->op) { in get_num_data_items()
715 WARN_ON(!osd_req_opcode_valid(op->op)); in get_num_data_items()
746 BUG_ON(which >= osd_req->r_num_ops); in osd_req_op_init()
749 op = &osd_req->r_ops[which]; in osd_req_op_init()
751 op->op = opcode; in osd_req_op_init()
752 op->flags = flags; in osd_req_op_init()
771 op->extent.offset = offset; in osd_req_op_extent_init()
772 op->extent.length = length; in osd_req_op_extent_init()
773 op->extent.truncate_size = truncate_size; in osd_req_op_extent_init()
774 op->extent.truncate_seq = truncate_seq; in osd_req_op_extent_init()
778 op->indata_len = payload_len; in osd_req_op_extent_init()
788 BUG_ON(which >= osd_req->r_num_ops); in osd_req_op_extent_update()
789 op = &osd_req->r_ops[which]; in osd_req_op_extent_update()
790 previous = op->extent.length; in osd_req_op_extent_update()
796 op->extent.length = length; in osd_req_op_extent_update()
797 if (op->op == CEPH_OSD_OP_WRITE || op->op == CEPH_OSD_OP_WRITEFULL) in osd_req_op_extent_update()
798 op->indata_len -= previous - length; in osd_req_op_extent_update()
807 BUG_ON(which + 1 >= osd_req->r_num_ops); in osd_req_op_extent_dup_last()
809 prev_op = &osd_req->r_ops[which]; in osd_req_op_extent_dup_last()
810 op = osd_req_op_init(osd_req, which + 1, prev_op->op, prev_op->flags); in osd_req_op_extent_dup_last()
812 op->indata_len = prev_op->indata_len; in osd_req_op_extent_dup_last()
813 op->outdata_len = prev_op->outdata_len; in osd_req_op_extent_dup_last()
814 op->extent = prev_op->extent; in osd_req_op_extent_dup_last()
816 op->extent.offset += offset_inc; in osd_req_op_extent_dup_last()
817 op->extent.length -= offset_inc; in osd_req_op_extent_dup_last()
819 if (op->op == CEPH_OSD_OP_WRITE || op->op == CEPH_OSD_OP_WRITEFULL) in osd_req_op_extent_dup_last()
820 op->indata_len -= offset_inc; in osd_req_op_extent_dup_last()
837 return -ENOMEM; in osd_req_op_cls_init()
839 op->cls.class_name = class; in osd_req_op_cls_init()
842 op->cls.class_len = size; in osd_req_op_cls_init()
848 op->cls.method_name = method; in osd_req_op_cls_init()
851 op->cls.method_len = size; in osd_req_op_cls_init()
858 op->indata_len = payload_len; in osd_req_op_cls_init()
881 return -ENOMEM; in osd_req_op_xattr_init()
884 op->xattr.name_len = payload_len; in osd_req_op_xattr_init()
889 op->xattr.value_len = size; in osd_req_op_xattr_init()
895 op->xattr.cmp_op = cmp_op; in osd_req_op_xattr_init()
896 op->xattr.cmp_mode = cmp_mode; in osd_req_op_xattr_init()
898 ceph_osd_data_pagelist_init(&op->xattr.osd_data, pagelist); in osd_req_op_xattr_init()
899 op->indata_len = payload_len; in osd_req_op_xattr_init()
912 u8 watch_opcode, u64 cookie, u32 gen) in osd_req_op_watch_init() argument
917 op->watch.cookie = cookie; in osd_req_op_watch_init()
918 op->watch.op = watch_opcode; in osd_req_op_watch_init()
919 op->watch.gen = gen; in osd_req_op_watch_init()
932 op->notify.cookie = cookie; in osd_req_op_notify_init()
934 ceph_osd_data_pagelist_init(&op->notify.request_data, request_pl); in osd_req_op_notify_init()
935 op->indata_len = request_pl->length; in osd_req_op_notify_init()
950 op->alloc_hint.expected_object_size = expected_object_size; in osd_req_op_alloc_hint_init()
951 op->alloc_hint.expected_write_size = expected_write_size; in osd_req_op_alloc_hint_init()
952 op->alloc_hint.flags = flags; in osd_req_op_alloc_hint_init()
956 * not worth a feature bit. Set FAILOK per-op flag to make in osd_req_op_alloc_hint_init()
959 op->flags |= CEPH_OSD_OP_FLAG_FAILOK; in osd_req_op_alloc_hint_init()
968 if (osd_data->type == CEPH_OSD_DATA_TYPE_PAGES) { in ceph_osdc_msg_data_add()
971 ceph_msg_data_add_pages(msg, osd_data->pages, in ceph_osdc_msg_data_add()
972 length, osd_data->alignment, false); in ceph_osdc_msg_data_add()
973 } else if (osd_data->type == CEPH_OSD_DATA_TYPE_PAGELIST) { in ceph_osdc_msg_data_add()
975 ceph_msg_data_add_pagelist(msg, osd_data->pagelist); in ceph_osdc_msg_data_add()
977 } else if (osd_data->type == CEPH_OSD_DATA_TYPE_BIO) { in ceph_osdc_msg_data_add()
978 ceph_msg_data_add_bio(msg, &osd_data->bio_pos, length); in ceph_osdc_msg_data_add()
980 } else if (osd_data->type == CEPH_OSD_DATA_TYPE_BVECS) { in ceph_osdc_msg_data_add()
981 ceph_msg_data_add_bvecs(msg, &osd_data->bvec_pos); in ceph_osdc_msg_data_add()
982 } else if (osd_data->type == CEPH_OSD_DATA_TYPE_ITER) { in ceph_osdc_msg_data_add()
983 ceph_msg_data_add_iter(msg, &osd_data->iter); in ceph_osdc_msg_data_add()
985 BUG_ON(osd_data->type != CEPH_OSD_DATA_TYPE_NONE); in ceph_osdc_msg_data_add()
992 switch (src->op) { in osd_req_encode_op()
1001 dst->extent.offset = cpu_to_le64(src->extent.offset); in osd_req_encode_op()
1002 dst->extent.length = cpu_to_le64(src->extent.length); in osd_req_encode_op()
1003 dst->extent.truncate_size = in osd_req_encode_op()
1004 cpu_to_le64(src->extent.truncate_size); in osd_req_encode_op()
1005 dst->extent.truncate_seq = in osd_req_encode_op()
1006 cpu_to_le32(src->extent.truncate_seq); in osd_req_encode_op()
1009 dst->cls.class_len = src->cls.class_len; in osd_req_encode_op()
1010 dst->cls.method_len = src->cls.method_len; in osd_req_encode_op()
1011 dst->cls.indata_len = cpu_to_le32(src->cls.indata_len); in osd_req_encode_op()
1014 dst->watch.cookie = cpu_to_le64(src->watch.cookie); in osd_req_encode_op()
1015 dst->watch.ver = cpu_to_le64(0); in osd_req_encode_op()
1016 dst->watch.op = src->watch.op; in osd_req_encode_op()
1017 dst->watch.gen = cpu_to_le32(src->watch.gen); in osd_req_encode_op()
1022 dst->notify.cookie = cpu_to_le64(src->notify.cookie); in osd_req_encode_op()
1027 dst->alloc_hint.expected_object_size = in osd_req_encode_op()
1028 cpu_to_le64(src->alloc_hint.expected_object_size); in osd_req_encode_op()
1029 dst->alloc_hint.expected_write_size = in osd_req_encode_op()
1030 cpu_to_le64(src->alloc_hint.expected_write_size); in osd_req_encode_op()
1031 dst->alloc_hint.flags = cpu_to_le32(src->alloc_hint.flags); in osd_req_encode_op()
1035 dst->xattr.name_len = cpu_to_le32(src->xattr.name_len); in osd_req_encode_op()
1036 dst->xattr.value_len = cpu_to_le32(src->xattr.value_len); in osd_req_encode_op()
1037 dst->xattr.cmp_op = src->xattr.cmp_op; in osd_req_encode_op()
1038 dst->xattr.cmp_mode = src->xattr.cmp_mode; in osd_req_encode_op()
1044 dst->copy_from.snapid = cpu_to_le64(src->copy_from.snapid); in osd_req_encode_op()
1045 dst->copy_from.src_version = in osd_req_encode_op()
1046 cpu_to_le64(src->copy_from.src_version); in osd_req_encode_op()
1047 dst->copy_from.flags = src->copy_from.flags; in osd_req_encode_op()
1048 dst->copy_from.src_fadvise_flags = in osd_req_encode_op()
1049 cpu_to_le32(src->copy_from.src_fadvise_flags); in osd_req_encode_op()
1052 dst->assert_ver.unused = cpu_to_le64(0); in osd_req_encode_op()
1053 dst->assert_ver.ver = cpu_to_le64(src->assert_ver.ver); in osd_req_encode_op()
1057 ceph_osd_op_name(src->op)); in osd_req_encode_op()
1063 dst->op = cpu_to_le16(src->op); in osd_req_encode_op()
1064 dst->flags = cpu_to_le32(src->flags); in osd_req_encode_op()
1065 dst->payload_len = cpu_to_le32(src->indata_len); in osd_req_encode_op()
1067 return src->indata_len; in osd_req_encode_op()
1103 r = -ENOMEM; in ceph_osdc_new_request()
1115 u32 object_size = layout->object_size; in ceph_osdc_new_request()
1116 u32 object_base = off - objoff; in ceph_osdc_new_request()
1117 if (!(truncate_seq == 1 && truncate_size == -1ULL)) { in ceph_osdc_new_request()
1121 truncate_size -= object_base; in ceph_osdc_new_request()
1130 req->r_base_oloc.pool = layout->pool_id; in ceph_osdc_new_request()
1131 req->r_base_oloc.pool_ns = ceph_try_get_string(layout->pool_ns); in ceph_osdc_new_request()
1132 ceph_oid_printf(&req->r_base_oid, "%llx.%08llx", vino.ino, objnum); in ceph_osdc_new_request()
1133 req->r_flags = flags | osdc->client->options->read_from_replica; in ceph_osdc_new_request()
1135 req->r_snapid = vino.snap; in ceph_osdc_new_request()
1137 req->r_data_offset = off; in ceph_osdc_new_request()
1143 * If this is a multi-op write request, assume that we'll need in ceph_osdc_new_request()
1144 * request ops. If it's a multi-op read then assume we'll need in ceph_osdc_new_request()
1145 * reply ops. Anything else and call it -EINVAL. in ceph_osdc_new_request()
1154 r = -EINVAL; in ceph_osdc_new_request()
1176 WARN_ON(op->op != CEPH_OSD_OP_SPARSE_READ); in __ceph_alloc_sparse_ext_map()
1178 op->extent.sparse_ext_cnt = cnt; in __ceph_alloc_sparse_ext_map()
1179 op->extent.sparse_ext = kmalloc_array(cnt, in __ceph_alloc_sparse_ext_map()
1180 sizeof(*op->extent.sparse_ext), in __ceph_alloc_sparse_ext_map()
1182 if (!op->extent.sparse_ext) in __ceph_alloc_sparse_ext_map()
1183 return -ENOMEM; in __ceph_alloc_sparse_ext_map()
1189 * We keep osd requests in an rbtree, sorted by ->r_tid.
1203 for (n = rb_first(&osdc->osds); n; n = rb_next(n)) { in DEFINE_RB_FUNCS()
1206 for (p = rb_first(&osd->o_requests); p; ) { in DEFINE_RB_FUNCS()
1216 for (p = rb_first(&osdc->homeless_osd.o_requests); p; ) { in DEFINE_RB_FUNCS()
1228 return osd->o_osd == CEPH_HOMELESS_OSD; in osd_homeless()
1233 verify_osdc_locked(osd->o_osdc); in osd_registered()
1235 return !RB_EMPTY_NODE(&osd->o_node); in osd_registered()
1239 * Assumes @osd is zero-initialized.
1243 refcount_set(&osd->o_ref, 1); in osd_init()
1244 RB_CLEAR_NODE(&osd->o_node); in osd_init()
1245 spin_lock_init(&osd->o_requests_lock); in osd_init()
1246 osd->o_requests = RB_ROOT; in osd_init()
1247 osd->o_linger_requests = RB_ROOT; in osd_init()
1248 osd->o_backoff_mappings = RB_ROOT; in osd_init()
1249 osd->o_backoffs_by_id = RB_ROOT; in osd_init()
1250 INIT_LIST_HEAD(&osd->o_osd_lru); in osd_init()
1251 INIT_LIST_HEAD(&osd->o_keepalive_item); in osd_init()
1252 osd->o_incarnation = 1; in osd_init()
1253 mutex_init(&osd->lock); in osd_init()
1258 kfree(sr->sr_extent); in ceph_init_sparse_read()
1260 sr->sr_state = CEPH_SPARSE_READ_HDR; in ceph_init_sparse_read()
1265 WARN_ON(!RB_EMPTY_NODE(&osd->o_node)); in osd_cleanup()
1266 WARN_ON(!RB_EMPTY_ROOT(&osd->o_requests)); in osd_cleanup()
1267 WARN_ON(!RB_EMPTY_ROOT(&osd->o_linger_requests)); in osd_cleanup()
1268 WARN_ON(!RB_EMPTY_ROOT(&osd->o_backoff_mappings)); in osd_cleanup()
1269 WARN_ON(!RB_EMPTY_ROOT(&osd->o_backoffs_by_id)); in osd_cleanup()
1270 WARN_ON(!list_empty(&osd->o_osd_lru)); in osd_cleanup()
1271 WARN_ON(!list_empty(&osd->o_keepalive_item)); in osd_cleanup()
1273 ceph_init_sparse_read(&osd->o_sparse_read); in osd_cleanup()
1275 if (osd->o_auth.authorizer) { in osd_cleanup()
1277 ceph_auth_destroy_authorizer(osd->o_auth.authorizer); in osd_cleanup()
1292 osd->o_osdc = osdc; in create_osd()
1293 osd->o_osd = onum; in create_osd()
1294 osd->o_sparse_op_idx = -1; in create_osd()
1296 ceph_init_sparse_read(&osd->o_sparse_read); in create_osd()
1298 ceph_con_init(&osd->o_con, osd, &osd_con_ops, &osdc->client->msgr); in create_osd()
1305 if (refcount_inc_not_zero(&osd->o_ref)) { in get_osd()
1306 dout("get_osd %p %d -> %d\n", osd, refcount_read(&osd->o_ref)-1, in get_osd()
1307 refcount_read(&osd->o_ref)); in get_osd()
1317 dout("put_osd %p %d -> %d\n", osd, refcount_read(&osd->o_ref), in put_osd()
1318 refcount_read(&osd->o_ref) - 1); in put_osd()
1319 if (refcount_dec_and_test(&osd->o_ref)) { in put_osd()
1329 struct ceph_osd_client *osdc = osd->o_osdc; in DEFINE_RB_FUNCS()
1331 dout("%s osd %p osd%d\n", __func__, osd, osd->o_osd); in DEFINE_RB_FUNCS()
1332 BUG_ON(!list_empty(&osd->o_osd_lru)); in DEFINE_RB_FUNCS()
1334 spin_lock(&osdc->osd_lru_lock); in DEFINE_RB_FUNCS()
1335 list_add_tail(&osd->o_osd_lru, &osdc->osd_lru); in DEFINE_RB_FUNCS()
1336 spin_unlock(&osdc->osd_lru_lock); in DEFINE_RB_FUNCS()
1338 osd->lru_ttl = jiffies + osdc->client->options->osd_idle_ttl; in DEFINE_RB_FUNCS()
1343 if (RB_EMPTY_ROOT(&osd->o_requests) && in maybe_move_osd_to_lru()
1344 RB_EMPTY_ROOT(&osd->o_linger_requests)) in maybe_move_osd_to_lru()
1350 struct ceph_osd_client *osdc = osd->o_osdc; in __remove_osd_from_lru()
1352 dout("%s osd %p osd%d\n", __func__, osd, osd->o_osd); in __remove_osd_from_lru()
1354 spin_lock(&osdc->osd_lru_lock); in __remove_osd_from_lru()
1355 if (!list_empty(&osd->o_osd_lru)) in __remove_osd_from_lru()
1356 list_del_init(&osd->o_osd_lru); in __remove_osd_from_lru()
1357 spin_unlock(&osdc->osd_lru_lock); in __remove_osd_from_lru()
1366 struct ceph_osd_client *osdc = osd->o_osdc; in close_osd()
1370 dout("%s osd %p osd%d\n", __func__, osd, osd->o_osd); in close_osd()
1372 ceph_con_close(&osd->o_con); in close_osd()
1374 for (n = rb_first(&osd->o_requests); n; ) { in close_osd()
1380 dout(" reassigning req %p tid %llu\n", req, req->r_tid); in close_osd()
1382 link_request(&osdc->homeless_osd, req); in close_osd()
1384 for (n = rb_first(&osd->o_linger_requests); n; ) { in close_osd()
1391 lreq->linger_id); in close_osd()
1393 link_linger(&osdc->homeless_osd, lreq); in close_osd()
1398 erase_osd(&osdc->osds, osd); in close_osd()
1409 dout("%s osd %p osd%d\n", __func__, osd, osd->o_osd); in reopen_osd()
1411 if (RB_EMPTY_ROOT(&osd->o_requests) && in reopen_osd()
1412 RB_EMPTY_ROOT(&osd->o_linger_requests)) { in reopen_osd()
1414 return -ENODEV; in reopen_osd()
1417 peer_addr = &osd->o_osdc->osdmap->osd_addr[osd->o_osd]; in reopen_osd()
1418 if (!memcmp(peer_addr, &osd->o_con.peer_addr, sizeof (*peer_addr)) && in reopen_osd()
1419 !ceph_con_opened(&osd->o_con)) { in reopen_osd()
1425 for (n = rb_first(&osd->o_requests); n; n = rb_next(n)) { in reopen_osd()
1428 req->r_stamp = jiffies; in reopen_osd()
1431 return -EAGAIN; in reopen_osd()
1434 ceph_con_close(&osd->o_con); in reopen_osd()
1435 ceph_con_open(&osd->o_con, CEPH_ENTITY_TYPE_OSD, osd->o_osd, peer_addr); in reopen_osd()
1436 osd->o_incarnation++; in reopen_osd()
1452 osd = lookup_osd(&osdc->osds, o); in lookup_create_osd()
1454 osd = &osdc->homeless_osd; in lookup_create_osd()
1457 return ERR_PTR(-EAGAIN); in lookup_create_osd()
1460 insert_osd(&osdc->osds, osd); in lookup_create_osd()
1461 ceph_con_open(&osd->o_con, CEPH_ENTITY_TYPE_OSD, osd->o_osd, in lookup_create_osd()
1462 &osdc->osdmap->osd_addr[osd->o_osd]); in lookup_create_osd()
1465 dout("%s osdc %p osd%d -> osd %p\n", __func__, osdc, o, osd); in lookup_create_osd()
1470 * Create request <-> OSD session relation.
1477 WARN_ON(!req->r_tid || req->r_osd); in link_request()
1478 dout("%s osd %p osd%d req %p tid %llu\n", __func__, osd, osd->o_osd, in link_request()
1479 req, req->r_tid); in link_request()
1484 atomic_inc(&osd->o_osdc->num_homeless); in link_request()
1487 spin_lock(&osd->o_requests_lock); in link_request()
1488 insert_request(&osd->o_requests, req); in link_request()
1489 spin_unlock(&osd->o_requests_lock); in link_request()
1490 req->r_osd = osd; in link_request()
1496 WARN_ON(req->r_osd != osd); in unlink_request()
1497 dout("%s osd %p osd%d req %p tid %llu\n", __func__, osd, osd->o_osd, in unlink_request()
1498 req, req->r_tid); in unlink_request()
1500 req->r_osd = NULL; in unlink_request()
1501 spin_lock(&osd->o_requests_lock); in unlink_request()
1502 erase_request(&osd->o_requests, req); in unlink_request()
1503 spin_unlock(&osd->o_requests_lock); in unlink_request()
1509 atomic_dec(&osd->o_osdc->num_homeless); in unlink_request()
1514 return pi->flags & CEPH_POOL_FLAG_FULL; in __pool_full()
1521 for (n = rb_first(&osdc->osdmap->pg_pools); n; n = rb_next(n)) { in have_pool_full()
1536 pi = ceph_pg_pool_by_id(osdc->osdmap, pool_id); in pool_full()
1556 WARN_ON(pi->id != t->target_oloc.pool); in target_should_be_paused()
1557 return ((t->flags & CEPH_OSD_FLAG_READ) && pauserd) || in target_should_be_paused()
1558 ((t->flags & CEPH_OSD_FLAG_WRITE) && pausewr) || in target_should_be_paused()
1559 (osdc->osdmap->epoch < osdc->epoch_barrier); in target_should_be_paused()
1564 int i = get_random_u32_below(acting->size); in pick_random_replica()
1567 acting->osds[i], acting->primary); in pick_random_replica()
1579 struct ceph_options *opt = osdc->client->options; in pick_closest_replica()
1584 locality = ceph_get_crush_locality(osdc->osdmap, in pick_closest_replica()
1585 acting->osds[i], in pick_closest_replica()
1586 &opt->crush_locs); in pick_closest_replica()
1594 } while (++i < acting->size); in pick_closest_replica()
1597 acting->osds[best_i], best_locality, acting->primary); in pick_closest_replica()
1614 bool is_read = t->flags & CEPH_OSD_FLAG_READ; in calc_target()
1615 bool is_write = t->flags & CEPH_OSD_FLAG_WRITE; in calc_target()
1625 t->epoch = osdc->osdmap->epoch; in calc_target()
1626 pi = ceph_pg_pool_by_id(osdc->osdmap, t->base_oloc.pool); in calc_target()
1628 t->osd = CEPH_HOMELESS_OSD; in calc_target()
1633 if (osdc->osdmap->epoch == pi->last_force_request_resend) { in calc_target()
1634 if (t->last_force_resend < pi->last_force_request_resend) { in calc_target()
1635 t->last_force_resend = pi->last_force_request_resend; in calc_target()
1637 } else if (t->last_force_resend == 0) { in calc_target()
1643 ceph_oid_copy(&t->target_oid, &t->base_oid); in calc_target()
1644 ceph_oloc_copy(&t->target_oloc, &t->base_oloc); in calc_target()
1645 if ((t->flags & CEPH_OSD_FLAG_IGNORE_OVERLAY) == 0) { in calc_target()
1646 if (is_read && pi->read_tier >= 0) in calc_target()
1647 t->target_oloc.pool = pi->read_tier; in calc_target()
1648 if (is_write && pi->write_tier >= 0) in calc_target()
1649 t->target_oloc.pool = pi->write_tier; in calc_target()
1651 pi = ceph_pg_pool_by_id(osdc->osdmap, t->target_oloc.pool); in calc_target()
1653 t->osd = CEPH_HOMELESS_OSD; in calc_target()
1659 __ceph_object_locator_to_pg(pi, &t->target_oid, &t->target_oloc, &pgid); in calc_target()
1661 last_pgid.seed = ceph_stable_mod(pgid.seed, t->pg_num, t->pg_num_mask); in calc_target()
1663 ceph_pg_to_up_acting_osds(osdc->osdmap, pi, &pgid, &up, &acting); in calc_target()
1665 ceph_is_new_interval(&t->acting, in calc_target()
1667 &t->up, in calc_target()
1669 t->size, in calc_target()
1670 pi->size, in calc_target()
1671 t->min_size, in calc_target()
1672 pi->min_size, in calc_target()
1673 t->pg_num, in calc_target()
1674 pi->pg_num, in calc_target()
1675 t->sort_bitwise, in calc_target()
1677 t->recovery_deletes, in calc_target()
1682 if (t->paused && !target_should_be_paused(osdc, t, pi)) { in calc_target()
1683 t->paused = false; in calc_target()
1686 legacy_change = ceph_pg_compare(&t->pgid, &pgid) || in calc_target()
1687 ceph_osds_changed(&t->acting, &acting, in calc_target()
1688 t->used_replica || any_change); in calc_target()
1689 if (t->pg_num) in calc_target()
1690 split = ceph_pg_is_split(&last_pgid, t->pg_num, pi->pg_num); in calc_target()
1693 t->pgid = pgid; /* struct */ in calc_target()
1694 ceph_pg_to_primary_shard(osdc->osdmap, pi, &pgid, &t->spgid); in calc_target()
1695 ceph_osds_copy(&t->acting, &acting); in calc_target()
1696 ceph_osds_copy(&t->up, &up); in calc_target()
1697 t->size = pi->size; in calc_target()
1698 t->min_size = pi->min_size; in calc_target()
1699 t->pg_num = pi->pg_num; in calc_target()
1700 t->pg_num_mask = pi->pg_num_mask; in calc_target()
1701 t->sort_bitwise = sort_bitwise; in calc_target()
1702 t->recovery_deletes = recovery_deletes; in calc_target()
1704 if ((t->flags & (CEPH_OSD_FLAG_BALANCE_READS | in calc_target()
1706 !is_write && pi->type == CEPH_POOL_TYPE_REP && in calc_target()
1711 if (t->flags & CEPH_OSD_FLAG_BALANCE_READS) { in calc_target()
1716 t->osd = acting.osds[pos]; in calc_target()
1717 t->used_replica = pos > 0; in calc_target()
1719 t->osd = acting.primary; in calc_target()
1720 t->used_replica = false; in calc_target()
1730 dout("%s t %p -> %d%d%d%d ct_res %d osd%d\n", __func__, t, unpaused, in calc_target()
1731 legacy_change, force_resend, split, ct_res, t->osd); in calc_target()
1743 RB_CLEAR_NODE(&spg->node); in alloc_spg_mapping()
1744 spg->backoffs = RB_ROOT; in alloc_spg_mapping()
1750 WARN_ON(!RB_EMPTY_NODE(&spg->node)); in free_spg_mapping()
1751 WARN_ON(!RB_EMPTY_ROOT(&spg->backoffs)); in free_spg_mapping()
1758 * ceph_pg_mapping. Used to track OSD backoffs -- a backoff [range] is
1767 return hoid->is_max ? 0x100000000ull : hoid->hash_reverse_bits; in DEFINE_RB_FUNCS2()
1773 if (hoid->key_len) { in hoid_get_effective_key()
1774 *pkey = hoid->key; in hoid_get_effective_key()
1775 *pkey_len = hoid->key_len; in hoid_get_effective_key()
1777 *pkey = hoid->oid; in hoid_get_effective_key()
1778 *pkey_len = hoid->oid_len; in hoid_get_effective_key()
1790 ret = -1; in compare_names()
1804 if (lhs->is_max < rhs->is_max) in hoid_compare()
1805 return -1; in hoid_compare()
1806 if (lhs->is_max > rhs->is_max) in hoid_compare()
1809 if (lhs->pool < rhs->pool) in hoid_compare()
1810 return -1; in hoid_compare()
1811 if (lhs->pool > rhs->pool) in hoid_compare()
1815 return -1; in hoid_compare()
1819 ret = compare_names(lhs->nspace, lhs->nspace_len, in hoid_compare()
1820 rhs->nspace, rhs->nspace_len); in hoid_compare()
1831 ret = compare_names(lhs->oid, lhs->oid_len, rhs->oid, rhs->oid_len); in hoid_compare()
1835 if (lhs->snapid < rhs->snapid) in hoid_compare()
1836 return -1; in hoid_compare()
1837 if (lhs->snapid > rhs->snapid) in hoid_compare()
1844 * For decoding ->begin and ->end of MOSDBackoff only -- no MIN/MAX
1847 * Assumes @hoid is zero-initialized.
1865 hoid->key = ceph_extract_encoded_string(p, end, &hoid->key_len, in decode_hoid()
1867 if (IS_ERR(hoid->key)) { in decode_hoid()
1868 ret = PTR_ERR(hoid->key); in decode_hoid()
1869 hoid->key = NULL; in decode_hoid()
1873 hoid->oid = ceph_extract_encoded_string(p, end, &hoid->oid_len, in decode_hoid()
1875 if (IS_ERR(hoid->oid)) { in decode_hoid()
1876 ret = PTR_ERR(hoid->oid); in decode_hoid()
1877 hoid->oid = NULL; in decode_hoid()
1881 ceph_decode_64_safe(p, end, hoid->snapid, e_inval); in decode_hoid()
1882 ceph_decode_32_safe(p, end, hoid->hash, e_inval); in decode_hoid()
1883 ceph_decode_8_safe(p, end, hoid->is_max, e_inval); in decode_hoid()
1885 hoid->nspace = ceph_extract_encoded_string(p, end, &hoid->nspace_len, in decode_hoid()
1887 if (IS_ERR(hoid->nspace)) { in decode_hoid()
1888 ret = PTR_ERR(hoid->nspace); in decode_hoid()
1889 hoid->nspace = NULL; in decode_hoid()
1893 ceph_decode_64_safe(p, end, hoid->pool, e_inval); in decode_hoid()
1899 return -EINVAL; in decode_hoid()
1905 4 + hoid->key_len + 4 + hoid->oid_len + 4 + hoid->nspace_len; in hoid_encoding_size()
1911 ceph_encode_string(p, end, hoid->key, hoid->key_len); in encode_hoid()
1912 ceph_encode_string(p, end, hoid->oid, hoid->oid_len); in encode_hoid()
1913 ceph_encode_64(p, hoid->snapid); in encode_hoid()
1914 ceph_encode_32(p, hoid->hash); in encode_hoid()
1915 ceph_encode_8(p, hoid->is_max); in encode_hoid()
1916 ceph_encode_string(p, end, hoid->nspace, hoid->nspace_len); in encode_hoid()
1917 ceph_encode_64(p, hoid->pool); in encode_hoid()
1923 kfree(hoid->key); in free_hoid()
1924 kfree(hoid->oid); in free_hoid()
1925 kfree(hoid->nspace); in free_hoid()
1938 RB_CLEAR_NODE(&backoff->spg_node); in alloc_backoff()
1939 RB_CLEAR_NODE(&backoff->id_node); in alloc_backoff()
1945 WARN_ON(!RB_EMPTY_NODE(&backoff->spg_node)); in free_backoff()
1946 WARN_ON(!RB_EMPTY_NODE(&backoff->id_node)); in free_backoff()
1948 free_hoid(backoff->begin); in free_backoff()
1949 free_hoid(backoff->end); in free_backoff()
1954 * Within a specific spgid, backoffs are managed by ->begin hoid.
1962 struct rb_node *n = root->rb_node; in lookup_containing_backoff()
1969 cmp = hoid_compare(hoid, cur->begin); in lookup_containing_backoff()
1971 n = n->rb_left; in lookup_containing_backoff()
1973 if (hoid_compare(hoid, cur->end) < 0) in lookup_containing_backoff()
1976 n = n->rb_right; in lookup_containing_backoff()
1992 while (!RB_EMPTY_ROOT(&osd->o_backoff_mappings)) { in DEFINE_RB_FUNCS()
1994 rb_entry(rb_first(&osd->o_backoff_mappings), in DEFINE_RB_FUNCS()
1997 while (!RB_EMPTY_ROOT(&spg->backoffs)) { in DEFINE_RB_FUNCS()
1999 rb_entry(rb_first(&spg->backoffs), in DEFINE_RB_FUNCS()
2002 erase_backoff(&spg->backoffs, backoff); in DEFINE_RB_FUNCS()
2003 erase_backoff_by_id(&osd->o_backoffs_by_id, backoff); in DEFINE_RB_FUNCS()
2006 erase_spg_mapping(&osd->o_backoff_mappings, spg); in DEFINE_RB_FUNCS()
2012 * Set up a temporary, non-owning view into @t.
2017 hoid->key = NULL; in hoid_fill_from_target()
2018 hoid->key_len = 0; in hoid_fill_from_target()
2019 hoid->oid = t->target_oid.name; in hoid_fill_from_target()
2020 hoid->oid_len = t->target_oid.name_len; in hoid_fill_from_target()
2021 hoid->snapid = CEPH_NOSNAP; in hoid_fill_from_target()
2022 hoid->hash = t->pgid.seed; in hoid_fill_from_target()
2023 hoid->is_max = false; in hoid_fill_from_target()
2024 if (t->target_oloc.pool_ns) { in hoid_fill_from_target()
2025 hoid->nspace = t->target_oloc.pool_ns->str; in hoid_fill_from_target()
2026 hoid->nspace_len = t->target_oloc.pool_ns->len; in hoid_fill_from_target()
2028 hoid->nspace = NULL; in hoid_fill_from_target()
2029 hoid->nspace_len = 0; in hoid_fill_from_target()
2031 hoid->pool = t->target_oloc.pool; in hoid_fill_from_target()
2037 struct ceph_osd *osd = req->r_osd; in should_plug_request()
2042 spg = lookup_spg_mapping(&osd->o_backoff_mappings, &req->r_t.spgid); in should_plug_request()
2046 hoid_fill_from_target(&hoid, &req->r_t); in should_plug_request()
2047 backoff = lookup_containing_backoff(&spg->backoffs, &hoid); in should_plug_request()
2052 __func__, req, req->r_tid, osd->o_osd, backoff->spgid.pgid.pool, in should_plug_request()
2053 backoff->spgid.pgid.seed, backoff->spgid.shard, backoff->id); in should_plug_request()
2062 struct ceph_msg *request_msg = req->r_request; in setup_request_data()
2063 struct ceph_msg *reply_msg = req->r_reply; in setup_request_data()
2066 if (req->r_request->num_data_items || req->r_reply->num_data_items) in setup_request_data()
2069 WARN_ON(request_msg->data_length || reply_msg->data_length); in setup_request_data()
2070 for (op = req->r_ops; op != &req->r_ops[req->r_num_ops]; op++) { in setup_request_data()
2071 switch (op->op) { in setup_request_data()
2075 WARN_ON(op->indata_len != op->extent.length); in setup_request_data()
2077 &op->extent.osd_data); in setup_request_data()
2081 WARN_ON(op->indata_len != op->xattr.name_len + in setup_request_data()
2082 op->xattr.value_len); in setup_request_data()
2084 &op->xattr.osd_data); in setup_request_data()
2088 &op->notify_ack.request_data); in setup_request_data()
2092 &op->copy_from.osd_data); in setup_request_data()
2098 &op->raw_data_in); in setup_request_data()
2103 &op->extent.osd_data); in setup_request_data()
2107 &op->list_watchers.response_data); in setup_request_data()
2112 WARN_ON(op->indata_len != op->cls.class_len + in setup_request_data()
2113 op->cls.method_len + in setup_request_data()
2114 op->cls.indata_len); in setup_request_data()
2116 &op->cls.request_info); in setup_request_data()
2119 &op->cls.request_data); in setup_request_data()
2122 &op->cls.response_data); in setup_request_data()
2126 &op->notify.request_data); in setup_request_data()
2128 &op->notify.response_data); in setup_request_data()
2137 ceph_encode_64(p, pgid->pool); in encode_pgid()
2138 ceph_encode_32(p, pgid->seed); in encode_pgid()
2139 ceph_encode_32(p, -1); /* preferred */ in encode_pgid()
2145 encode_pgid(p, &spgid->pgid); in encode_spgid()
2146 ceph_encode_8(p, spgid->shard); in encode_spgid()
2153 ceph_encode_64(p, oloc->pool); in encode_oloc()
2154 ceph_encode_32(p, -1); /* preferred */ in encode_oloc()
2156 if (oloc->pool_ns) in encode_oloc()
2157 ceph_encode_string(p, end, oloc->pool_ns->str, in encode_oloc()
2158 oloc->pool_ns->len); in encode_oloc()
2166 void *p = msg->front.iov_base; in encode_request_partial()
2167 void *const end = p + msg->front_alloc_len; in encode_request_partial()
2171 if (req->r_flags & CEPH_OSD_FLAG_WRITE) { in encode_request_partial()
2173 WARN_ON(req->r_snapid != CEPH_NOSNAP); in encode_request_partial()
2175 WARN_ON(req->r_mtime.tv_sec || req->r_mtime.tv_nsec || in encode_request_partial()
2176 req->r_data_offset || req->r_snapc); in encode_request_partial()
2181 encode_spgid(&p, &req->r_t.spgid); /* actual spg */ in encode_request_partial()
2182 ceph_encode_32(&p, req->r_t.pgid.seed); /* raw hash */ in encode_request_partial()
2183 ceph_encode_32(&p, req->r_osdc->osdmap->epoch); in encode_request_partial()
2184 ceph_encode_32(&p, req->r_flags); in encode_request_partial()
2196 ceph_encode_timespec64(p, &req->r_mtime); in encode_request_partial()
2199 encode_oloc(&p, end, &req->r_t.target_oloc); in encode_request_partial()
2200 ceph_encode_string(&p, end, req->r_t.target_oid.name, in encode_request_partial()
2201 req->r_t.target_oid.name_len); in encode_request_partial()
2204 ceph_encode_16(&p, req->r_num_ops); in encode_request_partial()
2205 for (i = 0; i < req->r_num_ops; i++) { in encode_request_partial()
2206 data_len += osd_req_encode_op(p, &req->r_ops[i]); in encode_request_partial()
2210 ceph_encode_64(&p, req->r_snapid); /* snapid */ in encode_request_partial()
2211 if (req->r_snapc) { in encode_request_partial()
2212 ceph_encode_64(&p, req->r_snapc->seq); in encode_request_partial()
2213 ceph_encode_32(&p, req->r_snapc->num_snaps); in encode_request_partial()
2214 for (i = 0; i < req->r_snapc->num_snaps; i++) in encode_request_partial()
2215 ceph_encode_64(&p, req->r_snapc->snaps[i]); in encode_request_partial()
2221 ceph_encode_32(&p, req->r_attempts); /* retry_attempt */ in encode_request_partial()
2222 BUG_ON(p > end - 8); /* space for features */ in encode_request_partial()
2224 msg->hdr.version = cpu_to_le16(8); /* MOSDOp v8 */ in encode_request_partial()
2226 msg->front.iov_len = p - msg->front.iov_base; in encode_request_partial()
2227 msg->hdr.front_len = cpu_to_le32(msg->front.iov_len); in encode_request_partial()
2228 msg->hdr.data_len = cpu_to_le32(data_len); in encode_request_partial()
2232 * need to re-copy it before writing it to disk (direct I/O). in encode_request_partial()
2234 msg->hdr.data_off = cpu_to_le16(req->r_data_offset); in encode_request_partial()
2237 req->r_t.target_oid.name, req->r_t.target_oid.name_len); in encode_request_partial()
2242 void *p = msg->front.iov_base; in encode_request_finish()
2243 void *const partial_end = p + msg->front.iov_len; in encode_request_finish()
2244 void *const end = p + msg->front_alloc_len; in encode_request_finish()
2246 if (CEPH_HAVE_FEATURE(msg->con->peer_features, RESEND_ON_SPLIT)) { in encode_request_finish()
2247 /* luminous OSD -- encode features and be done */ in encode_request_finish()
2249 ceph_encode_64(&p, msg->con->peer_features); in encode_request_finish()
2269 * Pre-luminous OSD -- reencode v8 into v4 using @head in encode_request_finish()
2283 oloc_len = p - oloc; in encode_request_finish()
2288 oid_len = p - oid; in encode_request_finish()
2291 tail_len = partial_end - p; in encode_request_finish()
2293 p = msg->front.iov_base; in encode_request_finish()
2314 /* tail -- ops, snapid, snapc, retry_attempt */ in encode_request_finish()
2319 msg->hdr.version = cpu_to_le16(4); /* MOSDOp v4 */ in encode_request_finish()
2323 msg->front.iov_len = p - msg->front.iov_base; in encode_request_finish()
2324 msg->hdr.front_len = cpu_to_le32(msg->front.iov_len); in encode_request_finish()
2327 le64_to_cpu(msg->hdr.tid), le32_to_cpu(msg->hdr.front_len), in encode_request_finish()
2328 le32_to_cpu(msg->hdr.middle_len), le32_to_cpu(msg->hdr.data_len), in encode_request_finish()
2329 le16_to_cpu(msg->hdr.version)); in encode_request_finish()
2337 struct ceph_osd *osd = req->r_osd; in send_request()
2340 WARN_ON(osd->o_osd != req->r_t.osd); in send_request()
2350 if (req->r_sent) in send_request()
2351 ceph_msg_revoke(req->r_request); in send_request()
2353 req->r_flags |= CEPH_OSD_FLAG_KNOWN_REDIR; in send_request()
2354 if (req->r_attempts) in send_request()
2355 req->r_flags |= CEPH_OSD_FLAG_RETRY; in send_request()
2357 WARN_ON(req->r_flags & CEPH_OSD_FLAG_RETRY); in send_request()
2359 encode_request_partial(req, req->r_request); in send_request()
2362 __func__, req, req->r_tid, req->r_t.pgid.pool, req->r_t.pgid.seed, in send_request()
2363 req->r_t.spgid.pgid.pool, req->r_t.spgid.pgid.seed, in send_request()
2364 req->r_t.spgid.shard, osd->o_osd, req->r_t.epoch, req->r_flags, in send_request()
2365 req->r_attempts); in send_request()
2367 req->r_t.paused = false; in send_request()
2368 req->r_stamp = jiffies; in send_request()
2369 req->r_attempts++; in send_request()
2371 req->r_sent = osd->o_incarnation; in send_request()
2372 req->r_request->hdr.tid = cpu_to_le64(req->r_tid); in send_request()
2373 ceph_con_send(&osd->o_con, ceph_msg_get(req->r_request)); in send_request()
2381 WARN_ON(!osdc->osdmap->epoch); in maybe_request_map()
2392 if (ceph_monc_want_map(&osdc->client->monc, CEPH_SUB_OSDMAP, in maybe_request_map()
2393 osdc->osdmap->epoch + 1, continuous)) in maybe_request_map()
2394 ceph_monc_renew_subs(&osdc->client->monc); in maybe_request_map()
2402 struct ceph_osd_client *osdc = req->r_osdc; in __submit_request()
2409 WARN_ON(req->r_tid); in __submit_request()
2413 ct_res = calc_target(osdc, &req->r_t, false); in __submit_request()
2417 osd = lookup_create_osd(osdc, req->r_t.osd, wrlocked); in __submit_request()
2419 WARN_ON(PTR_ERR(osd) != -EAGAIN || wrlocked); in __submit_request()
2423 if (osdc->abort_err) { in __submit_request()
2424 dout("req %p abort_err %d\n", req, osdc->abort_err); in __submit_request()
2425 err = osdc->abort_err; in __submit_request()
2426 } else if (osdc->osdmap->epoch < osdc->epoch_barrier) { in __submit_request()
2427 dout("req %p epoch %u barrier %u\n", req, osdc->osdmap->epoch, in __submit_request()
2428 osdc->epoch_barrier); in __submit_request()
2429 req->r_t.paused = true; in __submit_request()
2431 } else if ((req->r_flags & CEPH_OSD_FLAG_WRITE) && in __submit_request()
2434 req->r_t.paused = true; in __submit_request()
2436 } else if ((req->r_flags & CEPH_OSD_FLAG_READ) && in __submit_request()
2439 req->r_t.paused = true; in __submit_request()
2441 } else if ((req->r_flags & CEPH_OSD_FLAG_WRITE) && in __submit_request()
2442 !(req->r_flags & (CEPH_OSD_FLAG_FULL_TRY | in __submit_request()
2445 pool_full(osdc, req->r_t.base_oloc.pool))) { in __submit_request()
2447 if (ceph_test_opt(osdc->client, ABORT_ON_FULL)) { in __submit_request()
2448 err = -ENOSPC; in __submit_request()
2454 req->r_t.base_oloc.pool); in __submit_request()
2455 req->r_t.paused = true; in __submit_request()
2464 mutex_lock(&osd->lock); in __submit_request()
2470 req->r_tid = atomic64_inc_return(&osdc->last_tid); in __submit_request()
2476 mutex_unlock(&osd->lock); in __submit_request()
2482 downgrade_write(&osdc->lock); in __submit_request()
2486 up_read(&osdc->lock); in __submit_request()
2487 down_write(&osdc->lock); in __submit_request()
2495 WARN_ON(req->r_flags & (CEPH_OSD_FLAG_ACK | CEPH_OSD_FLAG_ONDISK)); in account_request()
2496 WARN_ON(!(req->r_flags & (CEPH_OSD_FLAG_READ | CEPH_OSD_FLAG_WRITE))); in account_request()
2498 req->r_flags |= CEPH_OSD_FLAG_ONDISK; in account_request()
2499 atomic_inc(&req->r_osdc->num_requests); in account_request()
2501 req->r_start_stamp = jiffies; in account_request()
2502 req->r_start_latency = ktime_get(); in account_request()
2514 struct ceph_osd_client *osdc = req->r_osdc; in finish_request()
2516 WARN_ON(lookup_request_mc(&osdc->map_checks, req->r_tid)); in finish_request()
2517 dout("%s req %p tid %llu\n", __func__, req, req->r_tid); in finish_request()
2519 req->r_end_latency = ktime_get(); in finish_request()
2521 if (req->r_osd) { in finish_request()
2522 ceph_init_sparse_read(&req->r_osd->o_sparse_read); in finish_request()
2523 unlink_request(req->r_osd, req); in finish_request()
2525 atomic_dec(&osdc->num_requests); in finish_request()
2533 ceph_msg_revoke(req->r_request); in finish_request()
2534 ceph_msg_revoke_incoming(req->r_reply); in finish_request()
2540 req->r_tid, req->r_callback, req->r_result); in __complete_request()
2542 if (req->r_callback) in __complete_request()
2543 req->r_callback(req); in __complete_request()
2544 complete_all(&req->r_completion); in __complete_request()
2557 * This is open-coded in handle_reply().
2561 dout("%s req %p tid %llu err %d\n", __func__, req, req->r_tid, err); in complete_request()
2563 req->r_result = err; in complete_request()
2566 INIT_WORK(&req->r_complete_work, complete_request_workfn); in complete_request()
2567 queue_work(req->r_osdc->completion_wq, &req->r_complete_work); in complete_request()
2572 struct ceph_osd_client *osdc = req->r_osdc; in cancel_map_check()
2577 lookup_req = lookup_request_mc(&osdc->map_checks, req->r_tid); in cancel_map_check()
2582 erase_request_mc(&osdc->map_checks, req); in cancel_map_check()
2588 dout("%s req %p tid %llu\n", __func__, req, req->r_tid); in cancel_request()
2592 complete_all(&req->r_completion); in cancel_request()
2598 dout("%s req %p tid %llu err %d\n", __func__, req, req->r_tid, err); in abort_request()
2613 * Abort all in-flight requests with @err and arrange for all future
2619 down_write(&osdc->lock); in ceph_osdc_abort_requests()
2621 osdc->abort_err = err; in ceph_osdc_abort_requests()
2622 up_write(&osdc->lock); in ceph_osdc_abort_requests()
2628 down_write(&osdc->lock); in ceph_osdc_clear_abort_err()
2629 osdc->abort_err = 0; in ceph_osdc_clear_abort_err()
2630 up_write(&osdc->lock); in ceph_osdc_clear_abort_err()
2636 if (likely(eb > osdc->epoch_barrier)) { in update_epoch_barrier()
2638 osdc->epoch_barrier, eb); in update_epoch_barrier()
2639 osdc->epoch_barrier = eb; in update_epoch_barrier()
2641 if (eb > osdc->osdmap->epoch) in update_epoch_barrier()
2648 down_read(&osdc->lock); in ceph_osdc_update_epoch_barrier()
2649 if (unlikely(eb > osdc->epoch_barrier)) { in ceph_osdc_update_epoch_barrier()
2650 up_read(&osdc->lock); in ceph_osdc_update_epoch_barrier()
2651 down_write(&osdc->lock); in ceph_osdc_update_epoch_barrier()
2653 up_write(&osdc->lock); in ceph_osdc_update_epoch_barrier()
2655 up_read(&osdc->lock); in ceph_osdc_update_epoch_barrier()
2668 struct ceph_osd_client *osdc = req->r_osdc; in abort_on_full_fn()
2671 if ((req->r_flags & CEPH_OSD_FLAG_WRITE) && in abort_on_full_fn()
2673 pool_full(osdc, req->r_t.base_oloc.pool))) { in abort_on_full_fn()
2675 update_epoch_barrier(osdc, osdc->osdmap->epoch); in abort_on_full_fn()
2678 abort_request(req, -ENOSPC); in abort_on_full_fn()
2687 * osdc->epoch_barrier to the latest map epoch that we've seen if any were
2694 if (ceph_test_opt(osdc->client, ABORT_ON_FULL) && in ceph_osdc_abort_on_full()
2701 struct ceph_osd_client *osdc = req->r_osdc; in check_pool_dne()
2702 struct ceph_osdmap *map = osdc->osdmap; in check_pool_dne()
2705 WARN_ON(!map->epoch); in check_pool_dne()
2707 if (req->r_attempts) { in check_pool_dne()
2713 req->r_map_dne_bound = map->epoch; in check_pool_dne()
2715 req->r_tid); in check_pool_dne()
2718 req, req->r_tid, req->r_map_dne_bound, map->epoch); in check_pool_dne()
2721 if (req->r_map_dne_bound) { in check_pool_dne()
2722 if (map->epoch >= req->r_map_dne_bound) { in check_pool_dne()
2725 req->r_tid); in check_pool_dne()
2726 complete_request(req, -ENOENT); in check_pool_dne()
2735 struct ceph_osd_client *osdc = &greq->monc->client->osdc; in map_check_cb()
2737 u64 tid = greq->private_data; in map_check_cb()
2739 WARN_ON(greq->result || !greq->u.newest); in map_check_cb()
2741 down_write(&osdc->lock); in map_check_cb()
2742 req = lookup_request_mc(&osdc->map_checks, tid); in map_check_cb()
2749 req, req->r_tid, req->r_map_dne_bound, greq->u.newest); in map_check_cb()
2750 if (!req->r_map_dne_bound) in map_check_cb()
2751 req->r_map_dne_bound = greq->u.newest; in map_check_cb()
2752 erase_request_mc(&osdc->map_checks, req); in map_check_cb()
2757 up_write(&osdc->lock); in map_check_cb()
2762 struct ceph_osd_client *osdc = req->r_osdc; in send_map_check()
2768 lookup_req = lookup_request_mc(&osdc->map_checks, req->r_tid); in send_map_check()
2775 insert_request_mc(&osdc->map_checks, req); in send_map_check()
2776 ret = ceph_monc_get_version_async(&osdc->client->monc, "osdmap", in send_map_check()
2777 map_check_cb, req->r_tid); in send_map_check()
2790 lreq->reg_req, lreq->ping_req); in linger_release()
2791 WARN_ON(!RB_EMPTY_NODE(&lreq->node)); in linger_release()
2792 WARN_ON(!RB_EMPTY_NODE(&lreq->osdc_node)); in linger_release()
2793 WARN_ON(!RB_EMPTY_NODE(&lreq->mc_node)); in linger_release()
2794 WARN_ON(!list_empty(&lreq->scan_item)); in linger_release()
2795 WARN_ON(!list_empty(&lreq->pending_lworks)); in linger_release()
2796 WARN_ON(lreq->osd); in linger_release()
2798 if (lreq->request_pl) in linger_release()
2799 ceph_pagelist_release(lreq->request_pl); in linger_release()
2800 if (lreq->notify_id_pages) in linger_release()
2801 ceph_release_page_vector(lreq->notify_id_pages, 1); in linger_release()
2803 ceph_osdc_put_request(lreq->reg_req); in linger_release()
2804 ceph_osdc_put_request(lreq->ping_req); in linger_release()
2805 target_destroy(&lreq->t); in linger_release()
2812 kref_put(&lreq->kref, linger_release); in linger_put()
2818 kref_get(&lreq->kref); in linger_get()
2831 kref_init(&lreq->kref); in linger_alloc()
2832 mutex_init(&lreq->lock); in linger_alloc()
2833 RB_CLEAR_NODE(&lreq->node); in linger_alloc()
2834 RB_CLEAR_NODE(&lreq->osdc_node); in linger_alloc()
2835 RB_CLEAR_NODE(&lreq->mc_node); in linger_alloc()
2836 INIT_LIST_HEAD(&lreq->scan_item); in linger_alloc()
2837 INIT_LIST_HEAD(&lreq->pending_lworks); in linger_alloc()
2838 init_completion(&lreq->reg_commit_wait); in linger_alloc()
2839 init_completion(&lreq->notify_finish_wait); in linger_alloc()
2841 lreq->osdc = osdc; in linger_alloc()
2842 target_init(&lreq->t); in linger_alloc()
2853 * Create linger request <-> OSD session relation. in DEFINE_RB_INSDEL_FUNCS()
2861 WARN_ON(!lreq->linger_id || lreq->osd); in DEFINE_RB_INSDEL_FUNCS()
2863 osd->o_osd, lreq, lreq->linger_id); in DEFINE_RB_INSDEL_FUNCS()
2868 atomic_inc(&osd->o_osdc->num_homeless); in DEFINE_RB_INSDEL_FUNCS()
2871 insert_linger(&osd->o_linger_requests, lreq); in DEFINE_RB_INSDEL_FUNCS()
2872 lreq->osd = osd; in DEFINE_RB_INSDEL_FUNCS()
2879 WARN_ON(lreq->osd != osd); in unlink_linger()
2881 osd->o_osd, lreq, lreq->linger_id); in unlink_linger()
2883 lreq->osd = NULL; in unlink_linger()
2884 erase_linger(&osd->o_linger_requests, lreq); in unlink_linger()
2890 atomic_dec(&osd->o_osdc->num_homeless); in unlink_linger()
2895 verify_osdc_locked(lreq->osdc); in __linger_registered()
2897 return !RB_EMPTY_NODE(&lreq->osdc_node); in __linger_registered()
2902 struct ceph_osd_client *osdc = lreq->osdc; in linger_registered()
2905 down_read(&osdc->lock); in linger_registered()
2907 up_read(&osdc->lock); in linger_registered()
2914 struct ceph_osd_client *osdc = lreq->osdc; in linger_register()
2917 WARN_ON(lreq->linger_id); in linger_register()
2920 lreq->linger_id = ++osdc->last_linger_id; in linger_register()
2921 insert_linger_osdc(&osdc->linger_requests, lreq); in linger_register()
2926 struct ceph_osd_client *osdc = lreq->osdc; in linger_unregister()
2930 erase_linger_osdc(&osdc->linger_requests, lreq); in linger_unregister()
2936 struct ceph_osd_linger_request *lreq = req->r_priv; in cancel_linger_request()
2938 WARN_ON(!req->r_linger); in cancel_linger_request()
2973 INIT_WORK(&lwork->work, workfn); in lwork_alloc()
2974 INIT_LIST_HEAD(&lwork->pending_item); in lwork_alloc()
2975 lwork->lreq = linger_get(lreq); in lwork_alloc()
2982 struct ceph_osd_linger_request *lreq = lwork->lreq; in lwork_free()
2984 mutex_lock(&lreq->lock); in lwork_free()
2985 list_del(&lwork->pending_item); in lwork_free()
2986 mutex_unlock(&lreq->lock); in lwork_free()
2994 struct ceph_osd_linger_request *lreq = lwork->lreq; in lwork_queue()
2995 struct ceph_osd_client *osdc = lreq->osdc; in lwork_queue()
2998 WARN_ON(!list_empty(&lwork->pending_item)); in lwork_queue()
3000 lwork->queued_stamp = jiffies; in lwork_queue()
3001 list_add_tail(&lwork->pending_item, &lreq->pending_lworks); in lwork_queue()
3002 queue_work(osdc->notify_wq, &lwork->work); in lwork_queue()
3008 struct ceph_osd_linger_request *lreq = lwork->lreq; in do_watch_notify()
3015 WARN_ON(!lreq->is_watch); in do_watch_notify()
3017 __func__, lreq, lwork->notify.notify_id, lwork->notify.notifier_id, in do_watch_notify()
3018 lwork->notify.payload_len); in do_watch_notify()
3019 lreq->wcb(lreq->data, lwork->notify.notify_id, lreq->linger_id, in do_watch_notify()
3020 lwork->notify.notifier_id, lwork->notify.payload, in do_watch_notify()
3021 lwork->notify.payload_len); in do_watch_notify()
3024 ceph_msg_put(lwork->notify.msg); in do_watch_notify()
3031 struct ceph_osd_linger_request *lreq = lwork->lreq; in do_watch_error()
3038 dout("%s lreq %p err %d\n", __func__, lreq, lwork->error.err); in do_watch_error()
3039 lreq->errcb(lreq->data, lreq->linger_id, lwork->error.err); in do_watch_error()
3051 pr_err("failed to allocate error-lwork\n"); in queue_watch_error()
3055 lwork->error.err = lreq->last_error; in queue_watch_error()
3062 if (!completion_done(&lreq->reg_commit_wait)) { in linger_reg_commit_complete()
3063 lreq->reg_commit_error = (result <= 0 ? result : 0); in linger_reg_commit_complete()
3064 complete_all(&lreq->reg_commit_wait); in linger_reg_commit_complete()
3070 struct ceph_osd_linger_request *lreq = req->r_priv; in linger_commit_cb()
3072 mutex_lock(&lreq->lock); in linger_commit_cb()
3073 if (req != lreq->reg_req) { in linger_commit_cb()
3075 __func__, lreq, lreq->linger_id, req, lreq->reg_req); in linger_commit_cb()
3080 lreq->linger_id, req->r_result); in linger_commit_cb()
3081 linger_reg_commit_complete(lreq, req->r_result); in linger_commit_cb()
3082 lreq->committed = true; in linger_commit_cb()
3084 if (!lreq->is_watch) { in linger_commit_cb()
3087 void *p = page_address(osd_data->pages[0]); in linger_commit_cb()
3089 WARN_ON(req->r_ops[0].op != CEPH_OSD_OP_NOTIFY || in linger_commit_cb()
3090 osd_data->type != CEPH_OSD_DATA_TYPE_PAGES); in linger_commit_cb()
3093 if (req->r_ops[0].outdata_len >= sizeof(u64)) { in linger_commit_cb()
3094 lreq->notify_id = ceph_decode_64(&p); in linger_commit_cb()
3096 lreq->notify_id); in linger_commit_cb()
3103 mutex_unlock(&lreq->lock); in linger_commit_cb()
3110 * Translate ENOENT -> ENOTCONN so that a delete->disconnection in normalize_watch_error()
3114 if (err == -ENOENT) in normalize_watch_error()
3115 err = -ENOTCONN; in normalize_watch_error()
3122 struct ceph_osd_linger_request *lreq = req->r_priv; in linger_reconnect_cb()
3124 mutex_lock(&lreq->lock); in linger_reconnect_cb()
3125 if (req != lreq->reg_req) { in linger_reconnect_cb()
3127 __func__, lreq, lreq->linger_id, req, lreq->reg_req); in linger_reconnect_cb()
3132 lreq, lreq->linger_id, req->r_result, lreq->last_error); in linger_reconnect_cb()
3133 if (req->r_result < 0) { in linger_reconnect_cb()
3134 if (!lreq->last_error) { in linger_reconnect_cb()
3135 lreq->last_error = normalize_watch_error(req->r_result); in linger_reconnect_cb()
3141 mutex_unlock(&lreq->lock); in linger_reconnect_cb()
3147 struct ceph_osd_client *osdc = lreq->osdc; in send_linger()
3152 mutex_lock(&lreq->lock); in send_linger()
3153 dout("%s lreq %p linger_id %llu\n", __func__, lreq, lreq->linger_id); in send_linger()
3155 if (lreq->reg_req) { in send_linger()
3156 if (lreq->reg_req->r_osd) in send_linger()
3157 cancel_linger_request(lreq->reg_req); in send_linger()
3158 ceph_osdc_put_request(lreq->reg_req); in send_linger()
3164 target_copy(&req->r_t, &lreq->t); in send_linger()
3165 req->r_mtime = lreq->mtime; in send_linger()
3167 if (lreq->is_watch && lreq->committed) { in send_linger()
3169 lreq->linger_id, ++lreq->register_gen); in send_linger()
3171 req->r_ops[0].watch.gen); in send_linger()
3172 req->r_callback = linger_reconnect_cb; in send_linger()
3174 if (lreq->is_watch) { in send_linger()
3176 lreq->linger_id, 0); in send_linger()
3178 lreq->notify_id = 0; in send_linger()
3180 refcount_inc(&lreq->request_pl->refcnt); in send_linger()
3181 osd_req_op_notify_init(req, 0, lreq->linger_id, in send_linger()
3182 lreq->request_pl); in send_linger()
3185 lreq->notify_id_pages, PAGE_SIZE, 0, false, false); in send_linger()
3188 req->r_callback = linger_commit_cb; in send_linger()
3194 req->r_priv = linger_get(lreq); in send_linger()
3195 req->r_linger = true; in send_linger()
3196 lreq->reg_req = req; in send_linger()
3197 mutex_unlock(&lreq->lock); in send_linger()
3204 struct ceph_osd_linger_request *lreq = req->r_priv; in linger_ping_cb()
3206 mutex_lock(&lreq->lock); in linger_ping_cb()
3207 if (req != lreq->ping_req) { in linger_ping_cb()
3209 __func__, lreq, lreq->linger_id, req, lreq->ping_req); in linger_ping_cb()
3214 __func__, lreq, lreq->linger_id, req->r_result, lreq->ping_sent, in linger_ping_cb()
3215 lreq->last_error); in linger_ping_cb()
3216 if (lreq->register_gen == req->r_ops[0].watch.gen) { in linger_ping_cb()
3217 if (!req->r_result) { in linger_ping_cb()
3218 lreq->watch_valid_thru = lreq->ping_sent; in linger_ping_cb()
3219 } else if (!lreq->last_error) { in linger_ping_cb()
3220 lreq->last_error = normalize_watch_error(req->r_result); in linger_ping_cb()
3225 lreq->register_gen, req->r_ops[0].watch.gen); in linger_ping_cb()
3229 mutex_unlock(&lreq->lock); in linger_ping_cb()
3235 struct ceph_osd_client *osdc = lreq->osdc; in send_linger_ping()
3244 lreq->ping_sent = jiffies; in send_linger_ping()
3246 __func__, lreq, lreq->linger_id, lreq->ping_sent, in send_linger_ping()
3247 lreq->register_gen); in send_linger_ping()
3249 if (lreq->ping_req) { in send_linger_ping()
3250 if (lreq->ping_req->r_osd) in send_linger_ping()
3251 cancel_linger_request(lreq->ping_req); in send_linger_ping()
3252 ceph_osdc_put_request(lreq->ping_req); in send_linger_ping()
3258 target_copy(&req->r_t, &lreq->t); in send_linger_ping()
3259 osd_req_op_watch_init(req, 0, CEPH_OSD_WATCH_OP_PING, lreq->linger_id, in send_linger_ping()
3260 lreq->register_gen); in send_linger_ping()
3261 req->r_callback = linger_ping_cb; in send_linger_ping()
3266 req->r_priv = linger_get(lreq); in send_linger_ping()
3267 req->r_linger = true; in send_linger_ping()
3268 lreq->ping_req = req; in send_linger_ping()
3272 req->r_tid = atomic64_inc_return(&osdc->last_tid); in send_linger_ping()
3273 link_request(lreq->osd, req); in send_linger_ping()
3279 struct ceph_osd_client *osdc = lreq->osdc; in linger_submit()
3282 down_write(&osdc->lock); in linger_submit()
3285 calc_target(osdc, &lreq->t, false); in linger_submit()
3286 osd = lookup_create_osd(osdc, lreq->t.osd, true); in linger_submit()
3290 up_write(&osdc->lock); in linger_submit()
3295 struct ceph_osd_client *osdc = lreq->osdc; in cancel_linger_map_check()
3300 lookup_lreq = lookup_linger_mc(&osdc->linger_map_checks, in cancel_linger_map_check()
3301 lreq->linger_id); in cancel_linger_map_check()
3306 erase_linger_mc(&osdc->linger_map_checks, lreq); in cancel_linger_map_check()
3315 if (lreq->ping_req && lreq->ping_req->r_osd) in __linger_cancel()
3316 cancel_linger_request(lreq->ping_req); in __linger_cancel()
3317 if (lreq->reg_req && lreq->reg_req->r_osd) in __linger_cancel()
3318 cancel_linger_request(lreq->reg_req); in __linger_cancel()
3320 unlink_linger(lreq->osd, lreq); in __linger_cancel()
3326 struct ceph_osd_client *osdc = lreq->osdc; in linger_cancel()
3328 down_write(&osdc->lock); in linger_cancel()
3331 up_write(&osdc->lock); in linger_cancel()
3338 struct ceph_osd_client *osdc = lreq->osdc; in check_linger_pool_dne()
3339 struct ceph_osdmap *map = osdc->osdmap; in check_linger_pool_dne()
3342 WARN_ON(!map->epoch); in check_linger_pool_dne()
3344 if (lreq->register_gen) { in check_linger_pool_dne()
3345 lreq->map_dne_bound = map->epoch; in check_linger_pool_dne()
3347 lreq, lreq->linger_id); in check_linger_pool_dne()
3350 __func__, lreq, lreq->linger_id, lreq->map_dne_bound, in check_linger_pool_dne()
3351 map->epoch); in check_linger_pool_dne()
3354 if (lreq->map_dne_bound) { in check_linger_pool_dne()
3355 if (map->epoch >= lreq->map_dne_bound) { in check_linger_pool_dne()
3358 lreq->linger_id); in check_linger_pool_dne()
3359 linger_reg_commit_complete(lreq, -ENOENT); in check_linger_pool_dne()
3369 struct ceph_osd_client *osdc = &greq->monc->client->osdc; in linger_map_check_cb()
3371 u64 linger_id = greq->private_data; in linger_map_check_cb()
3373 WARN_ON(greq->result || !greq->u.newest); in linger_map_check_cb()
3375 down_write(&osdc->lock); in linger_map_check_cb()
3376 lreq = lookup_linger_mc(&osdc->linger_map_checks, linger_id); in linger_map_check_cb()
3383 __func__, lreq, lreq->linger_id, lreq->map_dne_bound, in linger_map_check_cb()
3384 greq->u.newest); in linger_map_check_cb()
3385 if (!lreq->map_dne_bound) in linger_map_check_cb()
3386 lreq->map_dne_bound = greq->u.newest; in linger_map_check_cb()
3387 erase_linger_mc(&osdc->linger_map_checks, lreq); in linger_map_check_cb()
3392 up_write(&osdc->lock); in linger_map_check_cb()
3397 struct ceph_osd_client *osdc = lreq->osdc; in send_linger_map_check()
3403 lookup_lreq = lookup_linger_mc(&osdc->linger_map_checks, in send_linger_map_check()
3404 lreq->linger_id); in send_linger_map_check()
3411 insert_linger_mc(&osdc->linger_map_checks, lreq); in send_linger_map_check()
3412 ret = ceph_monc_get_version_async(&osdc->client->monc, "osdmap", in send_linger_map_check()
3413 linger_map_check_cb, lreq->linger_id); in send_linger_map_check()
3421 dout("%s lreq %p linger_id %llu\n", __func__, lreq, lreq->linger_id); in linger_reg_commit_wait()
3422 ret = wait_for_completion_killable(&lreq->reg_commit_wait); in linger_reg_commit_wait()
3423 return ret ?: lreq->reg_commit_error; in linger_reg_commit_wait()
3431 dout("%s lreq %p linger_id %llu\n", __func__, lreq, lreq->linger_id); in linger_notify_finish_wait()
3432 left = wait_for_completion_killable_timeout(&lreq->notify_finish_wait, in linger_notify_finish_wait()
3435 left = left ?: -ETIMEDOUT; in linger_notify_finish_wait()
3437 left = lreq->notify_finish_error; /* completed */ in linger_notify_finish_wait()
3452 struct ceph_options *opts = osdc->client->options; in handle_timeout()
3453 unsigned long cutoff = jiffies - opts->osd_keepalive_timeout; in handle_timeout()
3454 unsigned long expiry_cutoff = jiffies - opts->osd_request_timeout; in handle_timeout()
3459 down_write(&osdc->lock); in handle_timeout()
3466 for (n = rb_first(&osdc->osds); n; n = rb_next(n)) { in handle_timeout()
3470 for (p = rb_first(&osd->o_requests); p; ) { in handle_timeout()
3476 if (time_before(req->r_stamp, cutoff)) { in handle_timeout()
3478 req, req->r_tid, osd->o_osd); in handle_timeout()
3481 if (opts->osd_request_timeout && in handle_timeout()
3482 time_before(req->r_start_stamp, expiry_cutoff)) { in handle_timeout()
3484 req->r_tid, osd->o_osd); in handle_timeout()
3485 abort_request(req, -ETIMEDOUT); in handle_timeout()
3488 for (p = rb_first(&osd->o_linger_requests); p; p = rb_next(p)) { in handle_timeout()
3493 lreq, lreq->linger_id, osd->o_osd); in handle_timeout()
3496 mutex_lock(&lreq->lock); in handle_timeout()
3497 if (lreq->is_watch && lreq->committed && !lreq->last_error) in handle_timeout()
3499 mutex_unlock(&lreq->lock); in handle_timeout()
3503 list_move_tail(&osd->o_keepalive_item, &slow_osds); in handle_timeout()
3506 if (opts->osd_request_timeout) { in handle_timeout()
3507 for (p = rb_first(&osdc->homeless_osd.o_requests); p; ) { in handle_timeout()
3513 if (time_before(req->r_start_stamp, expiry_cutoff)) { in handle_timeout()
3515 req->r_tid, osdc->homeless_osd.o_osd); in handle_timeout()
3516 abort_request(req, -ETIMEDOUT); in handle_timeout()
3521 if (atomic_read(&osdc->num_homeless) || !list_empty(&slow_osds)) in handle_timeout()
3528 list_del_init(&osd->o_keepalive_item); in handle_timeout()
3529 ceph_con_keepalive(&osd->o_con); in handle_timeout()
3532 up_write(&osdc->lock); in handle_timeout()
3533 schedule_delayed_work(&osdc->timeout_work, in handle_timeout()
3534 osdc->client->options->osd_keepalive_timeout); in handle_timeout()
3542 unsigned long delay = osdc->client->options->osd_idle_ttl / 4; in handle_osds_timeout()
3546 down_write(&osdc->lock); in handle_osds_timeout()
3547 list_for_each_entry_safe(osd, nosd, &osdc->osd_lru, o_osd_lru) { in handle_osds_timeout()
3548 if (time_before(jiffies, osd->lru_ttl)) in handle_osds_timeout()
3551 WARN_ON(!RB_EMPTY_ROOT(&osd->o_requests)); in handle_osds_timeout()
3552 WARN_ON(!RB_EMPTY_ROOT(&osd->o_linger_requests)); in handle_osds_timeout()
3556 up_write(&osdc->lock); in handle_osds_timeout()
3557 schedule_delayed_work(&osdc->osds_timeout_work, in handle_osds_timeout()
3586 oloc->pool = ceph_decode_64(p); in ceph_oloc_decode()
3601 if (!oloc->pool_ns || in ceph_oloc_decode()
3602 ceph_compare_string(oloc->pool_ns, *p, len)) in ceph_oloc_decode()
3606 if (oloc->pool_ns) in ceph_oloc_decode()
3618 if (hash != -1) { in ceph_oloc_decode()
3630 ret = -EINVAL; in ceph_oloc_decode()
3654 ret = ceph_oloc_decode(p, end, &redir->oloc); in ceph_redirect_decode()
3670 ret = -EINVAL; in ceph_redirect_decode()
3690 void *p = msg->front.iov_base; in decode_MOSDOpReply()
3691 void *const end = p + msg->front.iov_len; in decode_MOSDOpReply()
3692 u16 version = le16_to_cpu(msg->hdr.version); in decode_MOSDOpReply()
3703 ret = ceph_decode_pgid(&p, end, &m->pgid); in decode_MOSDOpReply()
3707 ceph_decode_64_safe(&p, end, m->flags, e_inval); in decode_MOSDOpReply()
3708 ceph_decode_32_safe(&p, end, m->result, e_inval); in decode_MOSDOpReply()
3712 ceph_decode_32_safe(&p, end, m->epoch, e_inval); in decode_MOSDOpReply()
3714 ceph_decode_32_safe(&p, end, m->num_ops, e_inval); in decode_MOSDOpReply()
3715 if (m->num_ops > ARRAY_SIZE(m->outdata_len)) in decode_MOSDOpReply()
3718 ceph_decode_need(&p, end, m->num_ops * sizeof(struct ceph_osd_op), in decode_MOSDOpReply()
3720 for (i = 0; i < m->num_ops; i++) { in decode_MOSDOpReply()
3723 m->outdata_len[i] = le32_to_cpu(op->payload_len); in decode_MOSDOpReply()
3727 ceph_decode_32_safe(&p, end, m->retry_attempt, e_inval); in decode_MOSDOpReply()
3728 for (i = 0; i < m->num_ops; i++) in decode_MOSDOpReply()
3729 ceph_decode_32_safe(&p, end, m->rval[i], e_inval); in decode_MOSDOpReply()
3732 ceph_decode_need(&p, end, sizeof(m->replay_version), e_inval); in decode_MOSDOpReply()
3733 memcpy(&m->replay_version, p, sizeof(m->replay_version)); in decode_MOSDOpReply()
3734 p += sizeof(m->replay_version); in decode_MOSDOpReply()
3735 ceph_decode_64_safe(&p, end, m->user_version, e_inval); in decode_MOSDOpReply()
3737 m->replay_version = bad_replay_version; /* struct */ in decode_MOSDOpReply()
3738 m->user_version = le64_to_cpu(m->replay_version.version); in decode_MOSDOpReply()
3751 ret = ceph_redirect_decode(&p, end, &m->redirect); in decode_MOSDOpReply()
3755 ceph_oloc_init(&m->redirect.oloc); in decode_MOSDOpReply()
3761 return -EINVAL; in decode_MOSDOpReply()
3765 * Handle MOSDOpReply. Set ->r_result and call the callback if it is
3770 struct ceph_osd_client *osdc = osd->o_osdc; in handle_reply()
3773 u64 tid = le64_to_cpu(msg->hdr.tid); in handle_reply()
3780 down_read(&osdc->lock); in handle_reply()
3782 dout("%s osd%d unknown\n", __func__, osd->o_osd); in handle_reply()
3785 WARN_ON(osd->o_osd != le64_to_cpu(msg->hdr.src.num)); in handle_reply()
3787 mutex_lock(&osd->lock); in handle_reply()
3788 req = lookup_request(&osd->o_requests, tid); in handle_reply()
3790 dout("%s osd%d tid %llu unknown\n", __func__, osd->o_osd, tid); in handle_reply()
3794 m.redirect.oloc.pool_ns = req->r_t.target_oloc.pool_ns; in handle_reply()
3799 req->r_tid, ret); in handle_reply()
3804 __func__, req, req->r_tid, m.flags, m.pgid.pool, m.pgid.seed, in handle_reply()
3809 if (m.retry_attempt != req->r_attempts - 1) { in handle_reply()
3811 req, req->r_tid, m.retry_attempt, in handle_reply()
3812 req->r_attempts - 1); in handle_reply()
3820 dout("req %p tid %llu redirect pool %lld\n", req, req->r_tid, in handle_reply()
3823 mutex_unlock(&osd->lock); in handle_reply()
3826 * Not ceph_oloc_copy() - changing pool_ns is not in handle_reply()
3829 req->r_t.target_oloc.pool = m.redirect.oloc.pool; in handle_reply()
3830 req->r_flags |= CEPH_OSD_FLAG_REDIRECTED | in handle_reply()
3833 req->r_tid = 0; in handle_reply()
3838 if (m.result == -EAGAIN) { in handle_reply()
3839 dout("req %p tid %llu EAGAIN\n", req, req->r_tid); in handle_reply()
3841 mutex_unlock(&osd->lock); in handle_reply()
3848 req->r_t.pgid.pool = 0; in handle_reply()
3849 req->r_t.pgid.seed = 0; in handle_reply()
3850 WARN_ON(!req->r_t.used_replica); in handle_reply()
3851 req->r_flags &= ~(CEPH_OSD_FLAG_BALANCE_READS | in handle_reply()
3853 req->r_tid = 0; in handle_reply()
3858 if (m.num_ops != req->r_num_ops) { in handle_reply()
3860 req->r_num_ops, req->r_tid); in handle_reply()
3863 for (i = 0; i < req->r_num_ops; i++) { in handle_reply()
3865 req->r_tid, i, m.rval[i], m.outdata_len[i]); in handle_reply()
3866 req->r_ops[i].rval = m.rval[i]; in handle_reply()
3867 req->r_ops[i].outdata_len = m.outdata_len[i]; in handle_reply()
3870 if (data_len != le32_to_cpu(msg->hdr.data_len)) { in handle_reply()
3872 le32_to_cpu(msg->hdr.data_len), req->r_tid); in handle_reply()
3876 req, req->r_tid, m.result, data_len); in handle_reply()
3883 req->r_version = m.user_version; in handle_reply()
3884 req->r_result = m.result ?: data_len; in handle_reply()
3886 mutex_unlock(&osd->lock); in handle_reply()
3887 up_read(&osdc->lock); in handle_reply()
3893 complete_request(req, -EIO); in handle_reply()
3895 mutex_unlock(&osd->lock); in handle_reply()
3897 up_read(&osdc->lock); in handle_reply()
3904 for (n = rb_first(&osdc->osdmap->pg_pools); n; n = rb_next(n)) { in set_pool_was_full()
3908 pi->was_full = __pool_full(pi); in set_pool_was_full()
3916 pi = ceph_pg_pool_by_id(osdc->osdmap, pool_id); in pool_cleared_full()
3920 return pi->was_full && !__pool_full(pi); in pool_cleared_full()
3926 struct ceph_osd_client *osdc = lreq->osdc; in recalc_linger_target()
3929 ct_res = calc_target(osdc, &lreq->t, true); in recalc_linger_target()
3933 osd = lookup_create_osd(osdc, lreq->t.osd, true); in recalc_linger_target()
3934 if (osd != lreq->osd) { in recalc_linger_target()
3935 unlink_linger(lreq->osd, lreq); in recalc_linger_target()
3953 struct ceph_osd_client *osdc = osd->o_osdc; in scan_requests()
3957 for (n = rb_first(&osd->o_linger_requests); n; ) { in scan_requests()
3965 lreq->linger_id); in scan_requests()
3971 pool_cleared_full(osdc, lreq->t.base_oloc.pool)); in scan_requests()
3983 if (list_empty(&lreq->scan_item)) in scan_requests()
3984 list_add_tail(&lreq->scan_item, need_resend_linger); in scan_requests()
3987 list_del_init(&lreq->scan_item); in scan_requests()
3993 for (n = rb_first(&osd->o_requests); n; ) { in scan_requests()
4000 dout("%s req %p tid %llu\n", __func__, req, req->r_tid); in scan_requests()
4001 ct_res = calc_target(osdc, &req->r_t, false); in scan_requests()
4006 pool_cleared_full(osdc, req->r_t.base_oloc.pool)); in scan_requests()
4008 (!(req->r_flags & CEPH_OSD_FLAG_WRITE) || in scan_requests()
4040 ceph_msgr2(osdc->client), in handle_one_map()
4041 osdc->osdmap); in handle_one_map()
4043 newmap = ceph_osdmap_decode(&p, end, ceph_msgr2(osdc->client)); in handle_one_map()
4047 if (newmap != osdc->osdmap) { in handle_one_map()
4049 * Preserve ->was_full before destroying the old map. in handle_one_map()
4050 * For pools that weren't in the old map, ->was_full in handle_one_map()
4053 for (n = rb_first(&newmap->pg_pools); n; n = rb_next(n)) { in handle_one_map()
4058 old_pi = ceph_pg_pool_by_id(osdc->osdmap, pi->id); in handle_one_map()
4060 pi->was_full = old_pi->was_full; in handle_one_map()
4062 WARN_ON(pi->was_full); in handle_one_map()
4065 if (osdc->osdmap->epoch && in handle_one_map()
4066 osdc->osdmap->epoch + 1 < newmap->epoch) { in handle_one_map()
4071 ceph_osdmap_destroy(osdc->osdmap); in handle_one_map()
4072 osdc->osdmap = newmap; in handle_one_map()
4076 scan_requests(&osdc->homeless_osd, skipped_map, was_full, true, in handle_one_map()
4079 for (n = rb_first(&osdc->osds); n; ) { in handle_one_map()
4086 if (!ceph_osd_is_up(osdc->osdmap, osd->o_osd) || in handle_one_map()
4087 memcmp(&osd->o_con.peer_addr, in handle_one_map()
4088 ceph_osd_addr(osdc->osdmap, osd->o_osd), in handle_one_map()
4111 if (req->r_t.epoch < osdc->osdmap->epoch) { in kick_requests()
4112 ct_res = calc_target(osdc, &req->r_t, false); in kick_requests()
4128 osd = lookup_create_osd(osdc, req->r_t.osd, true); in kick_requests()
4130 if (!req->r_linger) { in kick_requests()
4131 if (!osd_homeless(osd) && !req->r_t.paused) in kick_requests()
4139 if (!osd_homeless(lreq->osd)) in kick_requests()
4142 list_del_init(&lreq->scan_item); in kick_requests()
4155 void *p = msg->front.iov_base; in ceph_osdc_handle_map()
4156 void *const end = p + msg->front.iov_len; in ceph_osdc_handle_map()
4167 dout("%s have %u\n", __func__, osdc->osdmap->epoch); in ceph_osdc_handle_map()
4168 down_write(&osdc->lock); in ceph_osdc_handle_map()
4173 if (ceph_check_fsid(osdc->client, &fsid) < 0) in ceph_osdc_handle_map()
4189 if (osdc->osdmap->epoch && in ceph_osdc_handle_map()
4190 osdc->osdmap->epoch + 1 == epoch) { in ceph_osdc_handle_map()
4203 nr_maps--; in ceph_osdc_handle_map()
4217 dout("skipping non-latest full map %u len %d\n", in ceph_osdc_handle_map()
4219 } else if (osdc->osdmap->epoch >= epoch) { in ceph_osdc_handle_map()
4222 osdc->osdmap->epoch); in ceph_osdc_handle_map()
4231 nr_maps--; in ceph_osdc_handle_map()
4245 osdc->osdmap->epoch < osdc->epoch_barrier) in ceph_osdc_handle_map()
4251 ceph_monc_got_map(&osdc->client->monc, CEPH_SUB_OSDMAP, in ceph_osdc_handle_map()
4252 osdc->osdmap->epoch); in ceph_osdc_handle_map()
4253 up_write(&osdc->lock); in ceph_osdc_handle_map()
4254 wake_up_all(&osdc->client->auth_wq); in ceph_osdc_handle_map()
4260 up_write(&osdc->lock); in ceph_osdc_handle_map()
4272 for (n = rb_first(&osd->o_requests); n; ) { in kick_osd_requests()
4278 if (!req->r_linger) { in kick_osd_requests()
4279 if (!req->r_t.paused) in kick_osd_requests()
4285 for (n = rb_first(&osd->o_linger_requests); n; n = rb_next(n)) { in kick_osd_requests()
4298 struct ceph_osd *osd = con->private; in osd_fault()
4299 struct ceph_osd_client *osdc = osd->o_osdc; in osd_fault()
4301 dout("%s osd %p osd%d\n", __func__, osd, osd->o_osd); in osd_fault()
4303 down_write(&osdc->lock); in osd_fault()
4305 dout("%s osd%d unknown\n", __func__, osd->o_osd); in osd_fault()
4314 up_write(&osdc->lock); in osd_fault()
4328 void *p = msg->front.iov_base; in decode_MOSDBackoff()
4329 void *const end = p + msg->front.iov_len; in decode_MOSDBackoff()
4338 ret = ceph_decode_pgid(&p, end, &m->spgid.pgid); in decode_MOSDBackoff()
4342 ceph_decode_8_safe(&p, end, m->spgid.shard, e_inval); in decode_MOSDBackoff()
4343 ceph_decode_32_safe(&p, end, m->map_epoch, e_inval); in decode_MOSDBackoff()
4344 ceph_decode_8_safe(&p, end, m->op, e_inval); in decode_MOSDBackoff()
4345 ceph_decode_64_safe(&p, end, m->id, e_inval); in decode_MOSDBackoff()
4347 m->begin = kzalloc(sizeof(*m->begin), GFP_NOIO); in decode_MOSDBackoff()
4348 if (!m->begin) in decode_MOSDBackoff()
4349 return -ENOMEM; in decode_MOSDBackoff()
4351 ret = decode_hoid(&p, end, m->begin); in decode_MOSDBackoff()
4353 free_hoid(m->begin); in decode_MOSDBackoff()
4357 m->end = kzalloc(sizeof(*m->end), GFP_NOIO); in decode_MOSDBackoff()
4358 if (!m->end) { in decode_MOSDBackoff()
4359 free_hoid(m->begin); in decode_MOSDBackoff()
4360 return -ENOMEM; in decode_MOSDBackoff()
4363 ret = decode_hoid(&p, end, m->end); in decode_MOSDBackoff()
4365 free_hoid(m->begin); in decode_MOSDBackoff()
4366 free_hoid(m->end); in decode_MOSDBackoff()
4373 return -EINVAL; in decode_MOSDBackoff()
4388 hoid_encoding_size(backoff->begin); in create_backoff_message()
4390 hoid_encoding_size(backoff->end); in create_backoff_message()
4396 p = msg->front.iov_base; in create_backoff_message()
4397 end = p + msg->front_alloc_len; in create_backoff_message()
4399 encode_spgid(&p, &backoff->spgid); in create_backoff_message()
4402 ceph_encode_64(&p, backoff->id); in create_backoff_message()
4403 encode_hoid(&p, end, backoff->begin); in create_backoff_message()
4404 encode_hoid(&p, end, backoff->end); in create_backoff_message()
4407 msg->front.iov_len = p - msg->front.iov_base; in create_backoff_message()
4408 msg->hdr.version = cpu_to_le16(1); /* MOSDBackoff v1 */ in create_backoff_message()
4409 msg->hdr.front_len = cpu_to_le32(msg->front.iov_len); in create_backoff_message()
4420 dout("%s osd%d spgid %llu.%xs%d id %llu\n", __func__, osd->o_osd, in handle_backoff_block()
4421 m->spgid.pgid.pool, m->spgid.pgid.seed, m->spgid.shard, m->id); in handle_backoff_block()
4423 spg = lookup_spg_mapping(&osd->o_backoff_mappings, &m->spgid); in handle_backoff_block()
4430 spg->spgid = m->spgid; /* struct */ in handle_backoff_block()
4431 insert_spg_mapping(&osd->o_backoff_mappings, spg); in handle_backoff_block()
4439 backoff->spgid = m->spgid; /* struct */ in handle_backoff_block()
4440 backoff->id = m->id; in handle_backoff_block()
4441 backoff->begin = m->begin; in handle_backoff_block()
4442 m->begin = NULL; /* backoff now owns this */ in handle_backoff_block()
4443 backoff->end = m->end; in handle_backoff_block()
4444 m->end = NULL; /* ditto */ in handle_backoff_block()
4446 insert_backoff(&spg->backoffs, backoff); in handle_backoff_block()
4447 insert_backoff_by_id(&osd->o_backoffs_by_id, backoff); in handle_backoff_block()
4453 msg = create_backoff_message(backoff, m->map_epoch); in handle_backoff_block()
4458 ceph_con_send(&osd->o_con, msg); in handle_backoff_block()
4480 dout("%s osd%d spgid %llu.%xs%d id %llu\n", __func__, osd->o_osd, in handle_backoff_unblock()
4481 m->spgid.pgid.pool, m->spgid.pgid.seed, m->spgid.shard, m->id); in handle_backoff_unblock()
4483 backoff = lookup_backoff_by_id(&osd->o_backoffs_by_id, m->id); in handle_backoff_unblock()
4486 __func__, osd->o_osd, m->spgid.pgid.pool, in handle_backoff_unblock()
4487 m->spgid.pgid.seed, m->spgid.shard, m->id); in handle_backoff_unblock()
4491 if (hoid_compare(backoff->begin, m->begin) && in handle_backoff_unblock()
4492 hoid_compare(backoff->end, m->end)) { in handle_backoff_unblock()
4494 __func__, osd->o_osd, m->spgid.pgid.pool, in handle_backoff_unblock()
4495 m->spgid.pgid.seed, m->spgid.shard, m->id); in handle_backoff_unblock()
4499 spg = lookup_spg_mapping(&osd->o_backoff_mappings, &backoff->spgid); in handle_backoff_unblock()
4502 erase_backoff(&spg->backoffs, backoff); in handle_backoff_unblock()
4503 erase_backoff_by_id(&osd->o_backoffs_by_id, backoff); in handle_backoff_unblock()
4506 if (RB_EMPTY_ROOT(&spg->backoffs)) { in handle_backoff_unblock()
4507 erase_spg_mapping(&osd->o_backoff_mappings, spg); in handle_backoff_unblock()
4511 for (n = rb_first(&osd->o_requests); n; n = rb_next(n)) { in handle_backoff_unblock()
4515 if (!ceph_spg_compare(&req->r_t.spgid, &m->spgid)) { in handle_backoff_unblock()
4517 * Match against @m, not @backoff -- the PG may in handle_backoff_unblock()
4520 if (target_contained_by(&req->r_t, m->begin, m->end)) { in handle_backoff_unblock()
4533 struct ceph_osd_client *osdc = osd->o_osdc; in handle_backoff()
4537 down_read(&osdc->lock); in handle_backoff()
4539 dout("%s osd%d unknown\n", __func__, osd->o_osd); in handle_backoff()
4540 up_read(&osdc->lock); in handle_backoff()
4543 WARN_ON(osd->o_osd != le64_to_cpu(msg->hdr.src.num)); in handle_backoff()
4545 mutex_lock(&osd->lock); in handle_backoff()
4561 pr_err("%s osd%d unknown op %d\n", __func__, osd->o_osd, m.op); in handle_backoff()
4568 mutex_unlock(&osd->lock); in handle_backoff()
4569 up_read(&osdc->lock); in handle_backoff()
4578 void *p = msg->front.iov_base; in handle_watch_notify()
4579 void *const end = p + msg->front.iov_len; in handle_watch_notify()
4602 if (le16_to_cpu(msg->hdr.version) >= 2) in handle_watch_notify()
4605 if (le16_to_cpu(msg->hdr.version) >= 3) in handle_watch_notify()
4608 down_read(&osdc->lock); in handle_watch_notify()
4609 lreq = lookup_linger_osdc(&osdc->linger_requests, cookie); in handle_watch_notify()
4616 mutex_lock(&lreq->lock); in handle_watch_notify()
4618 opcode, cookie, lreq, lreq->is_watch); in handle_watch_notify()
4620 if (!lreq->last_error) { in handle_watch_notify()
4621 lreq->last_error = -ENOTCONN; in handle_watch_notify()
4624 } else if (!lreq->is_watch) { in handle_watch_notify()
4626 if (lreq->notify_id && lreq->notify_id != notify_id) { in handle_watch_notify()
4628 lreq->notify_id, notify_id); in handle_watch_notify()
4629 } else if (!completion_done(&lreq->notify_finish_wait)) { in handle_watch_notify()
4631 msg->num_data_items ? &msg->data[0] : NULL; in handle_watch_notify()
4634 if (lreq->preply_pages) { in handle_watch_notify()
4635 WARN_ON(data->type != in handle_watch_notify()
4637 *lreq->preply_pages = data->pages; in handle_watch_notify()
4638 *lreq->preply_len = data->length; in handle_watch_notify()
4639 data->own_pages = false; in handle_watch_notify()
4642 lreq->notify_finish_error = return_code; in handle_watch_notify()
4643 complete_all(&lreq->notify_finish_wait); in handle_watch_notify()
4649 pr_err("failed to allocate notify-lwork\n"); in handle_watch_notify()
4653 lwork->notify.notify_id = notify_id; in handle_watch_notify()
4654 lwork->notify.notifier_id = notifier_id; in handle_watch_notify()
4655 lwork->notify.payload = payload; in handle_watch_notify()
4656 lwork->notify.payload_len = payload_len; in handle_watch_notify()
4657 lwork->notify.msg = ceph_msg_get(msg); in handle_watch_notify()
4662 mutex_unlock(&lreq->lock); in handle_watch_notify()
4664 up_read(&osdc->lock); in handle_watch_notify()
4677 down_read(&osdc->lock); in ceph_osdc_start_request()
4679 up_read(&osdc->lock); in ceph_osdc_start_request()
4694 struct ceph_osd_client *osdc = req->r_osdc; in ceph_osdc_cancel_request()
4696 down_write(&osdc->lock); in ceph_osdc_cancel_request()
4697 if (req->r_osd) in ceph_osdc_cancel_request()
4699 up_write(&osdc->lock); in ceph_osdc_cancel_request()
4711 dout("%s req %p tid %llu\n", __func__, req, req->r_tid); in wait_request_timeout()
4712 left = wait_for_completion_killable_timeout(&req->r_completion, in wait_request_timeout()
4715 left = left ?: -ETIMEDOUT; in wait_request_timeout()
4718 left = req->r_result; /* completed */ in wait_request_timeout()
4735 * sync - wait for all in-flight requests to flush. avoid starvation.
4740 u64 last_tid = atomic64_read(&osdc->last_tid); in ceph_osdc_sync()
4743 down_read(&osdc->lock); in ceph_osdc_sync()
4744 for (n = rb_first(&osdc->osds); n; n = rb_next(n)) { in ceph_osdc_sync()
4747 mutex_lock(&osd->lock); in ceph_osdc_sync()
4748 for (p = rb_first(&osd->o_requests); p; p = rb_next(p)) { in ceph_osdc_sync()
4752 if (req->r_tid > last_tid) in ceph_osdc_sync()
4755 if (!(req->r_flags & CEPH_OSD_FLAG_WRITE)) in ceph_osdc_sync()
4759 mutex_unlock(&osd->lock); in ceph_osdc_sync()
4760 up_read(&osdc->lock); in ceph_osdc_sync()
4762 __func__, req, req->r_tid, last_tid); in ceph_osdc_sync()
4763 wait_for_completion(&req->r_completion); in ceph_osdc_sync()
4768 mutex_unlock(&osd->lock); in ceph_osdc_sync()
4771 up_read(&osdc->lock); in ceph_osdc_sync()
4792 return ERR_PTR(-ENOMEM); in ceph_osdc_watch()
4794 lreq->is_watch = true; in ceph_osdc_watch()
4795 lreq->wcb = wcb; in ceph_osdc_watch()
4796 lreq->errcb = errcb; in ceph_osdc_watch()
4797 lreq->data = data; in ceph_osdc_watch()
4798 lreq->watch_valid_thru = jiffies; in ceph_osdc_watch()
4800 ceph_oid_copy(&lreq->t.base_oid, oid); in ceph_osdc_watch()
4801 ceph_oloc_copy(&lreq->t.base_oloc, oloc); in ceph_osdc_watch()
4802 lreq->t.flags = CEPH_OSD_FLAG_WRITE; in ceph_osdc_watch()
4803 ktime_get_real_ts64(&lreq->mtime); in ceph_osdc_watch()
4830 struct ceph_options *opts = osdc->client->options; in ceph_osdc_unwatch()
4836 return -ENOMEM; in ceph_osdc_unwatch()
4838 ceph_oid_copy(&req->r_base_oid, &lreq->t.base_oid); in ceph_osdc_unwatch()
4839 ceph_oloc_copy(&req->r_base_oloc, &lreq->t.base_oloc); in ceph_osdc_unwatch()
4840 req->r_flags = CEPH_OSD_FLAG_WRITE; in ceph_osdc_unwatch()
4841 ktime_get_real_ts64(&req->r_mtime); in ceph_osdc_unwatch()
4843 lreq->linger_id, 0); in ceph_osdc_unwatch()
4852 ret = wait_request_timeout(req, opts->mount_timeout); in ceph_osdc_unwatch()
4872 return -ENOMEM; in osd_req_op_notify_ack_init()
4884 return -ENOMEM; in osd_req_op_notify_ack_init()
4887 ceph_osd_data_pagelist_init(&op->notify_ack.request_data, pl); in osd_req_op_notify_ack_init()
4888 op->indata_len = pl->length; in osd_req_op_notify_ack_init()
4905 return -ENOMEM; in ceph_osdc_notify_ack()
4907 ceph_oid_copy(&req->r_base_oid, oid); in ceph_osdc_notify_ack()
4908 ceph_oloc_copy(&req->r_base_oloc, oloc); in ceph_osdc_notify_ack()
4909 req->r_flags = CEPH_OSD_FLAG_READ; in ceph_osdc_notify_ack()
4957 return -ENOMEM; in ceph_osdc_notify()
4959 lreq->request_pl = ceph_pagelist_alloc(GFP_NOIO); in ceph_osdc_notify()
4960 if (!lreq->request_pl) { in ceph_osdc_notify()
4961 ret = -ENOMEM; in ceph_osdc_notify()
4965 ret = ceph_pagelist_encode_32(lreq->request_pl, 1); /* prot_ver */ in ceph_osdc_notify()
4966 ret |= ceph_pagelist_encode_32(lreq->request_pl, timeout); in ceph_osdc_notify()
4967 ret |= ceph_pagelist_encode_32(lreq->request_pl, payload_len); in ceph_osdc_notify()
4968 ret |= ceph_pagelist_append(lreq->request_pl, payload, payload_len); in ceph_osdc_notify()
4970 ret = -ENOMEM; in ceph_osdc_notify()
4975 lreq->notify_id_pages = ceph_alloc_page_vector(1, GFP_NOIO); in ceph_osdc_notify()
4976 if (IS_ERR(lreq->notify_id_pages)) { in ceph_osdc_notify()
4977 ret = PTR_ERR(lreq->notify_id_pages); in ceph_osdc_notify()
4978 lreq->notify_id_pages = NULL; in ceph_osdc_notify()
4982 lreq->preply_pages = preply_pages; in ceph_osdc_notify()
4983 lreq->preply_len = preply_len; in ceph_osdc_notify()
4985 ceph_oid_copy(&lreq->t.base_oid, oid); in ceph_osdc_notify()
4986 ceph_oloc_copy(&lreq->t.base_oloc, oloc); in ceph_osdc_notify()
4987 lreq->t.flags = CEPH_OSD_FLAG_READ; in ceph_osdc_notify()
5015 ret = -EINVAL; in decode_watcher()
5016 ceph_decode_copy_safe(p, end, &item->name, sizeof(item->name), bad); in decode_watcher()
5017 ceph_decode_64_safe(p, end, item->cookie, bad); in decode_watcher()
5021 ret = ceph_decode_entity_addr(p, end, &item->addr); in decode_watcher()
5029 ENTITY_NAME(item->name), item->cookie, in decode_watcher()
5030 ceph_pr_addr(&item->addr)); in decode_watcher()
5052 return -ENOMEM; in decode_watchers()
5082 return -ENOMEM; in ceph_osdc_list_watchers()
5084 ceph_oid_copy(&req->r_base_oid, oid); in ceph_osdc_list_watchers()
5085 ceph_oloc_copy(&req->r_base_oloc, oloc); in ceph_osdc_list_watchers()
5086 req->r_flags = CEPH_OSD_FLAG_READ; in ceph_osdc_list_watchers()
5107 void *const end = p + req->r_ops[0].outdata_len; in ceph_osdc_list_watchers()
5119 * Call all pending notify callbacks - for use after a watch is
5125 flush_workqueue(osdc->notify_wq); in ceph_osdc_flush_notifies()
5131 down_read(&osdc->lock); in ceph_osdc_maybe_request_map()
5133 up_read(&osdc->lock); in ceph_osdc_maybe_request_map()
5155 return -E2BIG; in ceph_osdc_call()
5159 return -ENOMEM; in ceph_osdc_call()
5161 ceph_oid_copy(&req->r_base_oid, oid); in ceph_osdc_call()
5162 ceph_oloc_copy(&req->r_base_oloc, oloc); in ceph_osdc_call()
5163 req->r_flags = flags; in ceph_osdc_call()
5183 ret = req->r_ops[0].rval; in ceph_osdc_call()
5185 *resp_len = req->r_ops[0].outdata_len; in ceph_osdc_call()
5201 down_write(&osdc->lock); in ceph_osdc_reopen_osds()
5202 for (n = rb_first(&osdc->osds); n; ) { in ceph_osdc_reopen_osds()
5209 up_write(&osdc->lock); in ceph_osdc_reopen_osds()
5220 osdc->client = client; in ceph_osdc_init()
5221 init_rwsem(&osdc->lock); in ceph_osdc_init()
5222 osdc->osds = RB_ROOT; in ceph_osdc_init()
5223 INIT_LIST_HEAD(&osdc->osd_lru); in ceph_osdc_init()
5224 spin_lock_init(&osdc->osd_lru_lock); in ceph_osdc_init()
5225 osd_init(&osdc->homeless_osd); in ceph_osdc_init()
5226 osdc->homeless_osd.o_osdc = osdc; in ceph_osdc_init()
5227 osdc->homeless_osd.o_osd = CEPH_HOMELESS_OSD; in ceph_osdc_init()
5228 osdc->last_linger_id = CEPH_LINGER_ID_START; in ceph_osdc_init()
5229 osdc->linger_requests = RB_ROOT; in ceph_osdc_init()
5230 osdc->map_checks = RB_ROOT; in ceph_osdc_init()
5231 osdc->linger_map_checks = RB_ROOT; in ceph_osdc_init()
5232 INIT_DELAYED_WORK(&osdc->timeout_work, handle_timeout); in ceph_osdc_init()
5233 INIT_DELAYED_WORK(&osdc->osds_timeout_work, handle_osds_timeout); in ceph_osdc_init()
5235 err = -ENOMEM; in ceph_osdc_init()
5236 osdc->osdmap = ceph_osdmap_alloc(); in ceph_osdc_init()
5237 if (!osdc->osdmap) in ceph_osdc_init()
5240 osdc->req_mempool = mempool_create_slab_pool(10, in ceph_osdc_init()
5242 if (!osdc->req_mempool) in ceph_osdc_init()
5245 err = ceph_msgpool_init(&osdc->msgpool_op, CEPH_MSG_OSD_OP, in ceph_osdc_init()
5249 err = ceph_msgpool_init(&osdc->msgpool_op_reply, CEPH_MSG_OSD_OPREPLY, in ceph_osdc_init()
5255 err = -ENOMEM; in ceph_osdc_init()
5256 osdc->notify_wq = create_singlethread_workqueue("ceph-watch-notify"); in ceph_osdc_init()
5257 if (!osdc->notify_wq) in ceph_osdc_init()
5260 osdc->completion_wq = create_singlethread_workqueue("ceph-completion"); in ceph_osdc_init()
5261 if (!osdc->completion_wq) in ceph_osdc_init()
5264 schedule_delayed_work(&osdc->timeout_work, in ceph_osdc_init()
5265 osdc->client->options->osd_keepalive_timeout); in ceph_osdc_init()
5266 schedule_delayed_work(&osdc->osds_timeout_work, in ceph_osdc_init()
5267 round_jiffies_relative(osdc->client->options->osd_idle_ttl)); in ceph_osdc_init()
5272 destroy_workqueue(osdc->notify_wq); in ceph_osdc_init()
5274 ceph_msgpool_destroy(&osdc->msgpool_op_reply); in ceph_osdc_init()
5276 ceph_msgpool_destroy(&osdc->msgpool_op); in ceph_osdc_init()
5278 mempool_destroy(osdc->req_mempool); in ceph_osdc_init()
5280 ceph_osdmap_destroy(osdc->osdmap); in ceph_osdc_init()
5287 destroy_workqueue(osdc->completion_wq); in ceph_osdc_stop()
5288 destroy_workqueue(osdc->notify_wq); in ceph_osdc_stop()
5289 cancel_delayed_work_sync(&osdc->timeout_work); in ceph_osdc_stop()
5290 cancel_delayed_work_sync(&osdc->osds_timeout_work); in ceph_osdc_stop()
5292 down_write(&osdc->lock); in ceph_osdc_stop()
5293 while (!RB_EMPTY_ROOT(&osdc->osds)) { in ceph_osdc_stop()
5294 struct ceph_osd *osd = rb_entry(rb_first(&osdc->osds), in ceph_osdc_stop()
5298 up_write(&osdc->lock); in ceph_osdc_stop()
5299 WARN_ON(refcount_read(&osdc->homeless_osd.o_ref) != 1); in ceph_osdc_stop()
5300 osd_cleanup(&osdc->homeless_osd); in ceph_osdc_stop()
5302 WARN_ON(!list_empty(&osdc->osd_lru)); in ceph_osdc_stop()
5303 WARN_ON(!RB_EMPTY_ROOT(&osdc->linger_requests)); in ceph_osdc_stop()
5304 WARN_ON(!RB_EMPTY_ROOT(&osdc->map_checks)); in ceph_osdc_stop()
5305 WARN_ON(!RB_EMPTY_ROOT(&osdc->linger_map_checks)); in ceph_osdc_stop()
5306 WARN_ON(atomic_read(&osdc->num_requests)); in ceph_osdc_stop()
5307 WARN_ON(atomic_read(&osdc->num_homeless)); in ceph_osdc_stop()
5309 ceph_osdmap_destroy(osdc->osdmap); in ceph_osdc_stop()
5310 mempool_destroy(osdc->req_mempool); in ceph_osdc_stop()
5311 ceph_msgpool_destroy(&osdc->msgpool_op); in ceph_osdc_stop()
5312 ceph_msgpool_destroy(&osdc->msgpool_op_reply); in ceph_osdc_stop()
5334 op->copy_from.snapid = src_snapid; in osd_req_op_copy_from_init()
5335 op->copy_from.src_version = src_version; in osd_req_op_copy_from_init()
5336 op->copy_from.flags = copy_from_flags; in osd_req_op_copy_from_init()
5337 op->copy_from.src_fadvise_flags = src_fadvise_flags; in osd_req_op_copy_from_init()
5341 ceph_encode_string(&p, end, src_oid->name, src_oid->name_len); in osd_req_op_copy_from_init()
5345 op->indata_len = PAGE_SIZE - (end - p); in osd_req_op_copy_from_init()
5347 ceph_osd_data_pages_init(&op->copy_from.osd_data, pages, in osd_req_op_copy_from_init()
5348 op->indata_len, 0, false, true); in osd_req_op_copy_from_init()
5362 return ceph_osd_request_cache ? 0 : -ENOMEM; in ceph_osdc_setup()
5377 struct ceph_osd *osd = con->private; in osd_dispatch()
5378 struct ceph_osd_client *osdc = osd->o_osdc; in osd_dispatch()
5379 int type = le16_to_cpu(msg->hdr.type); in osd_dispatch()
5408 if (req->r_flags & CEPH_OSD_FLAG_READ) { in sparse_data_requested()
5411 for (i = 0; i < req->r_num_ops; ++i) { in sparse_data_requested()
5412 struct ceph_osd_req_op *op = &req->r_ops[i]; in sparse_data_requested()
5414 if (op->op == CEPH_OSD_OP_SPARSE_READ) in sparse_data_requested()
5415 len += op->extent.length; in sparse_data_requested()
5424 * message at the moment - for now, just skip the message.
5430 struct ceph_osd *osd = con->private; in get_reply()
5431 struct ceph_osd_client *osdc = osd->o_osdc; in get_reply()
5434 int front_len = le32_to_cpu(hdr->front_len); in get_reply()
5435 int data_len = le32_to_cpu(hdr->data_len); in get_reply()
5436 u64 tid = le64_to_cpu(hdr->tid); in get_reply()
5439 down_read(&osdc->lock); in get_reply()
5441 dout("%s osd%d unknown, skipping\n", __func__, osd->o_osd); in get_reply()
5445 WARN_ON(osd->o_osd != le64_to_cpu(hdr->src.num)); in get_reply()
5447 mutex_lock(&osd->lock); in get_reply()
5448 req = lookup_request(&osd->o_requests, tid); in get_reply()
5451 osd->o_osd, tid); in get_reply()
5456 ceph_msg_revoke_incoming(req->r_reply); in get_reply()
5458 if (front_len > req->r_reply->front_alloc_len) { in get_reply()
5460 __func__, osd->o_osd, req->r_tid, front_len, in get_reply()
5461 req->r_reply->front_alloc_len); in get_reply()
5466 ceph_msg_put(req->r_reply); in get_reply()
5467 req->r_reply = m; in get_reply()
5471 if (!srlen && data_len > req->r_reply->data_length) { in get_reply()
5473 __func__, osd->o_osd, req->r_tid, data_len, in get_reply()
5474 req->r_reply->data_length); in get_reply()
5480 m = ceph_msg_get(req->r_reply); in get_reply()
5481 m->sparse_read_total = srlen; in get_reply()
5486 mutex_unlock(&osd->lock); in get_reply()
5488 up_read(&osdc->lock); in get_reply()
5495 int type = le16_to_cpu(hdr->type); in alloc_msg_with_page_vector()
5496 u32 front_len = le32_to_cpu(hdr->front_len); in alloc_msg_with_page_vector()
5497 u32 data_len = le32_to_cpu(hdr->data_len); in alloc_msg_with_page_vector()
5523 struct ceph_osd *osd = con->private; in osd_alloc_msg()
5524 int type = le16_to_cpu(hdr->type); in osd_alloc_msg()
5536 osd->o_osd, type); in osd_alloc_msg()
5547 struct ceph_osd *osd = con->private; in osd_get_con()
5555 struct ceph_osd *osd = con->private; in osd_put_con()
5570 struct ceph_osd *o = con->private; in osd_get_authorizer()
5571 struct ceph_osd_client *osdc = o->o_osdc; in osd_get_authorizer()
5572 struct ceph_auth_client *ac = osdc->client->monc.auth; in osd_get_authorizer()
5573 struct ceph_auth_handshake *auth = &o->o_auth; in osd_get_authorizer()
5587 struct ceph_osd *o = con->private; in osd_add_authorizer_challenge()
5588 struct ceph_osd_client *osdc = o->o_osdc; in osd_add_authorizer_challenge()
5589 struct ceph_auth_client *ac = osdc->client->monc.auth; in osd_add_authorizer_challenge()
5591 return ceph_auth_add_authorizer_challenge(ac, o->o_auth.authorizer, in osd_add_authorizer_challenge()
5597 struct ceph_osd *o = con->private; in osd_verify_authorizer_reply()
5598 struct ceph_osd_client *osdc = o->o_osdc; in osd_verify_authorizer_reply()
5599 struct ceph_auth_client *ac = osdc->client->monc.auth; in osd_verify_authorizer_reply()
5600 struct ceph_auth_handshake *auth = &o->o_auth; in osd_verify_authorizer_reply()
5602 return ceph_auth_verify_authorizer_reply(ac, auth->authorizer, in osd_verify_authorizer_reply()
5603 auth->authorizer_reply_buf, auth->authorizer_reply_buf_len, in osd_verify_authorizer_reply()
5609 struct ceph_osd *o = con->private; in osd_invalidate_authorizer()
5610 struct ceph_osd_client *osdc = o->o_osdc; in osd_invalidate_authorizer()
5611 struct ceph_auth_client *ac = osdc->client->monc.auth; in osd_invalidate_authorizer()
5614 return ceph_monc_validate_auth(&osdc->client->monc); in osd_invalidate_authorizer()
5621 struct ceph_osd *o = con->private; in osd_get_auth_request()
5622 struct ceph_auth_client *ac = o->o_osdc->client->monc.auth; in osd_get_auth_request()
5623 struct ceph_auth_handshake *auth = &o->o_auth; in osd_get_auth_request()
5631 *authorizer = auth->authorizer_buf; in osd_get_auth_request()
5632 *authorizer_len = auth->authorizer_buf_len; in osd_get_auth_request()
5641 struct ceph_osd *o = con->private; in osd_handle_auth_reply_more()
5642 struct ceph_auth_client *ac = o->o_osdc->client->monc.auth; in osd_handle_auth_reply_more()
5643 struct ceph_auth_handshake *auth = &o->o_auth; in osd_handle_auth_reply_more()
5651 *authorizer = auth->authorizer_buf; in osd_handle_auth_reply_more()
5652 *authorizer_len = auth->authorizer_buf_len; in osd_handle_auth_reply_more()
5661 struct ceph_osd *o = con->private; in osd_handle_auth_done()
5662 struct ceph_auth_client *ac = o->o_osdc->client->monc.auth; in osd_handle_auth_done()
5663 struct ceph_auth_handshake *auth = &o->o_auth; in osd_handle_auth_done()
5675 struct ceph_osd *o = con->private; in osd_handle_auth_bad_method()
5676 struct ceph_mon_client *monc = &o->o_osdc->client->monc; in osd_handle_auth_bad_method()
5679 if (ceph_auth_handle_bad_authorizer(monc->auth, CEPH_ENTITY_TYPE_OSD, in osd_handle_auth_bad_method()
5688 return -EACCES; in osd_handle_auth_bad_method()
5693 int type = le16_to_cpu(msg->hdr.type); in osd_reencode_message()
5701 struct ceph_osd *o = msg->con->private; in osd_sign_message()
5702 struct ceph_auth_handshake *auth = &o->o_auth; in osd_sign_message()
5709 struct ceph_osd *o = msg->con->private; in osd_check_message_signature()
5710 struct ceph_auth_handshake *auth = &o->o_auth; in osd_check_message_signature()
5727 len -= plen; in advance_cursor()
5735 struct ceph_osd *o = con->private; in prep_next_sparse_read()
5736 struct ceph_sparse_read *sr = &o->o_sparse_read; in prep_next_sparse_read()
5740 spin_lock(&o->o_requests_lock); in prep_next_sparse_read()
5741 req = lookup_request(&o->o_requests, le64_to_cpu(con->in_msg->hdr.tid)); in prep_next_sparse_read()
5743 spin_unlock(&o->o_requests_lock); in prep_next_sparse_read()
5744 return -EBADR; in prep_next_sparse_read()
5747 if (o->o_sparse_op_idx < 0) { in prep_next_sparse_read()
5749 __func__, o->o_osd); in prep_next_sparse_read()
5753 op = &req->r_ops[o->o_sparse_op_idx]; in prep_next_sparse_read()
5755 WARN_ON_ONCE(op->extent.sparse_ext); in prep_next_sparse_read()
5758 op->extent.sparse_ext = sr->sr_extent; in prep_next_sparse_read()
5759 sr->sr_extent = NULL; in prep_next_sparse_read()
5760 op->extent.sparse_ext_cnt = sr->sr_count; in prep_next_sparse_read()
5761 sr->sr_ext_len = 0; in prep_next_sparse_read()
5762 dout("%s: [%d] completed extent array len %d cursor->resid %zd\n", in prep_next_sparse_read()
5763 __func__, o->o_osd, op->extent.sparse_ext_cnt, cursor->resid); in prep_next_sparse_read()
5766 if (end < sr->sr_req_len) in prep_next_sparse_read()
5767 advance_cursor(cursor, sr->sr_req_len - end, false); in prep_next_sparse_read()
5773 while (++o->o_sparse_op_idx < req->r_num_ops) { in prep_next_sparse_read()
5774 op = &req->r_ops[o->o_sparse_op_idx]; in prep_next_sparse_read()
5775 if (op->op == CEPH_OSD_OP_SPARSE_READ) in prep_next_sparse_read()
5780 spin_unlock(&o->o_requests_lock); in prep_next_sparse_read()
5781 o->o_sparse_op_idx = -1; in prep_next_sparse_read()
5784 sr->sr_req_off = op->extent.offset; in prep_next_sparse_read()
5785 sr->sr_req_len = op->extent.length; in prep_next_sparse_read()
5786 sr->sr_pos = sr->sr_req_off; in prep_next_sparse_read()
5788 o->o_osd, o->o_sparse_op_idx, sr->sr_req_off, sr->sr_req_len); in prep_next_sparse_read()
5791 sr->sr_ext_len = op->extent.sparse_ext_cnt; in prep_next_sparse_read()
5792 op->extent.sparse_ext_cnt = 0; in prep_next_sparse_read()
5793 sr->sr_extent = op->extent.sparse_ext; in prep_next_sparse_read()
5794 op->extent.sparse_ext = NULL; in prep_next_sparse_read()
5796 spin_unlock(&o->o_requests_lock); in prep_next_sparse_read()
5805 for (i = 0; i < sr->sr_count; i++) { in convert_extent_map()
5806 struct ceph_sparse_extent *ext = &sr->sr_extent[i]; in convert_extent_map() local
5808 ext->off = le64_to_cpu((__force __le64)ext->off); in convert_extent_map()
5809 ext->len = le64_to_cpu((__force __le64)ext->len); in convert_extent_map()
5822 struct ceph_osd *o = con->private; in osd_sparse_read()
5823 struct ceph_sparse_read *sr = &o->o_sparse_read; in osd_sparse_read()
5824 u32 count = sr->sr_count; in osd_sparse_read()
5828 switch (sr->sr_state) { in osd_sparse_read()
5836 ret = sizeof(sr->sr_count); in osd_sparse_read()
5837 *pbuf = (char *)&sr->sr_count; in osd_sparse_read()
5838 sr->sr_state = CEPH_SPARSE_READ_EXTENTS; in osd_sparse_read()
5841 /* Convert sr_count to host-endian */ in osd_sparse_read()
5842 count = le32_to_cpu((__force __le32)sr->sr_count); in osd_sparse_read()
5843 sr->sr_count = count; in osd_sparse_read()
5844 dout("[%d] got %u extents\n", o->o_osd, count); in osd_sparse_read()
5847 if (!sr->sr_extent || count > sr->sr_ext_len) { in osd_sparse_read()
5849 kfree(sr->sr_extent); in osd_sparse_read()
5850 sr->sr_extent = kmalloc_array(count, in osd_sparse_read()
5851 sizeof(*sr->sr_extent), in osd_sparse_read()
5853 if (!sr->sr_extent) { in osd_sparse_read()
5856 return -ENOMEM; in osd_sparse_read()
5858 sr->sr_ext_len = count; in osd_sparse_read()
5860 ret = count * sizeof(*sr->sr_extent); in osd_sparse_read()
5861 *pbuf = (char *)sr->sr_extent; in osd_sparse_read()
5862 sr->sr_state = CEPH_SPARSE_READ_DATA_LEN; in osd_sparse_read()
5869 ret = sizeof(sr->sr_datalen); in osd_sparse_read()
5870 *pbuf = (char *)&sr->sr_datalen; in osd_sparse_read()
5871 sr->sr_state = CEPH_SPARSE_READ_DATA_PRE; in osd_sparse_read()
5874 /* Convert sr_datalen to host-endian */ in osd_sparse_read()
5875 sr->sr_datalen = le32_to_cpu((__force __le32)sr->sr_datalen); in osd_sparse_read()
5877 len += sr->sr_extent[i].len; in osd_sparse_read()
5878 if (sr->sr_datalen != len) { in osd_sparse_read()
5880 sr->sr_datalen, len); in osd_sparse_read()
5881 return -EREMOTEIO; in osd_sparse_read()
5883 sr->sr_state = CEPH_SPARSE_READ_DATA; in osd_sparse_read()
5886 if (sr->sr_index >= count) { in osd_sparse_read()
5887 sr->sr_state = CEPH_SPARSE_READ_HDR; in osd_sparse_read()
5891 eoff = sr->sr_extent[sr->sr_index].off; in osd_sparse_read()
5892 elen = sr->sr_extent[sr->sr_index].len; in osd_sparse_read()
5894 dout("[%d] ext %d off 0x%llx len 0x%llx\n", in osd_sparse_read()
5895 o->o_osd, sr->sr_index, eoff, elen); in osd_sparse_read()
5900 return -EREMOTEIO; in osd_sparse_read()
5904 if (sr->sr_pos < eoff) in osd_sparse_read()
5905 advance_cursor(cursor, eoff - sr->sr_pos, true); in osd_sparse_read()
5908 sr->sr_pos = eoff + elen; in osd_sparse_read()
5911 cursor->sr_resid = elen; in osd_sparse_read()
5916 ++sr->sr_index; in osd_sparse_read()