1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/ceph/ceph_debug.h> 3 4 #include <linux/bug.h> 5 #include <linux/err.h> 6 #include <linux/overflow.h> 7 #include <linux/random.h> 8 #include <linux/slab.h> 9 #include <linux/types.h> 10 11 #include <linux/ceph/messenger.h> 12 #include <linux/ceph/decode.h> 13 14 #include "mdsmap.h" 15 #include "mds_client.h" 16 #include "super.h" 17 18 #define CEPH_MDS_IS_READY(i, ignore_laggy) \ 19 (m->m_info[i].state > 0 && (ignore_laggy ? true : !m->m_info[i].laggy)) 20 21 static int __mdsmap_get_random_mds(struct ceph_mdsmap *m, bool ignore_laggy) 22 { 23 int n = 0; 24 int i, j; 25 26 /* count */ 27 for (i = 0; i < m->possible_max_rank; i++) 28 if (CEPH_MDS_IS_READY(i, ignore_laggy)) 29 n++; 30 if (n == 0) 31 return -1; 32 33 /* pick */ 34 n = get_random_u32_below(n); 35 for (j = 0, i = 0; i < m->possible_max_rank; i++) { 36 if (CEPH_MDS_IS_READY(i, ignore_laggy)) 37 j++; 38 if (j > n) 39 break; 40 } 41 42 return i; 43 } 44 45 /* 46 * choose a random mds that is "up" (i.e. has a state > 0), or -1. 47 */ 48 int ceph_mdsmap_get_random_mds(struct ceph_mdsmap *m) 49 { 50 int mds; 51 52 mds = __mdsmap_get_random_mds(m, false); 53 if (mds == m->possible_max_rank || mds == -1) 54 mds = __mdsmap_get_random_mds(m, true); 55 56 return mds == m->possible_max_rank ? -1 : mds; 57 } 58 59 #define __decode_and_drop_type(p, end, type, bad) \ 60 do { \ 61 if (*p + sizeof(type) > end) \ 62 goto bad; \ 63 *p += sizeof(type); \ 64 } while (0) 65 66 #define __decode_and_drop_set(p, end, type, bad) \ 67 do { \ 68 u32 n; \ 69 size_t need; \ 70 ceph_decode_32_safe(p, end, n, bad); \ 71 need = sizeof(type) * n; \ 72 ceph_decode_need(p, end, need, bad); \ 73 *p += need; \ 74 } while (0) 75 76 #define __decode_and_drop_map(p, end, ktype, vtype, bad) \ 77 do { \ 78 u32 n; \ 79 size_t need; \ 80 ceph_decode_32_safe(p, end, n, bad); \ 81 need = (sizeof(ktype) + sizeof(vtype)) * n; \ 82 ceph_decode_need(p, end, need, bad); \ 83 *p += need; \ 84 } while (0) 85 86 87 static int __decode_and_drop_compat_set(void **p, void* end) 88 { 89 int i; 90 /* compat, ro_compat, incompat*/ 91 for (i = 0; i < 3; i++) { 92 u32 n; 93 ceph_decode_need(p, end, sizeof(u64) + sizeof(u32), bad); 94 /* mask */ 95 *p += sizeof(u64); 96 /* names (map<u64, string>) */ 97 n = ceph_decode_32(p); 98 while (n-- > 0) { 99 u32 len; 100 ceph_decode_need(p, end, sizeof(u64) + sizeof(u32), 101 bad); 102 *p += sizeof(u64); 103 len = ceph_decode_32(p); 104 ceph_decode_need(p, end, len, bad); 105 *p += len; 106 } 107 } 108 return 0; 109 bad: 110 return -1; 111 } 112 113 /* 114 * Decode an MDS map 115 * 116 * Ignore any fields we don't care about (there are quite a few of 117 * them). 118 */ 119 struct ceph_mdsmap *ceph_mdsmap_decode(struct ceph_mds_client *mdsc, void **p, 120 void *end, bool msgr2) 121 { 122 struct ceph_client *cl = mdsc->fsc->client; 123 struct ceph_mdsmap *m; 124 const void *start = *p; 125 int i, j, n; 126 int err; 127 u8 mdsmap_v; 128 u16 mdsmap_ev; 129 u32 target; 130 size_t export_targets_len; 131 132 m = kzalloc_obj(*m, GFP_NOFS); 133 if (!m) 134 return ERR_PTR(-ENOMEM); 135 136 ceph_decode_need(p, end, 1 + 1, bad); 137 mdsmap_v = ceph_decode_8(p); 138 *p += sizeof(u8); /* mdsmap_cv */ 139 if (mdsmap_v >= 4) { 140 u32 mdsmap_len; 141 ceph_decode_32_safe(p, end, mdsmap_len, bad); 142 if (end < *p + mdsmap_len) 143 goto bad; 144 end = *p + mdsmap_len; 145 } 146 147 ceph_decode_need(p, end, 8*sizeof(u32) + sizeof(u64), bad); 148 m->m_epoch = ceph_decode_32(p); 149 m->m_client_epoch = ceph_decode_32(p); 150 m->m_last_failure = ceph_decode_32(p); 151 m->m_root = ceph_decode_32(p); 152 m->m_session_timeout = ceph_decode_32(p); 153 m->m_session_autoclose = ceph_decode_32(p); 154 m->m_max_file_size = ceph_decode_64(p); 155 m->m_max_mds = ceph_decode_32(p); 156 157 /* 158 * pick out the active nodes as the m_num_active_mds, the 159 * m_num_active_mds maybe larger than m_max_mds when decreasing 160 * the max_mds in cluster side, in other case it should less 161 * than or equal to m_max_mds. 162 */ 163 m->m_num_active_mds = n = ceph_decode_32(p); 164 165 /* 166 * the possible max rank, it maybe larger than the m_num_active_mds, 167 * for example if the mds_max == 2 in the cluster, when the MDS(0) 168 * was laggy and being replaced by a new MDS, we will temporarily 169 * receive a new mds map with n_num_mds == 1 and the active MDS(1), 170 * and the mds rank >= m_num_active_mds. 171 */ 172 m->possible_max_rank = max(m->m_num_active_mds, m->m_max_mds); 173 174 m->m_info = kzalloc_objs(*m->m_info, m->possible_max_rank, GFP_NOFS); 175 if (!m->m_info) 176 goto nomem; 177 178 /* pick out active nodes from mds_info (state > 0) */ 179 for (i = 0; i < n; i++) { 180 u64 global_id; 181 u32 namelen; 182 s32 mds, inc, state; 183 u8 info_v; 184 void *info_end = NULL; 185 struct ceph_entity_addr addr; 186 u32 num_export_targets; 187 void *pexport_targets = NULL; 188 struct ceph_timespec laggy_since; 189 struct ceph_mds_info *info; 190 bool laggy; 191 192 ceph_decode_need(p, end, sizeof(u64) + 1, bad); 193 global_id = ceph_decode_64(p); 194 info_v= ceph_decode_8(p); 195 if (info_v >= 4) { 196 u32 info_len; 197 ceph_decode_need(p, end, 1 + sizeof(u32), bad); 198 *p += sizeof(u8); /* info_cv */ 199 info_len = ceph_decode_32(p); 200 info_end = *p + info_len; 201 if (info_end > end) 202 goto bad; 203 } 204 205 ceph_decode_need(p, end, sizeof(u64) + sizeof(u32), bad); 206 *p += sizeof(u64); 207 namelen = ceph_decode_32(p); /* skip mds name */ 208 *p += namelen; 209 210 ceph_decode_32_safe(p, end, mds, bad); 211 ceph_decode_32_safe(p, end, inc, bad); 212 ceph_decode_32_safe(p, end, state, bad); 213 *p += sizeof(u64); /* state_seq */ 214 if (info_v >= 8) 215 err = ceph_decode_entity_addrvec(p, end, msgr2, &addr); 216 else 217 err = ceph_decode_entity_addr(p, end, &addr); 218 if (err) 219 goto corrupt; 220 221 ceph_decode_copy_safe(p, end, &laggy_since, sizeof(laggy_since), 222 bad); 223 laggy = laggy_since.tv_sec != 0 || laggy_since.tv_nsec != 0; 224 *p += sizeof(u32); 225 ceph_decode_32_safe(p, end, namelen, bad); 226 *p += namelen; 227 if (info_v >= 2) { 228 ceph_decode_32_safe(p, end, num_export_targets, bad); 229 export_targets_len = size_mul(num_export_targets, 230 sizeof(u32)); 231 ceph_decode_need(p, end, export_targets_len, bad); 232 pexport_targets = *p; 233 *p += export_targets_len; 234 } else { 235 num_export_targets = 0; 236 } 237 238 if (info_end && *p != info_end) { 239 if (*p > info_end) 240 goto bad; 241 *p = info_end; 242 } 243 244 doutc(cl, "%d/%d %lld mds%d.%d %s %s%s\n", i+1, n, global_id, 245 mds, inc, ceph_pr_addr(&addr), 246 ceph_mds_state_name(state), laggy ? "(laggy)" : ""); 247 248 if (mds < 0 || mds >= m->possible_max_rank) { 249 pr_warn_client(cl, "got incorrect mds(%d)\n", mds); 250 continue; 251 } 252 253 if (state <= 0) { 254 doutc(cl, "got incorrect state(%s)\n", 255 ceph_mds_state_name(state)); 256 continue; 257 } 258 259 info = &m->m_info[mds]; 260 info->global_id = global_id; 261 info->state = state; 262 info->addr = addr; 263 info->laggy = laggy; 264 info->num_export_targets = num_export_targets; 265 if (num_export_targets) { 266 info->export_targets = kcalloc(num_export_targets, 267 sizeof(u32), GFP_NOFS); 268 if (!info->export_targets) 269 goto nomem; 270 for (j = 0; j < num_export_targets; j++) { 271 target = ceph_decode_32(&pexport_targets); 272 if (target >= CEPH_MAX_MDS) { 273 err = -EIO; 274 goto corrupt; 275 } 276 info->export_targets[j] = target; 277 } 278 } else { 279 info->export_targets = NULL; 280 } 281 } 282 283 /* pg_pools */ 284 ceph_decode_32_safe(p, end, n, bad); 285 m->m_num_data_pg_pools = n; 286 m->m_data_pg_pools = kcalloc(n, sizeof(u64), GFP_NOFS); 287 if (!m->m_data_pg_pools) 288 goto nomem; 289 ceph_decode_need(p, end, sizeof(u64)*(n+1), bad); 290 for (i = 0; i < n; i++) 291 m->m_data_pg_pools[i] = ceph_decode_64(p); 292 m->m_cas_pg_pool = ceph_decode_64(p); 293 m->m_enabled = m->m_epoch > 1; 294 295 mdsmap_ev = 1; 296 if (mdsmap_v >= 2) { 297 ceph_decode_16_safe(p, end, mdsmap_ev, bad_ext); 298 } 299 if (mdsmap_ev >= 3) { 300 if (__decode_and_drop_compat_set(p, end) < 0) 301 goto bad_ext; 302 } 303 /* metadata_pool */ 304 if (mdsmap_ev < 5) { 305 __decode_and_drop_type(p, end, u32, bad_ext); 306 } else { 307 __decode_and_drop_type(p, end, u64, bad_ext); 308 } 309 310 /* created + modified + tableserver */ 311 __decode_and_drop_type(p, end, struct ceph_timespec, bad_ext); 312 __decode_and_drop_type(p, end, struct ceph_timespec, bad_ext); 313 __decode_and_drop_type(p, end, u32, bad_ext); 314 315 /* in */ 316 { 317 int num_laggy = 0; 318 ceph_decode_32_safe(p, end, n, bad_ext); 319 ceph_decode_need(p, end, sizeof(u32) * n, bad_ext); 320 321 for (i = 0; i < n; i++) { 322 s32 mds = ceph_decode_32(p); 323 if (mds >= 0 && mds < m->possible_max_rank) { 324 if (m->m_info[mds].laggy) 325 num_laggy++; 326 } 327 } 328 m->m_num_laggy = num_laggy; 329 330 if (n > m->possible_max_rank) { 331 void *new_m_info = krealloc(m->m_info, 332 n * sizeof(*m->m_info), 333 GFP_NOFS | __GFP_ZERO); 334 if (!new_m_info) 335 goto nomem; 336 m->m_info = new_m_info; 337 } 338 m->possible_max_rank = n; 339 } 340 341 /* inc */ 342 __decode_and_drop_map(p, end, u32, u32, bad_ext); 343 /* up */ 344 __decode_and_drop_map(p, end, u32, u64, bad_ext); 345 /* failed */ 346 __decode_and_drop_set(p, end, u32, bad_ext); 347 /* stopped */ 348 __decode_and_drop_set(p, end, u32, bad_ext); 349 350 if (mdsmap_ev >= 4) { 351 /* last_failure_osd_epoch */ 352 __decode_and_drop_type(p, end, u32, bad_ext); 353 } 354 if (mdsmap_ev >= 6) { 355 /* ever_allowed_snaps */ 356 __decode_and_drop_type(p, end, u8, bad_ext); 357 /* explicitly_allowed_snaps */ 358 __decode_and_drop_type(p, end, u8, bad_ext); 359 } 360 if (mdsmap_ev >= 7) { 361 /* inline_data_enabled */ 362 __decode_and_drop_type(p, end, u8, bad_ext); 363 } 364 if (mdsmap_ev >= 8) { 365 size_t fsname_len; 366 367 /* enabled */ 368 ceph_decode_8_safe(p, end, m->m_enabled, bad_ext); 369 370 /* fs_name */ 371 m->m_fs_name = ceph_extract_encoded_string(p, end, 372 &fsname_len, 373 GFP_NOFS); 374 if (IS_ERR(m->m_fs_name)) { 375 m->m_fs_name = NULL; 376 goto nomem; 377 } 378 379 /* validate fsname against mds_namespace */ 380 if (!namespace_equals(mdsc->fsc->mount_options, m->m_fs_name, 381 fsname_len)) { 382 pr_warn_client(cl, "fsname %s doesn't match mds_namespace %s\n", 383 m->m_fs_name, 384 mdsc->fsc->mount_options->mds_namespace); 385 goto bad; 386 } 387 } else { 388 m->m_enabled = false; 389 m->m_fs_name = kstrdup(CEPH_OLD_FS_NAME, GFP_NOFS); 390 if (!m->m_fs_name) 391 goto nomem; 392 } 393 /* damaged */ 394 if (mdsmap_ev >= 9) { 395 size_t need; 396 ceph_decode_32_safe(p, end, n, bad_ext); 397 need = sizeof(u32) * n; 398 ceph_decode_need(p, end, need, bad_ext); 399 *p += need; 400 m->m_damaged = n > 0; 401 } else { 402 m->m_damaged = false; 403 } 404 if (mdsmap_ev >= 17) { 405 /* balancer */ 406 ceph_decode_skip_string(p, end, bad_ext); 407 /* standby_count_wanted */ 408 ceph_decode_skip_32(p, end, bad_ext); 409 /* old_max_mds */ 410 ceph_decode_skip_32(p, end, bad_ext); 411 /* min_compat_client */ 412 ceph_decode_skip_8(p, end, bad_ext); 413 /* required_client_features */ 414 ceph_decode_skip_set(p, end, 64, bad_ext); 415 /* bal_rank_mask */ 416 ceph_decode_skip_string(p, end, bad_ext); 417 } 418 if (mdsmap_ev >= 18) { 419 ceph_decode_64_safe(p, end, m->m_max_xattr_size, bad_ext); 420 } 421 bad_ext: 422 doutc(cl, "m_enabled: %d, m_damaged: %d, m_num_laggy: %d\n", 423 !!m->m_enabled, !!m->m_damaged, m->m_num_laggy); 424 *p = end; 425 doutc(cl, "success epoch %u\n", m->m_epoch); 426 return m; 427 nomem: 428 err = -ENOMEM; 429 goto out_err; 430 corrupt: 431 pr_err_client(cl, "corrupt mdsmap\n"); 432 print_hex_dump(KERN_DEBUG, "mdsmap: ", 433 DUMP_PREFIX_OFFSET, 16, 1, 434 start, end - start, true); 435 out_err: 436 ceph_mdsmap_destroy(m); 437 return ERR_PTR(err); 438 bad: 439 err = -EINVAL; 440 goto corrupt; 441 } 442 443 void ceph_mdsmap_destroy(struct ceph_mdsmap *m) 444 { 445 int i; 446 447 if (m->m_info) { 448 for (i = 0; i < m->possible_max_rank; i++) 449 kfree(m->m_info[i].export_targets); 450 kfree(m->m_info); 451 } 452 kfree(m->m_data_pg_pools); 453 kfree(m->m_fs_name); 454 kfree(m); 455 } 456 457 bool ceph_mdsmap_is_cluster_available(struct ceph_mdsmap *m) 458 { 459 int i, nr_active = 0; 460 if (!m->m_enabled) 461 return false; 462 if (m->m_damaged) 463 return false; 464 if (m->m_num_laggy == m->m_num_active_mds) 465 return false; 466 for (i = 0; i < m->possible_max_rank; i++) { 467 if (m->m_info[i].state == CEPH_MDS_STATE_ACTIVE) 468 nr_active++; 469 } 470 return nr_active > 0; 471 } 472