1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/ceph/ceph_debug.h>
3
4 #include <linux/bug.h>
5 #include <linux/err.h>
6 #include <linux/overflow.h>
7 #include <linux/random.h>
8 #include <linux/slab.h>
9 #include <linux/types.h>
10
11 #include <linux/ceph/messenger.h>
12 #include <linux/ceph/decode.h>
13
14 #include "mdsmap.h"
15 #include "mds_client.h"
16 #include "super.h"
17
18 #define CEPH_MDS_IS_READY(i, ignore_laggy) \
19 (m->m_info[i].state > 0 && (ignore_laggy ? true : !m->m_info[i].laggy))
20
__mdsmap_get_random_mds(struct ceph_mdsmap * m,bool ignore_laggy)21 static int __mdsmap_get_random_mds(struct ceph_mdsmap *m, bool ignore_laggy)
22 {
23 int n = 0;
24 int i, j;
25
26 /* count */
27 for (i = 0; i < m->possible_max_rank; i++)
28 if (CEPH_MDS_IS_READY(i, ignore_laggy))
29 n++;
30 if (n == 0)
31 return -1;
32
33 /* pick */
34 n = get_random_u32_below(n);
35 for (j = 0, i = 0; i < m->possible_max_rank; i++) {
36 if (CEPH_MDS_IS_READY(i, ignore_laggy))
37 j++;
38 if (j > n)
39 break;
40 }
41
42 return i;
43 }
44
45 /*
46 * choose a random mds that is "up" (i.e. has a state > 0), or -1.
47 */
ceph_mdsmap_get_random_mds(struct ceph_mdsmap * m)48 int ceph_mdsmap_get_random_mds(struct ceph_mdsmap *m)
49 {
50 int mds;
51
52 mds = __mdsmap_get_random_mds(m, false);
53 if (mds == m->possible_max_rank || mds == -1)
54 mds = __mdsmap_get_random_mds(m, true);
55
56 return mds == m->possible_max_rank ? -1 : mds;
57 }
58
59 #define __decode_and_drop_type(p, end, type, bad) \
60 do { \
61 if (*p + sizeof(type) > end) \
62 goto bad; \
63 *p += sizeof(type); \
64 } while (0)
65
66 #define __decode_and_drop_set(p, end, type, bad) \
67 do { \
68 u32 n; \
69 size_t need; \
70 ceph_decode_32_safe(p, end, n, bad); \
71 need = sizeof(type) * n; \
72 ceph_decode_need(p, end, need, bad); \
73 *p += need; \
74 } while (0)
75
76 #define __decode_and_drop_map(p, end, ktype, vtype, bad) \
77 do { \
78 u32 n; \
79 size_t need; \
80 ceph_decode_32_safe(p, end, n, bad); \
81 need = (sizeof(ktype) + sizeof(vtype)) * n; \
82 ceph_decode_need(p, end, need, bad); \
83 *p += need; \
84 } while (0)
85
86
__decode_and_drop_compat_set(void ** p,void * end)87 static int __decode_and_drop_compat_set(void **p, void* end)
88 {
89 int i;
90 /* compat, ro_compat, incompat*/
91 for (i = 0; i < 3; i++) {
92 u32 n;
93 ceph_decode_need(p, end, sizeof(u64) + sizeof(u32), bad);
94 /* mask */
95 *p += sizeof(u64);
96 /* names (map<u64, string>) */
97 n = ceph_decode_32(p);
98 while (n-- > 0) {
99 u32 len;
100 ceph_decode_need(p, end, sizeof(u64) + sizeof(u32),
101 bad);
102 *p += sizeof(u64);
103 len = ceph_decode_32(p);
104 ceph_decode_need(p, end, len, bad);
105 *p += len;
106 }
107 }
108 return 0;
109 bad:
110 return -1;
111 }
112
113 /*
114 * Decode an MDS map
115 *
116 * Ignore any fields we don't care about (there are quite a few of
117 * them).
118 */
ceph_mdsmap_decode(struct ceph_mds_client * mdsc,void ** p,void * end,bool msgr2)119 struct ceph_mdsmap *ceph_mdsmap_decode(struct ceph_mds_client *mdsc, void **p,
120 void *end, bool msgr2)
121 {
122 struct ceph_client *cl = mdsc->fsc->client;
123 struct ceph_mdsmap *m;
124 const void *start = *p;
125 int i, j, n;
126 int err;
127 u8 mdsmap_v;
128 u16 mdsmap_ev;
129 u32 target;
130 size_t export_targets_len;
131
132 m = kzalloc_obj(*m, GFP_NOFS);
133 if (!m)
134 return ERR_PTR(-ENOMEM);
135
136 ceph_decode_need(p, end, 1 + 1, bad);
137 mdsmap_v = ceph_decode_8(p);
138 *p += sizeof(u8); /* mdsmap_cv */
139 if (mdsmap_v >= 4) {
140 u32 mdsmap_len;
141 ceph_decode_32_safe(p, end, mdsmap_len, bad);
142 if (end < *p + mdsmap_len)
143 goto bad;
144 end = *p + mdsmap_len;
145 }
146
147 ceph_decode_need(p, end, 8*sizeof(u32) + sizeof(u64), bad);
148 m->m_epoch = ceph_decode_32(p);
149 m->m_client_epoch = ceph_decode_32(p);
150 m->m_last_failure = ceph_decode_32(p);
151 m->m_root = ceph_decode_32(p);
152 m->m_session_timeout = ceph_decode_32(p);
153 m->m_session_autoclose = ceph_decode_32(p);
154 m->m_max_file_size = ceph_decode_64(p);
155 m->m_max_mds = ceph_decode_32(p);
156
157 /*
158 * pick out the active nodes as the m_num_active_mds, the
159 * m_num_active_mds maybe larger than m_max_mds when decreasing
160 * the max_mds in cluster side, in other case it should less
161 * than or equal to m_max_mds.
162 */
163 m->m_num_active_mds = n = ceph_decode_32(p);
164
165 /*
166 * the possible max rank, it maybe larger than the m_num_active_mds,
167 * for example if the mds_max == 2 in the cluster, when the MDS(0)
168 * was laggy and being replaced by a new MDS, we will temporarily
169 * receive a new mds map with n_num_mds == 1 and the active MDS(1),
170 * and the mds rank >= m_num_active_mds.
171 */
172 m->possible_max_rank = max(m->m_num_active_mds, m->m_max_mds);
173
174 m->m_info = kzalloc_objs(*m->m_info, m->possible_max_rank, GFP_NOFS);
175 if (!m->m_info)
176 goto nomem;
177
178 /* pick out active nodes from mds_info (state > 0) */
179 for (i = 0; i < n; i++) {
180 u64 global_id;
181 u32 namelen;
182 s32 mds, inc, state;
183 u8 info_v;
184 void *info_end = NULL;
185 struct ceph_entity_addr addr;
186 u32 num_export_targets;
187 void *pexport_targets = NULL;
188 struct ceph_timespec laggy_since;
189 struct ceph_mds_info *info;
190 bool laggy;
191
192 ceph_decode_need(p, end, sizeof(u64) + 1, bad);
193 global_id = ceph_decode_64(p);
194 info_v= ceph_decode_8(p);
195 if (info_v >= 4) {
196 u32 info_len;
197 ceph_decode_need(p, end, 1 + sizeof(u32), bad);
198 *p += sizeof(u8); /* info_cv */
199 info_len = ceph_decode_32(p);
200 info_end = *p + info_len;
201 if (info_end > end)
202 goto bad;
203 }
204
205 ceph_decode_need(p, end, sizeof(u64) + sizeof(u32), bad);
206 *p += sizeof(u64);
207 namelen = ceph_decode_32(p); /* skip mds name */
208 *p += namelen;
209
210 ceph_decode_32_safe(p, end, mds, bad);
211 ceph_decode_32_safe(p, end, inc, bad);
212 ceph_decode_32_safe(p, end, state, bad);
213 *p += sizeof(u64); /* state_seq */
214 if (info_v >= 8)
215 err = ceph_decode_entity_addrvec(p, end, msgr2, &addr);
216 else
217 err = ceph_decode_entity_addr(p, end, &addr);
218 if (err)
219 goto corrupt;
220
221 ceph_decode_copy_safe(p, end, &laggy_since, sizeof(laggy_since),
222 bad);
223 laggy = laggy_since.tv_sec != 0 || laggy_since.tv_nsec != 0;
224 *p += sizeof(u32);
225 ceph_decode_32_safe(p, end, namelen, bad);
226 *p += namelen;
227 if (info_v >= 2) {
228 ceph_decode_32_safe(p, end, num_export_targets, bad);
229 export_targets_len = size_mul(num_export_targets,
230 sizeof(u32));
231 ceph_decode_need(p, end, export_targets_len, bad);
232 pexport_targets = *p;
233 *p += export_targets_len;
234 } else {
235 num_export_targets = 0;
236 }
237
238 if (info_end && *p != info_end) {
239 if (*p > info_end)
240 goto bad;
241 *p = info_end;
242 }
243
244 doutc(cl, "%d/%d %lld mds%d.%d %s %s%s\n", i+1, n, global_id,
245 mds, inc, ceph_pr_addr(&addr),
246 ceph_mds_state_name(state), laggy ? "(laggy)" : "");
247
248 if (mds < 0 || mds >= m->possible_max_rank) {
249 pr_warn_client(cl, "got incorrect mds(%d)\n", mds);
250 continue;
251 }
252
253 if (state <= 0) {
254 doutc(cl, "got incorrect state(%s)\n",
255 ceph_mds_state_name(state));
256 continue;
257 }
258
259 info = &m->m_info[mds];
260 info->global_id = global_id;
261 info->state = state;
262 info->addr = addr;
263 info->laggy = laggy;
264 info->num_export_targets = num_export_targets;
265 if (num_export_targets) {
266 info->export_targets = kcalloc(num_export_targets,
267 sizeof(u32), GFP_NOFS);
268 if (!info->export_targets)
269 goto nomem;
270 for (j = 0; j < num_export_targets; j++) {
271 target = ceph_decode_32(&pexport_targets);
272 if (target >= CEPH_MAX_MDS) {
273 err = -EIO;
274 goto corrupt;
275 }
276 info->export_targets[j] = target;
277 }
278 } else {
279 info->export_targets = NULL;
280 }
281 }
282
283 /* pg_pools */
284 ceph_decode_32_safe(p, end, n, bad);
285 m->m_num_data_pg_pools = n;
286 m->m_data_pg_pools = kcalloc(n, sizeof(u64), GFP_NOFS);
287 if (!m->m_data_pg_pools)
288 goto nomem;
289 ceph_decode_need(p, end, sizeof(u64)*(n+1), bad);
290 for (i = 0; i < n; i++)
291 m->m_data_pg_pools[i] = ceph_decode_64(p);
292 m->m_cas_pg_pool = ceph_decode_64(p);
293 m->m_enabled = m->m_epoch > 1;
294
295 mdsmap_ev = 1;
296 if (mdsmap_v >= 2) {
297 ceph_decode_16_safe(p, end, mdsmap_ev, bad_ext);
298 }
299 if (mdsmap_ev >= 3) {
300 if (__decode_and_drop_compat_set(p, end) < 0)
301 goto bad_ext;
302 }
303 /* metadata_pool */
304 if (mdsmap_ev < 5) {
305 __decode_and_drop_type(p, end, u32, bad_ext);
306 } else {
307 __decode_and_drop_type(p, end, u64, bad_ext);
308 }
309
310 /* created + modified + tableserver */
311 __decode_and_drop_type(p, end, struct ceph_timespec, bad_ext);
312 __decode_and_drop_type(p, end, struct ceph_timespec, bad_ext);
313 __decode_and_drop_type(p, end, u32, bad_ext);
314
315 /* in */
316 {
317 int num_laggy = 0;
318 ceph_decode_32_safe(p, end, n, bad_ext);
319 ceph_decode_need(p, end, sizeof(u32) * n, bad_ext);
320
321 for (i = 0; i < n; i++) {
322 s32 mds = ceph_decode_32(p);
323 if (mds >= 0 && mds < m->possible_max_rank) {
324 if (m->m_info[mds].laggy)
325 num_laggy++;
326 }
327 }
328 m->m_num_laggy = num_laggy;
329
330 if (n > m->possible_max_rank) {
331 void *new_m_info = krealloc(m->m_info,
332 n * sizeof(*m->m_info),
333 GFP_NOFS | __GFP_ZERO);
334 if (!new_m_info)
335 goto nomem;
336 m->m_info = new_m_info;
337 }
338 m->possible_max_rank = n;
339 }
340
341 /* inc */
342 __decode_and_drop_map(p, end, u32, u32, bad_ext);
343 /* up */
344 __decode_and_drop_map(p, end, u32, u64, bad_ext);
345 /* failed */
346 __decode_and_drop_set(p, end, u32, bad_ext);
347 /* stopped */
348 __decode_and_drop_set(p, end, u32, bad_ext);
349
350 if (mdsmap_ev >= 4) {
351 /* last_failure_osd_epoch */
352 __decode_and_drop_type(p, end, u32, bad_ext);
353 }
354 if (mdsmap_ev >= 6) {
355 /* ever_allowed_snaps */
356 __decode_and_drop_type(p, end, u8, bad_ext);
357 /* explicitly_allowed_snaps */
358 __decode_and_drop_type(p, end, u8, bad_ext);
359 }
360 if (mdsmap_ev >= 7) {
361 /* inline_data_enabled */
362 __decode_and_drop_type(p, end, u8, bad_ext);
363 }
364 if (mdsmap_ev >= 8) {
365 size_t fsname_len;
366
367 /* enabled */
368 ceph_decode_8_safe(p, end, m->m_enabled, bad_ext);
369
370 /* fs_name */
371 m->m_fs_name = ceph_extract_encoded_string(p, end,
372 &fsname_len,
373 GFP_NOFS);
374 if (IS_ERR(m->m_fs_name)) {
375 m->m_fs_name = NULL;
376 goto nomem;
377 }
378
379 /* validate fsname against mds_namespace */
380 if (!namespace_equals(mdsc->fsc->mount_options, m->m_fs_name,
381 fsname_len)) {
382 pr_warn_client(cl, "fsname %s doesn't match mds_namespace %s\n",
383 m->m_fs_name,
384 mdsc->fsc->mount_options->mds_namespace);
385 goto bad;
386 }
387 } else {
388 m->m_enabled = false;
389 m->m_fs_name = kstrdup(CEPH_OLD_FS_NAME, GFP_NOFS);
390 if (!m->m_fs_name)
391 goto nomem;
392 }
393 /* damaged */
394 if (mdsmap_ev >= 9) {
395 size_t need;
396 ceph_decode_32_safe(p, end, n, bad_ext);
397 need = sizeof(u32) * n;
398 ceph_decode_need(p, end, need, bad_ext);
399 *p += need;
400 m->m_damaged = n > 0;
401 } else {
402 m->m_damaged = false;
403 }
404 if (mdsmap_ev >= 17) {
405 /* balancer */
406 ceph_decode_skip_string(p, end, bad_ext);
407 /* standby_count_wanted */
408 ceph_decode_skip_32(p, end, bad_ext);
409 /* old_max_mds */
410 ceph_decode_skip_32(p, end, bad_ext);
411 /* min_compat_client */
412 ceph_decode_skip_8(p, end, bad_ext);
413 /* required_client_features */
414 ceph_decode_skip_set(p, end, 64, bad_ext);
415 /* bal_rank_mask */
416 ceph_decode_skip_string(p, end, bad_ext);
417 }
418 if (mdsmap_ev >= 18) {
419 ceph_decode_64_safe(p, end, m->m_max_xattr_size, bad_ext);
420 }
421 bad_ext:
422 doutc(cl, "m_enabled: %d, m_damaged: %d, m_num_laggy: %d\n",
423 !!m->m_enabled, !!m->m_damaged, m->m_num_laggy);
424 *p = end;
425 doutc(cl, "success epoch %u\n", m->m_epoch);
426 return m;
427 nomem:
428 err = -ENOMEM;
429 goto out_err;
430 corrupt:
431 pr_err_client(cl, "corrupt mdsmap\n");
432 print_hex_dump(KERN_DEBUG, "mdsmap: ",
433 DUMP_PREFIX_OFFSET, 16, 1,
434 start, end - start, true);
435 out_err:
436 ceph_mdsmap_destroy(m);
437 return ERR_PTR(err);
438 bad:
439 err = -EINVAL;
440 goto corrupt;
441 }
442
ceph_mdsmap_destroy(struct ceph_mdsmap * m)443 void ceph_mdsmap_destroy(struct ceph_mdsmap *m)
444 {
445 int i;
446
447 if (m->m_info) {
448 for (i = 0; i < m->possible_max_rank; i++)
449 kfree(m->m_info[i].export_targets);
450 kfree(m->m_info);
451 }
452 kfree(m->m_data_pg_pools);
453 kfree(m->m_fs_name);
454 kfree(m);
455 }
456
ceph_mdsmap_is_cluster_available(struct ceph_mdsmap * m)457 bool ceph_mdsmap_is_cluster_available(struct ceph_mdsmap *m)
458 {
459 int i, nr_active = 0;
460 if (!m->m_enabled)
461 return false;
462 if (m->m_damaged)
463 return false;
464 if (m->m_num_laggy == m->m_num_active_mds)
465 return false;
466 for (i = 0; i < m->possible_max_rank; i++) {
467 if (m->m_info[i].state == CEPH_MDS_STATE_ACTIVE)
468 nr_active++;
469 }
470 return nr_active > 0;
471 }
472