xref: /linux/net/devlink/health.c (revision 79997eda0d31bc68203c95ecb978773ee6ce7a1f)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Copyright (c) 2016 Mellanox Technologies. All rights reserved.
4  * Copyright (c) 2016 Jiri Pirko <jiri@mellanox.com>
5  */
6 
7 #include <net/genetlink.h>
8 #include <net/sock.h>
9 #include <trace/events/devlink.h>
10 #include "devl_internal.h"
11 
12 struct devlink_fmsg_item {
13 	struct list_head list;
14 	int attrtype;
15 	u8 nla_type;
16 	u16 len;
17 	int value[];
18 };
19 
20 struct devlink_fmsg {
21 	struct list_head item_list;
22 	int err; /* first error encountered on some devlink_fmsg_XXX() call */
23 	bool putting_binary; /* This flag forces enclosing of binary data
24 			      * in an array brackets. It forces using
25 			      * of designated API:
26 			      * devlink_fmsg_binary_pair_nest_start()
27 			      * devlink_fmsg_binary_pair_nest_end()
28 			      */
29 };
30 
31 static struct devlink_fmsg *devlink_fmsg_alloc(void)
32 {
33 	struct devlink_fmsg *fmsg;
34 
35 	fmsg = kzalloc(sizeof(*fmsg), GFP_KERNEL);
36 	if (!fmsg)
37 		return NULL;
38 
39 	INIT_LIST_HEAD(&fmsg->item_list);
40 
41 	return fmsg;
42 }
43 
44 static void devlink_fmsg_free(struct devlink_fmsg *fmsg)
45 {
46 	struct devlink_fmsg_item *item, *tmp;
47 
48 	list_for_each_entry_safe(item, tmp, &fmsg->item_list, list) {
49 		list_del(&item->list);
50 		kfree(item);
51 	}
52 	kfree(fmsg);
53 }
54 
55 struct devlink_health_reporter {
56 	struct list_head list;
57 	void *priv;
58 	const struct devlink_health_reporter_ops *ops;
59 	struct devlink *devlink;
60 	struct devlink_port *devlink_port;
61 	struct devlink_fmsg *dump_fmsg;
62 	u64 graceful_period;
63 	bool auto_recover;
64 	bool auto_dump;
65 	u8 health_state;
66 	u64 dump_ts;
67 	u64 dump_real_ts;
68 	u64 error_count;
69 	u64 recovery_count;
70 	u64 last_recovery_ts;
71 };
72 
73 void *
74 devlink_health_reporter_priv(struct devlink_health_reporter *reporter)
75 {
76 	return reporter->priv;
77 }
78 EXPORT_SYMBOL_GPL(devlink_health_reporter_priv);
79 
80 static struct devlink_health_reporter *
81 __devlink_health_reporter_find_by_name(struct list_head *reporter_list,
82 				       const char *reporter_name)
83 {
84 	struct devlink_health_reporter *reporter;
85 
86 	list_for_each_entry(reporter, reporter_list, list)
87 		if (!strcmp(reporter->ops->name, reporter_name))
88 			return reporter;
89 	return NULL;
90 }
91 
92 static struct devlink_health_reporter *
93 devlink_health_reporter_find_by_name(struct devlink *devlink,
94 				     const char *reporter_name)
95 {
96 	return __devlink_health_reporter_find_by_name(&devlink->reporter_list,
97 						      reporter_name);
98 }
99 
100 static struct devlink_health_reporter *
101 devlink_port_health_reporter_find_by_name(struct devlink_port *devlink_port,
102 					  const char *reporter_name)
103 {
104 	return __devlink_health_reporter_find_by_name(&devlink_port->reporter_list,
105 						      reporter_name);
106 }
107 
108 static struct devlink_health_reporter *
109 __devlink_health_reporter_create(struct devlink *devlink,
110 				 const struct devlink_health_reporter_ops *ops,
111 				 u64 graceful_period, void *priv)
112 {
113 	struct devlink_health_reporter *reporter;
114 
115 	if (WARN_ON(graceful_period && !ops->recover))
116 		return ERR_PTR(-EINVAL);
117 
118 	reporter = kzalloc(sizeof(*reporter), GFP_KERNEL);
119 	if (!reporter)
120 		return ERR_PTR(-ENOMEM);
121 
122 	reporter->priv = priv;
123 	reporter->ops = ops;
124 	reporter->devlink = devlink;
125 	reporter->graceful_period = graceful_period;
126 	reporter->auto_recover = !!ops->recover;
127 	reporter->auto_dump = !!ops->dump;
128 	return reporter;
129 }
130 
131 /**
132  * devl_port_health_reporter_create() - create devlink health reporter for
133  *                                      specified port instance
134  *
135  * @port: devlink_port to which health reports will relate
136  * @ops: devlink health reporter ops
137  * @graceful_period: min time (in msec) between recovery attempts
138  * @priv: driver priv pointer
139  */
140 struct devlink_health_reporter *
141 devl_port_health_reporter_create(struct devlink_port *port,
142 				 const struct devlink_health_reporter_ops *ops,
143 				 u64 graceful_period, void *priv)
144 {
145 	struct devlink_health_reporter *reporter;
146 
147 	devl_assert_locked(port->devlink);
148 
149 	if (__devlink_health_reporter_find_by_name(&port->reporter_list,
150 						   ops->name))
151 		return ERR_PTR(-EEXIST);
152 
153 	reporter = __devlink_health_reporter_create(port->devlink, ops,
154 						    graceful_period, priv);
155 	if (IS_ERR(reporter))
156 		return reporter;
157 
158 	reporter->devlink_port = port;
159 	list_add_tail(&reporter->list, &port->reporter_list);
160 	return reporter;
161 }
162 EXPORT_SYMBOL_GPL(devl_port_health_reporter_create);
163 
164 struct devlink_health_reporter *
165 devlink_port_health_reporter_create(struct devlink_port *port,
166 				    const struct devlink_health_reporter_ops *ops,
167 				    u64 graceful_period, void *priv)
168 {
169 	struct devlink_health_reporter *reporter;
170 	struct devlink *devlink = port->devlink;
171 
172 	devl_lock(devlink);
173 	reporter = devl_port_health_reporter_create(port, ops,
174 						    graceful_period, priv);
175 	devl_unlock(devlink);
176 	return reporter;
177 }
178 EXPORT_SYMBOL_GPL(devlink_port_health_reporter_create);
179 
180 /**
181  * devl_health_reporter_create - create devlink health reporter
182  *
183  * @devlink: devlink instance which the health reports will relate
184  * @ops: devlink health reporter ops
185  * @graceful_period: min time (in msec) between recovery attempts
186  * @priv: driver priv pointer
187  */
188 struct devlink_health_reporter *
189 devl_health_reporter_create(struct devlink *devlink,
190 			    const struct devlink_health_reporter_ops *ops,
191 			    u64 graceful_period, void *priv)
192 {
193 	struct devlink_health_reporter *reporter;
194 
195 	devl_assert_locked(devlink);
196 
197 	if (devlink_health_reporter_find_by_name(devlink, ops->name))
198 		return ERR_PTR(-EEXIST);
199 
200 	reporter = __devlink_health_reporter_create(devlink, ops,
201 						    graceful_period, priv);
202 	if (IS_ERR(reporter))
203 		return reporter;
204 
205 	list_add_tail(&reporter->list, &devlink->reporter_list);
206 	return reporter;
207 }
208 EXPORT_SYMBOL_GPL(devl_health_reporter_create);
209 
210 struct devlink_health_reporter *
211 devlink_health_reporter_create(struct devlink *devlink,
212 			       const struct devlink_health_reporter_ops *ops,
213 			       u64 graceful_period, void *priv)
214 {
215 	struct devlink_health_reporter *reporter;
216 
217 	devl_lock(devlink);
218 	reporter = devl_health_reporter_create(devlink, ops,
219 					       graceful_period, priv);
220 	devl_unlock(devlink);
221 	return reporter;
222 }
223 EXPORT_SYMBOL_GPL(devlink_health_reporter_create);
224 
225 static void
226 devlink_health_reporter_free(struct devlink_health_reporter *reporter)
227 {
228 	if (reporter->dump_fmsg)
229 		devlink_fmsg_free(reporter->dump_fmsg);
230 	kfree(reporter);
231 }
232 
233 /**
234  * devl_health_reporter_destroy() - destroy devlink health reporter
235  *
236  * @reporter: devlink health reporter to destroy
237  */
238 void
239 devl_health_reporter_destroy(struct devlink_health_reporter *reporter)
240 {
241 	devl_assert_locked(reporter->devlink);
242 
243 	list_del(&reporter->list);
244 	devlink_health_reporter_free(reporter);
245 }
246 EXPORT_SYMBOL_GPL(devl_health_reporter_destroy);
247 
248 void
249 devlink_health_reporter_destroy(struct devlink_health_reporter *reporter)
250 {
251 	struct devlink *devlink = reporter->devlink;
252 
253 	devl_lock(devlink);
254 	devl_health_reporter_destroy(reporter);
255 	devl_unlock(devlink);
256 }
257 EXPORT_SYMBOL_GPL(devlink_health_reporter_destroy);
258 
259 static int
260 devlink_nl_health_reporter_fill(struct sk_buff *msg,
261 				struct devlink_health_reporter *reporter,
262 				enum devlink_command cmd, u32 portid,
263 				u32 seq, int flags)
264 {
265 	struct devlink *devlink = reporter->devlink;
266 	struct nlattr *reporter_attr;
267 	void *hdr;
268 
269 	hdr = genlmsg_put(msg, portid, seq, &devlink_nl_family, flags, cmd);
270 	if (!hdr)
271 		return -EMSGSIZE;
272 
273 	if (devlink_nl_put_handle(msg, devlink))
274 		goto genlmsg_cancel;
275 
276 	if (reporter->devlink_port) {
277 		if (nla_put_u32(msg, DEVLINK_ATTR_PORT_INDEX, reporter->devlink_port->index))
278 			goto genlmsg_cancel;
279 	}
280 	reporter_attr = nla_nest_start_noflag(msg,
281 					      DEVLINK_ATTR_HEALTH_REPORTER);
282 	if (!reporter_attr)
283 		goto genlmsg_cancel;
284 	if (nla_put_string(msg, DEVLINK_ATTR_HEALTH_REPORTER_NAME,
285 			   reporter->ops->name))
286 		goto reporter_nest_cancel;
287 	if (nla_put_u8(msg, DEVLINK_ATTR_HEALTH_REPORTER_STATE,
288 		       reporter->health_state))
289 		goto reporter_nest_cancel;
290 	if (nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_ERR_COUNT,
291 			      reporter->error_count, DEVLINK_ATTR_PAD))
292 		goto reporter_nest_cancel;
293 	if (nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_RECOVER_COUNT,
294 			      reporter->recovery_count, DEVLINK_ATTR_PAD))
295 		goto reporter_nest_cancel;
296 	if (reporter->ops->recover &&
297 	    nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD,
298 			      reporter->graceful_period,
299 			      DEVLINK_ATTR_PAD))
300 		goto reporter_nest_cancel;
301 	if (reporter->ops->recover &&
302 	    nla_put_u8(msg, DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER,
303 		       reporter->auto_recover))
304 		goto reporter_nest_cancel;
305 	if (reporter->dump_fmsg &&
306 	    nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_DUMP_TS,
307 			      jiffies_to_msecs(reporter->dump_ts),
308 			      DEVLINK_ATTR_PAD))
309 		goto reporter_nest_cancel;
310 	if (reporter->dump_fmsg &&
311 	    nla_put_u64_64bit(msg, DEVLINK_ATTR_HEALTH_REPORTER_DUMP_TS_NS,
312 			      reporter->dump_real_ts, DEVLINK_ATTR_PAD))
313 		goto reporter_nest_cancel;
314 	if (reporter->ops->dump &&
315 	    nla_put_u8(msg, DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP,
316 		       reporter->auto_dump))
317 		goto reporter_nest_cancel;
318 
319 	nla_nest_end(msg, reporter_attr);
320 	genlmsg_end(msg, hdr);
321 	return 0;
322 
323 reporter_nest_cancel:
324 	nla_nest_cancel(msg, reporter_attr);
325 genlmsg_cancel:
326 	genlmsg_cancel(msg, hdr);
327 	return -EMSGSIZE;
328 }
329 
330 static struct devlink_health_reporter *
331 devlink_health_reporter_get_from_attrs(struct devlink *devlink,
332 				       struct nlattr **attrs)
333 {
334 	struct devlink_port *devlink_port;
335 	char *reporter_name;
336 
337 	if (!attrs[DEVLINK_ATTR_HEALTH_REPORTER_NAME])
338 		return NULL;
339 
340 	reporter_name = nla_data(attrs[DEVLINK_ATTR_HEALTH_REPORTER_NAME]);
341 	devlink_port = devlink_port_get_from_attrs(devlink, attrs);
342 	if (IS_ERR(devlink_port))
343 		return devlink_health_reporter_find_by_name(devlink,
344 							    reporter_name);
345 	else
346 		return devlink_port_health_reporter_find_by_name(devlink_port,
347 								 reporter_name);
348 }
349 
350 static struct devlink_health_reporter *
351 devlink_health_reporter_get_from_info(struct devlink *devlink,
352 				      struct genl_info *info)
353 {
354 	return devlink_health_reporter_get_from_attrs(devlink, info->attrs);
355 }
356 
357 int devlink_nl_health_reporter_get_doit(struct sk_buff *skb,
358 					struct genl_info *info)
359 {
360 	struct devlink *devlink = info->user_ptr[0];
361 	struct devlink_health_reporter *reporter;
362 	struct sk_buff *msg;
363 	int err;
364 
365 	reporter = devlink_health_reporter_get_from_info(devlink, info);
366 	if (!reporter)
367 		return -EINVAL;
368 
369 	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
370 	if (!msg)
371 		return -ENOMEM;
372 
373 	err = devlink_nl_health_reporter_fill(msg, reporter,
374 					      DEVLINK_CMD_HEALTH_REPORTER_GET,
375 					      info->snd_portid, info->snd_seq,
376 					      0);
377 	if (err) {
378 		nlmsg_free(msg);
379 		return err;
380 	}
381 
382 	return genlmsg_reply(msg, info);
383 }
384 
385 static int devlink_nl_health_reporter_get_dump_one(struct sk_buff *msg,
386 						   struct devlink *devlink,
387 						   struct netlink_callback *cb,
388 						   int flags)
389 {
390 	struct devlink_nl_dump_state *state = devlink_dump_state(cb);
391 	const struct genl_info *info = genl_info_dump(cb);
392 	struct devlink_health_reporter *reporter;
393 	unsigned long port_index_end = ULONG_MAX;
394 	struct nlattr **attrs = info->attrs;
395 	unsigned long port_index_start = 0;
396 	struct devlink_port *port;
397 	unsigned long port_index;
398 	int idx = 0;
399 	int err;
400 
401 	if (attrs && attrs[DEVLINK_ATTR_PORT_INDEX]) {
402 		port_index_start = nla_get_u32(attrs[DEVLINK_ATTR_PORT_INDEX]);
403 		port_index_end = port_index_start;
404 		flags |= NLM_F_DUMP_FILTERED;
405 		goto per_port_dump;
406 	}
407 
408 	list_for_each_entry(reporter, &devlink->reporter_list, list) {
409 		if (idx < state->idx) {
410 			idx++;
411 			continue;
412 		}
413 		err = devlink_nl_health_reporter_fill(msg, reporter,
414 						      DEVLINK_CMD_HEALTH_REPORTER_GET,
415 						      NETLINK_CB(cb->skb).portid,
416 						      cb->nlh->nlmsg_seq,
417 						      flags);
418 		if (err) {
419 			state->idx = idx;
420 			return err;
421 		}
422 		idx++;
423 	}
424 per_port_dump:
425 	xa_for_each_range(&devlink->ports, port_index, port,
426 			  port_index_start, port_index_end) {
427 		list_for_each_entry(reporter, &port->reporter_list, list) {
428 			if (idx < state->idx) {
429 				idx++;
430 				continue;
431 			}
432 			err = devlink_nl_health_reporter_fill(msg, reporter,
433 							      DEVLINK_CMD_HEALTH_REPORTER_GET,
434 							      NETLINK_CB(cb->skb).portid,
435 							      cb->nlh->nlmsg_seq,
436 							      flags);
437 			if (err) {
438 				state->idx = idx;
439 				return err;
440 			}
441 			idx++;
442 		}
443 	}
444 
445 	return 0;
446 }
447 
448 int devlink_nl_health_reporter_get_dumpit(struct sk_buff *skb,
449 					  struct netlink_callback *cb)
450 {
451 	return devlink_nl_dumpit(skb, cb,
452 				 devlink_nl_health_reporter_get_dump_one);
453 }
454 
455 int devlink_nl_health_reporter_set_doit(struct sk_buff *skb,
456 					struct genl_info *info)
457 {
458 	struct devlink *devlink = info->user_ptr[0];
459 	struct devlink_health_reporter *reporter;
460 
461 	reporter = devlink_health_reporter_get_from_info(devlink, info);
462 	if (!reporter)
463 		return -EINVAL;
464 
465 	if (!reporter->ops->recover &&
466 	    (info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD] ||
467 	     info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER]))
468 		return -EOPNOTSUPP;
469 
470 	if (!reporter->ops->dump &&
471 	    info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP])
472 		return -EOPNOTSUPP;
473 
474 	if (info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD])
475 		reporter->graceful_period =
476 			nla_get_u64(info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_GRACEFUL_PERIOD]);
477 
478 	if (info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER])
479 		reporter->auto_recover =
480 			nla_get_u8(info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_RECOVER]);
481 
482 	if (info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP])
483 		reporter->auto_dump =
484 		nla_get_u8(info->attrs[DEVLINK_ATTR_HEALTH_REPORTER_AUTO_DUMP]);
485 
486 	return 0;
487 }
488 
489 static void devlink_recover_notify(struct devlink_health_reporter *reporter,
490 				   enum devlink_command cmd)
491 {
492 	struct devlink *devlink = reporter->devlink;
493 	struct sk_buff *msg;
494 	int err;
495 
496 	WARN_ON(cmd != DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
497 	ASSERT_DEVLINK_REGISTERED(devlink);
498 
499 	msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
500 	if (!msg)
501 		return;
502 
503 	err = devlink_nl_health_reporter_fill(msg, reporter, cmd, 0, 0, 0);
504 	if (err) {
505 		nlmsg_free(msg);
506 		return;
507 	}
508 
509 	genlmsg_multicast_netns(&devlink_nl_family, devlink_net(devlink), msg,
510 				0, DEVLINK_MCGRP_CONFIG, GFP_KERNEL);
511 }
512 
513 void
514 devlink_health_reporter_recovery_done(struct devlink_health_reporter *reporter)
515 {
516 	reporter->recovery_count++;
517 	reporter->last_recovery_ts = jiffies;
518 }
519 EXPORT_SYMBOL_GPL(devlink_health_reporter_recovery_done);
520 
521 static int
522 devlink_health_reporter_recover(struct devlink_health_reporter *reporter,
523 				void *priv_ctx, struct netlink_ext_ack *extack)
524 {
525 	int err;
526 
527 	if (reporter->health_state == DEVLINK_HEALTH_REPORTER_STATE_HEALTHY)
528 		return 0;
529 
530 	if (!reporter->ops->recover)
531 		return -EOPNOTSUPP;
532 
533 	err = reporter->ops->recover(reporter, priv_ctx, extack);
534 	if (err)
535 		return err;
536 
537 	devlink_health_reporter_recovery_done(reporter);
538 	reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_HEALTHY;
539 	devlink_recover_notify(reporter, DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
540 
541 	return 0;
542 }
543 
544 static void
545 devlink_health_dump_clear(struct devlink_health_reporter *reporter)
546 {
547 	if (!reporter->dump_fmsg)
548 		return;
549 	devlink_fmsg_free(reporter->dump_fmsg);
550 	reporter->dump_fmsg = NULL;
551 }
552 
553 static int devlink_health_do_dump(struct devlink_health_reporter *reporter,
554 				  void *priv_ctx,
555 				  struct netlink_ext_ack *extack)
556 {
557 	int err;
558 
559 	if (!reporter->ops->dump)
560 		return 0;
561 
562 	if (reporter->dump_fmsg)
563 		return 0;
564 
565 	reporter->dump_fmsg = devlink_fmsg_alloc();
566 	if (!reporter->dump_fmsg)
567 		return -ENOMEM;
568 
569 	devlink_fmsg_obj_nest_start(reporter->dump_fmsg);
570 
571 	err = reporter->ops->dump(reporter, reporter->dump_fmsg,
572 				  priv_ctx, extack);
573 	if (err)
574 		goto dump_err;
575 
576 	devlink_fmsg_obj_nest_end(reporter->dump_fmsg);
577 	err = reporter->dump_fmsg->err;
578 	if (err)
579 		goto dump_err;
580 
581 	reporter->dump_ts = jiffies;
582 	reporter->dump_real_ts = ktime_get_real_ns();
583 
584 	return 0;
585 
586 dump_err:
587 	devlink_health_dump_clear(reporter);
588 	return err;
589 }
590 
591 int devlink_health_report(struct devlink_health_reporter *reporter,
592 			  const char *msg, void *priv_ctx)
593 {
594 	enum devlink_health_reporter_state prev_health_state;
595 	struct devlink *devlink = reporter->devlink;
596 	unsigned long recover_ts_threshold;
597 	int ret;
598 
599 	/* write a log message of the current error */
600 	WARN_ON(!msg);
601 	trace_devlink_health_report(devlink, reporter->ops->name, msg);
602 	reporter->error_count++;
603 	prev_health_state = reporter->health_state;
604 	reporter->health_state = DEVLINK_HEALTH_REPORTER_STATE_ERROR;
605 	devlink_recover_notify(reporter, DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
606 
607 	/* abort if the previous error wasn't recovered */
608 	recover_ts_threshold = reporter->last_recovery_ts +
609 			       msecs_to_jiffies(reporter->graceful_period);
610 	if (reporter->auto_recover &&
611 	    (prev_health_state != DEVLINK_HEALTH_REPORTER_STATE_HEALTHY ||
612 	     (reporter->last_recovery_ts && reporter->recovery_count &&
613 	      time_is_after_jiffies(recover_ts_threshold)))) {
614 		trace_devlink_health_recover_aborted(devlink,
615 						     reporter->ops->name,
616 						     reporter->health_state,
617 						     jiffies -
618 						     reporter->last_recovery_ts);
619 		return -ECANCELED;
620 	}
621 
622 	if (reporter->auto_dump) {
623 		devl_lock(devlink);
624 		/* store current dump of current error, for later analysis */
625 		devlink_health_do_dump(reporter, priv_ctx, NULL);
626 		devl_unlock(devlink);
627 	}
628 
629 	if (!reporter->auto_recover)
630 		return 0;
631 
632 	devl_lock(devlink);
633 	ret = devlink_health_reporter_recover(reporter, priv_ctx, NULL);
634 	devl_unlock(devlink);
635 
636 	return ret;
637 }
638 EXPORT_SYMBOL_GPL(devlink_health_report);
639 
640 void
641 devlink_health_reporter_state_update(struct devlink_health_reporter *reporter,
642 				     enum devlink_health_reporter_state state)
643 {
644 	if (WARN_ON(state != DEVLINK_HEALTH_REPORTER_STATE_HEALTHY &&
645 		    state != DEVLINK_HEALTH_REPORTER_STATE_ERROR))
646 		return;
647 
648 	if (reporter->health_state == state)
649 		return;
650 
651 	reporter->health_state = state;
652 	trace_devlink_health_reporter_state_update(reporter->devlink,
653 						   reporter->ops->name, state);
654 	devlink_recover_notify(reporter, DEVLINK_CMD_HEALTH_REPORTER_RECOVER);
655 }
656 EXPORT_SYMBOL_GPL(devlink_health_reporter_state_update);
657 
658 int devlink_nl_health_reporter_recover_doit(struct sk_buff *skb,
659 					    struct genl_info *info)
660 {
661 	struct devlink *devlink = info->user_ptr[0];
662 	struct devlink_health_reporter *reporter;
663 
664 	reporter = devlink_health_reporter_get_from_info(devlink, info);
665 	if (!reporter)
666 		return -EINVAL;
667 
668 	return devlink_health_reporter_recover(reporter, NULL, info->extack);
669 }
670 
671 static void devlink_fmsg_err_if_binary(struct devlink_fmsg *fmsg)
672 {
673 	if (!fmsg->err && fmsg->putting_binary)
674 		fmsg->err = -EINVAL;
675 }
676 
677 static void devlink_fmsg_nest_common(struct devlink_fmsg *fmsg, int attrtype)
678 {
679 	struct devlink_fmsg_item *item;
680 
681 	if (fmsg->err)
682 		return;
683 
684 	item = kzalloc(sizeof(*item), GFP_KERNEL);
685 	if (!item) {
686 		fmsg->err = -ENOMEM;
687 		return;
688 	}
689 
690 	item->attrtype = attrtype;
691 	list_add_tail(&item->list, &fmsg->item_list);
692 }
693 
694 void devlink_fmsg_obj_nest_start(struct devlink_fmsg *fmsg)
695 {
696 	devlink_fmsg_err_if_binary(fmsg);
697 	devlink_fmsg_nest_common(fmsg, DEVLINK_ATTR_FMSG_OBJ_NEST_START);
698 }
699 EXPORT_SYMBOL_GPL(devlink_fmsg_obj_nest_start);
700 
701 static void devlink_fmsg_nest_end(struct devlink_fmsg *fmsg)
702 {
703 	devlink_fmsg_err_if_binary(fmsg);
704 	devlink_fmsg_nest_common(fmsg, DEVLINK_ATTR_FMSG_NEST_END);
705 }
706 
707 void devlink_fmsg_obj_nest_end(struct devlink_fmsg *fmsg)
708 {
709 	devlink_fmsg_nest_end(fmsg);
710 }
711 EXPORT_SYMBOL_GPL(devlink_fmsg_obj_nest_end);
712 
713 #define DEVLINK_FMSG_MAX_SIZE (GENLMSG_DEFAULT_SIZE - GENL_HDRLEN - NLA_HDRLEN)
714 
715 static void devlink_fmsg_put_name(struct devlink_fmsg *fmsg, const char *name)
716 {
717 	struct devlink_fmsg_item *item;
718 
719 	devlink_fmsg_err_if_binary(fmsg);
720 	if (fmsg->err)
721 		return;
722 
723 	if (strlen(name) + 1 > DEVLINK_FMSG_MAX_SIZE) {
724 		fmsg->err = -EMSGSIZE;
725 		return;
726 	}
727 
728 	item = kzalloc(sizeof(*item) + strlen(name) + 1, GFP_KERNEL);
729 	if (!item) {
730 		fmsg->err = -ENOMEM;
731 		return;
732 	}
733 
734 	item->nla_type = NLA_NUL_STRING;
735 	item->len = strlen(name) + 1;
736 	item->attrtype = DEVLINK_ATTR_FMSG_OBJ_NAME;
737 	memcpy(&item->value, name, item->len);
738 	list_add_tail(&item->list, &fmsg->item_list);
739 }
740 
741 void devlink_fmsg_pair_nest_start(struct devlink_fmsg *fmsg, const char *name)
742 {
743 	devlink_fmsg_err_if_binary(fmsg);
744 	devlink_fmsg_nest_common(fmsg, DEVLINK_ATTR_FMSG_PAIR_NEST_START);
745 	devlink_fmsg_put_name(fmsg, name);
746 }
747 EXPORT_SYMBOL_GPL(devlink_fmsg_pair_nest_start);
748 
749 void devlink_fmsg_pair_nest_end(struct devlink_fmsg *fmsg)
750 {
751 	devlink_fmsg_nest_end(fmsg);
752 }
753 EXPORT_SYMBOL_GPL(devlink_fmsg_pair_nest_end);
754 
755 void devlink_fmsg_arr_pair_nest_start(struct devlink_fmsg *fmsg,
756 				      const char *name)
757 {
758 	devlink_fmsg_pair_nest_start(fmsg, name);
759 	devlink_fmsg_nest_common(fmsg, DEVLINK_ATTR_FMSG_ARR_NEST_START);
760 }
761 EXPORT_SYMBOL_GPL(devlink_fmsg_arr_pair_nest_start);
762 
763 void devlink_fmsg_arr_pair_nest_end(struct devlink_fmsg *fmsg)
764 {
765 	devlink_fmsg_nest_end(fmsg);
766 	devlink_fmsg_nest_end(fmsg);
767 }
768 EXPORT_SYMBOL_GPL(devlink_fmsg_arr_pair_nest_end);
769 
770 void devlink_fmsg_binary_pair_nest_start(struct devlink_fmsg *fmsg,
771 					 const char *name)
772 {
773 	devlink_fmsg_arr_pair_nest_start(fmsg, name);
774 	fmsg->putting_binary = true;
775 }
776 EXPORT_SYMBOL_GPL(devlink_fmsg_binary_pair_nest_start);
777 
778 void devlink_fmsg_binary_pair_nest_end(struct devlink_fmsg *fmsg)
779 {
780 	if (fmsg->err)
781 		return;
782 
783 	if (!fmsg->putting_binary)
784 		fmsg->err = -EINVAL;
785 
786 	fmsg->putting_binary = false;
787 	devlink_fmsg_arr_pair_nest_end(fmsg);
788 }
789 EXPORT_SYMBOL_GPL(devlink_fmsg_binary_pair_nest_end);
790 
791 static void devlink_fmsg_put_value(struct devlink_fmsg *fmsg,
792 				   const void *value, u16 value_len,
793 				   u8 value_nla_type)
794 {
795 	struct devlink_fmsg_item *item;
796 
797 	if (fmsg->err)
798 		return;
799 
800 	if (value_len > DEVLINK_FMSG_MAX_SIZE) {
801 		fmsg->err = -EMSGSIZE;
802 		return;
803 	}
804 
805 	item = kzalloc(sizeof(*item) + value_len, GFP_KERNEL);
806 	if (!item) {
807 		fmsg->err = -ENOMEM;
808 		return;
809 	}
810 
811 	item->nla_type = value_nla_type;
812 	item->len = value_len;
813 	item->attrtype = DEVLINK_ATTR_FMSG_OBJ_VALUE_DATA;
814 	memcpy(&item->value, value, item->len);
815 	list_add_tail(&item->list, &fmsg->item_list);
816 }
817 
818 static void devlink_fmsg_bool_put(struct devlink_fmsg *fmsg, bool value)
819 {
820 	devlink_fmsg_err_if_binary(fmsg);
821 	devlink_fmsg_put_value(fmsg, &value, sizeof(value), NLA_FLAG);
822 }
823 
824 static void devlink_fmsg_u8_put(struct devlink_fmsg *fmsg, u8 value)
825 {
826 	devlink_fmsg_err_if_binary(fmsg);
827 	devlink_fmsg_put_value(fmsg, &value, sizeof(value), NLA_U8);
828 }
829 
830 void devlink_fmsg_u32_put(struct devlink_fmsg *fmsg, u32 value)
831 {
832 	devlink_fmsg_err_if_binary(fmsg);
833 	devlink_fmsg_put_value(fmsg, &value, sizeof(value), NLA_U32);
834 }
835 EXPORT_SYMBOL_GPL(devlink_fmsg_u32_put);
836 
837 static void devlink_fmsg_u64_put(struct devlink_fmsg *fmsg, u64 value)
838 {
839 	devlink_fmsg_err_if_binary(fmsg);
840 	devlink_fmsg_put_value(fmsg, &value, sizeof(value), NLA_U64);
841 }
842 
843 void devlink_fmsg_string_put(struct devlink_fmsg *fmsg, const char *value)
844 {
845 	devlink_fmsg_err_if_binary(fmsg);
846 	devlink_fmsg_put_value(fmsg, value, strlen(value) + 1, NLA_NUL_STRING);
847 }
848 EXPORT_SYMBOL_GPL(devlink_fmsg_string_put);
849 
850 void devlink_fmsg_binary_put(struct devlink_fmsg *fmsg, const void *value,
851 			     u16 value_len)
852 {
853 	if (!fmsg->err && !fmsg->putting_binary)
854 		fmsg->err = -EINVAL;
855 
856 	devlink_fmsg_put_value(fmsg, value, value_len, NLA_BINARY);
857 }
858 EXPORT_SYMBOL_GPL(devlink_fmsg_binary_put);
859 
860 void devlink_fmsg_bool_pair_put(struct devlink_fmsg *fmsg, const char *name,
861 				bool value)
862 {
863 	devlink_fmsg_pair_nest_start(fmsg, name);
864 	devlink_fmsg_bool_put(fmsg, value);
865 	devlink_fmsg_pair_nest_end(fmsg);
866 }
867 EXPORT_SYMBOL_GPL(devlink_fmsg_bool_pair_put);
868 
869 void devlink_fmsg_u8_pair_put(struct devlink_fmsg *fmsg, const char *name,
870 			      u8 value)
871 {
872 	devlink_fmsg_pair_nest_start(fmsg, name);
873 	devlink_fmsg_u8_put(fmsg, value);
874 	devlink_fmsg_pair_nest_end(fmsg);
875 }
876 EXPORT_SYMBOL_GPL(devlink_fmsg_u8_pair_put);
877 
878 void devlink_fmsg_u32_pair_put(struct devlink_fmsg *fmsg, const char *name,
879 			       u32 value)
880 {
881 	devlink_fmsg_pair_nest_start(fmsg, name);
882 	devlink_fmsg_u32_put(fmsg, value);
883 	devlink_fmsg_pair_nest_end(fmsg);
884 }
885 EXPORT_SYMBOL_GPL(devlink_fmsg_u32_pair_put);
886 
887 void devlink_fmsg_u64_pair_put(struct devlink_fmsg *fmsg, const char *name,
888 			       u64 value)
889 {
890 	devlink_fmsg_pair_nest_start(fmsg, name);
891 	devlink_fmsg_u64_put(fmsg, value);
892 	devlink_fmsg_pair_nest_end(fmsg);
893 }
894 EXPORT_SYMBOL_GPL(devlink_fmsg_u64_pair_put);
895 
896 void devlink_fmsg_string_pair_put(struct devlink_fmsg *fmsg, const char *name,
897 				  const char *value)
898 {
899 	devlink_fmsg_pair_nest_start(fmsg, name);
900 	devlink_fmsg_string_put(fmsg, value);
901 	devlink_fmsg_pair_nest_end(fmsg);
902 }
903 EXPORT_SYMBOL_GPL(devlink_fmsg_string_pair_put);
904 
905 void devlink_fmsg_binary_pair_put(struct devlink_fmsg *fmsg, const char *name,
906 				  const void *value, u32 value_len)
907 {
908 	u32 data_size;
909 	u32 offset;
910 
911 	devlink_fmsg_binary_pair_nest_start(fmsg, name);
912 
913 	for (offset = 0; offset < value_len; offset += data_size) {
914 		data_size = value_len - offset;
915 		if (data_size > DEVLINK_FMSG_MAX_SIZE)
916 			data_size = DEVLINK_FMSG_MAX_SIZE;
917 
918 		devlink_fmsg_binary_put(fmsg, value + offset, data_size);
919 	}
920 
921 	devlink_fmsg_binary_pair_nest_end(fmsg);
922 	fmsg->putting_binary = false;
923 }
924 EXPORT_SYMBOL_GPL(devlink_fmsg_binary_pair_put);
925 
926 static int
927 devlink_fmsg_item_fill_type(struct devlink_fmsg_item *msg, struct sk_buff *skb)
928 {
929 	switch (msg->nla_type) {
930 	case NLA_FLAG:
931 	case NLA_U8:
932 	case NLA_U32:
933 	case NLA_U64:
934 	case NLA_NUL_STRING:
935 	case NLA_BINARY:
936 		return nla_put_u8(skb, DEVLINK_ATTR_FMSG_OBJ_VALUE_TYPE,
937 				  msg->nla_type);
938 	default:
939 		return -EINVAL;
940 	}
941 }
942 
943 static int
944 devlink_fmsg_item_fill_data(struct devlink_fmsg_item *msg, struct sk_buff *skb)
945 {
946 	int attrtype = DEVLINK_ATTR_FMSG_OBJ_VALUE_DATA;
947 	u8 tmp;
948 
949 	switch (msg->nla_type) {
950 	case NLA_FLAG:
951 		/* Always provide flag data, regardless of its value */
952 		tmp = *(bool *)msg->value;
953 
954 		return nla_put_u8(skb, attrtype, tmp);
955 	case NLA_U8:
956 		return nla_put_u8(skb, attrtype, *(u8 *)msg->value);
957 	case NLA_U32:
958 		return nla_put_u32(skb, attrtype, *(u32 *)msg->value);
959 	case NLA_U64:
960 		return nla_put_u64_64bit(skb, attrtype, *(u64 *)msg->value,
961 					 DEVLINK_ATTR_PAD);
962 	case NLA_NUL_STRING:
963 		return nla_put_string(skb, attrtype, (char *)&msg->value);
964 	case NLA_BINARY:
965 		return nla_put(skb, attrtype, msg->len, (void *)&msg->value);
966 	default:
967 		return -EINVAL;
968 	}
969 }
970 
971 static int
972 devlink_fmsg_prepare_skb(struct devlink_fmsg *fmsg, struct sk_buff *skb,
973 			 int *start)
974 {
975 	struct devlink_fmsg_item *item;
976 	struct nlattr *fmsg_nlattr;
977 	int err = 0;
978 	int i = 0;
979 
980 	fmsg_nlattr = nla_nest_start_noflag(skb, DEVLINK_ATTR_FMSG);
981 	if (!fmsg_nlattr)
982 		return -EMSGSIZE;
983 
984 	list_for_each_entry(item, &fmsg->item_list, list) {
985 		if (i < *start) {
986 			i++;
987 			continue;
988 		}
989 
990 		switch (item->attrtype) {
991 		case DEVLINK_ATTR_FMSG_OBJ_NEST_START:
992 		case DEVLINK_ATTR_FMSG_PAIR_NEST_START:
993 		case DEVLINK_ATTR_FMSG_ARR_NEST_START:
994 		case DEVLINK_ATTR_FMSG_NEST_END:
995 			err = nla_put_flag(skb, item->attrtype);
996 			break;
997 		case DEVLINK_ATTR_FMSG_OBJ_VALUE_DATA:
998 			err = devlink_fmsg_item_fill_type(item, skb);
999 			if (err)
1000 				break;
1001 			err = devlink_fmsg_item_fill_data(item, skb);
1002 			break;
1003 		case DEVLINK_ATTR_FMSG_OBJ_NAME:
1004 			err = nla_put_string(skb, item->attrtype,
1005 					     (char *)&item->value);
1006 			break;
1007 		default:
1008 			err = -EINVAL;
1009 			break;
1010 		}
1011 		if (!err)
1012 			*start = ++i;
1013 		else
1014 			break;
1015 	}
1016 
1017 	nla_nest_end(skb, fmsg_nlattr);
1018 	return err;
1019 }
1020 
1021 static int devlink_fmsg_snd(struct devlink_fmsg *fmsg,
1022 			    struct genl_info *info,
1023 			    enum devlink_command cmd, int flags)
1024 {
1025 	struct nlmsghdr *nlh;
1026 	struct sk_buff *skb;
1027 	bool last = false;
1028 	int index = 0;
1029 	void *hdr;
1030 	int err;
1031 
1032 	if (fmsg->err)
1033 		return fmsg->err;
1034 
1035 	while (!last) {
1036 		int tmp_index = index;
1037 
1038 		skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
1039 		if (!skb)
1040 			return -ENOMEM;
1041 
1042 		hdr = genlmsg_put(skb, info->snd_portid, info->snd_seq,
1043 				  &devlink_nl_family, flags | NLM_F_MULTI, cmd);
1044 		if (!hdr) {
1045 			err = -EMSGSIZE;
1046 			goto nla_put_failure;
1047 		}
1048 
1049 		err = devlink_fmsg_prepare_skb(fmsg, skb, &index);
1050 		if (!err)
1051 			last = true;
1052 		else if (err != -EMSGSIZE || tmp_index == index)
1053 			goto nla_put_failure;
1054 
1055 		genlmsg_end(skb, hdr);
1056 		err = genlmsg_reply(skb, info);
1057 		if (err)
1058 			return err;
1059 	}
1060 
1061 	skb = genlmsg_new(GENLMSG_DEFAULT_SIZE, GFP_KERNEL);
1062 	if (!skb)
1063 		return -ENOMEM;
1064 	nlh = nlmsg_put(skb, info->snd_portid, info->snd_seq,
1065 			NLMSG_DONE, 0, flags | NLM_F_MULTI);
1066 	if (!nlh) {
1067 		err = -EMSGSIZE;
1068 		goto nla_put_failure;
1069 	}
1070 
1071 	return genlmsg_reply(skb, info);
1072 
1073 nla_put_failure:
1074 	nlmsg_free(skb);
1075 	return err;
1076 }
1077 
1078 static int devlink_fmsg_dumpit(struct devlink_fmsg *fmsg, struct sk_buff *skb,
1079 			       struct netlink_callback *cb,
1080 			       enum devlink_command cmd)
1081 {
1082 	struct devlink_nl_dump_state *state = devlink_dump_state(cb);
1083 	int index = state->idx;
1084 	int tmp_index = index;
1085 	void *hdr;
1086 	int err;
1087 
1088 	if (fmsg->err)
1089 		return fmsg->err;
1090 
1091 	hdr = genlmsg_put(skb, NETLINK_CB(cb->skb).portid, cb->nlh->nlmsg_seq,
1092 			  &devlink_nl_family, NLM_F_ACK | NLM_F_MULTI, cmd);
1093 	if (!hdr) {
1094 		err = -EMSGSIZE;
1095 		goto nla_put_failure;
1096 	}
1097 
1098 	err = devlink_fmsg_prepare_skb(fmsg, skb, &index);
1099 	if ((err && err != -EMSGSIZE) || tmp_index == index)
1100 		goto nla_put_failure;
1101 
1102 	state->idx = index;
1103 	genlmsg_end(skb, hdr);
1104 	return skb->len;
1105 
1106 nla_put_failure:
1107 	genlmsg_cancel(skb, hdr);
1108 	return err;
1109 }
1110 
1111 int devlink_nl_health_reporter_diagnose_doit(struct sk_buff *skb,
1112 					     struct genl_info *info)
1113 {
1114 	struct devlink *devlink = info->user_ptr[0];
1115 	struct devlink_health_reporter *reporter;
1116 	struct devlink_fmsg *fmsg;
1117 	int err;
1118 
1119 	reporter = devlink_health_reporter_get_from_info(devlink, info);
1120 	if (!reporter)
1121 		return -EINVAL;
1122 
1123 	if (!reporter->ops->diagnose)
1124 		return -EOPNOTSUPP;
1125 
1126 	fmsg = devlink_fmsg_alloc();
1127 	if (!fmsg)
1128 		return -ENOMEM;
1129 
1130 	devlink_fmsg_obj_nest_start(fmsg);
1131 
1132 	err = reporter->ops->diagnose(reporter, fmsg, info->extack);
1133 	if (err)
1134 		goto out;
1135 
1136 	devlink_fmsg_obj_nest_end(fmsg);
1137 
1138 	err = devlink_fmsg_snd(fmsg, info,
1139 			       DEVLINK_CMD_HEALTH_REPORTER_DIAGNOSE, 0);
1140 
1141 out:
1142 	devlink_fmsg_free(fmsg);
1143 	return err;
1144 }
1145 
1146 static struct devlink_health_reporter *
1147 devlink_health_reporter_get_from_cb_lock(struct netlink_callback *cb)
1148 {
1149 	const struct genl_info *info = genl_info_dump(cb);
1150 	struct devlink_health_reporter *reporter;
1151 	struct nlattr **attrs = info->attrs;
1152 	struct devlink *devlink;
1153 
1154 	devlink = devlink_get_from_attrs_lock(sock_net(cb->skb->sk), attrs);
1155 	if (IS_ERR(devlink))
1156 		return NULL;
1157 
1158 	reporter = devlink_health_reporter_get_from_attrs(devlink, attrs);
1159 	if (!reporter) {
1160 		devl_unlock(devlink);
1161 		devlink_put(devlink);
1162 	}
1163 	return reporter;
1164 }
1165 
1166 int devlink_nl_health_reporter_dump_get_dumpit(struct sk_buff *skb,
1167 					       struct netlink_callback *cb)
1168 {
1169 	struct devlink_nl_dump_state *state = devlink_dump_state(cb);
1170 	struct devlink_health_reporter *reporter;
1171 	struct devlink *devlink;
1172 	int err;
1173 
1174 	reporter = devlink_health_reporter_get_from_cb_lock(cb);
1175 	if (!reporter)
1176 		return -EINVAL;
1177 
1178 	devlink = reporter->devlink;
1179 	if (!reporter->ops->dump) {
1180 		devl_unlock(devlink);
1181 		devlink_put(devlink);
1182 		return -EOPNOTSUPP;
1183 	}
1184 
1185 	if (!state->idx) {
1186 		err = devlink_health_do_dump(reporter, NULL, cb->extack);
1187 		if (err)
1188 			goto unlock;
1189 		state->dump_ts = reporter->dump_ts;
1190 	}
1191 	if (!reporter->dump_fmsg || state->dump_ts != reporter->dump_ts) {
1192 		NL_SET_ERR_MSG(cb->extack, "Dump trampled, please retry");
1193 		err = -EAGAIN;
1194 		goto unlock;
1195 	}
1196 
1197 	err = devlink_fmsg_dumpit(reporter->dump_fmsg, skb, cb,
1198 				  DEVLINK_CMD_HEALTH_REPORTER_DUMP_GET);
1199 unlock:
1200 	devl_unlock(devlink);
1201 	devlink_put(devlink);
1202 	return err;
1203 }
1204 
1205 int devlink_nl_health_reporter_dump_clear_doit(struct sk_buff *skb,
1206 					       struct genl_info *info)
1207 {
1208 	struct devlink *devlink = info->user_ptr[0];
1209 	struct devlink_health_reporter *reporter;
1210 
1211 	reporter = devlink_health_reporter_get_from_info(devlink, info);
1212 	if (!reporter)
1213 		return -EINVAL;
1214 
1215 	if (!reporter->ops->dump)
1216 		return -EOPNOTSUPP;
1217 
1218 	devlink_health_dump_clear(reporter);
1219 	return 0;
1220 }
1221 
1222 int devlink_nl_health_reporter_test_doit(struct sk_buff *skb,
1223 					 struct genl_info *info)
1224 {
1225 	struct devlink *devlink = info->user_ptr[0];
1226 	struct devlink_health_reporter *reporter;
1227 
1228 	reporter = devlink_health_reporter_get_from_info(devlink, info);
1229 	if (!reporter)
1230 		return -EINVAL;
1231 
1232 	if (!reporter->ops->test)
1233 		return -EOPNOTSUPP;
1234 
1235 	return reporter->ops->test(reporter, info->extack);
1236 }
1237