xref: /linux/fs/ocfs2/stack_o2cb.c (revision a1ff5a7d78a036d6c2178ee5acd6ba4946243800)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * stack_o2cb.c
4  *
5  * Code which interfaces ocfs2 with the o2cb stack.
6  *
7  * Copyright (C) 2007 Oracle.  All rights reserved.
8  */
9 
10 #include <linux/kernel.h>
11 #include <linux/crc32.h>
12 #include <linux/slab.h>
13 #include <linux/module.h>
14 
15 /* Needed for AOP_TRUNCATED_PAGE in mlog_errno() */
16 #include <linux/fs.h>
17 
18 #include "cluster/masklog.h"
19 #include "cluster/nodemanager.h"
20 #include "cluster/heartbeat.h"
21 #include "cluster/tcp.h"
22 
23 #include "stackglue.h"
24 
25 struct o2dlm_private {
26 	struct dlm_eviction_cb op_eviction_cb;
27 };
28 
29 static struct ocfs2_stack_plugin o2cb_stack;
30 
31 /* These should be identical */
32 #if (DLM_LOCK_IV != LKM_IVMODE)
33 # error Lock modes do not match
34 #endif
35 #if (DLM_LOCK_NL != LKM_NLMODE)
36 # error Lock modes do not match
37 #endif
38 #if (DLM_LOCK_CR != LKM_CRMODE)
39 # error Lock modes do not match
40 #endif
41 #if (DLM_LOCK_CW != LKM_CWMODE)
42 # error Lock modes do not match
43 #endif
44 #if (DLM_LOCK_PR != LKM_PRMODE)
45 # error Lock modes do not match
46 #endif
47 #if (DLM_LOCK_PW != LKM_PWMODE)
48 # error Lock modes do not match
49 #endif
50 #if (DLM_LOCK_EX != LKM_EXMODE)
51 # error Lock modes do not match
52 #endif
mode_to_o2dlm(int mode)53 static inline int mode_to_o2dlm(int mode)
54 {
55 	BUG_ON(mode > LKM_MAXMODE);
56 
57 	return mode;
58 }
59 
flags_to_o2dlm(u32 flags)60 static int flags_to_o2dlm(u32 flags)
61 {
62 	int o2dlm_flags = 0;
63 
64 	if (flags & DLM_LKF_NOQUEUE)
65 		o2dlm_flags |= LKM_NOQUEUE;
66 	if (flags & DLM_LKF_CANCEL)
67 		o2dlm_flags |= LKM_CANCEL;
68 	if (flags & DLM_LKF_CONVERT)
69 		o2dlm_flags |= LKM_CONVERT;
70 	if (flags & DLM_LKF_VALBLK)
71 		o2dlm_flags |= LKM_VALBLK;
72 	if (flags & DLM_LKF_IVVALBLK)
73 		o2dlm_flags |= LKM_INVVALBLK;
74 	if (flags & DLM_LKF_ORPHAN)
75 		o2dlm_flags |= LKM_ORPHAN;
76 	if (flags & DLM_LKF_FORCEUNLOCK)
77 		o2dlm_flags |= LKM_FORCE;
78 	if (flags & DLM_LKF_TIMEOUT)
79 		o2dlm_flags |= LKM_TIMEOUT;
80 	if (flags & DLM_LKF_LOCAL)
81 		o2dlm_flags |= LKM_LOCAL;
82 
83 	return o2dlm_flags;
84 }
85 
86 /*
87  * Map an o2dlm status to standard errno values.
88  *
89  * o2dlm only uses a handful of these, and returns even fewer to the
90  * caller. Still, we try to assign sane values to each error.
91  *
92  * The following value pairs have special meanings to dlmglue, thus
93  * the right hand side needs to stay unique - never duplicate the
94  * mapping elsewhere in the table!
95  *
96  * DLM_NORMAL:		0
97  * DLM_NOTQUEUED:	-EAGAIN
98  * DLM_CANCELGRANT:	-EBUSY
99  * DLM_CANCEL:		-DLM_ECANCEL
100  */
101 /* Keep in sync with dlmapi.h */
102 static int status_map[] = {
103 	[DLM_NORMAL]			= 0,		/* Success */
104 	[DLM_GRANTED]			= -EINVAL,
105 	[DLM_DENIED]			= -EACCES,
106 	[DLM_DENIED_NOLOCKS]		= -EACCES,
107 	[DLM_WORKING]			= -EACCES,
108 	[DLM_BLOCKED]			= -EINVAL,
109 	[DLM_BLOCKED_ORPHAN]		= -EINVAL,
110 	[DLM_DENIED_GRACE_PERIOD]	= -EACCES,
111 	[DLM_SYSERR]			= -ENOMEM,	/* It is what it is */
112 	[DLM_NOSUPPORT]			= -EPROTO,
113 	[DLM_CANCELGRANT]		= -EBUSY,	/* Cancel after grant */
114 	[DLM_IVLOCKID]			= -EINVAL,
115 	[DLM_SYNC]			= -EINVAL,
116 	[DLM_BADTYPE]			= -EINVAL,
117 	[DLM_BADRESOURCE]		= -EINVAL,
118 	[DLM_MAXHANDLES]		= -ENOMEM,
119 	[DLM_NOCLINFO]			= -EINVAL,
120 	[DLM_NOLOCKMGR]			= -EINVAL,
121 	[DLM_NOPURGED]			= -EINVAL,
122 	[DLM_BADARGS]			= -EINVAL,
123 	[DLM_VOID]			= -EINVAL,
124 	[DLM_NOTQUEUED]			= -EAGAIN,	/* Trylock failed */
125 	[DLM_IVBUFLEN]			= -EINVAL,
126 	[DLM_CVTUNGRANT]		= -EPERM,
127 	[DLM_BADPARAM]			= -EINVAL,
128 	[DLM_VALNOTVALID]		= -EINVAL,
129 	[DLM_REJECTED]			= -EPERM,
130 	[DLM_ABORT]			= -EINVAL,
131 	[DLM_CANCEL]			= -DLM_ECANCEL,	/* Successful cancel */
132 	[DLM_IVRESHANDLE]		= -EINVAL,
133 	[DLM_DEADLOCK]			= -EDEADLK,
134 	[DLM_DENIED_NOASTS]		= -EINVAL,
135 	[DLM_FORWARD]			= -EINVAL,
136 	[DLM_TIMEOUT]			= -ETIMEDOUT,
137 	[DLM_IVGROUPID]			= -EINVAL,
138 	[DLM_VERS_CONFLICT]		= -EOPNOTSUPP,
139 	[DLM_BAD_DEVICE_PATH]		= -ENOENT,
140 	[DLM_NO_DEVICE_PERMISSION]	= -EPERM,
141 	[DLM_NO_CONTROL_DEVICE]		= -ENOENT,
142 	[DLM_RECOVERING]		= -ENOTCONN,
143 	[DLM_MIGRATING]			= -ERESTART,
144 	[DLM_MAXSTATS]			= -EINVAL,
145 };
146 
dlm_status_to_errno(enum dlm_status status)147 static int dlm_status_to_errno(enum dlm_status status)
148 {
149 	BUG_ON(status < 0 || status >= ARRAY_SIZE(status_map));
150 
151 	return status_map[status];
152 }
153 
o2dlm_lock_ast_wrapper(void * astarg)154 static void o2dlm_lock_ast_wrapper(void *astarg)
155 {
156 	struct ocfs2_dlm_lksb *lksb = astarg;
157 
158 	lksb->lksb_conn->cc_proto->lp_lock_ast(lksb);
159 }
160 
o2dlm_blocking_ast_wrapper(void * astarg,int level)161 static void o2dlm_blocking_ast_wrapper(void *astarg, int level)
162 {
163 	struct ocfs2_dlm_lksb *lksb = astarg;
164 
165 	lksb->lksb_conn->cc_proto->lp_blocking_ast(lksb, level);
166 }
167 
o2dlm_unlock_ast_wrapper(void * astarg,enum dlm_status status)168 static void o2dlm_unlock_ast_wrapper(void *astarg, enum dlm_status status)
169 {
170 	struct ocfs2_dlm_lksb *lksb = astarg;
171 	int error = dlm_status_to_errno(status);
172 
173 	/*
174 	 * In o2dlm, you can get both the lock_ast() for the lock being
175 	 * granted and the unlock_ast() for the CANCEL failing.  A
176 	 * successful cancel sends DLM_NORMAL here.  If the
177 	 * lock grant happened before the cancel arrived, you get
178 	 * DLM_CANCELGRANT.
179 	 *
180 	 * There's no need for the double-ast.  If we see DLM_CANCELGRANT,
181 	 * we just ignore it.  We expect the lock_ast() to handle the
182 	 * granted lock.
183 	 */
184 	if (status == DLM_CANCELGRANT)
185 		return;
186 
187 	lksb->lksb_conn->cc_proto->lp_unlock_ast(lksb, error);
188 }
189 
o2cb_dlm_lock(struct ocfs2_cluster_connection * conn,int mode,struct ocfs2_dlm_lksb * lksb,u32 flags,void * name,unsigned int namelen)190 static int o2cb_dlm_lock(struct ocfs2_cluster_connection *conn,
191 			 int mode,
192 			 struct ocfs2_dlm_lksb *lksb,
193 			 u32 flags,
194 			 void *name,
195 			 unsigned int namelen)
196 {
197 	enum dlm_status status;
198 	int o2dlm_mode = mode_to_o2dlm(mode);
199 	int o2dlm_flags = flags_to_o2dlm(flags);
200 	int ret;
201 
202 	status = dlmlock(conn->cc_lockspace, o2dlm_mode, &lksb->lksb_o2dlm,
203 			 o2dlm_flags, name, namelen,
204 			 o2dlm_lock_ast_wrapper, lksb,
205 			 o2dlm_blocking_ast_wrapper);
206 	ret = dlm_status_to_errno(status);
207 	return ret;
208 }
209 
o2cb_dlm_unlock(struct ocfs2_cluster_connection * conn,struct ocfs2_dlm_lksb * lksb,u32 flags)210 static int o2cb_dlm_unlock(struct ocfs2_cluster_connection *conn,
211 			   struct ocfs2_dlm_lksb *lksb,
212 			   u32 flags)
213 {
214 	enum dlm_status status;
215 	int o2dlm_flags = flags_to_o2dlm(flags);
216 	int ret;
217 
218 	status = dlmunlock(conn->cc_lockspace, &lksb->lksb_o2dlm,
219 			   o2dlm_flags, o2dlm_unlock_ast_wrapper, lksb);
220 	ret = dlm_status_to_errno(status);
221 	return ret;
222 }
223 
o2cb_dlm_lock_status(struct ocfs2_dlm_lksb * lksb)224 static int o2cb_dlm_lock_status(struct ocfs2_dlm_lksb *lksb)
225 {
226 	return dlm_status_to_errno(lksb->lksb_o2dlm.status);
227 }
228 
229 /*
230  * o2dlm aways has a "valid" LVB. If the dlm loses track of the LVB
231  * contents, it will zero out the LVB.  Thus the caller can always trust
232  * the contents.
233  */
o2cb_dlm_lvb_valid(struct ocfs2_dlm_lksb * lksb)234 static int o2cb_dlm_lvb_valid(struct ocfs2_dlm_lksb *lksb)
235 {
236 	return 1;
237 }
238 
o2cb_dlm_lvb(struct ocfs2_dlm_lksb * lksb)239 static void *o2cb_dlm_lvb(struct ocfs2_dlm_lksb *lksb)
240 {
241 	return (void *)(lksb->lksb_o2dlm.lvb);
242 }
243 
o2cb_dump_lksb(struct ocfs2_dlm_lksb * lksb)244 static void o2cb_dump_lksb(struct ocfs2_dlm_lksb *lksb)
245 {
246 	dlm_print_one_lock(lksb->lksb_o2dlm.lockid);
247 }
248 
249 /*
250  * Check if this node is heartbeating and is connected to all other
251  * heartbeating nodes.
252  */
o2cb_cluster_check(void)253 static int o2cb_cluster_check(void)
254 {
255 	u8 node_num;
256 	int i;
257 	unsigned long hbmap[BITS_TO_LONGS(O2NM_MAX_NODES)];
258 	unsigned long netmap[BITS_TO_LONGS(O2NM_MAX_NODES)];
259 
260 	node_num = o2nm_this_node();
261 	if (node_num == O2NM_MAX_NODES) {
262 		printk(KERN_ERR "o2cb: This node has not been configured.\n");
263 		return -EINVAL;
264 	}
265 
266 	/*
267 	 * o2dlm expects o2net sockets to be created. If not, then
268 	 * dlm_join_domain() fails with a stack of errors which are both cryptic
269 	 * and incomplete. The idea here is to detect upfront whether we have
270 	 * managed to connect to all nodes or not. If not, then list the nodes
271 	 * to allow the user to check the configuration (incorrect IP, firewall,
272 	 * etc.) Yes, this is racy. But its not the end of the world.
273 	 */
274 #define	O2CB_MAP_STABILIZE_COUNT	60
275 	for (i = 0; i < O2CB_MAP_STABILIZE_COUNT; ++i) {
276 		o2hb_fill_node_map(hbmap, O2NM_MAX_NODES);
277 		if (!test_bit(node_num, hbmap)) {
278 			printk(KERN_ERR "o2cb: %s heartbeat has not been "
279 			       "started.\n", (o2hb_global_heartbeat_active() ?
280 					      "Global" : "Local"));
281 			return -EINVAL;
282 		}
283 		o2net_fill_node_map(netmap, O2NM_MAX_NODES);
284 		/* Force set the current node to allow easy compare */
285 		set_bit(node_num, netmap);
286 		if (bitmap_equal(hbmap, netmap, O2NM_MAX_NODES))
287 			return 0;
288 		if (i < O2CB_MAP_STABILIZE_COUNT - 1)
289 			msleep(1000);
290 	}
291 
292 	printk(KERN_ERR "o2cb: This node could not connect to nodes:");
293 	i = -1;
294 	while ((i = find_next_bit(hbmap, O2NM_MAX_NODES,
295 				  i + 1)) < O2NM_MAX_NODES) {
296 		if (!test_bit(i, netmap))
297 			printk(" %u", i);
298 	}
299 	printk(".\n");
300 
301 	return -ENOTCONN;
302 }
303 
304 /*
305  * Called from the dlm when it's about to evict a node. This is how the
306  * classic stack signals node death.
307  */
o2dlm_eviction_cb(int node_num,void * data)308 static void o2dlm_eviction_cb(int node_num, void *data)
309 {
310 	struct ocfs2_cluster_connection *conn = data;
311 
312 	printk(KERN_NOTICE "o2cb: o2dlm has evicted node %d from domain %.*s\n",
313 	       node_num, conn->cc_namelen, conn->cc_name);
314 
315 	conn->cc_recovery_handler(node_num, conn->cc_recovery_data);
316 }
317 
o2cb_cluster_connect(struct ocfs2_cluster_connection * conn)318 static int o2cb_cluster_connect(struct ocfs2_cluster_connection *conn)
319 {
320 	int rc = 0;
321 	u32 dlm_key;
322 	struct dlm_ctxt *dlm;
323 	struct o2dlm_private *priv;
324 	struct dlm_protocol_version fs_version;
325 
326 	BUG_ON(conn == NULL);
327 	BUG_ON(conn->cc_proto == NULL);
328 
329 	/* Ensure cluster stack is up and all nodes are connected */
330 	rc = o2cb_cluster_check();
331 	if (rc) {
332 		printk(KERN_ERR "o2cb: Cluster check failed. Fix errors "
333 		       "before retrying.\n");
334 		goto out;
335 	}
336 
337 	priv = kzalloc(sizeof(struct o2dlm_private), GFP_KERNEL);
338 	if (!priv) {
339 		rc = -ENOMEM;
340 		goto out_free;
341 	}
342 
343 	/* This just fills the structure in.  It is safe to pass conn. */
344 	dlm_setup_eviction_cb(&priv->op_eviction_cb, o2dlm_eviction_cb,
345 			      conn);
346 
347 	conn->cc_private = priv;
348 
349 	/* used by the dlm code to make message headers unique, each
350 	 * node in this domain must agree on this. */
351 	dlm_key = crc32_le(0, conn->cc_name, conn->cc_namelen);
352 	fs_version.pv_major = conn->cc_version.pv_major;
353 	fs_version.pv_minor = conn->cc_version.pv_minor;
354 
355 	dlm = dlm_register_domain(conn->cc_name, dlm_key, &fs_version);
356 	if (IS_ERR(dlm)) {
357 		rc = PTR_ERR(dlm);
358 		mlog_errno(rc);
359 		goto out_free;
360 	}
361 
362 	conn->cc_version.pv_major = fs_version.pv_major;
363 	conn->cc_version.pv_minor = fs_version.pv_minor;
364 	conn->cc_lockspace = dlm;
365 
366 	dlm_register_eviction_cb(dlm, &priv->op_eviction_cb);
367 
368 out_free:
369 	if (rc)
370 		kfree(conn->cc_private);
371 
372 out:
373 	return rc;
374 }
375 
o2cb_cluster_disconnect(struct ocfs2_cluster_connection * conn)376 static int o2cb_cluster_disconnect(struct ocfs2_cluster_connection *conn)
377 {
378 	struct dlm_ctxt *dlm = conn->cc_lockspace;
379 	struct o2dlm_private *priv = conn->cc_private;
380 
381 	dlm_unregister_eviction_cb(&priv->op_eviction_cb);
382 	conn->cc_private = NULL;
383 	kfree(priv);
384 
385 	dlm_unregister_domain(dlm);
386 	conn->cc_lockspace = NULL;
387 
388 	return 0;
389 }
390 
o2cb_cluster_this_node(struct ocfs2_cluster_connection * conn,unsigned int * node)391 static int o2cb_cluster_this_node(struct ocfs2_cluster_connection *conn,
392 				  unsigned int *node)
393 {
394 	int node_num;
395 
396 	node_num = o2nm_this_node();
397 	if (node_num == O2NM_INVALID_NODE_NUM)
398 		return -ENOENT;
399 
400 	if (node_num >= O2NM_MAX_NODES)
401 		return -EOVERFLOW;
402 
403 	*node = node_num;
404 	return 0;
405 }
406 
407 static const struct ocfs2_stack_operations o2cb_stack_ops = {
408 	.connect	= o2cb_cluster_connect,
409 	.disconnect	= o2cb_cluster_disconnect,
410 	.this_node	= o2cb_cluster_this_node,
411 	.dlm_lock	= o2cb_dlm_lock,
412 	.dlm_unlock	= o2cb_dlm_unlock,
413 	.lock_status	= o2cb_dlm_lock_status,
414 	.lvb_valid	= o2cb_dlm_lvb_valid,
415 	.lock_lvb	= o2cb_dlm_lvb,
416 	.dump_lksb	= o2cb_dump_lksb,
417 };
418 
419 static struct ocfs2_stack_plugin o2cb_stack = {
420 	.sp_name	= "o2cb",
421 	.sp_ops		= &o2cb_stack_ops,
422 	.sp_owner	= THIS_MODULE,
423 };
424 
o2cb_stack_init(void)425 static int __init o2cb_stack_init(void)
426 {
427 	return ocfs2_stack_glue_register(&o2cb_stack);
428 }
429 
o2cb_stack_exit(void)430 static void __exit o2cb_stack_exit(void)
431 {
432 	ocfs2_stack_glue_unregister(&o2cb_stack);
433 }
434 
435 MODULE_AUTHOR("Oracle");
436 MODULE_DESCRIPTION("ocfs2 driver for the classic o2cb stack");
437 MODULE_LICENSE("GPL");
438 module_init(o2cb_stack_init);
439 module_exit(o2cb_stack_exit);
440