1 /* -*- mode: c; c-basic-offset: 8; -*- 2 * vim: noexpandtab sw=8 ts=8 sts=0: 3 * 4 * heartbeat.c 5 * 6 * Register ourselves with the heartbaet service, keep our node maps 7 * up to date, and fire off recovery when needed. 8 * 9 * Copyright (C) 2002, 2004 Oracle. All rights reserved. 10 * 11 * This program is free software; you can redistribute it and/or 12 * modify it under the terms of the GNU General Public 13 * License as published by the Free Software Foundation; either 14 * version 2 of the License, or (at your option) any later version. 15 * 16 * This program is distributed in the hope that it will be useful, 17 * but WITHOUT ANY WARRANTY; without even the implied warranty of 18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 19 * General Public License for more details. 20 * 21 * You should have received a copy of the GNU General Public 22 * License along with this program; if not, write to the 23 * Free Software Foundation, Inc., 59 Temple Place - Suite 330, 24 * Boston, MA 021110-1307, USA. 25 */ 26 27 #include <linux/fs.h> 28 #include <linux/types.h> 29 #include <linux/slab.h> 30 #include <linux/highmem.h> 31 #include <linux/kmod.h> 32 33 #include <cluster/heartbeat.h> 34 #include <cluster/nodemanager.h> 35 36 #include <dlm/dlmapi.h> 37 38 #define MLOG_MASK_PREFIX ML_SUPER 39 #include <cluster/masklog.h> 40 41 #include "ocfs2.h" 42 43 #include "alloc.h" 44 #include "heartbeat.h" 45 #include "inode.h" 46 #include "journal.h" 47 #include "vote.h" 48 49 #include "buffer_head_io.h" 50 51 #define OCFS2_HB_NODE_DOWN_PRI (0x0000002) 52 #define OCFS2_HB_NODE_UP_PRI OCFS2_HB_NODE_DOWN_PRI 53 54 static inline void __ocfs2_node_map_set_bit(struct ocfs2_node_map *map, 55 int bit); 56 static inline void __ocfs2_node_map_clear_bit(struct ocfs2_node_map *map, 57 int bit); 58 static inline int __ocfs2_node_map_is_empty(struct ocfs2_node_map *map); 59 static void __ocfs2_node_map_dup(struct ocfs2_node_map *target, 60 struct ocfs2_node_map *from); 61 static void __ocfs2_node_map_set(struct ocfs2_node_map *target, 62 struct ocfs2_node_map *from); 63 64 void ocfs2_init_node_maps(struct ocfs2_super *osb) 65 { 66 spin_lock_init(&osb->node_map_lock); 67 ocfs2_node_map_init(&osb->mounted_map); 68 ocfs2_node_map_init(&osb->recovery_map); 69 ocfs2_node_map_init(&osb->umount_map); 70 ocfs2_node_map_init(&osb->osb_recovering_orphan_dirs); 71 } 72 73 static void ocfs2_do_node_down(int node_num, 74 struct ocfs2_super *osb) 75 { 76 BUG_ON(osb->node_num == node_num); 77 78 mlog(0, "ocfs2: node down event for %d\n", node_num); 79 80 if (!osb->dlm) { 81 /* 82 * No DLM means we're not even ready to participate yet. 83 * We check the slots after the DLM comes up, so we will 84 * notice the node death then. We can safely ignore it 85 * here. 86 */ 87 return; 88 } 89 90 if (ocfs2_node_map_test_bit(osb, &osb->umount_map, node_num)) { 91 /* If a node is in the umount map, then we've been 92 * expecting him to go down and we know ahead of time 93 * that recovery is not necessary. */ 94 ocfs2_node_map_clear_bit(osb, &osb->umount_map, node_num); 95 return; 96 } 97 98 ocfs2_recovery_thread(osb, node_num); 99 100 ocfs2_remove_node_from_vote_queues(osb, node_num); 101 } 102 103 static void ocfs2_hb_node_down_cb(struct o2nm_node *node, 104 int node_num, 105 void *data) 106 { 107 ocfs2_do_node_down(node_num, (struct ocfs2_super *) data); 108 } 109 110 /* Called from the dlm when it's about to evict a node. We may also 111 * get a heartbeat callback later. */ 112 static void ocfs2_dlm_eviction_cb(int node_num, 113 void *data) 114 { 115 struct ocfs2_super *osb = (struct ocfs2_super *) data; 116 struct super_block *sb = osb->sb; 117 118 mlog(ML_NOTICE, "device (%u,%u): dlm has evicted node %d\n", 119 MAJOR(sb->s_dev), MINOR(sb->s_dev), node_num); 120 121 ocfs2_do_node_down(node_num, osb); 122 } 123 124 static void ocfs2_hb_node_up_cb(struct o2nm_node *node, 125 int node_num, 126 void *data) 127 { 128 struct ocfs2_super *osb = data; 129 130 BUG_ON(osb->node_num == node_num); 131 132 mlog(0, "node up event for %d\n", node_num); 133 ocfs2_node_map_clear_bit(osb, &osb->umount_map, node_num); 134 } 135 136 void ocfs2_setup_hb_callbacks(struct ocfs2_super *osb) 137 { 138 o2hb_setup_callback(&osb->osb_hb_down, O2HB_NODE_DOWN_CB, 139 ocfs2_hb_node_down_cb, osb, 140 OCFS2_HB_NODE_DOWN_PRI); 141 142 o2hb_setup_callback(&osb->osb_hb_up, O2HB_NODE_UP_CB, 143 ocfs2_hb_node_up_cb, osb, OCFS2_HB_NODE_UP_PRI); 144 145 /* Not exactly a heartbeat callback, but leads to essentially 146 * the same path so we set it up here. */ 147 dlm_setup_eviction_cb(&osb->osb_eviction_cb, 148 ocfs2_dlm_eviction_cb, 149 osb); 150 } 151 152 /* Most functions here are just stubs for now... */ 153 int ocfs2_register_hb_callbacks(struct ocfs2_super *osb) 154 { 155 int status; 156 157 if (ocfs2_mount_local(osb)) 158 return 0; 159 160 status = o2hb_register_callback(&osb->osb_hb_down); 161 if (status < 0) { 162 mlog_errno(status); 163 goto bail; 164 } 165 166 status = o2hb_register_callback(&osb->osb_hb_up); 167 if (status < 0) 168 mlog_errno(status); 169 170 bail: 171 return status; 172 } 173 174 void ocfs2_clear_hb_callbacks(struct ocfs2_super *osb) 175 { 176 int status; 177 178 if (ocfs2_mount_local(osb)) 179 return; 180 181 status = o2hb_unregister_callback(&osb->osb_hb_down); 182 if (status < 0) 183 mlog_errno(status); 184 185 status = o2hb_unregister_callback(&osb->osb_hb_up); 186 if (status < 0) 187 mlog_errno(status); 188 } 189 190 void ocfs2_stop_heartbeat(struct ocfs2_super *osb) 191 { 192 int ret; 193 char *argv[5], *envp[3]; 194 195 if (ocfs2_mount_local(osb)) 196 return; 197 198 if (!osb->uuid_str) { 199 /* This can happen if we don't get far enough in mount... */ 200 mlog(0, "No UUID with which to stop heartbeat!\n\n"); 201 return; 202 } 203 204 argv[0] = (char *)o2nm_get_hb_ctl_path(); 205 argv[1] = "-K"; 206 argv[2] = "-u"; 207 argv[3] = osb->uuid_str; 208 argv[4] = NULL; 209 210 mlog(0, "Run: %s %s %s %s\n", argv[0], argv[1], argv[2], argv[3]); 211 212 /* minimal command environment taken from cpu_run_sbin_hotplug */ 213 envp[0] = "HOME=/"; 214 envp[1] = "PATH=/sbin:/bin:/usr/sbin:/usr/bin"; 215 envp[2] = NULL; 216 217 ret = call_usermodehelper(argv[0], argv, envp, 1); 218 if (ret < 0) 219 mlog_errno(ret); 220 } 221 222 /* special case -1 for now 223 * TODO: should *really* make sure the calling func never passes -1!! */ 224 void ocfs2_node_map_init(struct ocfs2_node_map *map) 225 { 226 map->num_nodes = OCFS2_NODE_MAP_MAX_NODES; 227 memset(map->map, 0, BITS_TO_LONGS(OCFS2_NODE_MAP_MAX_NODES) * 228 sizeof(unsigned long)); 229 } 230 231 static inline void __ocfs2_node_map_set_bit(struct ocfs2_node_map *map, 232 int bit) 233 { 234 set_bit(bit, map->map); 235 } 236 237 void ocfs2_node_map_set_bit(struct ocfs2_super *osb, 238 struct ocfs2_node_map *map, 239 int bit) 240 { 241 if (bit==-1) 242 return; 243 BUG_ON(bit >= map->num_nodes); 244 spin_lock(&osb->node_map_lock); 245 __ocfs2_node_map_set_bit(map, bit); 246 spin_unlock(&osb->node_map_lock); 247 } 248 249 static inline void __ocfs2_node_map_clear_bit(struct ocfs2_node_map *map, 250 int bit) 251 { 252 clear_bit(bit, map->map); 253 } 254 255 void ocfs2_node_map_clear_bit(struct ocfs2_super *osb, 256 struct ocfs2_node_map *map, 257 int bit) 258 { 259 if (bit==-1) 260 return; 261 BUG_ON(bit >= map->num_nodes); 262 spin_lock(&osb->node_map_lock); 263 __ocfs2_node_map_clear_bit(map, bit); 264 spin_unlock(&osb->node_map_lock); 265 } 266 267 int ocfs2_node_map_test_bit(struct ocfs2_super *osb, 268 struct ocfs2_node_map *map, 269 int bit) 270 { 271 int ret; 272 if (bit >= map->num_nodes) { 273 mlog(ML_ERROR, "bit=%d map->num_nodes=%d\n", bit, map->num_nodes); 274 BUG(); 275 } 276 spin_lock(&osb->node_map_lock); 277 ret = test_bit(bit, map->map); 278 spin_unlock(&osb->node_map_lock); 279 return ret; 280 } 281 282 static inline int __ocfs2_node_map_is_empty(struct ocfs2_node_map *map) 283 { 284 int bit; 285 bit = find_next_bit(map->map, map->num_nodes, 0); 286 if (bit < map->num_nodes) 287 return 0; 288 return 1; 289 } 290 291 int ocfs2_node_map_is_empty(struct ocfs2_super *osb, 292 struct ocfs2_node_map *map) 293 { 294 int ret; 295 BUG_ON(map->num_nodes == 0); 296 spin_lock(&osb->node_map_lock); 297 ret = __ocfs2_node_map_is_empty(map); 298 spin_unlock(&osb->node_map_lock); 299 return ret; 300 } 301 302 static void __ocfs2_node_map_dup(struct ocfs2_node_map *target, 303 struct ocfs2_node_map *from) 304 { 305 BUG_ON(from->num_nodes == 0); 306 ocfs2_node_map_init(target); 307 __ocfs2_node_map_set(target, from); 308 } 309 310 /* returns 1 if bit is the only bit set in target, 0 otherwise */ 311 int ocfs2_node_map_is_only(struct ocfs2_super *osb, 312 struct ocfs2_node_map *target, 313 int bit) 314 { 315 struct ocfs2_node_map temp; 316 int ret; 317 318 spin_lock(&osb->node_map_lock); 319 __ocfs2_node_map_dup(&temp, target); 320 __ocfs2_node_map_clear_bit(&temp, bit); 321 ret = __ocfs2_node_map_is_empty(&temp); 322 spin_unlock(&osb->node_map_lock); 323 324 return ret; 325 } 326 327 static void __ocfs2_node_map_set(struct ocfs2_node_map *target, 328 struct ocfs2_node_map *from) 329 { 330 int num_longs, i; 331 332 BUG_ON(target->num_nodes != from->num_nodes); 333 BUG_ON(target->num_nodes == 0); 334 335 num_longs = BITS_TO_LONGS(target->num_nodes); 336 for (i = 0; i < num_longs; i++) 337 target->map[i] = from->map[i]; 338 } 339 340 /* Returns whether the recovery bit was actually set - it may not be 341 * if a node is still marked as needing recovery */ 342 int ocfs2_recovery_map_set(struct ocfs2_super *osb, 343 int num) 344 { 345 int set = 0; 346 347 spin_lock(&osb->node_map_lock); 348 349 __ocfs2_node_map_clear_bit(&osb->mounted_map, num); 350 351 if (!test_bit(num, osb->recovery_map.map)) { 352 __ocfs2_node_map_set_bit(&osb->recovery_map, num); 353 set = 1; 354 } 355 356 spin_unlock(&osb->node_map_lock); 357 358 return set; 359 } 360 361 void ocfs2_recovery_map_clear(struct ocfs2_super *osb, 362 int num) 363 { 364 ocfs2_node_map_clear_bit(osb, &osb->recovery_map, num); 365 } 366 367 int ocfs2_node_map_iterate(struct ocfs2_super *osb, 368 struct ocfs2_node_map *map, 369 int idx) 370 { 371 int i = idx; 372 373 idx = O2NM_INVALID_NODE_NUM; 374 spin_lock(&osb->node_map_lock); 375 if ((i != O2NM_INVALID_NODE_NUM) && 376 (i >= 0) && 377 (i < map->num_nodes)) { 378 while(i < map->num_nodes) { 379 if (test_bit(i, map->map)) { 380 idx = i; 381 break; 382 } 383 i++; 384 } 385 } 386 spin_unlock(&osb->node_map_lock); 387 return idx; 388 } 389