David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 1 | /****************************************************************************** |
| 2 | ******************************************************************************* |
| 3 | ** |
| 4 | ** Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved. |
| 5 | ** Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved. |
| 6 | ** |
| 7 | ** This copyrighted material is made available to anyone wishing to use, |
| 8 | ** modify, copy, or redistribute it subject to the terms and conditions |
| 9 | ** of the GNU General Public License v.2. |
| 10 | ** |
| 11 | ******************************************************************************* |
| 12 | ******************************************************************************/ |
| 13 | |
| 14 | #include "dlm_internal.h" |
| 15 | #include "lockspace.h" |
| 16 | #include "member.h" |
| 17 | #include "dir.h" |
| 18 | #include "ast.h" |
| 19 | #include "recover.h" |
| 20 | #include "lowcomms.h" |
| 21 | #include "lock.h" |
| 22 | #include "requestqueue.h" |
| 23 | #include "recoverd.h" |
| 24 | |
| 25 | |
| 26 | /* If the start for which we're re-enabling locking (seq) has been superseded |
| 27 | by a newer stop (ls_recover_seq), we need to leave locking disabled. */ |
| 28 | |
| 29 | static int enable_locking(struct dlm_ls *ls, uint64_t seq) |
| 30 | { |
| 31 | int error = -EINTR; |
| 32 | |
| 33 | spin_lock(&ls->ls_recover_lock); |
| 34 | if (ls->ls_recover_seq == seq) { |
| 35 | set_bit(LSFL_RUNNING, &ls->ls_flags); |
| 36 | up_write(&ls->ls_in_recovery); |
| 37 | error = 0; |
| 38 | } |
| 39 | spin_unlock(&ls->ls_recover_lock); |
| 40 | return error; |
| 41 | } |
| 42 | |
| 43 | static int ls_recover(struct dlm_ls *ls, struct dlm_recover *rv) |
| 44 | { |
| 45 | unsigned long start; |
| 46 | int error, neg = 0; |
| 47 | |
David Teigland | 9013592 | 2006-01-20 08:47:07 +0000 | [diff] [blame] | 48 | log_debug(ls, "recover %llx", rv->seq); |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 49 | |
David Teigland | 9013592 | 2006-01-20 08:47:07 +0000 | [diff] [blame] | 50 | mutex_lock(&ls->ls_recoverd_active); |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 51 | |
| 52 | /* |
| 53 | * Suspending and resuming dlm_astd ensures that no lkb's from this ls |
| 54 | * will be processed by dlm_astd during recovery. |
| 55 | */ |
| 56 | |
| 57 | dlm_astd_suspend(); |
| 58 | dlm_astd_resume(); |
| 59 | |
| 60 | /* |
| 61 | * This list of root rsb's will be the basis of most of the recovery |
| 62 | * routines. |
| 63 | */ |
| 64 | |
| 65 | dlm_create_root_list(ls); |
| 66 | |
| 67 | /* |
| 68 | * Free all the tossed rsb's so we don't have to recover them. |
| 69 | */ |
| 70 | |
| 71 | dlm_clear_toss_list(ls); |
| 72 | |
| 73 | /* |
| 74 | * Add or remove nodes from the lockspace's ls_nodes list. |
| 75 | * Also waits for all nodes to complete dlm_recover_members. |
| 76 | */ |
| 77 | |
| 78 | error = dlm_recover_members(ls, rv, &neg); |
| 79 | if (error) { |
| 80 | log_error(ls, "recover_members failed %d", error); |
| 81 | goto fail; |
| 82 | } |
| 83 | start = jiffies; |
| 84 | |
| 85 | /* |
| 86 | * Rebuild our own share of the directory by collecting from all other |
| 87 | * nodes their master rsb names that hash to us. |
| 88 | */ |
| 89 | |
| 90 | error = dlm_recover_directory(ls); |
| 91 | if (error) { |
| 92 | log_error(ls, "recover_directory failed %d", error); |
| 93 | goto fail; |
| 94 | } |
| 95 | |
| 96 | /* |
| 97 | * Purge directory-related requests that are saved in requestqueue. |
| 98 | * All dir requests from before recovery are invalid now due to the dir |
| 99 | * rebuild and will be resent by the requesting nodes. |
| 100 | */ |
| 101 | |
| 102 | dlm_purge_requestqueue(ls); |
| 103 | |
| 104 | /* |
| 105 | * Wait for all nodes to complete directory rebuild. |
| 106 | */ |
| 107 | |
| 108 | error = dlm_recover_directory_wait(ls); |
| 109 | if (error) { |
| 110 | log_error(ls, "recover_directory_wait failed %d", error); |
| 111 | goto fail; |
| 112 | } |
| 113 | |
| 114 | /* |
| 115 | * We may have outstanding operations that are waiting for a reply from |
| 116 | * a failed node. Mark these to be resent after recovery. Unlock and |
| 117 | * cancel ops can just be completed. |
| 118 | */ |
| 119 | |
| 120 | dlm_recover_waiters_pre(ls); |
| 121 | |
| 122 | error = dlm_recovery_stopped(ls); |
| 123 | if (error) |
| 124 | goto fail; |
| 125 | |
| 126 | if (neg || dlm_no_directory(ls)) { |
| 127 | /* |
| 128 | * Clear lkb's for departed nodes. |
| 129 | */ |
| 130 | |
| 131 | dlm_purge_locks(ls); |
| 132 | |
| 133 | /* |
| 134 | * Get new master nodeid's for rsb's that were mastered on |
| 135 | * departed nodes. |
| 136 | */ |
| 137 | |
| 138 | error = dlm_recover_masters(ls); |
| 139 | if (error) { |
| 140 | log_error(ls, "recover_masters failed %d", error); |
| 141 | goto fail; |
| 142 | } |
| 143 | |
| 144 | /* |
| 145 | * Send our locks on remastered rsb's to the new masters. |
| 146 | */ |
| 147 | |
| 148 | error = dlm_recover_locks(ls); |
| 149 | if (error) { |
| 150 | log_error(ls, "recover_locks failed %d", error); |
| 151 | goto fail; |
| 152 | } |
| 153 | |
| 154 | error = dlm_recover_locks_wait(ls); |
| 155 | if (error) { |
| 156 | log_error(ls, "recover_locks_wait failed %d", error); |
| 157 | goto fail; |
| 158 | } |
| 159 | |
| 160 | /* |
| 161 | * Finalize state in master rsb's now that all locks can be |
| 162 | * checked. This includes conversion resolution and lvb |
| 163 | * settings. |
| 164 | */ |
| 165 | |
| 166 | dlm_recover_rsbs(ls); |
| 167 | } |
| 168 | |
| 169 | dlm_release_root_list(ls); |
| 170 | |
| 171 | dlm_set_recover_status(ls, DLM_RS_DONE); |
| 172 | error = dlm_recover_done_wait(ls); |
| 173 | if (error) { |
| 174 | log_error(ls, "recover_done_wait failed %d", error); |
| 175 | goto fail; |
| 176 | } |
| 177 | |
| 178 | dlm_clear_members_gone(ls); |
| 179 | |
| 180 | error = enable_locking(ls, rv->seq); |
| 181 | if (error) { |
| 182 | log_error(ls, "enable_locking failed %d", error); |
| 183 | goto fail; |
| 184 | } |
| 185 | |
| 186 | error = dlm_process_requestqueue(ls); |
| 187 | if (error) { |
| 188 | log_error(ls, "process_requestqueue failed %d", error); |
| 189 | goto fail; |
| 190 | } |
| 191 | |
| 192 | error = dlm_recover_waiters_post(ls); |
| 193 | if (error) { |
| 194 | log_error(ls, "recover_waiters_post failed %d", error); |
| 195 | goto fail; |
| 196 | } |
| 197 | |
| 198 | dlm_grant_after_purge(ls); |
| 199 | |
| 200 | dlm_astd_wake(); |
| 201 | |
David Teigland | 9013592 | 2006-01-20 08:47:07 +0000 | [diff] [blame] | 202 | log_debug(ls, "recover %llx done: %u ms", rv->seq, |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 203 | jiffies_to_msecs(jiffies - start)); |
David Teigland | 9013592 | 2006-01-20 08:47:07 +0000 | [diff] [blame] | 204 | mutex_unlock(&ls->ls_recoverd_active); |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 205 | |
| 206 | return 0; |
| 207 | |
| 208 | fail: |
| 209 | dlm_release_root_list(ls); |
David Teigland | 9013592 | 2006-01-20 08:47:07 +0000 | [diff] [blame] | 210 | log_debug(ls, "recover %llx error %d", rv->seq, error); |
| 211 | mutex_unlock(&ls->ls_recoverd_active); |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 212 | return error; |
| 213 | } |
| 214 | |
| 215 | static void do_ls_recovery(struct dlm_ls *ls) |
| 216 | { |
| 217 | struct dlm_recover *rv = NULL; |
| 218 | |
| 219 | spin_lock(&ls->ls_recover_lock); |
| 220 | rv = ls->ls_recover_args; |
| 221 | ls->ls_recover_args = NULL; |
| 222 | clear_bit(LSFL_RECOVERY_STOP, &ls->ls_flags); |
| 223 | spin_unlock(&ls->ls_recover_lock); |
| 224 | |
| 225 | if (rv) { |
| 226 | ls_recover(ls, rv); |
| 227 | kfree(rv->nodeids); |
| 228 | kfree(rv); |
| 229 | } |
| 230 | } |
| 231 | |
| 232 | static int dlm_recoverd(void *arg) |
| 233 | { |
| 234 | struct dlm_ls *ls; |
| 235 | |
| 236 | ls = dlm_find_lockspace_local(arg); |
David Teigland | 5f88f1e | 2006-08-24 14:47:20 -0500 | [diff] [blame] | 237 | if (!ls) { |
| 238 | log_print("dlm_recoverd: no lockspace %p", arg); |
| 239 | return -1; |
| 240 | } |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 241 | |
| 242 | while (!kthread_should_stop()) { |
| 243 | set_current_state(TASK_INTERRUPTIBLE); |
| 244 | if (!test_bit(LSFL_WORK, &ls->ls_flags)) |
| 245 | schedule(); |
| 246 | set_current_state(TASK_RUNNING); |
| 247 | |
| 248 | if (test_and_clear_bit(LSFL_WORK, &ls->ls_flags)) |
| 249 | do_ls_recovery(ls); |
| 250 | } |
| 251 | |
| 252 | dlm_put_lockspace(ls); |
| 253 | return 0; |
| 254 | } |
| 255 | |
| 256 | void dlm_recoverd_kick(struct dlm_ls *ls) |
| 257 | { |
| 258 | set_bit(LSFL_WORK, &ls->ls_flags); |
| 259 | wake_up_process(ls->ls_recoverd_task); |
| 260 | } |
| 261 | |
| 262 | int dlm_recoverd_start(struct dlm_ls *ls) |
| 263 | { |
| 264 | struct task_struct *p; |
| 265 | int error = 0; |
| 266 | |
| 267 | p = kthread_run(dlm_recoverd, ls, "dlm_recoverd"); |
| 268 | if (IS_ERR(p)) |
| 269 | error = PTR_ERR(p); |
| 270 | else |
| 271 | ls->ls_recoverd_task = p; |
| 272 | return error; |
| 273 | } |
| 274 | |
| 275 | void dlm_recoverd_stop(struct dlm_ls *ls) |
| 276 | { |
| 277 | kthread_stop(ls->ls_recoverd_task); |
| 278 | } |
| 279 | |
| 280 | void dlm_recoverd_suspend(struct dlm_ls *ls) |
| 281 | { |
David Teigland | f6db1b8 | 2006-08-08 17:06:07 -0500 | [diff] [blame] | 282 | wake_up(&ls->ls_wait_general); |
David Teigland | 9013592 | 2006-01-20 08:47:07 +0000 | [diff] [blame] | 283 | mutex_lock(&ls->ls_recoverd_active); |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 284 | } |
| 285 | |
| 286 | void dlm_recoverd_resume(struct dlm_ls *ls) |
| 287 | { |
David Teigland | 9013592 | 2006-01-20 08:47:07 +0000 | [diff] [blame] | 288 | mutex_unlock(&ls->ls_recoverd_active); |
David Teigland | e7fd417 | 2006-01-18 09:30:29 +0000 | [diff] [blame] | 289 | } |
| 290 | |