- merge from ronnie

- add a flag to check that recovery completed correctly. If not, re-trigger it in monitoring

(This used to be ctdb commit d5ed941d9bab4af30d8b5f9b77bdf43d9218d69b)

diff --cc ctdb/server/ctdb_recoverd.c
index 82d6f14a4b887617d773b0869587e40313541b2f,c39cfda68d1a6b4ecf9d18b61c10a11bbd303b08..b0f36679d97b53f0f780f017ce0e7aeafa36bd08
--- 1/ctdb/server/ctdb_recoverd.c
--- 2/ctdb/server/ctdb_recoverd.c
+++ b/ctdb/server/ctdb_recoverd.c
@@@ -43,6 -43,6 +43,7 @@@ struct ctdb_recoverd 
        struct ban_state **banned_nodes;
        struct timeval priority_time;
        bool need_takeover_run;
++      bool need_recovery;
  };
  
  #define CONTROL_TIMEOUT() timeval_current_ofs(ctdb->tunable.recover_timeout, 0)
@@@ -731,6 -731,6 +732,9 @@@ static int do_recovery(struct ctdb_reco
        uint32_t generation;
        struct ctdb_dbid_map *dbmap;
  
++      /* if recovery fails, force it again */
++      rec->need_recovery = true;
++
        if (rec->last_culprit != culprit ||
            timeval_elapsed(&rec->first_recover_time) > ctdb->tunable.recovery_grace_period) {
                /* either a new node is the culprit, or we've decide to forgive them */
@@@ -928,6 -929,6 +932,8 @@@
  
        DEBUG(0, (__location__ " Recovery complete\n"));
  
++      rec->need_recovery = false;
++
        /* We just finished a recovery successfully. 
           We now wait for rerecovery_timeout before we allow 
           another recovery to take place.
@@@ -1576,6 -1577,6 +1582,12 @@@ again
        }
  
  
++      if (rec->need_recovery) {
++              /* a previous recovery didn't finish */
++              do_recovery(rec, mem_ctx, pnn, num_active, nodemap, vnnmap, nodemap->nodes[j].pnn);
++              goto again;             
++      }
++
        /* verify that all active nodes are in normal mode 
           and not in recovery mode 
         */