ctdb/server/ctdb_takeover_helper.c

   1 /*
   2    CTDB IP takeover helper
   3
   4    Copyright (C) Martin Schwenke  2016
   5
   6    Based on ctdb_recovery_helper.c
   7    Copyright (C) Amitay Isaacs  2015
   8
   9    and ctdb_takeover.c
  10    Copyright (C) Ronnie Sahlberg  2007
  11    Copyright (C) Andrew Tridgell  2007
  12    Copyright (C) Martin Schwenke  2011
  13
  14    This program is free software; you can redistribute it and/or modify
  15    it under the terms of the GNU General Public License as published by
  16    the Free Software Foundation; either version 3 of the License, or
  17    (at your option) any later version.
  18
  19    This program is distributed in the hope that it will be useful,
  20    but WITHOUT ANY WARRANTY; without even the implied warranty of
  21    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  22    GNU General Public License for more details.
  23
  24    You should have received a copy of the GNU General Public License
  25    along with this program; if not, see <http://www.gnu.org/licenses/>.
  26 */
  27
  28 #include "replace.h"
  29 #include "system/network.h"
  30 #include "system/filesys.h"
  31
  32 #include <popt.h>
  33 #include <talloc.h>
  34 #include <tevent.h>
  35
  36 #include "lib/util/debug.h"
  37 #include "lib/util/strv.h"
  38 #include "lib/util/strv_util.h"
  39 #include "lib/util/sys_rw.h"
  40 #include "lib/util/time.h"
  41 #include "lib/util/tevent_unix.h"
  42
  43 #include "protocol/protocol.h"
  44 #include "protocol/protocol_api.h"
  45 #include "protocol/protocol_util.h"
  46 #include "client/client.h"
  47
  48 #include "common/logging.h"
  49
  50 #include "server/ipalloc.h"
  51
  52 static int takeover_timeout = 9;
  53
  54 #define TIMEOUT()       timeval_current_ofs(takeover_timeout, 0)
  55
  56 /*
  57  * Utility functions
  58  */
  59
  60 static bool generic_recv(struct tevent_req *req, int *perr)
  61 {
  62         int err;
  63
  64         if (tevent_req_is_unix_error(req, &err)) {
  65                 if (perr != NULL) {
  66                         *perr = err;
  67                 }
  68                 return false;
  69         }
  70
  71         return true;
  72 }
  73
  74 static enum ipalloc_algorithm
  75 determine_algorithm(const struct ctdb_tunable_list *tunables)
  76 {
  77         switch (tunables->ip_alloc_algorithm) {
  78         case 0:
  79                 return IPALLOC_DETERMINISTIC;
  80         case 1:
  81                 return IPALLOC_NONDETERMINISTIC;
  82         case 2:
  83                 return IPALLOC_LCP2;
  84         default:
  85                 return IPALLOC_LCP2;
  86         };
  87 }
  88
  89 /**********************************************************************/
  90
  91 struct get_public_ips_state {
  92         uint32_t *pnns;
  93         int count;
  94         struct ctdb_public_ip_list *ips;
  95         uint32_t *ban_credits;
  96 };
  97
  98 static void get_public_ips_done(struct tevent_req *subreq);
  99
 100 static struct tevent_req *get_public_ips_send(
 101                                 TALLOC_CTX *mem_ctx,
 102                                 struct tevent_context *ev,
 103                                 struct ctdb_client_context *client,
 104                                 uint32_t *pnns,
 105                                 int count, int num_nodes,
 106                                 uint32_t *ban_credits,
 107                                 bool available_only)
 108 {
 109         struct tevent_req *req, *subreq;
 110         struct get_public_ips_state *state;
 111         struct ctdb_req_control request;
 112
 113         req = tevent_req_create(mem_ctx, &state, struct get_public_ips_state);
 114         if (req == NULL) {
 115                 return NULL;
 116         }
 117
 118         state->pnns = pnns;
 119         state->count = count;
 120         state->ban_credits = ban_credits;
 121
 122         state->ips  = talloc_zero_array(state,
 123                                         struct ctdb_public_ip_list,
 124                                         num_nodes);
 125         if (tevent_req_nomem(state->ips, req)) {
 126                 return tevent_req_post(req, ev);
 127         }
 128
 129         /* Short circuit if no nodes being asked for IPs */
 130         if (state->count == 0) {
 131                 tevent_req_done(req);
 132                 return tevent_req_post(req, ev);
 133         }
 134
 135         ctdb_req_control_get_public_ips(&request, available_only);
 136         subreq = ctdb_client_control_multi_send(mem_ctx, ev, client,
 137                                                 state->pnns,
 138                                                 state->count,
 139                                                 TIMEOUT(), &request);
 140         if (tevent_req_nomem(subreq, req)) {
 141                 return tevent_req_post(req, ev);
 142         }
 143         tevent_req_set_callback(subreq, get_public_ips_done, req);
 144
 145         return req;
 146 }
 147
 148 static void get_public_ips_done(struct tevent_req *subreq)
 149 {
 150         struct tevent_req *req = tevent_req_callback_data(
 151                 subreq, struct tevent_req);
 152         struct get_public_ips_state *state = tevent_req_data(
 153                 req, struct get_public_ips_state);
 154         struct ctdb_reply_control **reply;
 155         int *err_list;
 156         int ret, i;
 157         bool status, found_errors;
 158
 159         status = ctdb_client_control_multi_recv(subreq, &ret, state, &err_list,
 160                                                 &reply);
 161         TALLOC_FREE(subreq);
 162         if (! status) {
 163                 found_errors = false;
 164                 for (i = 0; i < state->count; i++) {
 165                         if (err_list[i] != 0) {
 166                                 uint32_t pnn = state->pnns[i];
 167
 168                                 D_ERR("control GET_PUBLIC_IPS failed on "
 169                                       "node %u, ret=%d\n", pnn, err_list[i]);
 170
 171                                 state->ban_credits[pnn]++;
 172                                 found_errors = true;
 173                         }
 174                 }
 175
 176                 tevent_req_error(req, ret);
 177                 return;
 178         }
 179
 180         found_errors = false;
 181         for (i = 0; i < state->count; i++) {
 182                 uint32_t pnn;
 183                 struct ctdb_public_ip_list *ips;
 184
 185                 pnn = state->pnns[i];
 186                 ret = ctdb_reply_control_get_public_ips(reply[i], state->ips,
 187                                                         &ips);
 188                 if (ret != 0) {
 189                         D_ERR("control GET_PUBLIC_IPS failed on "
 190                               "node %u\n", pnn);
 191                         state->ban_credits[pnn]++;
 192                         found_errors = true;
 193                         continue;
 194                 }
 195
 196                 D_INFO("Fetched public IPs from node %u\n", pnn);
 197                 state->ips[pnn] = *ips;
 198         }
 199
 200         if (found_errors) {
 201                 tevent_req_error(req, EIO);
 202                 return;
 203         }
 204
 205         talloc_free(reply);
 206
 207         tevent_req_done(req);
 208 }
 209
 210 static bool get_public_ips_recv(struct tevent_req *req, int *perr,
 211                                 TALLOC_CTX *mem_ctx,
 212                                 struct ctdb_public_ip_list **ips)
 213 {
 214         struct get_public_ips_state *state = tevent_req_data(
 215                 req, struct get_public_ips_state);
 216         int err;
 217
 218         if (tevent_req_is_unix_error(req, &err)) {
 219                 if (perr != NULL) {
 220                         *perr = err;
 221                 }
 222                 return false;
 223         }
 224
 225         *ips = talloc_steal(mem_ctx, state->ips);
 226
 227         return true;
 228 }
 229
 230 /**********************************************************************/
 231
 232 struct release_ip_state {
 233         int num_sent;
 234         int num_replies;
 235         int num_fails;
 236         int err_any;
 237         uint32_t *ban_credits;
 238 };
 239
 240 struct release_ip_one_state {
 241         struct tevent_req *req;
 242         uint32_t *pnns;
 243         int count;
 244         const char *ip_str;
 245 };
 246
 247 static void release_ip_done(struct tevent_req *subreq);
 248
 249 static struct tevent_req *release_ip_send(TALLOC_CTX *mem_ctx,
 250                                           struct tevent_context *ev,
 251                                           struct ctdb_client_context *client,
 252                                           uint32_t *pnns,
 253                                           int count,
 254                                           struct timeval timeout,
 255                                           struct public_ip_list *all_ips,
 256                                           uint32_t *ban_credits)
 257 {
 258         struct tevent_req *req, *subreq;
 259         struct release_ip_state *state;
 260         struct ctdb_req_control request;
 261         struct public_ip_list *tmp_ip;
 262
 263         req = tevent_req_create(mem_ctx, &state, struct release_ip_state);
 264         if (req == NULL) {
 265                 return NULL;
 266         }
 267
 268         state->num_sent = 0;
 269         state->num_replies = 0;
 270         state->num_fails = 0;
 271         state->ban_credits = ban_credits;
 272
 273         /* Send a RELEASE_IP to all nodes that should not be hosting
 274          * each IP.  For each IP, all but one of these will be
 275          * redundant.  However, the redundant ones are used to tell
 276          * nodes which node should be hosting the IP so that commands
 277          * like "ctdb ip" can display a particular nodes idea of who
 278          * is hosting what. */
 279         for (tmp_ip = all_ips; tmp_ip != NULL; tmp_ip = tmp_ip->next) {
 280                 struct release_ip_one_state *substate;
 281                 struct ctdb_public_ip ip;
 282                 int i;
 283
 284                 substate = talloc_zero(state, struct release_ip_one_state);
 285                 if (tevent_req_nomem(substate, req)) {
 286                         return tevent_req_post(req, ev);
 287                 }
 288
 289                 substate->pnns = talloc_zero_array(substate, uint32_t, count);
 290                 if (tevent_req_nomem(substate->pnns, req)) {
 291                         return tevent_req_post(req, ev);
 292                 }
 293
 294                 substate->count = 0;
 295                 substate->req = req;
 296
 297                 substate->ip_str  = ctdb_sock_addr_to_string(substate,
 298                                                              &tmp_ip->addr,
 299                                                              false);
 300                 if (tevent_req_nomem(substate->ip_str, req)) {
 301                         return tevent_req_post(req, ev);
 302                 }
 303
 304                 for (i = 0; i < count; i++) {
 305                         uint32_t pnn = pnns[i];
 306
 307                         /* Skip this node if IP is not known */
 308                         if (! bitmap_query(tmp_ip->known_on, pnn)) {
 309                                 continue;
 310                         }
 311
 312                         /* If pnn is not the node that should be
 313                          * hosting the IP then add it to the list of
 314                          * nodes that need to do a release. */
 315                         if (tmp_ip->pnn != pnn) {
 316                                 substate->pnns[substate->count] = pnn;
 317                                 substate->count++;
 318                         }
 319                 }
 320
 321                 if (substate->count == 0) {
 322                         /* No releases to send for this address... */
 323                         TALLOC_FREE(substate);
 324                         continue;
 325                 }
 326
 327                 ip.pnn = tmp_ip->pnn;
 328                 ip.addr = tmp_ip->addr;
 329                 ctdb_req_control_release_ip(&request, &ip);
 330                 subreq = ctdb_client_control_multi_send(state, ev, client,
 331                                                         substate->pnns,
 332                                                         substate->count,
 333                                                         timeout,/* cumulative */
 334                                                         &request);
 335                 if (tevent_req_nomem(subreq, req)) {
 336                         return tevent_req_post(req, ev);
 337                 }
 338                 tevent_req_set_callback(subreq, release_ip_done, substate);
 339
 340                 state->num_sent++;
 341         }
 342
 343         /* None sent, finished... */
 344         if (state->num_sent == 0) {
 345                 tevent_req_done(req);
 346                 return tevent_req_post(req, ev);
 347         }
 348
 349         return req;
 350 }
 351
 352 static void release_ip_done(struct tevent_req *subreq)
 353 {
 354         struct release_ip_one_state *substate = tevent_req_callback_data(
 355                 subreq, struct release_ip_one_state);
 356         struct tevent_req *req = substate->req;
 357         struct release_ip_state *state = tevent_req_data(
 358                 req, struct release_ip_state);
 359         int ret, i;
 360         int *err_list;
 361         bool status, found_errors;
 362
 363         status = ctdb_client_control_multi_recv(subreq, &ret, state,
 364                                                 &err_list, NULL);
 365         TALLOC_FREE(subreq);
 366
 367         if (status) {
 368                 D_INFO("RELEASE_IP %s succeeded on %d nodes\n",
 369                        substate->ip_str, substate->count);
 370                 goto done;
 371         }
 372
 373         /* Get some clear error messages out of err_list and count
 374          * banning credits
 375          */
 376         found_errors = false;
 377         for (i = 0; i < substate->count; i++) {
 378                 int err = err_list[i];
 379                 if (err != 0) {
 380                         uint32_t pnn = substate->pnns[i];
 381
 382                         D_ERR("RELEASE_IP %s failed on node %u, "
 383                               "ret=%d\n", substate->ip_str, pnn, err);
 384
 385                         state->ban_credits[pnn]++;
 386                         state->err_any = err;
 387                         found_errors = true;
 388                 }
 389         }
 390         if (! found_errors) {
 391                 D_ERR("RELEASE_IP %s internal error, ret=%d\n",
 392                       substate->ip_str, ret);
 393                 state->err_any = EIO;
 394         }
 395
 396         state->num_fails++;
 397
 398 done:
 399         talloc_free(substate);
 400
 401         state->num_replies++;
 402
 403         if (state->num_replies < state->num_sent) {
 404                 /* Not all replies received, don't go further */
 405                 return;
 406         }
 407
 408         if (state->num_fails > 0) {
 409                 tevent_req_error(req, state->err_any);
 410                 return;
 411         }
 412
 413         tevent_req_done(req);
 414 }
 415
 416 static bool release_ip_recv(struct tevent_req *req, int *perr)
 417 {
 418         return generic_recv(req, perr);
 419 }
 420
 421 /**********************************************************************/
 422
 423 struct take_ip_state {
 424         int num_sent;
 425         int num_replies;
 426         int num_fails;
 427         int err_any;
 428         uint32_t *ban_credits;
 429 };
 430
 431 struct take_ip_one_state {
 432         struct tevent_req *req;
 433         uint32_t pnn;
 434         const char *ip_str;
 435 };
 436
 437 static void take_ip_done(struct tevent_req *subreq);
 438
 439 static struct tevent_req *take_ip_send(TALLOC_CTX *mem_ctx,
 440                                        struct tevent_context *ev,
 441                                        struct ctdb_client_context *client,
 442                                        struct timeval timeout,
 443                                        struct public_ip_list *all_ips,
 444                                        uint32_t *ban_credits)
 445 {
 446         struct tevent_req *req, *subreq;
 447         struct take_ip_state *state;
 448         struct ctdb_req_control request;
 449         struct public_ip_list *tmp_ip;
 450
 451         req = tevent_req_create(mem_ctx, &state, struct take_ip_state);
 452         if (req == NULL) {
 453                 return NULL;
 454         }
 455
 456         state->num_sent = 0;
 457         state->num_replies = 0;
 458         state->num_fails = 0;
 459         state->ban_credits = ban_credits;
 460
 461         /* For each IP, send a TAKOVER_IP to the node that should be
 462          * hosting it.  Many of these will often be redundant (since
 463          * the allocation won't have changed) but they can be useful
 464          * to recover from inconsistencies. */
 465         for (tmp_ip = all_ips; tmp_ip != NULL; tmp_ip = tmp_ip->next) {
 466                 struct take_ip_one_state *substate;
 467                 struct ctdb_public_ip ip;
 468
 469                 if (tmp_ip->pnn == -1) {
 470                         /* IP will be unassigned */
 471                         continue;
 472                 }
 473
 474                 substate = talloc_zero(state, struct take_ip_one_state);
 475                 if (tevent_req_nomem(substate, req)) {
 476                         return tevent_req_post(req, ev);
 477                 }
 478
 479                 substate->req = req;
 480                 substate->pnn = tmp_ip->pnn;
 481
 482                 substate->ip_str  = ctdb_sock_addr_to_string(substate,
 483                                                              &tmp_ip->addr,
 484                                                              false);
 485                 if (tevent_req_nomem(substate->ip_str, req)) {
 486                         return tevent_req_post(req, ev);
 487                 }
 488
 489                 ip.pnn = tmp_ip->pnn;
 490                 ip.addr = tmp_ip->addr;
 491                 ctdb_req_control_takeover_ip(&request, &ip);
 492                 subreq = ctdb_client_control_send(
 493                                         state, ev, client, tmp_ip->pnn,
 494                                         timeout, /* cumulative */
 495                                         &request);
 496                 if (tevent_req_nomem(subreq, req)) {
 497                         return tevent_req_post(req, ev);
 498                 }
 499                 tevent_req_set_callback(subreq, take_ip_done, substate);
 500
 501                 state->num_sent++;
 502         }
 503
 504         /* None sent, finished... */
 505         if (state->num_sent == 0) {
 506                 tevent_req_done(req);
 507                 return tevent_req_post(req, ev);
 508         }
 509
 510         return req;
 511 }
 512
 513 static void take_ip_done(struct tevent_req *subreq)
 514 {
 515         struct take_ip_one_state *substate = tevent_req_callback_data(
 516                 subreq, struct take_ip_one_state);
 517         struct tevent_req *req = substate->req;
 518         struct ctdb_reply_control *reply;
 519         struct take_ip_state *state = tevent_req_data(
 520                 req, struct take_ip_state);
 521         int ret = 0;
 522         bool status;
 523
 524         status = ctdb_client_control_recv(subreq, &ret, state, &reply);
 525         TALLOC_FREE(subreq);
 526
 527         if (! status) {
 528                 D_ERR("TAKEOVER_IP %s failed to node %u, ret=%d\n",
 529                       substate->ip_str, substate->pnn, ret);
 530                 goto fail;
 531         }
 532
 533         ret = ctdb_reply_control_takeover_ip(reply);
 534         if (ret != 0) {
 535                 D_ERR("TAKEOVER_IP %s failed on node %u, ret=%d\n",
 536                       substate->ip_str, substate->pnn, ret);
 537                 goto fail;
 538         }
 539
 540         D_INFO("TAKEOVER_IP %s succeeded on node %u\n",
 541                substate->ip_str, substate->pnn);
 542         goto done;
 543
 544 fail:
 545         state->ban_credits[substate->pnn]++;
 546         state->num_fails++;
 547         state->err_any = ret;
 548
 549 done:
 550         talloc_free(substate);
 551
 552         state->num_replies++;
 553
 554         if (state->num_replies < state->num_sent) {
 555                 /* Not all replies received, don't go further */
 556                 return;
 557         }
 558
 559         if (state->num_fails > 0) {
 560                 tevent_req_error(req, state->err_any);
 561                 return;
 562         }
 563
 564         tevent_req_done(req);
 565 }
 566
 567 static bool take_ip_recv(struct tevent_req *req, int *perr)
 568 {
 569         return generic_recv(req, perr);
 570 }
 571
 572 /**********************************************************************/
 573
 574 struct ipreallocated_state {
 575         uint32_t *pnns;
 576         int count;
 577         uint32_t *ban_credits;
 578 };
 579
 580 static void ipreallocated_done(struct tevent_req *subreq);
 581
 582 static struct tevent_req *ipreallocated_send(TALLOC_CTX *mem_ctx,
 583                                              struct tevent_context *ev,
 584                                              struct ctdb_client_context *client,
 585                                              uint32_t *pnns,
 586                                              int count,
 587                                              struct timeval timeout,
 588                                              uint32_t *ban_credits)
 589 {
 590         struct tevent_req *req, *subreq;
 591         struct ipreallocated_state *state;
 592         struct ctdb_req_control request;
 593
 594         req = tevent_req_create(mem_ctx, &state, struct ipreallocated_state);
 595         if (req == NULL) {
 596                 return NULL;
 597         }
 598
 599         state->pnns = pnns;
 600         state->count = count;
 601         state->ban_credits = ban_credits;
 602
 603         ctdb_req_control_ipreallocated(&request);
 604         subreq = ctdb_client_control_multi_send(state, ev, client,
 605                                                 pnns, count,
 606                                                 timeout, /* cumulative */
 607                                                 &request);
 608         if (tevent_req_nomem(subreq, req)) {
 609                 return tevent_req_post(req, ev);
 610         }
 611         tevent_req_set_callback(subreq, ipreallocated_done, req);
 612
 613         return req;
 614 }
 615
 616 static void ipreallocated_done(struct tevent_req *subreq)
 617 {
 618         struct tevent_req *req = tevent_req_callback_data(
 619                 subreq, struct tevent_req);
 620         struct ipreallocated_state *state = tevent_req_data(
 621                 req, struct ipreallocated_state);
 622         int *err_list = NULL;
 623         int ret, i;
 624         bool status, found_errors;
 625
 626         status = ctdb_client_control_multi_recv(subreq, &ret, state,
 627                                                 &err_list, NULL);
 628         TALLOC_FREE(subreq);
 629
 630         if (status) {
 631                 D_INFO("IPREALLOCATED succeeded on %d nodes\n", state->count);
 632                 tevent_req_done(req);
 633                 return;
 634         }
 635
 636         /* Get some clear error messages out of err_list and count
 637          * banning credits
 638          */
 639         found_errors = false;
 640         for (i = 0; i < state->count; i++) {
 641                 int err = err_list[i];
 642                 if (err != 0) {
 643                         uint32_t pnn = state->pnns[i];
 644
 645                         D_ERR("IPREALLOCATED failed on node %u, ret=%d\n",
 646                               pnn, err);
 647
 648                         state->ban_credits[pnn]++;
 649                         found_errors = true;
 650                 }
 651         }
 652
 653         if (! found_errors) {
 654                 D_ERR("IPREALLOCATED internal error, ret=%d\n", ret);
 655         }
 656
 657         tevent_req_error(req, ret);
 658 }
 659
 660 static bool ipreallocated_recv(struct tevent_req *req, int *perr)
 661 {
 662         return generic_recv(req, perr);
 663 }
 664
 665 /**********************************************************************/
 666
 667 /*
 668  * Recalculate the allocation of public IPs to nodes and have the
 669  * nodes host their allocated addresses.
 670  *
 671  * - Get tunables
 672  * - Get nodemap
 673  * - Initialise IP allocation state.  Pass:
 674  *   + algorithm to be used;
 675  *   + various tunables (NoIPTakeover, NoIPFailback)
 676  *   + list of nodes to force rebalance (internal structure, currently
 677  *     no way to fetch, only used by LCP2 for nodes that have had new
 678  *     IP addresses added).
 679  * - Set IP flags for IP allocation based on node map
 680  * - Retrieve known and available IP addresses (done separately so
 681  *   values can be faked in unit testing)
 682  * - Use ipalloc_set_public_ips() to set known and available IP
 683  *   addresses for allocation
 684  * - If cluster can't host IP addresses then jump to IPREALLOCATED
 685  * - Run IP allocation algorithm
 686  * - Send RELEASE_IP to all nodes for IPs they should not host
 687  * - Send TAKE_IP to all nodes for IPs they should host
 688  * - Send IPREALLOCATED to all nodes
 689  */
 690
 691 struct takeover_state {
 692         struct tevent_context *ev;
 693         struct ctdb_client_context *client;
 694         struct timeval timeout;
 695         int num_nodes;
 696         uint32_t *pnns_connected;
 697         int num_connected;
 698         uint32_t *pnns_active;
 699         int num_active;
 700         uint32_t destnode;
 701         uint32_t *force_rebalance_nodes;
 702         struct ctdb_tunable_list *tun_list;
 703         struct ipalloc_state *ipalloc_state;
 704         struct ctdb_public_ip_list *known_ips;
 705         struct public_ip_list *all_ips;
 706         uint32_t *ban_credits;
 707 };
 708
 709 static void takeover_tunables_done(struct tevent_req *subreq);
 710 static void takeover_nodemap_done(struct tevent_req *subreq);
 711 static void takeover_known_ips_done(struct tevent_req *subreq);
 712 static void takeover_avail_ips_done(struct tevent_req *subreq);
 713 static void takeover_release_ip_done(struct tevent_req *subreq);
 714 static void takeover_take_ip_done(struct tevent_req *subreq);
 715 static void takeover_ipreallocated(struct tevent_req *req);
 716 static void takeover_ipreallocated_done(struct tevent_req *subreq);
 717 static void takeover_failed(struct tevent_req *subreq, int ret);
 718 static void takeover_failed_done(struct tevent_req *subreq);
 719
 720 static struct tevent_req *takeover_send(TALLOC_CTX *mem_ctx,
 721                                         struct tevent_context *ev,
 722                                         struct ctdb_client_context *client,
 723                                         uint32_t *force_rebalance_nodes)
 724 {
 725         struct tevent_req *req, *subreq;
 726         struct takeover_state *state;
 727         struct ctdb_req_control request;
 728
 729         req = tevent_req_create(mem_ctx, &state, struct takeover_state);
 730         if (req == NULL) {
 731                 return NULL;
 732         }
 733
 734         state->ev = ev;
 735         state->client = client;
 736         state->force_rebalance_nodes = force_rebalance_nodes;
 737         state->destnode = ctdb_client_pnn(client);
 738
 739         ctdb_req_control_get_all_tunables(&request);
 740         subreq = ctdb_client_control_send(state, state->ev, state->client,
 741                                           state->destnode, TIMEOUT(),
 742                                           &request);
 743         if (tevent_req_nomem(subreq, req)) {
 744                 return tevent_req_post(req, ev);
 745         }
 746         tevent_req_set_callback(subreq, takeover_tunables_done, req);
 747
 748         return req;
 749 }
 750
 751 static void takeover_tunables_done(struct tevent_req *subreq)
 752 {
 753         struct tevent_req *req = tevent_req_callback_data(
 754                 subreq, struct tevent_req);
 755         struct takeover_state *state = tevent_req_data(
 756                 req, struct takeover_state);
 757         struct ctdb_reply_control *reply;
 758         struct ctdb_req_control request;
 759         int ret;
 760         bool status;
 761
 762         status = ctdb_client_control_recv(subreq, &ret, state, &reply);
 763         TALLOC_FREE(subreq);
 764         if (! status) {
 765                 D_ERR("control GET_ALL_TUNABLES failed, ret=%d\n", ret);
 766                 tevent_req_error(req, ret);
 767                 return;
 768         }
 769
 770         ret = ctdb_reply_control_get_all_tunables(reply, state,
 771                                                   &state->tun_list);
 772         if (ret != 0) {
 773                 D_ERR("control GET_ALL_TUNABLES failed, ret=%d\n", ret);
 774                 tevent_req_error(req, ret);
 775                 return;
 776         }
 777
 778         talloc_free(reply);
 779
 780         takeover_timeout = state->tun_list->takeover_timeout;
 781
 782         ctdb_req_control_get_nodemap(&request);
 783         subreq = ctdb_client_control_send(state, state->ev, state->client,
 784                                           state->destnode, TIMEOUT(),
 785                                           &request);
 786         if (tevent_req_nomem(subreq, req)) {
 787                 return;
 788         }
 789         tevent_req_set_callback(subreq, takeover_nodemap_done, req);
 790 }
 791
 792 static void takeover_nodemap_done(struct tevent_req *subreq)
 793 {
 794         struct tevent_req *req = tevent_req_callback_data(
 795                 subreq, struct tevent_req);
 796         struct takeover_state *state = tevent_req_data(
 797                 req, struct takeover_state);
 798         struct ctdb_reply_control *reply;
 799         bool status;
 800         int ret;
 801         struct ctdb_node_map *nodemap;
 802         const char *ptr;
 803
 804         status = ctdb_client_control_recv(subreq, &ret, state, &reply);
 805         TALLOC_FREE(subreq);
 806         if (! status) {
 807                 D_ERR("control GET_NODEMAP failed to node %u, ret=%d\n",
 808                         state->destnode, ret);
 809                 tevent_req_error(req, ret);
 810                 return;
 811         }
 812
 813         ret = ctdb_reply_control_get_nodemap(reply, state, &nodemap);
 814         if (ret != 0) {
 815                 D_ERR("control GET_NODEMAP failed, ret=%d\n", ret);
 816                 tevent_req_error(req, ret);
 817                 return;
 818         }
 819
 820         state->num_nodes = nodemap->num;
 821
 822         state->num_connected = list_of_connected_nodes(nodemap,
 823                                                        CTDB_UNKNOWN_PNN, state,
 824                                                        &state->pnns_connected);
 825         if (state->num_connected <= 0) {
 826                 tevent_req_error(req, ENOMEM);
 827                 return;
 828         }
 829
 830         state->num_active = list_of_active_nodes(nodemap,
 831                                                  CTDB_UNKNOWN_PNN, state,
 832                                                  &state->pnns_active);
 833         if (state->num_active <= 0) {
 834                 tevent_req_error(req, ENOMEM);
 835                 return;
 836         }
 837
 838         /* Default timeout for early jump to IPREALLOCATED.  See below
 839          * for explanation of 3 times...
 840          */
 841         state->timeout = timeval_current_ofs(3 * takeover_timeout, 0);
 842
 843         state->ban_credits = talloc_zero_array(state, uint32_t,
 844                                                state->num_nodes);
 845         if (tevent_req_nomem(state->ban_credits, req)) {
 846                 return;
 847         }
 848
 849         ptr = getenv("CTDB_DISABLE_IP_FAILOVER");
 850         if (ptr != NULL) {
 851                 /* IP failover is completely disabled so just send out
 852                  * ipreallocated event.
 853                  */
 854                 takeover_ipreallocated(req);
 855                 return;
 856         }
 857
 858         state->ipalloc_state =
 859                 ipalloc_state_init(
 860                         state, state->num_nodes,
 861                         determine_algorithm(state->tun_list),
 862                         (state->tun_list->no_ip_takeover != 0),
 863                         (state->tun_list->no_ip_failback != 0),
 864                         state->force_rebalance_nodes);
 865         if (tevent_req_nomem(state->ipalloc_state, req)) {
 866                 return;
 867         }
 868
 869         subreq = get_public_ips_send(state, state->ev, state->client,
 870                                      state->pnns_connected, state->num_connected,
 871                                      state->num_nodes, state->ban_credits,
 872                                      false);
 873         if (tevent_req_nomem(subreq, req)) {
 874                 return;
 875         }
 876
 877         tevent_req_set_callback(subreq, takeover_known_ips_done, req);
 878 }
 879
 880 static void takeover_known_ips_done(struct tevent_req *subreq)
 881 {
 882         struct tevent_req *req = tevent_req_callback_data(
 883                 subreq, struct tevent_req);
 884         struct takeover_state *state = tevent_req_data(
 885                 req, struct takeover_state);
 886         int ret;
 887         bool status;
 888         uint32_t *pnns = NULL;
 889         int count, i;
 890
 891         status = get_public_ips_recv(subreq, &ret, state, &state->known_ips);
 892         TALLOC_FREE(subreq);
 893
 894         if (! status) {
 895                 D_ERR("Failed to fetch known public IPs\n");
 896                 takeover_failed(req, ret);
 897                 return;
 898         }
 899
 900         /* Get available IPs from active nodes that actually have known IPs */
 901
 902         pnns = talloc_zero_array(state, uint32_t, state->num_active);
 903         if (tevent_req_nomem(pnns, req)) {
 904                 return;
 905         }
 906
 907         count = 0;
 908         for (i = 0; i < state->num_active; i++) {
 909                 uint32_t pnn = state->pnns_active[i];
 910
 911                 /* If pnn has IPs then fetch available IPs from it */
 912                 if (state->known_ips[pnn].num > 0) {
 913                         pnns[count] = pnn;
 914                         count++;
 915                 }
 916         }
 917
 918         subreq = get_public_ips_send(state, state->ev, state->client,
 919                                      pnns, count,
 920                                      state->num_nodes, state->ban_credits,
 921                                      true);
 922         if (tevent_req_nomem(subreq, req)) {
 923                 return;
 924         }
 925
 926         tevent_req_set_callback(subreq, takeover_avail_ips_done, req);
 927 }
 928
 929 static void takeover_avail_ips_done(struct tevent_req *subreq)
 930 {
 931         struct tevent_req *req = tevent_req_callback_data(
 932                 subreq, struct tevent_req);
 933         struct takeover_state *state = tevent_req_data(
 934                 req, struct takeover_state);
 935         bool status;
 936         int ret;
 937         struct ctdb_public_ip_list *available_ips;
 938
 939         status = get_public_ips_recv(subreq, &ret, state, &available_ips);
 940         TALLOC_FREE(subreq);
 941
 942         if (! status) {
 943                 D_ERR("Failed to fetch available public IPs\n");
 944                 takeover_failed(req, ret);
 945                 return;
 946         }
 947
 948         ipalloc_set_public_ips(state->ipalloc_state,
 949                                state->known_ips, available_ips);
 950
 951         if (! ipalloc_can_host_ips(state->ipalloc_state)) {
 952                 D_NOTICE("No nodes available to host public IPs yet\n");
 953                 takeover_ipreallocated(req);
 954                 return;
 955         }
 956
 957         /* Do the IP reassignment calculations */
 958         state->all_ips = ipalloc(state->ipalloc_state);
 959         if (tevent_req_nomem(state->all_ips, req)) {
 960                 return;
 961         }
 962
 963         /* Each of the following stages (RELEASE_IP, TAKEOVER_IP,
 964          * IPREALLOCATED) notionally has a timeout of TakeoverTimeout
 965          * seconds.  However, RELEASE_IP can take longer due to TCP
 966          * connection killing, so sometimes needs more time.
 967          * Therefore, use a cumulative timeout of TakeoverTimeout * 3
 968          * seconds across all 3 stages.  No explicit expiry checks are
 969          * needed before each stage because tevent is smart enough to
 970          * fire the timeouts even if they are in the past.  Initialise
 971          * this here so it explicitly covers the stages we're
 972          * interested in but, in particular, not the time taken by the
 973          * ipalloc().
 974          */
 975         state->timeout = timeval_current_ofs(3 * takeover_timeout, 0);
 976
 977         subreq = release_ip_send(state, state->ev, state->client,
 978                                  state->pnns_connected, state->num_connected,
 979                                  state->timeout, state->all_ips,
 980                                  state->ban_credits);
 981         if (tevent_req_nomem(subreq, req)) {
 982                 return;
 983         }
 984         tevent_req_set_callback(subreq, takeover_release_ip_done, req);
 985 }
 986
 987 static void takeover_release_ip_done(struct tevent_req *subreq)
 988 {
 989         struct tevent_req *req = tevent_req_callback_data(
 990                 subreq, struct tevent_req);
 991         struct takeover_state *state = tevent_req_data(
 992                 req, struct takeover_state);
 993         int ret;
 994         bool status;
 995
 996         status = release_ip_recv(subreq, &ret);
 997         TALLOC_FREE(subreq);
 998
 999         if (! status) {
1000                 takeover_failed(req, ret);
1001                 return;
1002         }
1003
1004         /* All released, now for takeovers */
1005
1006         subreq = take_ip_send(state, state->ev, state->client,
1007                               state->timeout, state->all_ips,
1008                               state->ban_credits);
1009         if (tevent_req_nomem(subreq, req)) {
1010                 return;
1011         }
1012         tevent_req_set_callback(subreq, takeover_take_ip_done, req);
1013 }
1014
1015 static void takeover_take_ip_done(struct tevent_req *subreq)
1016 {
1017         struct tevent_req *req = tevent_req_callback_data(
1018                 subreq, struct tevent_req);
1019         int ret = 0;
1020         bool status;
1021
1022         status = take_ip_recv(subreq, &ret);
1023         TALLOC_FREE(subreq);
1024
1025         if (! status) {
1026                 takeover_failed(req, ret);
1027                 return;
1028         }
1029
1030         takeover_ipreallocated(req);
1031 }
1032
1033 static void takeover_ipreallocated(struct tevent_req *req)
1034 {
1035         struct takeover_state *state = tevent_req_data(
1036                 req, struct takeover_state);
1037         struct tevent_req *subreq;
1038
1039         subreq = ipreallocated_send(state, state->ev, state->client,
1040                                     state->pnns_connected,
1041                                     state->num_connected,
1042                                     state->timeout,
1043                                     state->ban_credits);
1044         if (tevent_req_nomem(subreq, req)) {
1045                 return;
1046         }
1047         tevent_req_set_callback(subreq, takeover_ipreallocated_done, req);
1048 }
1049
1050 static void takeover_ipreallocated_done(struct tevent_req *subreq)
1051 {
1052         struct tevent_req *req = tevent_req_callback_data(
1053                 subreq, struct tevent_req);
1054         int ret;
1055         bool status;
1056
1057         status = ipreallocated_recv(subreq, &ret);
1058         TALLOC_FREE(subreq);
1059
1060         if (! status) {
1061                 takeover_failed(req, ret);
1062                 return;
1063         }
1064
1065         tevent_req_done(req);
1066 }
1067
1068 struct takeover_failed_state {
1069         struct tevent_req *req;
1070         int ret;
1071 };
1072
1073 void takeover_failed(struct tevent_req *req, int ret)
1074 {
1075         struct takeover_state *state = tevent_req_data(
1076                 req, struct takeover_state);
1077         struct tevent_req *subreq;
1078         uint32_t max_pnn = CTDB_UNKNOWN_PNN;
1079         int max_credits = 0;
1080         int pnn;
1081
1082         /* Check that bans are enabled */
1083         if (state->tun_list->enable_bans == 0) {
1084                 tevent_req_error(req, ret);
1085                 return;
1086         }
1087
1088         for (pnn = 0; pnn < state->num_nodes; pnn++) {
1089                 if (state->ban_credits[pnn] > max_credits) {
1090                         max_pnn = pnn;
1091                         max_credits = state->ban_credits[pnn];
1092                 }
1093         }
1094
1095         if (max_credits > 0) {
1096                 struct ctdb_req_message message;
1097                 struct takeover_failed_state *substate;
1098
1099                 D_WARNING("Assigning banning credits to node %u\n", max_pnn);
1100
1101                 substate = talloc_zero(state, struct takeover_failed_state);
1102                 if (tevent_req_nomem(substate, req)) {
1103                         return;
1104                 }
1105                 substate->req = req;
1106                 substate->ret = ret;
1107
1108                 message.srvid = CTDB_SRVID_BANNING;
1109                 message.data.pnn = max_pnn;
1110
1111                 subreq = ctdb_client_message_send(
1112                         state, state->ev, state->client,
1113                         ctdb_client_pnn(state->client),
1114                         &message);
1115                 if (subreq == NULL) {
1116                         D_ERR("failed to assign banning credits\n");
1117                         tevent_req_error(req, ret);
1118                         return;
1119                 }
1120                 tevent_req_set_callback(subreq, takeover_failed_done, substate);
1121         } else {
1122                 tevent_req_error(req, ret);
1123         }
1124 }
1125
1126 static void takeover_failed_done(struct tevent_req *subreq)
1127 {
1128         struct takeover_failed_state *substate = tevent_req_callback_data(
1129                 subreq, struct takeover_failed_state);
1130         struct tevent_req *req = substate->req;
1131         int ret;
1132         bool status;
1133
1134         status = ctdb_client_message_recv(subreq, &ret);
1135         TALLOC_FREE(subreq);
1136         if (! status) {
1137                 D_ERR("failed to assign banning credits, ret=%d\n", ret);
1138         }
1139
1140         ret = substate->ret;
1141         talloc_free(substate);
1142         tevent_req_error(req, ret);
1143 }
1144
1145 static void takeover_recv(struct tevent_req *req, int *perr)
1146 {
1147         generic_recv(req, perr);
1148 }
1149
1150 static uint32_t *parse_node_list(TALLOC_CTX *mem_ctx, const char* s)
1151 {
1152         char *strv = NULL;
1153         int num, i, ret;
1154         char *t;
1155         uint32_t *nodes;
1156
1157         ret = strv_split(mem_ctx, &strv, s, ",");
1158         if (ret != 0) {
1159                 D_ERR("out of memory\n");
1160                 return NULL;
1161         }
1162
1163         num = strv_count(strv);
1164
1165         nodes = talloc_array(mem_ctx, uint32_t, num);
1166         if (nodes == NULL) {
1167                 D_ERR("out of memory\n");
1168                 return NULL;
1169         }
1170
1171         t = NULL;
1172         for (i = 0; i < num; i++) {
1173                 t = strv_next(strv, t);
1174                 nodes[i] = atoi(t);
1175         }
1176
1177         return nodes;
1178 }
1179
1180 static void usage(const char *progname)
1181 {
1182         fprintf(stderr,
1183                 "\nUsage: %s <output-fd> <ctdb-socket-path> "
1184                 "[<force-rebalance-nodes>]\n",
1185                 progname);
1186 }
1187
1188 /*
1189  * Arguments - write fd, socket path
1190  */
1191 int main(int argc, const char *argv[])
1192 {
1193         int write_fd;
1194         const char *sockpath;
1195         TALLOC_CTX *mem_ctx;
1196         struct tevent_context *ev;
1197         struct ctdb_client_context *client;
1198         int ret;
1199         struct tevent_req *req;
1200         uint32_t *force_rebalance_nodes = NULL;
1201
1202         if (argc < 3 || argc > 4) {
1203                 usage(argv[0]);
1204                 exit(1);
1205         }
1206
1207         write_fd = atoi(argv[1]);
1208         sockpath = argv[2];
1209
1210         mem_ctx = talloc_new(NULL);
1211         if (mem_ctx == NULL) {
1212                 fprintf(stderr, "talloc_new() failed\n");
1213                 ret = ENOMEM;
1214                 goto done;
1215         }
1216
1217         if (argc == 4) {
1218                 force_rebalance_nodes = parse_node_list(mem_ctx, argv[3]);
1219                 if (force_rebalance_nodes == NULL) {
1220                         usage(argv[0]);
1221                         ret = EINVAL;
1222                         goto done;
1223                 }
1224         }
1225
1226         ret = logging_init(mem_ctx, NULL, NULL, "ctdb-takeover");
1227         if (ret != 0) {
1228                 fprintf(stderr,
1229                         "ctdb-takeover: Unable to initialize logging\n");
1230                 goto done;
1231         }
1232
1233         ev = tevent_context_init(mem_ctx);
1234         if (ev == NULL) {
1235                 D_ERR("tevent_context_init() failed\n");
1236                 ret = ENOMEM;
1237                 goto done;
1238         }
1239
1240         ret = ctdb_client_init(mem_ctx, ev, sockpath, &client);
1241         if (ret != 0) {
1242                 D_ERR("ctdb_client_init() failed, ret=%d\n", ret);
1243                 goto done;
1244         }
1245
1246         req = takeover_send(mem_ctx, ev, client, force_rebalance_nodes);
1247         if (req == NULL) {
1248                 D_ERR("takeover_send() failed\n");
1249                 ret = 1;
1250                 goto done;
1251         }
1252
1253         if (! tevent_req_poll(req, ev)) {
1254                 D_ERR("tevent_req_poll() failed\n");
1255                 ret = 1;
1256                 goto done;
1257         }
1258
1259         takeover_recv(req, &ret);
1260         TALLOC_FREE(req);
1261         if (ret != 0) {
1262                 D_ERR("takeover run failed, ret=%d\n", ret);
1263         }
1264
1265 done:
1266         sys_write_v(write_fd, &ret, sizeof(ret));
1267
1268         talloc_free(mem_ctx);
1269         return ret;
1270 }