exec/totemrrp.c

   1 /*
   2  * Copyright (c) 2005 MontaVista Software, Inc.
   3  * Copyright (c) 2006 Red Hat, Inc.
   4  * Copyright (c) 2006 Sun Microsystems, Inc.
   5  *
   6  * All rights reserved.
   7  *
   8  * Author: Steven Dake (sdake@mvista.com)
   9  *
  10  * This software licensed under BSD license, the text of which follows:
  11  *
  12  * Redistribution and use in source and binary forms, with or without
  13  * modification, are permitted provided that the following conditions are met:
  14  *
  15  * - Redistributions of source code must retain the above copyright notice,
  16  *   this list of conditions and the following disclaimer.
  17  * - Redistributions in binary form must reproduce the above copyright notice,
  18  *   this list of conditions and the following disclaimer in the documentation
  19  *   and/or other materials provided with the distribution.
  20  * - Neither the name of the MontaVista Software, Inc. nor the names of its
  21  *   contributors may be used to endorse or promote products derived from this
  22  *   software without specific prior written permission.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  25  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  26  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  27  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  28  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  29  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  30  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  31  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  32  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  33  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  34  * THE POSSIBILITY OF SUCH DAMAGE.
  35  */
  36
  37 #include <assert.h>
  38 #include <pthread.h>
  39 #include <sys/mman.h>
  40 #include <sys/types.h>
  41 #include <sys/stat.h>
  42 #include <sys/socket.h>
  43 #include <netdb.h>
  44 #include <sys/un.h>
  45 #include <sys/ioctl.h>
  46 #include <sys/param.h>
  47 #include <netinet/in.h>
  48 #include <arpa/inet.h>
  49 #include <unistd.h>
  50 #include <fcntl.h>
  51 #include <stdlib.h>
  52 #include <stdio.h>
  53 #include <errno.h>
  54 #include <signal.h>
  55 #include <sched.h>
  56 #include <time.h>
  57 #include <sys/time.h>
  58 #include <sys/poll.h>
  59
  60 #include "../include/queue.h"
  61 #include "../include/sq.h"
  62 #include "../include/list.h"
  63 #include "../include/hdb.h"
  64 #include "swab.h"
  65 #include "aispoll.h"
  66 #include "totemnet.h"
  67 #include "totemrrp.h"
  68
  69 struct totemrrp_instance;
  70 struct passive_instance {
  71         struct totemrrp_instance *rrp_instance;
  72         unsigned int *faulty;
  73         unsigned int *token_recv_count;
  74         unsigned int *mcast_recv_count;
  75         unsigned char token[15000];
  76         unsigned int token_len;
  77         poll_timer_handle timer_expired_token;
  78         poll_timer_handle timer_problem_decrementer;
  79         void *totemrrp_context;
  80         unsigned int token_xmit_iface;
  81         unsigned int msg_xmit_iface;
  82 };
  83
  84 struct active_instance {
  85         struct totemrrp_instance *rrp_instance;
  86         unsigned int *faulty;
  87         unsigned int *last_token_recv;
  88         unsigned int *counter_problems;
  89         unsigned char token[15000];
  90         unsigned int token_len;
  91         unsigned int last_token_seq;
  92         poll_timer_handle timer_expired_token;
  93         poll_timer_handle timer_problem_decrementer;
  94         void *totemrrp_context;
  95 };
  96
  97 struct rrp_algo {
  98         char *name;
  99
 100         void * (*initialize) (
 101                 struct totemrrp_instance *rrp_instance,
 102                 int interface_count);
 103
 104         void (*mcast_recv) (
 105                 struct totemrrp_instance *instance,
 106                 unsigned int iface_no,
 107                 void *context,
 108                 void *msg,
 109                 unsigned int msg_len);
 110
 111         void (*mcast_noflush_send) (
 112                 struct totemrrp_instance *instance,
 113                 struct iovec *iovec,
 114                 unsigned int iov_len);
 115
 116         void (*mcast_flush_send) (
 117                 struct totemrrp_instance *instance,
 118                 struct iovec *iovec,
 119                 unsigned int iov_len);
 120
 121         void (*token_recv) (
 122                 struct totemrrp_instance *instance,
 123                 unsigned int iface_no,
 124                 void *context,
 125                 void *msg,
 126                 unsigned int msg_len,
 127                 unsigned int token_seqid);
 128
 129         void (*token_send) (
 130                 struct totemrrp_instance *instance,
 131                 struct iovec *iovec,
 132                 unsigned int iov_len);
 133
 134         void (*recv_flush) (
 135                 struct totemrrp_instance *instance);
 136
 137         void (*send_flush) (
 138                 struct totemrrp_instance *instance);
 139
 140         void (*iface_check) (
 141                 struct totemrrp_instance *instance);
 142
 143         void (*processor_count_set) (
 144                 struct totemrrp_instance *instance,
 145                 unsigned int processor_count);
 146
 147         void (*token_target_set) (
 148                 struct totemrrp_instance *instance,
 149                 struct totem_ip_address *token_target,
 150                 unsigned int iface_no);
 151
 152         void (*ring_reenable) (
 153                 struct totemrrp_instance *instance);
 154 };
 155
 156 struct totemrrp_instance {
 157         poll_handle totemrrp_poll_handle;
 158
 159         struct totem_interface *interfaces;
 160
 161         struct rrp_algo *rrp_algo;
 162
 163         void *context;
 164
 165         char *status[INTERFACE_MAX];
 166
 167         void (*totemrrp_deliver_fn) (
 168                 void *context,
 169                 void *msg,
 170                 int msg_len);
 171
 172         void (*totemrrp_iface_change_fn) (
 173                 void *context,
 174                 struct totem_ip_address *iface_addr,
 175                 unsigned int iface_no);
 176
 177         void (*totemrrp_token_seqid_get) (
 178                 void *msg,
 179                 unsigned int *seqid,
 180                 unsigned int *token_is);
 181
 182         unsigned int (*totemrrp_msgs_missing) (void);
 183
 184         /*
 185          * Function and data used to log messages
 186          */
 187         int totemrrp_log_level_security;
 188
 189         int totemrrp_log_level_error;
 190
 191         int totemrrp_log_level_warning;
 192
 193         int totemrrp_log_level_notice;
 194
 195         int totemrrp_log_level_debug;
 196
 197         void (*totemrrp_log_printf) (char *file, int line, int level, char *format, ...) __attribute__((format(printf, 4, 5)));
 198
 199         totemrrp_handle handle;
 200
 201         totemnet_handle *net_handles;
 202
 203         void *rrp_algo_instance;
 204
 205         int interface_count;
 206
 207         int poll_handle;
 208
 209         int processor_count;
 210
 211         struct totem_config *totem_config;
 212 };
 213
 214 /*
 215  * None Replication Forward Declerations
 216  */
 217 static void none_mcast_recv (
 218         struct totemrrp_instance *instance,
 219         unsigned int iface_no,
 220         void *context,
 221         void *msg,
 222         unsigned int msg_len);
 223
 224 static void none_mcast_noflush_send (
 225         struct totemrrp_instance *instance,
 226         struct iovec *iovec,
 227         unsigned int iov_len);
 228
 229 static void none_mcast_flush_send (
 230         struct totemrrp_instance *instance,
 231         struct iovec *iovec,
 232         unsigned int iov_len);
 233
 234 static void none_token_recv (
 235         struct totemrrp_instance *instance,
 236         unsigned int iface_no,
 237         void *context,
 238         void *msg,
 239         unsigned int msg_len,
 240         unsigned int token_seqid);
 241
 242 static void none_token_send (
 243         struct totemrrp_instance *instance,
 244         struct iovec *iovec,
 245         unsigned int iov_len);
 246
 247 static void none_recv_flush (
 248         struct totemrrp_instance *instance);
 249
 250 static void none_send_flush (
 251         struct totemrrp_instance *instance);
 252
 253 static void none_iface_check (
 254         struct totemrrp_instance *instance);
 255
 256 static void none_processor_count_set (
 257         struct totemrrp_instance *instance,
 258         unsigned int processor_count_set);
 259
 260 static void none_token_target_set (
 261         struct totemrrp_instance *instance,
 262         struct totem_ip_address *token_target,
 263         unsigned int iface_no);
 264
 265 static void none_ring_reenable (
 266         struct totemrrp_instance *instance);
 267
 268 /*
 269  * Passive Replication Forward Declerations
 270  */
 271 static void *passive_instance_initialize (
 272         struct totemrrp_instance *rrp_instance,
 273         int interface_count);
 274
 275 static void passive_mcast_recv (
 276         struct totemrrp_instance *instance,
 277         unsigned int iface_no,
 278         void *context,
 279         void *msg,
 280         unsigned int msg_len);
 281
 282 static void passive_mcast_noflush_send (
 283         struct totemrrp_instance *instance,
 284         struct iovec *iovec,
 285         unsigned int iov_len);
 286
 287 static void passive_mcast_flush_send (
 288         struct totemrrp_instance *instance,
 289         struct iovec *iovec,
 290         unsigned int iov_len);
 291
 292 static void passive_token_recv (
 293         struct totemrrp_instance *instance,
 294         unsigned int iface_no,
 295         void *context,
 296         void *msg,
 297         unsigned int msg_len,
 298         unsigned int token_seqid);
 299
 300 static void passive_token_send (
 301         struct totemrrp_instance *instance,
 302         struct iovec *iovec,
 303         unsigned int iov_len);
 304
 305 static void passive_recv_flush (
 306         struct totemrrp_instance *instance);
 307
 308 static void passive_send_flush (
 309         struct totemrrp_instance *instance);
 310
 311 static void passive_iface_check (
 312         struct totemrrp_instance *instance);
 313
 314 static void passive_processor_count_set (
 315         struct totemrrp_instance *instance,
 316         unsigned int processor_count_set);
 317
 318 static void passive_token_target_set (
 319         struct totemrrp_instance *instance,
 320         struct totem_ip_address *token_target,
 321         unsigned int iface_no);
 322
 323 static void passive_ring_reenable (
 324         struct totemrrp_instance *instance);
 325
 326 /*
 327  * Active Replication Forward Definitions
 328  */
 329 static void *active_instance_initialize (
 330         struct totemrrp_instance *rrp_instance,
 331         int interface_count);
 332
 333 static void active_mcast_recv (
 334         struct totemrrp_instance *instance,
 335         unsigned int iface_no,
 336         void *context,
 337         void *msg,
 338         unsigned int msg_len);
 339
 340 static void active_mcast_noflush_send (
 341         struct totemrrp_instance *instance,
 342         struct iovec *iovec,
 343         unsigned int iov_len);
 344
 345 static void active_mcast_flush_send (
 346         struct totemrrp_instance *instance,
 347         struct iovec *iovec,
 348         unsigned int iov_len);
 349
 350 static void active_token_recv (
 351         struct totemrrp_instance *instance,
 352         unsigned int iface_no,
 353         void *context,
 354         void *msg,
 355         unsigned int msg_len,
 356         unsigned int token_seqid);
 357
 358 static void active_token_send (
 359         struct totemrrp_instance *instance,
 360         struct iovec *iovec,
 361         unsigned int iov_len);
 362
 363 static void active_recv_flush (
 364         struct totemrrp_instance *instance);
 365
 366 static void active_send_flush (
 367         struct totemrrp_instance *instance);
 368
 369 static void active_iface_check (
 370         struct totemrrp_instance *instance);
 371
 372 static void active_processor_count_set (
 373         struct totemrrp_instance *instance,
 374         unsigned int processor_count_set);
 375
 376 static void active_token_target_set (
 377         struct totemrrp_instance *instance,
 378         struct totem_ip_address *token_target,
 379         unsigned int iface_no);
 380
 381 static void active_ring_reenable (
 382         struct totemrrp_instance *instance);
 383
 384 static void active_timer_expired_token_start (
 385         struct active_instance *active_instance);
 386
 387 static void active_timer_expired_token_cancel (
 388         struct active_instance *active_instance);
 389
 390 static void active_timer_problem_decrementer_start (
 391         struct active_instance *active_instance);
 392
 393 static void active_timer_problem_decrementer_cancel (
 394         struct active_instance *active_instance);
 395
 396 struct rrp_algo none_algo = {
 397         .name                   = "none",
 398         .initialize             = NULL,
 399         .mcast_recv             = none_mcast_recv,
 400         .mcast_noflush_send     = none_mcast_noflush_send,
 401         .mcast_flush_send       = none_mcast_flush_send,
 402         .token_recv             = none_token_recv,
 403         .token_send             = none_token_send,
 404         .recv_flush             = none_recv_flush,
 405         .send_flush             = none_send_flush,
 406         .iface_check            = none_iface_check,
 407         .processor_count_set    = none_processor_count_set,
 408         .token_target_set       = none_token_target_set,
 409         .ring_reenable          = none_ring_reenable
 410 };
 411
 412 struct rrp_algo passive_algo = {
 413         .name                   = "passive",
 414         .initialize             = passive_instance_initialize,
 415         .mcast_recv             = passive_mcast_recv,
 416         .mcast_noflush_send     = passive_mcast_noflush_send,
 417         .mcast_flush_send       = passive_mcast_flush_send,
 418         .token_recv             = passive_token_recv,
 419         .token_send             = passive_token_send,
 420         .recv_flush             = passive_recv_flush,
 421         .send_flush             = passive_send_flush,
 422         .iface_check            = passive_iface_check,
 423         .processor_count_set    = passive_processor_count_set,
 424         .token_target_set       = passive_token_target_set,
 425         .ring_reenable          = passive_ring_reenable
 426 };
 427
 428 struct rrp_algo active_algo = {
 429         .name                   = "active",
 430         .initialize             = active_instance_initialize,
 431         .mcast_recv             = active_mcast_recv,
 432         .mcast_noflush_send     = active_mcast_noflush_send,
 433         .mcast_flush_send       = active_mcast_flush_send,
 434         .token_recv             = active_token_recv,
 435         .token_send             = active_token_send,
 436         .recv_flush             = active_recv_flush,
 437         .send_flush             = active_send_flush,
 438         .iface_check            = active_iface_check,
 439         .processor_count_set    = active_processor_count_set,
 440         .token_target_set       = active_token_target_set,
 441         .ring_reenable          = active_ring_reenable
 442 };
 443
 444 struct rrp_algo *rrp_algos[] = {
 445         &none_algo,
 446         &passive_algo,
 447         &active_algo
 448 };
 449
 450 #define RRP_ALGOS_COUNT 3
 451
 452 /*
 453  * All instances in one database
 454  */
 455 static struct hdb_handle_database totemrrp_instance_database = {
 456         .handle_count   = 0,
 457         .handles        = 0,
 458         .iterator       = 0,
 459         .mutex          = PTHREAD_MUTEX_INITIALIZER
 460 };
 461
 462 #define log_printf(level, format, args...) \
 463     rrp_instance->totemrrp_log_printf (__FILE__, __LINE__, level, format, ##args)
 464
 465 /*
 466  * None Replication Implementation
 467  */
 468
 469 static void none_mcast_recv (
 470         struct totemrrp_instance *rrp_instance,
 471         unsigned int iface_no,
 472         void *context,
 473         void *msg,
 474         unsigned int msg_len)
 475 {
 476         rrp_instance->totemrrp_deliver_fn (
 477                 context,
 478                 msg,
 479                 msg_len);
 480 }
 481
 482 static void none_mcast_flush_send (
 483         struct totemrrp_instance *instance,
 484         struct iovec *iovec,
 485         unsigned int iov_len)
 486 {
 487         totemnet_mcast_flush_send (instance->net_handles[0], iovec, iov_len);
 488 }
 489
 490 static void none_mcast_noflush_send (
 491         struct totemrrp_instance *instance,
 492         struct iovec *iovec,
 493         unsigned int iov_len)
 494 {
 495         totemnet_mcast_noflush_send (instance->net_handles[0], iovec, iov_len);
 496 }
 497
 498 static void none_token_recv (
 499         struct totemrrp_instance *rrp_instance,
 500         unsigned int iface_no,
 501         void *context,
 502         void *msg,
 503         unsigned int msg_len,
 504         unsigned int token_seq)
 505 {
 506         rrp_instance->totemrrp_deliver_fn (
 507                 context,
 508                 msg,
 509                 msg_len);
 510 }
 511
 512 static void none_token_send (
 513         struct totemrrp_instance *instance,
 514         struct iovec *iovec,
 515         unsigned int iov_len)
 516 {
 517         totemnet_token_send (
 518                 instance->net_handles[0],
 519                 iovec, iov_len);
 520 }
 521
 522 static void none_recv_flush (struct totemrrp_instance *instance)
 523 {
 524         totemnet_recv_flush (instance->net_handles[0]);
 525 }
 526
 527 static void none_send_flush (struct totemrrp_instance *instance)
 528 {
 529         totemnet_send_flush (instance->net_handles[0]);
 530 }
 531
 532 static void none_iface_check (struct totemrrp_instance *instance)
 533 {
 534         totemnet_iface_check (instance->net_handles[0]);
 535 }
 536
 537 static void none_processor_count_set (
 538         struct totemrrp_instance *instance,
 539         unsigned int processor_count)
 540 {
 541         totemnet_processor_count_set (instance->net_handles[0],
 542                 processor_count);
 543 }
 544
 545 static void none_token_target_set (
 546         struct totemrrp_instance *instance,
 547         struct totem_ip_address *token_target,
 548         unsigned int iface_no)
 549 {
 550         totemnet_token_target_set (instance->net_handles[0], token_target);
 551 }
 552
 553 static void none_ring_reenable (
 554         struct totemrrp_instance *instance)
 555 {
 556         /*
 557          * No operation
 558          */
 559 }
 560
 561 /*
 562  * Passive Replication Implementation
 563  */
 564 void *passive_instance_initialize (
 565         struct totemrrp_instance *rrp_instance,
 566         int interface_count)
 567 {
 568         struct passive_instance *instance;
 569
 570         instance = malloc (sizeof (struct passive_instance));
 571         if (instance == 0) {
 572                 goto error_exit;
 573         }
 574         memset (instance, 0, sizeof (struct passive_instance));
 575
 576         instance->faulty = malloc (sizeof (int) * interface_count);
 577         if (instance->faulty == 0) {
 578                 free (instance);
 579                 instance = 0;
 580                 goto error_exit;
 581         }
 582         memset (instance->faulty, 0, sizeof (int) * interface_count);
 583
 584         instance->token_recv_count = malloc (sizeof (int) * interface_count);
 585         if (instance->token_recv_count == 0) {
 586                 free (instance->faulty);
 587                 free (instance);
 588                 instance = 0;
 589                 goto error_exit;
 590         }
 591         memset (instance->token_recv_count, 0, sizeof (int) * interface_count);
 592
 593         instance->mcast_recv_count = malloc (sizeof (int) * interface_count);
 594         if (instance->mcast_recv_count == 0) {
 595                 free (instance->token_recv_count);
 596                 free (instance->faulty);
 597                 free (instance);
 598                 instance = 0;
 599                 goto error_exit;
 600         }
 601         memset (instance->mcast_recv_count, 0, sizeof (int) * interface_count);
 602
 603 error_exit:
 604         return ((void *)instance);
 605 }
 606
 607 static void timer_function_passive_token_expired (void *context)
 608 {
 609         struct passive_instance *passive_instance = (struct passive_instance *)context;
 610         struct totemrrp_instance *rrp_instance = passive_instance->rrp_instance;
 611
 612         rrp_instance->totemrrp_deliver_fn (
 613                 passive_instance->totemrrp_context,
 614                 passive_instance->token,
 615                 passive_instance->token_len);
 616 }
 617
 618 /* TODO
 619 static void timer_function_passive_problem_decrementer (void *context)
 620 {
 621 //      struct passive_instance *passive_instance = (struct passive_instance *)context;
 622 //      struct totemrrp_instance *rrp_instance = passive_instance->rrp_instance;
 623
 624 }
 625 */
 626
 627
 628 static void passive_timer_expired_token_start (
 629         struct passive_instance *passive_instance)
 630 {
 631         poll_timer_add (
 632                 passive_instance->rrp_instance->poll_handle,
 633                 passive_instance->rrp_instance->totem_config->rrp_token_expired_timeout,
 634                 (void *)passive_instance,
 635                 timer_function_passive_token_expired,
 636                 &passive_instance->timer_expired_token);
 637 }
 638
 639 static void passive_timer_expired_token_cancel (
 640         struct passive_instance *passive_instance)
 641 {
 642         poll_timer_delete (
 643                 passive_instance->rrp_instance->poll_handle,
 644                 passive_instance->timer_expired_token);
 645 }
 646
 647 /*
 648 static void passive_timer_problem_decrementer_start (
 649         struct passive_instance *passive_instance)
 650 {
 651         poll_timer_add (
 652                 passive_instance->rrp_instance->poll_handle,
 653                 passive_instance->rrp_instance->totem_config->rrp_problem_count_timeout,
 654                 (void *)passive_instance,
 655                 timer_function_passive_problem_decrementer,
 656                 &passive_instance->timer_problem_decrementer);
 657 }
 658
 659 static void passive_timer_problem_decrementer_cancel (
 660         struct passive_instance *passive_instance)
 661 {
 662         poll_timer_delete (
 663                 passive_instance->rrp_instance->poll_handle,
 664                 passive_instance->timer_problem_decrementer);
 665 }
 666 */
 667
 668
 669 static void passive_mcast_recv (
 670         struct totemrrp_instance *rrp_instance,
 671         unsigned int iface_no,
 672         void *context,
 673         void *msg,
 674         unsigned int msg_len)
 675 {
 676         struct passive_instance *passive_instance = (struct passive_instance *)rrp_instance->rrp_algo_instance;
 677         unsigned int max;
 678         unsigned int i;
 679
 680         rrp_instance->totemrrp_deliver_fn (
 681                 context,
 682                 msg,
 683                 msg_len);
 684
 685         if (rrp_instance->totemrrp_msgs_missing() == 0 &&
 686                 passive_instance->timer_expired_token) {
 687                 /*
 688                  * Delivers the last token
 689                  */
 690                 rrp_instance->totemrrp_deliver_fn (
 691                         passive_instance->totemrrp_context,
 692                         passive_instance->token,
 693                         passive_instance->token_len);
 694                 passive_timer_expired_token_cancel (passive_instance);
 695         }
 696
 697         /*
 698          * Monitor for failures
 699          * TODO doesn't handle wrap-around of the mcast recv count
 700          */
 701         passive_instance->mcast_recv_count[iface_no] += 1;
 702         max = 0;
 703         for (i = 0; i < rrp_instance->interface_count; i++) {
 704                 if (max < passive_instance->mcast_recv_count[i]) {
 705                         max = passive_instance->mcast_recv_count[i];
 706                 }
 707         }
 708
 709         for (i = 0; i < rrp_instance->interface_count; i++) {
 710                 if ((passive_instance->faulty[i] == 0) &&
 711                         (max - passive_instance->mcast_recv_count[i] >
 712                         rrp_instance->totem_config->rrp_problem_count_threshold)) {
 713                         passive_instance->faulty[i] = 1;
 714                         sprintf (rrp_instance->status[i],
 715                                 "Marking ringid %u interface %s FAULTY - adminisrtative intervention required.",
 716                                 i,
 717                                 totemnet_iface_print (rrp_instance->net_handles[i]));
 718                         log_printf (
 719                                 rrp_instance->totemrrp_log_level_error,
 720                                 rrp_instance->status[i]);
 721                 }
 722         }
 723 }
 724
 725 static void passive_mcast_flush_send (
 726         struct totemrrp_instance *instance,
 727         struct iovec *iovec,
 728         unsigned int iov_len)
 729 {
 730         struct passive_instance *passive_instance = (struct passive_instance *)instance->rrp_algo_instance;
 731
 732         do {
 733                 passive_instance->msg_xmit_iface = (passive_instance->msg_xmit_iface + 1) % instance->interface_count;
 734         } while (passive_instance->faulty[passive_instance->msg_xmit_iface] == 1);
 735
 736         totemnet_mcast_flush_send (instance->net_handles[passive_instance->msg_xmit_iface], iovec, iov_len);
 737 }
 738
 739 static void passive_mcast_noflush_send (
 740         struct totemrrp_instance *instance,
 741         struct iovec *iovec,
 742         unsigned int iov_len)
 743 {
 744         struct passive_instance *passive_instance = (struct passive_instance *)instance->rrp_algo_instance;
 745
 746         do {
 747                 passive_instance->msg_xmit_iface = (passive_instance->msg_xmit_iface + 1) % instance->interface_count;
 748         } while (passive_instance->faulty[passive_instance->msg_xmit_iface] == 1);
 749
 750
 751         totemnet_mcast_noflush_send (instance->net_handles[passive_instance->msg_xmit_iface], iovec, iov_len);
 752 }
 753
 754 static void passive_token_recv (
 755         struct totemrrp_instance *rrp_instance,
 756         unsigned int iface_no,
 757         void *context,
 758         void *msg,
 759         unsigned int msg_len,
 760         unsigned int token_seq)
 761 {
 762         struct passive_instance *passive_instance = (struct passive_instance *)rrp_instance->rrp_algo_instance;
 763         unsigned int max;
 764         unsigned int i;
 765
 766         passive_instance->totemrrp_context = context; // this should be in totemrrp_instance ? TODO
 767
 768         if (rrp_instance->totemrrp_msgs_missing() == 0) {
 769                 rrp_instance->totemrrp_deliver_fn (
 770                         context,
 771                         msg,
 772                         msg_len);
 773         } else {
 774                 memcpy (passive_instance->token, msg, msg_len);
 775                 passive_timer_expired_token_start (passive_instance);
 776
 777         }
 778
 779         /*
 780          * Monitor for failures
 781          * TODO doesn't handle wrap-around of the token
 782          */
 783         passive_instance->token_recv_count[iface_no] += 1;
 784         max = 0;
 785         for (i = 0; i < rrp_instance->interface_count; i++) {
 786                 if (max < passive_instance->token_recv_count[i]) {
 787                         max = passive_instance->token_recv_count[i];
 788                 }
 789         }
 790
 791         for (i = 0; i < rrp_instance->interface_count; i++) {
 792                 if ((passive_instance->faulty[i] == 0) &&
 793                         (max - passive_instance->token_recv_count[i] >
 794                         rrp_instance->totem_config->rrp_problem_count_threshold)) {
 795                         passive_instance->faulty[i] = 1;
 796                         sprintf (rrp_instance->status[i],
 797                                 "Marking seqid %d ringid %u interface %s FAULTY - adminisrtative intervention required.",
 798                                 token_seq,
 799                                 i,
 800                                 totemnet_iface_print (rrp_instance->net_handles[i]));
 801                         log_printf (
 802                                 rrp_instance->totemrrp_log_level_error,
 803                                 rrp_instance->status[i]);
 804                 }
 805         }
 806 }
 807
 808 static void passive_token_send (
 809         struct totemrrp_instance *instance,
 810         struct iovec *iovec,
 811         unsigned int iov_len)
 812 {
 813         struct passive_instance *passive_instance = (struct passive_instance *)instance->rrp_algo_instance;
 814
 815         do {
 816                 passive_instance->token_xmit_iface = (passive_instance->token_xmit_iface + 1) % instance->interface_count;
 817         } while (passive_instance->faulty[passive_instance->token_xmit_iface] == 1);
 818
 819         totemnet_token_send (
 820                 instance->net_handles[passive_instance->token_xmit_iface],
 821                 iovec, iov_len);
 822
 823 }
 824
 825 static void passive_recv_flush (struct totemrrp_instance *instance)
 826 {
 827         struct passive_instance *rrp_algo_instance = (struct passive_instance *)instance->rrp_algo_instance;
 828         unsigned int i;
 829
 830         for (i = 0; i < instance->interface_count; i++) {
 831                 if (rrp_algo_instance->faulty[i] == 0) {
 832
 833                         totemnet_recv_flush (instance->net_handles[i]);
 834                 }
 835         }
 836 }
 837
 838 static void passive_send_flush (struct totemrrp_instance *instance)
 839 {
 840         struct passive_instance *rrp_algo_instance = (struct passive_instance *)instance->rrp_algo_instance;
 841         unsigned int i;
 842
 843         for (i = 0; i < instance->interface_count; i++) {
 844                 if (rrp_algo_instance->faulty[i] == 0) {
 845
 846                         totemnet_send_flush (instance->net_handles[i]);
 847                 }
 848         }
 849 }
 850
 851 static void passive_iface_check (struct totemrrp_instance *instance)
 852 {
 853         struct passive_instance *rrp_algo_instance = (struct passive_instance *)instance->rrp_algo_instance;
 854         unsigned int i;
 855
 856         for (i = 0; i < instance->interface_count; i++) {
 857                 if (rrp_algo_instance->faulty[i] == 0) {
 858
 859                         totemnet_iface_check (instance->net_handles[i]);
 860                 }
 861         }
 862 }
 863
 864 static void passive_processor_count_set (
 865         struct totemrrp_instance *instance,
 866         unsigned int processor_count)
 867 {
 868         struct passive_instance *rrp_algo_instance = (struct passive_instance *)instance->rrp_algo_instance;
 869         unsigned int i;
 870
 871         for (i = 0; i < instance->interface_count; i++) {
 872                 if (rrp_algo_instance->faulty[i] == 0) {
 873
 874                         totemnet_processor_count_set (instance->net_handles[i],
 875                                 processor_count);
 876                 }
 877         }
 878 }
 879
 880 static void passive_token_target_set (
 881         struct totemrrp_instance *instance,
 882         struct totem_ip_address *token_target,
 883         unsigned int iface_no)
 884 {
 885         totemnet_token_target_set (instance->net_handles[iface_no], token_target);
 886 }
 887
 888 static void passive_ring_reenable (
 889         struct totemrrp_instance *instance)
 890 {
 891         struct passive_instance *rrp_algo_instance = (struct passive_instance *)instance->rrp_algo_instance;
 892
 893         memset (rrp_algo_instance->mcast_recv_count, 0, sizeof (unsigned int) *
 894                 instance->interface_count);
 895         memset (rrp_algo_instance->token_recv_count, 0, sizeof (unsigned int) *
 896                 instance->interface_count);
 897         memset (rrp_algo_instance->faulty, 0, sizeof (unsigned int) *
 898                 instance->interface_count);
 899 }
 900
 901 /*
 902  * Active Replication Implementation
 903  */
 904 void *active_instance_initialize (
 905         struct totemrrp_instance *rrp_instance,
 906         int interface_count)
 907 {
 908         struct active_instance *instance;
 909
 910         instance = malloc (sizeof (struct active_instance));
 911         if (instance == 0) {
 912                 goto error_exit;
 913         }
 914         memset (instance, 0, sizeof (struct active_instance));
 915
 916         instance->faulty = malloc (sizeof (int) * interface_count);
 917         if (instance->faulty == 0) {
 918                 free (instance);
 919                 instance = 0;
 920                 goto error_exit;
 921         }
 922         memset (instance->faulty, 0, sizeof (unsigned int) * interface_count);
 923
 924         instance->last_token_recv = malloc (sizeof (int) * interface_count);
 925         if (instance->last_token_recv == 0) {
 926                 free (instance->faulty);
 927                 free (instance);
 928                 instance = 0;
 929                 goto error_exit;
 930         }
 931         memset (instance->last_token_recv, 0, sizeof (unsigned int) * interface_count);
 932
 933         instance->counter_problems = malloc (sizeof (int) * interface_count);
 934         if (instance->counter_problems == 0) {
 935                 free (instance->last_token_recv);
 936                 free (instance->faulty);
 937                 free (instance);
 938                 instance = 0;
 939                 goto error_exit;
 940         }
 941         memset (instance->counter_problems, 0, sizeof (unsigned int) * interface_count);
 942
 943         instance->timer_expired_token = 0;
 944
 945         instance->timer_problem_decrementer = 0;
 946
 947         instance->rrp_instance = rrp_instance;
 948
 949 error_exit:
 950         return ((void *)instance);
 951 }
 952 static void timer_function_active_problem_decrementer (void *context)
 953 {
 954         struct active_instance *active_instance = (struct active_instance *)context;
 955         struct totemrrp_instance *rrp_instance = active_instance->rrp_instance;
 956         unsigned int problem_found = 0;
 957         unsigned int i;
 958
 959         for (i = 0; i < rrp_instance->interface_count; i++) {
 960                 if (active_instance->counter_problems[i] > 0) {
 961                         problem_found = 1;
 962                         active_instance->counter_problems[i] -= 1;
 963                         if (active_instance->counter_problems[i] == 0) {
 964                                 sprintf (rrp_instance->status[i],
 965                                         "ring %d active with no faults", i);
 966                         } else {
 967                                 sprintf (rrp_instance->status[i],
 968                                         "Decrementing problem counter for iface %s to [%d of %d]",
 969                                         totemnet_iface_print (rrp_instance->net_handles[i]),
 970                                         active_instance->counter_problems[i],
 971                                         rrp_instance->totem_config->rrp_problem_count_threshold);
 972                         }
 973                                 log_printf (
 974                                         rrp_instance->totemrrp_log_level_warning,
 975                                         rrp_instance->status[i]);
 976                 }
 977         }
 978         if (problem_found) {
 979                 active_timer_problem_decrementer_start (active_instance);
 980         } else {
 981                 active_instance->timer_problem_decrementer = 0;
 982         }
 983 }
 984
 985 static void timer_function_active_token_expired (void *context)
 986 {
 987         struct active_instance *active_instance = (struct active_instance *)context;
 988         struct totemrrp_instance *rrp_instance = active_instance->rrp_instance;
 989         unsigned int i;
 990
 991         for (i = 0; i < rrp_instance->interface_count; i++) {
 992                 if (active_instance->last_token_recv[i] == 0) {
 993                         active_instance->counter_problems[i] += 1;
 994
 995                         if (active_instance->timer_problem_decrementer == 0) {
 996                                 active_timer_problem_decrementer_start (active_instance);
 997                         }
 998                         sprintf (rrp_instance->status[i],
 999                                 "Incrementing problem counter for seqid %d iface %s to [%d of %d]",
1000                                 active_instance->last_token_seq,
1001                                 totemnet_iface_print (rrp_instance->net_handles[i]),
1002                                 active_instance->counter_problems[i],
1003                                 rrp_instance->totem_config->rrp_problem_count_threshold);
1004                         log_printf (
1005                                 rrp_instance->totemrrp_log_level_warning,
1006                                 rrp_instance->status[i]);
1007                 }
1008         }
1009         for (i = 0; i < rrp_instance->interface_count; i++) {
1010                 if (active_instance->counter_problems[i] >= rrp_instance->totem_config->rrp_problem_count_threshold)
1011                 {
1012                         active_instance->faulty[i] = 1;
1013                         sprintf (rrp_instance->status[i],
1014                                 "Marking seqid %d ringid %u interface %s FAULTY - adminisrtative intervention required.",
1015                                 active_instance->last_token_seq,
1016                                 i,
1017                                 totemnet_iface_print (rrp_instance->net_handles[i]));
1018                         log_printf (
1019                                 rrp_instance->totemrrp_log_level_error,
1020                                 rrp_instance->status[i]);
1021                         active_timer_problem_decrementer_cancel (active_instance);
1022                 }
1023         }
1024
1025         rrp_instance->totemrrp_deliver_fn (
1026                 active_instance->totemrrp_context,
1027                 active_instance->token,
1028                 active_instance->token_len);
1029 }
1030
1031 static void active_timer_expired_token_start (
1032         struct active_instance *active_instance)
1033 {
1034         poll_timer_add (
1035                 active_instance->rrp_instance->poll_handle,
1036                 active_instance->rrp_instance->totem_config->rrp_token_expired_timeout,
1037                 (void *)active_instance,
1038                 timer_function_active_token_expired,
1039                 &active_instance->timer_expired_token);
1040 }
1041
1042 static void active_timer_expired_token_cancel (
1043         struct active_instance *active_instance)
1044 {
1045         poll_timer_delete (
1046                 active_instance->rrp_instance->poll_handle,
1047                 active_instance->timer_expired_token);
1048 }
1049
1050 static void active_timer_problem_decrementer_start (
1051         struct active_instance *active_instance)
1052 {
1053         poll_timer_add (
1054                 active_instance->rrp_instance->poll_handle,
1055                 active_instance->rrp_instance->totem_config->rrp_problem_count_timeout,
1056                 (void *)active_instance,
1057                 timer_function_active_problem_decrementer,
1058                 &active_instance->timer_problem_decrementer);
1059 }
1060
1061 static void active_timer_problem_decrementer_cancel (
1062         struct active_instance *active_instance)
1063 {
1064         poll_timer_delete (
1065                 active_instance->rrp_instance->poll_handle,
1066                 active_instance->timer_problem_decrementer);
1067 }
1068
1069
1070 /*
1071  * active replication
1072  */
1073 static void active_mcast_recv (
1074         struct totemrrp_instance *instance,
1075         unsigned int iface_no,
1076         void *context,
1077         void *msg,
1078         unsigned int msg_len)
1079 {
1080         instance->totemrrp_deliver_fn (
1081                 context,
1082                 msg,
1083                 msg_len);
1084 }
1085
1086 static void active_mcast_flush_send (
1087         struct totemrrp_instance *instance,
1088         struct iovec *iovec,
1089         unsigned int iov_len)
1090 {
1091         int i;
1092         struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
1093
1094         for (i = 0; i < instance->interface_count; i++) {
1095                 if (rrp_algo_instance->faulty[i] == 0) {
1096                         totemnet_mcast_flush_send (instance->net_handles[i], iovec, iov_len);
1097                 }
1098         }
1099 }
1100
1101 static void active_mcast_noflush_send (
1102         struct totemrrp_instance *instance,
1103         struct iovec *iovec,
1104         unsigned int iov_len)
1105 {
1106         int i;
1107         struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
1108
1109         for (i = 0; i < instance->interface_count; i++) {
1110                 if (rrp_algo_instance->faulty[i] == 0) {
1111                         totemnet_mcast_noflush_send (instance->net_handles[i], iovec, iov_len);
1112                 }
1113         }
1114 }
1115
1116 static void active_token_recv (
1117         struct totemrrp_instance *instance,
1118         unsigned int iface_no,
1119         void *context,
1120         void *msg,
1121         unsigned int msg_len,
1122         unsigned int token_seq)
1123 {
1124         int i;
1125         struct active_instance *active_instance = (struct active_instance *)instance->rrp_algo_instance;
1126
1127         active_instance->totemrrp_context = context; // this should be in totemrrp_instance ?
1128         if (token_seq > active_instance->last_token_seq) {
1129                 memcpy (active_instance->token, msg, msg_len);
1130                 active_instance->token_len = msg_len;
1131                 for (i = 0; i < instance->interface_count; i++) {
1132                         active_instance->last_token_recv[i] = 0;
1133                 }
1134
1135                 active_instance->last_token_recv[iface_no] = 1;
1136                 active_timer_expired_token_start (active_instance);
1137         }
1138
1139         active_instance->last_token_seq = token_seq;
1140
1141         if (token_seq == active_instance->last_token_seq) {
1142                 active_instance->last_token_recv[iface_no] = 1;
1143                 for (i = 0; i < instance->interface_count; i++) {
1144                         if ((active_instance->last_token_recv[i] == 0) &&
1145                                 active_instance->faulty[i] == 0) {
1146                                 return; /* don't deliver token */
1147                         }
1148                 }
1149                 active_timer_expired_token_cancel (active_instance);
1150
1151                 instance->totemrrp_deliver_fn (
1152                         context,
1153                         msg,
1154                         msg_len);
1155         }
1156 }
1157
1158 static void active_token_send (
1159         struct totemrrp_instance *instance,
1160         struct iovec *iovec,
1161         unsigned int iov_len)
1162 {
1163         struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
1164         int i;
1165
1166         for (i = 0; i < instance->interface_count; i++) {
1167                 if (rrp_algo_instance->faulty[i] == 0) {
1168                         totemnet_token_send (
1169                                 instance->net_handles[i],
1170                                 iovec, iov_len);
1171
1172                 }
1173         }
1174 }
1175
1176 static void active_recv_flush (struct totemrrp_instance *instance)
1177 {
1178         struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
1179         unsigned int i;
1180
1181         for (i = 0; i < instance->interface_count; i++) {
1182                 if (rrp_algo_instance->faulty[i] == 0) {
1183
1184                         totemnet_recv_flush (instance->net_handles[i]);
1185                 }
1186         }
1187 }
1188
1189 static void active_send_flush (struct totemrrp_instance *instance)
1190 {
1191         struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
1192         unsigned int i;
1193
1194         for (i = 0; i < instance->interface_count; i++) {
1195                 if (rrp_algo_instance->faulty[i] == 0) {
1196
1197                         totemnet_send_flush (instance->net_handles[i]);
1198                 }
1199         }
1200 }
1201
1202 static void active_iface_check (struct totemrrp_instance *instance)
1203 {
1204         struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
1205         unsigned int i;
1206
1207         for (i = 0; i < instance->interface_count; i++) {
1208                 if (rrp_algo_instance->faulty[i] == 0) {
1209
1210                         totemnet_iface_check (instance->net_handles[i]);
1211                 }
1212         }
1213 }
1214
1215 static void active_processor_count_set (
1216         struct totemrrp_instance *instance,
1217         unsigned int processor_count)
1218 {
1219         struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
1220         unsigned int i;
1221
1222         for (i = 0; i < instance->interface_count; i++) {
1223                 if (rrp_algo_instance->faulty[i] == 0) {
1224
1225                         totemnet_processor_count_set (instance->net_handles[i],
1226                                 processor_count);
1227                 }
1228         }
1229 }
1230
1231 static void active_token_target_set (
1232         struct totemrrp_instance *instance,
1233         struct totem_ip_address *token_target,
1234         unsigned int iface_no)
1235 {
1236         totemnet_token_target_set (instance->net_handles[iface_no], token_target);
1237 }
1238
1239 static void active_ring_reenable (
1240         struct totemrrp_instance *instance)
1241 {
1242         struct active_instance *rrp_algo_instance = (struct active_instance *)instance->rrp_algo_instance;
1243
1244         memset (rrp_algo_instance->last_token_recv, 0, sizeof (unsigned int) *
1245                 instance->interface_count);
1246         memset (rrp_algo_instance->faulty, 0, sizeof (unsigned int) *
1247                 instance->interface_count);
1248         memset (rrp_algo_instance->counter_problems, 0, sizeof (unsigned int) *
1249                 instance->interface_count);
1250 }
1251
1252 struct deliver_fn_context {
1253         struct totemrrp_instance *instance;
1254         void *context;
1255         int iface_no;
1256 };
1257
1258 static void totemrrp_instance_initialize (struct totemrrp_instance *instance)
1259 {
1260         memset (instance, 0, sizeof (struct totemrrp_instance));
1261 }
1262
1263 static int totemrrp_algorithm_set (
1264         struct totem_config *totem_config,
1265         struct totemrrp_instance *instance)
1266 {
1267         unsigned int res = -1;
1268         unsigned int i;
1269
1270         for (i = 0; i < RRP_ALGOS_COUNT; i++) {
1271                 if (strcmp (totem_config->rrp_mode, rrp_algos[i]->name) == 0) {
1272                         instance->rrp_algo = rrp_algos[i];
1273                         if (rrp_algos[i]->initialize) {
1274                                 instance->rrp_algo_instance = rrp_algos[i]->initialize (
1275                                         instance,
1276                                         totem_config->interface_count);
1277                         }
1278                         res = 0;
1279                         break;
1280                 }
1281         }
1282         for (i = 0; i < totem_config->interface_count; i++) {
1283                 instance->status[i] = malloc (1024);
1284                 sprintf (instance->status[i], "ring %d active with no faults", i);
1285         }
1286         return (res);
1287 }
1288
1289 void rrp_deliver_fn (
1290         void *context,
1291         void *msg,
1292         int msg_len)
1293 {
1294         unsigned int token_seqid;
1295         unsigned int token_is;
1296
1297         struct deliver_fn_context *deliver_fn_context = (struct deliver_fn_context *)context;
1298
1299         deliver_fn_context->instance->totemrrp_token_seqid_get (
1300                 msg,
1301                 &token_seqid,
1302                 &token_is);
1303
1304         if (token_is) {
1305                 /*
1306                  * Deliver to the token receiver for this rrp algorithm
1307                  */
1308                 deliver_fn_context->instance->rrp_algo->token_recv (
1309                         deliver_fn_context->instance,
1310                         deliver_fn_context->iface_no,
1311                         deliver_fn_context->context,
1312                         msg,
1313                         msg_len,
1314                         token_seqid);
1315         } else {
1316                 /*
1317                  * Deliver to the mcast receiver for this rrp algorithm
1318                  */
1319                 deliver_fn_context->instance->rrp_algo->mcast_recv (
1320                         deliver_fn_context->instance,
1321                         deliver_fn_context->iface_no,
1322                         deliver_fn_context->context,
1323                         msg,
1324                         msg_len);
1325         }
1326 }
1327
1328 void rrp_iface_change_fn (
1329         void *context,
1330         struct totem_ip_address *iface_addr)
1331 {
1332         struct deliver_fn_context *deliver_fn_context = (struct deliver_fn_context *)context;
1333
1334         deliver_fn_context->instance->totemrrp_iface_change_fn (
1335                 deliver_fn_context->context,
1336                 iface_addr,
1337                 deliver_fn_context->iface_no);
1338 }
1339
1340 int totemrrp_finalize (
1341         totemrrp_handle handle)
1342 {
1343         struct totemrrp_instance *instance;
1344         int res = 0;
1345         int i;
1346
1347         res = hdb_handle_get (&totemrrp_instance_database, handle,
1348                 (void *)&instance);
1349         if (res != 0) {
1350                 res = ENOENT;
1351                 goto error_exit;
1352         }
1353
1354         for (i = 0; i < instance->interface_count; i++) {
1355                 totemnet_finalize (instance->net_handles[i]);
1356         }
1357
1358         hdb_handle_put (&totemrrp_instance_database, handle);
1359
1360 error_exit:
1361         return (res);
1362 }
1363
1364 /*
1365  * Totem Redundant Ring interface
1366  * depends on poll abstraction, POSIX, IPV4
1367  */
1368
1369 /*
1370  * Create an instance
1371  */
1372 int totemrrp_initialize (
1373         poll_handle poll_handle,
1374         totemrrp_handle *handle,
1375         struct totem_config *totem_config,
1376         void *context,
1377
1378         void (*deliver_fn) (
1379                 void *context,
1380                 void *msg,
1381                 int msg_len),
1382
1383         void (*iface_change_fn) (
1384                 void *context,
1385                 struct totem_ip_address *iface_addr,
1386                 unsigned int iface_no),
1387
1388         void (*token_seqid_get) (
1389                 void *msg,
1390                 unsigned int *seqid,
1391                 unsigned int *token_is),
1392
1393         unsigned int (*msgs_missing) (void))
1394 {
1395         struct totemrrp_instance *instance;
1396         unsigned int res;
1397         int i;
1398
1399         res = hdb_handle_create (&totemrrp_instance_database,
1400         sizeof (struct totemrrp_instance), handle);
1401         if (res != 0) {
1402                 goto error_exit;
1403         }
1404         res = hdb_handle_get (&totemrrp_instance_database, *handle,
1405                 (void *)&instance);
1406         if (res != 0) {
1407                 goto error_destroy;
1408         }
1409
1410         totemrrp_instance_initialize (instance);
1411
1412         instance->totem_config = totem_config;
1413
1414         res = totemrrp_algorithm_set (
1415                 instance->totem_config,
1416                 instance);
1417         if (res == -1) {
1418                 goto error_put;
1419         }
1420
1421         /*
1422         * Configure logging
1423         */
1424         instance->totemrrp_log_level_security = totem_config->totem_logging_configuration.log_level_security;
1425         instance->totemrrp_log_level_error = totem_config->totem_logging_configuration.log_level_error;
1426         instance->totemrrp_log_level_warning = totem_config->totem_logging_configuration.log_level_warning;
1427         instance->totemrrp_log_level_notice = totem_config->totem_logging_configuration.log_level_notice;
1428         instance->totemrrp_log_level_debug = totem_config->totem_logging_configuration.log_level_debug;
1429         instance->totemrrp_log_printf = totem_config->totem_logging_configuration.log_printf;
1430
1431         instance->interfaces = totem_config->interfaces;
1432
1433         instance->totemrrp_poll_handle = poll_handle;
1434
1435         instance->totemrrp_deliver_fn = deliver_fn;
1436
1437         instance->totemrrp_iface_change_fn = iface_change_fn;
1438
1439         instance->totemrrp_token_seqid_get = token_seqid_get;
1440
1441         instance->totemrrp_msgs_missing = msgs_missing;
1442
1443         instance->interface_count = totem_config->interface_count;
1444
1445         instance->net_handles = malloc (sizeof (totemnet_handle) * totem_config->interface_count);
1446
1447         instance->context = context;
1448
1449         instance->poll_handle = poll_handle;
1450
1451         for (i = 0; i < totem_config->interface_count; i++) {
1452                 struct deliver_fn_context *deliver_fn_context;
1453
1454                 deliver_fn_context = malloc (sizeof (struct deliver_fn_context));
1455                 assert (deliver_fn_context);
1456                 deliver_fn_context->instance = instance;
1457                 deliver_fn_context->context = context;
1458                 deliver_fn_context->iface_no = i;
1459
1460                 totemnet_initialize (
1461                         poll_handle,
1462                         &instance->net_handles[i],
1463                         totem_config,
1464                         i,
1465                         (void *)deliver_fn_context,
1466                         rrp_deliver_fn,
1467                         rrp_iface_change_fn);
1468         }
1469
1470         totemnet_net_mtu_adjust (totem_config);
1471
1472 error_exit:
1473         hdb_handle_put (&totemrrp_instance_database, *handle);
1474         return (0);
1475
1476 error_put:
1477         hdb_handle_put (&totemrrp_instance_database, *handle);
1478 error_destroy:
1479         hdb_handle_destroy (&totemrrp_instance_database, *handle);
1480         return (res);
1481 }
1482
1483 int totemrrp_processor_count_set (
1484         totemrrp_handle handle,
1485         unsigned int processor_count)
1486 {
1487         struct totemrrp_instance *instance;
1488         int res = 0;
1489
1490         res = hdb_handle_get (&totemrrp_instance_database, handle,
1491                 (void *)&instance);
1492         if (res != 0) {
1493                 res = ENOENT;
1494                 goto error_exit;
1495         }
1496
1497         instance->rrp_algo->processor_count_set (instance, processor_count);
1498
1499         instance->processor_count = processor_count;
1500
1501         hdb_handle_put (&totemrrp_instance_database, handle);
1502
1503 error_exit:
1504         return (res);
1505 }
1506
1507 int totemrrp_token_target_set (
1508         totemrrp_handle handle,
1509         struct totem_ip_address *addr,
1510         unsigned int iface_no)
1511 {
1512         struct totemrrp_instance *instance;
1513         int res = 0;
1514
1515         res = hdb_handle_get (&totemrrp_instance_database, handle,
1516                 (void *)&instance);
1517         if (res != 0) {
1518                 res = ENOENT;
1519                 goto error_exit;
1520         }
1521
1522         instance->rrp_algo->token_target_set (instance, addr, iface_no);
1523
1524         hdb_handle_put (&totemrrp_instance_database, handle);
1525
1526 error_exit:
1527         return (res);
1528 }
1529 int totemrrp_recv_flush (totemrrp_handle handle)
1530 {
1531         struct totemrrp_instance *instance;
1532         int res = 0;
1533
1534         res = hdb_handle_get (&totemrrp_instance_database, handle,
1535                 (void *)&instance);
1536         if (res != 0) {
1537                 res = ENOENT;
1538                 goto error_exit;
1539         }
1540
1541         instance->rrp_algo->recv_flush (instance);
1542
1543         hdb_handle_put (&totemrrp_instance_database, handle);
1544
1545 error_exit:
1546         return (res);
1547 }
1548
1549 int totemrrp_send_flush (totemrrp_handle handle)
1550 {
1551         struct totemrrp_instance *instance;
1552         int res = 0;
1553
1554         res = hdb_handle_get (&totemrrp_instance_database, handle,
1555                 (void *)&instance);
1556         if (res != 0) {
1557                 res = ENOENT;
1558                 goto error_exit;
1559         }
1560
1561         instance->rrp_algo->send_flush (instance);
1562
1563         hdb_handle_put (&totemrrp_instance_database, handle);
1564
1565 error_exit:
1566         return (res);
1567 }
1568
1569 int totemrrp_token_send (
1570         totemrrp_handle handle,
1571         struct iovec *iovec,
1572         unsigned int iov_len)
1573 {
1574         struct totemrrp_instance *instance;
1575         int res = 0;
1576
1577         res = hdb_handle_get (&totemrrp_instance_database, handle,
1578                 (void *)&instance);
1579         if (res != 0) {
1580                 res = ENOENT;
1581                 goto error_exit;
1582         }
1583
1584         instance->rrp_algo->token_send (instance, iovec, iov_len);
1585
1586         hdb_handle_put (&totemrrp_instance_database, handle);
1587
1588 error_exit:
1589         return (res);
1590 }
1591
1592 int totemrrp_mcast_flush_send (
1593         totemrrp_handle handle,
1594         struct iovec *iovec,
1595         unsigned int iov_len)
1596 {
1597         struct totemrrp_instance *instance;
1598         int res = 0;
1599
1600         res = hdb_handle_get (&totemrrp_instance_database, handle,
1601                 (void *)&instance);
1602         if (res != 0) {
1603                 res = ENOENT;
1604                 goto error_exit;
1605         }
1606
1607 // TODO this needs to return the result
1608         instance->rrp_algo->mcast_flush_send (instance, iovec, iov_len);
1609
1610         hdb_handle_put (&totemrrp_instance_database, handle);
1611 error_exit:
1612         return (res);
1613 }
1614
1615 int totemrrp_mcast_noflush_send (
1616         totemrrp_handle handle,
1617         struct iovec *iovec,
1618         unsigned int iov_len)
1619 {
1620         struct totemrrp_instance *instance;
1621         int res = 0;
1622
1623         res = hdb_handle_get (&totemrrp_instance_database, handle,
1624                 (void *)&instance);
1625         if (res != 0) {
1626                 res = ENOENT;
1627                 goto error_exit;
1628         }
1629
1630         /*
1631          * merge detects go out through mcast_flush_send so it is safe to
1632          * flush these messages if we are only one processor.  This avoids
1633          * an encryption/hmac and decryption/hmac
1634          */
1635         if (instance->processor_count > 1) {
1636
1637 // TODO this needs to return the result
1638                 instance->rrp_algo->mcast_noflush_send (instance, iovec, iov_len);
1639         }
1640
1641         hdb_handle_put (&totemrrp_instance_database, handle);
1642 error_exit:
1643         return (res);
1644 }
1645
1646 int totemrrp_iface_check (totemrrp_handle handle)
1647 {
1648         struct totemrrp_instance *instance;
1649         int res = 0;
1650
1651         res = hdb_handle_get (&totemrrp_instance_database, handle,
1652                 (void *)&instance);
1653         if (res != 0) {
1654                 res = ENOENT;
1655                 goto error_exit;
1656         }
1657
1658         instance->rrp_algo->iface_check (instance);
1659
1660         hdb_handle_put (&totemrrp_instance_database, handle);
1661 error_exit:
1662         return (res);
1663 }
1664
1665 int totemrrp_ifaces_get (
1666         totemrrp_handle handle,
1667         char ***status,
1668         unsigned int *iface_count)
1669 {
1670         struct totemrrp_instance *instance;
1671         int res = 0;
1672
1673         res = hdb_handle_get (&totemrrp_instance_database, handle,
1674                 (void *)&instance);
1675         if (res != 0) {
1676                 res = ENOENT;
1677                 goto error_exit;
1678         }
1679
1680         *status = instance->status;
1681
1682         if (iface_count) {
1683                 *iface_count = instance->interface_count;
1684         }
1685
1686         hdb_handle_put (&totemrrp_instance_database, handle);
1687
1688 error_exit:
1689         return (res);
1690 }
1691
1692 int totemrrp_ring_reenable (
1693         totemrrp_handle handle)
1694 {
1695         struct totemrrp_instance *instance;
1696         int res = 0;
1697         unsigned int i;
1698
1699 printf ("totemrrp ring reenable\n");
1700         res = hdb_handle_get (&totemrrp_instance_database, handle,
1701                 (void *)&instance);
1702         if (res != 0) {
1703                 res = ENOENT;
1704                 goto error_exit;
1705         }
1706
1707         instance->rrp_algo->ring_reenable (instance);
1708
1709         for (i = 0; i < instance->interface_count; i++) {
1710                 sprintf (instance->status[i], "ring %d active with no faults", i);
1711         }
1712
1713         hdb_handle_put (&totemrrp_instance_database, handle);
1714
1715 error_exit:
1716         return (res);
1717 }