exec/totemnet.c

   1 /*
   2  * Copyright (c) 2005 MontaVista Software, Inc.
   3  * Copyright (c) 2006 Red Hat, Inc.
   4  * Copyright (c) 2006 Sun Microsystems, Inc.
   5  *
   6  * All rights reserved.
   7  *
   8  * Author: Steven Dake (sdake@mvista.com)
   9
  10  * This software licensed under BSD license, the text of which follows:
  11  *
  12  * Redistribution and use in source and binary forms, with or without
  13  * modification, are permitted provided that the following conditions are met:
  14  *
  15  * - Redistributions of source code must retain the above copyright notice,
  16  *   this list of conditions and the following disclaimer.
  17  * - Redistributions in binary form must reproduce the above copyright notice,
  18  *   this list of conditions and the following disclaimer in the documentation
  19  *   and/or other materials provided with the distribution.
  20  * - Neither the name of the MontaVista Software, Inc. nor the names of its
  21  *   contributors may be used to endorse or promote products derived from this
  22  *   software without specific prior written permission.
  23  *
  24  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
  25  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  26  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  27  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
  28  * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
  29  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
  30  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
  31  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
  32  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
  33  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
  34  * THE POSSIBILITY OF SUCH DAMAGE.
  35  */
  36
  37 #include <assert.h>
  38 #include <pthread.h>
  39 #include <sys/mman.h>
  40 #include <sys/types.h>
  41 #include <sys/stat.h>
  42 #include <sys/socket.h>
  43 #include <netdb.h>
  44 #include <sys/un.h>
  45 #include <sys/ioctl.h>
  46 #include <sys/param.h>
  47 #include <netinet/in.h>
  48 #include <arpa/inet.h>
  49 #include <unistd.h>
  50 #include <fcntl.h>
  51 #include <stdlib.h>
  52 #include <stdio.h>
  53 #include <errno.h>
  54 #include <signal.h>
  55 #include <sched.h>
  56 #include <time.h>
  57 #include <sys/time.h>
  58 #include <sys/poll.h>
  59
  60 #include "aispoll.h"
  61 #include "totemnet.h"
  62 #include "wthread.h"
  63 #include "swab.h"
  64 #include "../include/queue.h"
  65 #include "../include/sq.h"
  66 #include "../include/list.h"
  67 #include "../include/hdb.h"
  68 #include "swab.h"
  69
  70 #include "crypto.h"
  71
  72 #ifdef OPENAIS_SOLARIS
  73 #define MSG_NOSIGNAL 0
  74 #endif
  75
  76 #define MCAST_SOCKET_BUFFER_SIZE (TRANSMITS_ALLOWED * FRAME_SIZE_MAX)
  77
  78 #define NETIF_STATE_REPORT_UP           1
  79 #define NETIF_STATE_REPORT_DOWN         2
  80
  81 #define BIND_STATE_UNBOUND      0
  82 #define BIND_STATE_REGULAR      1
  83 #define BIND_STATE_LOOPBACK     2
  84
  85 #define HMAC_HASH_SIZE 20
  86 struct security_header {
  87         unsigned char hash_digest[HMAC_HASH_SIZE]; /* The hash *MUST* be first in the data structure */
  88         unsigned char salt[16]; /* random number */
  89         char msg[0];
  90 } __attribute__((packed));
  91
  92 struct totemnet_mcast_thread_state {
  93         unsigned char iobuf[FRAME_SIZE_MAX];
  94         prng_state prng_state;
  95 };
  96
  97 struct totemnet_socket {
  98         int mcast_recv;
  99         int mcast_send;
 100         int token;
 101 };
 102
 103 struct totemnet_instance {
 104         hmac_state totemnet_hmac_state;
 105
 106         prng_state totemnet_prng_state;
 107
 108         unsigned char totemnet_private_key[1024];
 109
 110         unsigned int totemnet_private_key_len;
 111
 112         poll_handle totemnet_poll_handle;
 113
 114         struct totem_interface *totem_interface;
 115
 116         int netif_state_report;
 117
 118         int netif_bind_state;
 119
 120         struct worker_thread_group worker_thread_group;
 121
 122         void *context;
 123
 124         void (*totemnet_deliver_fn) (
 125                 void *context,
 126                 void *msg,
 127                 int msg_len);
 128
 129         void (*totemnet_iface_change_fn) (
 130                 void *context,
 131                 struct totem_ip_address *iface_address);
 132
 133         /*
 134          * Function and data used to log messages
 135          */
 136         int totemnet_log_level_security;
 137
 138         int totemnet_log_level_error;
 139
 140         int totemnet_log_level_warning;
 141
 142         int totemnet_log_level_notice;
 143
 144         int totemnet_log_level_debug;
 145
 146         void (*totemnet_log_printf) (char *file, int line, int level, char *format, ...) __attribute__((format(printf, 4, 5)));
 147
 148         totemnet_handle handle;
 149
 150         char iov_buffer[FRAME_SIZE_MAX];
 151
 152         char iov_buffer_flush[FRAME_SIZE_MAX];
 153
 154         struct iovec totemnet_iov_recv;
 155
 156         struct iovec totemnet_iov_recv_flush;
 157
 158         struct totemnet_socket totemnet_sockets;
 159
 160         struct totem_ip_address mcast_address;
 161
 162         int stats_sent;
 163
 164         int stats_recv;
 165
 166         int stats_delv;
 167
 168         int stats_remcasts;
 169
 170         int stats_orf_token;
 171
 172         struct timeval stats_tv_start;
 173
 174         struct totem_ip_address my_id;
 175
 176         int firstrun;
 177
 178         poll_timer_handle timer_netif_check_timeout;
 179
 180         unsigned int my_memb_entries;
 181
 182         int flushing;
 183
 184         struct totem_config *totem_config;
 185
 186         struct totem_ip_address token_target;
 187 };
 188
 189 struct work_item {
 190         struct iovec iovec[20];
 191         int iov_len;
 192         struct totemnet_instance *instance;
 193 };
 194
 195 static void netif_down_check (struct totemnet_instance *instance);
 196
 197 static int totemnet_build_sockets (
 198         struct totemnet_instance *instance,
 199         struct totem_ip_address *bindnet_address,
 200         struct totem_ip_address *mcastaddress,
 201         struct totemnet_socket *sockets,
 202         struct totem_ip_address *bound_to);
 203
 204 static struct totem_ip_address localhost;
 205
 206 /*
 207  * All instances in one database
 208  */
 209 static struct hdb_handle_database totemnet_instance_database = {
 210         .handle_count   = 0,
 211         .handles        = 0,
 212         .iterator       = 0,
 213         .mutex          = PTHREAD_MUTEX_INITIALIZER
 214 };
 215
 216 static void totemnet_instance_initialize (struct totemnet_instance *instance)
 217 {
 218         memset (instance, 0, sizeof (struct totemnet_instance));
 219
 220         instance->netif_state_report = NETIF_STATE_REPORT_UP | NETIF_STATE_REPORT_DOWN;
 221
 222         instance->totemnet_iov_recv.iov_base = instance->iov_buffer;
 223
 224         instance->totemnet_iov_recv.iov_len = FRAME_SIZE_MAX; //sizeof (instance->iov_buffer);
 225         instance->totemnet_iov_recv_flush.iov_base = instance->iov_buffer_flush;
 226
 227         instance->totemnet_iov_recv_flush.iov_len = FRAME_SIZE_MAX; //sizeof (instance->iov_buffer);
 228
 229         /*
 230          * There is always atleast 1 processor
 231          */
 232         instance->my_memb_entries = 1;
 233 }
 234
 235 #define log_printf(level, format, args...) \
 236     instance->totemnet_log_printf (__FILE__, __LINE__, level, format, ##args)
 237
 238 static int authenticate_and_decrypt (
 239         struct totemnet_instance *instance,
 240         struct iovec *iov)
 241 {
 242         unsigned char keys[48];
 243         struct security_header *header =
 244                 (struct security_header *)iov[0].iov_base;
 245         prng_state keygen_prng_state;
 246         prng_state stream_prng_state;
 247         unsigned char *hmac_key = &keys[32];
 248         unsigned char *cipher_key = &keys[16];
 249         unsigned char *initial_vector = &keys[0];
 250         unsigned char digest_comparison[HMAC_HASH_SIZE];
 251         unsigned long len;
 252
 253         /*
 254          * Generate MAC, CIPHER, IV keys from private key
 255          */
 256         memset (keys, 0, sizeof (keys));
 257         sober128_start (&keygen_prng_state);
 258         sober128_add_entropy (instance->totemnet_private_key,
 259                 instance->totemnet_private_key_len, &keygen_prng_state);
 260         sober128_add_entropy (header->salt, sizeof (header->salt), &keygen_prng_state);
 261
 262         sober128_read (keys, sizeof (keys), &keygen_prng_state);
 263
 264         /*
 265          * Setup stream cipher
 266          */
 267         sober128_start (&stream_prng_state);
 268         sober128_add_entropy (cipher_key, 16, &stream_prng_state);
 269         sober128_add_entropy (initial_vector, 16, &stream_prng_state);
 270
 271         /*
 272          * Authenticate contents of message
 273          */
 274         hmac_init (&instance->totemnet_hmac_state, DIGEST_SHA1, hmac_key, 16);
 275
 276         hmac_process (&instance->totemnet_hmac_state,
 277                 (unsigned char *)iov->iov_base + HMAC_HASH_SIZE,
 278                 iov->iov_len - HMAC_HASH_SIZE);
 279
 280         len = hash_descriptor[DIGEST_SHA1]->hashsize;
 281         assert (HMAC_HASH_SIZE >= len);
 282         hmac_done (&instance->totemnet_hmac_state, digest_comparison, &len);
 283
 284         if (memcmp (digest_comparison, header->hash_digest, len) != 0) {
 285                 log_printf (instance->totemnet_log_level_security, "Received message has invalid digest... ignoring.\n");
 286                 return (-1);
 287         }
 288
 289         /*
 290          * Decrypt the contents of the message with the cipher key
 291          */
 292         assert(iov->iov_len >= sizeof (struct security_header));
 293         sober128_read (
 294                 (unsigned char *)iov->iov_base + sizeof (struct security_header),
 295                 (unsigned long)iov->iov_len - sizeof (struct security_header),
 296                 &stream_prng_state);
 297
 298         return (0);
 299 }
 300 static void encrypt_and_sign_worker (
 301         struct totemnet_instance *instance,
 302         unsigned char *buf,
 303         int *buf_len,
 304         struct iovec *iovec,
 305         int iov_len,
 306         prng_state *prng_state_in)
 307 {
 308         int i;
 309         unsigned char *addr;
 310         unsigned char keys[48];
 311         struct security_header *header;
 312         unsigned char *hmac_key = &keys[32];
 313         unsigned char *cipher_key = &keys[16];
 314         unsigned char *initial_vector = &keys[0];
 315         unsigned long len;
 316         int outlen = 0;
 317         hmac_state hmac_state;
 318         prng_state keygen_prng_state;
 319         prng_state stream_prng_state;
 320
 321         header = (struct security_header *)buf;
 322         addr = buf + sizeof (struct security_header);
 323
 324         memset (keys, 0, sizeof (keys));
 325         memset (header->salt, 0, sizeof (header->salt));
 326
 327         /*
 328          * Generate MAC, CIPHER, IV keys from private key
 329          */
 330         sober128_read (header->salt, sizeof (header->salt), prng_state_in);
 331         sober128_start (&keygen_prng_state);
 332         sober128_add_entropy (instance->totemnet_private_key,
 333                 instance->totemnet_private_key_len,
 334                 &keygen_prng_state);
 335         sober128_add_entropy (header->salt, sizeof (header->salt),
 336                 &keygen_prng_state);
 337
 338         sober128_read (keys, sizeof (keys), &keygen_prng_state);
 339
 340         /*
 341          * Setup stream cipher
 342          */
 343         sober128_start (&stream_prng_state);
 344         sober128_add_entropy (cipher_key, 16, &stream_prng_state);
 345         sober128_add_entropy (initial_vector, 16, &stream_prng_state);
 346
 347         outlen = sizeof (struct security_header);
 348         /*
 349          * Copy remainder of message, then encrypt it
 350          */
 351         for (i = 1; i < iov_len; i++) {
 352                 memcpy (addr, iovec[i].iov_base, iovec[i].iov_len);
 353                 addr += iovec[i].iov_len;
 354                 outlen += iovec[i].iov_len;
 355         }
 356
 357         /*
 358          * Encrypt message by XORing stream cipher data
 359          */
 360         sober128_read (buf + sizeof (struct security_header),
 361                 outlen - sizeof (struct security_header),
 362                 &stream_prng_state);
 363
 364         memset (&hmac_state, 0, sizeof (hmac_state));
 365
 366         /*
 367          * Sign the contents of the message with the hmac key and store signature in message
 368          */
 369         hmac_init (&hmac_state, DIGEST_SHA1, hmac_key, 16);
 370
 371         hmac_process (&hmac_state,
 372                 buf + HMAC_HASH_SIZE,
 373                 outlen - HMAC_HASH_SIZE);
 374
 375         len = hash_descriptor[DIGEST_SHA1]->hashsize;
 376
 377         hmac_done (&hmac_state, header->hash_digest, &len);
 378
 379         *buf_len = outlen;
 380 }
 381
 382 static inline void ucast_sendmsg (
 383         struct totemnet_instance *instance,
 384         struct totem_ip_address *system_to,
 385         struct iovec *iovec_in,
 386         int iov_len_in)
 387 {
 388         struct msghdr msg_ucast;
 389         int res = 0;
 390         int buf_len;
 391         unsigned char sheader[sizeof (struct security_header)];
 392         unsigned char encrypt_data[FRAME_SIZE_MAX];
 393         struct iovec iovec_encrypt[20];
 394         struct iovec *iovec_sendmsg;
 395         struct sockaddr_storage sockaddr;
 396         int iov_len;
 397         int addrlen;
 398
 399         if (instance->totem_config->secauth == 1) {
 400
 401                 iovec_encrypt[0].iov_base = (char *)sheader;
 402                 iovec_encrypt[0].iov_len = sizeof (struct security_header);
 403                 memcpy (&iovec_encrypt[1], &iovec_in[0],
 404                         sizeof (struct iovec) * iov_len_in);
 405
 406                 /*
 407                  * Encrypt and digest the message
 408                  */
 409                 encrypt_and_sign_worker (
 410                         instance,
 411                         encrypt_data,
 412                         &buf_len,
 413                         iovec_encrypt,
 414                         iov_len_in + 1,
 415                         &instance->totemnet_prng_state);
 416
 417                 iovec_encrypt[0].iov_base = (char *)encrypt_data;
 418                 iovec_encrypt[0].iov_len = buf_len;
 419                 iovec_sendmsg = &iovec_encrypt[0];
 420                 iov_len = 1;
 421         } else {
 422                 iovec_sendmsg = iovec_in;
 423                 iov_len = iov_len_in;
 424         }
 425
 426         /*
 427          * Build unicast message
 428          */
 429         totemip_totemip_to_sockaddr_convert(system_to,
 430                 instance->totem_interface->ip_port, &sockaddr, &addrlen);
 431         msg_ucast.msg_name = &sockaddr;
 432         msg_ucast.msg_namelen = addrlen;
 433         msg_ucast.msg_iov = iovec_sendmsg;
 434         msg_ucast.msg_iovlen = iov_len;
 435 #ifndef OPENAIS_SOLARIS
 436         msg_ucast.msg_control = 0;
 437         msg_ucast.msg_controllen = 0;
 438         msg_ucast.msg_flags = 0;
 439 #else
 440         msg_ucast.msg_accrights = NULL;
 441         msg_ucast.msg_accrightslen = 0;
 442 #endif
 443
 444         /*
 445          * Transmit multicast message
 446          * An error here is recovered by totemsrp
 447          */
 448         res = sendmsg (instance->totemnet_sockets.mcast_send, &msg_ucast,
 449                 MSG_NOSIGNAL);
 450 }
 451
 452 static inline void mcast_sendmsg (
 453         struct totemnet_instance *instance,
 454         struct iovec *iovec_in,
 455         int iov_len_in)
 456 {
 457         struct msghdr msg_mcast;
 458         int res = 0;
 459         int buf_len;
 460         unsigned char sheader[sizeof (struct security_header)];
 461         unsigned char encrypt_data[FRAME_SIZE_MAX];
 462         struct iovec iovec_encrypt[20];
 463         struct iovec *iovec_sendmsg;
 464         struct sockaddr_storage sockaddr;
 465         int iov_len;
 466         int addrlen;
 467
 468         if (instance->totem_config->secauth == 1) {
 469
 470                 iovec_encrypt[0].iov_base = (char *)sheader;
 471                 iovec_encrypt[0].iov_len = sizeof (struct security_header);
 472                 memcpy (&iovec_encrypt[1], &iovec_in[0],
 473                         sizeof (struct iovec) * iov_len_in);
 474
 475                 /*
 476                  * Encrypt and digest the message
 477                  */
 478                 encrypt_and_sign_worker (
 479                         instance,
 480                         encrypt_data,
 481                         &buf_len,
 482                         iovec_encrypt,
 483                         iov_len_in + 1,
 484                         &instance->totemnet_prng_state);
 485
 486                 iovec_encrypt[0].iov_base = (char *)encrypt_data;
 487                 iovec_encrypt[0].iov_len = buf_len;
 488                 iovec_sendmsg = &iovec_encrypt[0];
 489                 iov_len = 1;
 490         } else {
 491                 iovec_sendmsg = iovec_in;
 492                 iov_len = iov_len_in;
 493         }
 494
 495         /*
 496          * Build multicast message
 497          */
 498         totemip_totemip_to_sockaddr_convert(&instance->mcast_address,
 499                 instance->totem_interface->ip_port, &sockaddr, &addrlen);
 500         msg_mcast.msg_name = &sockaddr;
 501         msg_mcast.msg_namelen = addrlen;
 502         msg_mcast.msg_iov = iovec_sendmsg;
 503         msg_mcast.msg_iovlen = iov_len;
 504 #ifndef OPENAIS_SOLARIS
 505         msg_mcast.msg_control = 0;
 506         msg_mcast.msg_controllen = 0;
 507         msg_mcast.msg_flags = 0;
 508 #else
 509         msg_mcast.msg_accrights = NULL;
 510         msg_mcast.msg_accrightslen = 0;
 511 #endif
 512
 513         /*
 514          * Transmit multicast message
 515          * An error here is recovered by totemsrp
 516          */
 517         res = sendmsg (instance->totemnet_sockets.mcast_send, &msg_mcast,
 518                 MSG_NOSIGNAL);
 519 }
 520
 521 static void totemnet_mcast_thread_state_constructor (
 522         void *totemnet_mcast_thread_state_in)
 523 {
 524         struct totemnet_mcast_thread_state *totemnet_mcast_thread_state =
 525                 (struct totemnet_mcast_thread_state *)totemnet_mcast_thread_state_in;
 526         memset (totemnet_mcast_thread_state, 0,
 527                 sizeof (totemnet_mcast_thread_state));
 528
 529         rng_make_prng (128, PRNG_SOBER,
 530                 &totemnet_mcast_thread_state->prng_state, NULL);
 531 }
 532
 533
 534 static void totemnet_mcast_worker_fn (void *thread_state, void *work_item_in)
 535 {
 536         struct work_item *work_item = (struct work_item *)work_item_in;
 537         struct totemnet_mcast_thread_state *totemnet_mcast_thread_state =
 538                 (struct totemnet_mcast_thread_state *)thread_state;
 539         struct totemnet_instance *instance = work_item->instance;
 540         struct msghdr msg_mcast;
 541         unsigned char sheader[sizeof (struct security_header)];
 542         int res = 0;
 543         int buf_len;
 544         struct iovec iovec_encrypted;
 545         struct iovec *iovec_sendmsg;
 546         struct sockaddr_storage sockaddr;
 547         unsigned int iovs;
 548         int addrlen;
 549
 550         if (instance->totem_config->secauth == 1) {
 551                 memmove (&work_item->iovec[1], &work_item->iovec[0],
 552                         work_item->iov_len * sizeof (struct iovec));
 553                 work_item->iovec[0].iov_base = (char *)sheader;
 554                 work_item->iovec[0].iov_len = sizeof (struct security_header);
 555
 556                 /*
 557                  * Encrypt and digest the message
 558                  */
 559                 encrypt_and_sign_worker (
 560                         instance,
 561                         totemnet_mcast_thread_state->iobuf, &buf_len,
 562                         work_item->iovec, work_item->iov_len + 1,
 563                         &totemnet_mcast_thread_state->prng_state);
 564
 565                         iovec_sendmsg = &iovec_encrypted;
 566                         iovec_sendmsg->iov_base = (char *)totemnet_mcast_thread_state->iobuf;
 567                         iovec_sendmsg->iov_len = buf_len;
 568                         iovs = 1;
 569         } else {
 570                 iovec_sendmsg = work_item->iovec;
 571                 iovs = work_item->iov_len;
 572         }
 573
 574         totemip_totemip_to_sockaddr_convert(&instance->mcast_address,
 575                 instance->totem_interface->ip_port, &sockaddr, &addrlen);
 576
 577         msg_mcast.msg_name = &sockaddr;
 578         msg_mcast.msg_namelen = addrlen;
 579         msg_mcast.msg_iov = iovec_sendmsg;
 580         msg_mcast.msg_iovlen = iovs;
 581 #ifndef OPENAIS_SOLARIS
 582         msg_mcast.msg_control = 0;
 583         msg_mcast.msg_controllen = 0;
 584         msg_mcast.msg_flags = 0;
 585 #else
 586         msg_mcast.msg_accrights = NULL;
 587         msg_mcast.msg_accrightslen = 0;
 588 #endif
 589
 590         /*
 591          * Transmit multicast message
 592          * An error here is recovered by totemnet
 593          */
 594         res = sendmsg (instance->totemnet_sockets.mcast_send, &msg_mcast,
 595                 MSG_NOSIGNAL);
 596         if (res > 0) {
 597                 instance->stats_sent += res;
 598         }
 599 }
 600
 601 int totemnet_finalize (
 602         totemnet_handle handle)
 603 {
 604         struct totemnet_instance *instance;
 605         int res = 0;
 606
 607         res = hdb_handle_get (&totemnet_instance_database, handle,
 608                 (void *)&instance);
 609         if (res != 0) {
 610                 res = ENOENT;
 611                 goto error_exit;
 612         }
 613
 614         worker_thread_group_exit (&instance->worker_thread_group);
 615
 616         hdb_handle_put (&totemnet_instance_database, handle);
 617
 618 error_exit:
 619         return (res);
 620 }
 621
 622 /*
 623  * Only designed to work with a message with one iov
 624  */
 625
 626 static int net_deliver_fn (
 627         poll_handle handle,
 628         int fd,
 629         int revents,
 630         void *data)
 631 {
 632         struct totemnet_instance *instance = (struct totemnet_instance *)data;
 633         struct msghdr msg_recv;
 634         struct iovec *iovec;
 635         struct security_header *security_header;
 636         struct sockaddr_storage system_from;
 637         int bytes_received;
 638         int res = 0;
 639         char *msg_offset;
 640         unsigned int size_delv;
 641
 642         if (instance->flushing == 1) {
 643                 iovec = &instance->totemnet_iov_recv_flush;
 644         } else {
 645                 iovec = &instance->totemnet_iov_recv;
 646         }
 647
 648         /*
 649          * Receive datagram
 650          */
 651         msg_recv.msg_name = &system_from;
 652         msg_recv.msg_namelen = sizeof (struct sockaddr_storage);
 653         msg_recv.msg_iov = iovec;
 654         msg_recv.msg_iovlen = 1;
 655 #ifndef OPENAIS_SOLARIS
 656         msg_recv.msg_control = 0;
 657         msg_recv.msg_controllen = 0;
 658         msg_recv.msg_flags = 0;
 659 #else
 660         msg_recv.msg_accrights = NULL;
 661         msg_recv.msg_accrightslen = 0;
 662 #endif
 663
 664         bytes_received = recvmsg (fd, &msg_recv, MSG_NOSIGNAL | MSG_DONTWAIT);
 665         if (bytes_received == -1) {
 666                 return (0);
 667         } else {
 668                 instance->stats_recv += bytes_received;
 669         }
 670
 671         if ((instance->totem_config->secauth == 1) &&
 672                 (bytes_received < sizeof (struct security_header))) {
 673
 674                 log_printf (instance->totemnet_log_level_security, "Received message is too short...  ignoring %d.\n", bytes_received);
 675                 return (0);
 676         }
 677
 678         security_header = (struct security_header *)iovec->iov_base;
 679
 680         iovec->iov_len = bytes_received;
 681         if (instance->totem_config->secauth == 1) {
 682                 /*
 683                  * Authenticate and if authenticated, decrypt datagram
 684                  */
 685
 686                 res = authenticate_and_decrypt (instance, iovec);
 687                 if (res == -1) {
 688                         log_printf (instance->totemnet_log_level_security,
 689                                 "Invalid packet data\n");
 690                         iovec->iov_len = FRAME_SIZE_MAX;
 691                         return 0;
 692                 }
 693                 msg_offset = iovec->iov_base +
 694                         sizeof (struct security_header);
 695                 size_delv = bytes_received - sizeof (struct security_header);
 696         } else {
 697                 msg_offset = iovec->iov_base;
 698                 size_delv = bytes_received;
 699         }
 700
 701         /*
 702          * Handle incoming message
 703          */
 704         instance->totemnet_deliver_fn (
 705                 instance->context,
 706                 msg_offset,
 707                 size_delv);
 708
 709         iovec->iov_len = FRAME_SIZE_MAX;
 710         return (0);
 711 }
 712
 713 static int netif_determine (
 714         struct totemnet_instance *instance,
 715         struct totem_ip_address *bindnet,
 716         struct totem_ip_address *bound_to,
 717         int *interface_up,
 718         int *interface_num)
 719 {
 720         int res;
 721
 722         res = totemip_iface_check (bindnet, bound_to,
 723                 interface_up, interface_num);
 724
 725         /*
 726          * If the desired binding is to an IPV4 network and nodeid isn't
 727          * specified, retrieve the node id from this_ip network address
 728          *
 729          * IPV6 networks must have a node ID specified since the node id
 730          * field is only 32 bits.
 731          */
 732         if (bound_to->family == AF_INET && bound_to->nodeid == 0) {
 733                 bound_to->nodeid = totemip_compute_nodeid_from_addr(bound_to);
 734         }
 735
 736         return (res);
 737 }
 738
 739
 740 /*
 741  * If the interface is up, the sockets for totem are built.  If the interface is down
 742  * this function is requeued in the timer list to retry building the sockets later.
 743  */
 744 static void timer_function_netif_check_timeout (
 745         void *data)
 746 {
 747         struct totemnet_instance *instance = (struct totemnet_instance *)data;
 748         int res;
 749         int interface_up;
 750         int interface_num;
 751         struct totem_ip_address *bind_address;
 752
 753         /*
 754          * Build sockets for every interface
 755          */
 756         netif_determine (instance,
 757                 &instance->totem_interface->bindnet,
 758                 &instance->totem_interface->boundto,
 759                 &interface_up, &interface_num);
 760         /*
 761          * If the network interface isn't back up and we are already
 762          * in loopback mode, add timer to check again and return
 763          */
 764         if ((instance->netif_bind_state == BIND_STATE_LOOPBACK &&
 765                 interface_up == 0) ||
 766
 767         (instance->my_memb_entries == 1 &&
 768                 instance->netif_bind_state == BIND_STATE_REGULAR &&
 769                 interface_up == 1)) {
 770
 771                 poll_timer_add (instance->totemnet_poll_handle,
 772                         instance->totem_config->downcheck_timeout,
 773                         (void *)instance,
 774                         timer_function_netif_check_timeout,
 775                         &instance->timer_netif_check_timeout);
 776
 777                 /*
 778                  * Add a timer to check for a downed regular interface
 779                  */
 780                 return;
 781         }
 782
 783         if (instance->totemnet_sockets.mcast_recv > 0) {
 784                 close (instance->totemnet_sockets.mcast_recv);
 785                 poll_dispatch_delete (instance->totemnet_poll_handle,
 786                         instance->totemnet_sockets.mcast_recv);
 787         }
 788         if (instance->totemnet_sockets.mcast_send > 0) {
 789                 close (instance->totemnet_sockets.mcast_send);
 790         }
 791         if (instance->totemnet_sockets.token > 0) {
 792                 close (instance->totemnet_sockets.token);
 793                 poll_dispatch_delete (instance->totemnet_poll_handle,
 794                         instance->totemnet_sockets.token);
 795         }
 796
 797         if (interface_up == 0) {
 798                 /*
 799                  * Interface is not up
 800                  */
 801                 instance->netif_bind_state = BIND_STATE_LOOPBACK;
 802                 bind_address = &localhost;
 803
 804                 /*
 805                  * Add a timer to retry building interfaces and request memb_gather_enter
 806                  */
 807                 poll_timer_add (instance->totemnet_poll_handle,
 808                         instance->totem_config->downcheck_timeout,
 809                         (void *)instance,
 810                         timer_function_netif_check_timeout,
 811                         &instance->timer_netif_check_timeout);
 812         } else {
 813                 /*
 814                  * Interface is up
 815                  */
 816                 instance->netif_bind_state = BIND_STATE_REGULAR;
 817                 bind_address = &instance->totem_interface->bindnet;
 818         }
 819         /*
 820          * Create and bind the multicast and unicast sockets
 821          */
 822         res = totemnet_build_sockets (instance,
 823                 &instance->mcast_address,
 824                 bind_address,
 825                 &instance->totemnet_sockets,
 826                 &instance->totem_interface->boundto);
 827
 828         poll_dispatch_add (
 829                 instance->totemnet_poll_handle,
 830                 instance->totemnet_sockets.mcast_recv,
 831                 POLLIN, instance, net_deliver_fn);
 832
 833         poll_dispatch_add (
 834                 instance->totemnet_poll_handle,
 835                 instance->totemnet_sockets.token,
 836                 POLLIN, instance, net_deliver_fn);
 837
 838         totemip_copy (&instance->my_id, &instance->totem_interface->boundto);
 839
 840         /*
 841          * This reports changes in the interface to the user and totemsrp
 842          */
 843         if (instance->netif_bind_state == BIND_STATE_REGULAR) {
 844                 if (instance->netif_state_report & NETIF_STATE_REPORT_UP) {
 845                         log_printf (instance->totemnet_log_level_notice,
 846                                 "The network interface [%s] is now up.\n",
 847                                 totemip_print (&instance->totem_interface->boundto));
 848                         instance->netif_state_report = NETIF_STATE_REPORT_DOWN;
 849                         instance->totemnet_iface_change_fn (instance->context, &instance->my_id);
 850                 }
 851                 /*
 852                  * Add a timer to check for interface going down in single membership
 853                  */
 854                 if (instance->my_memb_entries == 1) {
 855                         poll_timer_add (instance->totemnet_poll_handle,
 856                                 instance->totem_config->downcheck_timeout,
 857                                 (void *)instance,
 858                                 timer_function_netif_check_timeout,
 859                                 &instance->timer_netif_check_timeout);
 860                 }
 861
 862         } else {
 863                 if (instance->netif_state_report & NETIF_STATE_REPORT_DOWN) {
 864                         log_printf (instance->totemnet_log_level_notice,
 865                                 "The network interface is down.\n");
 866                         instance->totemnet_iface_change_fn (instance->context, &instance->my_id);
 867                 }
 868                 instance->netif_state_report = NETIF_STATE_REPORT_UP;
 869
 870         }
 871 }
 872
 873
 874 /*
 875  * Check if an interface is down and reconfigure
 876  * totemnet waiting for it to come back up
 877  */
 878 static void netif_down_check (struct totemnet_instance *instance)
 879 {
 880         timer_function_netif_check_timeout (instance);
 881 }
 882
 883 /* Set the socket priority to INTERACTIVE to ensure
 884    that our messages don't get queued behind anything else */
 885 static void totemnet_traffic_control_set(struct totemnet_instance *instance, int sock)
 886 {
 887 #ifdef SO_PRIORITY
 888     int prio = 6; /* TC_PRIO_INTERACTIVE */
 889
 890     if (setsockopt(sock, SOL_SOCKET, SO_PRIORITY, &prio, sizeof(int)))
 891                 log_printf (instance->totemnet_log_level_warning, "Could not set traffic priority. (%s)\n", strerror (errno));
 892 #endif
 893 }
 894
 895 static int totemnet_build_sockets_ip (
 896         struct totemnet_instance *instance,
 897         struct totem_ip_address *mcast_address,
 898         struct totem_ip_address *bindnet_address,
 899         struct totemnet_socket *sockets,
 900         struct totem_ip_address *bound_to,
 901         int interface_num)
 902 {
 903         struct sockaddr_storage sockaddr;
 904         struct ipv6_mreq mreq6;
 905         struct ip_mreq mreq;
 906         struct sockaddr_storage mcast_ss, boundto_ss;
 907         struct sockaddr_in6 *mcast_sin6 = (struct sockaddr_in6 *)&mcast_ss;
 908         struct sockaddr_in  *mcast_sin = (struct sockaddr_in *)&mcast_ss;
 909         struct sockaddr_in  *boundto_sin = (struct sockaddr_in *)&boundto_ss;
 910         unsigned int sendbuf_size;
 911         unsigned int recvbuf_size;
 912         unsigned int optlen = sizeof (sendbuf_size);
 913         int addrlen;
 914         int res;
 915         int flag;
 916
 917         /*
 918          * Create multicast recv socket
 919          */
 920         sockets->mcast_recv = socket (bindnet_address->family, SOCK_DGRAM, 0);
 921         if (sockets->mcast_recv == -1) {
 922                 perror ("socket");
 923                 return (-1);
 924         }
 925
 926         totemip_nosigpipe (sockets->mcast_recv);
 927         res = fcntl (sockets->mcast_recv, F_SETFL, O_NONBLOCK);
 928         if (res == -1) {
 929                 log_printf (instance->totemnet_log_level_warning, "Could not set non-blocking operation on multicast socket: %s\n", strerror (errno));
 930                 return (-1);
 931         }
 932
 933         /*
 934          * Force reuse
 935          */
 936          flag = 1;
 937          if ( setsockopt(sockets->mcast_recv, SOL_SOCKET, SO_REUSEADDR, (char *)&flag, sizeof (flag)) < 0) {
 938                 perror("setsockopt reuseaddr");
 939                 return (-1);
 940         }
 941
 942         /*
 943          * Bind to multicast socket used for multicast receives
 944          */
 945         totemip_totemip_to_sockaddr_convert(mcast_address,
 946                 instance->totem_interface->ip_port, &sockaddr, &addrlen);
 947         res = bind (sockets->mcast_recv, (struct sockaddr *)&sockaddr, addrlen);
 948         if (res == -1) {
 949                 perror ("bind mcast recv socket failed");
 950                 return (-1);
 951         }
 952
 953         /*
 954          * Setup mcast send socket
 955          */
 956         sockets->mcast_send = socket (bindnet_address->family, SOCK_DGRAM, 0);
 957         if (sockets->mcast_send == -1) {
 958                 perror ("socket");
 959                 return (-1);
 960         }
 961
 962         totemip_nosigpipe (sockets->mcast_send);
 963         res = fcntl (sockets->mcast_send, F_SETFL, O_NONBLOCK);
 964         if (res == -1) {
 965                 log_printf (instance->totemnet_log_level_warning, "Could not set non-blocking operation on multicast socket: %s\n", strerror (errno));
 966                 return (-1);
 967         }
 968
 969         /*
 970          * Force reuse
 971          */
 972          flag = 1;
 973          if ( setsockopt(sockets->mcast_send, SOL_SOCKET, SO_REUSEADDR, (char *)&flag, sizeof (flag)) < 0) {
 974                 perror("setsockopt reuseaddr");
 975                 return (-1);
 976         }
 977
 978         totemip_totemip_to_sockaddr_convert(bound_to, instance->totem_interface->ip_port - 1,
 979                 &sockaddr, &addrlen);
 980         res = bind (sockets->mcast_send, (struct sockaddr *)&sockaddr, addrlen);
 981         if (res == -1) {
 982                 perror ("bind mcast send socket failed");
 983                 return (-1);
 984         }
 985
 986         /*
 987          * Setup unicast socket
 988          */
 989         sockets->token = socket (bindnet_address->family, SOCK_DGRAM, 0);
 990         if (sockets->token == -1) {
 991                 perror ("socket2");
 992                 return (-1);
 993         }
 994
 995         totemip_nosigpipe (sockets->token);
 996         res = fcntl (sockets->token, F_SETFL, O_NONBLOCK);
 997         if (res == -1) {
 998                 log_printf (instance->totemnet_log_level_warning, "Could not set non-blocking operation on token socket: %s\n", strerror (errno));
 999                 return (-1);
1000         }
1001
1002         /*
1003          * Force reuse
1004          */
1005          flag = 1;
1006          if ( setsockopt(sockets->token, SOL_SOCKET, SO_REUSEADDR, (char *)&flag, sizeof (flag)) < 0) {
1007                 perror("setsockopt reuseaddr");
1008                 return (-1);
1009         }
1010
1011         /*
1012          * Bind to unicast socket used for token send/receives
1013          * This has the side effect of binding to the correct interface
1014          */
1015         totemip_totemip_to_sockaddr_convert(bound_to, instance->totem_interface->ip_port, &sockaddr, &addrlen);
1016         res = bind (sockets->token, (struct sockaddr *)&sockaddr, addrlen);
1017         if (res == -1) {
1018                 perror ("bind token socket failed");
1019                 return (-1);
1020         }
1021
1022         recvbuf_size = MCAST_SOCKET_BUFFER_SIZE;
1023         sendbuf_size = MCAST_SOCKET_BUFFER_SIZE;
1024         /*
1025          * Set buffer sizes to avoid overruns
1026          */
1027          res = setsockopt (sockets->mcast_recv, SOL_SOCKET, SO_RCVBUF, &recvbuf_size, optlen);
1028          res = setsockopt (sockets->mcast_send, SOL_SOCKET, SO_SNDBUF, &sendbuf_size, optlen);
1029
1030         res = getsockopt (sockets->mcast_recv, SOL_SOCKET, SO_RCVBUF, &recvbuf_size, &optlen);
1031         if (res == 0) {
1032                 log_printf (instance->totemnet_log_level_notice,
1033                         "Receive multicast socket recv buffer size (%d bytes).\n", recvbuf_size);
1034         }
1035
1036         res = getsockopt (sockets->mcast_send, SOL_SOCKET, SO_SNDBUF, &sendbuf_size, &optlen);
1037         if (res == 0) {
1038                 log_printf (instance->totemnet_log_level_notice,
1039                         "Transmit multicast socket send buffer size (%d bytes).\n", sendbuf_size);
1040         }
1041
1042         /*
1043          * Join group membership on socket
1044          */
1045         totemip_totemip_to_sockaddr_convert(mcast_address, instance->totem_interface->ip_port, &mcast_ss, &addrlen);
1046         totemip_totemip_to_sockaddr_convert(bound_to, instance->totem_interface->ip_port, &boundto_ss, &addrlen);
1047
1048         switch ( bindnet_address->family ) {
1049                 case AF_INET:
1050                 memset(&mreq, 0, sizeof(mreq));
1051                 mreq.imr_multiaddr.s_addr = mcast_sin->sin_addr.s_addr;
1052                 mreq.imr_interface.s_addr = boundto_sin->sin_addr.s_addr;
1053                 res = setsockopt (sockets->mcast_recv, IPPROTO_IP, IP_ADD_MEMBERSHIP,
1054                         &mreq, sizeof (mreq));
1055                 if (res == -1) {
1056                         perror ("join ipv4 multicast group failed");
1057                         return (-1);
1058                 }
1059                 break;
1060                 case AF_INET6:
1061                 memset(&mreq6, 0, sizeof(mreq6));
1062                 memcpy(&mreq6.ipv6mr_multiaddr, &mcast_sin6->sin6_addr, sizeof(struct in6_addr));
1063                 mreq6.ipv6mr_interface = interface_num;
1064
1065                 res = setsockopt (sockets->mcast_recv, IPPROTO_IPV6, IPV6_JOIN_GROUP,
1066                         &mreq6, sizeof (mreq6));
1067                 if (res == -1) {
1068                         perror ("join ipv6 multicast group failed");
1069                         return (-1);
1070                 }
1071                 break;
1072         }
1073
1074 #ifndef OPENAIS_SOLARIS
1075         /*
1076          * Turn on multicast loopback
1077          */
1078
1079         flag = 1;
1080         switch ( bindnet_address->family ) {
1081                 case AF_INET:
1082                 res = setsockopt (sockets->mcast_recv, IPPROTO_IP, IP_MULTICAST_LOOP,
1083                         &flag, sizeof (flag));
1084                 break;
1085                 case AF_INET6:
1086                 res = setsockopt (sockets->mcast_recv, IPPROTO_IPV6, IPV6_MULTICAST_LOOP,
1087                         &flag, sizeof (flag));
1088         }
1089         if (res == -1) {
1090                 perror ("turn off loopback");
1091                 return (-1);
1092         }
1093 #endif
1094
1095         /*
1096          * Set multicast packets TTL
1097          */
1098
1099         if ( bindnet_address->family == AF_INET6 )
1100         {
1101                 flag = 255;
1102                 res = setsockopt (sockets->mcast_send, IPPROTO_IPV6, IPV6_MULTICAST_HOPS,
1103                         &flag, sizeof (flag));
1104                 if (res == -1) {
1105                         perror ("setp mcast hops");
1106                         return (-1);
1107                 }
1108         }
1109
1110 #ifndef OPENAIS_SOLARIS
1111         /*
1112          * Bind to a specific interface for multicast send and receive
1113          */
1114         switch ( bindnet_address->family ) {
1115                 case AF_INET:
1116                 if (setsockopt (sockets->mcast_send, IPPROTO_IP, IP_MULTICAST_IF,
1117                         &boundto_sin->sin_addr, sizeof (boundto_sin->sin_addr)) < 0) {
1118                         perror ("cannot select interface");
1119                         return (-1);
1120                 }
1121                 if (setsockopt (sockets->mcast_recv, IPPROTO_IP, IP_MULTICAST_IF,
1122                         &boundto_sin->sin_addr, sizeof (boundto_sin->sin_addr)) < 0) {
1123                         perror ("cannot select interface");
1124                         return (-1);
1125                 }
1126                 break;
1127                 case AF_INET6:
1128                 if (setsockopt (sockets->mcast_send, IPPROTO_IPV6, IPV6_MULTICAST_IF,
1129                         &interface_num, sizeof (interface_num)) < 0) {
1130                         perror ("cannot select interface");
1131                         return (-1);
1132                 }
1133                 if (setsockopt (sockets->mcast_recv, IPPROTO_IPV6, IPV6_MULTICAST_IF,
1134                         &interface_num, sizeof (interface_num)) < 0) {
1135                         perror ("cannot select interface");
1136                         return (-1);
1137                 }
1138                 break;
1139         }
1140 #endif
1141
1142         return 0;
1143 }
1144
1145 static int totemnet_build_sockets (
1146         struct totemnet_instance *instance,
1147         struct totem_ip_address *mcast_address,
1148         struct totem_ip_address *bindnet_address,
1149         struct totemnet_socket *sockets,
1150         struct totem_ip_address *bound_to)
1151 {
1152         int interface_num;
1153         int interface_up;
1154         int res;
1155
1156         /*
1157          * Determine the ip address bound to and the interface name
1158          */
1159         res = netif_determine (instance,
1160                 bindnet_address,
1161                 bound_to,
1162                 &interface_up,
1163                 &interface_num);
1164
1165         if (res == -1) {
1166                 return (-1);
1167         }
1168
1169         totemip_copy(&instance->my_id, bound_to);
1170
1171         res = totemnet_build_sockets_ip (instance, mcast_address,
1172                 bindnet_address, sockets, bound_to, interface_num);
1173
1174         /* We only send out of the token socket */
1175         totemnet_traffic_control_set(instance, sockets->token);
1176         return res;
1177 }
1178
1179 /*
1180  * Totem Network interface - also does encryption/decryption
1181  * depends on poll abstraction, POSIX, IPV4
1182  */
1183
1184 /*
1185  * Create an instance
1186  */
1187 int totemnet_initialize (
1188         poll_handle poll_handle,
1189         totemnet_handle *handle,
1190         struct totem_config *totem_config,
1191         int interface_no,
1192         void *context,
1193
1194         void (*deliver_fn) (
1195                 void *context,
1196                 void *msg,
1197                 int msg_len),
1198
1199         void (*iface_change_fn) (
1200                 void *context,
1201                 struct totem_ip_address *iface_address))
1202 {
1203         struct totemnet_instance *instance;
1204         unsigned int res;
1205
1206         res = hdb_handle_create (&totemnet_instance_database,
1207                 sizeof (struct totemnet_instance), handle);
1208         if (res != 0) {
1209                 goto error_exit;
1210         }
1211         res = hdb_handle_get (&totemnet_instance_database, *handle,
1212                 (void *)&instance);
1213         if (res != 0) {
1214                 goto error_destroy;
1215         }
1216
1217         totemnet_instance_initialize (instance);
1218
1219         instance->totem_config = totem_config;
1220         /*
1221         * Configure logging
1222         */
1223         instance->totemnet_log_level_security = 1; //totem_config->totem_logging_configuration.log_level_security;
1224         instance->totemnet_log_level_error = totem_config->totem_logging_configuration.log_level_error;
1225         instance->totemnet_log_level_warning = totem_config->totem_logging_configuration.log_level_warning;
1226         instance->totemnet_log_level_notice = totem_config->totem_logging_configuration.log_level_notice;
1227         instance->totemnet_log_level_debug = totem_config->totem_logging_configuration.log_level_debug;
1228         instance->totemnet_log_printf = totem_config->totem_logging_configuration.log_printf;
1229
1230         /*
1231         * Initialize random number generator for later use to generate salt
1232         */
1233         memcpy (instance->totemnet_private_key, totem_config->private_key,
1234                 totem_config->private_key_len);
1235
1236         instance->totemnet_private_key_len = totem_config->private_key_len;
1237
1238         rng_make_prng (128, PRNG_SOBER, &instance->totemnet_prng_state, NULL);
1239
1240         /*
1241          * Initialize local variables for totemnet
1242          */
1243         instance->totem_interface = &totem_config->interfaces[interface_no];
1244         totemip_copy (&instance->mcast_address, &instance->totem_interface->mcast_addr);
1245         memset (instance->iov_buffer, 0, FRAME_SIZE_MAX);
1246
1247         /*
1248         * If threaded send requested, initialize thread group data structure
1249         */
1250         if (totem_config->threads) {
1251                 worker_thread_group_init (
1252                         &instance->worker_thread_group,
1253                         totem_config->threads, 128,
1254                         sizeof (struct work_item),
1255                         sizeof (struct totemnet_mcast_thread_state),
1256                         totemnet_mcast_thread_state_constructor,
1257                         totemnet_mcast_worker_fn);
1258         }
1259
1260         instance->totemnet_poll_handle = poll_handle;
1261
1262         instance->totem_interface->bindnet.nodeid = instance->totem_config->node_id;
1263
1264         instance->context = context;
1265         instance->totemnet_deliver_fn = deliver_fn;
1266
1267         instance->totemnet_iface_change_fn = iface_change_fn;
1268
1269         instance->handle = *handle;
1270
1271         rng_make_prng (128, PRNG_SOBER, &instance->totemnet_prng_state, NULL);
1272
1273         totemip_localhost (instance->mcast_address.family, &localhost);
1274
1275         netif_down_check (instance);
1276
1277 error_exit:
1278         hdb_handle_put (&totemnet_instance_database, *handle);
1279         return (0);
1280
1281 error_destroy:
1282         hdb_handle_destroy (&totemnet_instance_database, *handle);
1283         return (-1);
1284 }
1285
1286 int totemnet_processor_count_set (
1287         totemnet_handle handle,
1288         int processor_count)
1289 {
1290         struct totemnet_instance *instance;
1291         int res = 0;
1292
1293         res = hdb_handle_get (&totemnet_instance_database, handle,
1294                 (void *)&instance);
1295         if (res != 0) {
1296                 res = ENOENT;
1297                 goto error_exit;
1298         }
1299
1300         instance->my_memb_entries = processor_count;
1301         poll_timer_delete (instance->totemnet_poll_handle,
1302                 instance->timer_netif_check_timeout);
1303         if (processor_count == 1) {
1304                 poll_timer_add (instance->totemnet_poll_handle,
1305                         instance->totem_config->downcheck_timeout,
1306                         (void *)instance,
1307                         timer_function_netif_check_timeout,
1308                         &instance->timer_netif_check_timeout);
1309         }
1310         hdb_handle_put (&totemnet_instance_database, handle);
1311
1312 error_exit:
1313         return (res);
1314 }
1315
1316 int totemnet_recv_flush (totemnet_handle handle)
1317 {
1318         struct totemnet_instance *instance;
1319         struct pollfd ufd;
1320         int nfds;
1321         int res = 0;
1322
1323         res = hdb_handle_get (&totemnet_instance_database, handle,
1324                 (void *)&instance);
1325         if (res != 0) {
1326                 res = ENOENT;
1327                 goto error_exit;
1328         }
1329
1330         instance->flushing = 1;
1331
1332         do {
1333                 ufd.fd = instance->totemnet_sockets.mcast_recv;
1334                 ufd.events = POLLIN;
1335                 nfds = poll (&ufd, 1, 0);
1336                 if (nfds == 1 && ufd.revents & POLLIN) {
1337                 net_deliver_fn (0, instance->totemnet_sockets.mcast_recv,
1338                         ufd.revents, instance);
1339                 }
1340         } while (nfds == 1);
1341
1342         instance->flushing = 0;
1343
1344         hdb_handle_put (&totemnet_instance_database, handle);
1345
1346 error_exit:
1347         return (res);
1348 }
1349
1350 int totemnet_send_flush (totemnet_handle handle)
1351 {
1352         struct totemnet_instance *instance;
1353         int res = 0;
1354
1355         res = hdb_handle_get (&totemnet_instance_database, handle,
1356                 (void *)&instance);
1357         if (res != 0) {
1358                 res = ENOENT;
1359                 goto error_exit;
1360         }
1361
1362         worker_thread_group_wait (&instance->worker_thread_group);
1363
1364         hdb_handle_put (&totemnet_instance_database, handle);
1365
1366 error_exit:
1367         return (res);
1368 }
1369
1370 int totemnet_token_send (
1371         totemnet_handle handle,
1372         struct iovec *iovec,
1373         int iov_len)
1374 {
1375         struct totemnet_instance *instance;
1376         int res = 0;
1377
1378         res = hdb_handle_get (&totemnet_instance_database, handle,
1379                 (void *)&instance);
1380         if (res != 0) {
1381                 res = ENOENT;
1382                 goto error_exit;
1383         }
1384
1385         ucast_sendmsg (instance, &instance->token_target, iovec, iov_len);
1386
1387         hdb_handle_put (&totemnet_instance_database, handle);
1388
1389 error_exit:
1390         return (res);
1391 }
1392 int totemnet_mcast_flush_send (
1393         totemnet_handle handle,
1394         struct iovec *iovec,
1395         unsigned int iov_len)
1396 {
1397         struct totemnet_instance *instance;
1398         int res = 0;
1399
1400         res = hdb_handle_get (&totemnet_instance_database, handle,
1401                 (void *)&instance);
1402         if (res != 0) {
1403                 res = ENOENT;
1404                 goto error_exit;
1405         }
1406
1407         mcast_sendmsg (instance, iovec, iov_len);
1408
1409         hdb_handle_put (&totemnet_instance_database, handle);
1410
1411 error_exit:
1412         return (res);
1413 }
1414
1415 int totemnet_mcast_noflush_send (
1416         totemnet_handle handle,
1417         struct iovec *iovec,
1418         unsigned int iov_len)
1419 {
1420         struct totemnet_instance *instance;
1421         struct work_item work_item;
1422         int res = 0;
1423
1424         res = hdb_handle_get (&totemnet_instance_database, handle,
1425                 (void *)&instance);
1426         if (res != 0) {
1427                 res = ENOENT;
1428                 goto error_exit;
1429         }
1430
1431         if (instance->totem_config->threads) {
1432                 memcpy (&work_item.iovec[0], iovec, iov_len * sizeof (struct iovec));
1433                 work_item.iov_len = iov_len;
1434                 work_item.instance = instance;
1435
1436                 worker_thread_group_work_add (&instance->worker_thread_group,
1437                         &work_item);
1438         } else {
1439                 mcast_sendmsg (instance, iovec, iov_len);
1440         }
1441
1442         hdb_handle_put (&totemnet_instance_database, handle);
1443 error_exit:
1444         return (res);
1445 }
1446
1447 extern int totemnet_iface_check (totemnet_handle handle)
1448 {
1449         struct totemnet_instance *instance;
1450         int res = 0;
1451
1452         res = hdb_handle_get (&totemnet_instance_database, handle,
1453                 (void *)&instance);
1454         if (res != 0) {
1455                 res = ENOENT;
1456                 goto error_exit;
1457         }
1458
1459         timer_function_netif_check_timeout (instance);
1460
1461         hdb_handle_put (&totemnet_instance_database, handle);
1462 error_exit:
1463         return (res);
1464 }
1465
1466 extern void totemnet_net_mtu_adjust (struct totem_config *totem_config)
1467 {
1468 #define UDPIP_HEADER_SIZE (20 + 8) /* 20 bytes for ip 8 bytes for udp */
1469         if (totem_config->secauth == 1) {
1470                 totem_config->net_mtu -= sizeof (struct security_header) +
1471                         UDPIP_HEADER_SIZE;
1472         } else {
1473                 totem_config->net_mtu -= UDPIP_HEADER_SIZE;
1474         }
1475 }
1476
1477 char *totemnet_iface_print (totemnet_handle handle)  {
1478         struct totemnet_instance *instance;
1479         int res = 0;
1480         char *ret_char;
1481
1482         res = hdb_handle_get (&totemnet_instance_database, handle,
1483                 (void *)&instance);
1484         if (res != 0) {
1485                 ret_char = "Invalid totemnet handle";
1486                 goto error_exit;
1487         }
1488
1489         ret_char = (char *)totemip_print (&instance->my_id);
1490
1491         hdb_handle_put (&totemnet_instance_database, handle);
1492 error_exit:
1493         return (ret_char);
1494 }
1495
1496 int totemnet_iface_get (
1497         totemnet_handle handle,
1498         struct totem_ip_address *addr)
1499 {
1500         struct totemnet_instance *instance;
1501         unsigned int res;
1502
1503         res = hdb_handle_get (&totemnet_instance_database, handle,
1504                 (void *)&instance);
1505         if (res != 0) {
1506                 goto error_exit;
1507         }
1508
1509         memcpy (addr, &instance->my_id, sizeof (struct totem_ip_address));
1510
1511         hdb_handle_put (&totemnet_instance_database, handle);
1512
1513 error_exit:
1514         return (res);
1515 }
1516
1517 int totemnet_token_target_set (
1518         totemnet_handle handle,
1519         struct totem_ip_address *token_target)
1520 {
1521         struct totemnet_instance *instance;
1522         unsigned int res;
1523
1524         res = hdb_handle_get (&totemnet_instance_database, handle,
1525                 (void *)&instance);
1526         if (res != 0) {
1527                 goto error_exit;
1528         }
1529
1530         memcpy (&instance->token_target, token_target,
1531                 sizeof (struct totem_ip_address));
1532
1533         hdb_handle_put (&totemnet_instance_database, handle);
1534
1535 error_exit:
1536         return (res);
1537 }
1538
1539