contrib/ivshmem-server/ivshmem-server.c

   1 /*
   2  * Copyright 6WIND S.A., 2014
   3  *
   4  * This work is licensed under the terms of the GNU GPL, version 2 or
   5  * (at your option) any later version.  See the COPYING file in the
   6  * top-level directory.
   7  */
   8 #include "qemu-common.h"
   9 #include "qemu/sockets.h"
  10
  11 #include <sys/mman.h>
  12 #include <sys/types.h>
  13 #include <sys/socket.h>
  14 #include <sys/un.h>
  15
  16 #include "ivshmem-server.h"
  17
  18 /* log a message on stdout if verbose=1 */
  19 #define IVSHMEM_SERVER_DEBUG(server, fmt, ...) do { \
  20         if ((server)->verbose) {         \
  21             printf(fmt, ## __VA_ARGS__); \
  22         }                                \
  23     } while (0)
  24
  25 /** maximum size of a huge page, used by ivshmem_server_ftruncate() */
  26 #define IVSHMEM_SERVER_MAX_HUGEPAGE_SIZE (1024 * 1024 * 1024)
  27
  28 /** default listen backlog (number of sockets not accepted) */
  29 #define IVSHMEM_SERVER_LISTEN_BACKLOG 10
  30
  31 /* send message to a client unix socket */
  32 static int
  33 ivshmem_server_send_one_msg(int sock_fd, long peer_id, int fd)
  34 {
  35     int ret;
  36     struct msghdr msg;
  37     struct iovec iov[1];
  38     union {
  39         struct cmsghdr cmsg;
  40         char control[CMSG_SPACE(sizeof(int))];
  41     } msg_control;
  42     struct cmsghdr *cmsg;
  43
  44     iov[0].iov_base = &peer_id;
  45     iov[0].iov_len = sizeof(peer_id);
  46
  47     memset(&msg, 0, sizeof(msg));
  48     msg.msg_iov = iov;
  49     msg.msg_iovlen = 1;
  50
  51     /* if fd is specified, add it in a cmsg */
  52     if (fd >= 0) {
  53         memset(&msg_control, 0, sizeof(msg_control));
  54         msg.msg_control = &msg_control;
  55         msg.msg_controllen = sizeof(msg_control);
  56         cmsg = CMSG_FIRSTHDR(&msg);
  57         cmsg->cmsg_level = SOL_SOCKET;
  58         cmsg->cmsg_type = SCM_RIGHTS;
  59         cmsg->cmsg_len = CMSG_LEN(sizeof(int));
  60         memcpy(CMSG_DATA(cmsg), &fd, sizeof(fd));
  61     }
  62
  63     ret = sendmsg(sock_fd, &msg, 0);
  64     if (ret <= 0) {
  65         return -1;
  66     }
  67
  68     return 0;
  69 }
  70
  71 /* free a peer when the server advertises a disconnection or when the
  72  * server is freed */
  73 static void
  74 ivshmem_server_free_peer(IvshmemServer *server, IvshmemServerPeer *peer)
  75 {
  76     unsigned vector;
  77     IvshmemServerPeer *other_peer;
  78
  79     IVSHMEM_SERVER_DEBUG(server, "free peer %ld\n", peer->id);
  80     close(peer->sock_fd);
  81     QTAILQ_REMOVE(&server->peer_list, peer, next);
  82
  83     /* advertise the deletion to other peers */
  84     QTAILQ_FOREACH(other_peer, &server->peer_list, next) {
  85         ivshmem_server_send_one_msg(other_peer->sock_fd, peer->id, -1);
  86     }
  87
  88     for (vector = 0; vector < peer->vectors_count; vector++) {
  89         event_notifier_cleanup(&peer->vectors[vector]);
  90     }
  91
  92     g_free(peer);
  93 }
  94
  95 /* send the peer id and the shm_fd just after a new client connection */
  96 static int
  97 ivshmem_server_send_initial_info(IvshmemServer *server, IvshmemServerPeer *peer)
  98 {
  99     int ret;
 100
 101     /* send the peer id to the client */
 102     ret = ivshmem_server_send_one_msg(peer->sock_fd, peer->id, -1);
 103     if (ret < 0) {
 104         IVSHMEM_SERVER_DEBUG(server, "cannot send peer id: %s\n",
 105                              strerror(errno));
 106         return -1;
 107     }
 108
 109     /* send the shm_fd */
 110     ret = ivshmem_server_send_one_msg(peer->sock_fd, -1, server->shm_fd);
 111     if (ret < 0) {
 112         IVSHMEM_SERVER_DEBUG(server, "cannot send shm fd: %s\n",
 113                              strerror(errno));
 114         return -1;
 115     }
 116
 117     return 0;
 118 }
 119
 120 /* handle message on listening unix socket (new client connection) */
 121 static int
 122 ivshmem_server_handle_new_conn(IvshmemServer *server)
 123 {
 124     IvshmemServerPeer *peer, *other_peer;
 125     struct sockaddr_un unaddr;
 126     socklen_t unaddr_len;
 127     int newfd;
 128     unsigned i;
 129
 130     /* accept the incoming connection */
 131     unaddr_len = sizeof(unaddr);
 132     newfd = qemu_accept(server->sock_fd,
 133                         (struct sockaddr *)&unaddr, &unaddr_len);
 134
 135     if (newfd < 0) {
 136         IVSHMEM_SERVER_DEBUG(server, "cannot accept() %s\n", strerror(errno));
 137         return -1;
 138     }
 139
 140     qemu_set_nonblock(newfd);
 141     IVSHMEM_SERVER_DEBUG(server, "accept()=%d\n", newfd);
 142
 143     /* allocate new structure for this peer */
 144     peer = g_malloc0(sizeof(*peer));
 145     peer->sock_fd = newfd;
 146
 147     /* get an unused peer id */
 148     /* XXX: this could use id allocation such as Linux IDA, or simply
 149      * a free-list */
 150     for (i = 0; i < G_MAXUINT16; i++) {
 151         if (ivshmem_server_search_peer(server, server->cur_id) == NULL) {
 152             break;
 153         }
 154         server->cur_id++;
 155     }
 156     if (i == G_MAXUINT16) {
 157         IVSHMEM_SERVER_DEBUG(server, "cannot allocate new client id\n");
 158         goto fail;
 159     }
 160     peer->id = server->cur_id++;
 161
 162     /* create eventfd, one per vector */
 163     peer->vectors_count = server->n_vectors;
 164     for (i = 0; i < peer->vectors_count; i++) {
 165         if (event_notifier_init(&peer->vectors[i], FALSE) < 0) {
 166             IVSHMEM_SERVER_DEBUG(server, "cannot create eventfd\n");
 167             goto fail;
 168         }
 169     }
 170
 171     /* send peer id and shm fd */
 172     if (ivshmem_server_send_initial_info(server, peer) < 0) {
 173         IVSHMEM_SERVER_DEBUG(server, "cannot send initial info\n");
 174         goto fail;
 175     }
 176
 177     /* advertise the new peer to others */
 178     QTAILQ_FOREACH(other_peer, &server->peer_list, next) {
 179         for (i = 0; i < peer->vectors_count; i++) {
 180             ivshmem_server_send_one_msg(other_peer->sock_fd, peer->id,
 181                                         peer->vectors[i].wfd);
 182         }
 183     }
 184
 185     /* advertise the other peers to the new one */
 186     QTAILQ_FOREACH(other_peer, &server->peer_list, next) {
 187         for (i = 0; i < peer->vectors_count; i++) {
 188             ivshmem_server_send_one_msg(peer->sock_fd, other_peer->id,
 189                                         other_peer->vectors[i].wfd);
 190         }
 191     }
 192
 193     /* advertise the new peer to itself */
 194     for (i = 0; i < peer->vectors_count; i++) {
 195         ivshmem_server_send_one_msg(peer->sock_fd, peer->id,
 196                                     event_notifier_get_fd(&peer->vectors[i]));
 197     }
 198
 199     QTAILQ_INSERT_TAIL(&server->peer_list, peer, next);
 200     IVSHMEM_SERVER_DEBUG(server, "new peer id = %ld\n",
 201                          peer->id);
 202     return 0;
 203
 204 fail:
 205     while (i--) {
 206         event_notifier_cleanup(&peer->vectors[i]);
 207     }
 208     close(newfd);
 209     g_free(peer);
 210     return -1;
 211 }
 212
 213 /* Try to ftruncate a file to next power of 2 of shmsize.
 214  * If it fails; all power of 2 above shmsize are tested until
 215  * we reach the maximum huge page size. This is useful
 216  * if the shm file is in a hugetlbfs that cannot be truncated to the
 217  * shm_size value. */
 218 static int
 219 ivshmem_server_ftruncate(int fd, unsigned shmsize)
 220 {
 221     int ret;
 222     struct stat mapstat;
 223
 224     /* align shmsize to next power of 2 */
 225     shmsize = pow2ceil(shmsize);
 226
 227     if (fstat(fd, &mapstat) != -1 && mapstat.st_size == shmsize) {
 228         return 0;
 229     }
 230
 231     while (shmsize <= IVSHMEM_SERVER_MAX_HUGEPAGE_SIZE) {
 232         ret = ftruncate(fd, shmsize);
 233         if (ret == 0) {
 234             return ret;
 235         }
 236         shmsize *= 2;
 237     }
 238
 239     return -1;
 240 }
 241
 242 /* Init a new ivshmem server */
 243 int
 244 ivshmem_server_init(IvshmemServer *server, const char *unix_sock_path,
 245                     const char *shm_path, size_t shm_size, unsigned n_vectors,
 246                     bool verbose)
 247 {
 248     int ret;
 249
 250     memset(server, 0, sizeof(*server));
 251     server->verbose = verbose;
 252
 253     ret = snprintf(server->unix_sock_path, sizeof(server->unix_sock_path),
 254                    "%s", unix_sock_path);
 255     if (ret < 0 || ret >= sizeof(server->unix_sock_path)) {
 256         IVSHMEM_SERVER_DEBUG(server, "could not copy unix socket path\n");
 257         return -1;
 258     }
 259     ret = snprintf(server->shm_path, sizeof(server->shm_path),
 260                    "%s", shm_path);
 261     if (ret < 0 || ret >= sizeof(server->shm_path)) {
 262         IVSHMEM_SERVER_DEBUG(server, "could not copy shm path\n");
 263         return -1;
 264     }
 265
 266     server->shm_size = shm_size;
 267     server->n_vectors = n_vectors;
 268
 269     QTAILQ_INIT(&server->peer_list);
 270
 271     return 0;
 272 }
 273
 274 /* open shm, create and bind to the unix socket */
 275 int
 276 ivshmem_server_start(IvshmemServer *server)
 277 {
 278     struct sockaddr_un sun;
 279     int shm_fd, sock_fd, ret;
 280
 281     /* open shm file */
 282     shm_fd = shm_open(server->shm_path, O_CREAT|O_RDWR, S_IRWXU);
 283     if (shm_fd < 0) {
 284         fprintf(stderr, "cannot open shm file %s: %s\n", server->shm_path,
 285                 strerror(errno));
 286         return -1;
 287     }
 288     if (ivshmem_server_ftruncate(shm_fd, server->shm_size) < 0) {
 289         fprintf(stderr, "ftruncate(%s) failed: %s\n", server->shm_path,
 290                 strerror(errno));
 291         goto err_close_shm;
 292     }
 293
 294     IVSHMEM_SERVER_DEBUG(server, "create & bind socket %s\n",
 295                          server->unix_sock_path);
 296
 297     /* create the unix listening socket */
 298     sock_fd = socket(AF_UNIX, SOCK_STREAM, 0);
 299     if (sock_fd < 0) {
 300         IVSHMEM_SERVER_DEBUG(server, "cannot create socket: %s\n",
 301                              strerror(errno));
 302         goto err_close_shm;
 303     }
 304
 305     sun.sun_family = AF_UNIX;
 306     ret = snprintf(sun.sun_path, sizeof(sun.sun_path), "%s",
 307                    server->unix_sock_path);
 308     if (ret < 0 || ret >= sizeof(sun.sun_path)) {
 309         IVSHMEM_SERVER_DEBUG(server, "could not copy unix socket path\n");
 310         goto err_close_sock;
 311     }
 312     if (bind(sock_fd, (struct sockaddr *)&sun, sizeof(sun)) < 0) {
 313         IVSHMEM_SERVER_DEBUG(server, "cannot connect to %s: %s\n", sun.sun_path,
 314                              strerror(errno));
 315         goto err_close_sock;
 316     }
 317
 318     if (listen(sock_fd, IVSHMEM_SERVER_LISTEN_BACKLOG) < 0) {
 319         IVSHMEM_SERVER_DEBUG(server, "listen() failed: %s\n", strerror(errno));
 320         goto err_close_sock;
 321     }
 322
 323     server->sock_fd = sock_fd;
 324     server->shm_fd = shm_fd;
 325
 326     return 0;
 327
 328 err_close_sock:
 329     close(sock_fd);
 330 err_close_shm:
 331     close(shm_fd);
 332     return -1;
 333 }
 334
 335 /* close connections to clients, the unix socket and the shm fd */
 336 void
 337 ivshmem_server_close(IvshmemServer *server)
 338 {
 339     IvshmemServerPeer *peer, *npeer;
 340
 341     IVSHMEM_SERVER_DEBUG(server, "close server\n");
 342
 343     QTAILQ_FOREACH_SAFE(peer, &server->peer_list, next, npeer) {
 344         ivshmem_server_free_peer(server, peer);
 345     }
 346
 347     unlink(server->unix_sock_path);
 348     close(server->sock_fd);
 349     close(server->shm_fd);
 350     server->sock_fd = -1;
 351     server->shm_fd = -1;
 352 }
 353
 354 /* get the fd_set according to the unix socket and the peer list */
 355 void
 356 ivshmem_server_get_fds(const IvshmemServer *server, fd_set *fds, int *maxfd)
 357 {
 358     IvshmemServerPeer *peer;
 359
 360     if (server->sock_fd == -1) {
 361         return;
 362     }
 363
 364     FD_SET(server->sock_fd, fds);
 365     if (server->sock_fd >= *maxfd) {
 366         *maxfd = server->sock_fd + 1;
 367     }
 368
 369     QTAILQ_FOREACH(peer, &server->peer_list, next) {
 370         FD_SET(peer->sock_fd, fds);
 371         if (peer->sock_fd >= *maxfd) {
 372             *maxfd = peer->sock_fd + 1;
 373         }
 374     }
 375 }
 376
 377 /* process incoming messages on the sockets in fd_set */
 378 int
 379 ivshmem_server_handle_fds(IvshmemServer *server, fd_set *fds, int maxfd)
 380 {
 381     IvshmemServerPeer *peer, *peer_next;
 382
 383     if (server->sock_fd < maxfd && FD_ISSET(server->sock_fd, fds) &&
 384         ivshmem_server_handle_new_conn(server) < 0 && errno != EINTR) {
 385         IVSHMEM_SERVER_DEBUG(server, "ivshmem_server_handle_new_conn() "
 386                              "failed\n");
 387         return -1;
 388     }
 389
 390     QTAILQ_FOREACH_SAFE(peer, &server->peer_list, next, peer_next) {
 391         /* any message from a peer socket result in a close() */
 392         IVSHMEM_SERVER_DEBUG(server, "peer->sock_fd=%d\n", peer->sock_fd);
 393         if (peer->sock_fd < maxfd && FD_ISSET(peer->sock_fd, fds)) {
 394             ivshmem_server_free_peer(server, peer);
 395         }
 396     }
 397
 398     return 0;
 399 }
 400
 401 /* lookup peer from its id */
 402 IvshmemServerPeer *
 403 ivshmem_server_search_peer(IvshmemServer *server, long peer_id)
 404 {
 405     IvshmemServerPeer *peer;
 406
 407     QTAILQ_FOREACH(peer, &server->peer_list, next) {
 408         if (peer->id == peer_id) {
 409             return peer;
 410         }
 411     }
 412     return NULL;
 413 }
 414
 415 /* dump our info, the list of peers their vectors on stdout */
 416 void
 417 ivshmem_server_dump(const IvshmemServer *server)
 418 {
 419     const IvshmemServerPeer *peer;
 420     unsigned vector;
 421
 422     /* dump peers */
 423     QTAILQ_FOREACH(peer, &server->peer_list, next) {
 424         printf("peer_id = %ld\n", peer->id);
 425
 426         for (vector = 0; vector < peer->vectors_count; vector++) {
 427             printf("  vector %d is enabled (fd=%d)\n", vector,
 428                    event_notifier_get_fd(&peer->vectors[vector]));
 429         }
 430     }
 431 }