src/ct_server.c

   1 /*
   2  * curvetun - the cipherspace wormhole creator
   3  * Part of the netsniff-ng project
   4  * By Daniel Borkmann <daniel@netsniff-ng.org>
   5  * Copyright 2011 Daniel Borkmann <daniel@netsniff-ng.org>,
   6  * Subject to the GPL.
   7  */
   8
   9 #define _GNU_SOURCE
  10 #include <stdio.h>
  11 #include <stdlib.h>
  12 #include <fcntl.h>
  13 #include <errno.h>
  14 #include <string.h>
  15 #include <unistd.h>
  16 #include <pthread.h>
  17 #include <syslog.h>
  18 #include <signal.h>
  19 #include <netdb.h>
  20 #include <stdint.h>
  21 #include <netinet/in.h>
  22 #include <netinet/tcp.h>
  23 #include <netinet/udp.h>
  24 #include <sys/poll.h>
  25 #include <sys/types.h>
  26 #include <sys/socket.h>
  27 #include <sys/wait.h>
  28 #include <sys/epoll.h>
  29 #include <arpa/inet.h>
  30
  31 #include "die.h"
  32 #include "netdev.h"
  33 #include "write_or_die.h"
  34 #include "psched.h"
  35 #include "xmalloc.h"
  36 #include "curvetun.h"
  37 #include "compiler.h"
  38 #include "deflate.h"
  39 #include "cpusched.h"
  40 #include "trie.h"
  41
  42 struct parent_info {
  43         int efd;
  44         int refd;
  45         int tunfd;
  46         int ipv4;
  47         int udp;
  48 };
  49
  50 struct worker_struct {
  51         pthread_t trid;
  52         int efd[2];
  53         unsigned int cpu;
  54         struct parent_info parent;
  55         int (*handler)(int fd, const struct worker_struct *ws,
  56                        char *buff, size_t len);
  57 };
  58
  59 static struct worker_struct *threadpool = NULL;
  60
  61 extern sig_atomic_t sigint;
  62
  63 static int handler_udp_tun_to_net(int fd, const struct worker_struct *ws,
  64                                   char *buff, size_t len) __pure;
  65 static int handler_udp_net_to_tun(int fd, const struct worker_struct *ws,
  66                                   char *buff, size_t len) __pure;
  67 static int handler_udp(int fd, const struct worker_struct *ws,
  68                        char *buff, size_t len) __pure;
  69 static int handler_tcp_tun_to_net(int fd, const struct worker_struct *ws,
  70                                   char *buff, size_t len) __pure;
  71 static int handler_tcp_net_to_tun(int fd, const struct worker_struct *ws,
  72                                   char *buff, size_t len) __pure;
  73 static int handler_tcp(int fd, const struct worker_struct *ws,
  74                        char *buff, size_t len) __pure;
  75 static void *worker(void *self) __pure;
  76
  77 static int handler_udp_tun_to_net(int fd, const struct worker_struct *ws,
  78                                   char *buff, size_t len)
  79 {
  80         int dfd, state, keep = 1;
  81         char *pbuff;
  82         ssize_t rlen, err, plen;
  83         struct ct_proto *hdr;
  84         struct sockaddr_storage naddr;
  85         socklen_t nlen;
  86
  87         errno = 0;
  88         while ((rlen = read(fd, buff + sizeof(struct ct_proto),
  89                             len - sizeof(struct ct_proto))) > 0) {
  90                 dfd = -1;
  91                 nlen = 0;
  92                 memset(&naddr, 0, sizeof(naddr));
  93
  94                 hdr = (struct ct_proto *) buff;
  95                 hdr->canary = htons(CANARY);
  96                 hdr->flags = 0;
  97
  98                 trie_addr_lookup(buff + sizeof(struct ct_proto),
  99                                  rlen - sizeof(struct ct_proto),
 100                                  ws->parent.ipv4, &dfd, &naddr,
 101                                  (size_t *) &nlen);
 102
 103                 if (dfd < 0 || nlen == 0) {
 104                         syslog(LOG_INFO, "CPU%u: UDP tunnel lookup failed: "
 105                                "unknown destination\n", ws->cpu);
 106                         continue;
 107                 }
 108
 109                 plen = z_deflate(buff + sizeof(struct ct_proto),
 110                                  rlen - sizeof(struct ct_proto), &pbuff);
 111                 if (plen < 0) {
 112                         syslog(LOG_ERR, "CPU%u: UDP tunnel deflate error: %s\n",
 113                                ws->cpu, strerror(errno));
 114                         continue;
 115                 }
 116
 117                 hdr->payload = htons((uint16_t) plen);
 118
 119                 state = 1;
 120                 setsockopt(dfd, IPPROTO_UDP, UDP_CORK, &state, sizeof(state));
 121
 122                 err = sendto(dfd, hdr, sizeof(struct ct_proto), 0,
 123                              (struct sockaddr *) &naddr, nlen);
 124                 if (err < 0)
 125                         syslog(LOG_ERR, "CPU%u: UDP tunnel write error: %s\n",
 126                                ws->cpu, strerror(errno));
 127
 128                 err = sendto(dfd, pbuff, plen, 0, (struct sockaddr *) &naddr,
 129                              nlen);
 130                 if (err < 0)
 131                         syslog(LOG_ERR, "CPU%u: UDP tunnel write error: %s\n",
 132                                ws->cpu, strerror(errno));
 133
 134                 state = 0;
 135                 setsockopt(dfd, IPPROTO_UDP, UDP_CORK, &state, sizeof(state));
 136
 137                 errno = 0;
 138         }
 139
 140         if (rlen < 0 && errno != EAGAIN)
 141                 syslog(LOG_ERR, "CPU%u: UDP tunnel read error: %s\n",
 142                        ws->cpu, strerror(errno));
 143
 144         return keep;
 145 }
 146
 147 static void handler_udp_notify_close(int fd, struct sockaddr_storage *addr,
 148                                      socklen_t len)
 149 {
 150         ssize_t err;
 151         struct ct_proto hdr;
 152
 153         memset(&hdr, 0, sizeof(hdr));
 154         hdr.flags |= PROTO_FLAG_EXIT;
 155         hdr.payload = 0;
 156         hdr.canary = htons(CANARY);
 157
 158         err = sendto(fd, &hdr, sizeof(hdr), 0, (struct sockaddr *) addr, len);
 159 }
 160
 161 static int handler_udp_net_to_tun(int fd, const struct worker_struct *ws,
 162                                   char *buff, size_t len)
 163 {
 164         int keep = 1;
 165         char *pbuff;
 166         ssize_t rlen, err, plen;
 167         struct ct_proto *hdr;
 168         struct sockaddr_storage naddr;
 169         socklen_t nlen;
 170
 171         nlen = sizeof(naddr);
 172         memset(&naddr, 0, sizeof(naddr));
 173
 174         errno = 0;
 175         while ((rlen = recvfrom(fd, buff, len, 0, (struct sockaddr *) &naddr,
 176                                 &nlen)) > 0) {
 177                 hdr = (struct ct_proto *) buff;
 178
 179                 if (unlikely(rlen < sizeof(struct ct_proto)))
 180                         goto close;
 181                 if (unlikely(rlen - sizeof(*hdr) != ntohs(hdr->payload)))
 182                         goto close;
 183                 if (unlikely(ntohs(hdr->canary) != CANARY))
 184                         goto close;
 185                 if (unlikely(ntohs(hdr->payload) == 0))
 186                         goto close;
 187
 188                 err = trie_addr_maybe_update(buff + sizeof(struct ct_proto),
 189                                              rlen - sizeof(struct ct_proto),
 190                                              ws->parent.ipv4, fd,
 191                                              &naddr, nlen);
 192                 if (err) {
 193                         syslog(LOG_INFO, "CPU%u: Malicious packet dropped "
 194                                "from id %d\n", ws->cpu, fd);
 195                         continue;
 196                 }
 197
 198                 if (hdr->flags & PROTO_FLAG_EXIT) {
 199 close:
 200                         trie_addr_remove_addr(&naddr, nlen);
 201                         handler_udp_notify_close(fd, &naddr, nlen);
 202                         nlen = sizeof(naddr);
 203                         memset(&naddr, 0, sizeof(naddr));
 204                         continue;
 205                 }
 206
 207                 plen = z_inflate(buff + sizeof(struct ct_proto),
 208                                  rlen - sizeof(struct ct_proto), &pbuff);
 209                 if (plen < 0) {
 210                         syslog(LOG_ERR, "CPU%u: UDP net inflate error: %s\n",
 211                                ws->cpu, strerror(errno));
 212                         goto next;
 213                 }
 214
 215                 err = write(ws->parent.tunfd, pbuff, plen);
 216                 if (err < 0)
 217                         syslog(LOG_ERR, "CPU%u: UDP net write error: %s\n",
 218                                ws->cpu, strerror(errno));
 219
 220 next:
 221                 nlen = sizeof(naddr);
 222                 memset(&naddr, 0, sizeof(naddr));
 223                 errno = 0;
 224         }
 225
 226         if (rlen < 0 && errno != EAGAIN)
 227                 syslog(LOG_ERR, "CPU%u: UDP net read error: %s\n",
 228                        ws->cpu, strerror(errno));
 229
 230         return keep;
 231 }
 232
 233 static int handler_udp(int fd, const struct worker_struct *ws,
 234                        char *buff, size_t len)
 235 {
 236         int ret = 0;
 237         if (fd == ws->parent.tunfd)
 238                 ret = handler_udp_tun_to_net(fd, ws, buff, len);
 239         else
 240                 ret = handler_udp_net_to_tun(fd, ws, buff, len);
 241         return ret;
 242 }
 243
 244 static int handler_tcp_tun_to_net(int fd, const struct worker_struct *ws,
 245                                   char *buff, size_t len)
 246 {
 247         int dfd, state, keep = 1;
 248         char *pbuff;
 249         ssize_t rlen, err, plen;
 250         struct ct_proto *hdr;
 251         socklen_t nlen;
 252
 253         errno = 0;
 254         while ((rlen = read(fd, buff + sizeof(struct ct_proto),
 255                             len - sizeof(struct ct_proto))) > 0) {
 256                 dfd = -1;
 257
 258                 hdr = (struct ct_proto *) buff;
 259                 hdr->canary = htons(CANARY);
 260                 hdr->flags = 0;
 261
 262                 trie_addr_lookup(buff + sizeof(struct ct_proto),
 263                                  rlen - sizeof(struct ct_proto),
 264                                  ws->parent.ipv4, &dfd, NULL,
 265                                  (size_t *) &nlen);
 266
 267                 if (dfd < 0) {
 268                         syslog(LOG_INFO, "CPU%u: TCP tunnel lookup failed: "
 269                                "unknown destination\n", ws->cpu);
 270                         continue;
 271                 }
 272
 273                 plen = z_deflate(buff + sizeof(struct ct_proto),
 274                                  rlen - sizeof(struct ct_proto), &pbuff);
 275                 if (plen < 0) {
 276                         syslog(LOG_ERR, "CPU%u: TCP tunnel deflate error: %s\n",
 277                                ws->cpu, strerror(errno));
 278                         continue;
 279                 }
 280
 281                 hdr->payload = htons((uint16_t) plen);
 282
 283                 state = 1;
 284                 setsockopt(dfd, IPPROTO_TCP, TCP_CORK, &state, sizeof(state));
 285
 286                 err = write_exact(dfd, hdr, sizeof(struct ct_proto), 0);
 287                 if (err < 0)
 288                         syslog(LOG_ERR, "CPU%u: TCP tunnel write error: %s\n",
 289                                ws->cpu, strerror(errno));
 290
 291                 err = write_exact(dfd, pbuff, plen, 0);
 292                 if (err < 0)
 293                         syslog(LOG_ERR, "CPU%u: TCP tunnel write error: %s\n",
 294                                ws->cpu, strerror(errno));
 295
 296                 state = 0;
 297                 setsockopt(dfd, IPPROTO_TCP, TCP_CORK, &state, sizeof(state));
 298
 299                 errno = 0;
 300         }
 301
 302         if (rlen < 0 && errno != EAGAIN)
 303                 syslog(LOG_ERR, "CPU%u: TCP tunnel read error: %s\n",
 304                        ws->cpu, strerror(errno));
 305
 306         return keep;
 307 }
 308
 309 ssize_t handler_tcp_read(int fd, char *buff, size_t len)
 310 {
 311         ssize_t rlen;
 312         struct ct_proto *hdr = (struct ct_proto *) buff;
 313
 314         /* May exit on EAGAIN if 0 Byte read */
 315         rlen = read_exact(fd, buff, sizeof(struct ct_proto), 1);
 316         if (rlen < 0)
 317                 return rlen;
 318
 319         /* May not exit on EAGAIN if 0 Byte read */
 320         rlen = read_exact(fd, buff + sizeof(struct ct_proto),
 321                           ntohs(hdr->payload), 0);
 322         if (rlen < 0)
 323                 return rlen;
 324
 325         return sizeof(struct ct_proto) + rlen;
 326 }
 327
 328 static void handler_tcp_notify_close(int fd)
 329 {
 330         ssize_t err;
 331         struct ct_proto hdr;
 332
 333         memset(&hdr, 0, sizeof(hdr));
 334         hdr.flags |= PROTO_FLAG_EXIT;
 335         hdr.payload = 0;
 336         hdr.canary = htons(CANARY);
 337
 338         err = write(fd, &hdr, sizeof(hdr));
 339 }
 340
 341 static int handler_tcp_net_to_tun(int fd, const struct worker_struct *ws,
 342                                   char *buff, size_t len)
 343 {
 344         int keep = 1, count = 0;
 345         char *pbuff;
 346         ssize_t rlen, err, plen;
 347         struct ct_proto *hdr;
 348         uint64_t fd64;
 349
 350         errno = 0;
 351         while ((rlen = handler_tcp_read(fd, buff, len)) > 0) {
 352                 hdr = (struct ct_proto *) buff;
 353
 354                 if (unlikely(rlen < sizeof(struct ct_proto)))
 355                         goto close;
 356                 if (unlikely(rlen - sizeof(*hdr) != ntohs(hdr->payload)))
 357                         goto close;
 358                 if (unlikely(ntohs(hdr->canary) != CANARY))
 359                         goto close;
 360                 if (unlikely(ntohs(hdr->payload) == 0))
 361                         goto close;
 362
 363                 err = trie_addr_maybe_update(buff + sizeof(struct ct_proto),
 364                                              rlen - sizeof(struct ct_proto),
 365                                              ws->parent.ipv4, fd, NULL, 0);
 366                 if (err) {
 367                         syslog(LOG_INFO, "CPU%u: Malicious packet dropped "
 368                                "from id %d\n", ws->cpu, fd);
 369                         continue;
 370                 }
 371
 372                 if (hdr->flags & PROTO_FLAG_EXIT) {
 373 close:
 374                         fd64 = fd;
 375                         trie_addr_remove(fd);
 376                         handler_tcp_notify_close(fd);
 377                         rlen = write(ws->parent.efd, &fd64, sizeof(fd64));
 378                         if (rlen != sizeof(fd64))
 379                                 syslog(LOG_ERR, "CPU%u: TCP event write error: %s\n",
 380                                        ws->cpu, strerror(errno));
 381                         keep = 0;
 382                         return keep;
 383                 }
 384
 385                 plen = z_inflate(buff + sizeof(struct ct_proto),
 386                                  rlen - sizeof(struct ct_proto), &pbuff);
 387                 if (plen < 0) {
 388                         syslog(LOG_ERR, "CPU%u: TCP net inflate error: %s\n",
 389                                ws->cpu, strerror(errno));
 390                         continue;
 391                 }
 392
 393                 err = write(ws->parent.tunfd, pbuff, plen);
 394                 if (err < 0)
 395                         syslog(LOG_ERR, "CPU%u: TCP net write error: %s\n",
 396                                ws->cpu, strerror(errno));
 397
 398                 count++;
 399                 if (count == 10) {
 400                         err = write_exact(ws->efd[1], &fd, sizeof(fd), 1);
 401                         if (err != sizeof(fd))
 402                                 syslog(LOG_ERR, "CPU%u: TCP net put fd back in "
 403                                        "pipe error: %s\n", ws->cpu, strerror(errno));
 404                         return keep;
 405                 }
 406
 407                 errno = 0;
 408         }
 409
 410         if (rlen < 0 && errno != EAGAIN && errno != EBADF)
 411                 syslog(LOG_ERR, "CPU%u: TCP net read error: %s\n",
 412                        ws->cpu, strerror(errno));
 413
 414         return keep;
 415 }
 416
 417 static int handler_tcp(int fd, const struct worker_struct *ws,
 418                        char *buff, size_t len)
 419 {
 420         int ret = 0;
 421         if (fd == ws->parent.tunfd)
 422                 ret = handler_tcp_tun_to_net(fd, ws, buff, len);
 423         else
 424                 ret = handler_tcp_net_to_tun(fd, ws, buff, len);
 425         return ret;
 426 }
 427
 428 static void *worker(void *self)
 429 {
 430         int fd, old_state;
 431         ssize_t ret;
 432         size_t blen = TUNBUFF_SIZ; //FIXME
 433         const struct worker_struct *ws = self;
 434         struct pollfd fds;
 435         char *buff;
 436
 437         fds.fd = ws->efd[0];
 438         fds.events = POLLIN;
 439
 440         buff = xmalloc(blen);
 441         syslog(LOG_INFO, "curvetun thread on CPU%u up!\n", ws->cpu);
 442         pthread_cleanup_push(xfree, buff);
 443
 444         while (likely(!sigint)) {
 445                 poll(&fds, 1, -1);
 446                 if ((fds.revents & POLLIN) != POLLIN)
 447                         continue;
 448                 pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old_state);
 449                 while ((ret = read_exact(ws->efd[0], &fd, sizeof(fd), 1)) > 0) {
 450                         if (ret != sizeof(fd)) {
 451                                 syslog(LOG_ERR, "CPU%u: Thread could not read "
 452                                        "event descriptor!\n", ws->cpu);
 453                                 sched_yield();
 454                                 continue;
 455                         }
 456
 457                         ret = ws->handler(fd, ws, buff, blen);
 458                         if (ret) {
 459                                 ret = write_exact(ws->parent.refd, &fd, sizeof(fd), 1);
 460                                 if (ret != sizeof(fd))
 461                                         syslog(LOG_ERR, "CPU%u: Retriggering failed: "
 462                                                "%s\n", ws->cpu, strerror(errno));
 463                         }
 464                 }
 465                 pthread_setcancelstate(old_state, NULL);
 466         }
 467
 468         syslog(LOG_INFO, "curvetun thread on CPU%u down!\n", ws->cpu);
 469         pthread_cleanup_pop(1);
 470         pthread_exit((void *) ((long) ws->cpu));
 471 }
 472
 473 static void thread_spawn_or_panic(unsigned int cpus, int efd, int refd,
 474                                   int tunfd, int ipv4, int udp)
 475 {
 476         int i, ret;
 477         cpu_set_t cpuset;
 478         unsigned int threads;
 479
 480         threads = cpus * THREADS_PER_CPU;
 481         for (i = 0; i < threads; ++i) {
 482                 CPU_ZERO(&cpuset);
 483                 threadpool[i].cpu = i % cpus;
 484                 CPU_SET(threadpool[i].cpu, &cpuset);
 485
 486                 ret = pipe2(threadpool[i].efd, O_NONBLOCK);
 487                 if (ret < 0)
 488                         panic("Cannot create event socket!\n");
 489
 490                 threadpool[i].parent.efd = efd;
 491                 threadpool[i].parent.refd = refd;
 492                 threadpool[i].parent.tunfd = tunfd;
 493                 threadpool[i].parent.ipv4 = ipv4;
 494                 threadpool[i].parent.udp = udp;
 495                 threadpool[i].handler = udp ? handler_udp : handler_tcp;
 496
 497                 ret = pthread_create(&threadpool[i].trid, NULL,
 498                                      worker, &threadpool[i]);
 499                 if (ret < 0)
 500                         panic("Thread creation failed!\n");
 501
 502                 ret = pthread_setaffinity_np(threadpool[i].trid,
 503                                              sizeof(cpu_set_t), &cpuset);
 504                 if (ret < 0)
 505                         panic("Thread CPU migration failed!\n");
 506
 507                 pthread_detach(threadpool[i].trid);
 508         }
 509
 510         sleep(1);
 511 }
 512
 513 static void thread_finish(unsigned int cpus)
 514 {
 515         int i, ret;
 516         unsigned int threads;
 517
 518         threads = cpus * THREADS_PER_CPU;
 519         for (i = 0; i < threads; ++i) {
 520                 ret = pthread_join(threadpool[i].trid, NULL);
 521                 if (ret < 0)
 522                         continue;
 523                 close(threadpool[i].efd[0]);
 524                 close(threadpool[i].efd[1]);
 525         }
 526 }
 527
 528 int server_main(int port, int udp, int lnum)
 529 {
 530         int lfd = -1, kdpfd, nfds, nfd, curfds, efd[2], refd[2], tunfd;
 531         int ipv4 = 0, i;
 532         unsigned int cpus = 0, threads;
 533         ssize_t ret;
 534         struct epoll_event ev, *events;
 535         struct addrinfo hints, *ahead, *ai;
 536
 537         openlog("curvetun", LOG_PID | LOG_CONS | LOG_NDELAY, LOG_DAEMON);
 538         syslog(LOG_INFO, "curvetun server booting!\n");
 539
 540         ret = z_alloc_or_maybe_die(Z_DEFAULT_COMPRESSION);
 541         if (ret < 0)
 542                 panic("Cannot init zLib!\n");
 543
 544         memset(&hints, 0, sizeof(hints));
 545         hints.ai_family = PF_UNSPEC;
 546         hints.ai_socktype = udp ? SOCK_DGRAM : SOCK_STREAM;
 547         hints.ai_protocol = udp ? IPPROTO_UDP : IPPROTO_TCP;
 548         hints.ai_flags = AI_PASSIVE;
 549
 550         ret = getaddrinfo(NULL, "6666", &hints, &ahead);
 551         if (ret < 0)
 552                 panic("Cannot get address info!\n");
 553
 554         for (ai = ahead; ai != NULL && lfd < 0; ai = ai->ai_next) {
 555                 lfd = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol);
 556                 if (lfd < 0)
 557                         continue;
 558                 if (ai->ai_family == AF_INET6) {
 559                         int one = 1;
 560 #ifdef IPV6_V6ONLY
 561                         ret = setsockopt(lfd, IPPROTO_IPV6, IPV6_V6ONLY,
 562                                          &one, sizeof(one));
 563                         if (ret < 0) {
 564                                 close(lfd);
 565                                 lfd = -1;
 566                                 continue;
 567                         }
 568 #else
 569                         close(lfd);
 570                         lfd = -1;
 571                         continue;
 572 #endif /* IPV6_V6ONLY */
 573                 }
 574                 set_reuseaddr(lfd);
 575                 ret = bind(lfd, ai->ai_addr, ai->ai_addrlen);
 576                 if (ret < 0) {
 577                         close(lfd);
 578                         lfd = -1;
 579                         continue;
 580                 }
 581                 if (!udp) {
 582                         ret = listen(lfd, 5);
 583                         if (ret < 0) {
 584                                 close(lfd);
 585                                 lfd = -1;
 586                                 continue;
 587                         }
 588                 }
 589                 ipv4 = (ai->ai_family == AF_INET6 ? 0 :
 590                         (ai->ai_family == AF_INET ? 1 : -1));
 591                 syslog(LOG_INFO, "curvetun on IPv%d via %s!\n",
 592                        ipv4 ? 4 : 6, udp ? "UDP" : "TCP");
 593         }
 594
 595         freeaddrinfo(ahead);
 596         if (lfd < 0 || ipv4 < 0)
 597                 panic("Cannot create socket!\n");
 598
 599         tunfd = tun_open_or_die(DEVNAME_SERVER);
 600
 601         ret = pipe2(efd, O_NONBLOCK);
 602         if (ret < 0)
 603                 panic("Cannot create parent event fd!\n");
 604
 605         ret = pipe2(refd, O_NONBLOCK);
 606         if (ret < 0)
 607                 panic("Cannot create parent (r)event fd!\n");
 608
 609         set_nonblocking(lfd);
 610
 611         events = xzmalloc(MAX_EPOLL_SIZE * sizeof(*events));
 612         for (i = 0; i < MAX_EPOLL_SIZE; ++i)
 613                 events[i].data.fd = -1;
 614
 615         kdpfd = epoll_create(MAX_EPOLL_SIZE);
 616         if (kdpfd < 0)
 617                 panic("Cannot create socket!\n");
 618
 619         memset(&ev, 0, sizeof(ev));
 620         ev.events = udp ? EPOLLIN | EPOLLET | EPOLLONESHOT : EPOLLIN;
 621         ev.data.fd = lfd;
 622         ret = epoll_ctl(kdpfd, EPOLL_CTL_ADD, lfd, &ev);
 623         if (ret < 0)
 624                 panic("Cannot add socket for epoll!\n");
 625
 626         memset(&ev, 0, sizeof(ev));
 627         ev.events = EPOLLIN;
 628         ev.data.fd = efd[0];
 629         ret = epoll_ctl(kdpfd, EPOLL_CTL_ADD, efd[0], &ev);
 630         if (ret < 0)
 631                 panic("Cannot add socket for events!\n");
 632
 633         memset(&ev, 0, sizeof(ev));
 634         ev.events = EPOLLIN;
 635         ev.data.fd = refd[0];
 636         ret = epoll_ctl(kdpfd, EPOLL_CTL_ADD, refd[0], &ev);
 637         if (ret < 0)
 638                 panic("Cannot add socket for (r)events!\n");
 639
 640         memset(&ev, 0, sizeof(ev));
 641         ev.events = EPOLLIN | EPOLLET | EPOLLONESHOT;
 642         ev.data.fd = tunfd;
 643         ret = epoll_ctl(kdpfd, EPOLL_CTL_ADD, tunfd, &ev);
 644         if (ret < 0)
 645                 panic("Cannot add socket for tundev!\n");
 646
 647         curfds = 4;
 648
 649         trie_init();
 650
 651         cpus = get_number_cpus_online();
 652         threads = cpus * THREADS_PER_CPU;
 653         if (!((threads != 0) && ((threads & (threads - 1)) == 0)))
 654                 panic("thread number not power of two!\n");
 655         threadpool = xzmalloc(sizeof(*threadpool) * threads);
 656         thread_spawn_or_panic(cpus, efd[1], refd[1], tunfd, ipv4, udp);
 657
 658         init_cpusched(threads, MAX_EPOLL_SIZE);
 659         register_socket(tunfd);
 660         register_socket(lfd);
 661
 662         syslog(LOG_INFO, "tunnel id: %d, listener id: %d\n", tunfd, lfd);
 663         syslog(LOG_INFO, "curvetun up and running!\n");
 664
 665         while (likely(!sigint)) {
 666                 nfds = epoll_wait(kdpfd, events, curfds, -1);
 667                 if (nfds < 0) {
 668                         syslog(LOG_ERR, "epoll_wait error: %s\n",
 669                                strerror(errno));
 670                         break;
 671                 }
 672
 673                 for (i = 0; i < nfds; ++i) {
 674                         if (unlikely(events[i].data.fd < 0))
 675                                 continue;
 676                         if (events[i].data.fd == lfd && !udp) {
 677                                 int one, ncpu;
 678                                 char hbuff[256], sbuff[256];
 679                                 struct sockaddr_storage taddr;
 680                                 socklen_t tlen;
 681
 682                                 tlen = sizeof(taddr);
 683                                 nfd = accept(lfd, (struct sockaddr *) &taddr,
 684                                              &tlen);
 685                                 if (nfd < 0) {
 686                                         syslog(LOG_ERR, "accept error: %s\n",
 687                                                strerror(errno));
 688                                         continue;
 689                                 }
 690
 691                                 if (curfds + 1 > MAX_EPOLL_SIZE) {
 692                                         close(nfd);
 693                                         continue;
 694                                 }
 695
 696                                 curfds++;
 697                                 ncpu = register_socket(nfd);
 698
 699                                 memset(hbuff, 0, sizeof(hbuff));
 700                                 memset(sbuff, 0, sizeof(sbuff));
 701
 702                                 getnameinfo((struct sockaddr *) &taddr, tlen,
 703                                             hbuff, sizeof(hbuff),
 704                                             sbuff, sizeof(sbuff),
 705                                             NI_NUMERICHOST | NI_NUMERICSERV);
 706
 707                                 syslog(LOG_INFO, "New connection from %s:%s "
 708                                        "with id %d on CPU%d, %d active!\n",
 709                                        hbuff, sbuff, nfd, ncpu, curfds);
 710
 711                                 set_nonblocking(nfd);
 712
 713                                 one = 1;
 714                                 setsockopt(nfd, SOL_SOCKET, SO_KEEPALIVE,
 715                                            &one, sizeof(one));
 716                                 one = 1;
 717                                 setsockopt(nfd, IPPROTO_TCP, TCP_NODELAY,
 718                                            &one, sizeof(one));
 719
 720                                 memset(&ev, 0, sizeof(ev));
 721                                 ev.events = EPOLLIN | EPOLLET | EPOLLONESHOT;
 722                                 ev.data.fd = nfd;
 723                                 ret = epoll_ctl(kdpfd, EPOLL_CTL_ADD, nfd, &ev);
 724                                 if (ret < 0) {
 725                                         syslog(LOG_ERR, "Epoll ctl add error"
 726                                                "on id %d: %s\n", nfd,
 727                                                strerror(errno));
 728                                         close(nfd);
 729                                         curfds--;
 730                                         continue;
 731                                 }
 732                         } else if (events[i].data.fd == refd[0]) {
 733                                 int fd_one;
 734
 735                                 ret = read_exact(refd[0], &fd_one, sizeof(fd_one), 1);
 736                                 if (ret != sizeof(fd_one) || fd_one <= 0)
 737                                         continue;
 738
 739                                 memset(&ev, 0, sizeof(ev));
 740                                 ev.events = EPOLLIN | EPOLLET | EPOLLONESHOT;
 741                                 ev.data.fd = fd_one;
 742                                 ret = epoll_ctl(kdpfd, EPOLL_CTL_MOD, fd_one, &ev);
 743                                 if (ret < 0) {
 744                                         syslog(LOG_ERR, "Epoll ctl mod "
 745                                                "error on id %d: %s\n",
 746                                                fd_one, strerror(errno));
 747                                         close(fd_one);
 748                                         continue;
 749                                 }
 750                         } else if (events[i].data.fd == efd[0]) {
 751                                 int fd_del, test;
 752
 753                                 ret = read_exact(efd[0], &fd_del, sizeof(fd_del), 1);
 754                                 if (ret != sizeof(fd_del) || fd_del <= 0)
 755                                         continue;
 756
 757                                 ret = read(fd_del, &test, sizeof(test));
 758                                 if (ret < 0 && errno == EBADF)
 759                                         continue;
 760
 761                                 ret = epoll_ctl(kdpfd, EPOLL_CTL_DEL, fd_del, &ev);
 762                                 if (ret < 0) {
 763                                         syslog(LOG_ERR, "Epoll ctl del "
 764                                                "error on id %d: %s\n",
 765                                                fd_del, strerror(errno));
 766                                         close(fd_del);
 767                                         continue;
 768                                 }
 769                                 close(fd_del);
 770                                 curfds--;
 771                                 unregister_socket(fd_del);
 772
 773                                 syslog(LOG_INFO, "Closed connection with "
 774                                        "id %d, %d active!\n",
 775                                        fd_del, curfds);
 776                         } else {
 777                                 int cpu, fd_work = events[i].data.fd;
 778                                 cpu = socket_to_cpu(fd_work);
 779
 780                                 ret = write_exact(threadpool[cpu].efd[1],
 781                                                   &fd_work, sizeof(fd_work), 1);
 782                                 if (ret != sizeof(fd_work))
 783                                         syslog(LOG_ERR, "Write error on event "
 784                                                "dispatch: %s\n", strerror(errno));
 785                         }
 786                 }
 787         }
 788
 789         syslog(LOG_INFO, "curvetun prepare shut down!\n");
 790
 791         close(lfd);
 792         close(efd[0]);
 793         close(efd[1]);
 794         close(refd[0]);
 795         close(refd[1]);
 796         close(tunfd);
 797
 798         thread_finish(cpus);
 799         xfree(threadpool);
 800         xfree(events);
 801
 802         unregister_socket(lfd);
 803         unregister_socket(tunfd);
 804         destroy_cpusched();
 805         trie_cleanup();
 806         z_free();
 807
 808         syslog(LOG_INFO, "curvetun shut down!\n");
 809         closelog();
 810
 811         return 0;
 812 }
 813