src/ct_server.c

   1 /*
   2  * curvetun - the cipherspace wormhole creator
   3  * Part of the netsniff-ng project
   4  * By Daniel Borkmann <daniel@netsniff-ng.org>
   5  * Copyright 2011 Daniel Borkmann <daniel@netsniff-ng.org>,
   6  * Subject to the GPL, version 2.
   7  */
   8
   9 #define _GNU_SOURCE
  10 #include <stdio.h>
  11 #include <stdlib.h>
  12 #include <fcntl.h>
  13 #include <errno.h>
  14 #include <string.h>
  15 #include <unistd.h>
  16 #include <pthread.h>
  17 #include <syslog.h>
  18 #include <signal.h>
  19 #include <netdb.h>
  20 #include <stdint.h>
  21 #include <netinet/in.h>
  22 #include <netinet/tcp.h>
  23 #include <netinet/udp.h>
  24 #include <sys/poll.h>
  25 #include <sys/types.h>
  26 #include <sys/socket.h>
  27 #include <sys/wait.h>
  28 #include <sys/epoll.h>
  29 #include <arpa/inet.h>
  30 #include <linux/if_tun.h>
  31
  32 #include "die.h"
  33 #include "xutils.h"
  34 #include "xio.h"
  35 #include "xmalloc.h"
  36 #include "curvetun.h"
  37 #include "curve.h"
  38 #include "built_in.h"
  39 #include "ct_usermgmt.h"
  40 #include "ct_cpusched.h"
  41 #include "trie.h"
  42
  43 struct parent_info {
  44         int efd;
  45         int refd;
  46         int tunfd;
  47         int ipv4;
  48         int udp;
  49 };
  50
  51 struct worker_struct {
  52         pthread_t trid;
  53         int efd[2];
  54         unsigned int cpu;
  55         struct parent_info parent;
  56         int (*handler)(int fd, const struct worker_struct *ws,
  57                        char *buff, size_t len);
  58         struct curve25519_struct *c;
  59 };
  60
  61 static struct worker_struct *threadpool = NULL;
  62
  63 static int auth_log = 1;
  64
  65 extern volatile sig_atomic_t sigint;
  66
  67 static int handler_udp_tun_to_net(int fd, const struct worker_struct *ws,
  68                                   char *buff, size_t len) __pure;
  69 static int handler_udp_net_to_tun(int fd, const struct worker_struct *ws,
  70                                   char *buff, size_t len) __pure;
  71 static int handler_udp(int fd, const struct worker_struct *ws,
  72                        char *buff, size_t len) __pure;
  73 static int handler_tcp_tun_to_net(int fd, const struct worker_struct *ws,
  74                                   char *buff, size_t len) __pure;
  75 static int handler_tcp_net_to_tun(int fd, const struct worker_struct *ws,
  76                                   char *buff, size_t len) __pure;
  77 static int handler_tcp(int fd, const struct worker_struct *ws,
  78                        char *buff, size_t len) __pure;
  79 ssize_t handler_tcp_read(int fd, char *buff, size_t len);
  80 static void *worker(void *self) __pure;
  81
  82 static int handler_udp_tun_to_net(int fd, const struct worker_struct *ws,
  83                                   char *buff, size_t len)
  84 {
  85         int dfd, keep = 1;
  86         char *cbuff;
  87         ssize_t rlen, err, clen;
  88         struct ct_proto *hdr;
  89         struct curve25519_proto *p;
  90         struct sockaddr_storage naddr;
  91         socklen_t nlen;
  92         size_t off = sizeof(struct ct_proto) + crypto_box_zerobytes;
  93
  94         if (!buff || len <= off)
  95                 return 0;
  96
  97         memset(buff, 0, len);
  98         while ((rlen = read(fd, buff + off, len - off)) > 0) {
  99                 dfd = -1; nlen = 0; p = NULL;
 100
 101                 memset(&naddr, 0, sizeof(naddr));
 102
 103                 hdr = (struct ct_proto *) buff;
 104                 memset(hdr, 0, sizeof(*hdr));
 105                 hdr->flags = 0;
 106
 107                 trie_addr_lookup(buff + off, rlen, ws->parent.ipv4, &dfd, &naddr,
 108                                  (size_t *) &nlen);
 109                 if (unlikely(dfd < 0 || nlen == 0)) {
 110                         memset(buff, 0, len);
 111                         continue;
 112                 }
 113
 114                 err = get_user_by_sockaddr(&naddr, nlen, &p);
 115                 if (unlikely(err || !p)) {
 116                         memset(buff, 0, len);
 117                         continue;
 118                 }
 119
 120                 clen = curve25519_encode(ws->c, p, (unsigned char *) (buff + off -
 121                                          crypto_box_zerobytes), (rlen +
 122                                          crypto_box_zerobytes), (unsigned char **)
 123                                          &cbuff);
 124                 if (unlikely(clen <= 0)) {
 125                         memset(buff, 0, len);
 126                         continue;
 127                 }
 128
 129                 hdr->payload = htons((uint16_t) clen);
 130
 131                 set_udp_cork(dfd);
 132
 133                 sendto(dfd, hdr, sizeof(struct ct_proto), 0, (struct sockaddr *)
 134                        &naddr, nlen);
 135                 sendto(dfd, cbuff, clen, 0, (struct sockaddr *) &naddr, nlen);
 136
 137                 set_udp_uncork(dfd);
 138
 139                 memset(buff, 0, len);
 140         }
 141
 142         return keep;
 143 }
 144
 145 static void handler_udp_notify_close(int fd, struct sockaddr_storage *addr)
 146 {
 147         struct ct_proto hdr;
 148
 149         memset(&hdr, 0, sizeof(hdr));
 150         hdr.flags |= PROTO_FLAG_EXIT;
 151         hdr.payload = 0;
 152
 153         sendto(fd, &hdr, sizeof(hdr), 0, (struct sockaddr *) addr,
 154                sizeof(*addr));
 155 }
 156
 157 static int handler_udp_net_to_tun(int fd, const struct worker_struct *ws,
 158                                   char *buff, size_t len)
 159 {
 160         int keep = 1;
 161         char *cbuff;
 162         ssize_t rlen, err, clen;
 163         struct ct_proto *hdr;
 164         struct curve25519_proto *p;
 165         struct sockaddr_storage naddr;
 166         socklen_t nlen = sizeof(naddr);
 167
 168         if (!buff || !len)
 169                 return 0;
 170
 171         memset(&naddr, 0, sizeof(naddr));
 172         while ((rlen = recvfrom(fd, buff, len, 0, (struct sockaddr *) &naddr,
 173                                 &nlen)) > 0) {
 174                 p = NULL;
 175
 176                 hdr = (struct ct_proto *) buff;
 177
 178                 if (unlikely(rlen < sizeof(struct ct_proto)))
 179                         goto close;
 180                 if (unlikely(rlen - sizeof(*hdr) != ntohs(hdr->payload)))
 181                         goto close;
 182                 if (unlikely(ntohs(hdr->payload) == 0))
 183                         goto close;
 184                 if (hdr->flags & PROTO_FLAG_EXIT) {
 185 close:
 186                         remove_user_by_sockaddr(&naddr, nlen);
 187                         trie_addr_remove_addr(&naddr, nlen);
 188                         handler_udp_notify_close(fd, &naddr);
 189
 190                         return keep;
 191                 }
 192                 if (hdr->flags & PROTO_FLAG_INIT) {
 193                         syslog_maybe(auth_log, LOG_INFO, "Got initial userhash "
 194                                      "from remote end!\n");
 195
 196                         if (unlikely(rlen - sizeof(*hdr) <
 197                                      sizeof(struct username_struct)))
 198                                 goto close;
 199
 200                         err = try_register_user_by_sockaddr(ws->c,
 201                                         buff + sizeof(struct ct_proto),
 202                                         rlen - sizeof(struct ct_proto),
 203                                         &naddr, nlen, auth_log);
 204                         if (unlikely(err))
 205                                 goto close;
 206
 207                         goto next;
 208                 }
 209
 210                 err = get_user_by_sockaddr(&naddr, nlen, &p);
 211                 if (unlikely(err || !p))
 212                         goto close;
 213
 214                 clen = curve25519_decode(ws->c, p, (unsigned char *) buff +
 215                                          sizeof(struct ct_proto),
 216                                          rlen - sizeof(struct ct_proto),
 217                                          (unsigned char **) &cbuff, NULL);
 218                 if (unlikely(clen <= 0))
 219                         goto close;
 220
 221                 cbuff += crypto_box_zerobytes;
 222                 clen -= crypto_box_zerobytes;
 223
 224                 err = trie_addr_maybe_update(cbuff, clen, ws->parent.ipv4,
 225                                              fd, &naddr, nlen);
 226                 if (unlikely(err))
 227                         goto next;
 228
 229                 err = write(ws->parent.tunfd, cbuff, clen);
 230 next:
 231                 nlen = sizeof(naddr);
 232                 memset(&naddr, 0, sizeof(naddr));
 233         }
 234
 235         return keep;
 236 }
 237
 238 static int handler_udp(int fd, const struct worker_struct *ws,
 239                        char *buff, size_t len)
 240 {
 241         int ret = 0;
 242
 243         if (fd == ws->parent.tunfd)
 244                 ret = handler_udp_tun_to_net(fd, ws, buff, len);
 245         else
 246                 ret = handler_udp_net_to_tun(fd, ws, buff, len);
 247
 248         return ret;
 249 }
 250
 251 static int handler_tcp_tun_to_net(int fd, const struct worker_struct *ws,
 252                                   char *buff, size_t len)
 253 {
 254         int dfd, keep = 1;
 255         char *cbuff;
 256         ssize_t rlen, err, clen;
 257         struct ct_proto *hdr;
 258         struct curve25519_proto *p;
 259         socklen_t nlen;
 260         size_t off = sizeof(struct ct_proto) + crypto_box_zerobytes;
 261
 262         if (!buff || len <= off)
 263                 return 0;
 264
 265         memset(buff, 0, len);
 266         while ((rlen = read(fd, buff + off, len - off)) > 0) {
 267                 dfd = -1; p = NULL;
 268
 269                 hdr = (struct ct_proto *) buff;
 270                 memset(hdr, 0, sizeof(*hdr));
 271                 hdr->flags = 0;
 272
 273                 trie_addr_lookup(buff + off, rlen, ws->parent.ipv4, &dfd, NULL,
 274                                  (size_t *) &nlen);
 275                 if (unlikely(dfd < 0)) {
 276                         memset(buff, 0, len);
 277                         continue;
 278                 }
 279
 280                 err = get_user_by_socket(dfd, &p);
 281                 if (unlikely(err || !p)) {
 282                         memset(buff, 0, len);
 283                         continue;
 284                 }
 285
 286                 clen = curve25519_encode(ws->c, p, (unsigned char *) (buff + off -
 287                                          crypto_box_zerobytes), (rlen +
 288                                          crypto_box_zerobytes), (unsigned char **)
 289                                          &cbuff);
 290                 if (unlikely(clen <= 0)) {
 291                         memset(buff, 0, len);
 292                         continue;
 293                 }
 294
 295                 hdr->payload = htons((uint16_t) clen);
 296
 297                 set_tcp_cork(dfd);
 298
 299                 write_exact(dfd, hdr, sizeof(struct ct_proto), 0);
 300                 write_exact(dfd, cbuff, clen, 0);
 301
 302                 set_tcp_uncork(dfd);
 303
 304                 memset(buff, 0, len);
 305         }
 306
 307         return keep;
 308 }
 309
 310 ssize_t handler_tcp_read(int fd, char *buff, size_t len)
 311 {
 312         ssize_t rlen;
 313         struct ct_proto *hdr = (struct ct_proto *) buff;
 314
 315         if (!buff || !len)
 316                 return 0;
 317
 318         /* May exit on EAGAIN if 0 Byte read */
 319         rlen = read_exact(fd, buff, sizeof(struct ct_proto), 1);
 320         if (rlen < 0)
 321                 return rlen;
 322         if (unlikely(ntohs(hdr->payload) > len - sizeof(struct ct_proto))) {
 323                 errno = ENOMEM;
 324                 return 1; /* Force server to close connection */
 325         }
 326
 327         /* May not exit on EAGAIN if 0 Byte read */
 328         rlen = read_exact(fd, buff + sizeof(struct ct_proto),
 329                           ntohs(hdr->payload), 0);
 330         if (rlen < 0)
 331                 return rlen;
 332
 333         return sizeof(struct ct_proto) + rlen;
 334 }
 335
 336 static void handler_tcp_notify_close(int fd)
 337 {
 338         ssize_t err;
 339         struct ct_proto hdr;
 340
 341         memset(&hdr, 0, sizeof(hdr));
 342         hdr.flags |= PROTO_FLAG_EXIT;
 343         hdr.payload = 0;
 344
 345         err = write(fd, &hdr, sizeof(hdr));
 346 }
 347
 348 static int handler_tcp_net_to_tun(int fd, const struct worker_struct *ws,
 349                                   char *buff, size_t len)
 350 {
 351         int keep = 1, count = 0;
 352         char *cbuff;
 353         ssize_t rlen, err, clen;
 354         struct ct_proto *hdr;
 355         struct curve25519_proto *p;
 356
 357         if (!buff || !len)
 358                 return 0;
 359
 360         while ((rlen = handler_tcp_read(fd, buff, len)) > 0) {
 361                 p = NULL;
 362
 363                 hdr = (struct ct_proto *) buff;
 364
 365                 if (unlikely(rlen < sizeof(struct ct_proto)))
 366                         goto close;
 367                 if (unlikely(rlen - sizeof(*hdr) != ntohs(hdr->payload)))
 368                         goto close;
 369                 if (unlikely(ntohs(hdr->payload) == 0))
 370                         goto close;
 371                 if (hdr->flags & PROTO_FLAG_EXIT) {
 372 close:
 373                         remove_user_by_socket(fd);
 374                         trie_addr_remove(fd);
 375                         handler_tcp_notify_close(fd);
 376                         rlen = write(ws->parent.efd, &fd, sizeof(fd));
 377
 378                         keep = 0;
 379                         return keep;
 380                 }
 381                 if (hdr->flags & PROTO_FLAG_INIT) {
 382                         syslog_maybe(auth_log, LOG_INFO, "Got initial userhash "
 383                                      "from remote end!\n");
 384
 385                         if (unlikely(rlen - sizeof(*hdr) <
 386                                      sizeof(struct username_struct)))
 387                                 goto close;
 388
 389                         err = try_register_user_by_socket(ws->c,
 390                                         buff + sizeof(struct ct_proto),
 391                                         rlen - sizeof(struct ct_proto),
 392                                         fd, auth_log);
 393                         if (unlikely(err))
 394                                 goto close;
 395
 396                         continue;
 397                 }
 398
 399                 err = get_user_by_socket(fd, &p);
 400                 if (unlikely(err || !p))
 401                         continue;
 402
 403                 clen = curve25519_decode(ws->c, p, (unsigned char *) buff +
 404                                          sizeof(struct ct_proto),
 405                                          rlen - sizeof(struct ct_proto),
 406                                          (unsigned char **) &cbuff, NULL);
 407                 if (unlikely(clen <= 0))
 408                         continue;
 409
 410                 cbuff += crypto_box_zerobytes;
 411                 clen -= crypto_box_zerobytes;
 412
 413                 err = trie_addr_maybe_update(cbuff, clen, ws->parent.ipv4,
 414                                              fd, NULL, 0);
 415                 if (unlikely(err))
 416                         continue;
 417
 418                 err = write(ws->parent.tunfd, cbuff, clen);
 419
 420                 count++;
 421                 if (count == 10) {
 422                         write_exact(ws->efd[1], &fd, sizeof(fd), 1);
 423                         /* Read later next data and let others process */
 424                         return keep;
 425                 }
 426         }
 427
 428         return keep;
 429 }
 430
 431 static int handler_tcp(int fd, const struct worker_struct *ws,
 432                        char *buff, size_t len)
 433 {
 434         int ret = 0;
 435
 436         if (fd == ws->parent.tunfd)
 437                 ret = handler_tcp_tun_to_net(fd, ws, buff, len);
 438         else
 439                 ret = handler_tcp_net_to_tun(fd, ws, buff, len);
 440
 441         return ret;
 442 }
 443
 444 static void *worker(void *self)
 445 {
 446         int fd, old_state;
 447         ssize_t ret;
 448         size_t blen = TUNBUFF_SIZ; //FIXME
 449         const struct worker_struct *ws = self;
 450         struct pollfd fds;
 451         char *buff;
 452
 453         fds.fd = ws->efd[0];
 454         fds.events = POLLIN;
 455
 456         ret = curve25519_alloc_or_maybe_die(ws->c);
 457         if (ret < 0)
 458                 syslog_panic("Cannot init curve25519!\n");
 459
 460         buff = xmalloc_aligned(blen, 64);
 461
 462         syslog(LOG_INFO, "curvetun thread on CPU%u up!\n", ws->cpu);
 463
 464         pthread_cleanup_push(xfree_func, ws->c);
 465         pthread_cleanup_push(curve25519_free, ws->c);
 466         pthread_cleanup_push(xfree_func, buff);
 467
 468         while (likely(!sigint)) {
 469                 poll(&fds, 1, -1);
 470                 if ((fds.revents & POLLIN) != POLLIN)
 471                         continue;
 472
 473                 pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old_state);
 474
 475                 while ((ret = read_exact(ws->efd[0], &fd, sizeof(fd), 1)) > 0) {
 476                         if (ret != sizeof(fd)) {
 477                                 sched_yield();
 478                                 continue;
 479                         }
 480
 481                         ret = ws->handler(fd, ws, buff, blen);
 482                         if (ret)
 483                                 write_exact(ws->parent.refd, &fd, sizeof(fd), 1);
 484                 }
 485
 486                 pthread_setcancelstate(old_state, NULL);
 487         }
 488
 489         syslog(LOG_INFO, "curvetun thread on CPU%u down!\n", ws->cpu);
 490
 491         pthread_cleanup_pop(1);
 492         pthread_cleanup_pop(1);
 493         pthread_cleanup_pop(1);
 494
 495         pthread_exit((void *) ((long) ws->cpu));
 496 }
 497
 498 static void thread_spawn_or_panic(unsigned int cpus, int efd, int refd,
 499                                   int tunfd, int ipv4, int udp)
 500 {
 501         int i, ret;
 502         cpu_set_t cpuset;
 503         unsigned int threads;
 504
 505         threads = cpus * THREADS_PER_CPU;
 506
 507         for (i = 0; i < threads; ++i) {
 508                 CPU_ZERO(&cpuset);
 509                 threadpool[i].cpu = i % cpus;
 510                 CPU_SET(threadpool[i].cpu, &cpuset);
 511
 512                 ret = pipe2(threadpool[i].efd, O_NONBLOCK);
 513                 if (ret < 0)
 514                         syslog_panic("Cannot create event socket!\n");
 515
 516                 threadpool[i].c = xmalloc_aligned(sizeof(*threadpool[i].c), 64);
 517                 threadpool[i].parent.efd = efd;
 518                 threadpool[i].parent.refd = refd;
 519                 threadpool[i].parent.tunfd = tunfd;
 520                 threadpool[i].parent.ipv4 = ipv4;
 521                 threadpool[i].parent.udp = udp;
 522                 threadpool[i].handler = udp ? handler_udp : handler_tcp;
 523
 524                 ret = pthread_create(&threadpool[i].trid, NULL,
 525                                      worker, &threadpool[i]);
 526                 if (ret < 0)
 527                         syslog_panic("Thread creation failed!\n");
 528
 529                 ret = pthread_setaffinity_np(threadpool[i].trid,
 530                                              sizeof(cpuset), &cpuset);
 531                 if (ret < 0)
 532                         syslog_panic("Thread CPU migration failed!\n");
 533
 534                 pthread_detach(threadpool[i].trid);
 535         }
 536
 537         sleep(1);
 538 }
 539
 540 static void thread_finish(unsigned int cpus)
 541 {
 542         int i;
 543         unsigned int threads;
 544
 545         threads = cpus * THREADS_PER_CPU;
 546
 547         for (i = 0; i < threads; ++i) {
 548                 while (pthread_join(threadpool[i].trid, NULL) < 0)
 549                         ;
 550
 551                 close(threadpool[i].efd[0]);
 552                 close(threadpool[i].efd[1]);
 553         }
 554 }
 555
 556 int server_main(char *home, char *dev, char *port, int udp, int ipv4, int log)
 557 {
 558         int lfd = -1, kdpfd, nfds, nfd, curfds, efd[2], refd[2], tunfd, i;
 559         unsigned int cpus = 0, threads, udp_cpu = 0;
 560         ssize_t ret;
 561         struct epoll_event *events;
 562         struct addrinfo hints, *ahead, *ai;
 563
 564         auth_log = !!log;
 565         openlog("curvetun", LOG_PID | LOG_CONS | LOG_NDELAY, LOG_DAEMON);
 566
 567         syslog(LOG_INFO, "curvetun server booting!\n");
 568         syslog_maybe(!auth_log, LOG_INFO, "curvetun user logging disabled!\n");
 569
 570         parse_userfile_and_generate_user_store_or_die(home);
 571
 572         memset(&hints, 0, sizeof(hints));
 573         hints.ai_family = PF_UNSPEC;
 574         hints.ai_socktype = udp ? SOCK_DGRAM : SOCK_STREAM;
 575         hints.ai_protocol = udp ? IPPROTO_UDP : IPPROTO_TCP;
 576         hints.ai_flags = AI_PASSIVE;
 577
 578         ret = getaddrinfo(NULL, port, &hints, &ahead);
 579         if (ret < 0)
 580                 syslog_panic("Cannot get address info!\n");
 581
 582         for (ai = ahead; ai != NULL && lfd < 0; ai = ai->ai_next) {
 583                 lfd = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol);
 584                 if (lfd < 0)
 585                         continue;
 586                 if (ai->ai_family == AF_INET6) {
 587 #ifdef IPV6_V6ONLY
 588                         ret = set_ipv6_only(lfd);
 589                         if (ret < 0) {
 590                                 close(lfd);
 591                                 lfd = -1;
 592                                 continue;
 593                         }
 594 #else
 595                         close(lfd);
 596                         lfd = -1;
 597                         continue;
 598 #endif /* IPV6_V6ONLY */
 599                 }
 600
 601                 set_reuseaddr(lfd);
 602                 set_mtu_disc_dont(lfd);
 603
 604                 ret = bind(lfd, ai->ai_addr, ai->ai_addrlen);
 605                 if (ret < 0) {
 606                         close(lfd);
 607                         lfd = -1;
 608                         continue;
 609                 }
 610
 611                 if (!udp) {
 612                         ret = listen(lfd, 5);
 613                         if (ret < 0) {
 614                                 close(lfd);
 615                                 lfd = -1;
 616                                 continue;
 617                         }
 618                 }
 619
 620                 if (ipv4 == -1) {
 621                         ipv4 = (ai->ai_family == AF_INET6 ? 0 :
 622                                 (ai->ai_family == AF_INET ? 1 : -1));
 623                 }
 624
 625                 syslog_maybe(auth_log, LOG_INFO, "curvetun on IPv%d via %s "
 626                              "on port %s!\n", ai->ai_family == AF_INET ? 4 : 6,
 627                              udp ? "UDP" : "TCP", port);
 628                 syslog_maybe(auth_log, LOG_INFO, "Allowed overlay proto is "
 629                              "IPv%d!\n", ipv4 ? 4 : 6);
 630         }
 631
 632         freeaddrinfo(ahead);
 633
 634         if (lfd < 0 || ipv4 < 0)
 635                 syslog_panic("Cannot create socket!\n");
 636
 637         tunfd = tun_open_or_die(dev ? dev : DEVNAME_SERVER, IFF_TUN | IFF_NO_PI);
 638
 639         pipe_or_die(efd, O_NONBLOCK);
 640         pipe_or_die(refd, O_NONBLOCK);
 641
 642         set_nonblocking(lfd);
 643
 644         events = xzmalloc(MAX_EPOLL_SIZE * sizeof(*events));
 645         for (i = 0; i < MAX_EPOLL_SIZE; ++i)
 646                 events[i].data.fd = -1;
 647
 648         kdpfd = epoll_create(MAX_EPOLL_SIZE);
 649         if (kdpfd < 0)
 650                 syslog_panic("Cannot create socket!\n");
 651
 652         set_epoll_descriptor(kdpfd, EPOLL_CTL_ADD, lfd,
 653                              udp ? EPOLLIN | EPOLLET | EPOLLONESHOT : EPOLLIN);
 654         set_epoll_descriptor(kdpfd, EPOLL_CTL_ADD, efd[0], EPOLLIN);
 655         set_epoll_descriptor(kdpfd, EPOLL_CTL_ADD, refd[0], EPOLLIN);
 656         set_epoll_descriptor(kdpfd, EPOLL_CTL_ADD, tunfd,
 657                              EPOLLIN | EPOLLET | EPOLLONESHOT);
 658         curfds = 4;
 659
 660         trie_init();
 661
 662         cpus = get_number_cpus_online();
 663         threads = cpus * THREADS_PER_CPU;
 664         if (!ispow2(threads))
 665                 syslog_panic("Thread number not power of two!\n");
 666
 667         threadpool = xzmalloc(sizeof(*threadpool) * threads);
 668         thread_spawn_or_panic(cpus, efd[1], refd[1], tunfd, ipv4, udp);
 669
 670         init_cpusched(threads);
 671
 672         register_socket(tunfd);
 673         register_socket(lfd);
 674
 675         syslog(LOG_INFO, "curvetun up and running!\n");
 676
 677         while (likely(!sigint)) {
 678                 nfds = epoll_wait(kdpfd, events, curfds, -1);
 679                 if (nfds < 0) {
 680                         syslog(LOG_ERR, "epoll_wait error: %s\n",
 681                                strerror(errno));
 682                         break;
 683                 }
 684
 685                 for (i = 0; i < nfds; ++i) {
 686                         if (unlikely(events[i].data.fd < 0))
 687                                 continue;
 688
 689                         if (events[i].data.fd == lfd && !udp) {
 690                                 int ncpu;
 691                                 char hbuff[256], sbuff[256];
 692                                 struct sockaddr_storage taddr;
 693                                 socklen_t tlen;
 694
 695                                 tlen = sizeof(taddr);
 696                                 nfd = accept(lfd, (struct sockaddr *) &taddr,
 697                                              &tlen);
 698                                 if (nfd < 0) {
 699                                         syslog(LOG_ERR, "accept error: %s\n",
 700                                                strerror(errno));
 701                                         continue;
 702                                 }
 703
 704                                 if (curfds + 1 > MAX_EPOLL_SIZE) {
 705                                         close(nfd);
 706                                         continue;
 707                                 }
 708
 709                                 curfds++;
 710
 711                                 ncpu = register_socket(nfd);
 712
 713                                 memset(hbuff, 0, sizeof(hbuff));
 714                                 memset(sbuff, 0, sizeof(sbuff));
 715                                 getnameinfo((struct sockaddr *) &taddr, tlen,
 716                                             hbuff, sizeof(hbuff),
 717                                             sbuff, sizeof(sbuff),
 718                                             NI_NUMERICHOST | NI_NUMERICSERV);
 719
 720                                 syslog_maybe(auth_log, LOG_INFO, "New connection "
 721                                              "from %s:%s with id %d on CPU%d, %d "
 722                                              "active!\n", hbuff, sbuff, nfd, ncpu,
 723                                              curfds);
 724
 725                                 set_nonblocking(nfd);
 726                                 set_socket_keepalive(nfd);
 727                                 set_tcp_nodelay(nfd);
 728                                 ret = set_epoll_descriptor2(kdpfd, EPOLL_CTL_ADD,
 729                                                 nfd, EPOLLIN | EPOLLET | EPOLLONESHOT);
 730                                 if (ret < 0) {
 731                                         close(nfd);
 732                                         curfds--;
 733                                         continue;
 734                                 }
 735                         } else if (events[i].data.fd == refd[0]) {
 736                                 int fd_one;
 737
 738                                 ret = read_exact(refd[0], &fd_one,
 739                                                  sizeof(fd_one), 1);
 740                                 if (ret != sizeof(fd_one) || fd_one <= 0)
 741                                         continue;
 742
 743                                 ret = set_epoll_descriptor2(kdpfd, EPOLL_CTL_MOD,
 744                                                 fd_one, EPOLLIN | EPOLLET | EPOLLONESHOT);
 745                                 if (ret < 0) {
 746                                         close(fd_one);
 747                                         continue;
 748                                 }
 749                         } else if (events[i].data.fd == efd[0]) {
 750                                 int fd_del, test;
 751
 752                                 ret = read_exact(efd[0], &fd_del,
 753                                                  sizeof(fd_del), 1);
 754                                 if (ret != sizeof(fd_del) || fd_del <= 0)
 755                                         continue;
 756
 757                                 ret = read(fd_del, &test, sizeof(test));
 758                                 if (ret < 0 && errno == EBADF)
 759                                         continue;
 760
 761                                 ret = set_epoll_descriptor2(kdpfd, EPOLL_CTL_DEL,
 762                                                 fd_del, 0);
 763                                 if (ret < 0) {
 764                                         close(fd_del);
 765                                         continue;
 766                                 }
 767
 768                                 close(fd_del);
 769                                 curfds--;
 770                                 unregister_socket(fd_del);
 771
 772                                 syslog_maybe(auth_log, LOG_INFO, "Closed connection "
 773                                              "with id %d, %d active!\n", fd_del,
 774                                              curfds);
 775                         } else {
 776                                 int cpu, fd_work = events[i].data.fd;
 777
 778                                 if (!udp)
 779                                         cpu = socket_to_cpu(fd_work);
 780                                 else
 781                                         udp_cpu = (udp_cpu + 1) & (threads - 1);
 782
 783                                 write_exact(threadpool[udp ? udp_cpu : cpu].efd[1],
 784                                             &fd_work, sizeof(fd_work), 1);
 785                         }
 786                 }
 787         }
 788
 789         syslog(LOG_INFO, "curvetun prepare shut down!\n");
 790
 791         close(lfd);
 792         close(efd[0]);
 793         close(efd[1]);
 794         close(refd[0]);
 795         close(refd[1]);
 796         close(tunfd);
 797
 798         thread_finish(cpus);
 799
 800         xfree(threadpool);
 801         xfree(events);
 802
 803         unregister_socket(lfd);
 804         unregister_socket(tunfd);
 805
 806         destroy_cpusched();
 807
 808         trie_cleanup();
 809
 810         destroy_user_store();
 811
 812         syslog(LOG_INFO, "curvetun shut down!\n");
 813         closelog();
 814
 815         return 0;
 816 }