ct_server.c

   1 /*
   2  * curvetun - the cipherspace wormhole creator
   3  * Part of the netsniff-ng project
   4  * Copyright 2011 Daniel Borkmann <daniel@netsniff-ng.org>,
   5  * Subject to the GPL, version 2.
   6  */
   7
   8 #define _GNU_SOURCE
   9 #include <stdio.h>
  10 #include <stdlib.h>
  11 #include <fcntl.h>
  12 #include <errno.h>
  13 #include <string.h>
  14 #include <unistd.h>
  15 #include <pthread.h>
  16 #include <syslog.h>
  17 #include <signal.h>
  18 #include <netdb.h>
  19 #include <stdint.h>
  20 #include <netinet/in.h>
  21 #include <netinet/tcp.h>
  22 #include <netinet/udp.h>
  23 #include <sys/poll.h>
  24 #include <sys/types.h>
  25 #include <sys/socket.h>
  26 #include <sys/wait.h>
  27 #include <sys/epoll.h>
  28 #include <arpa/inet.h>
  29 #include <linux/if_tun.h>
  30
  31 #include "die.h"
  32 #include "xutils.h"
  33 #include "xio.h"
  34 #include "xmalloc.h"
  35 #include "curvetun.h"
  36 #include "curve.h"
  37 #include "corking.h"
  38 #include "cpus.h"
  39 #include "built_in.h"
  40 #include "ct_usermgmt.h"
  41 #include "cpusched.h"
  42 #include "trie.h"
  43
  44 struct parent_info {
  45         int efd;
  46         int refd;
  47         int tunfd;
  48         int ipv4;
  49         int udp;
  50 };
  51
  52 struct worker_struct {
  53         pthread_t trid;
  54         int efd[2];
  55         unsigned int cpu;
  56         struct parent_info parent;
  57         int (*handler)(int fd, const struct worker_struct *ws,
  58                        char *buff, size_t len);
  59         struct curve25519_struct *c;
  60 };
  61
  62 static struct worker_struct *threadpool = NULL;
  63
  64 static int auth_log = 1;
  65
  66 extern volatile sig_atomic_t sigint;
  67
  68 static int handler_udp_tun_to_net(int fd, const struct worker_struct *ws,
  69                                   char *buff, size_t len) __pure;
  70 static int handler_udp_net_to_tun(int fd, const struct worker_struct *ws,
  71                                   char *buff, size_t len) __pure;
  72 static int handler_udp(int fd, const struct worker_struct *ws,
  73                        char *buff, size_t len) __pure;
  74 static int handler_tcp_tun_to_net(int fd, const struct worker_struct *ws,
  75                                   char *buff, size_t len) __pure;
  76 static int handler_tcp_net_to_tun(int fd, const struct worker_struct *ws,
  77                                   char *buff, size_t len) __pure;
  78 static int handler_tcp(int fd, const struct worker_struct *ws,
  79                        char *buff, size_t len) __pure;
  80 ssize_t handler_tcp_read(int fd, char *buff, size_t len);
  81 static void *worker(void *self) __pure;
  82
  83 static int handler_udp_tun_to_net(int fd, const struct worker_struct *ws,
  84                                   char *buff, size_t len)
  85 {
  86         int dfd, keep = 1;
  87         char *cbuff;
  88         ssize_t rlen, err, clen;
  89         struct ct_proto *hdr;
  90         struct curve25519_proto *p;
  91         struct sockaddr_storage naddr;
  92         socklen_t nlen;
  93         size_t off = sizeof(struct ct_proto) + crypto_box_zerobytes;
  94
  95         if (!buff || len <= off)
  96                 return 0;
  97
  98         memset(buff, 0, len);
  99         while ((rlen = read(fd, buff + off, len - off)) > 0) {
 100                 dfd = -1; nlen = 0; p = NULL;
 101
 102                 memset(&naddr, 0, sizeof(naddr));
 103
 104                 hdr = (struct ct_proto *) buff;
 105                 memset(hdr, 0, sizeof(*hdr));
 106                 hdr->flags = 0;
 107
 108                 trie_addr_lookup(buff + off, rlen, ws->parent.ipv4, &dfd, &naddr,
 109                                  (size_t *) &nlen);
 110                 if (unlikely(dfd < 0 || nlen == 0)) {
 111                         memset(buff, 0, len);
 112                         continue;
 113                 }
 114
 115                 err = get_user_by_sockaddr(&naddr, nlen, &p);
 116                 if (unlikely(err || !p)) {
 117                         memset(buff, 0, len);
 118                         continue;
 119                 }
 120
 121                 clen = curve25519_encode(ws->c, p, (unsigned char *) (buff + off -
 122                                          crypto_box_zerobytes), (rlen +
 123                                          crypto_box_zerobytes), (unsigned char **)
 124                                          &cbuff);
 125                 if (unlikely(clen <= 0)) {
 126                         memset(buff, 0, len);
 127                         continue;
 128                 }
 129
 130                 hdr->payload = htons((uint16_t) clen);
 131
 132                 set_udp_cork(dfd);
 133
 134                 sendto(dfd, hdr, sizeof(struct ct_proto), 0, (struct sockaddr *)
 135                        &naddr, nlen);
 136                 sendto(dfd, cbuff, clen, 0, (struct sockaddr *) &naddr, nlen);
 137
 138                 set_udp_uncork(dfd);
 139
 140                 memset(buff, 0, len);
 141         }
 142
 143         return keep;
 144 }
 145
 146 static void handler_udp_notify_close(int fd, struct sockaddr_storage *addr)
 147 {
 148         struct ct_proto hdr;
 149
 150         memset(&hdr, 0, sizeof(hdr));
 151         hdr.flags |= PROTO_FLAG_EXIT;
 152         hdr.payload = 0;
 153
 154         sendto(fd, &hdr, sizeof(hdr), 0, (struct sockaddr *) addr,
 155                sizeof(*addr));
 156 }
 157
 158 static int handler_udp_net_to_tun(int fd, const struct worker_struct *ws,
 159                                   char *buff, size_t len)
 160 {
 161         int keep = 1;
 162         char *cbuff;
 163         ssize_t rlen, err, clen;
 164         struct ct_proto *hdr;
 165         struct curve25519_proto *p;
 166         struct sockaddr_storage naddr;
 167         socklen_t nlen = sizeof(naddr);
 168
 169         if (!buff || !len)
 170                 return 0;
 171
 172         memset(&naddr, 0, sizeof(naddr));
 173         while ((rlen = recvfrom(fd, buff, len, 0, (struct sockaddr *) &naddr,
 174                                 &nlen)) > 0) {
 175                 p = NULL;
 176
 177                 hdr = (struct ct_proto *) buff;
 178
 179                 if (unlikely(rlen < sizeof(struct ct_proto)))
 180                         goto close;
 181                 if (unlikely(rlen - sizeof(*hdr) != ntohs(hdr->payload)))
 182                         goto close;
 183                 if (unlikely(ntohs(hdr->payload) == 0))
 184                         goto close;
 185                 if (hdr->flags & PROTO_FLAG_EXIT) {
 186 close:
 187                         remove_user_by_sockaddr(&naddr, nlen);
 188                         trie_addr_remove_addr(&naddr, nlen);
 189                         handler_udp_notify_close(fd, &naddr);
 190
 191                         return keep;
 192                 }
 193                 if (hdr->flags & PROTO_FLAG_INIT) {
 194                         syslog_maybe(auth_log, LOG_INFO, "Got initial userhash "
 195                                      "from remote end!\n");
 196
 197                         if (unlikely(rlen - sizeof(*hdr) <
 198                                      sizeof(struct username_struct)))
 199                                 goto close;
 200
 201                         err = try_register_user_by_sockaddr(ws->c,
 202                                         buff + sizeof(struct ct_proto),
 203                                         rlen - sizeof(struct ct_proto),
 204                                         &naddr, nlen, auth_log);
 205                         if (unlikely(err))
 206                                 goto close;
 207
 208                         goto next;
 209                 }
 210
 211                 err = get_user_by_sockaddr(&naddr, nlen, &p);
 212                 if (unlikely(err || !p))
 213                         goto close;
 214
 215                 clen = curve25519_decode(ws->c, p, (unsigned char *) buff +
 216                                          sizeof(struct ct_proto),
 217                                          rlen - sizeof(struct ct_proto),
 218                                          (unsigned char **) &cbuff, NULL);
 219                 if (unlikely(clen <= 0))
 220                         goto close;
 221
 222                 cbuff += crypto_box_zerobytes;
 223                 clen -= crypto_box_zerobytes;
 224
 225                 err = trie_addr_maybe_update(cbuff, clen, ws->parent.ipv4,
 226                                              fd, &naddr, nlen);
 227                 if (unlikely(err))
 228                         goto next;
 229
 230                 err = write(ws->parent.tunfd, cbuff, clen);
 231 next:
 232                 nlen = sizeof(naddr);
 233                 memset(&naddr, 0, sizeof(naddr));
 234         }
 235
 236         return keep;
 237 }
 238
 239 static int handler_udp(int fd, const struct worker_struct *ws,
 240                        char *buff, size_t len)
 241 {
 242         int ret = 0;
 243
 244         if (fd == ws->parent.tunfd)
 245                 ret = handler_udp_tun_to_net(fd, ws, buff, len);
 246         else
 247                 ret = handler_udp_net_to_tun(fd, ws, buff, len);
 248
 249         return ret;
 250 }
 251
 252 static int handler_tcp_tun_to_net(int fd, const struct worker_struct *ws,
 253                                   char *buff, size_t len)
 254 {
 255         int dfd, keep = 1;
 256         char *cbuff;
 257         ssize_t rlen, err, clen;
 258         struct ct_proto *hdr;
 259         struct curve25519_proto *p;
 260         socklen_t nlen;
 261         size_t off = sizeof(struct ct_proto) + crypto_box_zerobytes;
 262
 263         if (!buff || len <= off)
 264                 return 0;
 265
 266         memset(buff, 0, len);
 267         while ((rlen = read(fd, buff + off, len - off)) > 0) {
 268                 dfd = -1; p = NULL;
 269
 270                 hdr = (struct ct_proto *) buff;
 271                 memset(hdr, 0, sizeof(*hdr));
 272                 hdr->flags = 0;
 273
 274                 trie_addr_lookup(buff + off, rlen, ws->parent.ipv4, &dfd, NULL,
 275                                  (size_t *) &nlen);
 276                 if (unlikely(dfd < 0)) {
 277                         memset(buff, 0, len);
 278                         continue;
 279                 }
 280
 281                 err = get_user_by_socket(dfd, &p);
 282                 if (unlikely(err || !p)) {
 283                         memset(buff, 0, len);
 284                         continue;
 285                 }
 286
 287                 clen = curve25519_encode(ws->c, p, (unsigned char *) (buff + off -
 288                                          crypto_box_zerobytes), (rlen +
 289                                          crypto_box_zerobytes), (unsigned char **)
 290                                          &cbuff);
 291                 if (unlikely(clen <= 0)) {
 292                         memset(buff, 0, len);
 293                         continue;
 294                 }
 295
 296                 hdr->payload = htons((uint16_t) clen);
 297
 298                 set_tcp_cork(dfd);
 299
 300                 write_exact(dfd, hdr, sizeof(struct ct_proto), 0);
 301                 write_exact(dfd, cbuff, clen, 0);
 302
 303                 set_tcp_uncork(dfd);
 304
 305                 memset(buff, 0, len);
 306         }
 307
 308         return keep;
 309 }
 310
 311 ssize_t handler_tcp_read(int fd, char *buff, size_t len)
 312 {
 313         ssize_t rlen;
 314         struct ct_proto *hdr = (struct ct_proto *) buff;
 315
 316         if (!buff || !len)
 317                 return 0;
 318
 319         /* May exit on EAGAIN if 0 Byte read */
 320         rlen = read_exact(fd, buff, sizeof(struct ct_proto), 1);
 321         if (rlen < 0)
 322                 return rlen;
 323         if (unlikely(ntohs(hdr->payload) > len - sizeof(struct ct_proto))) {
 324                 errno = ENOMEM;
 325                 return 1; /* Force server to close connection */
 326         }
 327
 328         /* May not exit on EAGAIN if 0 Byte read */
 329         rlen = read_exact(fd, buff + sizeof(struct ct_proto),
 330                           ntohs(hdr->payload), 0);
 331         if (rlen < 0)
 332                 return rlen;
 333
 334         return sizeof(struct ct_proto) + rlen;
 335 }
 336
 337 static void handler_tcp_notify_close(int fd)
 338 {
 339         struct ct_proto hdr;
 340
 341         memset(&hdr, 0, sizeof(hdr));
 342         hdr.flags |= PROTO_FLAG_EXIT;
 343         hdr.payload = 0;
 344
 345         if (write(fd, &hdr, sizeof(hdr))) { ; }
 346 }
 347
 348 static int handler_tcp_net_to_tun(int fd, const struct worker_struct *ws,
 349                                   char *buff, size_t len)
 350 {
 351         int keep = 1, count = 0;
 352         char *cbuff;
 353         ssize_t rlen, err, clen;
 354         struct ct_proto *hdr;
 355         struct curve25519_proto *p;
 356
 357         if (!buff || !len)
 358                 return 0;
 359
 360         while ((rlen = handler_tcp_read(fd, buff, len)) > 0) {
 361                 p = NULL;
 362
 363                 hdr = (struct ct_proto *) buff;
 364
 365                 if (unlikely(rlen < sizeof(struct ct_proto)))
 366                         goto close;
 367                 if (unlikely(rlen - sizeof(*hdr) != ntohs(hdr->payload)))
 368                         goto close;
 369                 if (unlikely(ntohs(hdr->payload) == 0))
 370                         goto close;
 371                 if (hdr->flags & PROTO_FLAG_EXIT) {
 372 close:
 373                         remove_user_by_socket(fd);
 374                         trie_addr_remove(fd);
 375                         handler_tcp_notify_close(fd);
 376                         rlen = write(ws->parent.efd, &fd, sizeof(fd));
 377
 378                         keep = 0;
 379                         return keep;
 380                 }
 381                 if (hdr->flags & PROTO_FLAG_INIT) {
 382                         syslog_maybe(auth_log, LOG_INFO, "Got initial userhash "
 383                                      "from remote end!\n");
 384
 385                         if (unlikely(rlen - sizeof(*hdr) <
 386                                      sizeof(struct username_struct)))
 387                                 goto close;
 388
 389                         err = try_register_user_by_socket(ws->c,
 390                                         buff + sizeof(struct ct_proto),
 391                                         rlen - sizeof(struct ct_proto),
 392                                         fd, auth_log);
 393                         if (unlikely(err))
 394                                 goto close;
 395
 396                         continue;
 397                 }
 398
 399                 err = get_user_by_socket(fd, &p);
 400                 if (unlikely(err || !p))
 401                         continue;
 402
 403                 clen = curve25519_decode(ws->c, p, (unsigned char *) buff +
 404                                          sizeof(struct ct_proto),
 405                                          rlen - sizeof(struct ct_proto),
 406                                          (unsigned char **) &cbuff, NULL);
 407                 if (unlikely(clen <= 0))
 408                         continue;
 409
 410                 cbuff += crypto_box_zerobytes;
 411                 clen -= crypto_box_zerobytes;
 412
 413                 err = trie_addr_maybe_update(cbuff, clen, ws->parent.ipv4,
 414                                              fd, NULL, 0);
 415                 if (unlikely(err))
 416                         continue;
 417
 418                 err = write(ws->parent.tunfd, cbuff, clen);
 419
 420                 count++;
 421                 if (count == 10) {
 422                         write_exact(ws->efd[1], &fd, sizeof(fd), 1);
 423                         /* Read later next data and let others process */
 424                         return keep;
 425                 }
 426         }
 427
 428         return keep;
 429 }
 430
 431 static int handler_tcp(int fd, const struct worker_struct *ws,
 432                        char *buff, size_t len)
 433 {
 434         int ret = 0;
 435
 436         if (fd == ws->parent.tunfd)
 437                 ret = handler_tcp_tun_to_net(fd, ws, buff, len);
 438         else
 439                 ret = handler_tcp_net_to_tun(fd, ws, buff, len);
 440
 441         return ret;
 442 }
 443
 444 static void *worker(void *self)
 445 {
 446         int fd, old_state;
 447         ssize_t ret;
 448         size_t blen = TUNBUFF_SIZ; //FIXME
 449         const struct worker_struct *ws = self;
 450         struct pollfd fds;
 451         char *buff;
 452
 453         fds.fd = ws->efd[0];
 454         fds.events = POLLIN;
 455
 456         curve25519_alloc_or_maybe_die(ws->c);
 457
 458         buff = xmalloc_aligned(blen, 64);
 459
 460         syslog(LOG_INFO, "curvetun thread on CPU%u up!\n", ws->cpu);
 461
 462         pthread_cleanup_push(xfree_func, ws->c);
 463         pthread_cleanup_push(curve25519_free, ws->c);
 464         pthread_cleanup_push(xfree_func, buff);
 465
 466         while (likely(!sigint)) {
 467                 poll(&fds, 1, -1);
 468                 if ((fds.revents & POLLIN) != POLLIN)
 469                         continue;
 470
 471                 pthread_setcancelstate(PTHREAD_CANCEL_DISABLE, &old_state);
 472
 473                 while ((ret = read_exact(ws->efd[0], &fd, sizeof(fd), 1)) > 0) {
 474                         if (ret != sizeof(fd)) {
 475                                 sched_yield();
 476                                 continue;
 477                         }
 478
 479                         ret = ws->handler(fd, ws, buff, blen);
 480                         if (ret)
 481                                 write_exact(ws->parent.refd, &fd, sizeof(fd), 1);
 482                 }
 483
 484                 pthread_setcancelstate(old_state, NULL);
 485         }
 486
 487         syslog(LOG_INFO, "curvetun thread on CPU%u down!\n", ws->cpu);
 488
 489         pthread_cleanup_pop(1);
 490         pthread_cleanup_pop(1);
 491         pthread_cleanup_pop(1);
 492
 493         pthread_exit((void *) ((long) ws->cpu));
 494 }
 495
 496 static void thread_spawn_or_panic(unsigned int cpus, int efd, int refd,
 497                                   int tunfd, int ipv4, int udp)
 498 {
 499         int i, ret;
 500         cpu_set_t cpuset;
 501         unsigned int threads;
 502
 503         threads = cpus * THREADS_PER_CPU;
 504
 505         for (i = 0; i < threads; ++i) {
 506                 CPU_ZERO(&cpuset);
 507                 threadpool[i].cpu = i % cpus;
 508                 CPU_SET(threadpool[i].cpu, &cpuset);
 509
 510                 ret = pipe2(threadpool[i].efd, O_NONBLOCK);
 511                 if (ret < 0)
 512                         syslog_panic("Cannot create event socket!\n");
 513
 514                 threadpool[i].c = xmalloc_aligned(sizeof(*threadpool[i].c), 64);
 515                 threadpool[i].parent.efd = efd;
 516                 threadpool[i].parent.refd = refd;
 517                 threadpool[i].parent.tunfd = tunfd;
 518                 threadpool[i].parent.ipv4 = ipv4;
 519                 threadpool[i].parent.udp = udp;
 520                 threadpool[i].handler = udp ? handler_udp : handler_tcp;
 521
 522                 ret = pthread_create(&threadpool[i].trid, NULL,
 523                                      worker, &threadpool[i]);
 524                 if (ret < 0)
 525                         syslog_panic("Thread creation failed!\n");
 526
 527                 ret = pthread_setaffinity_np(threadpool[i].trid,
 528                                              sizeof(cpuset), &cpuset);
 529                 if (ret < 0)
 530                         syslog_panic("Thread CPU migration failed!\n");
 531
 532                 pthread_detach(threadpool[i].trid);
 533         }
 534
 535         sleep(1);
 536 }
 537
 538 static void thread_finish(unsigned int cpus)
 539 {
 540         int i;
 541         unsigned int threads;
 542
 543         threads = cpus * THREADS_PER_CPU;
 544
 545         for (i = 0; i < threads; ++i) {
 546                 while (pthread_join(threadpool[i].trid, NULL) < 0)
 547                         ;
 548
 549                 close(threadpool[i].efd[0]);
 550                 close(threadpool[i].efd[1]);
 551         }
 552 }
 553
 554 int server_main(char *home, char *dev, char *port, int udp, int ipv4, int log)
 555 {
 556         int lfd = -1, kdpfd, nfds, nfd, curfds, efd[2], refd[2], tunfd, i;
 557         unsigned int cpus = 0, threads, udp_cpu = 0;
 558         ssize_t ret;
 559         struct epoll_event *events;
 560         struct addrinfo hints, *ahead, *ai;
 561
 562         auth_log = !!log;
 563         openlog("curvetun", LOG_PID | LOG_CONS | LOG_NDELAY, LOG_DAEMON);
 564
 565         syslog(LOG_INFO, "curvetun server booting!\n");
 566         syslog_maybe(!auth_log, LOG_INFO, "curvetun user logging disabled!\n");
 567
 568         parse_userfile_and_generate_user_store_or_die(home);
 569
 570         memset(&hints, 0, sizeof(hints));
 571         hints.ai_family = PF_UNSPEC;
 572         hints.ai_socktype = udp ? SOCK_DGRAM : SOCK_STREAM;
 573         hints.ai_protocol = udp ? IPPROTO_UDP : IPPROTO_TCP;
 574         hints.ai_flags = AI_PASSIVE;
 575
 576         ret = getaddrinfo(NULL, port, &hints, &ahead);
 577         if (ret < 0)
 578                 syslog_panic("Cannot get address info!\n");
 579
 580         for (ai = ahead; ai != NULL && lfd < 0; ai = ai->ai_next) {
 581                 lfd = socket(ai->ai_family, ai->ai_socktype, ai->ai_protocol);
 582                 if (lfd < 0)
 583                         continue;
 584                 if (ai->ai_family == AF_INET6) {
 585 #ifdef IPV6_V6ONLY
 586                         ret = set_ipv6_only(lfd);
 587                         if (ret < 0) {
 588                                 close(lfd);
 589                                 lfd = -1;
 590                                 continue;
 591                         }
 592 #else
 593                         close(lfd);
 594                         lfd = -1;
 595                         continue;
 596 #endif /* IPV6_V6ONLY */
 597                 }
 598
 599                 set_reuseaddr(lfd);
 600                 set_mtu_disc_dont(lfd);
 601
 602                 ret = bind(lfd, ai->ai_addr, ai->ai_addrlen);
 603                 if (ret < 0) {
 604                         close(lfd);
 605                         lfd = -1;
 606                         continue;
 607                 }
 608
 609                 if (!udp) {
 610                         ret = listen(lfd, 5);
 611                         if (ret < 0) {
 612                                 close(lfd);
 613                                 lfd = -1;
 614                                 continue;
 615                         }
 616                 }
 617
 618                 if (ipv4 == -1) {
 619                         ipv4 = (ai->ai_family == AF_INET6 ? 0 :
 620                                 (ai->ai_family == AF_INET ? 1 : -1));
 621                 }
 622
 623                 syslog_maybe(auth_log, LOG_INFO, "curvetun on IPv%d via %s "
 624                              "on port %s!\n", ai->ai_family == AF_INET ? 4 : 6,
 625                              udp ? "UDP" : "TCP", port);
 626                 syslog_maybe(auth_log, LOG_INFO, "Allowed overlay proto is "
 627                              "IPv%d!\n", ipv4 ? 4 : 6);
 628         }
 629
 630         freeaddrinfo(ahead);
 631
 632         if (lfd < 0 || ipv4 < 0)
 633                 syslog_panic("Cannot create socket!\n");
 634
 635         tunfd = tun_open_or_die(dev ? dev : DEVNAME_SERVER, IFF_TUN | IFF_NO_PI);
 636
 637         pipe_or_die(efd, O_NONBLOCK);
 638         pipe_or_die(refd, O_NONBLOCK);
 639
 640         set_nonblocking(lfd);
 641
 642         events = xzmalloc(MAX_EPOLL_SIZE * sizeof(*events));
 643         for (i = 0; i < MAX_EPOLL_SIZE; ++i)
 644                 events[i].data.fd = -1;
 645
 646         kdpfd = epoll_create(MAX_EPOLL_SIZE);
 647         if (kdpfd < 0)
 648                 syslog_panic("Cannot create socket!\n");
 649
 650         set_epoll_descriptor(kdpfd, EPOLL_CTL_ADD, lfd,
 651                              udp ? EPOLLIN | EPOLLET | EPOLLONESHOT : EPOLLIN);
 652         set_epoll_descriptor(kdpfd, EPOLL_CTL_ADD, efd[0], EPOLLIN);
 653         set_epoll_descriptor(kdpfd, EPOLL_CTL_ADD, refd[0], EPOLLIN);
 654         set_epoll_descriptor(kdpfd, EPOLL_CTL_ADD, tunfd,
 655                              EPOLLIN | EPOLLET | EPOLLONESHOT);
 656         curfds = 4;
 657
 658         trie_init();
 659
 660         cpus = get_number_cpus_online();
 661         threads = cpus * THREADS_PER_CPU;
 662         if (!ispow2(threads))
 663                 syslog_panic("Thread number not power of two!\n");
 664
 665         threadpool = xzmalloc(sizeof(*threadpool) * threads);
 666         thread_spawn_or_panic(cpus, efd[1], refd[1], tunfd, ipv4, udp);
 667
 668         init_cpusched(threads);
 669
 670         register_socket(tunfd);
 671         register_socket(lfd);
 672
 673         syslog(LOG_INFO, "curvetun up and running!\n");
 674
 675         while (likely(!sigint)) {
 676                 nfds = epoll_wait(kdpfd, events, curfds, -1);
 677                 if (nfds < 0) {
 678                         syslog(LOG_ERR, "epoll_wait error: %s\n",
 679                                strerror(errno));
 680                         break;
 681                 }
 682
 683                 for (i = 0; i < nfds; ++i) {
 684                         if (unlikely(events[i].data.fd < 0))
 685                                 continue;
 686
 687                         if (events[i].data.fd == lfd && !udp) {
 688                                 int ncpu;
 689                                 char hbuff[256], sbuff[256];
 690                                 struct sockaddr_storage taddr;
 691                                 socklen_t tlen;
 692
 693                                 tlen = sizeof(taddr);
 694                                 nfd = accept(lfd, (struct sockaddr *) &taddr,
 695                                              &tlen);
 696                                 if (nfd < 0) {
 697                                         syslog(LOG_ERR, "accept error: %s\n",
 698                                                strerror(errno));
 699                                         continue;
 700                                 }
 701
 702                                 if (curfds + 1 > MAX_EPOLL_SIZE) {
 703                                         close(nfd);
 704                                         continue;
 705                                 }
 706
 707                                 curfds++;
 708
 709                                 ncpu = register_socket(nfd);
 710
 711                                 memset(hbuff, 0, sizeof(hbuff));
 712                                 memset(sbuff, 0, sizeof(sbuff));
 713                                 getnameinfo((struct sockaddr *) &taddr, tlen,
 714                                             hbuff, sizeof(hbuff),
 715                                             sbuff, sizeof(sbuff),
 716                                             NI_NUMERICHOST | NI_NUMERICSERV);
 717
 718                                 syslog_maybe(auth_log, LOG_INFO, "New connection "
 719                                              "from %s:%s (%d active client connections) -  id %d on CPU%d",
 720                                              hbuff, sbuff, curfds-4, nfd, ncpu);
 721
 722                                 set_nonblocking(nfd);
 723                                 set_socket_keepalive(nfd);
 724                                 set_tcp_nodelay(nfd);
 725                                 ret = set_epoll_descriptor2(kdpfd, EPOLL_CTL_ADD,
 726                                                 nfd, EPOLLIN | EPOLLET | EPOLLONESHOT);
 727                                 if (ret < 0) {
 728                                         close(nfd);
 729                                         curfds--;
 730                                         continue;
 731                                 }
 732                         } else if (events[i].data.fd == refd[0]) {
 733                                 int fd_one;
 734
 735                                 ret = read_exact(refd[0], &fd_one,
 736                                                  sizeof(fd_one), 1);
 737                                 if (ret != sizeof(fd_one) || fd_one <= 0)
 738                                         continue;
 739
 740                                 ret = set_epoll_descriptor2(kdpfd, EPOLL_CTL_MOD,
 741                                                 fd_one, EPOLLIN | EPOLLET | EPOLLONESHOT);
 742                                 if (ret < 0) {
 743                                         close(fd_one);
 744                                         continue;
 745                                 }
 746                         } else if (events[i].data.fd == efd[0]) {
 747                                 int fd_del, test;
 748
 749                                 ret = read_exact(efd[0], &fd_del,
 750                                                  sizeof(fd_del), 1);
 751                                 if (ret != sizeof(fd_del) || fd_del <= 0)
 752                                         continue;
 753
 754                                 ret = read(fd_del, &test, sizeof(test));
 755                                 if (ret < 0 && errno == EBADF)
 756                                         continue;
 757
 758                                 ret = set_epoll_descriptor2(kdpfd, EPOLL_CTL_DEL,
 759                                                 fd_del, 0);
 760                                 if (ret < 0) {
 761                                         close(fd_del);
 762                                         continue;
 763                                 }
 764
 765                                 close(fd_del);
 766                                 curfds--;
 767                                 unregister_socket(fd_del);
 768
 769                                 syslog_maybe(auth_log, LOG_INFO, "Closed connection "
 770                                              "with id %d (%d active client connections remain)\n", fd_del,
 771                                              curfds-4);
 772                         } else {
 773                                 int cpu, fd_work = events[i].data.fd;
 774
 775                                 if (!udp)
 776                                         cpu = socket_to_cpu(fd_work);
 777                                 else
 778                                         udp_cpu = (udp_cpu + 1) & (threads - 1);
 779
 780                                 write_exact(threadpool[udp ? udp_cpu : cpu].efd[1],
 781                                             &fd_work, sizeof(fd_work), 1);
 782                         }
 783                 }
 784         }
 785
 786         syslog(LOG_INFO, "curvetun prepare shut down!\n");
 787
 788         close(lfd);
 789         close(efd[0]);
 790         close(efd[1]);
 791         close(refd[0]);
 792         close(refd[1]);
 793         close(tunfd);
 794
 795         thread_finish(cpus);
 796
 797         xfree(threadpool);
 798         xfree(events);
 799
 800         unregister_socket(lfd);
 801         unregister_socket(tunfd);
 802
 803         destroy_cpusched();
 804
 805         trie_cleanup();
 806
 807         destroy_user_store();
 808
 809         syslog(LOG_INFO, "curvetun shut down!\n");
 810         closelog();
 811
 812         return 0;
 813 }