main.c

   1 #define _POSIX_C_SOURCE 200112L
   2
   3 #include <stdio.h>
   4 #include <stdlib.h>
   5 #include <stdint.h>
   6 #include <stdbool.h>
   7 #include <string.h>
   8 #include <time.h>
   9 #include <pthread.h>
  10 #include <signal.h>
  11 #include <sodium/core.h>
  12 #include <sodium/randombytes.h>
  13 #ifdef PASSPHRASE
  14 #include <sodium/crypto_pwhash.h>
  15 #endif
  16 #include <sodium/utils.h>
  17
  18 #include "types.h"
  19 #include "vec.h"
  20 #include "base32.h"
  21 #include "cpucount.h"
  22 #include "keccak.h"
  23 #include "ioutil.h"
  24 #include "common.h"
  25 #include "yaml.h"
  26
  27 #include "filters.h"
  28
  29 #include "worker.h"
  30
  31 #ifndef _WIN32
  32 #define FSZ "%zu"
  33 #else
  34 #define FSZ "%Iu"
  35 #endif
  36
  37 // Argon2 hashed passphrase stretching settings
  38 // NOTE: changing these will break compatibility
  39 #define PWHASH_OPSLIMIT 48
  40 #define PWHASH_MEMLIMIT 64 * 1024 * 1024
  41 #define PWHASH_ALG      crypto_pwhash_ALG_ARGON2ID13
  42
  43 static int quietflag = 0;
  44 static int verboseflag = 0;
  45 #ifndef PCRE2FILTER
  46 static int wantdedup = 0;
  47 #endif
  48
  49 // 0, direndpos, onionendpos
  50 // printstartpos = either 0 or direndpos
  51 // printlen      = either onionendpos + 1 or ONION_LEN + 1 (additional 1 is for newline)
  52 size_t onionendpos;   // end of .onion within string
  53 size_t direndpos;     // end of dir before .onion within string
  54 size_t printstartpos; // where to start printing from
  55 size_t printlen;      // precalculated, related to printstartpos
  56
  57 pthread_mutex_t fout_mutex;
  58 FILE *fout;
  59
  60 static void termhandler(int sig)
  61 {
  62         switch (sig) {
  63         case SIGTERM:
  64         case SIGINT:
  65                 endwork = 1;
  66                 break;
  67         }
  68 }
  69
  70 #ifdef STATISTICS
  71 struct tstatstruct {
  72         u64 numcalc;
  73         u64 numsuccess;
  74         u64 numrestart;
  75         u32 oldnumcalc;
  76         u32 oldnumsuccess;
  77         u32 oldnumrestart;
  78 } ;
  79 VEC_STRUCT(tstatsvec,struct tstatstruct);
  80 #endif
  81
  82 static void printhelp(FILE *out,const char *progname)
  83 {
  84         fprintf(out,
  85                 "Usage: %s filter [filter...] [options]\n"
  86                 "       %s -f filterfile [options]\n"
  87                 "Options:\n"
  88                 "\t-h  - print help to stdout and quit\n"
  89                 "\t-f  - specify filter file which contains filters separated by newlines\n"
  90                 "\t-D  - deduplicate filters\n"
  91                 "\t-q  - do not print diagnostic output to stderr\n"
  92                 "\t-x  - do not print onion names\n"
  93                 "\t-v  - print more diagnostic data\n"
  94                 "\t-o filename  - output onion names to specified file (append)\n"
  95                 "\t-O filename  - output onion names to specified file (overwrite)\n"
  96                 "\t-F  - include directory names in onion names output\n"
  97                 "\t-d dirname  - output directory\n"
  98                 "\t-t numthreads  - specify number of threads to utilise (default - CPU core count or 1)\n"
  99                 "\t-j numthreads  - same as -t\n"
 100                 "\t-n numkeys  - specify number of keys (default - 0 - unlimited)\n"
 101                 "\t-N numwords  - specify number of words per key (default - 1)\n"
 102                 "\t-z  - use faster key generation method; this is now default\n"
 103                 "\t-Z  - use slower key generation method\n"
 104                 "\t-B  - use batching key generation method (>10x faster than -z, experimental)\n"
 105                 "\t-s  - print statistics each 10 seconds\n"
 106                 "\t-S t  - print statistics every specified ammount of seconds\n"
 107                 "\t-T  - do not reset statistics counters when printing\n"
 108                 "\t-y  - output generated keys in YAML format instead of dumping them to filesystem\n"
 109                 "\t-Y [filename [host.onion]]  - parse YAML encoded input and extract key(s) to filesystem\n"
 110                 "\t--rawyaml  - raw (unprefixed) public/secret keys for -y/-Y (may be useful for tor controller API)\n"
 111 #ifdef PASSPHRASE
 112                 "\t-p passphrase  - use passphrase to initialize the random seed with\n"
 113                 "\t-P  - same as -p, but takes passphrase from PASSPHRASE environment variable\n"
 114 #endif
 115                 ,progname,progname);
 116         fflush(out);
 117 }
 118
 119 static void e_additional(void)
 120 {
 121         fprintf(stderr,"additional argument required\n");
 122         exit(1);
 123 }
 124
 125 #ifndef STATISTICS
 126 static void e_nostatistics(void)
 127 {
 128         fprintf(stderr,"statistics support not compiled in\n");
 129         exit(1);
 130 }
 131 #endif
 132
 133 static void setworkdir(const char *wd)
 134 {
 135         free(workdir);
 136         size_t l = strlen(wd);
 137         if (!l) {
 138                 workdir = 0;
 139                 workdirlen = 0;
 140                 if (!quietflag)
 141                         fprintf(stderr,"unset workdir\n");
 142                 return;
 143         }
 144         unsigned needslash = 0;
 145         if (wd[l-1] != '/')
 146                 needslash = 1;
 147         char *s = (char *) malloc(l + needslash + 1);
 148         if (!s)
 149                 abort();
 150         memcpy(s,wd,l);
 151         if (needslash)
 152                 s[l++] = '/';
 153         s[l] = 0;
 154
 155         workdir = s;
 156         workdirlen = l;
 157         if (!quietflag)
 158                 fprintf(stderr,"set workdir: %s\n",workdir);
 159 }
 160
 161 #ifdef PASSPHRASE
 162 static void setpassphrase(const char *pass)
 163 {
 164         static u8 salt[crypto_pwhash_SALTBYTES] = {0};
 165         fprintf(stderr,"expanding passphrase (may take a while)...");
 166         if (crypto_pwhash(determseed,sizeof(determseed),
 167                 pass,strlen(pass),salt,
 168                 PWHASH_OPSLIMIT,PWHASH_MEMLIMIT,PWHASH_ALG) != 0)
 169         {
 170                 fprintf(stderr," out of memory!\n");
 171                 exit(1);
 172         }
 173         fprintf(stderr," done.\n");
 174 }
 175 #endif
 176
 177 VEC_STRUCT(threadvec, pthread_t);
 178
 179 #include "filters_main.inc.h"
 180
 181 int main(int argc,char **argv)
 182 {
 183         const char *outfile = 0;
 184         const char *infile = 0;
 185         const char *onehostname = 0;
 186         const char *arg;
 187         int ignoreargs = 0;
 188         int dirnameflag = 0;
 189         int numthreads = 0;
 190         int fastkeygen = 1;
 191         int batchkeygen = 0;
 192         int yamlinput = 0;
 193 #ifdef PASSPHRASE
 194         int deterministic = 0;
 195 #endif
 196         int outfileoverwrite = 0;
 197         struct threadvec threads;
 198 #ifdef STATISTICS
 199         struct statsvec stats;
 200         struct tstatsvec tstats;
 201         u64 reportdelay = 0;
 202         int realtimestats = 1;
 203 #endif
 204         int tret;
 205
 206         if (sodium_init() < 0) {
 207                 fprintf(stderr,"sodium_init() failed\n");
 208                 return 1;
 209         }
 210         worker_init();
 211         filters_init();
 212
 213         setvbuf(stderr,0,_IONBF,0);
 214         fout = stdout;
 215
 216         const char *progname = argv[0];
 217         if (argc <= 1) {
 218                 printhelp(stderr,progname);
 219                 exit(1);
 220         }
 221         argc--; argv++;
 222
 223         while (argc--) {
 224                 arg = *argv++;
 225                 if (!ignoreargs && *arg == '-') {
 226                         int numargit = 0;
 227                 nextarg:
 228                         ++arg;
 229                         ++numargit;
 230                         if (*arg == '-') {
 231                                 if (numargit > 1) {
 232                                         fprintf(stderr,"unrecognised argument: -\n");
 233                                         exit(1);
 234                                 }
 235                                 ++arg;
 236                                 if (!*arg)
 237                                         ignoreargs = 1;
 238                                 else if (!strcmp(arg,"help") || !strcmp(arg,"usage")) {
 239                                         printhelp(stdout,progname);
 240                                         exit(0);
 241                                 }
 242                                 else if (!strcmp(arg,"rawyaml"))
 243                                         yamlraw = 1;
 244                                 else {
 245                                         fprintf(stderr,"unrecognised argument: --%s\n",arg);
 246                                         exit(1);
 247                                 }
 248                                 numargit = 0;
 249                         }
 250                         else if (*arg == 0) {
 251                                 if (numargit == 1)
 252                                         ignoreargs = 1;
 253                                 continue;
 254                         }
 255                         else if (*arg == 'h') {
 256                                 printhelp(stdout,progname);
 257                                 exit(0);
 258                         }
 259                         else if (*arg == 'f') {
 260                                 if (argc--) {
 261                                         const char *filename = *argv++;
 262                                         if (!loadfilterfile(filename)) {
 263                                                 fprintf(stderr,"failed to load filter file %s\n",filename);
 264                                                 exit(1);
 265                                         }
 266                                 }
 267                                 else
 268                                         e_additional();
 269                         }
 270                         else if (*arg == 'D') {
 271 #ifndef PCRE2FILTER
 272                                 wantdedup = 1;
 273 #else
 274                                 fprintf(stderr,"WARNING: deduplication isn't supported with regex filters\n");
 275 #endif
 276                         }
 277                         else if (*arg == 'q')
 278                                 ++quietflag;
 279                         else if (*arg == 'x')
 280                                 fout = 0;
 281                         else if (*arg == 'v')
 282                                 verboseflag = 1;
 283                         else if (*arg == 'o') {
 284                                 outfileoverwrite = 0;
 285                                 if (argc--)
 286                                         outfile = *argv++;
 287                                 else
 288                                         e_additional();
 289                         }
 290                         else if (*arg == 'O') {
 291                                 outfileoverwrite = 1;
 292                                 if (argc--)
 293                                         outfile = *argv++;
 294                                 else
 295                                         e_additional();
 296                         }
 297                         else if (*arg == 'F')
 298                                 dirnameflag = 1;
 299                         else if (*arg == 'd') {
 300                                 if (argc--)
 301                                         setworkdir(*argv++);
 302                                 else
 303                                         e_additional();
 304                         }
 305                         else if (*arg == 't' || *arg == 'j') {
 306                                 if (argc--)
 307                                         numthreads = atoi(*argv++);
 308                                 else
 309                                         e_additional();
 310                         }
 311                         else if (*arg == 'n') {
 312                                 if (argc--)
 313                                         numneedgenerate = (size_t)atoll(*argv++);
 314                                 else
 315                                         e_additional();
 316                         }
 317                         else if (*arg == 'N') {
 318                                 if (argc--)
 319                                         numwords = atoi(*argv++);
 320                                 else
 321                                         e_additional();
 322                         }
 323                         else if (*arg == 'Z')
 324                                 fastkeygen = 0;
 325                         else if (*arg == 'z')
 326                                 fastkeygen = 1;
 327                         else if (*arg == 'B')
 328                                 batchkeygen = 1;
 329                         else if (*arg == 's') {
 330 #ifdef STATISTICS
 331                                 reportdelay = 10000000;
 332 #else
 333                                 e_nostatistics();
 334 #endif
 335                         }
 336                         else if (*arg == 'S') {
 337 #ifdef STATISTICS
 338                                 if (argc--)
 339                                         reportdelay = (u64)atoll(*argv++) * 1000000;
 340                                 else
 341                                         e_additional();
 342 #else
 343                                 e_nostatistics();
 344 #endif
 345                         }
 346                         else if (*arg == 'T') {
 347 #ifdef STATISTICS
 348                                 realtimestats = 0;
 349 #else
 350                                 e_nostatistics();
 351 #endif
 352                         }
 353                         else if (*arg == 'y')
 354                                 yamloutput = 1;
 355                         else if (*arg == 'Y') {
 356                                 yamlinput = 1;
 357                                 if (argc) {
 358                                         --argc;
 359                                         infile = *argv++;
 360                                         if (!*infile)
 361                                                 infile = 0;
 362                                         if (argc) {
 363                                                 --argc;
 364                                                 onehostname = *argv++;
 365                                                 if (!*onehostname)
 366                                                         onehostname = 0;
 367                                                 if (onehostname && strlen(onehostname) != ONION_LEN) {
 368                                                         fprintf(stderr,"bad onion argument length\n");
 369                                                         exit(1);
 370                                                 }
 371                                         }
 372                                 }
 373                         }
 374 #ifdef PASSPHRASE
 375                         else if (*arg == 'p') {
 376                                 if (argc--) {
 377                                         setpassphrase(*argv++);
 378                                         deterministic = 1;
 379                                 }
 380                                 else
 381                                         e_additional();
 382                         }
 383                         else if (*arg == 'P') {
 384                                 const char *pass = getenv("PASSPHRASE");
 385                                 if (!pass) {
 386                                         fprintf(stderr,"store passphrase in PASSPHRASE environment variable\n");
 387                                         exit(1);
 388                                 }
 389                                 setpassphrase(pass);
 390                                 deterministic = 1;
 391                         }
 392 #endif // PASSPHRASE
 393                         else {
 394                                 fprintf(stderr,"unrecognised argument: -%c\n",*arg);
 395                                 exit(1);
 396                         }
 397                         if (numargit)
 398                                 goto nextarg;
 399                 }
 400                 else
 401                         filters_add(arg);
 402         }
 403
 404         if (yamlinput && yamloutput) {
 405                 fprintf(stderr,"both -y and -Y does not make sense\n");
 406                 exit(1);
 407         }
 408
 409         if (yamlraw && !yamlinput && !yamloutput) {
 410                 fprintf(stderr,"--rawyaml requires either -y or -Y to do anything\n");
 411                 exit(1);
 412         }
 413
 414         if (outfile) {
 415                 fout = fopen(outfile,!outfileoverwrite ? "a" : "w");
 416                 if (!fout) {
 417                         perror("failed to open output file");
 418                         exit(1);
 419                 }
 420         }
 421
 422         if (!fout && yamloutput) {
 423                 fprintf(stderr,"nil output with yaml mode does not make sense\n");
 424                 exit(1);
 425         }
 426
 427         if (workdir)
 428                 createdir(workdir,1);
 429
 430         direndpos = workdirlen;
 431         onionendpos = workdirlen + ONION_LEN;
 432
 433         if (!dirnameflag) {
 434                 printstartpos = direndpos;
 435                 printlen = ONION_LEN + 1; // + '\n'
 436         } else {
 437                 printstartpos = 0;
 438                 printlen = onionendpos + 1; // + '\n'
 439         }
 440
 441         if (yamlinput) {
 442                 char *sname = makesname();
 443                 FILE *fin = stdin;
 444                 if (infile) {
 445                         fin = fopen(infile,"r");
 446                         if (!fin) {
 447                                 fprintf(stderr,"failed to open input file\n");
 448                                 return 1;
 449                         }
 450                 }
 451                 tret = yamlin_parseandcreate(fin,sname,onehostname,yamlraw);
 452                 if (infile) {
 453                         fclose(fin);
 454                         fin = 0;
 455                 }
 456                 free(sname);
 457
 458                 if (tret)
 459                         return tret;
 460
 461                 goto done;
 462         }
 463
 464         filters_prepare();
 465
 466         filters_print();
 467
 468 #ifdef STATISTICS
 469         if (!filters_count() && !reportdelay)
 470 #else
 471         if (!filters_count())
 472 #endif
 473                 return 0;
 474
 475 #ifdef EXPANDMASK
 476         if (numwords > 1 && flattened)
 477                 fprintf(stderr,"WARNING: -N switch will produce bogus results because we can't know filter width. reconfigure with --enable-besort and recompile.\n");
 478 #endif
 479
 480         if (yamloutput)
 481                 yamlout_init();
 482
 483         pthread_mutex_init(&keysgenerated_mutex,0);
 484         pthread_mutex_init(&fout_mutex,0);
 485 #ifdef PASSPHRASE
 486         pthread_mutex_init(&determseed_mutex,0);
 487 #endif
 488
 489         if (numthreads <= 0) {
 490                 numthreads = cpucount();
 491                 if (numthreads <= 0)
 492                         numthreads = 1;
 493         }
 494         if (!quietflag)
 495                 fprintf(stderr,"using %d %s\n",
 496                         numthreads,numthreads == 1 ? "thread" : "threads");
 497
 498 #ifdef PASSPHRASE
 499         if (!quietflag && deterministic && numneedgenerate != 1)
 500                 fprintf(stderr,"CAUTION: avoid using keys generated with same password for unrelated services, as single leaked key may help attacker to regenerate related keys.\n");
 501 #endif
 502
 503         signal(SIGTERM,termhandler);
 504         signal(SIGINT,termhandler);
 505
 506         VEC_INIT(threads);
 507         VEC_ADDN(threads,numthreads);
 508 #ifdef STATISTICS
 509         VEC_INIT(stats);
 510         VEC_ADDN(stats,numthreads);
 511         VEC_ZERO(stats);
 512         VEC_INIT(tstats);
 513         VEC_ADDN(tstats,numthreads);
 514         VEC_ZERO(tstats);
 515 #endif
 516
 517 #if 0
 518         pthread_attr_t tattr,*tattrp = &tattr;
 519         tret = pthread_attr_init(tattrp);
 520         if (tret) {
 521                 perror("pthread_attr_init");
 522                 tattrp = 0;
 523         }
 524         else {
 525                 tret = pthread_attr_setstacksize(tattrp,80<<10);
 526                 if (tret)
 527                         perror("pthread_attr_setstacksize");
 528         }
 529 #endif
 530
 531         for (size_t i = 0;i < VEC_LENGTH(threads);++i) {
 532                 void *tp = 0;
 533 #ifdef STATISTICS
 534                 tp = &VEC_BUF(stats,i);
 535 #endif
 536                 tret = pthread_create(&VEC_BUF(threads,i),0,
 537 #ifdef PASSPHRASE
 538                                 deterministic ? (
 539                                         batchkeygen ? worker_batch_pass : worker_fast_pass) :
 540 #endif
 541                                 batchkeygen ? worker_batch :
 542                                 (fastkeygen ? worker_fast : worker_slow),tp);
 543                 if (tret) {
 544                         fprintf(stderr,"error while making " FSZ "th thread: %s\n",i,strerror(tret));
 545                         exit(1);
 546                 }
 547         }
 548
 549 #if 0
 550         if (tattrp) {
 551                 tret = pthread_attr_destroy(tattrp);
 552                 if (tret)
 553                         perror("pthread_attr_destroy");
 554         }
 555 #endif
 556
 557 #ifdef STATISTICS
 558         struct timespec nowtime;
 559         u64 istarttime,inowtime,ireporttime = 0,elapsedoffset = 0;
 560         if (clock_gettime(CLOCK_MONOTONIC,&nowtime) < 0) {
 561                 perror("failed to get time");
 562                 exit(1);
 563         }
 564         istarttime = (1000000 * (u64)nowtime.tv_sec) + ((u64)nowtime.tv_nsec / 1000);
 565 #endif
 566         struct timespec ts;
 567         memset(&ts,0,sizeof(ts));
 568         ts.tv_nsec = 100000000;
 569         while (!endwork) {
 570                 if (numneedgenerate && keysgenerated >= numneedgenerate) {
 571                         endwork = 1;
 572                         break;
 573                 }
 574                 nanosleep(&ts,0);
 575
 576 #ifdef STATISTICS
 577                 clock_gettime(CLOCK_MONOTONIC,&nowtime);
 578                 inowtime = (1000000 * (u64)nowtime.tv_sec) + ((u64)nowtime.tv_nsec / 1000);
 579                 u64 sumcalc = 0,sumsuccess = 0,sumrestart = 0;
 580                 for (int i = 0;i < numthreads;++i) {
 581                         u32 newt,tdiff;
 582                         // numcalc
 583                         newt = VEC_BUF(stats,i).numcalc.v;
 584                         tdiff = newt - VEC_BUF(tstats,i).oldnumcalc;
 585                         VEC_BUF(tstats,i).oldnumcalc = newt;
 586                         VEC_BUF(tstats,i).numcalc += (u64)tdiff;
 587                         sumcalc += VEC_BUF(tstats,i).numcalc;
 588                         // numsuccess
 589                         newt = VEC_BUF(stats,i).numsuccess.v;
 590                         tdiff = newt - VEC_BUF(tstats,i).oldnumsuccess;
 591                         VEC_BUF(tstats,i).oldnumsuccess = newt;
 592                         VEC_BUF(tstats,i).numsuccess += (u64)tdiff;
 593                         sumsuccess += VEC_BUF(tstats,i).numsuccess;
 594                         // numrestart
 595                         newt = VEC_BUF(stats,i).numrestart.v;
 596                         tdiff = newt - VEC_BUF(tstats,i).oldnumrestart;
 597                         VEC_BUF(tstats,i).oldnumrestart = newt;
 598                         VEC_BUF(tstats,i).numrestart += (u64)tdiff;
 599                         sumrestart += VEC_BUF(tstats,i).numrestart;
 600                 }
 601                 if (reportdelay && (!ireporttime || inowtime - ireporttime >= reportdelay)) {
 602                         if (ireporttime)
 603                                 ireporttime += reportdelay;
 604                         else
 605                                 ireporttime = inowtime;
 606                         if (!ireporttime)
 607                                 ireporttime = 1;
 608
 609                         double calcpersec = (1000000.0 * sumcalc) / (inowtime - istarttime);
 610                         double succpersec = (1000000.0 * sumsuccess) / (inowtime - istarttime);
 611                         double restpersec = (1000000.0 * sumrestart) / (inowtime - istarttime);
 612                         fprintf(stderr,">calc/sec:%8lf, succ/sec:%8lf, rest/sec:%8lf, elapsed:%5.6lfsec\n",
 613                                 calcpersec,succpersec,restpersec,
 614                                 (inowtime - istarttime + elapsedoffset) / 1000000.0);
 615
 616                         if (realtimestats) {
 617                                 for (int i = 0;i < numthreads;++i) {
 618                                         VEC_BUF(tstats,i).numcalc = 0;
 619                                         VEC_BUF(tstats,i).numsuccess = 0;
 620                                         VEC_BUF(tstats,i).numrestart = 0;
 621                                 }
 622                                 elapsedoffset += inowtime - istarttime;
 623                                 istarttime = inowtime;
 624                         }
 625                 }
 626                 if (sumcalc > U64_MAX / 2) {
 627                         for (int i = 0;i < numthreads;++i) {
 628                                 VEC_BUF(tstats,i).numcalc /= 2;
 629                                 VEC_BUF(tstats,i).numsuccess /= 2;
 630                                 VEC_BUF(tstats,i).numrestart /= 2;
 631                         }
 632                         u64 timediff = (inowtime - istarttime + 1) / 2;
 633                         elapsedoffset += timediff;
 634                         istarttime += timediff;
 635                 }
 636 #endif
 637         }
 638
 639         if (!quietflag)
 640                 fprintf(stderr,"waiting for threads to finish...");
 641         for (size_t i = 0;i < VEC_LENGTH(threads);++i)
 642                 pthread_join(VEC_BUF(threads,i),0);
 643         if (!quietflag)
 644                 fprintf(stderr," done.\n");
 645
 646         if (yamloutput)
 647                 yamlout_clean();
 648
 649 #ifdef PASSPHRASE
 650         pthread_mutex_destroy(&determseed_mutex);
 651 #endif
 652         pthread_mutex_destroy(&fout_mutex);
 653         pthread_mutex_destroy(&keysgenerated_mutex);
 654
 655 done:
 656         filters_clean();
 657
 658         if (outfile)
 659                 fclose(fout);
 660
 661         return 0;
 662 }