main.c

   1 #define _POSIX_C_SOURCE 200112L
   2
   3 #include <stdio.h>
   4 #include <stdlib.h>
   5 #include <stdint.h>
   6 #include <string.h>
   7 #include <time.h>
   8 #include <pthread.h>
   9 #include <signal.h>
  10 #include <sodium/core.h>
  11 #include <sodium/randombytes.h>
  12 #ifdef PASSPHRASE
  13 #include <sodium/crypto_pwhash.h>
  14 #endif
  15 #include <sodium/utils.h>
  16
  17 #include "types.h"
  18 #include "vec.h"
  19 #include "base32.h"
  20 #include "cpucount.h"
  21 #include "keccak.h"
  22 #include "ioutil.h"
  23 #include "common.h"
  24 #include "yaml.h"
  25
  26 #include "filters.h"
  27
  28 #include "worker.h"
  29
  30 #ifndef _WIN32
  31 #define FSZ "%zu"
  32 #else
  33 #define FSZ "%Iu"
  34 #endif
  35
  36 // Argon2 hashed passphrase stretching settings
  37 // NOTE: changing these will break compatibility
  38 #define PWHASH_OPSLIMIT 48
  39 #define PWHASH_MEMLIMIT 64 * 1024 * 1024
  40 #define PWHASH_ALG      crypto_pwhash_ALG_ARGON2ID13
  41
  42 static int quietflag = 0;
  43 static int verboseflag = 0;
  44 #ifndef PCRE2FILTER
  45 static int wantdedup = 0;
  46 #endif
  47
  48 // 0, direndpos, onionendpos
  49 // printstartpos = either 0 or direndpos
  50 // printlen      = either onionendpos + 1 or ONION_LEN + 1 (additional 1 is for newline)
  51 size_t onionendpos;   // end of .onion within string
  52 size_t direndpos;     // end of dir before .onion within string
  53 size_t printstartpos; // where to start printing from
  54 size_t printlen;      // precalculated, related to printstartpos
  55
  56 pthread_mutex_t fout_mutex;
  57 FILE *fout;
  58
  59 static void termhandler(int sig)
  60 {
  61         switch (sig) {
  62         case SIGTERM:
  63         case SIGINT:
  64                 endwork = 1;
  65                 break;
  66         }
  67 }
  68
  69 #ifdef STATISTICS
  70 struct tstatstruct {
  71         u64 numcalc;
  72         u64 numsuccess;
  73         u64 numrestart;
  74         u32 oldnumcalc;
  75         u32 oldnumsuccess;
  76         u32 oldnumrestart;
  77 } ;
  78 VEC_STRUCT(tstatsvec,struct tstatstruct);
  79 #endif
  80
  81 static void printhelp(FILE *out,const char *progname)
  82 {
  83         fprintf(out,
  84                 "Usage: %s filter [filter...] [options]\n"
  85                 "       %s -f filterfile [options]\n"
  86                 "Options:\n"
  87                 "\t-h  - print help to stdout and quit\n"
  88                 "\t-f  - specify filter file which contains filters separated by newlines\n"
  89                 "\t-D  - deduplicate filters\n"
  90                 "\t-q  - do not print diagnostic output to stderr\n"
  91                 "\t-x  - do not print onion names\n"
  92                 "\t-v  - print more diagnostic data\n"
  93                 "\t-o filename  - output onion names to specified file (append)\n"
  94                 "\t-O filename  - output onion names to specified file (overwrite)\n"
  95                 "\t-F  - include directory names in onion names output\n"
  96                 "\t-d dirname  - output directory\n"
  97                 "\t-t numthreads  - specify number of threads to utilise (default - CPU core count or 1)\n"
  98                 "\t-j numthreads  - same as -t\n"
  99                 "\t-n numkeys  - specify number of keys (default - 0 - unlimited)\n"
 100                 "\t-N numwords  - specify number of words per key (default - 1)\n"
 101                 "\t-z  - use faster key generation method; this is now default\n"
 102                 "\t-Z  - use slower key generation method\n"
 103                 "\t-B  - use batching key generation method (>10x faster than -z, experimental)\n"
 104                 "\t-s  - print statistics each 10 seconds\n"
 105                 "\t-S t  - print statistics every specified ammount of seconds\n"
 106                 "\t-T  - do not reset statistics counters when printing\n"
 107                 "\t-y  - output generated keys in YAML format instead of dumping them to filesystem\n"
 108                 "\t-Y [filename [host.onion]]  - parse YAML encoded input and extract key(s) to filesystem\n"
 109 #ifdef PASSPHRASE
 110                 "\t-p passphrase  - use passphrase to initialize the random seed with\n"
 111                 "\t-P  - same as -p, but takes passphrase from PASSPHRASE environment variable\n"
 112 #endif
 113                 ,progname,progname);
 114         fflush(out);
 115 }
 116
 117 static void e_additional(void)
 118 {
 119         fprintf(stderr,"additional argument required\n");
 120         exit(1);
 121 }
 122
 123 #ifndef STATISTICS
 124 static void e_nostatistics(void)
 125 {
 126         fprintf(stderr,"statistics support not compiled in\n");
 127         exit(1);
 128 }
 129 #endif
 130
 131 static void setworkdir(const char *wd)
 132 {
 133         free(workdir);
 134         size_t l = strlen(wd);
 135         if (!l) {
 136                 workdir = 0;
 137                 workdirlen = 0;
 138                 if (!quietflag)
 139                         fprintf(stderr,"unset workdir\n");
 140                 return;
 141         }
 142         unsigned needslash = 0;
 143         if (wd[l-1] != '/')
 144                 needslash = 1;
 145         char *s = (char *) malloc(l + needslash + 1);
 146         if (!s)
 147                 abort();
 148         memcpy(s,wd,l);
 149         if (needslash)
 150                 s[l++] = '/';
 151         s[l] = 0;
 152
 153         workdir = s;
 154         workdirlen = l;
 155         if (!quietflag)
 156                 fprintf(stderr,"set workdir: %s\n",workdir);
 157 }
 158
 159 #ifdef PASSPHRASE
 160 static void setpassphrase(const char *pass)
 161 {
 162         static u8 salt[crypto_pwhash_SALTBYTES] = {0};
 163         fprintf(stderr,"expanding passphrase (may take a while)...");
 164         if (crypto_pwhash(determseed,sizeof(determseed),
 165                 pass,strlen(pass),salt,
 166                 PWHASH_OPSLIMIT,PWHASH_MEMLIMIT,PWHASH_ALG) != 0)
 167         {
 168                 fprintf(stderr," out of memory!\n");
 169                 exit(1);
 170         }
 171         fprintf(stderr," done.\n");
 172 }
 173 #endif
 174
 175 VEC_STRUCT(threadvec, pthread_t);
 176
 177 #include "filters_main.inc.h"
 178
 179 int main(int argc,char **argv)
 180 {
 181         const char *outfile = 0;
 182         const char *infile = 0;
 183         const char *hostname = 0;
 184         const char *arg;
 185         int ignoreargs = 0;
 186         int dirnameflag = 0;
 187         int numthreads = 0;
 188         int fastkeygen = 1;
 189         int batchkeygen = 0;
 190         int yamlinput = 0;
 191 #ifdef PASSPHRASE
 192         int deterministic = 0;
 193 #endif
 194         int outfileoverwrite = 0;
 195         struct threadvec threads;
 196 #ifdef STATISTICS
 197         struct statsvec stats;
 198         struct tstatsvec tstats;
 199         u64 reportdelay = 0;
 200         int realtimestats = 1;
 201 #endif
 202         int tret;
 203
 204         if (sodium_init() < 0) {
 205                 fprintf(stderr,"sodium_init() failed\n");
 206                 return 1;
 207         }
 208         worker_init();
 209         filters_init();
 210
 211         setvbuf(stderr,0,_IONBF,0);
 212         fout = stdout;
 213
 214         const char *progname = argv[0];
 215         if (argc <= 1) {
 216                 printhelp(stderr,progname);
 217                 exit(1);
 218         }
 219         argc--; argv++;
 220
 221         while (argc--) {
 222                 arg = *argv++;
 223                 if (!ignoreargs && *arg == '-') {
 224                         int numargit = 0;
 225                 nextarg:
 226                         ++arg;
 227                         ++numargit;
 228                         if (*arg == '-') {
 229                                 if (numargit > 1) {
 230                                         fprintf(stderr,"unrecognised argument: -\n");
 231                                         exit(1);
 232                                 }
 233                                 ++arg;
 234                                 if (!*arg)
 235                                         ignoreargs = 1;
 236                                 else if (!strcmp(arg,"help") || !strcmp(arg,"usage")) {
 237                                         printhelp(stdout,progname);
 238                                         exit(0);
 239                                 }
 240                                 else {
 241                                         fprintf(stderr,"unrecognised argument: --%s\n",arg);
 242                                         exit(1);
 243                                 }
 244                                 numargit = 0;
 245                         }
 246                         else if (*arg == 0) {
 247                                 if (numargit == 1)
 248                                         ignoreargs = 1;
 249                                 continue;
 250                         }
 251                         else if (*arg == 'h') {
 252                                 printhelp(stdout,progname);
 253                                 exit(0);
 254                         }
 255                         else if (*arg == 'f') {
 256                                 if (argc--)
 257                                         loadfilterfile(*argv++);
 258                                 else
 259                                         e_additional();
 260                         }
 261                         else if (*arg == 'D') {
 262 #ifndef PCRE2FILTER
 263                                 wantdedup = 1;
 264 #else
 265                                 fprintf(stderr,"WARNING: deduplication isn't supported with regex filters\n");
 266 #endif
 267                         }
 268                         else if (*arg == 'q')
 269                                 ++quietflag;
 270                         else if (*arg == 'x')
 271                                 fout = 0;
 272                         else if (*arg == 'v')
 273                                 verboseflag = 1;
 274                         else if (*arg == 'o') {
 275                                 outfileoverwrite = 0;
 276                                 if (argc--)
 277                                         outfile = *argv++;
 278                                 else
 279                                         e_additional();
 280                         }
 281                         else if (*arg == 'O') {
 282                                 outfileoverwrite = 1;
 283                                 if (argc--)
 284                                         outfile = *argv++;
 285                                 else
 286                                         e_additional();
 287                         }
 288                         else if (*arg == 'F')
 289                                 dirnameflag = 1;
 290                         else if (*arg == 'd') {
 291                                 if (argc--)
 292                                         setworkdir(*argv++);
 293                                 else
 294                                         e_additional();
 295                         }
 296                         else if (*arg == 't' || *arg == 'j') {
 297                                 if (argc--)
 298                                         numthreads = atoi(*argv++);
 299                                 else
 300                                         e_additional();
 301                         }
 302                         else if (*arg == 'n') {
 303                                 if (argc--)
 304                                         numneedgenerate = (size_t)atoll(*argv++);
 305                                 else
 306                                         e_additional();
 307                         }
 308                         else if (*arg == 'N') {
 309                                 if (argc--)
 310                                         numwords = atoi(*argv++);
 311                                 else
 312                                         e_additional();
 313                         }
 314                         else if (*arg == 'Z')
 315                                 fastkeygen = 0;
 316                         else if (*arg == 'z')
 317                                 fastkeygen = 1;
 318                         else if (*arg == 'B')
 319                                 batchkeygen = 1;
 320                         else if (*arg == 's') {
 321 #ifdef STATISTICS
 322                                 reportdelay = 10000000;
 323 #else
 324                                 e_nostatistics();
 325 #endif
 326                         }
 327                         else if (*arg == 'S') {
 328 #ifdef STATISTICS
 329                                 if (argc--)
 330                                         reportdelay = (u64)atoll(*argv++) * 1000000;
 331                                 else
 332                                         e_additional();
 333 #else
 334                                 e_nostatistics();
 335 #endif
 336                         }
 337                         else if (*arg == 'T') {
 338 #ifdef STATISTICS
 339                                 realtimestats = 0;
 340 #else
 341                                 e_nostatistics();
 342 #endif
 343                         }
 344                         else if (*arg == 'y')
 345                                 yamloutput = 1;
 346                         else if (*arg == 'Y') {
 347                                 yamlinput = 1;
 348                                 if (argc) {
 349                                         --argc;
 350                                         infile = *argv++;
 351                                         if (!*infile)
 352                                                 infile = 0;
 353                                         if (argc) {
 354                                                 --argc;
 355                                                 hostname = *argv++;
 356                                                 if (!*hostname)
 357                                                         hostname = 0;
 358                                                 if (hostname && strlen(hostname) != ONION_LEN) {
 359                                                         fprintf(stderr,"bad onion argument length\n");
 360                                                         exit(1);
 361                                                 }
 362                                         }
 363                                 }
 364                         }
 365 #ifdef PASSPHRASE
 366                         else if (*arg == 'p') {
 367                                 if (argc--) {
 368                                         setpassphrase(*argv++);
 369                                         deterministic = 1;
 370                                 }
 371                                 else
 372                                         e_additional();
 373                         }
 374                         else if (*arg == 'P') {
 375                                 const char *pass = getenv("PASSPHRASE");
 376                                 if (!pass) {
 377                                         fprintf(stderr,"store passphrase in PASSPHRASE environment variable\n");
 378                                         exit(1);
 379                                 }
 380                                 setpassphrase(pass);
 381                                 deterministic = 1;
 382                         }
 383 #endif // PASSPHRASE
 384                         else {
 385                                 fprintf(stderr,"unrecognised argument: -%c\n",*arg);
 386                                 exit(1);
 387                         }
 388                         if (numargit)
 389                                 goto nextarg;
 390                 }
 391                 else
 392                         filters_add(arg);
 393         }
 394
 395         if (outfile) {
 396                 fout = fopen(outfile,!outfileoverwrite ? "a" : "w");
 397                 if (!fout) {
 398                         perror("failed to open output file");
 399                         exit(1);
 400                 }
 401         }
 402
 403         if (!fout && yamloutput) {
 404                 fprintf(stderr,"nil output with yaml mode does not make sense\n");
 405                 exit(1);
 406         }
 407
 408         if (workdir)
 409                 createdir(workdir,1);
 410
 411         direndpos = workdirlen;
 412         onionendpos = workdirlen + ONION_LEN;
 413
 414         if (!dirnameflag) {
 415                 printstartpos = direndpos;
 416                 printlen = ONION_LEN + 1; // + '\n'
 417         } else {
 418                 printstartpos = 0;
 419                 printlen = onionendpos + 1; // + '\n'
 420         }
 421
 422         if (yamlinput) {
 423                 char *sname = makesname();
 424                 FILE *fin = stdin;
 425                 if (infile) {
 426                         fin = fopen(infile,"r");
 427                         if (!fin) {
 428                                 fprintf(stderr,"failed to open input file\n");
 429                                 return 1;
 430                         }
 431                 }
 432                 tret = yamlin_parseandcreate(fin,sname,hostname);
 433                 if (infile) {
 434                         fclose(fin);
 435                         fin = 0;
 436                 }
 437                 free(sname);
 438
 439                 if (tret)
 440                         return tret;
 441
 442                 goto done;
 443         }
 444
 445         filters_prepare();
 446
 447         filters_print();
 448
 449 #ifdef STATISTICS
 450         if (!filters_count() && !reportdelay)
 451 #else
 452         if (!filters_count())
 453 #endif
 454                 return 0;
 455
 456 #ifdef EXPANDMASK
 457         if (numwords > 1 && flattened)
 458                 fprintf(stderr,"WARNING: -N switch will produce bogus results because we can't know filter width. reconfigure with --enable-besort and recompile.\n");
 459 #endif
 460
 461         if (yamloutput)
 462                 yamlout_init();
 463
 464         pthread_mutex_init(&keysgenerated_mutex,0);
 465         pthread_mutex_init(&fout_mutex,0);
 466 #ifdef PASSPHRASE
 467         pthread_mutex_init(&determseed_mutex,0);
 468 #endif
 469
 470         if (numthreads <= 0) {
 471                 numthreads = cpucount();
 472                 if (numthreads <= 0)
 473                         numthreads = 1;
 474         }
 475         if (!quietflag)
 476                 fprintf(stderr,"using %d %s\n",
 477                         numthreads,numthreads == 1 ? "thread" : "threads");
 478
 479 #ifdef PASSPHRASE
 480         if (!quietflag && deterministic && numneedgenerate != 1)
 481                 fprintf(stderr,"CAUTION: avoid using keys generated with same password for unrelated services, as single leaked key may help attacker to regenerate related keys.\n");
 482 #endif
 483
 484         signal(SIGTERM,termhandler);
 485         signal(SIGINT,termhandler);
 486
 487         VEC_INIT(threads);
 488         VEC_ADDN(threads,numthreads);
 489 #ifdef STATISTICS
 490         VEC_INIT(stats);
 491         VEC_ADDN(stats,numthreads);
 492         VEC_ZERO(stats);
 493         VEC_INIT(tstats);
 494         VEC_ADDN(tstats,numthreads);
 495         VEC_ZERO(tstats);
 496 #endif
 497
 498 #if 0
 499         pthread_attr_t tattr,*tattrp = &tattr;
 500         tret = pthread_attr_init(tattrp);
 501         if (tret) {
 502                 perror("pthread_attr_init");
 503                 tattrp = 0;
 504         }
 505         else {
 506                 tret = pthread_attr_setstacksize(tattrp,80<<10);
 507                 if (tret)
 508                         perror("pthread_attr_setstacksize");
 509         }
 510 #endif
 511
 512         for (size_t i = 0;i < VEC_LENGTH(threads);++i) {
 513                 void *tp = 0;
 514 #ifdef STATISTICS
 515                 tp = &VEC_BUF(stats,i);
 516 #endif
 517                 tret = pthread_create(&VEC_BUF(threads,i),0,
 518 #ifdef PASSPHRASE
 519                                 deterministic ? (
 520                                         batchkeygen ? worker_batch_pass : worker_fast_pass) :
 521 #endif
 522                                 batchkeygen ? worker_batch :
 523                                 (fastkeygen ? worker_fast : worker_slow),tp);
 524                 if (tret) {
 525                         fprintf(stderr,"error while making " FSZ "th thread: %s\n",i,strerror(tret));
 526                         exit(1);
 527                 }
 528         }
 529
 530 #if 0
 531         if (tattrp) {
 532                 tret = pthread_attr_destroy(tattrp);
 533                 if (tret)
 534                         perror("pthread_attr_destroy");
 535         }
 536 #endif
 537
 538 #ifdef STATISTICS
 539         struct timespec nowtime;
 540         u64 istarttime,inowtime,ireporttime = 0,elapsedoffset = 0;
 541         if (clock_gettime(CLOCK_MONOTONIC,&nowtime) < 0) {
 542                 perror("failed to get time");
 543                 exit(1);
 544         }
 545         istarttime = (1000000 * (u64)nowtime.tv_sec) + ((u64)nowtime.tv_nsec / 1000);
 546 #endif
 547         struct timespec ts;
 548         memset(&ts,0,sizeof(ts));
 549         ts.tv_nsec = 100000000;
 550         while (!endwork) {
 551                 if (numneedgenerate && keysgenerated >= numneedgenerate) {
 552                         endwork = 1;
 553                         break;
 554                 }
 555                 nanosleep(&ts,0);
 556
 557 #ifdef STATISTICS
 558                 clock_gettime(CLOCK_MONOTONIC,&nowtime);
 559                 inowtime = (1000000 * (u64)nowtime.tv_sec) + ((u64)nowtime.tv_nsec / 1000);
 560                 u64 sumcalc = 0,sumsuccess = 0,sumrestart = 0;
 561                 for (int i = 0;i < numthreads;++i) {
 562                         u32 newt,tdiff;
 563                         // numcalc
 564                         newt = VEC_BUF(stats,i).numcalc.v;
 565                         tdiff = newt - VEC_BUF(tstats,i).oldnumcalc;
 566                         VEC_BUF(tstats,i).oldnumcalc = newt;
 567                         VEC_BUF(tstats,i).numcalc += (u64)tdiff;
 568                         sumcalc += VEC_BUF(tstats,i).numcalc;
 569                         // numsuccess
 570                         newt = VEC_BUF(stats,i).numsuccess.v;
 571                         tdiff = newt - VEC_BUF(tstats,i).oldnumsuccess;
 572                         VEC_BUF(tstats,i).oldnumsuccess = newt;
 573                         VEC_BUF(tstats,i).numsuccess += (u64)tdiff;
 574                         sumsuccess += VEC_BUF(tstats,i).numsuccess;
 575                         // numrestart
 576                         newt = VEC_BUF(stats,i).numrestart.v;
 577                         tdiff = newt - VEC_BUF(tstats,i).oldnumrestart;
 578                         VEC_BUF(tstats,i).oldnumrestart = newt;
 579                         VEC_BUF(tstats,i).numrestart += (u64)tdiff;
 580                         sumrestart += VEC_BUF(tstats,i).numrestart;
 581                 }
 582                 if (reportdelay && (!ireporttime || inowtime - ireporttime >= reportdelay)) {
 583                         if (ireporttime)
 584                                 ireporttime += reportdelay;
 585                         else
 586                                 ireporttime = inowtime;
 587                         if (!ireporttime)
 588                                 ireporttime = 1;
 589
 590                         double calcpersec = (1000000.0 * sumcalc) / (inowtime - istarttime);
 591                         double succpersec = (1000000.0 * sumsuccess) / (inowtime - istarttime);
 592                         double restpersec = (1000000.0 * sumrestart) / (inowtime - istarttime);
 593                         fprintf(stderr,">calc/sec:%8lf, succ/sec:%8lf, rest/sec:%8lf, elapsed:%5.6lfsec\n",
 594                                 calcpersec,succpersec,restpersec,
 595                                 (inowtime - istarttime + elapsedoffset) / 1000000.0);
 596
 597                         if (realtimestats) {
 598                                 for (int i = 0;i < numthreads;++i) {
 599                                         VEC_BUF(tstats,i).numcalc = 0;
 600                                         VEC_BUF(tstats,i).numsuccess = 0;
 601                                         VEC_BUF(tstats,i).numrestart = 0;
 602                                 }
 603                                 elapsedoffset += inowtime - istarttime;
 604                                 istarttime = inowtime;
 605                         }
 606                 }
 607                 if (sumcalc > U64_MAX / 2) {
 608                         for (int i = 0;i < numthreads;++i) {
 609                                 VEC_BUF(tstats,i).numcalc /= 2;
 610                                 VEC_BUF(tstats,i).numsuccess /= 2;
 611                                 VEC_BUF(tstats,i).numrestart /= 2;
 612                         }
 613                         u64 timediff = (inowtime - istarttime + 1) / 2;
 614                         elapsedoffset += timediff;
 615                         istarttime += timediff;
 616                 }
 617 #endif
 618         }
 619
 620         if (!quietflag)
 621                 fprintf(stderr,"waiting for threads to finish...");
 622         for (size_t i = 0;i < VEC_LENGTH(threads);++i)
 623                 pthread_join(VEC_BUF(threads,i),0);
 624         if (!quietflag)
 625                 fprintf(stderr," done.\n");
 626
 627         if (yamloutput)
 628                 yamlout_clean();
 629
 630 #ifdef PASSPHRASE
 631         pthread_mutex_destroy(&determseed_mutex);
 632 #endif
 633         pthread_mutex_destroy(&fout_mutex);
 634         pthread_mutex_destroy(&keysgenerated_mutex);
 635
 636 done:
 637         filters_clean();
 638
 639         if (outfile)
 640                 fclose(fout);
 641
 642         return 0;
 643 }