main.c

   1 #define _POSIX_C_SOURCE 200112L
   2
   3 #include <stdio.h>
   4 #include <stdlib.h>
   5 #include <errno.h>
   6 #include <stdint.h>
   7 #include <stdbool.h>
   8 #include <string.h>
   9 #include <time.h>
  10 #include <pthread.h>
  11 #include <signal.h>
  12 #include <sodium/core.h>
  13 #include <sodium/randombytes.h>
  14 #ifdef PASSPHRASE
  15 #include <sodium/crypto_pwhash.h>
  16 #endif
  17 #include <sodium/utils.h>
  18
  19 #include "types.h"
  20 #include "vec.h"
  21 #include "base32.h"
  22 #include "cpucount.h"
  23 #include "keccak.h"
  24 #include "ioutil.h"
  25 #include "common.h"
  26 #include "yaml.h"
  27
  28 #include "filters.h"
  29
  30 #include "worker.h"
  31
  32 #ifndef _WIN32
  33 #define FSZ "%zu"
  34 #else
  35 #define FSZ "%Iu"
  36 #endif
  37
  38 // Argon2 hashed passphrase stretching settings
  39 // NOTE: changing these will break compatibility
  40 #define PWHASH_OPSLIMIT 48
  41 #define PWHASH_MEMLIMIT 64 * 1024 * 1024
  42 #define PWHASH_ALG      crypto_pwhash_ALG_ARGON2ID13
  43
  44 static int quietflag = 0;
  45 static int verboseflag = 0;
  46 #ifndef PCRE2FILTER
  47 static int wantdedup = 0;
  48 #endif
  49
  50 // 0, direndpos, onionendpos
  51 // printstartpos = either 0 or direndpos
  52 // printlen      = either onionendpos + 1 or ONION_LEN + 1 (additional 1 is for newline)
  53 size_t onionendpos;   // end of .onion within string
  54 size_t direndpos;     // end of dir before .onion within string
  55 size_t printstartpos; // where to start printing from
  56 size_t printlen;      // precalculated, related to printstartpos
  57
  58 pthread_mutex_t fout_mutex;
  59 FILE *fout;
  60
  61 static void termhandler(int sig)
  62 {
  63         switch (sig) {
  64         case SIGTERM:
  65         case SIGINT:
  66                 endwork = 1;
  67                 break;
  68         }
  69 }
  70
  71 #ifdef STATISTICS
  72 struct tstatstruct {
  73         u64 numcalc;
  74         u64 numsuccess;
  75         u64 numrestart;
  76         u32 oldnumcalc;
  77         u32 oldnumsuccess;
  78         u32 oldnumrestart;
  79 } ;
  80 VEC_STRUCT(tstatsvec,struct tstatstruct);
  81 #endif
  82
  83 static void printhelp(FILE *out,const char *progname)
  84 {
  85         fprintf(out,
  86                 "Usage: %s filter [filter...] [options]\n"
  87                 "       %s -f filterfile [options]\n"
  88                 "Options:\n"
  89                 "\t-h  - print help to stdout and quit\n"
  90                 "\t-f  - specify filter file which contains filters separated by newlines\n"
  91                 "\t-D  - deduplicate filters\n"
  92                 "\t-q  - do not print diagnostic output to stderr\n"
  93                 "\t-x  - do not print onion names\n"
  94                 "\t-v  - print more diagnostic data\n"
  95                 "\t-o filename  - output onion names to specified file (append)\n"
  96                 "\t-O filename  - output onion names to specified file (overwrite)\n"
  97                 "\t-F  - include directory names in onion names output\n"
  98                 "\t-d dirname  - output directory\n"
  99                 "\t-t numthreads  - specify number of threads to utilise (default - CPU core count or 1)\n"
 100                 "\t-j numthreads  - same as -t\n"
 101                 "\t-n numkeys  - specify number of keys (default - 0 - unlimited)\n"
 102                 "\t-N numwords  - specify number of words per key (default - 1)\n"
 103                 "\t-z  - use faster key generation method; this is now default\n"
 104                 "\t-Z  - use slower key generation method\n"
 105                 "\t-B  - use batching key generation method (>10x faster than -z, experimental)\n"
 106                 "\t-s  - print statistics each 10 seconds\n"
 107                 "\t-S t  - print statistics every specified ammount of seconds\n"
 108                 "\t-T  - do not reset statistics counters when printing\n"
 109                 "\t-y  - output generated keys in YAML format instead of dumping them to filesystem\n"
 110                 "\t-Y [filename [host.onion]]  - parse YAML encoded input and extract key(s) to filesystem\n"
 111                 "\t--rawyaml  - raw (unprefixed) public/secret keys for -y/-Y (may be useful for tor controller API)\n"
 112 #ifdef PASSPHRASE
 113                 "\t-p passphrase  - use passphrase to initialize the random seed with\n"
 114                 "\t-P  - same as -p, but takes passphrase from PASSPHRASE environment variable\n"
 115 #endif
 116                 ,progname,progname);
 117         fflush(out);
 118 }
 119
 120 static void e_additional(void)
 121 {
 122         fprintf(stderr,"additional argument required\n");
 123         exit(1);
 124 }
 125
 126 #ifndef STATISTICS
 127 static void e_nostatistics(void)
 128 {
 129         fprintf(stderr,"statistics support not compiled in\n");
 130         exit(1);
 131 }
 132 #endif
 133
 134 static void setworkdir(const char *wd)
 135 {
 136         free(workdir);
 137         size_t l = strlen(wd);
 138         if (!l) {
 139                 workdir = 0;
 140                 workdirlen = 0;
 141                 if (!quietflag)
 142                         fprintf(stderr,"unset workdir\n");
 143                 return;
 144         }
 145         unsigned needslash = 0;
 146         if (wd[l-1] != '/')
 147                 needslash = 1;
 148         char *s = (char *) malloc(l + needslash + 1);
 149         if (!s)
 150                 abort();
 151         memcpy(s,wd,l);
 152         if (needslash)
 153                 s[l++] = '/';
 154         s[l] = 0;
 155
 156         workdir = s;
 157         workdirlen = l;
 158         if (!quietflag)
 159                 fprintf(stderr,"set workdir: %s\n",workdir);
 160 }
 161
 162 #ifdef PASSPHRASE
 163 static void setpassphrase(const char *pass)
 164 {
 165         static u8 salt[crypto_pwhash_SALTBYTES] = {0};
 166         fprintf(stderr,"expanding passphrase (may take a while)...");
 167         if (crypto_pwhash(determseed,sizeof(determseed),
 168                 pass,strlen(pass),salt,
 169                 PWHASH_OPSLIMIT,PWHASH_MEMLIMIT,PWHASH_ALG) != 0)
 170         {
 171                 fprintf(stderr," out of memory!\n");
 172                 exit(1);
 173         }
 174         fprintf(stderr," done.\n");
 175 }
 176 #endif
 177
 178 VEC_STRUCT(threadvec, pthread_t);
 179
 180 #include "filters_main.inc.h"
 181
 182 int main(int argc,char **argv)
 183 {
 184         const char *outfile = 0;
 185         const char *infile = 0;
 186         const char *onehostname = 0;
 187         const char *arg;
 188         int ignoreargs = 0;
 189         int dirnameflag = 0;
 190         int numthreads = 0;
 191         int fastkeygen = 1;
 192         int batchkeygen = 0;
 193         int yamlinput = 0;
 194 #ifdef PASSPHRASE
 195         int deterministic = 0;
 196 #endif
 197         int outfileoverwrite = 0;
 198         struct threadvec threads;
 199 #ifdef STATISTICS
 200         struct statsvec stats;
 201         struct tstatsvec tstats;
 202         u64 reportdelay = 0;
 203         int realtimestats = 1;
 204 #endif
 205         int tret;
 206
 207         if (sodium_init() < 0) {
 208                 fprintf(stderr,"sodium_init() failed\n");
 209                 return 1;
 210         }
 211         worker_init();
 212         filters_init();
 213
 214         setvbuf(stderr,0,_IONBF,0);
 215         fout = stdout;
 216
 217         const char *progname = argv[0];
 218         if (argc <= 1) {
 219                 printhelp(stderr,progname);
 220                 exit(1);
 221         }
 222         argc--; argv++;
 223
 224         while (argc--) {
 225                 arg = *argv++;
 226                 if (!ignoreargs && *arg == '-') {
 227                         int numargit = 0;
 228                 nextarg:
 229                         ++arg;
 230                         ++numargit;
 231                         if (*arg == '-') {
 232                                 if (numargit > 1) {
 233                                         fprintf(stderr,"unrecognised argument: -\n");
 234                                         exit(1);
 235                                 }
 236                                 ++arg;
 237                                 if (!*arg)
 238                                         ignoreargs = 1;
 239                                 else if (!strcmp(arg,"help") || !strcmp(arg,"usage")) {
 240                                         printhelp(stdout,progname);
 241                                         exit(0);
 242                                 }
 243                                 else if (!strcmp(arg,"rawyaml"))
 244                                         yamlraw = 1;
 245                                 else {
 246                                         fprintf(stderr,"unrecognised argument: --%s\n",arg);
 247                                         exit(1);
 248                                 }
 249                                 numargit = 0;
 250                         }
 251                         else if (*arg == 0) {
 252                                 if (numargit == 1)
 253                                         ignoreargs = 1;
 254                                 continue;
 255                         }
 256                         else if (*arg == 'h') {
 257                                 printhelp(stdout,progname);
 258                                 exit(0);
 259                         }
 260                         else if (*arg == 'f') {
 261                                 if (argc--) {
 262                                         if (!loadfilterfile(*argv++))
 263                                                 exit(1);
 264                                 }
 265                                 else
 266                                         e_additional();
 267                         }
 268                         else if (*arg == 'D') {
 269 #ifndef PCRE2FILTER
 270                                 wantdedup = 1;
 271 #else
 272                                 fprintf(stderr,"WARNING: deduplication isn't supported with regex filters\n");
 273 #endif
 274                         }
 275                         else if (*arg == 'q')
 276                                 ++quietflag;
 277                         else if (*arg == 'x')
 278                                 fout = 0;
 279                         else if (*arg == 'v')
 280                                 verboseflag = 1;
 281                         else if (*arg == 'o') {
 282                                 outfileoverwrite = 0;
 283                                 if (argc--)
 284                                         outfile = *argv++;
 285                                 else
 286                                         e_additional();
 287                         }
 288                         else if (*arg == 'O') {
 289                                 outfileoverwrite = 1;
 290                                 if (argc--)
 291                                         outfile = *argv++;
 292                                 else
 293                                         e_additional();
 294                         }
 295                         else if (*arg == 'F')
 296                                 dirnameflag = 1;
 297                         else if (*arg == 'd') {
 298                                 if (argc--)
 299                                         setworkdir(*argv++);
 300                                 else
 301                                         e_additional();
 302                         }
 303                         else if (*arg == 't' || *arg == 'j') {
 304                                 if (argc--)
 305                                         numthreads = atoi(*argv++);
 306                                 else
 307                                         e_additional();
 308                         }
 309                         else if (*arg == 'n') {
 310                                 if (argc--)
 311                                         numneedgenerate = (size_t)atoll(*argv++);
 312                                 else
 313                                         e_additional();
 314                         }
 315                         else if (*arg == 'N') {
 316                                 if (argc--)
 317                                         numwords = atoi(*argv++);
 318                                 else
 319                                         e_additional();
 320                         }
 321                         else if (*arg == 'Z')
 322                                 fastkeygen = 0;
 323                         else if (*arg == 'z')
 324                                 fastkeygen = 1;
 325                         else if (*arg == 'B')
 326                                 batchkeygen = 1;
 327                         else if (*arg == 's') {
 328 #ifdef STATISTICS
 329                                 reportdelay = 10000000;
 330 #else
 331                                 e_nostatistics();
 332 #endif
 333                         }
 334                         else if (*arg == 'S') {
 335 #ifdef STATISTICS
 336                                 if (argc--)
 337                                         reportdelay = (u64)atoll(*argv++) * 1000000;
 338                                 else
 339                                         e_additional();
 340 #else
 341                                 e_nostatistics();
 342 #endif
 343                         }
 344                         else if (*arg == 'T') {
 345 #ifdef STATISTICS
 346                                 realtimestats = 0;
 347 #else
 348                                 e_nostatistics();
 349 #endif
 350                         }
 351                         else if (*arg == 'y')
 352                                 yamloutput = 1;
 353                         else if (*arg == 'Y') {
 354                                 yamlinput = 1;
 355                                 if (argc) {
 356                                         --argc;
 357                                         infile = *argv++;
 358                                         if (!*infile)
 359                                                 infile = 0;
 360                                         if (argc) {
 361                                                 --argc;
 362                                                 onehostname = *argv++;
 363                                                 if (!*onehostname)
 364                                                         onehostname = 0;
 365                                                 if (onehostname && strlen(onehostname) != ONION_LEN) {
 366                                                         fprintf(stderr,"bad onion argument length\n");
 367                                                         exit(1);
 368                                                 }
 369                                         }
 370                                 }
 371                         }
 372 #ifdef PASSPHRASE
 373                         else if (*arg == 'p') {
 374                                 if (argc--) {
 375                                         setpassphrase(*argv++);
 376                                         deterministic = 1;
 377                                 }
 378                                 else
 379                                         e_additional();
 380                         }
 381                         else if (*arg == 'P') {
 382                                 const char *pass = getenv("PASSPHRASE");
 383                                 if (!pass) {
 384                                         fprintf(stderr,"store passphrase in PASSPHRASE environment variable\n");
 385                                         exit(1);
 386                                 }
 387                                 setpassphrase(pass);
 388                                 deterministic = 1;
 389                         }
 390 #endif // PASSPHRASE
 391                         else {
 392                                 fprintf(stderr,"unrecognised argument: -%c\n",*arg);
 393                                 exit(1);
 394                         }
 395                         if (numargit)
 396                                 goto nextarg;
 397                 }
 398                 else
 399                         filters_add(arg);
 400         }
 401
 402         if (yamlinput && yamloutput) {
 403                 fprintf(stderr,"both -y and -Y does not make sense\n");
 404                 exit(1);
 405         }
 406
 407         if (yamlraw && !yamlinput && !yamloutput) {
 408                 fprintf(stderr,"--rawyaml requires either -y or -Y to do anything\n");
 409                 exit(1);
 410         }
 411
 412         if (outfile) {
 413                 fout = fopen(outfile,!outfileoverwrite ? "a" : "w");
 414                 if (!fout) {
 415                         perror("failed to open output file");
 416                         exit(1);
 417                 }
 418         }
 419
 420         if (!fout && yamloutput) {
 421                 fprintf(stderr,"nil output with yaml mode does not make sense\n");
 422                 exit(1);
 423         }
 424
 425         if (workdir)
 426                 createdir(workdir,1);
 427
 428         direndpos = workdirlen;
 429         onionendpos = workdirlen + ONION_LEN;
 430
 431         if (!dirnameflag) {
 432                 printstartpos = direndpos;
 433                 printlen = ONION_LEN + 1; // + '\n'
 434         } else {
 435                 printstartpos = 0;
 436                 printlen = onionendpos + 1; // + '\n'
 437         }
 438
 439         if (yamlinput) {
 440                 char *sname = makesname();
 441                 FILE *fin = stdin;
 442                 if (infile) {
 443                         fin = fopen(infile,"r");
 444                         if (!fin) {
 445                                 fprintf(stderr,"failed to open input file\n");
 446                                 return 1;
 447                         }
 448                 }
 449                 tret = yamlin_parseandcreate(fin,sname,onehostname,yamlraw);
 450                 if (infile) {
 451                         fclose(fin);
 452                         fin = 0;
 453                 }
 454                 free(sname);
 455
 456                 if (tret)
 457                         return tret;
 458
 459                 goto done;
 460         }
 461
 462         filters_prepare();
 463
 464         filters_print();
 465
 466 #ifdef STATISTICS
 467         if (!filters_count() && !reportdelay)
 468 #else
 469         if (!filters_count())
 470 #endif
 471                 return 0;
 472
 473 #ifdef EXPANDMASK
 474         if (numwords > 1 && flattened)
 475                 fprintf(stderr,"WARNING: -N switch will produce bogus results because we can't know filter width. reconfigure with --enable-besort and recompile.\n");
 476 #endif
 477
 478         if (yamloutput)
 479                 yamlout_init();
 480
 481         pthread_mutex_init(&keysgenerated_mutex,0);
 482         pthread_mutex_init(&fout_mutex,0);
 483 #ifdef PASSPHRASE
 484         pthread_mutex_init(&determseed_mutex,0);
 485 #endif
 486
 487         if (numthreads <= 0) {
 488                 numthreads = cpucount();
 489                 if (numthreads <= 0)
 490                         numthreads = 1;
 491         }
 492         if (!quietflag)
 493                 fprintf(stderr,"using %d %s\n",
 494                         numthreads,numthreads == 1 ? "thread" : "threads");
 495
 496 #ifdef PASSPHRASE
 497         if (!quietflag && deterministic && numneedgenerate != 1)
 498                 fprintf(stderr,"CAUTION: avoid using keys generated with same password for unrelated services, as single leaked key may help attacker to regenerate related keys.\n");
 499 #endif
 500
 501         signal(SIGTERM,termhandler);
 502         signal(SIGINT,termhandler);
 503
 504         VEC_INIT(threads);
 505         VEC_ADDN(threads,numthreads);
 506 #ifdef STATISTICS
 507         VEC_INIT(stats);
 508         VEC_ADDN(stats,numthreads);
 509         VEC_ZERO(stats);
 510         VEC_INIT(tstats);
 511         VEC_ADDN(tstats,numthreads);
 512         VEC_ZERO(tstats);
 513 #endif
 514
 515 #if 0
 516         pthread_attr_t tattr,*tattrp = &tattr;
 517         tret = pthread_attr_init(tattrp);
 518         if (tret) {
 519                 perror("pthread_attr_init");
 520                 tattrp = 0;
 521         }
 522         else {
 523                 tret = pthread_attr_setstacksize(tattrp,80<<10);
 524                 if (tret)
 525                         perror("pthread_attr_setstacksize");
 526         }
 527 #endif
 528
 529         for (size_t i = 0;i < VEC_LENGTH(threads);++i) {
 530                 void *tp = 0;
 531 #ifdef STATISTICS
 532                 tp = &VEC_BUF(stats,i);
 533 #endif
 534                 tret = pthread_create(&VEC_BUF(threads,i),0,
 535 #ifdef PASSPHRASE
 536                                 deterministic ? (
 537                                         batchkeygen ? worker_batch_pass : worker_fast_pass) :
 538 #endif
 539                                 batchkeygen ? worker_batch :
 540                                 (fastkeygen ? worker_fast : worker_slow),tp);
 541                 if (tret) {
 542                         fprintf(stderr,"error while making " FSZ "th thread: %s\n",i,strerror(tret));
 543                         exit(1);
 544                 }
 545         }
 546
 547 #if 0
 548         if (tattrp) {
 549                 tret = pthread_attr_destroy(tattrp);
 550                 if (tret)
 551                         perror("pthread_attr_destroy");
 552         }
 553 #endif
 554
 555 #ifdef STATISTICS
 556         struct timespec nowtime;
 557         u64 istarttime,inowtime,ireporttime = 0,elapsedoffset = 0;
 558         if (clock_gettime(CLOCK_MONOTONIC,&nowtime) < 0) {
 559                 perror("failed to get time");
 560                 exit(1);
 561         }
 562         istarttime = (1000000 * (u64)nowtime.tv_sec) + ((u64)nowtime.tv_nsec / 1000);
 563 #endif
 564         struct timespec ts;
 565         memset(&ts,0,sizeof(ts));
 566         ts.tv_nsec = 100000000;
 567         while (!endwork) {
 568                 if (numneedgenerate && keysgenerated >= numneedgenerate) {
 569                         endwork = 1;
 570                         break;
 571                 }
 572                 nanosleep(&ts,0);
 573
 574 #ifdef STATISTICS
 575                 clock_gettime(CLOCK_MONOTONIC,&nowtime);
 576                 inowtime = (1000000 * (u64)nowtime.tv_sec) + ((u64)nowtime.tv_nsec / 1000);
 577                 u64 sumcalc = 0,sumsuccess = 0,sumrestart = 0;
 578                 for (int i = 0;i < numthreads;++i) {
 579                         u32 newt,tdiff;
 580                         // numcalc
 581                         newt = VEC_BUF(stats,i).numcalc.v;
 582                         tdiff = newt - VEC_BUF(tstats,i).oldnumcalc;
 583                         VEC_BUF(tstats,i).oldnumcalc = newt;
 584                         VEC_BUF(tstats,i).numcalc += (u64)tdiff;
 585                         sumcalc += VEC_BUF(tstats,i).numcalc;
 586                         // numsuccess
 587                         newt = VEC_BUF(stats,i).numsuccess.v;
 588                         tdiff = newt - VEC_BUF(tstats,i).oldnumsuccess;
 589                         VEC_BUF(tstats,i).oldnumsuccess = newt;
 590                         VEC_BUF(tstats,i).numsuccess += (u64)tdiff;
 591                         sumsuccess += VEC_BUF(tstats,i).numsuccess;
 592                         // numrestart
 593                         newt = VEC_BUF(stats,i).numrestart.v;
 594                         tdiff = newt - VEC_BUF(tstats,i).oldnumrestart;
 595                         VEC_BUF(tstats,i).oldnumrestart = newt;
 596                         VEC_BUF(tstats,i).numrestart += (u64)tdiff;
 597                         sumrestart += VEC_BUF(tstats,i).numrestart;
 598                 }
 599                 if (reportdelay && (!ireporttime || inowtime - ireporttime >= reportdelay)) {
 600                         if (ireporttime)
 601                                 ireporttime += reportdelay;
 602                         else
 603                                 ireporttime = inowtime;
 604                         if (!ireporttime)
 605                                 ireporttime = 1;
 606
 607                         double calcpersec = (1000000.0 * sumcalc) / (inowtime - istarttime);
 608                         double succpersec = (1000000.0 * sumsuccess) / (inowtime - istarttime);
 609                         double restpersec = (1000000.0 * sumrestart) / (inowtime - istarttime);
 610                         fprintf(stderr,">calc/sec:%8lf, succ/sec:%8lf, rest/sec:%8lf, elapsed:%5.6lfsec\n",
 611                                 calcpersec,succpersec,restpersec,
 612                                 (inowtime - istarttime + elapsedoffset) / 1000000.0);
 613
 614                         if (realtimestats) {
 615                                 for (int i = 0;i < numthreads;++i) {
 616                                         VEC_BUF(tstats,i).numcalc = 0;
 617                                         VEC_BUF(tstats,i).numsuccess = 0;
 618                                         VEC_BUF(tstats,i).numrestart = 0;
 619                                 }
 620                                 elapsedoffset += inowtime - istarttime;
 621                                 istarttime = inowtime;
 622                         }
 623                 }
 624                 if (sumcalc > U64_MAX / 2) {
 625                         for (int i = 0;i < numthreads;++i) {
 626                                 VEC_BUF(tstats,i).numcalc /= 2;
 627                                 VEC_BUF(tstats,i).numsuccess /= 2;
 628                                 VEC_BUF(tstats,i).numrestart /= 2;
 629                         }
 630                         u64 timediff = (inowtime - istarttime + 1) / 2;
 631                         elapsedoffset += timediff;
 632                         istarttime += timediff;
 633                 }
 634 #endif
 635         }
 636
 637         if (!quietflag)
 638                 fprintf(stderr,"waiting for threads to finish...");
 639         for (size_t i = 0;i < VEC_LENGTH(threads);++i)
 640                 pthread_join(VEC_BUF(threads,i),0);
 641         if (!quietflag)
 642                 fprintf(stderr," done.\n");
 643
 644         if (yamloutput)
 645                 yamlout_clean();
 646
 647 #ifdef PASSPHRASE
 648         pthread_mutex_destroy(&determseed_mutex);
 649 #endif
 650         pthread_mutex_destroy(&fout_mutex);
 651         pthread_mutex_destroy(&keysgenerated_mutex);
 652
 653 done:
 654         filters_clean();
 655
 656         if (outfile)
 657                 fclose(fout);
 658
 659         return 0;
 660 }