signduterre.py

   1 #!/usr/bin/python3.0
   2 #
   3 # ToC
   4 # 1. DOCUMENTATION
   5 # 2. IMPORT & INITIALIZATION
   6 # 3. OPTION HANDLING
   7 # 4. ARGUMENT PROCESSING
   8 # 5. SIGNATURE CREATION AND CHECKING
   9
  10 #############################################################################
  11 #                                                                           #
  12 #               DOCUMENTATION                                               #
  13 #                                                                           #
  14 #############################################################################
  15
  16 manual = """
  17 [[[!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd"]]][[[html]]][[[header]]][[[title]]]Signature-du-Terroir[[[/title]]][[[/header]]][[[body]]][[[h1]]]Signature-du-Terroir[[[/h1]]][[[p]]]
  18 Construct a signature of the installed software state or check the integrity of the installation
  19 using a previously made signature.
  20 [[[/p]]][[[p]]]
  21 Usage: signduterre.py [options] FILE1 FILE2 ...
  22 [[[/p]]][[[p]]]
  23 Options:[[[/p]]][[[pre]]]
  24   -h, --help            show this help message and exit
  25   -s HEX, --salt=HEX    Enter salt in cleartext. If not given, a hexadecimal
  26                         salt will be suggested. The SUGGESTED[=N] keyword will
  27                         cause the selection of the suggested string. N is the
  28                         number of salts generated (default N=1). If N>1, all
  29                         will be printed and a random one will be used to
  30                         generate the signature (selection printed to STDERR).
  31   -a, --all-salts-pattern
  32                         Use all salts in sequence, randomly replace salts with
  33                         incorrect ones in the output to create a pattern of
  34                         failing hashes indicated by a corresponding integer
  35                         number. Depends on '--salt SUGGESTED=N'.
  36                                                 Implies --total-only.
  37   -p TEXT, --passphrase=TEXT
  38                         Enter passphrase in cleartext, the keyword
  39                         SUGGESTED[=N] will cause the suggested passphrase to
  40                         be used. If N>1, N passphrases will be printed to
  41                         STDERR and a random one will be used (selection
  42                         printed to STDERR). Entering the name of an existing
  43                         file (or '-' for STDIN) will cause it to be read and a
  44                         random passphrase found in the file will be used
  45                         (creating a signature), or they will all be used in
  46                         sequence (--check-file).
  47   -c FILE, --check-file=FILE
  48                         Check contents with the output of a previous run from
  49                         file or STDIN ('-'). Except when the --quiet option is
  50                         given, the previous output will contain all
  51                         information needed for the program, but not the
  52                         passphrase and the --execute option.
  53   -i FILE, --input-file=FILE
  54                         Use names from FILE or STDIN ('-'), use one
  55                         filename per line
  56   -o FILE, --output-file=FILE
  57                         Print to FILE instead of STDOUT
  58   -P FILE, --Private-file=FILE
  59                         Print private information (passwords etc.) to FILE
  60                         instead of STDERR
  61   -u USER, --user=USER  Execute $(cmd) as USER, default 'nobody' (root/sudo
  62                         only)
  63   -S, --Status          For each file, add a line with unvarying file status
  64                         information: st_mode, st_ino, st_dev, st_uid, st_gid,
  65                         and st_size (like the '?' prefix, default False)
  66   --Status-values=MODE  Status values to print for --Status, default MODE is
  67                         'fmidlugs' (file, mode, inode, device, uid, gid, size).
  68                                                 Also available (n)l(inks), a(time), (m)t(ime), and
  69                                                 c(time).
  70   -t, --total-only      Only print the total hash, unsets --detailed-view
  71                         (default True)
  72   -d, --detailed-view   Print hashes of individual files, is unset by --total-
  73                         only (default False)
  74   -e, --execute         Interpret $(cmd) (default False)
  75   --execute-args=ARGS   Arguments for the $(cmd) commands ($1 ....)
  76   -n, --no-execute      Explicitely do NOT Interpret $(cmd)
  77   --print-execute       Print the results of $() command execution to STDERR
  78                         for debugging purposes
  79    -m, --manual          Print the manual and exit
  80   --manual-html         Print the manual in HTML format and exit
  81   --manual-make         Print the examples in the manual as a makefile and
  82                         exit
  83   -r, --release-notes   Print the release notes and exit
  84   -l, --license         Print license text and exit
  85   -v, --verbose         Print more information on output
  86   -q, --quiet           Print minimal information (hide filenames). If the
  87                         output is used with --check-file, the command line
  88                         options and arguments must be repeated.
  89 [[[/pre]]][[[p]]]
  90 FILE1 FILE2 ...
  91 Names and paths of one or more files to be checked. All file arguments in SdT accept '-' as the STDIN file
  92 (ie, piped data).
  93 [[[/p]]][[[p]]]
  94 Any name starting with a '$', eg, $PATH, will be interpreted as an environmental variable or a command
  95 according to the bash conventions: '$ENV' and '${ENV}' as variables, '$(cmd;cmd...)' as system commands
  96 (bash --restricted -c 'cmd;cmd...' PID). Where PID the current Process ID is (available as positional
  97 parameter $0). Other parameters can be entered with the --execute-args option ($1 etc). Do not forget to
  98 enclose the arguments in single ''-quotes! The commands are scanned for unwanted characters and these
  99 are removed (eg, ' and \\, however, escaping $ is allowed, eg, '\\$1'). The use of '$(cmd;cmd...)'
 100 requires explicit use of the -e or --execute option.
 101 [[[/p]]][[[p]]]
 102 If executed as root or sudo, $(cmd;cmd...) will be executed as 'sudo -H -u <user>' which defaults to
 103 --user nobody ('--user root' is at your own risk). This will obviously not work when invoked as non-root/sudo.
 104 --user root is necessary when you need to check privileged information, eg, you want to check the MBR with
 105 '$(dd if=/dev/hda bs=512 count=1 status=noxfer | od -X)'
 106 However, as you might use --check-file with files you did not create yourself, it is important to
 107 be warned if commands are to be executed.
 108 [[[/p]]][[[p]]]
 109 Interpretation of $() ONLY works if the -e or --execute options are entered. signduterre.py can easily
 110 be adapted to automatically use the setting in the check-file. However, this is deemed insecure and
 111 commented out in the distribution version.
 112 [[[/p]]][[[p]]]
 113 The -n or --no-execute option explicitely supress the interpretation of $(cmd) arguments.
 114 [[[/p]]][[[p]]]
 115 Meta information from stat() on files is signed when the filename is preceded by a '?'. '?./signduterre.py' will
 116 extract (st_mode, st_ino, st_dev, st_nlinks, st_uid, st_gid, st_size) and hash a line of these data (visible
 117 with --verbose). The --Status option will automatically add such a line in front of every file. Note that '?'
 118 is implied for directories. Both '/' and '?/' produce a hash of, eg,:
 119 [[[/p]]][[[pre]]]
 120 stat(/) = [st_mode=041775, st_ino=2, st_dev=234881026, st_uid=0, st_gid=80, st_size=1360]
 121 [[[/pre]]][[[p]]]
 122 The --Status-values=<mode> option selects which status values will be used: f(ile), m(ode), i(node),
 123 d(evice), u(id), g(id), s(ize), (n)l(inks), a(time), (m)t(ime), and c(time). Default is
 124 --Status-values='fmidugs'. Note that nlinks of a directory include every file in the directory, so this
 125 option can check whether files have been added to a directory.
 126 [[[/p]]][[[p]]]
 127 Arguments enclosed in []-brackets will be hidden in the output. That is, '[/proc/self/exe]' will show up as
 128 '[1]' in the output (or '[n]' with n the number of the hidden argument), equivalent to the use of the
 129 --quiet option. This means the hidden arguments must be entered again when using the --check-file (-c)
 130 option.
 131 [[[/p]]][[[p]]]
 132 Signature-du-Terroir
 133 [[[/p]]][[[p]]]
 134 A very simple tool to generate a signature that can be used to test the integrity of files and "states" in
 135 a running installation. signduterre.py constructs a signature of the current system state and checks
 136 installation state with a previously made signature. The files are hashed with a passphrase to allow detection
 137 of compromised systems while running on the same system. The signature checking can be subverted, but the
 138 flexibillity of signduterre.py and the fact that the output of any command can be tested should hamper
 139 automated root-kit attacks.
 140 [[[/p]]][[[p]]]
 141 signduterre.py writes a total SHA-256 hash to STDOUT of all the files and commands entered as arguments. It
 142 can also write a hash for each individual file (insecure). The output of a signature can be send to a file and
 143 later used to check with --check-file. Hashes are calculated with a hashed salt + passphrase sequence
 144 pre-pended to create unpredictable hashes. This procedure ensures that an attacker does not know whether or
 145 not the correct passphrase has been entered. An attacker can only know when to supply the requested hash
 146 values if she knows the passphrase or has copies available of all the tested files and output of commands to
 147 calculate the hashes on the fly.
 148 [[[/p]]][[[p]]]
 149 The Problem
 150 [[[/p]]][[[p]]]
 151 The problem SdT tries to solve is how to test whether your system has been compromised when you can only use
 152 the potentially compromised system? The solution is to store a password encrypted signature (or fingerprint)
 153 of your system when you are sure it is in a good state. Then you check whether the system can still
 154 distinguish between correct and incorrect passwords when it regenerates the signature. The trick is to use
 155 the right data (ie, questions) to generate the signature.
 156 [[[/p]]][[[p]]]
 157 As an illustration of the problem, take the sha256sum command which generates file hashes using the SHA256
 158 algorithm. Hashes can be generated and checked with this command:
 159 [[[/p]]][[[pre]]]
 160 # Use of sha256sum to check integrity of ps and ls commands
 161 $ sha256sum /bin/ps /bin/ls > ps-ls.sh256
 162 $ sha256sum -c ps-ls.sh256
 163 [[[/pre]]][[[p]]]
 164 A compromised file will show up as FAILED. This is ok for unintentional changes to the files. However, a malicious
 165 attacker could easily replace /usr/bin/sha256sum with a program that would replace the hash of malicious replacements
 166 of these files with the hash sums of the original files. There are three easy ways of doing that. Either simply say 'ok'
 167 when checking the file, print out the old hash whenever an altered file is requested, or look for the hash of the new,
 168 malicious replacement and print out the old hash sum instead. The former two are easy to circumvent, the last one is
 169 somewhat less easy.
 170 [[[/p]]][[[p]]]
 171 SECURITY
 172 [[[/p]]][[[p]]]
 173 When run on a compromised system, signduterre.py can be subverted if the attacker keeps a copy of all the
 174 files and command outputs, and reroutes the open() and stat() functions, or simply delegating signduterre.py
 175 to a chroot jail with the original system. In principle, signduterre.py only checks whether the computer
 176 responds identically to when the signature file was made. There is no theoretic barrier against a compromised
 177 computer perfectly simulating the original system when tested, but behaving adversely at other times. Except
 178 for running from clean boot media (USB?), I know of no theoretical sound solution to this problem.
 179 [[[/p]]][[[p]]]
 180 However, this scenario assumes the use of unlimited resources and time. Inside a limited, real computer system,
 181 the attacker must make compromises on what can and what cannot be simulated with the available time and
 182 hardware. The idea behind signduterre.py is to "ask difficult questions" that increase the cost of simulating
 183 the original system high enough to make detection of successful attacks likely.signduterre.py simply intends
 184 to raise the bar high enoug. One point is to store the times needed to create the original hashes. This timing
 185 can later be used to see whether the new timings are reasonable. If the same hardware takes considerably
 186 longer to perform the same calculations, or needs a much longer delay before it starts, the tester might want
 187 to see where this time is spent.
 188 [[[/p]]][[[p]]]
 189 Signature-du-Terroir works on the assumption that any attacker in control of a compromised system cannot
 190 predict whether the passphrase entered is correct or not. An attacker can always intercept the in- and output
 191 of signduterre. When running with --check-file, this means the program can be made to print out OK
 192 irrespective of the tests. A safe use of signduterre.py is to start with a random number of incorrect
 193 passphrases and see whether they fail. Alternatively, and easier, is to add a number of unused salts
 194 to the check-file and let the attacker guess which one is correct.
 195 [[[/p]]][[[p]]]
 196 THE CORRECT USE OF signduterre.py IS TO ENTER A RANDOM NUMBER OF INCORRECT PASSPHRASES OR SALTS FOR EACH
 197 TEST AND SEE WHETHER IT FAILS AT THE CORRECT INSTANCES!
 198 [[[/p]]][[[p]]]
 199 On a compromised system, signduterre.py's detailed file testing (--detailed-view) is easily subverted. With a
 200 matched file hash, the attacker will know that the correct passphrase has been entered and can print out the
 201 stored hashes or 'ok's  for the rest of the checks. So if the attacker keeps any entry in the signature file
 202 uncompromised, she can intercept the output, test the password on the unchanged entry and substitute the
 203 requested hashes for the output if the hash of that entry matches.
 204 [[[/p]]][[[p]]]
 205 When checking for root-kits and other malware, it is safest to compare the signature files from a different,
 206 clean, system. But then you would not need signduterre.py anyway. If you have to work on the system itself,
 207 only use the -t or --total-only options to create signatures with a total hash and without individual file
 208 hashes. Such a signature can be used to check whether the system is unchanged. Another signature file WITH A
 209 DIFFERENT PASSPHRASE can then be used to identify the individual files that have changed. If a detailed
 210 signature file has the same passphrase, an attacker could use that other file to read the individual file
 211 hashes to check whether the correct passphrase was entered.
 212 [[[/p]]][[[p]]]
 213 Using the --check-file option in itself is UNsafe. An attacker simply has to print out 'OK' to defeat the
 214 check. This attack can be foiled by making it unpredictable when signduterre.py should return 'OK'. This can
 215 be done by using a list of salts or passphrases where only one of them (or none!) is correct. Any attacker
 216 will have to guess when to return 'OK'.
 217 [[[/p]]][[[p]]]
 218 As generating and entering wrong passphrases and salts is tedious, users have to be supported in correct use
 219 of SdT. To assist users, the '--salt SUGGESTED=<N>' option will generate a number N of salts. When
 220 checking, each of these salts is tried in turn. An attacker that is unable to simulate the uncompromised
 221 system will have to guess which one of the salts is the correct one, and whether or not the passphrase
 222 is correct. This increases the chances of detecting compromised systems. If this is not enough guess
 223 work, the '-a', '--all-salts-pattern' option will use all salts in sequence to generate total hashes,
 224 but random salts will be changed in the output. This generates a pattern of failed salt tests. This pattern
 225 is translated into a bit pattern and printed as an integer ([Fail, Fail, OK, Fail, OK, OK, Fail, OK]
 226 = 00101101 (least significant first) = 10110100 (unsigned bin) = 180). On creation of a signature, this
 227 number is printed to STDERR, on checking (--check-file) it is printed to STDOUT (note that the number
 228 will never become 0 or all Fail). So for '--salt SUGGESTED=<N> --all-salts-pattern' the probability of
 229 guessing the correct output goes from 1/N to 1/(2^N - 1). Note that '--all-salts-pattern' will work,
 230 but is pointless, without '--salt SUGGESTED=<N>' with N>1.
 231 [[[/p]]][[[p]]]
 232 The '--passphrase SUGGESTED=N' option will generate and print N passphrases. One of these is chosen at
 233 random for the signature. The number of the chosen passphrase is printed on STDERR with the passwords.
 234 When checking a file, the stored passphrases can be read in again, either by entering the passphrase
 235 file after the --passphrase option ('--passphrase <passphrase file>'), or directly from the --check-file.
 236 signduterre.py will print out the result for each of the passphrases.
 237 [[[/p]]][[[p]]]
 238 Note, that storing passphrases in a file and feeding it to signduterre.py is MUCH less secure than just
 239 typing them in. Moreover, it might completely defeat the purpose of signduterre.py. If future experiences
 240 cast any more doubt on the security of this option, it will be removed.
 241 [[[/p]]][[[p]]]
 242 For those who want to know more about what an "ideal attacker" can do, see:[[[br]]]
 243 Ken Thompson "Reflections on Trusting Trust"[[[br]]]
 244 [[[a href="http://cm.bell-labs.com/who/ken/trust.html"]]]http://cm.bell-labs.com/who/ken/trust.html[[[/a]]][[[br]]]
 245 [[[a href="http://www.ece.cmu.edu/~ganger/712.fall02/papers/p761-thompson.pdf"]]]http://www.ece.cmu.edu/~ganger/712.fall02/papers/p761-thompson.pdf[[[/a]]]
 246 [[[/p]]][[[p]]]
 247 David A Wheeler "Countering Trusting Trust through Diverse Double-Compiling"[[[br]]]
 248 [[[a href="http://www.acsa-admin.org/2005/abstracts/47.html"]]]http://www.acsa-admin.org/2005/abstracts/47.html[[[/a]]]
 249 [[[/p]]][[[p]]]
 250 and the discussion of these at Bruce Schneier's 'Countering "Trusting Trust"'[[[br]]]
 251 [[[a href="http://www.schneier.com/blog/archives/2006/01/countering_trus.html"]]]http://www.schneier.com/blog/archives/2006/01/countering_trus.html[[[/a]]]
 252 [[[/p]]][[[p]]]
 253 Manual
 254 [[[/p]]][[[p]]]
 255 The intent of signduterre.py is to ensure that the signature cannot be subverted even if the system has been
 256 compromised by an attacker that has obtained root control over the computer and any existing signature files.
 257 [[[/p]]][[[p]]]
 258 signduterre.py asks for a passphrase which is PRE-pended to every file before the hash is constructed (unless
 259 the passphrase is entered with an option). As long as the passphrase is not compromised, the hashes cannot
 260 be reconstructed. A randomly generated, unpadded base-64 encoded 16 Byte password (ie, ~22 characters) is
 261 suggested in interactive use. If '--passphrase SUGGESTED' is entered on the command line or no passphrase is
 262 enetered when asked, the suggested value will be used. This value is printed to STDERR (the screen or 2) for
 263 safe keeping. Please, make sure you store the printed passphrase. For instance:
 264 [[[/p]]][[[pre make=example]]]
 265 #
 266 # Simple system sanity test using the 'which' command to establish the paths
 267 $ python3.0 signduterre.py --passphrase SUGGESTED --salt SUGGESTED --detailed-view \\
 268 `which python3.0 bash ps ls find stat` 2> test-20090630_11-14-03.pwd > test-20090630_11-14-03.sdt
 269 $ python3.0 signduterre.py --passphrase test-20090630_11-14-03.pwd --check-file test-20090630_11-14-03.sdt
 270 [[[/pre]]][[[p]]]
 271 The first command will store the passphrase (and all error messages) in a file 'Signature_20090630_11-14-03.pwd'
 272 and the check-file in 'Signature_20090630_11-14-03.sdt'. The second line will test the signature.
 273 The signature will be made of the files used for the commands python3.0, bash, ps, ls, find, and stat.
 274 These files are found using the 'which' command.
 275 [[[/p]]][[[p]]]
 276 It is not secure to store files with the passphrase on the system you want to check. However, you could
 277 pipe STDERR or --Private-file to some safe site.
 278 [[[/p]]][[[p]]]
 279 Good passphrases are difficult to remember, so their plaintext form should be protected. To protect the
 280 passphrase against rainbow and brute force attacks, the passphrase is concatenated to a salt phrase and
 281 hashed before use (SHA-256).
 282 [[[/p]]][[[p]]]
 283 The salt phrase is requested when constructing a signature. In interactive use, an 8 byte hexadecimal
 284 (= 16 character) salt from /dev/urandom is suggested. If '--salt SUGGESTED' is entered on the command line
 285 as the salt, the suggested value will be used. The salt is printed in plaintext to the output. The salt will
 286 make it more difficult to determine whether the same passphrase has been used to create different signatures.
 287 [[[/p]]][[[p]]]
 288 At the bottom, a 'TOTAL HASH' line will be printed that hashes all the lines printed for the files. This
 289 includes the file names as printed on the hash lines. It is not inconceivable that existing signature files
 290 could have been compromised in ways that might be missed when checking the signature. The total hash will
 291 point out such changes.
 292 [[[/p]]][[[p]]]
 293 Examples:[[[/p]]][[[pre make=example]]]
 294 #
 295 # Self test of root directory, python, and signduterre.py using the 'which' command to establish the paths
 296 $ python3.0 signduterre.py --detailed-view --salt 436a73e3 --passphrase liauwefa3251EWC -o test-self.sdt \\
 297  / `which python3.0 signduterre.py`
 298 $ python3.0 signduterre.py --passphrase liauwefa3251EWC -c test-self.sdt
 299 [[[/pre]]][[[p]]]
 300 Write a signature to the file test-self.sdt and test it with the --check-file option. The signature contains
 301 the SHA-256 hashes of the files, /usr/bin/python3.0, signduterre.py, and the status information on the root
 302 directory. The salt '436a73e3' and passphrase 'liauwefa3251EWC' are used.
 303 [[[/p]]][[[pre make=linux]]]
 304 #
 305 # Self test of root directory, python, and signduterre.py using the the /proc file system
 306 $ python3.0 signduterre.py --detailed-view --salt SUGGESTED --passphrase liauwefa3251EWC -o test-self_proc.sdt \\
 307  /proc/self/root /proc/self/exe `which signduterre.py`
 308 $ python3.0 signduterre.py --passphrase liauwefa3251EWC --check-file test-self_proc.sdt
 309 [[[/pre]]][[[p]]]
 310 Write a signature to the file test-self_proc.sdt and test it with the --check-file option. The signature
 311 contains the SHA-256 hashes of the same files as above, /usr/bin/python3.0, signduterre.py, and the status
 312 information on the root directory. However, the python executable and the root directory are now accessed
 313 through the /proc file system. The suggested salt is used (written to test-self_proc.sdt) and the passphrase
 314 is (again) 'liauwefa3251EWC'.
 315 [[[/p]]][[[pre make=example]]]
 316 #
 317 # Test of supporting commands for chkrootkit
 318 $ python3.0 signduterre.py --execute --total-only --salt SUGGESTED=8 --passphrase SUGGESTED --Status \\
 319   --output-file=test-chkrootkit.sdt --Private-file=test-chkrootkit.pwd \\
 320   signduterre.py `which bash awk cut egrep find head id ls netstat ps strings sed uname`
 321 $ python3.0 signduterre.py --execute --passphrase test-chkrootkit.pwd --check-file test-chkrootkit.sdt
 322 [[[/pre]]][[[p]]]
 323 Writes a signature of the requested files to test-chkrootkit.sdt (signature) and private information to
 324 test-chkrootkit.pwd (password and selected salt) and checks it in the next line. The files are those of
 325 commands required by the chkrootkit program (http://www.chkrootkit.org/), with bash added. The 'which'
 326 command will give the paths for the commands. Eight salts are generated, of which only 1 is actually
 327 used. When checking, the correct salt should match. This prevents a compromised program from simply
 328 printing out OK tot he check. A more comprehensive evation of guessing the correct salt can be obtained
 329 by using the '--all-salts-pattern' option.
 330 [[[/p]]][[[pre make=linux]]]
 331 #
 332 # Simply lump all "system" files, the PATH environment variable and the first 2 columns of the output of lsmod
 333 $ python3.0 signduterre.py --execute --detail --salt SUGGESTED --passphrase liauwefa3251EWC --Status --total-only \\
 334   signduterre.py /sbin/* /bin/* /usr/bin/find /usr/bin/stat /usr/bin/python* '${PATH}' \\
 335   '$(lsmod | awk "{print \$1, \$2}")' > test-20090625_14-31-54.sdt
 336 #
 337 # Failing check due to missing --execute option
 338 $ python3.0 signduterre.py --passphrase liauwefa3251EWC -c test-20090625_14-31-54.sdt
 339 $ python3.0 signduterre.py --passphrase liauwefa3251EWC -c test-20090625_14-31-54.sdt --no-execute
 340 #
 341 # Successful check
 342 $ python3.0 signduterre.py --execute --passphrase liauwefa3251EWC --check-file test-20090625_14-31-54.sdt
 343 [[[/pre]]][[[p]]]
 344 Prints a signature to the system test-20090625_14-31-54.sdt and the automatically generated password to
 345 test-20090625_14-31-54.pwd. The salt will be automatically determined. The signature contains the SHA-256
 346 hashes of the file status and file contents of signduterre.py, /sbin/*, /bin/*,  /usr/bin/find,
 347 /usr/bin/file, /usr/bin/python* on separate lines, and a hash of the PATH environment variable. Do not
 348 display the hash of every single file, which could be insecure, but only the total hash.
 349 The first two checks will both fail if test-20090625_14-31-54.sdt contains a $(cmd) entry.
 350 The --no-execute option is default and prevents the execute option (if reading the execute option from the
 351 signature file has been activated). The last check will succeed (if the files have not been changed).
 352 [[[/p]]][[[pre make=example]]]
 353 #
 354 # Use a list of generated passphrases
 355 $ python3.0 signduterre.py --salt SUGGESTED --passphrase SUGGESTED=20 signduterre.py \\
 356 2> test-20090630_16-44-34.pwd > test-20090630_16-44-34.sdt
 357 $ python3.0 signduterre.py -p test-20090630_16-44-34.pwd -c test-20090630_16-44-34.sdt
 358 [[[/pre]]][[[p]]]
 359 Will generate and print 20 passphrases and print a signature using one randomly chosen passphrase from these
 360 20. Everything is written to the files 'test-20090630_16-44-34.pwd' and 'test-20090630_16-44-34.sdt'.
 361 Such file names can easily be generated with 'test-`date "+%Y%m%d_%H-%M-%S"`.sdt'.
 362 The next command will check all 20 passphrases generated before from the Signature file and print the results.
 363 [[[/p]]][[[pre make=example]]]
 364 #
 365 # Use a list of generated salts with a pattern of correct salts
 366 $ python3.0 signduterre.py --salt SUGGESTED=16 --passphrase SUGGESTED --all-salts-pattern \\
 367 -P test-salt-pattern.pwd -o test-salt-pattern.sdt `which bash stat find ls ps id uname awk gawk perl`
 368 $ python3.0 signduterre.py -p test-salt-pattern.pwd -c test-salt-pattern.sdt
 369 # Compare to salt pattern number to the one from the check-file
 370 $ cat test-salt-pattern.pwd
 371 [[[/pre]]][[[p]]]
 372 [[[/p]]][[[pre make=sudo]]]
 373 #
 374 # Check MBR and current root directory (sudo and root user)
 375 $ sudo python3.0 signduterre.py -u root -s SUGGESTED -p SUGGESTED --Status-values='i' -v -e -t \\
 376 --output-file test-boot-sector.sdt --Private-file test-boot-sector.pwd --execute-args=sda \\
 377 '?/proc/self/root' `which dd` '$(dd if=/dev/$1 bs=512 count=1 status=noxfer | od -X)'
 378 $ sudo python3.0 signduterre.py -u root -e -p test-boot-sector.pwd -c test-boot-sector.sdt
 379 [[[/pre]]][[[p]]]
 380 Will hash the inode numbers of the effective root directory (eg, chroot) and the executable (python)
 381 together with the contents of the MBR (Master Boot Record) on /dev/sda in Hex. It uses suggested salt and
 382 passphrase. Accessing /dev/sda is only possible when root, so the command is entered with sudo and
 383 --user root. Use the '--print-execute' option if you want to check the output of the dd command.
 384 [[[/p]]][[[p]]]
 385 The main problem with intrusion detection by comparing file contents is the ability of an attacker
 386 to redirect attempts to read a compromised file to a stored copy of the original. So, sha256 or
 387 python could be changed to read '/home/attacker/old/ps' when the argument was '/bin/ps'. This would
 388 foil any scheme that depends on entering file names in programs. An answer to this threat is to
 389 read the bytes in files in as many ways as possible. Therefor, forcing an attacker to change many
 390 files which itself would increase the probability of detection of the attack. The following command
 391 will read the same (test) file, and generate identical hashes, in many different ways.
 392 [[[/p]]][[[pre make=example]]]
 393 #
 394 # Example generating identical signatures of the same text file in different ways
 395 $ dd if=signduterre.py 2>/dev/null | \\
 396 python3.0 signduterre.py -v -d -s 1234567890abcdef -p poiuytrewq \\
 397 --execute --execute-args='signduterre.py' \\
 398 signduterre.py - \\
 399 '$(cat $1)' \\
 400 '$(grep "" $1)' \\
 401 '$(awk "{print}" $1)' \\
 402 '$(cut -f 1-100 $1)' \\
 403 '$(perl -ane "{print \$_}" $1)' \\
 404 '$(python3.0 -c "import sys;f=open(sys.argv[1]);sys.stdout.buffer.write(f.buffer.read())" $1;)' \\
 405 '$(ruby -e "f=open(ARGV[0]);print f.read();" $1;)'
 406 [[[/pre]]][[[p]]]
 407 These "commands" do not always return the same bytes (awk), or any bytes at all (grep), from a text
 408 file as when used with a binary file. However, if the commands can print the bytes unaltered, the
 409 signatures will be identical. That is, the following arguments will work on a binary file:
 410 [[[/p]]][[[pre make=example]]]
 411 # Example generating identical signatures of the same file in different ways, now for binary files
 412 $ dd if=/bin/bash 2>/dev/null | \\
 413 python3.0 signduterre.py -v -d -s 1234567890abcdef -p poiuytrewq \\
 414 --execute --execute-args='/bin/bash' \\
 415 /bin/bash - \\
 416 '$(cat $1)' \\
 417 '$(perl -ane "{print \$_}" $1)' \\
 418 '$(python3.0 -c "import sys;f=open(sys.argv[1]);sys.stdout.buffer.write(f.buffer.read())" $1;)' \\
 419 '$(ruby -e "f=open(ARGV[0]);print f.read();" $1;)'
 420 [[[/pre]]][[[p]]]
 421 Will generate the same identical signatures for /bin/bash, STDIN, '$(cat /bin/bash)' etc.
 422 There are obviously many more ways to read out the bytes from the disk or memory. The main point
 423 being that it should be difficult to predict for an attacker which commands must be compromised
 424 to hide changes in the system.
 425 [[[/p]]][[[p]]]
 426 The examples can be run as a makefile using make. Use one of the following commands:
 427 [[[/p]]][[[pre]]]
 428 # General examples
 429 python3.0 signduterre.py --manual-make |make -f - example
 430 # Linux specific examples
 431 python3.0 signduterre.py --manual-make |make -f - linux
 432 # Examples requiring sudo
 433 python3.0 signduterre.py --manual-make | sudo make -f - sudo
 434 [[[/pre]]][[[/body]]][[[/html]]]
 435 """
 436 #
 437 license = """
 438 Signature-du-Terroir
 439 Construct a signature of the installed software state or check a previously made signature.
 440
 441 copyright 2009, R.J.J.H. van Son
 442
 443 This program is free software: you can redistribute it and/or modify
 444 it under the terms of the GNU General Public License as published by
 445 the Free Software Foundation, either version 3 of the License, or
 446 (at your option) any later version.
 447
 448 This program is distributed in the hope that it will be useful,
 449 but WITHOUT ANY WARRANTY; without even the implied warranty of
 450 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 451 GNU General Public License for more details.
 452
 453 You should have received a copy of the GNU General Public License
 454 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 455 """;
 456 #
 457 # Note that only release notes are put here
 458 # See git repository for detailed change comments:
 459 # git clone git://repo.or.cz/signduterre.git
 460 releasenotes = """
 461 20090730 - Release v0.4
 462 20090724 - Added '--all-salts-pattern' and HTML formatting in manual
 463 20090723 - Added URL support for all files. Does not yet work due to bug in Python 3.0
 464 20090723 - Added '-' for STDIN
 465 20090717 - Added --execute-args
 466 20090716 - Release v0.3
 467 20090713 - Added --quiet option
 468 20090712 - moved from /dev/random to /dev/urandom
 469 20090702 - Replaced -g with -p SUGGESTED[=N]
 470 20090702 - Generating and testing lists of random salts
 471 20090701 - Release v0.2
 472 20090630 - Generating and testing random passphrases
 473 20090630 - --execute works on $(cmd) only, nlinks in ?path and ? implied for directories
 474 20090630 - Ported to Python 3.0
 475
 476 20090628 - Release v0.1b
 477 20090628 - Added release-notes
 478
 479 20090626 - Release v0.1a
 480 20090626 - Initial commit to Git
 481 """;
 482
 483 #############################################################################
 484 #                                                                           #
 485 #               IMPORT & INITIALIZATION                                     #
 486 #                                                                           #
 487 #############################################################################
 488
 489 import sys;
 490 import os;
 491 import subprocess;
 492 import stat;
 493 import subprocess;
 494 # if sys.stdout.isatty(): import readline;
 495 import binascii;
 496 import hashlib;
 497 import re;
 498 import time;
 499 from optparse import OptionParser;
 500 import base64;
 501 import random;
 502 import struct;
 503 import urllib.request;
 504 import urllib.error;
 505
 506 # Limit the characters that can be used in $(cmd) commands
 507 # Only allow the escape of '$'
 508 not_allowed_chars = re.compile('([^\w\ \.\/\"\|\;\,\-\$\[\]\{\}\(\)\@\`\!\*\=\\\\]|([\\\\]+([^\$\"\\\\]|$)))');
 509
 510 programname = "Signature-du-Terroir";
 511 version = "0.4";
 512
 513 def open_infile(filename, mode):
 514         if filename == '-':
 515                 return sys.stdin;
 516         elif filename.find('://') > -1:
 517                 print("URL:", filename, file=current_private);
 518                 return urllib.request.urlopen(filename);
 519         else :
 520                 if not os.path.isfile(filename):
 521                         print(filename, "does not exist", file=sys.stderr)
 522                         quit();
 523                 return open(filename, mode);
 524
 525 def open_outfile(filename, mode):
 526         if filename == '-':
 527                 return sys.stdout;
 528         elif filename.find('://') > -1:
 529                 print("URL:", filename, file=current_private);
 530                 return urllib.request.urlopen(filename);
 531         else :
 532                 return open(filename, mode);
 533
 534 current_outfile = sys.stdout;
 535 current_private = sys.stderr;
 536
 537 #############################################################################
 538 #                                                                           #
 539 #               OPTION HANDLING                                             #
 540 #                                                                           #
 541 #############################################################################
 542
 543 parser = OptionParser()
 544 parser.add_option("-s", "--salt", metavar="HEX",
 545                                   dest="salt", default=False,
 546                   help="Enter salt in cleartext. If not given, a hexadecimal salt will be suggested. The SUGGESTED[=N] keyword will cause the selection of the suggested string. N is the number of salts generated (default N=1). If N>1, all will be printed and a random one will be used to generate the signature (selection printed to STDERR).")
 547 parser.add_option("-a", "--all-salts-pattern",
 548                   dest="allsalts", default=False, action="store_true",
 549                   help="Use all salts in sequence, randomly replace salts with incorrect ones in the output to create a pattern of failing hashes indicated by a corresponding integer number. Depends on '--salt SUGGESTED=N'. Implies --total-only.")
 550 parser.add_option("-p", "--passphrase", metavar="TEXT",
 551                   dest="passphrase", default=False,
 552                   help="Enter passphrase in cleartext, the keyword SUGGESTED[=N] will cause the suggested passphrase to be used. If N>1, N passphrases will be printed to STDERR and a random one will be used (selection printed to STDERR). Entering the name of an existing file (or '-' for STDIN) will cause it to be read and a random passphrase found in the file will be used (creating a signature), or they will all be used in sequence (--check-file).")
 553 parser.add_option("-c", "--check-file",
 554                   dest="check", default=False, metavar="FILE",
 555                   help="Check contents with the output of a previous run from file or STDIN ('-'). Except when the --quiet option is given, the previous output will contain all information needed for the program, but not the passphrase and the --execute option.")
 556 parser.add_option("-i", "--input-file",
 557                   dest="input", default=False, metavar="FILE",
 558                   help="Use names from FILE or STDIN ('-'), use one filename per line")
 559 parser.add_option("-o", "--output-file",
 560                   dest="output", default=False, metavar="FILE",
 561                   help="Print to FILE instead of STDOUT")
 562 parser.add_option("-P", "--Private-file",
 563                   dest="private", default=False, metavar="FILE",
 564                   help="Print private information (passwords etc.) to FILE instead of STDERR")
 565 parser.add_option("-u", "--user",
 566                   dest="user", default="nobody", metavar="USER",
 567                   help="Execute $(cmd) as USER, default 'nobody' (root/sudo only)")
 568 parser.add_option("-S", "--Status",
 569                   dest="status", default=False, action="store_true",
 570                   help="For each file, add a line with unvarying file status information: st_mode, st_ino, st_dev, st_uid, st_gid, and st_size (like the '?' prefix, default False)")
 571 parser.add_option("--Status-values",
 572                   dest="statusvalues", default="fmidugs", metavar="MODE",
 573                   help="Status values to print for --Status, default MODE is 'fmidugs' (file, mode, inode, device, uid, gid, size). Also available (n)l(inks) a(time), (m)t(ime), and c(time).")
 574 parser.add_option("-t", "--total-only",
 575                   dest="total", default=False, action="store_true",
 576                   help="Only print the total hash, unsets --detailed-view (default True)")
 577 parser.add_option("-d", "--detailed-view",
 578                   dest="detail", default=False, action="store_true",
 579                   help="Print hashes of individual files, is unset by --total-only (default False)")
 580 parser.add_option("-e", "--execute",
 581                   dest="execute", default=False, action="store_true",
 582                   help="Interpret $(cmd) (default False)")
 583 parser.add_option("--execute-args",
 584                   dest="executeargs", default='', metavar="ARGS",
 585                   help="Arguments for the $(cmd) commands ($1 ....)")
 586 parser.add_option("-n", "--no-execute",
 587                   dest="noexecute", default=False, action="store_true",
 588                   help="Explicitely do NOT Interpret $(cmd)")
 589 parser.add_option("--print-hexdump",
 590                   dest="printhexdump", default=False, action="store_true",
 591                   help="Print hexadecimal dump of input bytes to STDERR for debugging purposes")
 592 parser.add_option("-m", "--manual",
 593                   dest="manual", default=False, action="store_true",
 594                   help="Print the manual and exit")
 595 parser.add_option("--manual-html",
 596                   dest="manualhtml", default=False, action="store_true",
 597                   help="Print the manual in HTML format and exit")
 598 parser.add_option("--manual-make",
 599                   dest="manualmake", default=False, action="store_true",
 600                   help="Print the examples in the manual as a makefile and exit")
 601 parser.add_option("-r", "--release-notes",
 602                   dest="releasenotes", default=False, action="store_true",
 603                   help="Print the release notes and exit")
 604 parser.add_option("-l", "--license",
 605                   dest="license", default=False, action="store_true",
 606                   help="Print license text and exit")
 607 parser.add_option("-v", "--verbose",
 608                   dest="verbose", default=False, action="store_true",
 609                   help="Print more information on output")
 610 parser.add_option("-q", "--quiet",
 611                   dest="quiet", default=False, action="store_true",
 612                   help="Print minimal information (hide filenames). If the output is used with --check-file, the command line options and arguments must be repeated.")
 613
 614 (options, check_filenames) = parser.parse_args();
 615
 616
 617 # Start with opening any non-default output files
 618 my_output = False;
 619 if options.output:
 620         current_outfile = open_outfile(options.output, 'w');
 621         my_output = options.output;
 622
 623 my_private = False;
 624 if options.private:
 625         current_private = open_outfile(options.private, 'w');
 626         my_private = options.private;
 627
 628 print("# Program: "+programname + " version " + version, file=current_outfile);
 629 print("#", time.strftime("%Y/%m/%d %H:%M:%S", time.localtime()), "("+time.tzname[0]+")\n", file=current_outfile);
 630
 631 # Print license
 632 if options.license:
 633         print (license, file=sys.stderr);
 634         exit(0);
 635 # Print manual
 636 if options.manual:
 637         htmltags = re.compile('\[\[\[[^\]]*\]\]\]');
 638         cleartext_manual = htmltags.sub('', manual);
 639         print (cleartext_manual, file=sys.stdout);
 640         exit(0);
 641 # Print HTML manual
 642 if options.manualhtml:
 643         protleftanglesbracks = re.compile('\<');
 644         protrightanglesbracks = re.compile('\>');
 645         leftanglesbracks = re.compile('\[\[\[');
 646         rightanglesbracks = re.compile('\]\]\]');
 647         html_manual = protleftanglesbracks.sub('&lt;', manual);
 648         html_manual = protrightanglesbracks.sub('&gt;', html_manual);
 649         html_manual = leftanglesbracks.sub('<', html_manual);
 650         html_manual = rightanglesbracks.sub('>', html_manual);
 651         print (html_manual, file=sys.stdout);
 652         exit(0);
 653 # Print manual examples
 654 if options.manualmake:
 655         make_manual = re.sub("\$ ", "\t", manual);
 656         make_manual = re.sub("\#", "\t#", make_manual);
 657         make_manual = re.sub(r"\\\s*\n", '', make_manual);
 658         make_manual = re.sub(r"\$", r'$$', make_manual);
 659         # Protect "single" [ brackets
 660         make_manual = re.sub(r"(^|[^\[])\[([^\[]|$)", r"\1&#91;\2", make_manual);
 661         extrexamples = re.compile(r"\[\[\[pre\s+make\=?(\w*)\s*\]\]\]\n([^\[]*)\n\[\[\[/pre\s*\]\]\]", re.IGNORECASE|re.MULTILINE|re.DOTALL);
 662         exampleiter = extrexamples.finditer(make_manual);
 663         makefile_list = [];
 664         for match in exampleiter:
 665                 # We had to convert any '[' in the command. Now convert them back.
 666                 command_text = re.sub(r"\&\#91\;", '[', match.group(2));
 667                 makefile_list.append(match.group(1)+":\n"+command_text);
 668
 669         previous_cat = 'NOT A VALUE';
 670         makefile_list.sort()
 671         for line in makefile_list:
 672                 (category, commands) = line.split(':\n');
 673                 if category != previous_cat:
 674                         previous_cat = category;
 675                         print("\n"+previous_cat+":", file=sys.stdout);
 676                 print(commands, file=sys.stdout);
 677         # Clean option
 678         print("\nclean:\n\trm test-*.sdt test-*.pwd", file=sys.stdout);
 679         exit(0);
 680 # Print release notes
 681 if options.releasenotes:
 682         print ("Version: "+version, file=sys.stderr);
 683         print (releasenotes, file=sys.stderr);
 684         exit(0);
 685
 686 my_salt = options.salt;
 687 my_allsalts = options.allsalts;
 688 my_passphrase = options.passphrase;
 689 my_check = options.check;
 690 my_status = options.status;
 691 my_statusvalues = options.statusvalues;
 692 my_verbose = options.verbose and not options.quiet;
 693 my_quiet = options.quiet;
 694 execute = options.execute;
 695 execute_args = options.executeargs;
 696 if options.noexecute: execute = False;
 697 input_file = options.input;
 698
 699 # Set total-only with the correct default
 700 total_only = True;
 701 total_only = not options.detail;
 702 if options.total: total_only = options.total;
 703 if my_allsalts: total_only = my_allsalts; # All alts pattern only sensible with total-only
 704 if my_check: total_only = False;
 705
 706 my_user = options.user;
 707 # Things might be executed as another user
 708 user_change = '';
 709 if os.getuid() == 0:
 710         user_change = 'sudo -H -u '+my_user+' ';
 711         if not my_quiet: print("User: "+my_user, file=current_outfile);
 712
 713 # Execute option
 714 if execute:
 715         text_execute = "True";
 716 else:
 717         text_execute = "False";
 718
 719 if execute and not my_quiet:
 720         print("Execute system commands: "+text_execute+"\n", file=current_outfile);
 721         if execute_args != '': print("Execute arguments: '"+execute_args+"'\n", file=current_outfile);
 722
 723 # --quiet option
 724 if my_quiet: print("Quiet: True\n", file=current_outfile);
 725
 726 # --quiet option
 727 if my_statusvalues != 'fmidugs': print("Status-values: '"+my_statusvalues+"'\n", file=current_outfile);
 728
 729 #############################################################################
 730 #                                                                           #
 731 #               ARGUMENT PROCESSING                                         #
 732 #                                                                           #
 733 #############################################################################
 734
 735 # Measure time intervals
 736 start_time = time.time();
 737
 738 dev_random = open("/dev/urandom", 'rb');
 739
 740 # Read the check file
 741 passphrase_list = [];
 742 salt_list = [];
 743 check_hashes = {};
 744 total_hash_list = [];
 745 if my_check:
 746         highest_arg_used = 0;
 747         print("# Checking: "+my_check+"\n", file=current_outfile);
 748         arg_list = check_filenames;
 749         check_filenames = [];
 750         with open_infile(my_check, 'r') as c:
 751                 for line in c:
 752                         match = re.search("Execute system commands:\s+(True|False)", line);
 753                         if match != None:
 754                                 # Uncomment the next line if you want automatic --execute from the check-file (DANGEROUS)
 755                                 # execute = match.group(1).upper() == 'TRUE';
 756                                 continue;
 757
 758                         match = re.search("Execute arguments:\s+\'([\w\$\s\-\+\/]*)\'", line);
 759                         if match != None:
 760                                 execute_args = match.group(1);
 761                                 continue;
 762
 763                         match = re.search("Quiet:\s+(True|False)", line);
 764                         if match != None:
 765                                 my_quiet = match.group(1).upper() == 'TRUE';
 766                                 if my_quiet: my_verbose = False;
 767                                 continue;
 768
 769                         match = re.search("Salt\s*\+\s*TOTAL HASH\s*\:\s+\'([\w]*)\'\s+\'([\w]*)\'", line);
 770                         if match != None:
 771                                 salt_list.append(match.group(1));
 772                                 total_hash_list.append(match.group(2));
 773                                 my_allsalts = True; # Salt+TOTAL HASH imples all-salts-pattern
 774                                 continue;
 775
 776                         match = re.search("Salt\:\s+\'([\w]*)\'", line);
 777                         if match != None:
 778                                 salt_list.append(match.group(1));
 779                                 continue;
 780
 781                         match = re.search("Salt\s*\+\s*TOTAL HASH\s*\:\s+\'([\w]+)\'\s+\'([a-f0-9]+)\'", line);
 782                         if match != None:
 783                                 salt_list.append(match.group(1));
 784                                 total_hash_list.append(match.group(2));
 785                                 continue;
 786
 787                         match = re.search("User\:\s+\'([\w]*)\'", line);
 788                         if match != None:
 789                                 # Uncomment the next line if you want automatic --user from the check-file (DANGEROUS)
 790                                 # my_user = match.group(1);
 791                                 continue;
 792
 793                         match = re.search("Passphrase\:\s+\'([^\']*)\'", line);
 794                         if match != None:
 795                                 passphrase_list.append(match.group(1));
 796                                 continue;
 797
 798                         match = re.search("Status-values\:\s+\'([\w]*)\'", line);
 799                         if match != None:
 800                                 my_statusvalues = match.group(1);
 801                                 continue;
 802
 803                         match = re.search("^\s*([a-f0-9]+)\s+\*(TOTAL HASH)\s*$", line)
 804                         if match != None:
 805                                 total_hash_list.append(match.group(1));
 806                                 continue;
 807
 808                         match = re.search("^\s*([a-f0-9\-]+)\s+\*\[([0-9]+)\]\s*$", line)
 809                         if match != None:
 810                                 filenumber = int(match.group(2));
 811                                 if filenumber > highest_arg_used: highest_arg_used = filenumber;
 812                                 # Watch out, arguments count from 0
 813                                 check_filenames.append(arg_list[filenumber - 1]);
 814                                 check_hashes['['+match.group(2)+']'] = match.group(1);
 815                                 continue;
 816
 817                         match = re.search("^\s*([a-f0-9\-]+)\s+\*(.*)\s*$", line)
 818                         if match != None:
 819                                 check_filenames.append(match.group(2));
 820                                 # Catch --execute error as early as possible
 821                                 if match.group(2).startswith('$(') and not execute:
 822                                         error_message = "Executable argument \'"+match.group(2)+"\' only allowed with the --execute flag";
 823                                         print (error_message, file=sys.stderr);
 824                                         if not sys.stdout.isatty(): print(error_message, file=current_outfile);
 825                                         exit(0);
 826                                 check_hashes[match.group(2)] = match.group(1);
 827                                 continue;
 828         for i in range(highest_arg_used, len(arg_list)):
 829             check_filenames.append(arg_list[i]);
 830             check_hashes['['+str(i+1)+']'] = (64*'-');
 831
 832 # Read input-file
 833 if input_file:
 834         with open_infile(input_file, 'r') as i:
 835                 for line in i:
 836                         # Clean up filename
 837                         current_filename = re.sub('[^\w\-\.\/\$\{\(\)\}\?\[\]]', '', line);
 838                         check_filenames.append(current_filename);
 839                         if my_check: check_hashes['['+str(i+1)+']'] = (64*'-');
 840
 841 stat_list = [];
 842 for x in check_filenames:
 843         if os.path.isdir(x):
 844                 x = '?'+x;
 845         if my_status and not x.startswith(('?', '$')):
 846                 stat_list.append('?'+x);
 847         stat_list.append(x);
 848 check_filenames = stat_list;
 849
 850 # Seed Pseudo Random Number Generator
 851 seed = dev_random.read(16);
 852 random.seed(seed);
 853
 854 # Read suggested salts from /dev/(u)random if needed
 855 if my_salt:
 856         if my_salt.startswith('SUGGESTED'):
 857                 N=1;
 858                 match = re.search("([0-9][0-9]*)$", my_salt);
 859                 if match != None:
 860                         N = int(match.group(1));
 861                 for i in range(0,N):
 862                         salt = dev_random.read(8);
 863                         salt_list.append(str(binascii.hexlify(salt), 'ascii'));
 864         else:
 865                 salt_list.append(my_salt);
 866 elif len(salt_list) == 0:
 867         salt = dev_random.read(8);
 868         sys.stderr.write("Enter salt (suggest \'"+str(binascii.hexlify(salt), 'ascii')+"\'): ");
 869         new_salt = input();
 870         if not new_salt: new_salt = str(binascii.hexlify(salt), 'ascii');
 871         salt_list.append(new_salt);
 872
 873 # If not combining salts with TOTAL HASH, print salts now
 874 if not my_allsalts:
 875         for my_salt in salt_list:
 876                 print("Salt: \'"+my_salt+"\'", file=current_outfile);
 877
 878 # Get passphrase
 879 if my_passphrase and(my_passphrase == '-' or os.path.isfile(my_passphrase)):
 880     with open_infile(my_passphrase, 'r') as file:
 881         for line in file:
 882                 match = re.search("Passphrase\:\s+\'([^\']*)\'", line);
 883                 if match != None:
 884                         passphrase_list.append(match.group(1));
 885 elif not my_passphrase and len(passphrase_list) == 0:
 886         suggest_passphrase = dev_random.read(16);
 887         sys.stderr.write("Enter passphrase (suggest \'"+str(base64.b64encode(suggest_passphrase), 'ascii').rstrip('=')+"\'): ");
 888         # How kan we make this unreadable on input?
 889         current_passphrase = input();
 890         if not current_passphrase:
 891             current_passphrase = str(base64.b64encode(suggest_passphrase), 'ascii').rstrip('=');
 892             print("Passphrase: \'"+current_passphrase+"\'", file=current_private);
 893         passphrase_list.append(current_passphrase);
 894 elif my_passphrase.startswith('SUGGESTED'):
 895     N = 1;
 896     match = re.search("([0-9][0-9]*)$", my_passphrase);
 897     if match != None:
 898         N = int(match.group(1));
 899     j = int(random.random()*N);
 900     for i in range(0, N):
 901             suggest_passphrase = dev_random.read(16);
 902             current_passphrase = str(base64.b64encode(suggest_passphrase), 'ascii').rstrip('=');
 903             print("Passphrase: \'"+current_passphrase+"\'", file=current_private);
 904             passphrase_list.append(current_passphrase);
 905 else:
 906     passphrase_list.append(my_passphrase);
 907
 908 selected_salt = 1;
 909 fail_fraction = 0.5;
 910 if not my_check:
 911         if len(passphrase_list) > 1:
 912                 j = int(random.random()*len(passphrase_list));
 913                 passphrase_list = [passphrase_list[j]];
 914                 print("# Selected passphrase:", j+1, file=current_private);
 915         if len(salt_list) > 1:
 916                 j = int(random.random()*len(salt_list));
 917                 # Make sure at least 1 salt will match and print the selection if only one is used
 918                 selected_salt = j+1;
 919                 if not my_allsalts:
 920                         salt_list = [salt_list[selected_salt-1]];
 921                         print("# Selected salt:", selected_salt, file=current_private);
 922                 else:
 923                         salt_N = len(salt_list);
 924                         fail_fraction = (salt_N/2.0)/(salt_N - 1);
 925         else:
 926                 fail_fraction = 0;
 927
 928 # Close /dev/(u)random
 929 dev_random.close;
 930
 931 #############################################################################
 932 #                                                                           #
 933 #               SIGNATURE CREATION AND CHECKING                             #
 934 #                                                                           #
 935 #############################################################################
 936
 937 end_time = time.time();
 938 print("# Preparation time:", end_time - start_time, "seconds\n", file=current_outfile);
 939
 940 pnum = 1;
 941 snum = 1;
 942 corrpnum = 0;
 943 corrsnum = 0;
 944 matched_salt_pattern = -1;
 945 salt_pattern_number = -1;
 946
 947 for my_passphrase in passphrase_list:
 948         snum = 1;
 949         # Initialize salt pattern
 950         if my_allsalts:
 951                 salt_pattern_number = 0;
 952                 current_salt_power = 1;
 953
 954         for my_salt in salt_list:
 955                 print("# Start signature: ", end='', file=current_outfile);
 956                 if len(passphrase_list) > 1: print("passphrase -", pnum, end='', file=current_outfile);
 957                 if len(salt_list) > 1: print(" salt -", snum, end='', file=current_outfile);
 958                 print("", file=current_outfile);
 959
 960                 # Should everything be printed?
 961                 print_verbose = my_verbose and not (my_allsalts and snum > 1);
 962
 963                 file_argnum = 0;
 964                 start_time = time.time();
 965                 # Construct the passphrase hash
 966                 passphrase = hashlib.sha256();
 967
 968                 passphrase.update(bytes(my_salt, encoding='ascii'));
 969                 passphrase.update(bytes(my_passphrase, encoding='ascii'));
 970
 971                 # Create prefix which is a hash of the salt+passphrase
 972                 prefix = passphrase.hexdigest();
 973
 974                 # Create signature and write output
 975                 totalhash = hashlib.sha256();
 976                 totalhash.update(bytes(prefix, encoding='ascii'));
 977                 for org_filename in check_filenames:
 978                         # Create file hash object
 979                         filehash = hashlib.sha256();
 980                         filehash.update(bytes(prefix, encoding='ascii'));
 981                         # Remove []
 982                         filename = org_filename.strip('[').rstrip(']');
 983                         # Use system variables and commands
 984                         if filename.startswith('$'):
 985                                 # Commands $(command)
 986                                 match = re.search('^\$\((.+)\)$', filename);
 987                                 if match != None:
 988                                         if not execute :
 989                                                 error_message = "Executable argument \'"+filename+"\' only allowed with the --execute flag";
 990                                                 print (error_message, file=sys.stderr);
 991                                                 if not sys.stdout.isatty(): print(error_message, file=current_outfile);
 992                                                 exit(0);
 993                                         current_command = not_allowed_chars.sub(" ", match.group(1));
 994                                         current_command_line = user_change+"bash --restricted -c \'"+current_command+"\' "+str(os.getpid())+" "+execute_args;
 995                                         # Print command
 996                                         if print_verbose :
 997                                                 print("#", current_command_line, file=current_outfile);
 998                                         # Spawn command and open a pipe to the output
 999                                         pipe = subprocess.Popen(current_command_line, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE);
1000                                         for b in pipe.stdout:
1001                                                 if type(b).__name__ == 'str':
1002                                                         b = bytes(b, encoding='utf8');
1003                                                 filehash.update(b);
1004                                                 if options.printhexdump: # For debugging commands
1005                                                         print(str(binascii.hexlify(b), 'ascii'), file=sys.stderr);
1006                                         # See whether there was an error
1007                                         pipe.wait();
1008                                         if pipe.returncode:
1009                                                 error_message = pipe.stderr.read();
1010                                                 print('$('+current_command+')', "\n", str(error_message, encoding='UTF8'), file=sys.stderr);
1011                                                 exit(pipe.returncode);
1012                                 # ${ENV} environment variables
1013                                 match = re.search('^\$[\{]?([^\}\(\)]+)[\}]?$', filename);
1014                                 if match != None:
1015                                         current_var = not_allowed_chars.sub(" ", match.group(1));
1016                                         if print_verbose:
1017                                                 print("# echo $"+ current_var, file=current_outfile);
1018                                         b = os.environ[current_var];
1019                                         filehash.update(bytes(b, encoding='utf8'));
1020                         # lstat() meta information
1021                         elif filename.startswith('?'):
1022                                 if not os.path.exists(filename.lstrip('?')):
1023                                         print(filename, "does not exist", file=sys.stderr)
1024                                         quit();
1025                                 filestat = os.stat(filename.lstrip('?'));
1026                                 if my_statusvalues == "": my_statusvalues = 'fmidlugs'
1027                                 b = "";
1028                                 if 'f' in my_statusvalues:
1029                                         b += 'stat('+filename.lstrip('?')+') = '
1030                                 b += '[';
1031                                 if 'm' in my_statusvalues:
1032                                         b += 'st_mode='+str(oct(filestat.st_mode))+', ';
1033                                 if 'i' in my_statusvalues:
1034                                         b += 'st_ino='+str(filestat.st_ino)+', ';
1035                                 if 'd' in my_statusvalues:
1036                                         b += 'st_dev='+str(filestat.st_dev)+', '
1037                                 if 'l' in my_statusvalues:
1038                                         b += 'st_nlink='+str(filestat.st_nlink)+', '
1039                                 if 'u' in my_statusvalues:
1040                                         b += 'st_uid='+str(filestat.st_uid)+', '
1041                                 if 'g' in my_statusvalues:
1042                                         b += 'st_gid='+str(filestat.st_gid)+', '
1043                                 if 's' in my_statusvalues:
1044                                         b += 'st_size='+str(filestat.st_size)+', '
1045                                 if 'a' in my_statusvalues:
1046                                         b += 'st_atime='+str(filestat.st_atime)+', '
1047                                 if 't' in my_statusvalues:
1048                                         b += 'st_mtime='+str(filestat.st_mtime)+', '
1049                                 if 'c' in my_statusvalues:
1050                                         b += 'st_ctime='+str(filestat.st_ctime);
1051
1052                                 b = b.rstrip(', ') + ']';
1053                                 filehash.update(bytes(b, encoding='utf8'));
1054                                 if print_verbose:
1055                                         print ("# "+ b, file=current_outfile);
1056                         # Use file
1057                         else:
1058                                 # open and read the file
1059                                 if filename != '-' and filename.find('://') == -1 and not os.path.exists(filename):
1060                                         print(filename, "does not exist", file=sys.stderr)
1061                                         quit();
1062
1063                                 if filename == '-':
1064                                         for b in sys.stdin.buffer:
1065                                                 if type(b).__name__ == 'str':
1066                                                         b = bytes(b, encoding='utf8');
1067                                                 filehash.update(b);
1068                                                 if options.printhexdump: # For debugging commands
1069                                                         print(str(binascii.hexlify(b), 'ascii'), file=sys.stderr);
1070
1071                                 else:
1072                                         with open_infile(filename, 'rb') as file:
1073                                                 for b in file:
1074                                                         if type(b).__name__ == 'str':
1075                                                                 b = bytes(b, encoding='utf8');
1076                                                         filehash.update(b);
1077                                                         if options.printhexdump: # For debugging commands
1078                                                                 print(str(binascii.hexlify(b), 'ascii'), file=sys.stderr);
1079
1080                         current_digest = filehash.hexdigest();
1081                         print_name = filename;
1082                         if my_quiet or org_filename.startswith('['):
1083                             file_argnum += 1;
1084                             print_name = '['+str(file_argnum)+']';
1085                         current_hash_line = current_digest+" *"+print_name
1086                         totalhash.update(bytes(current_hash_line, encoding='ascii'));
1087
1088                         # Be careful to use this ONLY after totalhash has been updated!
1089                         if total_only:
1090                             current_hash_line = (len(current_digest)*'-')+" *"+print_name;
1091
1092                         # Write output
1093                         if not my_check:
1094                                 if not (my_quiet and total_only) and not (my_allsalts and snum > 1):
1095                                     print(current_hash_line, file=current_outfile);
1096                         elif not (my_quiet or my_allsalts):
1097                                 if check_hashes[print_name] ==  (len(current_digest)*'-'):
1098                                         # Suppress redundant output of empty, ----, lines
1099                                         if snum <= 1 and pnum <= 1:
1100                                                 print(check_hashes[print_name]+" *"+print_name, file=current_outfile);
1101                                 elif current_digest != check_hashes[print_name]:
1102                                         print("FAILED: "+current_hash_line, file=current_outfile);
1103                                 else:
1104                                         print("ok"+" *"+print_name, file=current_outfile);
1105
1106                 # Handle total hash
1107                 current_total_digest = totalhash.hexdigest();
1108                 # Write (in)correct salts with the TOTAL HASH
1109                 if my_allsalts:
1110                         output_salt = my_salt;
1111                         j = random.random();
1112                         # Randomly create an incorrect salt for failed output
1113                         if not my_check:
1114                                 if j < fail_fraction and snum != selected_salt:
1115                                         salt = dev_random.read(8);
1116                                         output_salt = str(binascii.hexlify(salt), 'ascii');
1117                                 else:
1118                                         salt_pattern_number += current_salt_power;
1119                         current_total_digest_line = "Salt+TOTAL HASH: '"+output_salt+"' '"+current_total_digest+"'";
1120                 else: # Standard TOTAL HASH line
1121                         current_total_digest_line = current_total_digest+" *"+"TOTAL HASH";
1122                 end_time = time.time();
1123                 print("# \n# Total hash - Time to completion:", end_time - start_time, "seconds", file=current_outfile);
1124                 total_hash_num = 0;
1125                 if my_allsalts: total_hash_num = snum-1; # Current TOTAL HASH number of more are used
1126                 if not my_check:
1127                         print(current_total_digest_line+"\n", file=current_outfile);
1128                 elif current_total_digest != total_hash_list[total_hash_num]:
1129                         if not my_allsalts: print("FAILED: "+current_total_digest_line+"\n", file=current_outfile);
1130                 else:
1131                         if my_allsalts: salt_pattern_number += current_salt_power;  # Update salt bit pattern
1132                         match_number = "";
1133                         if len(passphrase_list) > 1 or len(salt_list): match_number = " #"
1134                         if len(passphrase_list) > 1: match_number += " passphrase no: "+str(pnum);
1135                         if len(salt_list) > 1: match_number += " salt no: "+str(snum);
1136                         if not my_allsalts: print("OK"+" *"+"TOTAL HASH"+match_number+"\n", file=current_outfile);
1137                         corrsnum = snum;
1138                         corrpnum = pnum;
1139                 snum += 1;
1140                 if my_allsalts: current_salt_power *= 2;  # Update current bit position in salt pattern
1141         if my_check and corrpnum == pnum: matched_salt_pattern = salt_pattern_number;
1142         pnum += 1;
1143
1144 if my_check and len(passphrase_list) > 1:
1145         if corrpnum > 0:
1146                 print("Passphrase entry:",corrpnum,"matched", file=current_outfile);
1147         else:
1148                 print("No passphrase entry matched!", file=current_outfile);
1149 if my_check and (not my_allsalts) and len(salt_list) > 1:
1150         if corrpnum > 0:
1151                 if corrsnum > 0:
1152                         print("Salt entry:",corrsnum,"matched", file=current_outfile);
1153                 else:
1154                         print("No salt entry matched!", file=current_outfile);
1155         else:
1156                 print("No entry matched", file=current_outfile);
1157 # Print salt bit patterns
1158 elif my_check and my_allsalts:
1159         print("Salt pattern number:", matched_salt_pattern, file=current_outfile);
1160 elif not my_check and my_allsalts:
1161         print("# Salt pattern number:", salt_pattern_number, file=current_private);
1162
1163 # Close output files if necessary
1164 if my_output and my_output != '-':
1165         current_outfile.close();
1166 if my_private and my_private != '-':
1167         current_private.close();