?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Add files via upload
[PyCatFile.git] / pycatfile.py
blobfc3f579a8dd909abc9786722184c4750a68cb94d
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 '''
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the Revised BSD License.
7
8 This program is distributed in the hope that it will be useful,
9 but WITHOUT ANY WARRANTY; without even the implied warranty of
10 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 Revised BSD License for more details.
12
13 Copyright 2018-2024 Cool Dude 2k - http://idb.berlios.de/
14 Copyright 2018-2024 Game Maker 2k - http://intdb.sourceforge.net/
15 Copyright 2018-2024 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
16
17 $FileInfo: pycatfile.py - Last Update: 5/5/2024 Ver. 0.10.2 RC 1 - Author: cooldude2k $
18 '''
19
20 from __future__ import absolute_import, division, print_function, unicode_literals;
21 import io, os, re, sys, time, stat, zlib, base64, shutil, socket, hashlib, datetime, logging, binascii, tempfile, zipfile, platform;
22 from ftplib import FTP, FTP_TLS;
23 if(sys.version[0]=="2"):
24 from urlparse import urlparse, urlunparse;
25 elif(sys.version[0]>="3"):
26 from urllib.parse import urlunparse;
27 from urllib.parse import urlparse;
28
29 if os.name == 'nt': # Only modify if on Windows
30 if sys.version[0] == "2":
31 import codecs;
32 sys.stdout = codecs.getwriter('utf-8')(sys.stdout);
33 sys.stderr = codecs.getwriter('utf-8')(sys.stderr);
34 else:
35 import io;
36 sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace', line_buffering=True);
37 sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', errors='replace', line_buffering=True);
38
39 hashlib_guaranteed = False;
40 os.environ["PYTHONIOENCODING"] = "UTF-8";
41 os.environ["LC_CTYPE"] = "UTF-8";
42 try:
43 reload(sys);
44 try:
45 sys.setdefaultencoding('UTF-8');
46 except NameError:
47 pass;
48 except AttributeError:
49 pass;
50 except NameError:
51 pass;
52 except AttributeError:
53 pass;
54
55 try:
56 from zlib import crc32;
57 except ImportError:
58 from binascii import crc32;
59
60 if(sys.version_info[0]==2):
61 FileNotFoundError = IOError;
62
63 rarfile_support = False;
64 try:
65 import rarfile;
66 rarfile_support = True;
67 except ImportError:
68 rarfile_support = False;
69
70 py7zr_support = False;
71 try:
72 import py7zr;
73 py7zr_support = True;
74 except ImportError:
75 py7zr_support = False;
76
77 try:
78 from xtarfile import is_tarfile;
79 except ImportError:
80 try:
81 from safetar import is_tarfile;
82 except ImportError:
83 from tarfile import is_tarfile;
84
85 try:
86 import xtarfile as tarfile;
87 except ImportError:
88 try:
89 import safetar as tarfile;
90 except ImportError:
91 import tarfile;
92
93 haveparamiko = False;
94 try:
95 import paramiko;
96 haveparamiko = True;
97 except ImportError:
98 haveparamiko = False;
99
100 havepysftp = False;
101 try:
102 import pysftp;
103 havepysftp = True;
104 except ImportError:
105 havepysftp = False;
106
107 haverequests = False;
108 try:
109 import requests;
110 haverequests = True;
111 except ImportError:
112 haverequests = False;
113
114 if(haverequests):
115 import urllib3;
116 logging.getLogger("urllib3").setLevel(logging.WARNING);
117
118 try:
119 # Python 3 imports
120 from urllib.request import Request, build_opener, HTTPBasicAuthHandler;
121 from urllib.parse import urlparse;
122 except ImportError:
123 # Python 2 imports
124 from urllib2 import Request, build_opener, HTTPBasicAuthHandler;
125 from urlparse import urlparse;
126
127 if(sys.version[0]=="2"):
128 try:
129 from io import StringIO, BytesIO;
130 except ImportError:
131 try:
132 from cStringIO import StringIO;
133 from cStringIO import StringIO as BytesIO;
134 except ImportError:
135 from StringIO import StringIO;
136 from StringIO import StringIO as BytesIO;
137 elif(sys.version[0]>="3"):
138 from io import StringIO, BytesIO;
139 else:
140 teststringio = 0;
141 if(teststringio<=0):
142 try:
143 from cStringIO import StringIO as BytesIO;
144 teststringio = 1;
145 except ImportError:
146 teststringio = 0;
147 if(teststringio<=0):
148 try:
149 from StringIO import StringIO as BytesIO;
150 teststringio = 2;
151 except ImportError:
152 teststringio = 0;
153 if(teststringio<=0):
154 try:
155 from io import BytesIO;
156 teststringio = 3;
157 except ImportError:
158 teststringio = 0;
159
160 __use_pysftp__ = False;
161 if(not havepysftp):
162 __use_pysftp__ = False;
163 __file_format_name__ = "CatFile";
164 __program_name__ = "Py"+__file_format_name__;
165 __file_format_lower__ = __file_format_name__.lower();
166 __file_format_magic__ = __file_format_name__;
167 __file_format_len__ = len(__file_format_magic__);
168 __file_format_hex__ = binascii.hexlify(__file_format_magic__.encode("UTF-8")).decode("UTF-8");
169 __file_format_delimiter__ = "\x00";
170 __file_format_ver__ = "001";
171 __use_new_style__ = True;
172 __use_advanced_list__ = True;
173 __use_alt_inode__ = False;
174 __file_format_list__ = [__file_format_name__, __file_format_magic__, __file_format_lower__, __file_format_len__, __file_format_hex__, __file_format_delimiter__, __file_format_ver__, __use_new_style__, __use_advanced_list__, __use_alt_inode__];
175 __project__ = __program_name__;
176 __project_url__ = "https://github.com/GameMaker2k/PyCatFile";
177 __version_info__ = (0, 10, 2, "RC 1", 1);
178 __version_date_info__ = (2024, 5, 5, "RC 1", 1);
179 __version_date__ = str(__version_date_info__[0]) + "." + str(__version_date_info__[1]).zfill(2) + "." + str(__version_date_info__[2]).zfill(2);
180 __revision__ = __version_info__[3];
181 __revision_id__ = "$Id$";
182 if(__version_info__[4] is not None):
183 __version_date_plusrc__ = __version_date__ + "-" + str(__version_date_info__[4]);
184 if(__version_info__[4] is None):
185 __version_date_plusrc__ = __version_date__;
186 if(__version_info__[3] is not None):
187 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]) + " " + str(__version_info__[3]);
188 if(__version_info__[3] is None):
189 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]);
190
191 PyBitness = platform.architecture();
192 if(PyBitness=="32bit" or PyBitness=="32"):
193 PyBitness = "32";
194 elif(PyBitness=="64bit" or PyBitness=="64"):
195 PyBitness = "64";
196 else:
197 PyBitness = "32";
198
199 geturls_ua_pycatfile_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
200 if(platform.python_implementation()!=""):
201 py_implementation = platform.python_implementation();
202 if(platform.python_implementation()==""):
203 py_implementation = "Python";
204 geturls_ua_pycatfile_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=py_implementation, pyver=platform.python_version(), proname=__project__, prover=__version__);
205 geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
206 geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
207 geturls_headers_pycatfile_python = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
208 geturls_headers_pycatfile_python_alt = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python_alt, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
209 geturls_headers_googlebot_google = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
210 geturls_headers_googlebot_google_old = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
211
212 compressionlist = ['auto', 'gzip', 'bzip2', 'zstd', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
213 outextlist = ['gz', 'bz2', 'zst', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
214 outextlistwd = ['.gz', '.bz2', '.zst', '.lz4', '.lzo', '.lzop', '.lzma', '.xz'];
215
216 tarfile_mimetype = "application/tar";
217 tarfile_tar_mimetype = tarfile_mimetype;
218 zipfile_mimetype = "application/zip";
219 zipfile_zip_mimetype = zipfile_mimetype;
220 rarfile_mimetype = "application/rar";
221 rarfile_rar_mimetype = rarfile_mimetype;
222 archivefile_mimetype = "application/x-"+__file_format_list__[1]+"";
223 archivefile_cat_mimetype = archivefile_mimetype;
224 archivefile_gzip_mimetype = "application/x-"+__file_format_list__[1]+"+gzip";
225 archivefile_gz_mimetype = archivefile_gzip_mimetype;
226 archivefile_bzip2_mimetype = "application/x-"+__file_format_list__[1]+"+bzip2";
227 archivefile_bz2_mimetype = archivefile_bzip2_mimetype;
228 archivefile_lz4_mimetype = "application/x-"+__file_format_list__[1]+"+lz4";
229 archivefile_lzop_mimetype = "application/x-"+__file_format_list__[1]+"+lzop";
230 archivefile_lzo_mimetype = archivefile_lzop_mimetype;
231 archivefile_zstandard_mimetype = "application/x-"+__file_format_list__[1]+"+zstandard";
232 archivefile_zstd_mimetype = archivefile_zstandard_mimetype;
233 archivefile_lzma_mimetype = "application/x-"+__file_format_list__[1]+"+lzma";
234 archivefile_xz_mimetype = "application/x-"+__file_format_list__[1]+"+xz";
235 archivefile_extensions = ['.cat', '.cat.gz', '.cat.bz2', '.cat.zst', '.cat.lz4', '.cat.lzo', '.cat.lzop', '.cat.lzma', '.cat.xz'];
236
237 if __name__ == "__main__":
238 import subprocess;
239 curscrpath = os.path.dirname(sys.argv[0]);
240 if(curscrpath==""):
241 curscrpath = ".";
242 if(os.sep=="\\"):
243 curscrpath = curscrpath.replace(os.sep, "/");
244 curscrpath = curscrpath + "/";
245 scrfile = curscrpath + "catfile.py";
246 if(os.path.exists(scrfile) and os.path.isfile(scrfile)):
247 scrcmd = subprocess.Popen([sys.executable, scrfile] + sys.argv[1:]);
248 scrcmd.wait();
249
250 def VerbosePrintOut(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
251 if(not dbgenable):
252 return True;
253 log_functions = {
254 "print": print,
255 "log": logging.info,
256 "warning": logging.warning,
257 "error": logging.error,
258 "critical": logging.critical,
259 "exception": logging.exception,
260 "logalt": lambda x: logging.log(dgblevel, x),
261 "debug": logging.debug
262 };
263 log_function = log_functions.get(outtype);
264 if(log_function):
265 log_function(dbgtxt);
266 return True;
267 return False;
268
269 def VerbosePrintOutReturn(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
270 VerbosePrintOut(dbgtxt, outtype, dbgenable, dgblevel);
271 return dbgtxt;
272
273 def RemoveWindowsPath(dpath):
274 if(dpath is None):
275 dpath = "";
276 if(os.sep!="/"):
277 dpath = dpath.replace(os.path.sep, "/");
278 dpath = dpath.rstrip("/");
279 if(dpath=="." or dpath==".."):
280 dpath = dpath + "/";
281 return dpath;
282
283 def NormalizeRelativePath(inpath):
284 inpath = RemoveWindowsPath(inpath);
285 if(os.path.isabs(inpath)):
286 outpath = inpath;
287 else:
288 if(inpath.startswith("./") or inpath.startswith("../")):
289 outpath = inpath;
290 else:
291 outpath = "./" + inpath;
292 return outpath;
293
294 def PrependPath(base_dir, child_path):
295 # Check if base_dir is None or empty, if so, return child_path as is
296 if not base_dir:
297 return child_path;
298 # Ensure base_dir ends with exactly one slash
299 if not base_dir.endswith('/'):
300 base_dir += '/';
301 # Check if child_path starts with ./ or ../ (indicating a relative path)
302 if child_path.startswith('./') or child_path.startswith('../'):
303 # For relative paths, we don't alter the child_path
304 return base_dir + child_path;
305 else:
306 # For non-relative paths, ensure there's no starting slash on child_path to avoid double slashes
307 return base_dir + child_path.lstrip('/');
308
309 def ListDir(dirpath, followlink=False, duplicates=False):
310 if(isinstance(dirpath, (list, tuple, ))):
311 dirpath = list(filter(None, dirpath));
312 elif(isinstance(dirpath, (str, ))):
313 dirpath = list(filter(None, [dirpath]));
314 retlist = [];
315 for mydirfile in dirpath:
316 if(not os.path.exists(mydirfile)):
317 return False;
318 mydirfile = NormalizeRelativePath(mydirfile);
319 if(os.path.exists(mydirfile) and os.path.islink(mydirfile)):
320 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile));
321 if(os.path.exists(mydirfile) and os.path.isdir(mydirfile)):
322 for root, dirs, filenames in os.walk(mydirfile):
323 dpath = root;
324 dpath = RemoveWindowsPath(dpath);
325 if(dpath not in retlist and not duplicates):
326 retlist.append(dpath);
327 if(duplicates):
328 retlist.append(dpath);
329 for file in filenames:
330 fpath = os.path.join(root, file);
331 fpath = RemoveWindowsPath(fpath);
332 if(fpath not in retlist and not duplicates):
333 retlist.append(fpath);
334 if(duplicates):
335 retlist.append(fpath);
336 else:
337 retlist.append(RemoveWindowsPath(mydirfile));
338 return retlist;
339
340 def ListDirAdvanced(dirpath, followlink=False, duplicates=False):
341 if isinstance(dirpath, (list, tuple)):
342 dirpath = list(filter(None, dirpath));
343 elif isinstance(dirpath, str):
344 dirpath = list(filter(None, [dirpath]));
345 retlist = []
346 for mydirfile in dirpath:
347 if not os.path.exists(mydirfile):
348 return False;
349 mydirfile = NormalizeRelativePath(mydirfile);
350 if os.path.exists(mydirfile) and os.path.islink(mydirfile) and followlink:
351 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile))
352 if os.path.exists(mydirfile) and os.path.isdir(mydirfile):
353 for root, dirs, filenames in os.walk(mydirfile):
354 # Sort dirs and filenames alphabetically in place
355 dirs.sort(key=lambda x: x.lower());
356 filenames.sort(key=lambda x: x.lower());
357 dpath = RemoveWindowsPath(root);
358 if not duplicates and dpath not in retlist:
359 retlist.append(dpath);
360 elif duplicates:
361 retlist.append(dpath);
362 for file in filenames:
363 fpath = os.path.join(root, file);
364 fpath = RemoveWindowsPath(fpath);
365 if not duplicates and fpath not in retlist:
366 retlist.append(fpath);
367 elif duplicates:
368 retlist.append(fpath);
369 else:
370 retlist.append(RemoveWindowsPath(mydirfile));
371 return retlist;
372
373 def create_alias_function(prefix, base_name, suffix, target_function):
374 # Define a new function that wraps the target function
375 def alias_function(*args, **kwargs):
376 return target_function(*args, **kwargs);
377
378 # Create the function name by combining the prefix, base name, and the suffix
379 function_name = "{}{}{}".format(prefix, base_name, suffix);
380
381 # Add the new function to the global namespace
382 globals()[function_name] = alias_function;
383
384 def create_alias_function_alt(prefix, base_name, suffix, target_function):
385 # Create the function name by combining the prefix, base name, and the suffix
386 # Use the format method for string formatting, compatible with Python 2 and 3
387 function_name = "{}{}{}".format(prefix, base_name, suffix)
388 # Add the new function (alias of the target_function) to the global namespace
389 # This line is compatible as-is with both Python 2 and 3
390 globals()[function_name] = target_function
391
392 # initial_value can be 0xFFFF or 0x0000
393 def crc16_ansi(msg, initial_value=0xFFFF):
394 # CRC-16-IBM / CRC-16-ANSI polynomial and initial value
395 poly = 0x8005; # Polynomial for CRC-16-IBM / CRC-16-ANSI
396 crc = initial_value; # Initial value
397 for b in msg:
398 crc ^= b << 8; # XOR byte into CRC top byte
399 for _ in range(8): # Process each bit
400 if crc & 0x8000: # If the top bit is set
401 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
402 else:
403 crc = crc << 1; # Just shift left
404 crc &= 0xFFFF; # Ensure CRC remains 16-bit
405 return crc;
406
407 # initial_value can be 0xFFFF or 0x0000
408 def crc16_ibm(msg, initial_value=0xFFFF):
409 return crc16_ansi(msg, initial_value);
410
411 # initial_value is 0xFFFF
412 def crc16(msg):
413 return crc16_ansi(msg, 0xFFFF);
414
415 # initial_value can be 0xFFFF, 0x1D0F or 0x0000
416 def crc16_ccitt(msg, initial_value=0xFFFF):
417 # CRC-16-CCITT polynomial
418 poly = 0x1021; # Polynomial for CRC-16-CCITT
419 # Use the specified initial value
420 crc = initial_value;
421 for b in msg:
422 crc ^= b << 8; # XOR byte into CRC top byte
423 for _ in range(8): # Process each bit
424 if crc & 0x8000: # If the top bit is set
425 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
426 else:
427 crc = crc << 1; # Just shift left
428 crc &= 0xFFFF; # Ensure CRC remains 16-bit
429 return crc;
430
431 # initial_value can be 0x42F0E1EBA9EA3693 or 0x0000000000000000
432 def crc64_ecma(msg, initial_value=0x0000000000000000):
433 # CRC-64-ECMA polynomial and initial value
434 poly = 0x42F0E1EBA9EA3693;
435 crc = initial_value; # Initial value for CRC-64-ECMA
436 for b in msg:
437 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
438 for _ in range(8): # Process each bit
439 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
440 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
441 else:
442 crc <<= 1; # Just shift left if the MSB is 0
443 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
444 return crc;
445
446 # initial_value can be 0x000000000000001B or 0xFFFFFFFFFFFFFFFF
447 def crc64_iso(msg, initial_value=0xFFFFFFFFFFFFFFFF):
448 # CRC-64-ISO polynomial and initial value
449 poly = 0x000000000000001B;
450 crc = initial_value; # Common initial value for CRC-64-ISO
451 for b in msg:
452 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
453 for _ in range(8): # Process each bit
454 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
455 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
456 else:
457 crc <<= 1; # Just shift left if the MSB is 0
458 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
459 return crc;
460
461 def GetDataFromArray(data, path, default=None):
462 element = data;
463 try:
464 for key in path:
465 element = element[key];
466 return element;
467 except (KeyError, TypeError, IndexError):
468 return default;
469
470 def GetDataFromArrayAlt(structure, path, default=None):
471 element = structure;
472 for key in path:
473 if isinstance(element, dict) and key in element:
474 element = element[key];
475 elif isinstance(element, list) and isinstance(key, int) and -len(element) <= key < len(element):
476 element = element[key];
477 else:
478 return default;
479 return element;
480
481 def GetHeaderChecksum(inlist=[], checksumtype="crc32", encodedata=True, formatspecs=__file_format_list__):
482 if isinstance(inlist, list):
483 fileheader = AppendNullBytes(inlist, formatspecs[5]);
484 else:
485 fileheader = AppendNullByte(inlist, formatspecs[5]);
486 if(encodedata):
487 fileheader = fileheader.encode('UTF-8');
488 if(checksumtype=="none" or checksumtype==""):
489 catfileheadercshex = format(0, 'x').lower();
490 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
491 catfileheadercshex = format(crc16(fileheader) & 0xffff, '04x').lower();
492 elif(checksumtype=="crc16_ccitt"):
493 catfileheadercshex = format(crc16_ccitt(fileheader) & 0xffff, '04x').lower();
494 elif(checksumtype=="adler32"):
495 catfileheadercshex = format(zlib.adler32(fileheader) & 0xffffffff, '08x').lower();
496 elif(checksumtype=="crc32"):
497 catfileheadercshex = format(crc32(fileheader) & 0xffffffff, '08x').lower();
498 elif(checksumtype=="crc64_ecma"):
499 catfileheadercshex = format(crc64_ecma(fileheader) & 0xffffffffffffffff, '016x').lower();
500 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
501 catfileheadercshex = format(crc64_iso(fileheader) & 0xffffffffffffffff, '016x').lower();
502 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
503 checksumoutstr = hashlib.new(checksumtype);
504 checksumoutstr.update(fileheader);
505 catfileheadercshex = checksumoutstr.hexdigest().lower();
506 else:
507 catfileheadercshex = format(0, 'x').lower();
508 return catfileheadercshex;
509
510 def GetFileChecksum(instr, checksumtype="crc32", encodedata=True, formatspecs=__file_format_list__):
511 if(encodedata):
512 instr = instr.encode('UTF-8');
513 if(checksumtype=="none" or checksumtype==""):
514 catinstrcshex = format(0, 'x').lower();
515 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
516 catinstrcshex = format(crc16(instr) & 0xffff, '04x').lower();
517 elif(checksumtype=="crc16_ccitt"):
518 catinstrcshex = format(crc16_ccitt(instr) & 0xffff, '04x').lower();
519 elif(checksumtype=="adler32"):
520 catinstrcshex = format(zlib.adler32(instr) & 0xffffffff, '08x').lower();
521 elif(checksumtype=="crc32"):
522 catinstrcshex = format(crc32(instr) & 0xffffffff, '08x').lower();
523 elif(checksumtype=="crc64_ecma"):
524 catinstrcshex = format(crc64_ecma(instr) & 0xffffffffffffffff, '016x').lower();
525 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
526 catinstrcshex = format(crc64_iso(instr) & 0xffffffffffffffff, '016x').lower();
527 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
528 checksumoutstr = hashlib.new(checksumtype);
529 checksumoutstr.update(instr);
530 catinstrcshex = checksumoutstr.hexdigest().lower();
531 else:
532 catinstrcshex = format(0, 'x').lower();
533 return catinstrcshex;
534
535 def ValidateHeaderChecksum(inlist=[], checksumtype="crc32", inchecksum="0", formatspecs=__file_format_list__):
536 catfileheadercshex = GetHeaderChecksum(inlist, checksumtype, True, formatspecs);
537 inchecksum = inchecksum.lower();
538 catfileheadercshex = catfileheadercshex.lower();
539 if(inchecksum==catfileheadercshex):
540 return True;
541 else:
542 return False;
543
544 def ValidateFileChecksum(infile, checksumtype="crc32", inchecksum="0", formatspecs=__file_format_list__):
545 catinfilecshex = GetFileChecksum(infile, checksumtype, True, formatspecs);
546 inchecksum = inchecksum.lower();
547 catinfilecshex = catinfilecshex.lower();
548 if(inchecksum==catinfilecshex):
549 return True;
550 else:
551 return False;
552
553 def ReadTillNullByte(fp, delimiter=__file_format_delimiter__):
554 curbyte = b"";
555 curfullbyte = b"";
556 nullbyte = delimiter.encode("UTF-8");
557 while(True):
558 curbyte = fp.read(1);
559 if(curbyte==nullbyte or not curbyte):
560 break;
561 curfullbyte = curfullbyte + curbyte;
562 return curfullbyte.decode('UTF-8');
563
564 def ReadUntilNullByte(fp, delimiter=__file_format_delimiter__):
565 return ReadTillNullByte(fp, delimiter);
566
567 def SeekToEndOfFile(fp):
568 lasttell = 0;
569 while(True):
570 fp.seek(1, 1);
571 if(lasttell==fp.tell()):
572 break;
573 lasttell = fp.tell();
574 return True;
575
576 def ReadFileHeaderData(fp, rounds=0, delimiter=__file_format_delimiter__):
577 rocount = 0;
578 roend = int(rounds);
579 HeaderOut = [];
580 while(rocount<roend):
581 HeaderOut.append(ReadTillNullByte(fp, delimiter));
582 rocount = rocount + 1;
583 return HeaderOut;
584
585 def ReadFileHeaderDataBySize(fp, delimiter=__file_format_delimiter__):
586 headerpresize = ReadTillNullByte(fp, delimiter);
587 headersize = int(headerpresize, 16);
588 if(headersize<=0):
589 return [];
590 headercontent = str(fp.read(headersize).decode('UTF-8')).split(delimiter);
591 fp.seek(1, 1);
592 rocount = 0;
593 roend = int(len(headercontent));
594 HeaderOut = [headerpresize];
595 while(rocount<roend):
596 HeaderOut.append(headercontent[rocount]);
597 rocount = rocount + 1;
598 return HeaderOut;
599
600 def ReadFileHeaderDataWoSize(fp, delimiter=__file_format_delimiter__):
601 preheaderdata = ReadFileHeaderData(fp, 2, delimiter);
602 headersize = int(preheaderdata[0], 16);
603 headernumfields = int(preheaderdata[1], 16);
604 if(headersize<=0 or headernumfields<=0):
605 return [];
606 headerdata = ReadFileHeaderData(fp, headernumfields, delimiter);
607 HeaderOut = preheaderdata + headerdata;
608 return HeaderOut;
609
610 def ReadFileHeaderDataBySizeWithContent(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
611 delimiter = formatspecs[5];
612 fheaderstart = fp.tell();
613 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
614 if(len(HeaderOut)==0):
615 return False;
616 if(re.findall("^[.|/]", HeaderOut[3])):
617 fname = HeaderOut[3];
618 else:
619 fname = "./"+HeaderOut[3];
620 fchecksumtype = HeaderOut[-3].lower();
621 fcs = HeaderOut[-2].lower();
622 fccs = HeaderOut[-1].lower();
623 fsize = int(HeaderOut[5], 16);
624 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
625 if(fcs!=newfcs and not skipchecksum):
626 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
627 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
628 return False;
629 fhend = fp.tell() - 1;
630 fcontentstart = fp.tell();
631 fcontents = BytesIO();
632 if(fsize>0 and not listonly):
633 fcontents.write(fp.read(fsize));
634 elif(fsize>0 and listonly):
635 fp.seek(fsize, 1);
636 fcontents.seek(0, 0);
637 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
638 if(fccs!=newfccs and not skipchecksum and not listonly):
639 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
640 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
641 return False;
642 fp.seek(1, 1);
643 fcontentend = fp.tell() - 1;
644 HeaderOut.append(fcontents);
645 return HeaderOut;
646
647 def ReadFileHeaderDataBySizeWithContentToArray(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
648 delimiter = formatspecs[5];
649 fheaderstart = fp.tell();
650 if(formatspecs[7]):
651 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
652 else:
653 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
654 if(len(HeaderOut)==0):
655 return False;
656 fheadsize = int(HeaderOut[0], 16);
657 fnumfields = int(HeaderOut[1], 16);
658 ftype = int(HeaderOut[2], 16);
659 if(re.findall("^[.|/]", HeaderOut[3])):
660 fname = HeaderOut[3];
661 else:
662 fname = "./"+HeaderOut[3];
663 fbasedir = os.path.dirname(fname);
664 flinkname = HeaderOut[4];
665 fsize = int(HeaderOut[5], 16);
666 fatime = int(HeaderOut[6], 16);
667 fmtime = int(HeaderOut[7], 16);
668 fctime = int(HeaderOut[8], 16);
669 fbtime = int(HeaderOut[9], 16);
670 fmode = int(HeaderOut[10], 16);
671 fchmode = stat.S_IMODE(fmode);
672 ftypemod = stat.S_IFMT(fmode);
673 fwinattributes = int(HeaderOut[11], 16);
674 fuid = int(HeaderOut[12], 16);
675 funame = HeaderOut[13];
676 fgid = int(HeaderOut[14], 16);
677 fgname = HeaderOut[15];
678 fid = int(HeaderOut[16], 16);
679 finode = int(HeaderOut[17], 16);
680 flinkcount = int(HeaderOut[18], 16);
681 fdev_minor = int(HeaderOut[19], 16);
682 fdev_major = int(HeaderOut[20], 16);
683 frdev_minor = int(HeaderOut[21], 16);
684 frdev_major = int(HeaderOut[22], 16);
685 fextrasize = int(HeaderOut[23], 16);
686 fextrafields = int(HeaderOut[24], 16);
687 extrafieldslist = [];
688 extrastart = 25;
689 extraend = extrastart + fextrafields;
690 extrafieldslist = [];
691 if(extrastart<extraend):
692 extrafieldslist.append(HeaderOut[extrastart]);
693 extrastart = extrastart + 1;
694 fchecksumtype = HeaderOut[extrastart].lower();
695 fcs = HeaderOut[extrastart + 1].lower();
696 fccs = HeaderOut[extrastart + 2].lower();
697 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
698 if(fcs!=newfcs and not skipchecksum):
699 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
700 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
701 return False;
702 fhend = fp.tell() - 1;
703 fcontentstart = fp.tell();
704 fcontents = BytesIO();
705 pyhascontents = False;
706 if(fsize>0 and not listonly):
707 fcontents.write(fp.read(catfsize));
708 pyhascontents = True;
709 elif(fsize>0 and listonly):
710 fp.seek(fsize, 1);
711 pyhascontents = False;
712 fcontents.seek(0, 0);
713 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
714 if(fccs!=newfccs and not skipchecksum and not listonly):
715 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
716 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
717 return False;
718 fp.seek(1, 1);
719 fcontentend = fp.tell() - 1;
720 fcontents.seek(0, 0);
721 catlist = {'fheadersize': fheadsize, 'fhstart': fheaderstart, 'fhend': fhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': fchecksumtype, 'fnumfields': fnumfields + 2, 'frawheader': HeaderOut, 'fextrafields': fextrafields, 'fextrafieldsize': fextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': fcs, 'fcontentchecksum': fccs, 'fhascontents': pyhascontents, 'fcontentstart': fcontentstart, 'fcontentend': fcontentend, 'fcontents': fcontents};
722 return catlist;
723
724 def ReadFileHeaderDataBySizeWithContentToList(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
725 delimiter = formatspecs[5];
726 fheaderstart = fp.tell();
727 if(formatspecs[7]):
728 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
729 else:
730 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
731 if(len(HeaderOut)==0):
732 return False;
733 fheadsize = int(HeaderOut[0], 16);
734 fnumfields = int(HeaderOut[1], 16);
735 ftype = int(HeaderOut[2], 16);
736 if(re.findall("^[.|/]", HeaderOut[3])):
737 fname = HeaderOut[3];
738 else:
739 fname = "./"+HeaderOut[3];
740 fbasedir = os.path.dirname(fname);
741 flinkname = HeaderOut[4];
742 fsize = int(HeaderOut[5], 16);
743 fatime = int(HeaderOut[6], 16);
744 fmtime = int(HeaderOut[7], 16);
745 fctime = int(HeaderOut[8], 16);
746 fbtime = int(HeaderOut[9], 16);
747 fmode = int(HeaderOut[10], 16);
748 fchmode = stat.S_IMODE(fmode);
749 ftypemod = stat.S_IFMT(fmode);
750 fwinattributes = int(HeaderOut[11], 16);
751 fuid = int(HeaderOut[12], 16);
752 funame = HeaderOut[13];
753 fgid = int(HeaderOut[14], 16);
754 fgname = HeaderOut[15];
755 fid = int(HeaderOut[16], 16);
756 finode = int(HeaderOut[17], 16);
757 flinkcount = int(HeaderOut[18], 16);
758 fdev_minor = int(HeaderOut[19], 16);
759 fdev_major = int(HeaderOut[20], 16);
760 frdev_minor = int(HeaderOut[21], 16);
761 frdev_major = int(HeaderOut[22], 16);
762 fextrasize = int(HeaderOut[23], 16);
763 fextrafields = int(HeaderOut[24], 16);
764 extrafieldslist = [];
765 extrastart = 25;
766 extraend = extrastart + fextrafields;
767 extrafieldslist = [];
768 if(extrastart<extraend):
769 extrafieldslist.append(HeaderOut[extrastart]);
770 extrastart = extrastart + 1;
771 fchecksumtype = HeaderOut[extrastart].lower();
772 fcs = HeaderOut[extrastart + 1].lower();
773 fccs = HeaderOut[extrastart + 2].lower();
774 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
775 if(fcs!=newfcs and not skipchecksum):
776 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
777 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
778 return False;
779 fhend = fp.tell() - 1;
780 fcontentstart = fp.tell();
781 fcontents = BytesIO();
782 pyhascontents = False;
783 if(fsize>0 and not listonly):
784 fcontents.write(fp.read(fsize));
785 pyhascontents = True;
786 elif(fsize>0 and listonly):
787 fp.seek(fsize, 1);
788 pyhascontents = False;
789 fcontents.seek(0, 0);
790 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
791 if(fccs!=newfccs and not skipchecksum and not listonly):
792 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
793 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
794 return False;
795 fp.seek(1, 1);
796 fcontentend = fp.tell() - 1;
797 catlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, extrafieldslist, fchecksumtype, fcontents];
798 return catlist;
799
800 def ReadFileDataBySizeWithContent(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
801 delimiter = formatspecs[5];
802 curloc = fp.tell();
803 if(curloc>0):
804 fp.seek(0, 0);
805 catheader = ReadFileHeaderData(fp, 4, delimiter);
806 if(curloc>0):
807 fp.seek(curloc, 0);
808 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
809 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
810 if(not headercheck and not skipchecksum):
811 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
812 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
813 return False;
814 fnumfiles = int(catheader[1], 16);
815 countnum = 0;
816 flist = [];
817 while(countnum < fnumfiles):
818 HeaderOut = ReadFileHeaderDataBySizeWithContent(fp, listonly, skipchecksum, formatspecs)
819 if(len(HeaderOut)==0):
820 break;
821 flist.append(HeaderOut);
822 countnum = countnum + 1;
823 return flist;
824
825 def ReadFileDataBySizeWithContentToArray(fp, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
826 delimiter = formatspecs[5];
827 curloc = fp.tell();
828 if(curloc>0):
829 fp.seek(0, 0);
830 catheader = ReadFileHeaderData(fp, 4, delimiter);
831 if(curloc>0):
832 fp.seek(curloc, 0);
833 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
834 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
835 if(not headercheck and not skipchecksum):
836 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
837 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
838 return False;
839 catstring = catheader[0];
840 catversion = re.findall(r"([\d]+)$", catstring);
841 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
842 fprenumfiles = catheader[1];
843 fnumfiles = int(fprenumfiles, 16);
844 fprechecksumtype = catheader[2];
845 fprechecksum = catheader[3];
846 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
847 if(seekstart<0 and seekstart>fnumfiles):
848 seekstart = 0;
849 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
850 seekend = fnumfiles;
851 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
852 seekend = fnumfiles - abs(seekend);
853 if(seekstart>0):
854 il = 0;
855 while(il < seekstart):
856 prefhstart = fp.tell();
857 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs[5]);
858 if(len(preheaderdata)==0):
859 break;
860 prefsize = int(preheaderdata[5], 16);
861 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-3].lower(), True, formatspecs);
862 prefcs = preheaderdata[-2];
863 if(prefcs!=prenewfcs and not skipchecksum):
864 VVerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
865 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
866 return False;
867 valid_archive = False;
868 invalid_archive = True;
869 prefhend = fp.tell() - 1;
870 prefcontentstart = fp.tell();
871 prefcontents = BytesIO();
872 pyhascontents = False;
873 if(prefsize>0):
874 prefcontents.write(fp.read(prefsize));
875 prefcontents.seek(0, 0);
876 prenewfccs = GetFileChecksum(prefcontents.read(), preheaderdata[-3].lower(), False, formatspecs);
877 prefccs = preheaderdata[-1];
878 pyhascontents = True;
879 if(prefccs!=prenewfccs and not skipchecksum):
880 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
881 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
882 return False;
883 fp.seek(1, 1);
884 il = il + 1;
885 realidnum = 0;
886 countnum = seekstart;
887 while(countnum < seekend):
888 catlist['ffilelist'].update({realidnum: {'fid': realidnum, 'fidalt': realidnum}});
889 HeaderOut = ReadFileHeaderDataBySizeWithContentToArray(fp, listonly, skipchecksum, formatspecs);
890 if(len(HeaderOut)):
891 break;
892 catlist['ffilelist'][realidnum].update(HeaderOut);
893 countnum = countnum + 1;
894 realidnum = realidnum + 1;
895 return catlist;
896
897 def ReadFileDataBySizeWithContentToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
898 if(isinstance(infile, dict)):
899 listcatfiles = infile;
900 else:
901 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
902 return False;
903 listcatfiles = ReadFileDataBySizeWithContentToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs);
904 if(not listcatfiles):
905 return False;
906 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
907 lenlist = len(listcatfiles['ffilelist']);
908 lcfi = 0;
909 lcfx = int(listcatfiles['fnumfiles']);
910 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
911 lcfx = int(lenlist);
912 else:
913 lcfx = int(listcatfiles['fnumfiles']);
914 while(lcfi < lcfx):
915 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
916 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
917 catarray['filetoid'].update(filetoidarray);
918 catarray['idtofile'].update(idtofilearray);
919 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
920 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
921 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
922 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
923 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
924 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
925 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
926 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
927 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
928 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
929 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
930 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
931 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
932 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
933 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
934 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
935 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
936 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
937 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
938 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
939 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
940 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
941 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
942 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
943 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
944 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
945 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
946 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
947 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
948 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
949 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
950 lcfi = lcfi + 1;
951 return catarray;
952
953 def ReadFileDataBySizeWithContentToList(fp, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
954 delimiter = formatspecs[5];
955 curloc = fp.tell();
956 if(curloc>0):
957 fp.seek(0, 0);
958 catheader = ReadFileHeaderData(fp, 4, delimiter);
959 if(curloc>0):
960 fp.seek(curloc, 0);
961 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
962 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
963 if(not headercheck and not skipchecksum):
964 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
965 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
966 return False;
967 catstring = catheader[0];
968 catversion = re.findall(r"([\d]+)$", catstring);
969 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
970 fprenumfiles = catheader[1];
971 fnumfiles = int(fprenumfiles, 16);
972 fprechecksumtype = catheader[2];
973 fprechecksum = catheader[3];
974 catlist = [];
975 if(seekstart<0 and seekstart>fnumfiles):
976 seekstart = 0;
977 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
978 seekend = fnumfiles;
979 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
980 seekend = fnumfiles - abs(seekend);
981 if(seekstart>0):
982 il = 0;
983 while(il < seekstart):
984 prefhstart = fp.tell();
985 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs[5]);
986 if(len(preheaderdata)==0):
987 break;
988 prefsize = int(preheaderdata[5], 16);
989 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-3].lower(), True, formatspecs);
990 prefcs = preheaderdata[-2];
991 if(prefcs!=prenewfcs and not skipchecksum):
992 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
993 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
994 return False;
995 valid_archive = False;
996 invalid_archive = True;
997 prefhend = fp.tell() - 1;
998 prefcontentstart = fp.tell();
999 prefcontents = "";
1000 pyhascontents = False;
1001 if(prefsize>0):
1002 prefcontents = fp.read(prefsize);
1003 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
1004 prefccs = preheaderdata[-1];
1005 pyhascontents = True;
1006 if(prefccs!=prenewfccs and not skipchecksum):
1007 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
1008 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
1009 return False;
1010 fp.seek(1, 1);
1011 il = il + 1;
1012 realidnum = 0;
1013 countnum = seekstart;
1014 while(countnum < seekend):
1015 HeaderOut = ReadFileHeaderDataBySizeWithContentToList(fp, listonly, skipchecksum, formatspecs);
1016 if(len(HeaderOut)):
1017 break;
1018 catlist.append(HeaderOut);
1019 countnum = countnum + 1;
1020 realidnum = realidnum + 1;
1021 return catlist;
1022
1023 def ReadInFileBySizeWithContentToArray(infile, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
1024 delimiter = formatspecs[5];
1025 if(hasattr(infile, "read") or hasattr(infile, "write")):
1026 fp = infile;
1027 fp.seek(0, 0);
1028 fp = UncompressArchiveFile(fp, formatspecs);
1029 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1030 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
1031 return False;
1032 if(not fp):
1033 return False;
1034 fp.seek(0, 0);
1035 elif(infile=="-"):
1036 fp = BytesIO();
1037 if(hasattr(sys.stdin, "buffer")):
1038 shutil.copyfileobj(sys.stdin.buffer, fp);
1039 else:
1040 shutil.copyfileobj(sys.stdin, fp);
1041 fp.seek(0, 0);
1042 fp = UncompressArchiveFile(fp, formatspecs);
1043 if(not fp):
1044 return False;
1045 fp.seek(0, 0);
1046 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
1047 fp = download_file_from_internet_file(infile);
1048 fp = UncompressArchiveFile(fp, formatspecs);
1049 fp.seek(0, 0);
1050 if(not fp):
1051 return False;
1052 fp.seek(0, 0);
1053 else:
1054 infile = RemoveWindowsPath(infile);
1055 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
1056 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
1057 return False;
1058 compresscheck = CheckCompressionType(infile, formatspecs, True);
1059 if(not compresscheck):
1060 fextname = os.path.splitext(infile)[1];
1061 if(fextname==".gz"):
1062 compresscheck = "gzip";
1063 elif(fextname==".bz2"):
1064 compresscheck = "bzip2";
1065 elif(fextname==".zst"):
1066 compresscheck = "zstd";
1067 elif(fextname==".lz4" or fextname==".clz4"):
1068 compresscheck = "lz4";
1069 elif(fextname==".lzo" or fextname==".lzop"):
1070 compresscheck = "lzo";
1071 elif(fextname==".lzma" or fextname==".xz"):
1072 compresscheck = "lzma";
1073 else:
1074 return False;
1075 if(not compresscheck):
1076 return False;
1077 fp = UncompressFile(infile, formatspecs, "rb");
1078 return ReadFileDataBySizeWithContentToArray(fp, listonly, skipchecksum, formatspecs);
1079
1080 def ReadInFileBySizeWithContentToList(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
1081 delimiter = formatspecs[5];
1082 if(hasattr(infile, "read") or hasattr(infile, "write")):
1083 fp = infile;
1084 fp.seek(0, 0);
1085 fp = UncompressArchiveFile(fp, formatspecs);
1086 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1087 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
1088 return False;
1089 if(not fp):
1090 return False;
1091 fp.seek(0, 0);
1092 elif(infile=="-"):
1093 fp = BytesIO();
1094 if(hasattr(sys.stdin, "buffer")):
1095 shutil.copyfileobj(sys.stdin.buffer, fp);
1096 else:
1097 shutil.copyfileobj(sys.stdin, fp);
1098 fp.seek(0, 0);
1099 fp = UncompressArchiveFile(fp, formatspecs);
1100 if(not fp):
1101 return False;
1102 fp.seek(0, 0);
1103 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
1104 fp = download_file_from_internet_file(infile);
1105 fp = UncompressArchiveFile(fp, formatspecs);
1106 fp.seek(0, 0);
1107 if(not fp):
1108 return False;
1109 fp.seek(0, 0);
1110 else:
1111 infile = RemoveWindowsPath(infile);
1112 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
1113 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
1114 return False;
1115 compresscheck = CheckCompressionType(infile, formatspecs, True);
1116 if(not compresscheck):
1117 fextname = os.path.splitext(infile)[1];
1118 if(fextname==".gz"):
1119 compresscheck = "gzip";
1120 elif(fextname==".bz2"):
1121 compresscheck = "bzip2";
1122 elif(fextname==".zst"):
1123 compresscheck = "zstd";
1124 elif(fextname==".lz4" or fextname==".clz4"):
1125 compresscheck = "lz4";
1126 elif(fextname==".lzo" or fextname==".lzop"):
1127 compresscheck = "lzo";
1128 elif(fextname==".lzma" or fextname==".xz"):
1129 compresscheck = "lzma";
1130 else:
1131 return False;
1132 if(not compresscheck):
1133 return False;
1134 fp = UncompressFile(infile, formatspecs, "rb");
1135 return ReadFileDataBySizeWithContentToList(fp, seekstart, seekend, listonly, skipchecksum, formatspecs);
1136
1137 def ReadInFileBySizeWithContentToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
1138 if(isinstance(infile, dict)):
1139 listcatfiles = infile;
1140 else:
1141 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
1142 infile = RemoveWindowsPath(infile);
1143 listcatfiles = ReadInFileBySizeWithContentToArray(infile, listonly, skipchecksum, formatspecs);
1144 if(not listcatfiles):
1145 return False;
1146 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
1147 lenlist = len(listcatfiles['ffilelist']);
1148 lcfi = 0;
1149 lcfx = int(listcatfiles['fnumfiles']);
1150 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
1151 lcfx = int(lenlist);
1152 else:
1153 lcfx = int(listcatfiles['fnumfiles']);
1154 while(lcfi < lcfx):
1155 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
1156 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
1157 catarray['filetoid'].update(filetoidarray);
1158 catarray['idtofile'].update(idtofilearray);
1159 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
1160 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
1161 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
1162 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
1163 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
1164 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
1165 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
1166 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
1167 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
1168 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1169 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1170 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
1171 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
1172 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
1173 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
1174 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
1175 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1176 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1177 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
1178 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
1179 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
1180 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1181 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1182 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
1183 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
1184 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
1185 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
1186 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1187 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1188 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1189 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1190 lcfi = lcfi + 1;
1191 return catarray;
1192
1193 def AppendNullByte(indata, delimiter=__file_format_delimiter__):
1194 outdata = str(indata) + delimiter;
1195 return outdata;
1196
1197 def AppendNullBytes(indata=[], delimiter=__file_format_delimiter__):
1198 outdata = "";
1199 inum = 0;
1200 il = len(indata);
1201 while(inum < il):
1202 outdata = outdata + AppendNullByte(indata[inum], delimiter);
1203 inum = inum + 1;
1204 return outdata;
1205
1206 def AppendFileHeader(fp, numfiles, checksumtype="crc32", formatspecs=__file_format_list__):
1207 delimiter = formatspecs[5];
1208 catver = formatspecs[6];
1209 fileheaderver = str(int(catver.replace(".", "")));
1210 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
1211 fp.write(fileheader.encode('UTF-8'));
1212 fnumfiles = format(int(numfiles), 'x').lower();
1213 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs[5]);
1214 catfileheadercshex = GetFileChecksum(fileheader + fnumfilesa, checksumtype, True, formatspecs);
1215 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs[5]);
1216 fp.write(fnumfilesa.encode('UTF-8'));
1217 try:
1218 fp.flush();
1219 os.fsync(fp.fileno());
1220 except io.UnsupportedOperation:
1221 pass;
1222 except AttributeError:
1223 pass;
1224 except OSError as e:
1225 pass;
1226 return fp;
1227
1228 def MakeEmptyFilePointer(fp, checksumtype="crc32", formatspecs=__file_format_list__):
1229 AppendFileHeader(fp, 0, checksumtype, formatspecs);
1230 return fp;
1231
1232 def MakeEmptyFile(outfile, compression="auto", compressionlevel=None, checksumtype="crc32", formatspecs=__file_format_list__, returnfp=False):
1233 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1234 if(os.path.exists(outfile)):
1235 try:
1236 os.unlink(outfile);
1237 except OSError as e:
1238 pass;
1239 if(outfile=="-"):
1240 verbose = False;
1241 catfpfp = BytesIO();
1242 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1243 catfp = outfile;
1244 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1245 catfp = BytesIO();
1246 else:
1247 fbasename = os.path.splitext(outfile)[0];
1248 fextname = os.path.splitext(outfile)[1];
1249 catfp = CompressOpenFile(outfile, True, compressionlevel);
1250 catfp = AppendFileHeader(catfp, 0, checksumtype, formatspecs);
1251 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1252 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1253 try:
1254 catfp.flush();
1255 os.fsync(catfp.fileno());
1256 except io.UnsupportedOperation:
1257 pass;
1258 except AttributeError:
1259 pass;
1260 except OSError as e:
1261 pass;
1262 if(outfile=="-"):
1263 catfp.seek(0, 0);
1264 if(hasattr(sys.stdout, "buffer")):
1265 shutil.copyfileobj(catfp, sys.stdout.buffer);
1266 else:
1267 shutil.copyfileobj(catfp, sys.stdout);
1268 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1269 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1270 catfp.seek(0, 0);
1271 upload_file_to_internet_file(catfp, outfile);
1272 if(returnfp):
1273 catfp.seek(0, 0);
1274 return catfp;
1275 else:
1276 catfp.close();
1277 return True;
1278
1279 def AppendFileHeaderWithContent(fp, filevalues=[], extradata=[], filecontent="", checksumtype="crc32", formatspecs=__file_format_list__):
1280 extrafields = format(len(extradata), 'x').lower();
1281 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
1282 if(len(extradata)>0):
1283 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
1284 extrasizelen = format(len(extrasizestr), 'x').lower();
1285 catoutlen = len(filevalues) + len(extradata) + 5;
1286 catoutlenhex = format(catoutlen, 'x').lower();
1287 catoutlist = filevalues;
1288 catoutlist.insert(0, catoutlenhex);
1289 catoutlist.append(extrasizelen);
1290 catoutlist.append(extrafields);
1291 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
1292 if(len(extradata)>0):
1293 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
1294 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
1295 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
1296 catfilecontentcshex = GetFileChecksum(filecontent, checksumtype, False, formatspecs);
1297 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
1298 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
1299 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
1300 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
1301 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
1302 catfileoutstrecd = catfileoutstr.encode('UTF-8');
1303 nullstrecd = formatspecs[5].encode('UTF-8');
1304 catfileout = catfileoutstrecd + filecontent + nullstrecd;
1305 fp.write(catfileout);
1306 try:
1307 fp.flush();
1308 os.fsync(fp.fileno());
1309 except io.UnsupportedOperation:
1310 pass;
1311 except AttributeError:
1312 pass;
1313 except OSError as e:
1314 pass;
1315 return fp;
1316
1317 def AppendFilesWithContent(infiles, fp, dirlistfromtxt=False, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False):
1318 advancedlist = formatspecs[8];
1319 altinode = formatspecs[9];
1320 if(verbose):
1321 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
1322 if(infiles=="-"):
1323 for line in sys.stdin:
1324 infilelist.append(line.strip());
1325 infilelist = list(filter(None, infilelist));
1326 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
1327 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
1328 return False;
1329 with UncompressFile(infiles, formatspecs, "r") as finfile:
1330 for line in finfile:
1331 infilelist.append(line.strip());
1332 infilelist = list(filter(None, infilelist));
1333 else:
1334 if(isinstance(infiles, (list, tuple, ))):
1335 infilelist = list(filter(None, infiles));
1336 elif(isinstance(infiles, (str, ))):
1337 infilelist = list(filter(None, [infiles]));
1338 if(advancedlist):
1339 GetDirList = ListDirAdvanced(infilelist, followlink, False);
1340 else:
1341 GetDirList = ListDir(infilelist, followlink, False);
1342 if(not GetDirList):
1343 return False;
1344 curinode = 0;
1345 curfid = 0;
1346 inodelist = [];
1347 inodetofile = {};
1348 filetoinode = {};
1349 inodetocatinode = {};
1350 numfiles = int(len(GetDirList));
1351 fnumfiles = format(numfiles, 'x').lower();
1352 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
1353 for curfname in GetDirList:
1354 if(re.findall("^[.|/]", curfname)):
1355 fname = curfname;
1356 else:
1357 fname = "./"+curfname;
1358 if(verbose):
1359 VerbosePrintOut(fname);
1360 if(not followlink or followlink is None):
1361 fstatinfo = os.lstat(fname);
1362 else:
1363 fstatinfo = os.stat(fname);
1364 fpremode = fstatinfo.st_mode;
1365 finode = fstatinfo.st_ino;
1366 flinkcount = fstatinfo.st_nlink;
1367 ftype = 0;
1368 if(stat.S_ISREG(fpremode)):
1369 ftype = 0;
1370 elif(stat.S_ISLNK(fpremode)):
1371 ftype = 2;
1372 elif(stat.S_ISCHR(fpremode)):
1373 ftype = 3;
1374 elif(stat.S_ISBLK(fpremode)):
1375 ftype = 4;
1376 elif(stat.S_ISDIR(fpremode)):
1377 ftype = 5;
1378 elif(stat.S_ISFIFO(fpremode)):
1379 ftype = 6;
1380 elif(stat.S_ISSOCK(fpremode)):
1381 ftype = 8;
1382 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
1383 ftype = 9;
1384 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
1385 ftype = 10;
1386 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
1387 ftype = 111;
1388 else:
1389 ftype = 0;
1390 flinkname = "";
1391 fcurfid = format(int(curfid), 'x').lower();
1392 if(not followlink and finode!=0):
1393 if(ftype!=1):
1394 if(finode in inodelist):
1395 ftype = 1;
1396 flinkname = inodetofile[finode];
1397 if(altinode):
1398 fcurinode = format(int(finode), 'x').lower();
1399 else:
1400 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
1401 if(finode not in inodelist):
1402 inodelist.append(finode);
1403 inodetofile.update({finode: fname});
1404 inodetocatinode.update({finode: curinode});
1405 if(altinode):
1406 fcurinode = format(int(finode), 'x').lower();
1407 else:
1408 fcurinode = format(int(curinode), 'x').lower();
1409 curinode = curinode + 1;
1410 else:
1411 fcurinode = format(int(curinode), 'x').lower();
1412 curinode = curinode + 1;
1413 curfid = curfid + 1;
1414 if(ftype==2):
1415 flinkname = os.readlink(fname);
1416 fdev = fstatinfo.st_dev;
1417 getfdev = GetDevMajorMinor(fdev);
1418 fdev_minor = getfdev[0];
1419 fdev_major = getfdev[1];
1420 frdev = fstatinfo.st_dev;
1421 if(hasattr(fstatinfo, "st_rdev")):
1422 frdev = fstatinfo.st_rdev;
1423 else:
1424 frdev = fstatinfo.st_dev;
1425 getfrdev = GetDevMajorMinor(frdev);
1426 frdev_minor = getfrdev[0];
1427 frdev_major = getfrdev[1];
1428 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
1429 fsize = format(int("0"), 'x').lower();
1430 elif(ftype==0 or ftype==7):
1431 fsize = format(int(fstatinfo.st_size), 'x').lower();
1432 else:
1433 fsize = format(int(fstatinfo.st_size)).lower();
1434 fatime = format(int(fstatinfo.st_atime), 'x').lower();
1435 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
1436 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
1437 if(hasattr(fstatinfo, "st_birthtime")):
1438 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
1439 else:
1440 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
1441 fmode = format(int(fstatinfo.st_mode), 'x').lower();
1442 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
1443 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
1444 fuid = format(int(fstatinfo.st_uid), 'x').lower();
1445 fgid = format(int(fstatinfo.st_gid), 'x').lower();
1446 funame = "";
1447 try:
1448 import pwd;
1449 try:
1450 userinfo = pwd.getpwuid(fstatinfo.st_uid);
1451 funame = userinfo.pw_name;
1452 except KeyError:
1453 funame = "";
1454 except ImportError:
1455 funame = "";
1456 fgname = "";
1457 try:
1458 import grp;
1459 try:
1460 groupinfo = grp.getgrgid(fstatinfo.st_gid);
1461 fgname = groupinfo.gr_name;
1462 except KeyError:
1463 fgname = "";
1464 except ImportError:
1465 fgname = "";
1466 fdev_minor = format(int(fdev_minor), 'x').lower();
1467 fdev_major = format(int(fdev_major), 'x').lower();
1468 frdev_minor = format(int(frdev_minor), 'x').lower();
1469 frdev_major = format(int(frdev_major), 'x').lower();
1470 finode = format(int(finode), 'x').lower();
1471 flinkcount = format(int(flinkcount), 'x').lower();
1472 if(hasattr(fstatinfo, "st_file_attributes")):
1473 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
1474 else:
1475 fwinattributes = format(int(0), 'x').lower();
1476 fcontents = BytesIO();
1477 chunk_size = 1024;
1478 if(ftype==0 or ftype==7):
1479 with open(fname, "rb") as fpc:
1480 shutil.copyfileobj(fpc, fcontents);
1481 if(followlink and (ftype==1 or ftype==2)):
1482 flstatinfo = os.stat(flinkname);
1483 with open(flinkname, "rb") as fpc:
1484 shutil.copyfileobj(fpc, fcontents);
1485 fcontents.seek(0, 0);
1486 ftypehex = format(ftype, 'x').lower();
1487 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
1488 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
1489 if(numfiles>0):
1490 catfp.write(AppendNullBytes([0, 0], formatspecs[5]).encode("UTF-8"));
1491 fp.seek(0, 0);
1492 return fp;
1493
1494 def AppendListsWithContent(inlist, fp, dirlistfromtxt=False, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False):
1495 if(verbose):
1496 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
1497 GetDirList = inlist;
1498 if(not GetDirList):
1499 return False;
1500 curinode = 0;
1501 curfid = 0;
1502 inodelist = [];
1503 inodetofile = {};
1504 filetoinode = {};
1505 inodetocatinode = {};
1506 numfiles = int(len(GetDirList));
1507 fnumfiles = format(numfiles, 'x').lower();
1508 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
1509 for curfname in GetDirList:
1510 ftype = format(curfname[0], 'x').lower();
1511 if(re.findall("^[.|/]", curfname[1])):
1512 fname = curfname[1];
1513 else:
1514 fname = "./"+curfname[1];
1515 fbasedir = os.path.dirname(fname);
1516 flinkname = curfname[2];
1517 fsize = format(curfname[3], 'x').lower();
1518 fatime = format(curfname[4], 'x').lower();
1519 fmtime = format(curfname[5], 'x').lower();
1520 fctime = format(curfname[6], 'x').lower();
1521 fbtime = format(curfname[7], 'x').lower();
1522 fmode = format(curfname[8], 'x').lower();
1523 fwinattributes = format(curfname[9], 'x').lower();
1524 fuid = format(curfname[10], 'x').lower();
1525 funame = curfname[11];
1526 fgid = format(curfname[12], 'x').lower();
1527 fgname = curfname[13];
1528 fid = format(curfname[14], 'x').lower();
1529 finode = format(curfname[15], 'x').lower();
1530 flinkcount = format(curfname[16], 'x').lower();
1531 fdev_minor = format(curfname[17], 'x').lower();
1532 fdev_major = format(curfname[18], 'x').lower();
1533 frdev_minor = format(curfname[19], 'x').lower();
1534 frdev_major = format(curfname[20], 'x').lower();
1535 extradata = curfname[21];
1536 fchecksumtype = curfname[22];
1537 fcontents = curfname[23];
1538 catoutlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
1539 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents, checksumtype, formatspecs);
1540 if(numfiles>0):
1541 catfp.write(AppendNullBytes([0, 0], formatspecs[5]).encode("UTF-8"));
1542 fp.seek(0, 0);
1543 return fp;
1544
1545 def AppendInFileWithContent(infile, fp, dirlistfromtxt=False, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False):
1546 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, False, formatspecs);
1547 return AppendListsWithContent(inlist, fp, dirlistfromtxt, filevalues, extradata, followlink, checksumtype, formatspecs, verbose);
1548
1549 def AppendFilesWithContentToOutFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False, returnfp=False):
1550 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1551 if(os.path.exists(outfile)):
1552 try:
1553 os.unlink(outfile);
1554 except OSError as e:
1555 pass;
1556 if(outfile=="-"):
1557 verbose = False;
1558 catfpfp = BytesIO();
1559 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1560 catfp = outfile;
1561 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1562 catfp = BytesIO();
1563 else:
1564 fbasename = os.path.splitext(outfile)[0];
1565 fextname = os.path.splitext(outfile)[1];
1566 catfp = CompressOpenFile(outfile, True, compressionlevel);
1567 catfp = AppendFilesWithContent(infiles, catfp, dirlistfromtxt, filevalues, extradata, followlink, checksumtype, formatspecs, verbose);
1568 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1569 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1570 try:
1571 catfp.flush();
1572 os.fsync(catfp.fileno());
1573 except io.UnsupportedOperation:
1574 pass;
1575 except AttributeError:
1576 pass;
1577 except OSError as e:
1578 pass;
1579 if(outfile=="-"):
1580 catfp.seek(0, 0);
1581 if(hasattr(sys.stdout, "buffer")):
1582 shutil.copyfileobj(catfp, sys.stdout.buffer);
1583 else:
1584 shutil.copyfileobj(catfp, sys.stdout);
1585 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1586 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1587 catfp.seek(0, 0);
1588 upload_file_to_internet_file(catfp, outfile);
1589 if(returnfp):
1590 catfp.seek(0, 0);
1591 return catfp;
1592 else:
1593 catfp.close();
1594 return True;
1595
1596 def AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt=False, compression="auto", compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False, returnfp=False):
1597 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1598 if(os.path.exists(outfile)):
1599 try:
1600 os.unlink(outfile);
1601 except OSError as e:
1602 pass;
1603 if(outfile=="-"):
1604 verbose = False;
1605 catfpfp = BytesIO();
1606 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1607 catfp = outfile;
1608 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1609 catfp = BytesIO();
1610 else:
1611 fbasename = os.path.splitext(outfile)[0];
1612 fextname = os.path.splitext(outfile)[1];
1613 catfp = CompressOpenFile(outfile, True, compressionlevel);
1614 catfp = AppendListsWithContent(inlist, catfp, dirlistfromtxt, filevalues, extradata, followlink, checksumtype, formatspecs, verbose);
1615 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1616 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1617 try:
1618 catfp.flush();
1619 os.fsync(catfp.fileno());
1620 except io.UnsupportedOperation:
1621 pass;
1622 except AttributeError:
1623 pass;
1624 except OSError as e:
1625 pass;
1626 if(outfile=="-"):
1627 catfp.seek(0, 0);
1628 if(hasattr(sys.stdout, "buffer")):
1629 shutil.copyfileobj(catfp, sys.stdout.buffer);
1630 else:
1631 shutil.copyfileobj(catfp, sys.stdout);
1632 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1633 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1634 catfp.seek(0, 0);
1635 upload_file_to_internet_file(catfp, outfile);
1636 if(returnfp):
1637 catfp.seek(0, 0);
1638 return catfp;
1639 else:
1640 catfp.close();
1641 return True;
1642
1643 def AppendInFileWithContentToOutFile(infile, outfile, dirlistfromtxt=False, compression="auto", compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False, returnfp=False):
1644 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, False, formatspecs);
1645 return AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt, compression, compressionlevel, filevalues, extradata, followlink, checksumtype, formatspecs, verbose, returnfp);
1646
1647 def PrintPermissionString(fchmode, ftype):
1648 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
1649 permissionstr = "";
1650 for fmodval in str(oct(fchmode))[-3:]:
1651 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
1652 if(ftype==0 or ftype==7):
1653 permissionstr = "-" + permissionstr;
1654 if(ftype==1):
1655 permissionstr = "h" + permissionstr;
1656 if(ftype==2):
1657 permissionstr = "l" + permissionstr;
1658 if(ftype==3):
1659 permissionstr = "c" + permissionstr;
1660 if(ftype==4):
1661 permissionstr = "b" + permissionstr;
1662 if(ftype==5):
1663 permissionstr = "d" + permissionstr;
1664 if(ftype==6):
1665 permissionstr = "f" + permissionstr;
1666 if(ftype==8):
1667 permissionstr = "D" + permissionstr;
1668 if(ftype==9):
1669 permissionstr = "p" + permissionstr;
1670 if(ftype==10):
1671 permissionstr = "w" + permissionstr;
1672 try:
1673 permissionoutstr = stat.filemode(fchmode);
1674 except AttributeError:
1675 permissionoutstr = permissionstr;
1676 except KeyError:
1677 permissionoutstr = permissionstr;
1678 return permissionoutstr;
1679
1680 def PrintPermissionStringAlt(fchmode, ftype):
1681 permissions = {
1682 '0': '---', '1': '--x', '2': '-w-', '3': '-wx',
1683 '4': 'r--', '5': 'r-x', '6': 'rw-', '7': 'rwx'
1684 };
1685 # Translate file mode into permission string
1686 permissionstr = ''.join([permissions[i] for i in str(oct(fchmode))[-3:]]);
1687 # Append file type indicator
1688 type_indicators = {
1689 0: '-', 1: 'h', 2: 'l', 3: 'c', 4: 'b',
1690 5: 'd', 6: 'f', 8: 'D', 9: 'p', 10: 'w'
1691 };
1692 file_type = type_indicators.get(ftype, '-');
1693 permissionstr = file_type + permissionstr;
1694 try:
1695 permissionoutstr = stat.filemode(fchmode);
1696 except AttributeError:
1697 permissionoutstr = permissionstr;
1698 return permissionoutstr;
1699
1700 def CompressionSupport():
1701 compression_list = [];
1702 try:
1703 import gzip;
1704 compression_list.append("gz");
1705 compression_list.append("gzip");
1706 except ImportError:
1707 '''return False;'''
1708 try:
1709 import bz2;
1710 compression_list.append("bz2");
1711 compression_list.append("bzip2");
1712 except ImportError:
1713 '''return False;'''
1714 try:
1715 import lz4;
1716 compression_list.append("lz4");
1717 except ImportError:
1718 '''return False;'''
1719 try:
1720 import lzo;
1721 compression_list.append("lzo");
1722 compression_list.append("lzop");
1723 except ImportError:
1724 '''return False;'''
1725 try:
1726 import zstandard;
1727 compression_list.append("zstd");
1728 compression_list.append("zstandard");
1729 except ImportError:
1730 '''return False;'''
1731 try:
1732 import lzma;
1733 compression_list.append("lzma");
1734 compression_list.append("xz");
1735 except ImportError:
1736 '''return False;'''
1737 return compression_list;
1738
1739 def CheckCompressionType(infile, formatspecs=__file_format_list__, closefp=True):
1740 if(hasattr(infile, "read") or hasattr(infile, "write")):
1741 catfp = infile;
1742 else:
1743 try:
1744 catfp = open(infile, "rb");
1745 except FileNotFoundError:
1746 return False;
1747 catfp.seek(0, 0);
1748 prefp = catfp.read(2);
1749 filetype = False;
1750 if(prefp==binascii.unhexlify("1f8b")):
1751 filetype = "gzip";
1752 catfp.seek(0, 0);
1753 prefp = catfp.read(3);
1754 if(prefp==binascii.unhexlify("425a68")):
1755 filetype = "bzip2";
1756 if(prefp==binascii.unhexlify("5d0000")):
1757 filetype = "lzma";
1758 catfp.seek(0, 0);
1759 prefp = catfp.read(4);
1760 if(prefp==binascii.unhexlify("28b52ffd")):
1761 filetype = "zstd";
1762 if(prefp==binascii.unhexlify("04224d18")):
1763 filetype = "lz4";
1764 if(prefp==binascii.unhexlify("504B0304")):
1765 filetype = "zipfile";
1766 catfp.seek(0, 0);
1767 prefp = catfp.read(5);
1768 if(prefp==binascii.unhexlify("7573746172")):
1769 filetype = "tarfile";
1770 catfp.seek(0, 0);
1771 prefp = catfp.read(6);
1772 if(prefp==binascii.unhexlify("fd377a585a00")):
1773 filetype = "lzma";
1774 if(prefp==binascii.unhexlify("377abcaf271c")):
1775 filetype = "7zipfile";
1776 catfp.seek(0, 0);
1777 prefp = catfp.read(7);
1778 if(prefp==binascii.unhexlify("526172211a0700")):
1779 filetype = "rarfile";
1780 if(prefp==binascii.unhexlify("43617446696c65")):
1781 filetype = "catfile";
1782 catfp.seek(0, 0);
1783 prefp = catfp.read(8);
1784 if(prefp==binascii.unhexlify("526172211a070100")):
1785 filetype = "rarfile";
1786 catfp.seek(0, 0);
1787 prefp = catfp.read(formatspecs[3]);
1788 if(prefp==binascii.unhexlify(formatspecs[4])):
1789 filetype = formatspecs[2];
1790 catfp.seek(0, 0);
1791 prefp = catfp.read(9);
1792 if(prefp==binascii.unhexlify("894c5a4f000d0a1a0a")):
1793 filetype = "lzo";
1794 catfp.seek(0, 0);
1795 prefp = catfp.read(10);
1796 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
1797 filetype = "tarfile";
1798 catfp.seek(0, 0);
1799 if(filetype=="gzip" or filetype=="bzip2" or filetype=="lzma" or filetype=="zstd" or filetype=="lz4"):
1800 if(is_tarfile(catfp)):
1801 filetype = "tarfile";
1802 if(not filetype):
1803 if(is_tarfile(catfp)):
1804 filetype = "tarfile";
1805 elif(zipfile.is_zipfile(catfp)):
1806 filetype = "zipfile";
1807 elif(rarfile.is_rarfile(catfp) or rarfile.is_rarfile_sfx(catfp)):
1808 filetype = "rarile";
1809 else:
1810 filetype = False;
1811 catfp.seek(0, 0);
1812 if(closefp):
1813 catfp.close();
1814 return filetype;
1815
1816 def CheckCompressionTypeFromString(instring, formatspecs=__file_format_list__, closefp=True):
1817 try:
1818 instringsfile = BytesIO(instring);
1819 except TypeError:
1820 instringsfile = BytesIO(instring.encode("UTF-8"));
1821 return CheckCompressionType(instringsfile, formatspecs, closefp);
1822
1823 def GetCompressionMimeType(infile, formatspecs=__file_format_list__):
1824 compresscheck = CheckCompressionType(fp, formatspecs, False);
1825 if(compresscheck=="gzip" or compresscheck=="gz"):
1826 return archivefile_gzip_mimetype;
1827 if(compresscheck=="bzip2" or compresscheck=="bz2"):
1828 return archivefile_bzip2_mimetype;
1829 if(compresscheck=="zstd" or compresscheck=="zstandard"):
1830 return archivefile_zstandard_mimetype;
1831 if(compresscheck=="lz4"):
1832 return archivefile_lz4_mimetype;
1833 if(compresscheck=="lzo" or compresscheck=="lzop"):
1834 return archivefile_lzop_mimetype;
1835 if(compresscheck=="lzma"):
1836 return archivefile_lzma_mimetype;
1837 if(compresscheck=="xz"):
1838 return archivefile_xz_mimetype;
1839 if(compresscheck=="catfile" or compresscheck=="cat" or compresscheck==formatspecs[2]):
1840 return archivefile_cat_mimetype;
1841 if(not compresscheck):
1842 return False;
1843 return False;
1844
1845 def UncompressArchiveFile(fp, formatspecs=__file_format_list__):
1846 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
1847 return False;
1848 compresscheck = CheckCompressionType(fp, formatspecs, False);
1849 if(compresscheck=="gzip"):
1850 try:
1851 import gzip;
1852 except ImportError:
1853 return False;
1854 catfp = gzip.GzipFile(fileobj=fp, mode="rb");
1855 if(compresscheck=="bzip2"):
1856 try:
1857 import bz2;
1858 except ImportError:
1859 return False;
1860 catfp = BytesIO();
1861 catfp.write(bz2.decompress(fp.read()));
1862 if(compresscheck=="zstd"):
1863 try:
1864 import zstandard;
1865 except ImportError:
1866 return False;
1867 catfp = BytesIO();
1868 catfp.write(zstandard.decompress(fp.read()));
1869 if(compresscheck=="lz4"):
1870 try:
1871 import lz4.frame;
1872 except ImportError:
1873 return False;
1874 catfp = BytesIO();
1875 catfp.write(lz4.frame.decompress(fp.read()));
1876 if(compresscheck=="lzo" or compresscheck=="lzop"):
1877 try:
1878 import lzo;
1879 except ImportError:
1880 return False;
1881 catfp = BytesIO();
1882 catfp.write(lzo.decompress(fp.read()));
1883 if(compresscheck=="lzma" or compresscheck=="xz"):
1884 try:
1885 import lzma;
1886 except ImportError:
1887 return False;
1888 catfp = BytesIO();
1889 catfp.write(lzma.decompress(fp.read()));
1890 if(compresscheck=="catfile" or compresscheck==formatspecs[2]):
1891 catfp = fp;
1892 if(not compresscheck):
1893 try:
1894 import lzma;
1895 except ImportError:
1896 return False;
1897 catfp = BytesIO();
1898 with fp as fpcontent:
1899 try:
1900 catfp.write(lzma.decompress(fp.read()));
1901 except lzma.LZMAError:
1902 return False;
1903 if(compresscheck!="catfile" or compresscheck!=formatspecs[2]):
1904 fp.close();
1905 return catfp;
1906
1907 create_alias_function("Uncompress", __file_format_name__, "", UncompressArchiveFile);
1908
1909 def UncompressFile(infile, formatspecs=__file_format_list__, mode="rb"):
1910 compresscheck = CheckCompressionType(infile, formatspecs, False);
1911 if(sys.version_info[0]==2 and compresscheck):
1912 if(mode=="rt"):
1913 mode = "r";
1914 if(mode=="wt"):
1915 mode = "w";
1916 try:
1917 if(compresscheck=="gzip"):
1918 try:
1919 import gzip;
1920 except ImportError:
1921 return False;
1922 try:
1923 filefp = gzip.open(infile, mode, encoding="UTF-8");
1924 except (ValueError, TypeError) as e:
1925 filefp = gzip.open(infile, mode);
1926 if(compresscheck=="bzip2"):
1927 try:
1928 import bz2;
1929 except ImportError:
1930 return False;
1931 try:
1932 filefp = bz2.open(infile, mode, encoding="UTF-8");
1933 except (ValueError, TypeError) as e:
1934 filefp = bz2.open(infile, mode);
1935 if(compresscheck=="zstd"):
1936 try:
1937 import zstandard;
1938 except ImportError:
1939 return False;
1940 try:
1941 filefp = zstandard.open(infile, mode, encoding="UTF-8");
1942 except (ValueError, TypeError) as e:
1943 filefp = zstandard.open(infile, mode);
1944 if(compresscheck=="lz4"):
1945 try:
1946 import lz4.frame;
1947 except ImportError:
1948 return False;
1949 try:
1950 filefp = lz4.frame.open(infile, mode, encoding="UTF-8");
1951 except (ValueError, TypeError) as e:
1952 filefp = lz4.frame.open(infile, mode);
1953 if(compresscheck=="lzo"):
1954 try:
1955 import lzo;
1956 except ImportError:
1957 return False;
1958 try:
1959 filefp = lzo.open(infile, mode, encoding="UTF-8");
1960 except (ValueError, TypeError) as e:
1961 filefp = lzo.open(infile, mode);
1962 if(compresscheck=="lzma"):
1963 try:
1964 import lzma;
1965 except ImportError:
1966 return False;
1967 try:
1968 filefp = lzma.open(infile, mode, encoding="UTF-8");
1969 except (ValueError, TypeError) as e:
1970 filefp = lzma.open(infile, mode);
1971 if(compresscheck=="catfile" or compresscheck==formatspecs[2]):
1972 try:
1973 filefp = open(infile, mode, encoding="UTF-8");
1974 except (ValueError, TypeError) as e:
1975 filefp = open(infile, mode);
1976 if(not compresscheck):
1977 try:
1978 filefp = open(infile, mode, encoding="UTF-8");
1979 except (ValueError, TypeError) as e:
1980 filefp = open(infile, mode);
1981 except FileNotFoundError:
1982 return False;
1983 return filefp;
1984
1985 def UncompressString(infile):
1986 compresscheck = CheckCompressionTypeFromString(infile, formatspecs, False);
1987 if(compresscheck=="gzip"):
1988 try:
1989 import gzip;
1990 except ImportError:
1991 return False;
1992 fileuz = gzip.decompress(infile);
1993 if(compresscheck=="bzip2"):
1994 try:
1995 import bz2;
1996 except ImportError:
1997 return False;
1998 fileuz = bz2.decompress(infile);
1999 if(compresscheck=="zstd"):
2000 try:
2001 import zstandard;
2002 except ImportError:
2003 return False;
2004 fileuz = zstandard.decompress(infile);
2005 if(compresscheck=="lz4"):
2006 try:
2007 import lz4.frame;
2008 except ImportError:
2009 return False;
2010 fileuz = lz4.frame.decompress(infile);
2011 if(compresscheck=="lzo"):
2012 try:
2013 import lzo;
2014 except ImportError:
2015 return False;
2016 fileuz = lzo.decompress(infile);
2017 if(compresscheck=="lzma"):
2018 try:
2019 import lzma;
2020 except ImportError:
2021 return False;
2022 fileuz = lzma.decompress(infile);
2023 if(not compresscheck):
2024 fileuz = infile;
2025 if(hasattr(fileuz, 'decode')):
2026 fileuz = fileuz.decode("UTF-8");
2027 return fileuz;
2028
2029 def UncompressStringAlt(infile):
2030 filefp = StringIO();
2031 outstring = UncompressString(infile);
2032 filefp.write(outstring);
2033 filefp.seek(0, 0);
2034 return filefp;
2035
2036 def CheckCompressionSubType(infile, formatspecs=__file_format_list__, closefp=True):
2037 compresscheck = CheckCompressionType(infile, formatspecs, False);
2038 if(not compresscheck):
2039 fextname = os.path.splitext(infile)[1];
2040 if(fextname==".gz"):
2041 compresscheck = "gzip";
2042 elif(fextname==".bz2"):
2043 compresscheck = "bzip2";
2044 elif(fextname==".zst"):
2045 compresscheck = "zstd";
2046 elif(fextname==".lz4"):
2047 compresscheck = "lz4";
2048 elif(fextname==".lzo" or fextname==".lzop"):
2049 compresscheck = "lzo";
2050 elif(fextname==".lzma" or fextname==".xz"):
2051 compresscheck = "lzma";
2052 else:
2053 return False;
2054 if(compresscheck=="gzip" or compresscheck=="bzip2" or compresscheck=="lzma" or compresscheck=="zstd" or compresscheck=="lz4"):
2055 if(is_tarfile(infile)):
2056 filetype = "tarfile";
2057 if(not compresscheck):
2058 if(is_tarfile(infile)):
2059 return "tarfile";
2060 elif(zipfile.is_zipfile(infile)):
2061 return "zipfile";
2062 elif(rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile)):
2063 return "rarile";
2064 else:
2065 return False;
2066 return False;
2067 if(compresscheck=="catfile"):
2068 return "catfile";
2069 if(compresscheck==formatspecs[2]):
2070 return formatspecs[2];
2071 if(compresscheck=="tarfile"):
2072 return "tarfile";
2073 if(compresscheck=="zipfile"):
2074 return "zipfile";
2075 if(rarfile_support and compresscheck=="rarfile"):
2076 return "rarfile";
2077 if(py7zr_support and compresscheck=="7zipfile"):
2078 return "7zipfile";
2079 if(hasattr(infile, "read") or hasattr(infile, "write")):
2080 catfp = UncompressArchiveFile(infile, formatspecs[2]);
2081 else:
2082 try:
2083 if(compresscheck=="gzip"):
2084 try:
2085 import gzip;
2086 except ImportError:
2087 return False;
2088 catfp = gzip.GzipFile(infile, "rb");
2089 if(compresscheck=="bzip2"):
2090 try:
2091 import bz2;
2092 except ImportError:
2093 return False;
2094 catfp = bz2.BZ2File(infile, "rb");
2095 if(compresscheck=="lz4"):
2096 try:
2097 import lz4.frame;
2098 except ImportError:
2099 return False;
2100 catfp = lz4.frame.open(infile, "rb");
2101 if(compresscheck=="zstd"):
2102 try:
2103 import zstandard;
2104 except ImportError:
2105 return False;
2106 catfp = zstandard.open(infile, "rb");
2107 if(compresscheck=="lzma" or compresscheck=="xz"):
2108 try:
2109 import lzma;
2110 except ImportError:
2111 return False;
2112 catfp = lzma.open(infile, "rb");
2113 except FileNotFoundError:
2114 return False;
2115 filetype = False;
2116 prefp = catfp.read(5);
2117 if(prefp==binascii.unhexlify("7573746172")):
2118 filetype = "tarfile";
2119 catfp.seek(0, 0);
2120 prefp = catfp.read(7);
2121 if(prefp==binascii.unhexlify("43617446696c65")):
2122 filetype = "catfile";
2123 catfp.seek(0, 0);
2124 prefp = catfp.read(formatspecs[3]);
2125 if(prefp==binascii.unhexlify(formatspecs[4])):
2126 filetype = formatspecs[2];
2127 catfp.seek(0, 0);
2128 prefp = catfp.read(10);
2129 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
2130 filetype = "tarfile";
2131 catfp.seek(0, 0);
2132 if(closefp):
2133 catfp.close();
2134 return filetype;
2135
2136 def GZipCompress(data, compresslevel=9):
2137 try:
2138 import gzip;
2139 except ImportError:
2140 return False;
2141 tmpfp = tempfile.NamedTemporaryFile("wb", delete=False);
2142 tmpfp.close();
2143 tmpfp = gzip.GzipFile(tmpfp.name, mode="wb", compresslevel=compresslevel);
2144 tmpfp.write(data);
2145 tmpfp.close();
2146 try:
2147 catfp = open(tmpfp.name, "rb");
2148 except FileNotFoundError:
2149 return False;
2150 catdata = catfp.read();
2151 catfp.close();
2152 return catdata;
2153
2154 def CompressArchiveFile(fp, compression="auto", compressionlevel=None, formatspecs=__file_format_list__):
2155 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
2156 return False;
2157 fp.seek(0, 0);
2158 if(not compression or compression=="catfile" or compression==formatspecs[2]):
2159 compression = "auto";
2160 if(compression not in compressionlist and compression is None):
2161 compression = "auto";
2162 if(compression=="gzip"):
2163 try:
2164 import gzip;
2165 except ImportError:
2166 return False;
2167 catfp = BytesIO();
2168 if(compressionlevel is None):
2169 compressionlevel = 9;
2170 else:
2171 compressionlevel = int(compressionlevel);
2172 catfp.write(gzip.compress(fp.read(), compresslevel=compressionlevel));
2173 if(compression=="bzip2"):
2174 try:
2175 import bz2;
2176 except ImportError:
2177 return False;
2178 catfp = BytesIO();
2179 if(compressionlevel is None):
2180 compressionlevel = 9;
2181 else:
2182 compressionlevel = int(compressionlevel);
2183 catfp.write(bz2.compress(fp.read(), compresslevel=compressionlevel));
2184 if(compression=="lz4"):
2185 try:
2186 import lz4.frame;
2187 except ImportError:
2188 return False;
2189 catfp = BytesIO();
2190 if(compressionlevel is None):
2191 compressionlevel = 9;
2192 else:
2193 compressionlevel = int(compressionlevel);
2194 catfp.write(lz4.frame.compress(fp.read(), compression_level=compressionlevel));
2195 if(compression=="lzo" or compression=="lzop"):
2196 try:
2197 import lzo;
2198 except ImportError:
2199 return False;
2200 catfp = BytesIO();
2201 if(compressionlevel is None):
2202 compressionlevel = 9;
2203 else:
2204 compressionlevel = int(compressionlevel);
2205 catfp.write(lzo.compress(fp.read(), compresslevel=compressionlevel));
2206 if(compression=="zstd"):
2207 try:
2208 import zstandard;
2209 except ImportError:
2210 return False;
2211 catfp = BytesIO();
2212 if(compressionlevel is None):
2213 compressionlevel = 10;
2214 else:
2215 compressionlevel = int(compressionlevel);
2216 catfp.write(zstandard.compress(fp.read(), level=compressionlevel));
2217 if(compression=="lzma"):
2218 try:
2219 import lzma;
2220 except ImportError:
2221 return False;
2222 catfp = BytesIO();
2223 if(compressionlevel is None):
2224 compressionlevel = 9;
2225 else:
2226 compressionlevel = int(compressionlevel);
2227 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]));
2228 if(compression=="xz"):
2229 try:
2230 import lzma;
2231 except ImportError:
2232 return False;
2233 catfp = BytesIO();
2234 if(compressionlevel is None):
2235 compressionlevel = 9;
2236 else:
2237 compressionlevel = int(compressionlevel);
2238 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]));
2239 if(compression=="auto" or compression is None):
2240 catfp = fp;
2241 catfp.seek(0, 0);
2242 return catfp;
2243
2244 create_alias_function("Compress", __file_format_name__, "", CompressArchiveFile);
2245
2246 def CompressOpenFile(outfile, compressionenable=True, compressionlevel=None):
2247 if(outfile is None):
2248 return False;
2249 fbasename = os.path.splitext(outfile)[0];
2250 fextname = os.path.splitext(outfile)[1];
2251 if(compressionlevel is None and fextname!=".zst"):
2252 compressionlevel = 9;
2253 elif(compressionlevel is None and fextname==".zst"):
2254 compressionlevel = 10;
2255 else:
2256 compressionlevel = int(compressionlevel);
2257 if(sys.version_info[0]==2):
2258 mode = "w";
2259 else:
2260 mode = "wb";
2261 try:
2262 if(fextname not in outextlistwd or not compressionenable):
2263 try:
2264 outfp = open(outfile, "wb", encoding="UTF-8");
2265 except (ValueError, TypeError) as e:
2266 outfp = open(outfile, "wb");
2267 elif(fextname==".gz"):
2268 try:
2269 import gzip;
2270 except ImportError:
2271 return False;
2272 try:
2273 outfp = gzip.open(outfile, mode, compressionlevel, encoding="UTF-8");
2274 except (ValueError, TypeError) as e:
2275 outfp = gzip.open(outfile, mode, compressionlevel);
2276 elif(fextname==".bz2"):
2277 try:
2278 import bz2;
2279 except ImportError:
2280 return False;
2281 try:
2282 outfp = bz2.open(outfile, mode, compressionlevel, encoding="UTF-8");
2283 except (ValueError, TypeError) as e:
2284 outfp = bz2.open(outfile, mode, compressionlevel);
2285 elif(fextname==".zst"):
2286 try:
2287 import zstandard;
2288 except ImportError:
2289 return False;
2290 try:
2291 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel), encoding="UTF-8");
2292 except (ValueError, TypeError) as e:
2293 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel));
2294 elif(fextname==".xz"):
2295 try:
2296 import lzma;
2297 except ImportError:
2298 return False;
2299 try:
2300 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}], encoding="UTF-8");
2301 except (ValueError, TypeError) as e:
2302 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]);
2303 elif(fextname==".lz4"):
2304 try:
2305 import lz4.frame;
2306 except ImportError:
2307 return False;
2308 try:
2309 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel, encoding="UTF-8");
2310 except (ValueError, TypeError) as e:
2311 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel);
2312 elif(fextname==".lzo"):
2313 try:
2314 import lzo;
2315 except ImportError:
2316 return False;
2317 try:
2318 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel, encoding="UTF-8");
2319 except (ValueError, TypeError) as e:
2320 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel);
2321 elif(fextname==".lzma"):
2322 try:
2323 import lzma;
2324 except ImportError:
2325 return False;
2326 try:
2327 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}], encoding="UTF-8");
2328 except (ValueError, TypeError) as e:
2329 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]);
2330 except FileNotFoundError:
2331 return False;
2332 return outfp;
2333
2334 def GetDevMajorMinor(fdev):
2335 retdev = [];
2336 if(hasattr(os, "minor")):
2337 retdev.append(os.minor(fdev));
2338 else:
2339 retdev.append(0);
2340 if(hasattr(os, "major")):
2341 retdev.append(os.major(fdev));
2342 else:
2343 retdev.append(0);
2344 return retdev;
2345
2346 def CheckSumSupport(checkfor, guaranteed=True):
2347 if(guaranteed):
2348 hash_list = sorted(list(hashlib.algorithms_guaranteed));
2349 else:
2350 hash_list = sorted(list(hashlib.algorithms_available));
2351 checklistout = sorted(hash_list + ['adler32', 'crc16', 'crc16_ansi', 'crc16_ibm', 'crc16_ccitt', 'crc32', 'crc64', 'crc64_ecma', 'crc64_iso', 'none']);
2352 if(checkfor in checklistout):
2353 return True;
2354 else:
2355 return False;
2356
2357 def CheckSumSupportAlt(checkfor, guaranteed=True):
2358 if(guaranteed):
2359 hash_list = sorted(list(hashlib.algorithms_guaranteed));
2360 else:
2361 hash_list = sorted(list(hashlib.algorithms_available));
2362 checklistout = hash_list;
2363 if(checkfor in checklistout):
2364 return True;
2365 else:
2366 return False;
2367
2368 def PackArchiveFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
2369 advancedlist = formatspecs[8];
2370 altinode = formatspecs[9];
2371 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2372 outfile = RemoveWindowsPath(outfile);
2373 checksumtype = checksumtype.lower();
2374 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
2375 checksumtype="crc32";
2376 if(checksumtype=="none"):
2377 checksumtype = "";
2378 if(not compression or compression=="catfile" or compression==formatspecs[2]):
2379 compression = "auto";
2380 if(compression not in compressionlist and compression is None):
2381 compression = "auto";
2382 if(verbose):
2383 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2384 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2385 if(os.path.exists(outfile)):
2386 try:
2387 os.unlink(outfile);
2388 except OSError as e:
2389 pass;
2390 if(outfile=="-"):
2391 verbose = False;
2392 catfp = BytesIO();
2393 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2394 catfp = outfile;
2395 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2396 catfp = BytesIO();
2397 else:
2398 fbasename = os.path.splitext(outfile)[0];
2399 fextname = os.path.splitext(outfile)[1];
2400 catfp = CompressOpenFile(outfile, True, compressionlevel);
2401 catver = formatspecs[6];
2402 fileheaderver = str(int(catver.replace(".", "")));
2403 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
2404 catfp.write(fileheader.encode('UTF-8'));
2405 infilelist = [];
2406 if(infiles=="-"):
2407 for line in sys.stdin:
2408 infilelist.append(line.strip());
2409 infilelist = list(filter(None, infilelist));
2410 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
2411 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
2412 return False;
2413 with UncompressFile(infiles, formatspecs, "r") as finfile:
2414 for line in finfile:
2415 infilelist.append(line.strip());
2416 infilelist = list(filter(None, infilelist));
2417 else:
2418 if(isinstance(infiles, (list, tuple, ))):
2419 infilelist = list(filter(None, infiles));
2420 elif(isinstance(infiles, (str, ))):
2421 infilelist = list(filter(None, [infiles]));
2422 if(advancedlist):
2423 GetDirList = ListDirAdvanced(infilelist, followlink, False);
2424 else:
2425 GetDirList = ListDir(infilelist, followlink, False);
2426 if(not GetDirList):
2427 return False;
2428 curinode = 0;
2429 curfid = 0;
2430 inodelist = [];
2431 inodetofile = {};
2432 filetoinode = {};
2433 inodetocatinode = {};
2434 numfiles = int(len(GetDirList));
2435 fnumfiles = format(int(len(GetDirList)), 'x').lower();
2436 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs[5]);
2437 catfileheadercshex = GetFileChecksum(fileheader + fnumfilesa, checksumtype, True, formatspecs);
2438 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs[5]);
2439 catfp.write(fnumfilesa.encode('UTF-8'));
2440 try:
2441 catfp.flush();
2442 os.fsync(catfp.fileno());
2443 except io.UnsupportedOperation:
2444 pass;
2445 except AttributeError:
2446 pass;
2447 except OSError as e:
2448 pass;
2449 for curfname in GetDirList:
2450 if(re.findall("^[.|/]", curfname)):
2451 fname = curfname;
2452 else:
2453 fname = "./"+curfname;
2454 if(verbose):
2455 VerbosePrintOut(fname);
2456 if(not followlink or followlink is None):
2457 fstatinfo = os.lstat(fname);
2458 else:
2459 fstatinfo = os.stat(fname);
2460 fpremode = fstatinfo.st_mode;
2461 finode = fstatinfo.st_ino;
2462 flinkcount = fstatinfo.st_nlink;
2463 ftype = 0;
2464 if(stat.S_ISREG(fpremode)):
2465 ftype = 0;
2466 elif(stat.S_ISLNK(fpremode)):
2467 ftype = 2;
2468 elif(stat.S_ISCHR(fpremode)):
2469 ftype = 3;
2470 elif(stat.S_ISBLK(fpremode)):
2471 ftype = 4;
2472 elif(stat.S_ISDIR(fpremode)):
2473 ftype = 5;
2474 elif(stat.S_ISFIFO(fpremode)):
2475 ftype = 6;
2476 elif(stat.S_ISSOCK(fpremode)):
2477 ftype = 8;
2478 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
2479 ftype = 9;
2480 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
2481 ftype = 10;
2482 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
2483 ftype = 111;
2484 else:
2485 ftype = 0;
2486 flinkname = "";
2487 fcurfid = format(int(curfid), 'x').lower();
2488 if(not followlink and finode!=0):
2489 if(ftype!=1):
2490 if(finode in inodelist):
2491 ftype = 1;
2492 flinkname = inodetofile[finode];
2493 if(altinode):
2494 fcurinode = format(int(finode), 'x').lower();
2495 else:
2496 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
2497 if(finode not in inodelist):
2498 inodelist.append(finode);
2499 inodetofile.update({finode: fname});
2500 inodetocatinode.update({finode: curinode});
2501 if(altinode):
2502 fcurinode = format(int(finode), 'x').lower();
2503 else:
2504 fcurinode = format(int(curinode), 'x').lower();
2505 curinode = curinode + 1;
2506 else:
2507 fcurinode = format(int(curinode), 'x').lower();
2508 curinode = curinode + 1;
2509 curfid = curfid + 1;
2510 if(ftype==2):
2511 flinkname = os.readlink(fname);
2512 fdev = fstatinfo.st_dev;
2513 getfdev = GetDevMajorMinor(fdev);
2514 fdev_minor = getfdev[0];
2515 fdev_major = getfdev[1];
2516 frdev = fstatinfo.st_dev;
2517 if(hasattr(fstatinfo, "st_rdev")):
2518 frdev = fstatinfo.st_rdev;
2519 else:
2520 frdev = fstatinfo.st_dev;
2521 getfrdev = GetDevMajorMinor(frdev);
2522 frdev_minor = getfrdev[0];
2523 frdev_major = getfrdev[1];
2524 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
2525 fsize = format(int("0"), 'x').lower();
2526 elif(ftype==0 or ftype==7):
2527 fsize = format(int(fstatinfo.st_size), 'x').lower();
2528 else:
2529 fsize = format(int(fstatinfo.st_size)).lower();
2530 fatime = format(int(fstatinfo.st_atime), 'x').lower();
2531 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
2532 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
2533 if(hasattr(fstatinfo, "st_birthtime")):
2534 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
2535 else:
2536 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
2537 fmode = format(int(fstatinfo.st_mode), 'x').lower();
2538 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
2539 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
2540 fuid = format(int(fstatinfo.st_uid), 'x').lower();
2541 fgid = format(int(fstatinfo.st_gid), 'x').lower();
2542 funame = "";
2543 try:
2544 import pwd;
2545 try:
2546 userinfo = pwd.getpwuid(fstatinfo.st_uid);
2547 funame = userinfo.pw_name;
2548 except KeyError:
2549 funame = "";
2550 except ImportError:
2551 funame = "";
2552 fgname = "";
2553 try:
2554 import grp;
2555 try:
2556 groupinfo = grp.getgrgid(fstatinfo.st_gid);
2557 fgname = groupinfo.gr_name;
2558 except KeyError:
2559 fgname = "";
2560 except ImportError:
2561 fgname = "";
2562 fdev_minor = format(int(fdev_minor), 'x').lower();
2563 fdev_major = format(int(fdev_major), 'x').lower();
2564 frdev_minor = format(int(frdev_minor), 'x').lower();
2565 frdev_major = format(int(frdev_major), 'x').lower();
2566 finode = format(int(finode), 'x').lower();
2567 flinkcount = format(int(flinkcount), 'x').lower();
2568 if(hasattr(fstatinfo, "st_file_attributes")):
2569 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
2570 else:
2571 fwinattributes = format(int(0), 'x').lower();
2572 fcontents = BytesIO();
2573 if(ftype==0 or ftype==7):
2574 with open(fname, "rb") as fpc:
2575 shutil.copyfileobj(fpc, fcontents);
2576 if(followlink and (ftype==1 or ftype==2)):
2577 flstatinfo = os.stat(flinkname);
2578 with open(flinkname, "rb") as fpc:
2579 shutil.copyfileobj(fpc, fcontents);
2580 fcontents.seek(0, 0);
2581 ftypehex = format(ftype, 'x').lower();
2582 extrafields = format(len(extradata), 'x').lower();
2583 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
2584 if(len(extradata)>0):
2585 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
2586 extrasizelen = format(len(extrasizestr), 'x').lower();
2587 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, extrasizelen, extrafields];
2588 catoutlen = len(catoutlist) + len(extradata) + 3;
2589 catoutlenhex = format(catoutlen, 'x').lower();
2590 catoutlist.insert(0, catoutlenhex);
2591 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
2592 if(len(extradata)>0):
2593 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
2594 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
2595 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
2596 fcontents.seek(0, 0);
2597 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
2598 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
2599 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
2600 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
2601 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
2602 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
2603 catfileoutstrecd = catfileoutstr.encode('UTF-8');
2604 nullstrecd = formatspecs[5].encode('UTF-8');
2605 fcontents.seek(0, 0);
2606 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
2607 catfp.write(catfileout);
2608 try:
2609 catfp.flush();
2610 os.fsync(catfp.fileno());
2611 except io.UnsupportedOperation:
2612 pass;
2613 except AttributeError:
2614 pass;
2615 except OSError as e:
2616 pass;
2617 if(numfiles>0):
2618 catfp.write(AppendNullBytes([0, 0], formatspecs[5]).encode("UTF-8"));
2619 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
2620 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2621 try:
2622 catfp.flush();
2623 os.fsync(catfp.fileno());
2624 except io.UnsupportedOperation:
2625 pass;
2626 except AttributeError:
2627 pass;
2628 except OSError as e:
2629 pass;
2630 if(outfile=="-"):
2631 catfp.seek(0, 0);
2632 if(hasattr(sys.stdout, "buffer")):
2633 shutil.copyfileobj(catfp, sys.stdout.buffer);
2634 else:
2635 shutil.copyfileobj(catfp, sys.stdout);
2636 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2637 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2638 catfp.seek(0, 0);
2639 upload_file_to_internet_file(catfp, outfile);
2640 if(returnfp):
2641 catfp.seek(0, 0);
2642 return catfp;
2643 else:
2644 catfp.close();
2645 return True;
2646
2647 create_alias_function("Pack", __file_format_name__, "", PackArchiveFile);
2648
2649 if(hasattr(shutil, "register_archive_format")):
2650 def PackArchiveFileFunc(archive_name, source_dir, **kwargs):
2651 return PackArchiveFile(source_dir, archive_name, False, "auto", None, False, "crc32", [], __file_format_delimiter__, False, False);
2652 create_alias_function("Pack", __file_format_name__, "Func", PackArchiveFileFunc);
2653
2654 def PackArchiveFileFromDirList(infiles, outfile, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
2655 return PackArchiveFile(infiles, outfile, dirlistfromtxt, compression, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, returnfp);
2656
2657 create_alias_function("Pack", __file_format_name__, "FromDirList", PackArchiveFileFromDirList);
2658
2659 def PackArchiveFileFromTarFile(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
2660 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2661 outfile = RemoveWindowsPath(outfile);
2662 checksumtype = checksumtype.lower();
2663 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
2664 checksumtype="crc32";
2665 if(checksumtype=="none"):
2666 checksumtype = "";
2667 if(not compression or compression=="catfile" or compression==formatspecs[2]):
2668 compression = "auto";
2669 if(compression not in compressionlist and compression is None):
2670 compression = "auto";
2671 if(verbose):
2672 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2673 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2674 if(os.path.exists(outfile)):
2675 try:
2676 os.unlink(outfile);
2677 except OSError as e:
2678 pass;
2679 if(outfile=="-"):
2680 verbose = False;
2681 catfp = BytesIO();
2682 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2683 catfp = outfile;
2684 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2685 catfp = BytesIO();
2686 else:
2687 fbasename = os.path.splitext(outfile)[0];
2688 fextname = os.path.splitext(outfile)[1];
2689 catfp = CompressOpenFile(outfile, True, compressionlevel);
2690 catver = formatspecs[6];
2691 fileheaderver = str(int(catver.replace(".", "")));
2692 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
2693 catfp.write(fileheader.encode('UTF-8'));
2694 curinode = 0;
2695 curfid = 0;
2696 inodelist = [];
2697 inodetofile = {};
2698 filetoinode = {};
2699 inodetocatinode = {};
2700 if(infile=="-"):
2701 infile = BytesIO();
2702 if(hasattr(sys.stdin, "buffer")):
2703 shutil.copyfileobj(sys.stdin.buffer, infile);
2704 else:
2705 shutil.copyfileobj(sys.stdin, infile);
2706 infile.seek(0, 0);
2707 if(not infile):
2708 return False;
2709 infile.seek(0, 0);
2710 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
2711 infile = download_file_from_internet_file(infile);
2712 infile.seek(0, 0);
2713 if(not infile):
2714 return False;
2715 infile.seek(0, 0);
2716 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
2717 return False;
2718 elif(os.path.exists(infile) and os.path.isfile(infile)):
2719 try:
2720 if(not tarfile.is_tarfile(infile)):
2721 return False;
2722 except AttributeError:
2723 if(not is_tarfile(infile)):
2724 return False;
2725 else:
2726 return False;
2727 try:
2728 if(hasattr(infile, "read") or hasattr(infile, "write")):
2729 tarfp = tarfile.open(fileobj=infile, mode="r");
2730 else:
2731 tarfp = tarfile.open(infile, "r");
2732 except FileNotFoundError:
2733 return False;
2734 numfiles = int(len(tarfp.getmembers()));
2735 fnumfiles = format(int(len(tarfp.getmembers())), 'x').lower();
2736 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs[5]);
2737 catfileheadercshex = GetFileChecksum(fileheader + fnumfilesa, checksumtype, True, formatspecs);
2738 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs[5]);
2739 catfp.write(fnumfilesa.encode('UTF-8'));
2740 try:
2741 catfp.flush();
2742 os.fsync(catfp.fileno());
2743 except io.UnsupportedOperation:
2744 pass;
2745 except AttributeError:
2746 pass;
2747 except OSError as e:
2748 pass;
2749 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
2750 if(re.findall("^[.|/]", member.name)):
2751 fname = member.name;
2752 else:
2753 fname = "./"+member.name;
2754 if(verbose):
2755 VerbosePrintOut(fname);
2756 fpremode = member.mode;
2757 ffullmode = member.mode;
2758 flinkcount = 0;
2759 ftype = 0;
2760 if(member.isreg()):
2761 ffullmode = member.mode + stat.S_IFREG;
2762 ftype = 0;
2763 elif(member.isdev()):
2764 ffullmode = member.mode;
2765 ftype = 7;
2766 elif(member.islnk()):
2767 ffullmode = member.mode + stat.S_IFREG;
2768 ftype = 1;
2769 elif(member.issym()):
2770 ffullmode = member.mode + stat.S_IFLNK;
2771 ftype = 2;
2772 elif(member.ischr()):
2773 ffullmode = member.mode + stat.S_IFCHR;
2774 ftype = 3;
2775 elif(member.isblk()):
2776 ffullmode = member.mode + stat.S_IFBLK;
2777 ftype = 4;
2778 elif(member.isdir()):
2779 ffullmode = member.mode + stat.S_IFDIR;
2780 ftype = 5;
2781 elif(member.isfifo()):
2782 ffullmode = member.mode + stat.S_IFIFO;
2783 ftype = 6;
2784 elif(member.issparse()):
2785 ffullmode = member.mode;
2786 ftype = 12;
2787 else:
2788 ffullmode = member.mode;
2789 ftype = 0;
2790 flinkname = "";
2791 fcurfid = format(int(curfid), 'x').lower();
2792 fcurinode = format(int(curfid), 'x').lower();
2793 curfid = curfid + 1;
2794 if(ftype==2):
2795 flinkname = member.linkname;
2796 fdev_minor = format(int(member.devminor), 'x').lower();
2797 fdev_major = format(int(member.devmajor), 'x').lower();
2798 frdev_minor = format(int(member.devminor), 'x').lower();
2799 frdev_major = format(int(member.devmajor), 'x').lower();
2800 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
2801 fsize = format(int("0"), 'x').lower();
2802 elif(ftype==0 or ftype==7):
2803 fsize = format(int(member.size), 'x').lower();
2804 else:
2805 fsize = format(int(member.size), 'x').lower();
2806 fatime = format(int(member.mtime), 'x').lower();
2807 fmtime = format(int(member.mtime), 'x').lower();
2808 fctime = format(int(member.mtime), 'x').lower();
2809 fbtime = format(int(member.mtime), 'x').lower();
2810 fmode = format(int(ffullmode), 'x').lower();
2811 fchmode = format(int(stat.S_IMODE(ffullmode)), 'x').lower();
2812 ftypemod = format(int(stat.S_IFMT(ffullmode)), 'x').lower();
2813 fuid = format(int(member.uid), 'x').lower();
2814 fgid = format(int(member.gid), 'x').lower();
2815 funame = member.uname;
2816 fgname = member.gname;
2817 flinkcount = format(int(flinkcount), 'x').lower();
2818 fwinattributes = format(int(0), 'x').lower();
2819 fcontents = BytesIO();
2820 if(ftype==0 or ftype==7):
2821 with tarfp.extractfile(member) as fpc:
2822 shutil.copyfileobj(fpc, fcontents);
2823 fcontents.seek(0, 0);
2824 ftypehex = format(ftype, 'x').lower();
2825 extrafields = format(len(extradata), 'x').lower();
2826 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
2827 if(len(extradata)>0):
2828 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
2829 extrasizelen = format(len(extrasizestr), 'x').lower();
2830 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, extrasizelen, extrafields];
2831 catoutlen = len(catoutlist) + len(extradata) + 3;
2832 catoutlenhex = format(catoutlen, 'x').lower();
2833 catoutlist.insert(0, catoutlenhex);
2834 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
2835 if(len(extradata)>0):
2836 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
2837 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
2838 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
2839 fcontents.seek(0, 0);
2840 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
2841 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
2842 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
2843 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
2844 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
2845 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
2846 catfileoutstrecd = catfileoutstr.encode('UTF-8');
2847 nullstrecd = formatspecs[5].encode('UTF-8');
2848 fcontents.seek(0, 0);
2849 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
2850 catfp.write(catfileout);
2851 try:
2852 catfp.flush();
2853 os.fsync(catfp.fileno());
2854 except io.UnsupportedOperation:
2855 pass;
2856 except AttributeError:
2857 pass;
2858 except OSError as e:
2859 pass;
2860 if(numfiles>0):
2861 catfp.write(AppendNullBytes([0, 0], formatspecs[5]).encode("UTF-8"));
2862 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
2863 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2864 try:
2865 catfp.flush();
2866 os.fsync(catfp.fileno());
2867 except io.UnsupportedOperation:
2868 pass;
2869 except AttributeError:
2870 pass;
2871 except OSError as e:
2872 pass;
2873 if(outfile=="-"):
2874 catfp.seek(0, 0);
2875 if(hasattr(sys.stdout, "buffer")):
2876 shutil.copyfileobj(catfp, sys.stdout.buffer);
2877 else:
2878 shutil.copyfileobj(catfp, sys.stdout);
2879 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2880 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2881 catfp.seek(0, 0);
2882 upload_file_to_internet_file(catfp, outfile);
2883 if(returnfp):
2884 catfp.seek(0, 0);
2885 return catfp;
2886 else:
2887 catfp.close();
2888 return True;
2889
2890 create_alias_function("Pack", __file_format_name__, "FromTarFile", PackArchiveFileFromTarFile);
2891
2892 def PackArchiveFileFromZipFile(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
2893 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2894 outfile = RemoveWindowsPath(outfile);
2895 checksumtype = checksumtype.lower();
2896 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
2897 checksumtype="crc32";
2898 if(checksumtype=="none"):
2899 checksumtype = "";
2900 if(not compression or compression=="catfile" or compression==formatspecs[2]):
2901 compression = "auto";
2902 if(compression not in compressionlist and compression is None):
2903 compression = "auto";
2904 if(verbose):
2905 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2906 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2907 if(os.path.exists(outfile)):
2908 try:
2909 os.unlink(outfile);
2910 except OSError as e:
2911 pass;
2912 if(outfile=="-"):
2913 verbose = False;
2914 catfp = BytesIO();
2915 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2916 catfp = outfile;
2917 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2918 catfp = BytesIO();
2919 else:
2920 fbasename = os.path.splitext(outfile)[0];
2921 fextname = os.path.splitext(outfile)[1];
2922 catfp = CompressOpenFile(outfile, True, compressionlevel);
2923 catver = formatspecs[6];
2924 fileheaderver = str(int(catver.replace(".", "")));
2925 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
2926 catfp.write(fileheader.encode('UTF-8'));
2927 curinode = 0;
2928 curfid = 0;
2929 inodelist = [];
2930 inodetofile = {};
2931 filetoinode = {};
2932 inodetocatinode = {};
2933 if(infile=="-"):
2934 infile = BytesIO();
2935 if(hasattr(sys.stdin, "buffer")):
2936 shutil.copyfileobj(sys.stdin.buffer, infile);
2937 else:
2938 shutil.copyfileobj(sys.stdin, infile);
2939 infile.seek(0, 0);
2940 if(not infile):
2941 return False;
2942 infile.seek(0, 0);
2943 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
2944 infile = download_file_from_internet_file(infile);
2945 infile.seek(0, 0);
2946 if(not infile):
2947 return False;
2948 infile.seek(0, 0);
2949 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
2950 return False;
2951 else:
2952 return False;
2953 if(not zipfile.is_zipfile(infile)):
2954 return False;
2955 try:
2956 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
2957 except FileNotFoundError:
2958 return False;
2959 ziptest = zipfp.testzip();
2960 if(ziptest):
2961 VerbosePrintOut("Bad file found!");
2962 numfiles = int(len(zipfp.infolist()));
2963 fnumfiles = format(int(len(zipfp.infolist())), 'x').lower();
2964 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs[5]);
2965 catfileheadercshex = GetFileChecksum(fileheader + fnumfilesa, checksumtype, True, formatspecs);
2966 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs[5]);
2967 catfp.write(fnumfilesa.encode('UTF-8'));
2968 try:
2969 catfp.flush();
2970 os.fsync(catfp.fileno());
2971 except io.UnsupportedOperation:
2972 pass;
2973 except AttributeError:
2974 pass;
2975 except OSError as e:
2976 pass;
2977 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
2978 if(re.findall("^[.|/]", member.filename)):
2979 fname = member.filename;
2980 else:
2981 fname = "./"+member.filename;
2982 zipinfo = zipfp.getinfo(member.filename);
2983 if(verbose):
2984 VerbosePrintOut(fname);
2985 if(not member.is_dir()):
2986 fpremode = int(stat.S_IFREG + 438);
2987 elif(member.is_dir()):
2988 fpremode = int(stat.S_IFDIR + 511);
2989 flinkcount = 0;
2990 ftype = 0;
2991 if(not member.is_dir()):
2992 ftype = 0;
2993 elif(member.is_dir()):
2994 ftype = 5;
2995 flinkname = "";
2996 fcurfid = format(int(curfid), 'x').lower();
2997 fcurinode = format(int(curfid), 'x').lower();
2998 curfid = curfid + 1;
2999 fdev_minor = format(int(0), 'x').lower();
3000 fdev_major = format(int(0), 'x').lower();
3001 frdev_minor = format(int(0), 'x').lower();
3002 frdev_major = format(int(0), 'x').lower();
3003 if(ftype==5):
3004 fsize = format(int("0"), 'x').lower();
3005 elif(ftype==0):
3006 fsize = format(int(member.file_size), 'x').lower();
3007 else:
3008 fsize = format(int(member.file_size), 'x').lower();
3009 fatime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3010 fmtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3011 fctime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3012 fbtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3013 if(zipinfo.create_system==0 or zipinfo.create_system==10):
3014 fwinattributes = format(int(zipinfo.external_attr), 'x').lower();
3015 if(not member.is_dir()):
3016 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3017 fchmode = stat.S_IMODE(fmode);
3018 ftypemod = stat.S_IFMT(fmode);
3019 elif(member.is_dir()):
3020 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3021 fchmode = stat.S_IMODE(fmode);
3022 ftypemod = stat.S_IFMT(fmode);
3023 elif(zipinfo.create_system==3):
3024 fwinattributes = format(int(0), 'x').lower();
3025 fmode = format(int(zipinfo.external_attr), 'x').lower();
3026 fchmode = stat.S_IMODE(fmode);
3027 ftypemod = stat.S_IFMT(fmode);
3028 else:
3029 fwinattributes = format(int(0), 'x').lower();
3030 if(not member.is_dir()):
3031 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3032 fchmode = stat.S_IMODE(fmode);
3033 ftypemod = stat.S_IFMT(fmode);
3034 elif(member.is_dir()):
3035 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3036 fchmode = stat.S_IMODE(fmode);
3037 ftypemod = stat.S_IFMT(fmode);
3038 try:
3039 fuid = format(int(os.getuid()), 'x').lower();
3040 except AttributeError:
3041 fuid = format(int(0), 'x').lower();
3042 except KeyError:
3043 fuid = format(int(0), 'x').lower();
3044 try:
3045 fgid = format(int(os.getgid()), 'x').lower();
3046 except AttributeError:
3047 fgid = format(int(0), 'x').lower();
3048 except KeyError:
3049 fgid = format(int(0), 'x').lower();
3050 try:
3051 import pwd;
3052 try:
3053 userinfo = pwd.getpwuid(os.getuid());
3054 funame = userinfo.pw_name;
3055 except KeyError:
3056 funame = "";
3057 except AttributeError:
3058 funame = "";
3059 except ImportError:
3060 funame = "";
3061 fgname = "";
3062 try:
3063 import grp;
3064 try:
3065 groupinfo = grp.getgrgid(os.getgid());
3066 fgname = groupinfo.gr_name;
3067 except KeyError:
3068 fgname = "";
3069 except AttributeError:
3070 fgname = "";
3071 except ImportError:
3072 fgname = "";
3073 fcontents = BytesIO();
3074 if(ftype==0):
3075 fcontents.write(zipfp.read(member.filename));
3076 fcontents.seek(0, 0);
3077 ftypehex = format(ftype, 'x').lower();
3078 extrafields = format(len(extradata), 'x').lower();
3079 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
3080 if(len(extradata)>0):
3081 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
3082 extrasizelen = format(len(extrasizestr), 'x').lower();
3083 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, extrasizelen, extrafields];
3084 catoutlen = len(catoutlist) + len(extradata) + 3;
3085 catoutlenhex = format(catoutlen, 'x').lower();
3086 catoutlist.insert(0, catoutlenhex);
3087 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
3088 if(len(extradata)>0):
3089 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
3090 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
3091 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
3092 fcontents.seek(0, 0);
3093 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
3094 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
3095 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
3096 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
3097 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
3098 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
3099 catfileoutstrecd = catfileoutstr.encode('UTF-8');
3100 nullstrecd = formatspecs[5].encode('UTF-8');
3101 fcontents.seek(0, 0);
3102 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
3103 catfp.write(catfileout);
3104 try:
3105 catfp.flush();
3106 os.fsync(catfp.fileno());
3107 except io.UnsupportedOperation:
3108 pass;
3109 except AttributeError:
3110 pass;
3111 except OSError as e:
3112 pass;
3113 if(numfiles>0):
3114 catfp.write(AppendNullBytes([0, 0], formatspecs[5]).encode("UTF-8"));
3115 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3116 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3117 try:
3118 catfp.flush();
3119 os.fsync(catfp.fileno());
3120 except io.UnsupportedOperation:
3121 pass;
3122 except AttributeError:
3123 pass;
3124 except OSError as e:
3125 pass;
3126 if(outfile=="-"):
3127 catfp.seek(0, 0);
3128 if(hasattr(sys.stdout, "buffer")):
3129 shutil.copyfileobj(catfp, sys.stdout.buffer);
3130 else:
3131 shutil.copyfileobj(catfp, sys.stdout);
3132 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3133 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3134 catfp.seek(0, 0);
3135 upload_file_to_internet_file(catfp, outfile);
3136 if(returnfp):
3137 catfp.seek(0, 0);
3138 return catfp;
3139 else:
3140 catfp.close();
3141 return True;
3142
3143 create_alias_function("Pack", __file_format_name__, "FromZipFile", PackArchiveFileFromZipFile);
3144
3145 if(not rarfile_support):
3146 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
3147 return False
3148
3149 if(rarfile_support):
3150 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
3151 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3152 outfile = RemoveWindowsPath(outfile);
3153 checksumtype = checksumtype.lower();
3154 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3155 checksumtype="crc32";
3156 if(checksumtype=="none"):
3157 checksumtype = "";
3158 if(not compression or compression=="catfile" or compression==formatspecs[2]):
3159 compression = "auto";
3160 if(compression not in compressionlist and compression is None):
3161 compression = "auto";
3162 if(verbose):
3163 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3164 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3165 if(os.path.exists(outfile)):
3166 try:
3167 os.unlink(outfile);
3168 except OSError as e:
3169 pass;
3170 if(outfile=="-"):
3171 verbose = False;
3172 catfp = BytesIO();
3173 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3174 catfp = outfile;
3175 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3176 catfp = BytesIO();
3177 else:
3178 fbasename = os.path.splitext(outfile)[0];
3179 fextname = os.path.splitext(outfile)[1];
3180 catfp = CompressOpenFile(outfile, True, compressionlevel);
3181 catver = formatspecs[6];
3182 fileheaderver = str(int(catver.replace(".", "")));
3183 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
3184 catfp.write(fileheader.encode('UTF-8'));
3185 curinode = 0;
3186 curfid = 0;
3187 inodelist = [];
3188 inodetofile = {};
3189 filetoinode = {};
3190 inodetocatinode = {};
3191 if(not os.path.exists(infile) or not os.path.isfile(infile)):
3192 return False;
3193 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
3194 return False;
3195 rarfp = rarfile.RarFile(infile, "r");
3196 rartest = rarfp.testrar();
3197 if(rartest):
3198 VerbosePrintOut("Bad file found!");
3199 numfiles = int(len(rarfp.infolist()));
3200 fnumfiles = format(int(len(rarfp.infolist())), 'x').lower();
3201 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs[5]);
3202 catfileheadercshex = GetFileChecksum(fileheader + fnumfilesa, checksumtype, True, formatspecs);
3203 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs[5]);
3204 catfp.write(fnumfilesa.encode('UTF-8'));
3205 try:
3206 catfp.flush();
3207 os.fsync(catfp.fileno());
3208 except io.UnsupportedOperation:
3209 pass;
3210 except AttributeError:
3211 pass;
3212 except OSError as e:
3213 pass;
3214 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
3215 is_unix = False;
3216 is_windows = False;
3217 if(member.host_os==rarfile.RAR_OS_UNIX):
3218 is_windows = False;
3219 try:
3220 member.external_attr
3221 is_unix = True;
3222 except AttributeError:
3223 is_unix = False;
3224 elif(member.host_os==rarfile.RAR_OS_WIN32):
3225 is_unix = False;
3226 try:
3227 member.external_attr
3228 is_windows = True;
3229 except AttributeError:
3230 is_windows = False;
3231 else:
3232 is_unix = False;
3233 is_windows = False;
3234 if(re.findall("^[.|/]", member.filename)):
3235 fname = member.filename;
3236 else:
3237 fname = "./"+member.filename;
3238 rarinfo = rarfp.getinfo(member.filename);
3239 if(verbose):
3240 VerbosePrintOut(fname);
3241 if(is_unix and member.external_attr !=0):
3242 fpremode = int(member.external_attr);
3243 elif(member.is_file()):
3244 fpremode = int(stat.S_IFREG + 438);
3245 elif(member.is_symlink()):
3246 fpremode = int(stat.S_IFLNK + 438);
3247 elif(member.is_dir()):
3248 fpremode = int(stat.S_IFDIR + 511);
3249 if(is_windows and member.external_attr !=0):
3250 fwinattributes = format(int(member.external_attr), 'x').lower();
3251 else:
3252 fwinattributes = format(int(0), 'x').lower();
3253 flinkcount = 0;
3254 ftype = 0;
3255 if(member.is_file()):
3256 ftype = 0;
3257 elif(member.is_symlink()):
3258 ftype = 2;
3259 elif(member.is_dir()):
3260 ftype = 5;
3261 flinkname = "";
3262 if(ftype==2):
3263 flinkname = rarfp.read(member.filename).decode("UTF-8");
3264 fcurfid = format(int(curfid), 'x').lower();
3265 fcurinode = format(int(curfid), 'x').lower();
3266 curfid = curfid + 1;
3267 fdev_minor = format(int(0), 'x').lower();
3268 fdev_major = format(int(0), 'x').lower();
3269 frdev_minor = format(int(0), 'x').lower();
3270 frdev_major = format(int(0), 'x').lower();
3271 if(ftype==5):
3272 fsize = format(int("0"), 'x').lower();
3273 elif(ftype==0):
3274 fsize = format(int(member.file_size), 'x').lower();
3275 else:
3276 fsize = format(int(member.file_size), 'x').lower();
3277 try:
3278 if(member.atime):
3279 fatime = format(int(member.atime.timestamp()), 'x').lower();
3280 else:
3281 fatime = format(int(member.mtime.timestamp()), 'x').lower();
3282 except AttributeError:
3283 fatime = format(int(member.mtime.timestamp()), 'x').lower();
3284 fmtime = format(int(member.mtime.timestamp()), 'x').lower();
3285 try:
3286 if(member.ctime):
3287 fctime = format(int(member.ctime.timestamp()), 'x').lower();
3288 else:
3289 fctime = format(int(member.mtime.timestamp()), 'x').lower();
3290 except AttributeError:
3291 fctime = format(int(member.mtime.timestamp()), 'x').lower();
3292 fbtime = format(int(member.mtime.timestamp()), 'x').lower();
3293 if(is_unix and member.external_attr !=0):
3294 fmode = format(int(member.external_attr), 'x').lower();
3295 fchmode = format(int(stat.S_IMODE(member.external_attr)), 'x').lower();
3296 ftypemod = format(int(stat.S_IFMT(member.external_attr)), 'x').lower();
3297 elif(member.is_file()):
3298 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3299 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3300 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3301 elif(member.is_symlink()):
3302 fmode = format(int(stat.S_IFLNK + 438), 'x').lower();
3303 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3304 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3305 elif(member.is_dir()):
3306 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3307 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
3308 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
3309 try:
3310 fuid = format(int(os.getuid()), 'x').lower();
3311 except AttributeError:
3312 fuid = format(int(0), 'x').lower();
3313 except KeyError:
3314 fuid = format(int(0), 'x').lower();
3315 try:
3316 fgid = format(int(os.getgid()), 'x').lower();
3317 except AttributeError:
3318 fgid = format(int(0), 'x').lower();
3319 except KeyError:
3320 fgid = format(int(0), 'x').lower();
3321 try:
3322 import pwd;
3323 try:
3324 userinfo = pwd.getpwuid(os.getuid());
3325 funame = userinfo.pw_name;
3326 except KeyError:
3327 funame = "";
3328 except AttributeError:
3329 funame = "";
3330 except ImportError:
3331 funame = "";
3332 fgname = "";
3333 try:
3334 import grp;
3335 try:
3336 groupinfo = grp.getgrgid(os.getgid());
3337 fgname = groupinfo.gr_name;
3338 except KeyError:
3339 fgname = "";
3340 except AttributeError:
3341 fgname = "";
3342 except ImportError:
3343 fgname = "";
3344 fcontents = BytesIO();
3345 if(ftype==0):
3346 fcontents.write(rarfp.read(member.filename));
3347 fcontents.seek(0, 0);
3348 ftypehex = format(ftype, 'x').lower();
3349 extrafields = format(len(extradata), 'x').lower();
3350 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
3351 if(len(extradata)>0):
3352 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
3353 extrasizelen = format(len(extrasizestr), 'x').lower();
3354 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, extrasizelen, extrafields];
3355 catoutlen = len(catoutlist) + len(extradata) + 3;
3356 catoutlenhex = format(catoutlen, 'x').lower();
3357 catoutlist.insert(0, catoutlenhex);
3358 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
3359 if(len(extradata)>0):
3360 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
3361 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
3362 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
3363 fcontents.seek(0, 0);
3364 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
3365 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
3366 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
3367 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
3368 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
3369 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5])
3370 catfileoutstrecd = catfileoutstr.encode('UTF-8');
3371 nullstrecd = formatspecs[5].encode('UTF-8');
3372 fcontents.seek(0, 0);
3373 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
3374 catfp.write(catfileout);
3375 try:
3376 catfp.flush();
3377 os.fsync(catfp.fileno());
3378 except io.UnsupportedOperation:
3379 pass;
3380 except AttributeError:
3381 pass;
3382 except OSError as e:
3383 pass;
3384 if(numfiles>0):
3385 catfp.write(AppendNullBytes([0, 0], formatspecs[5]).encode("UTF-8"));
3386 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3387 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
3388 try:
3389 catfp.flush();
3390 os.fsync(catfp.fileno());
3391 except io.UnsupportedOperation:
3392 pass;
3393 except AttributeError:
3394 pass;
3395 except OSError as e:
3396 pass;
3397 if(outfile=="-"):
3398 catfp.seek(0, 0)
3399 if(hasattr(sys.stdout, "buffer")):
3400 shutil.copyfileobj(catfp, sys.stdout.buffer);
3401 else:
3402 shutil.copyfileobj(catfp, sys.stdout);
3403 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3404 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3405 catfp.seek(0, 0);
3406 upload_file_to_internet_file(catfp, outfile);
3407 if(returnfp):
3408 catfp.seek(0, 0)
3409 return catfp
3410 else:
3411 catfp.close()
3412 return True;
3413
3414 create_alias_function("Pack", __file_format_name__, "FromRarFile", PackArchiveFileFromRarFile);
3415
3416 if(not py7zr_support):
3417 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
3418 return False
3419
3420 if(py7zr_support):
3421 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
3422 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3423 outfile = RemoveWindowsPath(outfile);
3424 checksumtype = checksumtype.lower();
3425 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3426 checksumtype="crc32";
3427 if(checksumtype=="none"):
3428 checksumtype = "";
3429 if(not compression or compression=="catfile" or compression==formatspecs[2]):
3430 compression = "auto";
3431 if(compression not in compressionlist and compression is None):
3432 compression = "auto";
3433 if(verbose):
3434 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3435 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3436 if(os.path.exists(outfile)):
3437 try:
3438 os.unlink(outfile);
3439 except OSError as e:
3440 pass;
3441 if(outfile=="-"):
3442 verbose = False;
3443 catfp = BytesIO();
3444 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3445 catfp = outfile;
3446 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3447 catfp = BytesIO();
3448 else:
3449 fbasename = os.path.splitext(outfile)[0];
3450 fextname = os.path.splitext(outfile)[1];
3451 catfp = CompressOpenFile(outfile, True, compressionlevel);
3452 catver = formatspecs[6];
3453 fileheaderver = str(int(catver.replace(".", "")));
3454 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
3455 catfp.write(fileheader.encode('UTF-8'));
3456 curinode = 0;
3457 curfid = 0;
3458 inodelist = [];
3459 inodetofile = {};
3460 filetoinode = {};
3461 inodetocatinode = {};
3462 if(not os.path.exists(infile) or not os.path.isfile(infile)):
3463 return False;
3464 szpfp = py7zr.SevenZipFile(infile, mode="r");
3465 file_content = szpfp.readall();
3466 #sztest = szpfp.testzip();
3467 sztestalt = szpfp.test();
3468 if(sztestalt):
3469 VerbosePrintOut("Bad file found!");
3470 numfiles = int(len(szpfp.list()));
3471 fnumfiles = format(int(len(szpfp.list())), 'x').lower();
3472 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs[5]);
3473 catfileheadercshex = GetFileChecksum(fileheader + fnumfilesa, checksumtype, True, formatspecs);
3474 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs[5]);
3475 catfp.write(fnumfilesa.encode('UTF-8'));
3476 try:
3477 catfp.flush();
3478 os.fsync(catfp.fileno());
3479 except io.UnsupportedOperation:
3480 pass;
3481 except AttributeError:
3482 pass;
3483 except OSError as e:
3484 pass;
3485 for member in sorted(szpfp.list(), key=lambda x: x.filename):
3486 if(re.findall("^[.|/]", member.filename)):
3487 fname = member.filename;
3488 else:
3489 fname = "./"+member.filename;
3490 if(verbose):
3491 VerbosePrintOut(fname);
3492 if(not member.is_directory):
3493 fpremode = int(stat.S_IFREG + 438);
3494 elif(member.is_directory):
3495 fpremode = int(stat.S_IFDIR + 511);
3496 fwinattributes = format(int(0), 'x').lower();
3497 flinkcount = 0;
3498 ftype = 0;
3499 if(member.is_directory):
3500 ftype = 5;
3501 else:
3502 ftype = 0;
3503 flinkname = "";
3504 fcurfid = format(int(curfid), 'x').lower();
3505 fcurinode = format(int(curfid), 'x').lower();
3506 curfid = curfid + 1;
3507 fdev_minor = format(int(0), 'x').lower();
3508 fdev_major = format(int(0), 'x').lower();
3509 frdev_minor = format(int(0), 'x').lower();
3510 frdev_major = format(int(0), 'x').lower();
3511 if(ftype==5):
3512 fsize = format(int("0"), 'x').lower();
3513 fatime = format(int(member.creationtime.timestamp()), 'x').lower();
3514 fmtime = format(int(member.creationtime.timestamp()), 'x').lower();
3515 fctime = format(int(member.creationtime.timestamp()), 'x').lower();
3516 fbtime = format(int(member.creationtime.timestamp()), 'x').lower();
3517 if(member.is_directory):
3518 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3519 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
3520 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
3521 else:
3522 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3523 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3524 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3525 try:
3526 fuid = format(int(os.getuid()), 'x').lower();
3527 except AttributeError:
3528 fuid = format(int(0), 'x').lower();
3529 except KeyError:
3530 fuid = format(int(0), 'x').lower();
3531 try:
3532 fgid = format(int(os.getgid()), 'x').lower();
3533 except AttributeError:
3534 fgid = format(int(0), 'x').lower();
3535 except KeyError:
3536 fgid = format(int(0), 'x').lower();
3537 try:
3538 import pwd;
3539 try:
3540 userinfo = pwd.getpwuid(os.getuid());
3541 funame = userinfo.pw_name;
3542 except KeyError:
3543 funame = "";
3544 except AttributeError:
3545 funame = "";
3546 except ImportError:
3547 funame = "";
3548 fgname = "";
3549 try:
3550 import grp;
3551 try:
3552 groupinfo = grp.getgrgid(os.getgid());
3553 fgname = groupinfo.gr_name;
3554 except KeyError:
3555 fgname = "";
3556 except AttributeError:
3557 fgname = "";
3558 except ImportError:
3559 fgname = "";
3560 fcontents = BytesIO();
3561 if(ftype==0):
3562 fcontents.write(file_content[member.filename].read());
3563 fsize = format(fcontents.tell(), 'x').lower();
3564 file_content[member.filename].close();
3565 fcontents.seek(0, 0);
3566 ftypehex = format(ftype, 'x').lower();
3567 extrafields = format(len(extradata), 'x').lower();
3568 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
3569 if(len(extradata)>0):
3570 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
3571 extrasizelen = format(len(extrasizestr), 'x').lower();
3572 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, extrasizelen, extrafields];
3573 catoutlen = len(catoutlist) + len(extradata) + 3;
3574 catoutlenhex = format(catoutlen, 'x').lower();
3575 catoutlist.insert(0, catoutlenhex);
3576 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
3577 if(len(extradata)>0):
3578 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
3579 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
3580 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
3581 fcontents.seek(0, 0);
3582 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
3583 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
3584 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
3585 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
3586 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
3587 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5])
3588 catfileoutstrecd = catfileoutstr.encode('UTF-8');
3589 nullstrecd = formatspecs[5].encode('UTF-8');
3590 fcontents.seek(0, 0);
3591 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
3592 catfp.write(catfileout);
3593 try:
3594 catfp.flush();
3595 os.fsync(catfp.fileno());
3596 except io.UnsupportedOperation:
3597 pass;
3598 except AttributeError:
3599 pass;
3600 except OSError as e:
3601 pass;
3602 if(numfiles>0):
3603 catfp.write(AppendNullBytes([0, 0], formatspecs[5]).encode("UTF-8"));
3604 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3605 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
3606 try:
3607 catfp.flush();
3608 os.fsync(catfp.fileno());
3609 except io.UnsupportedOperation:
3610 pass;
3611 except AttributeError:
3612 pass;
3613 except OSError as e:
3614 pass;
3615 if(outfile=="-"):
3616 catfp.seek(0, 0);
3617 if(hasattr(sys.stdout, "buffer")):
3618 shutil.copyfileobj(catfp, sys.stdout.buffer);
3619 else:
3620 shutil.copyfileobj(catfp, sys.stdout);
3621 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3622 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3623 catfp.seek(0, 0);
3624 upload_file_to_internet_file(catfp, outfile);
3625 if(returnfp):
3626 catfp.seek(0, 0)
3627 return catfp
3628 else:
3629 catfp.close()
3630 return True;
3631
3632 create_alias_function("Pack", __file_format_name__, "FromSevenZipFile", PackArchiveFileFromSevenZipFile);
3633
3634 def PackArchiveFileFromInFile(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
3635 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
3636 if(verbose):
3637 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3638 if(checkcompressfile=="tarfile"):
3639 return PackArchiveFileFromTarFile(infile, outfile, compression, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3640 elif(checkcompressfile=="zipfile"):
3641 return PackArchiveFileFromZipFile(infile, outfile, compression, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3642 elif(checkcompressfile=="catfile"):
3643 return RePackArchiveFile(infile, outfile, compression, compressionlevel, False, 0, 0, checksumtype, False, extradata, formatspecs, verbose, returnfp);
3644 elif(rarfile_support and checkcompressfile=="rarfile"):
3645 return PackArchiveFileFromRarFile(infile, outfile, compression, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3646 elif(py7zr_support and checkcompressfile=="7zipfile"):
3647 return PackArchiveFileFromSevenZipFile(infile, outfile, compression, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3648 else:
3649 return False;
3650 return False;
3651
3652 create_alias_function("Pack", __file_format_name__, "FromInFile", PackArchiveFileFromInFile);
3653
3654 def ArchiveFileSeekToFileNum(infile, seekto=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
3655 if(hasattr(infile, "read") or hasattr(infile, "write")):
3656 catfp = infile;
3657 catfp.seek(0, 0);
3658 catfp = UncompressArchiveFile(catfp, formatspecs);
3659 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
3660 if(checkcompressfile=="tarfile"):
3661 return TarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3662 if(checkcompressfile=="zipfile"):
3663 return ZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3664 if(rarfile_support and checkcompressfile=="rarfile"):
3665 return RarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3666 if(py7zr_support and checkcompressfile=="7zipfile"):
3667 return SevenZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3668 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
3669 return False;
3670 if(not catfp):
3671 return False;
3672 catfp.seek(0, 0);
3673 elif(infile=="-"):
3674 catfp = BytesIO();
3675 if(hasattr(sys.stdin, "buffer")):
3676 shutil.copyfileobj(sys.stdin.buffer, catfp);
3677 else:
3678 shutil.copyfileobj(sys.stdin, catfp);
3679 catfp.seek(0, 0);
3680 catfp = UncompressArchiveFile(catfp, formatspecs);
3681 if(not catfp):
3682 return False;
3683 catfp.seek(0, 0);
3684 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3685 catfp = download_file_from_internet_file(infile);
3686 catfp.seek(0, 0);
3687 catfp = UncompressArchiveFile(catfp, formatspecs);
3688 if(not catfp):
3689 return False;
3690 catfp.seek(0, 0);
3691 else:
3692 infile = RemoveWindowsPath(infile);
3693 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
3694 if(checkcompressfile=="tarfile"):
3695 return TarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3696 if(checkcompressfile=="zipfile"):
3697 return ZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3698 if(rarfile_support and checkcompressfile=="rarfile"):
3699 return RarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3700 if(py7zr_support and checkcompressfile=="7zipfile"):
3701 return SevenZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3702 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
3703 return False;
3704 compresscheck = CheckCompressionType(infile, formatspecs, True);
3705 if(not compresscheck):
3706 fextname = os.path.splitext(infile)[1];
3707 if(fextname==".gz"):
3708 compresscheck = "gzip";
3709 elif(fextname==".bz2"):
3710 compresscheck = "bzip2";
3711 elif(fextname==".zst"):
3712 compresscheck = "zstd";
3713 elif(fextname==".lz4" or fextname==".clz4"):
3714 compresscheck = "lz4";
3715 elif(fextname==".lzo" or fextname==".lzop"):
3716 compresscheck = "lzo";
3717 elif(fextname==".lzma" or fextname==".xz"):
3718 compresscheck = "lzma";
3719 else:
3720 return False;
3721 if(not compresscheck):
3722 return False;
3723 catfp = UncompressFile(infile, formatspecs, "rb");
3724 '''
3725 try:
3726 catfp.seek(0, 2);
3727 except OSError:
3728 SeekToEndOfFile(catfp);
3729 except ValueError:
3730 SeekToEndOfFile(catfp);
3731 CatSize = catfp.tell();
3732 CatSizeEnd = CatSize;
3733 '''
3734 try:
3735 catfp.seek(0, 0);
3736 except OSError:
3737 return False;
3738 except ValueError:
3739 return False;
3740 curloc = catfp.tell();
3741 if(curloc>0):
3742 catfp.seek(0, 0);
3743 catheader = ReadFileHeaderData(catfp, 4, formatspecs[5]);
3744 if(curloc>0):
3745 catfp.seek(curloc, 0);
3746 catstring = catheader[0];
3747 catversion = re.findall(r"([\d]+)$", catstring);
3748 fprenumfiles = catheader[1];
3749 fnumfiles = int(fprenumfiles, 16);
3750 fprechecksumtype = catheader[2];
3751 fprechecksum = catheader[3];
3752 fileheader = AppendNullByte(catstring, formatspecs[5]);
3753 fnumfileshex = format(int(fnumfiles), 'x').lower();
3754 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs[5]);
3755 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
3756 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
3757 fheadtell = len(fileheader);
3758 if(fprechecksum!=catfileheadercshex and not skipchecksum):
3759 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
3760 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
3761 return False;
3762 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
3763 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
3764 if(seekto>=fnumfiles):
3765 seekto = fnumfiles - 1;
3766 if(seekto<0):
3767 seekto = 0;
3768 if(seekto>=0):
3769 il = -1;
3770 while(il < seekto):
3771 prefhstart = catfp.tell();
3772 if(formatspecs[7]):
3773 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs[5]);
3774 else:
3775 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs[5]);
3776 if(len(preheaderdata)==0):
3777 break;
3778 prefheadsize = int(preheaderdata[0], 16);
3779 prefnumfields = int(preheaderdata[1], 16);
3780 preftype = int(preheaderdata[2], 16);
3781 if(re.findall("^[.|/]", preheaderdata[3])):
3782 prefname = preheaderdata[3];
3783 else:
3784 prefname = "./"+preheaderdata[3];
3785 prefbasedir = os.path.dirname(prefname);
3786 preflinkname = preheaderdata[4];
3787 prefsize = int(preheaderdata[5], 16);
3788 prefatime = int(preheaderdata[6], 16);
3789 prefmtime = int(preheaderdata[7], 16);
3790 prefctime = int(preheaderdata[8], 16);
3791 prefbtime = int(preheaderdata[9], 16);
3792 prefmode = int(preheaderdata[10], 16);
3793 prefchmode = stat.S_IMODE(prefmode);
3794 preftypemod = stat.S_IFMT(prefmode);
3795 prefwinattributes = int(preheaderdata[11], 16);
3796 prefuid = int(preheaderdata[12], 16);
3797 prefuname = preheaderdata[13];
3798 prefgid = int(preheaderdata[14], 16);
3799 prefgname = preheaderdata[15];
3800 fid = int(preheaderdata[16], 16);
3801 finode = int(preheaderdata[17], 16);
3802 flinkcount = int(preheaderdata[18], 16);
3803 prefdev_minor = int(preheaderdata[19], 16);
3804 prefdev_major = int(preheaderdata[20], 16);
3805 prefrdev_minor = int(preheaderdata[22], 16);
3806 prefrdev_major = int(preheaderdata[23], 16);
3807 prefextrasize = int(preheaderdata[24], 16);
3808 prefextrafields = int(preheaderdata[25], 16);
3809 extrafieldslist = [];
3810 extrastart = 25;
3811 extraend = extrastart + prefextrafields;
3812 extrafieldslist = [];
3813 if(extrastart<extraend):
3814 extrafieldslist.append(preheaderdata[extrastart]);
3815 extrastart = extrastart + 1;
3816 prefchecksumtype = preheaderdata[extrastart].lower();
3817 prefcs = preheaderdata[extrastart + 1].lower();
3818 prefccs = preheaderdata[extrastart + 2].lower();
3819 hc = 0;
3820 hcmax = len(preheaderdata) - 2;
3821 hout = "";
3822 while(hc<hcmax):
3823 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs[5]);
3824 hc = hc + 1;
3825 prenewfcs = GetFileChecksum(hout, preheaderdata[-3].lower(), True, formatspecs);
3826 if(prefcs!=prenewfcs and not skipchecksum):
3827 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
3828 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
3829 return False;
3830 valid_archive = False;
3831 invalid_archive = True;
3832 prefhend = catfp.tell() - 1;
3833 prefcontentstart = catfp.tell();
3834 prefcontents = "";
3835 pyhascontents = False;
3836 if(prefsize>0):
3837 prefcontents = catfp.read(prefsize);
3838 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
3839 pyhascontents = True;
3840 if(prefccs!=prenewfccs and not skipchecksum):
3841 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
3842 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
3843 return False;
3844 catfp.seek(1, 1);
3845 il = il + 1;
3846 catfp.seek(seekstart, 0);
3847 fileidnum = il;
3848 catfheadsize = int(preheaderdata[0], 16);
3849 catfnumfields = int(preheaderdata[1], 16);
3850 catftype = int(preheaderdata[2], 16);
3851 if(re.findall("^[.|/]", preheaderdata[3])):
3852 catfname = preheaderdata[3];
3853 else:
3854 catfname = "./"+preheaderdata[3];
3855 catflinkname = preheaderdata[4];
3856 catfsize = int(preheaderdata[5], 16);
3857 catfbasedir = os.path.dirname(catfname);
3858 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
3859 if(returnfp):
3860 catlist.update({'catfp': catfp});
3861 else:
3862 catfp.close();
3863 return catlist;
3864
3865 create_alias_function("", __file_format_name__, "SeekToFileNum", ArchiveFileSeekToFileNum);
3866
3867 def ArchiveFileSeekToFileName(infile, seekfile=None, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
3868 if(hasattr(infile, "read") or hasattr(infile, "write")):
3869 catfp = infile;
3870 catfp.seek(0, 0);
3871 catfp = UncompressArchiveFile(catfp, formatspecs);
3872 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
3873 if(checkcompressfile=="tarfile"):
3874 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3875 if(checkcompressfile=="zipfile"):
3876 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3877 if(rarfile_support and checkcompressfile=="rarfile"):
3878 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3879 if(py7zr_support and checkcompressfile=="7zipfile"):
3880 return SevenZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3881 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
3882 return False;
3883 if(not catfp):
3884 return False;
3885 catfp.seek(0, 0);
3886 elif(infile=="-"):
3887 catfp = BytesIO();
3888 if(hasattr(sys.stdin, "buffer")):
3889 shutil.copyfileobj(sys.stdin.buffer, catfp);
3890 else:
3891 shutil.copyfileobj(sys.stdin, catfp);
3892 catfp.seek(0, 0);
3893 catfp = UncompressArchiveFile(catfp, formatspecs);
3894 if(not catfp):
3895 return False;
3896 catfp.seek(0, 0);
3897 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3898 catfp = download_file_from_internet_file(infile);
3899 catfp = UncompressArchiveFile(catfp, formatspecs);
3900 catfp.seek(0, 0);
3901 if(not catfp):
3902 return False;
3903 catfp.seek(0, 0);
3904 else:
3905 infile = RemoveWindowsPath(infile);
3906 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
3907 if(checkcompressfile=="tarfile"):
3908 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3909 if(checkcompressfile=="zipfile"):
3910 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3911 if(rarfile_support and checkcompressfile=="rarfile"):
3912 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3913 if(py7zr_support and checkcompressfile=="7zipfile"):
3914 return SevenZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3915 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
3916 return False;
3917 compresscheck = CheckCompressionType(infile, formatspecs, True);
3918 if(not compresscheck):
3919 fextname = os.path.splitext(infile)[1];
3920 if(fextname==".gz"):
3921 compresscheck = "gzip";
3922 elif(fextname==".bz2"):
3923 compresscheck = "bzip2";
3924 elif(fextname==".zst"):
3925 compresscheck = "zstd";
3926 elif(fextname==".lz4" or fextname==".clz4"):
3927 compresscheck = "lz4";
3928 elif(fextname==".lzo" or fextname==".lzop"):
3929 compresscheck = "lzo";
3930 elif(fextname==".lzma" or fextname==".xz"):
3931 compresscheck = "lzma";
3932 else:
3933 return False;
3934 if(not compresscheck):
3935 return False;
3936 catfp = UncompressFile(infile, formatspecs, "rb");
3937 '''
3938 try:
3939 catfp.seek(0, 2);
3940 except OSError:
3941 SeekToEndOfFile(catfp);
3942 except ValueError:
3943 SeekToEndOfFile(catfp);
3944 CatSize = catfp.tell();
3945 CatSizeEnd = CatSize;
3946 '''
3947 try:
3948 catfp.seek(0, 0);
3949 except OSError:
3950 return False;
3951 except ValueError:
3952 return False;
3953 curloc = catfp.tell();
3954 if(curloc>0):
3955 catfp.seek(0, 0);
3956 catheader = ReadFileHeaderData(catfp, 4, formatspecs[5]);
3957 if(curloc>0):
3958 catfp.seek(curloc, 0);
3959 catstring = catheader[0];
3960 catversion = re.findall(r"([\d]+)$", catstring);
3961 fprenumfiles = catheader[1];
3962 fnumfiles = int(fprenumfiles, 16);
3963 fprechecksumtype = catheader[2];
3964 fprechecksum = catheader[3];
3965 fileheader = AppendNullByte(catstring, formatspecs[5]);
3966 fnumfileshex = format(int(fnumfiles), 'x').lower();
3967 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs[5]);
3968 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
3969 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
3970 fheadtell = len(fileheader);
3971 if(fprechecksum!=catfileheadercshex and not skipchecksum):
3972 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
3973 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
3974 return False;
3975 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
3976 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
3977 seekto = fnumfiles - 1
3978 filefound = False;
3979 if(seekto>=0):
3980 il = -1;
3981 while(il < seekto):
3982 prefhstart = catfp.tell();
3983 if(formatspecs[7]):
3984 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs[5]);
3985 else:
3986 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs[5]);
3987 if(len(preheaderdata)==0):
3988 break;
3989 prefheadsize = int(preheaderdata[0], 16);
3990 prefnumfields = int(preheaderdata[1], 16);
3991 preftype = int(preheaderdata[2], 16);
3992 if(re.findall("^[.|/]", preheaderdata[3])):
3993 prefname = preheaderdata[3];
3994 else:
3995 prefname = "./"+preheaderdata[3];
3996 prefbasedir = os.path.dirname(prefname);
3997 preflinkname = preheaderdata[4];
3998 prefsize = int(preheaderdata[5], 16);
3999 prefatime = int(preheaderdata[6], 16);
4000 prefmtime = int(preheaderdata[7], 16);
4001 prefctime = int(preheaderdata[8], 16);
4002 prefbtime = int(preheaderdata[9], 16);
4003 prefmode = int(preheaderdata[10], 16);
4004 prefchmode = stat.S_IMODE(prefmode);
4005 preftypemod = stat.S_IFMT(prefmode);
4006 prefwinattributes = int(preheaderdata[11], 16);
4007 prefuid = int(preheaderdata[12], 16);
4008 prefuname = preheaderdata[13];
4009 prefgid = int(preheaderdata[14], 16);
4010 prefgname = preheaderdata[15];
4011 fid = int(preheaderdata[16], 16);
4012 finode = int(preheaderdata[17], 16);
4013 flinkcount = int(preheaderdata[18], 16);
4014 prefdev_minor = int(preheaderdata[19], 16);
4015 prefdev_major = int(preheaderdata[20], 16);
4016 prefrdev_minor = int(preheaderdata[22], 16);
4017 prefrdev_major = int(preheaderdata[23], 16);
4018 prefextrasize = int(preheaderdata[24], 16);
4019 prefextrafields = int(preheaderdata[25], 16);
4020 extrafieldslist = [];
4021 extrastart = 25;
4022 extraend = extrastart + prefextrafields;
4023 extrafieldslist = [];
4024 if(extrastart<extraend):
4025 extrafieldslist.append(preheaderdata[extrastart]);
4026 extrastart = extrastart + 1;
4027 prefchecksumtype = preheaderdata[extrastart].lower();
4028 prefcs = preheaderdata[extrastart + 1].lower();
4029 prefccs = preheaderdata[extrastart + 2].lower();
4030 hc = 0;
4031 hcmax = len(preheaderdata) - 2;
4032 hout = "";
4033 while(hc<hcmax):
4034 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs[5]);
4035 hc = hc + 1;
4036 prenewfcs = GetFileChecksum(hout, preheaderdata[-3].lower(), True, formatspecs);
4037 if(prefcs!=prenewfcs and not skipchecksum):
4038 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4039 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4040 return False;
4041 valid_archive = False;
4042 invalid_archive = True;
4043 prefhend = catfp.tell() - 1;
4044 prefcontentstart = catfp.tell();
4045 prefcontents = "";
4046 pyhascontents = False;
4047 if(prefsize>0):
4048 prefcontents = catfp.read(prefsize);
4049 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4050 pyhascontents = True;
4051 if(prefccs!=prenewfccs and not skipchecksum):
4052 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4053 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4054 return False;
4055 catfp.seek(1, 1);
4056 il = il + 1;
4057 filefound = False;
4058 prefname = preheaderdata[2];
4059 if(re.findall("^[.|/]", preheaderdata[2])):
4060 prefname = preheaderdata[2];
4061 else:
4062 prefname = "./"+preheaderdata[2];
4063 if(prefname==seekfile):
4064 filefound = True;
4065 break;
4066 catfp.seek(seekstart, 0);
4067 fileidnum = il;
4068 catfheadsize = int(preheaderdata[0], 16);
4069 catfnumfields = int(preheaderdata[1], 16);
4070 catftype = int(preheaderdata[2], 16);
4071 if(re.findall("^[.|/]", preheaderdata[3])):
4072 catfname = preheaderdata[3];
4073 else:
4074 catfname = "./"+preheaderdata[3];
4075 catflinkname = preheaderdata[4];
4076 catfsize = int(preheaderdata[5], 16);
4077 catfbasedir = os.path.dirname(catfname);
4078 if(filefound):
4079 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
4080 else:
4081 return False;
4082 if(returnfp):
4083 catlist.update({'catfp': catfp});
4084 else:
4085 catfp.close();
4086 return catlist;
4087
4088 create_alias_function("", __file_format_name__, "SeekToFileName", ArchiveFileSeekToFileName);
4089
4090 def ArchiveFileValidate(infile, formatspecs=__file_format_list__, verbose=False, returnfp=False):
4091 if(verbose):
4092 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
4093 if(hasattr(infile, "read") or hasattr(infile, "write")):
4094 catfp = infile;
4095 catfp.seek(0, 0);
4096 catfp = UncompressArchiveFile(catfp, formatspecs);
4097 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4098 if(checkcompressfile=="tarfile"):
4099 return TarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4100 if(checkcompressfile=="zipfile"):
4101 return ZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4102 if(rarfile_support and checkcompressfile=="rarfile"):
4103 return RarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4104 if(py7zr_support and checkcompressfile=="7zipfile"):
4105 return SevenZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4106 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
4107 return False;
4108 if(not catfp):
4109 return False;
4110 catfp.seek(0, 0);
4111 elif(infile=="-"):
4112 catfp = BytesIO();
4113 if(hasattr(sys.stdin, "buffer")):
4114 shutil.copyfileobj(sys.stdin.buffer, catfp);
4115 else:
4116 shutil.copyfileobj(sys.stdin, catfp);
4117 catfp.seek(0, 0);
4118 catfp = UncompressArchiveFile(catfp, formatspecs);
4119 if(not catfp):
4120 return False;
4121 catfp.seek(0, 0);
4122 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4123 catfp = download_file_from_internet_file(infile);
4124 catfp = UncompressArchiveFile(catfp, formatspecs);
4125 catfp.seek(0, 0);
4126 if(not catfp):
4127 return False;
4128 catfp.seek(0, 0);
4129 else:
4130 infile = RemoveWindowsPath(infile);
4131 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4132 if(checkcompressfile=="tarfile"):
4133 return TarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4134 if(checkcompressfile=="zipfile"):
4135 return ZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4136 if(rarfile_support and checkcompressfile=="rarfile"):
4137 return RarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4138 if(py7zr_support and checkcompressfile=="7zipfile"):
4139 return SevenZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4140 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
4141 return False;
4142 compresscheck = CheckCompressionType(infile, formatspecs, True);
4143 if(not compresscheck):
4144 fextname = os.path.splitext(infile)[1];
4145 if(fextname==".gz"):
4146 compresscheck = "gzip";
4147 elif(fextname==".bz2"):
4148 compresscheck = "bzip2";
4149 elif(fextname==".zst"):
4150 compresscheck = "zstd";
4151 elif(fextname==".lz4" or fextname==".clz4"):
4152 compresscheck = "lz4";
4153 elif(fextname==".lzo" or fextname==".lzop"):
4154 compresscheck = "lzo";
4155 elif(fextname==".lzma" or fextname==".xz"):
4156 compresscheck = "lzma";
4157 else:
4158 return False;
4159 if(not compresscheck):
4160 return False;
4161 catfp = UncompressFile(infile, formatspecs, "rb");
4162 '''
4163 try:
4164 catfp.seek(0, 2);
4165 except OSError:
4166 SeekToEndOfFile(catfp);
4167 except ValueError:
4168 SeekToEndOfFile(catfp);
4169 CatSize = catfp.tell();
4170 CatSizeEnd = CatSize;
4171 '''
4172 try:
4173 catfp.seek(0, 0);
4174 except OSError:
4175 return False;
4176 except ValueError:
4177 return False;
4178 curloc = catfp.tell();
4179 if(curloc>0):
4180 catfp.seek(0, 0);
4181 catheader = ReadFileHeaderData(catfp, 4, formatspecs[5]);
4182 if(curloc>0):
4183 catfp.seek(curloc, 0);
4184 catstring = catheader[0];
4185 catversion = re.findall(r"([\d]+)$", catstring);
4186 fprenumfiles = catheader[1];
4187 fnumfiles = int(fprenumfiles, 16);
4188 fprechecksumtype = catheader[2];
4189 fprechecksum = catheader[3];
4190 il = 0;
4191 fileheader = AppendNullByte(catstring, formatspecs[5]);
4192 fnumfileshex = format(int(fnumfiles), 'x').lower();
4193 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs[5]);
4194 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4195 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
4196 valid_archive = True;
4197 invalid_archive = False;
4198 if(verbose):
4199 VerbosePrintOut(infile);
4200 VerbosePrintOut("Number of Records " + str(fnumfiles));
4201 if(fprechecksum==catfileheadercshex):
4202 if(verbose):
4203 VerbosePrintOut("File Header Checksum Passed at offset " + str(0));
4204 VerbosePrintOut("'" + str(fprechecksum) + "' == " + "'" + str(catfileheadercshex) + "'");
4205 else:
4206 if(verbose):
4207 VerbosePrintOut("File Header Checksum Failed at offset " + str(0));
4208 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4209 valid_archive = False;
4210 invalid_archive = True;
4211 if(verbose):
4212 VerbosePrintOut("");
4213 while(il<fnumfiles):
4214 catfhstart = catfp.tell();
4215 if(formatspecs[7]):
4216 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs[5]);
4217 else:
4218 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs[5]);
4219 if(len(catheaderdata)==0):
4220 break;
4221 catfheadsize = int(catheaderdata[0], 16);
4222 catfnumfields = int(catheaderdata[1], 16);
4223 catftype = int(catheaderdata[2], 16);
4224 if(re.findall("^[.|/]", catheaderdata[3])):
4225 catfname = catheaderdata[3];
4226 else:
4227 catfname = "./"+catheaderdata[3];
4228 catfbasedir = os.path.dirname(catfname);
4229 catflinkname = catheaderdata[4];
4230 catfsize = int(catheaderdata[5], 16);
4231 catfatime = int(catheaderdata[6], 16);
4232 catfmtime = int(catheaderdata[7], 16);
4233 catfctime = int(catheaderdata[8], 16);
4234 catfbtime = int(catheaderdata[9], 16);
4235 catfmode = int(catheaderdata[10], 16);
4236 catfchmode = stat.S_IMODE(catfmode);
4237 catftypemod = stat.S_IFMT(catfmode);
4238 prefwinattributes = int(catheaderdata[11], 16);
4239 catfuid = int(catheaderdata[12], 16);
4240 catfuname = catheaderdata[13];
4241 catfgid = int(catheaderdata[14], 16);
4242 catfgname = catheaderdata[15];
4243 fid = int(catheaderdata[16], 16);
4244 finode = int(catheaderdata[17], 16);
4245 flinkcount = int(catheaderdata[18], 16);
4246 catfdev_minor = int(catheaderdata[19], 16);
4247 catfdev_major = int(catheaderdata[20], 16);
4248 catfrdev_minor = int(catheaderdata[21], 16);
4249 catfrdev_major = int(catheaderdata[22], 16);
4250 catfextrasize = int(catheaderdata[23], 16);
4251 catfextrafields = int(catheaderdata[24], 16);
4252 extrafieldslist = [];
4253 extrastart = 25;
4254 extraend = extrastart + catfextrafields;
4255 extrafieldslist = [];
4256 if(extrastart<extraend):
4257 extrafieldslist.append(catheaderdata[extrastart]);
4258 extrastart = extrastart + 1;
4259 catfchecksumtype = catheaderdata[extrastart].lower();
4260 catfcs = catheaderdata[extrastart + 1].lower();
4261 catfccs = catheaderdata[extrastart + 2].lower();
4262 hc = 0;
4263 hcmax = len(catheaderdata) - 2;
4264 hout = "";
4265 while(hc<hcmax):
4266 hout = hout + AppendNullByte(catheaderdata[hc], formatspecs[5]);
4267 hc = hc + 1;
4268 catnewfcs = GetFileChecksum(hout, catheaderdata[-3].lower(), True, formatspecs);
4269 if(verbose):
4270 VerbosePrintOut(catfname);
4271 VerbosePrintOut("Record Number " + str(il) + "; File ID " + str(fid) + "; iNode Number " + str(finode));
4272 if(catfcs==catnewfcs):
4273 if(verbose):
4274 VerbosePrintOut("File Header Checksum Passed at offset " + str(catfhstart));
4275 VerbosePrintOut("'" + str(catfcs) + "' == " + "'" + str(catnewfcs) + "'");
4276 else:
4277 if(verbose):
4278 VerbosePrintOut("File Header Checksum Failed at offset " + str(catfhstart));
4279 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
4280 valid_archive = False;
4281 invalid_archive = True;
4282 catfhend = catfp.tell() - 1;
4283 catfcontentstart = catfp.tell();
4284 catfcontents = "";
4285 pyhascontents = False;
4286 if(catfsize>0):
4287 catfcontents = catfp.read(catfsize);
4288 catnewfccs = GetFileChecksum(catfcontents, catheaderdata[-3].lower(), False, formatspecs);
4289 pyhascontents = True;
4290 if(catfccs==catnewfccs):
4291 if(verbose):
4292 VerbosePrintOut("File Content Checksum Passed at offset " + str(catfcontentstart));
4293 VerbosePrintOut("'" + str(catfccs) + "' == " + "'" + str(catnewfccs) + "'");
4294 else:
4295 if(verbose):
4296 VerbosePrintOut("File Content Checksum Failed at offset " + str(catfcontentstart));
4297 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
4298 valid_archive = False;
4299 invalid_archive = True;
4300 if(verbose):
4301 VerbosePrintOut("");
4302 catfp.seek(1, 1);
4303 il = il + 1;
4304 if(valid_archive):
4305 if(returnfp):
4306 return catfp;
4307 else:
4308 catfp.close();
4309 return True;
4310 else:
4311 catfp.close();
4312 return False;
4313
4314 create_alias_function("", __file_format_name__, "Validate", ArchiveFileValidate);
4315
4316 def ArchiveFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4317 if(hasattr(infile, "read") or hasattr(infile, "write")):
4318 catfp = infile;
4319 catfp.seek(0, 0);
4320 catfp = UncompressArchiveFile(catfp, formatspecs);
4321 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4322 if(checkcompressfile=="tarfile"):
4323 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4324 if(checkcompressfile=="zipfile"):
4325 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4326 if(rarfile_support and checkcompressfile=="rarfile"):
4327 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4328 if(py7zr_support and checkcompressfile=="7zipfile"):
4329 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4330 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
4331 return False;
4332 if(not catfp):
4333 return False;
4334 catfp.seek(0, 0);
4335 elif(infile=="-"):
4336 catfp = BytesIO();
4337 if(hasattr(sys.stdin, "buffer")):
4338 shutil.copyfileobj(sys.stdin.buffer, catfp);
4339 else:
4340 shutil.copyfileobj(sys.stdin, catfp);
4341 catfp.seek(0, 0);
4342 catfp = UncompressArchiveFile(catfp, formatspecs);
4343 if(not catfp):
4344 return False;
4345 catfp.seek(0, 0);
4346 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4347 catfp = download_file_from_internet_file(infile);
4348 catfp = UncompressArchiveFile(catfp, formatspecs);
4349 catfp.seek(0, 0);
4350 if(not catfp):
4351 return False;
4352 catfp.seek(0, 0);
4353 else:
4354 infile = RemoveWindowsPath(infile);
4355 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4356 if(checkcompressfile=="tarfile"):
4357 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4358 if(checkcompressfile=="zipfile"):
4359 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4360 if(rarfile_support and checkcompressfile=="rarfile"):
4361 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4362 if(py7zr_support and checkcompressfile=="7zipfile"):
4363 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4364 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
4365 return False;
4366 compresscheck = CheckCompressionType(infile, formatspecs, True);
4367 if(not compresscheck):
4368 fextname = os.path.splitext(infile)[1];
4369 if(fextname==".gz"):
4370 compresscheck = "gzip";
4371 elif(fextname==".bz2"):
4372 compresscheck = "bzip2";
4373 elif(fextname==".zst"):
4374 compresscheck = "zstd";
4375 elif(fextname==".lz4" or fextname==".clz4"):
4376 compresscheck = "lz4";
4377 elif(fextname==".lzo" or fextname==".lzop"):
4378 compresscheck = "lzo";
4379 elif(fextname==".lzma" or fextname==".xz"):
4380 compresscheck = "lzma";
4381 else:
4382 return False;
4383 if(not compresscheck):
4384 return False;
4385 catfp = UncompressFile(infile, formatspecs, "rb");
4386 '''
4387 try:
4388 catfp.seek(0, 2);
4389 except OSError:
4390 SeekToEndOfFile(catfp);
4391 except ValueError:
4392 SeekToEndOfFile(catfp);
4393 CatSize = catfp.tell();
4394 CatSizeEnd = CatSize;
4395 '''
4396 try:
4397 catfp.seek(0, 0);
4398 except OSError:
4399 return False;
4400 except ValueError:
4401 return False;
4402 curloc = catfp.tell();
4403 if(curloc>0):
4404 catfp.seek(0, 0);
4405 catheader = ReadFileHeaderData(catfp, 4, formatspecs[5]);
4406 if(curloc>0):
4407 catfp.seek(curloc, 0);
4408 catstring = catheader[0];
4409 catversion = re.findall(r"([\d]+)$", catstring);
4410 fprenumfiles = catheader[1];
4411 fnumfiles = int(fprenumfiles, 16);
4412 fprechecksumtype = catheader[2];
4413 fprechecksum = catheader[3];
4414 fileheader = AppendNullByte(catstring, formatspecs[5]);
4415 fnumfileshex = format(int(fnumfiles), 'x').lower();
4416 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs[5]);
4417 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4418 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
4419 fheadtell = len(fileheader);
4420 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4421 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
4422 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4423 return False;
4424 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
4425 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4426 if(seekstart<0 and seekstart>fnumfiles):
4427 seekstart = 0;
4428 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
4429 seekend = fnumfiles;
4430 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
4431 seekend = fnumfiles - abs(seekend);
4432 if(seekstart>0):
4433 il = 0;
4434 while(il < seekstart):
4435 prefhstart = catfp.tell();
4436 if(formatspecs[7]):
4437 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs[5]);
4438 else:
4439 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs[5]);
4440 if(len(preheaderdata)==0):
4441 break;
4442 prefheadsize = int(preheaderdata[0], 16);
4443 prefnumfields = int(preheaderdata[1], 16);
4444 if(re.findall("^[.|/]", preheaderdata[3])):
4445 prefname = preheaderdata[3];
4446 else:
4447 prefname = "./"+preheaderdata[3];
4448 prefsize = int(preheaderdata[5], 16);
4449 prefextrasize = int(preheaderdata[23], 16);
4450 prefextrafields = int(preheaderdata[24], 16);
4451 extrafieldslist = [];
4452 extrastart = 25;
4453 extraend = extrastart + prefextrafields;
4454 extrafieldslist = [];
4455 if(extrastart<extraend):
4456 extrafieldslist.append(preheaderdata[extrastart]);
4457 extrastart = extrastart + 1;
4458 prefchecksumtype = preheaderdata[extrastart].lower();
4459 prefcs = preheaderdata[extrastart + 1].lower();
4460 prefccs = preheaderdata[extrastart + 2].lower();
4461 hc = 0;
4462 hcmax = len(preheaderdata) - 2;
4463 hout = "";
4464 while(hc<hcmax):
4465 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs[5]);
4466 hc = hc + 1;
4467 prenewfcs = GetFileChecksum(hout, preheaderdata[-3].lower(), True, formatspecs);
4468 if(prefcs!=prenewfcs and not skipchecksum):
4469 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4470 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4471 return False;
4472 valid_archive = False;
4473 invalid_archive = True;
4474 prefhend = catfp.tell() - 1;
4475 prefcontentstart = catfp.tell();
4476 prefcontents = "";
4477 pyhascontents = False;
4478 if(prefsize>0):
4479 prefcontents = catfp.read(prefsize);
4480 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4481 pyhascontents = True;
4482 if(prefccs!=prenewfccs and not skipchecksum):
4483 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4484 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4485 return False;
4486 catfp.seek(1, 1);
4487 il = il + 1;
4488 fileidnum = seekstart;
4489 realidnum = 0;
4490 while(fileidnum<seekend):
4491 catfhstart = catfp.tell();
4492 if(formatspecs[7]):
4493 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs[5]);
4494 else:
4495 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs[5]);
4496 if(len(catheaderdata)==0):
4497 break;
4498 catfheadsize = int(catheaderdata[0], 16);
4499 catfnumfields = int(catheaderdata[1], 16);
4500 catftype = int(catheaderdata[2], 16);
4501 if(re.findall("^[.|/]", catheaderdata[3])):
4502 catfname = catheaderdata[3];
4503 else:
4504 catfname = "./"+catheaderdata[3];
4505 catfbasedir = os.path.dirname(catfname);
4506 catflinkname = catheaderdata[4];
4507 catfsize = int(catheaderdata[5], 16);
4508 catfatime = int(catheaderdata[6], 16);
4509 catfmtime = int(catheaderdata[7], 16);
4510 catfctime = int(catheaderdata[8], 16);
4511 catfbtime = int(catheaderdata[9], 16);
4512 catfmode = int(catheaderdata[10], 16);
4513 catfchmode = stat.S_IMODE(catfmode);
4514 catftypemod = stat.S_IFMT(catfmode);
4515 catfwinattributes = int(catheaderdata[11], 16);
4516 catfuid = int(catheaderdata[12], 16);
4517 catfuname = catheaderdata[13];
4518 catfgid = int(catheaderdata[14], 16);
4519 catfgname = catheaderdata[15];
4520 fid = int(catheaderdata[16], 16);
4521 finode = int(catheaderdata[17], 16);
4522 flinkcount = int(catheaderdata[18], 16);
4523 catfdev_minor = int(catheaderdata[19], 16);
4524 catfdev_major = int(catheaderdata[20], 16);
4525 catfrdev_minor = int(catheaderdata[21], 16);
4526 catfrdev_major = int(catheaderdata[22], 16);
4527 catfextrasize = int(catheaderdata[23], 16);
4528 catfextrafields = int(catheaderdata[24], 16);
4529 extrafieldslist = [];
4530 extrastart = 25;
4531 extraend = extrastart + catfextrafields;
4532 extrafieldslist = [];
4533 if(extrastart<extraend):
4534 extrafieldslist.append(catheaderdata[extrastart]);
4535 extrastart = extrastart + 1;
4536 catfchecksumtype = catheaderdata[extrastart].lower();
4537 catfcs = catheaderdata[extrastart + 1].lower();
4538 catfccs = catheaderdata[extrastart + 2].lower();
4539 hc = 0;
4540 hcmax = len(catheaderdata) - 2;
4541 hout = "";
4542 while(hc<hcmax):
4543 hout = hout + AppendNullByte(catheaderdata[hc], formatspecs[5]);
4544 hc = hc + 1;
4545 catnewfcs = GetFileChecksum(hout, catheaderdata[-3].lower(), True, formatspecs);
4546 if(catfcs!=catnewfcs and not skipchecksum):
4547 VerbosePrintOut("File Header Checksum Error with file " + catfname + " at offset " + str(catfhstart));
4548 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
4549 return False;
4550 catfhend = catfp.tell() - 1;
4551 catfcontentstart = catfp.tell();
4552 catfcontents = BytesIO();
4553 pyhascontents = False;
4554 catfcontents.seek(0, 0);
4555 if(catfsize>0 and not listonly):
4556 catfcontents.write(catfp.read(catfsize));
4557 catnewfccs = GetFileChecksum(catfcontents.read(), catheaderdata[-3].lower(), False, formatspecs);
4558 pyhascontents = True;
4559 if(catfccs!=catnewfccs and skipchecksum):
4560 VerbosePrintOut("File Content Checksum Error with file " + catfname + " at offset " + str(catfcontentstart));
4561 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
4562 return False;
4563 if(catfsize>0 and listonly):
4564 catfp.seek(catfsize, 1);
4565 pyhascontents = False;
4566 catfp.seek(1, 1);
4567 catfcontentend = catfp.tell() - 1;
4568 catfcontents.seek(0, 0);
4569 catlist['ffilelist'].update({realidnum: {'fid': realidnum, 'fidalt': fileidnum, 'fheadersize': catfheadsize, 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize, 'fatime': catfatime, 'fmtime': catfmtime, 'fctime': catfctime, 'fbtime': catfbtime, 'fmode': catfmode, 'fchmode': catfchmode, 'ftypemod': catftypemod, 'fwinattributes': catfwinattributes, 'fuid': catfuid, 'funame': catfuname, 'fgid': catfgid, 'fgname': catfgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': catfdev_minor, 'fmajor': catfdev_major, 'frminor': catfrdev_minor, 'frmajor': catfrdev_major, 'fchecksumtype': catfchecksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': catfextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': catfcs, 'fcontentchecksum': catfccs, 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': catfcontents} });
4570 fileidnum = fileidnum + 1;
4571 realidnum = realidnum + 1;
4572 if(returnfp):
4573 catlist.update({'catfp': catfp});
4574 else:
4575 catfp.close();
4576 return catlist;
4577
4578 create_alias_function("", __file_format_name__, "ToArray", ArchiveFileToArray);
4579
4580 def ArchiveFileStringToArray(catstr, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4581 catfp = BytesIO(catstr);
4582 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4583 return listcatfiles;
4584
4585 create_alias_function("", __file_format_name__, "StringToArray", ArchiveFileStringToArray);
4586
4587 def TarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4588 catfp = BytesIO();
4589 catfp = PackArchiveFileFromTarFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
4590 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4591 return listcatfiles;
4592
4593 def ZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4594 catfp = BytesIO();
4595 catfp = PackArchiveFileFromZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
4596 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4597 return listcatfiles;
4598
4599 if(not rarfile_support):
4600 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4601 return False;
4602
4603 if(rarfile_support):
4604 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4605 catfp = BytesIO();
4606 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
4607 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4608 return listcatfiles;
4609
4610 if(not py7zr_support):
4611 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4612 return False;
4613
4614 if(py7zr_support):
4615 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4616 catfp = BytesIO();
4617 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
4618 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4619 return listcatfiles;
4620
4621 def InFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4622 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4623 if(checkcompressfile=="tarfile"):
4624 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4625 elif(checkcompressfile=="zipfile"):
4626 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4627 elif(checkcompressfile=="catfile"):
4628 return ArchiveFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4629 elif(rarfile_support and checkcompressfile=="rarfile"):
4630 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4631 elif(py7zr_support and checkcompressfile=="7zipfile"):
4632 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4633 else:
4634 return False;
4635 return False;
4636
4637 def ListDirToArrayAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
4638 catver = formatspecs[6];
4639 fileheaderver = str(int(catver.replace(".", "")));
4640 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
4641 advancedlist = formatspecs[8];
4642 altinode = formatspecs[9];
4643 infilelist = [];
4644 if(infiles=="-"):
4645 for line in sys.stdin:
4646 infilelist.append(line.strip());
4647 infilelist = list(filter(None, infilelist));
4648 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
4649 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
4650 return False;
4651 with UncompressFile(infiles, formatspecs, "r") as finfile:
4652 for line in finfile:
4653 infilelist.append(line.strip());
4654 infilelist = list(filter(None, infilelist));
4655 else:
4656 if(isinstance(infiles, (list, tuple, ))):
4657 infilelist = list(filter(None, infiles));
4658 elif(isinstance(infiles, (str, ))):
4659 infilelist = list(filter(None, [infiles]));
4660 if(advancedlist):
4661 GetDirList = ListDirAdvanced(infilelist, followlink, False);
4662 else:
4663 GetDirList = ListDir(infilelist, followlink, False);
4664 if(not GetDirList):
4665 return False;
4666 curinode = 0;
4667 curfid = 0;
4668 inodelist = [];
4669 inodetofile = {};
4670 filetoinode = {};
4671 inodetocatinode = {};
4672 fileidnum = 0;
4673 fnumfiles = int(len(GetDirList));
4674 catver = formatspecs[6];
4675 fileheaderver = str(int(catver.replace(".", "")));
4676 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
4677 fnumfileshex = format(int(fnumfiles), 'x').lower();
4678 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs[5]);
4679 catversion = re.findall(r"([\d]+)$", fileheader);
4680 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
4681 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4682 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
4683 fheadtell = len(fileheader);
4684 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
4685 for curfname in GetDirList:
4686 if(re.findall("^[.|/]", curfname)):
4687 fname = curfname;
4688 else:
4689 fname = "./"+curfname;
4690 if(verbose):
4691 VerbosePrintOut(fname);
4692 if(not followlink or followlink is None):
4693 fstatinfo = os.lstat(fname);
4694 else:
4695 fstatinfo = os.stat(fname);
4696 fpremode = fstatinfo.st_mode;
4697 finode = fstatinfo.st_ino;
4698 flinkcount = fstatinfo.st_nlink;
4699 ftype = 0;
4700 if(stat.S_ISREG(fpremode)):
4701 ftype = 0;
4702 elif(stat.S_ISLNK(fpremode)):
4703 ftype = 2;
4704 elif(stat.S_ISCHR(fpremode)):
4705 ftype = 3;
4706 elif(stat.S_ISBLK(fpremode)):
4707 ftype = 4;
4708 elif(stat.S_ISDIR(fpremode)):
4709 ftype = 5;
4710 elif(stat.S_ISFIFO(fpremode)):
4711 ftype = 6;
4712 elif(stat.S_ISSOCK(fpremode)):
4713 ftype = 8;
4714 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
4715 ftype = 9;
4716 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
4717 ftype = 10;
4718 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
4719 ftype = 11;
4720 else:
4721 ftype = 0;
4722 flinkname = "";
4723 fbasedir = os.path.dirname(fname);
4724 fcurfid = curfid;
4725 if(not followlink and finode!=0):
4726 if(ftype!=1):
4727 if(finode in inodelist):
4728 ftype = 1;
4729 flinkname = inodetofile[finode];
4730 if(altinode):
4731 fcurinode = finode;
4732 else:
4733 fcurinode = inodetocatinode[finode];
4734 if(finode not in inodelist):
4735 inodelist.append(finode);
4736 inodetofile.update({finode: fname});
4737 inodetocatinode.update({finode: curinode});
4738 if(altinode):
4739 fcurinode = finode;
4740 else:
4741 fcurinode = curinode;
4742 curinode = curinode + 1;
4743 else:
4744 fcurinode = curinode;
4745 curinode = curinode + 1;
4746 curfid = curfid + 1;
4747 if(ftype==2):
4748 flinkname = os.readlink(fname);
4749 fdev = fstatinfo.st_dev;
4750 getfdev = GetDevMajorMinor(fdev);
4751 fdev_minor = getfdev[0];
4752 fdev_major = getfdev[1];
4753 frdev = fstatinfo.st_dev;
4754 if(hasattr(fstatinfo, "st_rdev")):
4755 frdev = fstatinfo.st_rdev;
4756 else:
4757 frdev = fstatinfo.st_dev;
4758 getfrdev = GetDevMajorMinor(frdev);
4759 frdev_minor = getfrdev[0];
4760 frdev_major = getfrdev[1];
4761 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
4762 fsize = "0";
4763 if(ftype==0 or ftype==7):
4764 fsize = fstatinfo.st_size;
4765 fatime = fstatinfo.st_atime;
4766 fmtime = fstatinfo.st_mtime;
4767 fctime = fstatinfo.st_ctime;
4768 if(hasattr(fstatinfo, "st_birthtime")):
4769 fbtime = fstatinfo.st_birthtime;
4770 else:
4771 fbtime = fstatinfo.st_ctime;
4772 fmode = fstatinfo.st_mode;
4773 fchmode = stat.S_IMODE(fstatinfo.st_mode);
4774 ftypemod = stat.S_IFMT(fstatinfo.st_mode);
4775 fuid = fstatinfo.st_uid;
4776 fgid = fstatinfo.st_gid;
4777 funame = "";
4778 try:
4779 import pwd;
4780 try:
4781 userinfo = pwd.getpwuid(fstatinfo.st_uid);
4782 funame = userinfo.pw_name;
4783 except KeyError:
4784 funame = "";
4785 except ImportError:
4786 funame = "";
4787 fgname = "";
4788 try:
4789 import grp;
4790 try:
4791 groupinfo = grp.getgrgid(fstatinfo.st_gid);
4792 fgname = groupinfo.gr_name;
4793 except KeyError:
4794 fgname = "";
4795 except ImportError:
4796 fgname = "";
4797 fdev_minor = fdev_minor;
4798 fdev_major = fdev_major;
4799 frdev_minor = frdev_minor;
4800 frdev_major = frdev_major;
4801 finode = finode;
4802 flinkcount = flinkcount;
4803 if(hasattr(fstatinfo, "st_file_attributes")):
4804 fwinattributes = fstatinfo.st_file_attributes;
4805 else:
4806 fwinattributes = 0;
4807 fcontents = BytesIO();
4808 if(ftype==0 or ftype==7):
4809 with open(fname, "rb") as fpc:
4810 shutil.copyfileobj(fpc, fcontents);
4811 if(followlink and (ftype==1 or ftype==2)):
4812 flstatinfo = os.stat(flinkname);
4813 with open(flinkname, "rb") as fpc:
4814 shutil.copyfileobj(fpc, fcontents);
4815 fcontents.seek(0, 0);
4816 ftypehex = format(ftype, 'x').lower();
4817 extrafields = len(extradata);
4818 extrafieldslist = extradata;
4819 catfextrafields = extrafields;
4820 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
4821 if(len(extradata)>0):
4822 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
4823 extrasizelen = len(extrasizestr);
4824 extrasizelenhex = format(extrasizelen, 'x').lower();
4825 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
4826 catoutlen = len(catoutlist) + len(extradata) + 3;
4827 catoutlenhex = format(catoutlen, 'x').lower();
4828 catoutlist.insert(0, catoutlenhex);
4829 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
4830 if(len(extradata)>0):
4831 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
4832 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
4833 catfnumfields = catoutlen;
4834 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
4835 fcontents.seek(0, 0);
4836 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
4837 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
4838 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
4839 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
4840 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
4841 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
4842 catfileoutstrecd = catfileoutstr.encode('UTF-8');
4843 nullstrecd = formatspecs[5].encode('UTF-8');
4844 fheadtell += len(catfileoutstr) + 1;
4845 catfcontentend = fheadtell - 1;
4846 fcontents.seek(0, 0);
4847 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
4848 pyhascontents = False;
4849 if(int(fsize)>0 and not listonly):
4850 pyhascontents = True;
4851 if(int(fsize)>0 and listonly):
4852 fcontents = BytesIO();
4853 pyhascontents = False;
4854 fcontents.seek(0, 0);
4855 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
4856 fileidnum = fileidnum + 1;
4857 return catlist;
4858
4859 def TarFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
4860 curinode = 0;
4861 curfid = 0;
4862 inodelist = [];
4863 inodetofile = {};
4864 filetoinode = {};
4865 inodetocatinode = {};
4866 fileidnum = 0;
4867 if(infile=="-"):
4868 infile = BytesIO();
4869 if(hasattr(sys.stdin, "buffer")):
4870 shutil.copyfileobj(sys.stdin.buffer, infile);
4871 else:
4872 shutil.copyfileobj(sys.stdin, infile);
4873 infile.seek(0, 0);
4874 if(not infile):
4875 return False;
4876 infile.seek(0, 0);
4877 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4878 infile = download_file_from_internet_file(infile);
4879 infile.seek(0, 0);
4880 if(not infile):
4881 return False;
4882 infile.seek(0, 0);
4883 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
4884 return False;
4885 elif(os.path.exists(infile) and os.path.isfile(infile)):
4886 try:
4887 if(not tarfile.is_tarfile(infile)):
4888 return False;
4889 except AttributeError:
4890 if(not is_tarfile(infile)):
4891 return False;
4892 else:
4893 return False;
4894 try:
4895 if(hasattr(infile, "read") or hasattr(infile, "write")):
4896 tarfp = tarfile.open(fileobj=infile, mode="r");
4897 else:
4898 tarfp = tarfile.open(infile, "r");
4899 except FileNotFoundError:
4900 return False;
4901 fnumfiles = int(len(tarfp.getmembers()));
4902 catver = formatspecs[6];
4903 fileheaderver = str(int(catver.replace(".", "")));
4904 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
4905 fnumfileshex = format(int(fnumfiles), 'x').lower();
4906 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs[5]);
4907 catversion = re.findall(r"([\d]+)$", fileheader);
4908 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
4909 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4910 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
4911 fheadtell = len(fileheader);
4912 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
4913 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
4914 if(re.findall("^[.|/]", member.name)):
4915 fname = member.name;
4916 else:
4917 fname = "./"+member.name;
4918 if(verbose):
4919 VerbosePrintOut(fname);
4920 fpremode = member.mode;
4921 ffullmode = member.mode;
4922 flinkcount = 0;
4923 ftype = 0;
4924 if(member.isreg()):
4925 ffullmode = member.mode + stat.S_IFREG;
4926 ftype = 0;
4927 elif(member.isdev()):
4928 ffullmode = member.mode;
4929 ftype = 7;
4930 elif(member.islnk()):
4931 ffullmode = member.mode + stat.S_IFREG;
4932 ftype = 1;
4933 elif(member.issym()):
4934 ffullmode = member.mode + stat.S_IFLNK;
4935 ftype = 2;
4936 elif(member.ischr()):
4937 ffullmode = member.mode + stat.S_IFCHR;
4938 ftype = 3;
4939 elif(member.isblk()):
4940 ffullmode = member.mode + stat.S_IFBLK;
4941 ftype = 4;
4942 elif(member.isdir()):
4943 ffullmode = member.mode + stat.S_IFDIR;
4944 ftype = 5;
4945 elif(member.isfifo()):
4946 ffullmode = member.mode + stat.S_IFIFO;
4947 ftype = 6;
4948 elif(member.issparse()):
4949 ffullmode = member.mode;
4950 ftype = 12;
4951 else:
4952 ffullmode = member.mode;
4953 ftype = 0;
4954 flinkname = "";
4955 fbasedir = os.path.dirname(fname);
4956 fcurfid = curfid;
4957 fcurinode = curfid;
4958 finode = fcurinode;
4959 curfid = curfid + 1;
4960 if(ftype==2):
4961 flinkname = member.linkname;
4962 fdev_minor = member.devminor;
4963 fdev_major = member.devmajor;
4964 frdev_minor = member.devminor;
4965 frdev_major = member.devmajor;
4966 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
4967 fsize = "0";
4968 elif(ftype==0 or ftype==7):
4969 fsize = member.size;
4970 else:
4971 fsize = member.size;
4972 fatime = member.mtime;
4973 fmtime = member.mtime;
4974 fctime = member.mtime;
4975 fbtime = member.mtime;
4976 fmode = ffullmode;
4977 fchmode = stat.S_IMODE(ffullmode);
4978 ftypemod = stat.S_IFMT(ffullmode);
4979 fuid = member.uid;
4980 fgid = member.gid;
4981 funame = member.uname;
4982 fgname = member.gname;
4983 flinkcount = flinkcount;
4984 fwinattributes = int(0);
4985 fcontents = BytesIO();
4986 if(ftype==0 or ftype==7):
4987 with tarfp.extractfile(member) as fpc:
4988 shutil.copyfileobj(fpc, fcontents);
4989 fcontents.seek(0, 0);
4990 ftypehex = format(ftype, 'x').lower();
4991 extrafields = len(extradata);
4992 extrafieldslist = extradata;
4993 catfextrafields = extrafields;
4994 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
4995 if(len(extradata)>0):
4996 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
4997 extrasizelen = len(extrasizestr);
4998 extrasizelenhex = format(extrasizelen, 'x').lower();
4999 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5000 catoutlen = len(catoutlist) + len(extradata) + 3;
5001 catoutlenhex = format(catoutlen, 'x').lower();
5002 catoutlist.insert(0, catoutlenhex);
5003 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
5004 if(len(extradata)>0):
5005 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
5006 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
5007 catfnumfields = catoutlen;
5008 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5009 fcontents.seek(0, 0);
5010 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5011 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5012 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5013 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
5014 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5015 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5016 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5017 nullstrecd = formatspecs[5].encode('UTF-8');
5018 fheadtell += len(catfileoutstr) + 1;
5019 catfcontentend = fheadtell - 1;
5020 fcontents.seek(0, 0);
5021 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5022 pyhascontents = False;
5023 if(int(fsize)>0 and not listonly):
5024 pyhascontents = True;
5025 if(int(fsize)>0 and listonly):
5026 fcontents = "";
5027 pyhascontents = False;
5028 fcontents.seek(0, 0);
5029 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5030 fileidnum = fileidnum + 1;
5031 return catlist;
5032
5033 def ZipFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5034 curinode = 0;
5035 curfid = 0;
5036 inodelist = [];
5037 inodetofile = {};
5038 filetoinode = {};
5039 inodetocatinode = {};
5040 fileidnum = 0;
5041 if(infile=="-"):
5042 infile = BytesIO();
5043 if(hasattr(sys.stdin, "buffer")):
5044 shutil.copyfileobj(sys.stdin.buffer, infile);
5045 else:
5046 shutil.copyfileobj(sys.stdin, infile);
5047 infile.seek(0, 0);
5048 if(not infile):
5049 return False;
5050 infile.seek(0, 0);
5051 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
5052 infile = download_file_from_internet_file(infile);
5053 infile.seek(0, 0);
5054 if(not infile):
5055 return False;
5056 infile.seek(0, 0);
5057 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
5058 return False;
5059 else:
5060 return False;
5061 if(not zipfile.is_zipfile(infile)):
5062 return False;
5063 try:
5064 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
5065 except FileNotFoundError:
5066 return False;
5067 ziptest = zipfp.testzip();
5068 if(ziptest):
5069 VerbosePrintOut("Bad file found!");
5070 fnumfiles = int(len(zipfp.infolist()));
5071 catver = formatspecs[6];
5072 fileheaderver = str(int(catver.replace(".", "")));
5073 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
5074 catversion = re.findall(r"([\d]+)$", fileheader);
5075 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
5076 fnumfileshex = format(int(fnumfiles), 'x').lower();
5077 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs[5]);
5078 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
5079 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
5080 fheadtell = len(fileheader);
5081 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5082 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
5083 if(re.findall("^[.|/]", member.filename)):
5084 fname = member.filename;
5085 else:
5086 fname = "./"+member.filename;
5087 zipinfo = zipfp.getinfo(member.filename);
5088 if(verbose):
5089 VerbosePrintOut(fname);
5090 if(not member.is_dir()):
5091 fpremode = stat.S_IFREG + 438;
5092 elif(member.is_dir()):
5093 fpremode = stat.S_IFDIR + 511;
5094 flinkcount = 0;
5095 ftype = 0;
5096 if(not member.is_dir()):
5097 ftype = 0;
5098 elif(member.is_dir()):
5099 ftype = 5;
5100 flinkname = "";
5101 fbasedir = os.path.dirname(fname);
5102 fcurfid = curfid;
5103 fcurinode = curfid;
5104 finode = fcurinode;
5105 curfid = curfid + 1;
5106 fdev_minor = 0;
5107 fdev_major = 0;
5108 frdev_minor = 0;
5109 frdev_major = 0;
5110 if(ftype==5):
5111 fsize = "0";
5112 elif(ftype==0):
5113 fsize = member.file_size;
5114 else:
5115 fsize = member.file_size;
5116 fatime = time.mktime(member.date_time + (0, 0, -1));
5117 fmtime = time.mktime(member.date_time + (0, 0, -1));
5118 fctime = time.mktime(member.date_time + (0, 0, -1));
5119 fbtime = time.mktime(member.date_time + (0, 0, -1));
5120 if(zipinfo.create_system==0 or zipinfo.create_system==10):
5121 fwinattributes = int(zipinfo.external_attr);
5122 if(not member.is_dir()):
5123 fmode = int(stat.S_IFREG + 438);
5124 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
5125 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
5126 elif(member.is_dir()):
5127 fmode = int(stat.S_IFDIR + 511);
5128 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
5129 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
5130 elif(zipinfo.create_system==3):
5131 fwinattributes = int(0);
5132 fmode = int(zipinfo.external_attr);
5133 else:
5134 fwinattributes = int(0);
5135 if(not member.is_dir()):
5136 fmode = int(stat.S_IFREG + 438);
5137 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
5138 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
5139 elif(member.is_dir()):
5140 fmode = int(stat.S_IFDIR + 511);
5141 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
5142 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
5143 try:
5144 fuid = os.getuid();
5145 except AttributeError:
5146 fuid = 0;
5147 except KeyError:
5148 fuid = 0;
5149 try:
5150 fgid = os.getgid();
5151 except AttributeError:
5152 fgid = 0;
5153 except KeyError:
5154 fgid = 0;
5155 try:
5156 import pwd;
5157 try:
5158 userinfo = pwd.getpwuid(os.getuid());
5159 funame = userinfo.pw_name;
5160 except KeyError:
5161 funame = "";
5162 except AttributeError:
5163 funame = "";
5164 except ImportError:
5165 funame = "";
5166 fgname = "";
5167 try:
5168 import grp;
5169 try:
5170 groupinfo = grp.getgrgid(os.getgid());
5171 fgname = groupinfo.gr_name;
5172 except KeyError:
5173 fgname = "";
5174 except AttributeError:
5175 fgname = "";
5176 except ImportError:
5177 fgname = "";
5178 fcontents = BytesIO();
5179 if(ftype==0):
5180 fcontents.write(zipfp.read(member.filename));
5181 fcontents.seek(0, 0);
5182 ftypehex = format(ftype, 'x').lower();
5183 extrafields = len(extradata);
5184 extrafieldslist = extradata;
5185 catfextrafields = extrafields;
5186 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
5187 if(len(extradata)>0):
5188 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
5189 extrasizelen = len(extrasizestr);
5190 extrasizelenhex = format(extrasizelen, 'x').lower();
5191 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5192 catoutlen = len(catoutlist) + len(extradata) + 3;
5193 catoutlenhex = format(catoutlen, 'x').lower();
5194 catoutlist.insert(0, catoutlenhex);
5195 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
5196 if(len(extradata)>0):
5197 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
5198 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
5199 catfnumfields = catoutlen;
5200 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5201 fcontents.seek(0, 0);
5202 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5203 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5204 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5205 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
5206 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5207 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5208 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5209 nullstrecd = formatspecs[5].encode('UTF-8');
5210 fheadtell += len(catfileoutstr) + 1;
5211 catfcontentend = fheadtell - 1;
5212 fcontents.seek(0, 0);
5213 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5214 pyhascontents = False;
5215 if(int(fsize)>0 and not listonly):
5216 pyhascontents = True;
5217 if(int(fsize)>0 and listonly):
5218 fcontents = BytesIO();
5219 pyhascontents = False;
5220 fcontents.seek(0, 0);
5221 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5222 fileidnum = fileidnum + 1;
5223 return catlist;
5224
5225 if(not rarfile_support):
5226 def RarFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5227 return False;
5228
5229 if(rarfile_support):
5230 def RarFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5231 curinode = 0;
5232 curfid = 0;
5233 inodelist = [];
5234 inodetofile = {};
5235 filetoinode = {};
5236 inodetocatinode = {};
5237 fileidnum = 0;
5238 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
5239 return False;
5240 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
5241 return False;
5242 rarfp = rarfile.RarFile(infile, "r");
5243 rartest = rarfp.testrar();
5244 if(rartest):
5245 VerbosePrintOut("Bad file found!");
5246 fnumfiles = int(len(rarfp.infolist()));
5247 catver = formatspecs[6];
5248 fileheaderver = str(int(catver.replace(".", "")));
5249 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
5250 catversion = re.findall(r"([\d]+)$", fileheader);
5251 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
5252 fnumfileshex = format(int(fnumfiles), 'x').lower();
5253 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs[5]);
5254 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
5255 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
5256 fheadtell = len(fileheader);
5257 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5258 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
5259 is_unix = False;
5260 is_windows = False;
5261 if(member.host_os==rarfile.RAR_OS_UNIX):
5262 is_windows = False;
5263 try:
5264 member.external_attr
5265 is_unix = True;
5266 except AttributeError:
5267 is_unix = False;
5268 elif(member.host_os==rarfile.RAR_OS_WIN32):
5269 is_unix = False;
5270 try:
5271 member.external_attr
5272 is_windows = True;
5273 except AttributeError:
5274 is_windows = False;
5275 else:
5276 is_unix = False;
5277 is_windows = False;
5278 if(re.findall("^[.|/]", member.filename)):
5279 fname = member.filename;
5280 else:
5281 fname = "./"+member.filename;
5282 rarinfo = rarfp.getinfo(member.filename);
5283 if(verbose):
5284 VerbosePrintOut(fname);
5285 if(is_unix and member.external_attr !=0):
5286 fpremode = int(member.external_attr);
5287 elif(member.is_file()):
5288 fpremode = stat.S_IFREG + 438;
5289 elif(member.is_symlink()):
5290 fpremode = stat.S_IFLNK + 438;
5291 elif(member.is_dir()):
5292 fpremode = stat.S_IFDIR + 511;
5293 if(is_windows and member.external_attr !=0):
5294 fwinattributes = int(member.external_attr);
5295 else:
5296 fwinattributes = int(0);
5297 flinkcount = 0;
5298 ftype = 0;
5299 if(member.is_file()):
5300 ftype = 0;
5301 elif(member.is_symlink()):
5302 ftype = 2;
5303 elif(member.is_dir()):
5304 ftype = 5;
5305 flinkname = "";
5306 if(ftype==2):
5307 flinkname = rarfp.read(member.filename).decode("UTF-8");
5308 fbasedir = os.path.dirname(fname);
5309 fcurfid = curfid;
5310 fcurinode = curfid;
5311 finode = fcurinode;
5312 curfid = curfid + 1;
5313 fdev_minor = 0;
5314 fdev_major = 0;
5315 frdev_minor = 0;
5316 frdev_major = 0;
5317 if(ftype==5):
5318 fsize = "0";
5319 if(ftype==0):
5320 fsize = member.file_size;
5321 try:
5322 if(member.atime):
5323 fatime = int(member.atime.timestamp());
5324 else:
5325 fatime = int(member.mtime.timestamp());
5326 except AttributeError:
5327 fatime = int(member.mtime.timestamp());
5328 fmtime = int(member.mtime.timestamp());
5329 try:
5330 if(member.ctime):
5331 fctime = int(member.ctime.timestamp());
5332 else:
5333 fctime = int(member.mtime.timestamp());
5334 except AttributeError:
5335 fctime = int(member.mtime.timestamp());
5336 fbtime = int(member.mtime.timestamp());
5337 if(is_unix and member.external_attr !=0):
5338 fmode = int(member.external_attr);
5339 fchmode = int(stat.S_IMODE(member.external_attr));
5340 ftypemod = int(stat.S_IFMT(member.external_attr));
5341 elif(member.is_file()):
5342 fmode = int(stat.S_IFREG + 438)
5343 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5344 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5345 elif(member.is_symlink()):
5346 fmode = int(stat.S_IFLNK + 438)
5347 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5348 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5349 elif(member.is_dir()):
5350 fmode = int(stat.S_IFDIR + 511)
5351 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
5352 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
5353 try:
5354 fuid = os.getuid();
5355 except AttributeError:
5356 fuid = 0;
5357 except KeyError:
5358 fuid = 0;
5359 try:
5360 fgid = os.getgid();
5361 except AttributeError:
5362 fgid = 0;
5363 except KeyError:
5364 fgid = 0;
5365 try:
5366 import pwd;
5367 try:
5368 userinfo = pwd.getpwuid(os.getuid());
5369 funame = userinfo.pw_name;
5370 except KeyError:
5371 funame = "";
5372 except AttributeError:
5373 funame = "";
5374 except ImportError:
5375 funame = "";
5376 fgname = "";
5377 try:
5378 import grp;
5379 try:
5380 groupinfo = grp.getgrgid(os.getgid());
5381 fgname = groupinfo.gr_name;
5382 except KeyError:
5383 fgname = "";
5384 except AttributeError:
5385 fgname = "";
5386 except ImportError:
5387 fgname = "";
5388 fcontents = BytesIO();
5389 if(ftype==0):
5390 fcontents.write(rarfp.read(member.filename));
5391 fcontents.seek(0, 0);
5392 ftypehex = format(ftype, 'x').lower();
5393 extrafields = len(extradata);
5394 extrafieldslist = extradata;
5395 catfextrafields = extrafields;
5396 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
5397 if(len(extradata)>0):
5398 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
5399 extrasizelen = len(extrasizestr);
5400 extrasizelenhex = format(extrasizelen, 'x').lower();
5401 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5402 catoutlen = len(catoutlist) + len(extradata) + 3;
5403 catoutlenhex = format(catoutlen, 'x').lower();
5404 catoutlist.insert(0, catoutlenhex);
5405 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
5406 if(len(extradata)>0):
5407 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
5408 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
5409 catfnumfields = 24 + catfextrafields;
5410 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5411 fcontents.seek(0, 0);
5412 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5413 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5414 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5415 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
5416 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5417 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5418 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5419 nullstrecd = formatspecs[5].encode('UTF-8');
5420 fheadtell += len(catfileoutstr) + 1;
5421 catfcontentend = fheadtell - 1;
5422 fcontents.seek(0, 0);
5423 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5424 pyhascontents = False;
5425 if(int(fsize)>0 and not listonly):
5426 pyhascontents = True;
5427 if(int(fsize)>0 and listonly):
5428 fcontents = BytesIO();
5429 pyhascontents = False;
5430 fcontents.seek(0, 0);
5431 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5432 fileidnum = fileidnum + 1;
5433 return catlist;
5434
5435 if(not py7zr_support):
5436 def SevenZipFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5437 return False;
5438
5439 if(py7zr_support):
5440 def SevenZipFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5441 curinode = 0;
5442 curfid = 0;
5443 inodelist = [];
5444 inodetofile = {};
5445 filetoinode = {};
5446 inodetocatinode = {};
5447 fileidnum = 0;
5448 szpfp = py7zr.SevenZipFile(infile, mode="r");
5449 file_content = szpfp.readall();
5450 #sztest = szpfp.testzip();
5451 sztestalt = szpfp.test();
5452 if(sztestalt):
5453 VerbosePrintOut("Bad file found!");
5454 numfiles = int(len(szpfp.list()));
5455 catver = formatspecs[6];
5456 fileheaderver = str(int(catver.replace(".", "")));
5457 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
5458 catversion = re.findall(r"([\d]+)$", fileheader);
5459 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
5460 fnumfileshex = format(int(fnumfiles), 'x').lower();
5461 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs[5]);
5462 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
5463 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
5464 fheadtell = len(fileheader);
5465 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5466 for member in sorted(szpfp.list(), key=lambda x: x.filename):
5467 if(re.findall("^[.|/]", member.filename)):
5468 fname = member.filename;
5469 else:
5470 fname = "./"+member.filename;
5471 if(not member.is_directory):
5472 fpremode = int(stat.S_IFREG + 438);
5473 elif(member.is_directory):
5474 fpremode = int(stat.S_IFDIR + 511);
5475 fwinattributes = int(0);
5476 flinkcount = 0;
5477 ftype = 0;
5478 if(member.is_directory):
5479 ftype = 5;
5480 else:
5481 ftype = 0;
5482 flinkname = "";
5483 fbasedir = os.path.dirname(fname);
5484 fcurfid = curfid;
5485 fcurinode = curfid;
5486 finode = fcurinode;
5487 curfid = curfid + 1;
5488 fdev_minor = 0;
5489 fdev_major = 0;
5490 frdev_minor = 0;
5491 frdev_major = 0;
5492 if(ftype==5):
5493 fsize = "0";
5494 fatime = int(member.creationtime.timestamp());
5495 fmtime = int(member.creationtime.timestamp());
5496 fctime = int(member.creationtime.timestamp());
5497 fbtime = int(member.creationtime.timestamp());
5498 if(member.is_directory):
5499 fmode = int(stat.S_IFDIR + 511)
5500 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
5501 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
5502 else:
5503 fmode = int(stat.S_IFLNK + 438)
5504 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5505 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5506 try:
5507 fuid = os.getuid();
5508 except AttributeError:
5509 fuid = 0;
5510 except KeyError:
5511 fuid = 0;
5512 try:
5513 fgid = os.getgid();
5514 except AttributeError:
5515 fgid = 0;
5516 except KeyError:
5517 fgid = 0;
5518 try:
5519 import pwd;
5520 try:
5521 userinfo = pwd.getpwuid(os.getuid());
5522 funame = userinfo.pw_name;
5523 except KeyError:
5524 funame = "";
5525 except AttributeError:
5526 funame = "";
5527 except ImportError:
5528 funame = "";
5529 fgname = "";
5530 try:
5531 import grp;
5532 try:
5533 groupinfo = grp.getgrgid(os.getgid());
5534 fgname = groupinfo.gr_name;
5535 except KeyError:
5536 fgname = "";
5537 except AttributeError:
5538 fgname = "";
5539 except ImportError:
5540 fgname = "";
5541 fcontents = BytesIO();
5542 if(ftype==0):
5543 fcontents.write(file_content[member.filename].read());
5544 fsize = format(fcontents.tell(), 'x').lower();
5545 fileop.close();
5546 fcontents.seek(0, 0);
5547 ftypehex = format(ftype, 'x').lower();
5548 extrafields = len(extradata);
5549 extrafieldslist = extradata;
5550 catfextrafields = extrafields;
5551 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
5552 if(len(extradata)>0):
5553 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
5554 extrasizelen = len(extrasizestr);
5555 extrasizelenhex = format(extrasizelen, 'x').lower();
5556 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5557 catoutlen = len(catoutlist) + len(extradata) + 3;
5558 catoutlenhex = format(catoutlen, 'x').lower();
5559 catoutlist.insert(0, catoutlenhex);
5560 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
5561 if(len(extradata)>0):
5562 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
5563 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
5564 catfnumfields = 24 + catfextrafields;
5565 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5566 fcontents.seek(0, 0);
5567 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5568 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5569 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5570 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
5571 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5572 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5573 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5574 nullstrecd = formatspecs[5].encode('UTF-8');
5575 fheadtell += len(catfileoutstr) + 1;
5576 catfcontentend = fheadtell - 1;
5577 fcontents.seek(0, 0);
5578 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5579 pyhascontents = False;
5580 if(int(fsize)>0 and not listonly):
5581 pyhascontents = True;
5582 if(int(fsize)>0 and listonly):
5583 fcontents = "";
5584 pyhascontents = False;
5585 fcontents.seek(0, 0);
5586 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5587 fileidnum = fileidnum + 1;
5588 return catlist;
5589
5590 def InFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5591 checkcompressfile = CheckCompressionSubType(infiles, formatspecs, True);
5592 if(checkcompressfile=="tarfile"):
5593 return TarFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5594 elif(checkcompressfile=="zipfile"):
5595 return ZipFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5596 elif(checkcompressfile=="catfile"):
5597 return ArchiveFileToArray(infile, 0, 0, listonly, False, formatspecs, False);
5598 elif(rarfile_support and checkcompressfile=="rarfile"):
5599 return RarFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5600 elif(py7zr_support and checkcompressfile=="7zipfile"):
5601 return SevenZipFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5602 else:
5603 return False;
5604 return False;
5605
5606 def ListDirToArray(infiles, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, seekstart=0, seekend=0, listonly=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
5607 outarray = BytesIO();
5608 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
5609 listcatfiles = ArchiveFileToArray(outarray, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5610 return listcatfiles;
5611
5612 def ArchiveFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5613 if(isinstance(infile, dict)):
5614 listcatfiles = infile;
5615 else:
5616 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
5617 infile = RemoveWindowsPath(infile);
5618 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5619 if(not listcatfiles):
5620 return False;
5621 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5622 if(returnfp):
5623 catarray.update({'catfp': listcatfiles['catfp']});
5624 lenlist = len(listcatfiles['ffilelist']);
5625 lcfi = 0;
5626 lcfx = int(listcatfiles['fnumfiles']);
5627 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5628 lcfx = int(lenlist);
5629 else:
5630 lcfx = int(listcatfiles['fnumfiles']);
5631 while(lcfi < lcfx):
5632 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5633 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5634 catarray['filetoid'].update(filetoidarray);
5635 catarray['idtofile'].update(idtofilearray);
5636 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5637 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5638 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5639 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5640 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5641 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5642 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5643 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5644 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5645 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5646 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5647 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5648 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5649 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5650 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5651 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5652 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5653 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5654 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5655 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5656 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5657 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5658 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5659 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5660 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5661 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5662 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
5663 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5664 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5665 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5666 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5667 lcfi = lcfi + 1;
5668 return catarray;
5669
5670 create_alias_function("", __file_format_name__, "ToArrayIndex", ArchiveFileToArrayIndex);
5671
5672 def ListDirToArrayIndexAlt(infiles, dirlistfromtxt=False, followlink=False, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5673 listcatfiles = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, listonly, checksumtype, extradata, formatspecs, verbose);
5674 if(not listcatfiles):
5675 return False;
5676 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5677 lenlist = len(listcatfiles['ffilelist']);
5678 lcfi = 0;
5679 lcfx = int(listcatfiles['fnumfiles']);
5680 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5681 lcfx = int(lenlist);
5682 else:
5683 lcfx = int(listcatfiles['fnumfiles']);
5684 while(lcfi < lcfx):
5685 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5686 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5687 catarray['filetoid'].update(filetoidarray);
5688 catarray['idtofile'].update(idtofilearray);
5689 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5690 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5691 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5692 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5693 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5694 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5695 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5696 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5697 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5698 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5699 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5700 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5701 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5702 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5703 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5704 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5705 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5706 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5707 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5708 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5709 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5710 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5711 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5712 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5713 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5714 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5715 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
5716 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5717 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5718 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5719 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5720 lcfi = lcfi + 1;
5721 return catarray;
5722
5723 def TarFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5724 listcatfiles = TarFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5725 if(not listcatfiles):
5726 return False;
5727 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5728 lenlist = len(listcatfiles['ffilelist']);
5729 lcfi = 0;
5730 lcfx = int(listcatfiles['fnumfiles']);
5731 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5732 lcfx = int(lenlist);
5733 else:
5734 lcfx = int(listcatfiles['fnumfiles']);
5735 while(lcfi < lcfx):
5736 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5737 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5738 catarray['filetoid'].update(filetoidarray);
5739 catarray['idtofile'].update(idtofilearray);
5740 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5741 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5742 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5743 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5744 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5745 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5746 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5747 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5748 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5749 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5750 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5751 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5752 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5753 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5754 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5755 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5756 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5757 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5758 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5759 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5760 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5761 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5762 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5763 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5764 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5765 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5766 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
5767 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5768 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5769 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5770 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5771 lcfi = lcfi + 1;
5772 return catarray;
5773
5774 def ZipFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5775 listcatfiles = ZipFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5776 if(not listcatfiles):
5777 return False;
5778 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5779 lenlist = len(listcatfiles['ffilelist']);
5780 lcfi = 0;
5781 lcfx = int(listcatfiles['fnumfiles']);
5782 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5783 lcfx = int(lenlist);
5784 else:
5785 lcfx = int(listcatfiles['fnumfiles']);
5786 while(lcfi < lcfx):
5787 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5788 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5789 catarray['filetoid'].update(filetoidarray);
5790 catarray['idtofile'].update(idtofilearray);
5791 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5792 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5793 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5794 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5795 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5796 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5797 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5798 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5799 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5800 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5801 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5802 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5803 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5804 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5805 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5806 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5807 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5808 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5809 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5810 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5811 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5812 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5813 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5814 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5815 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5816 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5817 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
5818 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5819 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5820 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5821 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5822 lcfi = lcfi + 1;
5823 return catarray;
5824
5825 if(not rarfile_support):
5826 def RarFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5827 return False;
5828
5829 if(rarfile_support):
5830 def RarFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5831 listcatfiles = RarFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5832 if(not listcatfiles):
5833 return False;
5834 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5835 lenlist = len(listcatfiles['ffilelist']);
5836 lcfi = 0;
5837 lcfx = int(listcatfiles['fnumfiles']);
5838 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5839 lcfx = int(lenlist);
5840 else:
5841 lcfx = int(listcatfiles['fnumfiles']);
5842 while(lcfi < lcfx):
5843 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5844 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5845 catarray['filetoid'].update(filetoidarray);
5846 catarray['idtofile'].update(idtofilearray);
5847 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5848 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5849 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5850 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5851 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5852 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5853 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5854 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5855 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5856 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5857 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5858 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5859 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5860 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5861 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5862 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5863 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5864 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5865 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5866 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5867 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5868 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5869 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5870 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5871 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5872 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5873 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
5874 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5875 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5876 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5877 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5878 lcfi = lcfi + 1;
5879 return catarray;
5880
5881 if(not py7zr_support):
5882 def SevenZipFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5883 return False;
5884
5885 if(py7zr_support):
5886 def SevenZipFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5887 listcatfiles = SevenZipFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5888 if(not listcatfiles):
5889 return False;
5890 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5891 lenlist = len(listcatfiles['ffilelist']);
5892 lcfi = 0;
5893 lcfx = int(listcatfiles['fnumfiles']);
5894 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5895 lcfx = int(lenlist);
5896 else:
5897 lcfx = int(listcatfiles['fnumfiles']);
5898 while(lcfi < lcfx):
5899 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5900 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5901 catarray['filetoid'].update(filetoidarray);
5902 catarray['idtofile'].update(idtofilearray);
5903 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5904 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5905 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5906 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5907 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5908 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5909 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5910 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5911 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5912 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5913 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5914 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5915 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5916 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5917 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5918 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5919 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5920 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5921 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5922 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5923 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5924 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5925 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5926 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5927 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5928 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5929 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
5930 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5931 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5932 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5933 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5934 lcfi = lcfi + 1;
5935 return catarray;
5936
5937 def ArchiveFileStringToArrayIndex(catstr, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5938 catfp = BytesIO(catstr);
5939 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5940 return listcatfiles;
5941
5942 create_alias_function("", __file_format_name__, "StringToArrayIndex", ArchiveFileStringToArrayIndex);
5943
5944 def TarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5945 catfp = BytesIO();
5946 catfp = PackArchiveFileFromTarFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
5947 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5948 return listcatfiles;
5949
5950 def ZipFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5951 catfp = BytesIO();
5952 catfp = PackArchiveFileFromZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
5953 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5954 return listcatfiles;
5955
5956 if(not rarfile_support):
5957 def RarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5958 return False;
5959
5960 if(rarfile_support):
5961 def RarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5962 catfp = BytesIO();
5963 catfp = PackArchiveFileFromRarFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
5964 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5965 return listcatfiles;
5966
5967 if(not py7zr_support):
5968 def SevenZipFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5969 return False;
5970
5971 if(py7zr_support):
5972 def SevenZipFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5973 catfp = BytesIO();
5974 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
5975 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5976 return listcatfiles;
5977
5978 def ListDirToArrayIndex(infiles, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, seekstart=0, seekend=0, listonly=False, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False, returnfp=False):
5979 outarray = BytesIO();
5980 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compressionlevel, followlink, checksumtype, formatspecs, verbose, True);
5981 listcatfiles = ArchiveFileToArrayIndex(outarray, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp)
5982 return listcatfiles;
5983
5984 def RePackArchiveFile(infile, outfile, compression="auto", compressionlevel=None, followlink=False, seekstart=0, seekend=0, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
5985 if(isinstance(infile, dict)):
5986 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
5987 listcatfiles = prelistcatfiles['list'];
5988 else:
5989 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
5990 infile = RemoveWindowsPath(infile);
5991 if(followlink):
5992 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
5993 listcatfiles = prelistcatfiles['list'];
5994 else:
5995 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
5996 if(outfile!="-" and not hasattr(infile, "read") and not hasattr(outfile, "write")):
5997 outfile = RemoveWindowsPath(outfile);
5998 checksumtype = checksumtype.lower();
5999 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
6000 checksumtype="crc32";
6001 if(checksumtype=="none"):
6002 checksumtype = "";
6003 if(not compression or compression=="catfile" or compression==formatspecs[2]):
6004 compression = "auto";
6005 if(compression not in compressionlist and compression is None):
6006 compression = "auto";
6007 if(verbose):
6008 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6009 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
6010 if(os.path.exists(outfile)):
6011 try:
6012 os.unlink(outfile);
6013 except OSError as e:
6014 pass;
6015 if(not listcatfiles):
6016 return False;
6017 if(outfile=="-"):
6018 verbose = False;
6019 catfp = BytesIO();
6020 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
6021 catfp = outfile;
6022 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
6023 catfp = BytesIO();
6024 else:
6025 fbasename = os.path.splitext(outfile)[0];
6026 fextname = os.path.splitext(outfile)[1];
6027 catfp = CompressOpenFile(outfile, True, compressionlevel);
6028 catver = formatspecs[6];
6029 fileheaderver = str(int(catver.replace(".", "")));
6030 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
6031 catfp.write(fileheader.encode('UTF-8'));
6032 lenlist = len(listcatfiles['ffilelist']);
6033 fnumfiles = int(listcatfiles['fnumfiles']);
6034 if(lenlist>fnumfiles or lenlist<fnumfiles):
6035 fnumfiles = lenlist;
6036 fnumfileshex = format(int(fnumfiles), 'x').lower();
6037 fnumfilesa = AppendNullBytes([fnumfileshex, checksumtype], formatspecs[5]);
6038 catfileheadercshex = GetFileChecksum(fileheader + fnumfilesa, checksumtype, True, formatspecs);
6039 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs[5]);
6040 catfp.write(fnumfilesa.encode('UTF-8'));
6041 try:
6042 catfp.flush();
6043 os.fsync(catfp.fileno());
6044 except io.UnsupportedOperation:
6045 pass;
6046 except AttributeError:
6047 pass;
6048 except OSError as e:
6049 pass;
6050 lenlist = len(listcatfiles['ffilelist']);
6051 fnumfiles = int(listcatfiles['fnumfiles']);
6052 lcfi = 0;
6053 lcfx = int(listcatfiles['fnumfiles']);
6054 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6055 lcfx = int(lenlist);
6056 else:
6057 lcfx = int(listcatfiles['fnumfiles']);
6058 curinode = 0;
6059 curfid = 0;
6060 inodelist = [];
6061 inodetofile = {};
6062 filetoinode = {};
6063 reallcfi = 0;
6064 while(lcfi < lcfx):
6065 if(re.findall("^[.|/]", listcatfiles['ffilelist'][reallcfi]['fname'])):
6066 fname = listcatfiles['ffilelist'][reallcfi]['fname'];
6067 else:
6068 fname = "./"+listcatfiles['ffilelist'][reallcfi]['fname'];
6069 if(verbose):
6070 VerbosePrintOut(fname);
6071 fheadersize = format(int(listcatfiles['ffilelist'][reallcfi]['fheadersize']), 'x').lower();
6072 fsize = format(int(listcatfiles['ffilelist'][reallcfi]['fsize']), 'x').lower();
6073 flinkname = listcatfiles['ffilelist'][reallcfi]['flinkname'];
6074 fatime = format(int(listcatfiles['ffilelist'][reallcfi]['fatime']), 'x').lower();
6075 fmtime = format(int(listcatfiles['ffilelist'][reallcfi]['fmtime']), 'x').lower();
6076 fctime = format(int(listcatfiles['ffilelist'][reallcfi]['fctime']), 'x').lower();
6077 fbtime = format(int(listcatfiles['ffilelist'][reallcfi]['fbtime']), 'x').lower();
6078 fmode = format(int(listcatfiles['ffilelist'][reallcfi]['fmode']), 'x').lower();
6079 fchmode = format(int(listcatfiles['ffilelist'][reallcfi]['fchmode']), 'x').lower();
6080 fuid = format(int(listcatfiles['ffilelist'][reallcfi]['fuid']), 'x').lower();
6081 funame = listcatfiles['ffilelist'][reallcfi]['funame'];
6082 fgid = format(int(listcatfiles['ffilelist'][reallcfi]['fgid']), 'x').lower();
6083 fgname = listcatfiles['ffilelist'][reallcfi]['fgname'];
6084 finode = listcatfiles['ffilelist'][reallcfi]['finode'];
6085 flinkcount = listcatfiles['ffilelist'][reallcfi]['flinkcount'];
6086 fwinattributes = flinkinfo['ffilelist'][reallcfi]['fwinattributes'];
6087 fdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['fminor']), 'x').lower();
6088 fdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['fmajor']), 'x').lower();
6089 frdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['frminor']), 'x').lower();
6090 frdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['frmajor']), 'x').lower();
6091 if(len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>listcatfiles['ffilelist'][reallcfi]['fextrafields'] and len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>0):
6092 listcatfiles['ffilelist'][reallcfi]['fextrafields'] = len(listcatfiles['ffilelist'][reallcfi]['fextralist']);
6093 if(len(extradata)>0):
6094 listcatfiles['ffilelist'][reallcfi]['fextrafields'] = len(extradata);
6095 listcatfiles['ffilelist'][reallcfi]['fextralist'] = extradata;
6096 extrafields = format(int(listcatfiles['ffilelist'][reallcfi]['fextrafields']), 'x').lower();
6097 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
6098 if(len(extradata)>0):
6099 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
6100 extrasizelen = format(len(extrasizestr), 'x').lower();
6101 fcontents = listcatfiles['ffilelist'][reallcfi]['fcontents'];
6102 if(followlink):
6103 if(listcatfiles['ffilelist'][reallcfi]['ftype']==1 or listcatfiles['ffilelist'][reallcfi]['ftype']==2):
6104 getflinkpath = listcatfiles['ffilelist'][reallcfi]['flinkname'];
6105 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6106 flinkinfo = listcatfiles['ffilelist'][flinkid];
6107 fheadersize = format(int(flinkinfo['fheadersize']), 'x').lower();
6108 fsize = format(int(flinkinfo['fsize']), 'x').lower();
6109 flinkname = flinkinfo['flinkname'];
6110 fatime = format(int(flinkinfo['fatime']), 'x').lower();
6111 fmtime = format(int(flinkinfo['fmtime']), 'x').lower();
6112 fctime = format(int(flinkinfo['fctime']), 'x').lower();
6113 fbtime = format(int(flinkinfo['fbtime']), 'x').lower();
6114 fmode = format(int(flinkinfo['fmode']), 'x').lower();
6115 fchmode = format(int(flinkinfo['fchmode']), 'x').lower();
6116 fuid = format(int(flinkinfo['fuid']), 'x').lower();
6117 funame = flinkinfo['funame'];
6118 fgid = format(int(flinkinfo['fgid']), 'x').lower();
6119 fgname = flinkinfo['fgname'];
6120 finode = flinkinfo['finode'];
6121 flinkcount = flinkinfo['flinkcount'];
6122 fwinattributes = flinkinfo['fwinattributes'];
6123 fdev_minor = format(int(flinkinfo['fminor']), 'x').lower();
6124 fdev_major = format(int(flinkinfo['fmajor']), 'x').lower();
6125 frdev_minor = format(int(flinkinfo['frminor']), 'x').lower();
6126 frdev_major = format(int(flinkinfo['frmajor']), 'x').lower();
6127 if(len(flinkinfo['fextralist'])>flinkinfo['fextrafields'] and len(flinkinfo['fextralist'])>0):
6128 flinkinfo['fextrafields'] = len(flinkinfo['fextralist']);
6129 if(len(extradata)>0):
6130 flinkinfo['fextrafields'] = len(extradata);
6131 flinkinfo['fextralist'] = extradata;
6132 extrafields = format(int(flinkinfo['fextrafields']), 'x').lower();
6133 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
6134 if(len(extradata)>0):
6135 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
6136 extrasizelen = format(len(extrasizestr), 'x').lower();
6137 fcontents = flinkinfo['fcontents'];
6138 ftypehex = format(flinkinfo['ftype'], 'x').lower();
6139 else:
6140 ftypehex = format(listcatfiles['ffilelist'][reallcfi]['ftype'], 'x').lower();
6141 fcurfid = format(curfid, 'x').lower();
6142 if(not followlink and finode!=0):
6143 if(listcatfiles['ffilelist'][reallcfi]['ftype']!=1):
6144 fcurinode = format(int(curinode), 'x').lower();
6145 inodetofile.update({curinode: fname});
6146 filetoinode.update({fname: curinode});
6147 curinode = curinode + 1;
6148 else:
6149 fcurinode = format(int(filetoinode[flinkname]), 'x').lower();
6150 else:
6151 fcurinode = format(int(curinode), 'x').lower();
6152 curinode = curinode + 1;
6153 curfid = curfid + 1;
6154 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, extrasizelen, extrafields];
6155 catoutlen = len(catoutlist) + len(extradata) + 3;
6156 catoutlenhex = format(catoutlen, 'x').lower();
6157 catoutlist.insert(0, catoutlenhex);
6158 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
6159 if(listcatfiles['ffilelist'][reallcfi]['fextrafields']>0):
6160 extrafieldslist = [];
6161 exi = 0;
6162 exil = listcatfiles['ffilelist'][reallcfi]['fextrafields'];
6163 while(exi<exil):
6164 extrafieldslist.append(listcatfiles['ffilelist'][reallcfi]['fextralist']);
6165 exi = exi + 1;
6166 catfileoutstr += AppendNullBytes([extrafieldslist], formatspecs[5]);
6167 catfileoutstr += AppendNullBytes([checksumtype], formatspecs[5]);
6168 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
6169 fcontents.seek(0, 0);
6170 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
6171 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
6172 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
6173 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
6174 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
6175 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
6176 catfileoutstrecd = catfileoutstr.encode('UTF-8');
6177 nullstrecd = formatspecs[5].encode('UTF-8');
6178 fcontents.seek(0, 0);
6179 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
6180 catfp.write(catfileout);
6181 try:
6182 catfp.flush();
6183 os.fsync(catfp.fileno());
6184 except io.UnsupportedOperation:
6185 pass;
6186 except AttributeError:
6187 pass;
6188 except OSError as e:
6189 pass;
6190 lcfi = lcfi + 1;
6191 reallcfi = reallcfi + 1;
6192 if(lcfx>0):
6193 catfp.write(AppendNullBytes([0, 0], formatspecs[5]).encode("UTF-8"));
6194 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
6195 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
6196 try:
6197 catfp.flush();
6198 os.fsync(catfp.fileno());
6199 except io.UnsupportedOperation:
6200 pass;
6201 except AttributeError:
6202 pass;
6203 except OSError as e:
6204 pass;
6205 if(outfile=="-"):
6206 catfp.seek(0, 0);
6207 if(hasattr(sys.stdout, "buffer")):
6208 shutil.copyfileobj(catfp, sys.stdout.buffer);
6209 else:
6210 shutil.copyfileobj(catfp, sys.stdout);
6211 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
6212 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
6213 catfp.seek(0, 0);
6214 upload_file_to_internet_file(catfp, outfile);
6215 if(returnfp):
6216 catfp.seek(0, 0);
6217 return catfp;
6218 else:
6219 catfp.close();
6220 return True;
6221
6222 create_alias_function("RePack", __file_format_name__, "", RePackArchiveFile);
6223
6224 def RePackArchiveFileFromString(catstr, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
6225 catfp = BytesIO(catstr);
6226 listcatfiles = RePackArchiveFile(catfp, compression, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6227 return listcatfiles;
6228
6229 create_alias_function("RePack", __file_format_name__, "FromString", RePackArchiveFileFromString);
6230
6231 def PackArchiveFileFromListDir(infiles, outfile, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
6232 outarray = BytesIO();
6233 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
6234 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6235 return listcatfiles;
6236
6237 create_alias_function("Pack", __file_format_name__, "FromListDir", PackArchiveFileFromListDir);
6238
6239 def UnPackArchiveFile(infile, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_list__, preservepermissions=True, preservetime=True, verbose=False, returnfp=False):
6240 if(outdir is not None):
6241 outdir = RemoveWindowsPath(outdir);
6242 if(verbose):
6243 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6244 if(isinstance(infile, dict)):
6245 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6246 listcatfiles = prelistcatfiles['list'];
6247 else:
6248 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6249 infile = RemoveWindowsPath(infile);
6250 if(followlink):
6251 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6252 listcatfiles = prelistcatfiles['list'];
6253 else:
6254 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6255 if(not listcatfiles):
6256 return False;
6257 lenlist = len(listcatfiles['ffilelist']);
6258 fnumfiles = int(listcatfiles['fnumfiles']);
6259 lcfi = 0;
6260 lcfx = int(listcatfiles['fnumfiles']);
6261 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6262 lcfx = int(lenlist);
6263 else:
6264 lcfx = int(listcatfiles['fnumfiles']);
6265 while(lcfi<lcfx):
6266 funame = "";
6267 try:
6268 import pwd;
6269 try:
6270 userinfo = pwd.getpwuid(listcatfiles['ffilelist'][lcfi]['fuid']);
6271 funame = userinfo.pw_name;
6272 except KeyError:
6273 funame = "";
6274 except ImportError:
6275 funame = "";
6276 fgname = "";
6277 try:
6278 import grp;
6279 try:
6280 groupinfo = grp.getgrgid(listcatfiles['ffilelist'][lcfi]['fgid']);
6281 fgname = groupinfo.gr_name;
6282 except KeyError:
6283 fgname = "";
6284 except ImportError:
6285 fgname = "";
6286 if(verbose):
6287 VerbosePrintOut(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6288 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6289 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6290 listcatfiles['ffilelist'][lcfi]['fcontents'].seek(0, 0);
6291 shutil.copyfileobj(listcatfiles['ffilelist'][lcfi]['fcontents'], fpc);
6292 try:
6293 fpc.flush();
6294 os.fsync(fpc.fileno());
6295 except io.UnsupportedOperation:
6296 pass;
6297 except AttributeError:
6298 pass;
6299 except OSError as e:
6300 pass;
6301 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
6302 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
6303 if(preservepermissions):
6304 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6305 if(preservetime):
6306 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
6307 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6308 if(followlink):
6309 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
6310 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6311 flinkinfo = listcatfiles['ffilelist'][flinkid];
6312 funame = "";
6313 try:
6314 import pwd;
6315 try:
6316 userinfo = pwd.getpwuid(flinkinfo['fuid']);
6317 funame = userinfo.pw_name;
6318 except KeyError:
6319 funame = "";
6320 except ImportError:
6321 funame = "";
6322 fgname = "";
6323 try:
6324 import grp;
6325 try:
6326 groupinfo = grp.getgrgid(flinkinfo['fgid']);
6327 fgname = groupinfo.gr_name;
6328 except KeyError:
6329 fgname = "";
6330 except ImportError:
6331 fgname = "";
6332 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
6333 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6334 flinkinfo['fcontents'].seek(0, 0);
6335 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
6336 try:
6337 fpc.flush();
6338 os.fsync(fpc.fileno());
6339 except io.UnsupportedOperation:
6340 pass;
6341 except AttributeError:
6342 pass;
6343 except OSError as e:
6344 pass;
6345 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6346 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6347 if(preservepermissions):
6348 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6349 if(preservetime):
6350 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6351 if(flinkinfo['ftype']==1):
6352 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6353 if(flinkinfo['ftype']==2):
6354 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6355 if(flinkinfo['ftype']==5):
6356 if(preservepermissions):
6357 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6358 else:
6359 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6360 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6361 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6362 if(preservepermissions):
6363 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6364 if(preservetime):
6365 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6366 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
6367 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6368 else:
6369 os.link(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6370 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6371 if(followlink):
6372 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
6373 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6374 flinkinfo = listcatfiles['ffilelist'][flinkid];
6375 funame = "";
6376 try:
6377 import pwd;
6378 try:
6379 userinfo = pwd.getpwuid(flinkinfo['fuid']);
6380 funame = userinfo.pw_name;
6381 except KeyError:
6382 funame = "";
6383 except ImportError:
6384 funame = "";
6385 fgname = "";
6386 try:
6387 import grp;
6388 try:
6389 groupinfo = grp.getgrgid(flinkinfo['fgid']);
6390 fgname = groupinfo.gr_name;
6391 except KeyError:
6392 fgname = "";
6393 except ImportError:
6394 fgname = "";
6395 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
6396 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6397 flinkinfo['fcontents'].seek(0, 0);
6398 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
6399 try:
6400 fpc.flush();
6401 os.fsync(fpc.fileno());
6402 except io.UnsupportedOperation:
6403 pass;
6404 except AttributeError:
6405 pass;
6406 except OSError as e:
6407 pass;
6408 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6409 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6410 if(preservepermissions):
6411 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6412 if(preservetime):
6413 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6414 if(flinkinfo['ftype']==1):
6415 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6416 if(flinkinfo['ftype']==2):
6417 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6418 if(flinkinfo['ftype']==5):
6419 if(preservepermissions):
6420 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6421 else:
6422 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6423 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6424 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6425 if(preservepermissions):
6426 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6427 if(preservetime):
6428 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6429 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
6430 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6431 else:
6432 os.symlink(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6433 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6434 if(preservepermissions):
6435 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6436 else:
6437 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6438 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
6439 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
6440 if(preservepermissions):
6441 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6442 if(preservetime):
6443 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
6444 if(listcatfiles['ffilelist'][lcfi]['ftype']==6 and hasattr(os, "mkfifo")):
6445 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6446 lcfi = lcfi + 1;
6447 if(returnfp):
6448 return listcatfiles['ffilelist']['catfp'];
6449 else:
6450 return True;
6451
6452 create_alias_function("UnPack", __file_format_name__, "", UnPackArchiveFile);
6453
6454 if(hasattr(shutil, "register_unpack_format")):
6455 def UnPackArchiveFileFunc(archive_name, extract_dir=None, **kwargs):
6456 return UnPackArchiveFile(archive_name, extract_dir, False, 0, 0, False, __file_format_delimiter__, False, False);
6457 create_alias_function("UnPack", __file_format_name__, "Func", UnPackArchiveFileFunc);
6458
6459 def UnPackArchiveFileString(catstr, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_list__, verbose=False, returnfp=False):
6460 catfp = BytesIO(catstr);
6461 listcatfiles = UnPackArchiveFile(catfp, outdir, followlink, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6462 return listcatfiles;
6463
6464 create_alias_function("UnPack", __file_format_name__, "String", UnPackArchiveFileString);
6465
6466 def ArchiveFileListFiles(infile, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_list__, verbose=False, returnfp=False):
6467 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6468 if(isinstance(infile, dict)):
6469 listcatfiles = infile;
6470 else:
6471 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6472 infile = RemoveWindowsPath(infile);
6473 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, True, skipchecksum, formatspecs, returnfp);
6474 if(not listcatfiles):
6475 return False;
6476 lenlist = len(listcatfiles['ffilelist']);
6477 fnumfiles = int(listcatfiles['fnumfiles']);
6478 lcfi = 0;
6479 lcfx = int(listcatfiles['fnumfiles']);
6480 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6481 lcfx = int(lenlist);
6482 else:
6483 lcfx = int(listcatfiles['fnumfiles']);
6484 returnval = {};
6485 while(lcfi<lcfx):
6486 returnval.update({lcfi: listcatfiles['ffilelist'][lcfi]['fname']});
6487 if(not verbose):
6488 VerbosePrintOut(listcatfiles['ffilelist'][lcfi]['fname']);
6489 if(verbose):
6490 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6491 printfname = listcatfiles['ffilelist'][lcfi]['fname'];
6492 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6493 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " link to " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6494 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6495 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " -> " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6496 fuprint = listcatfiles['ffilelist'][lcfi]['funame'];
6497 if(len(fuprint)<=0):
6498 fuprint = listcatfiles['ffilelist'][lcfi]['fuid'];
6499 fgprint = listcatfiles['ffilelist'][lcfi]['fgname'];
6500 if(len(fgprint)<=0):
6501 fgprint = listcatfiles['ffilelist'][lcfi]['fgid'];
6502 VerbosePrintOut(PrintPermissionString(listcatfiles['ffilelist'][lcfi]['fmode'], listcatfiles['ffilelist'][lcfi]['ftype']) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(listcatfiles['ffilelist'][lcfi]['fsize']).rjust(15) + " " + datetime.datetime.utcfromtimestamp(listcatfiles['ffilelist'][lcfi]['fmtime']).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6503 lcfi = lcfi + 1;
6504 if(returnfp):
6505 return listcatfiles['catfp'];
6506 else:
6507 return True;
6508
6509 create_alias_function("", __file_format_name__, "ListFiles", ArchiveFileListFiles);
6510
6511 def ArchiveFileStringListFiles(catstr, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_list__, verbose=False, returnfp=False):
6512 catfp = BytesIO(catstr);
6513 listcatfiles = ArchiveFileListFiles(catstr, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6514 return listcatfiles;
6515
6516 create_alias_function("", __file_format_name__, "StringListFiles", ArchiveFileStringListFiles);
6517
6518 def TarFileListFiles(infile, verbose=False, returnfp=False):
6519 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6520 if(infile=="-"):
6521 infile = BytesIO();
6522 if(hasattr(sys.stdin, "buffer")):
6523 shutil.copyfileobj(sys.stdin.buffer, infile);
6524 else:
6525 shutil.copyfileobj(sys.stdin, infile);
6526 infile.seek(0, 0);
6527 if(not infile):
6528 return False;
6529 infile.seek(0, 0);
6530 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
6531 infile = download_file_from_internet_file(infile);
6532 infile.seek(0, 0);
6533 if(not infile):
6534 return False;
6535 infile.seek(0, 0);
6536 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
6537 return False;
6538 elif(os.path.exists(infile) and os.path.isfile(infile)):
6539 try:
6540 if(not tarfile.is_tarfile(infile)):
6541 return False;
6542 except AttributeError:
6543 if(not is_tarfile(infile)):
6544 return False;
6545 else:
6546 return False;
6547 try:
6548 if(hasattr(infile, "read") or hasattr(infile, "write")):
6549 tarfp = tarfile.open(fileobj=infile, mode="r");
6550 else:
6551 tarfp = tarfile.open(infile, "r");
6552 except FileNotFoundError:
6553 return False;
6554 lcfi = 0
6555 returnval = {};
6556 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
6557 returnval.update({lcfi: member.name});
6558 fpremode = member.mode;
6559 ffullmode = member.mode;
6560 flinkcount = 0;
6561 ftype = 0;
6562 if(member.isreg()):
6563 ffullmode = member.mode + stat.S_IFREG;
6564 ftype = 0;
6565 elif(member.isdev()):
6566 ffullmode = member.mode;
6567 ftype = 7;
6568 elif(member.islnk()):
6569 ffullmode = member.mode + stat.S_IFREG;
6570 ftype = 1;
6571 elif(member.issym()):
6572 ffullmode = member.mode + stat.S_IFLNK;
6573 ftype = 2;
6574 elif(member.ischr()):
6575 ffullmode = member.mode + stat.S_IFCHR;
6576 ftype = 3;
6577 elif(member.isblk()):
6578 ffullmode = member.mode + stat.S_IFBLK;
6579 ftype = 4;
6580 elif(member.isdir()):
6581 ffullmode = member.mode + stat.S_IFDIR;
6582 ftype = 5;
6583 elif(member.isfifo()):
6584 ffullmode = member.mode + stat.S_IFIFO;
6585 ftype = 6;
6586 elif(member.issparse()):
6587 ffullmode = member.mode;
6588 ftype = 12;
6589 if(not verbose):
6590 VerbosePrintOut(member.name);
6591 elif(verbose):
6592 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6593 printfname = member.name;
6594 if(member.islnk()):
6595 printfname = member.name + " link to " + member.linkname;
6596 elif(member.issym()):
6597 printfname = member.name + " -> " + member.linkname;
6598 fuprint = member.uname;
6599 if(len(fuprint)<=0):
6600 fuprint = member.uid;
6601 fgprint = member.gname;
6602 if(len(fgprint)<=0):
6603 fgprint = member.gid;
6604 VerbosePrintOut(PrintPermissionString(ffullmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(member.mtime).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6605 lcfi = lcfi + 1;
6606 if(returnfp):
6607 return listcatfiles['catfp'];
6608 else:
6609 return True;
6610
6611 def ZipFileListFiles(infile, verbose=False, returnfp=False):
6612 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6613 if(infile=="-"):
6614 infile = BytesIO();
6615 if(hasattr(sys.stdin, "buffer")):
6616 shutil.copyfileobj(sys.stdin.buffer, infile);
6617 else:
6618 shutil.copyfileobj(sys.stdin, infile);
6619 infile.seek(0, 0);
6620 if(not infile):
6621 return False;
6622 infile.seek(0, 0);
6623 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
6624 infile = download_file_from_internet_file(infile);
6625 infile.seek(0, 0);
6626 if(not infile):
6627 return False;
6628 infile.seek(0, 0);
6629 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
6630 return False;
6631 else:
6632 return False;
6633 if(not zipfile.is_zipfile(infile)):
6634 return False;
6635 try:
6636 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
6637 except FileNotFoundError:
6638 return False;
6639 lcfi = 0;
6640 returnval = {};
6641 ziptest = zipfp.testzip();
6642 if(ziptest):
6643 VerbosePrintOut("Bad file found!");
6644 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
6645 if(zipinfo.create_system==0 or zipinfo.create_system==10):
6646 fwinattributes = int(zipinfo.external_attr);
6647 if(not member.is_dir()):
6648 fmode = int(stat.S_IFREG + 438);
6649 fchmode = int(stat.S_IMODE(fmode));
6650 ftypemod = int(stat.S_IFMT(fmode));
6651 elif(member.is_dir()):
6652 fmode = int(stat.S_IFDIR + 511);
6653 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6654 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6655 elif(zipinfo.create_system==3):
6656 fwinattributes =int(0);
6657 fmode = int(zipinfo.external_attr);
6658 fchmode = int(stat.S_IMODE(fmode));
6659 ftypemod = int(stat.S_IFMT(fmode));
6660 else:
6661 fwinattributes = int(0);
6662 if(not member.is_dir()):
6663 fmode = int(stat.S_IFREG + 438);
6664 fchmode = int(stat.S_IMODE(fmode));
6665 ftypemod = int(stat.S_IFMT(fmode));
6666 elif(member.is_dir()):
6667 fmode = int(stat.S_IFDIR + 511);
6668 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6669 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6670 returnval.update({lcfi: member.filename});
6671 if(not verbose):
6672 VerbosePrintOut(member.filename);
6673 if(verbose):
6674 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6675 permissionstr = "";
6676 for fmodval in str(oct(fmode))[-3:]:
6677 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
6678 if(not member.is_dir()):
6679 ftype = 0;
6680 permissionstr = "-" + permissionstr;
6681 elif(member.is_dir()):
6682 ftype = 5;
6683 permissionstr = "d" + permissionstr;
6684 printfname = member.filename;
6685 try:
6686 fuid = int(os.getuid());
6687 except AttributeError:
6688 fuid = int(0);
6689 except KeyError:
6690 fuid = int(0);
6691 try:
6692 fgid = int(os.getgid());
6693 except AttributeError:
6694 fgid = int(0);
6695 except KeyError:
6696 fgid = int(0);
6697 try:
6698 import pwd;
6699 try:
6700 userinfo = pwd.getpwuid(os.getuid());
6701 funame = userinfo.pw_name;
6702 except KeyError:
6703 funame = "";
6704 except AttributeError:
6705 funame = "";
6706 except ImportError:
6707 funame = "";
6708 fgname = "";
6709 try:
6710 import grp;
6711 try:
6712 groupinfo = grp.getgrgid(os.getgid());
6713 fgname = groupinfo.gr_name;
6714 except KeyError:
6715 fgname = "";
6716 except AttributeError:
6717 fgname = "";
6718 except ImportError:
6719 fgname = "";
6720 fuprint = funame;
6721 if(len(fuprint)<=0):
6722 fuprint = str(fuid);
6723 fgprint = fgname;
6724 if(len(fgprint)<=0):
6725 fgprint = str(fgid);
6726 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(int(time.mktime(member.date_time + (0, 0, -1)))).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6727 lcfi = lcfi + 1;
6728 if(returnfp):
6729 return listcatfiles['catfp'];
6730 else:
6731 return True;
6732
6733 if(not rarfile_support):
6734 def RarFileListFiles(infile, verbose=False, returnfp=False):
6735 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6736 if(not os.path.exists(infile) or not os.path.isfile(infile)):
6737 return False;
6738
6739 if(rarfile_support):
6740 def RarFileListFiles(infile, verbose=False, returnfp=False):
6741 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6742 if(not os.path.exists(infile) or not os.path.isfile(infile)):
6743 return False;
6744 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
6745 return False;
6746 lcfi = 0;
6747 returnval = {};
6748 rarfp = rarfile.RarFile(infile, "r");
6749 rartest = rarfp.testrar();
6750 if(rartest):
6751 VerbosePrintOut("Bad file found!");
6752 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
6753 is_unix = False;
6754 is_windows = False;
6755 if(member.host_os==rarfile.RAR_OS_UNIX):
6756 is_windows = False;
6757 try:
6758 member.external_attr
6759 is_unix = True;
6760 except AttributeError:
6761 is_unix = False;
6762 elif(member.host_os==rarfile.RAR_OS_WIN32):
6763 is_unix = False;
6764 try:
6765 member.external_attr
6766 is_windows = True;
6767 except AttributeError:
6768 is_windows = False;
6769 else:
6770 is_unix = False;
6771 is_windows = False;
6772 if(is_unix and member.external_attr !=0):
6773 fpremode = int(member.external_attr);
6774 elif(member.is_file()):
6775 fpremode = int(stat.S_IFREG + 438);
6776 elif(member.is_symlink()):
6777 fpremode = int(stat.S_IFLNK + 438);
6778 elif(member.is_dir()):
6779 fpremode = int(stat.S_IFDIR + 511);
6780 if(is_windows and member.external_attr !=0):
6781 fwinattributes = int(member.external_attr);
6782 else:
6783 fwinattributes = int(0);
6784 if(is_unix and member.external_attr !=0):
6785 fmode = int(member.external_attr);
6786 fchmode = int(stat.S_IMODE(member.external_attr));
6787 ftypemod = int(stat.S_IFMT(member.external_attr));
6788 elif(member.is_file()):
6789 fmode = int(stat.S_IFREG + 438);
6790 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
6791 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
6792 elif(member.is_symlink()):
6793 fmode = int(stat.S_IFLNK + 438);
6794 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
6795 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
6796 elif(member.is_dir()):
6797 fmode = int(stat.S_IFDIR + 511);
6798 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6799 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6800 returnval.update({lcfi: member.filename});
6801 if(not verbose):
6802 VerbosePrintOut(member.filename);
6803 if(verbose):
6804 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6805 permissionstr = "";
6806 for fmodval in str(oct(fmode))[-3:]:
6807 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
6808 if(member.is_file()):
6809 ftype = 0;
6810 permissionstr = "-" + permissionstr;
6811 printfname = member.filename;
6812 elif(member.is_symlink()):
6813 ftype = 2;
6814 permissionstr = "l" + permissionstr;
6815 printfname = member.name + " -> " + member.read().decode("UTF-8");
6816 elif(member.is_dir()):
6817 ftype = 5;
6818 permissionstr = "d" + permissionstr;
6819 printfname = member.filename;
6820 try:
6821 fuid = int(os.getuid());
6822 except AttributeError:
6823 fuid = int(0);
6824 except KeyError:
6825 fuid = int(0);
6826 try:
6827 fgid = int(os.getgid());
6828 except AttributeError:
6829 fgid = int(0);
6830 except KeyError:
6831 fgid = int(0);
6832 try:
6833 import pwd;
6834 try:
6835 userinfo = pwd.getpwuid(os.getuid());
6836 funame = userinfo.pw_name;
6837 except KeyError:
6838 funame = "";
6839 except AttributeError:
6840 funame = "";
6841 except ImportError:
6842 funame = "";
6843 fgname = "";
6844 try:
6845 import grp;
6846 try:
6847 groupinfo = grp.getgrgid(os.getgid());
6848 fgname = groupinfo.gr_name;
6849 except KeyError:
6850 fgname = "";
6851 except AttributeError:
6852 fgname = "";
6853 except ImportError:
6854 fgname = "";
6855 fuprint = funame;
6856 if(len(fuprint)<=0):
6857 fuprint = str(fuid);
6858 fgprint = fgname;
6859 if(len(fgprint)<=0):
6860 fgprint = str(fgid);
6861 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + member.mtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
6862 lcfi = lcfi + 1;
6863 if(returnfp):
6864 return listcatfiles['catfp'];
6865 else:
6866 return True;
6867
6868 if(not py7zr_support):
6869 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
6870 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6871 if(not os.path.exists(infile) or not os.path.isfile(infile)):
6872 return False;
6873
6874 if(py7zr_support):
6875 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
6876 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6877 if(not os.path.exists(infile) or not os.path.isfile(infile)):
6878 return False;
6879 lcfi = 0;
6880 returnval = {};
6881 szpfp = py7zr.SevenZipFile(infile, mode="r");
6882 file_content = szpfp.readall();
6883 #sztest = szpfp.testzip();
6884 sztestalt = szpfp.test();
6885 if(sztestalt):
6886 VerbosePrintOut("Bad file found!");
6887 for member in sorted(szpfp.list(), key=lambda x: x.filename):
6888 if(re.findall("^[.|/]", member.filename)):
6889 fname = member.filename;
6890 else:
6891 fname = "./"+member.filename;
6892 if(not member.is_directory):
6893 fpremode = int(stat.S_IFREG + 438);
6894 elif(member.is_directory):
6895 fpremode = int(stat.S_IFDIR + 511);
6896 fwinattributes = int(0);
6897 if(member.is_directory):
6898 fmode = int(stat.S_IFDIR + 511);
6899 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6900 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6901 else:
6902 fmode = int(stat.S_IFLNK + 438);
6903 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
6904 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
6905 returnval.update({lcfi: member.filename});
6906 if(not verbose):
6907 VerbosePrintOut(member.filename);
6908 if(verbose):
6909 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6910 permissionstr = "";
6911 for fmodval in str(oct(fmode))[-3:]:
6912 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
6913 fsize = int("0");
6914 if(not member.is_directory):
6915 ftype = 0;
6916 permissionstr = "-" + permissionstr;
6917 printfname = member.filename;
6918 elif(member.is_directory):
6919 ftype = 5;
6920 permissionstr = "d" + permissionstr;
6921 printfname = member.filename;
6922 if(ftype==0):
6923 fsize = len(file_content[member.filename].read());
6924 file_content[member.filename].close();
6925 try:
6926 fuid = int(os.getuid());
6927 except AttributeError:
6928 fuid = int(0);
6929 except KeyError:
6930 fuid = int(0);
6931 try:
6932 fgid = int(os.getgid());
6933 except AttributeError:
6934 fgid = int(0);
6935 except KeyError:
6936 fgid = int(0);
6937 try:
6938 import pwd;
6939 try:
6940 userinfo = pwd.getpwuid(os.getuid());
6941 funame = userinfo.pw_name;
6942 except KeyError:
6943 funame = "";
6944 except AttributeError:
6945 funame = "";
6946 except ImportError:
6947 funame = "";
6948 fgname = "";
6949 try:
6950 import grp;
6951 try:
6952 groupinfo = grp.getgrgid(os.getgid());
6953 fgname = groupinfo.gr_name;
6954 except KeyError:
6955 fgname = "";
6956 except AttributeError:
6957 fgname = "";
6958 except ImportError:
6959 fgname = "";
6960 fuprint = funame;
6961 if(len(fuprint)<=0):
6962 fuprint = str(fuid);
6963 fgprint = fgname;
6964 if(len(fgprint)<=0):
6965 fgprint = str(fgid);
6966 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(fsize).rjust(15) + " " + member.creationtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
6967 lcfi = lcfi + 1;
6968 if(returnfp):
6969 return listcatfiles['catfp'];
6970 else:
6971 return True;
6972
6973 def InFileListFiles(infile, verbose=False, formatspecs=__file_format_list__, returnfp=False):
6974 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6975 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
6976 if(checkcompressfile=="tarfile"):
6977 return TarFileListFiles(infile, verbose, returnfp);
6978 elif(checkcompressfile=="zipfile"):
6979 return ZipFileListFiles(infile, verbose, returnfp);
6980 elif(checkcompressfile=="catfile"):
6981 return ArchiveFileListFiles(infile, 0, 0, False, formatspecs, verbose, returnfp);
6982 elif(rarfile_support and checkcompressfile=="rarfile"):
6983 return RarFileListFiles(infile, verbose, returnfp);
6984 elif(py7zr_support and checkcompressfile=="7zipfile"):
6985 return SevenZipFileListFiles(infile, verbose, returnfp);
6986 else:
6987 return False;
6988 return False;
6989
6990 def ListDirListFiles(infiles, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False, returnfp=False):
6991 outarray = BytesIO();
6992 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compressionlevel, followlink, checksumtype, formatspecs, False, True);
6993 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6994 return listcatfiles;
6995
6996 def ListDirListFilesAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False, returnfp=False):
6997 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, listonly, checksumtype, formatspecs, verbose);
6998 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6999 return listcatfiles;
7000
7001 def PackArchiveFileFromListDirAlt(infiles, outfile, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
7002 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, False, checksumtype, extradata, formatspecs, False);
7003 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compressionlevel, followlink, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
7004 return listcatfiles;
7005
7006 create_alias_function("Pack", __file_format_name__, "FromListDirAlt", PackArchiveFileFromListDirAlt);
7007
7008 def PackArchiveFileFromTarFileAlt(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
7009 outarray = TarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7010 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7011 return listcatfiles;
7012
7013 create_alias_function("Pack", __file_format_name__, "FromTarFileAlt", PackArchiveFileFromTarFileAlt);
7014
7015 def PackArchiveFileFromZipFileAlt(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
7016 outarray = ZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7017 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7018 return listcatfiles;
7019
7020 create_alias_function("Pack", __file_format_name__, "FromZipFileAlt", PackArchiveFileFromZipFileAlt);
7021
7022 if(not rarfile_support):
7023 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
7024 return False;
7025
7026 if(rarfile_support):
7027 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
7028 outarray = RarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7029 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7030 return listcatfiles;
7031
7032 create_alias_function("Pack", __file_format_name__, "FromRarFileAlt", PackArchiveFileFromRarFileAlt);
7033
7034 if(not py7zr_support):
7035 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
7036 return False;
7037
7038 if(py7zr_support):
7039 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
7040 outarray = SevenZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7041 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7042 return listcatfiles;
7043
7044 create_alias_function("Pack", __file_format_name__, "FromSevenZipFileAlt", PackArchiveFileFromSevenZipFileAlt);
7045
7046 def download_file_from_ftp_file(url):
7047 urlparts = urlparse(url);
7048 file_name = os.path.basename(urlparts.path);
7049 file_dir = os.path.dirname(urlparts.path);
7050 if(urlparts.username is not None):
7051 ftp_username = urlparts.username;
7052 else:
7053 ftp_username = "anonymous";
7054 if(urlparts.password is not None):
7055 ftp_password = urlparts.password;
7056 elif(urlparts.password is None and urlparts.username=="anonymous"):
7057 ftp_password = "anonymous";
7058 else:
7059 ftp_password = "";
7060 if(urlparts.scheme=="ftp"):
7061 ftp = FTP();
7062 elif(urlparts.scheme=="ftps"):
7063 ftp = FTP_TLS();
7064 else:
7065 return False;
7066 if(urlparts.scheme=="sftp"):
7067 if(__use_pysftp__):
7068 return download_file_from_pysftp_file(url);
7069 else:
7070 return download_file_from_sftp_file(url);
7071 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7072 return download_file_from_http_file(url);
7073 ftp_port = urlparts.port;
7074 if(urlparts.port is None):
7075 ftp_port = 21;
7076 try:
7077 ftp.connect(urlparts.hostname, ftp_port);
7078 except socket.gaierror:
7079 log.info("Error With URL "+url);
7080 return False;
7081 except socket.timeout:
7082 log.info("Error With URL "+url);
7083 return False;
7084 ftp.login(urlparts.username, urlparts.password);
7085 if(urlparts.scheme=="ftps"):
7086 ftp.prot_p();
7087 ftpfile = BytesIO();
7088 ftp.retrbinary("RETR "+urlparts.path, ftpfile.write);
7089 #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
7090 ftp.close();
7091 ftpfile.seek(0, 0);
7092 return ftpfile;
7093
7094 def download_file_from_ftp_string(url):
7095 ftpfile = download_file_from_ftp_file(url);
7096 return ftpfile.read();
7097
7098 def upload_file_to_ftp_file(ftpfile, url):
7099 urlparts = urlparse(url);
7100 file_name = os.path.basename(urlparts.path);
7101 file_dir = os.path.dirname(urlparts.path);
7102 if(urlparts.username is not None):
7103 ftp_username = urlparts.username;
7104 else:
7105 ftp_username = "anonymous";
7106 if(urlparts.password is not None):
7107 ftp_password = urlparts.password;
7108 elif(urlparts.password is None and urlparts.username=="anonymous"):
7109 ftp_password = "anonymous";
7110 else:
7111 ftp_password = "";
7112 if(urlparts.scheme=="ftp"):
7113 ftp = FTP();
7114 elif(urlparts.scheme=="ftps"):
7115 ftp = FTP_TLS();
7116 else:
7117 return False;
7118 if(urlparts.scheme=="sftp"):
7119 if(__use_pysftp__):
7120 return upload_file_to_pysftp_file(url);
7121 else:
7122 return upload_file_to_sftp_file(url);
7123 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7124 return False;
7125 ftp_port = urlparts.port;
7126 if(urlparts.port is None):
7127 ftp_port = 21;
7128 try:
7129 ftp.connect(urlparts.hostname, ftp_port);
7130 except socket.gaierror:
7131 log.info("Error With URL "+url);
7132 return False;
7133 except socket.timeout:
7134 log.info("Error With URL "+url);
7135 return False;
7136 ftp.login(urlparts.username, urlparts.password);
7137 if(urlparts.scheme=="ftps"):
7138 ftp.prot_p();
7139 ftp.storbinary("STOR "+urlparts.path, ftpfile);
7140 ftp.close();
7141 ftpfile.seek(0, 0);
7142 return ftpfile;
7143
7144 def upload_file_to_ftp_string(ftpstring, url):
7145 ftpfileo = BytesIO(ftpstring);
7146 ftpfile = upload_file_to_ftp_file(ftpfileo, url);
7147 ftpfileo.close();
7148 return ftpfile;
7149
7150 def download_file_from_http_file(url, headers=geturls_headers_pycatfile_python_alt):
7151 # Parse the URL to extract username and password if present
7152 urlparts = urlparse(url);
7153 username = urlparts.username;
7154 password = urlparts.password;
7155 # Rebuild the URL without the username and password
7156 netloc = urlparts.hostname;
7157 if(urlparts.scheme=="sftp"):
7158 if(__use_pysftp__):
7159 return download_file_from_pysftp_file(url);
7160 else:
7161 return download_file_from_sftp_file(url);
7162 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7163 return download_file_from_ftp_file(url);
7164 if urlparts.port:
7165 netloc += ':' + str(urlparts.port);
7166 rebuilt_url = urlunparse((urlparts.scheme, netloc, urlparts.path, urlparts.params, urlparts.query, urlparts.fragment));
7167 # Create a temporary file object
7168 httpfile = BytesIO();
7169 if haverequests:
7170 # Use the requests library if available
7171 if username and password:
7172 response = requests.get(rebuilt_url, headers=headers, auth=(username, password), stream=True);
7173 else:
7174 response = requests.get(rebuilt_url, headers=headers, stream=True);
7175 response.raw.decode_content = True
7176 shutil.copyfileobj(response.raw, httpfile);
7177 else:
7178 # Build a Request object for urllib
7179 request = Request(rebuilt_url, headers=headers);
7180 # Create an opener object for handling URLs
7181 if username and password:
7182 # Create a password manager
7183 password_mgr = HTTPPasswordMgrWithDefaultRealm();
7184 # Add the username and password
7185 password_mgr.add_password(None, rebuilt_url, username, password);
7186 # Create an authentication handler using the password manager
7187 auth_handler = HTTPBasicAuthHandler(password_mgr);
7188 # Build the opener with the authentication handler
7189 opener = build_opener(auth_handler);
7190 else:
7191 opener = build_opener();
7192 with opener.open(request) as response:
7193 shutil.copyfileobj(response, httpfile);
7194 # Reset file pointer to the start
7195 httpfile.seek(0, 0);
7196 # Return the temporary file object
7197 return httpfile;
7198
7199 def download_file_from_http_string(url, headers=geturls_headers_pycatfile_python_alt):
7200 httpfile = download_file_from_http_file(url, headers);
7201 return ftpfile.read();
7202
7203 if(haveparamiko):
7204 def download_file_from_sftp_file(url):
7205 urlparts = urlparse(url);
7206 file_name = os.path.basename(urlparts.path);
7207 file_dir = os.path.dirname(urlparts.path);
7208 sftp_port = urlparts.port;
7209 if(urlparts.port is None):
7210 sftp_port = 22;
7211 else:
7212 sftp_port = urlparts.port;
7213 if(urlparts.username is not None):
7214 sftp_username = urlparts.username;
7215 else:
7216 sftp_username = "anonymous";
7217 if(urlparts.password is not None):
7218 sftp_password = urlparts.password;
7219 elif(urlparts.password is None and urlparts.username=="anonymous"):
7220 sftp_password = "anonymous";
7221 else:
7222 sftp_password = "";
7223 if(urlparts.scheme=="ftp"):
7224 return download_file_from_ftp_file(url);
7225 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7226 return download_file_from_http_file(url);
7227 if(urlparts.scheme!="sftp"):
7228 return False;
7229 ssh = paramiko.SSHClient();
7230 ssh.load_system_host_keys();
7231 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
7232 try:
7233 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7234 except paramiko.ssh_exception.SSHException:
7235 return False;
7236 except socket.gaierror:
7237 log.info("Error With URL "+url);
7238 return False;
7239 except socket.timeout:
7240 log.info("Error With URL "+url);
7241 return False;
7242 sftp = ssh.open_sftp();
7243 sftpfile = BytesIO();
7244 sftp.getfo(urlparts.path, sftpfile);
7245 sftp.close();
7246 ssh.close();
7247 sftpfile.seek(0, 0);
7248 return sftpfile;
7249 else:
7250 def download_file_from_sftp_file(url):
7251 return False;
7252
7253 if(haveparamiko):
7254 def download_file_from_sftp_string(url):
7255 sftpfile = download_file_from_sftp_file(url);
7256 return sftpfile.read();
7257 else:
7258 def download_file_from_ftp_string(url):
7259 return False;
7260
7261 if(haveparamiko):
7262 def upload_file_to_sftp_file(sftpfile, url):
7263 urlparts = urlparse(url);
7264 file_name = os.path.basename(urlparts.path);
7265 file_dir = os.path.dirname(urlparts.path);
7266 sftp_port = urlparts.port;
7267 if(urlparts.port is None):
7268 sftp_port = 22;
7269 else:
7270 sftp_port = urlparts.port;
7271 if(urlparts.username is not None):
7272 sftp_username = urlparts.username;
7273 else:
7274 sftp_username = "anonymous";
7275 if(urlparts.password is not None):
7276 sftp_password = urlparts.password;
7277 elif(urlparts.password is None and urlparts.username=="anonymous"):
7278 sftp_password = "anonymous";
7279 else:
7280 sftp_password = "";
7281 if(urlparts.scheme=="ftp"):
7282 return upload_file_to_ftp_file(url);
7283 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7284 return False;
7285 if(urlparts.scheme!="sftp"):
7286 return False;
7287 ssh = paramiko.SSHClient();
7288 ssh.load_system_host_keys();
7289 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
7290 try:
7291 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7292 except paramiko.ssh_exception.SSHException:
7293 return False;
7294 except socket.gaierror:
7295 log.info("Error With URL "+url);
7296 return False;
7297 except socket.timeout:
7298 log.info("Error With URL "+url);
7299 return False;
7300 sftp = ssh.open_sftp();
7301 sftp.putfo(sftpfile, urlparts.path);
7302 sftp.close();
7303 ssh.close();
7304 sftpfile.seek(0, 0);
7305 return sftpfile;
7306 else:
7307 def upload_file_to_sftp_file(sftpfile, url):
7308 return False;
7309
7310 if(haveparamiko):
7311 def upload_file_to_sftp_string(sftpstring, url):
7312 sftpfileo = BytesIO(sftpstring);
7313 sftpfile = upload_file_to_sftp_files(ftpfileo, url);
7314 sftpfileo.close();
7315 return sftpfile;
7316 else:
7317 def upload_file_to_sftp_string(url):
7318 return False;
7319
7320 if(havepysftp):
7321 def download_file_from_pysftp_file(url):
7322 urlparts = urlparse(url);
7323 file_name = os.path.basename(urlparts.path);
7324 file_dir = os.path.dirname(urlparts.path);
7325 sftp_port = urlparts.port;
7326 if(urlparts.port is None):
7327 sftp_port = 22;
7328 else:
7329 sftp_port = urlparts.port;
7330 if(urlparts.username is not None):
7331 sftp_username = urlparts.username;
7332 else:
7333 sftp_username = "anonymous";
7334 if(urlparts.password is not None):
7335 sftp_password = urlparts.password;
7336 elif(urlparts.password is None and urlparts.username=="anonymous"):
7337 sftp_password = "anonymous";
7338 else:
7339 sftp_password = "";
7340 if(urlparts.scheme=="ftp"):
7341 return download_file_from_ftp_file(url);
7342 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7343 return download_file_from_http_file(url);
7344 if(urlparts.scheme!="sftp"):
7345 return False;
7346 try:
7347 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7348 except paramiko.ssh_exception.SSHException:
7349 return False;
7350 except socket.gaierror:
7351 log.info("Error With URL "+url);
7352 return False;
7353 except socket.timeout:
7354 log.info("Error With URL "+url);
7355 return False;
7356 sftp = ssh.open_sftp();
7357 sftpfile = BytesIO();
7358 sftp.getfo(urlparts.path, sftpfile);
7359 sftp.close();
7360 ssh.close();
7361 sftpfile.seek(0, 0);
7362 return sftpfile;
7363 else:
7364 def download_file_from_pysftp_file(url):
7365 return False;
7366
7367 if(havepysftp):
7368 def download_file_from_pysftp_string(url):
7369 sftpfile = download_file_from_pysftp_file(url);
7370 return sftpfile.read();
7371 else:
7372 def download_file_from_ftp_string(url):
7373 return False;
7374
7375 if(havepysftp):
7376 def upload_file_to_pysftp_file(sftpfile, url):
7377 urlparts = urlparse(url);
7378 file_name = os.path.basename(urlparts.path);
7379 file_dir = os.path.dirname(urlparts.path);
7380 sftp_port = urlparts.port;
7381 if(urlparts.port is None):
7382 sftp_port = 22;
7383 else:
7384 sftp_port = urlparts.port;
7385 if(urlparts.username is not None):
7386 sftp_username = urlparts.username;
7387 else:
7388 sftp_username = "anonymous";
7389 if(urlparts.password is not None):
7390 sftp_password = urlparts.password;
7391 elif(urlparts.password is None and urlparts.username=="anonymous"):
7392 sftp_password = "anonymous";
7393 else:
7394 sftp_password = "";
7395 if(urlparts.scheme=="ftp"):
7396 return upload_file_to_ftp_file(url);
7397 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7398 return False;
7399 if(urlparts.scheme!="sftp"):
7400 return False;
7401 try:
7402 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7403 except paramiko.ssh_exception.SSHException:
7404 return False;
7405 except socket.gaierror:
7406 log.info("Error With URL "+url);
7407 return False;
7408 except socket.timeout:
7409 log.info("Error With URL "+url);
7410 return False;
7411 sftp = ssh.open_sftp();
7412 sftp.putfo(sftpfile, urlparts.path);
7413 sftp.close();
7414 ssh.close();
7415 sftpfile.seek(0, 0);
7416 return sftpfile;
7417 else:
7418 def upload_file_to_pysftp_file(sftpfile, url):
7419 return False;
7420
7421 if(havepysftp):
7422 def upload_file_to_pysftp_string(sftpstring, url):
7423 sftpfileo = BytesIO(sftpstring);
7424 sftpfile = upload_file_to_pysftp_files(ftpfileo, url);
7425 sftpfileo.close();
7426 return sftpfile;
7427 else:
7428 def upload_file_to_pysftp_string(url):
7429 return False;
7430
7431 def download_file_from_internet_file(url, headers=geturls_headers_pycatfile_python_alt):
7432 urlparts = urlparse(url);
7433 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7434 return download_file_from_http_file(url, headers);
7435 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7436 return download_file_from_ftp_file(url);
7437 elif(urlparts.scheme=="sftp"):
7438 if(__use_pysftp__ and havepysftp):
7439 return download_file_from_pysftp_file(url);
7440 else:
7441 return download_file_from_sftp_file(url);
7442 else:
7443 return False;
7444 return False;
7445
7446 def download_file_from_internet_uncompress_file(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_list__):
7447 fp = download_file_from_internet_file(url);
7448 fp = UncompressArchiveFile(fp, formatspecs);
7449 fp.seek(0, 0);
7450 if(not fp):
7451 return False;
7452 return fp;
7453
7454 def download_file_from_internet_string(url, headers=geturls_headers_pycatfile_python_alt):
7455 urlparts = urlparse(url);
7456 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7457 return download_file_from_http_string(url, headers);
7458 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7459 return download_file_from_ftp_string(url);
7460 elif(urlparts.scheme=="sftp"):
7461 if(__use_pysftp__ and havepysftp):
7462 return download_file_from_pysftp_string(url);
7463 else:
7464 return download_file_from_sftp_string(url);
7465 else:
7466 return False;
7467 return False;
7468
7469 def download_file_from_internet_uncompress_string(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_list__):
7470 fp = download_file_from_internet_string(url);
7471 fp = UncompressArchiveFile(fp, formatspecs);
7472 fp.seek(0, 0);
7473 if(not fp):
7474 return False;
7475 return fp;
7476
7477 def upload_file_to_internet_file(ifp, url):
7478 urlparts = urlparse(url);
7479 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7480 return False;
7481 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7482 return upload_file_to_ftp_file(ifp, url);
7483 elif(urlparts.scheme=="sftp"):
7484 if(__use_pysftp__ and havepysftp):
7485 return upload_file_to_pysftp_file(ifp, url);
7486 else:
7487 return upload_file_to_sftp_file(ifp, url);
7488 else:
7489 return False;
7490 return False;
7491
7492 def upload_file_to_internet_compress_file(ifp, url, formatspecs=__file_format_list__):
7493 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
7494 if(not catfileout):
7495 return False;
7496 catfp.seek(0, 0);
7497 upload_file_to_internet_file(catfp, outfile);
7498 return True;
7499
7500 def upload_file_to_internet_string(ifp, url):
7501 urlparts = urlparse(url);
7502 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7503 return False;
7504 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7505 return upload_file_to_ftp_string(ifp, url);
7506 elif(urlparts.scheme=="sftp"):
7507 if(__use_pysftp__ and havepysftp):
7508 return upload_file_to_pysftp_string(ifp, url);
7509 else:
7510 return upload_file_to_sftp_string(ifp, url);
7511 else:
7512 return False;
7513 return False;
7514
7515 def upload_file_to_internet_compress_string(ifp, url, formatspecs=__file_format_list__):
7516 catfp = CompressArchiveFile(BytesIO(ifp), compression, compressionlevel, formatspecs);
7517 if(not catfileout):
7518 return False;
7519 catfp.seek(0, 0);
7520 upload_file_to_internet_file(catfp, outfile);
7521 return True;
7522
7523 try:
7524 if(hasattr(shutil, "register_archive_format")):
7525 # Register the packing format
7526 shutil.register_archive_format(__file_format_name__, PackArchiveFileFunc, description='Pack concatenated files');
7527 except shutil.RegistryError:
7528 pass;
7529
7530 try:
7531 if(hasattr(shutil, "register_unpack_format")):
7532 # Register the unpacking format
7533 shutil.register_unpack_format(__file_format_name__, archivefile_extensions, UnPackArchiveFileFunc, description='UnPack concatenated files');
7534 except shutil.RegistryError:
7535 pass;
A tar like file format name catfile after unix cat command (concatenate files).