Add files via upload
[PyCatFile.git] / pycatfile.py
blobd9d1212038d0691eb50cb4ad83e8a9b2d76a5588
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
4 '''
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the Revised BSD License.
8 This program is distributed in the hope that it will be useful,
9 but WITHOUT ANY WARRANTY; without even the implied warranty of
10 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 Revised BSD License for more details.
13 Copyright 2018-2024 Cool Dude 2k - http://idb.berlios.de/
14 Copyright 2018-2024 Game Maker 2k - http://intdb.sourceforge.net/
15 Copyright 2018-2024 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
17 $FileInfo: pycatfile.py - Last Update: 5/15/2024 Ver. 0.11.2 RC 1 - Author: cooldude2k $
18 '''
20 from __future__ import absolute_import, division, print_function, unicode_literals;
21 import io, os, re, sys, time, stat, zlib, base64, shutil, socket, hashlib, datetime, logging, binascii, tempfile, zipfile, platform;
22 ftpssl = True;
23 try:
24 ftpssl = True;
25 from ftplib import FTP, FTP_TLS;
26 except ImportError:
27 ftpssl = False;
28 from ftplib import FTP;
29 if(sys.version[0]=="2"):
30 from urlparse import urlparse, urlunparse;
31 elif(sys.version[0]>="3"):
32 from urllib.parse import urlunparse;
33 from urllib.parse import urlparse;
35 if os.name == 'nt': # Only modify if on Windows
36 if sys.version[0] == "2":
37 import codecs;
38 sys.stdout = codecs.getwriter('utf-8')(sys.stdout);
39 sys.stderr = codecs.getwriter('utf-8')(sys.stderr);
40 else:
41 import io;
42 sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace', line_buffering=True);
43 sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', errors='replace', line_buffering=True);
45 hashlib_guaranteed = False;
46 os.environ["PYTHONIOENCODING"] = "UTF-8";
47 os.environ["LC_CTYPE"] = "UTF-8";
48 try:
49 reload(sys);
50 try:
51 sys.setdefaultencoding('UTF-8');
52 except NameError:
53 pass;
54 except AttributeError:
55 pass;
56 except NameError:
57 pass;
58 except AttributeError:
59 pass;
61 try:
62 from zlib import crc32;
63 except ImportError:
64 from binascii import crc32;
66 if(sys.version_info[0]==2):
67 FileNotFoundError = IOError;
69 rarfile_support = False;
70 try:
71 import rarfile;
72 rarfile_support = True;
73 except ImportError:
74 rarfile_support = False;
76 py7zr_support = False;
77 try:
78 import py7zr;
79 py7zr_support = True;
80 except ImportError:
81 py7zr_support = False;
83 try:
84 from xtarfile import is_tarfile;
85 except ImportError:
86 try:
87 from safetar import is_tarfile;
88 except ImportError:
89 from tarfile import is_tarfile;
91 try:
92 import xtarfile as tarfile;
93 except ImportError:
94 try:
95 import safetar as tarfile;
96 except ImportError:
97 import tarfile;
99 haveparamiko = False;
100 try:
101 import paramiko;
102 haveparamiko = True;
103 except ImportError:
104 haveparamiko = False;
106 havepysftp = False;
107 try:
108 import pysftp;
109 havepysftp = True;
110 except ImportError:
111 havepysftp = False;
113 haverequests = False;
114 try:
115 import requests;
116 haverequests = True;
117 except ImportError:
118 haverequests = False;
120 if(haverequests):
121 import urllib3;
122 logging.getLogger("urllib3").setLevel(logging.WARNING);
124 try:
125 # Python 3 imports
126 from urllib.request import Request, build_opener, HTTPBasicAuthHandler;
127 from urllib.parse import urlparse;
128 except ImportError:
129 # Python 2 imports
130 from urllib2 import Request, build_opener, HTTPBasicAuthHandler;
131 from urlparse import urlparse;
133 if(sys.version[0]=="2"):
134 try:
135 from io import StringIO, BytesIO;
136 except ImportError:
137 try:
138 from cStringIO import StringIO;
139 from cStringIO import StringIO as BytesIO;
140 except ImportError:
141 from StringIO import StringIO;
142 from StringIO import StringIO as BytesIO;
143 elif(sys.version[0]>="3"):
144 from io import StringIO, BytesIO;
145 else:
146 teststringio = 0;
147 if(teststringio<=0):
148 try:
149 from cStringIO import StringIO as BytesIO;
150 teststringio = 1;
151 except ImportError:
152 teststringio = 0;
153 if(teststringio<=0):
154 try:
155 from StringIO import StringIO as BytesIO;
156 teststringio = 2;
157 except ImportError:
158 teststringio = 0;
159 if(teststringio<=0):
160 try:
161 from io import BytesIO;
162 teststringio = 3;
163 except ImportError:
164 teststringio = 0;
166 __use_pysftp__ = False;
167 if(not havepysftp):
168 __use_pysftp__ = False;
169 __file_format_name__ = "CatFile";
170 __program_name__ = "Py"+__file_format_name__;
171 __file_format_lower__ = __file_format_name__.lower();
172 __file_format_magic__ = __file_format_name__;
173 __file_format_len__ = len(__file_format_magic__);
174 __file_format_hex__ = binascii.hexlify(__file_format_magic__.encode("UTF-8")).decode("UTF-8");
175 __file_format_delimiter__ = "\x00";
176 __file_format_ver__ = "001";
177 __use_new_style__ = True;
178 __use_advanced_list__ = True;
179 __use_alt_inode__ = False;
180 __file_format_extension__ = ".cat";
181 __file_format_list__ = [__file_format_name__, __file_format_magic__, __file_format_lower__, __file_format_len__, __file_format_hex__, __file_format_delimiter__, __file_format_ver__, __use_new_style__, __use_advanced_list__, __use_alt_inode__];
182 __file_format_dict__ = {'format_name': __file_format_name__, 'format_magic': __file_format_magic__, 'format_lower': __file_format_lower__, 'format_len': __file_format_len__, 'format_hex': __file_format_hex__, 'format_delimiter': __file_format_delimiter__, 'format_ver': __file_format_ver__, 'new_style': __use_new_style__, 'use_advanced_list': __use_advanced_list__, 'use_alt_inode': __use_alt_inode__};
183 __project__ = __program_name__;
184 __project_url__ = "https://github.com/GameMaker2k/PyCatFile";
185 __version_info__ = (0, 11, 2, "RC 1", 1);
186 __version_date_info__ = (2024, 5, 15, "RC 1", 1);
187 __version_date__ = str(__version_date_info__[0]) + "." + str(__version_date_info__[1]).zfill(2) + "." + str(__version_date_info__[2]).zfill(2);
188 __revision__ = __version_info__[3];
189 __revision_id__ = "$Id$";
190 if(__version_info__[4] is not None):
191 __version_date_plusrc__ = __version_date__ + "-" + str(__version_date_info__[4]);
192 if(__version_info__[4] is None):
193 __version_date_plusrc__ = __version_date__;
194 if(__version_info__[3] is not None):
195 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]) + " " + str(__version_info__[3]);
196 if(__version_info__[3] is None):
197 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]);
199 PyBitness = platform.architecture();
200 if(PyBitness=="32bit" or PyBitness=="32"):
201 PyBitness = "32";
202 elif(PyBitness=="64bit" or PyBitness=="64"):
203 PyBitness = "64";
204 else:
205 PyBitness = "32";
207 geturls_ua_pycatfile_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
208 if(platform.python_implementation()!=""):
209 py_implementation = platform.python_implementation();
210 if(platform.python_implementation()==""):
211 py_implementation = "Python";
212 geturls_ua_pycatfile_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=py_implementation, pyver=platform.python_version(), proname=__project__, prover=__version__);
213 geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
214 geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
215 geturls_headers_pycatfile_python = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
216 geturls_headers_pycatfile_python_alt = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python_alt, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
217 geturls_headers_googlebot_google = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
218 geturls_headers_googlebot_google_old = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
220 compressionsupport = CompressionSupport();
221 compressionlist = ['auto'];
222 compressionlistalt = [];
223 outextlist = [];
224 outextlistwd = [];
225 if('gzip' in compressionsupport):
226 compressionlist.append('gzip');
227 compressionlistalt.append('gzip');
228 outextlist.append('gz');
229 outextlistwd.append('.gz');
230 if('bzip2' in compressionsupport):
231 compressionlist.append('bzip2');
232 compressionlistalt.append('bzip2');
233 outextlist.append('bz2');
234 outextlistwd.append('.bz2');
235 if('zstd' in compressionsupport):
236 compressionlist.append('zstd');
237 compressionlistalt.append('zstd');
238 outextlist.append('zst');
239 outextlistwd.append('.zst');
240 if('lz4' in compressionsupport):
241 compressionlist.append('lz4');
242 compressionlistalt.append('lz4');
243 outextlist.append('lz4');
244 outextlistwd.append('.lz4');
245 if('lzo' in compressionsupport):
246 compressionlist.append('lzo');
247 compressionlistalt.append('lzo');
248 outextlist.append('lzo');
249 outextlistwd.append('.lzo');
250 if('lzop' in compressionsupport):
251 compressionlist.append('lzop');
252 compressionlistalt.append('lzop');
253 outextlist.append('lzop');
254 outextlistwd.append('.lzop');
255 if('lzma' in compressionsupport):
256 compressionlist.append('lzma');
257 compressionlistalt.append('lzma');
258 outextlist.append('lzma');
259 outextlistwd.append('.lzma');
260 if('xz' in compressionsupport):
261 compressionlist.append('xz');
262 compressionlistalt.append('xz');
263 outextlist.append('xz');
264 outextlistwd.append('.xz');
266 tarfile_mimetype = "application/tar";
267 tarfile_tar_mimetype = tarfile_mimetype;
268 zipfile_mimetype = "application/zip";
269 zipfile_zip_mimetype = zipfile_mimetype;
270 rarfile_mimetype = "application/rar";
271 rarfile_rar_mimetype = rarfile_mimetype;
272 archivefile_mimetype = "application/x-"+__file_format_dict__['format_lower']+"";
273 archivefile_cat_mimetype = archivefile_mimetype;
274 archivefile_gzip_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+gzip";
275 archivefile_gz_mimetype = archivefile_gzip_mimetype;
276 archivefile_bzip2_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+bzip2";
277 archivefile_bz2_mimetype = archivefile_bzip2_mimetype;
278 archivefile_lz4_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lz4";
279 archivefile_lzop_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lzop";
280 archivefile_lzo_mimetype = archivefile_lzop_mimetype;
281 archivefile_zstandard_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+zstandard";
282 archivefile_zstd_mimetype = archivefile_zstandard_mimetype;
283 archivefile_lzma_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lzma";
284 archivefile_xz_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+xz";
285 archivefile_extensions = [__file_format_extension__, __file_format_extension__+".gz", __file_format_extension__+".bz2", __file_format_extension__+".zst", __file_format_extension__+".lz4", __file_format_extension__+".lzo", __file_format_extension__+".lzop", __file_format_extension__+".lzma", __file_format_extension__+".xz"];
287 if __name__ == "__main__":
288 import subprocess;
289 curscrpath = os.path.dirname(sys.argv[0]);
290 if(curscrpath==""):
291 curscrpath = ".";
292 if(os.sep=="\\"):
293 curscrpath = curscrpath.replace(os.sep, "/");
294 curscrpath = curscrpath + "/";
295 scrfile = curscrpath + "catfile.py";
296 if(os.path.exists(scrfile) and os.path.isfile(scrfile)):
297 scrcmd = subprocess.Popen([sys.executable, scrfile] + sys.argv[1:]);
298 scrcmd.wait();
300 def VerbosePrintOut(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
301 if(not dbgenable):
302 return True;
303 log_functions = {
304 "print": print,
305 "log": logging.info,
306 "warning": logging.warning,
307 "error": logging.error,
308 "critical": logging.critical,
309 "exception": logging.exception,
310 "logalt": lambda x: logging.log(dgblevel, x),
311 "debug": logging.debug
313 log_function = log_functions.get(outtype);
314 if(log_function):
315 log_function(dbgtxt);
316 return True;
317 return False;
319 def VerbosePrintOutReturn(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
320 VerbosePrintOut(dbgtxt, outtype, dbgenable, dgblevel);
321 return dbgtxt;
323 def RemoveWindowsPath(dpath):
324 if(dpath is None):
325 dpath = "";
326 if(os.sep!="/"):
327 dpath = dpath.replace(os.path.sep, "/");
328 dpath = dpath.rstrip("/");
329 if(dpath=="." or dpath==".."):
330 dpath = dpath + "/";
331 return dpath;
333 def NormalizeRelativePath(inpath):
334 inpath = RemoveWindowsPath(inpath);
335 if(os.path.isabs(inpath)):
336 outpath = inpath;
337 else:
338 if(inpath.startswith("./") or inpath.startswith("../")):
339 outpath = inpath;
340 else:
341 outpath = "./" + inpath;
342 return outpath;
344 def PrependPath(base_dir, child_path):
345 # Check if base_dir is None or empty, if so, return child_path as is
346 if not base_dir:
347 return child_path;
348 # Ensure base_dir ends with exactly one slash
349 if not base_dir.endswith('/'):
350 base_dir += '/';
351 # Check if child_path starts with ./ or ../ (indicating a relative path)
352 if child_path.startswith('./') or child_path.startswith('../'):
353 # For relative paths, we don't alter the child_path
354 return base_dir + child_path;
355 else:
356 # For non-relative paths, ensure there's no starting slash on child_path to avoid double slashes
357 return base_dir + child_path.lstrip('/');
359 def ListDir(dirpath, followlink=False, duplicates=False):
360 if(isinstance(dirpath, (list, tuple, ))):
361 dirpath = list(filter(None, dirpath));
362 elif(isinstance(dirpath, (str, ))):
363 dirpath = list(filter(None, [dirpath]));
364 retlist = [];
365 for mydirfile in dirpath:
366 if(not os.path.exists(mydirfile)):
367 return False;
368 mydirfile = NormalizeRelativePath(mydirfile);
369 if(os.path.exists(mydirfile) and os.path.islink(mydirfile)):
370 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile));
371 if(os.path.exists(mydirfile) and os.path.isdir(mydirfile)):
372 for root, dirs, filenames in os.walk(mydirfile):
373 dpath = root;
374 dpath = RemoveWindowsPath(dpath);
375 if(dpath not in retlist and not duplicates):
376 retlist.append(dpath);
377 if(duplicates):
378 retlist.append(dpath);
379 for file in filenames:
380 fpath = os.path.join(root, file);
381 fpath = RemoveWindowsPath(fpath);
382 if(fpath not in retlist and not duplicates):
383 retlist.append(fpath);
384 if(duplicates):
385 retlist.append(fpath);
386 else:
387 retlist.append(RemoveWindowsPath(mydirfile));
388 return retlist;
390 def ListDirAdvanced(dirpath, followlink=False, duplicates=False):
391 if isinstance(dirpath, (list, tuple)):
392 dirpath = list(filter(None, dirpath));
393 elif isinstance(dirpath, str):
394 dirpath = list(filter(None, [dirpath]));
395 retlist = []
396 for mydirfile in dirpath:
397 if not os.path.exists(mydirfile):
398 return False;
399 mydirfile = NormalizeRelativePath(mydirfile);
400 if os.path.exists(mydirfile) and os.path.islink(mydirfile) and followlink:
401 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile))
402 if os.path.exists(mydirfile) and os.path.isdir(mydirfile):
403 for root, dirs, filenames in os.walk(mydirfile):
404 # Sort dirs and filenames alphabetically in place
405 dirs.sort(key=lambda x: x.lower());
406 filenames.sort(key=lambda x: x.lower());
407 dpath = RemoveWindowsPath(root);
408 if not duplicates and dpath not in retlist:
409 retlist.append(dpath);
410 elif duplicates:
411 retlist.append(dpath);
412 for file in filenames:
413 fpath = os.path.join(root, file);
414 fpath = RemoveWindowsPath(fpath);
415 if not duplicates and fpath not in retlist:
416 retlist.append(fpath);
417 elif duplicates:
418 retlist.append(fpath);
419 else:
420 retlist.append(RemoveWindowsPath(mydirfile));
421 return retlist;
423 def create_alias_function(prefix, base_name, suffix, target_function):
424 # Define a new function that wraps the target function
425 def alias_function(*args, **kwargs):
426 return target_function(*args, **kwargs);
428 # Create the function name by combining the prefix, base name, and the suffix
429 function_name = "{}{}{}".format(prefix, base_name, suffix);
431 # Add the new function to the global namespace
432 globals()[function_name] = alias_function;
434 def create_alias_function_alt(prefix, base_name, suffix, target_function):
435 # Create the function name by combining the prefix, base name, and the suffix
436 # Use the format method for string formatting, compatible with Python 2 and 3
437 function_name = "{}{}{}".format(prefix, base_name, suffix)
438 # Add the new function (alias of the target_function) to the global namespace
439 # This line is compatible as-is with both Python 2 and 3
440 globals()[function_name] = target_function
442 def FormatSpecsListToDict(formatspecs=__file_format_list__):
443 if(isinstance(formatspecs, (list, tuple, ))):
444 return {'format_name': formatspecs[0], 'format_magic': formatspecs[1], 'format_lower': formatspecs[2], 'format_len': formatspecs[3], 'format_hex': formatspecs[4], 'format_delimiter': formatspecs[5], 'format_ver': formatspecs[6], 'new_style': formatspecs[7], 'use_advanced_list': formatspecs[8], 'use_alt_inode': formatspecs[9]};
445 elif(isinstance(formatspecs, (dict, ))):
446 return formatspecs;
447 else:
448 return __file_format_dict__;
449 return __file_format_dict__;
451 # initial_value can be 0xFFFF or 0x0000
452 def crc16_ansi(msg, initial_value=0xFFFF):
453 # CRC-16-IBM / CRC-16-ANSI polynomial and initial value
454 poly = 0x8005; # Polynomial for CRC-16-IBM / CRC-16-ANSI
455 crc = initial_value; # Initial value
456 for b in msg:
457 crc ^= b << 8; # XOR byte into CRC top byte
458 for _ in range(8): # Process each bit
459 if crc & 0x8000: # If the top bit is set
460 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
461 else:
462 crc = crc << 1; # Just shift left
463 crc &= 0xFFFF; # Ensure CRC remains 16-bit
464 return crc;
466 # initial_value can be 0xFFFF or 0x0000
467 def crc16_ibm(msg, initial_value=0xFFFF):
468 return crc16_ansi(msg, initial_value);
470 # initial_value is 0xFFFF
471 def crc16(msg):
472 return crc16_ansi(msg, 0xFFFF);
474 # initial_value can be 0xFFFF, 0x1D0F or 0x0000
475 def crc16_ccitt(msg, initial_value=0xFFFF):
476 # CRC-16-CCITT polynomial
477 poly = 0x1021; # Polynomial for CRC-16-CCITT
478 # Use the specified initial value
479 crc = initial_value;
480 for b in msg:
481 crc ^= b << 8; # XOR byte into CRC top byte
482 for _ in range(8): # Process each bit
483 if crc & 0x8000: # If the top bit is set
484 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
485 else:
486 crc = crc << 1; # Just shift left
487 crc &= 0xFFFF; # Ensure CRC remains 16-bit
488 return crc;
490 # initial_value can be 0x42F0E1EBA9EA3693 or 0x0000000000000000
491 def crc64_ecma(msg, initial_value=0x0000000000000000):
492 # CRC-64-ECMA polynomial and initial value
493 poly = 0x42F0E1EBA9EA3693;
494 crc = initial_value; # Initial value for CRC-64-ECMA
495 for b in msg:
496 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
497 for _ in range(8): # Process each bit
498 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
499 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
500 else:
501 crc <<= 1; # Just shift left if the MSB is 0
502 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
503 return crc;
505 # initial_value can be 0x000000000000001B or 0xFFFFFFFFFFFFFFFF
506 def crc64_iso(msg, initial_value=0xFFFFFFFFFFFFFFFF):
507 # CRC-64-ISO polynomial and initial value
508 poly = 0x000000000000001B;
509 crc = initial_value; # Common initial value for CRC-64-ISO
510 for b in msg:
511 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
512 for _ in range(8): # Process each bit
513 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
514 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
515 else:
516 crc <<= 1; # Just shift left if the MSB is 0
517 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
518 return crc;
520 def GetDataFromArray(data, path, default=None):
521 element = data;
522 try:
523 for key in path:
524 element = element[key];
525 return element;
526 except (KeyError, TypeError, IndexError):
527 return default;
529 def GetDataFromArrayAlt(structure, path, default=None):
530 element = structure;
531 for key in path:
532 if isinstance(element, dict) and key in element:
533 element = element[key];
534 elif isinstance(element, list) and isinstance(key, int) and -len(element) <= key < len(element):
535 element = element[key];
536 else:
537 return default;
538 return element;
540 def GetHeaderChecksum(inlist=[], checksumtype="crc32", encodedata=True, formatspecs=__file_format_dict__):
541 formatspecs = FormatSpecsListToDict(formatspecs);
542 if isinstance(inlist, list):
543 fileheader = AppendNullBytes(inlist, formatspecs['format_delimiter']);
544 else:
545 fileheader = AppendNullByte(inlist, formatspecs['format_delimiter']);
546 if(encodedata):
547 fileheader = fileheader.encode('UTF-8');
548 if(checksumtype=="none" or checksumtype==""):
549 catfileheadercshex = format(0, 'x').lower();
550 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
551 catfileheadercshex = format(crc16(fileheader) & 0xffff, '04x').lower();
552 elif(checksumtype=="crc16_ccitt"):
553 catfileheadercshex = format(crc16_ccitt(fileheader) & 0xffff, '04x').lower();
554 elif(checksumtype=="adler32"):
555 catfileheadercshex = format(zlib.adler32(fileheader) & 0xffffffff, '08x').lower();
556 elif(checksumtype=="crc32"):
557 catfileheadercshex = format(crc32(fileheader) & 0xffffffff, '08x').lower();
558 elif(checksumtype=="crc64_ecma"):
559 catfileheadercshex = format(crc64_ecma(fileheader) & 0xffffffffffffffff, '016x').lower();
560 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
561 catfileheadercshex = format(crc64_iso(fileheader) & 0xffffffffffffffff, '016x').lower();
562 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
563 checksumoutstr = hashlib.new(checksumtype);
564 checksumoutstr.update(fileheader);
565 catfileheadercshex = checksumoutstr.hexdigest().lower();
566 else:
567 catfileheadercshex = format(0, 'x').lower();
568 return catfileheadercshex;
570 def GetFileChecksum(instr, checksumtype="crc32", encodedata=True, formatspecs=__file_format_dict__):
571 formatspecs = FormatSpecsListToDict(formatspecs);
572 if(encodedata):
573 instr = instr.encode('UTF-8');
574 if(checksumtype=="none" or checksumtype==""):
575 catinstrcshex = format(0, 'x').lower();
576 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
577 catinstrcshex = format(crc16(instr) & 0xffff, '04x').lower();
578 elif(checksumtype=="crc16_ccitt"):
579 catinstrcshex = format(crc16_ccitt(instr) & 0xffff, '04x').lower();
580 elif(checksumtype=="adler32"):
581 catinstrcshex = format(zlib.adler32(instr) & 0xffffffff, '08x').lower();
582 elif(checksumtype=="crc32"):
583 catinstrcshex = format(crc32(instr) & 0xffffffff, '08x').lower();
584 elif(checksumtype=="crc64_ecma"):
585 catinstrcshex = format(crc64_ecma(instr) & 0xffffffffffffffff, '016x').lower();
586 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
587 catinstrcshex = format(crc64_iso(instr) & 0xffffffffffffffff, '016x').lower();
588 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
589 checksumoutstr = hashlib.new(checksumtype);
590 checksumoutstr.update(instr);
591 catinstrcshex = checksumoutstr.hexdigest().lower();
592 else:
593 catinstrcshex = format(0, 'x').lower();
594 return catinstrcshex;
596 def ValidateHeaderChecksum(inlist=[], checksumtype="crc32", inchecksum="0", formatspecs=__file_format_dict__):
597 formatspecs = FormatSpecsListToDict(formatspecs);
598 catfileheadercshex = GetHeaderChecksum(inlist, checksumtype, True, formatspecs);
599 inchecksum = inchecksum.lower();
600 catfileheadercshex = catfileheadercshex.lower();
601 if(inchecksum==catfileheadercshex):
602 return True;
603 else:
604 return False;
606 def ValidateFileChecksum(infile, checksumtype="crc32", inchecksum="0", formatspecs=__file_format_dict__):
607 formatspecs = FormatSpecsListToDict(formatspecs);
608 catinfilecshex = GetFileChecksum(infile, checksumtype, True, formatspecs);
609 inchecksum = inchecksum.lower();
610 catinfilecshex = catinfilecshex.lower();
611 if(inchecksum==catinfilecshex):
612 return True;
613 else:
614 return False;
616 def ReadTillNullByteOld(fp, delimiter=__file_format_dict__['format_delimiter']):
617 curbyte = b"";
618 curfullbyte = b"";
619 nullbyte = delimiter.encode("UTF-8");
620 while(True):
621 curbyte = fp.read(1);
622 if(curbyte==nullbyte or not curbyte):
623 break;
624 curfullbyte = curfullbyte + curbyte;
625 return curfullbyte.decode('UTF-8');
627 def ReadUntilNullByteOld(fp, delimiter=__file_format_dict__['format_delimiter']):
628 return ReadTillNullByteOld(fp, delimiter);
630 def ReadTillNullByteAlt(fp, delimiter=__file_format_dict__['format_delimiter'], chunk_size=1024, max_read=1024000):
631 delimiter = delimiter.encode('UTF-8') # Ensure the delimiter is in bytes
632 buffer = bytearray();
633 total_read = 0;
634 delimiter_length = len(delimiter);
635 while True:
636 chunk = fp.read(chunk_size)
637 if not chunk:
638 # End of file reached without finding the delimiter
639 break;
640 buffer.extend(chunk);
641 total_read += len(chunk);
642 if delimiter in buffer:
643 # Delimiter found, calculate where to reset the file pointer
644 index = buffer.find(delimiter);
645 # Calculate how many extra bytes were read after the delimiter
646 extra_bytes_read = len(buffer) - (index + delimiter_length);
647 # Move the file pointer back to just after the delimiter
648 fp.seek(-extra_bytes_read, 1);
649 buffer = buffer[:index];
650 break;
651 if total_read >= max_read:
652 # Stop reading if max limit is reached to prevent excessive memory usage
653 raise MemoryError("Maximum read limit reached without finding the delimiter.");
654 # Check for incomplete UTF-8 sequences at the end of the buffer
655 if len(buffer) > 1 and 128 <= buffer[-1] < 192:
656 # This suggests that the last byte might be the start of a multi-byte character
657 # Try to read one more byte to complete the character
658 extra_byte = fp.read(1);
659 if extra_byte:
660 buffer.extend(extra_byte);
661 else:
662 # No more data available
663 break;
664 try:
665 return buffer.decode('UTF-8', errors='replace');
666 except UnicodeDecodeError:
667 return buffer.decode('UTF-8', errors='replace');
669 def ReadUntilNullByteAlt(fp, delimiter=__file_format_dict__['format_delimiter'], chunk_size=1024, max_read=1024000):
670 return ReadTillNullByteAlt(fp, delimiter, chunk_size, max_read);
672 def ReadTillNullByte(fp, delimiter=__file_format_dict__['format_delimiter'], max_read=1024000):
673 curfullbyte = bytearray();
674 nullbyte = delimiter.encode("UTF-8");
675 total_read = 0; # Track the total number of bytes read
676 while True:
677 curbyte = fp.read(1);
678 if curbyte == nullbyte or not curbyte:
679 break;
680 curfullbyte.extend(curbyte);
681 total_read += 1;
682 if total_read >= max_read:
683 raise MemoryError("Maximum read limit reached without finding the delimiter.");
684 # Decode the full byte array to string once out of the loop
685 try:
686 return curfullbyte.decode('UTF-8');
687 except UnicodeDecodeError:
688 # Handle potential partial UTF-8 characters
689 for i in range(1, 4):
690 try:
691 return curfullbyte[:-i].decode('UTF-8');
692 except UnicodeDecodeError:
693 continue;
694 raise; # Re-raise if decoding fails even after trimming
696 def ReadUntilNullByte(fp, delimiter=__file_format_dict__['format_delimiter'], max_read=1024000):
697 return ReadTillNullByte(fp, delimiter, max_read);
699 def ReadTillNullByteByNum(fp, delimiter=__file_format_dict__['format_delimiter'], num_delimiters=1, chunk_size=1024, max_read=1024000):
700 delimiter = delimiter.encode('UTF-8'); # Ensure the delimiter is in bytes
701 buffer = bytearray();
702 total_read = 0;
703 delimiter_length = len(delimiter);
704 results = [];
705 while len(results) < num_delimiters:
706 chunk = fp.read(chunk_size);
707 if not chunk:
708 # End of file reached; decode whatever is collected if it's the last needed part
709 if len(buffer) > 0:
710 results.append(buffer.decode('UTF-8', errors='replace'));
711 break;
712 buffer.extend(chunk)
713 total_read += len(chunk);
714 # Check if we have found the delimiter
715 while delimiter in buffer:
716 index = buffer.find(delimiter);
717 # Decode the section before the delimiter
718 results.append(buffer[:index].decode('UTF-8', errors='replace'));
719 # Remove the processed part from the buffer
720 buffer = buffer[index + delimiter_length:];
721 if len(results) == num_delimiters:
722 # If reached the required number of delimiters, adjust the file pointer and stop
723 fp.seek(-len(buffer), 1);
724 return results;
725 if total_read >= max_read:
726 # Stop reading if max limit is reached to prevent excessive memory usage
727 raise MemoryError("Maximum read limit reached without finding the delimiter.");
728 # Check for incomplete UTF-8 sequences at the end of the buffer
729 if len(buffer) > 1 and 128 <= buffer[-1] < 192:
730 # This suggests that the last byte might be the start of a multi-byte character
731 # Try to read one more byte to complete the character
732 extra_byte = fp.read(1);
733 if extra_byte:
734 buffer.extend(extra_byte);
735 else:
736 # No more data available
737 break;
738 # Process remaining buffer if less than the required number of delimiters were found
739 if len(buffer) > 0 and len(results) < num_delimiters:
740 results.append(buffer.decode('UTF-8', errors='replace'));
741 return results;
743 def ReadUntilNullByteByNum(fp, delimiter=__file_format_dict__['format_delimiter'], num_delimiters=1, chunk_size=1024, max_read=1024000):
744 return ReadTillNullByteByNum(fp, delimiter, num_delimiters, chunk_size, max_read);
746 def SeekToEndOfFile(fp):
747 lasttell = 0;
748 while(True):
749 fp.seek(1, 1);
750 if(lasttell==fp.tell()):
751 break;
752 lasttell = fp.tell();
753 return True;
755 def ReadFileHeaderData(fp, rounds=0, delimiter=__file_format_dict__['format_delimiter']):
756 rocount = 0;
757 roend = int(rounds);
758 HeaderOut = [];
759 while(rocount<roend):
760 HeaderOut.append(ReadTillNullByte(fp, delimiter));
761 rocount = rocount + 1;
762 return HeaderOut;
764 def ReadFileHeaderDataBySize(fp, delimiter=__file_format_dict__['format_delimiter']):
765 headerpresize = ReadTillNullByte(fp, delimiter);
766 headersize = int(headerpresize, 16);
767 if(headersize<=0):
768 return [];
769 headercontent = str(fp.read(headersize).decode('UTF-8')).split(delimiter);
770 fp.seek(1, 1);
771 rocount = 0;
772 roend = int(len(headercontent));
773 HeaderOut = [headerpresize];
774 while(rocount<roend):
775 HeaderOut.append(headercontent[rocount]);
776 rocount = rocount + 1;
777 return HeaderOut;
779 def ReadFileHeaderDataWoSize(fp, delimiter=__file_format_dict__['format_delimiter']):
780 preheaderdata = ReadFileHeaderData(fp, 2, delimiter);
781 headersize = int(preheaderdata[0], 16);
782 headernumfields = int(preheaderdata[1], 16);
783 if(headersize<=0 or headernumfields<=0):
784 return [];
785 headerdata = ReadTillNullByteByNum(fp, delimiter, headernumfields);
786 #headerdata = ReadFileHeaderData(fp, headernumfields, delimiter);
787 HeaderOut = preheaderdata + headerdata;
788 return HeaderOut;
790 def ReadFileHeaderDataBySizeWithContent(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
791 formatspecs = FormatSpecsListToDict(formatspecs);
792 delimiter = formatspecs['format_delimiter'];
793 fheaderstart = fp.tell();
794 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
795 if(len(HeaderOut)==0):
796 return False;
797 if(re.findall("^[.|/]", HeaderOut[3])):
798 fname = HeaderOut[3];
799 else:
800 fname = "./"+HeaderOut[3];
801 fchecksumtype = HeaderOut[-3].lower();
802 fcs = HeaderOut[-2].lower();
803 fccs = HeaderOut[-1].lower();
804 fsize = int(HeaderOut[5], 16);
805 fcompression = HeaderOut[12];
806 fcsize = int(HeaderOut[13], 16);
807 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
808 if(fcs!=newfcs and not skipchecksum):
809 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
810 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
811 return False;
812 fhend = fp.tell() - 1;
813 fcontentstart = fp.tell();
814 fcontents = BytesIO();
815 if(fsize>0 and not listonly):
816 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
817 fcontents.write(fp.read(fsize));
818 else:
819 fcontents.write(fp.read(fcsize));
820 elif(fsize>0 and listonly):
821 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
822 fp.seek(fsize, 1);
823 else:
824 fp.seek(fcsize, 1);
825 fcontents.seek(0, 0);
826 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
827 if(fccs!=newfccs and not skipchecksum and not listonly):
828 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
829 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
830 return False;
831 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
832 pass;
833 else:
834 fcontents.seek(0, 0);
835 if(uncompress):
836 fcontents = UncompressArchiveFile(fcontents, formatspecs);
837 fp.seek(1, 1);
838 fcontentend = fp.tell() - 1;
839 HeaderOut.append(fcontents);
840 return HeaderOut;
842 def ReadFileHeaderDataBySizeWithContentToArray(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
843 formatspecs = FormatSpecsListToDict(formatspecs);
844 delimiter = formatspecs['format_delimiter'];
845 fheaderstart = fp.tell();
846 if(formatspecs['new_style']):
847 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
848 else:
849 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
850 if(len(HeaderOut)==0):
851 return False;
852 fheadsize = int(HeaderOut[0], 16);
853 fnumfields = int(HeaderOut[1], 16);
854 ftype = int(HeaderOut[2], 16);
855 if(re.findall("^[.|/]", HeaderOut[3])):
856 fname = HeaderOut[3];
857 else:
858 fname = "./"+HeaderOut[3];
859 fbasedir = os.path.dirname(fname);
860 flinkname = HeaderOut[4];
861 fsize = int(HeaderOut[5], 16);
862 fatime = int(HeaderOut[6], 16);
863 fmtime = int(HeaderOut[7], 16);
864 fctime = int(HeaderOut[8], 16);
865 fbtime = int(HeaderOut[9], 16);
866 fmode = int(HeaderOut[10], 16);
867 fchmode = stat.S_IMODE(fmode);
868 ftypemod = stat.S_IFMT(fmode);
869 fwinattributes = int(HeaderOut[11], 16);
870 fcompression = HeaderOut[12];
871 fcsize = int(HeaderOut[13], 16);
872 fuid = int(HeaderOut[14], 16);
873 funame = HeaderOut[15];
874 fgid = int(HeaderOut[16], 16);
875 fgname = HeaderOut[17];
876 fid = int(HeaderOut[18], 16);
877 finode = int(HeaderOut[19], 16);
878 flinkcount = int(HeaderOut[20], 16);
879 fdev_minor = int(HeaderOut[21], 16);
880 fdev_major = int(HeaderOut[22], 16);
881 frdev_minor = int(HeaderOut[23], 16);
882 frdev_major = int(HeaderOut[24], 16);
883 fextrasize = int(HeaderOut[25], 16);
884 fextrafields = int(HeaderOut[26], 16);
885 extrafieldslist = [];
886 extrastart = 27;
887 extraend = extrastart + fextrafields;
888 extrafieldslist = [];
889 if(extrastart<extraend):
890 extrafieldslist.append(HeaderOut[extrastart]);
891 extrastart = extrastart + 1;
892 fchecksumtype = HeaderOut[extrastart].lower();
893 fcs = HeaderOut[extrastart + 1].lower();
894 fccs = HeaderOut[extrastart + 2].lower();
895 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
896 if(fcs!=newfcs and not skipchecksum):
897 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
898 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
899 return False;
900 fhend = fp.tell() - 1;
901 fcontentstart = fp.tell();
902 fcontents = BytesIO();
903 pyhascontents = False;
904 if(fsize>0 and not listonly):
905 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
906 fcontents.write(fp.read(fsize));
907 else:
908 fcontents.write(fp.read(fcsize));
909 pyhascontents = True;
910 elif(fsize>0 and listonly):
911 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
912 fp.seek(fsize, 1);
913 else:
914 fp.seek(fcsize, 1);
915 pyhascontents = False;
916 fcontents.seek(0, 0);
917 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
918 if(fccs!=newfccs and not skipchecksum and not listonly):
919 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
920 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
921 return False;
922 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
923 pass;
924 else:
925 fcontents.seek(0, 0);
926 if(uncompress):
927 fcontents = UncompressArchiveFile(fcontents, formatspecs);
928 fcontents.seek(0, 0);
929 fccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
930 fp.seek(1, 1);
931 fcontentend = fp.tell() - 1;
932 fcontents.seek(0, 0);
933 catlist = {'fheadersize': fheadsize, 'fhstart': fheaderstart, 'fhend': fhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': fchecksumtype, 'fnumfields': fnumfields + 2, 'frawheader': HeaderOut, 'fextrafields': fextrafields, 'fextrafieldsize': fextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': fcs, 'fcontentchecksum': fccs, 'fhascontents': pyhascontents, 'fcontentstart': fcontentstart, 'fcontentend': fcontentend, 'fcontents': fcontents};
934 return catlist;
936 def ReadFileHeaderDataBySizeWithContentToList(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
937 formatspecs = FormatSpecsListToDict(formatspecs);
938 delimiter = formatspecs['format_delimiter'];
939 fheaderstart = fp.tell();
940 if(formatspecs['new_style']):
941 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
942 else:
943 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
944 if(len(HeaderOut)==0):
945 return False;
946 fheadsize = int(HeaderOut[0], 16);
947 fnumfields = int(HeaderOut[1], 16);
948 ftype = int(HeaderOut[2], 16);
949 if(re.findall("^[.|/]", HeaderOut[3])):
950 fname = HeaderOut[3];
951 else:
952 fname = "./"+HeaderOut[3];
953 fbasedir = os.path.dirname(fname);
954 flinkname = HeaderOut[4];
955 fsize = int(HeaderOut[5], 16);
956 fatime = int(HeaderOut[6], 16);
957 fmtime = int(HeaderOut[7], 16);
958 fctime = int(HeaderOut[8], 16);
959 fbtime = int(HeaderOut[9], 16);
960 fmode = int(HeaderOut[10], 16);
961 fchmode = stat.S_IMODE(fmode);
962 ftypemod = stat.S_IFMT(fmode);
963 fwinattributes = int(HeaderOut[11], 16);
964 fcompression = HeaderOut[12];
965 fcsize = int(HeaderOut[13], 16);
966 fuid = int(HeaderOut[14], 16);
967 funame = HeaderOut[15];
968 fgid = int(HeaderOut[16], 16);
969 fgname = HeaderOut[17];
970 fid = int(HeaderOut[18], 16);
971 finode = int(HeaderOut[19], 16);
972 flinkcount = int(HeaderOut[20], 16);
973 fdev_minor = int(HeaderOut[21], 16);
974 fdev_major = int(HeaderOut[22], 16);
975 frdev_minor = int(HeaderOut[23], 16);
976 frdev_major = int(HeaderOut[24], 16);
977 fextrasize = int(HeaderOut[25], 16);
978 fextrafields = int(HeaderOut[26], 16);
979 extrafieldslist = [];
980 extrastart = 27;
981 extraend = extrastart + fextrafields;
982 extrafieldslist = [];
983 if(extrastart<extraend):
984 extrafieldslist.append(HeaderOut[extrastart]);
985 extrastart = extrastart + 1;
986 fchecksumtype = HeaderOut[extrastart].lower();
987 fcs = HeaderOut[extrastart + 1].lower();
988 fccs = HeaderOut[extrastart + 2].lower();
989 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
990 if(fcs!=newfcs and not skipchecksum):
991 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
992 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
993 return False;
994 fhend = fp.tell() - 1;
995 fcontentstart = fp.tell();
996 fcontents = BytesIO();
997 pyhascontents = False;
998 if(fsize>0 and not listonly):
999 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1000 fcontents.write(fp.read(fsize));
1001 else:
1002 fcontents.write(fp.read(fcsize));
1003 pyhascontents = True;
1004 elif(fsize>0 and listonly):
1005 if(fcompression=="none" or fcompression=="" or fcompression=="atuo"):
1006 fp.seek(fsize, 1);
1007 else:
1008 fp.seek(fcsize, 1);
1009 pyhascontents = False;
1010 fcontents.seek(0, 0);
1011 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
1012 if(fccs!=newfccs and not skipchecksum and not listonly):
1013 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
1014 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
1015 return False;
1016 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1017 pass;
1018 else:
1019 fcontents.seek(0, 0);
1020 if(uncompress):
1021 fcontents = UncompressArchiveFile(fcontents, formatspecs);
1022 fcontents.seek(0, 0);
1023 fp.seek(1, 1);
1024 fcontentend = fp.tell() - 1;
1025 catlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, extrafieldslist, fchecksumtype, fcontents];
1026 return catlist;
1028 def ReadFileDataBySizeWithContent(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1029 formatspecs = FormatSpecsListToDict(formatspecs);
1030 delimiter = formatspecs['format_delimiter'];
1031 curloc = fp.tell();
1032 if(curloc>0):
1033 fp.seek(0, 0);
1034 catheader = ReadFileHeaderData(fp, 4, delimiter);
1035 if(curloc>0):
1036 fp.seek(curloc, 0);
1037 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1038 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1039 if(not headercheck and not skipchecksum):
1040 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1041 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1042 return False;
1043 fnumfiles = int(catheader[1], 16);
1044 countnum = 0;
1045 flist = [];
1046 while(countnum < fnumfiles):
1047 HeaderOut = ReadFileHeaderDataBySizeWithContent(fp, listonly, uncompress, skipchecksum, formatspecs)
1048 if(len(HeaderOut)==0):
1049 break;
1050 flist.append(HeaderOut);
1051 countnum = countnum + 1;
1052 return flist;
1054 def ReadFileDataBySizeWithContentToArray(fp, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1055 formatspecs = FormatSpecsListToDict(formatspecs);
1056 delimiter = formatspecs['format_delimiter'];
1057 curloc = fp.tell();
1058 if(curloc>0):
1059 fp.seek(0, 0);
1060 catheader = ReadFileHeaderData(fp, 4, delimiter);
1061 if(curloc>0):
1062 fp.seek(curloc, 0);
1063 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1064 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1065 if(not headercheck and not skipchecksum):
1066 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1067 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1068 return False;
1069 catstring = catheader[0];
1070 catversion = re.findall(r"([\d]+)", catstring);
1071 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
1072 fprenumfiles = catheader[1];
1073 fnumfiles = int(fprenumfiles, 16);
1074 fprechecksumtype = catheader[2];
1075 fprechecksum = catheader[3];
1076 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
1077 if(seekstart<0 and seekstart>fnumfiles):
1078 seekstart = 0;
1079 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
1080 seekend = fnumfiles;
1081 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
1082 seekend = fnumfiles - abs(seekend);
1083 if(seekstart>0):
1084 il = 0;
1085 while(il < seekstart):
1086 prefhstart = fp.tell();
1087 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs['format_delimiter']);
1088 if(len(preheaderdata)==0):
1089 break;
1090 prefsize = int(preheaderdata[5], 16);
1091 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-3].lower(), True, formatspecs);
1092 prefcs = preheaderdata[-2];
1093 if(prefcs!=prenewfcs and not skipchecksum):
1094 VVerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
1095 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
1096 return False;
1097 valid_archive = False;
1098 invalid_archive = True;
1099 prefhend = fp.tell() - 1;
1100 prefcontentstart = fp.tell();
1101 prefcontents = BytesIO();
1102 pyhascontents = False;
1103 if(prefsize>0):
1104 prefcontents.write(fp.read(prefsize));
1105 prefcontents.seek(0, 0);
1106 prenewfccs = GetFileChecksum(prefcontents.read(), preheaderdata[-3].lower(), False, formatspecs);
1107 prefccs = preheaderdata[-1];
1108 pyhascontents = True;
1109 if(prefccs!=prenewfccs and not skipchecksum):
1110 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
1111 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
1112 return False;
1113 fp.seek(1, 1);
1114 il = il + 1;
1115 realidnum = 0;
1116 countnum = seekstart;
1117 while(countnum < seekend):
1118 catlist['ffilelist'].update({realidnum: {'fid': realidnum, 'fidalt': realidnum}});
1119 HeaderOut = ReadFileHeaderDataBySizeWithContentToArray(fp, listonly, uncompress, skipchecksum, formatspecs);
1120 if(len(HeaderOut)==0):
1121 break;
1122 catlist['ffilelist'][realidnum].update(HeaderOut);
1123 countnum = countnum + 1;
1124 realidnum = realidnum + 1;
1125 return catlist;
1127 def ReadFileDataBySizeWithContentToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1128 formatspecs = FormatSpecsListToDict(formatspecs);
1129 if(isinstance(infile, dict)):
1130 listcatfiles = infile;
1131 else:
1132 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
1133 return False;
1134 listcatfiles = ReadFileDataBySizeWithContentToArray(infile, seekstart, seekend, listonly, uncompress, skipchecksum, formatspecs);
1135 if(not listcatfiles):
1136 return False;
1137 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
1138 lenlist = len(listcatfiles['ffilelist']);
1139 lcfi = 0;
1140 lcfx = int(listcatfiles['fnumfiles']);
1141 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
1142 lcfx = int(lenlist);
1143 else:
1144 lcfx = int(listcatfiles['fnumfiles']);
1145 while(lcfi < lcfx):
1146 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
1147 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
1148 catarray['filetoid'].update(filetoidarray);
1149 catarray['idtofile'].update(idtofilearray);
1150 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
1151 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
1152 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
1153 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
1154 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
1155 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
1156 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
1157 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
1158 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
1159 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1160 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1161 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
1162 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
1163 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
1164 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
1165 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
1166 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1167 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1168 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
1169 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
1170 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
1171 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1172 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1173 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
1174 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
1175 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
1176 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
1177 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1178 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1179 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1180 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1181 lcfi = lcfi + 1;
1182 return catarray;
1184 def ReadFileDataBySizeWithContentToList(fp, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1185 formatspecs = FormatSpecsListToDict(formatspecs);
1186 delimiter = formatspecs['format_delimiter'];
1187 curloc = fp.tell();
1188 if(curloc>0):
1189 fp.seek(0, 0);
1190 catheader = ReadFileHeaderData(fp, 4, delimiter);
1191 if(curloc>0):
1192 fp.seek(curloc, 0);
1193 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1194 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1195 if(not headercheck and not skipchecksum):
1196 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1197 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1198 return False;
1199 catstring = catheader[0];
1200 catversion = re.findall(r"([\d]+)", catstring);
1201 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
1202 fprenumfiles = catheader[1];
1203 fnumfiles = int(fprenumfiles, 16);
1204 fprechecksumtype = catheader[2];
1205 fprechecksum = catheader[3];
1206 catlist = [];
1207 if(seekstart<0 and seekstart>fnumfiles):
1208 seekstart = 0;
1209 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
1210 seekend = fnumfiles;
1211 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
1212 seekend = fnumfiles - abs(seekend);
1213 if(seekstart>0):
1214 il = 0;
1215 while(il < seekstart):
1216 prefhstart = fp.tell();
1217 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs['format_delimiter']);
1218 if(len(preheaderdata)==0):
1219 break;
1220 prefsize = int(preheaderdata[5], 16);
1221 prefcompression = preheaderdata[12];
1222 prefcsize = int(preheaderdata[13], 16);
1223 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-3].lower(), True, formatspecs);
1224 prefcs = preheaderdata[-2];
1225 if(prefcs!=prenewfcs and not skipchecksum):
1226 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
1227 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
1228 return False;
1229 valid_archive = False;
1230 invalid_archive = True;
1231 prefhend = fp.tell() - 1;
1232 prefcontentstart = fp.tell();
1233 prefcontents = "";
1234 pyhascontents = False;
1235 if(prefsize>0):
1236 if(prefcompression=="none" or prefcompression=="" or prefcompression=="auto"):
1237 prefcontents = catfp.read(prefsize);
1238 else:
1239 prefcontents = catfp.read(prefcsize);
1240 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
1241 prefccs = preheaderdata[-1];
1242 pyhascontents = True;
1243 if(prefccs!=prenewfccs and not skipchecksum):
1244 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
1245 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
1246 return False;
1247 fp.seek(1, 1);
1248 il = il + 1;
1249 realidnum = 0;
1250 countnum = seekstart;
1251 while(countnum < seekend):
1252 HeaderOut = ReadFileHeaderDataBySizeWithContentToList(fp, listonly, uncompress, skipchecksum, formatspecs);
1253 if(len(HeaderOut)==0):
1254 break;
1255 catlist.append(HeaderOut);
1256 countnum = countnum + 1;
1257 realidnum = realidnum + 1;
1258 return catlist;
1260 def ReadInFileBySizeWithContentToArray(infile, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1261 formatspecs = FormatSpecsListToDict(formatspecs);
1262 delimiter = formatspecs['format_delimiter'];
1263 if(hasattr(infile, "read") or hasattr(infile, "write")):
1264 fp = infile;
1265 fp.seek(0, 0);
1266 fp = UncompressArchiveFile(fp, formatspecs);
1267 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1268 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1269 return False;
1270 if(not fp):
1271 return False;
1272 fp.seek(0, 0);
1273 elif(infile=="-"):
1274 fp = BytesIO();
1275 if(hasattr(sys.stdin, "buffer")):
1276 shutil.copyfileobj(sys.stdin.buffer, fp);
1277 else:
1278 shutil.copyfileobj(sys.stdin, fp);
1279 fp.seek(0, 0);
1280 fp = UncompressArchiveFile(fp, formatspecs);
1281 if(not fp):
1282 return False;
1283 fp.seek(0, 0);
1284 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
1285 fp = download_file_from_internet_file(infile);
1286 fp = UncompressArchiveFile(fp, formatspecs);
1287 fp.seek(0, 0);
1288 if(not fp):
1289 return False;
1290 fp.seek(0, 0);
1291 else:
1292 infile = RemoveWindowsPath(infile);
1293 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
1294 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1295 return False;
1296 compresscheck = CheckCompressionType(infile, formatspecs, True);
1297 if(not compresscheck):
1298 fextname = os.path.splitext(infile)[1];
1299 if(fextname==".gz"):
1300 compresscheck = "gzip";
1301 elif(fextname==".bz2"):
1302 compresscheck = "bzip2";
1303 elif(fextname==".zst"):
1304 compresscheck = "zstd";
1305 elif(fextname==".lz4" or fextname==".clz4"):
1306 compresscheck = "lz4";
1307 elif(fextname==".lzo" or fextname==".lzop"):
1308 compresscheck = "lzo";
1309 elif(fextname==".lzma" or fextname==".xz"):
1310 compresscheck = "lzma";
1311 else:
1312 return False;
1313 if(not compresscheck):
1314 return False;
1315 fp = UncompressFile(infile, formatspecs, "rb");
1316 return ReadFileDataBySizeWithContentToArray(fp, listonly, uncompress, skipchecksum, formatspecs);
1318 def ReadInFileBySizeWithContentToList(infile, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1319 formatspecs = FormatSpecsListToDict(formatspecs);
1320 delimiter = formatspecs['format_delimiter'];
1321 if(hasattr(infile, "read") or hasattr(infile, "write")):
1322 fp = infile;
1323 fp.seek(0, 0);
1324 fp = UncompressArchiveFile(fp, formatspecs);
1325 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1326 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1327 return False;
1328 if(not fp):
1329 return False;
1330 fp.seek(0, 0);
1331 elif(infile=="-"):
1332 fp = BytesIO();
1333 if(hasattr(sys.stdin, "buffer")):
1334 shutil.copyfileobj(sys.stdin.buffer, fp);
1335 else:
1336 shutil.copyfileobj(sys.stdin, fp);
1337 fp.seek(0, 0);
1338 fp = UncompressArchiveFile(fp, formatspecs);
1339 if(not fp):
1340 return False;
1341 fp.seek(0, 0);
1342 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
1343 fp = download_file_from_internet_file(infile);
1344 fp = UncompressArchiveFile(fp, formatspecs);
1345 fp.seek(0, 0);
1346 if(not fp):
1347 return False;
1348 fp.seek(0, 0);
1349 else:
1350 infile = RemoveWindowsPath(infile);
1351 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
1352 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1353 return False;
1354 compresscheck = CheckCompressionType(infile, formatspecs, True);
1355 if(not compresscheck):
1356 fextname = os.path.splitext(infile)[1];
1357 if(fextname==".gz"):
1358 compresscheck = "gzip";
1359 elif(fextname==".bz2"):
1360 compresscheck = "bzip2";
1361 elif(fextname==".zst"):
1362 compresscheck = "zstd";
1363 elif(fextname==".lz4" or fextname==".clz4"):
1364 compresscheck = "lz4";
1365 elif(fextname==".lzo" or fextname==".lzop"):
1366 compresscheck = "lzo";
1367 elif(fextname==".lzma" or fextname==".xz"):
1368 compresscheck = "lzma";
1369 else:
1370 return False;
1371 if(not compresscheck):
1372 return False;
1373 fp = UncompressFile(infile, formatspecs, "rb");
1374 return ReadFileDataBySizeWithContentToList(fp, seekstart, seekend, listonly, uncompress, skipchecksum, formatspecs);
1376 def ReadInFileBySizeWithContentToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
1377 formatspecs = FormatSpecsListToDict(formatspecs);
1378 if(isinstance(infile, dict)):
1379 listcatfiles = infile;
1380 else:
1381 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
1382 infile = RemoveWindowsPath(infile);
1383 listcatfiles = ReadInFileBySizeWithContentToArray(infile, listonly, skipchecksum, formatspecs);
1384 if(not listcatfiles):
1385 return False;
1386 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
1387 lenlist = len(listcatfiles['ffilelist']);
1388 lcfi = 0;
1389 lcfx = int(listcatfiles['fnumfiles']);
1390 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
1391 lcfx = int(lenlist);
1392 else:
1393 lcfx = int(listcatfiles['fnumfiles']);
1394 while(lcfi < lcfx):
1395 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
1396 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
1397 catarray['filetoid'].update(filetoidarray);
1398 catarray['idtofile'].update(idtofilearray);
1399 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
1400 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
1401 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
1402 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
1403 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
1404 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
1405 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
1406 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
1407 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
1408 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1409 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1410 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
1411 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
1412 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
1413 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
1414 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
1415 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1416 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1417 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
1418 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
1419 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
1420 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1421 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1422 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
1423 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
1424 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
1425 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
1426 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1427 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1428 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1429 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1430 lcfi = lcfi + 1;
1431 return catarray;
1433 def AppendNullByte(indata, delimiter=__file_format_dict__['format_delimiter']):
1434 outdata = str(indata) + delimiter;
1435 return outdata;
1437 def AppendNullBytes(indata=[], delimiter=__file_format_dict__['format_delimiter']):
1438 outdata = "";
1439 inum = 0;
1440 il = len(indata);
1441 while(inum < il):
1442 outdata = outdata + AppendNullByte(indata[inum], delimiter);
1443 inum = inum + 1;
1444 return outdata;
1446 def AppendFileHeader(fp, numfiles, checksumtype="crc32", formatspecs=__file_format_dict__):
1447 formatspecs = FormatSpecsListToDict(formatspecs);
1448 delimiter = formatspecs['format_delimiter'];
1449 catver = formatspecs['format_ver'];
1450 fileheaderver = str(int(catver.replace(".", "")));
1451 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
1452 fp.write(fileheader.encode('UTF-8'));
1453 fnumfiles = format(int(numfiles), 'x').lower();
1454 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs['format_delimiter']);
1455 catfileheadercshex = GetFileChecksum(fileheader + fnumfilesa, checksumtype, True, formatspecs);
1456 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
1457 fp.write(fnumfilesa.encode('UTF-8'));
1458 try:
1459 fp.flush();
1460 os.fsync(fp.fileno());
1461 except io.UnsupportedOperation:
1462 pass;
1463 except AttributeError:
1464 pass;
1465 except OSError as e:
1466 pass;
1467 return fp;
1469 def MakeEmptyFilePointer(fp, checksumtype="crc32", formatspecs=__file_format_dict__):
1470 formatspecs = FormatSpecsListToDict(formatspecs);
1471 AppendFileHeader(fp, 0, checksumtype, formatspecs);
1472 return fp;
1474 def MakeEmptyFile(outfile, compression="auto", compressionlevel=None, checksumtype="crc32", formatspecs=__file_format_dict__, returnfp=False):
1475 formatspecs = FormatSpecsListToDict(formatspecs);
1476 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1477 if(os.path.exists(outfile)):
1478 try:
1479 os.unlink(outfile);
1480 except OSError as e:
1481 pass;
1482 if(outfile=="-"):
1483 verbose = False;
1484 catfpfp = BytesIO();
1485 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1486 catfp = outfile;
1487 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1488 catfp = BytesIO();
1489 else:
1490 fbasename = os.path.splitext(outfile)[0];
1491 fextname = os.path.splitext(outfile)[1];
1492 catfp = CompressOpenFile(outfile, True, compressionlevel);
1493 catfp = AppendFileHeader(catfp, 0, checksumtype, formatspecs);
1494 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1495 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1496 try:
1497 catfp.flush();
1498 os.fsync(catfp.fileno());
1499 except io.UnsupportedOperation:
1500 pass;
1501 except AttributeError:
1502 pass;
1503 except OSError as e:
1504 pass;
1505 if(outfile=="-"):
1506 catfp.seek(0, 0);
1507 if(hasattr(sys.stdout, "buffer")):
1508 shutil.copyfileobj(catfp, sys.stdout.buffer);
1509 else:
1510 shutil.copyfileobj(catfp, sys.stdout);
1511 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1512 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1513 catfp.seek(0, 0);
1514 upload_file_to_internet_file(catfp, outfile);
1515 if(returnfp):
1516 catfp.seek(0, 0);
1517 return catfp;
1518 else:
1519 catfp.close();
1520 return True;
1522 def AppendFileHeaderWithContent(fp, filevalues=[], extradata=[], filecontent="", checksumtype="crc32", formatspecs=__file_format_dict__):
1523 formatspecs = FormatSpecsListToDict(formatspecs);
1524 extrafields = format(len(extradata), 'x').lower();
1525 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
1526 if(len(extradata)>0):
1527 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
1528 extrasizelen = format(len(extrasizestr), 'x').lower();
1529 catoutlen = len(filevalues) + len(extradata) + 5;
1530 catoutlenhex = format(catoutlen, 'x').lower();
1531 catoutlist = filevalues;
1532 catoutlist.insert(0, catoutlenhex);
1533 catoutlist.append(extrasizelen);
1534 catoutlist.append(extrafields);
1535 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
1536 if(len(extradata)>0):
1537 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
1538 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
1539 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
1540 catfilecontentcshex = GetFileChecksum(filecontent, checksumtype, False, formatspecs);
1541 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
1542 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
1543 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
1544 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
1545 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
1546 catfileoutstrecd = catfileoutstr.encode('UTF-8');
1547 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
1548 catfileout = catfileoutstrecd + filecontent + nullstrecd;
1549 fp.write(catfileout);
1550 try:
1551 fp.flush();
1552 os.fsync(fp.fileno());
1553 except io.UnsupportedOperation:
1554 pass;
1555 except AttributeError:
1556 pass;
1557 except OSError as e:
1558 pass;
1559 return fp;
1561 def AppendFilesWithContent(infiles, fp, dirlistfromtxt=False, filevalues=[], extradata=[], compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
1562 formatspecs = FormatSpecsListToDict(formatspecs);
1563 advancedlist = formatspecs['use_advanced_list'];
1564 altinode = formatspecs['use_alt_inode'];
1565 if(verbose):
1566 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
1567 if(infiles=="-"):
1568 for line in sys.stdin:
1569 infilelist.append(line.strip());
1570 infilelist = list(filter(None, infilelist));
1571 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
1572 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
1573 return False;
1574 with UncompressFile(infiles, formatspecs, "r") as finfile:
1575 for line in finfile:
1576 infilelist.append(line.strip());
1577 infilelist = list(filter(None, infilelist));
1578 else:
1579 if(isinstance(infiles, (list, tuple, ))):
1580 infilelist = list(filter(None, infiles));
1581 elif(isinstance(infiles, (str, ))):
1582 infilelist = list(filter(None, [infiles]));
1583 if(advancedlist):
1584 GetDirList = ListDirAdvanced(infilelist, followlink, False);
1585 else:
1586 GetDirList = ListDir(infilelist, followlink, False);
1587 if(not GetDirList):
1588 return False;
1589 curinode = 0;
1590 curfid = 0;
1591 inodelist = [];
1592 inodetofile = {};
1593 filetoinode = {};
1594 inodetocatinode = {};
1595 numfiles = int(len(GetDirList));
1596 fnumfiles = format(numfiles, 'x').lower();
1597 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
1598 for curfname in GetDirList:
1599 if(re.findall("^[.|/]", curfname)):
1600 fname = curfname;
1601 else:
1602 fname = "./"+curfname;
1603 if(verbose):
1604 VerbosePrintOut(fname);
1605 if(not followlink or followlink is None):
1606 fstatinfo = os.lstat(fname);
1607 else:
1608 fstatinfo = os.stat(fname);
1609 fpremode = fstatinfo.st_mode;
1610 finode = fstatinfo.st_ino;
1611 flinkcount = fstatinfo.st_nlink;
1612 ftype = 0;
1613 if(stat.S_ISREG(fpremode)):
1614 ftype = 0;
1615 elif(stat.S_ISLNK(fpremode)):
1616 ftype = 2;
1617 elif(stat.S_ISCHR(fpremode)):
1618 ftype = 3;
1619 elif(stat.S_ISBLK(fpremode)):
1620 ftype = 4;
1621 elif(stat.S_ISDIR(fpremode)):
1622 ftype = 5;
1623 elif(stat.S_ISFIFO(fpremode)):
1624 ftype = 6;
1625 elif(stat.S_ISSOCK(fpremode)):
1626 ftype = 8;
1627 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
1628 ftype = 9;
1629 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
1630 ftype = 10;
1631 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
1632 ftype = 111;
1633 else:
1634 ftype = 0;
1635 flinkname = "";
1636 fcurfid = format(int(curfid), 'x').lower();
1637 if(not followlink and finode!=0):
1638 if(ftype!=1):
1639 if(finode in inodelist):
1640 ftype = 1;
1641 flinkname = inodetofile[finode];
1642 if(altinode):
1643 fcurinode = format(int(finode), 'x').lower();
1644 else:
1645 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
1646 if(finode not in inodelist):
1647 inodelist.append(finode);
1648 inodetofile.update({finode: fname});
1649 inodetocatinode.update({finode: curinode});
1650 if(altinode):
1651 fcurinode = format(int(finode), 'x').lower();
1652 else:
1653 fcurinode = format(int(curinode), 'x').lower();
1654 curinode = curinode + 1;
1655 else:
1656 fcurinode = format(int(curinode), 'x').lower();
1657 curinode = curinode + 1;
1658 curfid = curfid + 1;
1659 if(ftype==2):
1660 flinkname = os.readlink(fname);
1661 fdev = fstatinfo.st_dev;
1662 getfdev = GetDevMajorMinor(fdev);
1663 fdev_minor = getfdev[0];
1664 fdev_major = getfdev[1];
1665 frdev = fstatinfo.st_dev;
1666 if(hasattr(fstatinfo, "st_rdev")):
1667 frdev = fstatinfo.st_rdev;
1668 else:
1669 frdev = fstatinfo.st_dev;
1670 getfrdev = GetDevMajorMinor(frdev);
1671 frdev_minor = getfrdev[0];
1672 frdev_major = getfrdev[1];
1673 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
1674 fsize = format(int("0"), 'x').lower();
1675 elif(ftype==0 or ftype==7):
1676 fsize = format(int(fstatinfo.st_size), 'x').lower();
1677 else:
1678 fsize = format(int(fstatinfo.st_size)).lower();
1679 fatime = format(int(fstatinfo.st_atime), 'x').lower();
1680 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
1681 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
1682 if(hasattr(fstatinfo, "st_birthtime")):
1683 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
1684 else:
1685 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
1686 fmode = format(int(fstatinfo.st_mode), 'x').lower();
1687 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
1688 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
1689 fuid = format(int(fstatinfo.st_uid), 'x').lower();
1690 fgid = format(int(fstatinfo.st_gid), 'x').lower();
1691 funame = "";
1692 try:
1693 import pwd;
1694 try:
1695 userinfo = pwd.getpwuid(fstatinfo.st_uid);
1696 funame = userinfo.pw_name;
1697 except KeyError:
1698 funame = "";
1699 except ImportError:
1700 funame = "";
1701 fgname = "";
1702 try:
1703 import grp;
1704 try:
1705 groupinfo = grp.getgrgid(fstatinfo.st_gid);
1706 fgname = groupinfo.gr_name;
1707 except KeyError:
1708 fgname = "";
1709 except ImportError:
1710 fgname = "";
1711 fdev_minor = format(int(fdev_minor), 'x').lower();
1712 fdev_major = format(int(fdev_major), 'x').lower();
1713 frdev_minor = format(int(frdev_minor), 'x').lower();
1714 frdev_major = format(int(frdev_major), 'x').lower();
1715 finode = format(int(finode), 'x').lower();
1716 flinkcount = format(int(flinkcount), 'x').lower();
1717 if(hasattr(fstatinfo, "st_file_attributes")):
1718 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
1719 else:
1720 fwinattributes = format(int(0), 'x').lower();
1721 fcompression = "";
1722 fcsize = format(int(0), 'x').lower();
1723 fcontents = BytesIO();
1724 chunk_size = 1024;
1725 if(ftype==0 or ftype==7):
1726 with open(fname, "rb") as fpc:
1727 shutil.copyfileobj(fpc, fcontents);
1728 if(not compresswholefile):
1729 fcontents.seek(0, 2);
1730 ucfsize = fcontents.tell();
1731 fcontents.seek(0, 0);
1732 if(compression=="auto"):
1733 ilsize = len(compressionlistalt);
1734 ilmin = 0;
1735 ilcsize = [];
1736 while(ilmin < ilsize):
1737 cfcontents = BytesIO();
1738 shutil.copyfileobj(fcontents, cfcontents);
1739 fcontents.seek(0, 0);
1740 cfcontents.seek(0, 0);
1741 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
1742 if(cfcontents):
1743 cfcontents.seek(0, 2);
1744 ilcsize.append(cfcontents.tell());
1745 cfcontents.close();
1746 else:
1747 try:
1748 ilcsize.append(sys.maxint);
1749 except AttributeError:
1750 ilcsize.append(sys.maxsize);
1751 ilmin = ilmin + 1;
1752 ilcmin = ilcsize.index(min(ilcsize));
1753 compression = compressionlistalt[ilcmin];
1754 fcontents.seek(0, 0);
1755 cfcontents = BytesIO();
1756 shutil.copyfileobj(fcontents, cfcontents);
1757 cfcontents.seek(0, 0);
1758 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
1759 cfcontents.seek(0, 2);
1760 cfsize = cfcontents.tell();
1761 if(ucfsize > cfsize):
1762 fcsize = format(int(cfsize), 'x').lower();
1763 fcompression = compression;
1764 fcontents.close();
1765 fcontents = cfcontents;
1766 if(followlink and (ftype==1 or ftype==2)):
1767 flstatinfo = os.stat(flinkname);
1768 with open(flinkname, "rb") as fpc:
1769 shutil.copyfileobj(fpc, fcontents);
1770 if(not compresswholefile):
1771 fcontents.seek(0, 2);
1772 ucfsize = fcontents.tell();
1773 fcontents.seek(0, 0);
1774 if(compression=="auto"):
1775 ilsize = len(compressionlistalt);
1776 ilmin = 0;
1777 ilcsize = [];
1778 while(ilmin < ilsize):
1779 cfcontents = BytesIO();
1780 shutil.copyfileobj(fcontents, cfcontents);
1781 fcontents.seek(0, 0);
1782 cfcontents.seek(0, 0);
1783 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
1784 if(cfcontents):
1785 cfcontents.seek(0, 2);
1786 ilcsize.append(cfcontents.tell());
1787 cfcontents.close();
1788 else:
1789 try:
1790 ilcsize.append(sys.maxint);
1791 except AttributeError:
1792 ilcsize.append(sys.maxsize);
1793 ilmin = ilmin + 1;
1794 ilcmin = ilcsize.index(min(ilcsize));
1795 compression = compressionlistalt[ilcmin];
1796 fcontents.seek(0, 0);
1797 cfcontents = BytesIO();
1798 shutil.copyfileobj(fcontents, cfcontents);
1799 cfcontents.seek(0, 0);
1800 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
1801 cfcontents.seek(0, 2);
1802 cfsize = cfcontents.tell();
1803 if(ucfsize > cfsize):
1804 fcsize = format(int(cfsize), 'x').lower();
1805 fcompression = compression;
1806 fcontents.close();
1807 fcontents = cfcontents;
1808 if(fcompression=="none"):
1809 fcompression = "";
1810 fcontents.seek(0, 0);
1811 ftypehex = format(ftype, 'x').lower();
1812 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
1813 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
1814 if(numfiles>0):
1815 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
1816 fp.seek(0, 0);
1817 return fp;
1819 def AppendListsWithContent(inlist, fp, dirlistfromtxt=False, filevalues=[], extradata=[], compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
1820 formatspecs = FormatSpecsListToDict(formatspecs);
1821 if(verbose):
1822 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
1823 GetDirList = inlist;
1824 if(not GetDirList):
1825 return False;
1826 curinode = 0;
1827 curfid = 0;
1828 inodelist = [];
1829 inodetofile = {};
1830 filetoinode = {};
1831 inodetocatinode = {};
1832 numfiles = int(len(GetDirList));
1833 fnumfiles = format(numfiles, 'x').lower();
1834 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
1835 for curfname in GetDirList:
1836 ftype = format(curfname[0], 'x').lower();
1837 if(re.findall("^[.|/]", curfname[1])):
1838 fname = curfname[1];
1839 else:
1840 fname = "./"+curfname[1];
1841 fbasedir = os.path.dirname(fname);
1842 flinkname = curfname[2];
1843 fsize = format(curfname[3], 'x').lower();
1844 fatime = format(curfname[4], 'x').lower();
1845 fmtime = format(curfname[5], 'x').lower();
1846 fctime = format(curfname[6], 'x').lower();
1847 fbtime = format(curfname[7], 'x').lower();
1848 fmode = format(curfname[8], 'x').lower();
1849 fwinattributes = format(curfname[9], 'x').lower();
1850 fcompression = curfname[10];
1851 fcsize = format(curfname[11], 'x').lower();
1852 fuid = format(curfname[12], 'x').lower();
1853 funame = curfname[13];
1854 fgid = format(curfname[14], 'x').lower();
1855 fgname = curfname[15];
1856 fid = format(curfname[16], 'x').lower();
1857 finode = format(curfname[17], 'x').lower();
1858 flinkcount = format(curfname[18], 'x').lower();
1859 fdev_minor = format(curfname[19], 'x').lower();
1860 fdev_major = format(curfname[20], 'x').lower();
1861 frdev_minor = format(curfname[21], 'x').lower();
1862 frdev_major = format(curfname[22], 'x').lower();
1863 extradata = curfname[23];
1864 fchecksumtype = curfname[24];
1865 fcontents = curfname[25];
1866 catoutlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
1867 fcontents.seek(0, 0);
1868 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
1869 if(numfiles>0):
1870 fp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
1871 return fp;
1873 def AppendInFileWithContent(infile, fp, dirlistfromtxt=False, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
1874 formatspecs = FormatSpecsListToDict(formatspecs);
1875 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, True, False, formatspecs);
1876 return AppendListsWithContent(inlist, fp, dirlistfromtxt, filevalues, extradata, followlink, checksumtype, formatspecs, verbose);
1878 def AppendFilesWithContentToOutFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
1879 formatspecs = FormatSpecsListToDict(formatspecs);
1880 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1881 if(os.path.exists(outfile)):
1882 try:
1883 os.unlink(outfile);
1884 except OSError as e:
1885 pass;
1886 if(outfile=="-"):
1887 verbose = False;
1888 catfpfp = BytesIO();
1889 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1890 catfp = outfile;
1891 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1892 catfp = BytesIO();
1893 else:
1894 fbasename = os.path.splitext(outfile)[0];
1895 fextname = os.path.splitext(outfile)[1];
1896 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
1897 catfp = AppendFilesWithContent(infiles, catfp, dirlistfromtxt, filevalues, extradata, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose);
1898 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1899 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1900 try:
1901 catfp.flush();
1902 os.fsync(catfp.fileno());
1903 except io.UnsupportedOperation:
1904 pass;
1905 except AttributeError:
1906 pass;
1907 except OSError as e:
1908 pass;
1909 if(outfile=="-"):
1910 catfp.seek(0, 0);
1911 if(hasattr(sys.stdout, "buffer")):
1912 shutil.copyfileobj(catfp, sys.stdout.buffer);
1913 else:
1914 shutil.copyfileobj(catfp, sys.stdout);
1915 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1916 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1917 catfp.seek(0, 0);
1918 upload_file_to_internet_file(catfp, outfile);
1919 if(returnfp):
1920 catfp.seek(0, 0);
1921 return catfp;
1922 else:
1923 catfp.close();
1924 return True;
1926 def AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
1927 formatspecs = FormatSpecsListToDict(formatspecs);
1928 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1929 if(os.path.exists(outfile)):
1930 try:
1931 os.unlink(outfile);
1932 except OSError as e:
1933 pass;
1934 if(outfile=="-"):
1935 verbose = False;
1936 catfpfp = BytesIO();
1937 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1938 catfp = outfile;
1939 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1940 catfp = BytesIO();
1941 else:
1942 fbasename = os.path.splitext(outfile)[0];
1943 fextname = os.path.splitext(outfile)[1];
1944 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
1945 catfp = AppendListsWithContent(inlist, catfp, dirlistfromtxt, filevalues, extradata, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose);
1946 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1947 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1948 try:
1949 catfp.flush();
1950 os.fsync(catfp.fileno());
1951 except io.UnsupportedOperation:
1952 pass;
1953 except AttributeError:
1954 pass;
1955 except OSError as e:
1956 pass;
1957 if(outfile=="-"):
1958 catfp.seek(0, 0);
1959 if(hasattr(sys.stdout, "buffer")):
1960 shutil.copyfileobj(catfp, sys.stdout.buffer);
1961 else:
1962 shutil.copyfileobj(catfp, sys.stdout);
1963 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1964 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1965 catfp.seek(0, 0);
1966 upload_file_to_internet_file(catfp, outfile);
1967 if(returnfp):
1968 catfp.seek(0, 0);
1969 return catfp;
1970 else:
1971 catfp.close();
1972 return True;
1974 def AppendInFileWithContentToOutFile(infile, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
1975 formatspecs = FormatSpecsListToDict(formatspecs);
1976 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, True, False, formatspecs);
1977 return AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt, compression, compresswholefile, compressionlevel, filevalues, extradata, followlink, checksumtype, formatspecs, verbose, returnfp);
1979 def PrintPermissionString(fchmode, ftype):
1980 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
1981 permissionstr = "";
1982 for fmodval in str(oct(fchmode))[-3:]:
1983 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
1984 if(ftype==0 or ftype==7):
1985 permissionstr = "-" + permissionstr;
1986 if(ftype==1):
1987 permissionstr = "h" + permissionstr;
1988 if(ftype==2):
1989 permissionstr = "l" + permissionstr;
1990 if(ftype==3):
1991 permissionstr = "c" + permissionstr;
1992 if(ftype==4):
1993 permissionstr = "b" + permissionstr;
1994 if(ftype==5):
1995 permissionstr = "d" + permissionstr;
1996 if(ftype==6):
1997 permissionstr = "f" + permissionstr;
1998 if(ftype==8):
1999 permissionstr = "D" + permissionstr;
2000 if(ftype==9):
2001 permissionstr = "p" + permissionstr;
2002 if(ftype==10):
2003 permissionstr = "w" + permissionstr;
2004 try:
2005 permissionoutstr = stat.filemode(fchmode);
2006 except AttributeError:
2007 permissionoutstr = permissionstr;
2008 except KeyError:
2009 permissionoutstr = permissionstr;
2010 return permissionoutstr;
2012 def PrintPermissionStringAlt(fchmode, ftype):
2013 permissions = {
2014 '0': '---', '1': '--x', '2': '-w-', '3': '-wx',
2015 '4': 'r--', '5': 'r-x', '6': 'rw-', '7': 'rwx'
2017 # Translate file mode into permission string
2018 permissionstr = ''.join([permissions[i] for i in str(oct(fchmode))[-3:]]);
2019 # Append file type indicator
2020 type_indicators = {
2021 0: '-', 1: 'h', 2: 'l', 3: 'c', 4: 'b',
2022 5: 'd', 6: 'f', 8: 'D', 9: 'p', 10: 'w'
2024 file_type = type_indicators.get(ftype, '-');
2025 permissionstr = file_type + permissionstr;
2026 try:
2027 permissionoutstr = stat.filemode(fchmode);
2028 except AttributeError:
2029 permissionoutstr = permissionstr;
2030 return permissionoutstr;
2032 def CompressionSupport():
2033 compression_list = [];
2034 try:
2035 import gzip;
2036 compression_list.append("gz");
2037 compression_list.append("gzip");
2038 except ImportError:
2039 '''return False;'''
2040 try:
2041 import bz2;
2042 compression_list.append("bz2");
2043 compression_list.append("bzip2");
2044 except ImportError:
2045 '''return False;'''
2046 try:
2047 import lz4;
2048 compression_list.append("lz4");
2049 except ImportError:
2050 '''return False;'''
2051 try:
2052 import lzo;
2053 compression_list.append("lzo");
2054 compression_list.append("lzop");
2055 except ImportError:
2056 '''return False;'''
2057 try:
2058 import zstandard;
2059 compression_list.append("zstd");
2060 compression_list.append("zstandard");
2061 except ImportError:
2062 '''return False;'''
2063 try:
2064 import lzma;
2065 compression_list.append("lzma");
2066 compression_list.append("xz");
2067 except ImportError:
2068 '''return False;'''
2069 return compression_list;
2071 def CheckCompressionType(infile, formatspecs=__file_format_dict__, closefp=True):
2072 formatspecs = FormatSpecsListToDict(formatspecs);
2073 if(hasattr(infile, "read") or hasattr(infile, "write")):
2074 catfp = infile;
2075 else:
2076 try:
2077 catfp = open(infile, "rb");
2078 except FileNotFoundError:
2079 return False;
2080 catfp.seek(0, 0);
2081 prefp = catfp.read(2);
2082 filetype = False;
2083 if(prefp==binascii.unhexlify("1f8b")):
2084 filetype = "gzip";
2085 catfp.seek(0, 0);
2086 prefp = catfp.read(3);
2087 if(prefp==binascii.unhexlify("425a68")):
2088 filetype = "bzip2";
2089 if(prefp==binascii.unhexlify("5d0000")):
2090 filetype = "lzma";
2091 catfp.seek(0, 0);
2092 prefp = catfp.read(4);
2093 if(prefp==binascii.unhexlify("28b52ffd")):
2094 filetype = "zstd";
2095 if(prefp==binascii.unhexlify("04224d18")):
2096 filetype = "lz4";
2097 if(prefp==binascii.unhexlify("504B0304")):
2098 filetype = "zipfile";
2099 catfp.seek(0, 0);
2100 prefp = catfp.read(5);
2101 if(prefp==binascii.unhexlify("7573746172")):
2102 filetype = "tarfile";
2103 catfp.seek(0, 0);
2104 prefp = catfp.read(6);
2105 if(prefp==binascii.unhexlify("fd377a585a00")):
2106 filetype = "lzma";
2107 if(prefp==binascii.unhexlify("377abcaf271c")):
2108 filetype = "7zipfile";
2109 catfp.seek(0, 0);
2110 prefp = catfp.read(7);
2111 if(prefp==binascii.unhexlify("526172211a0700")):
2112 filetype = "rarfile";
2113 if(prefp==binascii.unhexlify("43617446696c65")):
2114 filetype = "catfile";
2115 catfp.seek(0, 0);
2116 prefp = catfp.read(8);
2117 if(prefp==binascii.unhexlify("526172211a070100")):
2118 filetype = "rarfile";
2119 catfp.seek(0, 0);
2120 prefp = catfp.read(formatspecs['format_len']);
2121 if(prefp==binascii.unhexlify(formatspecs['format_hex'])):
2122 filetype = formatspecs['format_lower'];
2123 catfp.seek(0, 0);
2124 prefp = catfp.read(9);
2125 if(prefp==binascii.unhexlify("894c5a4f000d0a1a0a")):
2126 filetype = "lzo";
2127 catfp.seek(0, 0);
2128 prefp = catfp.read(10);
2129 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
2130 filetype = "tarfile";
2131 catfp.seek(0, 0);
2132 if(filetype=="gzip" or filetype=="bzip2" or filetype=="lzma" or filetype=="zstd" or filetype=="lz4"):
2133 if(is_tarfile(catfp)):
2134 filetype = "tarfile";
2135 if(not filetype):
2136 if(is_tarfile(catfp)):
2137 filetype = "tarfile";
2138 elif(zipfile.is_zipfile(catfp)):
2139 filetype = "zipfile";
2140 elif(rarfile.is_rarfile(catfp) or rarfile.is_rarfile_sfx(catfp)):
2141 filetype = "rarile";
2142 else:
2143 filetype = False;
2144 catfp.seek(0, 0);
2145 if(closefp):
2146 catfp.close();
2147 return filetype;
2149 def CheckCompressionTypeFromString(instring, formatspecs=__file_format_dict__, closefp=True):
2150 formatspecs = FormatSpecsListToDict(formatspecs);
2151 try:
2152 instringsfile = BytesIO(instring);
2153 except TypeError:
2154 instringsfile = BytesIO(instring.encode("UTF-8"));
2155 return CheckCompressionType(instringsfile, formatspecs, closefp);
2157 def GetCompressionMimeType(infile, formatspecs=__file_format_dict__):
2158 formatspecs = FormatSpecsListToDict(formatspecs);
2159 compresscheck = CheckCompressionType(fp, formatspecs, False);
2160 if(compresscheck=="gzip" or compresscheck=="gz"):
2161 return archivefile_gzip_mimetype;
2162 if(compresscheck=="bzip2" or compresscheck=="bz2"):
2163 return archivefile_bzip2_mimetype;
2164 if(compresscheck=="zstd" or compresscheck=="zstandard"):
2165 return archivefile_zstandard_mimetype;
2166 if(compresscheck=="lz4"):
2167 return archivefile_lz4_mimetype;
2168 if(compresscheck=="lzo" or compresscheck=="lzop"):
2169 return archivefile_lzop_mimetype;
2170 if(compresscheck=="lzma"):
2171 return archivefile_lzma_mimetype;
2172 if(compresscheck=="xz"):
2173 return archivefile_xz_mimetype;
2174 if(compresscheck=="catfile" or compresscheck=="cat" or compresscheck==formatspecs['format_lower']):
2175 return archivefile_cat_mimetype;
2176 if(not compresscheck):
2177 return False;
2178 return False;
2180 def UncompressArchiveFile(fp, formatspecs=__file_format_dict__):
2181 formatspecs = FormatSpecsListToDict(formatspecs);
2182 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
2183 return False;
2184 compresscheck = CheckCompressionType(fp, formatspecs, False);
2185 if(compresscheck=="gzip"):
2186 try:
2187 import gzip;
2188 except ImportError:
2189 return False;
2190 catfp = gzip.GzipFile(fileobj=fp, mode="rb");
2191 if(compresscheck=="bzip2"):
2192 try:
2193 import bz2;
2194 except ImportError:
2195 return False;
2196 catfp = BytesIO();
2197 catfp.write(bz2.decompress(fp.read()));
2198 if(compresscheck=="zstd"):
2199 try:
2200 import zstandard;
2201 except ImportError:
2202 return False;
2203 catfp = BytesIO();
2204 catfp.write(zstandard.decompress(fp.read()));
2205 if(compresscheck=="lz4"):
2206 try:
2207 import lz4.frame;
2208 except ImportError:
2209 return False;
2210 catfp = BytesIO();
2211 catfp.write(lz4.frame.decompress(fp.read()));
2212 if(compresscheck=="lzo" or compresscheck=="lzop"):
2213 try:
2214 import lzo;
2215 except ImportError:
2216 return False;
2217 catfp = BytesIO();
2218 catfp.write(lzo.decompress(fp.read()));
2219 if(compresscheck=="lzma" or compresscheck=="xz"):
2220 try:
2221 import lzma;
2222 except ImportError:
2223 return False;
2224 catfp = BytesIO();
2225 catfp.write(lzma.decompress(fp.read()));
2226 if(compresscheck=="catfile" or compresscheck==formatspecs['format_lower']):
2227 catfp = fp;
2228 if(not compresscheck):
2229 try:
2230 import lzma;
2231 except ImportError:
2232 return False;
2233 catfp = BytesIO();
2234 with fp as fpcontent:
2235 try:
2236 catfp.write(lzma.decompress(fp.read()));
2237 except lzma.LZMAError:
2238 return False;
2239 if(compresscheck!="catfile" or compresscheck!=formatspecs['format_lower']):
2240 fp.close();
2241 return catfp;
2243 create_alias_function("Uncompress", __file_format_name__, "", UncompressArchiveFile);
2245 def UncompressFile(infile, formatspecs=__file_format_dict__, mode="rb"):
2246 formatspecs = FormatSpecsListToDict(formatspecs);
2247 compresscheck = CheckCompressionType(infile, formatspecs, False);
2248 if(sys.version_info[0]==2 and compresscheck):
2249 if(mode=="rt"):
2250 mode = "r";
2251 if(mode=="wt"):
2252 mode = "w";
2253 try:
2254 if(compresscheck=="gzip"):
2255 try:
2256 import gzip;
2257 except ImportError:
2258 return False;
2259 try:
2260 filefp = gzip.open(infile, mode, encoding="UTF-8");
2261 except (ValueError, TypeError) as e:
2262 filefp = gzip.open(infile, mode);
2263 if(compresscheck=="bzip2"):
2264 try:
2265 import bz2;
2266 except ImportError:
2267 return False;
2268 try:
2269 filefp = bz2.open(infile, mode, encoding="UTF-8");
2270 except (ValueError, TypeError) as e:
2271 filefp = bz2.open(infile, mode);
2272 if(compresscheck=="zstd"):
2273 try:
2274 import zstandard;
2275 except ImportError:
2276 return False;
2277 try:
2278 filefp = zstandard.open(infile, mode, encoding="UTF-8");
2279 except (ValueError, TypeError) as e:
2280 filefp = zstandard.open(infile, mode);
2281 if(compresscheck=="lz4"):
2282 try:
2283 import lz4.frame;
2284 except ImportError:
2285 return False;
2286 try:
2287 filefp = lz4.frame.open(infile, mode, encoding="UTF-8");
2288 except (ValueError, TypeError) as e:
2289 filefp = lz4.frame.open(infile, mode);
2290 if(compresscheck=="lzo"):
2291 try:
2292 import lzo;
2293 except ImportError:
2294 return False;
2295 try:
2296 filefp = lzo.open(infile, mode, encoding="UTF-8");
2297 except (ValueError, TypeError) as e:
2298 filefp = lzo.open(infile, mode);
2299 if(compresscheck=="lzma"):
2300 try:
2301 import lzma;
2302 except ImportError:
2303 return False;
2304 try:
2305 filefp = lzma.open(infile, mode, encoding="UTF-8");
2306 except (ValueError, TypeError) as e:
2307 filefp = lzma.open(infile, mode);
2308 if(compresscheck=="catfile" or compresscheck==formatspecs['format_lower']):
2309 try:
2310 filefp = open(infile, mode, encoding="UTF-8");
2311 except (ValueError, TypeError) as e:
2312 filefp = open(infile, mode);
2313 if(not compresscheck):
2314 try:
2315 filefp = open(infile, mode, encoding="UTF-8");
2316 except (ValueError, TypeError) as e:
2317 filefp = open(infile, mode);
2318 except FileNotFoundError:
2319 return False;
2320 return filefp;
2322 def UncompressString(infile):
2323 compresscheck = CheckCompressionTypeFromString(infile, formatspecs, False);
2324 if(compresscheck=="gzip"):
2325 try:
2326 import gzip;
2327 except ImportError:
2328 return False;
2329 fileuz = gzip.decompress(infile);
2330 if(compresscheck=="bzip2"):
2331 try:
2332 import bz2;
2333 except ImportError:
2334 return False;
2335 fileuz = bz2.decompress(infile);
2336 if(compresscheck=="zstd"):
2337 try:
2338 import zstandard;
2339 except ImportError:
2340 return False;
2341 fileuz = zstandard.decompress(infile);
2342 if(compresscheck=="lz4"):
2343 try:
2344 import lz4.frame;
2345 except ImportError:
2346 return False;
2347 fileuz = lz4.frame.decompress(infile);
2348 if(compresscheck=="lzo"):
2349 try:
2350 import lzo;
2351 except ImportError:
2352 return False;
2353 fileuz = lzo.decompress(infile);
2354 if(compresscheck=="lzma"):
2355 try:
2356 import lzma;
2357 except ImportError:
2358 return False;
2359 fileuz = lzma.decompress(infile);
2360 if(not compresscheck):
2361 fileuz = infile;
2362 if(hasattr(fileuz, 'decode')):
2363 fileuz = fileuz.decode("UTF-8");
2364 return fileuz;
2366 def UncompressStringAlt(infile):
2367 filefp = StringIO();
2368 outstring = UncompressString(infile);
2369 filefp.write(outstring);
2370 filefp.seek(0, 0);
2371 return filefp;
2373 def CheckCompressionSubType(infile, formatspecs=__file_format_dict__, closefp=True):
2374 formatspecs = FormatSpecsListToDict(formatspecs);
2375 compresscheck = CheckCompressionType(infile, formatspecs, False);
2376 if(not compresscheck):
2377 fextname = os.path.splitext(infile)[1];
2378 if(fextname==".gz"):
2379 compresscheck = "gzip";
2380 elif(fextname==".bz2"):
2381 compresscheck = "bzip2";
2382 elif(fextname==".zst"):
2383 compresscheck = "zstd";
2384 elif(fextname==".lz4"):
2385 compresscheck = "lz4";
2386 elif(fextname==".lzo" or fextname==".lzop"):
2387 compresscheck = "lzo";
2388 elif(fextname==".lzma" or fextname==".xz"):
2389 compresscheck = "lzma";
2390 else:
2391 return False;
2392 if(compresscheck=="gzip" or compresscheck=="bzip2" or compresscheck=="lzma" or compresscheck=="zstd" or compresscheck=="lz4"):
2393 if(is_tarfile(infile)):
2394 filetype = "tarfile";
2395 if(not compresscheck):
2396 if(is_tarfile(infile)):
2397 return "tarfile";
2398 elif(zipfile.is_zipfile(infile)):
2399 return "zipfile";
2400 elif(rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile)):
2401 return "rarile";
2402 else:
2403 return False;
2404 return False;
2405 if(compresscheck=="catfile"):
2406 return "catfile";
2407 if(compresscheck==formatspecs['format_lower']):
2408 return formatspecs['format_lower'];
2409 if(compresscheck=="tarfile"):
2410 return "tarfile";
2411 if(compresscheck=="zipfile"):
2412 return "zipfile";
2413 if(rarfile_support and compresscheck=="rarfile"):
2414 return "rarfile";
2415 if(py7zr_support and compresscheck=="7zipfile"):
2416 return "7zipfile";
2417 if(hasattr(infile, "read") or hasattr(infile, "write")):
2418 catfp = UncompressArchiveFile(infile, formatspecs['format_lower']);
2419 else:
2420 try:
2421 if(compresscheck=="gzip"):
2422 try:
2423 import gzip;
2424 except ImportError:
2425 return False;
2426 catfp = gzip.GzipFile(infile, "rb");
2427 if(compresscheck=="bzip2"):
2428 try:
2429 import bz2;
2430 except ImportError:
2431 return False;
2432 catfp = bz2.BZ2File(infile, "rb");
2433 if(compresscheck=="lz4"):
2434 try:
2435 import lz4.frame;
2436 except ImportError:
2437 return False;
2438 catfp = lz4.frame.open(infile, "rb");
2439 if(compresscheck=="zstd"):
2440 try:
2441 import zstandard;
2442 except ImportError:
2443 return False;
2444 catfp = zstandard.open(infile, "rb");
2445 if(compresscheck=="lzma" or compresscheck=="xz"):
2446 try:
2447 import lzma;
2448 except ImportError:
2449 return False;
2450 catfp = lzma.open(infile, "rb");
2451 except FileNotFoundError:
2452 return False;
2453 filetype = False;
2454 prefp = catfp.read(5);
2455 if(prefp==binascii.unhexlify("7573746172")):
2456 filetype = "tarfile";
2457 catfp.seek(0, 0);
2458 prefp = catfp.read(7);
2459 if(prefp==binascii.unhexlify("43617446696c65")):
2460 filetype = "catfile";
2461 catfp.seek(0, 0);
2462 prefp = catfp.read(formatspecs['format_len']);
2463 if(prefp==binascii.unhexlify(formatspecs['format_hex'])):
2464 filetype = formatspecs['format_lower'];
2465 catfp.seek(0, 0);
2466 prefp = catfp.read(10);
2467 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
2468 filetype = "tarfile";
2469 catfp.seek(0, 0);
2470 if(closefp):
2471 catfp.close();
2472 return filetype;
2474 def GZipCompress(data, compresslevel=9):
2475 try:
2476 import gzip;
2477 except ImportError:
2478 return False;
2479 tmpfp = tempfile.NamedTemporaryFile("wb", delete=False);
2480 tmpfp.close();
2481 tmpfp = gzip.GzipFile(tmpfp.name, mode="wb", compresslevel=compresslevel);
2482 tmpfp.write(data);
2483 tmpfp.close();
2484 try:
2485 catfp = open(tmpfp.name, "rb");
2486 except FileNotFoundError:
2487 return False;
2488 catdata = catfp.read();
2489 catfp.close();
2490 return catdata;
2492 def CompressArchiveFile(fp, compression="auto", compressionlevel=None, formatspecs=__file_format_dict__):
2493 formatspecs = FormatSpecsListToDict(formatspecs);
2494 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
2495 return False;
2496 fp.seek(0, 0);
2497 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
2498 compression = "auto";
2499 if(compression not in compressionlist and compression is None):
2500 compression = "auto";
2501 if(compression=="gzip"):
2502 try:
2503 import gzip;
2504 except ImportError:
2505 return False;
2506 catfp = BytesIO();
2507 if(compressionlevel is None):
2508 compressionlevel = 9;
2509 else:
2510 compressionlevel = int(compressionlevel);
2511 catfp.write(gzip.compress(fp.read(), compresslevel=compressionlevel));
2512 if(compression=="bzip2"):
2513 try:
2514 import bz2;
2515 except ImportError:
2516 return False;
2517 catfp = BytesIO();
2518 if(compressionlevel is None):
2519 compressionlevel = 9;
2520 else:
2521 compressionlevel = int(compressionlevel);
2522 catfp.write(bz2.compress(fp.read(), compresslevel=compressionlevel));
2523 if(compression=="lz4"):
2524 try:
2525 import lz4.frame;
2526 except ImportError:
2527 return False;
2528 catfp = BytesIO();
2529 if(compressionlevel is None):
2530 compressionlevel = 9;
2531 else:
2532 compressionlevel = int(compressionlevel);
2533 catfp.write(lz4.frame.compress(fp.read(), compression_level=compressionlevel));
2534 if(compression=="lzo" or compression=="lzop"):
2535 try:
2536 import lzo;
2537 except ImportError:
2538 return False;
2539 catfp = BytesIO();
2540 if(compressionlevel is None):
2541 compressionlevel = 9;
2542 else:
2543 compressionlevel = int(compressionlevel);
2544 catfp.write(lzo.compress(fp.read(), compresslevel=compressionlevel));
2545 if(compression=="zstd"):
2546 try:
2547 import zstandard;
2548 except ImportError:
2549 return False;
2550 catfp = BytesIO();
2551 if(compressionlevel is None):
2552 compressionlevel = 10;
2553 else:
2554 compressionlevel = int(compressionlevel);
2555 catfp.write(zstandard.compress(fp.read(), level=compressionlevel));
2556 if(compression=="lzma"):
2557 try:
2558 import lzma;
2559 except ImportError:
2560 return False;
2561 catfp = BytesIO();
2562 if(compressionlevel is None):
2563 compressionlevel = 9;
2564 else:
2565 compressionlevel = int(compressionlevel);
2566 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]));
2567 if(compression=="xz"):
2568 try:
2569 import lzma;
2570 except ImportError:
2571 return False;
2572 catfp = BytesIO();
2573 if(compressionlevel is None):
2574 compressionlevel = 9;
2575 else:
2576 compressionlevel = int(compressionlevel);
2577 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]));
2578 if(compression=="auto" or compression is None):
2579 catfp = fp;
2580 catfp.seek(0, 0);
2581 return catfp;
2583 create_alias_function("Compress", __file_format_name__, "", CompressArchiveFile);
2585 def CompressOpenFile(outfile, compressionenable=True, compressionlevel=None):
2586 if(outfile is None):
2587 return False;
2588 fbasename = os.path.splitext(outfile)[0];
2589 fextname = os.path.splitext(outfile)[1];
2590 if(compressionlevel is None and fextname!=".zst"):
2591 compressionlevel = 9;
2592 elif(compressionlevel is None and fextname==".zst"):
2593 compressionlevel = 10;
2594 else:
2595 compressionlevel = int(compressionlevel);
2596 if(sys.version_info[0]==2):
2597 mode = "w";
2598 else:
2599 mode = "wb";
2600 try:
2601 if(fextname not in outextlistwd or not compressionenable):
2602 try:
2603 outfp = open(outfile, "wb", encoding="UTF-8");
2604 except (ValueError, TypeError) as e:
2605 outfp = open(outfile, "wb");
2606 elif(fextname==".gz"):
2607 try:
2608 import gzip;
2609 except ImportError:
2610 return False;
2611 try:
2612 outfp = gzip.open(outfile, mode, compressionlevel, encoding="UTF-8");
2613 except (ValueError, TypeError) as e:
2614 outfp = gzip.open(outfile, mode, compressionlevel);
2615 elif(fextname==".bz2"):
2616 try:
2617 import bz2;
2618 except ImportError:
2619 return False;
2620 try:
2621 outfp = bz2.open(outfile, mode, compressionlevel, encoding="UTF-8");
2622 except (ValueError, TypeError) as e:
2623 outfp = bz2.open(outfile, mode, compressionlevel);
2624 elif(fextname==".zst"):
2625 try:
2626 import zstandard;
2627 except ImportError:
2628 return False;
2629 try:
2630 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel), encoding="UTF-8");
2631 except (ValueError, TypeError) as e:
2632 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel));
2633 elif(fextname==".xz"):
2634 try:
2635 import lzma;
2636 except ImportError:
2637 return False;
2638 try:
2639 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}], encoding="UTF-8");
2640 except (ValueError, TypeError) as e:
2641 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]);
2642 elif(fextname==".lz4"):
2643 try:
2644 import lz4.frame;
2645 except ImportError:
2646 return False;
2647 try:
2648 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel, encoding="UTF-8");
2649 except (ValueError, TypeError) as e:
2650 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel);
2651 elif(fextname==".lzo"):
2652 try:
2653 import lzo;
2654 except ImportError:
2655 return False;
2656 try:
2657 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel, encoding="UTF-8");
2658 except (ValueError, TypeError) as e:
2659 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel);
2660 elif(fextname==".lzma"):
2661 try:
2662 import lzma;
2663 except ImportError:
2664 return False;
2665 try:
2666 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}], encoding="UTF-8");
2667 except (ValueError, TypeError) as e:
2668 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]);
2669 except FileNotFoundError:
2670 return False;
2671 return outfp;
2673 def GetDevMajorMinor(fdev):
2674 retdev = [];
2675 if(hasattr(os, "minor")):
2676 retdev.append(os.minor(fdev));
2677 else:
2678 retdev.append(0);
2679 if(hasattr(os, "major")):
2680 retdev.append(os.major(fdev));
2681 else:
2682 retdev.append(0);
2683 return retdev;
2685 def CheckSumSupport(checkfor, guaranteed=True):
2686 if(guaranteed):
2687 try:
2688 hash_list = sorted(list(hashlib.algorithms_guaranteed));
2689 except AttributeError:
2690 hash_list = sorted(list(hashlib.algorithms));
2691 else:
2692 try:
2693 hash_list = sorted(list(hashlib.algorithms_available));
2694 except AttributeError:
2695 hash_list = sorted(list(hashlib.algorithms));
2696 checklistout = sorted(hash_list + ['adler32', 'crc16', 'crc16_ansi', 'crc16_ibm', 'crc16_ccitt', 'crc32', 'crc64', 'crc64_ecma', 'crc64_iso', 'none']);
2697 if(checkfor in checklistout):
2698 return True;
2699 else:
2700 return False;
2702 def CheckSumSupportAlt(checkfor, guaranteed=True):
2703 if(guaranteed):
2704 try:
2705 hash_list = sorted(list(hashlib.algorithms_guaranteed));
2706 except AttributeError:
2707 hash_list = sorted(list(hashlib.algorithms));
2708 else:
2709 try:
2710 hash_list = sorted(list(hashlib.algorithms_available));
2711 except AttributeError:
2712 hash_list = sorted(list(hashlib.algorithms));
2713 checklistout = hash_list;
2714 if(checkfor in checklistout):
2715 return True;
2716 else:
2717 return False;
2719 def PackArchiveFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2720 formatspecs = FormatSpecsListToDict(formatspecs);
2721 advancedlist = formatspecs['use_advanced_list'];
2722 altinode = formatspecs['use_alt_inode'];
2723 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2724 outfile = RemoveWindowsPath(outfile);
2725 checksumtype = checksumtype.lower();
2726 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
2727 checksumtype="crc32";
2728 if(checksumtype=="none"):
2729 checksumtype = "";
2730 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
2731 compression = "auto";
2732 if(compression not in compressionlist and compression is None):
2733 compression = "auto";
2734 if(verbose):
2735 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2736 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2737 if(os.path.exists(outfile)):
2738 try:
2739 os.unlink(outfile);
2740 except OSError as e:
2741 pass;
2742 if(outfile=="-"):
2743 verbose = False;
2744 catfp = BytesIO();
2745 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2746 catfp = outfile;
2747 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2748 catfp = BytesIO();
2749 else:
2750 fbasename = os.path.splitext(outfile)[0];
2751 fextname = os.path.splitext(outfile)[1];
2752 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
2753 catver = formatspecs['format_ver'];
2754 fileheaderver = str(int(catver.replace(".", "")));
2755 infilelist = [];
2756 if(infiles=="-"):
2757 for line in sys.stdin:
2758 infilelist.append(line.strip());
2759 infilelist = list(filter(None, infilelist));
2760 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
2761 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
2762 return False;
2763 with UncompressFile(infiles, formatspecs, "r") as finfile:
2764 for line in finfile:
2765 infilelist.append(line.strip());
2766 infilelist = list(filter(None, infilelist));
2767 else:
2768 if(isinstance(infiles, (list, tuple, ))):
2769 infilelist = list(filter(None, infiles));
2770 elif(isinstance(infiles, (str, ))):
2771 infilelist = list(filter(None, [infiles]));
2772 if(advancedlist):
2773 GetDirList = ListDirAdvanced(infilelist, followlink, False);
2774 else:
2775 GetDirList = ListDir(infilelist, followlink, False);
2776 if(not GetDirList):
2777 return False;
2778 curinode = 0;
2779 curfid = 0;
2780 inodelist = [];
2781 inodetofile = {};
2782 filetoinode = {};
2783 inodetocatinode = {};
2784 numfiles = int(len(GetDirList));
2785 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
2786 for curfname in GetDirList:
2787 if(re.findall("^[.|/]", curfname)):
2788 fname = curfname;
2789 else:
2790 fname = "./"+curfname;
2791 if(verbose):
2792 VerbosePrintOut(fname);
2793 if(not followlink or followlink is None):
2794 fstatinfo = os.lstat(fname);
2795 else:
2796 fstatinfo = os.stat(fname);
2797 fpremode = fstatinfo.st_mode;
2798 finode = fstatinfo.st_ino;
2799 flinkcount = fstatinfo.st_nlink;
2800 ftype = 0;
2801 if(stat.S_ISREG(fpremode)):
2802 ftype = 0;
2803 elif(stat.S_ISLNK(fpremode)):
2804 ftype = 2;
2805 elif(stat.S_ISCHR(fpremode)):
2806 ftype = 3;
2807 elif(stat.S_ISBLK(fpremode)):
2808 ftype = 4;
2809 elif(stat.S_ISDIR(fpremode)):
2810 ftype = 5;
2811 elif(stat.S_ISFIFO(fpremode)):
2812 ftype = 6;
2813 elif(stat.S_ISSOCK(fpremode)):
2814 ftype = 8;
2815 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
2816 ftype = 9;
2817 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
2818 ftype = 10;
2819 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
2820 ftype = 111;
2821 else:
2822 ftype = 0;
2823 flinkname = "";
2824 fcurfid = format(int(curfid), 'x').lower();
2825 if(not followlink and finode!=0):
2826 if(ftype!=1):
2827 if(finode in inodelist):
2828 ftype = 1;
2829 flinkname = inodetofile[finode];
2830 if(altinode):
2831 fcurinode = format(int(finode), 'x').lower();
2832 else:
2833 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
2834 if(finode not in inodelist):
2835 inodelist.append(finode);
2836 inodetofile.update({finode: fname});
2837 inodetocatinode.update({finode: curinode});
2838 if(altinode):
2839 fcurinode = format(int(finode), 'x').lower();
2840 else:
2841 fcurinode = format(int(curinode), 'x').lower();
2842 curinode = curinode + 1;
2843 else:
2844 fcurinode = format(int(curinode), 'x').lower();
2845 curinode = curinode + 1;
2846 curfid = curfid + 1;
2847 if(ftype==2):
2848 flinkname = os.readlink(fname);
2849 fdev = fstatinfo.st_dev;
2850 getfdev = GetDevMajorMinor(fdev);
2851 fdev_minor = getfdev[0];
2852 fdev_major = getfdev[1];
2853 frdev = fstatinfo.st_dev;
2854 if(hasattr(fstatinfo, "st_rdev")):
2855 frdev = fstatinfo.st_rdev;
2856 else:
2857 frdev = fstatinfo.st_dev;
2858 getfrdev = GetDevMajorMinor(frdev);
2859 frdev_minor = getfrdev[0];
2860 frdev_major = getfrdev[1];
2861 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
2862 fsize = format(int("0"), 'x').lower();
2863 elif(ftype==0 or ftype==7):
2864 fsize = format(int(fstatinfo.st_size), 'x').lower();
2865 else:
2866 fsize = format(int(fstatinfo.st_size)).lower();
2867 fatime = format(int(fstatinfo.st_atime), 'x').lower();
2868 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
2869 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
2870 if(hasattr(fstatinfo, "st_birthtime")):
2871 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
2872 else:
2873 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
2874 fmode = format(int(fstatinfo.st_mode), 'x').lower();
2875 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
2876 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
2877 fuid = format(int(fstatinfo.st_uid), 'x').lower();
2878 fgid = format(int(fstatinfo.st_gid), 'x').lower();
2879 funame = "";
2880 try:
2881 import pwd;
2882 try:
2883 userinfo = pwd.getpwuid(fstatinfo.st_uid);
2884 funame = userinfo.pw_name;
2885 except KeyError:
2886 funame = "";
2887 except ImportError:
2888 funame = "";
2889 fgname = "";
2890 try:
2891 import grp;
2892 try:
2893 groupinfo = grp.getgrgid(fstatinfo.st_gid);
2894 fgname = groupinfo.gr_name;
2895 except KeyError:
2896 fgname = "";
2897 except ImportError:
2898 fgname = "";
2899 fdev_minor = format(int(fdev_minor), 'x').lower();
2900 fdev_major = format(int(fdev_major), 'x').lower();
2901 frdev_minor = format(int(frdev_minor), 'x').lower();
2902 frdev_major = format(int(frdev_major), 'x').lower();
2903 finode = format(int(finode), 'x').lower();
2904 flinkcount = format(int(flinkcount), 'x').lower();
2905 if(hasattr(fstatinfo, "st_file_attributes")):
2906 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
2907 else:
2908 fwinattributes = format(int(0), 'x').lower();
2909 fcompression = "";
2910 fcsize = format(int(0), 'x').lower();
2911 fcontents = BytesIO();
2912 if(ftype==0 or ftype==7):
2913 with open(fname, "rb") as fpc:
2914 shutil.copyfileobj(fpc, fcontents);
2915 if(not compresswholefile):
2916 fcontents.seek(0, 2);
2917 ucfsize = fcontents.tell();
2918 fcontents.seek(0, 0);
2919 if(compression=="auto"):
2920 ilsize = len(compressionlistalt);
2921 ilmin = 0;
2922 ilcsize = [];
2923 while(ilmin < ilsize):
2924 cfcontents = BytesIO();
2925 shutil.copyfileobj(fcontents, cfcontents);
2926 fcontents.seek(0, 0);
2927 cfcontents.seek(0, 0);
2928 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
2929 if(cfcontents):
2930 cfcontents.seek(0, 2);
2931 ilcsize.append(cfcontents.tell());
2932 cfcontents.close();
2933 else:
2934 try:
2935 ilcsize.append(sys.maxint);
2936 except AttributeError:
2937 ilcsize.append(sys.maxsize);
2938 ilmin = ilmin + 1;
2939 ilcmin = ilcsize.index(min(ilcsize));
2940 compression = compressionlistalt[ilcmin];
2941 fcontents.seek(0, 0);
2942 cfcontents = BytesIO();
2943 shutil.copyfileobj(fcontents, cfcontents);
2944 cfcontents.seek(0, 0);
2945 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
2946 cfcontents.seek(0, 2);
2947 cfsize = cfcontents.tell();
2948 if(ucfsize > cfsize):
2949 fcsize = format(int(cfsize), 'x').lower();
2950 fcompression = compression;
2951 fcontents.close();
2952 fcontents = cfcontents;
2953 if(fcompression=="none"):
2954 fcompression = "";
2955 if(followlink and (ftype==1 or ftype==2)):
2956 flstatinfo = os.stat(flinkname);
2957 with open(flinkname, "rb") as fpc:
2958 shutil.copyfileobj(fpc, fcontents);
2959 if(not compresswholefile):
2960 fcontents.seek(0, 2);
2961 ucfsize = fcontents.tell();
2962 fcontents.seek(0, 0);
2963 if(compression=="auto"):
2964 ilsize = len(compressionlistalt);
2965 ilmin = 0;
2966 ilcsize = [];
2967 while(ilmin < ilsize):
2968 cfcontents = BytesIO();
2969 shutil.copyfileobj(fcontents, cfcontents);
2970 fcontents.seek(0, 0);
2971 cfcontents.seek(0, 0);
2972 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
2973 if(cfcontents):
2974 cfcontents.seek(0, 2);
2975 ilcsize.append(cfcontents.tell());
2976 cfcontents.close();
2977 else:
2978 try:
2979 ilcsize.append(sys.maxint);
2980 except AttributeError:
2981 ilcsize.append(sys.maxsize);
2982 ilmin = ilmin + 1;
2983 ilcmin = ilcsize.index(min(ilcsize));
2984 compression = compressionlistalt[ilcmin];
2985 fcontents.seek(0, 0);
2986 cfcontents = BytesIO();
2987 shutil.copyfileobj(fcontents, cfcontents);
2988 cfcontents.seek(0, 0);
2989 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
2990 cfcontents.seek(0, 2);
2991 cfsize = cfcontents.tell();
2992 if(ucfsize > cfsize):
2993 fcsize = format(int(cfsize), 'x').lower();
2994 fcompression = compression;
2995 fcontents.close();
2996 fcontents = cfcontents;
2997 fcontents.seek(0, 0);
2998 ftypehex = format(ftype, 'x').lower();
2999 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3000 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3001 fcontents.close();
3002 if(numfiles>0):
3003 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3004 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3005 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3006 try:
3007 catfp.flush();
3008 os.fsync(catfp.fileno());
3009 except io.UnsupportedOperation:
3010 pass;
3011 except AttributeError:
3012 pass;
3013 except OSError as e:
3014 pass;
3015 if(outfile=="-"):
3016 catfp.seek(0, 0);
3017 if(hasattr(sys.stdout, "buffer")):
3018 shutil.copyfileobj(catfp, sys.stdout.buffer);
3019 else:
3020 shutil.copyfileobj(catfp, sys.stdout);
3021 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3022 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3023 catfp.seek(0, 0);
3024 upload_file_to_internet_file(catfp, outfile);
3025 if(returnfp):
3026 catfp.seek(0, 0);
3027 return catfp;
3028 else:
3029 catfp.close();
3030 return True;
3032 create_alias_function("Pack", __file_format_name__, "", PackArchiveFile);
3034 if(hasattr(shutil, "register_archive_format")):
3035 def PackArchiveFileFunc(archive_name, source_dir, **kwargs):
3036 return PackArchiveFile(source_dir, archive_name, False, "auto", True, None, False, "crc32", [], __file_format_dict__['format_delimiter'], False, False);
3037 create_alias_function("Pack", __file_format_name__, "Func", PackArchiveFileFunc);
3039 def PackArchiveFileFromDirList(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3040 formatspecs = FormatSpecsListToDict(formatspecs);
3041 return PackArchiveFile(infiles, outfile, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, returnfp);
3043 create_alias_function("Pack", __file_format_name__, "FromDirList", PackArchiveFileFromDirList);
3045 def PackArchiveFileFromTarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3046 formatspecs = FormatSpecsListToDict(formatspecs);
3047 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3048 outfile = RemoveWindowsPath(outfile);
3049 checksumtype = checksumtype.lower();
3050 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3051 checksumtype="crc32";
3052 if(checksumtype=="none"):
3053 checksumtype = "";
3054 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3055 compression = "auto";
3056 if(compression not in compressionlist and compression is None):
3057 compression = "auto";
3058 if(verbose):
3059 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3060 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3061 if(os.path.exists(outfile)):
3062 try:
3063 os.unlink(outfile);
3064 except OSError as e:
3065 pass;
3066 if(outfile=="-"):
3067 verbose = False;
3068 catfp = BytesIO();
3069 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3070 catfp = outfile;
3071 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3072 catfp = BytesIO();
3073 else:
3074 fbasename = os.path.splitext(outfile)[0];
3075 fextname = os.path.splitext(outfile)[1];
3076 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3077 catver = formatspecs['format_ver'];
3078 fileheaderver = str(int(catver.replace(".", "")));
3079 curinode = 0;
3080 curfid = 0;
3081 inodelist = [];
3082 inodetofile = {};
3083 filetoinode = {};
3084 inodetocatinode = {};
3085 if(infile=="-"):
3086 infile = BytesIO();
3087 if(hasattr(sys.stdin, "buffer")):
3088 shutil.copyfileobj(sys.stdin.buffer, infile);
3089 else:
3090 shutil.copyfileobj(sys.stdin, infile);
3091 infile.seek(0, 0);
3092 if(not infile):
3093 return False;
3094 infile.seek(0, 0);
3095 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3096 infile = download_file_from_internet_file(infile);
3097 infile.seek(0, 0);
3098 if(not infile):
3099 return False;
3100 infile.seek(0, 0);
3101 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
3102 return False;
3103 elif(os.path.exists(infile) and os.path.isfile(infile)):
3104 try:
3105 if(not tarfile.is_tarfile(infile)):
3106 return False;
3107 except AttributeError:
3108 if(not is_tarfile(infile)):
3109 return False;
3110 else:
3111 return False;
3112 try:
3113 if(hasattr(infile, "read") or hasattr(infile, "write")):
3114 tarfp = tarfile.open(fileobj=infile, mode="r");
3115 else:
3116 tarfp = tarfile.open(infile, "r");
3117 except FileNotFoundError:
3118 return False;
3119 numfiles = int(len(tarfp.getmembers()));
3120 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3121 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
3122 if(re.findall("^[.|/]", member.name)):
3123 fname = member.name;
3124 else:
3125 fname = "./"+member.name;
3126 if(verbose):
3127 VerbosePrintOut(fname);
3128 fpremode = member.mode;
3129 ffullmode = member.mode;
3130 flinkcount = 0;
3131 ftype = 0;
3132 if(member.isreg()):
3133 ffullmode = member.mode + stat.S_IFREG;
3134 ftype = 0;
3135 elif(member.isdev()):
3136 ffullmode = member.mode;
3137 ftype = 7;
3138 elif(member.islnk()):
3139 ffullmode = member.mode + stat.S_IFREG;
3140 ftype = 1;
3141 elif(member.issym()):
3142 ffullmode = member.mode + stat.S_IFLNK;
3143 ftype = 2;
3144 elif(member.ischr()):
3145 ffullmode = member.mode + stat.S_IFCHR;
3146 ftype = 3;
3147 elif(member.isblk()):
3148 ffullmode = member.mode + stat.S_IFBLK;
3149 ftype = 4;
3150 elif(member.isdir()):
3151 ffullmode = member.mode + stat.S_IFDIR;
3152 ftype = 5;
3153 elif(member.isfifo()):
3154 ffullmode = member.mode + stat.S_IFIFO;
3155 ftype = 6;
3156 elif(member.issparse()):
3157 ffullmode = member.mode;
3158 ftype = 12;
3159 else:
3160 ffullmode = member.mode;
3161 ftype = 0;
3162 flinkname = "";
3163 fcurfid = format(int(curfid), 'x').lower();
3164 fcurinode = format(int(curfid), 'x').lower();
3165 curfid = curfid + 1;
3166 if(ftype==2):
3167 flinkname = member.linkname;
3168 fdev_minor = format(int(member.devminor), 'x').lower();
3169 fdev_major = format(int(member.devmajor), 'x').lower();
3170 frdev_minor = format(int(member.devminor), 'x').lower();
3171 frdev_major = format(int(member.devmajor), 'x').lower();
3172 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
3173 fsize = format(int("0"), 'x').lower();
3174 elif(ftype==0 or ftype==7):
3175 fsize = format(int(member.size), 'x').lower();
3176 else:
3177 fsize = format(int(member.size), 'x').lower();
3178 fatime = format(int(member.mtime), 'x').lower();
3179 fmtime = format(int(member.mtime), 'x').lower();
3180 fctime = format(int(member.mtime), 'x').lower();
3181 fbtime = format(int(member.mtime), 'x').lower();
3182 fmode = format(int(ffullmode), 'x').lower();
3183 fchmode = format(int(stat.S_IMODE(ffullmode)), 'x').lower();
3184 ftypemod = format(int(stat.S_IFMT(ffullmode)), 'x').lower();
3185 fuid = format(int(member.uid), 'x').lower();
3186 fgid = format(int(member.gid), 'x').lower();
3187 funame = member.uname;
3188 fgname = member.gname;
3189 flinkcount = format(int(flinkcount), 'x').lower();
3190 fwinattributes = format(int(0), 'x').lower();
3191 fcompression = "";
3192 fcsize = format(int(0), 'x').lower();
3193 fcontents = BytesIO();
3194 if(ftype==0 or ftype==7):
3195 with tarfp.extractfile(member) as fpc:
3196 shutil.copyfileobj(fpc, fcontents);
3197 if(not compresswholefile):
3198 fcontents.seek(0, 2);
3199 ucfsize = fcontents.tell();
3200 fcontents.seek(0, 0);
3201 if(compression=="auto"):
3202 ilsize = len(compressionlistalt);
3203 ilmin = 0;
3204 ilcsize = [];
3205 while(ilmin < ilsize):
3206 cfcontents = BytesIO();
3207 shutil.copyfileobj(fcontents, cfcontents);
3208 fcontents.seek(0, 0);
3209 cfcontents.seek(0, 0);
3210 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3211 if(cfcontents):
3212 cfcontents.seek(0, 2);
3213 ilcsize.append(cfcontents.tell());
3214 cfcontents.close();
3215 else:
3216 try:
3217 ilcsize.append(sys.maxint);
3218 except AttributeError:
3219 ilcsize.append(sys.maxsize);
3220 ilmin = ilmin + 1;
3221 ilcmin = ilcsize.index(min(ilcsize));
3222 compression = compressionlistalt[ilcmin];
3223 fcontents.seek(0, 0);
3224 cfcontents = BytesIO();
3225 shutil.copyfileobj(fcontents, cfcontents);
3226 cfcontents.seek(0, 0);
3227 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3228 cfcontents.seek(0, 2);
3229 cfsize = cfcontents.tell();
3230 if(ucfsize > cfsize):
3231 fcsize = format(int(cfsize), 'x').lower();
3232 fcompression = compression;
3233 fcontents.close();
3234 fcontents = cfcontents;
3235 if(fcompression=="none"):
3236 fcompression = "";
3237 fcontents.seek(0, 0);
3238 ftypehex = format(ftype, 'x').lower();
3239 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3240 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3241 fcontents.close();
3242 if(numfiles>0):
3243 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3244 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3245 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3246 try:
3247 catfp.flush();
3248 os.fsync(catfp.fileno());
3249 except io.UnsupportedOperation:
3250 pass;
3251 except AttributeError:
3252 pass;
3253 except OSError as e:
3254 pass;
3255 if(outfile=="-"):
3256 catfp.seek(0, 0);
3257 if(hasattr(sys.stdout, "buffer")):
3258 shutil.copyfileobj(catfp, sys.stdout.buffer);
3259 else:
3260 shutil.copyfileobj(catfp, sys.stdout);
3261 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3262 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3263 catfp.seek(0, 0);
3264 upload_file_to_internet_file(catfp, outfile);
3265 if(returnfp):
3266 catfp.seek(0, 0);
3267 return catfp;
3268 else:
3269 catfp.close();
3270 return True;
3272 create_alias_function("Pack", __file_format_name__, "FromTarFile", PackArchiveFileFromTarFile);
3274 def PackArchiveFileFromZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3275 formatspecs = FormatSpecsListToDict(formatspecs);
3276 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3277 outfile = RemoveWindowsPath(outfile);
3278 checksumtype = checksumtype.lower();
3279 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3280 checksumtype="crc32";
3281 if(checksumtype=="none"):
3282 checksumtype = "";
3283 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3284 compression = "auto";
3285 if(compression not in compressionlist and compression is None):
3286 compression = "auto";
3287 if(verbose):
3288 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3289 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3290 if(os.path.exists(outfile)):
3291 try:
3292 os.unlink(outfile);
3293 except OSError as e:
3294 pass;
3295 if(outfile=="-"):
3296 verbose = False;
3297 catfp = BytesIO();
3298 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3299 catfp = outfile;
3300 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3301 catfp = BytesIO();
3302 else:
3303 fbasename = os.path.splitext(outfile)[0];
3304 fextname = os.path.splitext(outfile)[1];
3305 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3306 catver = formatspecs['format_ver'];
3307 fileheaderver = str(int(catver.replace(".", "")));
3308 curinode = 0;
3309 curfid = 0;
3310 inodelist = [];
3311 inodetofile = {};
3312 filetoinode = {};
3313 inodetocatinode = {};
3314 if(infile=="-"):
3315 infile = BytesIO();
3316 if(hasattr(sys.stdin, "buffer")):
3317 shutil.copyfileobj(sys.stdin.buffer, infile);
3318 else:
3319 shutil.copyfileobj(sys.stdin, infile);
3320 infile.seek(0, 0);
3321 if(not infile):
3322 return False;
3323 infile.seek(0, 0);
3324 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3325 infile = download_file_from_internet_file(infile);
3326 infile.seek(0, 0);
3327 if(not infile):
3328 return False;
3329 infile.seek(0, 0);
3330 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
3331 return False;
3332 else:
3333 return False;
3334 if(not zipfile.is_zipfile(infile)):
3335 return False;
3336 try:
3337 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
3338 except FileNotFoundError:
3339 return False;
3340 ziptest = zipfp.testzip();
3341 if(ziptest):
3342 VerbosePrintOut("Bad file found!");
3343 numfiles = int(len(zipfp.infolist()));
3344 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3345 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
3346 if(re.findall("^[.|/]", member.filename)):
3347 fname = member.filename;
3348 else:
3349 fname = "./"+member.filename;
3350 zipinfo = zipfp.getinfo(member.filename);
3351 if(verbose):
3352 VerbosePrintOut(fname);
3353 if(not member.is_dir()):
3354 fpremode = int(stat.S_IFREG + 438);
3355 elif(member.is_dir()):
3356 fpremode = int(stat.S_IFDIR + 511);
3357 flinkcount = 0;
3358 ftype = 0;
3359 if(not member.is_dir()):
3360 ftype = 0;
3361 elif(member.is_dir()):
3362 ftype = 5;
3363 flinkname = "";
3364 fcurfid = format(int(curfid), 'x').lower();
3365 fcurinode = format(int(curfid), 'x').lower();
3366 curfid = curfid + 1;
3367 fdev_minor = format(int(0), 'x').lower();
3368 fdev_major = format(int(0), 'x').lower();
3369 frdev_minor = format(int(0), 'x').lower();
3370 frdev_major = format(int(0), 'x').lower();
3371 if(ftype==5):
3372 fsize = format(int("0"), 'x').lower();
3373 elif(ftype==0):
3374 fsize = format(int(member.file_size), 'x').lower();
3375 else:
3376 fsize = format(int(member.file_size), 'x').lower();
3377 fatime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3378 fmtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3379 fctime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3380 fbtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3381 if(zipinfo.create_system==0 or zipinfo.create_system==10):
3382 fwinattributes = format(int(zipinfo.external_attr), 'x').lower();
3383 if(not member.is_dir()):
3384 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3385 fchmode = stat.S_IMODE(fmode);
3386 ftypemod = stat.S_IFMT(fmode);
3387 elif(member.is_dir()):
3388 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3389 fchmode = stat.S_IMODE(fmode);
3390 ftypemod = stat.S_IFMT(fmode);
3391 elif(zipinfo.create_system==3):
3392 fwinattributes = format(int(0), 'x').lower();
3393 fmode = format(int(zipinfo.external_attr), 'x').lower();
3394 fchmode = stat.S_IMODE(fmode);
3395 ftypemod = stat.S_IFMT(fmode);
3396 else:
3397 fwinattributes = format(int(0), 'x').lower();
3398 if(not member.is_dir()):
3399 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3400 fchmode = stat.S_IMODE(fmode);
3401 ftypemod = stat.S_IFMT(fmode);
3402 elif(member.is_dir()):
3403 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3404 fchmode = stat.S_IMODE(fmode);
3405 ftypemod = stat.S_IFMT(fmode);
3406 fcompression = "";
3407 fcsize = format(int(0), 'x').lower();
3408 try:
3409 fuid = format(int(os.getuid()), 'x').lower();
3410 except AttributeError:
3411 fuid = format(int(0), 'x').lower();
3412 except KeyError:
3413 fuid = format(int(0), 'x').lower();
3414 try:
3415 fgid = format(int(os.getgid()), 'x').lower();
3416 except AttributeError:
3417 fgid = format(int(0), 'x').lower();
3418 except KeyError:
3419 fgid = format(int(0), 'x').lower();
3420 try:
3421 import pwd;
3422 try:
3423 userinfo = pwd.getpwuid(os.getuid());
3424 funame = userinfo.pw_name;
3425 except KeyError:
3426 funame = "";
3427 except AttributeError:
3428 funame = "";
3429 except ImportError:
3430 funame = "";
3431 fgname = "";
3432 try:
3433 import grp;
3434 try:
3435 groupinfo = grp.getgrgid(os.getgid());
3436 fgname = groupinfo.gr_name;
3437 except KeyError:
3438 fgname = "";
3439 except AttributeError:
3440 fgname = "";
3441 except ImportError:
3442 fgname = "";
3443 fcontents = BytesIO();
3444 if(ftype==0):
3445 fcontents.write(zipfp.read(member.filename));
3446 if(not compresswholefile):
3447 fcontents.seek(0, 2);
3448 ucfsize = fcontents.tell();
3449 fcontents.seek(0, 0);
3450 if(compression=="auto"):
3451 ilsize = len(compressionlistalt);
3452 ilmin = 0;
3453 ilcsize = [];
3454 while(ilmin < ilsize):
3455 cfcontents = BytesIO();
3456 shutil.copyfileobj(fcontents, cfcontents);
3457 fcontents.seek(0, 0);
3458 cfcontents.seek(0, 0);
3459 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3460 cfcontents.seek(0, 2);
3461 ilcsize.append(cfcontents.tell());
3462 cfcontents.close();
3463 ilmin = ilmin + 1;
3464 ilcmin = ilcsize.index(min(ilcsize));
3465 compression = compressionlistalt[ilcmin];
3466 fcontents.seek(0, 0);
3467 cfcontents = BytesIO();
3468 shutil.copyfileobj(fcontents, cfcontents);
3469 cfcontents.seek(0, 0);
3470 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3471 cfcontents.seek(0, 2);
3472 cfsize = cfcontents.tell();
3473 if(ucfsize > cfsize):
3474 fcsize = format(int(cfsize), 'x').lower();
3475 fcompression = compression;
3476 fcontents.close();
3477 fcontents = cfcontents;
3478 if(fcompression=="none"):
3479 fcompression = "";
3480 fcontents.seek(0, 0);
3481 ftypehex = format(ftype, 'x').lower();
3482 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3483 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3484 fcontents.close();
3485 if(numfiles>0):
3486 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3487 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3488 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3489 try:
3490 catfp.flush();
3491 os.fsync(catfp.fileno());
3492 except io.UnsupportedOperation:
3493 pass;
3494 except AttributeError:
3495 pass;
3496 except OSError as e:
3497 pass;
3498 if(outfile=="-"):
3499 catfp.seek(0, 0);
3500 if(hasattr(sys.stdout, "buffer")):
3501 shutil.copyfileobj(catfp, sys.stdout.buffer);
3502 else:
3503 shutil.copyfileobj(catfp, sys.stdout);
3504 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3505 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3506 catfp.seek(0, 0);
3507 upload_file_to_internet_file(catfp, outfile);
3508 if(returnfp):
3509 catfp.seek(0, 0);
3510 return catfp;
3511 else:
3512 catfp.close();
3513 return True;
3515 create_alias_function("Pack", __file_format_name__, "FromZipFile", PackArchiveFileFromZipFile);
3517 if(not rarfile_support):
3518 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3519 return False
3521 if(rarfile_support):
3522 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3523 formatspecs = FormatSpecsListToDict(formatspecs);
3524 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3525 outfile = RemoveWindowsPath(outfile);
3526 checksumtype = checksumtype.lower();
3527 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3528 checksumtype="crc32";
3529 if(checksumtype=="none"):
3530 checksumtype = "";
3531 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3532 compression = "auto";
3533 if(compression not in compressionlist and compression is None):
3534 compression = "auto";
3535 if(verbose):
3536 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3537 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3538 if(os.path.exists(outfile)):
3539 try:
3540 os.unlink(outfile);
3541 except OSError as e:
3542 pass;
3543 if(outfile=="-"):
3544 verbose = False;
3545 catfp = BytesIO();
3546 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3547 catfp = outfile;
3548 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3549 catfp = BytesIO();
3550 else:
3551 fbasename = os.path.splitext(outfile)[0];
3552 fextname = os.path.splitext(outfile)[1];
3553 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3554 catver = formatspecs['format_ver'];
3555 fileheaderver = str(int(catver.replace(".", "")));
3556 curinode = 0;
3557 curfid = 0;
3558 inodelist = [];
3559 inodetofile = {};
3560 filetoinode = {};
3561 inodetocatinode = {};
3562 if(not os.path.exists(infile) or not os.path.isfile(infile)):
3563 return False;
3564 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
3565 return False;
3566 rarfp = rarfile.RarFile(infile, "r");
3567 rartest = rarfp.testrar();
3568 if(rartest):
3569 VerbosePrintOut("Bad file found!");
3570 numfiles = int(len(rarfp.infolist()));
3571 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3572 try:
3573 catfp.flush();
3574 os.fsync(catfp.fileno());
3575 except io.UnsupportedOperation:
3576 pass;
3577 except AttributeError:
3578 pass;
3579 except OSError as e:
3580 pass;
3581 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
3582 is_unix = False;
3583 is_windows = False;
3584 if(member.host_os==rarfile.RAR_OS_UNIX):
3585 is_windows = False;
3586 try:
3587 member.external_attr
3588 is_unix = True;
3589 except AttributeError:
3590 is_unix = False;
3591 elif(member.host_os==rarfile.RAR_OS_WIN32):
3592 is_unix = False;
3593 try:
3594 member.external_attr
3595 is_windows = True;
3596 except AttributeError:
3597 is_windows = False;
3598 else:
3599 is_unix = False;
3600 is_windows = False;
3601 if(re.findall("^[.|/]", member.filename)):
3602 fname = member.filename;
3603 else:
3604 fname = "./"+member.filename;
3605 rarinfo = rarfp.getinfo(member.filename);
3606 if(verbose):
3607 VerbosePrintOut(fname);
3608 if(is_unix and member.external_attr !=0):
3609 fpremode = int(member.external_attr);
3610 elif(member.is_file()):
3611 fpremode = int(stat.S_IFREG + 438);
3612 elif(member.is_symlink()):
3613 fpremode = int(stat.S_IFLNK + 438);
3614 elif(member.is_dir()):
3615 fpremode = int(stat.S_IFDIR + 511);
3616 if(is_windows and member.external_attr !=0):
3617 fwinattributes = format(int(member.external_attr), 'x').lower();
3618 else:
3619 fwinattributes = format(int(0), 'x').lower();
3620 fcompression = "";
3621 fcsize = format(int(0), 'x').lower();
3622 flinkcount = 0;
3623 ftype = 0;
3624 if(member.is_file()):
3625 ftype = 0;
3626 elif(member.is_symlink()):
3627 ftype = 2;
3628 elif(member.is_dir()):
3629 ftype = 5;
3630 flinkname = "";
3631 if(ftype==2):
3632 flinkname = rarfp.read(member.filename).decode("UTF-8");
3633 fcurfid = format(int(curfid), 'x').lower();
3634 fcurinode = format(int(curfid), 'x').lower();
3635 curfid = curfid + 1;
3636 fdev_minor = format(int(0), 'x').lower();
3637 fdev_major = format(int(0), 'x').lower();
3638 frdev_minor = format(int(0), 'x').lower();
3639 frdev_major = format(int(0), 'x').lower();
3640 if(ftype==5):
3641 fsize = format(int("0"), 'x').lower();
3642 elif(ftype==0):
3643 fsize = format(int(member.file_size), 'x').lower();
3644 else:
3645 fsize = format(int(member.file_size), 'x').lower();
3646 try:
3647 if(member.atime):
3648 fatime = format(int(member.atime.timestamp()), 'x').lower();
3649 else:
3650 fatime = format(int(member.mtime.timestamp()), 'x').lower();
3651 except AttributeError:
3652 fatime = format(int(member.mtime.timestamp()), 'x').lower();
3653 fmtime = format(int(member.mtime.timestamp()), 'x').lower();
3654 try:
3655 if(member.ctime):
3656 fctime = format(int(member.ctime.timestamp()), 'x').lower();
3657 else:
3658 fctime = format(int(member.mtime.timestamp()), 'x').lower();
3659 except AttributeError:
3660 fctime = format(int(member.mtime.timestamp()), 'x').lower();
3661 fbtime = format(int(member.mtime.timestamp()), 'x').lower();
3662 if(is_unix and member.external_attr !=0):
3663 fmode = format(int(member.external_attr), 'x').lower();
3664 fchmode = format(int(stat.S_IMODE(member.external_attr)), 'x').lower();
3665 ftypemod = format(int(stat.S_IFMT(member.external_attr)), 'x').lower();
3666 elif(member.is_file()):
3667 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3668 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3669 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3670 elif(member.is_symlink()):
3671 fmode = format(int(stat.S_IFLNK + 438), 'x').lower();
3672 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3673 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3674 elif(member.is_dir()):
3675 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3676 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
3677 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
3678 try:
3679 fuid = format(int(os.getuid()), 'x').lower();
3680 except AttributeError:
3681 fuid = format(int(0), 'x').lower();
3682 except KeyError:
3683 fuid = format(int(0), 'x').lower();
3684 try:
3685 fgid = format(int(os.getgid()), 'x').lower();
3686 except AttributeError:
3687 fgid = format(int(0), 'x').lower();
3688 except KeyError:
3689 fgid = format(int(0), 'x').lower();
3690 try:
3691 import pwd;
3692 try:
3693 userinfo = pwd.getpwuid(os.getuid());
3694 funame = userinfo.pw_name;
3695 except KeyError:
3696 funame = "";
3697 except AttributeError:
3698 funame = "";
3699 except ImportError:
3700 funame = "";
3701 fgname = "";
3702 try:
3703 import grp;
3704 try:
3705 groupinfo = grp.getgrgid(os.getgid());
3706 fgname = groupinfo.gr_name;
3707 except KeyError:
3708 fgname = "";
3709 except AttributeError:
3710 fgname = "";
3711 except ImportError:
3712 fgname = "";
3713 fcontents = BytesIO();
3714 if(ftype==0):
3715 fcontents.write(rarfp.read(member.filename));
3716 if(not compresswholefile):
3717 fcontents.seek(0, 2);
3718 ucfsize = fcontents.tell();
3719 fcontents.seek(0, 0);
3720 if(compression=="auto"):
3721 ilsize = len(compressionlistalt);
3722 ilmin = 0;
3723 ilcsize = [];
3724 while(ilmin < ilsize):
3725 cfcontents = BytesIO();
3726 shutil.copyfileobj(fcontents, cfcontents);
3727 fcontents.seek(0, 0);
3728 cfcontents.seek(0, 0);
3729 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3730 if(cfcontents):
3731 cfcontents.seek(0, 2);
3732 ilcsize.append(cfcontents.tell());
3733 cfcontents.close();
3734 else:
3735 try:
3736 ilcsize.append(sys.maxint);
3737 except AttributeError:
3738 ilcsize.append(sys.maxsize);
3739 ilmin = ilmin + 1;
3740 ilcmin = ilcsize.index(min(ilcsize));
3741 compression = compressionlistalt[ilcmin];
3742 fcontents.seek(0, 0);
3743 cfcontents = BytesIO();
3744 shutil.copyfileobj(fcontents, cfcontents);
3745 cfcontents.seek(0, 0);
3746 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3747 cfcontents.seek(0, 2);
3748 cfsize = cfcontents.tell();
3749 if(ucfsize > cfsize):
3750 fcsize = format(int(cfsize), 'x').lower();
3751 fcompression = compression;
3752 fcontents.close();
3753 fcontents = cfcontents;
3754 if(fcompression=="none"):
3755 fcompression = "";
3756 fcontents.seek(0, 0);
3757 ftypehex = format(ftype, 'x').lower();
3758 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3759 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3760 fcontents.close();
3761 if(numfiles>0):
3762 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3763 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3764 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
3765 try:
3766 catfp.flush();
3767 os.fsync(catfp.fileno());
3768 except io.UnsupportedOperation:
3769 pass;
3770 except AttributeError:
3771 pass;
3772 except OSError as e:
3773 pass;
3774 if(outfile=="-"):
3775 catfp.seek(0, 0)
3776 if(hasattr(sys.stdout, "buffer")):
3777 shutil.copyfileobj(catfp, sys.stdout.buffer);
3778 else:
3779 shutil.copyfileobj(catfp, sys.stdout);
3780 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3781 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3782 catfp.seek(0, 0);
3783 upload_file_to_internet_file(catfp, outfile);
3784 if(returnfp):
3785 catfp.seek(0, 0)
3786 return catfp
3787 else:
3788 catfp.close()
3789 return True;
3791 create_alias_function("Pack", __file_format_name__, "FromRarFile", PackArchiveFileFromRarFile);
3793 if(not py7zr_support):
3794 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3795 return False
3797 if(py7zr_support):
3798 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3799 formatspecs = FormatSpecsListToDict(formatspecs);
3800 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3801 outfile = RemoveWindowsPath(outfile);
3802 checksumtype = checksumtype.lower();
3803 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3804 checksumtype="crc32";
3805 if(checksumtype=="none"):
3806 checksumtype = "";
3807 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3808 compression = "auto";
3809 if(compression not in compressionlist and compression is None):
3810 compression = "auto";
3811 if(verbose):
3812 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3813 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3814 if(os.path.exists(outfile)):
3815 try:
3816 os.unlink(outfile);
3817 except OSError as e:
3818 pass;
3819 if(outfile=="-"):
3820 verbose = False;
3821 catfp = BytesIO();
3822 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3823 catfp = outfile;
3824 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3825 catfp = BytesIO();
3826 else:
3827 fbasename = os.path.splitext(outfile)[0];
3828 fextname = os.path.splitext(outfile)[1];
3829 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3830 catver = formatspecs['format_ver'];
3831 fileheaderver = str(int(catver.replace(".", "")));
3832 curinode = 0;
3833 curfid = 0;
3834 inodelist = [];
3835 inodetofile = {};
3836 filetoinode = {};
3837 inodetocatinode = {};
3838 if(not os.path.exists(infile) or not os.path.isfile(infile)):
3839 return False;
3840 szpfp = py7zr.SevenZipFile(infile, mode="r");
3841 file_content = szpfp.readall();
3842 #sztest = szpfp.testzip();
3843 sztestalt = szpfp.test();
3844 if(sztestalt):
3845 VerbosePrintOut("Bad file found!");
3846 numfiles = int(len(szpfp.list()));
3847 AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3848 for member in sorted(szpfp.list(), key=lambda x: x.filename):
3849 if(re.findall("^[.|/]", member.filename)):
3850 fname = member.filename;
3851 else:
3852 fname = "./"+member.filename;
3853 if(verbose):
3854 VerbosePrintOut(fname);
3855 if(not member.is_directory):
3856 fpremode = int(stat.S_IFREG + 438);
3857 elif(member.is_directory):
3858 fpremode = int(stat.S_IFDIR + 511);
3859 fwinattributes = format(int(0), 'x').lower();
3860 fcompression = "";
3861 fcsize = format(int(0), 'x').lower();
3862 flinkcount = 0;
3863 ftype = 0;
3864 if(member.is_directory):
3865 ftype = 5;
3866 else:
3867 ftype = 0;
3868 flinkname = "";
3869 fcurfid = format(int(curfid), 'x').lower();
3870 fcurinode = format(int(curfid), 'x').lower();
3871 curfid = curfid + 1;
3872 fdev_minor = format(int(0), 'x').lower();
3873 fdev_major = format(int(0), 'x').lower();
3874 frdev_minor = format(int(0), 'x').lower();
3875 frdev_major = format(int(0), 'x').lower();
3876 if(ftype==5):
3877 fsize = format(int("0"), 'x').lower();
3878 fatime = format(int(member.creationtime.timestamp()), 'x').lower();
3879 fmtime = format(int(member.creationtime.timestamp()), 'x').lower();
3880 fctime = format(int(member.creationtime.timestamp()), 'x').lower();
3881 fbtime = format(int(member.creationtime.timestamp()), 'x').lower();
3882 if(member.is_directory):
3883 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3884 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
3885 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
3886 else:
3887 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3888 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3889 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3890 try:
3891 fuid = format(int(os.getuid()), 'x').lower();
3892 except AttributeError:
3893 fuid = format(int(0), 'x').lower();
3894 except KeyError:
3895 fuid = format(int(0), 'x').lower();
3896 try:
3897 fgid = format(int(os.getgid()), 'x').lower();
3898 except AttributeError:
3899 fgid = format(int(0), 'x').lower();
3900 except KeyError:
3901 fgid = format(int(0), 'x').lower();
3902 try:
3903 import pwd;
3904 try:
3905 userinfo = pwd.getpwuid(os.getuid());
3906 funame = userinfo.pw_name;
3907 except KeyError:
3908 funame = "";
3909 except AttributeError:
3910 funame = "";
3911 except ImportError:
3912 funame = "";
3913 fgname = "";
3914 try:
3915 import grp;
3916 try:
3917 groupinfo = grp.getgrgid(os.getgid());
3918 fgname = groupinfo.gr_name;
3919 except KeyError:
3920 fgname = "";
3921 except AttributeError:
3922 fgname = "";
3923 except ImportError:
3924 fgname = "";
3925 fcontents = BytesIO();
3926 if(ftype==0):
3927 fcontents.write(file_content[member.filename].read());
3928 fsize = format(fcontents.tell(), 'x').lower();
3929 file_content[member.filename].close();
3930 if(not compresswholefile):
3931 fcontents.seek(0, 2);
3932 ucfsize = fcontents.tell();
3933 fcontents.seek(0, 0);
3934 if(compression=="auto"):
3935 ilsize = len(compressionlistalt);
3936 ilmin = 0;
3937 ilcsize = [];
3938 while(ilmin < ilsize):
3939 cfcontents = BytesIO();
3940 shutil.copyfileobj(fcontents, cfcontents);
3941 fcontents.seek(0, 0);
3942 cfcontents.seek(0, 0);
3943 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3944 if(cfcontents):
3945 cfcontents.seek(0, 2);
3946 ilcsize.append(cfcontents.tell());
3947 cfcontents.close();
3948 else:
3949 try:
3950 ilcsize.append(sys.maxint);
3951 except AttributeError:
3952 ilcsize.append(sys.maxsize);
3953 ilmin = ilmin + 1;
3954 ilcmin = ilcsize.index(min(ilcsize));
3955 compression = compressionlistalt[ilcmin];
3956 fcontents.seek(0, 0);
3957 cfcontents = BytesIO();
3958 shutil.copyfileobj(fcontents, cfcontents);
3959 cfcontents.seek(0, 0);
3960 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3961 cfcontents.seek(0, 2);
3962 cfsize = cfcontents.tell();
3963 if(ucfsize > cfsize):
3964 fcsize = format(int(cfsize), 'x').lower();
3965 fcompression = compression;
3966 fcontents.close();
3967 fcontents = cfcontents;
3968 if(fcompression=="none"):
3969 fcompression = "";
3970 fcontents.seek(0, 0);
3971 ftypehex = format(ftype, 'x').lower();
3972 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3973 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3974 fcontents.close();
3975 if(numfiles>0):
3976 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3977 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3978 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
3979 try:
3980 catfp.flush();
3981 os.fsync(catfp.fileno());
3982 except io.UnsupportedOperation:
3983 pass;
3984 except AttributeError:
3985 pass;
3986 except OSError as e:
3987 pass;
3988 if(outfile=="-"):
3989 catfp.seek(0, 0);
3990 if(hasattr(sys.stdout, "buffer")):
3991 shutil.copyfileobj(catfp, sys.stdout.buffer);
3992 else:
3993 shutil.copyfileobj(catfp, sys.stdout);
3994 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3995 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3996 catfp.seek(0, 0);
3997 upload_file_to_internet_file(catfp, outfile);
3998 if(returnfp):
3999 catfp.seek(0, 0)
4000 return catfp
4001 else:
4002 catfp.close()
4003 return True;
4005 create_alias_function("Pack", __file_format_name__, "FromSevenZipFile", PackArchiveFileFromSevenZipFile);
4007 def PackArchiveFileFromInFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
4008 formatspecs = FormatSpecsListToDict(formatspecs);
4009 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4010 if(verbose):
4011 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
4012 if(checkcompressfile=="tarfile"):
4013 return PackArchiveFileFromTarFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4014 elif(checkcompressfile=="zipfile"):
4015 return PackArchiveFileFromZipFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4016 elif(checkcompressfile=="catfile"):
4017 return RePackArchiveFile(infile, outfile, compression, compresswholefile, compressionlevel, False, 0, 0, checksumtype, False, extradata, formatspecs, verbose, returnfp);
4018 elif(rarfile_support and checkcompressfile=="rarfile"):
4019 return PackArchiveFileFromRarFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4020 elif(py7zr_support and checkcompressfile=="7zipfile"):
4021 return PackArchiveFileFromSevenZipFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4022 else:
4023 return False;
4024 return False;
4026 create_alias_function("Pack", __file_format_name__, "FromInFile", PackArchiveFileFromInFile);
4028 def ArchiveFileSeekToFileNum(infile, seekto=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4029 formatspecs = FormatSpecsListToDict(formatspecs);
4030 if(hasattr(infile, "read") or hasattr(infile, "write")):
4031 catfp = infile;
4032 catfp.seek(0, 0);
4033 catfp = UncompressArchiveFile(catfp, formatspecs);
4034 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4035 if(checkcompressfile=="tarfile"):
4036 return TarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4037 if(checkcompressfile=="zipfile"):
4038 return ZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4039 if(rarfile_support and checkcompressfile=="rarfile"):
4040 return RarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4041 if(py7zr_support and checkcompressfile=="7zipfile"):
4042 return SevenZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4043 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4044 return False;
4045 if(not catfp):
4046 return False;
4047 catfp.seek(0, 0);
4048 elif(infile=="-"):
4049 catfp = BytesIO();
4050 if(hasattr(sys.stdin, "buffer")):
4051 shutil.copyfileobj(sys.stdin.buffer, catfp);
4052 else:
4053 shutil.copyfileobj(sys.stdin, catfp);
4054 catfp.seek(0, 0);
4055 catfp = UncompressArchiveFile(catfp, formatspecs);
4056 if(not catfp):
4057 return False;
4058 catfp.seek(0, 0);
4059 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4060 catfp = download_file_from_internet_file(infile);
4061 catfp.seek(0, 0);
4062 catfp = UncompressArchiveFile(catfp, formatspecs);
4063 if(not catfp):
4064 return False;
4065 catfp.seek(0, 0);
4066 else:
4067 infile = RemoveWindowsPath(infile);
4068 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4069 if(checkcompressfile=="tarfile"):
4070 return TarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4071 if(checkcompressfile=="zipfile"):
4072 return ZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4073 if(rarfile_support and checkcompressfile=="rarfile"):
4074 return RarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4075 if(py7zr_support and checkcompressfile=="7zipfile"):
4076 return SevenZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4077 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4078 return False;
4079 compresscheck = CheckCompressionType(infile, formatspecs, True);
4080 if(not compresscheck):
4081 fextname = os.path.splitext(infile)[1];
4082 if(fextname==".gz"):
4083 compresscheck = "gzip";
4084 elif(fextname==".bz2"):
4085 compresscheck = "bzip2";
4086 elif(fextname==".zst"):
4087 compresscheck = "zstd";
4088 elif(fextname==".lz4" or fextname==".clz4"):
4089 compresscheck = "lz4";
4090 elif(fextname==".lzo" or fextname==".lzop"):
4091 compresscheck = "lzo";
4092 elif(fextname==".lzma" or fextname==".xz"):
4093 compresscheck = "lzma";
4094 else:
4095 return False;
4096 if(not compresscheck):
4097 return False;
4098 catfp = UncompressFile(infile, formatspecs, "rb");
4100 try:
4101 catfp.seek(0, 2);
4102 except OSError:
4103 SeekToEndOfFile(catfp);
4104 except ValueError:
4105 SeekToEndOfFile(catfp);
4106 CatSize = catfp.tell();
4107 CatSizeEnd = CatSize;
4109 try:
4110 catfp.seek(0, 0);
4111 except OSError:
4112 return False;
4113 except ValueError:
4114 return False;
4115 curloc = catfp.tell();
4116 if(curloc>0):
4117 catfp.seek(0, 0);
4118 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4119 if(curloc>0):
4120 catfp.seek(curloc, 0);
4121 catstring = catheader[0];
4122 catversion = re.findall(r"([\d]+)", catstring);
4123 fprenumfiles = catheader[1];
4124 fnumfiles = int(fprenumfiles, 16);
4125 fprechecksumtype = catheader[2];
4126 fprechecksum = catheader[3];
4127 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4128 fnumfileshex = format(int(fnumfiles), 'x').lower();
4129 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4130 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4131 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4132 fheadtell = len(fileheader);
4133 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4134 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
4135 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4136 return False;
4137 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
4138 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4139 if(seekto>=fnumfiles):
4140 seekto = fnumfiles - 1;
4141 if(seekto<0):
4142 seekto = 0;
4143 if(seekto>=0):
4144 il = -1;
4145 while(il < seekto):
4146 prefhstart = catfp.tell();
4147 if(formatspecs['new_style']):
4148 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4149 else:
4150 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4151 if(len(preheaderdata)==0):
4152 break;
4153 prefheadsize = int(preheaderdata[0], 16);
4154 prefnumfields = int(preheaderdata[1], 16);
4155 preftype = int(preheaderdata[2], 16);
4156 if(re.findall("^[.|/]", preheaderdata[3])):
4157 prefname = preheaderdata[3];
4158 else:
4159 prefname = "./"+preheaderdata[3];
4160 prefbasedir = os.path.dirname(prefname);
4161 preflinkname = preheaderdata[4];
4162 prefsize = int(preheaderdata[5], 16);
4163 prefatime = int(preheaderdata[6], 16);
4164 prefmtime = int(preheaderdata[7], 16);
4165 prefctime = int(preheaderdata[8], 16);
4166 prefbtime = int(preheaderdata[9], 16);
4167 prefmode = int(preheaderdata[10], 16);
4168 prefchmode = stat.S_IMODE(prefmode);
4169 preftypemod = stat.S_IFMT(prefmode);
4170 prefwinattributes = int(preheaderdata[11], 16);
4171 prefcompression = preheaderdata[12];
4172 prefcsize = int(preheaderdata[13], 16);
4173 prefuid = int(preheaderdata[14], 16);
4174 prefuname = preheaderdata[15];
4175 prefgid = int(preheaderdata[16], 16);
4176 prefgname = preheaderdata[17];
4177 fid = int(preheaderdata[18], 16);
4178 finode = int(preheaderdata[19], 16);
4179 flinkcount = int(preheaderdata[20], 16);
4180 prefdev_minor = int(preheaderdata[21], 16);
4181 prefdev_major = int(preheaderdata[22], 16);
4182 prefrdev_minor = int(preheaderdata[23], 16);
4183 prefrdev_major = int(preheaderdata[24], 16);
4184 prefextrasize = int(preheaderdata[25], 16);
4185 prefextrafields = int(preheaderdata[26], 16);
4186 extrafieldslist = [];
4187 extrastart = 27;
4188 extraend = extrastart + prefextrafields;
4189 extrafieldslist = [];
4190 if(extrastart<extraend):
4191 extrafieldslist.append(preheaderdata[extrastart]);
4192 extrastart = extrastart + 1;
4193 prefchecksumtype = preheaderdata[extrastart].lower();
4194 prefcs = preheaderdata[extrastart + 1].lower();
4195 prefccs = preheaderdata[extrastart + 2].lower();
4196 hc = 0;
4197 hcmax = len(preheaderdata) - 2;
4198 hout = "";
4199 while(hc<hcmax):
4200 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs['format_delimiter']);
4201 hc = hc + 1;
4202 prenewfcs = GetFileChecksum(hout, preheaderdata[-3].lower(), True, formatspecs);
4203 if(prefcs!=prenewfcs and not skipchecksum):
4204 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4205 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4206 return False;
4207 valid_archive = False;
4208 invalid_archive = True;
4209 prefhend = catfp.tell() - 1;
4210 prefcontentstart = catfp.tell();
4211 prefcontents = "";
4212 pyhascontents = False;
4213 if(prefsize>0):
4214 if(prefcompression):
4215 prefcontents = catfp.read(prefsize);
4216 else:
4217 prefcontents = catfp.read(prefcsize);
4218 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4219 pyhascontents = True;
4220 if(prefccs!=prenewfccs and not skipchecksum):
4221 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4222 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4223 return False;
4224 catfp.seek(1, 1);
4225 il = il + 1;
4226 catfp.seek(seekstart, 0);
4227 fileidnum = il;
4228 catfheadsize = int(preheaderdata[0], 16);
4229 catfnumfields = int(preheaderdata[1], 16);
4230 catftype = int(preheaderdata[2], 16);
4231 if(re.findall("^[.|/]", preheaderdata[3])):
4232 catfname = preheaderdata[3];
4233 else:
4234 catfname = "./"+preheaderdata[3];
4235 catflinkname = preheaderdata[4];
4236 catfsize = int(preheaderdata[5], 16);
4237 catfbasedir = os.path.dirname(catfname);
4238 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
4239 if(returnfp):
4240 catlist.update({'catfp': catfp});
4241 else:
4242 catfp.close();
4243 return catlist;
4245 create_alias_function("", __file_format_name__, "SeekToFileNum", ArchiveFileSeekToFileNum);
4247 def ArchiveFileSeekToFileName(infile, seekfile=None, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4248 formatspecs = FormatSpecsListToDict(formatspecs);
4249 if(hasattr(infile, "read") or hasattr(infile, "write")):
4250 catfp = infile;
4251 catfp.seek(0, 0);
4252 catfp = UncompressArchiveFile(catfp, formatspecs);
4253 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4254 if(checkcompressfile=="tarfile"):
4255 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4256 if(checkcompressfile=="zipfile"):
4257 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4258 if(rarfile_support and checkcompressfile=="rarfile"):
4259 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4260 if(py7zr_support and checkcompressfile=="7zipfile"):
4261 return SevenZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4262 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4263 return False;
4264 if(not catfp):
4265 return False;
4266 catfp.seek(0, 0);
4267 elif(infile=="-"):
4268 catfp = BytesIO();
4269 if(hasattr(sys.stdin, "buffer")):
4270 shutil.copyfileobj(sys.stdin.buffer, catfp);
4271 else:
4272 shutil.copyfileobj(sys.stdin, catfp);
4273 catfp.seek(0, 0);
4274 catfp = UncompressArchiveFile(catfp, formatspecs);
4275 if(not catfp):
4276 return False;
4277 catfp.seek(0, 0);
4278 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4279 catfp = download_file_from_internet_file(infile);
4280 catfp = UncompressArchiveFile(catfp, formatspecs);
4281 catfp.seek(0, 0);
4282 if(not catfp):
4283 return False;
4284 catfp.seek(0, 0);
4285 else:
4286 infile = RemoveWindowsPath(infile);
4287 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4288 if(checkcompressfile=="tarfile"):
4289 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4290 if(checkcompressfile=="zipfile"):
4291 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4292 if(rarfile_support and checkcompressfile=="rarfile"):
4293 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4294 if(py7zr_support and checkcompressfile=="7zipfile"):
4295 return SevenZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4296 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4297 return False;
4298 compresscheck = CheckCompressionType(infile, formatspecs, True);
4299 if(not compresscheck):
4300 fextname = os.path.splitext(infile)[1];
4301 if(fextname==".gz"):
4302 compresscheck = "gzip";
4303 elif(fextname==".bz2"):
4304 compresscheck = "bzip2";
4305 elif(fextname==".zst"):
4306 compresscheck = "zstd";
4307 elif(fextname==".lz4" or fextname==".clz4"):
4308 compresscheck = "lz4";
4309 elif(fextname==".lzo" or fextname==".lzop"):
4310 compresscheck = "lzo";
4311 elif(fextname==".lzma" or fextname==".xz"):
4312 compresscheck = "lzma";
4313 else:
4314 return False;
4315 if(not compresscheck):
4316 return False;
4317 catfp = UncompressFile(infile, formatspecs, "rb");
4319 try:
4320 catfp.seek(0, 2);
4321 except OSError:
4322 SeekToEndOfFile(catfp);
4323 except ValueError:
4324 SeekToEndOfFile(catfp);
4325 CatSize = catfp.tell();
4326 CatSizeEnd = CatSize;
4328 try:
4329 catfp.seek(0, 0);
4330 except OSError:
4331 return False;
4332 except ValueError:
4333 return False;
4334 curloc = catfp.tell();
4335 if(curloc>0):
4336 catfp.seek(0, 0);
4337 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4338 if(curloc>0):
4339 catfp.seek(curloc, 0);
4340 catstring = catheader[0];
4341 catversion = re.findall(r"([\d]+)", catstring);
4342 fprenumfiles = catheader[1];
4343 fnumfiles = int(fprenumfiles, 16);
4344 fprechecksumtype = catheader[2];
4345 fprechecksum = catheader[3];
4346 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4347 fnumfileshex = format(int(fnumfiles), 'x').lower();
4348 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4349 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4350 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4351 fheadtell = len(fileheader);
4352 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4353 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
4354 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4355 return False;
4356 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
4357 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4358 seekto = fnumfiles - 1
4359 filefound = False;
4360 if(seekto>=0):
4361 il = -1;
4362 while(il < seekto):
4363 prefhstart = catfp.tell();
4364 if(formatspecs['new_style']):
4365 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4366 else:
4367 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4368 if(len(preheaderdata)==0):
4369 break;
4370 prefheadsize = int(preheaderdata[0], 16);
4371 prefnumfields = int(preheaderdata[1], 16);
4372 preftype = int(preheaderdata[2], 16);
4373 if(re.findall("^[.|/]", preheaderdata[3])):
4374 prefname = preheaderdata[3];
4375 else:
4376 prefname = "./"+preheaderdata[3];
4377 prefbasedir = os.path.dirname(prefname);
4378 preflinkname = preheaderdata[4];
4379 prefsize = int(preheaderdata[5], 16);
4380 prefatime = int(preheaderdata[6], 16);
4381 prefmtime = int(preheaderdata[7], 16);
4382 prefctime = int(preheaderdata[8], 16);
4383 prefbtime = int(preheaderdata[9], 16);
4384 prefmode = int(preheaderdata[10], 16);
4385 prefchmode = stat.S_IMODE(prefmode);
4386 preftypemod = stat.S_IFMT(prefmode);
4387 prefwinattributes = int(preheaderdata[11], 16);
4388 prefcompression = preheaderdata[12];
4389 prefcsize = int(preheaderdata[13], 16);
4390 prefuid = int(preheaderdata[14], 16);
4391 prefuname = preheaderdata[15];
4392 prefgid = int(preheaderdata[16], 16);
4393 prefgname = preheaderdata[17];
4394 fid = int(preheaderdata[18], 16);
4395 finode = int(preheaderdata[19], 16);
4396 flinkcount = int(preheaderdata[20], 16);
4397 prefdev_minor = int(preheaderdata[21], 16);
4398 prefdev_major = int(preheaderdata[22], 16);
4399 prefrdev_minor = int(preheaderdata[23], 16);
4400 prefrdev_major = int(preheaderdata[24], 16);
4401 prefextrasize = int(preheaderdata[25], 16);
4402 prefextrafields = int(preheaderdata[26], 16);
4403 extrafieldslist = [];
4404 extrastart = 27;
4405 extraend = extrastart + prefextrafields;
4406 extrafieldslist = [];
4407 if(extrastart<extraend):
4408 extrafieldslist.append(preheaderdata[extrastart]);
4409 extrastart = extrastart + 1;
4410 prefchecksumtype = preheaderdata[extrastart].lower();
4411 prefcs = preheaderdata[extrastart + 1].lower();
4412 prefccs = preheaderdata[extrastart + 2].lower();
4413 hc = 0;
4414 hcmax = len(preheaderdata) - 2;
4415 hout = "";
4416 while(hc<hcmax):
4417 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs['format_delimiter']);
4418 hc = hc + 1;
4419 prenewfcs = GetFileChecksum(hout, preheaderdata[-3].lower(), True, formatspecs);
4420 if(prefcs!=prenewfcs and not skipchecksum):
4421 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4422 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4423 return False;
4424 valid_archive = False;
4425 invalid_archive = True;
4426 prefhend = catfp.tell() - 1;
4427 prefcontentstart = catfp.tell();
4428 prefcontents = "";
4429 pyhascontents = False;
4430 if(prefsize>0):
4431 if(prefcompression):
4432 prefcontents = catfp.read(prefsize);
4433 else:
4434 prefcontents = catfp.read(prefcsize);
4435 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4436 pyhascontents = True;
4437 if(prefccs!=prenewfccs and not skipchecksum):
4438 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4439 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4440 return False;
4441 catfp.seek(1, 1);
4442 il = il + 1;
4443 filefound = False;
4444 prefname = preheaderdata[2];
4445 if(re.findall("^[.|/]", preheaderdata[2])):
4446 prefname = preheaderdata[2];
4447 else:
4448 prefname = "./"+preheaderdata[2];
4449 if(prefname==seekfile):
4450 filefound = True;
4451 break;
4452 catfp.seek(seekstart, 0);
4453 fileidnum = il;
4454 catfheadsize = int(preheaderdata[0], 16);
4455 catfnumfields = int(preheaderdata[1], 16);
4456 catftype = int(preheaderdata[2], 16);
4457 if(re.findall("^[.|/]", preheaderdata[3])):
4458 catfname = preheaderdata[3];
4459 else:
4460 catfname = "./"+preheaderdata[3];
4461 catflinkname = preheaderdata[4];
4462 catfsize = int(preheaderdata[5], 16);
4463 catfbasedir = os.path.dirname(catfname);
4464 if(filefound):
4465 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
4466 else:
4467 return False;
4468 if(returnfp):
4469 catlist.update({'catfp': catfp});
4470 else:
4471 catfp.close();
4472 return catlist;
4474 create_alias_function("", __file_format_name__, "SeekToFileName", ArchiveFileSeekToFileName);
4476 def ArchiveFileValidate(infile, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
4477 formatspecs = FormatSpecsListToDict(formatspecs);
4478 if(verbose):
4479 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
4480 if(hasattr(infile, "read") or hasattr(infile, "write")):
4481 catfp = infile;
4482 catfp.seek(0, 0);
4483 catfp = UncompressArchiveFile(catfp, formatspecs);
4484 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4485 if(checkcompressfile=="tarfile"):
4486 return TarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4487 if(checkcompressfile=="zipfile"):
4488 return ZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4489 if(rarfile_support and checkcompressfile=="rarfile"):
4490 return RarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4491 if(py7zr_support and checkcompressfile=="7zipfile"):
4492 return SevenZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4493 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4494 return False;
4495 if(not catfp):
4496 return False;
4497 catfp.seek(0, 0);
4498 elif(infile=="-"):
4499 catfp = BytesIO();
4500 if(hasattr(sys.stdin, "buffer")):
4501 shutil.copyfileobj(sys.stdin.buffer, catfp);
4502 else:
4503 shutil.copyfileobj(sys.stdin, catfp);
4504 catfp.seek(0, 0);
4505 catfp = UncompressArchiveFile(catfp, formatspecs);
4506 if(not catfp):
4507 return False;
4508 catfp.seek(0, 0);
4509 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4510 catfp = download_file_from_internet_file(infile);
4511 catfp = UncompressArchiveFile(catfp, formatspecs);
4512 catfp.seek(0, 0);
4513 if(not catfp):
4514 return False;
4515 catfp.seek(0, 0);
4516 else:
4517 infile = RemoveWindowsPath(infile);
4518 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4519 if(checkcompressfile=="tarfile"):
4520 return TarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4521 if(checkcompressfile=="zipfile"):
4522 return ZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4523 if(rarfile_support and checkcompressfile=="rarfile"):
4524 return RarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4525 if(py7zr_support and checkcompressfile=="7zipfile"):
4526 return SevenZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4527 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4528 return False;
4529 compresscheck = CheckCompressionType(infile, formatspecs, True);
4530 if(not compresscheck):
4531 fextname = os.path.splitext(infile)[1];
4532 if(fextname==".gz"):
4533 compresscheck = "gzip";
4534 elif(fextname==".bz2"):
4535 compresscheck = "bzip2";
4536 elif(fextname==".zst"):
4537 compresscheck = "zstd";
4538 elif(fextname==".lz4" or fextname==".clz4"):
4539 compresscheck = "lz4";
4540 elif(fextname==".lzo" or fextname==".lzop"):
4541 compresscheck = "lzo";
4542 elif(fextname==".lzma" or fextname==".xz"):
4543 compresscheck = "lzma";
4544 else:
4545 return False;
4546 if(not compresscheck):
4547 return False;
4548 catfp = UncompressFile(infile, formatspecs, "rb");
4550 try:
4551 catfp.seek(0, 2);
4552 except OSError:
4553 SeekToEndOfFile(catfp);
4554 except ValueError:
4555 SeekToEndOfFile(catfp);
4556 CatSize = catfp.tell();
4557 CatSizeEnd = CatSize;
4559 try:
4560 catfp.seek(0, 0);
4561 except OSError:
4562 return False;
4563 except ValueError:
4564 return False;
4565 curloc = catfp.tell();
4566 if(curloc>0):
4567 catfp.seek(0, 0);
4568 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4569 if(curloc>0):
4570 catfp.seek(curloc, 0);
4571 catstring = catheader[0];
4572 catversion = re.findall(r"([\d]+)", catstring);
4573 fprenumfiles = catheader[1];
4574 fnumfiles = int(fprenumfiles, 16);
4575 fprechecksumtype = catheader[2];
4576 fprechecksum = catheader[3];
4577 il = 0;
4578 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4579 fnumfileshex = format(int(fnumfiles), 'x').lower();
4580 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4581 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4582 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4583 valid_archive = True;
4584 invalid_archive = False;
4585 if(verbose):
4586 VerbosePrintOut(infile);
4587 VerbosePrintOut("Number of Records " + str(fnumfiles));
4588 if(fprechecksum==catfileheadercshex):
4589 if(verbose):
4590 VerbosePrintOut("File Header Checksum Passed at offset " + str(0));
4591 VerbosePrintOut("'" + str(fprechecksum) + "' == " + "'" + str(catfileheadercshex) + "'");
4592 else:
4593 if(verbose):
4594 VerbosePrintOut("File Header Checksum Failed at offset " + str(0));
4595 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4596 valid_archive = False;
4597 invalid_archive = True;
4598 if(verbose):
4599 VerbosePrintOut("");
4600 while(il<fnumfiles):
4601 catfhstart = catfp.tell();
4602 if(formatspecs['new_style']):
4603 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4604 else:
4605 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4606 if(len(catheaderdata)==0):
4607 break;
4608 catfheadsize = int(catheaderdata[0], 16);
4609 catfnumfields = int(catheaderdata[1], 16);
4610 catftype = int(catheaderdata[2], 16);
4611 if(re.findall("^[.|/]", catheaderdata[3])):
4612 catfname = catheaderdata[3];
4613 else:
4614 catfname = "./"+catheaderdata[3];
4615 catfbasedir = os.path.dirname(catfname);
4616 catflinkname = catheaderdata[4];
4617 catfsize = int(catheaderdata[5], 16);
4618 catfatime = int(catheaderdata[6], 16);
4619 catfmtime = int(catheaderdata[7], 16);
4620 catfctime = int(catheaderdata[8], 16);
4621 catfbtime = int(catheaderdata[9], 16);
4622 catfmode = int(catheaderdata[10], 16);
4623 catfchmode = stat.S_IMODE(catfmode);
4624 catftypemod = stat.S_IFMT(catfmode);
4625 catfwinattributes = int(catheaderdata[11], 16);
4626 catfcompression = catheaderdata[12];
4627 catfcsize = int(catheaderdata[13], 16);
4628 catfuid = int(catheaderdata[14], 16);
4629 catfuname = catheaderdata[15];
4630 catfgid = int(catheaderdata[16], 16);
4631 catfgname = catheaderdata[17];
4632 fid = int(catheaderdata[18], 16);
4633 finode = int(catheaderdata[19], 16);
4634 flinkcount = int(catheaderdata[20], 16);
4635 catfdev_minor = int(catheaderdata[21], 16);
4636 catfdev_major = int(catheaderdata[22], 16);
4637 catfrdev_minor = int(catheaderdata[23], 16);
4638 catfrdev_major = int(catheaderdata[24], 16);
4639 catfextrasize = int(catheaderdata[25], 16);
4640 catfextrafields = int(catheaderdata[26], 16);
4641 extrafieldslist = [];
4642 extrastart = 27;
4643 extraend = extrastart + catfextrafields;
4644 extrafieldslist = [];
4645 if(extrastart<extraend):
4646 extrafieldslist.append(catheaderdata[extrastart]);
4647 extrastart = extrastart + 1;
4648 catfchecksumtype = catheaderdata[extrastart].lower();
4649 catfcs = catheaderdata[extrastart + 1].lower();
4650 catfccs = catheaderdata[extrastart + 2].lower();
4651 hc = 0;
4652 hcmax = len(catheaderdata) - 2;
4653 hout = "";
4654 while(hc<hcmax):
4655 hout = hout + AppendNullByte(catheaderdata[hc], formatspecs['format_delimiter']);
4656 hc = hc + 1;
4657 catnewfcs = GetFileChecksum(hout, catheaderdata[-3].lower(), True, formatspecs);
4658 if(verbose):
4659 VerbosePrintOut(catfname);
4660 VerbosePrintOut("Record Number " + str(il) + "; File ID " + str(fid) + "; iNode Number " + str(finode));
4661 if(catfcs==catnewfcs):
4662 if(verbose):
4663 VerbosePrintOut("File Header Checksum Passed at offset " + str(catfhstart));
4664 VerbosePrintOut("'" + str(catfcs) + "' == " + "'" + str(catnewfcs) + "'");
4665 else:
4666 if(verbose):
4667 VerbosePrintOut("File Header Checksum Failed at offset " + str(catfhstart));
4668 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
4669 valid_archive = False;
4670 invalid_archive = True;
4671 catfhend = catfp.tell() - 1;
4672 catfcontentstart = catfp.tell();
4673 catfcontents = "";
4674 pyhascontents = False;
4675 if(catfsize>0):
4676 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4677 catfcontents = catfp.read(catfsize);
4678 else:
4679 catfcontents = catfp.read(catfcsize);
4680 catnewfccs = GetFileChecksum(catfcontents, catheaderdata[-3].lower(), False, formatspecs);
4681 pyhascontents = True;
4682 if(catfccs==catnewfccs):
4683 if(verbose):
4684 VerbosePrintOut("File Content Checksum Passed at offset " + str(catfcontentstart));
4685 VerbosePrintOut("'" + str(catfccs) + "' == " + "'" + str(catnewfccs) + "'");
4686 else:
4687 if(verbose):
4688 VerbosePrintOut("File Content Checksum Failed at offset " + str(catfcontentstart));
4689 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
4690 valid_archive = False;
4691 invalid_archive = True;
4692 if(verbose):
4693 VerbosePrintOut("");
4694 catfp.seek(1, 1);
4695 il = il + 1;
4696 if(valid_archive):
4697 if(returnfp):
4698 return catfp;
4699 else:
4700 catfp.close();
4701 return True;
4702 else:
4703 catfp.close();
4704 return False;
4706 create_alias_function("", __file_format_name__, "Validate", ArchiveFileValidate);
4708 def ArchiveFileToArray(infile, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4709 formatspecs = FormatSpecsListToDict(formatspecs);
4710 if(hasattr(infile, "read") or hasattr(infile, "write")):
4711 catfp = infile;
4712 catfp.seek(0, 0);
4713 catfp = UncompressArchiveFile(catfp, formatspecs);
4714 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4715 if(checkcompressfile=="tarfile"):
4716 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4717 if(checkcompressfile=="zipfile"):
4718 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4719 if(rarfile_support and checkcompressfile=="rarfile"):
4720 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4721 if(py7zr_support and checkcompressfile=="7zipfile"):
4722 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4723 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4724 return False;
4725 if(not catfp):
4726 return False;
4727 catfp.seek(0, 0);
4728 elif(infile=="-"):
4729 catfp = BytesIO();
4730 if(hasattr(sys.stdin, "buffer")):
4731 shutil.copyfileobj(sys.stdin.buffer, catfp);
4732 else:
4733 shutil.copyfileobj(sys.stdin, catfp);
4734 catfp.seek(0, 0);
4735 catfp = UncompressArchiveFile(catfp, formatspecs);
4736 if(not catfp):
4737 return False;
4738 catfp.seek(0, 0);
4739 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4740 catfp = download_file_from_internet_file(infile);
4741 catfp = UncompressArchiveFile(catfp, formatspecs);
4742 catfp.seek(0, 0);
4743 if(not catfp):
4744 return False;
4745 catfp.seek(0, 0);
4746 else:
4747 infile = RemoveWindowsPath(infile);
4748 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4749 if(checkcompressfile=="tarfile"):
4750 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4751 if(checkcompressfile=="zipfile"):
4752 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4753 if(rarfile_support and checkcompressfile=="rarfile"):
4754 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4755 if(py7zr_support and checkcompressfile=="7zipfile"):
4756 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4757 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4758 return False;
4759 compresscheck = CheckCompressionType(infile, formatspecs, True);
4760 if(not compresscheck):
4761 fextname = os.path.splitext(infile)[1];
4762 if(fextname==".gz"):
4763 compresscheck = "gzip";
4764 elif(fextname==".bz2"):
4765 compresscheck = "bzip2";
4766 elif(fextname==".zst"):
4767 compresscheck = "zstd";
4768 elif(fextname==".lz4" or fextname==".clz4"):
4769 compresscheck = "lz4";
4770 elif(fextname==".lzo" or fextname==".lzop"):
4771 compresscheck = "lzo";
4772 elif(fextname==".lzma" or fextname==".xz"):
4773 compresscheck = "lzma";
4774 else:
4775 return False;
4776 if(not compresscheck):
4777 return False;
4778 catfp = UncompressFile(infile, formatspecs, "rb");
4780 try:
4781 catfp.seek(0, 2);
4782 except OSError:
4783 SeekToEndOfFile(catfp);
4784 except ValueError:
4785 SeekToEndOfFile(catfp);
4786 CatSize = catfp.tell();
4787 CatSizeEnd = CatSize;
4789 try:
4790 catfp.seek(0, 0);
4791 except OSError:
4792 return False;
4793 except ValueError:
4794 return False;
4795 curloc = catfp.tell();
4796 if(curloc>0):
4797 catfp.seek(0, 0);
4798 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4799 if(curloc>0):
4800 catfp.seek(curloc, 0);
4801 catstring = catheader[0];
4802 catversion = re.findall(r"([\d]+)", catstring);
4803 fprenumfiles = catheader[1];
4804 fnumfiles = int(fprenumfiles, 16);
4805 fprechecksumtype = catheader[2];
4806 fprechecksum = catheader[3];
4807 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4808 fnumfileshex = format(int(fnumfiles), 'x').lower();
4809 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4810 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4811 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4812 fheadtell = len(fileheader);
4813 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4814 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
4815 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4816 return False;
4817 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
4818 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4819 if(seekstart<0 and seekstart>fnumfiles):
4820 seekstart = 0;
4821 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
4822 seekend = fnumfiles;
4823 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
4824 seekend = fnumfiles - abs(seekend);
4825 if(seekstart>0):
4826 il = 0;
4827 while(il < seekstart):
4828 prefhstart = catfp.tell();
4829 if(formatspecs['new_style']):
4830 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4831 else:
4832 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4833 if(len(preheaderdata)==0):
4834 break;
4835 prefheadsize = int(preheaderdata[0], 16);
4836 prefnumfields = int(preheaderdata[1], 16);
4837 if(re.findall("^[.|/]", preheaderdata[3])):
4838 prefname = preheaderdata[3];
4839 else:
4840 prefname = "./"+preheaderdata[3];
4841 prefsize = int(preheaderdata[5], 16);
4842 prefcompression = preheaderdata[12];
4843 prefcsize = int(preheaderdata[13], 16);
4844 prefextrasize = int(preheaderdata[24], 16);
4845 prefextrafields = int(preheaderdata[25], 16);
4846 extrafieldslist = [];
4847 extrastart = 26;
4848 extraend = extrastart + prefextrafields;
4849 extrafieldslist = [];
4850 if(extrastart<extraend):
4851 extrafieldslist.append(preheaderdata[extrastart]);
4852 extrastart = extrastart + 1;
4853 prefchecksumtype = preheaderdata[extrastart].lower();
4854 prefcs = preheaderdata[extrastart + 1].lower();
4855 prefccs = preheaderdata[extrastart + 2].lower();
4856 hc = 0;
4857 hcmax = len(preheaderdata) - 2;
4858 hout = "";
4859 while(hc<hcmax):
4860 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs['format_delimiter']);
4861 hc = hc + 1;
4862 prenewfcs = GetFileChecksum(hout, preheaderdata[-3].lower(), True, formatspecs);
4863 if(prefcs!=prenewfcs and not skipchecksum):
4864 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4865 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4866 return False;
4867 valid_archive = False;
4868 invalid_archive = True;
4869 prefhend = catfp.tell() - 1;
4870 prefcontentstart = catfp.tell();
4871 prefcontents = "";
4872 pyhascontents = False;
4873 if(prefsize>0):
4874 if(prefcompression=="none" or prefcompression=="" or prefcompression=="auto"):
4875 prefcontents = catfp.read(prefsize);
4876 else:
4877 prefcontents = catfp.read(prefcsize);
4878 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4879 pyhascontents = True;
4880 if(prefccs!=prenewfccs and not skipchecksum):
4881 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4882 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4883 return False;
4884 catfp.seek(1, 1);
4885 il = il + 1;
4886 fileidnum = seekstart;
4887 realidnum = 0;
4888 while(fileidnum<seekend):
4889 catfhstart = catfp.tell();
4890 if(formatspecs['new_style']):
4891 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4892 else:
4893 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4894 if(len(catheaderdata)==0):
4895 break;
4896 catfheadsize = int(catheaderdata[0], 16);
4897 catfnumfields = int(catheaderdata[1], 16);
4898 catftype = int(catheaderdata[2], 16);
4899 if(re.findall("^[.|/]", catheaderdata[3])):
4900 catfname = catheaderdata[3];
4901 else:
4902 catfname = "./"+catheaderdata[3];
4903 catfbasedir = os.path.dirname(catfname);
4904 catflinkname = catheaderdata[4];
4905 catfsize = int(catheaderdata[5], 16);
4906 catfatime = int(catheaderdata[6], 16);
4907 catfmtime = int(catheaderdata[7], 16);
4908 catfctime = int(catheaderdata[8], 16);
4909 catfbtime = int(catheaderdata[9], 16);
4910 catfmode = int(catheaderdata[10], 16);
4911 catfchmode = stat.S_IMODE(catfmode);
4912 catftypemod = stat.S_IFMT(catfmode);
4913 catfwinattributes = int(catheaderdata[11], 16);
4914 catfcompression = catheaderdata[12];
4915 catfcsize = int(catheaderdata[13], 16);
4916 catfuid = int(catheaderdata[14], 16);
4917 catfuname = catheaderdata[15];
4918 catfgid = int(catheaderdata[16], 16);
4919 catfgname = catheaderdata[17];
4920 catfid = int(catheaderdata[18], 16);
4921 catfinode = int(catheaderdata[19], 16);
4922 catflinkcount = int(catheaderdata[20], 16);
4923 catfdev_minor = int(catheaderdata[21], 16);
4924 catfdev_major = int(catheaderdata[22], 16);
4925 catfrdev_minor = int(catheaderdata[23], 16);
4926 catfrdev_major = int(catheaderdata[24], 16);
4927 catfextrasize = int(catheaderdata[25], 16);
4928 catfextrafields = int(catheaderdata[26], 16);
4929 extrafieldslist = [];
4930 extrastart = 27;
4931 extraend = extrastart + catfextrafields;
4932 extrafieldslist = [];
4933 if(extrastart<extraend):
4934 extrafieldslist.append(catheaderdata[extrastart]);
4935 extrastart = extrastart + 1;
4936 catfchecksumtype = catheaderdata[extrastart].lower();
4937 catfcs = catheaderdata[extrastart + 1].lower();
4938 catfccs = catheaderdata[extrastart + 2].lower();
4939 hc = 0;
4940 hcmax = len(catheaderdata) - 2;
4941 hout = "";
4942 while(hc<hcmax):
4943 hout = hout + AppendNullByte(catheaderdata[hc], formatspecs['format_delimiter']);
4944 hc = hc + 1;
4945 catnewfcs = GetFileChecksum(hout, catheaderdata[-3].lower(), True, formatspecs);
4946 if(catfcs!=catnewfcs and not skipchecksum):
4947 VerbosePrintOut("File Header Checksum Error with file " + catfname + " at offset " + str(catfhstart));
4948 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
4949 return False;
4950 catfhend = catfp.tell() - 1;
4951 catfcontentstart = catfp.tell();
4952 catfcontents = BytesIO();
4953 pyhascontents = False;
4954 if(catfsize>0 and not listonly):
4955 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4956 catfcontents.write(catfp.read(catfsize));
4957 else:
4958 catfcontents.write(catfp.read(catfcsize));
4959 catfcontents.seek(0, 0);
4960 catnewfccs = GetFileChecksum(catfcontents.read(), catheaderdata[-3].lower(), False, formatspecs);
4961 pyhascontents = True;
4962 if(catfccs!=catnewfccs and skipchecksum):
4963 VerbosePrintOut("File Content Checksum Error with file " + catfname + " at offset " + str(catfcontentstart));
4964 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
4965 return False;
4966 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4967 pass;
4968 else:
4969 catfcontents.seek(0, 0);
4970 if(uncompress):
4971 catfcontents = UncompressArchiveFile(catfcontents, formatspecs);
4972 catfcontents.seek(0, 0);
4973 catfccs = GetFileChecksum(catfcontents.read(), catheaderdata[-3].lower(), False, formatspecs);
4974 if(catfsize>0 and listonly):
4975 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4976 catfp.seek(catfsize, 1);
4977 else:
4978 catfp.seek(catfcsize, 1);
4979 pyhascontents = False;
4980 catfp.seek(1, 1);
4981 catfcontentend = catfp.tell() - 1;
4982 catfcontents.seek(0, 0);
4983 catlist['ffilelist'].update({realidnum: {'fid': realidnum, 'fidalt': fileidnum, 'fheadersize': catfheadsize, 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize, 'fatime': catfatime, 'fmtime': catfmtime, 'fctime': catfctime, 'fbtime': catfbtime, 'fmode': catfmode, 'fchmode': catfchmode, 'ftypemod': catftypemod, 'fwinattributes': catfwinattributes, 'fcompression': catfcompression, 'fcsize': catfcsize, 'fuid': catfuid, 'funame': catfuname, 'fgid': catfgid, 'fgname': catfgname, 'finode': catfinode, 'flinkcount': catflinkcount, 'fminor': catfdev_minor, 'fmajor': catfdev_major, 'frminor': catfrdev_minor, 'frmajor': catfrdev_major, 'fchecksumtype': catfchecksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': catfextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': catfcs, 'fcontentchecksum': catfccs, 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': catfcontents} });
4984 fileidnum = fileidnum + 1;
4985 realidnum = realidnum + 1;
4986 if(returnfp):
4987 catlist.update({'catfp': catfp});
4988 else:
4989 catfp.close();
4990 return catlist;
4992 create_alias_function("", __file_format_name__, "ToArray", ArchiveFileToArray);
4994 def ArchiveFileStringToArray(catstr, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4995 formatspecs = FormatSpecsListToDict(formatspecs);
4996 catfp = BytesIO(catstr);
4997 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
4998 return listcatfiles;
5000 create_alias_function("", __file_format_name__, "StringToArray", ArchiveFileStringToArray);
5002 def TarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5003 formatspecs = FormatSpecsListToDict(formatspecs);
5004 catfp = BytesIO();
5005 catfp = PackArchiveFileFromTarFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5006 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5007 return listcatfiles;
5009 def ZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5010 formatspecs = FormatSpecsListToDict(formatspecs);
5011 catfp = BytesIO();
5012 catfp = PackArchiveFileFromZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5013 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5014 return listcatfiles;
5016 if(not rarfile_support):
5017 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5018 return False;
5020 if(rarfile_support):
5021 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5022 formatspecs = FormatSpecsListToDict(formatspecs);
5023 catfp = BytesIO();
5024 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5025 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5026 return listcatfiles;
5028 if(not py7zr_support):
5029 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5030 return False;
5032 if(py7zr_support):
5033 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5034 formatspecs = FormatSpecsListToDict(formatspecs);
5035 catfp = BytesIO();
5036 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5037 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5038 return listcatfiles;
5040 def InFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5041 formatspecs = FormatSpecsListToDict(formatspecs);
5042 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
5043 if(checkcompressfile=="tarfile"):
5044 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5045 elif(checkcompressfile=="zipfile"):
5046 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5047 elif(checkcompressfile=="catfile"):
5048 return ArchiveFileToArray(infile, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5049 elif(rarfile_support and checkcompressfile=="rarfile"):
5050 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5051 elif(py7zr_support and checkcompressfile=="7zipfile"):
5052 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5053 else:
5054 return False;
5055 return False;
5057 def ListDirToArrayAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5058 formatspecs = FormatSpecsListToDict(formatspecs);
5059 catver = formatspecs['format_ver'];
5060 fileheaderver = str(int(catver.replace(".", "")));
5061 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5062 advancedlist = formatspecs['use_advanced_list'];
5063 altinode = formatspecs['use_alt_inode'];
5064 infilelist = [];
5065 if(infiles=="-"):
5066 for line in sys.stdin:
5067 infilelist.append(line.strip());
5068 infilelist = list(filter(None, infilelist));
5069 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
5070 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
5071 return False;
5072 with UncompressFile(infiles, formatspecs, "r") as finfile:
5073 for line in finfile:
5074 infilelist.append(line.strip());
5075 infilelist = list(filter(None, infilelist));
5076 else:
5077 if(isinstance(infiles, (list, tuple, ))):
5078 infilelist = list(filter(None, infiles));
5079 elif(isinstance(infiles, (str, ))):
5080 infilelist = list(filter(None, [infiles]));
5081 if(advancedlist):
5082 GetDirList = ListDirAdvanced(infilelist, followlink, False);
5083 else:
5084 GetDirList = ListDir(infilelist, followlink, False);
5085 if(not GetDirList):
5086 return False;
5087 curinode = 0;
5088 curfid = 0;
5089 inodelist = [];
5090 inodetofile = {};
5091 filetoinode = {};
5092 inodetocatinode = {};
5093 fileidnum = 0;
5094 fnumfiles = int(len(GetDirList));
5095 catver = formatspecs['format_ver'];
5096 fileheaderver = str(int(catver.replace(".", "")));
5097 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5098 fnumfileshex = format(int(fnumfiles), 'x').lower();
5099 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5100 catversion = re.findall(r"([\d]+)", fileheader);
5101 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5102 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5103 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5104 fheadtell = len(fileheader);
5105 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5106 for curfname in GetDirList:
5107 catfhstart = fheadtell;
5108 if(re.findall("^[.|/]", curfname)):
5109 fname = curfname;
5110 else:
5111 fname = "./"+curfname;
5112 if(verbose):
5113 VerbosePrintOut(fname);
5114 if(not followlink or followlink is None):
5115 fstatinfo = os.lstat(fname);
5116 else:
5117 fstatinfo = os.stat(fname);
5118 fpremode = fstatinfo.st_mode;
5119 finode = fstatinfo.st_ino;
5120 flinkcount = fstatinfo.st_nlink;
5121 ftype = 0;
5122 if(stat.S_ISREG(fpremode)):
5123 ftype = 0;
5124 elif(stat.S_ISLNK(fpremode)):
5125 ftype = 2;
5126 elif(stat.S_ISCHR(fpremode)):
5127 ftype = 3;
5128 elif(stat.S_ISBLK(fpremode)):
5129 ftype = 4;
5130 elif(stat.S_ISDIR(fpremode)):
5131 ftype = 5;
5132 elif(stat.S_ISFIFO(fpremode)):
5133 ftype = 6;
5134 elif(stat.S_ISSOCK(fpremode)):
5135 ftype = 8;
5136 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
5137 ftype = 9;
5138 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
5139 ftype = 10;
5140 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
5141 ftype = 11;
5142 else:
5143 ftype = 0;
5144 flinkname = "";
5145 fbasedir = os.path.dirname(fname);
5146 fcurfid = curfid;
5147 if(not followlink and finode!=0):
5148 if(ftype!=1):
5149 if(finode in inodelist):
5150 ftype = 1;
5151 flinkname = inodetofile[finode];
5152 if(altinode):
5153 fcurinode = finode;
5154 else:
5155 fcurinode = inodetocatinode[finode];
5156 if(finode not in inodelist):
5157 inodelist.append(finode);
5158 inodetofile.update({finode: fname});
5159 inodetocatinode.update({finode: curinode});
5160 if(altinode):
5161 fcurinode = finode;
5162 else:
5163 fcurinode = curinode;
5164 curinode = curinode + 1;
5165 else:
5166 fcurinode = curinode;
5167 curinode = curinode + 1;
5168 curfid = curfid + 1;
5169 if(ftype==2):
5170 flinkname = os.readlink(fname);
5171 fdev = fstatinfo.st_dev;
5172 getfdev = GetDevMajorMinor(fdev);
5173 fdev_minor = getfdev[0];
5174 fdev_major = getfdev[1];
5175 frdev = fstatinfo.st_dev;
5176 if(hasattr(fstatinfo, "st_rdev")):
5177 frdev = fstatinfo.st_rdev;
5178 else:
5179 frdev = fstatinfo.st_dev;
5180 getfrdev = GetDevMajorMinor(frdev);
5181 frdev_minor = getfrdev[0];
5182 frdev_major = getfrdev[1];
5183 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
5184 fsize = "0";
5185 if(ftype==0 or ftype==7):
5186 fsize = fstatinfo.st_size;
5187 fatime = fstatinfo.st_atime;
5188 fmtime = fstatinfo.st_mtime;
5189 fctime = fstatinfo.st_ctime;
5190 if(hasattr(fstatinfo, "st_birthtime")):
5191 fbtime = fstatinfo.st_birthtime;
5192 else:
5193 fbtime = fstatinfo.st_ctime;
5194 fmode = fstatinfo.st_mode;
5195 fchmode = stat.S_IMODE(fstatinfo.st_mode);
5196 ftypemod = stat.S_IFMT(fstatinfo.st_mode);
5197 fuid = fstatinfo.st_uid;
5198 fgid = fstatinfo.st_gid;
5199 funame = "";
5200 try:
5201 import pwd;
5202 try:
5203 userinfo = pwd.getpwuid(fstatinfo.st_uid);
5204 funame = userinfo.pw_name;
5205 except KeyError:
5206 funame = "";
5207 except ImportError:
5208 funame = "";
5209 fgname = "";
5210 try:
5211 import grp;
5212 try:
5213 groupinfo = grp.getgrgid(fstatinfo.st_gid);
5214 fgname = groupinfo.gr_name;
5215 except KeyError:
5216 fgname = "";
5217 except ImportError:
5218 fgname = "";
5219 fdev_minor = fdev_minor;
5220 fdev_major = fdev_major;
5221 frdev_minor = frdev_minor;
5222 frdev_major = frdev_major;
5223 flinkcount = flinkcount;
5224 if(hasattr(fstatinfo, "st_file_attributes")):
5225 fwinattributes = fstatinfo.st_file_attributes;
5226 else:
5227 fwinattributes = 0;
5228 fcompression = "";
5229 fcsize = 0;
5230 fcontents = BytesIO();
5231 if(ftype==0 or ftype==7):
5232 with open(fname, "rb") as fpc:
5233 shutil.copyfileobj(fpc, fcontents);
5234 if(followlink and (ftype==1 or ftype==2)):
5235 flstatinfo = os.stat(flinkname);
5236 with open(flinkname, "rb") as fpc:
5237 shutil.copyfileobj(fpc, fcontents);
5238 fcontents.seek(0, 0);
5239 ftypehex = format(ftype, 'x').lower();
5240 extrafields = len(extradata);
5241 extrafieldslist = extradata;
5242 catfextrafields = extrafields;
5243 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5244 if(len(extradata)>0):
5245 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5246 extrasizelen = len(extrasizestr);
5247 extrasizelenhex = format(extrasizelen, 'x').lower();
5248 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5249 catoutlen = len(catoutlist) + len(extradata) + 3;
5250 catoutlenhex = format(catoutlen, 'x').lower();
5251 catoutlist.insert(0, catoutlenhex);
5252 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5253 catheaderdata = catoutlist;
5254 if(len(extradata)>0):
5255 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5256 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
5257 catfnumfields = catoutlen;
5258 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5259 fcontents.seek(0, 0);
5260 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5261 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5262 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5263 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5264 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5265 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5266 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5267 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5268 catfcontentstart = fheadtell;
5269 fheadtell += len(catfileoutstr) + 1;
5270 catfcontentend = fheadtell - 1;
5271 catfhend = catfcontentend;
5272 fcontents.seek(0, 0);
5273 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5274 pyhascontents = False;
5275 if(int(fsize)>0 and not listonly):
5276 pyhascontents = True;
5277 if(int(fsize)>0 and listonly):
5278 fcontents = BytesIO();
5279 pyhascontents = False;
5280 fcontents.seek(0, 0);
5281 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5282 fileidnum = fileidnum + 1;
5283 return catlist;
5285 def TarFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5286 formatspecs = FormatSpecsListToDict(formatspecs);
5287 curinode = 0;
5288 curfid = 0;
5289 inodelist = [];
5290 inodetofile = {};
5291 filetoinode = {};
5292 inodetocatinode = {};
5293 fileidnum = 0;
5294 if(infile=="-"):
5295 infile = BytesIO();
5296 if(hasattr(sys.stdin, "buffer")):
5297 shutil.copyfileobj(sys.stdin.buffer, infile);
5298 else:
5299 shutil.copyfileobj(sys.stdin, infile);
5300 infile.seek(0, 0);
5301 if(not infile):
5302 return False;
5303 infile.seek(0, 0);
5304 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
5305 infile = download_file_from_internet_file(infile);
5306 infile.seek(0, 0);
5307 if(not infile):
5308 return False;
5309 infile.seek(0, 0);
5310 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
5311 return False;
5312 elif(os.path.exists(infile) and os.path.isfile(infile)):
5313 try:
5314 if(not tarfile.is_tarfile(infile)):
5315 return False;
5316 except AttributeError:
5317 if(not is_tarfile(infile)):
5318 return False;
5319 else:
5320 return False;
5321 try:
5322 if(hasattr(infile, "read") or hasattr(infile, "write")):
5323 tarfp = tarfile.open(fileobj=infile, mode="r");
5324 else:
5325 tarfp = tarfile.open(infile, "r");
5326 except FileNotFoundError:
5327 return False;
5328 fnumfiles = int(len(tarfp.getmembers()));
5329 catver = formatspecs['format_ver'];
5330 fileheaderver = str(int(catver.replace(".", "")));
5331 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5332 fnumfileshex = format(int(fnumfiles), 'x').lower();
5333 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5334 catversion = re.findall(r"([\d]+)", fileheader);
5335 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5336 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5337 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5338 fheadtell = len(fileheader);
5339 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5340 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
5341 catfhstart = fheadtell;
5342 if(re.findall("^[.|/]", member.name)):
5343 fname = member.name;
5344 else:
5345 fname = "./"+member.name;
5346 if(verbose):
5347 VerbosePrintOut(fname);
5348 fpremode = member.mode;
5349 ffullmode = member.mode;
5350 flinkcount = 0;
5351 ftype = 0;
5352 if(member.isreg()):
5353 ffullmode = member.mode + stat.S_IFREG;
5354 ftype = 0;
5355 elif(member.isdev()):
5356 ffullmode = member.mode;
5357 ftype = 7;
5358 elif(member.islnk()):
5359 ffullmode = member.mode + stat.S_IFREG;
5360 ftype = 1;
5361 elif(member.issym()):
5362 ffullmode = member.mode + stat.S_IFLNK;
5363 ftype = 2;
5364 elif(member.ischr()):
5365 ffullmode = member.mode + stat.S_IFCHR;
5366 ftype = 3;
5367 elif(member.isblk()):
5368 ffullmode = member.mode + stat.S_IFBLK;
5369 ftype = 4;
5370 elif(member.isdir()):
5371 ffullmode = member.mode + stat.S_IFDIR;
5372 ftype = 5;
5373 elif(member.isfifo()):
5374 ffullmode = member.mode + stat.S_IFIFO;
5375 ftype = 6;
5376 elif(member.issparse()):
5377 ffullmode = member.mode;
5378 ftype = 12;
5379 else:
5380 ffullmode = member.mode;
5381 ftype = 0;
5382 flinkname = "";
5383 fbasedir = os.path.dirname(fname);
5384 fcurfid = curfid;
5385 fcurinode = curfid;
5386 finode = fcurinode;
5387 curfid = curfid + 1;
5388 if(ftype==2):
5389 flinkname = member.linkname;
5390 fdev_minor = member.devminor;
5391 fdev_major = member.devmajor;
5392 frdev_minor = member.devminor;
5393 frdev_major = member.devmajor;
5394 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
5395 fsize = "0";
5396 elif(ftype==0 or ftype==7):
5397 fsize = member.size;
5398 else:
5399 fsize = member.size;
5400 fatime = member.mtime;
5401 fmtime = member.mtime;
5402 fctime = member.mtime;
5403 fbtime = member.mtime;
5404 fmode = ffullmode;
5405 fchmode = stat.S_IMODE(ffullmode);
5406 ftypemod = stat.S_IFMT(ffullmode);
5407 fuid = member.uid;
5408 fgid = member.gid;
5409 funame = member.uname;
5410 fgname = member.gname;
5411 flinkcount = flinkcount;
5412 fwinattributes = int(0);
5413 fcompression = "";
5414 fcsize = 0;
5415 fcontents = BytesIO();
5416 if(ftype==0 or ftype==7):
5417 with tarfp.extractfile(member) as fpc:
5418 shutil.copyfileobj(fpc, fcontents);
5419 fcontents.seek(0, 0);
5420 ftypehex = format(ftype, 'x').lower();
5421 extrafields = len(extradata);
5422 extrafieldslist = extradata;
5423 catfextrafields = extrafields;
5424 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5425 if(len(extradata)>0):
5426 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5427 extrasizelen = len(extrasizestr);
5428 extrasizelenhex = format(extrasizelen, 'x').lower();
5429 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5430 catoutlen = len(catoutlist) + len(extradata) + 3;
5431 catoutlenhex = format(catoutlen, 'x').lower();
5432 catoutlist.insert(0, catoutlenhex);
5433 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5434 catheaderdata = catoutlist;
5435 if(len(extradata)>0):
5436 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5437 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
5438 catfnumfields = catoutlen;
5439 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5440 fcontents.seek(0, 0);
5441 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5442 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5443 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5444 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5445 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5446 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5447 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5448 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5449 catfcontentstart = fheadtell;
5450 fheadtell += len(catfileoutstr) + 1;
5451 catfcontentend = fheadtell - 1;
5452 catfhend = catfcontentend;
5453 fcontents.seek(0, 0);
5454 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5455 pyhascontents = False;
5456 if(int(fsize)>0 and not listonly):
5457 pyhascontents = True;
5458 if(int(fsize)>0 and listonly):
5459 fcontents = BytesIO();
5460 pyhascontents = False;
5461 fcontents.seek(0, 0);
5462 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5463 fileidnum = fileidnum + 1;
5464 return catlist;
5466 def ZipFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5467 formatspecs = FormatSpecsListToDict(formatspecs);
5468 curinode = 0;
5469 curfid = 0;
5470 inodelist = [];
5471 inodetofile = {};
5472 filetoinode = {};
5473 inodetocatinode = {};
5474 fileidnum = 0;
5475 if(infile=="-"):
5476 infile = BytesIO();
5477 if(hasattr(sys.stdin, "buffer")):
5478 shutil.copyfileobj(sys.stdin.buffer, infile);
5479 else:
5480 shutil.copyfileobj(sys.stdin, infile);
5481 infile.seek(0, 0);
5482 if(not infile):
5483 return False;
5484 infile.seek(0, 0);
5485 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
5486 infile = download_file_from_internet_file(infile);
5487 infile.seek(0, 0);
5488 if(not infile):
5489 return False;
5490 infile.seek(0, 0);
5491 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
5492 return False;
5493 else:
5494 return False;
5495 if(not zipfile.is_zipfile(infile)):
5496 return False;
5497 try:
5498 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
5499 except FileNotFoundError:
5500 return False;
5501 ziptest = zipfp.testzip();
5502 if(ziptest):
5503 VerbosePrintOut("Bad file found!");
5504 fnumfiles = int(len(zipfp.infolist()));
5505 catver = formatspecs['format_ver'];
5506 fileheaderver = str(int(catver.replace(".", "")));
5507 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5508 catversion = re.findall(r"([\d]+)", fileheader);
5509 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5510 fnumfileshex = format(int(fnumfiles), 'x').lower();
5511 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5512 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5513 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5514 fheadtell = len(fileheader);
5515 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5516 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
5517 catfhstart = fheadtell;
5518 if(re.findall("^[.|/]", member.filename)):
5519 fname = member.filename;
5520 else:
5521 fname = "./"+member.filename;
5522 zipinfo = zipfp.getinfo(member.filename);
5523 if(verbose):
5524 VerbosePrintOut(fname);
5525 if(not member.is_dir()):
5526 fpremode = stat.S_IFREG + 438;
5527 elif(member.is_dir()):
5528 fpremode = stat.S_IFDIR + 511;
5529 flinkcount = 0;
5530 ftype = 0;
5531 if(not member.is_dir()):
5532 ftype = 0;
5533 elif(member.is_dir()):
5534 ftype = 5;
5535 flinkname = "";
5536 fbasedir = os.path.dirname(fname);
5537 fcurfid = curfid;
5538 fcurinode = curfid;
5539 finode = fcurinode;
5540 curfid = curfid + 1;
5541 fdev_minor = 0;
5542 fdev_major = 0;
5543 frdev_minor = 0;
5544 frdev_major = 0;
5545 if(ftype==5):
5546 fsize = "0";
5547 elif(ftype==0):
5548 fsize = member.file_size;
5549 else:
5550 fsize = member.file_size;
5551 fatime = time.mktime(member.date_time + (0, 0, -1));
5552 fmtime = time.mktime(member.date_time + (0, 0, -1));
5553 fctime = time.mktime(member.date_time + (0, 0, -1));
5554 fbtime = time.mktime(member.date_time + (0, 0, -1));
5555 if(zipinfo.create_system==0 or zipinfo.create_system==10):
5556 fwinattributes = int(zipinfo.external_attr);
5557 if(not member.is_dir()):
5558 fmode = int(stat.S_IFREG + 438);
5559 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
5560 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
5561 elif(member.is_dir()):
5562 fmode = int(stat.S_IFDIR + 511);
5563 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
5564 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
5565 elif(zipinfo.create_system==3):
5566 fwinattributes = int(0);
5567 fmode = int(zipinfo.external_attr);
5568 else:
5569 fwinattributes = int(0);
5570 if(not member.is_dir()):
5571 fmode = int(stat.S_IFREG + 438);
5572 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
5573 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
5574 elif(member.is_dir()):
5575 fmode = int(stat.S_IFDIR + 511);
5576 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
5577 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
5578 fcompression = "";
5579 fcsize = 0;
5580 try:
5581 fuid = os.getuid();
5582 except AttributeError:
5583 fuid = 0;
5584 except KeyError:
5585 fuid = 0;
5586 try:
5587 fgid = os.getgid();
5588 except AttributeError:
5589 fgid = 0;
5590 except KeyError:
5591 fgid = 0;
5592 try:
5593 import pwd;
5594 try:
5595 userinfo = pwd.getpwuid(os.getuid());
5596 funame = userinfo.pw_name;
5597 except KeyError:
5598 funame = "";
5599 except AttributeError:
5600 funame = "";
5601 except ImportError:
5602 funame = "";
5603 fgname = "";
5604 try:
5605 import grp;
5606 try:
5607 groupinfo = grp.getgrgid(os.getgid());
5608 fgname = groupinfo.gr_name;
5609 except KeyError:
5610 fgname = "";
5611 except AttributeError:
5612 fgname = "";
5613 except ImportError:
5614 fgname = "";
5615 fcontents = BytesIO();
5616 if(ftype==0):
5617 fcontents.write(zipfp.read(member.filename));
5618 fcontents.seek(0, 0);
5619 ftypehex = format(ftype, 'x').lower();
5620 extrafields = len(extradata);
5621 extrafieldslist = extradata;
5622 catfextrafields = extrafields;
5623 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5624 if(len(extradata)>0):
5625 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5626 extrasizelen = len(extrasizestr);
5627 extrasizelenhex = format(extrasizelen, 'x').lower();
5628 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5629 catoutlen = len(catoutlist) + len(extradata) + 3;
5630 catoutlenhex = format(catoutlen, 'x').lower();
5631 catoutlist.insert(0, catoutlenhex);
5632 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5633 catheaderdata = catoutlist;
5634 if(len(extradata)>0):
5635 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5636 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
5637 catfnumfields = catoutlen;
5638 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5639 fcontents.seek(0, 0);
5640 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5641 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5642 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5643 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5644 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5645 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5646 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5647 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5648 catfcontentstart = fheadtell;
5649 fheadtell += len(catfileoutstr) + 1;
5650 catfcontentend = fheadtell - 1;
5651 catfhend = catfcontentend;
5652 fcontents.seek(0, 0);
5653 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5654 pyhascontents = False;
5655 if(int(fsize)>0 and not listonly):
5656 pyhascontents = True;
5657 if(int(fsize)>0 and listonly):
5658 fcontents = BytesIO();
5659 pyhascontents = False;
5660 fcontents.seek(0, 0);
5661 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5662 fileidnum = fileidnum + 1;
5663 return catlist;
5665 if(not rarfile_support):
5666 def RarFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5667 return False;
5669 if(rarfile_support):
5670 def RarFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5671 formatspecs = FormatSpecsListToDict(formatspecs);
5672 curinode = 0;
5673 curfid = 0;
5674 inodelist = [];
5675 inodetofile = {};
5676 filetoinode = {};
5677 inodetocatinode = {};
5678 fileidnum = 0;
5679 if(not os.path.exists(infile,) or not os.path.isfile(infile,)):
5680 return False;
5681 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
5682 return False;
5683 rarfp = rarfile.RarFile(infile, "r");
5684 rartest = rarfp.testrar();
5685 if(rartest):
5686 VerbosePrintOut("Bad file found!");
5687 fnumfiles = int(len(rarfp.infolist()));
5688 catver = formatspecs['format_ver'];
5689 fileheaderver = str(int(catver.replace(".", "")));
5690 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5691 catversion = re.findall(r"([\d]+)", fileheader);
5692 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5693 fnumfileshex = format(int(fnumfiles), 'x').lower();
5694 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5695 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5696 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5697 fheadtell = len(fileheader);
5698 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5699 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
5700 catfhstart = fheadtell;
5701 is_unix = False;
5702 is_windows = False;
5703 if(member.host_os==rarfile.RAR_OS_UNIX):
5704 is_windows = False;
5705 try:
5706 member.external_attr
5707 is_unix = True;
5708 except AttributeError:
5709 is_unix = False;
5710 elif(member.host_os==rarfile.RAR_OS_WIN32):
5711 is_unix = False;
5712 try:
5713 member.external_attr
5714 is_windows = True;
5715 except AttributeError:
5716 is_windows = False;
5717 else:
5718 is_unix = False;
5719 is_windows = False;
5720 if(re.findall("^[.|/]", member.filename)):
5721 fname = member.filename;
5722 else:
5723 fname = "./"+member.filename;
5724 rarinfo = rarfp.getinfo(member.filename);
5725 if(verbose):
5726 VerbosePrintOut(fname);
5727 if(is_unix and member.external_attr !=0):
5728 fpremode = int(member.external_attr);
5729 elif(member.is_file()):
5730 fpremode = stat.S_IFREG + 438;
5731 elif(member.is_symlink()):
5732 fpremode = stat.S_IFLNK + 438;
5733 elif(member.is_dir()):
5734 fpremode = stat.S_IFDIR + 511;
5735 if(is_windows and member.external_attr !=0):
5736 fwinattributes = int(member.external_attr);
5737 else:
5738 fwinattributes = int(0);
5739 fcompression = "";
5740 fcsize = 0;
5741 flinkcount = 0;
5742 ftype = 0;
5743 if(member.is_file()):
5744 ftype = 0;
5745 elif(member.is_symlink()):
5746 ftype = 2;
5747 elif(member.is_dir()):
5748 ftype = 5;
5749 flinkname = "";
5750 if(ftype==2):
5751 flinkname = rarfp.read(member.filename).decode("UTF-8");
5752 fbasedir = os.path.dirname(fname);
5753 fcurfid = curfid;
5754 fcurinode = curfid;
5755 finode = fcurinode;
5756 curfid = curfid + 1;
5757 fdev_minor = 0;
5758 fdev_major = 0;
5759 frdev_minor = 0;
5760 frdev_major = 0;
5761 if(ftype==5):
5762 fsize = "0";
5763 if(ftype==0):
5764 fsize = member.file_size;
5765 try:
5766 if(member.atime):
5767 fatime = int(member.atime.timestamp());
5768 else:
5769 fatime = int(member.mtime.timestamp());
5770 except AttributeError:
5771 fatime = int(member.mtime.timestamp());
5772 fmtime = int(member.mtime.timestamp());
5773 try:
5774 if(member.ctime):
5775 fctime = int(member.ctime.timestamp());
5776 else:
5777 fctime = int(member.mtime.timestamp());
5778 except AttributeError:
5779 fctime = int(member.mtime.timestamp());
5780 fbtime = int(member.mtime.timestamp());
5781 if(is_unix and member.external_attr !=0):
5782 fmode = int(member.external_attr);
5783 fchmode = int(stat.S_IMODE(member.external_attr));
5784 ftypemod = int(stat.S_IFMT(member.external_attr));
5785 elif(member.is_file()):
5786 fmode = int(stat.S_IFREG + 438)
5787 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5788 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5789 elif(member.is_symlink()):
5790 fmode = int(stat.S_IFLNK + 438)
5791 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5792 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5793 elif(member.is_dir()):
5794 fmode = int(stat.S_IFDIR + 511)
5795 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
5796 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
5797 try:
5798 fuid = os.getuid();
5799 except AttributeError:
5800 fuid = 0;
5801 except KeyError:
5802 fuid = 0;
5803 try:
5804 fgid = os.getgid();
5805 except AttributeError:
5806 fgid = 0;
5807 except KeyError:
5808 fgid = 0;
5809 try:
5810 import pwd;
5811 try:
5812 userinfo = pwd.getpwuid(os.getuid());
5813 funame = userinfo.pw_name;
5814 except KeyError:
5815 funame = "";
5816 except AttributeError:
5817 funame = "";
5818 except ImportError:
5819 funame = "";
5820 fgname = "";
5821 try:
5822 import grp;
5823 try:
5824 groupinfo = grp.getgrgid(os.getgid());
5825 fgname = groupinfo.gr_name;
5826 except KeyError:
5827 fgname = "";
5828 except AttributeError:
5829 fgname = "";
5830 except ImportError:
5831 fgname = "";
5832 fcontents = BytesIO();
5833 if(ftype==0):
5834 fcontents.write(rarfp.read(member.filename));
5835 fcontents.seek(0, 0);
5836 ftypehex = format(ftype, 'x').lower();
5837 extrafields = len(extradata);
5838 extrafieldslist = extradata;
5839 catfextrafields = extrafields;
5840 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5841 if(len(extradata)>0):
5842 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5843 extrasizelen = len(extrasizestr);
5844 extrasizelenhex = format(extrasizelen, 'x').lower();
5845 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5846 catoutlen = len(catoutlist) + len(extradata) + 3;
5847 catoutlenhex = format(catoutlen, 'x').lower();
5848 catoutlist.insert(0, catoutlenhex);
5849 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5850 if(len(extradata)>0):
5851 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5852 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
5853 catfnumfields = 24 + catfextrafields;
5854 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5855 fcontents.seek(0, 0);
5856 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5857 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5858 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5859 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5860 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5861 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5862 catheaderdata = catoutlist;
5863 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5864 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5865 catfcontentstart = fheadtell;
5866 fheadtell += len(catfileoutstr) + 1;
5867 catfcontentend = fheadtell - 1;
5868 catfhend = catfcontentend;
5869 fcontents.seek(0, 0);
5870 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5871 pyhascontents = False;
5872 if(int(fsize)>0 and not listonly):
5873 pyhascontents = True;
5874 if(int(fsize)>0 and listonly):
5875 fcontents = BytesIO();
5876 pyhascontents = False;
5877 fcontents.seek(0, 0);
5878 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5879 fileidnum = fileidnum + 1;
5880 return catlist;
5882 if(not py7zr_support):
5883 def SevenZipFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5884 return False;
5886 if(py7zr_support):
5887 def SevenZipFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5888 formatspecs = FormatSpecsListToDict(formatspecs);
5889 curinode = 0;
5890 curfid = 0;
5891 inodelist = [];
5892 inodetofile = {};
5893 filetoinode = {};
5894 inodetocatinode = {};
5895 fileidnum = 0;
5896 szpfp = py7zr.SevenZipFile(infile, mode="r");
5897 file_content = szpfp.readall();
5898 #sztest = szpfp.testzip();
5899 sztestalt = szpfp.test();
5900 if(sztestalt):
5901 VerbosePrintOut("Bad file found!");
5902 numfiles = int(len(szpfp.list()));
5903 catver = formatspecs['format_ver'];
5904 fileheaderver = str(int(catver.replace(".", "")));
5905 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5906 catversion = re.findall(r"([\d]+)", fileheader);
5907 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5908 fnumfileshex = format(int(fnumfiles), 'x').lower();
5909 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5910 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5911 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5912 fheadtell = len(fileheader);
5913 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5914 for member in sorted(szpfp.list(), key=lambda x: x.filename):
5915 catfhstart = fheadtell;
5916 if(re.findall("^[.|/]", member.filename)):
5917 fname = member.filename;
5918 else:
5919 fname = "./"+member.filename;
5920 if(not member.is_directory):
5921 fpremode = int(stat.S_IFREG + 438);
5922 elif(member.is_directory):
5923 fpremode = int(stat.S_IFDIR + 511);
5924 fwinattributes = int(0);
5925 fcompression = "";
5926 fcsize = 0;
5927 flinkcount = 0;
5928 ftype = 0;
5929 if(member.is_directory):
5930 ftype = 5;
5931 else:
5932 ftype = 0;
5933 flinkname = "";
5934 fbasedir = os.path.dirname(fname);
5935 fcurfid = curfid;
5936 fcurinode = curfid;
5937 finode = fcurinode;
5938 curfid = curfid + 1;
5939 fdev_minor = 0;
5940 fdev_major = 0;
5941 frdev_minor = 0;
5942 frdev_major = 0;
5943 if(ftype==5):
5944 fsize = "0";
5945 fatime = int(member.creationtime.timestamp());
5946 fmtime = int(member.creationtime.timestamp());
5947 fctime = int(member.creationtime.timestamp());
5948 fbtime = int(member.creationtime.timestamp());
5949 if(member.is_directory):
5950 fmode = int(stat.S_IFDIR + 511)
5951 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
5952 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
5953 else:
5954 fmode = int(stat.S_IFLNK + 438)
5955 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5956 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5957 try:
5958 fuid = os.getuid();
5959 except AttributeError:
5960 fuid = 0;
5961 except KeyError:
5962 fuid = 0;
5963 try:
5964 fgid = os.getgid();
5965 except AttributeError:
5966 fgid = 0;
5967 except KeyError:
5968 fgid = 0;
5969 try:
5970 import pwd;
5971 try:
5972 userinfo = pwd.getpwuid(os.getuid());
5973 funame = userinfo.pw_name;
5974 except KeyError:
5975 funame = "";
5976 except AttributeError:
5977 funame = "";
5978 except ImportError:
5979 funame = "";
5980 fgname = "";
5981 try:
5982 import grp;
5983 try:
5984 groupinfo = grp.getgrgid(os.getgid());
5985 fgname = groupinfo.gr_name;
5986 except KeyError:
5987 fgname = "";
5988 except AttributeError:
5989 fgname = "";
5990 except ImportError:
5991 fgname = "";
5992 fcontents = BytesIO();
5993 if(ftype==0):
5994 fcontents.write(file_content[member.filename].read());
5995 fsize = format(fcontents.tell(), 'x').lower();
5996 fileop.close();
5997 fcontents.seek(0, 0);
5998 ftypehex = format(ftype, 'x').lower();
5999 extrafields = len(extradata);
6000 extrafieldslist = extradata;
6001 catfextrafields = extrafields;
6002 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
6003 if(len(extradata)>0):
6004 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
6005 extrasizelen = len(extrasizestr);
6006 extrasizelenhex = format(extrasizelen, 'x').lower();
6007 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
6008 catoutlen = len(catoutlist) + len(extradata) + 3;
6009 catoutlenhex = format(catoutlen, 'x').lower();
6010 catoutlist.insert(0, catoutlenhex);
6011 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
6012 catheaderdata = catoutlist;
6013 if(len(extradata)>0):
6014 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
6015 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
6016 catfnumfields = 24 + catfextrafields;
6017 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
6018 fcontents.seek(0, 0);
6019 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
6020 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
6021 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
6022 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
6023 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
6024 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
6025 catfileoutstrecd = catfileoutstr.encode('UTF-8');
6026 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
6027 catfcontentstart = fheadtell;
6028 fheadtell += len(catfileoutstr) + 1;
6029 catfcontentend = fheadtell - 1;
6030 catfhend = catfcontentend;
6031 fcontents.seek(0, 0);
6032 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
6033 pyhascontents = False;
6034 if(int(fsize)>0 and not listonly):
6035 pyhascontents = True;
6036 if(int(fsize)>0 and listonly):
6037 fcontents = BytesIO();
6038 pyhascontents = False;
6039 fcontents.seek(0, 0);
6040 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
6041 fileidnum = fileidnum + 1;
6042 return catlist;
6044 def InFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6045 formatspecs = FormatSpecsListToDict(formatspecs);
6046 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
6047 if(checkcompressfile=="tarfile"):
6048 return TarFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6049 elif(checkcompressfile=="zipfile"):
6050 return ZipFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6051 elif(checkcompressfile=="catfile"):
6052 return ArchiveFileToArray(infile, 0, 0, listonly, True, False, formatspecs, False);
6053 elif(rarfile_support and checkcompressfile=="rarfile"):
6054 return RarFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6055 elif(py7zr_support and checkcompressfile=="7zipfile"):
6056 return SevenZipFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6057 else:
6058 return False;
6059 return False;
6061 def ListDirToArray(infiles, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, listonly=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6062 formatspecs = FormatSpecsListToDict(formatspecs);
6063 outarray = BytesIO();
6064 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
6065 listcatfiles = ArchiveFileToArray(outarray, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
6066 return listcatfiles;
6068 def ArchiveFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6069 formatspecs = FormatSpecsListToDict(formatspecs);
6070 if(isinstance(infile, dict)):
6071 listcatfiles = infile;
6072 else:
6073 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6074 infile = RemoveWindowsPath(infile);
6075 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, listonly, uncompress, skipchecksum, formatspecs, returnfp);
6076 if(not listcatfiles):
6077 return False;
6078 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6079 if(returnfp):
6080 catarray.update({'catfp': listcatfiles['catfp']});
6081 lenlist = len(listcatfiles['ffilelist']);
6082 lcfi = 0;
6083 lcfx = int(listcatfiles['fnumfiles']);
6084 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6085 lcfx = int(lenlist);
6086 else:
6087 lcfx = int(listcatfiles['fnumfiles']);
6088 while(lcfi < lcfx):
6089 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6090 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6091 catarray['filetoid'].update(filetoidarray);
6092 catarray['idtofile'].update(idtofilearray);
6093 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6094 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6095 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6096 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6097 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6098 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6099 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6100 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6101 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6102 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6103 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6104 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6105 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6106 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6107 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6108 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6109 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6110 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6111 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6112 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6113 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6114 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6115 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6116 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6117 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6118 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6119 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6120 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6121 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6122 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6123 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6124 lcfi = lcfi + 1;
6125 return catarray;
6127 create_alias_function("", __file_format_name__, "ToArrayIndex", ArchiveFileToArrayIndex);
6129 def ListDirToArrayIndexAlt(infiles, dirlistfromtxt=False, followlink=False, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6130 formatspecs = FormatSpecsListToDict(formatspecs);
6131 listcatfiles = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, listonly, checksumtype, extradata, formatspecs, verbose);
6132 if(not listcatfiles):
6133 return False;
6134 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6135 lenlist = len(listcatfiles['ffilelist']);
6136 lcfi = 0;
6137 lcfx = int(listcatfiles['fnumfiles']);
6138 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6139 lcfx = int(lenlist);
6140 else:
6141 lcfx = int(listcatfiles['fnumfiles']);
6142 while(lcfi < lcfx):
6143 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6144 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6145 catarray['filetoid'].update(filetoidarray);
6146 catarray['idtofile'].update(idtofilearray);
6147 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6148 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6149 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6150 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6151 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6152 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6153 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6154 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6155 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6156 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6157 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6158 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6159 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6160 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6161 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6162 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6163 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6164 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6165 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6166 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6167 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6168 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6169 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6170 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6171 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6172 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6173 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6174 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6175 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6176 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6177 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6178 lcfi = lcfi + 1;
6179 return catarray;
6181 def TarFileToArrayIndexAlt(infile, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6182 formatspecs = FormatSpecsListToDict(formatspecs);
6183 listcatfiles = TarFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6184 if(not listcatfiles):
6185 return False;
6186 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6187 lenlist = len(listcatfiles['ffilelist']);
6188 lcfi = 0;
6189 lcfx = int(listcatfiles['fnumfiles']);
6190 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6191 lcfx = int(lenlist);
6192 else:
6193 lcfx = int(listcatfiles['fnumfiles']);
6194 while(lcfi < lcfx):
6195 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6196 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6197 catarray['filetoid'].update(filetoidarray);
6198 catarray['idtofile'].update(idtofilearray);
6199 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6200 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6201 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6202 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6203 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6204 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6205 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6206 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6207 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6208 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6209 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6210 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6211 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6212 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6213 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6214 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6215 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6216 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6217 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6218 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6219 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6220 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6221 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6222 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6223 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6224 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6225 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6226 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6227 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6228 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6229 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6230 lcfi = lcfi + 1;
6231 return catarray;
6233 def ZipFileToArrayIndexAlt(infile, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6234 formatspecs = FormatSpecsListToDict(formatspecs);
6235 listcatfiles = ZipFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6236 if(not listcatfiles):
6237 return False;
6238 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6239 lenlist = len(listcatfiles['ffilelist']);
6240 lcfi = 0;
6241 lcfx = int(listcatfiles['fnumfiles']);
6242 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6243 lcfx = int(lenlist);
6244 else:
6245 lcfx = int(listcatfiles['fnumfiles']);
6246 while(lcfi < lcfx):
6247 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6248 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6249 catarray['filetoid'].update(filetoidarray);
6250 catarray['idtofile'].update(idtofilearray);
6251 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6252 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6253 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6254 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6255 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6256 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6257 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6258 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6259 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6260 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6261 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6262 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6263 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6264 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6265 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6266 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6267 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6268 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6269 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6270 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6271 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6272 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6273 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6274 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6275 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6276 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6277 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6278 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6279 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6280 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6281 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6282 lcfi = lcfi + 1;
6283 return catarray;
6285 if(not rarfile_support):
6286 def RarFileToArrayIndexAlt(infile, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6287 return False;
6289 if(rarfile_support):
6290 def RarFileToArrayIndexAlt(infile, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6291 formatspecs = FormatSpecsListToDict(formatspecs);
6292 listcatfiles = RarFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6293 if(not listcatfiles):
6294 return False;
6295 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6296 lenlist = len(listcatfiles['ffilelist']);
6297 lcfi = 0;
6298 lcfx = int(listcatfiles['fnumfiles']);
6299 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6300 lcfx = int(lenlist);
6301 else:
6302 lcfx = int(listcatfiles['fnumfiles']);
6303 while(lcfi < lcfx):
6304 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6305 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6306 catarray['filetoid'].update(filetoidarray);
6307 catarray['idtofile'].update(idtofilearray);
6308 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6309 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6310 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6311 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6312 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6313 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6314 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6315 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6316 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6317 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6318 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6319 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6320 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6321 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6322 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6323 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6324 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6325 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6326 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6327 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6328 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6329 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6330 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6331 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6332 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6333 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6334 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6335 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6336 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6337 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6338 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6339 lcfi = lcfi + 1;
6340 return catarray;
6342 if(not py7zr_support):
6343 def SevenZipFileToArrayIndexAlt(infile, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6344 return False;
6346 if(py7zr_support):
6347 def SevenZipFileToArrayIndexAlt(infile, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6348 formatspecs = FormatSpecsListToDict(formatspecs);
6349 listcatfiles = SevenZipFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6350 if(not listcatfiles):
6351 return False;
6352 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6353 lenlist = len(listcatfiles['ffilelist']);
6354 lcfi = 0;
6355 lcfx = int(listcatfiles['fnumfiles']);
6356 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6357 lcfx = int(lenlist);
6358 else:
6359 lcfx = int(listcatfiles['fnumfiles']);
6360 while(lcfi < lcfx):
6361 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6362 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6363 catarray['filetoid'].update(filetoidarray);
6364 catarray['idtofile'].update(idtofilearray);
6365 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6366 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6367 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6368 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6369 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6370 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6371 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6372 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6373 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6374 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6375 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6376 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6377 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6378 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6379 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6380 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6381 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6382 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6383 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6384 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6385 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6386 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6387 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6388 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6389 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6390 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6391 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6392 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6393 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6394 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6395 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6396 lcfi = lcfi + 1;
6397 return catarray;
6399 def ArchiveFileStringToArrayIndex(catstr, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6400 formatspecs = FormatSpecsListToDict(formatspecs);
6401 catfp = BytesIO(catstr);
6402 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, uncompress, skipchecksum, formatspecs, returnfp);
6403 return listcatfiles;
6405 create_alias_function("", __file_format_name__, "StringToArrayIndex", ArchiveFileStringToArrayIndex);
6407 def TarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6408 formatspecs = FormatSpecsListToDict(formatspecs);
6409 catfp = BytesIO();
6410 catfp = PackArchiveFileFromTarFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
6411 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
6412 return listcatfiles;
6414 def ZipFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6415 formatspecs = FormatSpecsListToDict(formatspecs);
6416 catfp = BytesIO();
6417 catfp = PackArchiveFileFromZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
6418 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
6419 return listcatfiles;
6421 if(not rarfile_support):
6422 def RarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6423 return False;
6425 if(rarfile_support):
6426 def RarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6427 formatspecs = FormatSpecsListToDict(formatspecs);
6428 catfp = BytesIO();
6429 catfp = PackArchiveFileFromRarFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
6430 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
6431 return listcatfiles;
6433 if(not py7zr_support):
6434 def SevenZipFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6435 return False;
6437 if(py7zr_support):
6438 def SevenZipFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6439 formatspecs = FormatSpecsListToDict(formatspecs);
6440 catfp = BytesIO();
6441 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
6442 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
6443 return listcatfiles;
6445 def ListDirToArrayIndex(infiles, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, listonly=False, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6446 formatspecs = FormatSpecsListToDict(formatspecs);
6447 outarray = BytesIO();
6448 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose, True);
6449 listcatfiles = ArchiveFileToArrayIndex(outarray, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp)
6450 return listcatfiles;
6452 def RePackArchiveFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6453 formatspecs = FormatSpecsListToDict(formatspecs);
6454 if(isinstance(infile, dict)):
6455 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, True, skipchecksum, formatspecs, returnfp);
6456 listcatfiles = prelistcatfiles['list'];
6457 else:
6458 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6459 infile = RemoveWindowsPath(infile);
6460 if(followlink):
6461 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, True, skipchecksum, formatspecs, returnfp);
6462 listcatfiles = prelistcatfiles['list'];
6463 else:
6464 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, True, skipchecksum, formatspecs, returnfp);
6465 if(outfile!="-" and not hasattr(infile, "read") and not hasattr(outfile, "write")):
6466 outfile = RemoveWindowsPath(outfile);
6467 checksumtype = checksumtype.lower();
6468 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
6469 checksumtype="crc32";
6470 if(checksumtype=="none"):
6471 checksumtype = "";
6472 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
6473 compression = "auto";
6474 if(compression not in compressionlist and compression is None):
6475 compression = "auto";
6476 if(verbose):
6477 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6478 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
6479 if(os.path.exists(outfile)):
6480 try:
6481 os.unlink(outfile);
6482 except OSError as e:
6483 pass;
6484 if(not listcatfiles):
6485 return False;
6486 if(outfile=="-"):
6487 verbose = False;
6488 catfp = BytesIO();
6489 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
6490 catfp = outfile;
6491 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
6492 catfp = BytesIO();
6493 else:
6494 fbasename = os.path.splitext(outfile)[0];
6495 fextname = os.path.splitext(outfile)[1];
6496 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
6497 catver = formatspecs['format_ver'];
6498 fileheaderver = str(int(catver.replace(".", "")));
6499 lenlist = len(listcatfiles['ffilelist']);
6500 fnumfiles = int(listcatfiles['fnumfiles']);
6501 if(lenlist>fnumfiles or lenlist<fnumfiles):
6502 fnumfiles = lenlist;
6503 AppendFileHeader(catfp, fnumfiles, checksumtype, formatspecs);
6504 lenlist = len(listcatfiles['ffilelist']);
6505 fnumfiles = int(listcatfiles['fnumfiles']);
6506 lcfi = 0;
6507 lcfx = int(listcatfiles['fnumfiles']);
6508 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6509 lcfx = int(lenlist);
6510 else:
6511 lcfx = int(listcatfiles['fnumfiles']);
6512 curinode = 0;
6513 curfid = 0;
6514 inodelist = [];
6515 inodetofile = {};
6516 filetoinode = {};
6517 reallcfi = 0;
6518 while(lcfi < lcfx):
6519 if(re.findall("^[.|/]", listcatfiles['ffilelist'][reallcfi]['fname'])):
6520 fname = listcatfiles['ffilelist'][reallcfi]['fname'];
6521 else:
6522 fname = "./"+listcatfiles['ffilelist'][reallcfi]['fname'];
6523 if(verbose):
6524 VerbosePrintOut(fname);
6525 fheadersize = format(int(listcatfiles['ffilelist'][reallcfi]['fheadersize']), 'x').lower();
6526 fsize = format(int(listcatfiles['ffilelist'][reallcfi]['fsize']), 'x').lower();
6527 flinkname = listcatfiles['ffilelist'][reallcfi]['flinkname'];
6528 fatime = format(int(listcatfiles['ffilelist'][reallcfi]['fatime']), 'x').lower();
6529 fmtime = format(int(listcatfiles['ffilelist'][reallcfi]['fmtime']), 'x').lower();
6530 fctime = format(int(listcatfiles['ffilelist'][reallcfi]['fctime']), 'x').lower();
6531 fbtime = format(int(listcatfiles['ffilelist'][reallcfi]['fbtime']), 'x').lower();
6532 fmode = format(int(listcatfiles['ffilelist'][reallcfi]['fmode']), 'x').lower();
6533 fchmode = format(int(listcatfiles['ffilelist'][reallcfi]['fchmode']), 'x').lower();
6534 fuid = format(int(listcatfiles['ffilelist'][reallcfi]['fuid']), 'x').lower();
6535 funame = listcatfiles['ffilelist'][reallcfi]['funame'];
6536 fgid = format(int(listcatfiles['ffilelist'][reallcfi]['fgid']), 'x').lower();
6537 fgname = listcatfiles['ffilelist'][reallcfi]['fgname'];
6538 finode = format(int(listcatfiles['ffilelist'][reallcfi]['finode']), 'x').lower();
6539 flinkcount = format(int(listcatfiles['ffilelist'][reallcfi]['flinkcount']), 'x').lower();
6540 fwinattributes = format(int(listcatfiles['ffilelist'][reallcfi]['fwinattributes']), 'x').lower();
6541 fcompression = listcatfiles['ffilelist'][reallcfi]['fcompression'];
6542 fcsize = format(int(listcatfiles['ffilelist'][reallcfi]['fcsize']), 'x').lower();
6543 fdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['fminor']), 'x').lower();
6544 fdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['fmajor']), 'x').lower();
6545 frdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['frminor']), 'x').lower();
6546 frdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['frmajor']), 'x').lower();
6547 if(len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>listcatfiles['ffilelist'][reallcfi]['fextrafields'] and len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>0):
6548 listcatfiles['ffilelist'][reallcfi]['fextrafields'] = len(listcatfiles['ffilelist'][reallcfi]['fextralist']);
6549 if(not followlink and len(extradata)<0):
6550 extradata = listcatfiles['ffilelist'][reallcfi]['fextralist'];
6551 fcontents = listcatfiles['ffilelist'][reallcfi]['fcontents'];
6552 fcompression = "";
6553 fcsize = format(int(0), 'x').lower();
6554 if(not compresswholefile):
6555 fcontents.seek(0, 2);
6556 ucfsize = fcontents.tell();
6557 fcontents.seek(0, 0);
6558 if(compression=="auto"):
6559 ilsize = len(compressionlistalt);
6560 ilmin = 0;
6561 ilcsize = [];
6562 while(ilmin < ilsize):
6563 cfcontents = BytesIO();
6564 shutil.copyfileobj(fcontents, cfcontents);
6565 fcontents.seek(0, 0);
6566 cfcontents.seek(0, 0);
6567 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
6568 if(cfcontents):
6569 cfcontents.seek(0, 2);
6570 ilcsize.append(cfcontents.tell());
6571 cfcontents.close();
6572 else:
6573 try:
6574 ilcsize.append(sys.maxint);
6575 except AttributeError:
6576 ilcsize.append(sys.maxsize);
6577 ilmin = ilmin + 1;
6578 ilcmin = ilcsize.index(min(ilcsize));
6579 compression = compressionlistalt[ilcmin];
6580 fcontents.seek(0, 0);
6581 cfcontents = BytesIO();
6582 shutil.copyfileobj(fcontents, cfcontents);
6583 cfcontents.seek(0, 0);
6584 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
6585 cfcontents.seek(0, 2);
6586 cfsize = cfcontents.tell();
6587 if(ucfsize > cfsize):
6588 fcsize = format(int(cfsize), 'x').lower();
6589 fcompression = compression;
6590 fcontents.close();
6591 fcontents = cfcontents;
6592 if(followlink):
6593 if(listcatfiles['ffilelist'][reallcfi]['ftype']==1 or listcatfiles['ffilelist'][reallcfi]['ftype']==2):
6594 getflinkpath = listcatfiles['ffilelist'][reallcfi]['flinkname'];
6595 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6596 flinkinfo = listcatfiles['ffilelist'][flinkid];
6597 fheadersize = format(int(flinkinfo['fheadersize']), 'x').lower();
6598 fsize = format(int(flinkinfo['fsize']), 'x').lower();
6599 flinkname = flinkinfo['flinkname'];
6600 fatime = format(int(flinkinfo['fatime']), 'x').lower();
6601 fmtime = format(int(flinkinfo['fmtime']), 'x').lower();
6602 fctime = format(int(flinkinfo['fctime']), 'x').lower();
6603 fbtime = format(int(flinkinfo['fbtime']), 'x').lower();
6604 fmode = format(int(flinkinfo['fmode']), 'x').lower();
6605 fchmode = format(int(flinkinfo['fchmode']), 'x').lower();
6606 fuid = format(int(flinkinfo['fuid']), 'x').lower();
6607 funame = flinkinfo['funame'];
6608 fgid = format(int(flinkinfo['fgid']), 'x').lower();
6609 fgname = flinkinfo['fgname'];
6610 finode = format(int(flinkinfo['finode']), 'x').lower();
6611 flinkcount = format(int(flinkinfo['flinkcount']), 'x').lower();
6612 fwinattributes = format(int(flinkinfo['fwinattributes']), 'x').lower();
6613 fcompression = flinkinfo['fcompression'];
6614 fcsize = format(int(flinkinfo['fcsize']), 'x').lower();
6615 fdev_minor = format(int(flinkinfo['fminor']), 'x').lower();
6616 fdev_major = format(int(flinkinfo['fmajor']), 'x').lower();
6617 frdev_minor = format(int(flinkinfo['frminor']), 'x').lower();
6618 frdev_major = format(int(flinkinfo['frmajor']), 'x').lower();
6619 if(len(flinkinfo['fextralist'])>flinkinfo['fextrafields'] and len(flinkinfo['fextralist'])>0):
6620 flinkinfo['fextrafields'] = len(flinkinfo['fextralist']);
6621 if(len(extradata)<0):
6622 extradata = flinkinfo['fextralist'];
6623 fcontents = flinkinfo['fcontents'];
6624 ftypehex = format(flinkinfo['ftype'], 'x').lower();
6625 else:
6626 ftypehex = format(listcatfiles['ffilelist'][reallcfi]['ftype'], 'x').lower();
6627 fcurfid = format(curfid, 'x').lower();
6628 if(not followlink and finode!=0):
6629 if(listcatfiles['ffilelist'][reallcfi]['ftype']!=1):
6630 fcurinode = format(int(curinode), 'x').lower();
6631 inodetofile.update({curinode: fname});
6632 filetoinode.update({fname: curinode});
6633 curinode = curinode + 1;
6634 else:
6635 fcurinode = format(int(filetoinode[flinkname]), 'x').lower();
6636 else:
6637 fcurinode = format(int(curinode), 'x').lower();
6638 curinode = curinode + 1;
6639 curfid = curfid + 1;
6640 if(fcompression=="none"):
6641 fcompression = "";
6642 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
6643 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
6644 fcontents.close();
6645 lcfi = lcfi + 1;
6646 reallcfi = reallcfi + 1;
6647 if(lcfx>0):
6648 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
6649 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
6650 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
6651 try:
6652 catfp.flush();
6653 os.fsync(catfp.fileno());
6654 except io.UnsupportedOperation:
6655 pass;
6656 except AttributeError:
6657 pass;
6658 except OSError as e:
6659 pass;
6660 if(outfile=="-"):
6661 catfp.seek(0, 0);
6662 if(hasattr(sys.stdout, "buffer")):
6663 shutil.copyfileobj(catfp, sys.stdout.buffer);
6664 else:
6665 shutil.copyfileobj(catfp, sys.stdout);
6666 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
6667 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
6668 catfp.seek(0, 0);
6669 upload_file_to_internet_file(catfp, outfile);
6670 if(returnfp):
6671 catfp.seek(0, 0);
6672 return catfp;
6673 else:
6674 catfp.close();
6675 return True;
6677 create_alias_function("RePack", __file_format_name__, "", RePackArchiveFile);
6679 def RePackArchiveFileFromString(catstr, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6680 formatspecs = FormatSpecsListToDict(formatspecs);
6681 catfp = BytesIO(catstr);
6682 listcatfiles = RePackArchiveFile(catfp, compression, compresswholefile, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6683 return listcatfiles;
6685 create_alias_function("RePack", __file_format_name__, "FromString", RePackArchiveFileFromString);
6687 def PackArchiveFileFromListDir(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6688 formatspecs = FormatSpecsListToDict(formatspecs);
6689 outarray = BytesIO();
6690 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
6691 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6692 return listcatfiles;
6694 create_alias_function("Pack", __file_format_name__, "FromListDir", PackArchiveFileFromListDir);
6696 def UnPackArchiveFile(infile, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, preservepermissions=True, preservetime=True, verbose=False, returnfp=False):
6697 formatspecs = FormatSpecsListToDict(formatspecs);
6698 if(outdir is not None):
6699 outdir = RemoveWindowsPath(outdir);
6700 if(verbose):
6701 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6702 if(isinstance(infile, dict)):
6703 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, True, skipchecksum, formatspecs, returnfp);
6704 listcatfiles = prelistcatfiles['list'];
6705 else:
6706 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6707 infile = RemoveWindowsPath(infile);
6708 if(followlink):
6709 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, True, skipchecksum, formatspecs, returnfp);
6710 listcatfiles = prelistcatfiles['list'];
6711 else:
6712 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, True, skipchecksum, formatspecs, returnfp);
6713 if(not listcatfiles):
6714 return False;
6715 lenlist = len(listcatfiles['ffilelist']);
6716 fnumfiles = int(listcatfiles['fnumfiles']);
6717 lcfi = 0;
6718 lcfx = int(listcatfiles['fnumfiles']);
6719 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6720 lcfx = int(lenlist);
6721 else:
6722 lcfx = int(listcatfiles['fnumfiles']);
6723 while(lcfi<lcfx):
6724 funame = "";
6725 try:
6726 import pwd;
6727 try:
6728 userinfo = pwd.getpwuid(listcatfiles['ffilelist'][lcfi]['fuid']);
6729 funame = userinfo.pw_name;
6730 except KeyError:
6731 funame = "";
6732 except ImportError:
6733 funame = "";
6734 fgname = "";
6735 try:
6736 import grp;
6737 try:
6738 groupinfo = grp.getgrgid(listcatfiles['ffilelist'][lcfi]['fgid']);
6739 fgname = groupinfo.gr_name;
6740 except KeyError:
6741 fgname = "";
6742 except ImportError:
6743 fgname = "";
6744 if(verbose):
6745 VerbosePrintOut(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6746 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6747 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6748 listcatfiles['ffilelist'][lcfi]['fcontents'].seek(0, 0);
6749 shutil.copyfileobj(listcatfiles['ffilelist'][lcfi]['fcontents'], fpc);
6750 try:
6751 fpc.flush();
6752 os.fsync(fpc.fileno());
6753 except io.UnsupportedOperation:
6754 pass;
6755 except AttributeError:
6756 pass;
6757 except OSError as e:
6758 pass;
6759 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
6760 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
6761 if(preservepermissions):
6762 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6763 if(preservetime):
6764 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
6765 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6766 if(followlink):
6767 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
6768 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6769 flinkinfo = listcatfiles['ffilelist'][flinkid];
6770 funame = "";
6771 try:
6772 import pwd;
6773 try:
6774 userinfo = pwd.getpwuid(flinkinfo['fuid']);
6775 funame = userinfo.pw_name;
6776 except KeyError:
6777 funame = "";
6778 except ImportError:
6779 funame = "";
6780 fgname = "";
6781 try:
6782 import grp;
6783 try:
6784 groupinfo = grp.getgrgid(flinkinfo['fgid']);
6785 fgname = groupinfo.gr_name;
6786 except KeyError:
6787 fgname = "";
6788 except ImportError:
6789 fgname = "";
6790 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
6791 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6792 flinkinfo['fcontents'].seek(0, 0);
6793 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
6794 try:
6795 fpc.flush();
6796 os.fsync(fpc.fileno());
6797 except io.UnsupportedOperation:
6798 pass;
6799 except AttributeError:
6800 pass;
6801 except OSError as e:
6802 pass;
6803 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6804 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6805 if(preservepermissions):
6806 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6807 if(preservetime):
6808 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6809 if(flinkinfo['ftype']==1):
6810 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6811 if(flinkinfo['ftype']==2):
6812 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6813 if(flinkinfo['ftype']==5):
6814 if(preservepermissions):
6815 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6816 else:
6817 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6818 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6819 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6820 if(preservepermissions):
6821 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6822 if(preservetime):
6823 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6824 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
6825 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6826 else:
6827 os.link(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6828 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6829 if(followlink):
6830 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
6831 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6832 flinkinfo = listcatfiles['ffilelist'][flinkid];
6833 funame = "";
6834 try:
6835 import pwd;
6836 try:
6837 userinfo = pwd.getpwuid(flinkinfo['fuid']);
6838 funame = userinfo.pw_name;
6839 except KeyError:
6840 funame = "";
6841 except ImportError:
6842 funame = "";
6843 fgname = "";
6844 try:
6845 import grp;
6846 try:
6847 groupinfo = grp.getgrgid(flinkinfo['fgid']);
6848 fgname = groupinfo.gr_name;
6849 except KeyError:
6850 fgname = "";
6851 except ImportError:
6852 fgname = "";
6853 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
6854 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6855 flinkinfo['fcontents'].seek(0, 0);
6856 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
6857 try:
6858 fpc.flush();
6859 os.fsync(fpc.fileno());
6860 except io.UnsupportedOperation:
6861 pass;
6862 except AttributeError:
6863 pass;
6864 except OSError as e:
6865 pass;
6866 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6867 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6868 if(preservepermissions):
6869 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6870 if(preservetime):
6871 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6872 if(flinkinfo['ftype']==1):
6873 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6874 if(flinkinfo['ftype']==2):
6875 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6876 if(flinkinfo['ftype']==5):
6877 if(preservepermissions):
6878 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6879 else:
6880 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6881 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6882 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6883 if(preservepermissions):
6884 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6885 if(preservetime):
6886 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6887 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
6888 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6889 else:
6890 os.symlink(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6891 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6892 if(preservepermissions):
6893 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6894 else:
6895 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6896 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
6897 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
6898 if(preservepermissions):
6899 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6900 if(preservetime):
6901 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
6902 if(listcatfiles['ffilelist'][lcfi]['ftype']==6 and hasattr(os, "mkfifo")):
6903 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6904 lcfi = lcfi + 1;
6905 if(returnfp):
6906 return listcatfiles['ffilelist']['catfp'];
6907 else:
6908 return True;
6910 create_alias_function("UnPack", __file_format_name__, "", UnPackArchiveFile);
6912 if(hasattr(shutil, "register_unpack_format")):
6913 def UnPackArchiveFileFunc(archive_name, extract_dir=None, **kwargs):
6914 return UnPackArchiveFile(archive_name, extract_dir, False, 0, 0, False, __file_format_dict__['format_delimiter'], False, False);
6915 create_alias_function("UnPack", __file_format_name__, "Func", UnPackArchiveFileFunc);
6917 def UnPackArchiveFileString(catstr, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6918 formatspecs = FormatSpecsListToDict(formatspecs);
6919 catfp = BytesIO(catstr);
6920 listcatfiles = UnPackArchiveFile(catfp, outdir, followlink, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6921 return listcatfiles;
6923 create_alias_function("UnPack", __file_format_name__, "String", UnPackArchiveFileString);
6925 def ArchiveFileListFiles(infile, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6926 formatspecs = FormatSpecsListToDict(formatspecs);
6927 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6928 if(isinstance(infile, dict)):
6929 listcatfiles = infile;
6930 else:
6931 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6932 infile = RemoveWindowsPath(infile);
6933 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, True, False, skipchecksum, formatspecs, returnfp);
6934 if(not listcatfiles):
6935 return False;
6936 lenlist = len(listcatfiles['ffilelist']);
6937 fnumfiles = int(listcatfiles['fnumfiles']);
6938 lcfi = 0;
6939 lcfx = int(listcatfiles['fnumfiles']);
6940 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6941 lcfx = int(lenlist);
6942 else:
6943 lcfx = int(listcatfiles['fnumfiles']);
6944 returnval = {};
6945 while(lcfi<lcfx):
6946 returnval.update({lcfi: listcatfiles['ffilelist'][lcfi]['fname']});
6947 if(not verbose):
6948 VerbosePrintOut(listcatfiles['ffilelist'][lcfi]['fname']);
6949 if(verbose):
6950 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6951 printfname = listcatfiles['ffilelist'][lcfi]['fname'];
6952 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6953 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " link to " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6954 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6955 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " -> " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6956 fuprint = listcatfiles['ffilelist'][lcfi]['funame'];
6957 if(len(fuprint)<=0):
6958 fuprint = listcatfiles['ffilelist'][lcfi]['fuid'];
6959 fgprint = listcatfiles['ffilelist'][lcfi]['fgname'];
6960 if(len(fgprint)<=0):
6961 fgprint = listcatfiles['ffilelist'][lcfi]['fgid'];
6962 VerbosePrintOut(PrintPermissionString(listcatfiles['ffilelist'][lcfi]['fmode'], listcatfiles['ffilelist'][lcfi]['ftype']) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(listcatfiles['ffilelist'][lcfi]['fsize']).rjust(15) + " " + datetime.datetime.utcfromtimestamp(listcatfiles['ffilelist'][lcfi]['fmtime']).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6963 lcfi = lcfi + 1;
6964 if(returnfp):
6965 return listcatfiles['catfp'];
6966 else:
6967 return True;
6969 create_alias_function("", __file_format_name__, "ListFiles", ArchiveFileListFiles);
6971 def ArchiveFileStringListFiles(catstr, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6972 formatspecs = FormatSpecsListToDict(formatspecs);
6973 catfp = BytesIO(catstr);
6974 listcatfiles = ArchiveFileListFiles(catstr, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6975 return listcatfiles;
6977 create_alias_function("", __file_format_name__, "StringListFiles", ArchiveFileStringListFiles);
6979 def TarFileListFiles(infile, verbose=False, returnfp=False):
6980 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6981 if(infile=="-"):
6982 infile = BytesIO();
6983 if(hasattr(sys.stdin, "buffer")):
6984 shutil.copyfileobj(sys.stdin.buffer, infile);
6985 else:
6986 shutil.copyfileobj(sys.stdin, infile);
6987 infile.seek(0, 0);
6988 if(not infile):
6989 return False;
6990 infile.seek(0, 0);
6991 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
6992 infile = download_file_from_internet_file(infile);
6993 infile.seek(0, 0);
6994 if(not infile):
6995 return False;
6996 infile.seek(0, 0);
6997 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
6998 return False;
6999 elif(os.path.exists(infile) and os.path.isfile(infile)):
7000 try:
7001 if(not tarfile.is_tarfile(infile)):
7002 return False;
7003 except AttributeError:
7004 if(not is_tarfile(infile)):
7005 return False;
7006 else:
7007 return False;
7008 try:
7009 if(hasattr(infile, "read") or hasattr(infile, "write")):
7010 tarfp = tarfile.open(fileobj=infile, mode="r");
7011 else:
7012 tarfp = tarfile.open(infile, "r");
7013 except FileNotFoundError:
7014 return False;
7015 lcfi = 0
7016 returnval = {};
7017 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
7018 returnval.update({lcfi: member.name});
7019 fpremode = member.mode;
7020 ffullmode = member.mode;
7021 flinkcount = 0;
7022 ftype = 0;
7023 if(member.isreg()):
7024 ffullmode = member.mode + stat.S_IFREG;
7025 ftype = 0;
7026 elif(member.isdev()):
7027 ffullmode = member.mode;
7028 ftype = 7;
7029 elif(member.islnk()):
7030 ffullmode = member.mode + stat.S_IFREG;
7031 ftype = 1;
7032 elif(member.issym()):
7033 ffullmode = member.mode + stat.S_IFLNK;
7034 ftype = 2;
7035 elif(member.ischr()):
7036 ffullmode = member.mode + stat.S_IFCHR;
7037 ftype = 3;
7038 elif(member.isblk()):
7039 ffullmode = member.mode + stat.S_IFBLK;
7040 ftype = 4;
7041 elif(member.isdir()):
7042 ffullmode = member.mode + stat.S_IFDIR;
7043 ftype = 5;
7044 elif(member.isfifo()):
7045 ffullmode = member.mode + stat.S_IFIFO;
7046 ftype = 6;
7047 elif(member.issparse()):
7048 ffullmode = member.mode;
7049 ftype = 12;
7050 if(not verbose):
7051 VerbosePrintOut(member.name);
7052 elif(verbose):
7053 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7054 printfname = member.name;
7055 if(member.islnk()):
7056 printfname = member.name + " link to " + member.linkname;
7057 elif(member.issym()):
7058 printfname = member.name + " -> " + member.linkname;
7059 fuprint = member.uname;
7060 if(len(fuprint)<=0):
7061 fuprint = member.uid;
7062 fgprint = member.gname;
7063 if(len(fgprint)<=0):
7064 fgprint = member.gid;
7065 VerbosePrintOut(PrintPermissionString(ffullmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(member.mtime).strftime('%Y-%m-%d %H:%M') + " " + printfname));
7066 lcfi = lcfi + 1;
7067 if(returnfp):
7068 return listcatfiles['catfp'];
7069 else:
7070 return True;
7072 def ZipFileListFiles(infile, verbose=False, returnfp=False):
7073 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7074 if(infile=="-"):
7075 infile = BytesIO();
7076 if(hasattr(sys.stdin, "buffer")):
7077 shutil.copyfileobj(sys.stdin.buffer, infile);
7078 else:
7079 shutil.copyfileobj(sys.stdin, infile);
7080 infile.seek(0, 0);
7081 if(not infile):
7082 return False;
7083 infile.seek(0, 0);
7084 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
7085 infile = download_file_from_internet_file(infile);
7086 infile.seek(0, 0);
7087 if(not infile):
7088 return False;
7089 infile.seek(0, 0);
7090 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
7091 return False;
7092 else:
7093 return False;
7094 if(not zipfile.is_zipfile(infile)):
7095 return False;
7096 try:
7097 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
7098 except FileNotFoundError:
7099 return False;
7100 lcfi = 0;
7101 returnval = {};
7102 ziptest = zipfp.testzip();
7103 if(ziptest):
7104 VerbosePrintOut("Bad file found!");
7105 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
7106 if(zipinfo.create_system==0 or zipinfo.create_system==10):
7107 fwinattributes = int(zipinfo.external_attr);
7108 if(not member.is_dir()):
7109 fmode = int(stat.S_IFREG + 438);
7110 fchmode = int(stat.S_IMODE(fmode));
7111 ftypemod = int(stat.S_IFMT(fmode));
7112 elif(member.is_dir()):
7113 fmode = int(stat.S_IFDIR + 511);
7114 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7115 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7116 elif(zipinfo.create_system==3):
7117 fwinattributes =int(0);
7118 fmode = int(zipinfo.external_attr);
7119 fchmode = int(stat.S_IMODE(fmode));
7120 ftypemod = int(stat.S_IFMT(fmode));
7121 else:
7122 fwinattributes = int(0);
7123 if(not member.is_dir()):
7124 fmode = int(stat.S_IFREG + 438);
7125 fchmode = int(stat.S_IMODE(fmode));
7126 ftypemod = int(stat.S_IFMT(fmode));
7127 elif(member.is_dir()):
7128 fmode = int(stat.S_IFDIR + 511);
7129 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7130 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7131 returnval.update({lcfi: member.filename});
7132 if(not verbose):
7133 VerbosePrintOut(member.filename);
7134 if(verbose):
7135 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7136 permissionstr = "";
7137 for fmodval in str(oct(fmode))[-3:]:
7138 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
7139 if(not member.is_dir()):
7140 ftype = 0;
7141 permissionstr = "-" + permissionstr;
7142 elif(member.is_dir()):
7143 ftype = 5;
7144 permissionstr = "d" + permissionstr;
7145 printfname = member.filename;
7146 try:
7147 fuid = int(os.getuid());
7148 except AttributeError:
7149 fuid = int(0);
7150 except KeyError:
7151 fuid = int(0);
7152 try:
7153 fgid = int(os.getgid());
7154 except AttributeError:
7155 fgid = int(0);
7156 except KeyError:
7157 fgid = int(0);
7158 try:
7159 import pwd;
7160 try:
7161 userinfo = pwd.getpwuid(os.getuid());
7162 funame = userinfo.pw_name;
7163 except KeyError:
7164 funame = "";
7165 except AttributeError:
7166 funame = "";
7167 except ImportError:
7168 funame = "";
7169 fgname = "";
7170 try:
7171 import grp;
7172 try:
7173 groupinfo = grp.getgrgid(os.getgid());
7174 fgname = groupinfo.gr_name;
7175 except KeyError:
7176 fgname = "";
7177 except AttributeError:
7178 fgname = "";
7179 except ImportError:
7180 fgname = "";
7181 fuprint = funame;
7182 if(len(fuprint)<=0):
7183 fuprint = str(fuid);
7184 fgprint = fgname;
7185 if(len(fgprint)<=0):
7186 fgprint = str(fgid);
7187 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(int(time.mktime(member.date_time + (0, 0, -1)))).strftime('%Y-%m-%d %H:%M') + " " + printfname));
7188 lcfi = lcfi + 1;
7189 if(returnfp):
7190 return listcatfiles['catfp'];
7191 else:
7192 return True;
7194 if(not rarfile_support):
7195 def RarFileListFiles(infile, verbose=False, returnfp=False):
7196 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7197 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7198 return False;
7200 if(rarfile_support):
7201 def RarFileListFiles(infile, verbose=False, returnfp=False):
7202 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7203 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7204 return False;
7205 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
7206 return False;
7207 lcfi = 0;
7208 returnval = {};
7209 rarfp = rarfile.RarFile(infile, "r");
7210 rartest = rarfp.testrar();
7211 if(rartest):
7212 VerbosePrintOut("Bad file found!");
7213 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
7214 is_unix = False;
7215 is_windows = False;
7216 if(member.host_os==rarfile.RAR_OS_UNIX):
7217 is_windows = False;
7218 try:
7219 member.external_attr
7220 is_unix = True;
7221 except AttributeError:
7222 is_unix = False;
7223 elif(member.host_os==rarfile.RAR_OS_WIN32):
7224 is_unix = False;
7225 try:
7226 member.external_attr
7227 is_windows = True;
7228 except AttributeError:
7229 is_windows = False;
7230 else:
7231 is_unix = False;
7232 is_windows = False;
7233 if(is_unix and member.external_attr !=0):
7234 fpremode = int(member.external_attr);
7235 elif(member.is_file()):
7236 fpremode = int(stat.S_IFREG + 438);
7237 elif(member.is_symlink()):
7238 fpremode = int(stat.S_IFLNK + 438);
7239 elif(member.is_dir()):
7240 fpremode = int(stat.S_IFDIR + 511);
7241 if(is_windows and member.external_attr !=0):
7242 fwinattributes = int(member.external_attr);
7243 else:
7244 fwinattributes = int(0);
7245 if(is_unix and member.external_attr !=0):
7246 fmode = int(member.external_attr);
7247 fchmode = int(stat.S_IMODE(member.external_attr));
7248 ftypemod = int(stat.S_IFMT(member.external_attr));
7249 elif(member.is_file()):
7250 fmode = int(stat.S_IFREG + 438);
7251 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
7252 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
7253 elif(member.is_symlink()):
7254 fmode = int(stat.S_IFLNK + 438);
7255 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
7256 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
7257 elif(member.is_dir()):
7258 fmode = int(stat.S_IFDIR + 511);
7259 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7260 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7261 returnval.update({lcfi: member.filename});
7262 if(not verbose):
7263 VerbosePrintOut(member.filename);
7264 if(verbose):
7265 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7266 permissionstr = "";
7267 for fmodval in str(oct(fmode))[-3:]:
7268 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
7269 if(member.is_file()):
7270 ftype = 0;
7271 permissionstr = "-" + permissionstr;
7272 printfname = member.filename;
7273 elif(member.is_symlink()):
7274 ftype = 2;
7275 permissionstr = "l" + permissionstr;
7276 printfname = member.name + " -> " + member.read().decode("UTF-8");
7277 elif(member.is_dir()):
7278 ftype = 5;
7279 permissionstr = "d" + permissionstr;
7280 printfname = member.filename;
7281 try:
7282 fuid = int(os.getuid());
7283 except AttributeError:
7284 fuid = int(0);
7285 except KeyError:
7286 fuid = int(0);
7287 try:
7288 fgid = int(os.getgid());
7289 except AttributeError:
7290 fgid = int(0);
7291 except KeyError:
7292 fgid = int(0);
7293 try:
7294 import pwd;
7295 try:
7296 userinfo = pwd.getpwuid(os.getuid());
7297 funame = userinfo.pw_name;
7298 except KeyError:
7299 funame = "";
7300 except AttributeError:
7301 funame = "";
7302 except ImportError:
7303 funame = "";
7304 fgname = "";
7305 try:
7306 import grp;
7307 try:
7308 groupinfo = grp.getgrgid(os.getgid());
7309 fgname = groupinfo.gr_name;
7310 except KeyError:
7311 fgname = "";
7312 except AttributeError:
7313 fgname = "";
7314 except ImportError:
7315 fgname = "";
7316 fuprint = funame;
7317 if(len(fuprint)<=0):
7318 fuprint = str(fuid);
7319 fgprint = fgname;
7320 if(len(fgprint)<=0):
7321 fgprint = str(fgid);
7322 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + member.mtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
7323 lcfi = lcfi + 1;
7324 if(returnfp):
7325 return listcatfiles['catfp'];
7326 else:
7327 return True;
7329 if(not py7zr_support):
7330 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
7331 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7332 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7333 return False;
7335 if(py7zr_support):
7336 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
7337 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7338 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7339 return False;
7340 lcfi = 0;
7341 returnval = {};
7342 szpfp = py7zr.SevenZipFile(infile, mode="r");
7343 file_content = szpfp.readall();
7344 #sztest = szpfp.testzip();
7345 sztestalt = szpfp.test();
7346 if(sztestalt):
7347 VerbosePrintOut("Bad file found!");
7348 for member in sorted(szpfp.list(), key=lambda x: x.filename):
7349 if(re.findall("^[.|/]", member.filename)):
7350 fname = member.filename;
7351 else:
7352 fname = "./"+member.filename;
7353 if(not member.is_directory):
7354 fpremode = int(stat.S_IFREG + 438);
7355 elif(member.is_directory):
7356 fpremode = int(stat.S_IFDIR + 511);
7357 fwinattributes = int(0);
7358 if(member.is_directory):
7359 fmode = int(stat.S_IFDIR + 511);
7360 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7361 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7362 else:
7363 fmode = int(stat.S_IFLNK + 438);
7364 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
7365 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
7366 returnval.update({lcfi: member.filename});
7367 if(not verbose):
7368 VerbosePrintOut(member.filename);
7369 if(verbose):
7370 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7371 permissionstr = "";
7372 for fmodval in str(oct(fmode))[-3:]:
7373 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
7374 fsize = int("0");
7375 if(not member.is_directory):
7376 ftype = 0;
7377 permissionstr = "-" + permissionstr;
7378 printfname = member.filename;
7379 elif(member.is_directory):
7380 ftype = 5;
7381 permissionstr = "d" + permissionstr;
7382 printfname = member.filename;
7383 if(ftype==0):
7384 fsize = len(file_content[member.filename].read());
7385 file_content[member.filename].close();
7386 try:
7387 fuid = int(os.getuid());
7388 except AttributeError:
7389 fuid = int(0);
7390 except KeyError:
7391 fuid = int(0);
7392 try:
7393 fgid = int(os.getgid());
7394 except AttributeError:
7395 fgid = int(0);
7396 except KeyError:
7397 fgid = int(0);
7398 try:
7399 import pwd;
7400 try:
7401 userinfo = pwd.getpwuid(os.getuid());
7402 funame = userinfo.pw_name;
7403 except KeyError:
7404 funame = "";
7405 except AttributeError:
7406 funame = "";
7407 except ImportError:
7408 funame = "";
7409 fgname = "";
7410 try:
7411 import grp;
7412 try:
7413 groupinfo = grp.getgrgid(os.getgid());
7414 fgname = groupinfo.gr_name;
7415 except KeyError:
7416 fgname = "";
7417 except AttributeError:
7418 fgname = "";
7419 except ImportError:
7420 fgname = "";
7421 fuprint = funame;
7422 if(len(fuprint)<=0):
7423 fuprint = str(fuid);
7424 fgprint = fgname;
7425 if(len(fgprint)<=0):
7426 fgprint = str(fgid);
7427 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(fsize).rjust(15) + " " + member.creationtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
7428 lcfi = lcfi + 1;
7429 if(returnfp):
7430 return listcatfiles['catfp'];
7431 else:
7432 return True;
7434 def InFileListFiles(infile, verbose=False, formatspecs=__file_format_dict__, returnfp=False):
7435 formatspecs = FormatSpecsListToDict(formatspecs);
7436 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7437 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
7438 if(checkcompressfile=="tarfile"):
7439 return TarFileListFiles(infile, verbose, returnfp);
7440 elif(checkcompressfile=="zipfile"):
7441 return ZipFileListFiles(infile, verbose, returnfp);
7442 elif(checkcompressfile=="catfile"):
7443 return ArchiveFileListFiles(infile, 0, 0, False, formatspecs, verbose, returnfp);
7444 elif(rarfile_support and checkcompressfile=="rarfile"):
7445 return RarFileListFiles(infile, verbose, returnfp);
7446 elif(py7zr_support and checkcompressfile=="7zipfile"):
7447 return SevenZipFileListFiles(infile, verbose, returnfp);
7448 else:
7449 return False;
7450 return False;
7452 def ListDirListFiles(infiles, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7453 formatspecs = FormatSpecsListToDict(formatspecs);
7454 outarray = BytesIO();
7455 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, False, True);
7456 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
7457 return listcatfiles;
7459 def ListDirListFilesAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7460 formatspecs = FormatSpecsListToDict(formatspecs);
7461 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, listonly, checksumtype, formatspecs, verbose);
7462 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
7463 return listcatfiles;
7465 def PackArchiveFileFromListDirAlt(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7466 formatspecs = FormatSpecsListToDict(formatspecs);
7467 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, False, checksumtype, extradata, formatspecs, False);
7468 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, followlink, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
7469 return listcatfiles;
7471 create_alias_function("Pack", __file_format_name__, "FromListDirAlt", PackArchiveFileFromListDirAlt);
7473 def PackArchiveFileFromTarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7474 formatspecs = FormatSpecsListToDict(formatspecs);
7475 outarray = TarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7476 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7477 return listcatfiles;
7479 create_alias_function("Pack", __file_format_name__, "FromTarFileAlt", PackArchiveFileFromTarFileAlt);
7481 def PackArchiveFileFromZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7482 formatspecs = FormatSpecsListToDict(formatspecs);
7483 outarray = ZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7484 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7485 return listcatfiles;
7487 create_alias_function("Pack", __file_format_name__, "FromZipFileAlt", PackArchiveFileFromZipFileAlt);
7489 if(not rarfile_support):
7490 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7491 return False;
7493 if(rarfile_support):
7494 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7495 formatspecs = FormatSpecsListToDict(formatspecs);
7496 outarray = RarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7497 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7498 return listcatfiles;
7500 create_alias_function("Pack", __file_format_name__, "FromRarFileAlt", PackArchiveFileFromRarFileAlt);
7502 if(not py7zr_support):
7503 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7504 return False;
7506 if(py7zr_support):
7507 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7508 formatspecs = FormatSpecsListToDict(formatspecs);
7509 outarray = SevenZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7510 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7511 return listcatfiles;
7513 create_alias_function("Pack", __file_format_name__, "FromSevenZipFileAlt", PackArchiveFileFromSevenZipFileAlt);
7515 def download_file_from_ftp_file(url):
7516 urlparts = urlparse(url);
7517 file_name = os.path.basename(urlparts.path);
7518 file_dir = os.path.dirname(urlparts.path);
7519 if(urlparts.username is not None):
7520 ftp_username = urlparts.username;
7521 else:
7522 ftp_username = "anonymous";
7523 if(urlparts.password is not None):
7524 ftp_password = urlparts.password;
7525 elif(urlparts.password is None and urlparts.username=="anonymous"):
7526 ftp_password = "anonymous";
7527 else:
7528 ftp_password = "";
7529 if(urlparts.scheme=="ftp"):
7530 ftp = FTP();
7531 elif(urlparts.scheme=="ftps" and ftpssl):
7532 ftp = FTP_TLS();
7533 else:
7534 return False;
7535 if(urlparts.scheme=="sftp"):
7536 if(__use_pysftp__):
7537 return download_file_from_pysftp_file(url);
7538 else:
7539 return download_file_from_sftp_file(url);
7540 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7541 return download_file_from_http_file(url);
7542 ftp_port = urlparts.port;
7543 if(urlparts.port is None):
7544 ftp_port = 21;
7545 try:
7546 ftp.connect(urlparts.hostname, ftp_port);
7547 except socket.gaierror:
7548 log.info("Error With URL "+url);
7549 return False;
7550 except socket.timeout:
7551 log.info("Error With URL "+url);
7552 return False;
7553 ftp.login(urlparts.username, urlparts.password);
7554 if(urlparts.scheme=="ftps"):
7555 ftp.prot_p();
7556 ftpfile = BytesIO();
7557 ftp.retrbinary("RETR "+urlparts.path, ftpfile.write);
7558 #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
7559 ftp.close();
7560 ftpfile.seek(0, 0);
7561 return ftpfile;
7563 def download_file_from_ftp_string(url):
7564 ftpfile = download_file_from_ftp_file(url);
7565 return ftpfile.read();
7567 def upload_file_to_ftp_file(ftpfile, url):
7568 urlparts = urlparse(url);
7569 file_name = os.path.basename(urlparts.path);
7570 file_dir = os.path.dirname(urlparts.path);
7571 if(urlparts.username is not None):
7572 ftp_username = urlparts.username;
7573 else:
7574 ftp_username = "anonymous";
7575 if(urlparts.password is not None):
7576 ftp_password = urlparts.password;
7577 elif(urlparts.password is None and urlparts.username=="anonymous"):
7578 ftp_password = "anonymous";
7579 else:
7580 ftp_password = "";
7581 if(urlparts.scheme=="ftp"):
7582 ftp = FTP();
7583 elif(urlparts.scheme=="ftps" and ftpssl):
7584 ftp = FTP_TLS();
7585 else:
7586 return False;
7587 if(urlparts.scheme=="sftp"):
7588 if(__use_pysftp__):
7589 return upload_file_to_pysftp_file(url);
7590 else:
7591 return upload_file_to_sftp_file(url);
7592 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7593 return False;
7594 ftp_port = urlparts.port;
7595 if(urlparts.port is None):
7596 ftp_port = 21;
7597 try:
7598 ftp.connect(urlparts.hostname, ftp_port);
7599 except socket.gaierror:
7600 log.info("Error With URL "+url);
7601 return False;
7602 except socket.timeout:
7603 log.info("Error With URL "+url);
7604 return False;
7605 ftp.login(urlparts.username, urlparts.password);
7606 if(urlparts.scheme=="ftps"):
7607 ftp.prot_p();
7608 ftp.storbinary("STOR "+urlparts.path, ftpfile);
7609 ftp.close();
7610 ftpfile.seek(0, 0);
7611 return ftpfile;
7613 def upload_file_to_ftp_string(ftpstring, url):
7614 ftpfileo = BytesIO(ftpstring);
7615 ftpfile = upload_file_to_ftp_file(ftpfileo, url);
7616 ftpfileo.close();
7617 return ftpfile;
7619 def download_file_from_http_file(url, headers=geturls_headers_pycatfile_python_alt):
7620 # Parse the URL to extract username and password if present
7621 urlparts = urlparse(url);
7622 username = urlparts.username;
7623 password = urlparts.password;
7624 # Rebuild the URL without the username and password
7625 netloc = urlparts.hostname;
7626 if(urlparts.scheme=="sftp"):
7627 if(__use_pysftp__):
7628 return download_file_from_pysftp_file(url);
7629 else:
7630 return download_file_from_sftp_file(url);
7631 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7632 return download_file_from_ftp_file(url);
7633 if urlparts.port:
7634 netloc += ':' + str(urlparts.port);
7635 rebuilt_url = urlunparse((urlparts.scheme, netloc, urlparts.path, urlparts.params, urlparts.query, urlparts.fragment));
7636 # Create a temporary file object
7637 httpfile = BytesIO();
7638 if haverequests:
7639 # Use the requests library if available
7640 if username and password:
7641 response = requests.get(rebuilt_url, headers=headers, auth=(username, password), stream=True);
7642 else:
7643 response = requests.get(rebuilt_url, headers=headers, stream=True);
7644 response.raw.decode_content = True
7645 shutil.copyfileobj(response.raw, httpfile);
7646 else:
7647 # Build a Request object for urllib
7648 request = Request(rebuilt_url, headers=headers);
7649 # Create an opener object for handling URLs
7650 if username and password:
7651 # Create a password manager
7652 password_mgr = HTTPPasswordMgrWithDefaultRealm();
7653 # Add the username and password
7654 password_mgr.add_password(None, rebuilt_url, username, password);
7655 # Create an authentication handler using the password manager
7656 auth_handler = HTTPBasicAuthHandler(password_mgr);
7657 # Build the opener with the authentication handler
7658 opener = build_opener(auth_handler);
7659 else:
7660 opener = build_opener();
7661 with opener.open(request) as response:
7662 shutil.copyfileobj(response, httpfile);
7663 # Reset file pointer to the start
7664 httpfile.seek(0, 0);
7665 # Return the temporary file object
7666 return httpfile;
7668 def download_file_from_http_string(url, headers=geturls_headers_pycatfile_python_alt):
7669 httpfile = download_file_from_http_file(url, headers);
7670 return ftpfile.read();
7672 if(haveparamiko):
7673 def download_file_from_sftp_file(url):
7674 urlparts = urlparse(url);
7675 file_name = os.path.basename(urlparts.path);
7676 file_dir = os.path.dirname(urlparts.path);
7677 sftp_port = urlparts.port;
7678 if(urlparts.port is None):
7679 sftp_port = 22;
7680 else:
7681 sftp_port = urlparts.port;
7682 if(urlparts.username is not None):
7683 sftp_username = urlparts.username;
7684 else:
7685 sftp_username = "anonymous";
7686 if(urlparts.password is not None):
7687 sftp_password = urlparts.password;
7688 elif(urlparts.password is None and urlparts.username=="anonymous"):
7689 sftp_password = "anonymous";
7690 else:
7691 sftp_password = "";
7692 if(urlparts.scheme=="ftp"):
7693 return download_file_from_ftp_file(url);
7694 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7695 return download_file_from_http_file(url);
7696 if(urlparts.scheme!="sftp"):
7697 return False;
7698 ssh = paramiko.SSHClient();
7699 ssh.load_system_host_keys();
7700 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
7701 try:
7702 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7703 except paramiko.ssh_exception.SSHException:
7704 return False;
7705 except socket.gaierror:
7706 log.info("Error With URL "+url);
7707 return False;
7708 except socket.timeout:
7709 log.info("Error With URL "+url);
7710 return False;
7711 sftp = ssh.open_sftp();
7712 sftpfile = BytesIO();
7713 sftp.getfo(urlparts.path, sftpfile);
7714 sftp.close();
7715 ssh.close();
7716 sftpfile.seek(0, 0);
7717 return sftpfile;
7718 else:
7719 def download_file_from_sftp_file(url):
7720 return False;
7722 if(haveparamiko):
7723 def download_file_from_sftp_string(url):
7724 sftpfile = download_file_from_sftp_file(url);
7725 return sftpfile.read();
7726 else:
7727 def download_file_from_ftp_string(url):
7728 return False;
7730 if(haveparamiko):
7731 def upload_file_to_sftp_file(sftpfile, url):
7732 urlparts = urlparse(url);
7733 file_name = os.path.basename(urlparts.path);
7734 file_dir = os.path.dirname(urlparts.path);
7735 sftp_port = urlparts.port;
7736 if(urlparts.port is None):
7737 sftp_port = 22;
7738 else:
7739 sftp_port = urlparts.port;
7740 if(urlparts.username is not None):
7741 sftp_username = urlparts.username;
7742 else:
7743 sftp_username = "anonymous";
7744 if(urlparts.password is not None):
7745 sftp_password = urlparts.password;
7746 elif(urlparts.password is None and urlparts.username=="anonymous"):
7747 sftp_password = "anonymous";
7748 else:
7749 sftp_password = "";
7750 if(urlparts.scheme=="ftp"):
7751 return upload_file_to_ftp_file(url);
7752 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7753 return False;
7754 if(urlparts.scheme!="sftp"):
7755 return False;
7756 ssh = paramiko.SSHClient();
7757 ssh.load_system_host_keys();
7758 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
7759 try:
7760 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7761 except paramiko.ssh_exception.SSHException:
7762 return False;
7763 except socket.gaierror:
7764 log.info("Error With URL "+url);
7765 return False;
7766 except socket.timeout:
7767 log.info("Error With URL "+url);
7768 return False;
7769 sftp = ssh.open_sftp();
7770 sftp.putfo(sftpfile, urlparts.path);
7771 sftp.close();
7772 ssh.close();
7773 sftpfile.seek(0, 0);
7774 return sftpfile;
7775 else:
7776 def upload_file_to_sftp_file(sftpfile, url):
7777 return False;
7779 if(haveparamiko):
7780 def upload_file_to_sftp_string(sftpstring, url):
7781 sftpfileo = BytesIO(sftpstring);
7782 sftpfile = upload_file_to_sftp_files(ftpfileo, url);
7783 sftpfileo.close();
7784 return sftpfile;
7785 else:
7786 def upload_file_to_sftp_string(url):
7787 return False;
7789 if(havepysftp):
7790 def download_file_from_pysftp_file(url):
7791 urlparts = urlparse(url);
7792 file_name = os.path.basename(urlparts.path);
7793 file_dir = os.path.dirname(urlparts.path);
7794 sftp_port = urlparts.port;
7795 if(urlparts.port is None):
7796 sftp_port = 22;
7797 else:
7798 sftp_port = urlparts.port;
7799 if(urlparts.username is not None):
7800 sftp_username = urlparts.username;
7801 else:
7802 sftp_username = "anonymous";
7803 if(urlparts.password is not None):
7804 sftp_password = urlparts.password;
7805 elif(urlparts.password is None and urlparts.username=="anonymous"):
7806 sftp_password = "anonymous";
7807 else:
7808 sftp_password = "";
7809 if(urlparts.scheme=="ftp"):
7810 return download_file_from_ftp_file(url);
7811 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7812 return download_file_from_http_file(url);
7813 if(urlparts.scheme!="sftp"):
7814 return False;
7815 try:
7816 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7817 except paramiko.ssh_exception.SSHException:
7818 return False;
7819 except socket.gaierror:
7820 log.info("Error With URL "+url);
7821 return False;
7822 except socket.timeout:
7823 log.info("Error With URL "+url);
7824 return False;
7825 sftp = ssh.open_sftp();
7826 sftpfile = BytesIO();
7827 sftp.getfo(urlparts.path, sftpfile);
7828 sftp.close();
7829 ssh.close();
7830 sftpfile.seek(0, 0);
7831 return sftpfile;
7832 else:
7833 def download_file_from_pysftp_file(url):
7834 return False;
7836 if(havepysftp):
7837 def download_file_from_pysftp_string(url):
7838 sftpfile = download_file_from_pysftp_file(url);
7839 return sftpfile.read();
7840 else:
7841 def download_file_from_ftp_string(url):
7842 return False;
7844 if(havepysftp):
7845 def upload_file_to_pysftp_file(sftpfile, url):
7846 urlparts = urlparse(url);
7847 file_name = os.path.basename(urlparts.path);
7848 file_dir = os.path.dirname(urlparts.path);
7849 sftp_port = urlparts.port;
7850 if(urlparts.port is None):
7851 sftp_port = 22;
7852 else:
7853 sftp_port = urlparts.port;
7854 if(urlparts.username is not None):
7855 sftp_username = urlparts.username;
7856 else:
7857 sftp_username = "anonymous";
7858 if(urlparts.password is not None):
7859 sftp_password = urlparts.password;
7860 elif(urlparts.password is None and urlparts.username=="anonymous"):
7861 sftp_password = "anonymous";
7862 else:
7863 sftp_password = "";
7864 if(urlparts.scheme=="ftp"):
7865 return upload_file_to_ftp_file(url);
7866 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7867 return False;
7868 if(urlparts.scheme!="sftp"):
7869 return False;
7870 try:
7871 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7872 except paramiko.ssh_exception.SSHException:
7873 return False;
7874 except socket.gaierror:
7875 log.info("Error With URL "+url);
7876 return False;
7877 except socket.timeout:
7878 log.info("Error With URL "+url);
7879 return False;
7880 sftp = ssh.open_sftp();
7881 sftp.putfo(sftpfile, urlparts.path);
7882 sftp.close();
7883 ssh.close();
7884 sftpfile.seek(0, 0);
7885 return sftpfile;
7886 else:
7887 def upload_file_to_pysftp_file(sftpfile, url):
7888 return False;
7890 if(havepysftp):
7891 def upload_file_to_pysftp_string(sftpstring, url):
7892 sftpfileo = BytesIO(sftpstring);
7893 sftpfile = upload_file_to_pysftp_files(ftpfileo, url);
7894 sftpfileo.close();
7895 return sftpfile;
7896 else:
7897 def upload_file_to_pysftp_string(url):
7898 return False;
7900 def download_file_from_internet_file(url, headers=geturls_headers_pycatfile_python_alt):
7901 urlparts = urlparse(url);
7902 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7903 return download_file_from_http_file(url, headers);
7904 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7905 return download_file_from_ftp_file(url);
7906 elif(urlparts.scheme=="sftp"):
7907 if(__use_pysftp__ and havepysftp):
7908 return download_file_from_pysftp_file(url);
7909 else:
7910 return download_file_from_sftp_file(url);
7911 else:
7912 return False;
7913 return False;
7915 def download_file_from_internet_uncompress_file(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_dict__):
7916 formatspecs = FormatSpecsListToDict(formatspecs);
7917 fp = download_file_from_internet_file(url);
7918 fp = UncompressArchiveFile(fp, formatspecs);
7919 fp.seek(0, 0);
7920 if(not fp):
7921 return False;
7922 return fp;
7924 def download_file_from_internet_string(url, headers=geturls_headers_pycatfile_python_alt):
7925 urlparts = urlparse(url);
7926 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7927 return download_file_from_http_string(url, headers);
7928 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7929 return download_file_from_ftp_string(url);
7930 elif(urlparts.scheme=="sftp"):
7931 if(__use_pysftp__ and havepysftp):
7932 return download_file_from_pysftp_string(url);
7933 else:
7934 return download_file_from_sftp_string(url);
7935 else:
7936 return False;
7937 return False;
7939 def download_file_from_internet_uncompress_string(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_dict__):
7940 formatspecs = FormatSpecsListToDict(formatspecs);
7941 fp = download_file_from_internet_string(url);
7942 fp = UncompressArchiveFile(fp, formatspecs);
7943 fp.seek(0, 0);
7944 if(not fp):
7945 return False;
7946 return fp;
7948 def upload_file_to_internet_file(ifp, url):
7949 urlparts = urlparse(url);
7950 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7951 return False;
7952 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7953 return upload_file_to_ftp_file(ifp, url);
7954 elif(urlparts.scheme=="sftp"):
7955 if(__use_pysftp__ and havepysftp):
7956 return upload_file_to_pysftp_file(ifp, url);
7957 else:
7958 return upload_file_to_sftp_file(ifp, url);
7959 else:
7960 return False;
7961 return False;
7963 def upload_file_to_internet_compress_file(ifp, url, formatspecs=__file_format_dict__):
7964 formatspecs = FormatSpecsListToDict(formatspecs);
7965 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
7966 if(not catfileout):
7967 return False;
7968 catfp.seek(0, 0);
7969 upload_file_to_internet_file(catfp, outfile);
7970 return True;
7972 def upload_file_to_internet_string(ifp, url):
7973 urlparts = urlparse(url);
7974 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7975 return False;
7976 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7977 return upload_file_to_ftp_string(ifp, url);
7978 elif(urlparts.scheme=="sftp"):
7979 if(__use_pysftp__ and havepysftp):
7980 return upload_file_to_pysftp_string(ifp, url);
7981 else:
7982 return upload_file_to_sftp_string(ifp, url);
7983 else:
7984 return False;
7985 return False;
7987 def upload_file_to_internet_compress_string(ifp, url, formatspecs=__file_format_dict__):
7988 formatspecs = FormatSpecsListToDict(formatspecs);
7989 catfp = CompressArchiveFile(BytesIO(ifp), compression, compressionlevel, formatspecs);
7990 if(not catfileout):
7991 return False;
7992 catfp.seek(0, 0);
7993 upload_file_to_internet_file(catfp, outfile);
7994 return True;
7996 try:
7997 if(hasattr(shutil, "register_archive_format")):
7998 # Register the packing format
7999 shutil.register_archive_format(__file_format_name__, PackArchiveFileFunc, description='Pack concatenated files');
8000 except shutil.RegistryError:
8001 pass;
8003 try:
8004 if(hasattr(shutil, "register_unpack_format")):
8005 # Register the unpacking format
8006 shutil.register_unpack_format(__file_format_name__, archivefile_extensions, UnPackArchiveFileFunc, description='UnPack concatenated files');
8007 except shutil.RegistryError:
8008 pass;