Add files via upload
[PyCatFile.git] / pycatfile.py
blobe0a210109ac8090020891311f1bd86067e9c682d
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
4 '''
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the Revised BSD License.
8 This program is distributed in the hope that it will be useful,
9 but WITHOUT ANY WARRANTY; without even the implied warranty of
10 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 Revised BSD License for more details.
13 Copyright 2018-2024 Cool Dude 2k - http://idb.berlios.de/
14 Copyright 2018-2024 Game Maker 2k - http://intdb.sourceforge.net/
15 Copyright 2018-2024 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
17 $FileInfo: pycatfile.py - Last Update: 5/12/2024 Ver. 0.11.0 RC 1 - Author: cooldude2k $
18 '''
20 from __future__ import absolute_import, division, print_function, unicode_literals;
21 import io, os, re, sys, time, stat, zlib, base64, shutil, socket, hashlib, datetime, logging, binascii, tempfile, zipfile, platform;
22 ftpssl = True;
23 try:
24 ftpssl = True;
25 from ftplib import FTP, FTP_TLS;
26 except ImportError:
27 ftpssl = False;
28 from ftplib import FTP;
29 if(sys.version[0]=="2"):
30 from urlparse import urlparse, urlunparse;
31 elif(sys.version[0]>="3"):
32 from urllib.parse import urlunparse;
33 from urllib.parse import urlparse;
35 if os.name == 'nt': # Only modify if on Windows
36 if sys.version[0] == "2":
37 import codecs;
38 sys.stdout = codecs.getwriter('utf-8')(sys.stdout);
39 sys.stderr = codecs.getwriter('utf-8')(sys.stderr);
40 else:
41 import io;
42 sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace', line_buffering=True);
43 sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', errors='replace', line_buffering=True);
45 hashlib_guaranteed = False;
46 os.environ["PYTHONIOENCODING"] = "UTF-8";
47 os.environ["LC_CTYPE"] = "UTF-8";
48 try:
49 reload(sys);
50 try:
51 sys.setdefaultencoding('UTF-8');
52 except NameError:
53 pass;
54 except AttributeError:
55 pass;
56 except NameError:
57 pass;
58 except AttributeError:
59 pass;
61 try:
62 from zlib import crc32;
63 except ImportError:
64 from binascii import crc32;
66 if(sys.version_info[0]==2):
67 FileNotFoundError = IOError;
69 rarfile_support = False;
70 try:
71 import rarfile;
72 rarfile_support = True;
73 except ImportError:
74 rarfile_support = False;
76 py7zr_support = False;
77 try:
78 import py7zr;
79 py7zr_support = True;
80 except ImportError:
81 py7zr_support = False;
83 try:
84 from xtarfile import is_tarfile;
85 except ImportError:
86 try:
87 from safetar import is_tarfile;
88 except ImportError:
89 from tarfile import is_tarfile;
91 try:
92 import xtarfile as tarfile;
93 except ImportError:
94 try:
95 import safetar as tarfile;
96 except ImportError:
97 import tarfile;
99 haveparamiko = False;
100 try:
101 import paramiko;
102 haveparamiko = True;
103 except ImportError:
104 haveparamiko = False;
106 havepysftp = False;
107 try:
108 import pysftp;
109 havepysftp = True;
110 except ImportError:
111 havepysftp = False;
113 haverequests = False;
114 try:
115 import requests;
116 haverequests = True;
117 except ImportError:
118 haverequests = False;
120 if(haverequests):
121 import urllib3;
122 logging.getLogger("urllib3").setLevel(logging.WARNING);
124 try:
125 # Python 3 imports
126 from urllib.request import Request, build_opener, HTTPBasicAuthHandler;
127 from urllib.parse import urlparse;
128 except ImportError:
129 # Python 2 imports
130 from urllib2 import Request, build_opener, HTTPBasicAuthHandler;
131 from urlparse import urlparse;
133 if(sys.version[0]=="2"):
134 try:
135 from io import StringIO, BytesIO;
136 except ImportError:
137 try:
138 from cStringIO import StringIO;
139 from cStringIO import StringIO as BytesIO;
140 except ImportError:
141 from StringIO import StringIO;
142 from StringIO import StringIO as BytesIO;
143 elif(sys.version[0]>="3"):
144 from io import StringIO, BytesIO;
145 else:
146 teststringio = 0;
147 if(teststringio<=0):
148 try:
149 from cStringIO import StringIO as BytesIO;
150 teststringio = 1;
151 except ImportError:
152 teststringio = 0;
153 if(teststringio<=0):
154 try:
155 from StringIO import StringIO as BytesIO;
156 teststringio = 2;
157 except ImportError:
158 teststringio = 0;
159 if(teststringio<=0):
160 try:
161 from io import BytesIO;
162 teststringio = 3;
163 except ImportError:
164 teststringio = 0;
166 __use_pysftp__ = False;
167 if(not havepysftp):
168 __use_pysftp__ = False;
169 __file_format_name__ = "CatFile";
170 __program_name__ = "Py"+__file_format_name__;
171 __file_format_lower__ = __file_format_name__.lower();
172 __file_format_magic__ = __file_format_name__;
173 __file_format_len__ = len(__file_format_magic__);
174 __file_format_hex__ = binascii.hexlify(__file_format_magic__.encode("UTF-8")).decode("UTF-8");
175 __file_format_delimiter__ = "\x00";
176 __file_format_ver__ = "001";
177 __use_new_style__ = True;
178 __use_advanced_list__ = True;
179 __use_alt_inode__ = False;
180 __file_format_list__ = [__file_format_name__, __file_format_magic__, __file_format_lower__, __file_format_len__, __file_format_hex__, __file_format_delimiter__, __file_format_ver__, __use_new_style__, __use_advanced_list__, __use_alt_inode__];
181 __file_format_dict__ = {'format_name': __file_format_name__, 'format_magic': __file_format_magic__, 'format_lower': __file_format_lower__, 'format_len': __file_format_len__, 'format_hex': __file_format_hex__, 'format_delimiter': __file_format_delimiter__, 'format_ver': __file_format_ver__, 'new_style': __use_new_style__, 'use_advanced_list': __use_advanced_list__, 'use_alt_inode': __use_alt_inode__};
182 __project__ = __program_name__;
183 __project_url__ = "https://github.com/GameMaker2k/PyCatFile";
184 __version_info__ = (0, 11, 0, "RC 1", 1);
185 __version_date_info__ = (2024, 5, 12, "RC 1", 1);
186 __version_date__ = str(__version_date_info__[0]) + "." + str(__version_date_info__[1]).zfill(2) + "." + str(__version_date_info__[2]).zfill(2);
187 __revision__ = __version_info__[3];
188 __revision_id__ = "$Id$";
189 if(__version_info__[4] is not None):
190 __version_date_plusrc__ = __version_date__ + "-" + str(__version_date_info__[4]);
191 if(__version_info__[4] is None):
192 __version_date_plusrc__ = __version_date__;
193 if(__version_info__[3] is not None):
194 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]) + " " + str(__version_info__[3]);
195 if(__version_info__[3] is None):
196 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]);
198 PyBitness = platform.architecture();
199 if(PyBitness=="32bit" or PyBitness=="32"):
200 PyBitness = "32";
201 elif(PyBitness=="64bit" or PyBitness=="64"):
202 PyBitness = "64";
203 else:
204 PyBitness = "32";
206 geturls_ua_pycatfile_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
207 if(platform.python_implementation()!=""):
208 py_implementation = platform.python_implementation();
209 if(platform.python_implementation()==""):
210 py_implementation = "Python";
211 geturls_ua_pycatfile_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=py_implementation, pyver=platform.python_version(), proname=__project__, prover=__version__);
212 geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
213 geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
214 geturls_headers_pycatfile_python = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
215 geturls_headers_pycatfile_python_alt = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python_alt, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
216 geturls_headers_googlebot_google = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
217 geturls_headers_googlebot_google_old = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
219 compressionlist = ['auto', 'gzip', 'bzip2', 'zstd', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
220 compressionlistalt = ['gzip', 'bzip2', 'zstd', 'lz4', 'lzo', 'lzma', 'xz'];
221 outextlist = ['gz', 'bz2', 'zst', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
222 outextlistwd = ['.gz', '.bz2', '.zst', '.lz4', '.lzo', '.lzop', '.lzma', '.xz'];
224 tarfile_mimetype = "application/tar";
225 tarfile_tar_mimetype = tarfile_mimetype;
226 zipfile_mimetype = "application/zip";
227 zipfile_zip_mimetype = zipfile_mimetype;
228 rarfile_mimetype = "application/rar";
229 rarfile_rar_mimetype = rarfile_mimetype;
230 archivefile_mimetype = "application/x-"+__file_format_dict__['format_lower']+"";
231 archivefile_cat_mimetype = archivefile_mimetype;
232 archivefile_gzip_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+gzip";
233 archivefile_gz_mimetype = archivefile_gzip_mimetype;
234 archivefile_bzip2_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+bzip2";
235 archivefile_bz2_mimetype = archivefile_bzip2_mimetype;
236 archivefile_lz4_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lz4";
237 archivefile_lzop_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lzop";
238 archivefile_lzo_mimetype = archivefile_lzop_mimetype;
239 archivefile_zstandard_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+zstandard";
240 archivefile_zstd_mimetype = archivefile_zstandard_mimetype;
241 archivefile_lzma_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lzma";
242 archivefile_xz_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+xz";
243 archivefile_extensions = ['.cat', '.cat.gz', '.cat.bz2', '.cat.zst', '.cat.lz4', '.cat.lzo', '.cat.lzop', '.cat.lzma', '.cat.xz'];
245 if __name__ == "__main__":
246 import subprocess;
247 curscrpath = os.path.dirname(sys.argv[0]);
248 if(curscrpath==""):
249 curscrpath = ".";
250 if(os.sep=="\\"):
251 curscrpath = curscrpath.replace(os.sep, "/");
252 curscrpath = curscrpath + "/";
253 scrfile = curscrpath + "catfile.py";
254 if(os.path.exists(scrfile) and os.path.isfile(scrfile)):
255 scrcmd = subprocess.Popen([sys.executable, scrfile] + sys.argv[1:]);
256 scrcmd.wait();
258 def VerbosePrintOut(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
259 if(not dbgenable):
260 return True;
261 log_functions = {
262 "print": print,
263 "log": logging.info,
264 "warning": logging.warning,
265 "error": logging.error,
266 "critical": logging.critical,
267 "exception": logging.exception,
268 "logalt": lambda x: logging.log(dgblevel, x),
269 "debug": logging.debug
271 log_function = log_functions.get(outtype);
272 if(log_function):
273 log_function(dbgtxt);
274 return True;
275 return False;
277 def VerbosePrintOutReturn(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
278 VerbosePrintOut(dbgtxt, outtype, dbgenable, dgblevel);
279 return dbgtxt;
281 def RemoveWindowsPath(dpath):
282 if(dpath is None):
283 dpath = "";
284 if(os.sep!="/"):
285 dpath = dpath.replace(os.path.sep, "/");
286 dpath = dpath.rstrip("/");
287 if(dpath=="." or dpath==".."):
288 dpath = dpath + "/";
289 return dpath;
291 def NormalizeRelativePath(inpath):
292 inpath = RemoveWindowsPath(inpath);
293 if(os.path.isabs(inpath)):
294 outpath = inpath;
295 else:
296 if(inpath.startswith("./") or inpath.startswith("../")):
297 outpath = inpath;
298 else:
299 outpath = "./" + inpath;
300 return outpath;
302 def PrependPath(base_dir, child_path):
303 # Check if base_dir is None or empty, if so, return child_path as is
304 if not base_dir:
305 return child_path;
306 # Ensure base_dir ends with exactly one slash
307 if not base_dir.endswith('/'):
308 base_dir += '/';
309 # Check if child_path starts with ./ or ../ (indicating a relative path)
310 if child_path.startswith('./') or child_path.startswith('../'):
311 # For relative paths, we don't alter the child_path
312 return base_dir + child_path;
313 else:
314 # For non-relative paths, ensure there's no starting slash on child_path to avoid double slashes
315 return base_dir + child_path.lstrip('/');
317 def ListDir(dirpath, followlink=False, duplicates=False):
318 if(isinstance(dirpath, (list, tuple, ))):
319 dirpath = list(filter(None, dirpath));
320 elif(isinstance(dirpath, (str, ))):
321 dirpath = list(filter(None, [dirpath]));
322 retlist = [];
323 for mydirfile in dirpath:
324 if(not os.path.exists(mydirfile)):
325 return False;
326 mydirfile = NormalizeRelativePath(mydirfile);
327 if(os.path.exists(mydirfile) and os.path.islink(mydirfile)):
328 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile));
329 if(os.path.exists(mydirfile) and os.path.isdir(mydirfile)):
330 for root, dirs, filenames in os.walk(mydirfile):
331 dpath = root;
332 dpath = RemoveWindowsPath(dpath);
333 if(dpath not in retlist and not duplicates):
334 retlist.append(dpath);
335 if(duplicates):
336 retlist.append(dpath);
337 for file in filenames:
338 fpath = os.path.join(root, file);
339 fpath = RemoveWindowsPath(fpath);
340 if(fpath not in retlist and not duplicates):
341 retlist.append(fpath);
342 if(duplicates):
343 retlist.append(fpath);
344 else:
345 retlist.append(RemoveWindowsPath(mydirfile));
346 return retlist;
348 def ListDirAdvanced(dirpath, followlink=False, duplicates=False):
349 if isinstance(dirpath, (list, tuple)):
350 dirpath = list(filter(None, dirpath));
351 elif isinstance(dirpath, str):
352 dirpath = list(filter(None, [dirpath]));
353 retlist = []
354 for mydirfile in dirpath:
355 if not os.path.exists(mydirfile):
356 return False;
357 mydirfile = NormalizeRelativePath(mydirfile);
358 if os.path.exists(mydirfile) and os.path.islink(mydirfile) and followlink:
359 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile))
360 if os.path.exists(mydirfile) and os.path.isdir(mydirfile):
361 for root, dirs, filenames in os.walk(mydirfile):
362 # Sort dirs and filenames alphabetically in place
363 dirs.sort(key=lambda x: x.lower());
364 filenames.sort(key=lambda x: x.lower());
365 dpath = RemoveWindowsPath(root);
366 if not duplicates and dpath not in retlist:
367 retlist.append(dpath);
368 elif duplicates:
369 retlist.append(dpath);
370 for file in filenames:
371 fpath = os.path.join(root, file);
372 fpath = RemoveWindowsPath(fpath);
373 if not duplicates and fpath not in retlist:
374 retlist.append(fpath);
375 elif duplicates:
376 retlist.append(fpath);
377 else:
378 retlist.append(RemoveWindowsPath(mydirfile));
379 return retlist;
381 def create_alias_function(prefix, base_name, suffix, target_function):
382 # Define a new function that wraps the target function
383 def alias_function(*args, **kwargs):
384 return target_function(*args, **kwargs);
386 # Create the function name by combining the prefix, base name, and the suffix
387 function_name = "{}{}{}".format(prefix, base_name, suffix);
389 # Add the new function to the global namespace
390 globals()[function_name] = alias_function;
392 def create_alias_function_alt(prefix, base_name, suffix, target_function):
393 # Create the function name by combining the prefix, base name, and the suffix
394 # Use the format method for string formatting, compatible with Python 2 and 3
395 function_name = "{}{}{}".format(prefix, base_name, suffix)
396 # Add the new function (alias of the target_function) to the global namespace
397 # This line is compatible as-is with both Python 2 and 3
398 globals()[function_name] = target_function
400 # initial_value can be 0xFFFF or 0x0000
401 def crc16_ansi(msg, initial_value=0xFFFF):
402 # CRC-16-IBM / CRC-16-ANSI polynomial and initial value
403 poly = 0x8005; # Polynomial for CRC-16-IBM / CRC-16-ANSI
404 crc = initial_value; # Initial value
405 for b in msg:
406 crc ^= b << 8; # XOR byte into CRC top byte
407 for _ in range(8): # Process each bit
408 if crc & 0x8000: # If the top bit is set
409 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
410 else:
411 crc = crc << 1; # Just shift left
412 crc &= 0xFFFF; # Ensure CRC remains 16-bit
413 return crc;
415 # initial_value can be 0xFFFF or 0x0000
416 def crc16_ibm(msg, initial_value=0xFFFF):
417 return crc16_ansi(msg, initial_value);
419 # initial_value is 0xFFFF
420 def crc16(msg):
421 return crc16_ansi(msg, 0xFFFF);
423 # initial_value can be 0xFFFF, 0x1D0F or 0x0000
424 def crc16_ccitt(msg, initial_value=0xFFFF):
425 # CRC-16-CCITT polynomial
426 poly = 0x1021; # Polynomial for CRC-16-CCITT
427 # Use the specified initial value
428 crc = initial_value;
429 for b in msg:
430 crc ^= b << 8; # XOR byte into CRC top byte
431 for _ in range(8): # Process each bit
432 if crc & 0x8000: # If the top bit is set
433 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
434 else:
435 crc = crc << 1; # Just shift left
436 crc &= 0xFFFF; # Ensure CRC remains 16-bit
437 return crc;
439 # initial_value can be 0x42F0E1EBA9EA3693 or 0x0000000000000000
440 def crc64_ecma(msg, initial_value=0x0000000000000000):
441 # CRC-64-ECMA polynomial and initial value
442 poly = 0x42F0E1EBA9EA3693;
443 crc = initial_value; # Initial value for CRC-64-ECMA
444 for b in msg:
445 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
446 for _ in range(8): # Process each bit
447 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
448 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
449 else:
450 crc <<= 1; # Just shift left if the MSB is 0
451 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
452 return crc;
454 # initial_value can be 0x000000000000001B or 0xFFFFFFFFFFFFFFFF
455 def crc64_iso(msg, initial_value=0xFFFFFFFFFFFFFFFF):
456 # CRC-64-ISO polynomial and initial value
457 poly = 0x000000000000001B;
458 crc = initial_value; # Common initial value for CRC-64-ISO
459 for b in msg:
460 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
461 for _ in range(8): # Process each bit
462 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
463 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
464 else:
465 crc <<= 1; # Just shift left if the MSB is 0
466 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
467 return crc;
469 def GetDataFromArray(data, path, default=None):
470 element = data;
471 try:
472 for key in path:
473 element = element[key];
474 return element;
475 except (KeyError, TypeError, IndexError):
476 return default;
478 def GetDataFromArrayAlt(structure, path, default=None):
479 element = structure;
480 for key in path:
481 if isinstance(element, dict) and key in element:
482 element = element[key];
483 elif isinstance(element, list) and isinstance(key, int) and -len(element) <= key < len(element):
484 element = element[key];
485 else:
486 return default;
487 return element;
489 def GetHeaderChecksum(inlist=[], checksumtype="crc32", encodedata=True, formatspecs=__file_format_dict__):
490 if isinstance(inlist, list):
491 fileheader = AppendNullBytes(inlist, formatspecs['format_delimiter']);
492 else:
493 fileheader = AppendNullByte(inlist, formatspecs['format_delimiter']);
494 if(encodedata):
495 fileheader = fileheader.encode('UTF-8');
496 if(checksumtype=="none" or checksumtype==""):
497 catfileheadercshex = format(0, 'x').lower();
498 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
499 catfileheadercshex = format(crc16(fileheader) & 0xffff, '04x').lower();
500 elif(checksumtype=="crc16_ccitt"):
501 catfileheadercshex = format(crc16_ccitt(fileheader) & 0xffff, '04x').lower();
502 elif(checksumtype=="adler32"):
503 catfileheadercshex = format(zlib.adler32(fileheader) & 0xffffffff, '08x').lower();
504 elif(checksumtype=="crc32"):
505 catfileheadercshex = format(crc32(fileheader) & 0xffffffff, '08x').lower();
506 elif(checksumtype=="crc64_ecma"):
507 catfileheadercshex = format(crc64_ecma(fileheader) & 0xffffffffffffffff, '016x').lower();
508 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
509 catfileheadercshex = format(crc64_iso(fileheader) & 0xffffffffffffffff, '016x').lower();
510 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
511 checksumoutstr = hashlib.new(checksumtype);
512 checksumoutstr.update(fileheader);
513 catfileheadercshex = checksumoutstr.hexdigest().lower();
514 else:
515 catfileheadercshex = format(0, 'x').lower();
516 return catfileheadercshex;
518 def GetFileChecksum(instr, checksumtype="crc32", encodedata=True, formatspecs=__file_format_dict__):
519 if(encodedata):
520 instr = instr.encode('UTF-8');
521 if(checksumtype=="none" or checksumtype==""):
522 catinstrcshex = format(0, 'x').lower();
523 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
524 catinstrcshex = format(crc16(instr) & 0xffff, '04x').lower();
525 elif(checksumtype=="crc16_ccitt"):
526 catinstrcshex = format(crc16_ccitt(instr) & 0xffff, '04x').lower();
527 elif(checksumtype=="adler32"):
528 catinstrcshex = format(zlib.adler32(instr) & 0xffffffff, '08x').lower();
529 elif(checksumtype=="crc32"):
530 catinstrcshex = format(crc32(instr) & 0xffffffff, '08x').lower();
531 elif(checksumtype=="crc64_ecma"):
532 catinstrcshex = format(crc64_ecma(instr) & 0xffffffffffffffff, '016x').lower();
533 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
534 catinstrcshex = format(crc64_iso(instr) & 0xffffffffffffffff, '016x').lower();
535 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
536 checksumoutstr = hashlib.new(checksumtype);
537 checksumoutstr.update(instr);
538 catinstrcshex = checksumoutstr.hexdigest().lower();
539 else:
540 catinstrcshex = format(0, 'x').lower();
541 return catinstrcshex;
543 def ValidateHeaderChecksum(inlist=[], checksumtype="crc32", inchecksum="0", formatspecs=__file_format_dict__):
544 catfileheadercshex = GetHeaderChecksum(inlist, checksumtype, True, formatspecs);
545 inchecksum = inchecksum.lower();
546 catfileheadercshex = catfileheadercshex.lower();
547 if(inchecksum==catfileheadercshex):
548 return True;
549 else:
550 return False;
552 def ValidateFileChecksum(infile, checksumtype="crc32", inchecksum="0", formatspecs=__file_format_dict__):
553 catinfilecshex = GetFileChecksum(infile, checksumtype, True, formatspecs);
554 inchecksum = inchecksum.lower();
555 catinfilecshex = catinfilecshex.lower();
556 if(inchecksum==catinfilecshex):
557 return True;
558 else:
559 return False;
561 def ReadTillNullByteOld(fp, delimiter=__file_format_dict__['format_delimiter']):
562 curbyte = b"";
563 curfullbyte = b"";
564 nullbyte = delimiter.encode("UTF-8");
565 while(True):
566 curbyte = fp.read(1);
567 if(curbyte==nullbyte or not curbyte):
568 break;
569 curfullbyte = curfullbyte + curbyte;
570 return curfullbyte.decode('UTF-8');
572 def ReadUntilNullByteOld(fp, delimiter=__file_format_dict__['format_delimiter']):
573 return ReadTillNullByteOld(fp, delimiter);
575 def ReadTillNullByteAlt(fp, delimiter=__file_format_dict__['format_delimiter'], chunk_size=1024, max_read=1024000):
576 delimiter = delimiter.encode('UTF-8') # Ensure the delimiter is in bytes
577 buffer = bytearray();
578 total_read = 0;
579 delimiter_length = len(delimiter);
580 while True:
581 chunk = fp.read(chunk_size)
582 if not chunk:
583 # End of file reached without finding the delimiter
584 break;
585 buffer.extend(chunk);
586 total_read += len(chunk);
587 if delimiter in buffer:
588 # Delimiter found, calculate where to reset the file pointer
589 index = buffer.find(delimiter);
590 # Calculate how many extra bytes were read after the delimiter
591 extra_bytes_read = len(buffer) - (index + delimiter_length);
592 # Move the file pointer back to just after the delimiter
593 fp.seek(-extra_bytes_read, 1);
594 buffer = buffer[:index];
595 break;
596 if total_read >= max_read:
597 # Stop reading if max limit is reached to prevent excessive memory usage
598 raise MemoryError("Maximum read limit reached without finding the delimiter.");
599 # Check for incomplete UTF-8 sequences at the end of the buffer
600 if len(buffer) > 1 and 128 <= buffer[-1] < 192:
601 # This suggests that the last byte might be the start of a multi-byte character
602 # Try to read one more byte to complete the character
603 extra_byte = fp.read(1);
604 if extra_byte:
605 buffer.extend(extra_byte);
606 else:
607 # No more data available
608 break;
609 try:
610 return buffer.decode('UTF-8', errors='replace');
611 except UnicodeDecodeError:
612 return buffer.decode('UTF-8', errors='replace');
614 def ReadUntilNullByteAlt(fp, delimiter=__file_format_dict__['format_delimiter'], chunk_size=1024, max_read=1024000):
615 return ReadTillNullByteAlt(fp, delimiter, chunk_size, max_read);
617 def ReadTillNullByte(fp, delimiter=__file_format_dict__['format_delimiter'], max_read=1024000):
618 curfullbyte = bytearray();
619 nullbyte = delimiter.encode("UTF-8");
620 total_read = 0; # Track the total number of bytes read
621 while True:
622 curbyte = fp.read(1);
623 if curbyte == nullbyte or not curbyte:
624 break;
625 curfullbyte.extend(curbyte);
626 total_read += 1;
627 if total_read >= max_read:
628 raise MemoryError("Maximum read limit reached without finding the delimiter.");
629 # Decode the full byte array to string once out of the loop
630 try:
631 return curfullbyte.decode('UTF-8');
632 except UnicodeDecodeError:
633 # Handle potential partial UTF-8 characters
634 for i in range(1, 4):
635 try:
636 return curfullbyte[:-i].decode('UTF-8');
637 except UnicodeDecodeError:
638 continue;
639 raise; # Re-raise if decoding fails even after trimming
641 def ReadUntilNullByte(fp, delimiter=__file_format_dict__['format_delimiter'], max_read=1024000):
642 return ReadTillNullByte(fp, delimiter, max_read);
644 def ReadTillNullByteByNum(fp, delimiter=__file_format_dict__['format_delimiter'], num_delimiters=1, chunk_size=1024, max_read=1024000):
645 delimiter = delimiter.encode('UTF-8'); # Ensure the delimiter is in bytes
646 buffer = bytearray();
647 total_read = 0;
648 delimiter_length = len(delimiter);
649 results = [];
650 while len(results) < num_delimiters:
651 chunk = fp.read(chunk_size);
652 if not chunk:
653 # End of file reached; decode whatever is collected if it's the last needed part
654 if len(buffer) > 0:
655 results.append(buffer.decode('UTF-8', errors='replace'));
656 break;
657 buffer.extend(chunk)
658 total_read += len(chunk);
659 # Check if we have found the delimiter
660 while delimiter in buffer:
661 index = buffer.find(delimiter);
662 # Decode the section before the delimiter
663 results.append(buffer[:index].decode('UTF-8', errors='replace'));
664 # Remove the processed part from the buffer
665 buffer = buffer[index + delimiter_length:];
666 if len(results) == num_delimiters:
667 # If reached the required number of delimiters, adjust the file pointer and stop
668 fp.seek(-len(buffer), 1);
669 return results;
670 if total_read >= max_read:
671 # Stop reading if max limit is reached to prevent excessive memory usage
672 raise MemoryError("Maximum read limit reached without finding the delimiter.");
673 # Check for incomplete UTF-8 sequences at the end of the buffer
674 if len(buffer) > 1 and 128 <= buffer[-1] < 192:
675 # This suggests that the last byte might be the start of a multi-byte character
676 # Try to read one more byte to complete the character
677 extra_byte = fp.read(1);
678 if extra_byte:
679 buffer.extend(extra_byte);
680 else:
681 # No more data available
682 break;
683 # Process remaining buffer if less than the required number of delimiters were found
684 if len(buffer) > 0 and len(results) < num_delimiters:
685 results.append(buffer.decode('UTF-8', errors='replace'));
686 return results;
688 def ReadUntilNullByteByNum(fp, delimiter=__file_format_dict__['format_delimiter'], num_delimiters=1, chunk_size=1024, max_read=1024000):
689 return ReadTillNullByteByNum(fp, delimiter, num_delimiters, chunk_size, max_read);
691 def SeekToEndOfFile(fp):
692 lasttell = 0;
693 while(True):
694 fp.seek(1, 1);
695 if(lasttell==fp.tell()):
696 break;
697 lasttell = fp.tell();
698 return True;
700 def ReadFileHeaderData(fp, rounds=0, delimiter=__file_format_dict__['format_delimiter']):
701 rocount = 0;
702 roend = int(rounds);
703 HeaderOut = [];
704 while(rocount<roend):
705 HeaderOut.append(ReadTillNullByte(fp, delimiter));
706 rocount = rocount + 1;
707 return HeaderOut;
709 def ReadFileHeaderDataBySize(fp, delimiter=__file_format_dict__['format_delimiter']):
710 headerpresize = ReadTillNullByte(fp, delimiter);
711 headersize = int(headerpresize, 16);
712 if(headersize<=0):
713 return [];
714 headercontent = str(fp.read(headersize).decode('UTF-8')).split(delimiter);
715 fp.seek(1, 1);
716 rocount = 0;
717 roend = int(len(headercontent));
718 HeaderOut = [headerpresize];
719 while(rocount<roend):
720 HeaderOut.append(headercontent[rocount]);
721 rocount = rocount + 1;
722 return HeaderOut;
724 def ReadFileHeaderDataWoSize(fp, delimiter=__file_format_dict__['format_delimiter']):
725 preheaderdata = ReadFileHeaderData(fp, 2, delimiter);
726 headersize = int(preheaderdata[0], 16);
727 headernumfields = int(preheaderdata[1], 16);
728 if(headersize<=0 or headernumfields<=0):
729 return [];
730 headerdata = ReadTillNullByteByNum(fp, delimiter, headernumfields);
731 #headerdata = ReadFileHeaderData(fp, headernumfields, delimiter);
732 HeaderOut = preheaderdata + headerdata;
733 return HeaderOut;
735 def ReadFileHeaderDataBySizeWithContent(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
736 delimiter = formatspecs['format_delimiter'];
737 fheaderstart = fp.tell();
738 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
739 if(len(HeaderOut)==0):
740 return False;
741 if(re.findall("^[.|/]", HeaderOut[3])):
742 fname = HeaderOut[3];
743 else:
744 fname = "./"+HeaderOut[3];
745 fchecksumtype = HeaderOut[-3].lower();
746 fcs = HeaderOut[-2].lower();
747 fccs = HeaderOut[-1].lower();
748 fsize = int(HeaderOut[5], 16);
749 fcompression = HeaderOut[12];
750 fcsize = int(HeaderOut[13], 16);
751 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
752 if(fcs!=newfcs and not skipchecksum):
753 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
754 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
755 return False;
756 fhend = fp.tell() - 1;
757 fcontentstart = fp.tell();
758 fcontents = BytesIO();
759 if(fsize>0 and not listonly):
760 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
761 fcontents.write(fp.read(fsize));
762 else:
763 fcontents.write(fp.read(fcsize));
764 elif(fsize>0 and listonly):
765 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
766 fp.seek(fsize, 1);
767 else:
768 fp.seek(fcsize, 1);
769 fcontents.seek(0, 0);
770 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
771 if(fccs!=newfccs and not skipchecksum and not listonly):
772 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
773 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
774 return False;
775 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
776 pass;
777 else:
778 fcontents.seek(0, 0);
779 fcontents = UncompressArchiveFile(fcontents, formatspecs);
780 fp.seek(1, 1);
781 fcontentend = fp.tell() - 1;
782 HeaderOut.append(fcontents);
783 return HeaderOut;
785 def ReadFileHeaderDataBySizeWithContentToArray(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
786 delimiter = formatspecs['format_delimiter'];
787 fheaderstart = fp.tell();
788 if(formatspecs['new_style']):
789 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
790 else:
791 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
792 if(len(HeaderOut)==0):
793 return False;
794 fheadsize = int(HeaderOut[0], 16);
795 fnumfields = int(HeaderOut[1], 16);
796 ftype = int(HeaderOut[2], 16);
797 if(re.findall("^[.|/]", HeaderOut[3])):
798 fname = HeaderOut[3];
799 else:
800 fname = "./"+HeaderOut[3];
801 fbasedir = os.path.dirname(fname);
802 flinkname = HeaderOut[4];
803 fsize = int(HeaderOut[5], 16);
804 fatime = int(HeaderOut[6], 16);
805 fmtime = int(HeaderOut[7], 16);
806 fctime = int(HeaderOut[8], 16);
807 fbtime = int(HeaderOut[9], 16);
808 fmode = int(HeaderOut[10], 16);
809 fchmode = stat.S_IMODE(fmode);
810 ftypemod = stat.S_IFMT(fmode);
811 fwinattributes = int(HeaderOut[11], 16);
812 fcompression = HeaderOut[12];
813 fcsize = int(HeaderOut[13], 16);
814 fuid = int(HeaderOut[14], 16);
815 funame = HeaderOut[15];
816 fgid = int(HeaderOut[16], 16);
817 fgname = HeaderOut[17];
818 fid = int(HeaderOut[18], 16);
819 finode = int(HeaderOut[19], 16);
820 flinkcount = int(HeaderOut[20], 16);
821 fdev_minor = int(HeaderOut[21], 16);
822 fdev_major = int(HeaderOut[22], 16);
823 frdev_minor = int(HeaderOut[23], 16);
824 frdev_major = int(HeaderOut[24], 16);
825 fextrasize = int(HeaderOut[25], 16);
826 fextrafields = int(HeaderOut[26], 16);
827 extrafieldslist = [];
828 extrastart = 27;
829 extraend = extrastart + fextrafields;
830 extrafieldslist = [];
831 if(extrastart<extraend):
832 extrafieldslist.append(HeaderOut[extrastart]);
833 extrastart = extrastart + 1;
834 fchecksumtype = HeaderOut[extrastart].lower();
835 fcs = HeaderOut[extrastart + 1].lower();
836 fccs = HeaderOut[extrastart + 2].lower();
837 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
838 if(fcs!=newfcs and not skipchecksum):
839 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
840 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
841 return False;
842 fhend = fp.tell() - 1;
843 fcontentstart = fp.tell();
844 fcontents = BytesIO();
845 pyhascontents = False;
846 if(fsize>0 and not listonly):
847 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
848 fcontents.write(fp.read(fsize));
849 else:
850 fcontents.write(fp.read(fcsize));
851 pyhascontents = True;
852 elif(fsize>0 and listonly):
853 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
854 fp.seek(fsize, 1);
855 else:
856 fp.seek(fcsize, 1);
857 pyhascontents = False;
858 fcontents.seek(0, 0);
859 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
860 if(fccs!=newfccs and not skipchecksum and not listonly):
861 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
862 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
863 return False;
864 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
865 pass;
866 else:
867 fcontents.seek(0, 0);
868 fcontents = UncompressArchiveFile(fcontents, formatspecs);
869 fp.seek(1, 1);
870 fcontentend = fp.tell() - 1;
871 fcontents.seek(0, 0);
872 catlist = {'fheadersize': fheadsize, 'fhstart': fheaderstart, 'fhend': fhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': fchecksumtype, 'fnumfields': fnumfields + 2, 'frawheader': HeaderOut, 'fextrafields': fextrafields, 'fextrafieldsize': fextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': fcs, 'fcontentchecksum': fccs, 'fhascontents': pyhascontents, 'fcontentstart': fcontentstart, 'fcontentend': fcontentend, 'fcontents': fcontents};
873 return catlist;
875 def ReadFileHeaderDataBySizeWithContentToList(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
876 delimiter = formatspecs['format_delimiter'];
877 fheaderstart = fp.tell();
878 if(formatspecs['new_style']):
879 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
880 else:
881 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
882 if(len(HeaderOut)==0):
883 return False;
884 fheadsize = int(HeaderOut[0], 16);
885 fnumfields = int(HeaderOut[1], 16);
886 ftype = int(HeaderOut[2], 16);
887 if(re.findall("^[.|/]", HeaderOut[3])):
888 fname = HeaderOut[3];
889 else:
890 fname = "./"+HeaderOut[3];
891 fbasedir = os.path.dirname(fname);
892 flinkname = HeaderOut[4];
893 fsize = int(HeaderOut[5], 16);
894 fatime = int(HeaderOut[6], 16);
895 fmtime = int(HeaderOut[7], 16);
896 fctime = int(HeaderOut[8], 16);
897 fbtime = int(HeaderOut[9], 16);
898 fmode = int(HeaderOut[10], 16);
899 fchmode = stat.S_IMODE(fmode);
900 ftypemod = stat.S_IFMT(fmode);
901 fwinattributes = int(HeaderOut[11], 16);
902 fcompression = HeaderOut[12];
903 fcsize = int(HeaderOut[13], 16);
904 fuid = int(HeaderOut[14], 16);
905 funame = HeaderOut[15];
906 fgid = int(HeaderOut[16], 16);
907 fgname = HeaderOut[17];
908 fid = int(HeaderOut[18], 16);
909 finode = int(HeaderOut[19], 16);
910 flinkcount = int(HeaderOut[20], 16);
911 fdev_minor = int(HeaderOut[21], 16);
912 fdev_major = int(HeaderOut[22], 16);
913 frdev_minor = int(HeaderOut[23], 16);
914 frdev_major = int(HeaderOut[24], 16);
915 fextrasize = int(HeaderOut[25], 16);
916 fextrafields = int(HeaderOut[26], 16);
917 extrafieldslist = [];
918 extrastart = 27;
919 extraend = extrastart + fextrafields;
920 extrafieldslist = [];
921 if(extrastart<extraend):
922 extrafieldslist.append(HeaderOut[extrastart]);
923 extrastart = extrastart + 1;
924 fchecksumtype = HeaderOut[extrastart].lower();
925 fcs = HeaderOut[extrastart + 1].lower();
926 fccs = HeaderOut[extrastart + 2].lower();
927 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
928 if(fcs!=newfcs and not skipchecksum):
929 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
930 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
931 return False;
932 fhend = fp.tell() - 1;
933 fcontentstart = fp.tell();
934 fcontents = BytesIO();
935 pyhascontents = False;
936 if(fsize>0 and not listonly):
937 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
938 fcontents.write(fp.read(fsize));
939 else:
940 fcontents.write(fp.read(fcsize));
941 pyhascontents = True;
942 elif(fsize>0 and listonly):
943 if(fcompression=="none" or fcompression=="" or fcompression=="atuo"):
944 fp.seek(fsize, 1);
945 else:
946 fp.seek(fcsize, 1);
947 pyhascontents = False;
948 fcontents.seek(0, 0);
949 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
950 if(fccs!=newfccs and not skipchecksum and not listonly):
951 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
952 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
953 return False;
954 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
955 pass;
956 else:
957 fcontents.seek(0, 0);
958 fcontents = UncompressArchiveFile(fcontents, formatspecs);
959 fp.seek(1, 1);
960 fcontentend = fp.tell() - 1;
961 catlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, extrafieldslist, fchecksumtype, fcontents];
962 return catlist;
964 def ReadFileDataBySizeWithContent(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
965 delimiter = formatspecs['format_delimiter'];
966 curloc = fp.tell();
967 if(curloc>0):
968 fp.seek(0, 0);
969 catheader = ReadFileHeaderData(fp, 4, delimiter);
970 if(curloc>0):
971 fp.seek(curloc, 0);
972 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
973 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
974 if(not headercheck and not skipchecksum):
975 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
976 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
977 return False;
978 fnumfiles = int(catheader[1], 16);
979 countnum = 0;
980 flist = [];
981 while(countnum < fnumfiles):
982 HeaderOut = ReadFileHeaderDataBySizeWithContent(fp, listonly, skipchecksum, formatspecs)
983 if(len(HeaderOut)==0):
984 break;
985 flist.append(HeaderOut);
986 countnum = countnum + 1;
987 return flist;
989 def ReadFileDataBySizeWithContentToArray(fp, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
990 delimiter = formatspecs['format_delimiter'];
991 curloc = fp.tell();
992 if(curloc>0):
993 fp.seek(0, 0);
994 catheader = ReadFileHeaderData(fp, 4, delimiter);
995 if(curloc>0):
996 fp.seek(curloc, 0);
997 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
998 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
999 if(not headercheck and not skipchecksum):
1000 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1001 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1002 return False;
1003 catstring = catheader[0];
1004 catversion = re.findall(r"([\d]+)$", catstring);
1005 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
1006 fprenumfiles = catheader[1];
1007 fnumfiles = int(fprenumfiles, 16);
1008 fprechecksumtype = catheader[2];
1009 fprechecksum = catheader[3];
1010 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
1011 if(seekstart<0 and seekstart>fnumfiles):
1012 seekstart = 0;
1013 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
1014 seekend = fnumfiles;
1015 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
1016 seekend = fnumfiles - abs(seekend);
1017 if(seekstart>0):
1018 il = 0;
1019 while(il < seekstart):
1020 prefhstart = fp.tell();
1021 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs['format_delimiter']);
1022 if(len(preheaderdata)==0):
1023 break;
1024 prefsize = int(preheaderdata[5], 16);
1025 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-3].lower(), True, formatspecs);
1026 prefcs = preheaderdata[-2];
1027 if(prefcs!=prenewfcs and not skipchecksum):
1028 VVerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
1029 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
1030 return False;
1031 valid_archive = False;
1032 invalid_archive = True;
1033 prefhend = fp.tell() - 1;
1034 prefcontentstart = fp.tell();
1035 prefcontents = BytesIO();
1036 pyhascontents = False;
1037 if(prefsize>0):
1038 prefcontents.write(fp.read(prefsize));
1039 prefcontents.seek(0, 0);
1040 prenewfccs = GetFileChecksum(prefcontents.read(), preheaderdata[-3].lower(), False, formatspecs);
1041 prefccs = preheaderdata[-1];
1042 pyhascontents = True;
1043 if(prefccs!=prenewfccs and not skipchecksum):
1044 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
1045 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
1046 return False;
1047 fp.seek(1, 1);
1048 il = il + 1;
1049 realidnum = 0;
1050 countnum = seekstart;
1051 while(countnum < seekend):
1052 catlist['ffilelist'].update({realidnum: {'fid': realidnum, 'fidalt': realidnum}});
1053 HeaderOut = ReadFileHeaderDataBySizeWithContentToArray(fp, listonly, skipchecksum, formatspecs);
1054 if(len(HeaderOut)==0):
1055 break;
1056 catlist['ffilelist'][realidnum].update(HeaderOut);
1057 countnum = countnum + 1;
1058 realidnum = realidnum + 1;
1059 return catlist;
1061 def ReadFileDataBySizeWithContentToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
1062 if(isinstance(infile, dict)):
1063 listcatfiles = infile;
1064 else:
1065 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
1066 return False;
1067 listcatfiles = ReadFileDataBySizeWithContentToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs);
1068 if(not listcatfiles):
1069 return False;
1070 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
1071 lenlist = len(listcatfiles['ffilelist']);
1072 lcfi = 0;
1073 lcfx = int(listcatfiles['fnumfiles']);
1074 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
1075 lcfx = int(lenlist);
1076 else:
1077 lcfx = int(listcatfiles['fnumfiles']);
1078 while(lcfi < lcfx):
1079 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
1080 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
1081 catarray['filetoid'].update(filetoidarray);
1082 catarray['idtofile'].update(idtofilearray);
1083 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
1084 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
1085 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
1086 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
1087 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
1088 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
1089 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
1090 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
1091 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
1092 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1093 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1094 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
1095 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
1096 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
1097 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
1098 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
1099 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1100 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1101 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
1102 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
1103 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
1104 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1105 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1106 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
1107 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
1108 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
1109 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
1110 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1111 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1112 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1113 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1114 lcfi = lcfi + 1;
1115 return catarray;
1117 def ReadFileDataBySizeWithContentToList(fp, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
1118 delimiter = formatspecs['format_delimiter'];
1119 curloc = fp.tell();
1120 if(curloc>0):
1121 fp.seek(0, 0);
1122 catheader = ReadFileHeaderData(fp, 4, delimiter);
1123 if(curloc>0):
1124 fp.seek(curloc, 0);
1125 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1126 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1127 if(not headercheck and not skipchecksum):
1128 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1129 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1130 return False;
1131 catstring = catheader[0];
1132 catversion = re.findall(r"([\d]+)$", catstring);
1133 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
1134 fprenumfiles = catheader[1];
1135 fnumfiles = int(fprenumfiles, 16);
1136 fprechecksumtype = catheader[2];
1137 fprechecksum = catheader[3];
1138 catlist = [];
1139 if(seekstart<0 and seekstart>fnumfiles):
1140 seekstart = 0;
1141 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
1142 seekend = fnumfiles;
1143 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
1144 seekend = fnumfiles - abs(seekend);
1145 if(seekstart>0):
1146 il = 0;
1147 while(il < seekstart):
1148 prefhstart = fp.tell();
1149 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs['format_delimiter']);
1150 if(len(preheaderdata)==0):
1151 break;
1152 prefsize = int(preheaderdata[5], 16);
1153 prefcompression = preheaderdata[12];
1154 prefcsize = int(preheaderdata[13], 16);
1155 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-3].lower(), True, formatspecs);
1156 prefcs = preheaderdata[-2];
1157 if(prefcs!=prenewfcs and not skipchecksum):
1158 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
1159 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
1160 return False;
1161 valid_archive = False;
1162 invalid_archive = True;
1163 prefhend = fp.tell() - 1;
1164 prefcontentstart = fp.tell();
1165 prefcontents = "";
1166 pyhascontents = False;
1167 if(prefsize>0):
1168 if(prefcompression=="none" or prefcompression=="" or prefcompression=="auto"):
1169 prefcontents = catfp.read(prefsize);
1170 else:
1171 prefcontents = catfp.read(prefcsize);
1172 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
1173 prefccs = preheaderdata[-1];
1174 pyhascontents = True;
1175 if(prefccs!=prenewfccs and not skipchecksum):
1176 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
1177 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
1178 return False;
1179 fp.seek(1, 1);
1180 il = il + 1;
1181 realidnum = 0;
1182 countnum = seekstart;
1183 while(countnum < seekend):
1184 HeaderOut = ReadFileHeaderDataBySizeWithContentToList(fp, listonly, skipchecksum, formatspecs);
1185 if(len(HeaderOut)==0):
1186 break;
1187 catlist.append(HeaderOut);
1188 countnum = countnum + 1;
1189 realidnum = realidnum + 1;
1190 return catlist;
1192 def ReadInFileBySizeWithContentToArray(infile, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
1193 delimiter = formatspecs['format_delimiter'];
1194 if(hasattr(infile, "read") or hasattr(infile, "write")):
1195 fp = infile;
1196 fp.seek(0, 0);
1197 fp = UncompressArchiveFile(fp, formatspecs);
1198 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1199 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1200 return False;
1201 if(not fp):
1202 return False;
1203 fp.seek(0, 0);
1204 elif(infile=="-"):
1205 fp = BytesIO();
1206 if(hasattr(sys.stdin, "buffer")):
1207 shutil.copyfileobj(sys.stdin.buffer, fp);
1208 else:
1209 shutil.copyfileobj(sys.stdin, fp);
1210 fp.seek(0, 0);
1211 fp = UncompressArchiveFile(fp, formatspecs);
1212 if(not fp):
1213 return False;
1214 fp.seek(0, 0);
1215 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
1216 fp = download_file_from_internet_file(infile);
1217 fp = UncompressArchiveFile(fp, formatspecs);
1218 fp.seek(0, 0);
1219 if(not fp):
1220 return False;
1221 fp.seek(0, 0);
1222 else:
1223 infile = RemoveWindowsPath(infile);
1224 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
1225 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1226 return False;
1227 compresscheck = CheckCompressionType(infile, formatspecs, True);
1228 if(not compresscheck):
1229 fextname = os.path.splitext(infile)[1];
1230 if(fextname==".gz"):
1231 compresscheck = "gzip";
1232 elif(fextname==".bz2"):
1233 compresscheck = "bzip2";
1234 elif(fextname==".zst"):
1235 compresscheck = "zstd";
1236 elif(fextname==".lz4" or fextname==".clz4"):
1237 compresscheck = "lz4";
1238 elif(fextname==".lzo" or fextname==".lzop"):
1239 compresscheck = "lzo";
1240 elif(fextname==".lzma" or fextname==".xz"):
1241 compresscheck = "lzma";
1242 else:
1243 return False;
1244 if(not compresscheck):
1245 return False;
1246 fp = UncompressFile(infile, formatspecs, "rb");
1247 return ReadFileDataBySizeWithContentToArray(fp, listonly, skipchecksum, formatspecs);
1249 def ReadInFileBySizeWithContentToList(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
1250 delimiter = formatspecs['format_delimiter'];
1251 if(hasattr(infile, "read") or hasattr(infile, "write")):
1252 fp = infile;
1253 fp.seek(0, 0);
1254 fp = UncompressArchiveFile(fp, formatspecs);
1255 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1256 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1257 return False;
1258 if(not fp):
1259 return False;
1260 fp.seek(0, 0);
1261 elif(infile=="-"):
1262 fp = BytesIO();
1263 if(hasattr(sys.stdin, "buffer")):
1264 shutil.copyfileobj(sys.stdin.buffer, fp);
1265 else:
1266 shutil.copyfileobj(sys.stdin, fp);
1267 fp.seek(0, 0);
1268 fp = UncompressArchiveFile(fp, formatspecs);
1269 if(not fp):
1270 return False;
1271 fp.seek(0, 0);
1272 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
1273 fp = download_file_from_internet_file(infile);
1274 fp = UncompressArchiveFile(fp, formatspecs);
1275 fp.seek(0, 0);
1276 if(not fp):
1277 return False;
1278 fp.seek(0, 0);
1279 else:
1280 infile = RemoveWindowsPath(infile);
1281 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
1282 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1283 return False;
1284 compresscheck = CheckCompressionType(infile, formatspecs, True);
1285 if(not compresscheck):
1286 fextname = os.path.splitext(infile)[1];
1287 if(fextname==".gz"):
1288 compresscheck = "gzip";
1289 elif(fextname==".bz2"):
1290 compresscheck = "bzip2";
1291 elif(fextname==".zst"):
1292 compresscheck = "zstd";
1293 elif(fextname==".lz4" or fextname==".clz4"):
1294 compresscheck = "lz4";
1295 elif(fextname==".lzo" or fextname==".lzop"):
1296 compresscheck = "lzo";
1297 elif(fextname==".lzma" or fextname==".xz"):
1298 compresscheck = "lzma";
1299 else:
1300 return False;
1301 if(not compresscheck):
1302 return False;
1303 fp = UncompressFile(infile, formatspecs, "rb");
1304 return ReadFileDataBySizeWithContentToList(fp, seekstart, seekend, listonly, skipchecksum, formatspecs);
1306 def ReadInFileBySizeWithContentToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
1307 if(isinstance(infile, dict)):
1308 listcatfiles = infile;
1309 else:
1310 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
1311 infile = RemoveWindowsPath(infile);
1312 listcatfiles = ReadInFileBySizeWithContentToArray(infile, listonly, skipchecksum, formatspecs);
1313 if(not listcatfiles):
1314 return False;
1315 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
1316 lenlist = len(listcatfiles['ffilelist']);
1317 lcfi = 0;
1318 lcfx = int(listcatfiles['fnumfiles']);
1319 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
1320 lcfx = int(lenlist);
1321 else:
1322 lcfx = int(listcatfiles['fnumfiles']);
1323 while(lcfi < lcfx):
1324 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
1325 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
1326 catarray['filetoid'].update(filetoidarray);
1327 catarray['idtofile'].update(idtofilearray);
1328 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
1329 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
1330 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
1331 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
1332 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
1333 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
1334 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
1335 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
1336 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
1337 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1338 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1339 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
1340 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
1341 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
1342 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
1343 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
1344 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1345 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1346 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
1347 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
1348 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
1349 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1350 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1351 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
1352 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
1353 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
1354 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
1355 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1356 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1357 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1358 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1359 lcfi = lcfi + 1;
1360 return catarray;
1362 def AppendNullByte(indata, delimiter=__file_format_dict__['format_delimiter']):
1363 outdata = str(indata) + delimiter;
1364 return outdata;
1366 def AppendNullBytes(indata=[], delimiter=__file_format_dict__['format_delimiter']):
1367 outdata = "";
1368 inum = 0;
1369 il = len(indata);
1370 while(inum < il):
1371 outdata = outdata + AppendNullByte(indata[inum], delimiter);
1372 inum = inum + 1;
1373 return outdata;
1375 def AppendFileHeader(fp, numfiles, checksumtype="crc32", formatspecs=__file_format_dict__):
1376 delimiter = formatspecs['format_delimiter'];
1377 catver = formatspecs['format_ver'];
1378 fileheaderver = str(int(catver.replace(".", "")));
1379 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
1380 fp.write(fileheader.encode('UTF-8'));
1381 fnumfiles = format(int(numfiles), 'x').lower();
1382 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs['format_delimiter']);
1383 catfileheadercshex = GetFileChecksum(fileheader + fnumfilesa, checksumtype, True, formatspecs);
1384 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
1385 fp.write(fnumfilesa.encode('UTF-8'));
1386 try:
1387 fp.flush();
1388 os.fsync(fp.fileno());
1389 except io.UnsupportedOperation:
1390 pass;
1391 except AttributeError:
1392 pass;
1393 except OSError as e:
1394 pass;
1395 return fp;
1397 def MakeEmptyFilePointer(fp, checksumtype="crc32", formatspecs=__file_format_dict__):
1398 AppendFileHeader(fp, 0, checksumtype, formatspecs);
1399 return fp;
1401 def MakeEmptyFile(outfile, compression="auto", compressionlevel=None, checksumtype="crc32", formatspecs=__file_format_dict__, returnfp=False):
1402 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1403 if(os.path.exists(outfile)):
1404 try:
1405 os.unlink(outfile);
1406 except OSError as e:
1407 pass;
1408 if(outfile=="-"):
1409 verbose = False;
1410 catfpfp = BytesIO();
1411 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1412 catfp = outfile;
1413 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1414 catfp = BytesIO();
1415 else:
1416 fbasename = os.path.splitext(outfile)[0];
1417 fextname = os.path.splitext(outfile)[1];
1418 catfp = CompressOpenFile(outfile, True, compressionlevel);
1419 catfp = AppendFileHeader(catfp, 0, checksumtype, formatspecs);
1420 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1421 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1422 try:
1423 catfp.flush();
1424 os.fsync(catfp.fileno());
1425 except io.UnsupportedOperation:
1426 pass;
1427 except AttributeError:
1428 pass;
1429 except OSError as e:
1430 pass;
1431 if(outfile=="-"):
1432 catfp.seek(0, 0);
1433 if(hasattr(sys.stdout, "buffer")):
1434 shutil.copyfileobj(catfp, sys.stdout.buffer);
1435 else:
1436 shutil.copyfileobj(catfp, sys.stdout);
1437 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1438 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1439 catfp.seek(0, 0);
1440 upload_file_to_internet_file(catfp, outfile);
1441 if(returnfp):
1442 catfp.seek(0, 0);
1443 return catfp;
1444 else:
1445 catfp.close();
1446 return True;
1448 def AppendFileHeaderWithContent(fp, filevalues=[], extradata=[], filecontent="", checksumtype="crc32", formatspecs=__file_format_dict__):
1449 extrafields = format(len(extradata), 'x').lower();
1450 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
1451 if(len(extradata)>0):
1452 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
1453 extrasizelen = format(len(extrasizestr), 'x').lower();
1454 catoutlen = len(filevalues) + len(extradata) + 5;
1455 catoutlenhex = format(catoutlen, 'x').lower();
1456 catoutlist = filevalues;
1457 catoutlist.insert(0, catoutlenhex);
1458 catoutlist.append(extrasizelen);
1459 catoutlist.append(extrafields);
1460 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
1461 if(len(extradata)>0):
1462 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
1463 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
1464 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
1465 catfilecontentcshex = GetFileChecksum(filecontent, checksumtype, False, formatspecs);
1466 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
1467 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
1468 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
1469 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
1470 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
1471 catfileoutstrecd = catfileoutstr.encode('UTF-8');
1472 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
1473 catfileout = catfileoutstrecd + filecontent + nullstrecd;
1474 fp.write(catfileout);
1475 try:
1476 fp.flush();
1477 os.fsync(fp.fileno());
1478 except io.UnsupportedOperation:
1479 pass;
1480 except AttributeError:
1481 pass;
1482 except OSError as e:
1483 pass;
1484 return fp;
1486 def AppendFilesWithContent(infiles, fp, dirlistfromtxt=False, filevalues=[], extradata=[], compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
1487 advancedlist = formatspecs['use_advanced_list'];
1488 altinode = formatspecs['use_alt_inode'];
1489 if(verbose):
1490 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
1491 if(infiles=="-"):
1492 for line in sys.stdin:
1493 infilelist.append(line.strip());
1494 infilelist = list(filter(None, infilelist));
1495 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
1496 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
1497 return False;
1498 with UncompressFile(infiles, formatspecs, "r") as finfile:
1499 for line in finfile:
1500 infilelist.append(line.strip());
1501 infilelist = list(filter(None, infilelist));
1502 else:
1503 if(isinstance(infiles, (list, tuple, ))):
1504 infilelist = list(filter(None, infiles));
1505 elif(isinstance(infiles, (str, ))):
1506 infilelist = list(filter(None, [infiles]));
1507 if(advancedlist):
1508 GetDirList = ListDirAdvanced(infilelist, followlink, False);
1509 else:
1510 GetDirList = ListDir(infilelist, followlink, False);
1511 if(not GetDirList):
1512 return False;
1513 curinode = 0;
1514 curfid = 0;
1515 inodelist = [];
1516 inodetofile = {};
1517 filetoinode = {};
1518 inodetocatinode = {};
1519 numfiles = int(len(GetDirList));
1520 fnumfiles = format(numfiles, 'x').lower();
1521 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
1522 for curfname in GetDirList:
1523 if(re.findall("^[.|/]", curfname)):
1524 fname = curfname;
1525 else:
1526 fname = "./"+curfname;
1527 if(verbose):
1528 VerbosePrintOut(fname);
1529 if(not followlink or followlink is None):
1530 fstatinfo = os.lstat(fname);
1531 else:
1532 fstatinfo = os.stat(fname);
1533 fpremode = fstatinfo.st_mode;
1534 finode = fstatinfo.st_ino;
1535 flinkcount = fstatinfo.st_nlink;
1536 ftype = 0;
1537 if(stat.S_ISREG(fpremode)):
1538 ftype = 0;
1539 elif(stat.S_ISLNK(fpremode)):
1540 ftype = 2;
1541 elif(stat.S_ISCHR(fpremode)):
1542 ftype = 3;
1543 elif(stat.S_ISBLK(fpremode)):
1544 ftype = 4;
1545 elif(stat.S_ISDIR(fpremode)):
1546 ftype = 5;
1547 elif(stat.S_ISFIFO(fpremode)):
1548 ftype = 6;
1549 elif(stat.S_ISSOCK(fpremode)):
1550 ftype = 8;
1551 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
1552 ftype = 9;
1553 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
1554 ftype = 10;
1555 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
1556 ftype = 111;
1557 else:
1558 ftype = 0;
1559 flinkname = "";
1560 fcurfid = format(int(curfid), 'x').lower();
1561 if(not followlink and finode!=0):
1562 if(ftype!=1):
1563 if(finode in inodelist):
1564 ftype = 1;
1565 flinkname = inodetofile[finode];
1566 if(altinode):
1567 fcurinode = format(int(finode), 'x').lower();
1568 else:
1569 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
1570 if(finode not in inodelist):
1571 inodelist.append(finode);
1572 inodetofile.update({finode: fname});
1573 inodetocatinode.update({finode: curinode});
1574 if(altinode):
1575 fcurinode = format(int(finode), 'x').lower();
1576 else:
1577 fcurinode = format(int(curinode), 'x').lower();
1578 curinode = curinode + 1;
1579 else:
1580 fcurinode = format(int(curinode), 'x').lower();
1581 curinode = curinode + 1;
1582 curfid = curfid + 1;
1583 if(ftype==2):
1584 flinkname = os.readlink(fname);
1585 fdev = fstatinfo.st_dev;
1586 getfdev = GetDevMajorMinor(fdev);
1587 fdev_minor = getfdev[0];
1588 fdev_major = getfdev[1];
1589 frdev = fstatinfo.st_dev;
1590 if(hasattr(fstatinfo, "st_rdev")):
1591 frdev = fstatinfo.st_rdev;
1592 else:
1593 frdev = fstatinfo.st_dev;
1594 getfrdev = GetDevMajorMinor(frdev);
1595 frdev_minor = getfrdev[0];
1596 frdev_major = getfrdev[1];
1597 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
1598 fsize = format(int("0"), 'x').lower();
1599 elif(ftype==0 or ftype==7):
1600 fsize = format(int(fstatinfo.st_size), 'x').lower();
1601 else:
1602 fsize = format(int(fstatinfo.st_size)).lower();
1603 fatime = format(int(fstatinfo.st_atime), 'x').lower();
1604 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
1605 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
1606 if(hasattr(fstatinfo, "st_birthtime")):
1607 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
1608 else:
1609 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
1610 fmode = format(int(fstatinfo.st_mode), 'x').lower();
1611 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
1612 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
1613 fuid = format(int(fstatinfo.st_uid), 'x').lower();
1614 fgid = format(int(fstatinfo.st_gid), 'x').lower();
1615 funame = "";
1616 try:
1617 import pwd;
1618 try:
1619 userinfo = pwd.getpwuid(fstatinfo.st_uid);
1620 funame = userinfo.pw_name;
1621 except KeyError:
1622 funame = "";
1623 except ImportError:
1624 funame = "";
1625 fgname = "";
1626 try:
1627 import grp;
1628 try:
1629 groupinfo = grp.getgrgid(fstatinfo.st_gid);
1630 fgname = groupinfo.gr_name;
1631 except KeyError:
1632 fgname = "";
1633 except ImportError:
1634 fgname = "";
1635 fdev_minor = format(int(fdev_minor), 'x').lower();
1636 fdev_major = format(int(fdev_major), 'x').lower();
1637 frdev_minor = format(int(frdev_minor), 'x').lower();
1638 frdev_major = format(int(frdev_major), 'x').lower();
1639 finode = format(int(finode), 'x').lower();
1640 flinkcount = format(int(flinkcount), 'x').lower();
1641 if(hasattr(fstatinfo, "st_file_attributes")):
1642 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
1643 else:
1644 fwinattributes = format(int(0), 'x').lower();
1645 fcompression = "";
1646 fcsize = format(int(0), 'x').lower();
1647 fcontents = BytesIO();
1648 chunk_size = 1024;
1649 if(ftype==0 or ftype==7):
1650 with open(fname, "rb") as fpc:
1651 shutil.copyfileobj(fpc, fcontents);
1652 if(not compresswholefile):
1653 fcontents.seek(0, 2);
1654 ucfsize = fcontents.tell();
1655 fcontents.seek(0, 0);
1656 if(compression=="auto"):
1657 ilsize = len(compressionlistalt);
1658 ilmin = 0;
1659 ilcsize = [];
1660 while(ilmin < ilsize):
1661 cfcontents = BytesIO();
1662 shutil.copyfileobj(fcontents, cfcontents);
1663 fcontents.seek(0, 0);
1664 cfcontents.seek(0, 0);
1665 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
1666 if(cfcontents):
1667 cfcontents.seek(0, 2);
1668 ilcsize.append(cfcontents.tell());
1669 cfcontents.close();
1670 else:
1671 try:
1672 ilcsize.append(sys.maxint);
1673 except AttributeError:
1674 ilcsize.append(sys.maxsize);
1675 ilmin = ilmin + 1;
1676 ilcmin = ilcsize.index(min(ilcsize));
1677 compression = compressionlistalt[ilcmin];
1678 fcontents.seek(0, 0);
1679 cfcontents = BytesIO();
1680 shutil.copyfileobj(fcontents, cfcontents);
1681 cfcontents.seek(0, 0);
1682 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
1683 cfcontents.seek(0, 2);
1684 cfsize = cfcontents.tell();
1685 if(ucfsize > cfsize):
1686 fcsize = format(int(cfsize), 'x').lower();
1687 fcompression = compression;
1688 fcontents.close();
1689 fcontents = cfcontents;
1690 if(followlink and (ftype==1 or ftype==2)):
1691 flstatinfo = os.stat(flinkname);
1692 with open(flinkname, "rb") as fpc:
1693 shutil.copyfileobj(fpc, fcontents);
1694 if(not compresswholefile):
1695 fcontents.seek(0, 2);
1696 ucfsize = fcontents.tell();
1697 fcontents.seek(0, 0);
1698 if(compression=="auto"):
1699 ilsize = len(compressionlistalt);
1700 ilmin = 0;
1701 ilcsize = [];
1702 while(ilmin < ilsize):
1703 cfcontents = BytesIO();
1704 shutil.copyfileobj(fcontents, cfcontents);
1705 fcontents.seek(0, 0);
1706 cfcontents.seek(0, 0);
1707 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
1708 if(cfcontents):
1709 cfcontents.seek(0, 2);
1710 ilcsize.append(cfcontents.tell());
1711 cfcontents.close();
1712 else:
1713 try:
1714 ilcsize.append(sys.maxint);
1715 except AttributeError:
1716 ilcsize.append(sys.maxsize);
1717 ilmin = ilmin + 1;
1718 ilcmin = ilcsize.index(min(ilcsize));
1719 compression = compressionlistalt[ilcmin];
1720 fcontents.seek(0, 0);
1721 cfcontents = BytesIO();
1722 shutil.copyfileobj(fcontents, cfcontents);
1723 cfcontents.seek(0, 0);
1724 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
1725 cfcontents.seek(0, 2);
1726 cfsize = cfcontents.tell();
1727 if(ucfsize > cfsize):
1728 fcsize = format(int(cfsize), 'x').lower();
1729 fcompression = compression;
1730 fcontents.close();
1731 fcontents = cfcontents;
1732 if(fcompression=="none"):
1733 fcompression = "";
1734 fcontents.seek(0, 0);
1735 ftypehex = format(ftype, 'x').lower();
1736 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
1737 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
1738 if(numfiles>0):
1739 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
1740 fp.seek(0, 0);
1741 return fp;
1743 def AppendListsWithContent(inlist, fp, dirlistfromtxt=False, filevalues=[], extradata=[], compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
1744 if(verbose):
1745 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
1746 GetDirList = inlist;
1747 if(not GetDirList):
1748 return False;
1749 curinode = 0;
1750 curfid = 0;
1751 inodelist = [];
1752 inodetofile = {};
1753 filetoinode = {};
1754 inodetocatinode = {};
1755 numfiles = int(len(GetDirList));
1756 fnumfiles = format(numfiles, 'x').lower();
1757 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
1758 for curfname in GetDirList:
1759 ftype = format(curfname[0], 'x').lower();
1760 if(re.findall("^[.|/]", curfname[1])):
1761 fname = curfname[1];
1762 else:
1763 fname = "./"+curfname[1];
1764 fbasedir = os.path.dirname(fname);
1765 flinkname = curfname[2];
1766 fsize = format(curfname[3], 'x').lower();
1767 fatime = format(curfname[4], 'x').lower();
1768 fmtime = format(curfname[5], 'x').lower();
1769 fctime = format(curfname[6], 'x').lower();
1770 fbtime = format(curfname[7], 'x').lower();
1771 fmode = format(curfname[8], 'x').lower();
1772 fwinattributes = format(curfname[9], 'x').lower();
1773 fcompression = curfname[10];
1774 fcsize = format(curfname[11], 'x').lower();
1775 fuid = format(curfname[12], 'x').lower();
1776 funame = curfname[13];
1777 fgid = format(curfname[14], 'x').lower();
1778 fgname = curfname[15];
1779 fid = format(curfname[16], 'x').lower();
1780 finode = format(curfname[17], 'x').lower();
1781 flinkcount = format(curfname[18], 'x').lower();
1782 fdev_minor = format(curfname[19], 'x').lower();
1783 fdev_major = format(curfname[20], 'x').lower();
1784 frdev_minor = format(curfname[21], 'x').lower();
1785 frdev_major = format(curfname[22], 'x').lower();
1786 extradata = curfname[23];
1787 fchecksumtype = curfname[24];
1788 fcontents = curfname[25];
1789 catoutlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
1790 fcontents.seek(0, 0);
1791 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
1792 if(numfiles>0):
1793 fp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
1794 return fp;
1796 def AppendInFileWithContent(infile, fp, dirlistfromtxt=False, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
1797 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, False, formatspecs);
1798 return AppendListsWithContent(inlist, fp, dirlistfromtxt, filevalues, extradata, followlink, checksumtype, formatspecs, verbose);
1800 def AppendFilesWithContentToOutFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
1801 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1802 if(os.path.exists(outfile)):
1803 try:
1804 os.unlink(outfile);
1805 except OSError as e:
1806 pass;
1807 if(outfile=="-"):
1808 verbose = False;
1809 catfpfp = BytesIO();
1810 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1811 catfp = outfile;
1812 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1813 catfp = BytesIO();
1814 else:
1815 fbasename = os.path.splitext(outfile)[0];
1816 fextname = os.path.splitext(outfile)[1];
1817 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
1818 catfp = AppendFilesWithContent(infiles, catfp, dirlistfromtxt, filevalues, extradata, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose);
1819 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1820 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1821 try:
1822 catfp.flush();
1823 os.fsync(catfp.fileno());
1824 except io.UnsupportedOperation:
1825 pass;
1826 except AttributeError:
1827 pass;
1828 except OSError as e:
1829 pass;
1830 if(outfile=="-"):
1831 catfp.seek(0, 0);
1832 if(hasattr(sys.stdout, "buffer")):
1833 shutil.copyfileobj(catfp, sys.stdout.buffer);
1834 else:
1835 shutil.copyfileobj(catfp, sys.stdout);
1836 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1837 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1838 catfp.seek(0, 0);
1839 upload_file_to_internet_file(catfp, outfile);
1840 if(returnfp):
1841 catfp.seek(0, 0);
1842 return catfp;
1843 else:
1844 catfp.close();
1845 return True;
1847 def AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
1848 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1849 if(os.path.exists(outfile)):
1850 try:
1851 os.unlink(outfile);
1852 except OSError as e:
1853 pass;
1854 if(outfile=="-"):
1855 verbose = False;
1856 catfpfp = BytesIO();
1857 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1858 catfp = outfile;
1859 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1860 catfp = BytesIO();
1861 else:
1862 fbasename = os.path.splitext(outfile)[0];
1863 fextname = os.path.splitext(outfile)[1];
1864 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
1865 catfp = AppendListsWithContent(inlist, catfp, dirlistfromtxt, filevalues, extradata, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose);
1866 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1867 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1868 try:
1869 catfp.flush();
1870 os.fsync(catfp.fileno());
1871 except io.UnsupportedOperation:
1872 pass;
1873 except AttributeError:
1874 pass;
1875 except OSError as e:
1876 pass;
1877 if(outfile=="-"):
1878 catfp.seek(0, 0);
1879 if(hasattr(sys.stdout, "buffer")):
1880 shutil.copyfileobj(catfp, sys.stdout.buffer);
1881 else:
1882 shutil.copyfileobj(catfp, sys.stdout);
1883 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1884 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1885 catfp.seek(0, 0);
1886 upload_file_to_internet_file(catfp, outfile);
1887 if(returnfp):
1888 catfp.seek(0, 0);
1889 return catfp;
1890 else:
1891 catfp.close();
1892 return True;
1894 def AppendInFileWithContentToOutFile(infile, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
1895 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, False, formatspecs);
1896 return AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt, compression, compresswholefile, compressionlevel, filevalues, extradata, followlink, checksumtype, formatspecs, verbose, returnfp);
1898 def PrintPermissionString(fchmode, ftype):
1899 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
1900 permissionstr = "";
1901 for fmodval in str(oct(fchmode))[-3:]:
1902 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
1903 if(ftype==0 or ftype==7):
1904 permissionstr = "-" + permissionstr;
1905 if(ftype==1):
1906 permissionstr = "h" + permissionstr;
1907 if(ftype==2):
1908 permissionstr = "l" + permissionstr;
1909 if(ftype==3):
1910 permissionstr = "c" + permissionstr;
1911 if(ftype==4):
1912 permissionstr = "b" + permissionstr;
1913 if(ftype==5):
1914 permissionstr = "d" + permissionstr;
1915 if(ftype==6):
1916 permissionstr = "f" + permissionstr;
1917 if(ftype==8):
1918 permissionstr = "D" + permissionstr;
1919 if(ftype==9):
1920 permissionstr = "p" + permissionstr;
1921 if(ftype==10):
1922 permissionstr = "w" + permissionstr;
1923 try:
1924 permissionoutstr = stat.filemode(fchmode);
1925 except AttributeError:
1926 permissionoutstr = permissionstr;
1927 except KeyError:
1928 permissionoutstr = permissionstr;
1929 return permissionoutstr;
1931 def PrintPermissionStringAlt(fchmode, ftype):
1932 permissions = {
1933 '0': '---', '1': '--x', '2': '-w-', '3': '-wx',
1934 '4': 'r--', '5': 'r-x', '6': 'rw-', '7': 'rwx'
1936 # Translate file mode into permission string
1937 permissionstr = ''.join([permissions[i] for i in str(oct(fchmode))[-3:]]);
1938 # Append file type indicator
1939 type_indicators = {
1940 0: '-', 1: 'h', 2: 'l', 3: 'c', 4: 'b',
1941 5: 'd', 6: 'f', 8: 'D', 9: 'p', 10: 'w'
1943 file_type = type_indicators.get(ftype, '-');
1944 permissionstr = file_type + permissionstr;
1945 try:
1946 permissionoutstr = stat.filemode(fchmode);
1947 except AttributeError:
1948 permissionoutstr = permissionstr;
1949 return permissionoutstr;
1951 def CompressionSupport():
1952 compression_list = [];
1953 try:
1954 import gzip;
1955 compression_list.append("gz");
1956 compression_list.append("gzip");
1957 except ImportError:
1958 '''return False;'''
1959 try:
1960 import bz2;
1961 compression_list.append("bz2");
1962 compression_list.append("bzip2");
1963 except ImportError:
1964 '''return False;'''
1965 try:
1966 import lz4;
1967 compression_list.append("lz4");
1968 except ImportError:
1969 '''return False;'''
1970 try:
1971 import lzo;
1972 compression_list.append("lzo");
1973 compression_list.append("lzop");
1974 except ImportError:
1975 '''return False;'''
1976 try:
1977 import zstandard;
1978 compression_list.append("zstd");
1979 compression_list.append("zstandard");
1980 except ImportError:
1981 '''return False;'''
1982 try:
1983 import lzma;
1984 compression_list.append("lzma");
1985 compression_list.append("xz");
1986 except ImportError:
1987 '''return False;'''
1988 return compression_list;
1990 def CheckCompressionType(infile, formatspecs=__file_format_dict__, closefp=True):
1991 if(hasattr(infile, "read") or hasattr(infile, "write")):
1992 catfp = infile;
1993 else:
1994 try:
1995 catfp = open(infile, "rb");
1996 except FileNotFoundError:
1997 return False;
1998 catfp.seek(0, 0);
1999 prefp = catfp.read(2);
2000 filetype = False;
2001 if(prefp==binascii.unhexlify("1f8b")):
2002 filetype = "gzip";
2003 catfp.seek(0, 0);
2004 prefp = catfp.read(3);
2005 if(prefp==binascii.unhexlify("425a68")):
2006 filetype = "bzip2";
2007 if(prefp==binascii.unhexlify("5d0000")):
2008 filetype = "lzma";
2009 catfp.seek(0, 0);
2010 prefp = catfp.read(4);
2011 if(prefp==binascii.unhexlify("28b52ffd")):
2012 filetype = "zstd";
2013 if(prefp==binascii.unhexlify("04224d18")):
2014 filetype = "lz4";
2015 if(prefp==binascii.unhexlify("504B0304")):
2016 filetype = "zipfile";
2017 catfp.seek(0, 0);
2018 prefp = catfp.read(5);
2019 if(prefp==binascii.unhexlify("7573746172")):
2020 filetype = "tarfile";
2021 catfp.seek(0, 0);
2022 prefp = catfp.read(6);
2023 if(prefp==binascii.unhexlify("fd377a585a00")):
2024 filetype = "lzma";
2025 if(prefp==binascii.unhexlify("377abcaf271c")):
2026 filetype = "7zipfile";
2027 catfp.seek(0, 0);
2028 prefp = catfp.read(7);
2029 if(prefp==binascii.unhexlify("526172211a0700")):
2030 filetype = "rarfile";
2031 if(prefp==binascii.unhexlify("43617446696c65")):
2032 filetype = "catfile";
2033 catfp.seek(0, 0);
2034 prefp = catfp.read(8);
2035 if(prefp==binascii.unhexlify("526172211a070100")):
2036 filetype = "rarfile";
2037 catfp.seek(0, 0);
2038 prefp = catfp.read(formatspecs['format_len']);
2039 if(prefp==binascii.unhexlify(formatspecs['format_hex'])):
2040 filetype = formatspecs['format_lower'];
2041 catfp.seek(0, 0);
2042 prefp = catfp.read(9);
2043 if(prefp==binascii.unhexlify("894c5a4f000d0a1a0a")):
2044 filetype = "lzo";
2045 catfp.seek(0, 0);
2046 prefp = catfp.read(10);
2047 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
2048 filetype = "tarfile";
2049 catfp.seek(0, 0);
2050 if(filetype=="gzip" or filetype=="bzip2" or filetype=="lzma" or filetype=="zstd" or filetype=="lz4"):
2051 if(is_tarfile(catfp)):
2052 filetype = "tarfile";
2053 if(not filetype):
2054 if(is_tarfile(catfp)):
2055 filetype = "tarfile";
2056 elif(zipfile.is_zipfile(catfp)):
2057 filetype = "zipfile";
2058 elif(rarfile.is_rarfile(catfp) or rarfile.is_rarfile_sfx(catfp)):
2059 filetype = "rarile";
2060 else:
2061 filetype = False;
2062 catfp.seek(0, 0);
2063 if(closefp):
2064 catfp.close();
2065 return filetype;
2067 def CheckCompressionTypeFromString(instring, formatspecs=__file_format_dict__, closefp=True):
2068 try:
2069 instringsfile = BytesIO(instring);
2070 except TypeError:
2071 instringsfile = BytesIO(instring.encode("UTF-8"));
2072 return CheckCompressionType(instringsfile, formatspecs, closefp);
2074 def GetCompressionMimeType(infile, formatspecs=__file_format_dict__):
2075 compresscheck = CheckCompressionType(fp, formatspecs, False);
2076 if(compresscheck=="gzip" or compresscheck=="gz"):
2077 return archivefile_gzip_mimetype;
2078 if(compresscheck=="bzip2" or compresscheck=="bz2"):
2079 return archivefile_bzip2_mimetype;
2080 if(compresscheck=="zstd" or compresscheck=="zstandard"):
2081 return archivefile_zstandard_mimetype;
2082 if(compresscheck=="lz4"):
2083 return archivefile_lz4_mimetype;
2084 if(compresscheck=="lzo" or compresscheck=="lzop"):
2085 return archivefile_lzop_mimetype;
2086 if(compresscheck=="lzma"):
2087 return archivefile_lzma_mimetype;
2088 if(compresscheck=="xz"):
2089 return archivefile_xz_mimetype;
2090 if(compresscheck=="catfile" or compresscheck=="cat" or compresscheck==formatspecs['format_lower']):
2091 return archivefile_cat_mimetype;
2092 if(not compresscheck):
2093 return False;
2094 return False;
2096 def UncompressArchiveFile(fp, formatspecs=__file_format_dict__):
2097 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
2098 return False;
2099 compresscheck = CheckCompressionType(fp, formatspecs, False);
2100 if(compresscheck=="gzip"):
2101 try:
2102 import gzip;
2103 except ImportError:
2104 return False;
2105 catfp = gzip.GzipFile(fileobj=fp, mode="rb");
2106 if(compresscheck=="bzip2"):
2107 try:
2108 import bz2;
2109 except ImportError:
2110 return False;
2111 catfp = BytesIO();
2112 catfp.write(bz2.decompress(fp.read()));
2113 if(compresscheck=="zstd"):
2114 try:
2115 import zstandard;
2116 except ImportError:
2117 return False;
2118 catfp = BytesIO();
2119 catfp.write(zstandard.decompress(fp.read()));
2120 if(compresscheck=="lz4"):
2121 try:
2122 import lz4.frame;
2123 except ImportError:
2124 return False;
2125 catfp = BytesIO();
2126 catfp.write(lz4.frame.decompress(fp.read()));
2127 if(compresscheck=="lzo" or compresscheck=="lzop"):
2128 try:
2129 import lzo;
2130 except ImportError:
2131 return False;
2132 catfp = BytesIO();
2133 catfp.write(lzo.decompress(fp.read()));
2134 if(compresscheck=="lzma" or compresscheck=="xz"):
2135 try:
2136 import lzma;
2137 except ImportError:
2138 return False;
2139 catfp = BytesIO();
2140 catfp.write(lzma.decompress(fp.read()));
2141 if(compresscheck=="catfile" or compresscheck==formatspecs['format_lower']):
2142 catfp = fp;
2143 if(not compresscheck):
2144 try:
2145 import lzma;
2146 except ImportError:
2147 return False;
2148 catfp = BytesIO();
2149 with fp as fpcontent:
2150 try:
2151 catfp.write(lzma.decompress(fp.read()));
2152 except lzma.LZMAError:
2153 return False;
2154 if(compresscheck!="catfile" or compresscheck!=formatspecs['format_lower']):
2155 fp.close();
2156 return catfp;
2158 create_alias_function("Uncompress", __file_format_name__, "", UncompressArchiveFile);
2160 def UncompressFile(infile, formatspecs=__file_format_dict__, mode="rb"):
2161 compresscheck = CheckCompressionType(infile, formatspecs, False);
2162 if(sys.version_info[0]==2 and compresscheck):
2163 if(mode=="rt"):
2164 mode = "r";
2165 if(mode=="wt"):
2166 mode = "w";
2167 try:
2168 if(compresscheck=="gzip"):
2169 try:
2170 import gzip;
2171 except ImportError:
2172 return False;
2173 try:
2174 filefp = gzip.open(infile, mode, encoding="UTF-8");
2175 except (ValueError, TypeError) as e:
2176 filefp = gzip.open(infile, mode);
2177 if(compresscheck=="bzip2"):
2178 try:
2179 import bz2;
2180 except ImportError:
2181 return False;
2182 try:
2183 filefp = bz2.open(infile, mode, encoding="UTF-8");
2184 except (ValueError, TypeError) as e:
2185 filefp = bz2.open(infile, mode);
2186 if(compresscheck=="zstd"):
2187 try:
2188 import zstandard;
2189 except ImportError:
2190 return False;
2191 try:
2192 filefp = zstandard.open(infile, mode, encoding="UTF-8");
2193 except (ValueError, TypeError) as e:
2194 filefp = zstandard.open(infile, mode);
2195 if(compresscheck=="lz4"):
2196 try:
2197 import lz4.frame;
2198 except ImportError:
2199 return False;
2200 try:
2201 filefp = lz4.frame.open(infile, mode, encoding="UTF-8");
2202 except (ValueError, TypeError) as e:
2203 filefp = lz4.frame.open(infile, mode);
2204 if(compresscheck=="lzo"):
2205 try:
2206 import lzo;
2207 except ImportError:
2208 return False;
2209 try:
2210 filefp = lzo.open(infile, mode, encoding="UTF-8");
2211 except (ValueError, TypeError) as e:
2212 filefp = lzo.open(infile, mode);
2213 if(compresscheck=="lzma"):
2214 try:
2215 import lzma;
2216 except ImportError:
2217 return False;
2218 try:
2219 filefp = lzma.open(infile, mode, encoding="UTF-8");
2220 except (ValueError, TypeError) as e:
2221 filefp = lzma.open(infile, mode);
2222 if(compresscheck=="catfile" or compresscheck==formatspecs['format_lower']):
2223 try:
2224 filefp = open(infile, mode, encoding="UTF-8");
2225 except (ValueError, TypeError) as e:
2226 filefp = open(infile, mode);
2227 if(not compresscheck):
2228 try:
2229 filefp = open(infile, mode, encoding="UTF-8");
2230 except (ValueError, TypeError) as e:
2231 filefp = open(infile, mode);
2232 except FileNotFoundError:
2233 return False;
2234 return filefp;
2236 def UncompressString(infile):
2237 compresscheck = CheckCompressionTypeFromString(infile, formatspecs, False);
2238 if(compresscheck=="gzip"):
2239 try:
2240 import gzip;
2241 except ImportError:
2242 return False;
2243 fileuz = gzip.decompress(infile);
2244 if(compresscheck=="bzip2"):
2245 try:
2246 import bz2;
2247 except ImportError:
2248 return False;
2249 fileuz = bz2.decompress(infile);
2250 if(compresscheck=="zstd"):
2251 try:
2252 import zstandard;
2253 except ImportError:
2254 return False;
2255 fileuz = zstandard.decompress(infile);
2256 if(compresscheck=="lz4"):
2257 try:
2258 import lz4.frame;
2259 except ImportError:
2260 return False;
2261 fileuz = lz4.frame.decompress(infile);
2262 if(compresscheck=="lzo"):
2263 try:
2264 import lzo;
2265 except ImportError:
2266 return False;
2267 fileuz = lzo.decompress(infile);
2268 if(compresscheck=="lzma"):
2269 try:
2270 import lzma;
2271 except ImportError:
2272 return False;
2273 fileuz = lzma.decompress(infile);
2274 if(not compresscheck):
2275 fileuz = infile;
2276 if(hasattr(fileuz, 'decode')):
2277 fileuz = fileuz.decode("UTF-8");
2278 return fileuz;
2280 def UncompressStringAlt(infile):
2281 filefp = StringIO();
2282 outstring = UncompressString(infile);
2283 filefp.write(outstring);
2284 filefp.seek(0, 0);
2285 return filefp;
2287 def CheckCompressionSubType(infile, formatspecs=__file_format_dict__, closefp=True):
2288 compresscheck = CheckCompressionType(infile, formatspecs, False);
2289 if(not compresscheck):
2290 fextname = os.path.splitext(infile)[1];
2291 if(fextname==".gz"):
2292 compresscheck = "gzip";
2293 elif(fextname==".bz2"):
2294 compresscheck = "bzip2";
2295 elif(fextname==".zst"):
2296 compresscheck = "zstd";
2297 elif(fextname==".lz4"):
2298 compresscheck = "lz4";
2299 elif(fextname==".lzo" or fextname==".lzop"):
2300 compresscheck = "lzo";
2301 elif(fextname==".lzma" or fextname==".xz"):
2302 compresscheck = "lzma";
2303 else:
2304 return False;
2305 if(compresscheck=="gzip" or compresscheck=="bzip2" or compresscheck=="lzma" or compresscheck=="zstd" or compresscheck=="lz4"):
2306 if(is_tarfile(infile)):
2307 filetype = "tarfile";
2308 if(not compresscheck):
2309 if(is_tarfile(infile)):
2310 return "tarfile";
2311 elif(zipfile.is_zipfile(infile)):
2312 return "zipfile";
2313 elif(rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile)):
2314 return "rarile";
2315 else:
2316 return False;
2317 return False;
2318 if(compresscheck=="catfile"):
2319 return "catfile";
2320 if(compresscheck==formatspecs['format_lower']):
2321 return formatspecs['format_lower'];
2322 if(compresscheck=="tarfile"):
2323 return "tarfile";
2324 if(compresscheck=="zipfile"):
2325 return "zipfile";
2326 if(rarfile_support and compresscheck=="rarfile"):
2327 return "rarfile";
2328 if(py7zr_support and compresscheck=="7zipfile"):
2329 return "7zipfile";
2330 if(hasattr(infile, "read") or hasattr(infile, "write")):
2331 catfp = UncompressArchiveFile(infile, formatspecs['format_lower']);
2332 else:
2333 try:
2334 if(compresscheck=="gzip"):
2335 try:
2336 import gzip;
2337 except ImportError:
2338 return False;
2339 catfp = gzip.GzipFile(infile, "rb");
2340 if(compresscheck=="bzip2"):
2341 try:
2342 import bz2;
2343 except ImportError:
2344 return False;
2345 catfp = bz2.BZ2File(infile, "rb");
2346 if(compresscheck=="lz4"):
2347 try:
2348 import lz4.frame;
2349 except ImportError:
2350 return False;
2351 catfp = lz4.frame.open(infile, "rb");
2352 if(compresscheck=="zstd"):
2353 try:
2354 import zstandard;
2355 except ImportError:
2356 return False;
2357 catfp = zstandard.open(infile, "rb");
2358 if(compresscheck=="lzma" or compresscheck=="xz"):
2359 try:
2360 import lzma;
2361 except ImportError:
2362 return False;
2363 catfp = lzma.open(infile, "rb");
2364 except FileNotFoundError:
2365 return False;
2366 filetype = False;
2367 prefp = catfp.read(5);
2368 if(prefp==binascii.unhexlify("7573746172")):
2369 filetype = "tarfile";
2370 catfp.seek(0, 0);
2371 prefp = catfp.read(7);
2372 if(prefp==binascii.unhexlify("43617446696c65")):
2373 filetype = "catfile";
2374 catfp.seek(0, 0);
2375 prefp = catfp.read(formatspecs['format_len']);
2376 if(prefp==binascii.unhexlify(formatspecs['format_hex'])):
2377 filetype = formatspecs['format_lower'];
2378 catfp.seek(0, 0);
2379 prefp = catfp.read(10);
2380 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
2381 filetype = "tarfile";
2382 catfp.seek(0, 0);
2383 if(closefp):
2384 catfp.close();
2385 return filetype;
2387 def GZipCompress(data, compresslevel=9):
2388 try:
2389 import gzip;
2390 except ImportError:
2391 return False;
2392 tmpfp = tempfile.NamedTemporaryFile("wb", delete=False);
2393 tmpfp.close();
2394 tmpfp = gzip.GzipFile(tmpfp.name, mode="wb", compresslevel=compresslevel);
2395 tmpfp.write(data);
2396 tmpfp.close();
2397 try:
2398 catfp = open(tmpfp.name, "rb");
2399 except FileNotFoundError:
2400 return False;
2401 catdata = catfp.read();
2402 catfp.close();
2403 return catdata;
2405 def CompressArchiveFile(fp, compression="auto", compressionlevel=None, formatspecs=__file_format_dict__):
2406 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
2407 return False;
2408 fp.seek(0, 0);
2409 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
2410 compression = "auto";
2411 if(compression not in compressionlist and compression is None):
2412 compression = "auto";
2413 if(compression=="gzip"):
2414 try:
2415 import gzip;
2416 except ImportError:
2417 return False;
2418 catfp = BytesIO();
2419 if(compressionlevel is None):
2420 compressionlevel = 9;
2421 else:
2422 compressionlevel = int(compressionlevel);
2423 catfp.write(gzip.compress(fp.read(), compresslevel=compressionlevel));
2424 if(compression=="bzip2"):
2425 try:
2426 import bz2;
2427 except ImportError:
2428 return False;
2429 catfp = BytesIO();
2430 if(compressionlevel is None):
2431 compressionlevel = 9;
2432 else:
2433 compressionlevel = int(compressionlevel);
2434 catfp.write(bz2.compress(fp.read(), compresslevel=compressionlevel));
2435 if(compression=="lz4"):
2436 try:
2437 import lz4.frame;
2438 except ImportError:
2439 return False;
2440 catfp = BytesIO();
2441 if(compressionlevel is None):
2442 compressionlevel = 9;
2443 else:
2444 compressionlevel = int(compressionlevel);
2445 catfp.write(lz4.frame.compress(fp.read(), compression_level=compressionlevel));
2446 if(compression=="lzo" or compression=="lzop"):
2447 try:
2448 import lzo;
2449 except ImportError:
2450 return False;
2451 catfp = BytesIO();
2452 if(compressionlevel is None):
2453 compressionlevel = 9;
2454 else:
2455 compressionlevel = int(compressionlevel);
2456 catfp.write(lzo.compress(fp.read(), compresslevel=compressionlevel));
2457 if(compression=="zstd"):
2458 try:
2459 import zstandard;
2460 except ImportError:
2461 return False;
2462 catfp = BytesIO();
2463 if(compressionlevel is None):
2464 compressionlevel = 10;
2465 else:
2466 compressionlevel = int(compressionlevel);
2467 catfp.write(zstandard.compress(fp.read(), level=compressionlevel));
2468 if(compression=="lzma"):
2469 try:
2470 import lzma;
2471 except ImportError:
2472 return False;
2473 catfp = BytesIO();
2474 if(compressionlevel is None):
2475 compressionlevel = 9;
2476 else:
2477 compressionlevel = int(compressionlevel);
2478 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]));
2479 if(compression=="xz"):
2480 try:
2481 import lzma;
2482 except ImportError:
2483 return False;
2484 catfp = BytesIO();
2485 if(compressionlevel is None):
2486 compressionlevel = 9;
2487 else:
2488 compressionlevel = int(compressionlevel);
2489 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]));
2490 if(compression=="auto" or compression is None):
2491 catfp = fp;
2492 catfp.seek(0, 0);
2493 return catfp;
2495 create_alias_function("Compress", __file_format_name__, "", CompressArchiveFile);
2497 def CompressOpenFile(outfile, compressionenable=True, compressionlevel=None):
2498 if(outfile is None):
2499 return False;
2500 fbasename = os.path.splitext(outfile)[0];
2501 fextname = os.path.splitext(outfile)[1];
2502 if(compressionlevel is None and fextname!=".zst"):
2503 compressionlevel = 9;
2504 elif(compressionlevel is None and fextname==".zst"):
2505 compressionlevel = 10;
2506 else:
2507 compressionlevel = int(compressionlevel);
2508 if(sys.version_info[0]==2):
2509 mode = "w";
2510 else:
2511 mode = "wb";
2512 try:
2513 if(fextname not in outextlistwd or not compressionenable):
2514 try:
2515 outfp = open(outfile, "wb", encoding="UTF-8");
2516 except (ValueError, TypeError) as e:
2517 outfp = open(outfile, "wb");
2518 elif(fextname==".gz"):
2519 try:
2520 import gzip;
2521 except ImportError:
2522 return False;
2523 try:
2524 outfp = gzip.open(outfile, mode, compressionlevel, encoding="UTF-8");
2525 except (ValueError, TypeError) as e:
2526 outfp = gzip.open(outfile, mode, compressionlevel);
2527 elif(fextname==".bz2"):
2528 try:
2529 import bz2;
2530 except ImportError:
2531 return False;
2532 try:
2533 outfp = bz2.open(outfile, mode, compressionlevel, encoding="UTF-8");
2534 except (ValueError, TypeError) as e:
2535 outfp = bz2.open(outfile, mode, compressionlevel);
2536 elif(fextname==".zst"):
2537 try:
2538 import zstandard;
2539 except ImportError:
2540 return False;
2541 try:
2542 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel), encoding="UTF-8");
2543 except (ValueError, TypeError) as e:
2544 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel));
2545 elif(fextname==".xz"):
2546 try:
2547 import lzma;
2548 except ImportError:
2549 return False;
2550 try:
2551 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}], encoding="UTF-8");
2552 except (ValueError, TypeError) as e:
2553 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]);
2554 elif(fextname==".lz4"):
2555 try:
2556 import lz4.frame;
2557 except ImportError:
2558 return False;
2559 try:
2560 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel, encoding="UTF-8");
2561 except (ValueError, TypeError) as e:
2562 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel);
2563 elif(fextname==".lzo"):
2564 try:
2565 import lzo;
2566 except ImportError:
2567 return False;
2568 try:
2569 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel, encoding="UTF-8");
2570 except (ValueError, TypeError) as e:
2571 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel);
2572 elif(fextname==".lzma"):
2573 try:
2574 import lzma;
2575 except ImportError:
2576 return False;
2577 try:
2578 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}], encoding="UTF-8");
2579 except (ValueError, TypeError) as e:
2580 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]);
2581 except FileNotFoundError:
2582 return False;
2583 return outfp;
2585 def GetDevMajorMinor(fdev):
2586 retdev = [];
2587 if(hasattr(os, "minor")):
2588 retdev.append(os.minor(fdev));
2589 else:
2590 retdev.append(0);
2591 if(hasattr(os, "major")):
2592 retdev.append(os.major(fdev));
2593 else:
2594 retdev.append(0);
2595 return retdev;
2597 def CheckSumSupport(checkfor, guaranteed=True):
2598 if(guaranteed):
2599 try:
2600 hash_list = sorted(list(hashlib.algorithms_guaranteed));
2601 except AttributeError:
2602 hash_list = sorted(list(hashlib.algorithms));
2603 else:
2604 try:
2605 hash_list = sorted(list(hashlib.algorithms_available));
2606 except AttributeError:
2607 hash_list = sorted(list(hashlib.algorithms));
2608 checklistout = sorted(hash_list + ['adler32', 'crc16', 'crc16_ansi', 'crc16_ibm', 'crc16_ccitt', 'crc32', 'crc64', 'crc64_ecma', 'crc64_iso', 'none']);
2609 if(checkfor in checklistout):
2610 return True;
2611 else:
2612 return False;
2614 def CheckSumSupportAlt(checkfor, guaranteed=True):
2615 if(guaranteed):
2616 try:
2617 hash_list = sorted(list(hashlib.algorithms_guaranteed));
2618 except AttributeError:
2619 hash_list = sorted(list(hashlib.algorithms));
2620 else:
2621 try:
2622 hash_list = sorted(list(hashlib.algorithms_available));
2623 except AttributeError:
2624 hash_list = sorted(list(hashlib.algorithms));
2625 checklistout = hash_list;
2626 if(checkfor in checklistout):
2627 return True;
2628 else:
2629 return False;
2631 def PackArchiveFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2632 advancedlist = formatspecs['use_advanced_list'];
2633 altinode = formatspecs['use_alt_inode'];
2634 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2635 outfile = RemoveWindowsPath(outfile);
2636 checksumtype = checksumtype.lower();
2637 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
2638 checksumtype="crc32";
2639 if(checksumtype=="none"):
2640 checksumtype = "";
2641 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
2642 compression = "auto";
2643 if(compression not in compressionlist and compression is None):
2644 compression = "auto";
2645 if(verbose):
2646 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2647 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2648 if(os.path.exists(outfile)):
2649 try:
2650 os.unlink(outfile);
2651 except OSError as e:
2652 pass;
2653 if(outfile=="-"):
2654 verbose = False;
2655 catfp = BytesIO();
2656 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2657 catfp = outfile;
2658 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2659 catfp = BytesIO();
2660 else:
2661 fbasename = os.path.splitext(outfile)[0];
2662 fextname = os.path.splitext(outfile)[1];
2663 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
2664 catver = formatspecs['format_ver'];
2665 fileheaderver = str(int(catver.replace(".", "")));
2666 infilelist = [];
2667 if(infiles=="-"):
2668 for line in sys.stdin:
2669 infilelist.append(line.strip());
2670 infilelist = list(filter(None, infilelist));
2671 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
2672 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
2673 return False;
2674 with UncompressFile(infiles, formatspecs, "r") as finfile:
2675 for line in finfile:
2676 infilelist.append(line.strip());
2677 infilelist = list(filter(None, infilelist));
2678 else:
2679 if(isinstance(infiles, (list, tuple, ))):
2680 infilelist = list(filter(None, infiles));
2681 elif(isinstance(infiles, (str, ))):
2682 infilelist = list(filter(None, [infiles]));
2683 if(advancedlist):
2684 GetDirList = ListDirAdvanced(infilelist, followlink, False);
2685 else:
2686 GetDirList = ListDir(infilelist, followlink, False);
2687 if(not GetDirList):
2688 return False;
2689 curinode = 0;
2690 curfid = 0;
2691 inodelist = [];
2692 inodetofile = {};
2693 filetoinode = {};
2694 inodetocatinode = {};
2695 numfiles = int(len(GetDirList));
2696 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
2697 for curfname in GetDirList:
2698 if(re.findall("^[.|/]", curfname)):
2699 fname = curfname;
2700 else:
2701 fname = "./"+curfname;
2702 if(verbose):
2703 VerbosePrintOut(fname);
2704 if(not followlink or followlink is None):
2705 fstatinfo = os.lstat(fname);
2706 else:
2707 fstatinfo = os.stat(fname);
2708 fpremode = fstatinfo.st_mode;
2709 finode = fstatinfo.st_ino;
2710 flinkcount = fstatinfo.st_nlink;
2711 ftype = 0;
2712 if(stat.S_ISREG(fpremode)):
2713 ftype = 0;
2714 elif(stat.S_ISLNK(fpremode)):
2715 ftype = 2;
2716 elif(stat.S_ISCHR(fpremode)):
2717 ftype = 3;
2718 elif(stat.S_ISBLK(fpremode)):
2719 ftype = 4;
2720 elif(stat.S_ISDIR(fpremode)):
2721 ftype = 5;
2722 elif(stat.S_ISFIFO(fpremode)):
2723 ftype = 6;
2724 elif(stat.S_ISSOCK(fpremode)):
2725 ftype = 8;
2726 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
2727 ftype = 9;
2728 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
2729 ftype = 10;
2730 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
2731 ftype = 111;
2732 else:
2733 ftype = 0;
2734 flinkname = "";
2735 fcurfid = format(int(curfid), 'x').lower();
2736 if(not followlink and finode!=0):
2737 if(ftype!=1):
2738 if(finode in inodelist):
2739 ftype = 1;
2740 flinkname = inodetofile[finode];
2741 if(altinode):
2742 fcurinode = format(int(finode), 'x').lower();
2743 else:
2744 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
2745 if(finode not in inodelist):
2746 inodelist.append(finode);
2747 inodetofile.update({finode: fname});
2748 inodetocatinode.update({finode: curinode});
2749 if(altinode):
2750 fcurinode = format(int(finode), 'x').lower();
2751 else:
2752 fcurinode = format(int(curinode), 'x').lower();
2753 curinode = curinode + 1;
2754 else:
2755 fcurinode = format(int(curinode), 'x').lower();
2756 curinode = curinode + 1;
2757 curfid = curfid + 1;
2758 if(ftype==2):
2759 flinkname = os.readlink(fname);
2760 fdev = fstatinfo.st_dev;
2761 getfdev = GetDevMajorMinor(fdev);
2762 fdev_minor = getfdev[0];
2763 fdev_major = getfdev[1];
2764 frdev = fstatinfo.st_dev;
2765 if(hasattr(fstatinfo, "st_rdev")):
2766 frdev = fstatinfo.st_rdev;
2767 else:
2768 frdev = fstatinfo.st_dev;
2769 getfrdev = GetDevMajorMinor(frdev);
2770 frdev_minor = getfrdev[0];
2771 frdev_major = getfrdev[1];
2772 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
2773 fsize = format(int("0"), 'x').lower();
2774 elif(ftype==0 or ftype==7):
2775 fsize = format(int(fstatinfo.st_size), 'x').lower();
2776 else:
2777 fsize = format(int(fstatinfo.st_size)).lower();
2778 fatime = format(int(fstatinfo.st_atime), 'x').lower();
2779 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
2780 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
2781 if(hasattr(fstatinfo, "st_birthtime")):
2782 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
2783 else:
2784 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
2785 fmode = format(int(fstatinfo.st_mode), 'x').lower();
2786 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
2787 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
2788 fuid = format(int(fstatinfo.st_uid), 'x').lower();
2789 fgid = format(int(fstatinfo.st_gid), 'x').lower();
2790 funame = "";
2791 try:
2792 import pwd;
2793 try:
2794 userinfo = pwd.getpwuid(fstatinfo.st_uid);
2795 funame = userinfo.pw_name;
2796 except KeyError:
2797 funame = "";
2798 except ImportError:
2799 funame = "";
2800 fgname = "";
2801 try:
2802 import grp;
2803 try:
2804 groupinfo = grp.getgrgid(fstatinfo.st_gid);
2805 fgname = groupinfo.gr_name;
2806 except KeyError:
2807 fgname = "";
2808 except ImportError:
2809 fgname = "";
2810 fdev_minor = format(int(fdev_minor), 'x').lower();
2811 fdev_major = format(int(fdev_major), 'x').lower();
2812 frdev_minor = format(int(frdev_minor), 'x').lower();
2813 frdev_major = format(int(frdev_major), 'x').lower();
2814 finode = format(int(finode), 'x').lower();
2815 flinkcount = format(int(flinkcount), 'x').lower();
2816 if(hasattr(fstatinfo, "st_file_attributes")):
2817 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
2818 else:
2819 fwinattributes = format(int(0), 'x').lower();
2820 fcompression = "";
2821 fcsize = format(int(0), 'x').lower();
2822 fcontents = BytesIO();
2823 if(ftype==0 or ftype==7):
2824 with open(fname, "rb") as fpc:
2825 shutil.copyfileobj(fpc, fcontents);
2826 if(not compresswholefile):
2827 fcontents.seek(0, 2);
2828 ucfsize = fcontents.tell();
2829 fcontents.seek(0, 0);
2830 if(compression=="auto"):
2831 ilsize = len(compressionlistalt);
2832 ilmin = 0;
2833 ilcsize = [];
2834 while(ilmin < ilsize):
2835 cfcontents = BytesIO();
2836 shutil.copyfileobj(fcontents, cfcontents);
2837 fcontents.seek(0, 0);
2838 cfcontents.seek(0, 0);
2839 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
2840 if(cfcontents):
2841 cfcontents.seek(0, 2);
2842 ilcsize.append(cfcontents.tell());
2843 cfcontents.close();
2844 else:
2845 try:
2846 ilcsize.append(sys.maxint);
2847 except AttributeError:
2848 ilcsize.append(sys.maxsize);
2849 ilmin = ilmin + 1;
2850 ilcmin = ilcsize.index(min(ilcsize));
2851 compression = compressionlistalt[ilcmin];
2852 fcontents.seek(0, 0);
2853 cfcontents = BytesIO();
2854 shutil.copyfileobj(fcontents, cfcontents);
2855 cfcontents.seek(0, 0);
2856 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
2857 cfcontents.seek(0, 2);
2858 cfsize = cfcontents.tell();
2859 if(ucfsize > cfsize):
2860 fcsize = format(int(cfsize), 'x').lower();
2861 fcompression = compression;
2862 fcontents.close();
2863 fcontents = cfcontents;
2864 if(fcompression=="none"):
2865 fcompression = "";
2866 if(followlink and (ftype==1 or ftype==2)):
2867 flstatinfo = os.stat(flinkname);
2868 with open(flinkname, "rb") as fpc:
2869 shutil.copyfileobj(fpc, fcontents);
2870 if(not compresswholefile):
2871 fcontents.seek(0, 2);
2872 ucfsize = fcontents.tell();
2873 fcontents.seek(0, 0);
2874 if(compression=="auto"):
2875 ilsize = len(compressionlistalt);
2876 ilmin = 0;
2877 ilcsize = [];
2878 while(ilmin < ilsize):
2879 cfcontents = BytesIO();
2880 shutil.copyfileobj(fcontents, cfcontents);
2881 fcontents.seek(0, 0);
2882 cfcontents.seek(0, 0);
2883 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
2884 if(cfcontents):
2885 cfcontents.seek(0, 2);
2886 ilcsize.append(cfcontents.tell());
2887 cfcontents.close();
2888 else:
2889 try:
2890 ilcsize.append(sys.maxint);
2891 except AttributeError:
2892 ilcsize.append(sys.maxsize);
2893 ilmin = ilmin + 1;
2894 ilcmin = ilcsize.index(min(ilcsize));
2895 compression = compressionlistalt[ilcmin];
2896 fcontents.seek(0, 0);
2897 cfcontents = BytesIO();
2898 shutil.copyfileobj(fcontents, cfcontents);
2899 cfcontents.seek(0, 0);
2900 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
2901 cfcontents.seek(0, 2);
2902 cfsize = cfcontents.tell();
2903 if(ucfsize > cfsize):
2904 fcsize = format(int(cfsize), 'x').lower();
2905 fcompression = compression;
2906 fcontents.close();
2907 fcontents = cfcontents;
2908 fcontents.seek(0, 0);
2909 ftypehex = format(ftype, 'x').lower();
2910 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
2911 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
2912 fcontents.close();
2913 if(numfiles>0):
2914 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
2915 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
2916 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2917 try:
2918 catfp.flush();
2919 os.fsync(catfp.fileno());
2920 except io.UnsupportedOperation:
2921 pass;
2922 except AttributeError:
2923 pass;
2924 except OSError as e:
2925 pass;
2926 if(outfile=="-"):
2927 catfp.seek(0, 0);
2928 if(hasattr(sys.stdout, "buffer")):
2929 shutil.copyfileobj(catfp, sys.stdout.buffer);
2930 else:
2931 shutil.copyfileobj(catfp, sys.stdout);
2932 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2933 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2934 catfp.seek(0, 0);
2935 upload_file_to_internet_file(catfp, outfile);
2936 if(returnfp):
2937 catfp.seek(0, 0);
2938 return catfp;
2939 else:
2940 catfp.close();
2941 return True;
2943 create_alias_function("Pack", __file_format_name__, "", PackArchiveFile);
2945 if(hasattr(shutil, "register_archive_format")):
2946 def PackArchiveFileFunc(archive_name, source_dir, **kwargs):
2947 return PackArchiveFile(source_dir, archive_name, False, "auto", True, None, False, "crc32", [], __file_format_dict__['format_delimiter'], False, False);
2948 create_alias_function("Pack", __file_format_name__, "Func", PackArchiveFileFunc);
2950 def PackArchiveFileFromDirList(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2951 return PackArchiveFile(infiles, outfile, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, returnfp);
2953 create_alias_function("Pack", __file_format_name__, "FromDirList", PackArchiveFileFromDirList);
2955 def PackArchiveFileFromTarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2956 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2957 outfile = RemoveWindowsPath(outfile);
2958 checksumtype = checksumtype.lower();
2959 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
2960 checksumtype="crc32";
2961 if(checksumtype=="none"):
2962 checksumtype = "";
2963 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
2964 compression = "auto";
2965 if(compression not in compressionlist and compression is None):
2966 compression = "auto";
2967 if(verbose):
2968 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2969 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2970 if(os.path.exists(outfile)):
2971 try:
2972 os.unlink(outfile);
2973 except OSError as e:
2974 pass;
2975 if(outfile=="-"):
2976 verbose = False;
2977 catfp = BytesIO();
2978 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2979 catfp = outfile;
2980 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2981 catfp = BytesIO();
2982 else:
2983 fbasename = os.path.splitext(outfile)[0];
2984 fextname = os.path.splitext(outfile)[1];
2985 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
2986 catver = formatspecs['format_ver'];
2987 fileheaderver = str(int(catver.replace(".", "")));
2988 curinode = 0;
2989 curfid = 0;
2990 inodelist = [];
2991 inodetofile = {};
2992 filetoinode = {};
2993 inodetocatinode = {};
2994 if(infile=="-"):
2995 infile = BytesIO();
2996 if(hasattr(sys.stdin, "buffer")):
2997 shutil.copyfileobj(sys.stdin.buffer, infile);
2998 else:
2999 shutil.copyfileobj(sys.stdin, infile);
3000 infile.seek(0, 0);
3001 if(not infile):
3002 return False;
3003 infile.seek(0, 0);
3004 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3005 infile = download_file_from_internet_file(infile);
3006 infile.seek(0, 0);
3007 if(not infile):
3008 return False;
3009 infile.seek(0, 0);
3010 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
3011 return False;
3012 elif(os.path.exists(infile) and os.path.isfile(infile)):
3013 try:
3014 if(not tarfile.is_tarfile(infile)):
3015 return False;
3016 except AttributeError:
3017 if(not is_tarfile(infile)):
3018 return False;
3019 else:
3020 return False;
3021 try:
3022 if(hasattr(infile, "read") or hasattr(infile, "write")):
3023 tarfp = tarfile.open(fileobj=infile, mode="r");
3024 else:
3025 tarfp = tarfile.open(infile, "r");
3026 except FileNotFoundError:
3027 return False;
3028 numfiles = int(len(tarfp.getmembers()));
3029 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3030 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
3031 if(re.findall("^[.|/]", member.name)):
3032 fname = member.name;
3033 else:
3034 fname = "./"+member.name;
3035 if(verbose):
3036 VerbosePrintOut(fname);
3037 fpremode = member.mode;
3038 ffullmode = member.mode;
3039 flinkcount = 0;
3040 ftype = 0;
3041 if(member.isreg()):
3042 ffullmode = member.mode + stat.S_IFREG;
3043 ftype = 0;
3044 elif(member.isdev()):
3045 ffullmode = member.mode;
3046 ftype = 7;
3047 elif(member.islnk()):
3048 ffullmode = member.mode + stat.S_IFREG;
3049 ftype = 1;
3050 elif(member.issym()):
3051 ffullmode = member.mode + stat.S_IFLNK;
3052 ftype = 2;
3053 elif(member.ischr()):
3054 ffullmode = member.mode + stat.S_IFCHR;
3055 ftype = 3;
3056 elif(member.isblk()):
3057 ffullmode = member.mode + stat.S_IFBLK;
3058 ftype = 4;
3059 elif(member.isdir()):
3060 ffullmode = member.mode + stat.S_IFDIR;
3061 ftype = 5;
3062 elif(member.isfifo()):
3063 ffullmode = member.mode + stat.S_IFIFO;
3064 ftype = 6;
3065 elif(member.issparse()):
3066 ffullmode = member.mode;
3067 ftype = 12;
3068 else:
3069 ffullmode = member.mode;
3070 ftype = 0;
3071 flinkname = "";
3072 fcurfid = format(int(curfid), 'x').lower();
3073 fcurinode = format(int(curfid), 'x').lower();
3074 curfid = curfid + 1;
3075 if(ftype==2):
3076 flinkname = member.linkname;
3077 fdev_minor = format(int(member.devminor), 'x').lower();
3078 fdev_major = format(int(member.devmajor), 'x').lower();
3079 frdev_minor = format(int(member.devminor), 'x').lower();
3080 frdev_major = format(int(member.devmajor), 'x').lower();
3081 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
3082 fsize = format(int("0"), 'x').lower();
3083 elif(ftype==0 or ftype==7):
3084 fsize = format(int(member.size), 'x').lower();
3085 else:
3086 fsize = format(int(member.size), 'x').lower();
3087 fatime = format(int(member.mtime), 'x').lower();
3088 fmtime = format(int(member.mtime), 'x').lower();
3089 fctime = format(int(member.mtime), 'x').lower();
3090 fbtime = format(int(member.mtime), 'x').lower();
3091 fmode = format(int(ffullmode), 'x').lower();
3092 fchmode = format(int(stat.S_IMODE(ffullmode)), 'x').lower();
3093 ftypemod = format(int(stat.S_IFMT(ffullmode)), 'x').lower();
3094 fuid = format(int(member.uid), 'x').lower();
3095 fgid = format(int(member.gid), 'x').lower();
3096 funame = member.uname;
3097 fgname = member.gname;
3098 flinkcount = format(int(flinkcount), 'x').lower();
3099 fwinattributes = format(int(0), 'x').lower();
3100 fcompression = "";
3101 fcsize = format(int(0), 'x').lower();
3102 fcontents = BytesIO();
3103 if(ftype==0 or ftype==7):
3104 with tarfp.extractfile(member) as fpc:
3105 shutil.copyfileobj(fpc, fcontents);
3106 if(not compresswholefile):
3107 fcontents.seek(0, 2);
3108 ucfsize = fcontents.tell();
3109 fcontents.seek(0, 0);
3110 if(compression=="auto"):
3111 ilsize = len(compressionlistalt);
3112 ilmin = 0;
3113 ilcsize = [];
3114 while(ilmin < ilsize):
3115 cfcontents = BytesIO();
3116 shutil.copyfileobj(fcontents, cfcontents);
3117 fcontents.seek(0, 0);
3118 cfcontents.seek(0, 0);
3119 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3120 if(cfcontents):
3121 cfcontents.seek(0, 2);
3122 ilcsize.append(cfcontents.tell());
3123 cfcontents.close();
3124 else:
3125 try:
3126 ilcsize.append(sys.maxint);
3127 except AttributeError:
3128 ilcsize.append(sys.maxsize);
3129 ilmin = ilmin + 1;
3130 ilcmin = ilcsize.index(min(ilcsize));
3131 compression = compressionlistalt[ilcmin];
3132 fcontents.seek(0, 0);
3133 cfcontents = BytesIO();
3134 shutil.copyfileobj(fcontents, cfcontents);
3135 cfcontents.seek(0, 0);
3136 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3137 cfcontents.seek(0, 2);
3138 cfsize = cfcontents.tell();
3139 if(ucfsize > cfsize):
3140 fcsize = format(int(cfsize), 'x').lower();
3141 fcompression = compression;
3142 fcontents.close();
3143 fcontents = cfcontents;
3144 if(fcompression=="none"):
3145 fcompression = "";
3146 fcontents.seek(0, 0);
3147 ftypehex = format(ftype, 'x').lower();
3148 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3149 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3150 fcontents.close();
3151 if(numfiles>0):
3152 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3153 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3154 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3155 try:
3156 catfp.flush();
3157 os.fsync(catfp.fileno());
3158 except io.UnsupportedOperation:
3159 pass;
3160 except AttributeError:
3161 pass;
3162 except OSError as e:
3163 pass;
3164 if(outfile=="-"):
3165 catfp.seek(0, 0);
3166 if(hasattr(sys.stdout, "buffer")):
3167 shutil.copyfileobj(catfp, sys.stdout.buffer);
3168 else:
3169 shutil.copyfileobj(catfp, sys.stdout);
3170 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3171 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3172 catfp.seek(0, 0);
3173 upload_file_to_internet_file(catfp, outfile);
3174 if(returnfp):
3175 catfp.seek(0, 0);
3176 return catfp;
3177 else:
3178 catfp.close();
3179 return True;
3181 create_alias_function("Pack", __file_format_name__, "FromTarFile", PackArchiveFileFromTarFile);
3183 def PackArchiveFileFromZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3184 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3185 outfile = RemoveWindowsPath(outfile);
3186 checksumtype = checksumtype.lower();
3187 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3188 checksumtype="crc32";
3189 if(checksumtype=="none"):
3190 checksumtype = "";
3191 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3192 compression = "auto";
3193 if(compression not in compressionlist and compression is None):
3194 compression = "auto";
3195 if(verbose):
3196 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3197 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3198 if(os.path.exists(outfile)):
3199 try:
3200 os.unlink(outfile);
3201 except OSError as e:
3202 pass;
3203 if(outfile=="-"):
3204 verbose = False;
3205 catfp = BytesIO();
3206 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3207 catfp = outfile;
3208 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3209 catfp = BytesIO();
3210 else:
3211 fbasename = os.path.splitext(outfile)[0];
3212 fextname = os.path.splitext(outfile)[1];
3213 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3214 catver = formatspecs['format_ver'];
3215 fileheaderver = str(int(catver.replace(".", "")));
3216 curinode = 0;
3217 curfid = 0;
3218 inodelist = [];
3219 inodetofile = {};
3220 filetoinode = {};
3221 inodetocatinode = {};
3222 if(infile=="-"):
3223 infile = BytesIO();
3224 if(hasattr(sys.stdin, "buffer")):
3225 shutil.copyfileobj(sys.stdin.buffer, infile);
3226 else:
3227 shutil.copyfileobj(sys.stdin, infile);
3228 infile.seek(0, 0);
3229 if(not infile):
3230 return False;
3231 infile.seek(0, 0);
3232 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3233 infile = download_file_from_internet_file(infile);
3234 infile.seek(0, 0);
3235 if(not infile):
3236 return False;
3237 infile.seek(0, 0);
3238 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
3239 return False;
3240 else:
3241 return False;
3242 if(not zipfile.is_zipfile(infile)):
3243 return False;
3244 try:
3245 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
3246 except FileNotFoundError:
3247 return False;
3248 ziptest = zipfp.testzip();
3249 if(ziptest):
3250 VerbosePrintOut("Bad file found!");
3251 numfiles = int(len(zipfp.infolist()));
3252 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3253 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
3254 if(re.findall("^[.|/]", member.filename)):
3255 fname = member.filename;
3256 else:
3257 fname = "./"+member.filename;
3258 zipinfo = zipfp.getinfo(member.filename);
3259 if(verbose):
3260 VerbosePrintOut(fname);
3261 if(not member.is_dir()):
3262 fpremode = int(stat.S_IFREG + 438);
3263 elif(member.is_dir()):
3264 fpremode = int(stat.S_IFDIR + 511);
3265 flinkcount = 0;
3266 ftype = 0;
3267 if(not member.is_dir()):
3268 ftype = 0;
3269 elif(member.is_dir()):
3270 ftype = 5;
3271 flinkname = "";
3272 fcurfid = format(int(curfid), 'x').lower();
3273 fcurinode = format(int(curfid), 'x').lower();
3274 curfid = curfid + 1;
3275 fdev_minor = format(int(0), 'x').lower();
3276 fdev_major = format(int(0), 'x').lower();
3277 frdev_minor = format(int(0), 'x').lower();
3278 frdev_major = format(int(0), 'x').lower();
3279 if(ftype==5):
3280 fsize = format(int("0"), 'x').lower();
3281 elif(ftype==0):
3282 fsize = format(int(member.file_size), 'x').lower();
3283 else:
3284 fsize = format(int(member.file_size), 'x').lower();
3285 fatime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3286 fmtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3287 fctime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3288 fbtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3289 if(zipinfo.create_system==0 or zipinfo.create_system==10):
3290 fwinattributes = format(int(zipinfo.external_attr), 'x').lower();
3291 if(not member.is_dir()):
3292 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3293 fchmode = stat.S_IMODE(fmode);
3294 ftypemod = stat.S_IFMT(fmode);
3295 elif(member.is_dir()):
3296 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3297 fchmode = stat.S_IMODE(fmode);
3298 ftypemod = stat.S_IFMT(fmode);
3299 elif(zipinfo.create_system==3):
3300 fwinattributes = format(int(0), 'x').lower();
3301 fmode = format(int(zipinfo.external_attr), 'x').lower();
3302 fchmode = stat.S_IMODE(fmode);
3303 ftypemod = stat.S_IFMT(fmode);
3304 else:
3305 fwinattributes = format(int(0), 'x').lower();
3306 if(not member.is_dir()):
3307 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3308 fchmode = stat.S_IMODE(fmode);
3309 ftypemod = stat.S_IFMT(fmode);
3310 elif(member.is_dir()):
3311 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3312 fchmode = stat.S_IMODE(fmode);
3313 ftypemod = stat.S_IFMT(fmode);
3314 fcompression = "";
3315 fcsize = format(int(0), 'x').lower();
3316 try:
3317 fuid = format(int(os.getuid()), 'x').lower();
3318 except AttributeError:
3319 fuid = format(int(0), 'x').lower();
3320 except KeyError:
3321 fuid = format(int(0), 'x').lower();
3322 try:
3323 fgid = format(int(os.getgid()), 'x').lower();
3324 except AttributeError:
3325 fgid = format(int(0), 'x').lower();
3326 except KeyError:
3327 fgid = format(int(0), 'x').lower();
3328 try:
3329 import pwd;
3330 try:
3331 userinfo = pwd.getpwuid(os.getuid());
3332 funame = userinfo.pw_name;
3333 except KeyError:
3334 funame = "";
3335 except AttributeError:
3336 funame = "";
3337 except ImportError:
3338 funame = "";
3339 fgname = "";
3340 try:
3341 import grp;
3342 try:
3343 groupinfo = grp.getgrgid(os.getgid());
3344 fgname = groupinfo.gr_name;
3345 except KeyError:
3346 fgname = "";
3347 except AttributeError:
3348 fgname = "";
3349 except ImportError:
3350 fgname = "";
3351 fcontents = BytesIO();
3352 if(ftype==0):
3353 fcontents.write(zipfp.read(member.filename));
3354 if(not compresswholefile):
3355 fcontents.seek(0, 2);
3356 ucfsize = fcontents.tell();
3357 fcontents.seek(0, 0);
3358 if(compression=="auto"):
3359 ilsize = len(compressionlistalt);
3360 ilmin = 0;
3361 ilcsize = [];
3362 while(ilmin < ilsize):
3363 cfcontents = BytesIO();
3364 shutil.copyfileobj(fcontents, cfcontents);
3365 fcontents.seek(0, 0);
3366 cfcontents.seek(0, 0);
3367 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3368 cfcontents.seek(0, 2);
3369 ilcsize.append(cfcontents.tell());
3370 cfcontents.close();
3371 ilmin = ilmin + 1;
3372 ilcmin = ilcsize.index(min(ilcsize));
3373 compression = compressionlistalt[ilcmin];
3374 fcontents.seek(0, 0);
3375 cfcontents = BytesIO();
3376 shutil.copyfileobj(fcontents, cfcontents);
3377 cfcontents.seek(0, 0);
3378 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3379 cfcontents.seek(0, 2);
3380 cfsize = cfcontents.tell();
3381 if(ucfsize > cfsize):
3382 fcsize = format(int(cfsize), 'x').lower();
3383 fcompression = compression;
3384 fcontents.close();
3385 fcontents = cfcontents;
3386 if(fcompression=="none"):
3387 fcompression = "";
3388 fcontents.seek(0, 0);
3389 ftypehex = format(ftype, 'x').lower();
3390 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3391 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3392 fcontents.close();
3393 if(numfiles>0):
3394 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3395 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3396 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3397 try:
3398 catfp.flush();
3399 os.fsync(catfp.fileno());
3400 except io.UnsupportedOperation:
3401 pass;
3402 except AttributeError:
3403 pass;
3404 except OSError as e:
3405 pass;
3406 if(outfile=="-"):
3407 catfp.seek(0, 0);
3408 if(hasattr(sys.stdout, "buffer")):
3409 shutil.copyfileobj(catfp, sys.stdout.buffer);
3410 else:
3411 shutil.copyfileobj(catfp, sys.stdout);
3412 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3413 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3414 catfp.seek(0, 0);
3415 upload_file_to_internet_file(catfp, outfile);
3416 if(returnfp):
3417 catfp.seek(0, 0);
3418 return catfp;
3419 else:
3420 catfp.close();
3421 return True;
3423 create_alias_function("Pack", __file_format_name__, "FromZipFile", PackArchiveFileFromZipFile);
3425 if(not rarfile_support):
3426 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3427 return False
3429 if(rarfile_support):
3430 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3431 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3432 outfile = RemoveWindowsPath(outfile);
3433 checksumtype = checksumtype.lower();
3434 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3435 checksumtype="crc32";
3436 if(checksumtype=="none"):
3437 checksumtype = "";
3438 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3439 compression = "auto";
3440 if(compression not in compressionlist and compression is None):
3441 compression = "auto";
3442 if(verbose):
3443 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3444 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3445 if(os.path.exists(outfile)):
3446 try:
3447 os.unlink(outfile);
3448 except OSError as e:
3449 pass;
3450 if(outfile=="-"):
3451 verbose = False;
3452 catfp = BytesIO();
3453 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3454 catfp = outfile;
3455 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3456 catfp = BytesIO();
3457 else:
3458 fbasename = os.path.splitext(outfile)[0];
3459 fextname = os.path.splitext(outfile)[1];
3460 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3461 catver = formatspecs['format_ver'];
3462 fileheaderver = str(int(catver.replace(".", "")));
3463 curinode = 0;
3464 curfid = 0;
3465 inodelist = [];
3466 inodetofile = {};
3467 filetoinode = {};
3468 inodetocatinode = {};
3469 if(not os.path.exists(infile) or not os.path.isfile(infile)):
3470 return False;
3471 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
3472 return False;
3473 rarfp = rarfile.RarFile(infile, "r");
3474 rartest = rarfp.testrar();
3475 if(rartest):
3476 VerbosePrintOut("Bad file found!");
3477 numfiles = int(len(rarfp.infolist()));
3478 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3479 try:
3480 catfp.flush();
3481 os.fsync(catfp.fileno());
3482 except io.UnsupportedOperation:
3483 pass;
3484 except AttributeError:
3485 pass;
3486 except OSError as e:
3487 pass;
3488 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
3489 is_unix = False;
3490 is_windows = False;
3491 if(member.host_os==rarfile.RAR_OS_UNIX):
3492 is_windows = False;
3493 try:
3494 member.external_attr
3495 is_unix = True;
3496 except AttributeError:
3497 is_unix = False;
3498 elif(member.host_os==rarfile.RAR_OS_WIN32):
3499 is_unix = False;
3500 try:
3501 member.external_attr
3502 is_windows = True;
3503 except AttributeError:
3504 is_windows = False;
3505 else:
3506 is_unix = False;
3507 is_windows = False;
3508 if(re.findall("^[.|/]", member.filename)):
3509 fname = member.filename;
3510 else:
3511 fname = "./"+member.filename;
3512 rarinfo = rarfp.getinfo(member.filename);
3513 if(verbose):
3514 VerbosePrintOut(fname);
3515 if(is_unix and member.external_attr !=0):
3516 fpremode = int(member.external_attr);
3517 elif(member.is_file()):
3518 fpremode = int(stat.S_IFREG + 438);
3519 elif(member.is_symlink()):
3520 fpremode = int(stat.S_IFLNK + 438);
3521 elif(member.is_dir()):
3522 fpremode = int(stat.S_IFDIR + 511);
3523 if(is_windows and member.external_attr !=0):
3524 fwinattributes = format(int(member.external_attr), 'x').lower();
3525 else:
3526 fwinattributes = format(int(0), 'x').lower();
3527 fcompression = "";
3528 fcsize = format(int(0), 'x').lower();
3529 flinkcount = 0;
3530 ftype = 0;
3531 if(member.is_file()):
3532 ftype = 0;
3533 elif(member.is_symlink()):
3534 ftype = 2;
3535 elif(member.is_dir()):
3536 ftype = 5;
3537 flinkname = "";
3538 if(ftype==2):
3539 flinkname = rarfp.read(member.filename).decode("UTF-8");
3540 fcurfid = format(int(curfid), 'x').lower();
3541 fcurinode = format(int(curfid), 'x').lower();
3542 curfid = curfid + 1;
3543 fdev_minor = format(int(0), 'x').lower();
3544 fdev_major = format(int(0), 'x').lower();
3545 frdev_minor = format(int(0), 'x').lower();
3546 frdev_major = format(int(0), 'x').lower();
3547 if(ftype==5):
3548 fsize = format(int("0"), 'x').lower();
3549 elif(ftype==0):
3550 fsize = format(int(member.file_size), 'x').lower();
3551 else:
3552 fsize = format(int(member.file_size), 'x').lower();
3553 try:
3554 if(member.atime):
3555 fatime = format(int(member.atime.timestamp()), 'x').lower();
3556 else:
3557 fatime = format(int(member.mtime.timestamp()), 'x').lower();
3558 except AttributeError:
3559 fatime = format(int(member.mtime.timestamp()), 'x').lower();
3560 fmtime = format(int(member.mtime.timestamp()), 'x').lower();
3561 try:
3562 if(member.ctime):
3563 fctime = format(int(member.ctime.timestamp()), 'x').lower();
3564 else:
3565 fctime = format(int(member.mtime.timestamp()), 'x').lower();
3566 except AttributeError:
3567 fctime = format(int(member.mtime.timestamp()), 'x').lower();
3568 fbtime = format(int(member.mtime.timestamp()), 'x').lower();
3569 if(is_unix and member.external_attr !=0):
3570 fmode = format(int(member.external_attr), 'x').lower();
3571 fchmode = format(int(stat.S_IMODE(member.external_attr)), 'x').lower();
3572 ftypemod = format(int(stat.S_IFMT(member.external_attr)), 'x').lower();
3573 elif(member.is_file()):
3574 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3575 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3576 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3577 elif(member.is_symlink()):
3578 fmode = format(int(stat.S_IFLNK + 438), 'x').lower();
3579 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3580 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3581 elif(member.is_dir()):
3582 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3583 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
3584 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
3585 try:
3586 fuid = format(int(os.getuid()), 'x').lower();
3587 except AttributeError:
3588 fuid = format(int(0), 'x').lower();
3589 except KeyError:
3590 fuid = format(int(0), 'x').lower();
3591 try:
3592 fgid = format(int(os.getgid()), 'x').lower();
3593 except AttributeError:
3594 fgid = format(int(0), 'x').lower();
3595 except KeyError:
3596 fgid = format(int(0), 'x').lower();
3597 try:
3598 import pwd;
3599 try:
3600 userinfo = pwd.getpwuid(os.getuid());
3601 funame = userinfo.pw_name;
3602 except KeyError:
3603 funame = "";
3604 except AttributeError:
3605 funame = "";
3606 except ImportError:
3607 funame = "";
3608 fgname = "";
3609 try:
3610 import grp;
3611 try:
3612 groupinfo = grp.getgrgid(os.getgid());
3613 fgname = groupinfo.gr_name;
3614 except KeyError:
3615 fgname = "";
3616 except AttributeError:
3617 fgname = "";
3618 except ImportError:
3619 fgname = "";
3620 fcontents = BytesIO();
3621 if(ftype==0):
3622 fcontents.write(rarfp.read(member.filename));
3623 if(not compresswholefile):
3624 fcontents.seek(0, 2);
3625 ucfsize = fcontents.tell();
3626 fcontents.seek(0, 0);
3627 if(compression=="auto"):
3628 ilsize = len(compressionlistalt);
3629 ilmin = 0;
3630 ilcsize = [];
3631 while(ilmin < ilsize):
3632 cfcontents = BytesIO();
3633 shutil.copyfileobj(fcontents, cfcontents);
3634 fcontents.seek(0, 0);
3635 cfcontents.seek(0, 0);
3636 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3637 if(cfcontents):
3638 cfcontents.seek(0, 2);
3639 ilcsize.append(cfcontents.tell());
3640 cfcontents.close();
3641 else:
3642 try:
3643 ilcsize.append(sys.maxint);
3644 except AttributeError:
3645 ilcsize.append(sys.maxsize);
3646 ilmin = ilmin + 1;
3647 ilcmin = ilcsize.index(min(ilcsize));
3648 compression = compressionlistalt[ilcmin];
3649 fcontents.seek(0, 0);
3650 cfcontents = BytesIO();
3651 shutil.copyfileobj(fcontents, cfcontents);
3652 cfcontents.seek(0, 0);
3653 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3654 cfcontents.seek(0, 2);
3655 cfsize = cfcontents.tell();
3656 if(ucfsize > cfsize):
3657 fcsize = format(int(cfsize), 'x').lower();
3658 fcompression = compression;
3659 fcontents.close();
3660 fcontents = cfcontents;
3661 if(fcompression=="none"):
3662 fcompression = "";
3663 fcontents.seek(0, 0);
3664 ftypehex = format(ftype, 'x').lower();
3665 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3666 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3667 fcontents.close();
3668 if(numfiles>0):
3669 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3670 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3671 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
3672 try:
3673 catfp.flush();
3674 os.fsync(catfp.fileno());
3675 except io.UnsupportedOperation:
3676 pass;
3677 except AttributeError:
3678 pass;
3679 except OSError as e:
3680 pass;
3681 if(outfile=="-"):
3682 catfp.seek(0, 0)
3683 if(hasattr(sys.stdout, "buffer")):
3684 shutil.copyfileobj(catfp, sys.stdout.buffer);
3685 else:
3686 shutil.copyfileobj(catfp, sys.stdout);
3687 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3688 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3689 catfp.seek(0, 0);
3690 upload_file_to_internet_file(catfp, outfile);
3691 if(returnfp):
3692 catfp.seek(0, 0)
3693 return catfp
3694 else:
3695 catfp.close()
3696 return True;
3698 create_alias_function("Pack", __file_format_name__, "FromRarFile", PackArchiveFileFromRarFile);
3700 if(not py7zr_support):
3701 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3702 return False
3704 if(py7zr_support):
3705 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3706 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3707 outfile = RemoveWindowsPath(outfile);
3708 checksumtype = checksumtype.lower();
3709 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3710 checksumtype="crc32";
3711 if(checksumtype=="none"):
3712 checksumtype = "";
3713 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3714 compression = "auto";
3715 if(compression not in compressionlist and compression is None):
3716 compression = "auto";
3717 if(verbose):
3718 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3719 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3720 if(os.path.exists(outfile)):
3721 try:
3722 os.unlink(outfile);
3723 except OSError as e:
3724 pass;
3725 if(outfile=="-"):
3726 verbose = False;
3727 catfp = BytesIO();
3728 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3729 catfp = outfile;
3730 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3731 catfp = BytesIO();
3732 else:
3733 fbasename = os.path.splitext(outfile)[0];
3734 fextname = os.path.splitext(outfile)[1];
3735 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3736 catver = formatspecs['format_ver'];
3737 fileheaderver = str(int(catver.replace(".", "")));
3738 curinode = 0;
3739 curfid = 0;
3740 inodelist = [];
3741 inodetofile = {};
3742 filetoinode = {};
3743 inodetocatinode = {};
3744 if(not os.path.exists(infile) or not os.path.isfile(infile)):
3745 return False;
3746 szpfp = py7zr.SevenZipFile(infile, mode="r");
3747 file_content = szpfp.readall();
3748 #sztest = szpfp.testzip();
3749 sztestalt = szpfp.test();
3750 if(sztestalt):
3751 VerbosePrintOut("Bad file found!");
3752 numfiles = int(len(szpfp.list()));
3753 AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3754 for member in sorted(szpfp.list(), key=lambda x: x.filename):
3755 if(re.findall("^[.|/]", member.filename)):
3756 fname = member.filename;
3757 else:
3758 fname = "./"+member.filename;
3759 if(verbose):
3760 VerbosePrintOut(fname);
3761 if(not member.is_directory):
3762 fpremode = int(stat.S_IFREG + 438);
3763 elif(member.is_directory):
3764 fpremode = int(stat.S_IFDIR + 511);
3765 fwinattributes = format(int(0), 'x').lower();
3766 fcompression = "";
3767 fcsize = format(int(0), 'x').lower();
3768 flinkcount = 0;
3769 ftype = 0;
3770 if(member.is_directory):
3771 ftype = 5;
3772 else:
3773 ftype = 0;
3774 flinkname = "";
3775 fcurfid = format(int(curfid), 'x').lower();
3776 fcurinode = format(int(curfid), 'x').lower();
3777 curfid = curfid + 1;
3778 fdev_minor = format(int(0), 'x').lower();
3779 fdev_major = format(int(0), 'x').lower();
3780 frdev_minor = format(int(0), 'x').lower();
3781 frdev_major = format(int(0), 'x').lower();
3782 if(ftype==5):
3783 fsize = format(int("0"), 'x').lower();
3784 fatime = format(int(member.creationtime.timestamp()), 'x').lower();
3785 fmtime = format(int(member.creationtime.timestamp()), 'x').lower();
3786 fctime = format(int(member.creationtime.timestamp()), 'x').lower();
3787 fbtime = format(int(member.creationtime.timestamp()), 'x').lower();
3788 if(member.is_directory):
3789 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3790 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
3791 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
3792 else:
3793 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3794 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3795 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3796 try:
3797 fuid = format(int(os.getuid()), 'x').lower();
3798 except AttributeError:
3799 fuid = format(int(0), 'x').lower();
3800 except KeyError:
3801 fuid = format(int(0), 'x').lower();
3802 try:
3803 fgid = format(int(os.getgid()), 'x').lower();
3804 except AttributeError:
3805 fgid = format(int(0), 'x').lower();
3806 except KeyError:
3807 fgid = format(int(0), 'x').lower();
3808 try:
3809 import pwd;
3810 try:
3811 userinfo = pwd.getpwuid(os.getuid());
3812 funame = userinfo.pw_name;
3813 except KeyError:
3814 funame = "";
3815 except AttributeError:
3816 funame = "";
3817 except ImportError:
3818 funame = "";
3819 fgname = "";
3820 try:
3821 import grp;
3822 try:
3823 groupinfo = grp.getgrgid(os.getgid());
3824 fgname = groupinfo.gr_name;
3825 except KeyError:
3826 fgname = "";
3827 except AttributeError:
3828 fgname = "";
3829 except ImportError:
3830 fgname = "";
3831 fcontents = BytesIO();
3832 if(ftype==0):
3833 fcontents.write(file_content[member.filename].read());
3834 fsize = format(fcontents.tell(), 'x').lower();
3835 file_content[member.filename].close();
3836 if(not compresswholefile):
3837 fcontents.seek(0, 2);
3838 ucfsize = fcontents.tell();
3839 fcontents.seek(0, 0);
3840 if(compression=="auto"):
3841 ilsize = len(compressionlistalt);
3842 ilmin = 0;
3843 ilcsize = [];
3844 while(ilmin < ilsize):
3845 cfcontents = BytesIO();
3846 shutil.copyfileobj(fcontents, cfcontents);
3847 fcontents.seek(0, 0);
3848 cfcontents.seek(0, 0);
3849 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3850 if(cfcontents):
3851 cfcontents.seek(0, 2);
3852 ilcsize.append(cfcontents.tell());
3853 cfcontents.close();
3854 else:
3855 try:
3856 ilcsize.append(sys.maxint);
3857 except AttributeError:
3858 ilcsize.append(sys.maxsize);
3859 ilmin = ilmin + 1;
3860 ilcmin = ilcsize.index(min(ilcsize));
3861 compression = compressionlistalt[ilcmin];
3862 fcontents.seek(0, 0);
3863 cfcontents = BytesIO();
3864 shutil.copyfileobj(fcontents, cfcontents);
3865 cfcontents.seek(0, 0);
3866 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3867 cfcontents.seek(0, 2);
3868 cfsize = cfcontents.tell();
3869 if(ucfsize > cfsize):
3870 fcsize = format(int(cfsize), 'x').lower();
3871 fcompression = compression;
3872 fcontents.close();
3873 fcontents = cfcontents;
3874 if(fcompression=="none"):
3875 fcompression = "";
3876 fcontents.seek(0, 0);
3877 ftypehex = format(ftype, 'x').lower();
3878 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3879 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3880 fcontents.close();
3881 if(numfiles>0):
3882 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3883 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3884 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
3885 try:
3886 catfp.flush();
3887 os.fsync(catfp.fileno());
3888 except io.UnsupportedOperation:
3889 pass;
3890 except AttributeError:
3891 pass;
3892 except OSError as e:
3893 pass;
3894 if(outfile=="-"):
3895 catfp.seek(0, 0);
3896 if(hasattr(sys.stdout, "buffer")):
3897 shutil.copyfileobj(catfp, sys.stdout.buffer);
3898 else:
3899 shutil.copyfileobj(catfp, sys.stdout);
3900 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3901 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3902 catfp.seek(0, 0);
3903 upload_file_to_internet_file(catfp, outfile);
3904 if(returnfp):
3905 catfp.seek(0, 0)
3906 return catfp
3907 else:
3908 catfp.close()
3909 return True;
3911 create_alias_function("Pack", __file_format_name__, "FromSevenZipFile", PackArchiveFileFromSevenZipFile);
3913 def PackArchiveFileFromInFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3914 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
3915 if(verbose):
3916 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3917 if(checkcompressfile=="tarfile"):
3918 return PackArchiveFileFromTarFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3919 elif(checkcompressfile=="zipfile"):
3920 return PackArchiveFileFromZipFile(infile, outfile, compression, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3921 elif(checkcompressfile=="catfile"):
3922 return RePackArchiveFile(infile, outfile, compression, compressionlevel, False, 0, 0, checksumtype, False, extradata, formatspecs, verbose, returnfp);
3923 elif(rarfile_support and checkcompressfile=="rarfile"):
3924 return PackArchiveFileFromRarFile(infile, outfile, compression, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3925 elif(py7zr_support and checkcompressfile=="7zipfile"):
3926 return PackArchiveFileFromSevenZipFile(infile, outfile, compression, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3927 else:
3928 return False;
3929 return False;
3931 create_alias_function("Pack", __file_format_name__, "FromInFile", PackArchiveFileFromInFile);
3933 def ArchiveFileSeekToFileNum(infile, seekto=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
3934 if(hasattr(infile, "read") or hasattr(infile, "write")):
3935 catfp = infile;
3936 catfp.seek(0, 0);
3937 catfp = UncompressArchiveFile(catfp, formatspecs);
3938 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
3939 if(checkcompressfile=="tarfile"):
3940 return TarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3941 if(checkcompressfile=="zipfile"):
3942 return ZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3943 if(rarfile_support and checkcompressfile=="rarfile"):
3944 return RarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3945 if(py7zr_support and checkcompressfile=="7zipfile"):
3946 return SevenZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3947 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
3948 return False;
3949 if(not catfp):
3950 return False;
3951 catfp.seek(0, 0);
3952 elif(infile=="-"):
3953 catfp = BytesIO();
3954 if(hasattr(sys.stdin, "buffer")):
3955 shutil.copyfileobj(sys.stdin.buffer, catfp);
3956 else:
3957 shutil.copyfileobj(sys.stdin, catfp);
3958 catfp.seek(0, 0);
3959 catfp = UncompressArchiveFile(catfp, formatspecs);
3960 if(not catfp):
3961 return False;
3962 catfp.seek(0, 0);
3963 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3964 catfp = download_file_from_internet_file(infile);
3965 catfp.seek(0, 0);
3966 catfp = UncompressArchiveFile(catfp, formatspecs);
3967 if(not catfp):
3968 return False;
3969 catfp.seek(0, 0);
3970 else:
3971 infile = RemoveWindowsPath(infile);
3972 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
3973 if(checkcompressfile=="tarfile"):
3974 return TarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3975 if(checkcompressfile=="zipfile"):
3976 return ZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3977 if(rarfile_support and checkcompressfile=="rarfile"):
3978 return RarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3979 if(py7zr_support and checkcompressfile=="7zipfile"):
3980 return SevenZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3981 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
3982 return False;
3983 compresscheck = CheckCompressionType(infile, formatspecs, True);
3984 if(not compresscheck):
3985 fextname = os.path.splitext(infile)[1];
3986 if(fextname==".gz"):
3987 compresscheck = "gzip";
3988 elif(fextname==".bz2"):
3989 compresscheck = "bzip2";
3990 elif(fextname==".zst"):
3991 compresscheck = "zstd";
3992 elif(fextname==".lz4" or fextname==".clz4"):
3993 compresscheck = "lz4";
3994 elif(fextname==".lzo" or fextname==".lzop"):
3995 compresscheck = "lzo";
3996 elif(fextname==".lzma" or fextname==".xz"):
3997 compresscheck = "lzma";
3998 else:
3999 return False;
4000 if(not compresscheck):
4001 return False;
4002 catfp = UncompressFile(infile, formatspecs, "rb");
4004 try:
4005 catfp.seek(0, 2);
4006 except OSError:
4007 SeekToEndOfFile(catfp);
4008 except ValueError:
4009 SeekToEndOfFile(catfp);
4010 CatSize = catfp.tell();
4011 CatSizeEnd = CatSize;
4013 try:
4014 catfp.seek(0, 0);
4015 except OSError:
4016 return False;
4017 except ValueError:
4018 return False;
4019 curloc = catfp.tell();
4020 if(curloc>0):
4021 catfp.seek(0, 0);
4022 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4023 if(curloc>0):
4024 catfp.seek(curloc, 0);
4025 catstring = catheader[0];
4026 catversion = re.findall(r"([\d]+)$", catstring);
4027 fprenumfiles = catheader[1];
4028 fnumfiles = int(fprenumfiles, 16);
4029 fprechecksumtype = catheader[2];
4030 fprechecksum = catheader[3];
4031 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4032 fnumfileshex = format(int(fnumfiles), 'x').lower();
4033 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4034 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4035 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4036 fheadtell = len(fileheader);
4037 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4038 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
4039 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4040 return False;
4041 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
4042 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4043 if(seekto>=fnumfiles):
4044 seekto = fnumfiles - 1;
4045 if(seekto<0):
4046 seekto = 0;
4047 if(seekto>=0):
4048 il = -1;
4049 while(il < seekto):
4050 prefhstart = catfp.tell();
4051 if(formatspecs['new_style']):
4052 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4053 else:
4054 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4055 if(len(preheaderdata)==0):
4056 break;
4057 prefheadsize = int(preheaderdata[0], 16);
4058 prefnumfields = int(preheaderdata[1], 16);
4059 preftype = int(preheaderdata[2], 16);
4060 if(re.findall("^[.|/]", preheaderdata[3])):
4061 prefname = preheaderdata[3];
4062 else:
4063 prefname = "./"+preheaderdata[3];
4064 prefbasedir = os.path.dirname(prefname);
4065 preflinkname = preheaderdata[4];
4066 prefsize = int(preheaderdata[5], 16);
4067 prefatime = int(preheaderdata[6], 16);
4068 prefmtime = int(preheaderdata[7], 16);
4069 prefctime = int(preheaderdata[8], 16);
4070 prefbtime = int(preheaderdata[9], 16);
4071 prefmode = int(preheaderdata[10], 16);
4072 prefchmode = stat.S_IMODE(prefmode);
4073 preftypemod = stat.S_IFMT(prefmode);
4074 prefwinattributes = int(preheaderdata[11], 16);
4075 prefcompression = preheaderdata[12];
4076 prefcsize = int(preheaderdata[13], 16);
4077 prefuid = int(preheaderdata[14], 16);
4078 prefuname = preheaderdata[15];
4079 prefgid = int(preheaderdata[16], 16);
4080 prefgname = preheaderdata[17];
4081 fid = int(preheaderdata[18], 16);
4082 finode = int(preheaderdata[19], 16);
4083 flinkcount = int(preheaderdata[20], 16);
4084 prefdev_minor = int(preheaderdata[21], 16);
4085 prefdev_major = int(preheaderdata[22], 16);
4086 prefrdev_minor = int(preheaderdata[23], 16);
4087 prefrdev_major = int(preheaderdata[24], 16);
4088 prefextrasize = int(preheaderdata[25], 16);
4089 prefextrafields = int(preheaderdata[26], 16);
4090 extrafieldslist = [];
4091 extrastart = 27;
4092 extraend = extrastart + prefextrafields;
4093 extrafieldslist = [];
4094 if(extrastart<extraend):
4095 extrafieldslist.append(preheaderdata[extrastart]);
4096 extrastart = extrastart + 1;
4097 prefchecksumtype = preheaderdata[extrastart].lower();
4098 prefcs = preheaderdata[extrastart + 1].lower();
4099 prefccs = preheaderdata[extrastart + 2].lower();
4100 hc = 0;
4101 hcmax = len(preheaderdata) - 2;
4102 hout = "";
4103 while(hc<hcmax):
4104 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs['format_delimiter']);
4105 hc = hc + 1;
4106 prenewfcs = GetFileChecksum(hout, preheaderdata[-3].lower(), True, formatspecs);
4107 if(prefcs!=prenewfcs and not skipchecksum):
4108 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4109 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4110 return False;
4111 valid_archive = False;
4112 invalid_archive = True;
4113 prefhend = catfp.tell() - 1;
4114 prefcontentstart = catfp.tell();
4115 prefcontents = "";
4116 pyhascontents = False;
4117 if(prefsize>0):
4118 if(prefcompression):
4119 prefcontents = catfp.read(prefsize);
4120 else:
4121 prefcontents = catfp.read(prefcsize);
4122 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4123 pyhascontents = True;
4124 if(prefccs!=prenewfccs and not skipchecksum):
4125 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4126 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4127 return False;
4128 catfp.seek(1, 1);
4129 il = il + 1;
4130 catfp.seek(seekstart, 0);
4131 fileidnum = il;
4132 catfheadsize = int(preheaderdata[0], 16);
4133 catfnumfields = int(preheaderdata[1], 16);
4134 catftype = int(preheaderdata[2], 16);
4135 if(re.findall("^[.|/]", preheaderdata[3])):
4136 catfname = preheaderdata[3];
4137 else:
4138 catfname = "./"+preheaderdata[3];
4139 catflinkname = preheaderdata[4];
4140 catfsize = int(preheaderdata[5], 16);
4141 catfbasedir = os.path.dirname(catfname);
4142 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
4143 if(returnfp):
4144 catlist.update({'catfp': catfp});
4145 else:
4146 catfp.close();
4147 return catlist;
4149 create_alias_function("", __file_format_name__, "SeekToFileNum", ArchiveFileSeekToFileNum);
4151 def ArchiveFileSeekToFileName(infile, seekfile=None, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4152 if(hasattr(infile, "read") or hasattr(infile, "write")):
4153 catfp = infile;
4154 catfp.seek(0, 0);
4155 catfp = UncompressArchiveFile(catfp, formatspecs);
4156 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4157 if(checkcompressfile=="tarfile"):
4158 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4159 if(checkcompressfile=="zipfile"):
4160 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4161 if(rarfile_support and checkcompressfile=="rarfile"):
4162 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4163 if(py7zr_support and checkcompressfile=="7zipfile"):
4164 return SevenZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4165 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4166 return False;
4167 if(not catfp):
4168 return False;
4169 catfp.seek(0, 0);
4170 elif(infile=="-"):
4171 catfp = BytesIO();
4172 if(hasattr(sys.stdin, "buffer")):
4173 shutil.copyfileobj(sys.stdin.buffer, catfp);
4174 else:
4175 shutil.copyfileobj(sys.stdin, catfp);
4176 catfp.seek(0, 0);
4177 catfp = UncompressArchiveFile(catfp, formatspecs);
4178 if(not catfp):
4179 return False;
4180 catfp.seek(0, 0);
4181 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4182 catfp = download_file_from_internet_file(infile);
4183 catfp = UncompressArchiveFile(catfp, formatspecs);
4184 catfp.seek(0, 0);
4185 if(not catfp):
4186 return False;
4187 catfp.seek(0, 0);
4188 else:
4189 infile = RemoveWindowsPath(infile);
4190 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4191 if(checkcompressfile=="tarfile"):
4192 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4193 if(checkcompressfile=="zipfile"):
4194 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4195 if(rarfile_support and checkcompressfile=="rarfile"):
4196 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4197 if(py7zr_support and checkcompressfile=="7zipfile"):
4198 return SevenZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4199 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4200 return False;
4201 compresscheck = CheckCompressionType(infile, formatspecs, True);
4202 if(not compresscheck):
4203 fextname = os.path.splitext(infile)[1];
4204 if(fextname==".gz"):
4205 compresscheck = "gzip";
4206 elif(fextname==".bz2"):
4207 compresscheck = "bzip2";
4208 elif(fextname==".zst"):
4209 compresscheck = "zstd";
4210 elif(fextname==".lz4" or fextname==".clz4"):
4211 compresscheck = "lz4";
4212 elif(fextname==".lzo" or fextname==".lzop"):
4213 compresscheck = "lzo";
4214 elif(fextname==".lzma" or fextname==".xz"):
4215 compresscheck = "lzma";
4216 else:
4217 return False;
4218 if(not compresscheck):
4219 return False;
4220 catfp = UncompressFile(infile, formatspecs, "rb");
4222 try:
4223 catfp.seek(0, 2);
4224 except OSError:
4225 SeekToEndOfFile(catfp);
4226 except ValueError:
4227 SeekToEndOfFile(catfp);
4228 CatSize = catfp.tell();
4229 CatSizeEnd = CatSize;
4231 try:
4232 catfp.seek(0, 0);
4233 except OSError:
4234 return False;
4235 except ValueError:
4236 return False;
4237 curloc = catfp.tell();
4238 if(curloc>0):
4239 catfp.seek(0, 0);
4240 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4241 if(curloc>0):
4242 catfp.seek(curloc, 0);
4243 catstring = catheader[0];
4244 catversion = re.findall(r"([\d]+)$", catstring);
4245 fprenumfiles = catheader[1];
4246 fnumfiles = int(fprenumfiles, 16);
4247 fprechecksumtype = catheader[2];
4248 fprechecksum = catheader[3];
4249 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4250 fnumfileshex = format(int(fnumfiles), 'x').lower();
4251 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4252 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4253 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4254 fheadtell = len(fileheader);
4255 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4256 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
4257 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4258 return False;
4259 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
4260 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4261 seekto = fnumfiles - 1
4262 filefound = False;
4263 if(seekto>=0):
4264 il = -1;
4265 while(il < seekto):
4266 prefhstart = catfp.tell();
4267 if(formatspecs['new_style']):
4268 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4269 else:
4270 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4271 if(len(preheaderdata)==0):
4272 break;
4273 prefheadsize = int(preheaderdata[0], 16);
4274 prefnumfields = int(preheaderdata[1], 16);
4275 preftype = int(preheaderdata[2], 16);
4276 if(re.findall("^[.|/]", preheaderdata[3])):
4277 prefname = preheaderdata[3];
4278 else:
4279 prefname = "./"+preheaderdata[3];
4280 prefbasedir = os.path.dirname(prefname);
4281 preflinkname = preheaderdata[4];
4282 prefsize = int(preheaderdata[5], 16);
4283 prefatime = int(preheaderdata[6], 16);
4284 prefmtime = int(preheaderdata[7], 16);
4285 prefctime = int(preheaderdata[8], 16);
4286 prefbtime = int(preheaderdata[9], 16);
4287 prefmode = int(preheaderdata[10], 16);
4288 prefchmode = stat.S_IMODE(prefmode);
4289 preftypemod = stat.S_IFMT(prefmode);
4290 prefwinattributes = int(preheaderdata[11], 16);
4291 prefcompression = preheaderdata[12];
4292 prefcsize = int(preheaderdata[13], 16);
4293 prefuid = int(preheaderdata[14], 16);
4294 prefuname = preheaderdata[15];
4295 prefgid = int(preheaderdata[16], 16);
4296 prefgname = preheaderdata[17];
4297 fid = int(preheaderdata[18], 16);
4298 finode = int(preheaderdata[19], 16);
4299 flinkcount = int(preheaderdata[20], 16);
4300 prefdev_minor = int(preheaderdata[21], 16);
4301 prefdev_major = int(preheaderdata[22], 16);
4302 prefrdev_minor = int(preheaderdata[23], 16);
4303 prefrdev_major = int(preheaderdata[24], 16);
4304 prefextrasize = int(preheaderdata[25], 16);
4305 prefextrafields = int(preheaderdata[26], 16);
4306 extrafieldslist = [];
4307 extrastart = 27;
4308 extraend = extrastart + prefextrafields;
4309 extrafieldslist = [];
4310 if(extrastart<extraend):
4311 extrafieldslist.append(preheaderdata[extrastart]);
4312 extrastart = extrastart + 1;
4313 prefchecksumtype = preheaderdata[extrastart].lower();
4314 prefcs = preheaderdata[extrastart + 1].lower();
4315 prefccs = preheaderdata[extrastart + 2].lower();
4316 hc = 0;
4317 hcmax = len(preheaderdata) - 2;
4318 hout = "";
4319 while(hc<hcmax):
4320 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs['format_delimiter']);
4321 hc = hc + 1;
4322 prenewfcs = GetFileChecksum(hout, preheaderdata[-3].lower(), True, formatspecs);
4323 if(prefcs!=prenewfcs and not skipchecksum):
4324 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4325 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4326 return False;
4327 valid_archive = False;
4328 invalid_archive = True;
4329 prefhend = catfp.tell() - 1;
4330 prefcontentstart = catfp.tell();
4331 prefcontents = "";
4332 pyhascontents = False;
4333 if(prefsize>0):
4334 if(prefcompression):
4335 prefcontents = catfp.read(prefsize);
4336 else:
4337 prefcontents = catfp.read(prefcsize);
4338 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4339 pyhascontents = True;
4340 if(prefccs!=prenewfccs and not skipchecksum):
4341 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4342 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4343 return False;
4344 catfp.seek(1, 1);
4345 il = il + 1;
4346 filefound = False;
4347 prefname = preheaderdata[2];
4348 if(re.findall("^[.|/]", preheaderdata[2])):
4349 prefname = preheaderdata[2];
4350 else:
4351 prefname = "./"+preheaderdata[2];
4352 if(prefname==seekfile):
4353 filefound = True;
4354 break;
4355 catfp.seek(seekstart, 0);
4356 fileidnum = il;
4357 catfheadsize = int(preheaderdata[0], 16);
4358 catfnumfields = int(preheaderdata[1], 16);
4359 catftype = int(preheaderdata[2], 16);
4360 if(re.findall("^[.|/]", preheaderdata[3])):
4361 catfname = preheaderdata[3];
4362 else:
4363 catfname = "./"+preheaderdata[3];
4364 catflinkname = preheaderdata[4];
4365 catfsize = int(preheaderdata[5], 16);
4366 catfbasedir = os.path.dirname(catfname);
4367 if(filefound):
4368 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
4369 else:
4370 return False;
4371 if(returnfp):
4372 catlist.update({'catfp': catfp});
4373 else:
4374 catfp.close();
4375 return catlist;
4377 create_alias_function("", __file_format_name__, "SeekToFileName", ArchiveFileSeekToFileName);
4379 def ArchiveFileValidate(infile, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
4380 if(verbose):
4381 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
4382 if(hasattr(infile, "read") or hasattr(infile, "write")):
4383 catfp = infile;
4384 catfp.seek(0, 0);
4385 catfp = UncompressArchiveFile(catfp, formatspecs);
4386 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4387 if(checkcompressfile=="tarfile"):
4388 return TarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4389 if(checkcompressfile=="zipfile"):
4390 return ZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4391 if(rarfile_support and checkcompressfile=="rarfile"):
4392 return RarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4393 if(py7zr_support and checkcompressfile=="7zipfile"):
4394 return SevenZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4395 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4396 return False;
4397 if(not catfp):
4398 return False;
4399 catfp.seek(0, 0);
4400 elif(infile=="-"):
4401 catfp = BytesIO();
4402 if(hasattr(sys.stdin, "buffer")):
4403 shutil.copyfileobj(sys.stdin.buffer, catfp);
4404 else:
4405 shutil.copyfileobj(sys.stdin, catfp);
4406 catfp.seek(0, 0);
4407 catfp = UncompressArchiveFile(catfp, formatspecs);
4408 if(not catfp):
4409 return False;
4410 catfp.seek(0, 0);
4411 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4412 catfp = download_file_from_internet_file(infile);
4413 catfp = UncompressArchiveFile(catfp, formatspecs);
4414 catfp.seek(0, 0);
4415 if(not catfp):
4416 return False;
4417 catfp.seek(0, 0);
4418 else:
4419 infile = RemoveWindowsPath(infile);
4420 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4421 if(checkcompressfile=="tarfile"):
4422 return TarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4423 if(checkcompressfile=="zipfile"):
4424 return ZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4425 if(rarfile_support and checkcompressfile=="rarfile"):
4426 return RarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4427 if(py7zr_support and checkcompressfile=="7zipfile"):
4428 return SevenZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4429 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4430 return False;
4431 compresscheck = CheckCompressionType(infile, formatspecs, True);
4432 if(not compresscheck):
4433 fextname = os.path.splitext(infile)[1];
4434 if(fextname==".gz"):
4435 compresscheck = "gzip";
4436 elif(fextname==".bz2"):
4437 compresscheck = "bzip2";
4438 elif(fextname==".zst"):
4439 compresscheck = "zstd";
4440 elif(fextname==".lz4" or fextname==".clz4"):
4441 compresscheck = "lz4";
4442 elif(fextname==".lzo" or fextname==".lzop"):
4443 compresscheck = "lzo";
4444 elif(fextname==".lzma" or fextname==".xz"):
4445 compresscheck = "lzma";
4446 else:
4447 return False;
4448 if(not compresscheck):
4449 return False;
4450 catfp = UncompressFile(infile, formatspecs, "rb");
4452 try:
4453 catfp.seek(0, 2);
4454 except OSError:
4455 SeekToEndOfFile(catfp);
4456 except ValueError:
4457 SeekToEndOfFile(catfp);
4458 CatSize = catfp.tell();
4459 CatSizeEnd = CatSize;
4461 try:
4462 catfp.seek(0, 0);
4463 except OSError:
4464 return False;
4465 except ValueError:
4466 return False;
4467 curloc = catfp.tell();
4468 if(curloc>0):
4469 catfp.seek(0, 0);
4470 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4471 if(curloc>0):
4472 catfp.seek(curloc, 0);
4473 catstring = catheader[0];
4474 catversion = re.findall(r"([\d]+)$", catstring);
4475 fprenumfiles = catheader[1];
4476 fnumfiles = int(fprenumfiles, 16);
4477 fprechecksumtype = catheader[2];
4478 fprechecksum = catheader[3];
4479 il = 0;
4480 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4481 fnumfileshex = format(int(fnumfiles), 'x').lower();
4482 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4483 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4484 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4485 valid_archive = True;
4486 invalid_archive = False;
4487 if(verbose):
4488 VerbosePrintOut(infile);
4489 VerbosePrintOut("Number of Records " + str(fnumfiles));
4490 if(fprechecksum==catfileheadercshex):
4491 if(verbose):
4492 VerbosePrintOut("File Header Checksum Passed at offset " + str(0));
4493 VerbosePrintOut("'" + str(fprechecksum) + "' == " + "'" + str(catfileheadercshex) + "'");
4494 else:
4495 if(verbose):
4496 VerbosePrintOut("File Header Checksum Failed at offset " + str(0));
4497 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4498 valid_archive = False;
4499 invalid_archive = True;
4500 if(verbose):
4501 VerbosePrintOut("");
4502 while(il<fnumfiles):
4503 catfhstart = catfp.tell();
4504 if(formatspecs['new_style']):
4505 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4506 else:
4507 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4508 if(len(catheaderdata)==0):
4509 break;
4510 catfheadsize = int(catheaderdata[0], 16);
4511 catfnumfields = int(catheaderdata[1], 16);
4512 catftype = int(catheaderdata[2], 16);
4513 if(re.findall("^[.|/]", catheaderdata[3])):
4514 catfname = catheaderdata[3];
4515 else:
4516 catfname = "./"+catheaderdata[3];
4517 catfbasedir = os.path.dirname(catfname);
4518 catflinkname = catheaderdata[4];
4519 catfsize = int(catheaderdata[5], 16);
4520 catfatime = int(catheaderdata[6], 16);
4521 catfmtime = int(catheaderdata[7], 16);
4522 catfctime = int(catheaderdata[8], 16);
4523 catfbtime = int(catheaderdata[9], 16);
4524 catfmode = int(catheaderdata[10], 16);
4525 catfchmode = stat.S_IMODE(catfmode);
4526 catftypemod = stat.S_IFMT(catfmode);
4527 catfwinattributes = int(catheaderdata[11], 16);
4528 catfcompression = catheaderdata[12];
4529 catfcsize = int(catheaderdata[13], 16);
4530 catfuid = int(catheaderdata[14], 16);
4531 catfuname = catheaderdata[15];
4532 catfgid = int(catheaderdata[16], 16);
4533 catfgname = catheaderdata[17];
4534 fid = int(catheaderdata[18], 16);
4535 finode = int(catheaderdata[19], 16);
4536 flinkcount = int(catheaderdata[20], 16);
4537 catfdev_minor = int(catheaderdata[21], 16);
4538 catfdev_major = int(catheaderdata[22], 16);
4539 catfrdev_minor = int(catheaderdata[23], 16);
4540 catfrdev_major = int(catheaderdata[24], 16);
4541 catfextrasize = int(catheaderdata[25], 16);
4542 catfextrafields = int(catheaderdata[26], 16);
4543 extrafieldslist = [];
4544 extrastart = 27;
4545 extraend = extrastart + catfextrafields;
4546 extrafieldslist = [];
4547 if(extrastart<extraend):
4548 extrafieldslist.append(catheaderdata[extrastart]);
4549 extrastart = extrastart + 1;
4550 catfchecksumtype = catheaderdata[extrastart].lower();
4551 catfcs = catheaderdata[extrastart + 1].lower();
4552 catfccs = catheaderdata[extrastart + 2].lower();
4553 hc = 0;
4554 hcmax = len(catheaderdata) - 2;
4555 hout = "";
4556 while(hc<hcmax):
4557 hout = hout + AppendNullByte(catheaderdata[hc], formatspecs['format_delimiter']);
4558 hc = hc + 1;
4559 catnewfcs = GetFileChecksum(hout, catheaderdata[-3].lower(), True, formatspecs);
4560 if(verbose):
4561 VerbosePrintOut(catfname);
4562 VerbosePrintOut("Record Number " + str(il) + "; File ID " + str(fid) + "; iNode Number " + str(finode));
4563 if(catfcs==catnewfcs):
4564 if(verbose):
4565 VerbosePrintOut("File Header Checksum Passed at offset " + str(catfhstart));
4566 VerbosePrintOut("'" + str(catfcs) + "' == " + "'" + str(catnewfcs) + "'");
4567 else:
4568 if(verbose):
4569 VerbosePrintOut("File Header Checksum Failed at offset " + str(catfhstart));
4570 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
4571 valid_archive = False;
4572 invalid_archive = True;
4573 catfhend = catfp.tell() - 1;
4574 catfcontentstart = catfp.tell();
4575 catfcontents = "";
4576 pyhascontents = False;
4577 if(catfsize>0):
4578 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4579 catfcontents = catfp.read(catfsize);
4580 else:
4581 catfcontents = catfp.read(catfcsize);
4582 catnewfccs = GetFileChecksum(catfcontents, catheaderdata[-3].lower(), False, formatspecs);
4583 pyhascontents = True;
4584 if(catfccs==catnewfccs):
4585 if(verbose):
4586 VerbosePrintOut("File Content Checksum Passed at offset " + str(catfcontentstart));
4587 VerbosePrintOut("'" + str(catfccs) + "' == " + "'" + str(catnewfccs) + "'");
4588 else:
4589 if(verbose):
4590 VerbosePrintOut("File Content Checksum Failed at offset " + str(catfcontentstart));
4591 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
4592 valid_archive = False;
4593 invalid_archive = True;
4594 if(verbose):
4595 VerbosePrintOut("");
4596 catfp.seek(1, 1);
4597 il = il + 1;
4598 if(valid_archive):
4599 if(returnfp):
4600 return catfp;
4601 else:
4602 catfp.close();
4603 return True;
4604 else:
4605 catfp.close();
4606 return False;
4608 create_alias_function("", __file_format_name__, "Validate", ArchiveFileValidate);
4610 def ArchiveFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4611 if(hasattr(infile, "read") or hasattr(infile, "write")):
4612 catfp = infile;
4613 catfp.seek(0, 0);
4614 catfp = UncompressArchiveFile(catfp, formatspecs);
4615 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4616 if(checkcompressfile=="tarfile"):
4617 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4618 if(checkcompressfile=="zipfile"):
4619 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4620 if(rarfile_support and checkcompressfile=="rarfile"):
4621 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4622 if(py7zr_support and checkcompressfile=="7zipfile"):
4623 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4624 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4625 return False;
4626 if(not catfp):
4627 return False;
4628 catfp.seek(0, 0);
4629 elif(infile=="-"):
4630 catfp = BytesIO();
4631 if(hasattr(sys.stdin, "buffer")):
4632 shutil.copyfileobj(sys.stdin.buffer, catfp);
4633 else:
4634 shutil.copyfileobj(sys.stdin, catfp);
4635 catfp.seek(0, 0);
4636 catfp = UncompressArchiveFile(catfp, formatspecs);
4637 if(not catfp):
4638 return False;
4639 catfp.seek(0, 0);
4640 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4641 catfp = download_file_from_internet_file(infile);
4642 catfp = UncompressArchiveFile(catfp, formatspecs);
4643 catfp.seek(0, 0);
4644 if(not catfp):
4645 return False;
4646 catfp.seek(0, 0);
4647 else:
4648 infile = RemoveWindowsPath(infile);
4649 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4650 if(checkcompressfile=="tarfile"):
4651 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4652 if(checkcompressfile=="zipfile"):
4653 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4654 if(rarfile_support and checkcompressfile=="rarfile"):
4655 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4656 if(py7zr_support and checkcompressfile=="7zipfile"):
4657 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4658 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4659 return False;
4660 compresscheck = CheckCompressionType(infile, formatspecs, True);
4661 if(not compresscheck):
4662 fextname = os.path.splitext(infile)[1];
4663 if(fextname==".gz"):
4664 compresscheck = "gzip";
4665 elif(fextname==".bz2"):
4666 compresscheck = "bzip2";
4667 elif(fextname==".zst"):
4668 compresscheck = "zstd";
4669 elif(fextname==".lz4" or fextname==".clz4"):
4670 compresscheck = "lz4";
4671 elif(fextname==".lzo" or fextname==".lzop"):
4672 compresscheck = "lzo";
4673 elif(fextname==".lzma" or fextname==".xz"):
4674 compresscheck = "lzma";
4675 else:
4676 return False;
4677 if(not compresscheck):
4678 return False;
4679 catfp = UncompressFile(infile, formatspecs, "rb");
4681 try:
4682 catfp.seek(0, 2);
4683 except OSError:
4684 SeekToEndOfFile(catfp);
4685 except ValueError:
4686 SeekToEndOfFile(catfp);
4687 CatSize = catfp.tell();
4688 CatSizeEnd = CatSize;
4690 try:
4691 catfp.seek(0, 0);
4692 except OSError:
4693 return False;
4694 except ValueError:
4695 return False;
4696 curloc = catfp.tell();
4697 if(curloc>0):
4698 catfp.seek(0, 0);
4699 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4700 if(curloc>0):
4701 catfp.seek(curloc, 0);
4702 catstring = catheader[0];
4703 catversion = re.findall(r"([\d]+)$", catstring);
4704 fprenumfiles = catheader[1];
4705 fnumfiles = int(fprenumfiles, 16);
4706 fprechecksumtype = catheader[2];
4707 fprechecksum = catheader[3];
4708 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4709 fnumfileshex = format(int(fnumfiles), 'x').lower();
4710 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4711 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4712 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4713 fheadtell = len(fileheader);
4714 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4715 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
4716 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4717 return False;
4718 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
4719 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4720 if(seekstart<0 and seekstart>fnumfiles):
4721 seekstart = 0;
4722 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
4723 seekend = fnumfiles;
4724 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
4725 seekend = fnumfiles - abs(seekend);
4726 if(seekstart>0):
4727 il = 0;
4728 while(il < seekstart):
4729 prefhstart = catfp.tell();
4730 if(formatspecs['new_style']):
4731 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4732 else:
4733 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4734 if(len(preheaderdata)==0):
4735 break;
4736 prefheadsize = int(preheaderdata[0], 16);
4737 prefnumfields = int(preheaderdata[1], 16);
4738 if(re.findall("^[.|/]", preheaderdata[3])):
4739 prefname = preheaderdata[3];
4740 else:
4741 prefname = "./"+preheaderdata[3];
4742 prefsize = int(preheaderdata[5], 16);
4743 prefcompression = preheaderdata[12];
4744 prefcsize = int(preheaderdata[13], 16);
4745 prefextrasize = int(preheaderdata[24], 16);
4746 prefextrafields = int(preheaderdata[25], 16);
4747 extrafieldslist = [];
4748 extrastart = 26;
4749 extraend = extrastart + prefextrafields;
4750 extrafieldslist = [];
4751 if(extrastart<extraend):
4752 extrafieldslist.append(preheaderdata[extrastart]);
4753 extrastart = extrastart + 1;
4754 prefchecksumtype = preheaderdata[extrastart].lower();
4755 prefcs = preheaderdata[extrastart + 1].lower();
4756 prefccs = preheaderdata[extrastart + 2].lower();
4757 hc = 0;
4758 hcmax = len(preheaderdata) - 2;
4759 hout = "";
4760 while(hc<hcmax):
4761 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs['format_delimiter']);
4762 hc = hc + 1;
4763 prenewfcs = GetFileChecksum(hout, preheaderdata[-3].lower(), True, formatspecs);
4764 if(prefcs!=prenewfcs and not skipchecksum):
4765 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4766 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4767 return False;
4768 valid_archive = False;
4769 invalid_archive = True;
4770 prefhend = catfp.tell() - 1;
4771 prefcontentstart = catfp.tell();
4772 prefcontents = "";
4773 pyhascontents = False;
4774 if(prefsize>0):
4775 if(prefcompression=="none" or prefcompression=="" or prefcompression=="auto"):
4776 prefcontents = catfp.read(prefsize);
4777 else:
4778 prefcontents = catfp.read(prefcsize);
4779 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4780 pyhascontents = True;
4781 if(prefccs!=prenewfccs and not skipchecksum):
4782 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4783 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4784 return False;
4785 catfp.seek(1, 1);
4786 il = il + 1;
4787 fileidnum = seekstart;
4788 realidnum = 0;
4789 while(fileidnum<seekend):
4790 catfhstart = catfp.tell();
4791 if(formatspecs['new_style']):
4792 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4793 else:
4794 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4795 if(len(catheaderdata)==0):
4796 break;
4797 catfheadsize = int(catheaderdata[0], 16);
4798 catfnumfields = int(catheaderdata[1], 16);
4799 catftype = int(catheaderdata[2], 16);
4800 if(re.findall("^[.|/]", catheaderdata[3])):
4801 catfname = catheaderdata[3];
4802 else:
4803 catfname = "./"+catheaderdata[3];
4804 catfbasedir = os.path.dirname(catfname);
4805 catflinkname = catheaderdata[4];
4806 catfsize = int(catheaderdata[5], 16);
4807 catfatime = int(catheaderdata[6], 16);
4808 catfmtime = int(catheaderdata[7], 16);
4809 catfctime = int(catheaderdata[8], 16);
4810 catfbtime = int(catheaderdata[9], 16);
4811 catfmode = int(catheaderdata[10], 16);
4812 catfchmode = stat.S_IMODE(catfmode);
4813 catftypemod = stat.S_IFMT(catfmode);
4814 catfwinattributes = int(catheaderdata[11], 16);
4815 catfcompression = catheaderdata[12];
4816 catfcsize = int(catheaderdata[13], 16);
4817 catfuid = int(catheaderdata[14], 16);
4818 catfuname = catheaderdata[15];
4819 catfgid = int(catheaderdata[16], 16);
4820 catfgname = catheaderdata[17];
4821 catfid = int(catheaderdata[18], 16);
4822 catfinode = int(catheaderdata[19], 16);
4823 catflinkcount = int(catheaderdata[20], 16);
4824 catfdev_minor = int(catheaderdata[21], 16);
4825 catfdev_major = int(catheaderdata[22], 16);
4826 catfrdev_minor = int(catheaderdata[23], 16);
4827 catfrdev_major = int(catheaderdata[24], 16);
4828 catfextrasize = int(catheaderdata[25], 16);
4829 catfextrafields = int(catheaderdata[26], 16);
4830 extrafieldslist = [];
4831 extrastart = 27;
4832 extraend = extrastart + catfextrafields;
4833 extrafieldslist = [];
4834 if(extrastart<extraend):
4835 extrafieldslist.append(catheaderdata[extrastart]);
4836 extrastart = extrastart + 1;
4837 catfchecksumtype = catheaderdata[extrastart].lower();
4838 catfcs = catheaderdata[extrastart + 1].lower();
4839 catfccs = catheaderdata[extrastart + 2].lower();
4840 hc = 0;
4841 hcmax = len(catheaderdata) - 2;
4842 hout = "";
4843 while(hc<hcmax):
4844 hout = hout + AppendNullByte(catheaderdata[hc], formatspecs['format_delimiter']);
4845 hc = hc + 1;
4846 catnewfcs = GetFileChecksum(hout, catheaderdata[-3].lower(), True, formatspecs);
4847 if(catfcs!=catnewfcs and not skipchecksum):
4848 VerbosePrintOut("File Header Checksum Error with file " + catfname + " at offset " + str(catfhstart));
4849 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
4850 return False;
4851 catfhend = catfp.tell() - 1;
4852 catfcontentstart = catfp.tell();
4853 catfcontents = BytesIO();
4854 pyhascontents = False;
4855 if(catfsize>0 and not listonly):
4856 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4857 catfcontents.write(catfp.read(catfsize));
4858 else:
4859 catfcontents.write(catfp.read(catfcsize));
4860 catfcontents.seek(0, 0);
4861 catnewfccs = GetFileChecksum(catfcontents.read(), catheaderdata[-3].lower(), False, formatspecs);
4862 pyhascontents = True;
4863 if(catfccs!=catnewfccs and skipchecksum):
4864 VerbosePrintOut("File Content Checksum Error with file " + catfname + " at offset " + str(catfcontentstart));
4865 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
4866 return False;
4867 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4868 pass;
4869 else:
4870 catfcontents.seek(0, 0);
4871 catfcontents = UncompressArchiveFile(catfcontents, formatspecs);
4872 if(catfsize>0 and listonly):
4873 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4874 catfp.seek(catfsize, 1);
4875 else:
4876 catfp.seek(catfcsize, 1);
4877 pyhascontents = False;
4878 catfp.seek(1, 1);
4879 catfcontentend = catfp.tell() - 1;
4880 catfcontents.seek(0, 0);
4881 catlist['ffilelist'].update({realidnum: {'fid': realidnum, 'fidalt': fileidnum, 'fheadersize': catfheadsize, 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize, 'fatime': catfatime, 'fmtime': catfmtime, 'fctime': catfctime, 'fbtime': catfbtime, 'fmode': catfmode, 'fchmode': catfchmode, 'ftypemod': catftypemod, 'fwinattributes': catfwinattributes, 'fcompression': catfcompression, 'fcsize': catfcsize, 'fuid': catfuid, 'funame': catfuname, 'fgid': catfgid, 'fgname': catfgname, 'finode': catfinode, 'flinkcount': catflinkcount, 'fminor': catfdev_minor, 'fmajor': catfdev_major, 'frminor': catfrdev_minor, 'frmajor': catfrdev_major, 'fchecksumtype': catfchecksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': catfextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': catfcs, 'fcontentchecksum': catfccs, 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': catfcontents} });
4882 fileidnum = fileidnum + 1;
4883 realidnum = realidnum + 1;
4884 if(returnfp):
4885 catlist.update({'catfp': catfp});
4886 else:
4887 catfp.close();
4888 return catlist;
4890 create_alias_function("", __file_format_name__, "ToArray", ArchiveFileToArray);
4892 def ArchiveFileStringToArray(catstr, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4893 catfp = BytesIO(catstr);
4894 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4895 return listcatfiles;
4897 create_alias_function("", __file_format_name__, "StringToArray", ArchiveFileStringToArray);
4899 def TarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4900 catfp = BytesIO();
4901 catfp = PackArchiveFileFromTarFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
4902 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4903 return listcatfiles;
4905 def ZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4906 catfp = BytesIO();
4907 catfp = PackArchiveFileFromZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
4908 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4909 return listcatfiles;
4911 if(not rarfile_support):
4912 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4913 return False;
4915 if(rarfile_support):
4916 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4917 catfp = BytesIO();
4918 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
4919 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4920 return listcatfiles;
4922 if(not py7zr_support):
4923 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4924 return False;
4926 if(py7zr_support):
4927 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4928 catfp = BytesIO();
4929 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
4930 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4931 return listcatfiles;
4933 def InFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4934 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4935 if(checkcompressfile=="tarfile"):
4936 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4937 elif(checkcompressfile=="zipfile"):
4938 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4939 elif(checkcompressfile=="catfile"):
4940 return ArchiveFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4941 elif(rarfile_support and checkcompressfile=="rarfile"):
4942 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4943 elif(py7zr_support and checkcompressfile=="7zipfile"):
4944 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4945 else:
4946 return False;
4947 return False;
4949 def ListDirToArrayAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
4950 catver = formatspecs['format_ver'];
4951 fileheaderver = str(int(catver.replace(".", "")));
4952 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
4953 advancedlist = formatspecs['use_advanced_list'];
4954 altinode = formatspecs['use_alt_inode'];
4955 infilelist = [];
4956 if(infiles=="-"):
4957 for line in sys.stdin:
4958 infilelist.append(line.strip());
4959 infilelist = list(filter(None, infilelist));
4960 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
4961 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
4962 return False;
4963 with UncompressFile(infiles, formatspecs, "r") as finfile:
4964 for line in finfile:
4965 infilelist.append(line.strip());
4966 infilelist = list(filter(None, infilelist));
4967 else:
4968 if(isinstance(infiles, (list, tuple, ))):
4969 infilelist = list(filter(None, infiles));
4970 elif(isinstance(infiles, (str, ))):
4971 infilelist = list(filter(None, [infiles]));
4972 if(advancedlist):
4973 GetDirList = ListDirAdvanced(infilelist, followlink, False);
4974 else:
4975 GetDirList = ListDir(infilelist, followlink, False);
4976 if(not GetDirList):
4977 return False;
4978 curinode = 0;
4979 curfid = 0;
4980 inodelist = [];
4981 inodetofile = {};
4982 filetoinode = {};
4983 inodetocatinode = {};
4984 fileidnum = 0;
4985 fnumfiles = int(len(GetDirList));
4986 catver = formatspecs['format_ver'];
4987 fileheaderver = str(int(catver.replace(".", "")));
4988 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
4989 fnumfileshex = format(int(fnumfiles), 'x').lower();
4990 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
4991 catversion = re.findall(r"([\d]+)$", fileheader);
4992 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
4993 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4994 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4995 fheadtell = len(fileheader);
4996 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
4997 for curfname in GetDirList:
4998 if(re.findall("^[.|/]", curfname)):
4999 fname = curfname;
5000 else:
5001 fname = "./"+curfname;
5002 if(verbose):
5003 VerbosePrintOut(fname);
5004 if(not followlink or followlink is None):
5005 fstatinfo = os.lstat(fname);
5006 else:
5007 fstatinfo = os.stat(fname);
5008 fpremode = fstatinfo.st_mode;
5009 finode = fstatinfo.st_ino;
5010 flinkcount = fstatinfo.st_nlink;
5011 ftype = 0;
5012 if(stat.S_ISREG(fpremode)):
5013 ftype = 0;
5014 elif(stat.S_ISLNK(fpremode)):
5015 ftype = 2;
5016 elif(stat.S_ISCHR(fpremode)):
5017 ftype = 3;
5018 elif(stat.S_ISBLK(fpremode)):
5019 ftype = 4;
5020 elif(stat.S_ISDIR(fpremode)):
5021 ftype = 5;
5022 elif(stat.S_ISFIFO(fpremode)):
5023 ftype = 6;
5024 elif(stat.S_ISSOCK(fpremode)):
5025 ftype = 8;
5026 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
5027 ftype = 9;
5028 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
5029 ftype = 10;
5030 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
5031 ftype = 11;
5032 else:
5033 ftype = 0;
5034 flinkname = "";
5035 fbasedir = os.path.dirname(fname);
5036 fcurfid = curfid;
5037 if(not followlink and finode!=0):
5038 if(ftype!=1):
5039 if(finode in inodelist):
5040 ftype = 1;
5041 flinkname = inodetofile[finode];
5042 if(altinode):
5043 fcurinode = finode;
5044 else:
5045 fcurinode = inodetocatinode[finode];
5046 if(finode not in inodelist):
5047 inodelist.append(finode);
5048 inodetofile.update({finode: fname});
5049 inodetocatinode.update({finode: curinode});
5050 if(altinode):
5051 fcurinode = finode;
5052 else:
5053 fcurinode = curinode;
5054 curinode = curinode + 1;
5055 else:
5056 fcurinode = curinode;
5057 curinode = curinode + 1;
5058 curfid = curfid + 1;
5059 if(ftype==2):
5060 flinkname = os.readlink(fname);
5061 fdev = fstatinfo.st_dev;
5062 getfdev = GetDevMajorMinor(fdev);
5063 fdev_minor = getfdev[0];
5064 fdev_major = getfdev[1];
5065 frdev = fstatinfo.st_dev;
5066 if(hasattr(fstatinfo, "st_rdev")):
5067 frdev = fstatinfo.st_rdev;
5068 else:
5069 frdev = fstatinfo.st_dev;
5070 getfrdev = GetDevMajorMinor(frdev);
5071 frdev_minor = getfrdev[0];
5072 frdev_major = getfrdev[1];
5073 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
5074 fsize = "0";
5075 if(ftype==0 or ftype==7):
5076 fsize = fstatinfo.st_size;
5077 fatime = fstatinfo.st_atime;
5078 fmtime = fstatinfo.st_mtime;
5079 fctime = fstatinfo.st_ctime;
5080 if(hasattr(fstatinfo, "st_birthtime")):
5081 fbtime = fstatinfo.st_birthtime;
5082 else:
5083 fbtime = fstatinfo.st_ctime;
5084 fmode = fstatinfo.st_mode;
5085 fchmode = stat.S_IMODE(fstatinfo.st_mode);
5086 ftypemod = stat.S_IFMT(fstatinfo.st_mode);
5087 fuid = fstatinfo.st_uid;
5088 fgid = fstatinfo.st_gid;
5089 funame = "";
5090 try:
5091 import pwd;
5092 try:
5093 userinfo = pwd.getpwuid(fstatinfo.st_uid);
5094 funame = userinfo.pw_name;
5095 except KeyError:
5096 funame = "";
5097 except ImportError:
5098 funame = "";
5099 fgname = "";
5100 try:
5101 import grp;
5102 try:
5103 groupinfo = grp.getgrgid(fstatinfo.st_gid);
5104 fgname = groupinfo.gr_name;
5105 except KeyError:
5106 fgname = "";
5107 except ImportError:
5108 fgname = "";
5109 fdev_minor = fdev_minor;
5110 fdev_major = fdev_major;
5111 frdev_minor = frdev_minor;
5112 frdev_major = frdev_major;
5113 flinkcount = flinkcount;
5114 if(hasattr(fstatinfo, "st_file_attributes")):
5115 fwinattributes = fstatinfo.st_file_attributes;
5116 else:
5117 fwinattributes = 0;
5118 fcompression = "";
5119 fcsize = 0;
5120 fcontents = BytesIO();
5121 if(ftype==0 or ftype==7):
5122 with open(fname, "rb") as fpc:
5123 shutil.copyfileobj(fpc, fcontents);
5124 if(followlink and (ftype==1 or ftype==2)):
5125 flstatinfo = os.stat(flinkname);
5126 with open(flinkname, "rb") as fpc:
5127 shutil.copyfileobj(fpc, fcontents);
5128 fcontents.seek(0, 0);
5129 ftypehex = format(ftype, 'x').lower();
5130 extrafields = len(extradata);
5131 extrafieldslist = extradata;
5132 catfextrafields = extrafields;
5133 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5134 if(len(extradata)>0):
5135 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5136 extrasizelen = len(extrasizestr);
5137 extrasizelenhex = format(extrasizelen, 'x').lower();
5138 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5139 catoutlen = len(catoutlist) + len(extradata) + 3;
5140 catoutlenhex = format(catoutlen, 'x').lower();
5141 catoutlist.insert(0, catoutlenhex);
5142 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5143 if(len(extradata)>0):
5144 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5145 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
5146 catfnumfields = catoutlen;
5147 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5148 fcontents.seek(0, 0);
5149 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5150 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5151 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5152 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5153 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5154 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5155 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5156 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5157 fheadtell += len(catfileoutstr) + 1;
5158 catfcontentend = fheadtell - 1;
5159 fcontents.seek(0, 0);
5160 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5161 pyhascontents = False;
5162 if(int(fsize)>0 and not listonly):
5163 pyhascontents = True;
5164 if(int(fsize)>0 and listonly):
5165 fcontents = BytesIO();
5166 pyhascontents = False;
5167 fcontents.seek(0, 0);
5168 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5169 fileidnum = fileidnum + 1;
5170 return catlist;
5172 def TarFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5173 curinode = 0;
5174 curfid = 0;
5175 inodelist = [];
5176 inodetofile = {};
5177 filetoinode = {};
5178 inodetocatinode = {};
5179 fileidnum = 0;
5180 if(infile=="-"):
5181 infile = BytesIO();
5182 if(hasattr(sys.stdin, "buffer")):
5183 shutil.copyfileobj(sys.stdin.buffer, infile);
5184 else:
5185 shutil.copyfileobj(sys.stdin, infile);
5186 infile.seek(0, 0);
5187 if(not infile):
5188 return False;
5189 infile.seek(0, 0);
5190 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
5191 infile = download_file_from_internet_file(infile);
5192 infile.seek(0, 0);
5193 if(not infile):
5194 return False;
5195 infile.seek(0, 0);
5196 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
5197 return False;
5198 elif(os.path.exists(infile) and os.path.isfile(infile)):
5199 try:
5200 if(not tarfile.is_tarfile(infile)):
5201 return False;
5202 except AttributeError:
5203 if(not is_tarfile(infile)):
5204 return False;
5205 else:
5206 return False;
5207 try:
5208 if(hasattr(infile, "read") or hasattr(infile, "write")):
5209 tarfp = tarfile.open(fileobj=infile, mode="r");
5210 else:
5211 tarfp = tarfile.open(infile, "r");
5212 except FileNotFoundError:
5213 return False;
5214 fnumfiles = int(len(tarfp.getmembers()));
5215 catver = formatspecs['format_ver'];
5216 fileheaderver = str(int(catver.replace(".", "")));
5217 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5218 fnumfileshex = format(int(fnumfiles), 'x').lower();
5219 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5220 catversion = re.findall(r"([\d]+)$", fileheader);
5221 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
5222 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
5223 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5224 fheadtell = len(fileheader);
5225 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5226 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
5227 if(re.findall("^[.|/]", member.name)):
5228 fname = member.name;
5229 else:
5230 fname = "./"+member.name;
5231 if(verbose):
5232 VerbosePrintOut(fname);
5233 fpremode = member.mode;
5234 ffullmode = member.mode;
5235 flinkcount = 0;
5236 ftype = 0;
5237 if(member.isreg()):
5238 ffullmode = member.mode + stat.S_IFREG;
5239 ftype = 0;
5240 elif(member.isdev()):
5241 ffullmode = member.mode;
5242 ftype = 7;
5243 elif(member.islnk()):
5244 ffullmode = member.mode + stat.S_IFREG;
5245 ftype = 1;
5246 elif(member.issym()):
5247 ffullmode = member.mode + stat.S_IFLNK;
5248 ftype = 2;
5249 elif(member.ischr()):
5250 ffullmode = member.mode + stat.S_IFCHR;
5251 ftype = 3;
5252 elif(member.isblk()):
5253 ffullmode = member.mode + stat.S_IFBLK;
5254 ftype = 4;
5255 elif(member.isdir()):
5256 ffullmode = member.mode + stat.S_IFDIR;
5257 ftype = 5;
5258 elif(member.isfifo()):
5259 ffullmode = member.mode + stat.S_IFIFO;
5260 ftype = 6;
5261 elif(member.issparse()):
5262 ffullmode = member.mode;
5263 ftype = 12;
5264 else:
5265 ffullmode = member.mode;
5266 ftype = 0;
5267 flinkname = "";
5268 fbasedir = os.path.dirname(fname);
5269 fcurfid = curfid;
5270 fcurinode = curfid;
5271 finode = fcurinode;
5272 curfid = curfid + 1;
5273 if(ftype==2):
5274 flinkname = member.linkname;
5275 fdev_minor = member.devminor;
5276 fdev_major = member.devmajor;
5277 frdev_minor = member.devminor;
5278 frdev_major = member.devmajor;
5279 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
5280 fsize = "0";
5281 elif(ftype==0 or ftype==7):
5282 fsize = member.size;
5283 else:
5284 fsize = member.size;
5285 fatime = member.mtime;
5286 fmtime = member.mtime;
5287 fctime = member.mtime;
5288 fbtime = member.mtime;
5289 fmode = ffullmode;
5290 fchmode = stat.S_IMODE(ffullmode);
5291 ftypemod = stat.S_IFMT(ffullmode);
5292 fuid = member.uid;
5293 fgid = member.gid;
5294 funame = member.uname;
5295 fgname = member.gname;
5296 flinkcount = flinkcount;
5297 fwinattributes = int(0);
5298 fcompression = "";
5299 fcsize = 0;
5300 fcontents = BytesIO();
5301 if(ftype==0 or ftype==7):
5302 with tarfp.extractfile(member) as fpc:
5303 shutil.copyfileobj(fpc, fcontents);
5304 fcontents.seek(0, 0);
5305 ftypehex = format(ftype, 'x').lower();
5306 extrafields = len(extradata);
5307 extrafieldslist = extradata;
5308 catfextrafields = extrafields;
5309 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5310 if(len(extradata)>0):
5311 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5312 extrasizelen = len(extrasizestr);
5313 extrasizelenhex = format(extrasizelen, 'x').lower();
5314 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5315 catoutlen = len(catoutlist) + len(extradata) + 3;
5316 catoutlenhex = format(catoutlen, 'x').lower();
5317 catoutlist.insert(0, catoutlenhex);
5318 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5319 if(len(extradata)>0):
5320 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5321 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
5322 catfnumfields = catoutlen;
5323 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5324 fcontents.seek(0, 0);
5325 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5326 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5327 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5328 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5329 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5330 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5331 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5332 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5333 fheadtell += len(catfileoutstr) + 1;
5334 catfcontentend = fheadtell - 1;
5335 fcontents.seek(0, 0);
5336 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5337 pyhascontents = False;
5338 if(int(fsize)>0 and not listonly):
5339 pyhascontents = True;
5340 if(int(fsize)>0 and listonly):
5341 fcontents = BytesIO();
5342 pyhascontents = False;
5343 fcontents.seek(0, 0);
5344 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5345 fileidnum = fileidnum + 1;
5346 return catlist;
5348 def ZipFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5349 curinode = 0;
5350 curfid = 0;
5351 inodelist = [];
5352 inodetofile = {};
5353 filetoinode = {};
5354 inodetocatinode = {};
5355 fileidnum = 0;
5356 if(infile=="-"):
5357 infile = BytesIO();
5358 if(hasattr(sys.stdin, "buffer")):
5359 shutil.copyfileobj(sys.stdin.buffer, infile);
5360 else:
5361 shutil.copyfileobj(sys.stdin, infile);
5362 infile.seek(0, 0);
5363 if(not infile):
5364 return False;
5365 infile.seek(0, 0);
5366 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
5367 infile = download_file_from_internet_file(infile);
5368 infile.seek(0, 0);
5369 if(not infile):
5370 return False;
5371 infile.seek(0, 0);
5372 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
5373 return False;
5374 else:
5375 return False;
5376 if(not zipfile.is_zipfile(infile)):
5377 return False;
5378 try:
5379 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
5380 except FileNotFoundError:
5381 return False;
5382 ziptest = zipfp.testzip();
5383 if(ziptest):
5384 VerbosePrintOut("Bad file found!");
5385 fnumfiles = int(len(zipfp.infolist()));
5386 catver = formatspecs['format_ver'];
5387 fileheaderver = str(int(catver.replace(".", "")));
5388 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5389 catversion = re.findall(r"([\d]+)$", fileheader);
5390 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
5391 fnumfileshex = format(int(fnumfiles), 'x').lower();
5392 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5393 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
5394 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5395 fheadtell = len(fileheader);
5396 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5397 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
5398 if(re.findall("^[.|/]", member.filename)):
5399 fname = member.filename;
5400 else:
5401 fname = "./"+member.filename;
5402 zipinfo = zipfp.getinfo(member.filename);
5403 if(verbose):
5404 VerbosePrintOut(fname);
5405 if(not member.is_dir()):
5406 fpremode = stat.S_IFREG + 438;
5407 elif(member.is_dir()):
5408 fpremode = stat.S_IFDIR + 511;
5409 flinkcount = 0;
5410 ftype = 0;
5411 if(not member.is_dir()):
5412 ftype = 0;
5413 elif(member.is_dir()):
5414 ftype = 5;
5415 flinkname = "";
5416 fbasedir = os.path.dirname(fname);
5417 fcurfid = curfid;
5418 fcurinode = curfid;
5419 finode = fcurinode;
5420 curfid = curfid + 1;
5421 fdev_minor = 0;
5422 fdev_major = 0;
5423 frdev_minor = 0;
5424 frdev_major = 0;
5425 if(ftype==5):
5426 fsize = "0";
5427 elif(ftype==0):
5428 fsize = member.file_size;
5429 else:
5430 fsize = member.file_size;
5431 fatime = time.mktime(member.date_time + (0, 0, -1));
5432 fmtime = time.mktime(member.date_time + (0, 0, -1));
5433 fctime = time.mktime(member.date_time + (0, 0, -1));
5434 fbtime = time.mktime(member.date_time + (0, 0, -1));
5435 if(zipinfo.create_system==0 or zipinfo.create_system==10):
5436 fwinattributes = int(zipinfo.external_attr);
5437 if(not member.is_dir()):
5438 fmode = int(stat.S_IFREG + 438);
5439 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
5440 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
5441 elif(member.is_dir()):
5442 fmode = int(stat.S_IFDIR + 511);
5443 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
5444 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
5445 elif(zipinfo.create_system==3):
5446 fwinattributes = int(0);
5447 fmode = int(zipinfo.external_attr);
5448 else:
5449 fwinattributes = int(0);
5450 if(not member.is_dir()):
5451 fmode = int(stat.S_IFREG + 438);
5452 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
5453 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
5454 elif(member.is_dir()):
5455 fmode = int(stat.S_IFDIR + 511);
5456 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
5457 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
5458 fcompression = "";
5459 fcsize = 0;
5460 try:
5461 fuid = os.getuid();
5462 except AttributeError:
5463 fuid = 0;
5464 except KeyError:
5465 fuid = 0;
5466 try:
5467 fgid = os.getgid();
5468 except AttributeError:
5469 fgid = 0;
5470 except KeyError:
5471 fgid = 0;
5472 try:
5473 import pwd;
5474 try:
5475 userinfo = pwd.getpwuid(os.getuid());
5476 funame = userinfo.pw_name;
5477 except KeyError:
5478 funame = "";
5479 except AttributeError:
5480 funame = "";
5481 except ImportError:
5482 funame = "";
5483 fgname = "";
5484 try:
5485 import grp;
5486 try:
5487 groupinfo = grp.getgrgid(os.getgid());
5488 fgname = groupinfo.gr_name;
5489 except KeyError:
5490 fgname = "";
5491 except AttributeError:
5492 fgname = "";
5493 except ImportError:
5494 fgname = "";
5495 fcontents = BytesIO();
5496 if(ftype==0):
5497 fcontents.write(zipfp.read(member.filename));
5498 fcontents.seek(0, 0);
5499 ftypehex = format(ftype, 'x').lower();
5500 extrafields = len(extradata);
5501 extrafieldslist = extradata;
5502 catfextrafields = extrafields;
5503 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5504 if(len(extradata)>0):
5505 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5506 extrasizelen = len(extrasizestr);
5507 extrasizelenhex = format(extrasizelen, 'x').lower();
5508 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5509 catoutlen = len(catoutlist) + len(extradata) + 3;
5510 catoutlenhex = format(catoutlen, 'x').lower();
5511 catoutlist.insert(0, catoutlenhex);
5512 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5513 if(len(extradata)>0):
5514 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5515 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
5516 catfnumfields = catoutlen;
5517 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5518 fcontents.seek(0, 0);
5519 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5520 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5521 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5522 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5523 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5524 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5525 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5526 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5527 fheadtell += len(catfileoutstr) + 1;
5528 catfcontentend = fheadtell - 1;
5529 fcontents.seek(0, 0);
5530 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5531 pyhascontents = False;
5532 if(int(fsize)>0 and not listonly):
5533 pyhascontents = True;
5534 if(int(fsize)>0 and listonly):
5535 fcontents = BytesIO();
5536 pyhascontents = False;
5537 fcontents.seek(0, 0);
5538 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5539 fileidnum = fileidnum + 1;
5540 return catlist;
5542 if(not rarfile_support):
5543 def RarFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5544 return False;
5546 if(rarfile_support):
5547 def RarFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5548 curinode = 0;
5549 curfid = 0;
5550 inodelist = [];
5551 inodetofile = {};
5552 filetoinode = {};
5553 inodetocatinode = {};
5554 fileidnum = 0;
5555 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
5556 return False;
5557 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
5558 return False;
5559 rarfp = rarfile.RarFile(infile, "r");
5560 rartest = rarfp.testrar();
5561 if(rartest):
5562 VerbosePrintOut("Bad file found!");
5563 fnumfiles = int(len(rarfp.infolist()));
5564 catver = formatspecs['format_ver'];
5565 fileheaderver = str(int(catver.replace(".", "")));
5566 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5567 catversion = re.findall(r"([\d]+)$", fileheader);
5568 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
5569 fnumfileshex = format(int(fnumfiles), 'x').lower();
5570 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5571 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
5572 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5573 fheadtell = len(fileheader);
5574 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5575 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
5576 is_unix = False;
5577 is_windows = False;
5578 if(member.host_os==rarfile.RAR_OS_UNIX):
5579 is_windows = False;
5580 try:
5581 member.external_attr
5582 is_unix = True;
5583 except AttributeError:
5584 is_unix = False;
5585 elif(member.host_os==rarfile.RAR_OS_WIN32):
5586 is_unix = False;
5587 try:
5588 member.external_attr
5589 is_windows = True;
5590 except AttributeError:
5591 is_windows = False;
5592 else:
5593 is_unix = False;
5594 is_windows = False;
5595 if(re.findall("^[.|/]", member.filename)):
5596 fname = member.filename;
5597 else:
5598 fname = "./"+member.filename;
5599 rarinfo = rarfp.getinfo(member.filename);
5600 if(verbose):
5601 VerbosePrintOut(fname);
5602 if(is_unix and member.external_attr !=0):
5603 fpremode = int(member.external_attr);
5604 elif(member.is_file()):
5605 fpremode = stat.S_IFREG + 438;
5606 elif(member.is_symlink()):
5607 fpremode = stat.S_IFLNK + 438;
5608 elif(member.is_dir()):
5609 fpremode = stat.S_IFDIR + 511;
5610 if(is_windows and member.external_attr !=0):
5611 fwinattributes = int(member.external_attr);
5612 else:
5613 fwinattributes = int(0);
5614 fcompression = "";
5615 fcsize = 0;
5616 flinkcount = 0;
5617 ftype = 0;
5618 if(member.is_file()):
5619 ftype = 0;
5620 elif(member.is_symlink()):
5621 ftype = 2;
5622 elif(member.is_dir()):
5623 ftype = 5;
5624 flinkname = "";
5625 if(ftype==2):
5626 flinkname = rarfp.read(member.filename).decode("UTF-8");
5627 fbasedir = os.path.dirname(fname);
5628 fcurfid = curfid;
5629 fcurinode = curfid;
5630 finode = fcurinode;
5631 curfid = curfid + 1;
5632 fdev_minor = 0;
5633 fdev_major = 0;
5634 frdev_minor = 0;
5635 frdev_major = 0;
5636 if(ftype==5):
5637 fsize = "0";
5638 if(ftype==0):
5639 fsize = member.file_size;
5640 try:
5641 if(member.atime):
5642 fatime = int(member.atime.timestamp());
5643 else:
5644 fatime = int(member.mtime.timestamp());
5645 except AttributeError:
5646 fatime = int(member.mtime.timestamp());
5647 fmtime = int(member.mtime.timestamp());
5648 try:
5649 if(member.ctime):
5650 fctime = int(member.ctime.timestamp());
5651 else:
5652 fctime = int(member.mtime.timestamp());
5653 except AttributeError:
5654 fctime = int(member.mtime.timestamp());
5655 fbtime = int(member.mtime.timestamp());
5656 if(is_unix and member.external_attr !=0):
5657 fmode = int(member.external_attr);
5658 fchmode = int(stat.S_IMODE(member.external_attr));
5659 ftypemod = int(stat.S_IFMT(member.external_attr));
5660 elif(member.is_file()):
5661 fmode = int(stat.S_IFREG + 438)
5662 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5663 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5664 elif(member.is_symlink()):
5665 fmode = int(stat.S_IFLNK + 438)
5666 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5667 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5668 elif(member.is_dir()):
5669 fmode = int(stat.S_IFDIR + 511)
5670 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
5671 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
5672 try:
5673 fuid = os.getuid();
5674 except AttributeError:
5675 fuid = 0;
5676 except KeyError:
5677 fuid = 0;
5678 try:
5679 fgid = os.getgid();
5680 except AttributeError:
5681 fgid = 0;
5682 except KeyError:
5683 fgid = 0;
5684 try:
5685 import pwd;
5686 try:
5687 userinfo = pwd.getpwuid(os.getuid());
5688 funame = userinfo.pw_name;
5689 except KeyError:
5690 funame = "";
5691 except AttributeError:
5692 funame = "";
5693 except ImportError:
5694 funame = "";
5695 fgname = "";
5696 try:
5697 import grp;
5698 try:
5699 groupinfo = grp.getgrgid(os.getgid());
5700 fgname = groupinfo.gr_name;
5701 except KeyError:
5702 fgname = "";
5703 except AttributeError:
5704 fgname = "";
5705 except ImportError:
5706 fgname = "";
5707 fcontents = BytesIO();
5708 if(ftype==0):
5709 fcontents.write(rarfp.read(member.filename));
5710 fcontents.seek(0, 0);
5711 ftypehex = format(ftype, 'x').lower();
5712 extrafields = len(extradata);
5713 extrafieldslist = extradata;
5714 catfextrafields = extrafields;
5715 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5716 if(len(extradata)>0):
5717 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5718 extrasizelen = len(extrasizestr);
5719 extrasizelenhex = format(extrasizelen, 'x').lower();
5720 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5721 catoutlen = len(catoutlist) + len(extradata) + 3;
5722 catoutlenhex = format(catoutlen, 'x').lower();
5723 catoutlist.insert(0, catoutlenhex);
5724 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5725 if(len(extradata)>0):
5726 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5727 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
5728 catfnumfields = 24 + catfextrafields;
5729 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5730 fcontents.seek(0, 0);
5731 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5732 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5733 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5734 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5735 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5736 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5737 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5738 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5739 fheadtell += len(catfileoutstr) + 1;
5740 catfcontentend = fheadtell - 1;
5741 fcontents.seek(0, 0);
5742 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5743 pyhascontents = False;
5744 if(int(fsize)>0 and not listonly):
5745 pyhascontents = True;
5746 if(int(fsize)>0 and listonly):
5747 fcontents = BytesIO();
5748 pyhascontents = False;
5749 fcontents.seek(0, 0);
5750 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5751 fileidnum = fileidnum + 1;
5752 return catlist;
5754 if(not py7zr_support):
5755 def SevenZipFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5756 return False;
5758 if(py7zr_support):
5759 def SevenZipFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5760 curinode = 0;
5761 curfid = 0;
5762 inodelist = [];
5763 inodetofile = {};
5764 filetoinode = {};
5765 inodetocatinode = {};
5766 fileidnum = 0;
5767 szpfp = py7zr.SevenZipFile(infile, mode="r");
5768 file_content = szpfp.readall();
5769 #sztest = szpfp.testzip();
5770 sztestalt = szpfp.test();
5771 if(sztestalt):
5772 VerbosePrintOut("Bad file found!");
5773 numfiles = int(len(szpfp.list()));
5774 catver = formatspecs['format_ver'];
5775 fileheaderver = str(int(catver.replace(".", "")));
5776 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5777 catversion = re.findall(r"([\d]+)$", fileheader);
5778 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
5779 fnumfileshex = format(int(fnumfiles), 'x').lower();
5780 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5781 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
5782 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5783 fheadtell = len(fileheader);
5784 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5785 for member in sorted(szpfp.list(), key=lambda x: x.filename):
5786 if(re.findall("^[.|/]", member.filename)):
5787 fname = member.filename;
5788 else:
5789 fname = "./"+member.filename;
5790 if(not member.is_directory):
5791 fpremode = int(stat.S_IFREG + 438);
5792 elif(member.is_directory):
5793 fpremode = int(stat.S_IFDIR + 511);
5794 fwinattributes = int(0);
5795 fcompression = "";
5796 fcsize = 0;
5797 flinkcount = 0;
5798 ftype = 0;
5799 if(member.is_directory):
5800 ftype = 5;
5801 else:
5802 ftype = 0;
5803 flinkname = "";
5804 fbasedir = os.path.dirname(fname);
5805 fcurfid = curfid;
5806 fcurinode = curfid;
5807 finode = fcurinode;
5808 curfid = curfid + 1;
5809 fdev_minor = 0;
5810 fdev_major = 0;
5811 frdev_minor = 0;
5812 frdev_major = 0;
5813 if(ftype==5):
5814 fsize = "0";
5815 fatime = int(member.creationtime.timestamp());
5816 fmtime = int(member.creationtime.timestamp());
5817 fctime = int(member.creationtime.timestamp());
5818 fbtime = int(member.creationtime.timestamp());
5819 if(member.is_directory):
5820 fmode = int(stat.S_IFDIR + 511)
5821 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
5822 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
5823 else:
5824 fmode = int(stat.S_IFLNK + 438)
5825 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5826 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5827 try:
5828 fuid = os.getuid();
5829 except AttributeError:
5830 fuid = 0;
5831 except KeyError:
5832 fuid = 0;
5833 try:
5834 fgid = os.getgid();
5835 except AttributeError:
5836 fgid = 0;
5837 except KeyError:
5838 fgid = 0;
5839 try:
5840 import pwd;
5841 try:
5842 userinfo = pwd.getpwuid(os.getuid());
5843 funame = userinfo.pw_name;
5844 except KeyError:
5845 funame = "";
5846 except AttributeError:
5847 funame = "";
5848 except ImportError:
5849 funame = "";
5850 fgname = "";
5851 try:
5852 import grp;
5853 try:
5854 groupinfo = grp.getgrgid(os.getgid());
5855 fgname = groupinfo.gr_name;
5856 except KeyError:
5857 fgname = "";
5858 except AttributeError:
5859 fgname = "";
5860 except ImportError:
5861 fgname = "";
5862 fcontents = BytesIO();
5863 if(ftype==0):
5864 fcontents.write(file_content[member.filename].read());
5865 fsize = format(fcontents.tell(), 'x').lower();
5866 fileop.close();
5867 fcontents.seek(0, 0);
5868 ftypehex = format(ftype, 'x').lower();
5869 extrafields = len(extradata);
5870 extrafieldslist = extradata;
5871 catfextrafields = extrafields;
5872 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5873 if(len(extradata)>0):
5874 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5875 extrasizelen = len(extrasizestr);
5876 extrasizelenhex = format(extrasizelen, 'x').lower();
5877 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5878 catoutlen = len(catoutlist) + len(extradata) + 3;
5879 catoutlenhex = format(catoutlen, 'x').lower();
5880 catoutlist.insert(0, catoutlenhex);
5881 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5882 if(len(extradata)>0):
5883 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5884 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
5885 catfnumfields = 24 + catfextrafields;
5886 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5887 fcontents.seek(0, 0);
5888 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5889 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5890 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5891 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5892 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5893 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5894 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5895 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5896 fheadtell += len(catfileoutstr) + 1;
5897 catfcontentend = fheadtell - 1;
5898 fcontents.seek(0, 0);
5899 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5900 pyhascontents = False;
5901 if(int(fsize)>0 and not listonly):
5902 pyhascontents = True;
5903 if(int(fsize)>0 and listonly):
5904 fcontents = BytesIO();
5905 pyhascontents = False;
5906 fcontents.seek(0, 0);
5907 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5908 fileidnum = fileidnum + 1;
5909 return catlist;
5911 def InFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5912 checkcompressfile = CheckCompressionSubType(infiles, formatspecs, True);
5913 if(checkcompressfile=="tarfile"):
5914 return TarFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5915 elif(checkcompressfile=="zipfile"):
5916 return ZipFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5917 elif(checkcompressfile=="catfile"):
5918 return ArchiveFileToArray(infile, 0, 0, listonly, False, formatspecs, False);
5919 elif(rarfile_support and checkcompressfile=="rarfile"):
5920 return RarFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5921 elif(py7zr_support and checkcompressfile=="7zipfile"):
5922 return SevenZipFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5923 else:
5924 return False;
5925 return False;
5927 def ListDirToArray(infiles, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, listonly=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
5928 outarray = BytesIO();
5929 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
5930 listcatfiles = ArchiveFileToArray(outarray, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5931 return listcatfiles;
5933 def ArchiveFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5934 if(isinstance(infile, dict)):
5935 listcatfiles = infile;
5936 else:
5937 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
5938 infile = RemoveWindowsPath(infile);
5939 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5940 if(not listcatfiles):
5941 return False;
5942 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5943 if(returnfp):
5944 catarray.update({'catfp': listcatfiles['catfp']});
5945 lenlist = len(listcatfiles['ffilelist']);
5946 lcfi = 0;
5947 lcfx = int(listcatfiles['fnumfiles']);
5948 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5949 lcfx = int(lenlist);
5950 else:
5951 lcfx = int(listcatfiles['fnumfiles']);
5952 while(lcfi < lcfx):
5953 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5954 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5955 catarray['filetoid'].update(filetoidarray);
5956 catarray['idtofile'].update(idtofilearray);
5957 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5958 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5959 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5960 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5961 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5962 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5963 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5964 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5965 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5966 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5967 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5968 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5969 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5970 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5971 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5972 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5973 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5974 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5975 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5976 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5977 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5978 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5979 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5980 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5981 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5982 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5983 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
5984 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5985 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5986 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5987 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5988 lcfi = lcfi + 1;
5989 return catarray;
5991 create_alias_function("", __file_format_name__, "ToArrayIndex", ArchiveFileToArrayIndex);
5993 def ListDirToArrayIndexAlt(infiles, dirlistfromtxt=False, followlink=False, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5994 listcatfiles = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, listonly, checksumtype, extradata, formatspecs, verbose);
5995 if(not listcatfiles):
5996 return False;
5997 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5998 lenlist = len(listcatfiles['ffilelist']);
5999 lcfi = 0;
6000 lcfx = int(listcatfiles['fnumfiles']);
6001 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6002 lcfx = int(lenlist);
6003 else:
6004 lcfx = int(listcatfiles['fnumfiles']);
6005 while(lcfi < lcfx):
6006 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6007 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6008 catarray['filetoid'].update(filetoidarray);
6009 catarray['idtofile'].update(idtofilearray);
6010 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6011 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6012 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6013 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6014 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6015 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6016 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6017 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6018 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6019 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6020 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6021 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6022 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6023 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6024 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6025 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6026 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6027 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6028 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6029 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6030 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6031 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6032 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6033 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6034 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6035 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6036 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6037 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6038 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6039 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6040 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6041 lcfi = lcfi + 1;
6042 return catarray;
6044 def TarFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6045 listcatfiles = TarFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
6046 if(not listcatfiles):
6047 return False;
6048 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6049 lenlist = len(listcatfiles['ffilelist']);
6050 lcfi = 0;
6051 lcfx = int(listcatfiles['fnumfiles']);
6052 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6053 lcfx = int(lenlist);
6054 else:
6055 lcfx = int(listcatfiles['fnumfiles']);
6056 while(lcfi < lcfx):
6057 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6058 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6059 catarray['filetoid'].update(filetoidarray);
6060 catarray['idtofile'].update(idtofilearray);
6061 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6062 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6063 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6064 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6065 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6066 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6067 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6068 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6069 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6070 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6071 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6072 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6073 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6074 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6075 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6076 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6077 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6078 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6079 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6080 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6081 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6082 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6083 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6084 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6085 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6086 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6087 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6088 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6089 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6090 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6091 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6092 lcfi = lcfi + 1;
6093 return catarray;
6095 def ZipFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6096 listcatfiles = ZipFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
6097 if(not listcatfiles):
6098 return False;
6099 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6100 lenlist = len(listcatfiles['ffilelist']);
6101 lcfi = 0;
6102 lcfx = int(listcatfiles['fnumfiles']);
6103 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6104 lcfx = int(lenlist);
6105 else:
6106 lcfx = int(listcatfiles['fnumfiles']);
6107 while(lcfi < lcfx):
6108 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6109 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6110 catarray['filetoid'].update(filetoidarray);
6111 catarray['idtofile'].update(idtofilearray);
6112 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6113 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6114 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6115 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6116 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6117 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6118 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6119 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6120 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6121 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6122 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6123 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6124 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6125 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6126 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6127 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6128 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6129 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6130 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6131 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6132 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6133 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6134 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6135 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6136 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6137 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6138 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6139 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6140 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6141 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6142 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6143 lcfi = lcfi + 1;
6144 return catarray;
6146 if(not rarfile_support):
6147 def RarFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6148 return False;
6150 if(rarfile_support):
6151 def RarFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6152 listcatfiles = RarFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
6153 if(not listcatfiles):
6154 return False;
6155 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6156 lenlist = len(listcatfiles['ffilelist']);
6157 lcfi = 0;
6158 lcfx = int(listcatfiles['fnumfiles']);
6159 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6160 lcfx = int(lenlist);
6161 else:
6162 lcfx = int(listcatfiles['fnumfiles']);
6163 while(lcfi < lcfx):
6164 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6165 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6166 catarray['filetoid'].update(filetoidarray);
6167 catarray['idtofile'].update(idtofilearray);
6168 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6169 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6170 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6171 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6172 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6173 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6174 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6175 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6176 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6177 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6178 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6179 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6180 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6181 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6182 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6183 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6184 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6185 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6186 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6187 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6188 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6189 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6190 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6191 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6192 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6193 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6194 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6195 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6196 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6197 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6198 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6199 lcfi = lcfi + 1;
6200 return catarray;
6202 if(not py7zr_support):
6203 def SevenZipFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6204 return False;
6206 if(py7zr_support):
6207 def SevenZipFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6208 listcatfiles = SevenZipFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
6209 if(not listcatfiles):
6210 return False;
6211 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6212 lenlist = len(listcatfiles['ffilelist']);
6213 lcfi = 0;
6214 lcfx = int(listcatfiles['fnumfiles']);
6215 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6216 lcfx = int(lenlist);
6217 else:
6218 lcfx = int(listcatfiles['fnumfiles']);
6219 while(lcfi < lcfx):
6220 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6221 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6222 catarray['filetoid'].update(filetoidarray);
6223 catarray['idtofile'].update(idtofilearray);
6224 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6225 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6226 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6227 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6228 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6229 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6230 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6231 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6232 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6233 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6234 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6235 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6236 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6237 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6238 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6239 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6240 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6241 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6242 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6243 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6244 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6245 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6246 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6247 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6248 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6249 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6250 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6251 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6252 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6253 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6254 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6255 lcfi = lcfi + 1;
6256 return catarray;
6258 def ArchiveFileStringToArrayIndex(catstr, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6259 catfp = BytesIO(catstr);
6260 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
6261 return listcatfiles;
6263 create_alias_function("", __file_format_name__, "StringToArrayIndex", ArchiveFileStringToArrayIndex);
6265 def TarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6266 catfp = BytesIO();
6267 catfp = PackArchiveFileFromTarFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
6268 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
6269 return listcatfiles;
6271 def ZipFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6272 catfp = BytesIO();
6273 catfp = PackArchiveFileFromZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
6274 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
6275 return listcatfiles;
6277 if(not rarfile_support):
6278 def RarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6279 return False;
6281 if(rarfile_support):
6282 def RarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6283 catfp = BytesIO();
6284 catfp = PackArchiveFileFromRarFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
6285 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
6286 return listcatfiles;
6288 if(not py7zr_support):
6289 def SevenZipFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6290 return False;
6292 if(py7zr_support):
6293 def SevenZipFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6294 catfp = BytesIO();
6295 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
6296 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
6297 return listcatfiles;
6299 def ListDirToArrayIndex(infiles, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, listonly=False, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6300 outarray = BytesIO();
6301 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose, True);
6302 listcatfiles = ArchiveFileToArrayIndex(outarray, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp)
6303 return listcatfiles;
6305 def RePackArchiveFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6306 if(isinstance(infile, dict)):
6307 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6308 listcatfiles = prelistcatfiles['list'];
6309 else:
6310 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6311 infile = RemoveWindowsPath(infile);
6312 if(followlink):
6313 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6314 listcatfiles = prelistcatfiles['list'];
6315 else:
6316 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6317 if(outfile!="-" and not hasattr(infile, "read") and not hasattr(outfile, "write")):
6318 outfile = RemoveWindowsPath(outfile);
6319 checksumtype = checksumtype.lower();
6320 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
6321 checksumtype="crc32";
6322 if(checksumtype=="none"):
6323 checksumtype = "";
6324 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
6325 compression = "auto";
6326 if(compression not in compressionlist and compression is None):
6327 compression = "auto";
6328 if(verbose):
6329 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6330 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
6331 if(os.path.exists(outfile)):
6332 try:
6333 os.unlink(outfile);
6334 except OSError as e:
6335 pass;
6336 if(not listcatfiles):
6337 return False;
6338 if(outfile=="-"):
6339 verbose = False;
6340 catfp = BytesIO();
6341 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
6342 catfp = outfile;
6343 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
6344 catfp = BytesIO();
6345 else:
6346 fbasename = os.path.splitext(outfile)[0];
6347 fextname = os.path.splitext(outfile)[1];
6348 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
6349 catver = formatspecs['format_ver'];
6350 fileheaderver = str(int(catver.replace(".", "")));
6351 lenlist = len(listcatfiles['ffilelist']);
6352 fnumfiles = int(listcatfiles['fnumfiles']);
6353 if(lenlist>fnumfiles or lenlist<fnumfiles):
6354 fnumfiles = lenlist;
6355 AppendFileHeader(catfp, fnumfiles, checksumtype, formatspecs);
6356 lenlist = len(listcatfiles['ffilelist']);
6357 fnumfiles = int(listcatfiles['fnumfiles']);
6358 lcfi = 0;
6359 lcfx = int(listcatfiles['fnumfiles']);
6360 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6361 lcfx = int(lenlist);
6362 else:
6363 lcfx = int(listcatfiles['fnumfiles']);
6364 curinode = 0;
6365 curfid = 0;
6366 inodelist = [];
6367 inodetofile = {};
6368 filetoinode = {};
6369 reallcfi = 0;
6370 while(lcfi < lcfx):
6371 if(re.findall("^[.|/]", listcatfiles['ffilelist'][reallcfi]['fname'])):
6372 fname = listcatfiles['ffilelist'][reallcfi]['fname'];
6373 else:
6374 fname = "./"+listcatfiles['ffilelist'][reallcfi]['fname'];
6375 if(verbose):
6376 VerbosePrintOut(fname);
6377 fheadersize = format(int(listcatfiles['ffilelist'][reallcfi]['fheadersize']), 'x').lower();
6378 fsize = format(int(listcatfiles['ffilelist'][reallcfi]['fsize']), 'x').lower();
6379 flinkname = listcatfiles['ffilelist'][reallcfi]['flinkname'];
6380 fatime = format(int(listcatfiles['ffilelist'][reallcfi]['fatime']), 'x').lower();
6381 fmtime = format(int(listcatfiles['ffilelist'][reallcfi]['fmtime']), 'x').lower();
6382 fctime = format(int(listcatfiles['ffilelist'][reallcfi]['fctime']), 'x').lower();
6383 fbtime = format(int(listcatfiles['ffilelist'][reallcfi]['fbtime']), 'x').lower();
6384 fmode = format(int(listcatfiles['ffilelist'][reallcfi]['fmode']), 'x').lower();
6385 fchmode = format(int(listcatfiles['ffilelist'][reallcfi]['fchmode']), 'x').lower();
6386 fuid = format(int(listcatfiles['ffilelist'][reallcfi]['fuid']), 'x').lower();
6387 funame = listcatfiles['ffilelist'][reallcfi]['funame'];
6388 fgid = format(int(listcatfiles['ffilelist'][reallcfi]['fgid']), 'x').lower();
6389 fgname = listcatfiles['ffilelist'][reallcfi]['fgname'];
6390 finode = format(int(listcatfiles['ffilelist'][reallcfi]['finode']), 'x').lower();
6391 flinkcount = format(int(listcatfiles['ffilelist'][reallcfi]['flinkcount']), 'x').lower();
6392 fwinattributes = format(int(listcatfiles['ffilelist'][reallcfi]['fwinattributes']), 'x').lower();
6393 fcompression = listcatfiles['ffilelist'][reallcfi]['fcompression'];
6394 fcsize = format(int(listcatfiles['ffilelist'][reallcfi]['fcsize']), 'x').lower();
6395 fdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['fminor']), 'x').lower();
6396 fdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['fmajor']), 'x').lower();
6397 frdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['frminor']), 'x').lower();
6398 frdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['frmajor']), 'x').lower();
6399 if(len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>listcatfiles['ffilelist'][reallcfi]['fextrafields'] and len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>0):
6400 listcatfiles['ffilelist'][reallcfi]['fextrafields'] = len(listcatfiles['ffilelist'][reallcfi]['fextralist']);
6401 if(not followlink and len(extradata)<0):
6402 extradata = listcatfiles['ffilelist'][reallcfi]['fextralist'];
6403 fcontents = listcatfiles['ffilelist'][reallcfi]['fcontents'];
6404 fcompression = "";
6405 fcsize = format(int(0), 'x').lower();
6406 if(not compresswholefile):
6407 fcontents.seek(0, 2);
6408 ucfsize = fcontents.tell();
6409 fcontents.seek(0, 0);
6410 if(compression=="auto"):
6411 ilsize = len(compressionlistalt);
6412 ilmin = 0;
6413 ilcsize = [];
6414 while(ilmin < ilsize):
6415 cfcontents = BytesIO();
6416 shutil.copyfileobj(fcontents, cfcontents);
6417 fcontents.seek(0, 0);
6418 cfcontents.seek(0, 0);
6419 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
6420 if(cfcontents):
6421 cfcontents.seek(0, 2);
6422 ilcsize.append(cfcontents.tell());
6423 cfcontents.close();
6424 else:
6425 try:
6426 ilcsize.append(sys.maxint);
6427 except AttributeError:
6428 ilcsize.append(sys.maxsize);
6429 ilmin = ilmin + 1;
6430 ilcmin = ilcsize.index(min(ilcsize));
6431 compression = compressionlistalt[ilcmin];
6432 fcontents.seek(0, 0);
6433 cfcontents = BytesIO();
6434 shutil.copyfileobj(fcontents, cfcontents);
6435 cfcontents.seek(0, 0);
6436 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
6437 cfcontents.seek(0, 2);
6438 cfsize = cfcontents.tell();
6439 if(ucfsize > cfsize):
6440 fcsize = format(int(cfsize), 'x').lower();
6441 fcompression = compression;
6442 fcontents.close();
6443 fcontents = cfcontents;
6444 if(followlink):
6445 if(listcatfiles['ffilelist'][reallcfi]['ftype']==1 or listcatfiles['ffilelist'][reallcfi]['ftype']==2):
6446 getflinkpath = listcatfiles['ffilelist'][reallcfi]['flinkname'];
6447 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6448 flinkinfo = listcatfiles['ffilelist'][flinkid];
6449 fheadersize = format(int(flinkinfo['fheadersize']), 'x').lower();
6450 fsize = format(int(flinkinfo['fsize']), 'x').lower();
6451 flinkname = flinkinfo['flinkname'];
6452 fatime = format(int(flinkinfo['fatime']), 'x').lower();
6453 fmtime = format(int(flinkinfo['fmtime']), 'x').lower();
6454 fctime = format(int(flinkinfo['fctime']), 'x').lower();
6455 fbtime = format(int(flinkinfo['fbtime']), 'x').lower();
6456 fmode = format(int(flinkinfo['fmode']), 'x').lower();
6457 fchmode = format(int(flinkinfo['fchmode']), 'x').lower();
6458 fuid = format(int(flinkinfo['fuid']), 'x').lower();
6459 funame = flinkinfo['funame'];
6460 fgid = format(int(flinkinfo['fgid']), 'x').lower();
6461 fgname = flinkinfo['fgname'];
6462 finode = format(int(flinkinfo['finode']), 'x').lower();
6463 flinkcount = format(int(flinkinfo['flinkcount']), 'x').lower();
6464 fwinattributes = format(int(flinkinfo['fwinattributes']), 'x').lower();
6465 fcompression = flinkinfo['fcompression'];
6466 fcsize = format(int(flinkinfo['fcsize']), 'x').lower();
6467 fdev_minor = format(int(flinkinfo['fminor']), 'x').lower();
6468 fdev_major = format(int(flinkinfo['fmajor']), 'x').lower();
6469 frdev_minor = format(int(flinkinfo['frminor']), 'x').lower();
6470 frdev_major = format(int(flinkinfo['frmajor']), 'x').lower();
6471 if(len(flinkinfo['fextralist'])>flinkinfo['fextrafields'] and len(flinkinfo['fextralist'])>0):
6472 flinkinfo['fextrafields'] = len(flinkinfo['fextralist']);
6473 if(len(extradata)<0):
6474 extradata = flinkinfo['fextralist'];
6475 fcontents = flinkinfo['fcontents'];
6476 ftypehex = format(flinkinfo['ftype'], 'x').lower();
6477 else:
6478 ftypehex = format(listcatfiles['ffilelist'][reallcfi]['ftype'], 'x').lower();
6479 fcurfid = format(curfid, 'x').lower();
6480 if(not followlink and finode!=0):
6481 if(listcatfiles['ffilelist'][reallcfi]['ftype']!=1):
6482 fcurinode = format(int(curinode), 'x').lower();
6483 inodetofile.update({curinode: fname});
6484 filetoinode.update({fname: curinode});
6485 curinode = curinode + 1;
6486 else:
6487 fcurinode = format(int(filetoinode[flinkname]), 'x').lower();
6488 else:
6489 fcurinode = format(int(curinode), 'x').lower();
6490 curinode = curinode + 1;
6491 curfid = curfid + 1;
6492 if(fcompression=="none"):
6493 fcompression = "";
6494 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
6495 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
6496 fcontents.close();
6497 lcfi = lcfi + 1;
6498 reallcfi = reallcfi + 1;
6499 if(lcfx>0):
6500 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
6501 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
6502 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
6503 try:
6504 catfp.flush();
6505 os.fsync(catfp.fileno());
6506 except io.UnsupportedOperation:
6507 pass;
6508 except AttributeError:
6509 pass;
6510 except OSError as e:
6511 pass;
6512 if(outfile=="-"):
6513 catfp.seek(0, 0);
6514 if(hasattr(sys.stdout, "buffer")):
6515 shutil.copyfileobj(catfp, sys.stdout.buffer);
6516 else:
6517 shutil.copyfileobj(catfp, sys.stdout);
6518 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
6519 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
6520 catfp.seek(0, 0);
6521 upload_file_to_internet_file(catfp, outfile);
6522 if(returnfp):
6523 catfp.seek(0, 0);
6524 return catfp;
6525 else:
6526 catfp.close();
6527 return True;
6529 create_alias_function("RePack", __file_format_name__, "", RePackArchiveFile);
6531 def RePackArchiveFileFromString(catstr, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6532 catfp = BytesIO(catstr);
6533 listcatfiles = RePackArchiveFile(catfp, compression, compresswholefile, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6534 return listcatfiles;
6536 create_alias_function("RePack", __file_format_name__, "FromString", RePackArchiveFileFromString);
6538 def PackArchiveFileFromListDir(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6539 outarray = BytesIO();
6540 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
6541 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6542 return listcatfiles;
6544 create_alias_function("Pack", __file_format_name__, "FromListDir", PackArchiveFileFromListDir);
6546 def UnPackArchiveFile(infile, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, preservepermissions=True, preservetime=True, verbose=False, returnfp=False):
6547 if(outdir is not None):
6548 outdir = RemoveWindowsPath(outdir);
6549 if(verbose):
6550 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6551 if(isinstance(infile, dict)):
6552 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6553 listcatfiles = prelistcatfiles['list'];
6554 else:
6555 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6556 infile = RemoveWindowsPath(infile);
6557 if(followlink):
6558 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6559 listcatfiles = prelistcatfiles['list'];
6560 else:
6561 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6562 if(not listcatfiles):
6563 return False;
6564 lenlist = len(listcatfiles['ffilelist']);
6565 fnumfiles = int(listcatfiles['fnumfiles']);
6566 lcfi = 0;
6567 lcfx = int(listcatfiles['fnumfiles']);
6568 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6569 lcfx = int(lenlist);
6570 else:
6571 lcfx = int(listcatfiles['fnumfiles']);
6572 while(lcfi<lcfx):
6573 funame = "";
6574 try:
6575 import pwd;
6576 try:
6577 userinfo = pwd.getpwuid(listcatfiles['ffilelist'][lcfi]['fuid']);
6578 funame = userinfo.pw_name;
6579 except KeyError:
6580 funame = "";
6581 except ImportError:
6582 funame = "";
6583 fgname = "";
6584 try:
6585 import grp;
6586 try:
6587 groupinfo = grp.getgrgid(listcatfiles['ffilelist'][lcfi]['fgid']);
6588 fgname = groupinfo.gr_name;
6589 except KeyError:
6590 fgname = "";
6591 except ImportError:
6592 fgname = "";
6593 if(verbose):
6594 VerbosePrintOut(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6595 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6596 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6597 listcatfiles['ffilelist'][lcfi]['fcontents'].seek(0, 0);
6598 shutil.copyfileobj(listcatfiles['ffilelist'][lcfi]['fcontents'], fpc);
6599 try:
6600 fpc.flush();
6601 os.fsync(fpc.fileno());
6602 except io.UnsupportedOperation:
6603 pass;
6604 except AttributeError:
6605 pass;
6606 except OSError as e:
6607 pass;
6608 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
6609 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
6610 if(preservepermissions):
6611 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6612 if(preservetime):
6613 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
6614 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6615 if(followlink):
6616 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
6617 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6618 flinkinfo = listcatfiles['ffilelist'][flinkid];
6619 funame = "";
6620 try:
6621 import pwd;
6622 try:
6623 userinfo = pwd.getpwuid(flinkinfo['fuid']);
6624 funame = userinfo.pw_name;
6625 except KeyError:
6626 funame = "";
6627 except ImportError:
6628 funame = "";
6629 fgname = "";
6630 try:
6631 import grp;
6632 try:
6633 groupinfo = grp.getgrgid(flinkinfo['fgid']);
6634 fgname = groupinfo.gr_name;
6635 except KeyError:
6636 fgname = "";
6637 except ImportError:
6638 fgname = "";
6639 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
6640 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6641 flinkinfo['fcontents'].seek(0, 0);
6642 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
6643 try:
6644 fpc.flush();
6645 os.fsync(fpc.fileno());
6646 except io.UnsupportedOperation:
6647 pass;
6648 except AttributeError:
6649 pass;
6650 except OSError as e:
6651 pass;
6652 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6653 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6654 if(preservepermissions):
6655 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6656 if(preservetime):
6657 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6658 if(flinkinfo['ftype']==1):
6659 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6660 if(flinkinfo['ftype']==2):
6661 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6662 if(flinkinfo['ftype']==5):
6663 if(preservepermissions):
6664 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6665 else:
6666 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6667 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6668 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6669 if(preservepermissions):
6670 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6671 if(preservetime):
6672 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6673 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
6674 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6675 else:
6676 os.link(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6677 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6678 if(followlink):
6679 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
6680 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6681 flinkinfo = listcatfiles['ffilelist'][flinkid];
6682 funame = "";
6683 try:
6684 import pwd;
6685 try:
6686 userinfo = pwd.getpwuid(flinkinfo['fuid']);
6687 funame = userinfo.pw_name;
6688 except KeyError:
6689 funame = "";
6690 except ImportError:
6691 funame = "";
6692 fgname = "";
6693 try:
6694 import grp;
6695 try:
6696 groupinfo = grp.getgrgid(flinkinfo['fgid']);
6697 fgname = groupinfo.gr_name;
6698 except KeyError:
6699 fgname = "";
6700 except ImportError:
6701 fgname = "";
6702 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
6703 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6704 flinkinfo['fcontents'].seek(0, 0);
6705 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
6706 try:
6707 fpc.flush();
6708 os.fsync(fpc.fileno());
6709 except io.UnsupportedOperation:
6710 pass;
6711 except AttributeError:
6712 pass;
6713 except OSError as e:
6714 pass;
6715 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6716 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6717 if(preservepermissions):
6718 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6719 if(preservetime):
6720 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6721 if(flinkinfo['ftype']==1):
6722 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6723 if(flinkinfo['ftype']==2):
6724 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6725 if(flinkinfo['ftype']==5):
6726 if(preservepermissions):
6727 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6728 else:
6729 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6730 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6731 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6732 if(preservepermissions):
6733 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6734 if(preservetime):
6735 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6736 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
6737 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6738 else:
6739 os.symlink(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6740 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6741 if(preservepermissions):
6742 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6743 else:
6744 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6745 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
6746 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
6747 if(preservepermissions):
6748 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6749 if(preservetime):
6750 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
6751 if(listcatfiles['ffilelist'][lcfi]['ftype']==6 and hasattr(os, "mkfifo")):
6752 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6753 lcfi = lcfi + 1;
6754 if(returnfp):
6755 return listcatfiles['ffilelist']['catfp'];
6756 else:
6757 return True;
6759 create_alias_function("UnPack", __file_format_name__, "", UnPackArchiveFile);
6761 if(hasattr(shutil, "register_unpack_format")):
6762 def UnPackArchiveFileFunc(archive_name, extract_dir=None, **kwargs):
6763 return UnPackArchiveFile(archive_name, extract_dir, False, 0, 0, False, __file_format_dict__['format_delimiter'], False, False);
6764 create_alias_function("UnPack", __file_format_name__, "Func", UnPackArchiveFileFunc);
6766 def UnPackArchiveFileString(catstr, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6767 catfp = BytesIO(catstr);
6768 listcatfiles = UnPackArchiveFile(catfp, outdir, followlink, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6769 return listcatfiles;
6771 create_alias_function("UnPack", __file_format_name__, "String", UnPackArchiveFileString);
6773 def ArchiveFileListFiles(infile, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6774 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6775 if(isinstance(infile, dict)):
6776 listcatfiles = infile;
6777 else:
6778 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6779 infile = RemoveWindowsPath(infile);
6780 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, True, skipchecksum, formatspecs, returnfp);
6781 if(not listcatfiles):
6782 return False;
6783 lenlist = len(listcatfiles['ffilelist']);
6784 fnumfiles = int(listcatfiles['fnumfiles']);
6785 lcfi = 0;
6786 lcfx = int(listcatfiles['fnumfiles']);
6787 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6788 lcfx = int(lenlist);
6789 else:
6790 lcfx = int(listcatfiles['fnumfiles']);
6791 returnval = {};
6792 while(lcfi<lcfx):
6793 returnval.update({lcfi: listcatfiles['ffilelist'][lcfi]['fname']});
6794 if(not verbose):
6795 VerbosePrintOut(listcatfiles['ffilelist'][lcfi]['fname']);
6796 if(verbose):
6797 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6798 printfname = listcatfiles['ffilelist'][lcfi]['fname'];
6799 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6800 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " link to " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6801 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6802 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " -> " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6803 fuprint = listcatfiles['ffilelist'][lcfi]['funame'];
6804 if(len(fuprint)<=0):
6805 fuprint = listcatfiles['ffilelist'][lcfi]['fuid'];
6806 fgprint = listcatfiles['ffilelist'][lcfi]['fgname'];
6807 if(len(fgprint)<=0):
6808 fgprint = listcatfiles['ffilelist'][lcfi]['fgid'];
6809 VerbosePrintOut(PrintPermissionString(listcatfiles['ffilelist'][lcfi]['fmode'], listcatfiles['ffilelist'][lcfi]['ftype']) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(listcatfiles['ffilelist'][lcfi]['fsize']).rjust(15) + " " + datetime.datetime.utcfromtimestamp(listcatfiles['ffilelist'][lcfi]['fmtime']).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6810 lcfi = lcfi + 1;
6811 if(returnfp):
6812 return listcatfiles['catfp'];
6813 else:
6814 return True;
6816 create_alias_function("", __file_format_name__, "ListFiles", ArchiveFileListFiles);
6818 def ArchiveFileStringListFiles(catstr, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6819 catfp = BytesIO(catstr);
6820 listcatfiles = ArchiveFileListFiles(catstr, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6821 return listcatfiles;
6823 create_alias_function("", __file_format_name__, "StringListFiles", ArchiveFileStringListFiles);
6825 def TarFileListFiles(infile, verbose=False, returnfp=False):
6826 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6827 if(infile=="-"):
6828 infile = BytesIO();
6829 if(hasattr(sys.stdin, "buffer")):
6830 shutil.copyfileobj(sys.stdin.buffer, infile);
6831 else:
6832 shutil.copyfileobj(sys.stdin, infile);
6833 infile.seek(0, 0);
6834 if(not infile):
6835 return False;
6836 infile.seek(0, 0);
6837 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
6838 infile = download_file_from_internet_file(infile);
6839 infile.seek(0, 0);
6840 if(not infile):
6841 return False;
6842 infile.seek(0, 0);
6843 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
6844 return False;
6845 elif(os.path.exists(infile) and os.path.isfile(infile)):
6846 try:
6847 if(not tarfile.is_tarfile(infile)):
6848 return False;
6849 except AttributeError:
6850 if(not is_tarfile(infile)):
6851 return False;
6852 else:
6853 return False;
6854 try:
6855 if(hasattr(infile, "read") or hasattr(infile, "write")):
6856 tarfp = tarfile.open(fileobj=infile, mode="r");
6857 else:
6858 tarfp = tarfile.open(infile, "r");
6859 except FileNotFoundError:
6860 return False;
6861 lcfi = 0
6862 returnval = {};
6863 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
6864 returnval.update({lcfi: member.name});
6865 fpremode = member.mode;
6866 ffullmode = member.mode;
6867 flinkcount = 0;
6868 ftype = 0;
6869 if(member.isreg()):
6870 ffullmode = member.mode + stat.S_IFREG;
6871 ftype = 0;
6872 elif(member.isdev()):
6873 ffullmode = member.mode;
6874 ftype = 7;
6875 elif(member.islnk()):
6876 ffullmode = member.mode + stat.S_IFREG;
6877 ftype = 1;
6878 elif(member.issym()):
6879 ffullmode = member.mode + stat.S_IFLNK;
6880 ftype = 2;
6881 elif(member.ischr()):
6882 ffullmode = member.mode + stat.S_IFCHR;
6883 ftype = 3;
6884 elif(member.isblk()):
6885 ffullmode = member.mode + stat.S_IFBLK;
6886 ftype = 4;
6887 elif(member.isdir()):
6888 ffullmode = member.mode + stat.S_IFDIR;
6889 ftype = 5;
6890 elif(member.isfifo()):
6891 ffullmode = member.mode + stat.S_IFIFO;
6892 ftype = 6;
6893 elif(member.issparse()):
6894 ffullmode = member.mode;
6895 ftype = 12;
6896 if(not verbose):
6897 VerbosePrintOut(member.name);
6898 elif(verbose):
6899 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6900 printfname = member.name;
6901 if(member.islnk()):
6902 printfname = member.name + " link to " + member.linkname;
6903 elif(member.issym()):
6904 printfname = member.name + " -> " + member.linkname;
6905 fuprint = member.uname;
6906 if(len(fuprint)<=0):
6907 fuprint = member.uid;
6908 fgprint = member.gname;
6909 if(len(fgprint)<=0):
6910 fgprint = member.gid;
6911 VerbosePrintOut(PrintPermissionString(ffullmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(member.mtime).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6912 lcfi = lcfi + 1;
6913 if(returnfp):
6914 return listcatfiles['catfp'];
6915 else:
6916 return True;
6918 def ZipFileListFiles(infile, verbose=False, returnfp=False):
6919 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6920 if(infile=="-"):
6921 infile = BytesIO();
6922 if(hasattr(sys.stdin, "buffer")):
6923 shutil.copyfileobj(sys.stdin.buffer, infile);
6924 else:
6925 shutil.copyfileobj(sys.stdin, infile);
6926 infile.seek(0, 0);
6927 if(not infile):
6928 return False;
6929 infile.seek(0, 0);
6930 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
6931 infile = download_file_from_internet_file(infile);
6932 infile.seek(0, 0);
6933 if(not infile):
6934 return False;
6935 infile.seek(0, 0);
6936 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
6937 return False;
6938 else:
6939 return False;
6940 if(not zipfile.is_zipfile(infile)):
6941 return False;
6942 try:
6943 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
6944 except FileNotFoundError:
6945 return False;
6946 lcfi = 0;
6947 returnval = {};
6948 ziptest = zipfp.testzip();
6949 if(ziptest):
6950 VerbosePrintOut("Bad file found!");
6951 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
6952 if(zipinfo.create_system==0 or zipinfo.create_system==10):
6953 fwinattributes = int(zipinfo.external_attr);
6954 if(not member.is_dir()):
6955 fmode = int(stat.S_IFREG + 438);
6956 fchmode = int(stat.S_IMODE(fmode));
6957 ftypemod = int(stat.S_IFMT(fmode));
6958 elif(member.is_dir()):
6959 fmode = int(stat.S_IFDIR + 511);
6960 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6961 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6962 elif(zipinfo.create_system==3):
6963 fwinattributes =int(0);
6964 fmode = int(zipinfo.external_attr);
6965 fchmode = int(stat.S_IMODE(fmode));
6966 ftypemod = int(stat.S_IFMT(fmode));
6967 else:
6968 fwinattributes = int(0);
6969 if(not member.is_dir()):
6970 fmode = int(stat.S_IFREG + 438);
6971 fchmode = int(stat.S_IMODE(fmode));
6972 ftypemod = int(stat.S_IFMT(fmode));
6973 elif(member.is_dir()):
6974 fmode = int(stat.S_IFDIR + 511);
6975 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6976 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6977 returnval.update({lcfi: member.filename});
6978 if(not verbose):
6979 VerbosePrintOut(member.filename);
6980 if(verbose):
6981 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6982 permissionstr = "";
6983 for fmodval in str(oct(fmode))[-3:]:
6984 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
6985 if(not member.is_dir()):
6986 ftype = 0;
6987 permissionstr = "-" + permissionstr;
6988 elif(member.is_dir()):
6989 ftype = 5;
6990 permissionstr = "d" + permissionstr;
6991 printfname = member.filename;
6992 try:
6993 fuid = int(os.getuid());
6994 except AttributeError:
6995 fuid = int(0);
6996 except KeyError:
6997 fuid = int(0);
6998 try:
6999 fgid = int(os.getgid());
7000 except AttributeError:
7001 fgid = int(0);
7002 except KeyError:
7003 fgid = int(0);
7004 try:
7005 import pwd;
7006 try:
7007 userinfo = pwd.getpwuid(os.getuid());
7008 funame = userinfo.pw_name;
7009 except KeyError:
7010 funame = "";
7011 except AttributeError:
7012 funame = "";
7013 except ImportError:
7014 funame = "";
7015 fgname = "";
7016 try:
7017 import grp;
7018 try:
7019 groupinfo = grp.getgrgid(os.getgid());
7020 fgname = groupinfo.gr_name;
7021 except KeyError:
7022 fgname = "";
7023 except AttributeError:
7024 fgname = "";
7025 except ImportError:
7026 fgname = "";
7027 fuprint = funame;
7028 if(len(fuprint)<=0):
7029 fuprint = str(fuid);
7030 fgprint = fgname;
7031 if(len(fgprint)<=0):
7032 fgprint = str(fgid);
7033 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(int(time.mktime(member.date_time + (0, 0, -1)))).strftime('%Y-%m-%d %H:%M') + " " + printfname));
7034 lcfi = lcfi + 1;
7035 if(returnfp):
7036 return listcatfiles['catfp'];
7037 else:
7038 return True;
7040 if(not rarfile_support):
7041 def RarFileListFiles(infile, verbose=False, returnfp=False):
7042 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7043 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7044 return False;
7046 if(rarfile_support):
7047 def RarFileListFiles(infile, verbose=False, returnfp=False):
7048 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7049 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7050 return False;
7051 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
7052 return False;
7053 lcfi = 0;
7054 returnval = {};
7055 rarfp = rarfile.RarFile(infile, "r");
7056 rartest = rarfp.testrar();
7057 if(rartest):
7058 VerbosePrintOut("Bad file found!");
7059 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
7060 is_unix = False;
7061 is_windows = False;
7062 if(member.host_os==rarfile.RAR_OS_UNIX):
7063 is_windows = False;
7064 try:
7065 member.external_attr
7066 is_unix = True;
7067 except AttributeError:
7068 is_unix = False;
7069 elif(member.host_os==rarfile.RAR_OS_WIN32):
7070 is_unix = False;
7071 try:
7072 member.external_attr
7073 is_windows = True;
7074 except AttributeError:
7075 is_windows = False;
7076 else:
7077 is_unix = False;
7078 is_windows = False;
7079 if(is_unix and member.external_attr !=0):
7080 fpremode = int(member.external_attr);
7081 elif(member.is_file()):
7082 fpremode = int(stat.S_IFREG + 438);
7083 elif(member.is_symlink()):
7084 fpremode = int(stat.S_IFLNK + 438);
7085 elif(member.is_dir()):
7086 fpremode = int(stat.S_IFDIR + 511);
7087 if(is_windows and member.external_attr !=0):
7088 fwinattributes = int(member.external_attr);
7089 else:
7090 fwinattributes = int(0);
7091 if(is_unix and member.external_attr !=0):
7092 fmode = int(member.external_attr);
7093 fchmode = int(stat.S_IMODE(member.external_attr));
7094 ftypemod = int(stat.S_IFMT(member.external_attr));
7095 elif(member.is_file()):
7096 fmode = int(stat.S_IFREG + 438);
7097 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
7098 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
7099 elif(member.is_symlink()):
7100 fmode = int(stat.S_IFLNK + 438);
7101 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
7102 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
7103 elif(member.is_dir()):
7104 fmode = int(stat.S_IFDIR + 511);
7105 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7106 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7107 returnval.update({lcfi: member.filename});
7108 if(not verbose):
7109 VerbosePrintOut(member.filename);
7110 if(verbose):
7111 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7112 permissionstr = "";
7113 for fmodval in str(oct(fmode))[-3:]:
7114 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
7115 if(member.is_file()):
7116 ftype = 0;
7117 permissionstr = "-" + permissionstr;
7118 printfname = member.filename;
7119 elif(member.is_symlink()):
7120 ftype = 2;
7121 permissionstr = "l" + permissionstr;
7122 printfname = member.name + " -> " + member.read().decode("UTF-8");
7123 elif(member.is_dir()):
7124 ftype = 5;
7125 permissionstr = "d" + permissionstr;
7126 printfname = member.filename;
7127 try:
7128 fuid = int(os.getuid());
7129 except AttributeError:
7130 fuid = int(0);
7131 except KeyError:
7132 fuid = int(0);
7133 try:
7134 fgid = int(os.getgid());
7135 except AttributeError:
7136 fgid = int(0);
7137 except KeyError:
7138 fgid = int(0);
7139 try:
7140 import pwd;
7141 try:
7142 userinfo = pwd.getpwuid(os.getuid());
7143 funame = userinfo.pw_name;
7144 except KeyError:
7145 funame = "";
7146 except AttributeError:
7147 funame = "";
7148 except ImportError:
7149 funame = "";
7150 fgname = "";
7151 try:
7152 import grp;
7153 try:
7154 groupinfo = grp.getgrgid(os.getgid());
7155 fgname = groupinfo.gr_name;
7156 except KeyError:
7157 fgname = "";
7158 except AttributeError:
7159 fgname = "";
7160 except ImportError:
7161 fgname = "";
7162 fuprint = funame;
7163 if(len(fuprint)<=0):
7164 fuprint = str(fuid);
7165 fgprint = fgname;
7166 if(len(fgprint)<=0):
7167 fgprint = str(fgid);
7168 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + member.mtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
7169 lcfi = lcfi + 1;
7170 if(returnfp):
7171 return listcatfiles['catfp'];
7172 else:
7173 return True;
7175 if(not py7zr_support):
7176 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
7177 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7178 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7179 return False;
7181 if(py7zr_support):
7182 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
7183 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7184 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7185 return False;
7186 lcfi = 0;
7187 returnval = {};
7188 szpfp = py7zr.SevenZipFile(infile, mode="r");
7189 file_content = szpfp.readall();
7190 #sztest = szpfp.testzip();
7191 sztestalt = szpfp.test();
7192 if(sztestalt):
7193 VerbosePrintOut("Bad file found!");
7194 for member in sorted(szpfp.list(), key=lambda x: x.filename):
7195 if(re.findall("^[.|/]", member.filename)):
7196 fname = member.filename;
7197 else:
7198 fname = "./"+member.filename;
7199 if(not member.is_directory):
7200 fpremode = int(stat.S_IFREG + 438);
7201 elif(member.is_directory):
7202 fpremode = int(stat.S_IFDIR + 511);
7203 fwinattributes = int(0);
7204 if(member.is_directory):
7205 fmode = int(stat.S_IFDIR + 511);
7206 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7207 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7208 else:
7209 fmode = int(stat.S_IFLNK + 438);
7210 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
7211 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
7212 returnval.update({lcfi: member.filename});
7213 if(not verbose):
7214 VerbosePrintOut(member.filename);
7215 if(verbose):
7216 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7217 permissionstr = "";
7218 for fmodval in str(oct(fmode))[-3:]:
7219 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
7220 fsize = int("0");
7221 if(not member.is_directory):
7222 ftype = 0;
7223 permissionstr = "-" + permissionstr;
7224 printfname = member.filename;
7225 elif(member.is_directory):
7226 ftype = 5;
7227 permissionstr = "d" + permissionstr;
7228 printfname = member.filename;
7229 if(ftype==0):
7230 fsize = len(file_content[member.filename].read());
7231 file_content[member.filename].close();
7232 try:
7233 fuid = int(os.getuid());
7234 except AttributeError:
7235 fuid = int(0);
7236 except KeyError:
7237 fuid = int(0);
7238 try:
7239 fgid = int(os.getgid());
7240 except AttributeError:
7241 fgid = int(0);
7242 except KeyError:
7243 fgid = int(0);
7244 try:
7245 import pwd;
7246 try:
7247 userinfo = pwd.getpwuid(os.getuid());
7248 funame = userinfo.pw_name;
7249 except KeyError:
7250 funame = "";
7251 except AttributeError:
7252 funame = "";
7253 except ImportError:
7254 funame = "";
7255 fgname = "";
7256 try:
7257 import grp;
7258 try:
7259 groupinfo = grp.getgrgid(os.getgid());
7260 fgname = groupinfo.gr_name;
7261 except KeyError:
7262 fgname = "";
7263 except AttributeError:
7264 fgname = "";
7265 except ImportError:
7266 fgname = "";
7267 fuprint = funame;
7268 if(len(fuprint)<=0):
7269 fuprint = str(fuid);
7270 fgprint = fgname;
7271 if(len(fgprint)<=0):
7272 fgprint = str(fgid);
7273 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(fsize).rjust(15) + " " + member.creationtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
7274 lcfi = lcfi + 1;
7275 if(returnfp):
7276 return listcatfiles['catfp'];
7277 else:
7278 return True;
7280 def InFileListFiles(infile, verbose=False, formatspecs=__file_format_dict__, returnfp=False):
7281 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7282 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
7283 if(checkcompressfile=="tarfile"):
7284 return TarFileListFiles(infile, verbose, returnfp);
7285 elif(checkcompressfile=="zipfile"):
7286 return ZipFileListFiles(infile, verbose, returnfp);
7287 elif(checkcompressfile=="catfile"):
7288 return ArchiveFileListFiles(infile, 0, 0, False, formatspecs, verbose, returnfp);
7289 elif(rarfile_support and checkcompressfile=="rarfile"):
7290 return RarFileListFiles(infile, verbose, returnfp);
7291 elif(py7zr_support and checkcompressfile=="7zipfile"):
7292 return SevenZipFileListFiles(infile, verbose, returnfp);
7293 else:
7294 return False;
7295 return False;
7297 def ListDirListFiles(infiles, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7298 outarray = BytesIO();
7299 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, False, True);
7300 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
7301 return listcatfiles;
7303 def ListDirListFilesAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7304 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, listonly, checksumtype, formatspecs, verbose);
7305 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
7306 return listcatfiles;
7308 def PackArchiveFileFromListDirAlt(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7309 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, False, checksumtype, extradata, formatspecs, False);
7310 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, followlink, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
7311 return listcatfiles;
7313 create_alias_function("Pack", __file_format_name__, "FromListDirAlt", PackArchiveFileFromListDirAlt);
7315 def PackArchiveFileFromTarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7316 outarray = TarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7317 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7318 return listcatfiles;
7320 create_alias_function("Pack", __file_format_name__, "FromTarFileAlt", PackArchiveFileFromTarFileAlt);
7322 def PackArchiveFileFromZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7323 outarray = ZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7324 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7325 return listcatfiles;
7327 create_alias_function("Pack", __file_format_name__, "FromZipFileAlt", PackArchiveFileFromZipFileAlt);
7329 if(not rarfile_support):
7330 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7331 return False;
7333 if(rarfile_support):
7334 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7335 outarray = RarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7336 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7337 return listcatfiles;
7339 create_alias_function("Pack", __file_format_name__, "FromRarFileAlt", PackArchiveFileFromRarFileAlt);
7341 if(not py7zr_support):
7342 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7343 return False;
7345 if(py7zr_support):
7346 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7347 outarray = SevenZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7348 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7349 return listcatfiles;
7351 create_alias_function("Pack", __file_format_name__, "FromSevenZipFileAlt", PackArchiveFileFromSevenZipFileAlt);
7353 def download_file_from_ftp_file(url):
7354 urlparts = urlparse(url);
7355 file_name = os.path.basename(urlparts.path);
7356 file_dir = os.path.dirname(urlparts.path);
7357 if(urlparts.username is not None):
7358 ftp_username = urlparts.username;
7359 else:
7360 ftp_username = "anonymous";
7361 if(urlparts.password is not None):
7362 ftp_password = urlparts.password;
7363 elif(urlparts.password is None and urlparts.username=="anonymous"):
7364 ftp_password = "anonymous";
7365 else:
7366 ftp_password = "";
7367 if(urlparts.scheme=="ftp"):
7368 ftp = FTP();
7369 elif(urlparts.scheme=="ftps" and ftpssl):
7370 ftp = FTP_TLS();
7371 else:
7372 return False;
7373 if(urlparts.scheme=="sftp"):
7374 if(__use_pysftp__):
7375 return download_file_from_pysftp_file(url);
7376 else:
7377 return download_file_from_sftp_file(url);
7378 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7379 return download_file_from_http_file(url);
7380 ftp_port = urlparts.port;
7381 if(urlparts.port is None):
7382 ftp_port = 21;
7383 try:
7384 ftp.connect(urlparts.hostname, ftp_port);
7385 except socket.gaierror:
7386 log.info("Error With URL "+url);
7387 return False;
7388 except socket.timeout:
7389 log.info("Error With URL "+url);
7390 return False;
7391 ftp.login(urlparts.username, urlparts.password);
7392 if(urlparts.scheme=="ftps"):
7393 ftp.prot_p();
7394 ftpfile = BytesIO();
7395 ftp.retrbinary("RETR "+urlparts.path, ftpfile.write);
7396 #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
7397 ftp.close();
7398 ftpfile.seek(0, 0);
7399 return ftpfile;
7401 def download_file_from_ftp_string(url):
7402 ftpfile = download_file_from_ftp_file(url);
7403 return ftpfile.read();
7405 def upload_file_to_ftp_file(ftpfile, url):
7406 urlparts = urlparse(url);
7407 file_name = os.path.basename(urlparts.path);
7408 file_dir = os.path.dirname(urlparts.path);
7409 if(urlparts.username is not None):
7410 ftp_username = urlparts.username;
7411 else:
7412 ftp_username = "anonymous";
7413 if(urlparts.password is not None):
7414 ftp_password = urlparts.password;
7415 elif(urlparts.password is None and urlparts.username=="anonymous"):
7416 ftp_password = "anonymous";
7417 else:
7418 ftp_password = "";
7419 if(urlparts.scheme=="ftp"):
7420 ftp = FTP();
7421 elif(urlparts.scheme=="ftps" and ftpssl):
7422 ftp = FTP_TLS();
7423 else:
7424 return False;
7425 if(urlparts.scheme=="sftp"):
7426 if(__use_pysftp__):
7427 return upload_file_to_pysftp_file(url);
7428 else:
7429 return upload_file_to_sftp_file(url);
7430 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7431 return False;
7432 ftp_port = urlparts.port;
7433 if(urlparts.port is None):
7434 ftp_port = 21;
7435 try:
7436 ftp.connect(urlparts.hostname, ftp_port);
7437 except socket.gaierror:
7438 log.info("Error With URL "+url);
7439 return False;
7440 except socket.timeout:
7441 log.info("Error With URL "+url);
7442 return False;
7443 ftp.login(urlparts.username, urlparts.password);
7444 if(urlparts.scheme=="ftps"):
7445 ftp.prot_p();
7446 ftp.storbinary("STOR "+urlparts.path, ftpfile);
7447 ftp.close();
7448 ftpfile.seek(0, 0);
7449 return ftpfile;
7451 def upload_file_to_ftp_string(ftpstring, url):
7452 ftpfileo = BytesIO(ftpstring);
7453 ftpfile = upload_file_to_ftp_file(ftpfileo, url);
7454 ftpfileo.close();
7455 return ftpfile;
7457 def download_file_from_http_file(url, headers=geturls_headers_pycatfile_python_alt):
7458 # Parse the URL to extract username and password if present
7459 urlparts = urlparse(url);
7460 username = urlparts.username;
7461 password = urlparts.password;
7462 # Rebuild the URL without the username and password
7463 netloc = urlparts.hostname;
7464 if(urlparts.scheme=="sftp"):
7465 if(__use_pysftp__):
7466 return download_file_from_pysftp_file(url);
7467 else:
7468 return download_file_from_sftp_file(url);
7469 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7470 return download_file_from_ftp_file(url);
7471 if urlparts.port:
7472 netloc += ':' + str(urlparts.port);
7473 rebuilt_url = urlunparse((urlparts.scheme, netloc, urlparts.path, urlparts.params, urlparts.query, urlparts.fragment));
7474 # Create a temporary file object
7475 httpfile = BytesIO();
7476 if haverequests:
7477 # Use the requests library if available
7478 if username and password:
7479 response = requests.get(rebuilt_url, headers=headers, auth=(username, password), stream=True);
7480 else:
7481 response = requests.get(rebuilt_url, headers=headers, stream=True);
7482 response.raw.decode_content = True
7483 shutil.copyfileobj(response.raw, httpfile);
7484 else:
7485 # Build a Request object for urllib
7486 request = Request(rebuilt_url, headers=headers);
7487 # Create an opener object for handling URLs
7488 if username and password:
7489 # Create a password manager
7490 password_mgr = HTTPPasswordMgrWithDefaultRealm();
7491 # Add the username and password
7492 password_mgr.add_password(None, rebuilt_url, username, password);
7493 # Create an authentication handler using the password manager
7494 auth_handler = HTTPBasicAuthHandler(password_mgr);
7495 # Build the opener with the authentication handler
7496 opener = build_opener(auth_handler);
7497 else:
7498 opener = build_opener();
7499 with opener.open(request) as response:
7500 shutil.copyfileobj(response, httpfile);
7501 # Reset file pointer to the start
7502 httpfile.seek(0, 0);
7503 # Return the temporary file object
7504 return httpfile;
7506 def download_file_from_http_string(url, headers=geturls_headers_pycatfile_python_alt):
7507 httpfile = download_file_from_http_file(url, headers);
7508 return ftpfile.read();
7510 if(haveparamiko):
7511 def download_file_from_sftp_file(url):
7512 urlparts = urlparse(url);
7513 file_name = os.path.basename(urlparts.path);
7514 file_dir = os.path.dirname(urlparts.path);
7515 sftp_port = urlparts.port;
7516 if(urlparts.port is None):
7517 sftp_port = 22;
7518 else:
7519 sftp_port = urlparts.port;
7520 if(urlparts.username is not None):
7521 sftp_username = urlparts.username;
7522 else:
7523 sftp_username = "anonymous";
7524 if(urlparts.password is not None):
7525 sftp_password = urlparts.password;
7526 elif(urlparts.password is None and urlparts.username=="anonymous"):
7527 sftp_password = "anonymous";
7528 else:
7529 sftp_password = "";
7530 if(urlparts.scheme=="ftp"):
7531 return download_file_from_ftp_file(url);
7532 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7533 return download_file_from_http_file(url);
7534 if(urlparts.scheme!="sftp"):
7535 return False;
7536 ssh = paramiko.SSHClient();
7537 ssh.load_system_host_keys();
7538 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
7539 try:
7540 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7541 except paramiko.ssh_exception.SSHException:
7542 return False;
7543 except socket.gaierror:
7544 log.info("Error With URL "+url);
7545 return False;
7546 except socket.timeout:
7547 log.info("Error With URL "+url);
7548 return False;
7549 sftp = ssh.open_sftp();
7550 sftpfile = BytesIO();
7551 sftp.getfo(urlparts.path, sftpfile);
7552 sftp.close();
7553 ssh.close();
7554 sftpfile.seek(0, 0);
7555 return sftpfile;
7556 else:
7557 def download_file_from_sftp_file(url):
7558 return False;
7560 if(haveparamiko):
7561 def download_file_from_sftp_string(url):
7562 sftpfile = download_file_from_sftp_file(url);
7563 return sftpfile.read();
7564 else:
7565 def download_file_from_ftp_string(url):
7566 return False;
7568 if(haveparamiko):
7569 def upload_file_to_sftp_file(sftpfile, url):
7570 urlparts = urlparse(url);
7571 file_name = os.path.basename(urlparts.path);
7572 file_dir = os.path.dirname(urlparts.path);
7573 sftp_port = urlparts.port;
7574 if(urlparts.port is None):
7575 sftp_port = 22;
7576 else:
7577 sftp_port = urlparts.port;
7578 if(urlparts.username is not None):
7579 sftp_username = urlparts.username;
7580 else:
7581 sftp_username = "anonymous";
7582 if(urlparts.password is not None):
7583 sftp_password = urlparts.password;
7584 elif(urlparts.password is None and urlparts.username=="anonymous"):
7585 sftp_password = "anonymous";
7586 else:
7587 sftp_password = "";
7588 if(urlparts.scheme=="ftp"):
7589 return upload_file_to_ftp_file(url);
7590 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7591 return False;
7592 if(urlparts.scheme!="sftp"):
7593 return False;
7594 ssh = paramiko.SSHClient();
7595 ssh.load_system_host_keys();
7596 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
7597 try:
7598 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7599 except paramiko.ssh_exception.SSHException:
7600 return False;
7601 except socket.gaierror:
7602 log.info("Error With URL "+url);
7603 return False;
7604 except socket.timeout:
7605 log.info("Error With URL "+url);
7606 return False;
7607 sftp = ssh.open_sftp();
7608 sftp.putfo(sftpfile, urlparts.path);
7609 sftp.close();
7610 ssh.close();
7611 sftpfile.seek(0, 0);
7612 return sftpfile;
7613 else:
7614 def upload_file_to_sftp_file(sftpfile, url):
7615 return False;
7617 if(haveparamiko):
7618 def upload_file_to_sftp_string(sftpstring, url):
7619 sftpfileo = BytesIO(sftpstring);
7620 sftpfile = upload_file_to_sftp_files(ftpfileo, url);
7621 sftpfileo.close();
7622 return sftpfile;
7623 else:
7624 def upload_file_to_sftp_string(url):
7625 return False;
7627 if(havepysftp):
7628 def download_file_from_pysftp_file(url):
7629 urlparts = urlparse(url);
7630 file_name = os.path.basename(urlparts.path);
7631 file_dir = os.path.dirname(urlparts.path);
7632 sftp_port = urlparts.port;
7633 if(urlparts.port is None):
7634 sftp_port = 22;
7635 else:
7636 sftp_port = urlparts.port;
7637 if(urlparts.username is not None):
7638 sftp_username = urlparts.username;
7639 else:
7640 sftp_username = "anonymous";
7641 if(urlparts.password is not None):
7642 sftp_password = urlparts.password;
7643 elif(urlparts.password is None and urlparts.username=="anonymous"):
7644 sftp_password = "anonymous";
7645 else:
7646 sftp_password = "";
7647 if(urlparts.scheme=="ftp"):
7648 return download_file_from_ftp_file(url);
7649 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7650 return download_file_from_http_file(url);
7651 if(urlparts.scheme!="sftp"):
7652 return False;
7653 try:
7654 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7655 except paramiko.ssh_exception.SSHException:
7656 return False;
7657 except socket.gaierror:
7658 log.info("Error With URL "+url);
7659 return False;
7660 except socket.timeout:
7661 log.info("Error With URL "+url);
7662 return False;
7663 sftp = ssh.open_sftp();
7664 sftpfile = BytesIO();
7665 sftp.getfo(urlparts.path, sftpfile);
7666 sftp.close();
7667 ssh.close();
7668 sftpfile.seek(0, 0);
7669 return sftpfile;
7670 else:
7671 def download_file_from_pysftp_file(url):
7672 return False;
7674 if(havepysftp):
7675 def download_file_from_pysftp_string(url):
7676 sftpfile = download_file_from_pysftp_file(url);
7677 return sftpfile.read();
7678 else:
7679 def download_file_from_ftp_string(url):
7680 return False;
7682 if(havepysftp):
7683 def upload_file_to_pysftp_file(sftpfile, url):
7684 urlparts = urlparse(url);
7685 file_name = os.path.basename(urlparts.path);
7686 file_dir = os.path.dirname(urlparts.path);
7687 sftp_port = urlparts.port;
7688 if(urlparts.port is None):
7689 sftp_port = 22;
7690 else:
7691 sftp_port = urlparts.port;
7692 if(urlparts.username is not None):
7693 sftp_username = urlparts.username;
7694 else:
7695 sftp_username = "anonymous";
7696 if(urlparts.password is not None):
7697 sftp_password = urlparts.password;
7698 elif(urlparts.password is None and urlparts.username=="anonymous"):
7699 sftp_password = "anonymous";
7700 else:
7701 sftp_password = "";
7702 if(urlparts.scheme=="ftp"):
7703 return upload_file_to_ftp_file(url);
7704 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7705 return False;
7706 if(urlparts.scheme!="sftp"):
7707 return False;
7708 try:
7709 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7710 except paramiko.ssh_exception.SSHException:
7711 return False;
7712 except socket.gaierror:
7713 log.info("Error With URL "+url);
7714 return False;
7715 except socket.timeout:
7716 log.info("Error With URL "+url);
7717 return False;
7718 sftp = ssh.open_sftp();
7719 sftp.putfo(sftpfile, urlparts.path);
7720 sftp.close();
7721 ssh.close();
7722 sftpfile.seek(0, 0);
7723 return sftpfile;
7724 else:
7725 def upload_file_to_pysftp_file(sftpfile, url):
7726 return False;
7728 if(havepysftp):
7729 def upload_file_to_pysftp_string(sftpstring, url):
7730 sftpfileo = BytesIO(sftpstring);
7731 sftpfile = upload_file_to_pysftp_files(ftpfileo, url);
7732 sftpfileo.close();
7733 return sftpfile;
7734 else:
7735 def upload_file_to_pysftp_string(url):
7736 return False;
7738 def download_file_from_internet_file(url, headers=geturls_headers_pycatfile_python_alt):
7739 urlparts = urlparse(url);
7740 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7741 return download_file_from_http_file(url, headers);
7742 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7743 return download_file_from_ftp_file(url);
7744 elif(urlparts.scheme=="sftp"):
7745 if(__use_pysftp__ and havepysftp):
7746 return download_file_from_pysftp_file(url);
7747 else:
7748 return download_file_from_sftp_file(url);
7749 else:
7750 return False;
7751 return False;
7753 def download_file_from_internet_uncompress_file(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_dict__):
7754 fp = download_file_from_internet_file(url);
7755 fp = UncompressArchiveFile(fp, formatspecs);
7756 fp.seek(0, 0);
7757 if(not fp):
7758 return False;
7759 return fp;
7761 def download_file_from_internet_string(url, headers=geturls_headers_pycatfile_python_alt):
7762 urlparts = urlparse(url);
7763 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7764 return download_file_from_http_string(url, headers);
7765 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7766 return download_file_from_ftp_string(url);
7767 elif(urlparts.scheme=="sftp"):
7768 if(__use_pysftp__ and havepysftp):
7769 return download_file_from_pysftp_string(url);
7770 else:
7771 return download_file_from_sftp_string(url);
7772 else:
7773 return False;
7774 return False;
7776 def download_file_from_internet_uncompress_string(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_dict__):
7777 fp = download_file_from_internet_string(url);
7778 fp = UncompressArchiveFile(fp, formatspecs);
7779 fp.seek(0, 0);
7780 if(not fp):
7781 return False;
7782 return fp;
7784 def upload_file_to_internet_file(ifp, url):
7785 urlparts = urlparse(url);
7786 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7787 return False;
7788 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7789 return upload_file_to_ftp_file(ifp, url);
7790 elif(urlparts.scheme=="sftp"):
7791 if(__use_pysftp__ and havepysftp):
7792 return upload_file_to_pysftp_file(ifp, url);
7793 else:
7794 return upload_file_to_sftp_file(ifp, url);
7795 else:
7796 return False;
7797 return False;
7799 def upload_file_to_internet_compress_file(ifp, url, formatspecs=__file_format_dict__):
7800 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
7801 if(not catfileout):
7802 return False;
7803 catfp.seek(0, 0);
7804 upload_file_to_internet_file(catfp, outfile);
7805 return True;
7807 def upload_file_to_internet_string(ifp, url):
7808 urlparts = urlparse(url);
7809 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7810 return False;
7811 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7812 return upload_file_to_ftp_string(ifp, url);
7813 elif(urlparts.scheme=="sftp"):
7814 if(__use_pysftp__ and havepysftp):
7815 return upload_file_to_pysftp_string(ifp, url);
7816 else:
7817 return upload_file_to_sftp_string(ifp, url);
7818 else:
7819 return False;
7820 return False;
7822 def upload_file_to_internet_compress_string(ifp, url, formatspecs=__file_format_dict__):
7823 catfp = CompressArchiveFile(BytesIO(ifp), compression, compressionlevel, formatspecs);
7824 if(not catfileout):
7825 return False;
7826 catfp.seek(0, 0);
7827 upload_file_to_internet_file(catfp, outfile);
7828 return True;
7830 try:
7831 if(hasattr(shutil, "register_archive_format")):
7832 # Register the packing format
7833 shutil.register_archive_format(__file_format_name__, PackArchiveFileFunc, description='Pack concatenated files');
7834 except shutil.RegistryError:
7835 pass;
7837 try:
7838 if(hasattr(shutil, "register_unpack_format")):
7839 # Register the unpacking format
7840 shutil.register_unpack_format(__file_format_name__, archivefile_extensions, UnPackArchiveFileFunc, description='UnPack concatenated files');
7841 except shutil.RegistryError:
7842 pass;