?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Add files via upload
[PyCatFile.git] / pycatfile.py
blobb9b25f405d7672fb9cc62b2aa786e394924b445a
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 '''
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the Revised BSD License.
7
8 This program is distributed in the hope that it will be useful,
9 but WITHOUT ANY WARRANTY; without even the implied warranty of
10 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 Revised BSD License for more details.
12
13 Copyright 2018-2024 Cool Dude 2k - http://idb.berlios.de/
14 Copyright 2018-2024 Game Maker 2k - http://intdb.sourceforge.net/
15 Copyright 2018-2024 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
16
17 $FileInfo: pycatfile.py - Last Update: 5/8/2024 Ver. 0.10.4 RC 1 - Author: cooldude2k $
18 '''
19
20 from __future__ import absolute_import, division, print_function, unicode_literals;
21 import io, os, re, sys, time, stat, zlib, base64, shutil, socket, hashlib, datetime, logging, binascii, tempfile, zipfile, platform;
22 from ftplib import FTP, FTP_TLS;
23 if(sys.version[0]=="2"):
24 from urlparse import urlparse, urlunparse;
25 elif(sys.version[0]>="3"):
26 from urllib.parse import urlunparse;
27 from urllib.parse import urlparse;
28
29 if os.name == 'nt': # Only modify if on Windows
30 if sys.version[0] == "2":
31 import codecs;
32 sys.stdout = codecs.getwriter('utf-8')(sys.stdout);
33 sys.stderr = codecs.getwriter('utf-8')(sys.stderr);
34 else:
35 import io;
36 sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace', line_buffering=True);
37 sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', errors='replace', line_buffering=True);
38
39 hashlib_guaranteed = False;
40 os.environ["PYTHONIOENCODING"] = "UTF-8";
41 os.environ["LC_CTYPE"] = "UTF-8";
42 try:
43 reload(sys);
44 try:
45 sys.setdefaultencoding('UTF-8');
46 except NameError:
47 pass;
48 except AttributeError:
49 pass;
50 except NameError:
51 pass;
52 except AttributeError:
53 pass;
54
55 try:
56 from zlib import crc32;
57 except ImportError:
58 from binascii import crc32;
59
60 if(sys.version_info[0]==2):
61 FileNotFoundError = IOError;
62
63 rarfile_support = False;
64 try:
65 import rarfile;
66 rarfile_support = True;
67 except ImportError:
68 rarfile_support = False;
69
70 py7zr_support = False;
71 try:
72 import py7zr;
73 py7zr_support = True;
74 except ImportError:
75 py7zr_support = False;
76
77 try:
78 from xtarfile import is_tarfile;
79 except ImportError:
80 try:
81 from safetar import is_tarfile;
82 except ImportError:
83 from tarfile import is_tarfile;
84
85 try:
86 import xtarfile as tarfile;
87 except ImportError:
88 try:
89 import safetar as tarfile;
90 except ImportError:
91 import tarfile;
92
93 haveparamiko = False;
94 try:
95 import paramiko;
96 haveparamiko = True;
97 except ImportError:
98 haveparamiko = False;
99
100 havepysftp = False;
101 try:
102 import pysftp;
103 havepysftp = True;
104 except ImportError:
105 havepysftp = False;
106
107 haverequests = False;
108 try:
109 import requests;
110 haverequests = True;
111 except ImportError:
112 haverequests = False;
113
114 if(haverequests):
115 import urllib3;
116 logging.getLogger("urllib3").setLevel(logging.WARNING);
117
118 try:
119 # Python 3 imports
120 from urllib.request import Request, build_opener, HTTPBasicAuthHandler;
121 from urllib.parse import urlparse;
122 except ImportError:
123 # Python 2 imports
124 from urllib2 import Request, build_opener, HTTPBasicAuthHandler;
125 from urlparse import urlparse;
126
127 if(sys.version[0]=="2"):
128 try:
129 from io import StringIO, BytesIO;
130 except ImportError:
131 try:
132 from cStringIO import StringIO;
133 from cStringIO import StringIO as BytesIO;
134 except ImportError:
135 from StringIO import StringIO;
136 from StringIO import StringIO as BytesIO;
137 elif(sys.version[0]>="3"):
138 from io import StringIO, BytesIO;
139 else:
140 teststringio = 0;
141 if(teststringio<=0):
142 try:
143 from cStringIO import StringIO as BytesIO;
144 teststringio = 1;
145 except ImportError:
146 teststringio = 0;
147 if(teststringio<=0):
148 try:
149 from StringIO import StringIO as BytesIO;
150 teststringio = 2;
151 except ImportError:
152 teststringio = 0;
153 if(teststringio<=0):
154 try:
155 from io import BytesIO;
156 teststringio = 3;
157 except ImportError:
158 teststringio = 0;
159
160 __use_pysftp__ = False;
161 if(not havepysftp):
162 __use_pysftp__ = False;
163 __file_format_name__ = "CatFile";
164 __program_name__ = "Py"+__file_format_name__;
165 __file_format_lower__ = __file_format_name__.lower();
166 __file_format_magic__ = __file_format_name__;
167 __file_format_len__ = len(__file_format_magic__);
168 __file_format_hex__ = binascii.hexlify(__file_format_magic__.encode("UTF-8")).decode("UTF-8");
169 __file_format_delimiter__ = "\x00";
170 __file_format_ver__ = "001";
171 __use_new_style__ = True;
172 __use_advanced_list__ = True;
173 __use_alt_inode__ = False;
174 __file_format_list__ = [__file_format_name__, __file_format_magic__, __file_format_lower__, __file_format_len__, __file_format_hex__, __file_format_delimiter__, __file_format_ver__, __use_new_style__, __use_advanced_list__, __use_alt_inode__];
175 __project__ = __program_name__;
176 __project_url__ = "https://github.com/GameMaker2k/PyCatFile";
177 __version_info__ = (0, 10, 4, "RC 1", 1);
178 __version_date_info__ = (2024, 5, 8, "RC 1", 1);
179 __version_date__ = str(__version_date_info__[0]) + "." + str(__version_date_info__[1]).zfill(2) + "." + str(__version_date_info__[2]).zfill(2);
180 __revision__ = __version_info__[3];
181 __revision_id__ = "$Id$";
182 if(__version_info__[4] is not None):
183 __version_date_plusrc__ = __version_date__ + "-" + str(__version_date_info__[4]);
184 if(__version_info__[4] is None):
185 __version_date_plusrc__ = __version_date__;
186 if(__version_info__[3] is not None):
187 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]) + " " + str(__version_info__[3]);
188 if(__version_info__[3] is None):
189 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]);
190
191 PyBitness = platform.architecture();
192 if(PyBitness=="32bit" or PyBitness=="32"):
193 PyBitness = "32";
194 elif(PyBitness=="64bit" or PyBitness=="64"):
195 PyBitness = "64";
196 else:
197 PyBitness = "32";
198
199 geturls_ua_pycatfile_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
200 if(platform.python_implementation()!=""):
201 py_implementation = platform.python_implementation();
202 if(platform.python_implementation()==""):
203 py_implementation = "Python";
204 geturls_ua_pycatfile_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=py_implementation, pyver=platform.python_version(), proname=__project__, prover=__version__);
205 geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
206 geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
207 geturls_headers_pycatfile_python = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
208 geturls_headers_pycatfile_python_alt = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python_alt, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
209 geturls_headers_googlebot_google = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
210 geturls_headers_googlebot_google_old = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
211
212 compressionlist = ['auto', 'gzip', 'bzip2', 'zstd', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
213 compressionlistalt = ['gzip', 'bzip2', 'zstd', 'lz4', 'lzo', 'lzma', 'xz'];
214 outextlist = ['gz', 'bz2', 'zst', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
215 outextlistwd = ['.gz', '.bz2', '.zst', '.lz4', '.lzo', '.lzop', '.lzma', '.xz'];
216
217 tarfile_mimetype = "application/tar";
218 tarfile_tar_mimetype = tarfile_mimetype;
219 zipfile_mimetype = "application/zip";
220 zipfile_zip_mimetype = zipfile_mimetype;
221 rarfile_mimetype = "application/rar";
222 rarfile_rar_mimetype = rarfile_mimetype;
223 archivefile_mimetype = "application/x-"+__file_format_list__[1]+"";
224 archivefile_cat_mimetype = archivefile_mimetype;
225 archivefile_gzip_mimetype = "application/x-"+__file_format_list__[1]+"+gzip";
226 archivefile_gz_mimetype = archivefile_gzip_mimetype;
227 archivefile_bzip2_mimetype = "application/x-"+__file_format_list__[1]+"+bzip2";
228 archivefile_bz2_mimetype = archivefile_bzip2_mimetype;
229 archivefile_lz4_mimetype = "application/x-"+__file_format_list__[1]+"+lz4";
230 archivefile_lzop_mimetype = "application/x-"+__file_format_list__[1]+"+lzop";
231 archivefile_lzo_mimetype = archivefile_lzop_mimetype;
232 archivefile_zstandard_mimetype = "application/x-"+__file_format_list__[1]+"+zstandard";
233 archivefile_zstd_mimetype = archivefile_zstandard_mimetype;
234 archivefile_lzma_mimetype = "application/x-"+__file_format_list__[1]+"+lzma";
235 archivefile_xz_mimetype = "application/x-"+__file_format_list__[1]+"+xz";
236 archivefile_extensions = ['.cat', '.cat.gz', '.cat.bz2', '.cat.zst', '.cat.lz4', '.cat.lzo', '.cat.lzop', '.cat.lzma', '.cat.xz'];
237
238 if __name__ == "__main__":
239 import subprocess;
240 curscrpath = os.path.dirname(sys.argv[0]);
241 if(curscrpath==""):
242 curscrpath = ".";
243 if(os.sep=="\\"):
244 curscrpath = curscrpath.replace(os.sep, "/");
245 curscrpath = curscrpath + "/";
246 scrfile = curscrpath + "catfile.py";
247 if(os.path.exists(scrfile) and os.path.isfile(scrfile)):
248 scrcmd = subprocess.Popen([sys.executable, scrfile] + sys.argv[1:]);
249 scrcmd.wait();
250
251 def VerbosePrintOut(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
252 if(not dbgenable):
253 return True;
254 log_functions = {
255 "print": print,
256 "log": logging.info,
257 "warning": logging.warning,
258 "error": logging.error,
259 "critical": logging.critical,
260 "exception": logging.exception,
261 "logalt": lambda x: logging.log(dgblevel, x),
262 "debug": logging.debug
263 };
264 log_function = log_functions.get(outtype);
265 if(log_function):
266 log_function(dbgtxt);
267 return True;
268 return False;
269
270 def VerbosePrintOutReturn(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
271 VerbosePrintOut(dbgtxt, outtype, dbgenable, dgblevel);
272 return dbgtxt;
273
274 def RemoveWindowsPath(dpath):
275 if(dpath is None):
276 dpath = "";
277 if(os.sep!="/"):
278 dpath = dpath.replace(os.path.sep, "/");
279 dpath = dpath.rstrip("/");
280 if(dpath=="." or dpath==".."):
281 dpath = dpath + "/";
282 return dpath;
283
284 def NormalizeRelativePath(inpath):
285 inpath = RemoveWindowsPath(inpath);
286 if(os.path.isabs(inpath)):
287 outpath = inpath;
288 else:
289 if(inpath.startswith("./") or inpath.startswith("../")):
290 outpath = inpath;
291 else:
292 outpath = "./" + inpath;
293 return outpath;
294
295 def PrependPath(base_dir, child_path):
296 # Check if base_dir is None or empty, if so, return child_path as is
297 if not base_dir:
298 return child_path;
299 # Ensure base_dir ends with exactly one slash
300 if not base_dir.endswith('/'):
301 base_dir += '/';
302 # Check if child_path starts with ./ or ../ (indicating a relative path)
303 if child_path.startswith('./') or child_path.startswith('../'):
304 # For relative paths, we don't alter the child_path
305 return base_dir + child_path;
306 else:
307 # For non-relative paths, ensure there's no starting slash on child_path to avoid double slashes
308 return base_dir + child_path.lstrip('/');
309
310 def ListDir(dirpath, followlink=False, duplicates=False):
311 if(isinstance(dirpath, (list, tuple, ))):
312 dirpath = list(filter(None, dirpath));
313 elif(isinstance(dirpath, (str, ))):
314 dirpath = list(filter(None, [dirpath]));
315 retlist = [];
316 for mydirfile in dirpath:
317 if(not os.path.exists(mydirfile)):
318 return False;
319 mydirfile = NormalizeRelativePath(mydirfile);
320 if(os.path.exists(mydirfile) and os.path.islink(mydirfile)):
321 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile));
322 if(os.path.exists(mydirfile) and os.path.isdir(mydirfile)):
323 for root, dirs, filenames in os.walk(mydirfile):
324 dpath = root;
325 dpath = RemoveWindowsPath(dpath);
326 if(dpath not in retlist and not duplicates):
327 retlist.append(dpath);
328 if(duplicates):
329 retlist.append(dpath);
330 for file in filenames:
331 fpath = os.path.join(root, file);
332 fpath = RemoveWindowsPath(fpath);
333 if(fpath not in retlist and not duplicates):
334 retlist.append(fpath);
335 if(duplicates):
336 retlist.append(fpath);
337 else:
338 retlist.append(RemoveWindowsPath(mydirfile));
339 return retlist;
340
341 def ListDirAdvanced(dirpath, followlink=False, duplicates=False):
342 if isinstance(dirpath, (list, tuple)):
343 dirpath = list(filter(None, dirpath));
344 elif isinstance(dirpath, str):
345 dirpath = list(filter(None, [dirpath]));
346 retlist = []
347 for mydirfile in dirpath:
348 if not os.path.exists(mydirfile):
349 return False;
350 mydirfile = NormalizeRelativePath(mydirfile);
351 if os.path.exists(mydirfile) and os.path.islink(mydirfile) and followlink:
352 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile))
353 if os.path.exists(mydirfile) and os.path.isdir(mydirfile):
354 for root, dirs, filenames in os.walk(mydirfile):
355 # Sort dirs and filenames alphabetically in place
356 dirs.sort(key=lambda x: x.lower());
357 filenames.sort(key=lambda x: x.lower());
358 dpath = RemoveWindowsPath(root);
359 if not duplicates and dpath not in retlist:
360 retlist.append(dpath);
361 elif duplicates:
362 retlist.append(dpath);
363 for file in filenames:
364 fpath = os.path.join(root, file);
365 fpath = RemoveWindowsPath(fpath);
366 if not duplicates and fpath not in retlist:
367 retlist.append(fpath);
368 elif duplicates:
369 retlist.append(fpath);
370 else:
371 retlist.append(RemoveWindowsPath(mydirfile));
372 return retlist;
373
374 def create_alias_function(prefix, base_name, suffix, target_function):
375 # Define a new function that wraps the target function
376 def alias_function(*args, **kwargs):
377 return target_function(*args, **kwargs);
378
379 # Create the function name by combining the prefix, base name, and the suffix
380 function_name = "{}{}{}".format(prefix, base_name, suffix);
381
382 # Add the new function to the global namespace
383 globals()[function_name] = alias_function;
384
385 def create_alias_function_alt(prefix, base_name, suffix, target_function):
386 # Create the function name by combining the prefix, base name, and the suffix
387 # Use the format method for string formatting, compatible with Python 2 and 3
388 function_name = "{}{}{}".format(prefix, base_name, suffix)
389 # Add the new function (alias of the target_function) to the global namespace
390 # This line is compatible as-is with both Python 2 and 3
391 globals()[function_name] = target_function
392
393 # initial_value can be 0xFFFF or 0x0000
394 def crc16_ansi(msg, initial_value=0xFFFF):
395 # CRC-16-IBM / CRC-16-ANSI polynomial and initial value
396 poly = 0x8005; # Polynomial for CRC-16-IBM / CRC-16-ANSI
397 crc = initial_value; # Initial value
398 for b in msg:
399 crc ^= b << 8; # XOR byte into CRC top byte
400 for _ in range(8): # Process each bit
401 if crc & 0x8000: # If the top bit is set
402 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
403 else:
404 crc = crc << 1; # Just shift left
405 crc &= 0xFFFF; # Ensure CRC remains 16-bit
406 return crc;
407
408 # initial_value can be 0xFFFF or 0x0000
409 def crc16_ibm(msg, initial_value=0xFFFF):
410 return crc16_ansi(msg, initial_value);
411
412 # initial_value is 0xFFFF
413 def crc16(msg):
414 return crc16_ansi(msg, 0xFFFF);
415
416 # initial_value can be 0xFFFF, 0x1D0F or 0x0000
417 def crc16_ccitt(msg, initial_value=0xFFFF):
418 # CRC-16-CCITT polynomial
419 poly = 0x1021; # Polynomial for CRC-16-CCITT
420 # Use the specified initial value
421 crc = initial_value;
422 for b in msg:
423 crc ^= b << 8; # XOR byte into CRC top byte
424 for _ in range(8): # Process each bit
425 if crc & 0x8000: # If the top bit is set
426 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
427 else:
428 crc = crc << 1; # Just shift left
429 crc &= 0xFFFF; # Ensure CRC remains 16-bit
430 return crc;
431
432 # initial_value can be 0x42F0E1EBA9EA3693 or 0x0000000000000000
433 def crc64_ecma(msg, initial_value=0x0000000000000000):
434 # CRC-64-ECMA polynomial and initial value
435 poly = 0x42F0E1EBA9EA3693;
436 crc = initial_value; # Initial value for CRC-64-ECMA
437 for b in msg:
438 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
439 for _ in range(8): # Process each bit
440 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
441 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
442 else:
443 crc <<= 1; # Just shift left if the MSB is 0
444 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
445 return crc;
446
447 # initial_value can be 0x000000000000001B or 0xFFFFFFFFFFFFFFFF
448 def crc64_iso(msg, initial_value=0xFFFFFFFFFFFFFFFF):
449 # CRC-64-ISO polynomial and initial value
450 poly = 0x000000000000001B;
451 crc = initial_value; # Common initial value for CRC-64-ISO
452 for b in msg:
453 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
454 for _ in range(8): # Process each bit
455 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
456 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
457 else:
458 crc <<= 1; # Just shift left if the MSB is 0
459 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
460 return crc;
461
462 def GetDataFromArray(data, path, default=None):
463 element = data;
464 try:
465 for key in path:
466 element = element[key];
467 return element;
468 except (KeyError, TypeError, IndexError):
469 return default;
470
471 def GetDataFromArrayAlt(structure, path, default=None):
472 element = structure;
473 for key in path:
474 if isinstance(element, dict) and key in element:
475 element = element[key];
476 elif isinstance(element, list) and isinstance(key, int) and -len(element) <= key < len(element):
477 element = element[key];
478 else:
479 return default;
480 return element;
481
482 def GetHeaderChecksum(inlist=[], checksumtype="crc32", encodedata=True, formatspecs=__file_format_list__):
483 if isinstance(inlist, list):
484 fileheader = AppendNullBytes(inlist, formatspecs[5]);
485 else:
486 fileheader = AppendNullByte(inlist, formatspecs[5]);
487 if(encodedata):
488 fileheader = fileheader.encode('UTF-8');
489 if(checksumtype=="none" or checksumtype==""):
490 catfileheadercshex = format(0, 'x').lower();
491 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
492 catfileheadercshex = format(crc16(fileheader) & 0xffff, '04x').lower();
493 elif(checksumtype=="crc16_ccitt"):
494 catfileheadercshex = format(crc16_ccitt(fileheader) & 0xffff, '04x').lower();
495 elif(checksumtype=="adler32"):
496 catfileheadercshex = format(zlib.adler32(fileheader) & 0xffffffff, '08x').lower();
497 elif(checksumtype=="crc32"):
498 catfileheadercshex = format(crc32(fileheader) & 0xffffffff, '08x').lower();
499 elif(checksumtype=="crc64_ecma"):
500 catfileheadercshex = format(crc64_ecma(fileheader) & 0xffffffffffffffff, '016x').lower();
501 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
502 catfileheadercshex = format(crc64_iso(fileheader) & 0xffffffffffffffff, '016x').lower();
503 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
504 checksumoutstr = hashlib.new(checksumtype);
505 checksumoutstr.update(fileheader);
506 catfileheadercshex = checksumoutstr.hexdigest().lower();
507 else:
508 catfileheadercshex = format(0, 'x').lower();
509 return catfileheadercshex;
510
511 def GetFileChecksum(instr, checksumtype="crc32", encodedata=True, formatspecs=__file_format_list__):
512 if(encodedata):
513 instr = instr.encode('UTF-8');
514 if(checksumtype=="none" or checksumtype==""):
515 catinstrcshex = format(0, 'x').lower();
516 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
517 catinstrcshex = format(crc16(instr) & 0xffff, '04x').lower();
518 elif(checksumtype=="crc16_ccitt"):
519 catinstrcshex = format(crc16_ccitt(instr) & 0xffff, '04x').lower();
520 elif(checksumtype=="adler32"):
521 catinstrcshex = format(zlib.adler32(instr) & 0xffffffff, '08x').lower();
522 elif(checksumtype=="crc32"):
523 catinstrcshex = format(crc32(instr) & 0xffffffff, '08x').lower();
524 elif(checksumtype=="crc64_ecma"):
525 catinstrcshex = format(crc64_ecma(instr) & 0xffffffffffffffff, '016x').lower();
526 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
527 catinstrcshex = format(crc64_iso(instr) & 0xffffffffffffffff, '016x').lower();
528 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
529 checksumoutstr = hashlib.new(checksumtype);
530 checksumoutstr.update(instr);
531 catinstrcshex = checksumoutstr.hexdigest().lower();
532 else:
533 catinstrcshex = format(0, 'x').lower();
534 return catinstrcshex;
535
536 def ValidateHeaderChecksum(inlist=[], checksumtype="crc32", inchecksum="0", formatspecs=__file_format_list__):
537 catfileheadercshex = GetHeaderChecksum(inlist, checksumtype, True, formatspecs);
538 inchecksum = inchecksum.lower();
539 catfileheadercshex = catfileheadercshex.lower();
540 if(inchecksum==catfileheadercshex):
541 return True;
542 else:
543 return False;
544
545 def ValidateFileChecksum(infile, checksumtype="crc32", inchecksum="0", formatspecs=__file_format_list__):
546 catinfilecshex = GetFileChecksum(infile, checksumtype, True, formatspecs);
547 inchecksum = inchecksum.lower();
548 catinfilecshex = catinfilecshex.lower();
549 if(inchecksum==catinfilecshex):
550 return True;
551 else:
552 return False;
553
554 def ReadTillNullByte(fp, delimiter=__file_format_delimiter__):
555 curbyte = b"";
556 curfullbyte = b"";
557 nullbyte = delimiter.encode("UTF-8");
558 while(True):
559 curbyte = fp.read(1);
560 if(curbyte==nullbyte or not curbyte):
561 break;
562 curfullbyte = curfullbyte + curbyte;
563 return curfullbyte.decode('UTF-8');
564
565 def ReadUntilNullByte(fp, delimiter=__file_format_delimiter__):
566 return ReadTillNullByte(fp, delimiter);
567
568 def SeekToEndOfFile(fp):
569 lasttell = 0;
570 while(True):
571 fp.seek(1, 1);
572 if(lasttell==fp.tell()):
573 break;
574 lasttell = fp.tell();
575 return True;
576
577 def ReadFileHeaderData(fp, rounds=0, delimiter=__file_format_delimiter__):
578 rocount = 0;
579 roend = int(rounds);
580 HeaderOut = [];
581 while(rocount<roend):
582 HeaderOut.append(ReadTillNullByte(fp, delimiter));
583 rocount = rocount + 1;
584 return HeaderOut;
585
586 def ReadFileHeaderDataBySize(fp, delimiter=__file_format_delimiter__):
587 headerpresize = ReadTillNullByte(fp, delimiter);
588 headersize = int(headerpresize, 16);
589 if(headersize<=0):
590 return [];
591 headercontent = str(fp.read(headersize).decode('UTF-8')).split(delimiter);
592 fp.seek(1, 1);
593 rocount = 0;
594 roend = int(len(headercontent));
595 HeaderOut = [headerpresize];
596 while(rocount<roend):
597 HeaderOut.append(headercontent[rocount]);
598 rocount = rocount + 1;
599 return HeaderOut;
600
601 def ReadFileHeaderDataWoSize(fp, delimiter=__file_format_delimiter__):
602 preheaderdata = ReadFileHeaderData(fp, 2, delimiter);
603 headersize = int(preheaderdata[0], 16);
604 headernumfields = int(preheaderdata[1], 16);
605 if(headersize<=0 or headernumfields<=0):
606 return [];
607 headerdata = ReadFileHeaderData(fp, headernumfields, delimiter);
608 HeaderOut = preheaderdata + headerdata;
609 return HeaderOut;
610
611 def ReadFileHeaderDataBySizeWithContent(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
612 delimiter = formatspecs[5];
613 fheaderstart = fp.tell();
614 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
615 if(len(HeaderOut)==0):
616 return False;
617 if(re.findall("^[.|/]", HeaderOut[3])):
618 fname = HeaderOut[3];
619 else:
620 fname = "./"+HeaderOut[3];
621 fchecksumtype = HeaderOut[-3].lower();
622 fcs = HeaderOut[-2].lower();
623 fccs = HeaderOut[-1].lower();
624 fsize = int(HeaderOut[5], 16);
625 fcompression = HeaderOut[12];
626 fcsize = int(HeaderOut[13], 16);
627 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
628 if(fcs!=newfcs and not skipchecksum):
629 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
630 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
631 return False;
632 fhend = fp.tell() - 1;
633 fcontentstart = fp.tell();
634 fcontents = BytesIO();
635 if(fsize>0 and not listonly):
636 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
637 fcontents.write(fp.read(fsize));
638 else:
639 fcontents.write(fp.read(fcsize));
640 elif(fsize>0 and listonly):
641 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
642 fp.seek(fsize, 1);
643 else:
644 fp.seek(fcsize, 1);
645 fcontents.seek(0, 0);
646 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
647 if(fccs!=newfccs and not skipchecksum and not listonly):
648 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
649 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
650 return False;
651 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
652 pass;
653 else:
654 fcontents.seek(0, 0);
655 fcontents = UncompressArchiveFile(fcontents, formatspecs);
656 fp.seek(1, 1);
657 fcontentend = fp.tell() - 1;
658 HeaderOut.append(fcontents);
659 return HeaderOut;
660
661 def ReadFileHeaderDataBySizeWithContentToArray(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
662 delimiter = formatspecs[5];
663 fheaderstart = fp.tell();
664 if(formatspecs[7]):
665 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
666 else:
667 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
668 if(len(HeaderOut)==0):
669 return False;
670 fheadsize = int(HeaderOut[0], 16);
671 fnumfields = int(HeaderOut[1], 16);
672 ftype = int(HeaderOut[2], 16);
673 if(re.findall("^[.|/]", HeaderOut[3])):
674 fname = HeaderOut[3];
675 else:
676 fname = "./"+HeaderOut[3];
677 fbasedir = os.path.dirname(fname);
678 flinkname = HeaderOut[4];
679 fsize = int(HeaderOut[5], 16);
680 fatime = int(HeaderOut[6], 16);
681 fmtime = int(HeaderOut[7], 16);
682 fctime = int(HeaderOut[8], 16);
683 fbtime = int(HeaderOut[9], 16);
684 fmode = int(HeaderOut[10], 16);
685 fchmode = stat.S_IMODE(fmode);
686 ftypemod = stat.S_IFMT(fmode);
687 fwinattributes = int(HeaderOut[11], 16);
688 fcompression = HeaderOut[12];
689 fcsize = int(HeaderOut[13], 16);
690 fuid = int(HeaderOut[14], 16);
691 funame = HeaderOut[15];
692 fgid = int(HeaderOut[16], 16);
693 fgname = HeaderOut[17];
694 fid = int(HeaderOut[18], 16);
695 finode = int(HeaderOut[19], 16);
696 flinkcount = int(HeaderOut[20], 16);
697 fdev_minor = int(HeaderOut[21], 16);
698 fdev_major = int(HeaderOut[22], 16);
699 frdev_minor = int(HeaderOut[23], 16);
700 frdev_major = int(HeaderOut[24], 16);
701 fextrasize = int(HeaderOut[25], 16);
702 fextrafields = int(HeaderOut[26], 16);
703 extrafieldslist = [];
704 extrastart = 27;
705 extraend = extrastart + fextrafields;
706 extrafieldslist = [];
707 if(extrastart<extraend):
708 extrafieldslist.append(HeaderOut[extrastart]);
709 extrastart = extrastart + 1;
710 fchecksumtype = HeaderOut[extrastart].lower();
711 fcs = HeaderOut[extrastart + 1].lower();
712 fccs = HeaderOut[extrastart + 2].lower();
713 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
714 if(fcs!=newfcs and not skipchecksum):
715 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
716 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
717 return False;
718 fhend = fp.tell() - 1;
719 fcontentstart = fp.tell();
720 fcontents = BytesIO();
721 pyhascontents = False;
722 if(fsize>0 and not listonly):
723 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
724 fcontents.write(fp.read(fsize));
725 else:
726 fcontents.write(fp.read(fcsize));
727 pyhascontents = True;
728 elif(fsize>0 and listonly):
729 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
730 fp.seek(fsize, 1);
731 else:
732 fp.seek(fcsize, 1);
733 pyhascontents = False;
734 fcontents.seek(0, 0);
735 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
736 if(fccs!=newfccs and not skipchecksum and not listonly):
737 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
738 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
739 return False;
740 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
741 pass;
742 else:
743 fcontents.seek(0, 0);
744 fcontents = UncompressArchiveFile(fcontents, formatspecs);
745 fp.seek(1, 1);
746 fcontentend = fp.tell() - 1;
747 fcontents.seek(0, 0);
748 catlist = {'fheadersize': fheadsize, 'fhstart': fheaderstart, 'fhend': fhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': fchecksumtype, 'fnumfields': fnumfields + 2, 'frawheader': HeaderOut, 'fextrafields': fextrafields, 'fextrafieldsize': fextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': fcs, 'fcontentchecksum': fccs, 'fhascontents': pyhascontents, 'fcontentstart': fcontentstart, 'fcontentend': fcontentend, 'fcontents': fcontents};
749 return catlist;
750
751 def ReadFileHeaderDataBySizeWithContentToList(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
752 delimiter = formatspecs[5];
753 fheaderstart = fp.tell();
754 if(formatspecs[7]):
755 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
756 else:
757 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
758 if(len(HeaderOut)==0):
759 return False;
760 fheadsize = int(HeaderOut[0], 16);
761 fnumfields = int(HeaderOut[1], 16);
762 ftype = int(HeaderOut[2], 16);
763 if(re.findall("^[.|/]", HeaderOut[3])):
764 fname = HeaderOut[3];
765 else:
766 fname = "./"+HeaderOut[3];
767 fbasedir = os.path.dirname(fname);
768 flinkname = HeaderOut[4];
769 fsize = int(HeaderOut[5], 16);
770 fatime = int(HeaderOut[6], 16);
771 fmtime = int(HeaderOut[7], 16);
772 fctime = int(HeaderOut[8], 16);
773 fbtime = int(HeaderOut[9], 16);
774 fmode = int(HeaderOut[10], 16);
775 fchmode = stat.S_IMODE(fmode);
776 ftypemod = stat.S_IFMT(fmode);
777 fwinattributes = int(HeaderOut[11], 16);
778 fcompression = HeaderOut[12];
779 fcsize = int(HeaderOut[13], 16);
780 fuid = int(HeaderOut[14], 16);
781 funame = HeaderOut[15];
782 fgid = int(HeaderOut[16], 16);
783 fgname = HeaderOut[17];
784 fid = int(HeaderOut[18], 16);
785 finode = int(HeaderOut[19], 16);
786 flinkcount = int(HeaderOut[20], 16);
787 fdev_minor = int(HeaderOut[21], 16);
788 fdev_major = int(HeaderOut[22], 16);
789 frdev_minor = int(HeaderOut[23], 16);
790 frdev_major = int(HeaderOut[24], 16);
791 fextrasize = int(HeaderOut[25], 16);
792 fextrafields = int(HeaderOut[26], 16);
793 extrafieldslist = [];
794 extrastart = 27;
795 extraend = extrastart + fextrafields;
796 extrafieldslist = [];
797 if(extrastart<extraend):
798 extrafieldslist.append(HeaderOut[extrastart]);
799 extrastart = extrastart + 1;
800 fchecksumtype = HeaderOut[extrastart].lower();
801 fcs = HeaderOut[extrastart + 1].lower();
802 fccs = HeaderOut[extrastart + 2].lower();
803 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
804 if(fcs!=newfcs and not skipchecksum):
805 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
806 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
807 return False;
808 fhend = fp.tell() - 1;
809 fcontentstart = fp.tell();
810 fcontents = BytesIO();
811 pyhascontents = False;
812 if(fsize>0 and not listonly):
813 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
814 fcontents.write(fp.read(fsize));
815 else:
816 fcontents.write(fp.read(fcsize));
817 pyhascontents = True;
818 elif(fsize>0 and listonly):
819 if(fcompression=="none" or fcompression=="" or fcompression=="atuo"):
820 fp.seek(fsize, 1);
821 else:
822 fp.seek(fcsize, 1);
823 pyhascontents = False;
824 fcontents.seek(0, 0);
825 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
826 if(fccs!=newfccs and not skipchecksum and not listonly):
827 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
828 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
829 return False;
830 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
831 pass;
832 else:
833 fcontents.seek(0, 0);
834 fcontents = UncompressArchiveFile(fcontents, formatspecs);
835 fp.seek(1, 1);
836 fcontentend = fp.tell() - 1;
837 catlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, extrafieldslist, fchecksumtype, fcontents];
838 return catlist;
839
840 def ReadFileDataBySizeWithContent(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
841 delimiter = formatspecs[5];
842 curloc = fp.tell();
843 if(curloc>0):
844 fp.seek(0, 0);
845 catheader = ReadFileHeaderData(fp, 4, delimiter);
846 if(curloc>0):
847 fp.seek(curloc, 0);
848 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
849 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
850 if(not headercheck and not skipchecksum):
851 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
852 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
853 return False;
854 fnumfiles = int(catheader[1], 16);
855 countnum = 0;
856 flist = [];
857 while(countnum < fnumfiles):
858 HeaderOut = ReadFileHeaderDataBySizeWithContent(fp, listonly, skipchecksum, formatspecs)
859 if(len(HeaderOut)==0):
860 break;
861 flist.append(HeaderOut);
862 countnum = countnum + 1;
863 return flist;
864
865 def ReadFileDataBySizeWithContentToArray(fp, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
866 delimiter = formatspecs[5];
867 curloc = fp.tell();
868 if(curloc>0):
869 fp.seek(0, 0);
870 catheader = ReadFileHeaderData(fp, 4, delimiter);
871 if(curloc>0):
872 fp.seek(curloc, 0);
873 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
874 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
875 if(not headercheck and not skipchecksum):
876 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
877 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
878 return False;
879 catstring = catheader[0];
880 catversion = re.findall(r"([\d]+)$", catstring);
881 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
882 fprenumfiles = catheader[1];
883 fnumfiles = int(fprenumfiles, 16);
884 fprechecksumtype = catheader[2];
885 fprechecksum = catheader[3];
886 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
887 if(seekstart<0 and seekstart>fnumfiles):
888 seekstart = 0;
889 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
890 seekend = fnumfiles;
891 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
892 seekend = fnumfiles - abs(seekend);
893 if(seekstart>0):
894 il = 0;
895 while(il < seekstart):
896 prefhstart = fp.tell();
897 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs[5]);
898 if(len(preheaderdata)==0):
899 break;
900 prefsize = int(preheaderdata[5], 16);
901 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-3].lower(), True, formatspecs);
902 prefcs = preheaderdata[-2];
903 if(prefcs!=prenewfcs and not skipchecksum):
904 VVerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
905 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
906 return False;
907 valid_archive = False;
908 invalid_archive = True;
909 prefhend = fp.tell() - 1;
910 prefcontentstart = fp.tell();
911 prefcontents = BytesIO();
912 pyhascontents = False;
913 if(prefsize>0):
914 prefcontents.write(fp.read(prefsize));
915 prefcontents.seek(0, 0);
916 prenewfccs = GetFileChecksum(prefcontents.read(), preheaderdata[-3].lower(), False, formatspecs);
917 prefccs = preheaderdata[-1];
918 pyhascontents = True;
919 if(prefccs!=prenewfccs and not skipchecksum):
920 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
921 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
922 return False;
923 fp.seek(1, 1);
924 il = il + 1;
925 realidnum = 0;
926 countnum = seekstart;
927 while(countnum < seekend):
928 catlist['ffilelist'].update({realidnum: {'fid': realidnum, 'fidalt': realidnum}});
929 HeaderOut = ReadFileHeaderDataBySizeWithContentToArray(fp, listonly, skipchecksum, formatspecs);
930 if(len(HeaderOut)==0):
931 break;
932 catlist['ffilelist'][realidnum].update(HeaderOut);
933 countnum = countnum + 1;
934 realidnum = realidnum + 1;
935 return catlist;
936
937 def ReadFileDataBySizeWithContentToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
938 if(isinstance(infile, dict)):
939 listcatfiles = infile;
940 else:
941 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
942 return False;
943 listcatfiles = ReadFileDataBySizeWithContentToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs);
944 if(not listcatfiles):
945 return False;
946 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
947 lenlist = len(listcatfiles['ffilelist']);
948 lcfi = 0;
949 lcfx = int(listcatfiles['fnumfiles']);
950 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
951 lcfx = int(lenlist);
952 else:
953 lcfx = int(listcatfiles['fnumfiles']);
954 while(lcfi < lcfx):
955 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
956 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
957 catarray['filetoid'].update(filetoidarray);
958 catarray['idtofile'].update(idtofilearray);
959 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
960 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
961 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
962 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
963 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
964 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
965 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
966 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
967 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
968 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
969 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
970 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
971 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
972 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
973 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
974 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
975 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
976 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
977 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
978 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
979 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
980 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
981 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
982 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
983 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
984 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
985 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
986 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
987 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
988 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
989 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
990 lcfi = lcfi + 1;
991 return catarray;
992
993 def ReadFileDataBySizeWithContentToList(fp, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
994 delimiter = formatspecs[5];
995 curloc = fp.tell();
996 if(curloc>0):
997 fp.seek(0, 0);
998 catheader = ReadFileHeaderData(fp, 4, delimiter);
999 if(curloc>0):
1000 fp.seek(curloc, 0);
1001 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1002 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1003 if(not headercheck and not skipchecksum):
1004 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1005 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1006 return False;
1007 catstring = catheader[0];
1008 catversion = re.findall(r"([\d]+)$", catstring);
1009 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
1010 fprenumfiles = catheader[1];
1011 fnumfiles = int(fprenumfiles, 16);
1012 fprechecksumtype = catheader[2];
1013 fprechecksum = catheader[3];
1014 catlist = [];
1015 if(seekstart<0 and seekstart>fnumfiles):
1016 seekstart = 0;
1017 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
1018 seekend = fnumfiles;
1019 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
1020 seekend = fnumfiles - abs(seekend);
1021 if(seekstart>0):
1022 il = 0;
1023 while(il < seekstart):
1024 prefhstart = fp.tell();
1025 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs[5]);
1026 if(len(preheaderdata)==0):
1027 break;
1028 prefsize = int(preheaderdata[5], 16);
1029 prefcompression = preheaderdata[12];
1030 prefcsize = int(preheaderdata[13], 16);
1031 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-3].lower(), True, formatspecs);
1032 prefcs = preheaderdata[-2];
1033 if(prefcs!=prenewfcs and not skipchecksum):
1034 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
1035 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
1036 return False;
1037 valid_archive = False;
1038 invalid_archive = True;
1039 prefhend = fp.tell() - 1;
1040 prefcontentstart = fp.tell();
1041 prefcontents = "";
1042 pyhascontents = False;
1043 if(prefsize>0):
1044 if(prefcompression=="none" or prefcompression=="" or prefcompression=="auto"):
1045 prefcontents = catfp.read(prefsize);
1046 else:
1047 prefcontents = catfp.read(prefcsize);
1048 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
1049 prefccs = preheaderdata[-1];
1050 pyhascontents = True;
1051 if(prefccs!=prenewfccs and not skipchecksum):
1052 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
1053 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
1054 return False;
1055 fp.seek(1, 1);
1056 il = il + 1;
1057 realidnum = 0;
1058 countnum = seekstart;
1059 while(countnum < seekend):
1060 HeaderOut = ReadFileHeaderDataBySizeWithContentToList(fp, listonly, skipchecksum, formatspecs);
1061 if(len(HeaderOut)==0):
1062 break;
1063 catlist.append(HeaderOut);
1064 countnum = countnum + 1;
1065 realidnum = realidnum + 1;
1066 return catlist;
1067
1068 def ReadInFileBySizeWithContentToArray(infile, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
1069 delimiter = formatspecs[5];
1070 if(hasattr(infile, "read") or hasattr(infile, "write")):
1071 fp = infile;
1072 fp.seek(0, 0);
1073 fp = UncompressArchiveFile(fp, formatspecs);
1074 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1075 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
1076 return False;
1077 if(not fp):
1078 return False;
1079 fp.seek(0, 0);
1080 elif(infile=="-"):
1081 fp = BytesIO();
1082 if(hasattr(sys.stdin, "buffer")):
1083 shutil.copyfileobj(sys.stdin.buffer, fp);
1084 else:
1085 shutil.copyfileobj(sys.stdin, fp);
1086 fp.seek(0, 0);
1087 fp = UncompressArchiveFile(fp, formatspecs);
1088 if(not fp):
1089 return False;
1090 fp.seek(0, 0);
1091 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
1092 fp = download_file_from_internet_file(infile);
1093 fp = UncompressArchiveFile(fp, formatspecs);
1094 fp.seek(0, 0);
1095 if(not fp):
1096 return False;
1097 fp.seek(0, 0);
1098 else:
1099 infile = RemoveWindowsPath(infile);
1100 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
1101 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
1102 return False;
1103 compresscheck = CheckCompressionType(infile, formatspecs, True);
1104 if(not compresscheck):
1105 fextname = os.path.splitext(infile)[1];
1106 if(fextname==".gz"):
1107 compresscheck = "gzip";
1108 elif(fextname==".bz2"):
1109 compresscheck = "bzip2";
1110 elif(fextname==".zst"):
1111 compresscheck = "zstd";
1112 elif(fextname==".lz4" or fextname==".clz4"):
1113 compresscheck = "lz4";
1114 elif(fextname==".lzo" or fextname==".lzop"):
1115 compresscheck = "lzo";
1116 elif(fextname==".lzma" or fextname==".xz"):
1117 compresscheck = "lzma";
1118 else:
1119 return False;
1120 if(not compresscheck):
1121 return False;
1122 fp = UncompressFile(infile, formatspecs, "rb");
1123 return ReadFileDataBySizeWithContentToArray(fp, listonly, skipchecksum, formatspecs);
1124
1125 def ReadInFileBySizeWithContentToList(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
1126 delimiter = formatspecs[5];
1127 if(hasattr(infile, "read") or hasattr(infile, "write")):
1128 fp = infile;
1129 fp.seek(0, 0);
1130 fp = UncompressArchiveFile(fp, formatspecs);
1131 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1132 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
1133 return False;
1134 if(not fp):
1135 return False;
1136 fp.seek(0, 0);
1137 elif(infile=="-"):
1138 fp = BytesIO();
1139 if(hasattr(sys.stdin, "buffer")):
1140 shutil.copyfileobj(sys.stdin.buffer, fp);
1141 else:
1142 shutil.copyfileobj(sys.stdin, fp);
1143 fp.seek(0, 0);
1144 fp = UncompressArchiveFile(fp, formatspecs);
1145 if(not fp):
1146 return False;
1147 fp.seek(0, 0);
1148 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
1149 fp = download_file_from_internet_file(infile);
1150 fp = UncompressArchiveFile(fp, formatspecs);
1151 fp.seek(0, 0);
1152 if(not fp):
1153 return False;
1154 fp.seek(0, 0);
1155 else:
1156 infile = RemoveWindowsPath(infile);
1157 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
1158 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
1159 return False;
1160 compresscheck = CheckCompressionType(infile, formatspecs, True);
1161 if(not compresscheck):
1162 fextname = os.path.splitext(infile)[1];
1163 if(fextname==".gz"):
1164 compresscheck = "gzip";
1165 elif(fextname==".bz2"):
1166 compresscheck = "bzip2";
1167 elif(fextname==".zst"):
1168 compresscheck = "zstd";
1169 elif(fextname==".lz4" or fextname==".clz4"):
1170 compresscheck = "lz4";
1171 elif(fextname==".lzo" or fextname==".lzop"):
1172 compresscheck = "lzo";
1173 elif(fextname==".lzma" or fextname==".xz"):
1174 compresscheck = "lzma";
1175 else:
1176 return False;
1177 if(not compresscheck):
1178 return False;
1179 fp = UncompressFile(infile, formatspecs, "rb");
1180 return ReadFileDataBySizeWithContentToList(fp, seekstart, seekend, listonly, skipchecksum, formatspecs);
1181
1182 def ReadInFileBySizeWithContentToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
1183 if(isinstance(infile, dict)):
1184 listcatfiles = infile;
1185 else:
1186 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
1187 infile = RemoveWindowsPath(infile);
1188 listcatfiles = ReadInFileBySizeWithContentToArray(infile, listonly, skipchecksum, formatspecs);
1189 if(not listcatfiles):
1190 return False;
1191 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
1192 lenlist = len(listcatfiles['ffilelist']);
1193 lcfi = 0;
1194 lcfx = int(listcatfiles['fnumfiles']);
1195 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
1196 lcfx = int(lenlist);
1197 else:
1198 lcfx = int(listcatfiles['fnumfiles']);
1199 while(lcfi < lcfx):
1200 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
1201 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
1202 catarray['filetoid'].update(filetoidarray);
1203 catarray['idtofile'].update(idtofilearray);
1204 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
1205 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
1206 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
1207 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
1208 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
1209 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
1210 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
1211 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
1212 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
1213 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1214 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1215 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
1216 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
1217 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
1218 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
1219 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
1220 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1221 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1222 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
1223 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
1224 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
1225 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1226 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1227 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
1228 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
1229 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
1230 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
1231 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1232 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1233 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1234 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1235 lcfi = lcfi + 1;
1236 return catarray;
1237
1238 def AppendNullByte(indata, delimiter=__file_format_delimiter__):
1239 outdata = str(indata) + delimiter;
1240 return outdata;
1241
1242 def AppendNullBytes(indata=[], delimiter=__file_format_delimiter__):
1243 outdata = "";
1244 inum = 0;
1245 il = len(indata);
1246 while(inum < il):
1247 outdata = outdata + AppendNullByte(indata[inum], delimiter);
1248 inum = inum + 1;
1249 return outdata;
1250
1251 def AppendFileHeader(fp, numfiles, checksumtype="crc32", formatspecs=__file_format_list__):
1252 delimiter = formatspecs[5];
1253 catver = formatspecs[6];
1254 fileheaderver = str(int(catver.replace(".", "")));
1255 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
1256 fp.write(fileheader.encode('UTF-8'));
1257 fnumfiles = format(int(numfiles), 'x').lower();
1258 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs[5]);
1259 catfileheadercshex = GetFileChecksum(fileheader + fnumfilesa, checksumtype, True, formatspecs);
1260 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs[5]);
1261 fp.write(fnumfilesa.encode('UTF-8'));
1262 try:
1263 fp.flush();
1264 os.fsync(fp.fileno());
1265 except io.UnsupportedOperation:
1266 pass;
1267 except AttributeError:
1268 pass;
1269 except OSError as e:
1270 pass;
1271 return fp;
1272
1273 def MakeEmptyFilePointer(fp, checksumtype="crc32", formatspecs=__file_format_list__):
1274 AppendFileHeader(fp, 0, checksumtype, formatspecs);
1275 return fp;
1276
1277 def MakeEmptyFile(outfile, compression="auto", compressionlevel=None, checksumtype="crc32", formatspecs=__file_format_list__, returnfp=False):
1278 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1279 if(os.path.exists(outfile)):
1280 try:
1281 os.unlink(outfile);
1282 except OSError as e:
1283 pass;
1284 if(outfile=="-"):
1285 verbose = False;
1286 catfpfp = BytesIO();
1287 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1288 catfp = outfile;
1289 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1290 catfp = BytesIO();
1291 else:
1292 fbasename = os.path.splitext(outfile)[0];
1293 fextname = os.path.splitext(outfile)[1];
1294 catfp = CompressOpenFile(outfile, True, compressionlevel);
1295 catfp = AppendFileHeader(catfp, 0, checksumtype, formatspecs);
1296 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1297 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1298 try:
1299 catfp.flush();
1300 os.fsync(catfp.fileno());
1301 except io.UnsupportedOperation:
1302 pass;
1303 except AttributeError:
1304 pass;
1305 except OSError as e:
1306 pass;
1307 if(outfile=="-"):
1308 catfp.seek(0, 0);
1309 if(hasattr(sys.stdout, "buffer")):
1310 shutil.copyfileobj(catfp, sys.stdout.buffer);
1311 else:
1312 shutil.copyfileobj(catfp, sys.stdout);
1313 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1314 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1315 catfp.seek(0, 0);
1316 upload_file_to_internet_file(catfp, outfile);
1317 if(returnfp):
1318 catfp.seek(0, 0);
1319 return catfp;
1320 else:
1321 catfp.close();
1322 return True;
1323
1324 def AppendFileHeaderWithContent(fp, filevalues=[], extradata=[], filecontent="", checksumtype="crc32", formatspecs=__file_format_list__):
1325 extrafields = format(len(extradata), 'x').lower();
1326 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
1327 if(len(extradata)>0):
1328 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
1329 extrasizelen = format(len(extrasizestr), 'x').lower();
1330 catoutlen = len(filevalues) + len(extradata) + 5;
1331 catoutlenhex = format(catoutlen, 'x').lower();
1332 catoutlist = filevalues;
1333 catoutlist.insert(0, catoutlenhex);
1334 catoutlist.append(extrasizelen);
1335 catoutlist.append(extrafields);
1336 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
1337 if(len(extradata)>0):
1338 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
1339 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
1340 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
1341 catfilecontentcshex = GetFileChecksum(filecontent, checksumtype, False, formatspecs);
1342 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
1343 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
1344 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
1345 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
1346 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
1347 catfileoutstrecd = catfileoutstr.encode('UTF-8');
1348 nullstrecd = formatspecs[5].encode('UTF-8');
1349 catfileout = catfileoutstrecd + filecontent + nullstrecd;
1350 fp.write(catfileout);
1351 try:
1352 fp.flush();
1353 os.fsync(fp.fileno());
1354 except io.UnsupportedOperation:
1355 pass;
1356 except AttributeError:
1357 pass;
1358 except OSError as e:
1359 pass;
1360 return fp;
1361
1362 def AppendFilesWithContent(infiles, fp, dirlistfromtxt=False, filevalues=[], extradata=[], compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False):
1363 advancedlist = formatspecs[8];
1364 altinode = formatspecs[9];
1365 if(verbose):
1366 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
1367 if(infiles=="-"):
1368 for line in sys.stdin:
1369 infilelist.append(line.strip());
1370 infilelist = list(filter(None, infilelist));
1371 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
1372 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
1373 return False;
1374 with UncompressFile(infiles, formatspecs, "r") as finfile:
1375 for line in finfile:
1376 infilelist.append(line.strip());
1377 infilelist = list(filter(None, infilelist));
1378 else:
1379 if(isinstance(infiles, (list, tuple, ))):
1380 infilelist = list(filter(None, infiles));
1381 elif(isinstance(infiles, (str, ))):
1382 infilelist = list(filter(None, [infiles]));
1383 if(advancedlist):
1384 GetDirList = ListDirAdvanced(infilelist, followlink, False);
1385 else:
1386 GetDirList = ListDir(infilelist, followlink, False);
1387 if(not GetDirList):
1388 return False;
1389 curinode = 0;
1390 curfid = 0;
1391 inodelist = [];
1392 inodetofile = {};
1393 filetoinode = {};
1394 inodetocatinode = {};
1395 numfiles = int(len(GetDirList));
1396 fnumfiles = format(numfiles, 'x').lower();
1397 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
1398 for curfname in GetDirList:
1399 if(re.findall("^[.|/]", curfname)):
1400 fname = curfname;
1401 else:
1402 fname = "./"+curfname;
1403 if(verbose):
1404 VerbosePrintOut(fname);
1405 if(not followlink or followlink is None):
1406 fstatinfo = os.lstat(fname);
1407 else:
1408 fstatinfo = os.stat(fname);
1409 fpremode = fstatinfo.st_mode;
1410 finode = fstatinfo.st_ino;
1411 flinkcount = fstatinfo.st_nlink;
1412 ftype = 0;
1413 if(stat.S_ISREG(fpremode)):
1414 ftype = 0;
1415 elif(stat.S_ISLNK(fpremode)):
1416 ftype = 2;
1417 elif(stat.S_ISCHR(fpremode)):
1418 ftype = 3;
1419 elif(stat.S_ISBLK(fpremode)):
1420 ftype = 4;
1421 elif(stat.S_ISDIR(fpremode)):
1422 ftype = 5;
1423 elif(stat.S_ISFIFO(fpremode)):
1424 ftype = 6;
1425 elif(stat.S_ISSOCK(fpremode)):
1426 ftype = 8;
1427 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
1428 ftype = 9;
1429 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
1430 ftype = 10;
1431 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
1432 ftype = 111;
1433 else:
1434 ftype = 0;
1435 flinkname = "";
1436 fcurfid = format(int(curfid), 'x').lower();
1437 if(not followlink and finode!=0):
1438 if(ftype!=1):
1439 if(finode in inodelist):
1440 ftype = 1;
1441 flinkname = inodetofile[finode];
1442 if(altinode):
1443 fcurinode = format(int(finode), 'x').lower();
1444 else:
1445 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
1446 if(finode not in inodelist):
1447 inodelist.append(finode);
1448 inodetofile.update({finode: fname});
1449 inodetocatinode.update({finode: curinode});
1450 if(altinode):
1451 fcurinode = format(int(finode), 'x').lower();
1452 else:
1453 fcurinode = format(int(curinode), 'x').lower();
1454 curinode = curinode + 1;
1455 else:
1456 fcurinode = format(int(curinode), 'x').lower();
1457 curinode = curinode + 1;
1458 curfid = curfid + 1;
1459 if(ftype==2):
1460 flinkname = os.readlink(fname);
1461 fdev = fstatinfo.st_dev;
1462 getfdev = GetDevMajorMinor(fdev);
1463 fdev_minor = getfdev[0];
1464 fdev_major = getfdev[1];
1465 frdev = fstatinfo.st_dev;
1466 if(hasattr(fstatinfo, "st_rdev")):
1467 frdev = fstatinfo.st_rdev;
1468 else:
1469 frdev = fstatinfo.st_dev;
1470 getfrdev = GetDevMajorMinor(frdev);
1471 frdev_minor = getfrdev[0];
1472 frdev_major = getfrdev[1];
1473 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
1474 fsize = format(int("0"), 'x').lower();
1475 elif(ftype==0 or ftype==7):
1476 fsize = format(int(fstatinfo.st_size), 'x').lower();
1477 else:
1478 fsize = format(int(fstatinfo.st_size)).lower();
1479 fatime = format(int(fstatinfo.st_atime), 'x').lower();
1480 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
1481 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
1482 if(hasattr(fstatinfo, "st_birthtime")):
1483 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
1484 else:
1485 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
1486 fmode = format(int(fstatinfo.st_mode), 'x').lower();
1487 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
1488 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
1489 fuid = format(int(fstatinfo.st_uid), 'x').lower();
1490 fgid = format(int(fstatinfo.st_gid), 'x').lower();
1491 funame = "";
1492 try:
1493 import pwd;
1494 try:
1495 userinfo = pwd.getpwuid(fstatinfo.st_uid);
1496 funame = userinfo.pw_name;
1497 except KeyError:
1498 funame = "";
1499 except ImportError:
1500 funame = "";
1501 fgname = "";
1502 try:
1503 import grp;
1504 try:
1505 groupinfo = grp.getgrgid(fstatinfo.st_gid);
1506 fgname = groupinfo.gr_name;
1507 except KeyError:
1508 fgname = "";
1509 except ImportError:
1510 fgname = "";
1511 fdev_minor = format(int(fdev_minor), 'x').lower();
1512 fdev_major = format(int(fdev_major), 'x').lower();
1513 frdev_minor = format(int(frdev_minor), 'x').lower();
1514 frdev_major = format(int(frdev_major), 'x').lower();
1515 finode = format(int(finode), 'x').lower();
1516 flinkcount = format(int(flinkcount), 'x').lower();
1517 if(hasattr(fstatinfo, "st_file_attributes")):
1518 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
1519 else:
1520 fwinattributes = format(int(0), 'x').lower();
1521 fcompression = "";
1522 fcsize = format(int(0), 'x').lower();
1523 fcontents = BytesIO();
1524 chunk_size = 1024;
1525 if(ftype==0 or ftype==7):
1526 with open(fname, "rb") as fpc:
1527 shutil.copyfileobj(fpc, fcontents);
1528 if(not compresswholefile):
1529 fcontents.seek(0, 2);
1530 ucfsize = fcontents.tell();
1531 fcontents.seek(0, 0);
1532 if(compression=="auto"):
1533 ilsize = len(compressionlistalt);
1534 ilmin = 0;
1535 ilcsize = [];
1536 while(ilmin < ilsize):
1537 cfcontents = BytesIO();
1538 shutil.copyfileobj(fcontents, cfcontents);
1539 fcontents.seek(0, 0);
1540 cfcontents.seek(0, 0);
1541 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
1542 if(cfcontents):
1543 cfcontents.seek(0, 2);
1544 ilcsize.append(cfcontents.tell());
1545 cfcontents.close();
1546 else:
1547 try:
1548 ilcsize.append(sys.maxint);
1549 except AttributeError:
1550 ilcsize.append(sys.maxsize);
1551 ilmin = ilmin + 1;
1552 ilcmin = ilcsize.index(min(ilcsize));
1553 compression = compressionlistalt[ilcmin];
1554 fcontents.seek(0, 0);
1555 cfcontents = BytesIO();
1556 shutil.copyfileobj(fcontents, cfcontents);
1557 cfcontents.seek(0, 0);
1558 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
1559 cfcontents.seek(0, 2);
1560 cfsize = cfcontents.tell();
1561 if(ucfsize > cfsize):
1562 fcsize = format(int(cfsize), 'x').lower();
1563 fcompression = compression;
1564 fcontents.close();
1565 fcontents = cfcontents;
1566 if(followlink and (ftype==1 or ftype==2)):
1567 flstatinfo = os.stat(flinkname);
1568 with open(flinkname, "rb") as fpc:
1569 shutil.copyfileobj(fpc, fcontents);
1570 if(not compresswholefile):
1571 fcontents.seek(0, 2);
1572 ucfsize = fcontents.tell();
1573 fcontents.seek(0, 0);
1574 if(compression=="auto"):
1575 ilsize = len(compressionlistalt);
1576 ilmin = 0;
1577 ilcsize = [];
1578 while(ilmin < ilsize):
1579 cfcontents = BytesIO();
1580 shutil.copyfileobj(fcontents, cfcontents);
1581 fcontents.seek(0, 0);
1582 cfcontents.seek(0, 0);
1583 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
1584 if(cfcontents):
1585 cfcontents.seek(0, 2);
1586 ilcsize.append(cfcontents.tell());
1587 cfcontents.close();
1588 else:
1589 try:
1590 ilcsize.append(sys.maxint);
1591 except AttributeError:
1592 ilcsize.append(sys.maxsize);
1593 ilmin = ilmin + 1;
1594 ilcmin = ilcsize.index(min(ilcsize));
1595 compression = compressionlistalt[ilcmin];
1596 fcontents.seek(0, 0);
1597 cfcontents = BytesIO();
1598 shutil.copyfileobj(fcontents, cfcontents);
1599 cfcontents.seek(0, 0);
1600 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
1601 cfcontents.seek(0, 2);
1602 cfsize = cfcontents.tell();
1603 if(ucfsize > cfsize):
1604 fcsize = format(int(cfsize), 'x').lower();
1605 fcompression = compression;
1606 fcontents.close();
1607 fcontents = cfcontents;
1608 if(fcompression=="none"):
1609 fcompression = "";
1610 fcontents.seek(0, 0);
1611 ftypehex = format(ftype, 'x').lower();
1612 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
1613 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
1614 if(numfiles>0):
1615 catfp.write(AppendNullBytes([0, 0], formatspecs[5]).encode("UTF-8"));
1616 fp.seek(0, 0);
1617 return fp;
1618
1619 def AppendListsWithContent(inlist, fp, dirlistfromtxt=False, filevalues=[], extradata=[], compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False):
1620 if(verbose):
1621 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
1622 GetDirList = inlist;
1623 if(not GetDirList):
1624 return False;
1625 curinode = 0;
1626 curfid = 0;
1627 inodelist = [];
1628 inodetofile = {};
1629 filetoinode = {};
1630 inodetocatinode = {};
1631 numfiles = int(len(GetDirList));
1632 fnumfiles = format(numfiles, 'x').lower();
1633 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
1634 for curfname in GetDirList:
1635 ftype = format(curfname[0], 'x').lower();
1636 if(re.findall("^[.|/]", curfname[1])):
1637 fname = curfname[1];
1638 else:
1639 fname = "./"+curfname[1];
1640 fbasedir = os.path.dirname(fname);
1641 flinkname = curfname[2];
1642 fsize = format(curfname[3], 'x').lower();
1643 fatime = format(curfname[4], 'x').lower();
1644 fmtime = format(curfname[5], 'x').lower();
1645 fctime = format(curfname[6], 'x').lower();
1646 fbtime = format(curfname[7], 'x').lower();
1647 fmode = format(curfname[8], 'x').lower();
1648 fwinattributes = format(curfname[9], 'x').lower();
1649 fcompression = curfname[10];
1650 fcsize = format(curfname[11], 'x').lower();
1651 fuid = format(curfname[12], 'x').lower();
1652 funame = curfname[13];
1653 fgid = format(curfname[14], 'x').lower();
1654 fgname = curfname[15];
1655 fid = format(curfname[16], 'x').lower();
1656 finode = format(curfname[17], 'x').lower();
1657 flinkcount = format(curfname[18], 'x').lower();
1658 fdev_minor = format(curfname[19], 'x').lower();
1659 fdev_major = format(curfname[20], 'x').lower();
1660 frdev_minor = format(curfname[21], 'x').lower();
1661 frdev_major = format(curfname[22], 'x').lower();
1662 extradata = curfname[23];
1663 fchecksumtype = curfname[24];
1664 fcontents = curfname[25];
1665 catoutlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
1666 fcontents.seek(0, 0);
1667 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
1668 if(numfiles>0):
1669 fp.write(AppendNullBytes([0, 0], formatspecs[5]).encode("UTF-8"));
1670 return fp;
1671
1672 def AppendInFileWithContent(infile, fp, dirlistfromtxt=False, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False):
1673 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, False, formatspecs);
1674 return AppendListsWithContent(inlist, fp, dirlistfromtxt, filevalues, extradata, followlink, checksumtype, formatspecs, verbose);
1675
1676 def AppendFilesWithContentToOutFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False, returnfp=False):
1677 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1678 if(os.path.exists(outfile)):
1679 try:
1680 os.unlink(outfile);
1681 except OSError as e:
1682 pass;
1683 if(outfile=="-"):
1684 verbose = False;
1685 catfpfp = BytesIO();
1686 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1687 catfp = outfile;
1688 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1689 catfp = BytesIO();
1690 else:
1691 fbasename = os.path.splitext(outfile)[0];
1692 fextname = os.path.splitext(outfile)[1];
1693 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
1694 catfp = AppendFilesWithContent(infiles, catfp, dirlistfromtxt, filevalues, extradata, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose);
1695 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1696 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1697 try:
1698 catfp.flush();
1699 os.fsync(catfp.fileno());
1700 except io.UnsupportedOperation:
1701 pass;
1702 except AttributeError:
1703 pass;
1704 except OSError as e:
1705 pass;
1706 if(outfile=="-"):
1707 catfp.seek(0, 0);
1708 if(hasattr(sys.stdout, "buffer")):
1709 shutil.copyfileobj(catfp, sys.stdout.buffer);
1710 else:
1711 shutil.copyfileobj(catfp, sys.stdout);
1712 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1713 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1714 catfp.seek(0, 0);
1715 upload_file_to_internet_file(catfp, outfile);
1716 if(returnfp):
1717 catfp.seek(0, 0);
1718 return catfp;
1719 else:
1720 catfp.close();
1721 return True;
1722
1723 def AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False, returnfp=False):
1724 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1725 if(os.path.exists(outfile)):
1726 try:
1727 os.unlink(outfile);
1728 except OSError as e:
1729 pass;
1730 if(outfile=="-"):
1731 verbose = False;
1732 catfpfp = BytesIO();
1733 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1734 catfp = outfile;
1735 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1736 catfp = BytesIO();
1737 else:
1738 fbasename = os.path.splitext(outfile)[0];
1739 fextname = os.path.splitext(outfile)[1];
1740 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
1741 catfp = AppendListsWithContent(inlist, catfp, dirlistfromtxt, filevalues, extradata, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose);
1742 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1743 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1744 try:
1745 catfp.flush();
1746 os.fsync(catfp.fileno());
1747 except io.UnsupportedOperation:
1748 pass;
1749 except AttributeError:
1750 pass;
1751 except OSError as e:
1752 pass;
1753 if(outfile=="-"):
1754 catfp.seek(0, 0);
1755 if(hasattr(sys.stdout, "buffer")):
1756 shutil.copyfileobj(catfp, sys.stdout.buffer);
1757 else:
1758 shutil.copyfileobj(catfp, sys.stdout);
1759 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1760 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1761 catfp.seek(0, 0);
1762 upload_file_to_internet_file(catfp, outfile);
1763 if(returnfp):
1764 catfp.seek(0, 0);
1765 return catfp;
1766 else:
1767 catfp.close();
1768 return True;
1769
1770 def AppendInFileWithContentToOutFile(infile, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False, returnfp=False):
1771 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, False, formatspecs);
1772 return AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt, compression, compresswholefile, compressionlevel, filevalues, extradata, followlink, checksumtype, formatspecs, verbose, returnfp);
1773
1774 def PrintPermissionString(fchmode, ftype):
1775 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
1776 permissionstr = "";
1777 for fmodval in str(oct(fchmode))[-3:]:
1778 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
1779 if(ftype==0 or ftype==7):
1780 permissionstr = "-" + permissionstr;
1781 if(ftype==1):
1782 permissionstr = "h" + permissionstr;
1783 if(ftype==2):
1784 permissionstr = "l" + permissionstr;
1785 if(ftype==3):
1786 permissionstr = "c" + permissionstr;
1787 if(ftype==4):
1788 permissionstr = "b" + permissionstr;
1789 if(ftype==5):
1790 permissionstr = "d" + permissionstr;
1791 if(ftype==6):
1792 permissionstr = "f" + permissionstr;
1793 if(ftype==8):
1794 permissionstr = "D" + permissionstr;
1795 if(ftype==9):
1796 permissionstr = "p" + permissionstr;
1797 if(ftype==10):
1798 permissionstr = "w" + permissionstr;
1799 try:
1800 permissionoutstr = stat.filemode(fchmode);
1801 except AttributeError:
1802 permissionoutstr = permissionstr;
1803 except KeyError:
1804 permissionoutstr = permissionstr;
1805 return permissionoutstr;
1806
1807 def PrintPermissionStringAlt(fchmode, ftype):
1808 permissions = {
1809 '0': '---', '1': '--x', '2': '-w-', '3': '-wx',
1810 '4': 'r--', '5': 'r-x', '6': 'rw-', '7': 'rwx'
1811 };
1812 # Translate file mode into permission string
1813 permissionstr = ''.join([permissions[i] for i in str(oct(fchmode))[-3:]]);
1814 # Append file type indicator
1815 type_indicators = {
1816 0: '-', 1: 'h', 2: 'l', 3: 'c', 4: 'b',
1817 5: 'd', 6: 'f', 8: 'D', 9: 'p', 10: 'w'
1818 };
1819 file_type = type_indicators.get(ftype, '-');
1820 permissionstr = file_type + permissionstr;
1821 try:
1822 permissionoutstr = stat.filemode(fchmode);
1823 except AttributeError:
1824 permissionoutstr = permissionstr;
1825 return permissionoutstr;
1826
1827 def CompressionSupport():
1828 compression_list = [];
1829 try:
1830 import gzip;
1831 compression_list.append("gz");
1832 compression_list.append("gzip");
1833 except ImportError:
1834 '''return False;'''
1835 try:
1836 import bz2;
1837 compression_list.append("bz2");
1838 compression_list.append("bzip2");
1839 except ImportError:
1840 '''return False;'''
1841 try:
1842 import lz4;
1843 compression_list.append("lz4");
1844 except ImportError:
1845 '''return False;'''
1846 try:
1847 import lzo;
1848 compression_list.append("lzo");
1849 compression_list.append("lzop");
1850 except ImportError:
1851 '''return False;'''
1852 try:
1853 import zstandard;
1854 compression_list.append("zstd");
1855 compression_list.append("zstandard");
1856 except ImportError:
1857 '''return False;'''
1858 try:
1859 import lzma;
1860 compression_list.append("lzma");
1861 compression_list.append("xz");
1862 except ImportError:
1863 '''return False;'''
1864 return compression_list;
1865
1866 def CheckCompressionType(infile, formatspecs=__file_format_list__, closefp=True):
1867 if(hasattr(infile, "read") or hasattr(infile, "write")):
1868 catfp = infile;
1869 else:
1870 try:
1871 catfp = open(infile, "rb");
1872 except FileNotFoundError:
1873 return False;
1874 catfp.seek(0, 0);
1875 prefp = catfp.read(2);
1876 filetype = False;
1877 if(prefp==binascii.unhexlify("1f8b")):
1878 filetype = "gzip";
1879 catfp.seek(0, 0);
1880 prefp = catfp.read(3);
1881 if(prefp==binascii.unhexlify("425a68")):
1882 filetype = "bzip2";
1883 if(prefp==binascii.unhexlify("5d0000")):
1884 filetype = "lzma";
1885 catfp.seek(0, 0);
1886 prefp = catfp.read(4);
1887 if(prefp==binascii.unhexlify("28b52ffd")):
1888 filetype = "zstd";
1889 if(prefp==binascii.unhexlify("04224d18")):
1890 filetype = "lz4";
1891 if(prefp==binascii.unhexlify("504B0304")):
1892 filetype = "zipfile";
1893 catfp.seek(0, 0);
1894 prefp = catfp.read(5);
1895 if(prefp==binascii.unhexlify("7573746172")):
1896 filetype = "tarfile";
1897 catfp.seek(0, 0);
1898 prefp = catfp.read(6);
1899 if(prefp==binascii.unhexlify("fd377a585a00")):
1900 filetype = "lzma";
1901 if(prefp==binascii.unhexlify("377abcaf271c")):
1902 filetype = "7zipfile";
1903 catfp.seek(0, 0);
1904 prefp = catfp.read(7);
1905 if(prefp==binascii.unhexlify("526172211a0700")):
1906 filetype = "rarfile";
1907 if(prefp==binascii.unhexlify("43617446696c65")):
1908 filetype = "catfile";
1909 catfp.seek(0, 0);
1910 prefp = catfp.read(8);
1911 if(prefp==binascii.unhexlify("526172211a070100")):
1912 filetype = "rarfile";
1913 catfp.seek(0, 0);
1914 prefp = catfp.read(formatspecs[3]);
1915 if(prefp==binascii.unhexlify(formatspecs[4])):
1916 filetype = formatspecs[2];
1917 catfp.seek(0, 0);
1918 prefp = catfp.read(9);
1919 if(prefp==binascii.unhexlify("894c5a4f000d0a1a0a")):
1920 filetype = "lzo";
1921 catfp.seek(0, 0);
1922 prefp = catfp.read(10);
1923 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
1924 filetype = "tarfile";
1925 catfp.seek(0, 0);
1926 if(filetype=="gzip" or filetype=="bzip2" or filetype=="lzma" or filetype=="zstd" or filetype=="lz4"):
1927 if(is_tarfile(catfp)):
1928 filetype = "tarfile";
1929 if(not filetype):
1930 if(is_tarfile(catfp)):
1931 filetype = "tarfile";
1932 elif(zipfile.is_zipfile(catfp)):
1933 filetype = "zipfile";
1934 elif(rarfile.is_rarfile(catfp) or rarfile.is_rarfile_sfx(catfp)):
1935 filetype = "rarile";
1936 else:
1937 filetype = False;
1938 catfp.seek(0, 0);
1939 if(closefp):
1940 catfp.close();
1941 return filetype;
1942
1943 def CheckCompressionTypeFromString(instring, formatspecs=__file_format_list__, closefp=True):
1944 try:
1945 instringsfile = BytesIO(instring);
1946 except TypeError:
1947 instringsfile = BytesIO(instring.encode("UTF-8"));
1948 return CheckCompressionType(instringsfile, formatspecs, closefp);
1949
1950 def GetCompressionMimeType(infile, formatspecs=__file_format_list__):
1951 compresscheck = CheckCompressionType(fp, formatspecs, False);
1952 if(compresscheck=="gzip" or compresscheck=="gz"):
1953 return archivefile_gzip_mimetype;
1954 if(compresscheck=="bzip2" or compresscheck=="bz2"):
1955 return archivefile_bzip2_mimetype;
1956 if(compresscheck=="zstd" or compresscheck=="zstandard"):
1957 return archivefile_zstandard_mimetype;
1958 if(compresscheck=="lz4"):
1959 return archivefile_lz4_mimetype;
1960 if(compresscheck=="lzo" or compresscheck=="lzop"):
1961 return archivefile_lzop_mimetype;
1962 if(compresscheck=="lzma"):
1963 return archivefile_lzma_mimetype;
1964 if(compresscheck=="xz"):
1965 return archivefile_xz_mimetype;
1966 if(compresscheck=="catfile" or compresscheck=="cat" or compresscheck==formatspecs[2]):
1967 return archivefile_cat_mimetype;
1968 if(not compresscheck):
1969 return False;
1970 return False;
1971
1972 def UncompressArchiveFile(fp, formatspecs=__file_format_list__):
1973 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
1974 return False;
1975 compresscheck = CheckCompressionType(fp, formatspecs, False);
1976 if(compresscheck=="gzip"):
1977 try:
1978 import gzip;
1979 except ImportError:
1980 return False;
1981 catfp = gzip.GzipFile(fileobj=fp, mode="rb");
1982 if(compresscheck=="bzip2"):
1983 try:
1984 import bz2;
1985 except ImportError:
1986 return False;
1987 catfp = BytesIO();
1988 catfp.write(bz2.decompress(fp.read()));
1989 if(compresscheck=="zstd"):
1990 try:
1991 import zstandard;
1992 except ImportError:
1993 return False;
1994 catfp = BytesIO();
1995 catfp.write(zstandard.decompress(fp.read()));
1996 if(compresscheck=="lz4"):
1997 try:
1998 import lz4.frame;
1999 except ImportError:
2000 return False;
2001 catfp = BytesIO();
2002 catfp.write(lz4.frame.decompress(fp.read()));
2003 if(compresscheck=="lzo" or compresscheck=="lzop"):
2004 try:
2005 import lzo;
2006 except ImportError:
2007 return False;
2008 catfp = BytesIO();
2009 catfp.write(lzo.decompress(fp.read()));
2010 if(compresscheck=="lzma" or compresscheck=="xz"):
2011 try:
2012 import lzma;
2013 except ImportError:
2014 return False;
2015 catfp = BytesIO();
2016 catfp.write(lzma.decompress(fp.read()));
2017 if(compresscheck=="catfile" or compresscheck==formatspecs[2]):
2018 catfp = fp;
2019 if(not compresscheck):
2020 try:
2021 import lzma;
2022 except ImportError:
2023 return False;
2024 catfp = BytesIO();
2025 with fp as fpcontent:
2026 try:
2027 catfp.write(lzma.decompress(fp.read()));
2028 except lzma.LZMAError:
2029 return False;
2030 if(compresscheck!="catfile" or compresscheck!=formatspecs[2]):
2031 fp.close();
2032 return catfp;
2033
2034 create_alias_function("Uncompress", __file_format_name__, "", UncompressArchiveFile);
2035
2036 def UncompressFile(infile, formatspecs=__file_format_list__, mode="rb"):
2037 compresscheck = CheckCompressionType(infile, formatspecs, False);
2038 if(sys.version_info[0]==2 and compresscheck):
2039 if(mode=="rt"):
2040 mode = "r";
2041 if(mode=="wt"):
2042 mode = "w";
2043 try:
2044 if(compresscheck=="gzip"):
2045 try:
2046 import gzip;
2047 except ImportError:
2048 return False;
2049 try:
2050 filefp = gzip.open(infile, mode, encoding="UTF-8");
2051 except (ValueError, TypeError) as e:
2052 filefp = gzip.open(infile, mode);
2053 if(compresscheck=="bzip2"):
2054 try:
2055 import bz2;
2056 except ImportError:
2057 return False;
2058 try:
2059 filefp = bz2.open(infile, mode, encoding="UTF-8");
2060 except (ValueError, TypeError) as e:
2061 filefp = bz2.open(infile, mode);
2062 if(compresscheck=="zstd"):
2063 try:
2064 import zstandard;
2065 except ImportError:
2066 return False;
2067 try:
2068 filefp = zstandard.open(infile, mode, encoding="UTF-8");
2069 except (ValueError, TypeError) as e:
2070 filefp = zstandard.open(infile, mode);
2071 if(compresscheck=="lz4"):
2072 try:
2073 import lz4.frame;
2074 except ImportError:
2075 return False;
2076 try:
2077 filefp = lz4.frame.open(infile, mode, encoding="UTF-8");
2078 except (ValueError, TypeError) as e:
2079 filefp = lz4.frame.open(infile, mode);
2080 if(compresscheck=="lzo"):
2081 try:
2082 import lzo;
2083 except ImportError:
2084 return False;
2085 try:
2086 filefp = lzo.open(infile, mode, encoding="UTF-8");
2087 except (ValueError, TypeError) as e:
2088 filefp = lzo.open(infile, mode);
2089 if(compresscheck=="lzma"):
2090 try:
2091 import lzma;
2092 except ImportError:
2093 return False;
2094 try:
2095 filefp = lzma.open(infile, mode, encoding="UTF-8");
2096 except (ValueError, TypeError) as e:
2097 filefp = lzma.open(infile, mode);
2098 if(compresscheck=="catfile" or compresscheck==formatspecs[2]):
2099 try:
2100 filefp = open(infile, mode, encoding="UTF-8");
2101 except (ValueError, TypeError) as e:
2102 filefp = open(infile, mode);
2103 if(not compresscheck):
2104 try:
2105 filefp = open(infile, mode, encoding="UTF-8");
2106 except (ValueError, TypeError) as e:
2107 filefp = open(infile, mode);
2108 except FileNotFoundError:
2109 return False;
2110 return filefp;
2111
2112 def UncompressString(infile):
2113 compresscheck = CheckCompressionTypeFromString(infile, formatspecs, False);
2114 if(compresscheck=="gzip"):
2115 try:
2116 import gzip;
2117 except ImportError:
2118 return False;
2119 fileuz = gzip.decompress(infile);
2120 if(compresscheck=="bzip2"):
2121 try:
2122 import bz2;
2123 except ImportError:
2124 return False;
2125 fileuz = bz2.decompress(infile);
2126 if(compresscheck=="zstd"):
2127 try:
2128 import zstandard;
2129 except ImportError:
2130 return False;
2131 fileuz = zstandard.decompress(infile);
2132 if(compresscheck=="lz4"):
2133 try:
2134 import lz4.frame;
2135 except ImportError:
2136 return False;
2137 fileuz = lz4.frame.decompress(infile);
2138 if(compresscheck=="lzo"):
2139 try:
2140 import lzo;
2141 except ImportError:
2142 return False;
2143 fileuz = lzo.decompress(infile);
2144 if(compresscheck=="lzma"):
2145 try:
2146 import lzma;
2147 except ImportError:
2148 return False;
2149 fileuz = lzma.decompress(infile);
2150 if(not compresscheck):
2151 fileuz = infile;
2152 if(hasattr(fileuz, 'decode')):
2153 fileuz = fileuz.decode("UTF-8");
2154 return fileuz;
2155
2156 def UncompressStringAlt(infile):
2157 filefp = StringIO();
2158 outstring = UncompressString(infile);
2159 filefp.write(outstring);
2160 filefp.seek(0, 0);
2161 return filefp;
2162
2163 def CheckCompressionSubType(infile, formatspecs=__file_format_list__, closefp=True):
2164 compresscheck = CheckCompressionType(infile, formatspecs, False);
2165 if(not compresscheck):
2166 fextname = os.path.splitext(infile)[1];
2167 if(fextname==".gz"):
2168 compresscheck = "gzip";
2169 elif(fextname==".bz2"):
2170 compresscheck = "bzip2";
2171 elif(fextname==".zst"):
2172 compresscheck = "zstd";
2173 elif(fextname==".lz4"):
2174 compresscheck = "lz4";
2175 elif(fextname==".lzo" or fextname==".lzop"):
2176 compresscheck = "lzo";
2177 elif(fextname==".lzma" or fextname==".xz"):
2178 compresscheck = "lzma";
2179 else:
2180 return False;
2181 if(compresscheck=="gzip" or compresscheck=="bzip2" or compresscheck=="lzma" or compresscheck=="zstd" or compresscheck=="lz4"):
2182 if(is_tarfile(infile)):
2183 filetype = "tarfile";
2184 if(not compresscheck):
2185 if(is_tarfile(infile)):
2186 return "tarfile";
2187 elif(zipfile.is_zipfile(infile)):
2188 return "zipfile";
2189 elif(rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile)):
2190 return "rarile";
2191 else:
2192 return False;
2193 return False;
2194 if(compresscheck=="catfile"):
2195 return "catfile";
2196 if(compresscheck==formatspecs[2]):
2197 return formatspecs[2];
2198 if(compresscheck=="tarfile"):
2199 return "tarfile";
2200 if(compresscheck=="zipfile"):
2201 return "zipfile";
2202 if(rarfile_support and compresscheck=="rarfile"):
2203 return "rarfile";
2204 if(py7zr_support and compresscheck=="7zipfile"):
2205 return "7zipfile";
2206 if(hasattr(infile, "read") or hasattr(infile, "write")):
2207 catfp = UncompressArchiveFile(infile, formatspecs[2]);
2208 else:
2209 try:
2210 if(compresscheck=="gzip"):
2211 try:
2212 import gzip;
2213 except ImportError:
2214 return False;
2215 catfp = gzip.GzipFile(infile, "rb");
2216 if(compresscheck=="bzip2"):
2217 try:
2218 import bz2;
2219 except ImportError:
2220 return False;
2221 catfp = bz2.BZ2File(infile, "rb");
2222 if(compresscheck=="lz4"):
2223 try:
2224 import lz4.frame;
2225 except ImportError:
2226 return False;
2227 catfp = lz4.frame.open(infile, "rb");
2228 if(compresscheck=="zstd"):
2229 try:
2230 import zstandard;
2231 except ImportError:
2232 return False;
2233 catfp = zstandard.open(infile, "rb");
2234 if(compresscheck=="lzma" or compresscheck=="xz"):
2235 try:
2236 import lzma;
2237 except ImportError:
2238 return False;
2239 catfp = lzma.open(infile, "rb");
2240 except FileNotFoundError:
2241 return False;
2242 filetype = False;
2243 prefp = catfp.read(5);
2244 if(prefp==binascii.unhexlify("7573746172")):
2245 filetype = "tarfile";
2246 catfp.seek(0, 0);
2247 prefp = catfp.read(7);
2248 if(prefp==binascii.unhexlify("43617446696c65")):
2249 filetype = "catfile";
2250 catfp.seek(0, 0);
2251 prefp = catfp.read(formatspecs[3]);
2252 if(prefp==binascii.unhexlify(formatspecs[4])):
2253 filetype = formatspecs[2];
2254 catfp.seek(0, 0);
2255 prefp = catfp.read(10);
2256 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
2257 filetype = "tarfile";
2258 catfp.seek(0, 0);
2259 if(closefp):
2260 catfp.close();
2261 return filetype;
2262
2263 def GZipCompress(data, compresslevel=9):
2264 try:
2265 import gzip;
2266 except ImportError:
2267 return False;
2268 tmpfp = tempfile.NamedTemporaryFile("wb", delete=False);
2269 tmpfp.close();
2270 tmpfp = gzip.GzipFile(tmpfp.name, mode="wb", compresslevel=compresslevel);
2271 tmpfp.write(data);
2272 tmpfp.close();
2273 try:
2274 catfp = open(tmpfp.name, "rb");
2275 except FileNotFoundError:
2276 return False;
2277 catdata = catfp.read();
2278 catfp.close();
2279 return catdata;
2280
2281 def CompressArchiveFile(fp, compression="auto", compressionlevel=None, formatspecs=__file_format_list__):
2282 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
2283 return False;
2284 fp.seek(0, 0);
2285 if(not compression or compression=="catfile" or compression==formatspecs[2]):
2286 compression = "auto";
2287 if(compression not in compressionlist and compression is None):
2288 compression = "auto";
2289 if(compression=="gzip"):
2290 try:
2291 import gzip;
2292 except ImportError:
2293 return False;
2294 catfp = BytesIO();
2295 if(compressionlevel is None):
2296 compressionlevel = 9;
2297 else:
2298 compressionlevel = int(compressionlevel);
2299 catfp.write(gzip.compress(fp.read(), compresslevel=compressionlevel));
2300 if(compression=="bzip2"):
2301 try:
2302 import bz2;
2303 except ImportError:
2304 return False;
2305 catfp = BytesIO();
2306 if(compressionlevel is None):
2307 compressionlevel = 9;
2308 else:
2309 compressionlevel = int(compressionlevel);
2310 catfp.write(bz2.compress(fp.read(), compresslevel=compressionlevel));
2311 if(compression=="lz4"):
2312 try:
2313 import lz4.frame;
2314 except ImportError:
2315 return False;
2316 catfp = BytesIO();
2317 if(compressionlevel is None):
2318 compressionlevel = 9;
2319 else:
2320 compressionlevel = int(compressionlevel);
2321 catfp.write(lz4.frame.compress(fp.read(), compression_level=compressionlevel));
2322 if(compression=="lzo" or compression=="lzop"):
2323 try:
2324 import lzo;
2325 except ImportError:
2326 return False;
2327 catfp = BytesIO();
2328 if(compressionlevel is None):
2329 compressionlevel = 9;
2330 else:
2331 compressionlevel = int(compressionlevel);
2332 catfp.write(lzo.compress(fp.read(), compresslevel=compressionlevel));
2333 if(compression=="zstd"):
2334 try:
2335 import zstandard;
2336 except ImportError:
2337 return False;
2338 catfp = BytesIO();
2339 if(compressionlevel is None):
2340 compressionlevel = 10;
2341 else:
2342 compressionlevel = int(compressionlevel);
2343 catfp.write(zstandard.compress(fp.read(), level=compressionlevel));
2344 if(compression=="lzma"):
2345 try:
2346 import lzma;
2347 except ImportError:
2348 return False;
2349 catfp = BytesIO();
2350 if(compressionlevel is None):
2351 compressionlevel = 9;
2352 else:
2353 compressionlevel = int(compressionlevel);
2354 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]));
2355 if(compression=="xz"):
2356 try:
2357 import lzma;
2358 except ImportError:
2359 return False;
2360 catfp = BytesIO();
2361 if(compressionlevel is None):
2362 compressionlevel = 9;
2363 else:
2364 compressionlevel = int(compressionlevel);
2365 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]));
2366 if(compression=="auto" or compression is None):
2367 catfp = fp;
2368 catfp.seek(0, 0);
2369 return catfp;
2370
2371 create_alias_function("Compress", __file_format_name__, "", CompressArchiveFile);
2372
2373 def CompressOpenFile(outfile, compressionenable=True, compressionlevel=None):
2374 if(outfile is None):
2375 return False;
2376 fbasename = os.path.splitext(outfile)[0];
2377 fextname = os.path.splitext(outfile)[1];
2378 if(compressionlevel is None and fextname!=".zst"):
2379 compressionlevel = 9;
2380 elif(compressionlevel is None and fextname==".zst"):
2381 compressionlevel = 10;
2382 else:
2383 compressionlevel = int(compressionlevel);
2384 if(sys.version_info[0]==2):
2385 mode = "w";
2386 else:
2387 mode = "wb";
2388 try:
2389 if(fextname not in outextlistwd or not compressionenable):
2390 try:
2391 outfp = open(outfile, "wb", encoding="UTF-8");
2392 except (ValueError, TypeError) as e:
2393 outfp = open(outfile, "wb");
2394 elif(fextname==".gz"):
2395 try:
2396 import gzip;
2397 except ImportError:
2398 return False;
2399 try:
2400 outfp = gzip.open(outfile, mode, compressionlevel, encoding="UTF-8");
2401 except (ValueError, TypeError) as e:
2402 outfp = gzip.open(outfile, mode, compressionlevel);
2403 elif(fextname==".bz2"):
2404 try:
2405 import bz2;
2406 except ImportError:
2407 return False;
2408 try:
2409 outfp = bz2.open(outfile, mode, compressionlevel, encoding="UTF-8");
2410 except (ValueError, TypeError) as e:
2411 outfp = bz2.open(outfile, mode, compressionlevel);
2412 elif(fextname==".zst"):
2413 try:
2414 import zstandard;
2415 except ImportError:
2416 return False;
2417 try:
2418 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel), encoding="UTF-8");
2419 except (ValueError, TypeError) as e:
2420 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel));
2421 elif(fextname==".xz"):
2422 try:
2423 import lzma;
2424 except ImportError:
2425 return False;
2426 try:
2427 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}], encoding="UTF-8");
2428 except (ValueError, TypeError) as e:
2429 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]);
2430 elif(fextname==".lz4"):
2431 try:
2432 import lz4.frame;
2433 except ImportError:
2434 return False;
2435 try:
2436 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel, encoding="UTF-8");
2437 except (ValueError, TypeError) as e:
2438 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel);
2439 elif(fextname==".lzo"):
2440 try:
2441 import lzo;
2442 except ImportError:
2443 return False;
2444 try:
2445 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel, encoding="UTF-8");
2446 except (ValueError, TypeError) as e:
2447 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel);
2448 elif(fextname==".lzma"):
2449 try:
2450 import lzma;
2451 except ImportError:
2452 return False;
2453 try:
2454 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}], encoding="UTF-8");
2455 except (ValueError, TypeError) as e:
2456 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]);
2457 except FileNotFoundError:
2458 return False;
2459 return outfp;
2460
2461 def GetDevMajorMinor(fdev):
2462 retdev = [];
2463 if(hasattr(os, "minor")):
2464 retdev.append(os.minor(fdev));
2465 else:
2466 retdev.append(0);
2467 if(hasattr(os, "major")):
2468 retdev.append(os.major(fdev));
2469 else:
2470 retdev.append(0);
2471 return retdev;
2472
2473 def CheckSumSupport(checkfor, guaranteed=True):
2474 if(guaranteed):
2475 hash_list = sorted(list(hashlib.algorithms_guaranteed));
2476 else:
2477 hash_list = sorted(list(hashlib.algorithms_available));
2478 checklistout = sorted(hash_list + ['adler32', 'crc16', 'crc16_ansi', 'crc16_ibm', 'crc16_ccitt', 'crc32', 'crc64', 'crc64_ecma', 'crc64_iso', 'none']);
2479 if(checkfor in checklistout):
2480 return True;
2481 else:
2482 return False;
2483
2484 def CheckSumSupportAlt(checkfor, guaranteed=True):
2485 if(guaranteed):
2486 hash_list = sorted(list(hashlib.algorithms_guaranteed));
2487 else:
2488 hash_list = sorted(list(hashlib.algorithms_available));
2489 checklistout = hash_list;
2490 if(checkfor in checklistout):
2491 return True;
2492 else:
2493 return False;
2494
2495 def PackArchiveFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
2496 advancedlist = formatspecs[8];
2497 altinode = formatspecs[9];
2498 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2499 outfile = RemoveWindowsPath(outfile);
2500 checksumtype = checksumtype.lower();
2501 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
2502 checksumtype="crc32";
2503 if(checksumtype=="none"):
2504 checksumtype = "";
2505 if(not compression or compression=="catfile" or compression==formatspecs[2]):
2506 compression = "auto";
2507 if(compression not in compressionlist and compression is None):
2508 compression = "auto";
2509 if(verbose):
2510 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2511 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2512 if(os.path.exists(outfile)):
2513 try:
2514 os.unlink(outfile);
2515 except OSError as e:
2516 pass;
2517 if(outfile=="-"):
2518 verbose = False;
2519 catfp = BytesIO();
2520 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2521 catfp = outfile;
2522 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2523 catfp = BytesIO();
2524 else:
2525 fbasename = os.path.splitext(outfile)[0];
2526 fextname = os.path.splitext(outfile)[1];
2527 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
2528 catver = formatspecs[6];
2529 fileheaderver = str(int(catver.replace(".", "")));
2530 infilelist = [];
2531 if(infiles=="-"):
2532 for line in sys.stdin:
2533 infilelist.append(line.strip());
2534 infilelist = list(filter(None, infilelist));
2535 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
2536 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
2537 return False;
2538 with UncompressFile(infiles, formatspecs, "r") as finfile:
2539 for line in finfile:
2540 infilelist.append(line.strip());
2541 infilelist = list(filter(None, infilelist));
2542 else:
2543 if(isinstance(infiles, (list, tuple, ))):
2544 infilelist = list(filter(None, infiles));
2545 elif(isinstance(infiles, (str, ))):
2546 infilelist = list(filter(None, [infiles]));
2547 if(advancedlist):
2548 GetDirList = ListDirAdvanced(infilelist, followlink, False);
2549 else:
2550 GetDirList = ListDir(infilelist, followlink, False);
2551 if(not GetDirList):
2552 return False;
2553 curinode = 0;
2554 curfid = 0;
2555 inodelist = [];
2556 inodetofile = {};
2557 filetoinode = {};
2558 inodetocatinode = {};
2559 numfiles = int(len(GetDirList));
2560 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
2561 for curfname in GetDirList:
2562 if(re.findall("^[.|/]", curfname)):
2563 fname = curfname;
2564 else:
2565 fname = "./"+curfname;
2566 if(verbose):
2567 VerbosePrintOut(fname);
2568 if(not followlink or followlink is None):
2569 fstatinfo = os.lstat(fname);
2570 else:
2571 fstatinfo = os.stat(fname);
2572 fpremode = fstatinfo.st_mode;
2573 finode = fstatinfo.st_ino;
2574 flinkcount = fstatinfo.st_nlink;
2575 ftype = 0;
2576 if(stat.S_ISREG(fpremode)):
2577 ftype = 0;
2578 elif(stat.S_ISLNK(fpremode)):
2579 ftype = 2;
2580 elif(stat.S_ISCHR(fpremode)):
2581 ftype = 3;
2582 elif(stat.S_ISBLK(fpremode)):
2583 ftype = 4;
2584 elif(stat.S_ISDIR(fpremode)):
2585 ftype = 5;
2586 elif(stat.S_ISFIFO(fpremode)):
2587 ftype = 6;
2588 elif(stat.S_ISSOCK(fpremode)):
2589 ftype = 8;
2590 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
2591 ftype = 9;
2592 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
2593 ftype = 10;
2594 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
2595 ftype = 111;
2596 else:
2597 ftype = 0;
2598 flinkname = "";
2599 fcurfid = format(int(curfid), 'x').lower();
2600 if(not followlink and finode!=0):
2601 if(ftype!=1):
2602 if(finode in inodelist):
2603 ftype = 1;
2604 flinkname = inodetofile[finode];
2605 if(altinode):
2606 fcurinode = format(int(finode), 'x').lower();
2607 else:
2608 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
2609 if(finode not in inodelist):
2610 inodelist.append(finode);
2611 inodetofile.update({finode: fname});
2612 inodetocatinode.update({finode: curinode});
2613 if(altinode):
2614 fcurinode = format(int(finode), 'x').lower();
2615 else:
2616 fcurinode = format(int(curinode), 'x').lower();
2617 curinode = curinode + 1;
2618 else:
2619 fcurinode = format(int(curinode), 'x').lower();
2620 curinode = curinode + 1;
2621 curfid = curfid + 1;
2622 if(ftype==2):
2623 flinkname = os.readlink(fname);
2624 fdev = fstatinfo.st_dev;
2625 getfdev = GetDevMajorMinor(fdev);
2626 fdev_minor = getfdev[0];
2627 fdev_major = getfdev[1];
2628 frdev = fstatinfo.st_dev;
2629 if(hasattr(fstatinfo, "st_rdev")):
2630 frdev = fstatinfo.st_rdev;
2631 else:
2632 frdev = fstatinfo.st_dev;
2633 getfrdev = GetDevMajorMinor(frdev);
2634 frdev_minor = getfrdev[0];
2635 frdev_major = getfrdev[1];
2636 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
2637 fsize = format(int("0"), 'x').lower();
2638 elif(ftype==0 or ftype==7):
2639 fsize = format(int(fstatinfo.st_size), 'x').lower();
2640 else:
2641 fsize = format(int(fstatinfo.st_size)).lower();
2642 fatime = format(int(fstatinfo.st_atime), 'x').lower();
2643 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
2644 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
2645 if(hasattr(fstatinfo, "st_birthtime")):
2646 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
2647 else:
2648 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
2649 fmode = format(int(fstatinfo.st_mode), 'x').lower();
2650 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
2651 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
2652 fuid = format(int(fstatinfo.st_uid), 'x').lower();
2653 fgid = format(int(fstatinfo.st_gid), 'x').lower();
2654 funame = "";
2655 try:
2656 import pwd;
2657 try:
2658 userinfo = pwd.getpwuid(fstatinfo.st_uid);
2659 funame = userinfo.pw_name;
2660 except KeyError:
2661 funame = "";
2662 except ImportError:
2663 funame = "";
2664 fgname = "";
2665 try:
2666 import grp;
2667 try:
2668 groupinfo = grp.getgrgid(fstatinfo.st_gid);
2669 fgname = groupinfo.gr_name;
2670 except KeyError:
2671 fgname = "";
2672 except ImportError:
2673 fgname = "";
2674 fdev_minor = format(int(fdev_minor), 'x').lower();
2675 fdev_major = format(int(fdev_major), 'x').lower();
2676 frdev_minor = format(int(frdev_minor), 'x').lower();
2677 frdev_major = format(int(frdev_major), 'x').lower();
2678 finode = format(int(finode), 'x').lower();
2679 flinkcount = format(int(flinkcount), 'x').lower();
2680 if(hasattr(fstatinfo, "st_file_attributes")):
2681 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
2682 else:
2683 fwinattributes = format(int(0), 'x').lower();
2684 fcompression = "";
2685 fcsize = format(int(0), 'x').lower();
2686 fcontents = BytesIO();
2687 if(ftype==0 or ftype==7):
2688 with open(fname, "rb") as fpc:
2689 shutil.copyfileobj(fpc, fcontents);
2690 if(not compresswholefile):
2691 fcontents.seek(0, 2);
2692 ucfsize = fcontents.tell();
2693 fcontents.seek(0, 0);
2694 if(compression=="auto"):
2695 ilsize = len(compressionlistalt);
2696 ilmin = 0;
2697 ilcsize = [];
2698 while(ilmin < ilsize):
2699 cfcontents = BytesIO();
2700 shutil.copyfileobj(fcontents, cfcontents);
2701 fcontents.seek(0, 0);
2702 cfcontents.seek(0, 0);
2703 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
2704 if(cfcontents):
2705 cfcontents.seek(0, 2);
2706 ilcsize.append(cfcontents.tell());
2707 cfcontents.close();
2708 else:
2709 try:
2710 ilcsize.append(sys.maxint);
2711 except AttributeError:
2712 ilcsize.append(sys.maxsize);
2713 ilmin = ilmin + 1;
2714 ilcmin = ilcsize.index(min(ilcsize));
2715 compression = compressionlistalt[ilcmin];
2716 fcontents.seek(0, 0);
2717 cfcontents = BytesIO();
2718 shutil.copyfileobj(fcontents, cfcontents);
2719 cfcontents.seek(0, 0);
2720 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
2721 cfcontents.seek(0, 2);
2722 cfsize = cfcontents.tell();
2723 if(ucfsize > cfsize):
2724 fcsize = format(int(cfsize), 'x').lower();
2725 fcompression = compression;
2726 fcontents.close();
2727 fcontents = cfcontents;
2728 if(fcompression=="none"):
2729 fcompression = "";
2730 if(followlink and (ftype==1 or ftype==2)):
2731 flstatinfo = os.stat(flinkname);
2732 with open(flinkname, "rb") as fpc:
2733 shutil.copyfileobj(fpc, fcontents);
2734 if(not compresswholefile):
2735 fcontents.seek(0, 2);
2736 ucfsize = fcontents.tell();
2737 fcontents.seek(0, 0);
2738 if(compression=="auto"):
2739 ilsize = len(compressionlistalt);
2740 ilmin = 0;
2741 ilcsize = [];
2742 while(ilmin < ilsize):
2743 cfcontents = BytesIO();
2744 shutil.copyfileobj(fcontents, cfcontents);
2745 fcontents.seek(0, 0);
2746 cfcontents.seek(0, 0);
2747 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
2748 if(cfcontents):
2749 cfcontents.seek(0, 2);
2750 ilcsize.append(cfcontents.tell());
2751 cfcontents.close();
2752 else:
2753 try:
2754 ilcsize.append(sys.maxint);
2755 except AttributeError:
2756 ilcsize.append(sys.maxsize);
2757 ilmin = ilmin + 1;
2758 ilcmin = ilcsize.index(min(ilcsize));
2759 compression = compressionlistalt[ilcmin];
2760 fcontents.seek(0, 0);
2761 cfcontents = BytesIO();
2762 shutil.copyfileobj(fcontents, cfcontents);
2763 cfcontents.seek(0, 0);
2764 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
2765 cfcontents.seek(0, 2);
2766 cfsize = cfcontents.tell();
2767 if(ucfsize > cfsize):
2768 fcsize = format(int(cfsize), 'x').lower();
2769 fcompression = compression;
2770 fcontents.close();
2771 fcontents = cfcontents;
2772 fcontents.seek(0, 0);
2773 ftypehex = format(ftype, 'x').lower();
2774 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
2775 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
2776 fcontents.close();
2777 if(numfiles>0):
2778 catfp.write(AppendNullBytes([0, 0], formatspecs[5]).encode("UTF-8"));
2779 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
2780 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2781 try:
2782 catfp.flush();
2783 os.fsync(catfp.fileno());
2784 except io.UnsupportedOperation:
2785 pass;
2786 except AttributeError:
2787 pass;
2788 except OSError as e:
2789 pass;
2790 if(outfile=="-"):
2791 catfp.seek(0, 0);
2792 if(hasattr(sys.stdout, "buffer")):
2793 shutil.copyfileobj(catfp, sys.stdout.buffer);
2794 else:
2795 shutil.copyfileobj(catfp, sys.stdout);
2796 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2797 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2798 catfp.seek(0, 0);
2799 upload_file_to_internet_file(catfp, outfile);
2800 if(returnfp):
2801 catfp.seek(0, 0);
2802 return catfp;
2803 else:
2804 catfp.close();
2805 return True;
2806
2807 create_alias_function("Pack", __file_format_name__, "", PackArchiveFile);
2808
2809 if(hasattr(shutil, "register_archive_format")):
2810 def PackArchiveFileFunc(archive_name, source_dir, **kwargs):
2811 return PackArchiveFile(source_dir, archive_name, False, "auto", True, None, False, "crc32", [], __file_format_delimiter__, False, False);
2812 create_alias_function("Pack", __file_format_name__, "Func", PackArchiveFileFunc);
2813
2814 def PackArchiveFileFromDirList(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
2815 return PackArchiveFile(infiles, outfile, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, returnfp);
2816
2817 create_alias_function("Pack", __file_format_name__, "FromDirList", PackArchiveFileFromDirList);
2818
2819 def PackArchiveFileFromTarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
2820 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2821 outfile = RemoveWindowsPath(outfile);
2822 checksumtype = checksumtype.lower();
2823 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
2824 checksumtype="crc32";
2825 if(checksumtype=="none"):
2826 checksumtype = "";
2827 if(not compression or compression=="catfile" or compression==formatspecs[2]):
2828 compression = "auto";
2829 if(compression not in compressionlist and compression is None):
2830 compression = "auto";
2831 if(verbose):
2832 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2833 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2834 if(os.path.exists(outfile)):
2835 try:
2836 os.unlink(outfile);
2837 except OSError as e:
2838 pass;
2839 if(outfile=="-"):
2840 verbose = False;
2841 catfp = BytesIO();
2842 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2843 catfp = outfile;
2844 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2845 catfp = BytesIO();
2846 else:
2847 fbasename = os.path.splitext(outfile)[0];
2848 fextname = os.path.splitext(outfile)[1];
2849 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
2850 catver = formatspecs[6];
2851 fileheaderver = str(int(catver.replace(".", "")));
2852 curinode = 0;
2853 curfid = 0;
2854 inodelist = [];
2855 inodetofile = {};
2856 filetoinode = {};
2857 inodetocatinode = {};
2858 if(infile=="-"):
2859 infile = BytesIO();
2860 if(hasattr(sys.stdin, "buffer")):
2861 shutil.copyfileobj(sys.stdin.buffer, infile);
2862 else:
2863 shutil.copyfileobj(sys.stdin, infile);
2864 infile.seek(0, 0);
2865 if(not infile):
2866 return False;
2867 infile.seek(0, 0);
2868 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
2869 infile = download_file_from_internet_file(infile);
2870 infile.seek(0, 0);
2871 if(not infile):
2872 return False;
2873 infile.seek(0, 0);
2874 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
2875 return False;
2876 elif(os.path.exists(infile) and os.path.isfile(infile)):
2877 try:
2878 if(not tarfile.is_tarfile(infile)):
2879 return False;
2880 except AttributeError:
2881 if(not is_tarfile(infile)):
2882 return False;
2883 else:
2884 return False;
2885 try:
2886 if(hasattr(infile, "read") or hasattr(infile, "write")):
2887 tarfp = tarfile.open(fileobj=infile, mode="r");
2888 else:
2889 tarfp = tarfile.open(infile, "r");
2890 except FileNotFoundError:
2891 return False;
2892 numfiles = int(len(tarfp.getmembers()));
2893 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
2894 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
2895 if(re.findall("^[.|/]", member.name)):
2896 fname = member.name;
2897 else:
2898 fname = "./"+member.name;
2899 if(verbose):
2900 VerbosePrintOut(fname);
2901 fpremode = member.mode;
2902 ffullmode = member.mode;
2903 flinkcount = 0;
2904 ftype = 0;
2905 if(member.isreg()):
2906 ffullmode = member.mode + stat.S_IFREG;
2907 ftype = 0;
2908 elif(member.isdev()):
2909 ffullmode = member.mode;
2910 ftype = 7;
2911 elif(member.islnk()):
2912 ffullmode = member.mode + stat.S_IFREG;
2913 ftype = 1;
2914 elif(member.issym()):
2915 ffullmode = member.mode + stat.S_IFLNK;
2916 ftype = 2;
2917 elif(member.ischr()):
2918 ffullmode = member.mode + stat.S_IFCHR;
2919 ftype = 3;
2920 elif(member.isblk()):
2921 ffullmode = member.mode + stat.S_IFBLK;
2922 ftype = 4;
2923 elif(member.isdir()):
2924 ffullmode = member.mode + stat.S_IFDIR;
2925 ftype = 5;
2926 elif(member.isfifo()):
2927 ffullmode = member.mode + stat.S_IFIFO;
2928 ftype = 6;
2929 elif(member.issparse()):
2930 ffullmode = member.mode;
2931 ftype = 12;
2932 else:
2933 ffullmode = member.mode;
2934 ftype = 0;
2935 flinkname = "";
2936 fcurfid = format(int(curfid), 'x').lower();
2937 fcurinode = format(int(curfid), 'x').lower();
2938 curfid = curfid + 1;
2939 if(ftype==2):
2940 flinkname = member.linkname;
2941 fdev_minor = format(int(member.devminor), 'x').lower();
2942 fdev_major = format(int(member.devmajor), 'x').lower();
2943 frdev_minor = format(int(member.devminor), 'x').lower();
2944 frdev_major = format(int(member.devmajor), 'x').lower();
2945 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
2946 fsize = format(int("0"), 'x').lower();
2947 elif(ftype==0 or ftype==7):
2948 fsize = format(int(member.size), 'x').lower();
2949 else:
2950 fsize = format(int(member.size), 'x').lower();
2951 fatime = format(int(member.mtime), 'x').lower();
2952 fmtime = format(int(member.mtime), 'x').lower();
2953 fctime = format(int(member.mtime), 'x').lower();
2954 fbtime = format(int(member.mtime), 'x').lower();
2955 fmode = format(int(ffullmode), 'x').lower();
2956 fchmode = format(int(stat.S_IMODE(ffullmode)), 'x').lower();
2957 ftypemod = format(int(stat.S_IFMT(ffullmode)), 'x').lower();
2958 fuid = format(int(member.uid), 'x').lower();
2959 fgid = format(int(member.gid), 'x').lower();
2960 funame = member.uname;
2961 fgname = member.gname;
2962 flinkcount = format(int(flinkcount), 'x').lower();
2963 fwinattributes = format(int(0), 'x').lower();
2964 fcompression = "";
2965 fcsize = format(int(0), 'x').lower();
2966 fcontents = BytesIO();
2967 if(ftype==0 or ftype==7):
2968 with tarfp.extractfile(member) as fpc:
2969 shutil.copyfileobj(fpc, fcontents);
2970 if(not compresswholefile):
2971 fcontents.seek(0, 2);
2972 ucfsize = fcontents.tell();
2973 fcontents.seek(0, 0);
2974 if(compression=="auto"):
2975 ilsize = len(compressionlistalt);
2976 ilmin = 0;
2977 ilcsize = [];
2978 while(ilmin < ilsize):
2979 cfcontents = BytesIO();
2980 shutil.copyfileobj(fcontents, cfcontents);
2981 fcontents.seek(0, 0);
2982 cfcontents.seek(0, 0);
2983 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
2984 if(cfcontents):
2985 cfcontents.seek(0, 2);
2986 ilcsize.append(cfcontents.tell());
2987 cfcontents.close();
2988 else:
2989 try:
2990 ilcsize.append(sys.maxint);
2991 except AttributeError:
2992 ilcsize.append(sys.maxsize);
2993 ilmin = ilmin + 1;
2994 ilcmin = ilcsize.index(min(ilcsize));
2995 compression = compressionlistalt[ilcmin];
2996 fcontents.seek(0, 0);
2997 cfcontents = BytesIO();
2998 shutil.copyfileobj(fcontents, cfcontents);
2999 cfcontents.seek(0, 0);
3000 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3001 cfcontents.seek(0, 2);
3002 cfsize = cfcontents.tell();
3003 if(ucfsize > cfsize):
3004 fcsize = format(int(cfsize), 'x').lower();
3005 fcompression = compression;
3006 fcontents.close();
3007 fcontents = cfcontents;
3008 if(fcompression=="none"):
3009 fcompression = "";
3010 fcontents.seek(0, 0);
3011 ftypehex = format(ftype, 'x').lower();
3012 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3013 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3014 fcontents.close();
3015 if(numfiles>0):
3016 catfp.write(AppendNullBytes([0, 0], formatspecs[5]).encode("UTF-8"));
3017 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3018 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3019 try:
3020 catfp.flush();
3021 os.fsync(catfp.fileno());
3022 except io.UnsupportedOperation:
3023 pass;
3024 except AttributeError:
3025 pass;
3026 except OSError as e:
3027 pass;
3028 if(outfile=="-"):
3029 catfp.seek(0, 0);
3030 if(hasattr(sys.stdout, "buffer")):
3031 shutil.copyfileobj(catfp, sys.stdout.buffer);
3032 else:
3033 shutil.copyfileobj(catfp, sys.stdout);
3034 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3035 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3036 catfp.seek(0, 0);
3037 upload_file_to_internet_file(catfp, outfile);
3038 if(returnfp):
3039 catfp.seek(0, 0);
3040 return catfp;
3041 else:
3042 catfp.close();
3043 return True;
3044
3045 create_alias_function("Pack", __file_format_name__, "FromTarFile", PackArchiveFileFromTarFile);
3046
3047 def PackArchiveFileFromZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
3048 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3049 outfile = RemoveWindowsPath(outfile);
3050 checksumtype = checksumtype.lower();
3051 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3052 checksumtype="crc32";
3053 if(checksumtype=="none"):
3054 checksumtype = "";
3055 if(not compression or compression=="catfile" or compression==formatspecs[2]):
3056 compression = "auto";
3057 if(compression not in compressionlist and compression is None):
3058 compression = "auto";
3059 if(verbose):
3060 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3061 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3062 if(os.path.exists(outfile)):
3063 try:
3064 os.unlink(outfile);
3065 except OSError as e:
3066 pass;
3067 if(outfile=="-"):
3068 verbose = False;
3069 catfp = BytesIO();
3070 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3071 catfp = outfile;
3072 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3073 catfp = BytesIO();
3074 else:
3075 fbasename = os.path.splitext(outfile)[0];
3076 fextname = os.path.splitext(outfile)[1];
3077 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3078 catver = formatspecs[6];
3079 fileheaderver = str(int(catver.replace(".", "")));
3080 curinode = 0;
3081 curfid = 0;
3082 inodelist = [];
3083 inodetofile = {};
3084 filetoinode = {};
3085 inodetocatinode = {};
3086 if(infile=="-"):
3087 infile = BytesIO();
3088 if(hasattr(sys.stdin, "buffer")):
3089 shutil.copyfileobj(sys.stdin.buffer, infile);
3090 else:
3091 shutil.copyfileobj(sys.stdin, infile);
3092 infile.seek(0, 0);
3093 if(not infile):
3094 return False;
3095 infile.seek(0, 0);
3096 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3097 infile = download_file_from_internet_file(infile);
3098 infile.seek(0, 0);
3099 if(not infile):
3100 return False;
3101 infile.seek(0, 0);
3102 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
3103 return False;
3104 else:
3105 return False;
3106 if(not zipfile.is_zipfile(infile)):
3107 return False;
3108 try:
3109 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
3110 except FileNotFoundError:
3111 return False;
3112 ziptest = zipfp.testzip();
3113 if(ziptest):
3114 VerbosePrintOut("Bad file found!");
3115 numfiles = int(len(zipfp.infolist()));
3116 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3117 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
3118 if(re.findall("^[.|/]", member.filename)):
3119 fname = member.filename;
3120 else:
3121 fname = "./"+member.filename;
3122 zipinfo = zipfp.getinfo(member.filename);
3123 if(verbose):
3124 VerbosePrintOut(fname);
3125 if(not member.is_dir()):
3126 fpremode = int(stat.S_IFREG + 438);
3127 elif(member.is_dir()):
3128 fpremode = int(stat.S_IFDIR + 511);
3129 flinkcount = 0;
3130 ftype = 0;
3131 if(not member.is_dir()):
3132 ftype = 0;
3133 elif(member.is_dir()):
3134 ftype = 5;
3135 flinkname = "";
3136 fcurfid = format(int(curfid), 'x').lower();
3137 fcurinode = format(int(curfid), 'x').lower();
3138 curfid = curfid + 1;
3139 fdev_minor = format(int(0), 'x').lower();
3140 fdev_major = format(int(0), 'x').lower();
3141 frdev_minor = format(int(0), 'x').lower();
3142 frdev_major = format(int(0), 'x').lower();
3143 if(ftype==5):
3144 fsize = format(int("0"), 'x').lower();
3145 elif(ftype==0):
3146 fsize = format(int(member.file_size), 'x').lower();
3147 else:
3148 fsize = format(int(member.file_size), 'x').lower();
3149 fatime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3150 fmtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3151 fctime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3152 fbtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3153 if(zipinfo.create_system==0 or zipinfo.create_system==10):
3154 fwinattributes = format(int(zipinfo.external_attr), 'x').lower();
3155 if(not member.is_dir()):
3156 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3157 fchmode = stat.S_IMODE(fmode);
3158 ftypemod = stat.S_IFMT(fmode);
3159 elif(member.is_dir()):
3160 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3161 fchmode = stat.S_IMODE(fmode);
3162 ftypemod = stat.S_IFMT(fmode);
3163 elif(zipinfo.create_system==3):
3164 fwinattributes = format(int(0), 'x').lower();
3165 fmode = format(int(zipinfo.external_attr), 'x').lower();
3166 fchmode = stat.S_IMODE(fmode);
3167 ftypemod = stat.S_IFMT(fmode);
3168 else:
3169 fwinattributes = format(int(0), 'x').lower();
3170 if(not member.is_dir()):
3171 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3172 fchmode = stat.S_IMODE(fmode);
3173 ftypemod = stat.S_IFMT(fmode);
3174 elif(member.is_dir()):
3175 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3176 fchmode = stat.S_IMODE(fmode);
3177 ftypemod = stat.S_IFMT(fmode);
3178 fcompression = "";
3179 fcsize = format(int(0), 'x').lower();
3180 try:
3181 fuid = format(int(os.getuid()), 'x').lower();
3182 except AttributeError:
3183 fuid = format(int(0), 'x').lower();
3184 except KeyError:
3185 fuid = format(int(0), 'x').lower();
3186 try:
3187 fgid = format(int(os.getgid()), 'x').lower();
3188 except AttributeError:
3189 fgid = format(int(0), 'x').lower();
3190 except KeyError:
3191 fgid = format(int(0), 'x').lower();
3192 try:
3193 import pwd;
3194 try:
3195 userinfo = pwd.getpwuid(os.getuid());
3196 funame = userinfo.pw_name;
3197 except KeyError:
3198 funame = "";
3199 except AttributeError:
3200 funame = "";
3201 except ImportError:
3202 funame = "";
3203 fgname = "";
3204 try:
3205 import grp;
3206 try:
3207 groupinfo = grp.getgrgid(os.getgid());
3208 fgname = groupinfo.gr_name;
3209 except KeyError:
3210 fgname = "";
3211 except AttributeError:
3212 fgname = "";
3213 except ImportError:
3214 fgname = "";
3215 fcontents = BytesIO();
3216 if(ftype==0):
3217 fcontents.write(zipfp.read(member.filename));
3218 if(not compresswholefile):
3219 fcontents.seek(0, 2);
3220 ucfsize = fcontents.tell();
3221 fcontents.seek(0, 0);
3222 if(compression=="auto"):
3223 ilsize = len(compressionlistalt);
3224 ilmin = 0;
3225 ilcsize = [];
3226 while(ilmin < ilsize):
3227 cfcontents = BytesIO();
3228 shutil.copyfileobj(fcontents, cfcontents);
3229 fcontents.seek(0, 0);
3230 cfcontents.seek(0, 0);
3231 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3232 cfcontents.seek(0, 2);
3233 ilcsize.append(cfcontents.tell());
3234 cfcontents.close();
3235 ilmin = ilmin + 1;
3236 ilcmin = ilcsize.index(min(ilcsize));
3237 compression = compressionlistalt[ilcmin];
3238 fcontents.seek(0, 0);
3239 cfcontents = BytesIO();
3240 shutil.copyfileobj(fcontents, cfcontents);
3241 cfcontents.seek(0, 0);
3242 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3243 cfcontents.seek(0, 2);
3244 cfsize = cfcontents.tell();
3245 if(ucfsize > cfsize):
3246 fcsize = format(int(cfsize), 'x').lower();
3247 fcompression = compression;
3248 fcontents.close();
3249 fcontents = cfcontents;
3250 if(fcompression=="none"):
3251 fcompression = "";
3252 fcontents.seek(0, 0);
3253 ftypehex = format(ftype, 'x').lower();
3254 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3255 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3256 fcontents.close();
3257 if(numfiles>0):
3258 catfp.write(AppendNullBytes([0, 0], formatspecs[5]).encode("UTF-8"));
3259 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3260 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3261 try:
3262 catfp.flush();
3263 os.fsync(catfp.fileno());
3264 except io.UnsupportedOperation:
3265 pass;
3266 except AttributeError:
3267 pass;
3268 except OSError as e:
3269 pass;
3270 if(outfile=="-"):
3271 catfp.seek(0, 0);
3272 if(hasattr(sys.stdout, "buffer")):
3273 shutil.copyfileobj(catfp, sys.stdout.buffer);
3274 else:
3275 shutil.copyfileobj(catfp, sys.stdout);
3276 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3277 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3278 catfp.seek(0, 0);
3279 upload_file_to_internet_file(catfp, outfile);
3280 if(returnfp):
3281 catfp.seek(0, 0);
3282 return catfp;
3283 else:
3284 catfp.close();
3285 return True;
3286
3287 create_alias_function("Pack", __file_format_name__, "FromZipFile", PackArchiveFileFromZipFile);
3288
3289 if(not rarfile_support):
3290 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
3291 return False
3292
3293 if(rarfile_support):
3294 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
3295 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3296 outfile = RemoveWindowsPath(outfile);
3297 checksumtype = checksumtype.lower();
3298 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3299 checksumtype="crc32";
3300 if(checksumtype=="none"):
3301 checksumtype = "";
3302 if(not compression or compression=="catfile" or compression==formatspecs[2]):
3303 compression = "auto";
3304 if(compression not in compressionlist and compression is None):
3305 compression = "auto";
3306 if(verbose):
3307 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3308 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3309 if(os.path.exists(outfile)):
3310 try:
3311 os.unlink(outfile);
3312 except OSError as e:
3313 pass;
3314 if(outfile=="-"):
3315 verbose = False;
3316 catfp = BytesIO();
3317 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3318 catfp = outfile;
3319 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3320 catfp = BytesIO();
3321 else:
3322 fbasename = os.path.splitext(outfile)[0];
3323 fextname = os.path.splitext(outfile)[1];
3324 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3325 catver = formatspecs[6];
3326 fileheaderver = str(int(catver.replace(".", "")));
3327 curinode = 0;
3328 curfid = 0;
3329 inodelist = [];
3330 inodetofile = {};
3331 filetoinode = {};
3332 inodetocatinode = {};
3333 if(not os.path.exists(infile) or not os.path.isfile(infile)):
3334 return False;
3335 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
3336 return False;
3337 rarfp = rarfile.RarFile(infile, "r");
3338 rartest = rarfp.testrar();
3339 if(rartest):
3340 VerbosePrintOut("Bad file found!");
3341 numfiles = int(len(rarfp.infolist()));
3342 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3343 try:
3344 catfp.flush();
3345 os.fsync(catfp.fileno());
3346 except io.UnsupportedOperation:
3347 pass;
3348 except AttributeError:
3349 pass;
3350 except OSError as e:
3351 pass;
3352 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
3353 is_unix = False;
3354 is_windows = False;
3355 if(member.host_os==rarfile.RAR_OS_UNIX):
3356 is_windows = False;
3357 try:
3358 member.external_attr
3359 is_unix = True;
3360 except AttributeError:
3361 is_unix = False;
3362 elif(member.host_os==rarfile.RAR_OS_WIN32):
3363 is_unix = False;
3364 try:
3365 member.external_attr
3366 is_windows = True;
3367 except AttributeError:
3368 is_windows = False;
3369 else:
3370 is_unix = False;
3371 is_windows = False;
3372 if(re.findall("^[.|/]", member.filename)):
3373 fname = member.filename;
3374 else:
3375 fname = "./"+member.filename;
3376 rarinfo = rarfp.getinfo(member.filename);
3377 if(verbose):
3378 VerbosePrintOut(fname);
3379 if(is_unix and member.external_attr !=0):
3380 fpremode = int(member.external_attr);
3381 elif(member.is_file()):
3382 fpremode = int(stat.S_IFREG + 438);
3383 elif(member.is_symlink()):
3384 fpremode = int(stat.S_IFLNK + 438);
3385 elif(member.is_dir()):
3386 fpremode = int(stat.S_IFDIR + 511);
3387 if(is_windows and member.external_attr !=0):
3388 fwinattributes = format(int(member.external_attr), 'x').lower();
3389 else:
3390 fwinattributes = format(int(0), 'x').lower();
3391 fcompression = "";
3392 fcsize = format(int(0), 'x').lower();
3393 flinkcount = 0;
3394 ftype = 0;
3395 if(member.is_file()):
3396 ftype = 0;
3397 elif(member.is_symlink()):
3398 ftype = 2;
3399 elif(member.is_dir()):
3400 ftype = 5;
3401 flinkname = "";
3402 if(ftype==2):
3403 flinkname = rarfp.read(member.filename).decode("UTF-8");
3404 fcurfid = format(int(curfid), 'x').lower();
3405 fcurinode = format(int(curfid), 'x').lower();
3406 curfid = curfid + 1;
3407 fdev_minor = format(int(0), 'x').lower();
3408 fdev_major = format(int(0), 'x').lower();
3409 frdev_minor = format(int(0), 'x').lower();
3410 frdev_major = format(int(0), 'x').lower();
3411 if(ftype==5):
3412 fsize = format(int("0"), 'x').lower();
3413 elif(ftype==0):
3414 fsize = format(int(member.file_size), 'x').lower();
3415 else:
3416 fsize = format(int(member.file_size), 'x').lower();
3417 try:
3418 if(member.atime):
3419 fatime = format(int(member.atime.timestamp()), 'x').lower();
3420 else:
3421 fatime = format(int(member.mtime.timestamp()), 'x').lower();
3422 except AttributeError:
3423 fatime = format(int(member.mtime.timestamp()), 'x').lower();
3424 fmtime = format(int(member.mtime.timestamp()), 'x').lower();
3425 try:
3426 if(member.ctime):
3427 fctime = format(int(member.ctime.timestamp()), 'x').lower();
3428 else:
3429 fctime = format(int(member.mtime.timestamp()), 'x').lower();
3430 except AttributeError:
3431 fctime = format(int(member.mtime.timestamp()), 'x').lower();
3432 fbtime = format(int(member.mtime.timestamp()), 'x').lower();
3433 if(is_unix and member.external_attr !=0):
3434 fmode = format(int(member.external_attr), 'x').lower();
3435 fchmode = format(int(stat.S_IMODE(member.external_attr)), 'x').lower();
3436 ftypemod = format(int(stat.S_IFMT(member.external_attr)), 'x').lower();
3437 elif(member.is_file()):
3438 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3439 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3440 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3441 elif(member.is_symlink()):
3442 fmode = format(int(stat.S_IFLNK + 438), 'x').lower();
3443 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3444 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3445 elif(member.is_dir()):
3446 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3447 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
3448 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
3449 try:
3450 fuid = format(int(os.getuid()), 'x').lower();
3451 except AttributeError:
3452 fuid = format(int(0), 'x').lower();
3453 except KeyError:
3454 fuid = format(int(0), 'x').lower();
3455 try:
3456 fgid = format(int(os.getgid()), 'x').lower();
3457 except AttributeError:
3458 fgid = format(int(0), 'x').lower();
3459 except KeyError:
3460 fgid = format(int(0), 'x').lower();
3461 try:
3462 import pwd;
3463 try:
3464 userinfo = pwd.getpwuid(os.getuid());
3465 funame = userinfo.pw_name;
3466 except KeyError:
3467 funame = "";
3468 except AttributeError:
3469 funame = "";
3470 except ImportError:
3471 funame = "";
3472 fgname = "";
3473 try:
3474 import grp;
3475 try:
3476 groupinfo = grp.getgrgid(os.getgid());
3477 fgname = groupinfo.gr_name;
3478 except KeyError:
3479 fgname = "";
3480 except AttributeError:
3481 fgname = "";
3482 except ImportError:
3483 fgname = "";
3484 fcontents = BytesIO();
3485 if(ftype==0):
3486 fcontents.write(rarfp.read(member.filename));
3487 if(not compresswholefile):
3488 fcontents.seek(0, 2);
3489 ucfsize = fcontents.tell();
3490 fcontents.seek(0, 0);
3491 if(compression=="auto"):
3492 ilsize = len(compressionlistalt);
3493 ilmin = 0;
3494 ilcsize = [];
3495 while(ilmin < ilsize):
3496 cfcontents = BytesIO();
3497 shutil.copyfileobj(fcontents, cfcontents);
3498 fcontents.seek(0, 0);
3499 cfcontents.seek(0, 0);
3500 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3501 if(cfcontents):
3502 cfcontents.seek(0, 2);
3503 ilcsize.append(cfcontents.tell());
3504 cfcontents.close();
3505 else:
3506 try:
3507 ilcsize.append(sys.maxint);
3508 except AttributeError:
3509 ilcsize.append(sys.maxsize);
3510 ilmin = ilmin + 1;
3511 ilcmin = ilcsize.index(min(ilcsize));
3512 compression = compressionlistalt[ilcmin];
3513 fcontents.seek(0, 0);
3514 cfcontents = BytesIO();
3515 shutil.copyfileobj(fcontents, cfcontents);
3516 cfcontents.seek(0, 0);
3517 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3518 cfcontents.seek(0, 2);
3519 cfsize = cfcontents.tell();
3520 if(ucfsize > cfsize):
3521 fcsize = format(int(cfsize), 'x').lower();
3522 fcompression = compression;
3523 fcontents.close();
3524 fcontents = cfcontents;
3525 if(fcompression=="none"):
3526 fcompression = "";
3527 fcontents.seek(0, 0);
3528 ftypehex = format(ftype, 'x').lower();
3529 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3530 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3531 fcontents.close();
3532 if(numfiles>0):
3533 catfp.write(AppendNullBytes([0, 0], formatspecs[5]).encode("UTF-8"));
3534 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3535 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
3536 try:
3537 catfp.flush();
3538 os.fsync(catfp.fileno());
3539 except io.UnsupportedOperation:
3540 pass;
3541 except AttributeError:
3542 pass;
3543 except OSError as e:
3544 pass;
3545 if(outfile=="-"):
3546 catfp.seek(0, 0)
3547 if(hasattr(sys.stdout, "buffer")):
3548 shutil.copyfileobj(catfp, sys.stdout.buffer);
3549 else:
3550 shutil.copyfileobj(catfp, sys.stdout);
3551 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3552 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3553 catfp.seek(0, 0);
3554 upload_file_to_internet_file(catfp, outfile);
3555 if(returnfp):
3556 catfp.seek(0, 0)
3557 return catfp
3558 else:
3559 catfp.close()
3560 return True;
3561
3562 create_alias_function("Pack", __file_format_name__, "FromRarFile", PackArchiveFileFromRarFile);
3563
3564 if(not py7zr_support):
3565 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
3566 return False
3567
3568 if(py7zr_support):
3569 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
3570 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3571 outfile = RemoveWindowsPath(outfile);
3572 checksumtype = checksumtype.lower();
3573 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3574 checksumtype="crc32";
3575 if(checksumtype=="none"):
3576 checksumtype = "";
3577 if(not compression or compression=="catfile" or compression==formatspecs[2]):
3578 compression = "auto";
3579 if(compression not in compressionlist and compression is None):
3580 compression = "auto";
3581 if(verbose):
3582 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3583 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3584 if(os.path.exists(outfile)):
3585 try:
3586 os.unlink(outfile);
3587 except OSError as e:
3588 pass;
3589 if(outfile=="-"):
3590 verbose = False;
3591 catfp = BytesIO();
3592 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3593 catfp = outfile;
3594 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3595 catfp = BytesIO();
3596 else:
3597 fbasename = os.path.splitext(outfile)[0];
3598 fextname = os.path.splitext(outfile)[1];
3599 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3600 catver = formatspecs[6];
3601 fileheaderver = str(int(catver.replace(".", "")));
3602 curinode = 0;
3603 curfid = 0;
3604 inodelist = [];
3605 inodetofile = {};
3606 filetoinode = {};
3607 inodetocatinode = {};
3608 if(not os.path.exists(infile) or not os.path.isfile(infile)):
3609 return False;
3610 szpfp = py7zr.SevenZipFile(infile, mode="r");
3611 file_content = szpfp.readall();
3612 #sztest = szpfp.testzip();
3613 sztestalt = szpfp.test();
3614 if(sztestalt):
3615 VerbosePrintOut("Bad file found!");
3616 numfiles = int(len(szpfp.list()));
3617 AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3618 for member in sorted(szpfp.list(), key=lambda x: x.filename):
3619 if(re.findall("^[.|/]", member.filename)):
3620 fname = member.filename;
3621 else:
3622 fname = "./"+member.filename;
3623 if(verbose):
3624 VerbosePrintOut(fname);
3625 if(not member.is_directory):
3626 fpremode = int(stat.S_IFREG + 438);
3627 elif(member.is_directory):
3628 fpremode = int(stat.S_IFDIR + 511);
3629 fwinattributes = format(int(0), 'x').lower();
3630 fcompression = "";
3631 fcsize = format(int(0), 'x').lower();
3632 flinkcount = 0;
3633 ftype = 0;
3634 if(member.is_directory):
3635 ftype = 5;
3636 else:
3637 ftype = 0;
3638 flinkname = "";
3639 fcurfid = format(int(curfid), 'x').lower();
3640 fcurinode = format(int(curfid), 'x').lower();
3641 curfid = curfid + 1;
3642 fdev_minor = format(int(0), 'x').lower();
3643 fdev_major = format(int(0), 'x').lower();
3644 frdev_minor = format(int(0), 'x').lower();
3645 frdev_major = format(int(0), 'x').lower();
3646 if(ftype==5):
3647 fsize = format(int("0"), 'x').lower();
3648 fatime = format(int(member.creationtime.timestamp()), 'x').lower();
3649 fmtime = format(int(member.creationtime.timestamp()), 'x').lower();
3650 fctime = format(int(member.creationtime.timestamp()), 'x').lower();
3651 fbtime = format(int(member.creationtime.timestamp()), 'x').lower();
3652 if(member.is_directory):
3653 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3654 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
3655 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
3656 else:
3657 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3658 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3659 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3660 try:
3661 fuid = format(int(os.getuid()), 'x').lower();
3662 except AttributeError:
3663 fuid = format(int(0), 'x').lower();
3664 except KeyError:
3665 fuid = format(int(0), 'x').lower();
3666 try:
3667 fgid = format(int(os.getgid()), 'x').lower();
3668 except AttributeError:
3669 fgid = format(int(0), 'x').lower();
3670 except KeyError:
3671 fgid = format(int(0), 'x').lower();
3672 try:
3673 import pwd;
3674 try:
3675 userinfo = pwd.getpwuid(os.getuid());
3676 funame = userinfo.pw_name;
3677 except KeyError:
3678 funame = "";
3679 except AttributeError:
3680 funame = "";
3681 except ImportError:
3682 funame = "";
3683 fgname = "";
3684 try:
3685 import grp;
3686 try:
3687 groupinfo = grp.getgrgid(os.getgid());
3688 fgname = groupinfo.gr_name;
3689 except KeyError:
3690 fgname = "";
3691 except AttributeError:
3692 fgname = "";
3693 except ImportError:
3694 fgname = "";
3695 fcontents = BytesIO();
3696 if(ftype==0):
3697 fcontents.write(file_content[member.filename].read());
3698 fsize = format(fcontents.tell(), 'x').lower();
3699 file_content[member.filename].close();
3700 if(not compresswholefile):
3701 fcontents.seek(0, 2);
3702 ucfsize = fcontents.tell();
3703 fcontents.seek(0, 0);
3704 if(compression=="auto"):
3705 ilsize = len(compressionlistalt);
3706 ilmin = 0;
3707 ilcsize = [];
3708 while(ilmin < ilsize):
3709 cfcontents = BytesIO();
3710 shutil.copyfileobj(fcontents, cfcontents);
3711 fcontents.seek(0, 0);
3712 cfcontents.seek(0, 0);
3713 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3714 if(cfcontents):
3715 cfcontents.seek(0, 2);
3716 ilcsize.append(cfcontents.tell());
3717 cfcontents.close();
3718 else:
3719 try:
3720 ilcsize.append(sys.maxint);
3721 except AttributeError:
3722 ilcsize.append(sys.maxsize);
3723 ilmin = ilmin + 1;
3724 ilcmin = ilcsize.index(min(ilcsize));
3725 compression = compressionlistalt[ilcmin];
3726 fcontents.seek(0, 0);
3727 cfcontents = BytesIO();
3728 shutil.copyfileobj(fcontents, cfcontents);
3729 cfcontents.seek(0, 0);
3730 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3731 cfcontents.seek(0, 2);
3732 cfsize = cfcontents.tell();
3733 if(ucfsize > cfsize):
3734 fcsize = format(int(cfsize), 'x').lower();
3735 fcompression = compression;
3736 fcontents.close();
3737 fcontents = cfcontents;
3738 if(fcompression=="none"):
3739 fcompression = "";
3740 fcontents.seek(0, 0);
3741 ftypehex = format(ftype, 'x').lower();
3742 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3743 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3744 fcontents.close();
3745 if(numfiles>0):
3746 catfp.write(AppendNullBytes([0, 0], formatspecs[5]).encode("UTF-8"));
3747 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3748 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
3749 try:
3750 catfp.flush();
3751 os.fsync(catfp.fileno());
3752 except io.UnsupportedOperation:
3753 pass;
3754 except AttributeError:
3755 pass;
3756 except OSError as e:
3757 pass;
3758 if(outfile=="-"):
3759 catfp.seek(0, 0);
3760 if(hasattr(sys.stdout, "buffer")):
3761 shutil.copyfileobj(catfp, sys.stdout.buffer);
3762 else:
3763 shutil.copyfileobj(catfp, sys.stdout);
3764 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3765 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3766 catfp.seek(0, 0);
3767 upload_file_to_internet_file(catfp, outfile);
3768 if(returnfp):
3769 catfp.seek(0, 0)
3770 return catfp
3771 else:
3772 catfp.close()
3773 return True;
3774
3775 create_alias_function("Pack", __file_format_name__, "FromSevenZipFile", PackArchiveFileFromSevenZipFile);
3776
3777 def PackArchiveFileFromInFile(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
3778 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
3779 if(verbose):
3780 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3781 if(checkcompressfile=="tarfile"):
3782 return PackArchiveFileFromTarFile(infile, outfile, compression, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3783 elif(checkcompressfile=="zipfile"):
3784 return PackArchiveFileFromZipFile(infile, outfile, compression, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3785 elif(checkcompressfile=="catfile"):
3786 return RePackArchiveFile(infile, outfile, compression, compressionlevel, False, 0, 0, checksumtype, False, extradata, formatspecs, verbose, returnfp);
3787 elif(rarfile_support and checkcompressfile=="rarfile"):
3788 return PackArchiveFileFromRarFile(infile, outfile, compression, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3789 elif(py7zr_support and checkcompressfile=="7zipfile"):
3790 return PackArchiveFileFromSevenZipFile(infile, outfile, compression, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3791 else:
3792 return False;
3793 return False;
3794
3795 create_alias_function("Pack", __file_format_name__, "FromInFile", PackArchiveFileFromInFile);
3796
3797 def ArchiveFileSeekToFileNum(infile, seekto=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
3798 if(hasattr(infile, "read") or hasattr(infile, "write")):
3799 catfp = infile;
3800 catfp.seek(0, 0);
3801 catfp = UncompressArchiveFile(catfp, formatspecs);
3802 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
3803 if(checkcompressfile=="tarfile"):
3804 return TarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3805 if(checkcompressfile=="zipfile"):
3806 return ZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3807 if(rarfile_support and checkcompressfile=="rarfile"):
3808 return RarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3809 if(py7zr_support and checkcompressfile=="7zipfile"):
3810 return SevenZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3811 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
3812 return False;
3813 if(not catfp):
3814 return False;
3815 catfp.seek(0, 0);
3816 elif(infile=="-"):
3817 catfp = BytesIO();
3818 if(hasattr(sys.stdin, "buffer")):
3819 shutil.copyfileobj(sys.stdin.buffer, catfp);
3820 else:
3821 shutil.copyfileobj(sys.stdin, catfp);
3822 catfp.seek(0, 0);
3823 catfp = UncompressArchiveFile(catfp, formatspecs);
3824 if(not catfp):
3825 return False;
3826 catfp.seek(0, 0);
3827 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3828 catfp = download_file_from_internet_file(infile);
3829 catfp.seek(0, 0);
3830 catfp = UncompressArchiveFile(catfp, formatspecs);
3831 if(not catfp):
3832 return False;
3833 catfp.seek(0, 0);
3834 else:
3835 infile = RemoveWindowsPath(infile);
3836 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
3837 if(checkcompressfile=="tarfile"):
3838 return TarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3839 if(checkcompressfile=="zipfile"):
3840 return ZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3841 if(rarfile_support and checkcompressfile=="rarfile"):
3842 return RarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3843 if(py7zr_support and checkcompressfile=="7zipfile"):
3844 return SevenZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3845 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
3846 return False;
3847 compresscheck = CheckCompressionType(infile, formatspecs, True);
3848 if(not compresscheck):
3849 fextname = os.path.splitext(infile)[1];
3850 if(fextname==".gz"):
3851 compresscheck = "gzip";
3852 elif(fextname==".bz2"):
3853 compresscheck = "bzip2";
3854 elif(fextname==".zst"):
3855 compresscheck = "zstd";
3856 elif(fextname==".lz4" or fextname==".clz4"):
3857 compresscheck = "lz4";
3858 elif(fextname==".lzo" or fextname==".lzop"):
3859 compresscheck = "lzo";
3860 elif(fextname==".lzma" or fextname==".xz"):
3861 compresscheck = "lzma";
3862 else:
3863 return False;
3864 if(not compresscheck):
3865 return False;
3866 catfp = UncompressFile(infile, formatspecs, "rb");
3867 '''
3868 try:
3869 catfp.seek(0, 2);
3870 except OSError:
3871 SeekToEndOfFile(catfp);
3872 except ValueError:
3873 SeekToEndOfFile(catfp);
3874 CatSize = catfp.tell();
3875 CatSizeEnd = CatSize;
3876 '''
3877 try:
3878 catfp.seek(0, 0);
3879 except OSError:
3880 return False;
3881 except ValueError:
3882 return False;
3883 curloc = catfp.tell();
3884 if(curloc>0):
3885 catfp.seek(0, 0);
3886 catheader = ReadFileHeaderData(catfp, 4, formatspecs[5]);
3887 if(curloc>0):
3888 catfp.seek(curloc, 0);
3889 catstring = catheader[0];
3890 catversion = re.findall(r"([\d]+)$", catstring);
3891 fprenumfiles = catheader[1];
3892 fnumfiles = int(fprenumfiles, 16);
3893 fprechecksumtype = catheader[2];
3894 fprechecksum = catheader[3];
3895 fileheader = AppendNullByte(catstring, formatspecs[5]);
3896 fnumfileshex = format(int(fnumfiles), 'x').lower();
3897 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs[5]);
3898 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
3899 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
3900 fheadtell = len(fileheader);
3901 if(fprechecksum!=catfileheadercshex and not skipchecksum):
3902 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
3903 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
3904 return False;
3905 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
3906 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
3907 if(seekto>=fnumfiles):
3908 seekto = fnumfiles - 1;
3909 if(seekto<0):
3910 seekto = 0;
3911 if(seekto>=0):
3912 il = -1;
3913 while(il < seekto):
3914 prefhstart = catfp.tell();
3915 if(formatspecs[7]):
3916 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs[5]);
3917 else:
3918 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs[5]);
3919 if(len(preheaderdata)==0):
3920 break;
3921 prefheadsize = int(preheaderdata[0], 16);
3922 prefnumfields = int(preheaderdata[1], 16);
3923 preftype = int(preheaderdata[2], 16);
3924 if(re.findall("^[.|/]", preheaderdata[3])):
3925 prefname = preheaderdata[3];
3926 else:
3927 prefname = "./"+preheaderdata[3];
3928 prefbasedir = os.path.dirname(prefname);
3929 preflinkname = preheaderdata[4];
3930 prefsize = int(preheaderdata[5], 16);
3931 prefatime = int(preheaderdata[6], 16);
3932 prefmtime = int(preheaderdata[7], 16);
3933 prefctime = int(preheaderdata[8], 16);
3934 prefbtime = int(preheaderdata[9], 16);
3935 prefmode = int(preheaderdata[10], 16);
3936 prefchmode = stat.S_IMODE(prefmode);
3937 preftypemod = stat.S_IFMT(prefmode);
3938 prefwinattributes = int(preheaderdata[11], 16);
3939 prefcompression = preheaderdata[12];
3940 prefcsize = int(preheaderdata[13], 16);
3941 prefuid = int(preheaderdata[14], 16);
3942 prefuname = preheaderdata[15];
3943 prefgid = int(preheaderdata[16], 16);
3944 prefgname = preheaderdata[17];
3945 fid = int(preheaderdata[18], 16);
3946 finode = int(preheaderdata[19], 16);
3947 flinkcount = int(preheaderdata[20], 16);
3948 prefdev_minor = int(preheaderdata[21], 16);
3949 prefdev_major = int(preheaderdata[22], 16);
3950 prefrdev_minor = int(preheaderdata[23], 16);
3951 prefrdev_major = int(preheaderdata[24], 16);
3952 prefextrasize = int(preheaderdata[25], 16);
3953 prefextrafields = int(preheaderdata[26], 16);
3954 extrafieldslist = [];
3955 extrastart = 27;
3956 extraend = extrastart + prefextrafields;
3957 extrafieldslist = [];
3958 if(extrastart<extraend):
3959 extrafieldslist.append(preheaderdata[extrastart]);
3960 extrastart = extrastart + 1;
3961 prefchecksumtype = preheaderdata[extrastart].lower();
3962 prefcs = preheaderdata[extrastart + 1].lower();
3963 prefccs = preheaderdata[extrastart + 2].lower();
3964 hc = 0;
3965 hcmax = len(preheaderdata) - 2;
3966 hout = "";
3967 while(hc<hcmax):
3968 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs[5]);
3969 hc = hc + 1;
3970 prenewfcs = GetFileChecksum(hout, preheaderdata[-3].lower(), True, formatspecs);
3971 if(prefcs!=prenewfcs and not skipchecksum):
3972 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
3973 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
3974 return False;
3975 valid_archive = False;
3976 invalid_archive = True;
3977 prefhend = catfp.tell() - 1;
3978 prefcontentstart = catfp.tell();
3979 prefcontents = "";
3980 pyhascontents = False;
3981 if(prefsize>0):
3982 if(prefcompression):
3983 prefcontents = catfp.read(prefsize);
3984 else:
3985 prefcontents = catfp.read(prefcsize);
3986 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
3987 pyhascontents = True;
3988 if(prefccs!=prenewfccs and not skipchecksum):
3989 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
3990 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
3991 return False;
3992 catfp.seek(1, 1);
3993 il = il + 1;
3994 catfp.seek(seekstart, 0);
3995 fileidnum = il;
3996 catfheadsize = int(preheaderdata[0], 16);
3997 catfnumfields = int(preheaderdata[1], 16);
3998 catftype = int(preheaderdata[2], 16);
3999 if(re.findall("^[.|/]", preheaderdata[3])):
4000 catfname = preheaderdata[3];
4001 else:
4002 catfname = "./"+preheaderdata[3];
4003 catflinkname = preheaderdata[4];
4004 catfsize = int(preheaderdata[5], 16);
4005 catfbasedir = os.path.dirname(catfname);
4006 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
4007 if(returnfp):
4008 catlist.update({'catfp': catfp});
4009 else:
4010 catfp.close();
4011 return catlist;
4012
4013 create_alias_function("", __file_format_name__, "SeekToFileNum", ArchiveFileSeekToFileNum);
4014
4015 def ArchiveFileSeekToFileName(infile, seekfile=None, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4016 if(hasattr(infile, "read") or hasattr(infile, "write")):
4017 catfp = infile;
4018 catfp.seek(0, 0);
4019 catfp = UncompressArchiveFile(catfp, formatspecs);
4020 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4021 if(checkcompressfile=="tarfile"):
4022 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4023 if(checkcompressfile=="zipfile"):
4024 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4025 if(rarfile_support and checkcompressfile=="rarfile"):
4026 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4027 if(py7zr_support and checkcompressfile=="7zipfile"):
4028 return SevenZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4029 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
4030 return False;
4031 if(not catfp):
4032 return False;
4033 catfp.seek(0, 0);
4034 elif(infile=="-"):
4035 catfp = BytesIO();
4036 if(hasattr(sys.stdin, "buffer")):
4037 shutil.copyfileobj(sys.stdin.buffer, catfp);
4038 else:
4039 shutil.copyfileobj(sys.stdin, catfp);
4040 catfp.seek(0, 0);
4041 catfp = UncompressArchiveFile(catfp, formatspecs);
4042 if(not catfp):
4043 return False;
4044 catfp.seek(0, 0);
4045 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4046 catfp = download_file_from_internet_file(infile);
4047 catfp = UncompressArchiveFile(catfp, formatspecs);
4048 catfp.seek(0, 0);
4049 if(not catfp):
4050 return False;
4051 catfp.seek(0, 0);
4052 else:
4053 infile = RemoveWindowsPath(infile);
4054 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4055 if(checkcompressfile=="tarfile"):
4056 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4057 if(checkcompressfile=="zipfile"):
4058 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4059 if(rarfile_support and checkcompressfile=="rarfile"):
4060 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4061 if(py7zr_support and checkcompressfile=="7zipfile"):
4062 return SevenZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4063 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
4064 return False;
4065 compresscheck = CheckCompressionType(infile, formatspecs, True);
4066 if(not compresscheck):
4067 fextname = os.path.splitext(infile)[1];
4068 if(fextname==".gz"):
4069 compresscheck = "gzip";
4070 elif(fextname==".bz2"):
4071 compresscheck = "bzip2";
4072 elif(fextname==".zst"):
4073 compresscheck = "zstd";
4074 elif(fextname==".lz4" or fextname==".clz4"):
4075 compresscheck = "lz4";
4076 elif(fextname==".lzo" or fextname==".lzop"):
4077 compresscheck = "lzo";
4078 elif(fextname==".lzma" or fextname==".xz"):
4079 compresscheck = "lzma";
4080 else:
4081 return False;
4082 if(not compresscheck):
4083 return False;
4084 catfp = UncompressFile(infile, formatspecs, "rb");
4085 '''
4086 try:
4087 catfp.seek(0, 2);
4088 except OSError:
4089 SeekToEndOfFile(catfp);
4090 except ValueError:
4091 SeekToEndOfFile(catfp);
4092 CatSize = catfp.tell();
4093 CatSizeEnd = CatSize;
4094 '''
4095 try:
4096 catfp.seek(0, 0);
4097 except OSError:
4098 return False;
4099 except ValueError:
4100 return False;
4101 curloc = catfp.tell();
4102 if(curloc>0):
4103 catfp.seek(0, 0);
4104 catheader = ReadFileHeaderData(catfp, 4, formatspecs[5]);
4105 if(curloc>0):
4106 catfp.seek(curloc, 0);
4107 catstring = catheader[0];
4108 catversion = re.findall(r"([\d]+)$", catstring);
4109 fprenumfiles = catheader[1];
4110 fnumfiles = int(fprenumfiles, 16);
4111 fprechecksumtype = catheader[2];
4112 fprechecksum = catheader[3];
4113 fileheader = AppendNullByte(catstring, formatspecs[5]);
4114 fnumfileshex = format(int(fnumfiles), 'x').lower();
4115 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs[5]);
4116 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4117 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
4118 fheadtell = len(fileheader);
4119 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4120 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
4121 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4122 return False;
4123 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
4124 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4125 seekto = fnumfiles - 1
4126 filefound = False;
4127 if(seekto>=0):
4128 il = -1;
4129 while(il < seekto):
4130 prefhstart = catfp.tell();
4131 if(formatspecs[7]):
4132 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs[5]);
4133 else:
4134 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs[5]);
4135 if(len(preheaderdata)==0):
4136 break;
4137 prefheadsize = int(preheaderdata[0], 16);
4138 prefnumfields = int(preheaderdata[1], 16);
4139 preftype = int(preheaderdata[2], 16);
4140 if(re.findall("^[.|/]", preheaderdata[3])):
4141 prefname = preheaderdata[3];
4142 else:
4143 prefname = "./"+preheaderdata[3];
4144 prefbasedir = os.path.dirname(prefname);
4145 preflinkname = preheaderdata[4];
4146 prefsize = int(preheaderdata[5], 16);
4147 prefatime = int(preheaderdata[6], 16);
4148 prefmtime = int(preheaderdata[7], 16);
4149 prefctime = int(preheaderdata[8], 16);
4150 prefbtime = int(preheaderdata[9], 16);
4151 prefmode = int(preheaderdata[10], 16);
4152 prefchmode = stat.S_IMODE(prefmode);
4153 preftypemod = stat.S_IFMT(prefmode);
4154 prefwinattributes = int(preheaderdata[11], 16);
4155 prefcompression = preheaderdata[12];
4156 prefcsize = int(preheaderdata[13], 16);
4157 prefuid = int(preheaderdata[14], 16);
4158 prefuname = preheaderdata[15];
4159 prefgid = int(preheaderdata[16], 16);
4160 prefgname = preheaderdata[17];
4161 fid = int(preheaderdata[18], 16);
4162 finode = int(preheaderdata[19], 16);
4163 flinkcount = int(preheaderdata[20], 16);
4164 prefdev_minor = int(preheaderdata[21], 16);
4165 prefdev_major = int(preheaderdata[22], 16);
4166 prefrdev_minor = int(preheaderdata[23], 16);
4167 prefrdev_major = int(preheaderdata[24], 16);
4168 prefextrasize = int(preheaderdata[25], 16);
4169 prefextrafields = int(preheaderdata[26], 16);
4170 extrafieldslist = [];
4171 extrastart = 27;
4172 extraend = extrastart + prefextrafields;
4173 extrafieldslist = [];
4174 if(extrastart<extraend):
4175 extrafieldslist.append(preheaderdata[extrastart]);
4176 extrastart = extrastart + 1;
4177 prefchecksumtype = preheaderdata[extrastart].lower();
4178 prefcs = preheaderdata[extrastart + 1].lower();
4179 prefccs = preheaderdata[extrastart + 2].lower();
4180 hc = 0;
4181 hcmax = len(preheaderdata) - 2;
4182 hout = "";
4183 while(hc<hcmax):
4184 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs[5]);
4185 hc = hc + 1;
4186 prenewfcs = GetFileChecksum(hout, preheaderdata[-3].lower(), True, formatspecs);
4187 if(prefcs!=prenewfcs and not skipchecksum):
4188 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4189 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4190 return False;
4191 valid_archive = False;
4192 invalid_archive = True;
4193 prefhend = catfp.tell() - 1;
4194 prefcontentstart = catfp.tell();
4195 prefcontents = "";
4196 pyhascontents = False;
4197 if(prefsize>0):
4198 if(prefcompression):
4199 prefcontents = catfp.read(prefsize);
4200 else:
4201 prefcontents = catfp.read(prefcsize);
4202 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4203 pyhascontents = True;
4204 if(prefccs!=prenewfccs and not skipchecksum):
4205 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4206 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4207 return False;
4208 catfp.seek(1, 1);
4209 il = il + 1;
4210 filefound = False;
4211 prefname = preheaderdata[2];
4212 if(re.findall("^[.|/]", preheaderdata[2])):
4213 prefname = preheaderdata[2];
4214 else:
4215 prefname = "./"+preheaderdata[2];
4216 if(prefname==seekfile):
4217 filefound = True;
4218 break;
4219 catfp.seek(seekstart, 0);
4220 fileidnum = il;
4221 catfheadsize = int(preheaderdata[0], 16);
4222 catfnumfields = int(preheaderdata[1], 16);
4223 catftype = int(preheaderdata[2], 16);
4224 if(re.findall("^[.|/]", preheaderdata[3])):
4225 catfname = preheaderdata[3];
4226 else:
4227 catfname = "./"+preheaderdata[3];
4228 catflinkname = preheaderdata[4];
4229 catfsize = int(preheaderdata[5], 16);
4230 catfbasedir = os.path.dirname(catfname);
4231 if(filefound):
4232 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
4233 else:
4234 return False;
4235 if(returnfp):
4236 catlist.update({'catfp': catfp});
4237 else:
4238 catfp.close();
4239 return catlist;
4240
4241 create_alias_function("", __file_format_name__, "SeekToFileName", ArchiveFileSeekToFileName);
4242
4243 def ArchiveFileValidate(infile, formatspecs=__file_format_list__, verbose=False, returnfp=False):
4244 if(verbose):
4245 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
4246 if(hasattr(infile, "read") or hasattr(infile, "write")):
4247 catfp = infile;
4248 catfp.seek(0, 0);
4249 catfp = UncompressArchiveFile(catfp, formatspecs);
4250 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4251 if(checkcompressfile=="tarfile"):
4252 return TarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4253 if(checkcompressfile=="zipfile"):
4254 return ZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4255 if(rarfile_support and checkcompressfile=="rarfile"):
4256 return RarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4257 if(py7zr_support and checkcompressfile=="7zipfile"):
4258 return SevenZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4259 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
4260 return False;
4261 if(not catfp):
4262 return False;
4263 catfp.seek(0, 0);
4264 elif(infile=="-"):
4265 catfp = BytesIO();
4266 if(hasattr(sys.stdin, "buffer")):
4267 shutil.copyfileobj(sys.stdin.buffer, catfp);
4268 else:
4269 shutil.copyfileobj(sys.stdin, catfp);
4270 catfp.seek(0, 0);
4271 catfp = UncompressArchiveFile(catfp, formatspecs);
4272 if(not catfp):
4273 return False;
4274 catfp.seek(0, 0);
4275 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4276 catfp = download_file_from_internet_file(infile);
4277 catfp = UncompressArchiveFile(catfp, formatspecs);
4278 catfp.seek(0, 0);
4279 if(not catfp):
4280 return False;
4281 catfp.seek(0, 0);
4282 else:
4283 infile = RemoveWindowsPath(infile);
4284 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4285 if(checkcompressfile=="tarfile"):
4286 return TarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4287 if(checkcompressfile=="zipfile"):
4288 return ZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4289 if(rarfile_support and checkcompressfile=="rarfile"):
4290 return RarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4291 if(py7zr_support and checkcompressfile=="7zipfile"):
4292 return SevenZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4293 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
4294 return False;
4295 compresscheck = CheckCompressionType(infile, formatspecs, True);
4296 if(not compresscheck):
4297 fextname = os.path.splitext(infile)[1];
4298 if(fextname==".gz"):
4299 compresscheck = "gzip";
4300 elif(fextname==".bz2"):
4301 compresscheck = "bzip2";
4302 elif(fextname==".zst"):
4303 compresscheck = "zstd";
4304 elif(fextname==".lz4" or fextname==".clz4"):
4305 compresscheck = "lz4";
4306 elif(fextname==".lzo" or fextname==".lzop"):
4307 compresscheck = "lzo";
4308 elif(fextname==".lzma" or fextname==".xz"):
4309 compresscheck = "lzma";
4310 else:
4311 return False;
4312 if(not compresscheck):
4313 return False;
4314 catfp = UncompressFile(infile, formatspecs, "rb");
4315 '''
4316 try:
4317 catfp.seek(0, 2);
4318 except OSError:
4319 SeekToEndOfFile(catfp);
4320 except ValueError:
4321 SeekToEndOfFile(catfp);
4322 CatSize = catfp.tell();
4323 CatSizeEnd = CatSize;
4324 '''
4325 try:
4326 catfp.seek(0, 0);
4327 except OSError:
4328 return False;
4329 except ValueError:
4330 return False;
4331 curloc = catfp.tell();
4332 if(curloc>0):
4333 catfp.seek(0, 0);
4334 catheader = ReadFileHeaderData(catfp, 4, formatspecs[5]);
4335 if(curloc>0):
4336 catfp.seek(curloc, 0);
4337 catstring = catheader[0];
4338 catversion = re.findall(r"([\d]+)$", catstring);
4339 fprenumfiles = catheader[1];
4340 fnumfiles = int(fprenumfiles, 16);
4341 fprechecksumtype = catheader[2];
4342 fprechecksum = catheader[3];
4343 il = 0;
4344 fileheader = AppendNullByte(catstring, formatspecs[5]);
4345 fnumfileshex = format(int(fnumfiles), 'x').lower();
4346 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs[5]);
4347 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4348 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
4349 valid_archive = True;
4350 invalid_archive = False;
4351 if(verbose):
4352 VerbosePrintOut(infile);
4353 VerbosePrintOut("Number of Records " + str(fnumfiles));
4354 if(fprechecksum==catfileheadercshex):
4355 if(verbose):
4356 VerbosePrintOut("File Header Checksum Passed at offset " + str(0));
4357 VerbosePrintOut("'" + str(fprechecksum) + "' == " + "'" + str(catfileheadercshex) + "'");
4358 else:
4359 if(verbose):
4360 VerbosePrintOut("File Header Checksum Failed at offset " + str(0));
4361 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4362 valid_archive = False;
4363 invalid_archive = True;
4364 if(verbose):
4365 VerbosePrintOut("");
4366 while(il<fnumfiles):
4367 catfhstart = catfp.tell();
4368 if(formatspecs[7]):
4369 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs[5]);
4370 else:
4371 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs[5]);
4372 if(len(catheaderdata)==0):
4373 break;
4374 catfheadsize = int(catheaderdata[0], 16);
4375 catfnumfields = int(catheaderdata[1], 16);
4376 catftype = int(catheaderdata[2], 16);
4377 if(re.findall("^[.|/]", catheaderdata[3])):
4378 catfname = catheaderdata[3];
4379 else:
4380 catfname = "./"+catheaderdata[3];
4381 catfbasedir = os.path.dirname(catfname);
4382 catflinkname = catheaderdata[4];
4383 catfsize = int(catheaderdata[5], 16);
4384 catfatime = int(catheaderdata[6], 16);
4385 catfmtime = int(catheaderdata[7], 16);
4386 catfctime = int(catheaderdata[8], 16);
4387 catfbtime = int(catheaderdata[9], 16);
4388 catfmode = int(catheaderdata[10], 16);
4389 catfchmode = stat.S_IMODE(catfmode);
4390 catftypemod = stat.S_IFMT(catfmode);
4391 catfwinattributes = int(catheaderdata[11], 16);
4392 catfcompression = catheaderdata[12];
4393 catfcsize = int(catheaderdata[13], 16);
4394 catfuid = int(catheaderdata[14], 16);
4395 catfuname = catheaderdata[15];
4396 catfgid = int(catheaderdata[16], 16);
4397 catfgname = catheaderdata[17];
4398 fid = int(catheaderdata[18], 16);
4399 finode = int(catheaderdata[19], 16);
4400 flinkcount = int(catheaderdata[20], 16);
4401 catfdev_minor = int(catheaderdata[21], 16);
4402 catfdev_major = int(catheaderdata[22], 16);
4403 catfrdev_minor = int(catheaderdata[23], 16);
4404 catfrdev_major = int(catheaderdata[24], 16);
4405 catfextrasize = int(catheaderdata[25], 16);
4406 catfextrafields = int(catheaderdata[26], 16);
4407 extrafieldslist = [];
4408 extrastart = 27;
4409 extraend = extrastart + catfextrafields;
4410 extrafieldslist = [];
4411 if(extrastart<extraend):
4412 extrafieldslist.append(catheaderdata[extrastart]);
4413 extrastart = extrastart + 1;
4414 catfchecksumtype = catheaderdata[extrastart].lower();
4415 catfcs = catheaderdata[extrastart + 1].lower();
4416 catfccs = catheaderdata[extrastart + 2].lower();
4417 hc = 0;
4418 hcmax = len(catheaderdata) - 2;
4419 hout = "";
4420 while(hc<hcmax):
4421 hout = hout + AppendNullByte(catheaderdata[hc], formatspecs[5]);
4422 hc = hc + 1;
4423 catnewfcs = GetFileChecksum(hout, catheaderdata[-3].lower(), True, formatspecs);
4424 if(verbose):
4425 VerbosePrintOut(catfname);
4426 VerbosePrintOut("Record Number " + str(il) + "; File ID " + str(fid) + "; iNode Number " + str(finode));
4427 if(catfcs==catnewfcs):
4428 if(verbose):
4429 VerbosePrintOut("File Header Checksum Passed at offset " + str(catfhstart));
4430 VerbosePrintOut("'" + str(catfcs) + "' == " + "'" + str(catnewfcs) + "'");
4431 else:
4432 if(verbose):
4433 VerbosePrintOut("File Header Checksum Failed at offset " + str(catfhstart));
4434 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
4435 valid_archive = False;
4436 invalid_archive = True;
4437 catfhend = catfp.tell() - 1;
4438 catfcontentstart = catfp.tell();
4439 catfcontents = "";
4440 pyhascontents = False;
4441 if(catfsize>0):
4442 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4443 catfcontents = catfp.read(catfsize);
4444 else:
4445 catfcontents = catfp.read(catfcsize);
4446 catnewfccs = GetFileChecksum(catfcontents, catheaderdata[-3].lower(), False, formatspecs);
4447 pyhascontents = True;
4448 if(catfccs==catnewfccs):
4449 if(verbose):
4450 VerbosePrintOut("File Content Checksum Passed at offset " + str(catfcontentstart));
4451 VerbosePrintOut("'" + str(catfccs) + "' == " + "'" + str(catnewfccs) + "'");
4452 else:
4453 if(verbose):
4454 VerbosePrintOut("File Content Checksum Failed at offset " + str(catfcontentstart));
4455 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
4456 valid_archive = False;
4457 invalid_archive = True;
4458 if(verbose):
4459 VerbosePrintOut("");
4460 catfp.seek(1, 1);
4461 il = il + 1;
4462 if(valid_archive):
4463 if(returnfp):
4464 return catfp;
4465 else:
4466 catfp.close();
4467 return True;
4468 else:
4469 catfp.close();
4470 return False;
4471
4472 create_alias_function("", __file_format_name__, "Validate", ArchiveFileValidate);
4473
4474 def ArchiveFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4475 if(hasattr(infile, "read") or hasattr(infile, "write")):
4476 catfp = infile;
4477 catfp.seek(0, 0);
4478 catfp = UncompressArchiveFile(catfp, formatspecs);
4479 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4480 if(checkcompressfile=="tarfile"):
4481 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4482 if(checkcompressfile=="zipfile"):
4483 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4484 if(rarfile_support and checkcompressfile=="rarfile"):
4485 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4486 if(py7zr_support and checkcompressfile=="7zipfile"):
4487 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4488 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
4489 return False;
4490 if(not catfp):
4491 return False;
4492 catfp.seek(0, 0);
4493 elif(infile=="-"):
4494 catfp = BytesIO();
4495 if(hasattr(sys.stdin, "buffer")):
4496 shutil.copyfileobj(sys.stdin.buffer, catfp);
4497 else:
4498 shutil.copyfileobj(sys.stdin, catfp);
4499 catfp.seek(0, 0);
4500 catfp = UncompressArchiveFile(catfp, formatspecs);
4501 if(not catfp):
4502 return False;
4503 catfp.seek(0, 0);
4504 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4505 catfp = download_file_from_internet_file(infile);
4506 catfp = UncompressArchiveFile(catfp, formatspecs);
4507 catfp.seek(0, 0);
4508 if(not catfp):
4509 return False;
4510 catfp.seek(0, 0);
4511 else:
4512 infile = RemoveWindowsPath(infile);
4513 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4514 if(checkcompressfile=="tarfile"):
4515 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4516 if(checkcompressfile=="zipfile"):
4517 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4518 if(rarfile_support and checkcompressfile=="rarfile"):
4519 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4520 if(py7zr_support and checkcompressfile=="7zipfile"):
4521 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4522 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
4523 return False;
4524 compresscheck = CheckCompressionType(infile, formatspecs, True);
4525 if(not compresscheck):
4526 fextname = os.path.splitext(infile)[1];
4527 if(fextname==".gz"):
4528 compresscheck = "gzip";
4529 elif(fextname==".bz2"):
4530 compresscheck = "bzip2";
4531 elif(fextname==".zst"):
4532 compresscheck = "zstd";
4533 elif(fextname==".lz4" or fextname==".clz4"):
4534 compresscheck = "lz4";
4535 elif(fextname==".lzo" or fextname==".lzop"):
4536 compresscheck = "lzo";
4537 elif(fextname==".lzma" or fextname==".xz"):
4538 compresscheck = "lzma";
4539 else:
4540 return False;
4541 if(not compresscheck):
4542 return False;
4543 catfp = UncompressFile(infile, formatspecs, "rb");
4544 '''
4545 try:
4546 catfp.seek(0, 2);
4547 except OSError:
4548 SeekToEndOfFile(catfp);
4549 except ValueError:
4550 SeekToEndOfFile(catfp);
4551 CatSize = catfp.tell();
4552 CatSizeEnd = CatSize;
4553 '''
4554 try:
4555 catfp.seek(0, 0);
4556 except OSError:
4557 return False;
4558 except ValueError:
4559 return False;
4560 curloc = catfp.tell();
4561 if(curloc>0):
4562 catfp.seek(0, 0);
4563 catheader = ReadFileHeaderData(catfp, 4, formatspecs[5]);
4564 if(curloc>0):
4565 catfp.seek(curloc, 0);
4566 catstring = catheader[0];
4567 catversion = re.findall(r"([\d]+)$", catstring);
4568 fprenumfiles = catheader[1];
4569 fnumfiles = int(fprenumfiles, 16);
4570 fprechecksumtype = catheader[2];
4571 fprechecksum = catheader[3];
4572 fileheader = AppendNullByte(catstring, formatspecs[5]);
4573 fnumfileshex = format(int(fnumfiles), 'x').lower();
4574 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs[5]);
4575 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4576 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
4577 fheadtell = len(fileheader);
4578 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4579 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
4580 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4581 return False;
4582 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
4583 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4584 if(seekstart<0 and seekstart>fnumfiles):
4585 seekstart = 0;
4586 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
4587 seekend = fnumfiles;
4588 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
4589 seekend = fnumfiles - abs(seekend);
4590 if(seekstart>0):
4591 il = 0;
4592 while(il < seekstart):
4593 prefhstart = catfp.tell();
4594 if(formatspecs[7]):
4595 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs[5]);
4596 else:
4597 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs[5]);
4598 if(len(preheaderdata)==0):
4599 break;
4600 prefheadsize = int(preheaderdata[0], 16);
4601 prefnumfields = int(preheaderdata[1], 16);
4602 if(re.findall("^[.|/]", preheaderdata[3])):
4603 prefname = preheaderdata[3];
4604 else:
4605 prefname = "./"+preheaderdata[3];
4606 prefsize = int(preheaderdata[5], 16);
4607 prefcompression = preheaderdata[12];
4608 prefcsize = int(preheaderdata[13], 16);
4609 prefextrasize = int(preheaderdata[24], 16);
4610 prefextrafields = int(preheaderdata[25], 16);
4611 extrafieldslist = [];
4612 extrastart = 26;
4613 extraend = extrastart + prefextrafields;
4614 extrafieldslist = [];
4615 if(extrastart<extraend):
4616 extrafieldslist.append(preheaderdata[extrastart]);
4617 extrastart = extrastart + 1;
4618 prefchecksumtype = preheaderdata[extrastart].lower();
4619 prefcs = preheaderdata[extrastart + 1].lower();
4620 prefccs = preheaderdata[extrastart + 2].lower();
4621 hc = 0;
4622 hcmax = len(preheaderdata) - 2;
4623 hout = "";
4624 while(hc<hcmax):
4625 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs[5]);
4626 hc = hc + 1;
4627 prenewfcs = GetFileChecksum(hout, preheaderdata[-3].lower(), True, formatspecs);
4628 if(prefcs!=prenewfcs and not skipchecksum):
4629 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4630 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4631 return False;
4632 valid_archive = False;
4633 invalid_archive = True;
4634 prefhend = catfp.tell() - 1;
4635 prefcontentstart = catfp.tell();
4636 prefcontents = "";
4637 pyhascontents = False;
4638 if(prefsize>0):
4639 if(prefcompression=="none" or prefcompression=="" or prefcompression=="auto"):
4640 prefcontents = catfp.read(prefsize);
4641 else:
4642 prefcontents = catfp.read(prefcsize);
4643 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4644 pyhascontents = True;
4645 if(prefccs!=prenewfccs and not skipchecksum):
4646 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4647 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4648 return False;
4649 catfp.seek(1, 1);
4650 il = il + 1;
4651 fileidnum = seekstart;
4652 realidnum = 0;
4653 while(fileidnum<seekend):
4654 catfhstart = catfp.tell();
4655 if(formatspecs[7]):
4656 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs[5]);
4657 else:
4658 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs[5]);
4659 if(len(catheaderdata)==0):
4660 break;
4661 catfheadsize = int(catheaderdata[0], 16);
4662 catfnumfields = int(catheaderdata[1], 16);
4663 catftype = int(catheaderdata[2], 16);
4664 if(re.findall("^[.|/]", catheaderdata[3])):
4665 catfname = catheaderdata[3];
4666 else:
4667 catfname = "./"+catheaderdata[3];
4668 catfbasedir = os.path.dirname(catfname);
4669 catflinkname = catheaderdata[4];
4670 catfsize = int(catheaderdata[5], 16);
4671 catfatime = int(catheaderdata[6], 16);
4672 catfmtime = int(catheaderdata[7], 16);
4673 catfctime = int(catheaderdata[8], 16);
4674 catfbtime = int(catheaderdata[9], 16);
4675 catfmode = int(catheaderdata[10], 16);
4676 catfchmode = stat.S_IMODE(catfmode);
4677 catftypemod = stat.S_IFMT(catfmode);
4678 catfwinattributes = int(catheaderdata[11], 16);
4679 catfcompression = catheaderdata[12];
4680 catfcsize = int(catheaderdata[13], 16);
4681 catfuid = int(catheaderdata[14], 16);
4682 catfuname = catheaderdata[15];
4683 catfgid = int(catheaderdata[16], 16);
4684 catfgname = catheaderdata[17];
4685 catfid = int(catheaderdata[18], 16);
4686 catfinode = int(catheaderdata[19], 16);
4687 catflinkcount = int(catheaderdata[20], 16);
4688 catfdev_minor = int(catheaderdata[21], 16);
4689 catfdev_major = int(catheaderdata[22], 16);
4690 catfrdev_minor = int(catheaderdata[23], 16);
4691 catfrdev_major = int(catheaderdata[24], 16);
4692 catfextrasize = int(catheaderdata[25], 16);
4693 catfextrafields = int(catheaderdata[26], 16);
4694 extrafieldslist = [];
4695 extrastart = 27;
4696 extraend = extrastart + catfextrafields;
4697 extrafieldslist = [];
4698 if(extrastart<extraend):
4699 extrafieldslist.append(catheaderdata[extrastart]);
4700 extrastart = extrastart + 1;
4701 catfchecksumtype = catheaderdata[extrastart].lower();
4702 catfcs = catheaderdata[extrastart + 1].lower();
4703 catfccs = catheaderdata[extrastart + 2].lower();
4704 hc = 0;
4705 hcmax = len(catheaderdata) - 2;
4706 hout = "";
4707 while(hc<hcmax):
4708 hout = hout + AppendNullByte(catheaderdata[hc], formatspecs[5]);
4709 hc = hc + 1;
4710 catnewfcs = GetFileChecksum(hout, catheaderdata[-3].lower(), True, formatspecs);
4711 if(catfcs!=catnewfcs and not skipchecksum):
4712 VerbosePrintOut("File Header Checksum Error with file " + catfname + " at offset " + str(catfhstart));
4713 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
4714 return False;
4715 catfhend = catfp.tell() - 1;
4716 catfcontentstart = catfp.tell();
4717 catfcontents = BytesIO();
4718 pyhascontents = False;
4719 if(catfsize>0 and not listonly):
4720 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4721 catfcontents.write(catfp.read(catfsize));
4722 else:
4723 catfcontents.write(catfp.read(catfcsize));
4724 catfcontents.seek(0, 0);
4725 catnewfccs = GetFileChecksum(catfcontents.read(), catheaderdata[-3].lower(), False, formatspecs);
4726 pyhascontents = True;
4727 if(catfccs!=catnewfccs and skipchecksum):
4728 VerbosePrintOut("File Content Checksum Error with file " + catfname + " at offset " + str(catfcontentstart));
4729 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
4730 return False;
4731 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4732 pass;
4733 else:
4734 catfcontents.seek(0, 0);
4735 catfcontents = UncompressArchiveFile(catfcontents, formatspecs);
4736 if(catfsize>0 and listonly):
4737 catfp.seek(catfsize, 1);
4738 pyhascontents = False;
4739 catfp.seek(1, 1);
4740 catfcontentend = catfp.tell() - 1;
4741 catfcontents.seek(0, 0);
4742 catlist['ffilelist'].update({realidnum: {'fid': realidnum, 'fidalt': fileidnum, 'fheadersize': catfheadsize, 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize, 'fatime': catfatime, 'fmtime': catfmtime, 'fctime': catfctime, 'fbtime': catfbtime, 'fmode': catfmode, 'fchmode': catfchmode, 'ftypemod': catftypemod, 'fwinattributes': catfwinattributes, 'fcompression': catfcompression, 'fcsize': catfcsize, 'fuid': catfuid, 'funame': catfuname, 'fgid': catfgid, 'fgname': catfgname, 'finode': catfinode, 'flinkcount': catflinkcount, 'fminor': catfdev_minor, 'fmajor': catfdev_major, 'frminor': catfrdev_minor, 'frmajor': catfrdev_major, 'fchecksumtype': catfchecksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': catfextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': catfcs, 'fcontentchecksum': catfccs, 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': catfcontents} });
4743 fileidnum = fileidnum + 1;
4744 realidnum = realidnum + 1;
4745 if(returnfp):
4746 catlist.update({'catfp': catfp});
4747 else:
4748 catfp.close();
4749 return catlist;
4750
4751 create_alias_function("", __file_format_name__, "ToArray", ArchiveFileToArray);
4752
4753 def ArchiveFileStringToArray(catstr, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4754 catfp = BytesIO(catstr);
4755 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4756 return listcatfiles;
4757
4758 create_alias_function("", __file_format_name__, "StringToArray", ArchiveFileStringToArray);
4759
4760 def TarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4761 catfp = BytesIO();
4762 catfp = PackArchiveFileFromTarFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
4763 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4764 return listcatfiles;
4765
4766 def ZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4767 catfp = BytesIO();
4768 catfp = PackArchiveFileFromZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
4769 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4770 return listcatfiles;
4771
4772 if(not rarfile_support):
4773 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4774 return False;
4775
4776 if(rarfile_support):
4777 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4778 catfp = BytesIO();
4779 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
4780 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4781 return listcatfiles;
4782
4783 if(not py7zr_support):
4784 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4785 return False;
4786
4787 if(py7zr_support):
4788 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4789 catfp = BytesIO();
4790 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
4791 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4792 return listcatfiles;
4793
4794 def InFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4795 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4796 if(checkcompressfile=="tarfile"):
4797 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4798 elif(checkcompressfile=="zipfile"):
4799 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4800 elif(checkcompressfile=="catfile"):
4801 return ArchiveFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4802 elif(rarfile_support and checkcompressfile=="rarfile"):
4803 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4804 elif(py7zr_support and checkcompressfile=="7zipfile"):
4805 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4806 else:
4807 return False;
4808 return False;
4809
4810 def ListDirToArrayAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
4811 catver = formatspecs[6];
4812 fileheaderver = str(int(catver.replace(".", "")));
4813 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
4814 advancedlist = formatspecs[8];
4815 altinode = formatspecs[9];
4816 infilelist = [];
4817 if(infiles=="-"):
4818 for line in sys.stdin:
4819 infilelist.append(line.strip());
4820 infilelist = list(filter(None, infilelist));
4821 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
4822 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
4823 return False;
4824 with UncompressFile(infiles, formatspecs, "r") as finfile:
4825 for line in finfile:
4826 infilelist.append(line.strip());
4827 infilelist = list(filter(None, infilelist));
4828 else:
4829 if(isinstance(infiles, (list, tuple, ))):
4830 infilelist = list(filter(None, infiles));
4831 elif(isinstance(infiles, (str, ))):
4832 infilelist = list(filter(None, [infiles]));
4833 if(advancedlist):
4834 GetDirList = ListDirAdvanced(infilelist, followlink, False);
4835 else:
4836 GetDirList = ListDir(infilelist, followlink, False);
4837 if(not GetDirList):
4838 return False;
4839 curinode = 0;
4840 curfid = 0;
4841 inodelist = [];
4842 inodetofile = {};
4843 filetoinode = {};
4844 inodetocatinode = {};
4845 fileidnum = 0;
4846 fnumfiles = int(len(GetDirList));
4847 catver = formatspecs[6];
4848 fileheaderver = str(int(catver.replace(".", "")));
4849 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
4850 fnumfileshex = format(int(fnumfiles), 'x').lower();
4851 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs[5]);
4852 catversion = re.findall(r"([\d]+)$", fileheader);
4853 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
4854 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4855 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
4856 fheadtell = len(fileheader);
4857 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
4858 for curfname in GetDirList:
4859 if(re.findall("^[.|/]", curfname)):
4860 fname = curfname;
4861 else:
4862 fname = "./"+curfname;
4863 if(verbose):
4864 VerbosePrintOut(fname);
4865 if(not followlink or followlink is None):
4866 fstatinfo = os.lstat(fname);
4867 else:
4868 fstatinfo = os.stat(fname);
4869 fpremode = fstatinfo.st_mode;
4870 finode = fstatinfo.st_ino;
4871 flinkcount = fstatinfo.st_nlink;
4872 ftype = 0;
4873 if(stat.S_ISREG(fpremode)):
4874 ftype = 0;
4875 elif(stat.S_ISLNK(fpremode)):
4876 ftype = 2;
4877 elif(stat.S_ISCHR(fpremode)):
4878 ftype = 3;
4879 elif(stat.S_ISBLK(fpremode)):
4880 ftype = 4;
4881 elif(stat.S_ISDIR(fpremode)):
4882 ftype = 5;
4883 elif(stat.S_ISFIFO(fpremode)):
4884 ftype = 6;
4885 elif(stat.S_ISSOCK(fpremode)):
4886 ftype = 8;
4887 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
4888 ftype = 9;
4889 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
4890 ftype = 10;
4891 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
4892 ftype = 11;
4893 else:
4894 ftype = 0;
4895 flinkname = "";
4896 fbasedir = os.path.dirname(fname);
4897 fcurfid = curfid;
4898 if(not followlink and finode!=0):
4899 if(ftype!=1):
4900 if(finode in inodelist):
4901 ftype = 1;
4902 flinkname = inodetofile[finode];
4903 if(altinode):
4904 fcurinode = finode;
4905 else:
4906 fcurinode = inodetocatinode[finode];
4907 if(finode not in inodelist):
4908 inodelist.append(finode);
4909 inodetofile.update({finode: fname});
4910 inodetocatinode.update({finode: curinode});
4911 if(altinode):
4912 fcurinode = finode;
4913 else:
4914 fcurinode = curinode;
4915 curinode = curinode + 1;
4916 else:
4917 fcurinode = curinode;
4918 curinode = curinode + 1;
4919 curfid = curfid + 1;
4920 if(ftype==2):
4921 flinkname = os.readlink(fname);
4922 fdev = fstatinfo.st_dev;
4923 getfdev = GetDevMajorMinor(fdev);
4924 fdev_minor = getfdev[0];
4925 fdev_major = getfdev[1];
4926 frdev = fstatinfo.st_dev;
4927 if(hasattr(fstatinfo, "st_rdev")):
4928 frdev = fstatinfo.st_rdev;
4929 else:
4930 frdev = fstatinfo.st_dev;
4931 getfrdev = GetDevMajorMinor(frdev);
4932 frdev_minor = getfrdev[0];
4933 frdev_major = getfrdev[1];
4934 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
4935 fsize = "0";
4936 if(ftype==0 or ftype==7):
4937 fsize = fstatinfo.st_size;
4938 fatime = fstatinfo.st_atime;
4939 fmtime = fstatinfo.st_mtime;
4940 fctime = fstatinfo.st_ctime;
4941 if(hasattr(fstatinfo, "st_birthtime")):
4942 fbtime = fstatinfo.st_birthtime;
4943 else:
4944 fbtime = fstatinfo.st_ctime;
4945 fmode = fstatinfo.st_mode;
4946 fchmode = stat.S_IMODE(fstatinfo.st_mode);
4947 ftypemod = stat.S_IFMT(fstatinfo.st_mode);
4948 fuid = fstatinfo.st_uid;
4949 fgid = fstatinfo.st_gid;
4950 funame = "";
4951 try:
4952 import pwd;
4953 try:
4954 userinfo = pwd.getpwuid(fstatinfo.st_uid);
4955 funame = userinfo.pw_name;
4956 except KeyError:
4957 funame = "";
4958 except ImportError:
4959 funame = "";
4960 fgname = "";
4961 try:
4962 import grp;
4963 try:
4964 groupinfo = grp.getgrgid(fstatinfo.st_gid);
4965 fgname = groupinfo.gr_name;
4966 except KeyError:
4967 fgname = "";
4968 except ImportError:
4969 fgname = "";
4970 fdev_minor = fdev_minor;
4971 fdev_major = fdev_major;
4972 frdev_minor = frdev_minor;
4973 frdev_major = frdev_major;
4974 flinkcount = flinkcount;
4975 if(hasattr(fstatinfo, "st_file_attributes")):
4976 fwinattributes = fstatinfo.st_file_attributes;
4977 else:
4978 fwinattributes = 0;
4979 fcompression = "";
4980 fcsize = 0;
4981 fcontents = BytesIO();
4982 if(ftype==0 or ftype==7):
4983 with open(fname, "rb") as fpc:
4984 shutil.copyfileobj(fpc, fcontents);
4985 if(followlink and (ftype==1 or ftype==2)):
4986 flstatinfo = os.stat(flinkname);
4987 with open(flinkname, "rb") as fpc:
4988 shutil.copyfileobj(fpc, fcontents);
4989 fcontents.seek(0, 0);
4990 ftypehex = format(ftype, 'x').lower();
4991 extrafields = len(extradata);
4992 extrafieldslist = extradata;
4993 catfextrafields = extrafields;
4994 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
4995 if(len(extradata)>0):
4996 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
4997 extrasizelen = len(extrasizestr);
4998 extrasizelenhex = format(extrasizelen, 'x').lower();
4999 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5000 catoutlen = len(catoutlist) + len(extradata) + 3;
5001 catoutlenhex = format(catoutlen, 'x').lower();
5002 catoutlist.insert(0, catoutlenhex);
5003 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
5004 if(len(extradata)>0):
5005 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
5006 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
5007 catfnumfields = catoutlen;
5008 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5009 fcontents.seek(0, 0);
5010 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5011 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5012 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5013 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
5014 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5015 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5016 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5017 nullstrecd = formatspecs[5].encode('UTF-8');
5018 fheadtell += len(catfileoutstr) + 1;
5019 catfcontentend = fheadtell - 1;
5020 fcontents.seek(0, 0);
5021 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5022 pyhascontents = False;
5023 if(int(fsize)>0 and not listonly):
5024 pyhascontents = True;
5025 if(int(fsize)>0 and listonly):
5026 fcontents = BytesIO();
5027 pyhascontents = False;
5028 fcontents.seek(0, 0);
5029 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5030 fileidnum = fileidnum + 1;
5031 return catlist;
5032
5033 def TarFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5034 curinode = 0;
5035 curfid = 0;
5036 inodelist = [];
5037 inodetofile = {};
5038 filetoinode = {};
5039 inodetocatinode = {};
5040 fileidnum = 0;
5041 if(infile=="-"):
5042 infile = BytesIO();
5043 if(hasattr(sys.stdin, "buffer")):
5044 shutil.copyfileobj(sys.stdin.buffer, infile);
5045 else:
5046 shutil.copyfileobj(sys.stdin, infile);
5047 infile.seek(0, 0);
5048 if(not infile):
5049 return False;
5050 infile.seek(0, 0);
5051 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
5052 infile = download_file_from_internet_file(infile);
5053 infile.seek(0, 0);
5054 if(not infile):
5055 return False;
5056 infile.seek(0, 0);
5057 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
5058 return False;
5059 elif(os.path.exists(infile) and os.path.isfile(infile)):
5060 try:
5061 if(not tarfile.is_tarfile(infile)):
5062 return False;
5063 except AttributeError:
5064 if(not is_tarfile(infile)):
5065 return False;
5066 else:
5067 return False;
5068 try:
5069 if(hasattr(infile, "read") or hasattr(infile, "write")):
5070 tarfp = tarfile.open(fileobj=infile, mode="r");
5071 else:
5072 tarfp = tarfile.open(infile, "r");
5073 except FileNotFoundError:
5074 return False;
5075 fnumfiles = int(len(tarfp.getmembers()));
5076 catver = formatspecs[6];
5077 fileheaderver = str(int(catver.replace(".", "")));
5078 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
5079 fnumfileshex = format(int(fnumfiles), 'x').lower();
5080 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs[5]);
5081 catversion = re.findall(r"([\d]+)$", fileheader);
5082 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
5083 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
5084 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
5085 fheadtell = len(fileheader);
5086 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5087 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
5088 if(re.findall("^[.|/]", member.name)):
5089 fname = member.name;
5090 else:
5091 fname = "./"+member.name;
5092 if(verbose):
5093 VerbosePrintOut(fname);
5094 fpremode = member.mode;
5095 ffullmode = member.mode;
5096 flinkcount = 0;
5097 ftype = 0;
5098 if(member.isreg()):
5099 ffullmode = member.mode + stat.S_IFREG;
5100 ftype = 0;
5101 elif(member.isdev()):
5102 ffullmode = member.mode;
5103 ftype = 7;
5104 elif(member.islnk()):
5105 ffullmode = member.mode + stat.S_IFREG;
5106 ftype = 1;
5107 elif(member.issym()):
5108 ffullmode = member.mode + stat.S_IFLNK;
5109 ftype = 2;
5110 elif(member.ischr()):
5111 ffullmode = member.mode + stat.S_IFCHR;
5112 ftype = 3;
5113 elif(member.isblk()):
5114 ffullmode = member.mode + stat.S_IFBLK;
5115 ftype = 4;
5116 elif(member.isdir()):
5117 ffullmode = member.mode + stat.S_IFDIR;
5118 ftype = 5;
5119 elif(member.isfifo()):
5120 ffullmode = member.mode + stat.S_IFIFO;
5121 ftype = 6;
5122 elif(member.issparse()):
5123 ffullmode = member.mode;
5124 ftype = 12;
5125 else:
5126 ffullmode = member.mode;
5127 ftype = 0;
5128 flinkname = "";
5129 fbasedir = os.path.dirname(fname);
5130 fcurfid = curfid;
5131 fcurinode = curfid;
5132 finode = fcurinode;
5133 curfid = curfid + 1;
5134 if(ftype==2):
5135 flinkname = member.linkname;
5136 fdev_minor = member.devminor;
5137 fdev_major = member.devmajor;
5138 frdev_minor = member.devminor;
5139 frdev_major = member.devmajor;
5140 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
5141 fsize = "0";
5142 elif(ftype==0 or ftype==7):
5143 fsize = member.size;
5144 else:
5145 fsize = member.size;
5146 fatime = member.mtime;
5147 fmtime = member.mtime;
5148 fctime = member.mtime;
5149 fbtime = member.mtime;
5150 fmode = ffullmode;
5151 fchmode = stat.S_IMODE(ffullmode);
5152 ftypemod = stat.S_IFMT(ffullmode);
5153 fuid = member.uid;
5154 fgid = member.gid;
5155 funame = member.uname;
5156 fgname = member.gname;
5157 flinkcount = flinkcount;
5158 fwinattributes = int(0);
5159 fcompression = "";
5160 fcsize = 0;
5161 fcontents = BytesIO();
5162 if(ftype==0 or ftype==7):
5163 with tarfp.extractfile(member) as fpc:
5164 shutil.copyfileobj(fpc, fcontents);
5165 fcontents.seek(0, 0);
5166 ftypehex = format(ftype, 'x').lower();
5167 extrafields = len(extradata);
5168 extrafieldslist = extradata;
5169 catfextrafields = extrafields;
5170 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
5171 if(len(extradata)>0):
5172 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
5173 extrasizelen = len(extrasizestr);
5174 extrasizelenhex = format(extrasizelen, 'x').lower();
5175 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5176 catoutlen = len(catoutlist) + len(extradata) + 3;
5177 catoutlenhex = format(catoutlen, 'x').lower();
5178 catoutlist.insert(0, catoutlenhex);
5179 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
5180 if(len(extradata)>0):
5181 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
5182 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
5183 catfnumfields = catoutlen;
5184 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5185 fcontents.seek(0, 0);
5186 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5187 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5188 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5189 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
5190 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5191 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5192 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5193 nullstrecd = formatspecs[5].encode('UTF-8');
5194 fheadtell += len(catfileoutstr) + 1;
5195 catfcontentend = fheadtell - 1;
5196 fcontents.seek(0, 0);
5197 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5198 pyhascontents = False;
5199 if(int(fsize)>0 and not listonly):
5200 pyhascontents = True;
5201 if(int(fsize)>0 and listonly):
5202 fcontents = "";
5203 pyhascontents = False;
5204 fcontents.seek(0, 0);
5205 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5206 fileidnum = fileidnum + 1;
5207 return catlist;
5208
5209 def ZipFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5210 curinode = 0;
5211 curfid = 0;
5212 inodelist = [];
5213 inodetofile = {};
5214 filetoinode = {};
5215 inodetocatinode = {};
5216 fileidnum = 0;
5217 if(infile=="-"):
5218 infile = BytesIO();
5219 if(hasattr(sys.stdin, "buffer")):
5220 shutil.copyfileobj(sys.stdin.buffer, infile);
5221 else:
5222 shutil.copyfileobj(sys.stdin, infile);
5223 infile.seek(0, 0);
5224 if(not infile):
5225 return False;
5226 infile.seek(0, 0);
5227 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
5228 infile = download_file_from_internet_file(infile);
5229 infile.seek(0, 0);
5230 if(not infile):
5231 return False;
5232 infile.seek(0, 0);
5233 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
5234 return False;
5235 else:
5236 return False;
5237 if(not zipfile.is_zipfile(infile)):
5238 return False;
5239 try:
5240 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
5241 except FileNotFoundError:
5242 return False;
5243 ziptest = zipfp.testzip();
5244 if(ziptest):
5245 VerbosePrintOut("Bad file found!");
5246 fnumfiles = int(len(zipfp.infolist()));
5247 catver = formatspecs[6];
5248 fileheaderver = str(int(catver.replace(".", "")));
5249 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
5250 catversion = re.findall(r"([\d]+)$", fileheader);
5251 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
5252 fnumfileshex = format(int(fnumfiles), 'x').lower();
5253 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs[5]);
5254 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
5255 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
5256 fheadtell = len(fileheader);
5257 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5258 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
5259 if(re.findall("^[.|/]", member.filename)):
5260 fname = member.filename;
5261 else:
5262 fname = "./"+member.filename;
5263 zipinfo = zipfp.getinfo(member.filename);
5264 if(verbose):
5265 VerbosePrintOut(fname);
5266 if(not member.is_dir()):
5267 fpremode = stat.S_IFREG + 438;
5268 elif(member.is_dir()):
5269 fpremode = stat.S_IFDIR + 511;
5270 flinkcount = 0;
5271 ftype = 0;
5272 if(not member.is_dir()):
5273 ftype = 0;
5274 elif(member.is_dir()):
5275 ftype = 5;
5276 flinkname = "";
5277 fbasedir = os.path.dirname(fname);
5278 fcurfid = curfid;
5279 fcurinode = curfid;
5280 finode = fcurinode;
5281 curfid = curfid + 1;
5282 fdev_minor = 0;
5283 fdev_major = 0;
5284 frdev_minor = 0;
5285 frdev_major = 0;
5286 if(ftype==5):
5287 fsize = "0";
5288 elif(ftype==0):
5289 fsize = member.file_size;
5290 else:
5291 fsize = member.file_size;
5292 fatime = time.mktime(member.date_time + (0, 0, -1));
5293 fmtime = time.mktime(member.date_time + (0, 0, -1));
5294 fctime = time.mktime(member.date_time + (0, 0, -1));
5295 fbtime = time.mktime(member.date_time + (0, 0, -1));
5296 if(zipinfo.create_system==0 or zipinfo.create_system==10):
5297 fwinattributes = int(zipinfo.external_attr);
5298 if(not member.is_dir()):
5299 fmode = int(stat.S_IFREG + 438);
5300 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
5301 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
5302 elif(member.is_dir()):
5303 fmode = int(stat.S_IFDIR + 511);
5304 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
5305 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
5306 elif(zipinfo.create_system==3):
5307 fwinattributes = int(0);
5308 fmode = int(zipinfo.external_attr);
5309 else:
5310 fwinattributes = int(0);
5311 if(not member.is_dir()):
5312 fmode = int(stat.S_IFREG + 438);
5313 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
5314 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
5315 elif(member.is_dir()):
5316 fmode = int(stat.S_IFDIR + 511);
5317 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
5318 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
5319 fcompression = "";
5320 fcsize = 0;
5321 try:
5322 fuid = os.getuid();
5323 except AttributeError:
5324 fuid = 0;
5325 except KeyError:
5326 fuid = 0;
5327 try:
5328 fgid = os.getgid();
5329 except AttributeError:
5330 fgid = 0;
5331 except KeyError:
5332 fgid = 0;
5333 try:
5334 import pwd;
5335 try:
5336 userinfo = pwd.getpwuid(os.getuid());
5337 funame = userinfo.pw_name;
5338 except KeyError:
5339 funame = "";
5340 except AttributeError:
5341 funame = "";
5342 except ImportError:
5343 funame = "";
5344 fgname = "";
5345 try:
5346 import grp;
5347 try:
5348 groupinfo = grp.getgrgid(os.getgid());
5349 fgname = groupinfo.gr_name;
5350 except KeyError:
5351 fgname = "";
5352 except AttributeError:
5353 fgname = "";
5354 except ImportError:
5355 fgname = "";
5356 fcontents = BytesIO();
5357 if(ftype==0):
5358 fcontents.write(zipfp.read(member.filename));
5359 fcontents.seek(0, 0);
5360 ftypehex = format(ftype, 'x').lower();
5361 extrafields = len(extradata);
5362 extrafieldslist = extradata;
5363 catfextrafields = extrafields;
5364 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
5365 if(len(extradata)>0):
5366 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
5367 extrasizelen = len(extrasizestr);
5368 extrasizelenhex = format(extrasizelen, 'x').lower();
5369 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5370 catoutlen = len(catoutlist) + len(extradata) + 3;
5371 catoutlenhex = format(catoutlen, 'x').lower();
5372 catoutlist.insert(0, catoutlenhex);
5373 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
5374 if(len(extradata)>0):
5375 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
5376 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
5377 catfnumfields = catoutlen;
5378 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5379 fcontents.seek(0, 0);
5380 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5381 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5382 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5383 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
5384 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5385 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5386 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5387 nullstrecd = formatspecs[5].encode('UTF-8');
5388 fheadtell += len(catfileoutstr) + 1;
5389 catfcontentend = fheadtell - 1;
5390 fcontents.seek(0, 0);
5391 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5392 pyhascontents = False;
5393 if(int(fsize)>0 and not listonly):
5394 pyhascontents = True;
5395 if(int(fsize)>0 and listonly):
5396 fcontents = BytesIO();
5397 pyhascontents = False;
5398 fcontents.seek(0, 0);
5399 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5400 fileidnum = fileidnum + 1;
5401 return catlist;
5402
5403 if(not rarfile_support):
5404 def RarFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5405 return False;
5406
5407 if(rarfile_support):
5408 def RarFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5409 curinode = 0;
5410 curfid = 0;
5411 inodelist = [];
5412 inodetofile = {};
5413 filetoinode = {};
5414 inodetocatinode = {};
5415 fileidnum = 0;
5416 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
5417 return False;
5418 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
5419 return False;
5420 rarfp = rarfile.RarFile(infile, "r");
5421 rartest = rarfp.testrar();
5422 if(rartest):
5423 VerbosePrintOut("Bad file found!");
5424 fnumfiles = int(len(rarfp.infolist()));
5425 catver = formatspecs[6];
5426 fileheaderver = str(int(catver.replace(".", "")));
5427 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
5428 catversion = re.findall(r"([\d]+)$", fileheader);
5429 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
5430 fnumfileshex = format(int(fnumfiles), 'x').lower();
5431 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs[5]);
5432 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
5433 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
5434 fheadtell = len(fileheader);
5435 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5436 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
5437 is_unix = False;
5438 is_windows = False;
5439 if(member.host_os==rarfile.RAR_OS_UNIX):
5440 is_windows = False;
5441 try:
5442 member.external_attr
5443 is_unix = True;
5444 except AttributeError:
5445 is_unix = False;
5446 elif(member.host_os==rarfile.RAR_OS_WIN32):
5447 is_unix = False;
5448 try:
5449 member.external_attr
5450 is_windows = True;
5451 except AttributeError:
5452 is_windows = False;
5453 else:
5454 is_unix = False;
5455 is_windows = False;
5456 if(re.findall("^[.|/]", member.filename)):
5457 fname = member.filename;
5458 else:
5459 fname = "./"+member.filename;
5460 rarinfo = rarfp.getinfo(member.filename);
5461 if(verbose):
5462 VerbosePrintOut(fname);
5463 if(is_unix and member.external_attr !=0):
5464 fpremode = int(member.external_attr);
5465 elif(member.is_file()):
5466 fpremode = stat.S_IFREG + 438;
5467 elif(member.is_symlink()):
5468 fpremode = stat.S_IFLNK + 438;
5469 elif(member.is_dir()):
5470 fpremode = stat.S_IFDIR + 511;
5471 if(is_windows and member.external_attr !=0):
5472 fwinattributes = int(member.external_attr);
5473 else:
5474 fwinattributes = int(0);
5475 fcompression = "";
5476 fcsize = 0;
5477 flinkcount = 0;
5478 ftype = 0;
5479 if(member.is_file()):
5480 ftype = 0;
5481 elif(member.is_symlink()):
5482 ftype = 2;
5483 elif(member.is_dir()):
5484 ftype = 5;
5485 flinkname = "";
5486 if(ftype==2):
5487 flinkname = rarfp.read(member.filename).decode("UTF-8");
5488 fbasedir = os.path.dirname(fname);
5489 fcurfid = curfid;
5490 fcurinode = curfid;
5491 finode = fcurinode;
5492 curfid = curfid + 1;
5493 fdev_minor = 0;
5494 fdev_major = 0;
5495 frdev_minor = 0;
5496 frdev_major = 0;
5497 if(ftype==5):
5498 fsize = "0";
5499 if(ftype==0):
5500 fsize = member.file_size;
5501 try:
5502 if(member.atime):
5503 fatime = int(member.atime.timestamp());
5504 else:
5505 fatime = int(member.mtime.timestamp());
5506 except AttributeError:
5507 fatime = int(member.mtime.timestamp());
5508 fmtime = int(member.mtime.timestamp());
5509 try:
5510 if(member.ctime):
5511 fctime = int(member.ctime.timestamp());
5512 else:
5513 fctime = int(member.mtime.timestamp());
5514 except AttributeError:
5515 fctime = int(member.mtime.timestamp());
5516 fbtime = int(member.mtime.timestamp());
5517 if(is_unix and member.external_attr !=0):
5518 fmode = int(member.external_attr);
5519 fchmode = int(stat.S_IMODE(member.external_attr));
5520 ftypemod = int(stat.S_IFMT(member.external_attr));
5521 elif(member.is_file()):
5522 fmode = int(stat.S_IFREG + 438)
5523 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5524 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5525 elif(member.is_symlink()):
5526 fmode = int(stat.S_IFLNK + 438)
5527 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5528 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5529 elif(member.is_dir()):
5530 fmode = int(stat.S_IFDIR + 511)
5531 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
5532 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
5533 try:
5534 fuid = os.getuid();
5535 except AttributeError:
5536 fuid = 0;
5537 except KeyError:
5538 fuid = 0;
5539 try:
5540 fgid = os.getgid();
5541 except AttributeError:
5542 fgid = 0;
5543 except KeyError:
5544 fgid = 0;
5545 try:
5546 import pwd;
5547 try:
5548 userinfo = pwd.getpwuid(os.getuid());
5549 funame = userinfo.pw_name;
5550 except KeyError:
5551 funame = "";
5552 except AttributeError:
5553 funame = "";
5554 except ImportError:
5555 funame = "";
5556 fgname = "";
5557 try:
5558 import grp;
5559 try:
5560 groupinfo = grp.getgrgid(os.getgid());
5561 fgname = groupinfo.gr_name;
5562 except KeyError:
5563 fgname = "";
5564 except AttributeError:
5565 fgname = "";
5566 except ImportError:
5567 fgname = "";
5568 fcontents = BytesIO();
5569 if(ftype==0):
5570 fcontents.write(rarfp.read(member.filename));
5571 fcontents.seek(0, 0);
5572 ftypehex = format(ftype, 'x').lower();
5573 extrafields = len(extradata);
5574 extrafieldslist = extradata;
5575 catfextrafields = extrafields;
5576 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
5577 if(len(extradata)>0):
5578 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
5579 extrasizelen = len(extrasizestr);
5580 extrasizelenhex = format(extrasizelen, 'x').lower();
5581 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5582 catoutlen = len(catoutlist) + len(extradata) + 3;
5583 catoutlenhex = format(catoutlen, 'x').lower();
5584 catoutlist.insert(0, catoutlenhex);
5585 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
5586 if(len(extradata)>0):
5587 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
5588 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
5589 catfnumfields = 24 + catfextrafields;
5590 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5591 fcontents.seek(0, 0);
5592 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5593 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5594 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5595 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
5596 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5597 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5598 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5599 nullstrecd = formatspecs[5].encode('UTF-8');
5600 fheadtell += len(catfileoutstr) + 1;
5601 catfcontentend = fheadtell - 1;
5602 fcontents.seek(0, 0);
5603 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5604 pyhascontents = False;
5605 if(int(fsize)>0 and not listonly):
5606 pyhascontents = True;
5607 if(int(fsize)>0 and listonly):
5608 fcontents = BytesIO();
5609 pyhascontents = False;
5610 fcontents.seek(0, 0);
5611 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5612 fileidnum = fileidnum + 1;
5613 return catlist;
5614
5615 if(not py7zr_support):
5616 def SevenZipFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5617 return False;
5618
5619 if(py7zr_support):
5620 def SevenZipFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5621 curinode = 0;
5622 curfid = 0;
5623 inodelist = [];
5624 inodetofile = {};
5625 filetoinode = {};
5626 inodetocatinode = {};
5627 fileidnum = 0;
5628 szpfp = py7zr.SevenZipFile(infile, mode="r");
5629 file_content = szpfp.readall();
5630 #sztest = szpfp.testzip();
5631 sztestalt = szpfp.test();
5632 if(sztestalt):
5633 VerbosePrintOut("Bad file found!");
5634 numfiles = int(len(szpfp.list()));
5635 catver = formatspecs[6];
5636 fileheaderver = str(int(catver.replace(".", "")));
5637 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
5638 catversion = re.findall(r"([\d]+)$", fileheader);
5639 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
5640 fnumfileshex = format(int(fnumfiles), 'x').lower();
5641 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs[5]);
5642 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
5643 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
5644 fheadtell = len(fileheader);
5645 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5646 for member in sorted(szpfp.list(), key=lambda x: x.filename):
5647 if(re.findall("^[.|/]", member.filename)):
5648 fname = member.filename;
5649 else:
5650 fname = "./"+member.filename;
5651 if(not member.is_directory):
5652 fpremode = int(stat.S_IFREG + 438);
5653 elif(member.is_directory):
5654 fpremode = int(stat.S_IFDIR + 511);
5655 fwinattributes = int(0);
5656 fcompression = "";
5657 fcsize = 0;
5658 flinkcount = 0;
5659 ftype = 0;
5660 if(member.is_directory):
5661 ftype = 5;
5662 else:
5663 ftype = 0;
5664 flinkname = "";
5665 fbasedir = os.path.dirname(fname);
5666 fcurfid = curfid;
5667 fcurinode = curfid;
5668 finode = fcurinode;
5669 curfid = curfid + 1;
5670 fdev_minor = 0;
5671 fdev_major = 0;
5672 frdev_minor = 0;
5673 frdev_major = 0;
5674 if(ftype==5):
5675 fsize = "0";
5676 fatime = int(member.creationtime.timestamp());
5677 fmtime = int(member.creationtime.timestamp());
5678 fctime = int(member.creationtime.timestamp());
5679 fbtime = int(member.creationtime.timestamp());
5680 if(member.is_directory):
5681 fmode = int(stat.S_IFDIR + 511)
5682 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
5683 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
5684 else:
5685 fmode = int(stat.S_IFLNK + 438)
5686 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5687 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5688 try:
5689 fuid = os.getuid();
5690 except AttributeError:
5691 fuid = 0;
5692 except KeyError:
5693 fuid = 0;
5694 try:
5695 fgid = os.getgid();
5696 except AttributeError:
5697 fgid = 0;
5698 except KeyError:
5699 fgid = 0;
5700 try:
5701 import pwd;
5702 try:
5703 userinfo = pwd.getpwuid(os.getuid());
5704 funame = userinfo.pw_name;
5705 except KeyError:
5706 funame = "";
5707 except AttributeError:
5708 funame = "";
5709 except ImportError:
5710 funame = "";
5711 fgname = "";
5712 try:
5713 import grp;
5714 try:
5715 groupinfo = grp.getgrgid(os.getgid());
5716 fgname = groupinfo.gr_name;
5717 except KeyError:
5718 fgname = "";
5719 except AttributeError:
5720 fgname = "";
5721 except ImportError:
5722 fgname = "";
5723 fcontents = BytesIO();
5724 if(ftype==0):
5725 fcontents.write(file_content[member.filename].read());
5726 fsize = format(fcontents.tell(), 'x').lower();
5727 fileop.close();
5728 fcontents.seek(0, 0);
5729 ftypehex = format(ftype, 'x').lower();
5730 extrafields = len(extradata);
5731 extrafieldslist = extradata;
5732 catfextrafields = extrafields;
5733 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
5734 if(len(extradata)>0):
5735 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
5736 extrasizelen = len(extrasizestr);
5737 extrasizelenhex = format(extrasizelen, 'x').lower();
5738 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5739 catoutlen = len(catoutlist) + len(extradata) + 3;
5740 catoutlenhex = format(catoutlen, 'x').lower();
5741 catoutlist.insert(0, catoutlenhex);
5742 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
5743 if(len(extradata)>0):
5744 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
5745 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
5746 catfnumfields = 24 + catfextrafields;
5747 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5748 fcontents.seek(0, 0);
5749 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5750 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5751 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5752 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
5753 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5754 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5755 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5756 nullstrecd = formatspecs[5].encode('UTF-8');
5757 fheadtell += len(catfileoutstr) + 1;
5758 catfcontentend = fheadtell - 1;
5759 fcontents.seek(0, 0);
5760 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5761 pyhascontents = False;
5762 if(int(fsize)>0 and not listonly):
5763 pyhascontents = True;
5764 if(int(fsize)>0 and listonly):
5765 fcontents = "";
5766 pyhascontents = False;
5767 fcontents.seek(0, 0);
5768 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5769 fileidnum = fileidnum + 1;
5770 return catlist;
5771
5772 def InFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5773 checkcompressfile = CheckCompressionSubType(infiles, formatspecs, True);
5774 if(checkcompressfile=="tarfile"):
5775 return TarFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5776 elif(checkcompressfile=="zipfile"):
5777 return ZipFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5778 elif(checkcompressfile=="catfile"):
5779 return ArchiveFileToArray(infile, 0, 0, listonly, False, formatspecs, False);
5780 elif(rarfile_support and checkcompressfile=="rarfile"):
5781 return RarFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5782 elif(py7zr_support and checkcompressfile=="7zipfile"):
5783 return SevenZipFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5784 else:
5785 return False;
5786 return False;
5787
5788 def ListDirToArray(infiles, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, seekstart=0, seekend=0, listonly=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
5789 outarray = BytesIO();
5790 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
5791 listcatfiles = ArchiveFileToArray(outarray, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5792 return listcatfiles;
5793
5794 def ArchiveFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5795 if(isinstance(infile, dict)):
5796 listcatfiles = infile;
5797 else:
5798 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
5799 infile = RemoveWindowsPath(infile);
5800 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5801 if(not listcatfiles):
5802 return False;
5803 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5804 if(returnfp):
5805 catarray.update({'catfp': listcatfiles['catfp']});
5806 lenlist = len(listcatfiles['ffilelist']);
5807 lcfi = 0;
5808 lcfx = int(listcatfiles['fnumfiles']);
5809 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5810 lcfx = int(lenlist);
5811 else:
5812 lcfx = int(listcatfiles['fnumfiles']);
5813 while(lcfi < lcfx):
5814 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5815 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5816 catarray['filetoid'].update(filetoidarray);
5817 catarray['idtofile'].update(idtofilearray);
5818 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5819 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5820 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5821 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5822 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5823 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5824 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5825 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5826 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5827 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5828 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5829 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5830 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5831 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5832 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5833 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5834 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5835 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5836 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5837 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5838 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5839 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5840 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5841 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5842 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5843 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5844 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
5845 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5846 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5847 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5848 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5849 lcfi = lcfi + 1;
5850 return catarray;
5851
5852 create_alias_function("", __file_format_name__, "ToArrayIndex", ArchiveFileToArrayIndex);
5853
5854 def ListDirToArrayIndexAlt(infiles, dirlistfromtxt=False, followlink=False, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5855 listcatfiles = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, listonly, checksumtype, extradata, formatspecs, verbose);
5856 if(not listcatfiles):
5857 return False;
5858 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5859 lenlist = len(listcatfiles['ffilelist']);
5860 lcfi = 0;
5861 lcfx = int(listcatfiles['fnumfiles']);
5862 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5863 lcfx = int(lenlist);
5864 else:
5865 lcfx = int(listcatfiles['fnumfiles']);
5866 while(lcfi < lcfx):
5867 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5868 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5869 catarray['filetoid'].update(filetoidarray);
5870 catarray['idtofile'].update(idtofilearray);
5871 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5872 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5873 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5874 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5875 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5876 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5877 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5878 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5879 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5880 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5881 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5882 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5883 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5884 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5885 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5886 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5887 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5888 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5889 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5890 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5891 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5892 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5893 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5894 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5895 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5896 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5897 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
5898 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5899 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5900 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5901 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5902 lcfi = lcfi + 1;
5903 return catarray;
5904
5905 def TarFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5906 listcatfiles = TarFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5907 if(not listcatfiles):
5908 return False;
5909 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5910 lenlist = len(listcatfiles['ffilelist']);
5911 lcfi = 0;
5912 lcfx = int(listcatfiles['fnumfiles']);
5913 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5914 lcfx = int(lenlist);
5915 else:
5916 lcfx = int(listcatfiles['fnumfiles']);
5917 while(lcfi < lcfx):
5918 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5919 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5920 catarray['filetoid'].update(filetoidarray);
5921 catarray['idtofile'].update(idtofilearray);
5922 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5923 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5924 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5925 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5926 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5927 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5928 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5929 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5930 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5931 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5932 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5933 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5934 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5935 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5936 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5937 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5938 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5939 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5940 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5941 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5942 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5943 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5944 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5945 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5946 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5947 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5948 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
5949 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5950 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5951 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5952 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5953 lcfi = lcfi + 1;
5954 return catarray;
5955
5956 def ZipFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5957 listcatfiles = ZipFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5958 if(not listcatfiles):
5959 return False;
5960 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5961 lenlist = len(listcatfiles['ffilelist']);
5962 lcfi = 0;
5963 lcfx = int(listcatfiles['fnumfiles']);
5964 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5965 lcfx = int(lenlist);
5966 else:
5967 lcfx = int(listcatfiles['fnumfiles']);
5968 while(lcfi < lcfx):
5969 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5970 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5971 catarray['filetoid'].update(filetoidarray);
5972 catarray['idtofile'].update(idtofilearray);
5973 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5974 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5975 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5976 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5977 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5978 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5979 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5980 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5981 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5982 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5983 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5984 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5985 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5986 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5987 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5988 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5989 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5990 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5991 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5992 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5993 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5994 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5995 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5996 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5997 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5998 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5999 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6000 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6001 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6002 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6003 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6004 lcfi = lcfi + 1;
6005 return catarray;
6006
6007 if(not rarfile_support):
6008 def RarFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
6009 return False;
6010
6011 if(rarfile_support):
6012 def RarFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
6013 listcatfiles = RarFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
6014 if(not listcatfiles):
6015 return False;
6016 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6017 lenlist = len(listcatfiles['ffilelist']);
6018 lcfi = 0;
6019 lcfx = int(listcatfiles['fnumfiles']);
6020 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6021 lcfx = int(lenlist);
6022 else:
6023 lcfx = int(listcatfiles['fnumfiles']);
6024 while(lcfi < lcfx):
6025 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6026 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6027 catarray['filetoid'].update(filetoidarray);
6028 catarray['idtofile'].update(idtofilearray);
6029 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6030 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6031 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6032 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6033 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6034 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6035 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6036 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6037 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6038 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6039 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6040 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6041 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6042 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6043 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6044 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6045 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6046 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6047 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6048 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6049 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6050 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6051 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6052 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6053 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6054 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6055 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6056 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6057 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6058 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6059 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6060 lcfi = lcfi + 1;
6061 return catarray;
6062
6063 if(not py7zr_support):
6064 def SevenZipFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
6065 return False;
6066
6067 if(py7zr_support):
6068 def SevenZipFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
6069 listcatfiles = SevenZipFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
6070 if(not listcatfiles):
6071 return False;
6072 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6073 lenlist = len(listcatfiles['ffilelist']);
6074 lcfi = 0;
6075 lcfx = int(listcatfiles['fnumfiles']);
6076 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6077 lcfx = int(lenlist);
6078 else:
6079 lcfx = int(listcatfiles['fnumfiles']);
6080 while(lcfi < lcfx):
6081 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6082 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6083 catarray['filetoid'].update(filetoidarray);
6084 catarray['idtofile'].update(idtofilearray);
6085 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6086 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6087 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6088 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6089 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6090 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6091 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6092 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6093 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6094 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6095 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6096 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6097 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6098 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6099 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6100 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6101 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6102 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6103 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6104 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6105 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6106 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6107 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6108 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6109 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6110 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6111 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6112 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6113 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6114 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6115 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6116 lcfi = lcfi + 1;
6117 return catarray;
6118
6119 def ArchiveFileStringToArrayIndex(catstr, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
6120 catfp = BytesIO(catstr);
6121 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
6122 return listcatfiles;
6123
6124 create_alias_function("", __file_format_name__, "StringToArrayIndex", ArchiveFileStringToArrayIndex);
6125
6126 def TarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
6127 catfp = BytesIO();
6128 catfp = PackArchiveFileFromTarFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
6129 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
6130 return listcatfiles;
6131
6132 def ZipFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
6133 catfp = BytesIO();
6134 catfp = PackArchiveFileFromZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
6135 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
6136 return listcatfiles;
6137
6138 if(not rarfile_support):
6139 def RarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
6140 return False;
6141
6142 if(rarfile_support):
6143 def RarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
6144 catfp = BytesIO();
6145 catfp = PackArchiveFileFromRarFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
6146 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
6147 return listcatfiles;
6148
6149 if(not py7zr_support):
6150 def SevenZipFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
6151 return False;
6152
6153 if(py7zr_support):
6154 def SevenZipFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
6155 catfp = BytesIO();
6156 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
6157 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
6158 return listcatfiles;
6159
6160 def ListDirToArrayIndex(infiles, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, seekstart=0, seekend=0, listonly=False, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False, returnfp=False):
6161 outarray = BytesIO();
6162 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compressionlevel, followlink, checksumtype, formatspecs, verbose, True);
6163 listcatfiles = ArchiveFileToArrayIndex(outarray, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp)
6164 return listcatfiles;
6165
6166 def RePackArchiveFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
6167 if(isinstance(infile, dict)):
6168 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6169 listcatfiles = prelistcatfiles['list'];
6170 else:
6171 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6172 infile = RemoveWindowsPath(infile);
6173 if(followlink):
6174 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6175 listcatfiles = prelistcatfiles['list'];
6176 else:
6177 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6178 if(outfile!="-" and not hasattr(infile, "read") and not hasattr(outfile, "write")):
6179 outfile = RemoveWindowsPath(outfile);
6180 checksumtype = checksumtype.lower();
6181 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
6182 checksumtype="crc32";
6183 if(checksumtype=="none"):
6184 checksumtype = "";
6185 if(not compression or compression=="catfile" or compression==formatspecs[2]):
6186 compression = "auto";
6187 if(compression not in compressionlist and compression is None):
6188 compression = "auto";
6189 if(verbose):
6190 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6191 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
6192 if(os.path.exists(outfile)):
6193 try:
6194 os.unlink(outfile);
6195 except OSError as e:
6196 pass;
6197 if(not listcatfiles):
6198 return False;
6199 if(outfile=="-"):
6200 verbose = False;
6201 catfp = BytesIO();
6202 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
6203 catfp = outfile;
6204 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
6205 catfp = BytesIO();
6206 else:
6207 fbasename = os.path.splitext(outfile)[0];
6208 fextname = os.path.splitext(outfile)[1];
6209 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
6210 catver = formatspecs[6];
6211 fileheaderver = str(int(catver.replace(".", "")));
6212 lenlist = len(listcatfiles['ffilelist']);
6213 fnumfiles = int(listcatfiles['fnumfiles']);
6214 if(lenlist>fnumfiles or lenlist<fnumfiles):
6215 fnumfiles = lenlist;
6216 AppendFileHeader(catfp, fnumfiles, checksumtype, formatspecs);
6217 lenlist = len(listcatfiles['ffilelist']);
6218 fnumfiles = int(listcatfiles['fnumfiles']);
6219 lcfi = 0;
6220 lcfx = int(listcatfiles['fnumfiles']);
6221 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6222 lcfx = int(lenlist);
6223 else:
6224 lcfx = int(listcatfiles['fnumfiles']);
6225 curinode = 0;
6226 curfid = 0;
6227 inodelist = [];
6228 inodetofile = {};
6229 filetoinode = {};
6230 reallcfi = 0;
6231 while(lcfi < lcfx):
6232 if(re.findall("^[.|/]", listcatfiles['ffilelist'][reallcfi]['fname'])):
6233 fname = listcatfiles['ffilelist'][reallcfi]['fname'];
6234 else:
6235 fname = "./"+listcatfiles['ffilelist'][reallcfi]['fname'];
6236 if(verbose):
6237 VerbosePrintOut(fname);
6238 fheadersize = format(int(listcatfiles['ffilelist'][reallcfi]['fheadersize']), 'x').lower();
6239 fsize = format(int(listcatfiles['ffilelist'][reallcfi]['fsize']), 'x').lower();
6240 flinkname = listcatfiles['ffilelist'][reallcfi]['flinkname'];
6241 fatime = format(int(listcatfiles['ffilelist'][reallcfi]['fatime']), 'x').lower();
6242 fmtime = format(int(listcatfiles['ffilelist'][reallcfi]['fmtime']), 'x').lower();
6243 fctime = format(int(listcatfiles['ffilelist'][reallcfi]['fctime']), 'x').lower();
6244 fbtime = format(int(listcatfiles['ffilelist'][reallcfi]['fbtime']), 'x').lower();
6245 fmode = format(int(listcatfiles['ffilelist'][reallcfi]['fmode']), 'x').lower();
6246 fchmode = format(int(listcatfiles['ffilelist'][reallcfi]['fchmode']), 'x').lower();
6247 fuid = format(int(listcatfiles['ffilelist'][reallcfi]['fuid']), 'x').lower();
6248 funame = listcatfiles['ffilelist'][reallcfi]['funame'];
6249 fgid = format(int(listcatfiles['ffilelist'][reallcfi]['fgid']), 'x').lower();
6250 fgname = listcatfiles['ffilelist'][reallcfi]['fgname'];
6251 finode = format(int(listcatfiles['ffilelist'][reallcfi]['finode']), 'x').lower();
6252 flinkcount = format(int(listcatfiles['ffilelist'][reallcfi]['flinkcount']), 'x').lower();
6253 fwinattributes = format(int(listcatfiles['ffilelist'][reallcfi]['fwinattributes']), 'x').lower();
6254 fcompression = listcatfiles['ffilelist'][reallcfi]['fcompression'];
6255 fcsize = format(int(listcatfiles['ffilelist'][reallcfi]['fcsize']), 'x').lower();
6256 fdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['fminor']), 'x').lower();
6257 fdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['fmajor']), 'x').lower();
6258 frdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['frminor']), 'x').lower();
6259 frdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['frmajor']), 'x').lower();
6260 if(len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>listcatfiles['ffilelist'][reallcfi]['fextrafields'] and len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>0):
6261 listcatfiles['ffilelist'][reallcfi]['fextrafields'] = len(listcatfiles['ffilelist'][reallcfi]['fextralist']);
6262 if(not followlink and len(extradata)<0):
6263 extradata = listcatfiles['ffilelist'][reallcfi]['fextralist'];
6264 fcontents = listcatfiles['ffilelist'][reallcfi]['fcontents'];
6265 fcompression = "";
6266 fcsize = format(int(0), 'x').lower();
6267 if(not compresswholefile):
6268 fcontents.seek(0, 2);
6269 ucfsize = fcontents.tell();
6270 fcontents.seek(0, 0);
6271 if(compression=="auto"):
6272 ilsize = len(compressionlistalt);
6273 ilmin = 0;
6274 ilcsize = [];
6275 while(ilmin < ilsize):
6276 cfcontents = BytesIO();
6277 shutil.copyfileobj(fcontents, cfcontents);
6278 fcontents.seek(0, 0);
6279 cfcontents.seek(0, 0);
6280 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
6281 if(cfcontents):
6282 cfcontents.seek(0, 2);
6283 ilcsize.append(cfcontents.tell());
6284 cfcontents.close();
6285 else:
6286 try:
6287 ilcsize.append(sys.maxint);
6288 except AttributeError:
6289 ilcsize.append(sys.maxsize);
6290 ilmin = ilmin + 1;
6291 ilcmin = ilcsize.index(min(ilcsize));
6292 compression = compressionlistalt[ilcmin];
6293 fcontents.seek(0, 0);
6294 cfcontents = BytesIO();
6295 shutil.copyfileobj(fcontents, cfcontents);
6296 cfcontents.seek(0, 0);
6297 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
6298 cfcontents.seek(0, 2);
6299 cfsize = cfcontents.tell();
6300 if(ucfsize > cfsize):
6301 fcsize = format(int(cfsize), 'x').lower();
6302 fcompression = compression;
6303 fcontents.close();
6304 fcontents = cfcontents;
6305 if(followlink):
6306 if(listcatfiles['ffilelist'][reallcfi]['ftype']==1 or listcatfiles['ffilelist'][reallcfi]['ftype']==2):
6307 getflinkpath = listcatfiles['ffilelist'][reallcfi]['flinkname'];
6308 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6309 flinkinfo = listcatfiles['ffilelist'][flinkid];
6310 fheadersize = format(int(flinkinfo['fheadersize']), 'x').lower();
6311 fsize = format(int(flinkinfo['fsize']), 'x').lower();
6312 flinkname = flinkinfo['flinkname'];
6313 fatime = format(int(flinkinfo['fatime']), 'x').lower();
6314 fmtime = format(int(flinkinfo['fmtime']), 'x').lower();
6315 fctime = format(int(flinkinfo['fctime']), 'x').lower();
6316 fbtime = format(int(flinkinfo['fbtime']), 'x').lower();
6317 fmode = format(int(flinkinfo['fmode']), 'x').lower();
6318 fchmode = format(int(flinkinfo['fchmode']), 'x').lower();
6319 fuid = format(int(flinkinfo['fuid']), 'x').lower();
6320 funame = flinkinfo['funame'];
6321 fgid = format(int(flinkinfo['fgid']), 'x').lower();
6322 fgname = flinkinfo['fgname'];
6323 finode = format(int(flinkinfo['finode']), 'x').lower();
6324 flinkcount = format(int(flinkinfo['flinkcount']), 'x').lower();
6325 fwinattributes = format(int(flinkinfo['fwinattributes']), 'x').lower();
6326 fcompression = flinkinfo['fcompression'];
6327 fcsize = format(int(flinkinfo['fcsize']), 'x').lower();
6328 fdev_minor = format(int(flinkinfo['fminor']), 'x').lower();
6329 fdev_major = format(int(flinkinfo['fmajor']), 'x').lower();
6330 frdev_minor = format(int(flinkinfo['frminor']), 'x').lower();
6331 frdev_major = format(int(flinkinfo['frmajor']), 'x').lower();
6332 if(len(flinkinfo['fextralist'])>flinkinfo['fextrafields'] and len(flinkinfo['fextralist'])>0):
6333 flinkinfo['fextrafields'] = len(flinkinfo['fextralist']);
6334 if(len(extradata)<0):
6335 extradata = flinkinfo['fextralist'];
6336 fcontents = flinkinfo['fcontents'];
6337 ftypehex = format(flinkinfo['ftype'], 'x').lower();
6338 else:
6339 ftypehex = format(listcatfiles['ffilelist'][reallcfi]['ftype'], 'x').lower();
6340 fcurfid = format(curfid, 'x').lower();
6341 if(not followlink and finode!=0):
6342 if(listcatfiles['ffilelist'][reallcfi]['ftype']!=1):
6343 fcurinode = format(int(curinode), 'x').lower();
6344 inodetofile.update({curinode: fname});
6345 filetoinode.update({fname: curinode});
6346 curinode = curinode + 1;
6347 else:
6348 fcurinode = format(int(filetoinode[flinkname]), 'x').lower();
6349 else:
6350 fcurinode = format(int(curinode), 'x').lower();
6351 curinode = curinode + 1;
6352 curfid = curfid + 1;
6353 if(fcompression=="none"):
6354 fcompression = "";
6355 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
6356 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
6357 fcontents.close();
6358 lcfi = lcfi + 1;
6359 reallcfi = reallcfi + 1;
6360 if(lcfx>0):
6361 catfp.write(AppendNullBytes([0, 0], formatspecs[5]).encode("UTF-8"));
6362 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
6363 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
6364 try:
6365 catfp.flush();
6366 os.fsync(catfp.fileno());
6367 except io.UnsupportedOperation:
6368 pass;
6369 except AttributeError:
6370 pass;
6371 except OSError as e:
6372 pass;
6373 if(outfile=="-"):
6374 catfp.seek(0, 0);
6375 if(hasattr(sys.stdout, "buffer")):
6376 shutil.copyfileobj(catfp, sys.stdout.buffer);
6377 else:
6378 shutil.copyfileobj(catfp, sys.stdout);
6379 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
6380 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
6381 catfp.seek(0, 0);
6382 upload_file_to_internet_file(catfp, outfile);
6383 if(returnfp):
6384 catfp.seek(0, 0);
6385 return catfp;
6386 else:
6387 catfp.close();
6388 return True;
6389
6390 create_alias_function("RePack", __file_format_name__, "", RePackArchiveFile);
6391
6392 def RePackArchiveFileFromString(catstr, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
6393 catfp = BytesIO(catstr);
6394 listcatfiles = RePackArchiveFile(catfp, compression, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6395 return listcatfiles;
6396
6397 create_alias_function("RePack", __file_format_name__, "FromString", RePackArchiveFileFromString);
6398
6399 def PackArchiveFileFromListDir(infiles, outfile, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
6400 outarray = BytesIO();
6401 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
6402 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6403 return listcatfiles;
6404
6405 create_alias_function("Pack", __file_format_name__, "FromListDir", PackArchiveFileFromListDir);
6406
6407 def UnPackArchiveFile(infile, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_list__, preservepermissions=True, preservetime=True, verbose=False, returnfp=False):
6408 if(outdir is not None):
6409 outdir = RemoveWindowsPath(outdir);
6410 if(verbose):
6411 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6412 if(isinstance(infile, dict)):
6413 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6414 listcatfiles = prelistcatfiles['list'];
6415 else:
6416 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6417 infile = RemoveWindowsPath(infile);
6418 if(followlink):
6419 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6420 listcatfiles = prelistcatfiles['list'];
6421 else:
6422 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6423 if(not listcatfiles):
6424 return False;
6425 lenlist = len(listcatfiles['ffilelist']);
6426 fnumfiles = int(listcatfiles['fnumfiles']);
6427 lcfi = 0;
6428 lcfx = int(listcatfiles['fnumfiles']);
6429 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6430 lcfx = int(lenlist);
6431 else:
6432 lcfx = int(listcatfiles['fnumfiles']);
6433 while(lcfi<lcfx):
6434 funame = "";
6435 try:
6436 import pwd;
6437 try:
6438 userinfo = pwd.getpwuid(listcatfiles['ffilelist'][lcfi]['fuid']);
6439 funame = userinfo.pw_name;
6440 except KeyError:
6441 funame = "";
6442 except ImportError:
6443 funame = "";
6444 fgname = "";
6445 try:
6446 import grp;
6447 try:
6448 groupinfo = grp.getgrgid(listcatfiles['ffilelist'][lcfi]['fgid']);
6449 fgname = groupinfo.gr_name;
6450 except KeyError:
6451 fgname = "";
6452 except ImportError:
6453 fgname = "";
6454 if(verbose):
6455 VerbosePrintOut(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6456 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6457 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6458 listcatfiles['ffilelist'][lcfi]['fcontents'].seek(0, 0);
6459 shutil.copyfileobj(listcatfiles['ffilelist'][lcfi]['fcontents'], fpc);
6460 try:
6461 fpc.flush();
6462 os.fsync(fpc.fileno());
6463 except io.UnsupportedOperation:
6464 pass;
6465 except AttributeError:
6466 pass;
6467 except OSError as e:
6468 pass;
6469 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
6470 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
6471 if(preservepermissions):
6472 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6473 if(preservetime):
6474 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
6475 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6476 if(followlink):
6477 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
6478 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6479 flinkinfo = listcatfiles['ffilelist'][flinkid];
6480 funame = "";
6481 try:
6482 import pwd;
6483 try:
6484 userinfo = pwd.getpwuid(flinkinfo['fuid']);
6485 funame = userinfo.pw_name;
6486 except KeyError:
6487 funame = "";
6488 except ImportError:
6489 funame = "";
6490 fgname = "";
6491 try:
6492 import grp;
6493 try:
6494 groupinfo = grp.getgrgid(flinkinfo['fgid']);
6495 fgname = groupinfo.gr_name;
6496 except KeyError:
6497 fgname = "";
6498 except ImportError:
6499 fgname = "";
6500 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
6501 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6502 flinkinfo['fcontents'].seek(0, 0);
6503 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
6504 try:
6505 fpc.flush();
6506 os.fsync(fpc.fileno());
6507 except io.UnsupportedOperation:
6508 pass;
6509 except AttributeError:
6510 pass;
6511 except OSError as e:
6512 pass;
6513 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6514 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6515 if(preservepermissions):
6516 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6517 if(preservetime):
6518 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6519 if(flinkinfo['ftype']==1):
6520 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6521 if(flinkinfo['ftype']==2):
6522 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6523 if(flinkinfo['ftype']==5):
6524 if(preservepermissions):
6525 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6526 else:
6527 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6528 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6529 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6530 if(preservepermissions):
6531 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6532 if(preservetime):
6533 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6534 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
6535 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6536 else:
6537 os.link(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6538 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6539 if(followlink):
6540 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
6541 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6542 flinkinfo = listcatfiles['ffilelist'][flinkid];
6543 funame = "";
6544 try:
6545 import pwd;
6546 try:
6547 userinfo = pwd.getpwuid(flinkinfo['fuid']);
6548 funame = userinfo.pw_name;
6549 except KeyError:
6550 funame = "";
6551 except ImportError:
6552 funame = "";
6553 fgname = "";
6554 try:
6555 import grp;
6556 try:
6557 groupinfo = grp.getgrgid(flinkinfo['fgid']);
6558 fgname = groupinfo.gr_name;
6559 except KeyError:
6560 fgname = "";
6561 except ImportError:
6562 fgname = "";
6563 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
6564 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6565 flinkinfo['fcontents'].seek(0, 0);
6566 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
6567 try:
6568 fpc.flush();
6569 os.fsync(fpc.fileno());
6570 except io.UnsupportedOperation:
6571 pass;
6572 except AttributeError:
6573 pass;
6574 except OSError as e:
6575 pass;
6576 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6577 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6578 if(preservepermissions):
6579 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6580 if(preservetime):
6581 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6582 if(flinkinfo['ftype']==1):
6583 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6584 if(flinkinfo['ftype']==2):
6585 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6586 if(flinkinfo['ftype']==5):
6587 if(preservepermissions):
6588 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6589 else:
6590 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6591 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6592 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6593 if(preservepermissions):
6594 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6595 if(preservetime):
6596 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6597 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
6598 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6599 else:
6600 os.symlink(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6601 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6602 if(preservepermissions):
6603 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6604 else:
6605 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6606 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
6607 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
6608 if(preservepermissions):
6609 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6610 if(preservetime):
6611 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
6612 if(listcatfiles['ffilelist'][lcfi]['ftype']==6 and hasattr(os, "mkfifo")):
6613 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6614 lcfi = lcfi + 1;
6615 if(returnfp):
6616 return listcatfiles['ffilelist']['catfp'];
6617 else:
6618 return True;
6619
6620 create_alias_function("UnPack", __file_format_name__, "", UnPackArchiveFile);
6621
6622 if(hasattr(shutil, "register_unpack_format")):
6623 def UnPackArchiveFileFunc(archive_name, extract_dir=None, **kwargs):
6624 return UnPackArchiveFile(archive_name, extract_dir, False, 0, 0, False, __file_format_delimiter__, False, False);
6625 create_alias_function("UnPack", __file_format_name__, "Func", UnPackArchiveFileFunc);
6626
6627 def UnPackArchiveFileString(catstr, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_list__, verbose=False, returnfp=False):
6628 catfp = BytesIO(catstr);
6629 listcatfiles = UnPackArchiveFile(catfp, outdir, followlink, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6630 return listcatfiles;
6631
6632 create_alias_function("UnPack", __file_format_name__, "String", UnPackArchiveFileString);
6633
6634 def ArchiveFileListFiles(infile, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_list__, verbose=False, returnfp=False):
6635 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6636 if(isinstance(infile, dict)):
6637 listcatfiles = infile;
6638 else:
6639 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6640 infile = RemoveWindowsPath(infile);
6641 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, True, skipchecksum, formatspecs, returnfp);
6642 if(not listcatfiles):
6643 return False;
6644 lenlist = len(listcatfiles['ffilelist']);
6645 fnumfiles = int(listcatfiles['fnumfiles']);
6646 lcfi = 0;
6647 lcfx = int(listcatfiles['fnumfiles']);
6648 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6649 lcfx = int(lenlist);
6650 else:
6651 lcfx = int(listcatfiles['fnumfiles']);
6652 returnval = {};
6653 while(lcfi<lcfx):
6654 returnval.update({lcfi: listcatfiles['ffilelist'][lcfi]['fname']});
6655 if(not verbose):
6656 VerbosePrintOut(listcatfiles['ffilelist'][lcfi]['fname']);
6657 if(verbose):
6658 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6659 printfname = listcatfiles['ffilelist'][lcfi]['fname'];
6660 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6661 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " link to " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6662 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6663 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " -> " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6664 fuprint = listcatfiles['ffilelist'][lcfi]['funame'];
6665 if(len(fuprint)<=0):
6666 fuprint = listcatfiles['ffilelist'][lcfi]['fuid'];
6667 fgprint = listcatfiles['ffilelist'][lcfi]['fgname'];
6668 if(len(fgprint)<=0):
6669 fgprint = listcatfiles['ffilelist'][lcfi]['fgid'];
6670 VerbosePrintOut(PrintPermissionString(listcatfiles['ffilelist'][lcfi]['fmode'], listcatfiles['ffilelist'][lcfi]['ftype']) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(listcatfiles['ffilelist'][lcfi]['fsize']).rjust(15) + " " + datetime.datetime.utcfromtimestamp(listcatfiles['ffilelist'][lcfi]['fmtime']).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6671 lcfi = lcfi + 1;
6672 if(returnfp):
6673 return listcatfiles['catfp'];
6674 else:
6675 return True;
6676
6677 create_alias_function("", __file_format_name__, "ListFiles", ArchiveFileListFiles);
6678
6679 def ArchiveFileStringListFiles(catstr, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_list__, verbose=False, returnfp=False):
6680 catfp = BytesIO(catstr);
6681 listcatfiles = ArchiveFileListFiles(catstr, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6682 return listcatfiles;
6683
6684 create_alias_function("", __file_format_name__, "StringListFiles", ArchiveFileStringListFiles);
6685
6686 def TarFileListFiles(infile, verbose=False, returnfp=False):
6687 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6688 if(infile=="-"):
6689 infile = BytesIO();
6690 if(hasattr(sys.stdin, "buffer")):
6691 shutil.copyfileobj(sys.stdin.buffer, infile);
6692 else:
6693 shutil.copyfileobj(sys.stdin, infile);
6694 infile.seek(0, 0);
6695 if(not infile):
6696 return False;
6697 infile.seek(0, 0);
6698 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
6699 infile = download_file_from_internet_file(infile);
6700 infile.seek(0, 0);
6701 if(not infile):
6702 return False;
6703 infile.seek(0, 0);
6704 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
6705 return False;
6706 elif(os.path.exists(infile) and os.path.isfile(infile)):
6707 try:
6708 if(not tarfile.is_tarfile(infile)):
6709 return False;
6710 except AttributeError:
6711 if(not is_tarfile(infile)):
6712 return False;
6713 else:
6714 return False;
6715 try:
6716 if(hasattr(infile, "read") or hasattr(infile, "write")):
6717 tarfp = tarfile.open(fileobj=infile, mode="r");
6718 else:
6719 tarfp = tarfile.open(infile, "r");
6720 except FileNotFoundError:
6721 return False;
6722 lcfi = 0
6723 returnval = {};
6724 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
6725 returnval.update({lcfi: member.name});
6726 fpremode = member.mode;
6727 ffullmode = member.mode;
6728 flinkcount = 0;
6729 ftype = 0;
6730 if(member.isreg()):
6731 ffullmode = member.mode + stat.S_IFREG;
6732 ftype = 0;
6733 elif(member.isdev()):
6734 ffullmode = member.mode;
6735 ftype = 7;
6736 elif(member.islnk()):
6737 ffullmode = member.mode + stat.S_IFREG;
6738 ftype = 1;
6739 elif(member.issym()):
6740 ffullmode = member.mode + stat.S_IFLNK;
6741 ftype = 2;
6742 elif(member.ischr()):
6743 ffullmode = member.mode + stat.S_IFCHR;
6744 ftype = 3;
6745 elif(member.isblk()):
6746 ffullmode = member.mode + stat.S_IFBLK;
6747 ftype = 4;
6748 elif(member.isdir()):
6749 ffullmode = member.mode + stat.S_IFDIR;
6750 ftype = 5;
6751 elif(member.isfifo()):
6752 ffullmode = member.mode + stat.S_IFIFO;
6753 ftype = 6;
6754 elif(member.issparse()):
6755 ffullmode = member.mode;
6756 ftype = 12;
6757 if(not verbose):
6758 VerbosePrintOut(member.name);
6759 elif(verbose):
6760 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6761 printfname = member.name;
6762 if(member.islnk()):
6763 printfname = member.name + " link to " + member.linkname;
6764 elif(member.issym()):
6765 printfname = member.name + " -> " + member.linkname;
6766 fuprint = member.uname;
6767 if(len(fuprint)<=0):
6768 fuprint = member.uid;
6769 fgprint = member.gname;
6770 if(len(fgprint)<=0):
6771 fgprint = member.gid;
6772 VerbosePrintOut(PrintPermissionString(ffullmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(member.mtime).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6773 lcfi = lcfi + 1;
6774 if(returnfp):
6775 return listcatfiles['catfp'];
6776 else:
6777 return True;
6778
6779 def ZipFileListFiles(infile, verbose=False, returnfp=False):
6780 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6781 if(infile=="-"):
6782 infile = BytesIO();
6783 if(hasattr(sys.stdin, "buffer")):
6784 shutil.copyfileobj(sys.stdin.buffer, infile);
6785 else:
6786 shutil.copyfileobj(sys.stdin, infile);
6787 infile.seek(0, 0);
6788 if(not infile):
6789 return False;
6790 infile.seek(0, 0);
6791 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
6792 infile = download_file_from_internet_file(infile);
6793 infile.seek(0, 0);
6794 if(not infile):
6795 return False;
6796 infile.seek(0, 0);
6797 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
6798 return False;
6799 else:
6800 return False;
6801 if(not zipfile.is_zipfile(infile)):
6802 return False;
6803 try:
6804 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
6805 except FileNotFoundError:
6806 return False;
6807 lcfi = 0;
6808 returnval = {};
6809 ziptest = zipfp.testzip();
6810 if(ziptest):
6811 VerbosePrintOut("Bad file found!");
6812 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
6813 if(zipinfo.create_system==0 or zipinfo.create_system==10):
6814 fwinattributes = int(zipinfo.external_attr);
6815 if(not member.is_dir()):
6816 fmode = int(stat.S_IFREG + 438);
6817 fchmode = int(stat.S_IMODE(fmode));
6818 ftypemod = int(stat.S_IFMT(fmode));
6819 elif(member.is_dir()):
6820 fmode = int(stat.S_IFDIR + 511);
6821 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6822 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6823 elif(zipinfo.create_system==3):
6824 fwinattributes =int(0);
6825 fmode = int(zipinfo.external_attr);
6826 fchmode = int(stat.S_IMODE(fmode));
6827 ftypemod = int(stat.S_IFMT(fmode));
6828 else:
6829 fwinattributes = int(0);
6830 if(not member.is_dir()):
6831 fmode = int(stat.S_IFREG + 438);
6832 fchmode = int(stat.S_IMODE(fmode));
6833 ftypemod = int(stat.S_IFMT(fmode));
6834 elif(member.is_dir()):
6835 fmode = int(stat.S_IFDIR + 511);
6836 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6837 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6838 returnval.update({lcfi: member.filename});
6839 if(not verbose):
6840 VerbosePrintOut(member.filename);
6841 if(verbose):
6842 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6843 permissionstr = "";
6844 for fmodval in str(oct(fmode))[-3:]:
6845 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
6846 if(not member.is_dir()):
6847 ftype = 0;
6848 permissionstr = "-" + permissionstr;
6849 elif(member.is_dir()):
6850 ftype = 5;
6851 permissionstr = "d" + permissionstr;
6852 printfname = member.filename;
6853 try:
6854 fuid = int(os.getuid());
6855 except AttributeError:
6856 fuid = int(0);
6857 except KeyError:
6858 fuid = int(0);
6859 try:
6860 fgid = int(os.getgid());
6861 except AttributeError:
6862 fgid = int(0);
6863 except KeyError:
6864 fgid = int(0);
6865 try:
6866 import pwd;
6867 try:
6868 userinfo = pwd.getpwuid(os.getuid());
6869 funame = userinfo.pw_name;
6870 except KeyError:
6871 funame = "";
6872 except AttributeError:
6873 funame = "";
6874 except ImportError:
6875 funame = "";
6876 fgname = "";
6877 try:
6878 import grp;
6879 try:
6880 groupinfo = grp.getgrgid(os.getgid());
6881 fgname = groupinfo.gr_name;
6882 except KeyError:
6883 fgname = "";
6884 except AttributeError:
6885 fgname = "";
6886 except ImportError:
6887 fgname = "";
6888 fuprint = funame;
6889 if(len(fuprint)<=0):
6890 fuprint = str(fuid);
6891 fgprint = fgname;
6892 if(len(fgprint)<=0):
6893 fgprint = str(fgid);
6894 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(int(time.mktime(member.date_time + (0, 0, -1)))).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6895 lcfi = lcfi + 1;
6896 if(returnfp):
6897 return listcatfiles['catfp'];
6898 else:
6899 return True;
6900
6901 if(not rarfile_support):
6902 def RarFileListFiles(infile, verbose=False, returnfp=False):
6903 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6904 if(not os.path.exists(infile) or not os.path.isfile(infile)):
6905 return False;
6906
6907 if(rarfile_support):
6908 def RarFileListFiles(infile, verbose=False, returnfp=False):
6909 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6910 if(not os.path.exists(infile) or not os.path.isfile(infile)):
6911 return False;
6912 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
6913 return False;
6914 lcfi = 0;
6915 returnval = {};
6916 rarfp = rarfile.RarFile(infile, "r");
6917 rartest = rarfp.testrar();
6918 if(rartest):
6919 VerbosePrintOut("Bad file found!");
6920 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
6921 is_unix = False;
6922 is_windows = False;
6923 if(member.host_os==rarfile.RAR_OS_UNIX):
6924 is_windows = False;
6925 try:
6926 member.external_attr
6927 is_unix = True;
6928 except AttributeError:
6929 is_unix = False;
6930 elif(member.host_os==rarfile.RAR_OS_WIN32):
6931 is_unix = False;
6932 try:
6933 member.external_attr
6934 is_windows = True;
6935 except AttributeError:
6936 is_windows = False;
6937 else:
6938 is_unix = False;
6939 is_windows = False;
6940 if(is_unix and member.external_attr !=0):
6941 fpremode = int(member.external_attr);
6942 elif(member.is_file()):
6943 fpremode = int(stat.S_IFREG + 438);
6944 elif(member.is_symlink()):
6945 fpremode = int(stat.S_IFLNK + 438);
6946 elif(member.is_dir()):
6947 fpremode = int(stat.S_IFDIR + 511);
6948 if(is_windows and member.external_attr !=0):
6949 fwinattributes = int(member.external_attr);
6950 else:
6951 fwinattributes = int(0);
6952 if(is_unix and member.external_attr !=0):
6953 fmode = int(member.external_attr);
6954 fchmode = int(stat.S_IMODE(member.external_attr));
6955 ftypemod = int(stat.S_IFMT(member.external_attr));
6956 elif(member.is_file()):
6957 fmode = int(stat.S_IFREG + 438);
6958 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
6959 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
6960 elif(member.is_symlink()):
6961 fmode = int(stat.S_IFLNK + 438);
6962 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
6963 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
6964 elif(member.is_dir()):
6965 fmode = int(stat.S_IFDIR + 511);
6966 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6967 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6968 returnval.update({lcfi: member.filename});
6969 if(not verbose):
6970 VerbosePrintOut(member.filename);
6971 if(verbose):
6972 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6973 permissionstr = "";
6974 for fmodval in str(oct(fmode))[-3:]:
6975 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
6976 if(member.is_file()):
6977 ftype = 0;
6978 permissionstr = "-" + permissionstr;
6979 printfname = member.filename;
6980 elif(member.is_symlink()):
6981 ftype = 2;
6982 permissionstr = "l" + permissionstr;
6983 printfname = member.name + " -> " + member.read().decode("UTF-8");
6984 elif(member.is_dir()):
6985 ftype = 5;
6986 permissionstr = "d" + permissionstr;
6987 printfname = member.filename;
6988 try:
6989 fuid = int(os.getuid());
6990 except AttributeError:
6991 fuid = int(0);
6992 except KeyError:
6993 fuid = int(0);
6994 try:
6995 fgid = int(os.getgid());
6996 except AttributeError:
6997 fgid = int(0);
6998 except KeyError:
6999 fgid = int(0);
7000 try:
7001 import pwd;
7002 try:
7003 userinfo = pwd.getpwuid(os.getuid());
7004 funame = userinfo.pw_name;
7005 except KeyError:
7006 funame = "";
7007 except AttributeError:
7008 funame = "";
7009 except ImportError:
7010 funame = "";
7011 fgname = "";
7012 try:
7013 import grp;
7014 try:
7015 groupinfo = grp.getgrgid(os.getgid());
7016 fgname = groupinfo.gr_name;
7017 except KeyError:
7018 fgname = "";
7019 except AttributeError:
7020 fgname = "";
7021 except ImportError:
7022 fgname = "";
7023 fuprint = funame;
7024 if(len(fuprint)<=0):
7025 fuprint = str(fuid);
7026 fgprint = fgname;
7027 if(len(fgprint)<=0):
7028 fgprint = str(fgid);
7029 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + member.mtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
7030 lcfi = lcfi + 1;
7031 if(returnfp):
7032 return listcatfiles['catfp'];
7033 else:
7034 return True;
7035
7036 if(not py7zr_support):
7037 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
7038 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7039 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7040 return False;
7041
7042 if(py7zr_support):
7043 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
7044 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7045 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7046 return False;
7047 lcfi = 0;
7048 returnval = {};
7049 szpfp = py7zr.SevenZipFile(infile, mode="r");
7050 file_content = szpfp.readall();
7051 #sztest = szpfp.testzip();
7052 sztestalt = szpfp.test();
7053 if(sztestalt):
7054 VerbosePrintOut("Bad file found!");
7055 for member in sorted(szpfp.list(), key=lambda x: x.filename):
7056 if(re.findall("^[.|/]", member.filename)):
7057 fname = member.filename;
7058 else:
7059 fname = "./"+member.filename;
7060 if(not member.is_directory):
7061 fpremode = int(stat.S_IFREG + 438);
7062 elif(member.is_directory):
7063 fpremode = int(stat.S_IFDIR + 511);
7064 fwinattributes = int(0);
7065 if(member.is_directory):
7066 fmode = int(stat.S_IFDIR + 511);
7067 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7068 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7069 else:
7070 fmode = int(stat.S_IFLNK + 438);
7071 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
7072 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
7073 returnval.update({lcfi: member.filename});
7074 if(not verbose):
7075 VerbosePrintOut(member.filename);
7076 if(verbose):
7077 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7078 permissionstr = "";
7079 for fmodval in str(oct(fmode))[-3:]:
7080 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
7081 fsize = int("0");
7082 if(not member.is_directory):
7083 ftype = 0;
7084 permissionstr = "-" + permissionstr;
7085 printfname = member.filename;
7086 elif(member.is_directory):
7087 ftype = 5;
7088 permissionstr = "d" + permissionstr;
7089 printfname = member.filename;
7090 if(ftype==0):
7091 fsize = len(file_content[member.filename].read());
7092 file_content[member.filename].close();
7093 try:
7094 fuid = int(os.getuid());
7095 except AttributeError:
7096 fuid = int(0);
7097 except KeyError:
7098 fuid = int(0);
7099 try:
7100 fgid = int(os.getgid());
7101 except AttributeError:
7102 fgid = int(0);
7103 except KeyError:
7104 fgid = int(0);
7105 try:
7106 import pwd;
7107 try:
7108 userinfo = pwd.getpwuid(os.getuid());
7109 funame = userinfo.pw_name;
7110 except KeyError:
7111 funame = "";
7112 except AttributeError:
7113 funame = "";
7114 except ImportError:
7115 funame = "";
7116 fgname = "";
7117 try:
7118 import grp;
7119 try:
7120 groupinfo = grp.getgrgid(os.getgid());
7121 fgname = groupinfo.gr_name;
7122 except KeyError:
7123 fgname = "";
7124 except AttributeError:
7125 fgname = "";
7126 except ImportError:
7127 fgname = "";
7128 fuprint = funame;
7129 if(len(fuprint)<=0):
7130 fuprint = str(fuid);
7131 fgprint = fgname;
7132 if(len(fgprint)<=0):
7133 fgprint = str(fgid);
7134 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(fsize).rjust(15) + " " + member.creationtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
7135 lcfi = lcfi + 1;
7136 if(returnfp):
7137 return listcatfiles['catfp'];
7138 else:
7139 return True;
7140
7141 def InFileListFiles(infile, verbose=False, formatspecs=__file_format_list__, returnfp=False):
7142 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7143 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
7144 if(checkcompressfile=="tarfile"):
7145 return TarFileListFiles(infile, verbose, returnfp);
7146 elif(checkcompressfile=="zipfile"):
7147 return ZipFileListFiles(infile, verbose, returnfp);
7148 elif(checkcompressfile=="catfile"):
7149 return ArchiveFileListFiles(infile, 0, 0, False, formatspecs, verbose, returnfp);
7150 elif(rarfile_support and checkcompressfile=="rarfile"):
7151 return RarFileListFiles(infile, verbose, returnfp);
7152 elif(py7zr_support and checkcompressfile=="7zipfile"):
7153 return SevenZipFileListFiles(infile, verbose, returnfp);
7154 else:
7155 return False;
7156 return False;
7157
7158 def ListDirListFiles(infiles, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False, returnfp=False):
7159 outarray = BytesIO();
7160 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compressionlevel, followlink, checksumtype, formatspecs, False, True);
7161 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
7162 return listcatfiles;
7163
7164 def ListDirListFilesAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False, returnfp=False):
7165 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, listonly, checksumtype, formatspecs, verbose);
7166 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
7167 return listcatfiles;
7168
7169 def PackArchiveFileFromListDirAlt(infiles, outfile, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
7170 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, False, checksumtype, extradata, formatspecs, False);
7171 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compressionlevel, followlink, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
7172 return listcatfiles;
7173
7174 create_alias_function("Pack", __file_format_name__, "FromListDirAlt", PackArchiveFileFromListDirAlt);
7175
7176 def PackArchiveFileFromTarFileAlt(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
7177 outarray = TarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7178 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7179 return listcatfiles;
7180
7181 create_alias_function("Pack", __file_format_name__, "FromTarFileAlt", PackArchiveFileFromTarFileAlt);
7182
7183 def PackArchiveFileFromZipFileAlt(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
7184 outarray = ZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7185 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7186 return listcatfiles;
7187
7188 create_alias_function("Pack", __file_format_name__, "FromZipFileAlt", PackArchiveFileFromZipFileAlt);
7189
7190 if(not rarfile_support):
7191 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
7192 return False;
7193
7194 if(rarfile_support):
7195 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
7196 outarray = RarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7197 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7198 return listcatfiles;
7199
7200 create_alias_function("Pack", __file_format_name__, "FromRarFileAlt", PackArchiveFileFromRarFileAlt);
7201
7202 if(not py7zr_support):
7203 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
7204 return False;
7205
7206 if(py7zr_support):
7207 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
7208 outarray = SevenZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7209 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7210 return listcatfiles;
7211
7212 create_alias_function("Pack", __file_format_name__, "FromSevenZipFileAlt", PackArchiveFileFromSevenZipFileAlt);
7213
7214 def download_file_from_ftp_file(url):
7215 urlparts = urlparse(url);
7216 file_name = os.path.basename(urlparts.path);
7217 file_dir = os.path.dirname(urlparts.path);
7218 if(urlparts.username is not None):
7219 ftp_username = urlparts.username;
7220 else:
7221 ftp_username = "anonymous";
7222 if(urlparts.password is not None):
7223 ftp_password = urlparts.password;
7224 elif(urlparts.password is None and urlparts.username=="anonymous"):
7225 ftp_password = "anonymous";
7226 else:
7227 ftp_password = "";
7228 if(urlparts.scheme=="ftp"):
7229 ftp = FTP();
7230 elif(urlparts.scheme=="ftps"):
7231 ftp = FTP_TLS();
7232 else:
7233 return False;
7234 if(urlparts.scheme=="sftp"):
7235 if(__use_pysftp__):
7236 return download_file_from_pysftp_file(url);
7237 else:
7238 return download_file_from_sftp_file(url);
7239 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7240 return download_file_from_http_file(url);
7241 ftp_port = urlparts.port;
7242 if(urlparts.port is None):
7243 ftp_port = 21;
7244 try:
7245 ftp.connect(urlparts.hostname, ftp_port);
7246 except socket.gaierror:
7247 log.info("Error With URL "+url);
7248 return False;
7249 except socket.timeout:
7250 log.info("Error With URL "+url);
7251 return False;
7252 ftp.login(urlparts.username, urlparts.password);
7253 if(urlparts.scheme=="ftps"):
7254 ftp.prot_p();
7255 ftpfile = BytesIO();
7256 ftp.retrbinary("RETR "+urlparts.path, ftpfile.write);
7257 #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
7258 ftp.close();
7259 ftpfile.seek(0, 0);
7260 return ftpfile;
7261
7262 def download_file_from_ftp_string(url):
7263 ftpfile = download_file_from_ftp_file(url);
7264 return ftpfile.read();
7265
7266 def upload_file_to_ftp_file(ftpfile, url):
7267 urlparts = urlparse(url);
7268 file_name = os.path.basename(urlparts.path);
7269 file_dir = os.path.dirname(urlparts.path);
7270 if(urlparts.username is not None):
7271 ftp_username = urlparts.username;
7272 else:
7273 ftp_username = "anonymous";
7274 if(urlparts.password is not None):
7275 ftp_password = urlparts.password;
7276 elif(urlparts.password is None and urlparts.username=="anonymous"):
7277 ftp_password = "anonymous";
7278 else:
7279 ftp_password = "";
7280 if(urlparts.scheme=="ftp"):
7281 ftp = FTP();
7282 elif(urlparts.scheme=="ftps"):
7283 ftp = FTP_TLS();
7284 else:
7285 return False;
7286 if(urlparts.scheme=="sftp"):
7287 if(__use_pysftp__):
7288 return upload_file_to_pysftp_file(url);
7289 else:
7290 return upload_file_to_sftp_file(url);
7291 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7292 return False;
7293 ftp_port = urlparts.port;
7294 if(urlparts.port is None):
7295 ftp_port = 21;
7296 try:
7297 ftp.connect(urlparts.hostname, ftp_port);
7298 except socket.gaierror:
7299 log.info("Error With URL "+url);
7300 return False;
7301 except socket.timeout:
7302 log.info("Error With URL "+url);
7303 return False;
7304 ftp.login(urlparts.username, urlparts.password);
7305 if(urlparts.scheme=="ftps"):
7306 ftp.prot_p();
7307 ftp.storbinary("STOR "+urlparts.path, ftpfile);
7308 ftp.close();
7309 ftpfile.seek(0, 0);
7310 return ftpfile;
7311
7312 def upload_file_to_ftp_string(ftpstring, url):
7313 ftpfileo = BytesIO(ftpstring);
7314 ftpfile = upload_file_to_ftp_file(ftpfileo, url);
7315 ftpfileo.close();
7316 return ftpfile;
7317
7318 def download_file_from_http_file(url, headers=geturls_headers_pycatfile_python_alt):
7319 # Parse the URL to extract username and password if present
7320 urlparts = urlparse(url);
7321 username = urlparts.username;
7322 password = urlparts.password;
7323 # Rebuild the URL without the username and password
7324 netloc = urlparts.hostname;
7325 if(urlparts.scheme=="sftp"):
7326 if(__use_pysftp__):
7327 return download_file_from_pysftp_file(url);
7328 else:
7329 return download_file_from_sftp_file(url);
7330 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7331 return download_file_from_ftp_file(url);
7332 if urlparts.port:
7333 netloc += ':' + str(urlparts.port);
7334 rebuilt_url = urlunparse((urlparts.scheme, netloc, urlparts.path, urlparts.params, urlparts.query, urlparts.fragment));
7335 # Create a temporary file object
7336 httpfile = BytesIO();
7337 if haverequests:
7338 # Use the requests library if available
7339 if username and password:
7340 response = requests.get(rebuilt_url, headers=headers, auth=(username, password), stream=True);
7341 else:
7342 response = requests.get(rebuilt_url, headers=headers, stream=True);
7343 response.raw.decode_content = True
7344 shutil.copyfileobj(response.raw, httpfile);
7345 else:
7346 # Build a Request object for urllib
7347 request = Request(rebuilt_url, headers=headers);
7348 # Create an opener object for handling URLs
7349 if username and password:
7350 # Create a password manager
7351 password_mgr = HTTPPasswordMgrWithDefaultRealm();
7352 # Add the username and password
7353 password_mgr.add_password(None, rebuilt_url, username, password);
7354 # Create an authentication handler using the password manager
7355 auth_handler = HTTPBasicAuthHandler(password_mgr);
7356 # Build the opener with the authentication handler
7357 opener = build_opener(auth_handler);
7358 else:
7359 opener = build_opener();
7360 with opener.open(request) as response:
7361 shutil.copyfileobj(response, httpfile);
7362 # Reset file pointer to the start
7363 httpfile.seek(0, 0);
7364 # Return the temporary file object
7365 return httpfile;
7366
7367 def download_file_from_http_string(url, headers=geturls_headers_pycatfile_python_alt):
7368 httpfile = download_file_from_http_file(url, headers);
7369 return ftpfile.read();
7370
7371 if(haveparamiko):
7372 def download_file_from_sftp_file(url):
7373 urlparts = urlparse(url);
7374 file_name = os.path.basename(urlparts.path);
7375 file_dir = os.path.dirname(urlparts.path);
7376 sftp_port = urlparts.port;
7377 if(urlparts.port is None):
7378 sftp_port = 22;
7379 else:
7380 sftp_port = urlparts.port;
7381 if(urlparts.username is not None):
7382 sftp_username = urlparts.username;
7383 else:
7384 sftp_username = "anonymous";
7385 if(urlparts.password is not None):
7386 sftp_password = urlparts.password;
7387 elif(urlparts.password is None and urlparts.username=="anonymous"):
7388 sftp_password = "anonymous";
7389 else:
7390 sftp_password = "";
7391 if(urlparts.scheme=="ftp"):
7392 return download_file_from_ftp_file(url);
7393 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7394 return download_file_from_http_file(url);
7395 if(urlparts.scheme!="sftp"):
7396 return False;
7397 ssh = paramiko.SSHClient();
7398 ssh.load_system_host_keys();
7399 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
7400 try:
7401 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7402 except paramiko.ssh_exception.SSHException:
7403 return False;
7404 except socket.gaierror:
7405 log.info("Error With URL "+url);
7406 return False;
7407 except socket.timeout:
7408 log.info("Error With URL "+url);
7409 return False;
7410 sftp = ssh.open_sftp();
7411 sftpfile = BytesIO();
7412 sftp.getfo(urlparts.path, sftpfile);
7413 sftp.close();
7414 ssh.close();
7415 sftpfile.seek(0, 0);
7416 return sftpfile;
7417 else:
7418 def download_file_from_sftp_file(url):
7419 return False;
7420
7421 if(haveparamiko):
7422 def download_file_from_sftp_string(url):
7423 sftpfile = download_file_from_sftp_file(url);
7424 return sftpfile.read();
7425 else:
7426 def download_file_from_ftp_string(url):
7427 return False;
7428
7429 if(haveparamiko):
7430 def upload_file_to_sftp_file(sftpfile, url):
7431 urlparts = urlparse(url);
7432 file_name = os.path.basename(urlparts.path);
7433 file_dir = os.path.dirname(urlparts.path);
7434 sftp_port = urlparts.port;
7435 if(urlparts.port is None):
7436 sftp_port = 22;
7437 else:
7438 sftp_port = urlparts.port;
7439 if(urlparts.username is not None):
7440 sftp_username = urlparts.username;
7441 else:
7442 sftp_username = "anonymous";
7443 if(urlparts.password is not None):
7444 sftp_password = urlparts.password;
7445 elif(urlparts.password is None and urlparts.username=="anonymous"):
7446 sftp_password = "anonymous";
7447 else:
7448 sftp_password = "";
7449 if(urlparts.scheme=="ftp"):
7450 return upload_file_to_ftp_file(url);
7451 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7452 return False;
7453 if(urlparts.scheme!="sftp"):
7454 return False;
7455 ssh = paramiko.SSHClient();
7456 ssh.load_system_host_keys();
7457 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
7458 try:
7459 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7460 except paramiko.ssh_exception.SSHException:
7461 return False;
7462 except socket.gaierror:
7463 log.info("Error With URL "+url);
7464 return False;
7465 except socket.timeout:
7466 log.info("Error With URL "+url);
7467 return False;
7468 sftp = ssh.open_sftp();
7469 sftp.putfo(sftpfile, urlparts.path);
7470 sftp.close();
7471 ssh.close();
7472 sftpfile.seek(0, 0);
7473 return sftpfile;
7474 else:
7475 def upload_file_to_sftp_file(sftpfile, url):
7476 return False;
7477
7478 if(haveparamiko):
7479 def upload_file_to_sftp_string(sftpstring, url):
7480 sftpfileo = BytesIO(sftpstring);
7481 sftpfile = upload_file_to_sftp_files(ftpfileo, url);
7482 sftpfileo.close();
7483 return sftpfile;
7484 else:
7485 def upload_file_to_sftp_string(url):
7486 return False;
7487
7488 if(havepysftp):
7489 def download_file_from_pysftp_file(url):
7490 urlparts = urlparse(url);
7491 file_name = os.path.basename(urlparts.path);
7492 file_dir = os.path.dirname(urlparts.path);
7493 sftp_port = urlparts.port;
7494 if(urlparts.port is None):
7495 sftp_port = 22;
7496 else:
7497 sftp_port = urlparts.port;
7498 if(urlparts.username is not None):
7499 sftp_username = urlparts.username;
7500 else:
7501 sftp_username = "anonymous";
7502 if(urlparts.password is not None):
7503 sftp_password = urlparts.password;
7504 elif(urlparts.password is None and urlparts.username=="anonymous"):
7505 sftp_password = "anonymous";
7506 else:
7507 sftp_password = "";
7508 if(urlparts.scheme=="ftp"):
7509 return download_file_from_ftp_file(url);
7510 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7511 return download_file_from_http_file(url);
7512 if(urlparts.scheme!="sftp"):
7513 return False;
7514 try:
7515 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7516 except paramiko.ssh_exception.SSHException:
7517 return False;
7518 except socket.gaierror:
7519 log.info("Error With URL "+url);
7520 return False;
7521 except socket.timeout:
7522 log.info("Error With URL "+url);
7523 return False;
7524 sftp = ssh.open_sftp();
7525 sftpfile = BytesIO();
7526 sftp.getfo(urlparts.path, sftpfile);
7527 sftp.close();
7528 ssh.close();
7529 sftpfile.seek(0, 0);
7530 return sftpfile;
7531 else:
7532 def download_file_from_pysftp_file(url):
7533 return False;
7534
7535 if(havepysftp):
7536 def download_file_from_pysftp_string(url):
7537 sftpfile = download_file_from_pysftp_file(url);
7538 return sftpfile.read();
7539 else:
7540 def download_file_from_ftp_string(url):
7541 return False;
7542
7543 if(havepysftp):
7544 def upload_file_to_pysftp_file(sftpfile, url):
7545 urlparts = urlparse(url);
7546 file_name = os.path.basename(urlparts.path);
7547 file_dir = os.path.dirname(urlparts.path);
7548 sftp_port = urlparts.port;
7549 if(urlparts.port is None):
7550 sftp_port = 22;
7551 else:
7552 sftp_port = urlparts.port;
7553 if(urlparts.username is not None):
7554 sftp_username = urlparts.username;
7555 else:
7556 sftp_username = "anonymous";
7557 if(urlparts.password is not None):
7558 sftp_password = urlparts.password;
7559 elif(urlparts.password is None and urlparts.username=="anonymous"):
7560 sftp_password = "anonymous";
7561 else:
7562 sftp_password = "";
7563 if(urlparts.scheme=="ftp"):
7564 return upload_file_to_ftp_file(url);
7565 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7566 return False;
7567 if(urlparts.scheme!="sftp"):
7568 return False;
7569 try:
7570 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7571 except paramiko.ssh_exception.SSHException:
7572 return False;
7573 except socket.gaierror:
7574 log.info("Error With URL "+url);
7575 return False;
7576 except socket.timeout:
7577 log.info("Error With URL "+url);
7578 return False;
7579 sftp = ssh.open_sftp();
7580 sftp.putfo(sftpfile, urlparts.path);
7581 sftp.close();
7582 ssh.close();
7583 sftpfile.seek(0, 0);
7584 return sftpfile;
7585 else:
7586 def upload_file_to_pysftp_file(sftpfile, url):
7587 return False;
7588
7589 if(havepysftp):
7590 def upload_file_to_pysftp_string(sftpstring, url):
7591 sftpfileo = BytesIO(sftpstring);
7592 sftpfile = upload_file_to_pysftp_files(ftpfileo, url);
7593 sftpfileo.close();
7594 return sftpfile;
7595 else:
7596 def upload_file_to_pysftp_string(url):
7597 return False;
7598
7599 def download_file_from_internet_file(url, headers=geturls_headers_pycatfile_python_alt):
7600 urlparts = urlparse(url);
7601 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7602 return download_file_from_http_file(url, headers);
7603 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7604 return download_file_from_ftp_file(url);
7605 elif(urlparts.scheme=="sftp"):
7606 if(__use_pysftp__ and havepysftp):
7607 return download_file_from_pysftp_file(url);
7608 else:
7609 return download_file_from_sftp_file(url);
7610 else:
7611 return False;
7612 return False;
7613
7614 def download_file_from_internet_uncompress_file(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_list__):
7615 fp = download_file_from_internet_file(url);
7616 fp = UncompressArchiveFile(fp, formatspecs);
7617 fp.seek(0, 0);
7618 if(not fp):
7619 return False;
7620 return fp;
7621
7622 def download_file_from_internet_string(url, headers=geturls_headers_pycatfile_python_alt):
7623 urlparts = urlparse(url);
7624 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7625 return download_file_from_http_string(url, headers);
7626 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7627 return download_file_from_ftp_string(url);
7628 elif(urlparts.scheme=="sftp"):
7629 if(__use_pysftp__ and havepysftp):
7630 return download_file_from_pysftp_string(url);
7631 else:
7632 return download_file_from_sftp_string(url);
7633 else:
7634 return False;
7635 return False;
7636
7637 def download_file_from_internet_uncompress_string(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_list__):
7638 fp = download_file_from_internet_string(url);
7639 fp = UncompressArchiveFile(fp, formatspecs);
7640 fp.seek(0, 0);
7641 if(not fp):
7642 return False;
7643 return fp;
7644
7645 def upload_file_to_internet_file(ifp, url):
7646 urlparts = urlparse(url);
7647 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7648 return False;
7649 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7650 return upload_file_to_ftp_file(ifp, url);
7651 elif(urlparts.scheme=="sftp"):
7652 if(__use_pysftp__ and havepysftp):
7653 return upload_file_to_pysftp_file(ifp, url);
7654 else:
7655 return upload_file_to_sftp_file(ifp, url);
7656 else:
7657 return False;
7658 return False;
7659
7660 def upload_file_to_internet_compress_file(ifp, url, formatspecs=__file_format_list__):
7661 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
7662 if(not catfileout):
7663 return False;
7664 catfp.seek(0, 0);
7665 upload_file_to_internet_file(catfp, outfile);
7666 return True;
7667
7668 def upload_file_to_internet_string(ifp, url):
7669 urlparts = urlparse(url);
7670 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7671 return False;
7672 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7673 return upload_file_to_ftp_string(ifp, url);
7674 elif(urlparts.scheme=="sftp"):
7675 if(__use_pysftp__ and havepysftp):
7676 return upload_file_to_pysftp_string(ifp, url);
7677 else:
7678 return upload_file_to_sftp_string(ifp, url);
7679 else:
7680 return False;
7681 return False;
7682
7683 def upload_file_to_internet_compress_string(ifp, url, formatspecs=__file_format_list__):
7684 catfp = CompressArchiveFile(BytesIO(ifp), compression, compressionlevel, formatspecs);
7685 if(not catfileout):
7686 return False;
7687 catfp.seek(0, 0);
7688 upload_file_to_internet_file(catfp, outfile);
7689 return True;
7690
7691 try:
7692 if(hasattr(shutil, "register_archive_format")):
7693 # Register the packing format
7694 shutil.register_archive_format(__file_format_name__, PackArchiveFileFunc, description='Pack concatenated files');
7695 except shutil.RegistryError:
7696 pass;
7697
7698 try:
7699 if(hasattr(shutil, "register_unpack_format")):
7700 # Register the unpacking format
7701 shutil.register_unpack_format(__file_format_name__, archivefile_extensions, UnPackArchiveFileFunc, description='UnPack concatenated files');
7702 except shutil.RegistryError:
7703 pass;
A tar like file format name catfile after unix cat command (concatenate files).