?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Add files via upload
[PyCatFile.git] / pycatfile.py
bloba720bff106d8b71867c04a7a44c46974fae87cd6
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 '''
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the Revised BSD License.
7
8 This program is distributed in the hope that it will be useful,
9 but WITHOUT ANY WARRANTY; without even the implied warranty of
10 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 Revised BSD License for more details.
12
13 Copyright 2018-2024 Cool Dude 2k - http://idb.berlios.de/
14 Copyright 2018-2024 Game Maker 2k - http://intdb.sourceforge.net/
15 Copyright 2018-2024 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
16
17 $FileInfo: pycatfile.py - Last Update: 5/12/2024 Ver. 0.11.0 RC 1 - Author: cooldude2k $
18 '''
19
20 from __future__ import absolute_import, division, print_function, unicode_literals;
21 import io, os, re, sys, time, stat, zlib, base64, shutil, socket, hashlib, datetime, logging, binascii, tempfile, zipfile, platform;
22 ftpssl = True;
23 try:
24 ftpssl = True;
25 from ftplib import FTP, FTP_TLS;
26 except ImportError:
27 ftpssl = False;
28 from ftplib import FTP;
29 if(sys.version[0]=="2"):
30 from urlparse import urlparse, urlunparse;
31 elif(sys.version[0]>="3"):
32 from urllib.parse import urlunparse;
33 from urllib.parse import urlparse;
34
35 if os.name == 'nt': # Only modify if on Windows
36 if sys.version[0] == "2":
37 import codecs;
38 sys.stdout = codecs.getwriter('utf-8')(sys.stdout);
39 sys.stderr = codecs.getwriter('utf-8')(sys.stderr);
40 else:
41 import io;
42 sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace', line_buffering=True);
43 sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', errors='replace', line_buffering=True);
44
45 hashlib_guaranteed = False;
46 os.environ["PYTHONIOENCODING"] = "UTF-8";
47 os.environ["LC_CTYPE"] = "UTF-8";
48 try:
49 reload(sys);
50 try:
51 sys.setdefaultencoding('UTF-8');
52 except NameError:
53 pass;
54 except AttributeError:
55 pass;
56 except NameError:
57 pass;
58 except AttributeError:
59 pass;
60
61 try:
62 from zlib import crc32;
63 except ImportError:
64 from binascii import crc32;
65
66 if(sys.version_info[0]==2):
67 FileNotFoundError = IOError;
68
69 rarfile_support = False;
70 try:
71 import rarfile;
72 rarfile_support = True;
73 except ImportError:
74 rarfile_support = False;
75
76 py7zr_support = False;
77 try:
78 import py7zr;
79 py7zr_support = True;
80 except ImportError:
81 py7zr_support = False;
82
83 try:
84 from xtarfile import is_tarfile;
85 except ImportError:
86 try:
87 from safetar import is_tarfile;
88 except ImportError:
89 from tarfile import is_tarfile;
90
91 try:
92 import xtarfile as tarfile;
93 except ImportError:
94 try:
95 import safetar as tarfile;
96 except ImportError:
97 import tarfile;
98
99 haveparamiko = False;
100 try:
101 import paramiko;
102 haveparamiko = True;
103 except ImportError:
104 haveparamiko = False;
105
106 havepysftp = False;
107 try:
108 import pysftp;
109 havepysftp = True;
110 except ImportError:
111 havepysftp = False;
112
113 haverequests = False;
114 try:
115 import requests;
116 haverequests = True;
117 except ImportError:
118 haverequests = False;
119
120 if(haverequests):
121 import urllib3;
122 logging.getLogger("urllib3").setLevel(logging.WARNING);
123
124 try:
125 # Python 3 imports
126 from urllib.request import Request, build_opener, HTTPBasicAuthHandler;
127 from urllib.parse import urlparse;
128 except ImportError:
129 # Python 2 imports
130 from urllib2 import Request, build_opener, HTTPBasicAuthHandler;
131 from urlparse import urlparse;
132
133 if(sys.version[0]=="2"):
134 try:
135 from io import StringIO, BytesIO;
136 except ImportError:
137 try:
138 from cStringIO import StringIO;
139 from cStringIO import StringIO as BytesIO;
140 except ImportError:
141 from StringIO import StringIO;
142 from StringIO import StringIO as BytesIO;
143 elif(sys.version[0]>="3"):
144 from io import StringIO, BytesIO;
145 else:
146 teststringio = 0;
147 if(teststringio<=0):
148 try:
149 from cStringIO import StringIO as BytesIO;
150 teststringio = 1;
151 except ImportError:
152 teststringio = 0;
153 if(teststringio<=0):
154 try:
155 from StringIO import StringIO as BytesIO;
156 teststringio = 2;
157 except ImportError:
158 teststringio = 0;
159 if(teststringio<=0):
160 try:
161 from io import BytesIO;
162 teststringio = 3;
163 except ImportError:
164 teststringio = 0;
165
166 __use_pysftp__ = False;
167 if(not havepysftp):
168 __use_pysftp__ = False;
169 __file_format_name__ = "CatFile";
170 __program_name__ = "Py"+__file_format_name__;
171 __file_format_lower__ = __file_format_name__.lower();
172 __file_format_magic__ = __file_format_name__;
173 __file_format_len__ = len(__file_format_magic__);
174 __file_format_hex__ = binascii.hexlify(__file_format_magic__.encode("UTF-8")).decode("UTF-8");
175 __file_format_delimiter__ = "\x00";
176 __file_format_ver__ = "001";
177 __use_new_style__ = True;
178 __use_advanced_list__ = True;
179 __use_alt_inode__ = False;
180 __file_format_list__ = [__file_format_name__, __file_format_magic__, __file_format_lower__, __file_format_len__, __file_format_hex__, __file_format_delimiter__, __file_format_ver__, __use_new_style__, __use_advanced_list__, __use_alt_inode__];
181 __file_format_dict__ = {'format_name': __file_format_name__, 'format_magic': __file_format_magic__, 'format_lower': __file_format_lower__, 'format_len': __file_format_len__, 'format_hex': __file_format_hex__, 'format_delimiter': __file_format_delimiter__, 'format_ver': __file_format_ver__, 'new_style': __use_new_style__, 'use_advanced_list': __use_advanced_list__, 'use_alt_inode': __use_alt_inode__};
182 __project__ = __program_name__;
183 __project_url__ = "https://github.com/GameMaker2k/PyCatFile";
184 __version_info__ = (0, 11, 0, "RC 1", 1);
185 __version_date_info__ = (2024, 5, 12, "RC 1", 1);
186 __version_date__ = str(__version_date_info__[0]) + "." + str(__version_date_info__[1]).zfill(2) + "." + str(__version_date_info__[2]).zfill(2);
187 __revision__ = __version_info__[3];
188 __revision_id__ = "$Id$";
189 if(__version_info__[4] is not None):
190 __version_date_plusrc__ = __version_date__ + "-" + str(__version_date_info__[4]);
191 if(__version_info__[4] is None):
192 __version_date_plusrc__ = __version_date__;
193 if(__version_info__[3] is not None):
194 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]) + " " + str(__version_info__[3]);
195 if(__version_info__[3] is None):
196 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]);
197
198 PyBitness = platform.architecture();
199 if(PyBitness=="32bit" or PyBitness=="32"):
200 PyBitness = "32";
201 elif(PyBitness=="64bit" or PyBitness=="64"):
202 PyBitness = "64";
203 else:
204 PyBitness = "32";
205
206 geturls_ua_pycatfile_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
207 if(platform.python_implementation()!=""):
208 py_implementation = platform.python_implementation();
209 if(platform.python_implementation()==""):
210 py_implementation = "Python";
211 geturls_ua_pycatfile_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=py_implementation, pyver=platform.python_version(), proname=__project__, prover=__version__);
212 geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
213 geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
214 geturls_headers_pycatfile_python = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
215 geturls_headers_pycatfile_python_alt = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python_alt, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
216 geturls_headers_googlebot_google = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
217 geturls_headers_googlebot_google_old = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
218
219 compressionlist = ['auto', 'gzip', 'bzip2', 'zstd', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
220 compressionlistalt = ['gzip', 'bzip2', 'zstd', 'lz4', 'lzo', 'lzma', 'xz'];
221 outextlist = ['gz', 'bz2', 'zst', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
222 outextlistwd = ['.gz', '.bz2', '.zst', '.lz4', '.lzo', '.lzop', '.lzma', '.xz'];
223
224 tarfile_mimetype = "application/tar";
225 tarfile_tar_mimetype = tarfile_mimetype;
226 zipfile_mimetype = "application/zip";
227 zipfile_zip_mimetype = zipfile_mimetype;
228 rarfile_mimetype = "application/rar";
229 rarfile_rar_mimetype = rarfile_mimetype;
230 archivefile_mimetype = "application/x-"+__file_format_dict__['format_lower']+"";
231 archivefile_cat_mimetype = archivefile_mimetype;
232 archivefile_gzip_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+gzip";
233 archivefile_gz_mimetype = archivefile_gzip_mimetype;
234 archivefile_bzip2_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+bzip2";
235 archivefile_bz2_mimetype = archivefile_bzip2_mimetype;
236 archivefile_lz4_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lz4";
237 archivefile_lzop_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lzop";
238 archivefile_lzo_mimetype = archivefile_lzop_mimetype;
239 archivefile_zstandard_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+zstandard";
240 archivefile_zstd_mimetype = archivefile_zstandard_mimetype;
241 archivefile_lzma_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lzma";
242 archivefile_xz_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+xz";
243 archivefile_extensions = ['.cat', '.cat.gz', '.cat.bz2', '.cat.zst', '.cat.lz4', '.cat.lzo', '.cat.lzop', '.cat.lzma', '.cat.xz'];
244
245 if __name__ == "__main__":
246 import subprocess;
247 curscrpath = os.path.dirname(sys.argv[0]);
248 if(curscrpath==""):
249 curscrpath = ".";
250 if(os.sep=="\\"):
251 curscrpath = curscrpath.replace(os.sep, "/");
252 curscrpath = curscrpath + "/";
253 scrfile = curscrpath + "catfile.py";
254 if(os.path.exists(scrfile) and os.path.isfile(scrfile)):
255 scrcmd = subprocess.Popen([sys.executable, scrfile] + sys.argv[1:]);
256 scrcmd.wait();
257
258 def VerbosePrintOut(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
259 if(not dbgenable):
260 return True;
261 log_functions = {
262 "print": print,
263 "log": logging.info,
264 "warning": logging.warning,
265 "error": logging.error,
266 "critical": logging.critical,
267 "exception": logging.exception,
268 "logalt": lambda x: logging.log(dgblevel, x),
269 "debug": logging.debug
270 };
271 log_function = log_functions.get(outtype);
272 if(log_function):
273 log_function(dbgtxt);
274 return True;
275 return False;
276
277 def VerbosePrintOutReturn(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
278 VerbosePrintOut(dbgtxt, outtype, dbgenable, dgblevel);
279 return dbgtxt;
280
281 def RemoveWindowsPath(dpath):
282 if(dpath is None):
283 dpath = "";
284 if(os.sep!="/"):
285 dpath = dpath.replace(os.path.sep, "/");
286 dpath = dpath.rstrip("/");
287 if(dpath=="." or dpath==".."):
288 dpath = dpath + "/";
289 return dpath;
290
291 def NormalizeRelativePath(inpath):
292 inpath = RemoveWindowsPath(inpath);
293 if(os.path.isabs(inpath)):
294 outpath = inpath;
295 else:
296 if(inpath.startswith("./") or inpath.startswith("../")):
297 outpath = inpath;
298 else:
299 outpath = "./" + inpath;
300 return outpath;
301
302 def PrependPath(base_dir, child_path):
303 # Check if base_dir is None or empty, if so, return child_path as is
304 if not base_dir:
305 return child_path;
306 # Ensure base_dir ends with exactly one slash
307 if not base_dir.endswith('/'):
308 base_dir += '/';
309 # Check if child_path starts with ./ or ../ (indicating a relative path)
310 if child_path.startswith('./') or child_path.startswith('../'):
311 # For relative paths, we don't alter the child_path
312 return base_dir + child_path;
313 else:
314 # For non-relative paths, ensure there's no starting slash on child_path to avoid double slashes
315 return base_dir + child_path.lstrip('/');
316
317 def ListDir(dirpath, followlink=False, duplicates=False):
318 if(isinstance(dirpath, (list, tuple, ))):
319 dirpath = list(filter(None, dirpath));
320 elif(isinstance(dirpath, (str, ))):
321 dirpath = list(filter(None, [dirpath]));
322 retlist = [];
323 for mydirfile in dirpath:
324 if(not os.path.exists(mydirfile)):
325 return False;
326 mydirfile = NormalizeRelativePath(mydirfile);
327 if(os.path.exists(mydirfile) and os.path.islink(mydirfile)):
328 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile));
329 if(os.path.exists(mydirfile) and os.path.isdir(mydirfile)):
330 for root, dirs, filenames in os.walk(mydirfile):
331 dpath = root;
332 dpath = RemoveWindowsPath(dpath);
333 if(dpath not in retlist and not duplicates):
334 retlist.append(dpath);
335 if(duplicates):
336 retlist.append(dpath);
337 for file in filenames:
338 fpath = os.path.join(root, file);
339 fpath = RemoveWindowsPath(fpath);
340 if(fpath not in retlist and not duplicates):
341 retlist.append(fpath);
342 if(duplicates):
343 retlist.append(fpath);
344 else:
345 retlist.append(RemoveWindowsPath(mydirfile));
346 return retlist;
347
348 def ListDirAdvanced(dirpath, followlink=False, duplicates=False):
349 if isinstance(dirpath, (list, tuple)):
350 dirpath = list(filter(None, dirpath));
351 elif isinstance(dirpath, str):
352 dirpath = list(filter(None, [dirpath]));
353 retlist = []
354 for mydirfile in dirpath:
355 if not os.path.exists(mydirfile):
356 return False;
357 mydirfile = NormalizeRelativePath(mydirfile);
358 if os.path.exists(mydirfile) and os.path.islink(mydirfile) and followlink:
359 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile))
360 if os.path.exists(mydirfile) and os.path.isdir(mydirfile):
361 for root, dirs, filenames in os.walk(mydirfile):
362 # Sort dirs and filenames alphabetically in place
363 dirs.sort(key=lambda x: x.lower());
364 filenames.sort(key=lambda x: x.lower());
365 dpath = RemoveWindowsPath(root);
366 if not duplicates and dpath not in retlist:
367 retlist.append(dpath);
368 elif duplicates:
369 retlist.append(dpath);
370 for file in filenames:
371 fpath = os.path.join(root, file);
372 fpath = RemoveWindowsPath(fpath);
373 if not duplicates and fpath not in retlist:
374 retlist.append(fpath);
375 elif duplicates:
376 retlist.append(fpath);
377 else:
378 retlist.append(RemoveWindowsPath(mydirfile));
379 return retlist;
380
381 def create_alias_function(prefix, base_name, suffix, target_function):
382 # Define a new function that wraps the target function
383 def alias_function(*args, **kwargs):
384 return target_function(*args, **kwargs);
385
386 # Create the function name by combining the prefix, base name, and the suffix
387 function_name = "{}{}{}".format(prefix, base_name, suffix);
388
389 # Add the new function to the global namespace
390 globals()[function_name] = alias_function;
391
392 def create_alias_function_alt(prefix, base_name, suffix, target_function):
393 # Create the function name by combining the prefix, base name, and the suffix
394 # Use the format method for string formatting, compatible with Python 2 and 3
395 function_name = "{}{}{}".format(prefix, base_name, suffix)
396 # Add the new function (alias of the target_function) to the global namespace
397 # This line is compatible as-is with both Python 2 and 3
398 globals()[function_name] = target_function
399
400 def FormatSpecsListToDict(formatspecs=__file_format_list__):
401 if(isinstance(formatspecs, (list, tuple, ))):
402 return {'format_name': formatspecs[0], 'format_magic': formatspecs[1], 'format_lower': formatspecs[2], 'format_len': formatspecs[3], 'format_hex': formatspecs[4], 'format_delimiter': formatspecs[5], 'format_ver': formatspecs[6], 'new_style': formatspecs[7], 'use_advanced_list': formatspecs[8], 'use_alt_inode': formatspecs[9]};
403 elif(isinstance(formatspecs, (dict, ))):
404 return formatspecs;
405 else:
406 return __file_format_dict__;
407 return __file_format_dict__;
408
409 # initial_value can be 0xFFFF or 0x0000
410 def crc16_ansi(msg, initial_value=0xFFFF):
411 # CRC-16-IBM / CRC-16-ANSI polynomial and initial value
412 poly = 0x8005; # Polynomial for CRC-16-IBM / CRC-16-ANSI
413 crc = initial_value; # Initial value
414 for b in msg:
415 crc ^= b << 8; # XOR byte into CRC top byte
416 for _ in range(8): # Process each bit
417 if crc & 0x8000: # If the top bit is set
418 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
419 else:
420 crc = crc << 1; # Just shift left
421 crc &= 0xFFFF; # Ensure CRC remains 16-bit
422 return crc;
423
424 # initial_value can be 0xFFFF or 0x0000
425 def crc16_ibm(msg, initial_value=0xFFFF):
426 return crc16_ansi(msg, initial_value);
427
428 # initial_value is 0xFFFF
429 def crc16(msg):
430 return crc16_ansi(msg, 0xFFFF);
431
432 # initial_value can be 0xFFFF, 0x1D0F or 0x0000
433 def crc16_ccitt(msg, initial_value=0xFFFF):
434 # CRC-16-CCITT polynomial
435 poly = 0x1021; # Polynomial for CRC-16-CCITT
436 # Use the specified initial value
437 crc = initial_value;
438 for b in msg:
439 crc ^= b << 8; # XOR byte into CRC top byte
440 for _ in range(8): # Process each bit
441 if crc & 0x8000: # If the top bit is set
442 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
443 else:
444 crc = crc << 1; # Just shift left
445 crc &= 0xFFFF; # Ensure CRC remains 16-bit
446 return crc;
447
448 # initial_value can be 0x42F0E1EBA9EA3693 or 0x0000000000000000
449 def crc64_ecma(msg, initial_value=0x0000000000000000):
450 # CRC-64-ECMA polynomial and initial value
451 poly = 0x42F0E1EBA9EA3693;
452 crc = initial_value; # Initial value for CRC-64-ECMA
453 for b in msg:
454 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
455 for _ in range(8): # Process each bit
456 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
457 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
458 else:
459 crc <<= 1; # Just shift left if the MSB is 0
460 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
461 return crc;
462
463 # initial_value can be 0x000000000000001B or 0xFFFFFFFFFFFFFFFF
464 def crc64_iso(msg, initial_value=0xFFFFFFFFFFFFFFFF):
465 # CRC-64-ISO polynomial and initial value
466 poly = 0x000000000000001B;
467 crc = initial_value; # Common initial value for CRC-64-ISO
468 for b in msg:
469 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
470 for _ in range(8): # Process each bit
471 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
472 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
473 else:
474 crc <<= 1; # Just shift left if the MSB is 0
475 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
476 return crc;
477
478 def GetDataFromArray(data, path, default=None):
479 element = data;
480 try:
481 for key in path:
482 element = element[key];
483 return element;
484 except (KeyError, TypeError, IndexError):
485 return default;
486
487 def GetDataFromArrayAlt(structure, path, default=None):
488 element = structure;
489 for key in path:
490 if isinstance(element, dict) and key in element:
491 element = element[key];
492 elif isinstance(element, list) and isinstance(key, int) and -len(element) <= key < len(element):
493 element = element[key];
494 else:
495 return default;
496 return element;
497
498 def GetHeaderChecksum(inlist=[], checksumtype="crc32", encodedata=True, formatspecs=__file_format_dict__):
499 formatspecs = FormatSpecsListToDict(formatspecs);
500 if isinstance(inlist, list):
501 fileheader = AppendNullBytes(inlist, formatspecs['format_delimiter']);
502 else:
503 fileheader = AppendNullByte(inlist, formatspecs['format_delimiter']);
504 if(encodedata):
505 fileheader = fileheader.encode('UTF-8');
506 if(checksumtype=="none" or checksumtype==""):
507 catfileheadercshex = format(0, 'x').lower();
508 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
509 catfileheadercshex = format(crc16(fileheader) & 0xffff, '04x').lower();
510 elif(checksumtype=="crc16_ccitt"):
511 catfileheadercshex = format(crc16_ccitt(fileheader) & 0xffff, '04x').lower();
512 elif(checksumtype=="adler32"):
513 catfileheadercshex = format(zlib.adler32(fileheader) & 0xffffffff, '08x').lower();
514 elif(checksumtype=="crc32"):
515 catfileheadercshex = format(crc32(fileheader) & 0xffffffff, '08x').lower();
516 elif(checksumtype=="crc64_ecma"):
517 catfileheadercshex = format(crc64_ecma(fileheader) & 0xffffffffffffffff, '016x').lower();
518 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
519 catfileheadercshex = format(crc64_iso(fileheader) & 0xffffffffffffffff, '016x').lower();
520 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
521 checksumoutstr = hashlib.new(checksumtype);
522 checksumoutstr.update(fileheader);
523 catfileheadercshex = checksumoutstr.hexdigest().lower();
524 else:
525 catfileheadercshex = format(0, 'x').lower();
526 return catfileheadercshex;
527
528 def GetFileChecksum(instr, checksumtype="crc32", encodedata=True, formatspecs=__file_format_dict__):
529 formatspecs = FormatSpecsListToDict(formatspecs);
530 if(encodedata):
531 instr = instr.encode('UTF-8');
532 if(checksumtype=="none" or checksumtype==""):
533 catinstrcshex = format(0, 'x').lower();
534 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
535 catinstrcshex = format(crc16(instr) & 0xffff, '04x').lower();
536 elif(checksumtype=="crc16_ccitt"):
537 catinstrcshex = format(crc16_ccitt(instr) & 0xffff, '04x').lower();
538 elif(checksumtype=="adler32"):
539 catinstrcshex = format(zlib.adler32(instr) & 0xffffffff, '08x').lower();
540 elif(checksumtype=="crc32"):
541 catinstrcshex = format(crc32(instr) & 0xffffffff, '08x').lower();
542 elif(checksumtype=="crc64_ecma"):
543 catinstrcshex = format(crc64_ecma(instr) & 0xffffffffffffffff, '016x').lower();
544 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
545 catinstrcshex = format(crc64_iso(instr) & 0xffffffffffffffff, '016x').lower();
546 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
547 checksumoutstr = hashlib.new(checksumtype);
548 checksumoutstr.update(instr);
549 catinstrcshex = checksumoutstr.hexdigest().lower();
550 else:
551 catinstrcshex = format(0, 'x').lower();
552 return catinstrcshex;
553
554 def ValidateHeaderChecksum(inlist=[], checksumtype="crc32", inchecksum="0", formatspecs=__file_format_dict__):
555 formatspecs = FormatSpecsListToDict(formatspecs);
556 catfileheadercshex = GetHeaderChecksum(inlist, checksumtype, True, formatspecs);
557 inchecksum = inchecksum.lower();
558 catfileheadercshex = catfileheadercshex.lower();
559 if(inchecksum==catfileheadercshex):
560 return True;
561 else:
562 return False;
563
564 def ValidateFileChecksum(infile, checksumtype="crc32", inchecksum="0", formatspecs=__file_format_dict__):
565 formatspecs = FormatSpecsListToDict(formatspecs);
566 catinfilecshex = GetFileChecksum(infile, checksumtype, True, formatspecs);
567 inchecksum = inchecksum.lower();
568 catinfilecshex = catinfilecshex.lower();
569 if(inchecksum==catinfilecshex):
570 return True;
571 else:
572 return False;
573
574 def ReadTillNullByteOld(fp, delimiter=__file_format_dict__['format_delimiter']):
575 curbyte = b"";
576 curfullbyte = b"";
577 nullbyte = delimiter.encode("UTF-8");
578 while(True):
579 curbyte = fp.read(1);
580 if(curbyte==nullbyte or not curbyte):
581 break;
582 curfullbyte = curfullbyte + curbyte;
583 return curfullbyte.decode('UTF-8');
584
585 def ReadUntilNullByteOld(fp, delimiter=__file_format_dict__['format_delimiter']):
586 return ReadTillNullByteOld(fp, delimiter);
587
588 def ReadTillNullByteAlt(fp, delimiter=__file_format_dict__['format_delimiter'], chunk_size=1024, max_read=1024000):
589 delimiter = delimiter.encode('UTF-8') # Ensure the delimiter is in bytes
590 buffer = bytearray();
591 total_read = 0;
592 delimiter_length = len(delimiter);
593 while True:
594 chunk = fp.read(chunk_size)
595 if not chunk:
596 # End of file reached without finding the delimiter
597 break;
598 buffer.extend(chunk);
599 total_read += len(chunk);
600 if delimiter in buffer:
601 # Delimiter found, calculate where to reset the file pointer
602 index = buffer.find(delimiter);
603 # Calculate how many extra bytes were read after the delimiter
604 extra_bytes_read = len(buffer) - (index + delimiter_length);
605 # Move the file pointer back to just after the delimiter
606 fp.seek(-extra_bytes_read, 1);
607 buffer = buffer[:index];
608 break;
609 if total_read >= max_read:
610 # Stop reading if max limit is reached to prevent excessive memory usage
611 raise MemoryError("Maximum read limit reached without finding the delimiter.");
612 # Check for incomplete UTF-8 sequences at the end of the buffer
613 if len(buffer) > 1 and 128 <= buffer[-1] < 192:
614 # This suggests that the last byte might be the start of a multi-byte character
615 # Try to read one more byte to complete the character
616 extra_byte = fp.read(1);
617 if extra_byte:
618 buffer.extend(extra_byte);
619 else:
620 # No more data available
621 break;
622 try:
623 return buffer.decode('UTF-8', errors='replace');
624 except UnicodeDecodeError:
625 return buffer.decode('UTF-8', errors='replace');
626
627 def ReadUntilNullByteAlt(fp, delimiter=__file_format_dict__['format_delimiter'], chunk_size=1024, max_read=1024000):
628 return ReadTillNullByteAlt(fp, delimiter, chunk_size, max_read);
629
630 def ReadTillNullByte(fp, delimiter=__file_format_dict__['format_delimiter'], max_read=1024000):
631 curfullbyte = bytearray();
632 nullbyte = delimiter.encode("UTF-8");
633 total_read = 0; # Track the total number of bytes read
634 while True:
635 curbyte = fp.read(1);
636 if curbyte == nullbyte or not curbyte:
637 break;
638 curfullbyte.extend(curbyte);
639 total_read += 1;
640 if total_read >= max_read:
641 raise MemoryError("Maximum read limit reached without finding the delimiter.");
642 # Decode the full byte array to string once out of the loop
643 try:
644 return curfullbyte.decode('UTF-8');
645 except UnicodeDecodeError:
646 # Handle potential partial UTF-8 characters
647 for i in range(1, 4):
648 try:
649 return curfullbyte[:-i].decode('UTF-8');
650 except UnicodeDecodeError:
651 continue;
652 raise; # Re-raise if decoding fails even after trimming
653
654 def ReadUntilNullByte(fp, delimiter=__file_format_dict__['format_delimiter'], max_read=1024000):
655 return ReadTillNullByte(fp, delimiter, max_read);
656
657 def ReadTillNullByteByNum(fp, delimiter=__file_format_dict__['format_delimiter'], num_delimiters=1, chunk_size=1024, max_read=1024000):
658 delimiter = delimiter.encode('UTF-8'); # Ensure the delimiter is in bytes
659 buffer = bytearray();
660 total_read = 0;
661 delimiter_length = len(delimiter);
662 results = [];
663 while len(results) < num_delimiters:
664 chunk = fp.read(chunk_size);
665 if not chunk:
666 # End of file reached; decode whatever is collected if it's the last needed part
667 if len(buffer) > 0:
668 results.append(buffer.decode('UTF-8', errors='replace'));
669 break;
670 buffer.extend(chunk)
671 total_read += len(chunk);
672 # Check if we have found the delimiter
673 while delimiter in buffer:
674 index = buffer.find(delimiter);
675 # Decode the section before the delimiter
676 results.append(buffer[:index].decode('UTF-8', errors='replace'));
677 # Remove the processed part from the buffer
678 buffer = buffer[index + delimiter_length:];
679 if len(results) == num_delimiters:
680 # If reached the required number of delimiters, adjust the file pointer and stop
681 fp.seek(-len(buffer), 1);
682 return results;
683 if total_read >= max_read:
684 # Stop reading if max limit is reached to prevent excessive memory usage
685 raise MemoryError("Maximum read limit reached without finding the delimiter.");
686 # Check for incomplete UTF-8 sequences at the end of the buffer
687 if len(buffer) > 1 and 128 <= buffer[-1] < 192:
688 # This suggests that the last byte might be the start of a multi-byte character
689 # Try to read one more byte to complete the character
690 extra_byte = fp.read(1);
691 if extra_byte:
692 buffer.extend(extra_byte);
693 else:
694 # No more data available
695 break;
696 # Process remaining buffer if less than the required number of delimiters were found
697 if len(buffer) > 0 and len(results) < num_delimiters:
698 results.append(buffer.decode('UTF-8', errors='replace'));
699 return results;
700
701 def ReadUntilNullByteByNum(fp, delimiter=__file_format_dict__['format_delimiter'], num_delimiters=1, chunk_size=1024, max_read=1024000):
702 return ReadTillNullByteByNum(fp, delimiter, num_delimiters, chunk_size, max_read);
703
704 def SeekToEndOfFile(fp):
705 lasttell = 0;
706 while(True):
707 fp.seek(1, 1);
708 if(lasttell==fp.tell()):
709 break;
710 lasttell = fp.tell();
711 return True;
712
713 def ReadFileHeaderData(fp, rounds=0, delimiter=__file_format_dict__['format_delimiter']):
714 rocount = 0;
715 roend = int(rounds);
716 HeaderOut = [];
717 while(rocount<roend):
718 HeaderOut.append(ReadTillNullByte(fp, delimiter));
719 rocount = rocount + 1;
720 return HeaderOut;
721
722 def ReadFileHeaderDataBySize(fp, delimiter=__file_format_dict__['format_delimiter']):
723 headerpresize = ReadTillNullByte(fp, delimiter);
724 headersize = int(headerpresize, 16);
725 if(headersize<=0):
726 return [];
727 headercontent = str(fp.read(headersize).decode('UTF-8')).split(delimiter);
728 fp.seek(1, 1);
729 rocount = 0;
730 roend = int(len(headercontent));
731 HeaderOut = [headerpresize];
732 while(rocount<roend):
733 HeaderOut.append(headercontent[rocount]);
734 rocount = rocount + 1;
735 return HeaderOut;
736
737 def ReadFileHeaderDataWoSize(fp, delimiter=__file_format_dict__['format_delimiter']):
738 preheaderdata = ReadFileHeaderData(fp, 2, delimiter);
739 headersize = int(preheaderdata[0], 16);
740 headernumfields = int(preheaderdata[1], 16);
741 if(headersize<=0 or headernumfields<=0):
742 return [];
743 headerdata = ReadTillNullByteByNum(fp, delimiter, headernumfields);
744 #headerdata = ReadFileHeaderData(fp, headernumfields, delimiter);
745 HeaderOut = preheaderdata + headerdata;
746 return HeaderOut;
747
748 def ReadFileHeaderDataBySizeWithContent(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
749 formatspecs = FormatSpecsListToDict(formatspecs);
750 delimiter = formatspecs['format_delimiter'];
751 fheaderstart = fp.tell();
752 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
753 if(len(HeaderOut)==0):
754 return False;
755 if(re.findall("^[.|/]", HeaderOut[3])):
756 fname = HeaderOut[3];
757 else:
758 fname = "./"+HeaderOut[3];
759 fchecksumtype = HeaderOut[-3].lower();
760 fcs = HeaderOut[-2].lower();
761 fccs = HeaderOut[-1].lower();
762 fsize = int(HeaderOut[5], 16);
763 fcompression = HeaderOut[12];
764 fcsize = int(HeaderOut[13], 16);
765 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
766 if(fcs!=newfcs and not skipchecksum):
767 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
768 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
769 return False;
770 fhend = fp.tell() - 1;
771 fcontentstart = fp.tell();
772 fcontents = BytesIO();
773 if(fsize>0 and not listonly):
774 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
775 fcontents.write(fp.read(fsize));
776 else:
777 fcontents.write(fp.read(fcsize));
778 elif(fsize>0 and listonly):
779 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
780 fp.seek(fsize, 1);
781 else:
782 fp.seek(fcsize, 1);
783 fcontents.seek(0, 0);
784 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
785 if(fccs!=newfccs and not skipchecksum and not listonly):
786 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
787 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
788 return False;
789 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
790 pass;
791 else:
792 fcontents.seek(0, 0);
793 fcontents = UncompressArchiveFile(fcontents, formatspecs);
794 fp.seek(1, 1);
795 fcontentend = fp.tell() - 1;
796 HeaderOut.append(fcontents);
797 return HeaderOut;
798
799 def ReadFileHeaderDataBySizeWithContentToArray(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
800 formatspecs = FormatSpecsListToDict(formatspecs);
801 delimiter = formatspecs['format_delimiter'];
802 fheaderstart = fp.tell();
803 if(formatspecs['new_style']):
804 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
805 else:
806 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
807 if(len(HeaderOut)==0):
808 return False;
809 fheadsize = int(HeaderOut[0], 16);
810 fnumfields = int(HeaderOut[1], 16);
811 ftype = int(HeaderOut[2], 16);
812 if(re.findall("^[.|/]", HeaderOut[3])):
813 fname = HeaderOut[3];
814 else:
815 fname = "./"+HeaderOut[3];
816 fbasedir = os.path.dirname(fname);
817 flinkname = HeaderOut[4];
818 fsize = int(HeaderOut[5], 16);
819 fatime = int(HeaderOut[6], 16);
820 fmtime = int(HeaderOut[7], 16);
821 fctime = int(HeaderOut[8], 16);
822 fbtime = int(HeaderOut[9], 16);
823 fmode = int(HeaderOut[10], 16);
824 fchmode = stat.S_IMODE(fmode);
825 ftypemod = stat.S_IFMT(fmode);
826 fwinattributes = int(HeaderOut[11], 16);
827 fcompression = HeaderOut[12];
828 fcsize = int(HeaderOut[13], 16);
829 fuid = int(HeaderOut[14], 16);
830 funame = HeaderOut[15];
831 fgid = int(HeaderOut[16], 16);
832 fgname = HeaderOut[17];
833 fid = int(HeaderOut[18], 16);
834 finode = int(HeaderOut[19], 16);
835 flinkcount = int(HeaderOut[20], 16);
836 fdev_minor = int(HeaderOut[21], 16);
837 fdev_major = int(HeaderOut[22], 16);
838 frdev_minor = int(HeaderOut[23], 16);
839 frdev_major = int(HeaderOut[24], 16);
840 fextrasize = int(HeaderOut[25], 16);
841 fextrafields = int(HeaderOut[26], 16);
842 extrafieldslist = [];
843 extrastart = 27;
844 extraend = extrastart + fextrafields;
845 extrafieldslist = [];
846 if(extrastart<extraend):
847 extrafieldslist.append(HeaderOut[extrastart]);
848 extrastart = extrastart + 1;
849 fchecksumtype = HeaderOut[extrastart].lower();
850 fcs = HeaderOut[extrastart + 1].lower();
851 fccs = HeaderOut[extrastart + 2].lower();
852 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
853 if(fcs!=newfcs and not skipchecksum):
854 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
855 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
856 return False;
857 fhend = fp.tell() - 1;
858 fcontentstart = fp.tell();
859 fcontents = BytesIO();
860 pyhascontents = False;
861 if(fsize>0 and not listonly):
862 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
863 fcontents.write(fp.read(fsize));
864 else:
865 fcontents.write(fp.read(fcsize));
866 pyhascontents = True;
867 elif(fsize>0 and listonly):
868 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
869 fp.seek(fsize, 1);
870 else:
871 fp.seek(fcsize, 1);
872 pyhascontents = False;
873 fcontents.seek(0, 0);
874 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
875 if(fccs!=newfccs and not skipchecksum and not listonly):
876 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
877 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
878 return False;
879 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
880 pass;
881 else:
882 fcontents.seek(0, 0);
883 fcontents = UncompressArchiveFile(fcontents, formatspecs);
884 fp.seek(1, 1);
885 fcontentend = fp.tell() - 1;
886 fcontents.seek(0, 0);
887 catlist = {'fheadersize': fheadsize, 'fhstart': fheaderstart, 'fhend': fhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': fchecksumtype, 'fnumfields': fnumfields + 2, 'frawheader': HeaderOut, 'fextrafields': fextrafields, 'fextrafieldsize': fextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': fcs, 'fcontentchecksum': fccs, 'fhascontents': pyhascontents, 'fcontentstart': fcontentstart, 'fcontentend': fcontentend, 'fcontents': fcontents};
888 return catlist;
889
890 def ReadFileHeaderDataBySizeWithContentToList(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
891 formatspecs = FormatSpecsListToDict(formatspecs);
892 delimiter = formatspecs['format_delimiter'];
893 fheaderstart = fp.tell();
894 if(formatspecs['new_style']):
895 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
896 else:
897 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
898 if(len(HeaderOut)==0):
899 return False;
900 fheadsize = int(HeaderOut[0], 16);
901 fnumfields = int(HeaderOut[1], 16);
902 ftype = int(HeaderOut[2], 16);
903 if(re.findall("^[.|/]", HeaderOut[3])):
904 fname = HeaderOut[3];
905 else:
906 fname = "./"+HeaderOut[3];
907 fbasedir = os.path.dirname(fname);
908 flinkname = HeaderOut[4];
909 fsize = int(HeaderOut[5], 16);
910 fatime = int(HeaderOut[6], 16);
911 fmtime = int(HeaderOut[7], 16);
912 fctime = int(HeaderOut[8], 16);
913 fbtime = int(HeaderOut[9], 16);
914 fmode = int(HeaderOut[10], 16);
915 fchmode = stat.S_IMODE(fmode);
916 ftypemod = stat.S_IFMT(fmode);
917 fwinattributes = int(HeaderOut[11], 16);
918 fcompression = HeaderOut[12];
919 fcsize = int(HeaderOut[13], 16);
920 fuid = int(HeaderOut[14], 16);
921 funame = HeaderOut[15];
922 fgid = int(HeaderOut[16], 16);
923 fgname = HeaderOut[17];
924 fid = int(HeaderOut[18], 16);
925 finode = int(HeaderOut[19], 16);
926 flinkcount = int(HeaderOut[20], 16);
927 fdev_minor = int(HeaderOut[21], 16);
928 fdev_major = int(HeaderOut[22], 16);
929 frdev_minor = int(HeaderOut[23], 16);
930 frdev_major = int(HeaderOut[24], 16);
931 fextrasize = int(HeaderOut[25], 16);
932 fextrafields = int(HeaderOut[26], 16);
933 extrafieldslist = [];
934 extrastart = 27;
935 extraend = extrastart + fextrafields;
936 extrafieldslist = [];
937 if(extrastart<extraend):
938 extrafieldslist.append(HeaderOut[extrastart]);
939 extrastart = extrastart + 1;
940 fchecksumtype = HeaderOut[extrastart].lower();
941 fcs = HeaderOut[extrastart + 1].lower();
942 fccs = HeaderOut[extrastart + 2].lower();
943 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
944 if(fcs!=newfcs and not skipchecksum):
945 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
946 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
947 return False;
948 fhend = fp.tell() - 1;
949 fcontentstart = fp.tell();
950 fcontents = BytesIO();
951 pyhascontents = False;
952 if(fsize>0 and not listonly):
953 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
954 fcontents.write(fp.read(fsize));
955 else:
956 fcontents.write(fp.read(fcsize));
957 pyhascontents = True;
958 elif(fsize>0 and listonly):
959 if(fcompression=="none" or fcompression=="" or fcompression=="atuo"):
960 fp.seek(fsize, 1);
961 else:
962 fp.seek(fcsize, 1);
963 pyhascontents = False;
964 fcontents.seek(0, 0);
965 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
966 if(fccs!=newfccs and not skipchecksum and not listonly):
967 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
968 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
969 return False;
970 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
971 pass;
972 else:
973 fcontents.seek(0, 0);
974 fcontents = UncompressArchiveFile(fcontents, formatspecs);
975 fp.seek(1, 1);
976 fcontentend = fp.tell() - 1;
977 catlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, extrafieldslist, fchecksumtype, fcontents];
978 return catlist;
979
980 def ReadFileDataBySizeWithContent(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
981 formatspecs = FormatSpecsListToDict(formatspecs);
982 delimiter = formatspecs['format_delimiter'];
983 curloc = fp.tell();
984 if(curloc>0):
985 fp.seek(0, 0);
986 catheader = ReadFileHeaderData(fp, 4, delimiter);
987 if(curloc>0):
988 fp.seek(curloc, 0);
989 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
990 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
991 if(not headercheck and not skipchecksum):
992 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
993 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
994 return False;
995 fnumfiles = int(catheader[1], 16);
996 countnum = 0;
997 flist = [];
998 while(countnum < fnumfiles):
999 HeaderOut = ReadFileHeaderDataBySizeWithContent(fp, listonly, skipchecksum, formatspecs)
1000 if(len(HeaderOut)==0):
1001 break;
1002 flist.append(HeaderOut);
1003 countnum = countnum + 1;
1004 return flist;
1005
1006 def ReadFileDataBySizeWithContentToArray(fp, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
1007 formatspecs = FormatSpecsListToDict(formatspecs);
1008 delimiter = formatspecs['format_delimiter'];
1009 curloc = fp.tell();
1010 if(curloc>0):
1011 fp.seek(0, 0);
1012 catheader = ReadFileHeaderData(fp, 4, delimiter);
1013 if(curloc>0):
1014 fp.seek(curloc, 0);
1015 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1016 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1017 if(not headercheck and not skipchecksum):
1018 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1019 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1020 return False;
1021 catstring = catheader[0];
1022 catversion = re.findall(r"([\d]+)$", catstring);
1023 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
1024 fprenumfiles = catheader[1];
1025 fnumfiles = int(fprenumfiles, 16);
1026 fprechecksumtype = catheader[2];
1027 fprechecksum = catheader[3];
1028 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
1029 if(seekstart<0 and seekstart>fnumfiles):
1030 seekstart = 0;
1031 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
1032 seekend = fnumfiles;
1033 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
1034 seekend = fnumfiles - abs(seekend);
1035 if(seekstart>0):
1036 il = 0;
1037 while(il < seekstart):
1038 prefhstart = fp.tell();
1039 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs['format_delimiter']);
1040 if(len(preheaderdata)==0):
1041 break;
1042 prefsize = int(preheaderdata[5], 16);
1043 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-3].lower(), True, formatspecs);
1044 prefcs = preheaderdata[-2];
1045 if(prefcs!=prenewfcs and not skipchecksum):
1046 VVerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
1047 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
1048 return False;
1049 valid_archive = False;
1050 invalid_archive = True;
1051 prefhend = fp.tell() - 1;
1052 prefcontentstart = fp.tell();
1053 prefcontents = BytesIO();
1054 pyhascontents = False;
1055 if(prefsize>0):
1056 prefcontents.write(fp.read(prefsize));
1057 prefcontents.seek(0, 0);
1058 prenewfccs = GetFileChecksum(prefcontents.read(), preheaderdata[-3].lower(), False, formatspecs);
1059 prefccs = preheaderdata[-1];
1060 pyhascontents = True;
1061 if(prefccs!=prenewfccs and not skipchecksum):
1062 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
1063 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
1064 return False;
1065 fp.seek(1, 1);
1066 il = il + 1;
1067 realidnum = 0;
1068 countnum = seekstart;
1069 while(countnum < seekend):
1070 catlist['ffilelist'].update({realidnum: {'fid': realidnum, 'fidalt': realidnum}});
1071 HeaderOut = ReadFileHeaderDataBySizeWithContentToArray(fp, listonly, skipchecksum, formatspecs);
1072 if(len(HeaderOut)==0):
1073 break;
1074 catlist['ffilelist'][realidnum].update(HeaderOut);
1075 countnum = countnum + 1;
1076 realidnum = realidnum + 1;
1077 return catlist;
1078
1079 def ReadFileDataBySizeWithContentToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
1080 formatspecs = FormatSpecsListToDict(formatspecs);
1081 if(isinstance(infile, dict)):
1082 listcatfiles = infile;
1083 else:
1084 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
1085 return False;
1086 listcatfiles = ReadFileDataBySizeWithContentToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs);
1087 if(not listcatfiles):
1088 return False;
1089 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
1090 lenlist = len(listcatfiles['ffilelist']);
1091 lcfi = 0;
1092 lcfx = int(listcatfiles['fnumfiles']);
1093 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
1094 lcfx = int(lenlist);
1095 else:
1096 lcfx = int(listcatfiles['fnumfiles']);
1097 while(lcfi < lcfx):
1098 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
1099 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
1100 catarray['filetoid'].update(filetoidarray);
1101 catarray['idtofile'].update(idtofilearray);
1102 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
1103 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
1104 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
1105 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
1106 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
1107 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
1108 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
1109 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
1110 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
1111 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1112 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1113 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
1114 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
1115 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
1116 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
1117 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
1118 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1119 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1120 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
1121 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
1122 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
1123 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1124 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1125 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
1126 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
1127 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
1128 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
1129 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1130 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1131 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1132 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1133 lcfi = lcfi + 1;
1134 return catarray;
1135
1136 def ReadFileDataBySizeWithContentToList(fp, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
1137 formatspecs = FormatSpecsListToDict(formatspecs);
1138 delimiter = formatspecs['format_delimiter'];
1139 curloc = fp.tell();
1140 if(curloc>0):
1141 fp.seek(0, 0);
1142 catheader = ReadFileHeaderData(fp, 4, delimiter);
1143 if(curloc>0):
1144 fp.seek(curloc, 0);
1145 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1146 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1147 if(not headercheck and not skipchecksum):
1148 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1149 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1150 return False;
1151 catstring = catheader[0];
1152 catversion = re.findall(r"([\d]+)$", catstring);
1153 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
1154 fprenumfiles = catheader[1];
1155 fnumfiles = int(fprenumfiles, 16);
1156 fprechecksumtype = catheader[2];
1157 fprechecksum = catheader[3];
1158 catlist = [];
1159 if(seekstart<0 and seekstart>fnumfiles):
1160 seekstart = 0;
1161 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
1162 seekend = fnumfiles;
1163 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
1164 seekend = fnumfiles - abs(seekend);
1165 if(seekstart>0):
1166 il = 0;
1167 while(il < seekstart):
1168 prefhstart = fp.tell();
1169 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs['format_delimiter']);
1170 if(len(preheaderdata)==0):
1171 break;
1172 prefsize = int(preheaderdata[5], 16);
1173 prefcompression = preheaderdata[12];
1174 prefcsize = int(preheaderdata[13], 16);
1175 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-3].lower(), True, formatspecs);
1176 prefcs = preheaderdata[-2];
1177 if(prefcs!=prenewfcs and not skipchecksum):
1178 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
1179 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
1180 return False;
1181 valid_archive = False;
1182 invalid_archive = True;
1183 prefhend = fp.tell() - 1;
1184 prefcontentstart = fp.tell();
1185 prefcontents = "";
1186 pyhascontents = False;
1187 if(prefsize>0):
1188 if(prefcompression=="none" or prefcompression=="" or prefcompression=="auto"):
1189 prefcontents = catfp.read(prefsize);
1190 else:
1191 prefcontents = catfp.read(prefcsize);
1192 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
1193 prefccs = preheaderdata[-1];
1194 pyhascontents = True;
1195 if(prefccs!=prenewfccs and not skipchecksum):
1196 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
1197 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
1198 return False;
1199 fp.seek(1, 1);
1200 il = il + 1;
1201 realidnum = 0;
1202 countnum = seekstart;
1203 while(countnum < seekend):
1204 HeaderOut = ReadFileHeaderDataBySizeWithContentToList(fp, listonly, skipchecksum, formatspecs);
1205 if(len(HeaderOut)==0):
1206 break;
1207 catlist.append(HeaderOut);
1208 countnum = countnum + 1;
1209 realidnum = realidnum + 1;
1210 return catlist;
1211
1212 def ReadInFileBySizeWithContentToArray(infile, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
1213 formatspecs = FormatSpecsListToDict(formatspecs);
1214 delimiter = formatspecs['format_delimiter'];
1215 if(hasattr(infile, "read") or hasattr(infile, "write")):
1216 fp = infile;
1217 fp.seek(0, 0);
1218 fp = UncompressArchiveFile(fp, formatspecs);
1219 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1220 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1221 return False;
1222 if(not fp):
1223 return False;
1224 fp.seek(0, 0);
1225 elif(infile=="-"):
1226 fp = BytesIO();
1227 if(hasattr(sys.stdin, "buffer")):
1228 shutil.copyfileobj(sys.stdin.buffer, fp);
1229 else:
1230 shutil.copyfileobj(sys.stdin, fp);
1231 fp.seek(0, 0);
1232 fp = UncompressArchiveFile(fp, formatspecs);
1233 if(not fp):
1234 return False;
1235 fp.seek(0, 0);
1236 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
1237 fp = download_file_from_internet_file(infile);
1238 fp = UncompressArchiveFile(fp, formatspecs);
1239 fp.seek(0, 0);
1240 if(not fp):
1241 return False;
1242 fp.seek(0, 0);
1243 else:
1244 infile = RemoveWindowsPath(infile);
1245 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
1246 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1247 return False;
1248 compresscheck = CheckCompressionType(infile, formatspecs, True);
1249 if(not compresscheck):
1250 fextname = os.path.splitext(infile)[1];
1251 if(fextname==".gz"):
1252 compresscheck = "gzip";
1253 elif(fextname==".bz2"):
1254 compresscheck = "bzip2";
1255 elif(fextname==".zst"):
1256 compresscheck = "zstd";
1257 elif(fextname==".lz4" or fextname==".clz4"):
1258 compresscheck = "lz4";
1259 elif(fextname==".lzo" or fextname==".lzop"):
1260 compresscheck = "lzo";
1261 elif(fextname==".lzma" or fextname==".xz"):
1262 compresscheck = "lzma";
1263 else:
1264 return False;
1265 if(not compresscheck):
1266 return False;
1267 fp = UncompressFile(infile, formatspecs, "rb");
1268 return ReadFileDataBySizeWithContentToArray(fp, listonly, skipchecksum, formatspecs);
1269
1270 def ReadInFileBySizeWithContentToList(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
1271 formatspecs = FormatSpecsListToDict(formatspecs);
1272 delimiter = formatspecs['format_delimiter'];
1273 if(hasattr(infile, "read") or hasattr(infile, "write")):
1274 fp = infile;
1275 fp.seek(0, 0);
1276 fp = UncompressArchiveFile(fp, formatspecs);
1277 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1278 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1279 return False;
1280 if(not fp):
1281 return False;
1282 fp.seek(0, 0);
1283 elif(infile=="-"):
1284 fp = BytesIO();
1285 if(hasattr(sys.stdin, "buffer")):
1286 shutil.copyfileobj(sys.stdin.buffer, fp);
1287 else:
1288 shutil.copyfileobj(sys.stdin, fp);
1289 fp.seek(0, 0);
1290 fp = UncompressArchiveFile(fp, formatspecs);
1291 if(not fp):
1292 return False;
1293 fp.seek(0, 0);
1294 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
1295 fp = download_file_from_internet_file(infile);
1296 fp = UncompressArchiveFile(fp, formatspecs);
1297 fp.seek(0, 0);
1298 if(not fp):
1299 return False;
1300 fp.seek(0, 0);
1301 else:
1302 infile = RemoveWindowsPath(infile);
1303 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
1304 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1305 return False;
1306 compresscheck = CheckCompressionType(infile, formatspecs, True);
1307 if(not compresscheck):
1308 fextname = os.path.splitext(infile)[1];
1309 if(fextname==".gz"):
1310 compresscheck = "gzip";
1311 elif(fextname==".bz2"):
1312 compresscheck = "bzip2";
1313 elif(fextname==".zst"):
1314 compresscheck = "zstd";
1315 elif(fextname==".lz4" or fextname==".clz4"):
1316 compresscheck = "lz4";
1317 elif(fextname==".lzo" or fextname==".lzop"):
1318 compresscheck = "lzo";
1319 elif(fextname==".lzma" or fextname==".xz"):
1320 compresscheck = "lzma";
1321 else:
1322 return False;
1323 if(not compresscheck):
1324 return False;
1325 fp = UncompressFile(infile, formatspecs, "rb");
1326 return ReadFileDataBySizeWithContentToList(fp, seekstart, seekend, listonly, skipchecksum, formatspecs);
1327
1328 def ReadInFileBySizeWithContentToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
1329 formatspecs = FormatSpecsListToDict(formatspecs);
1330 if(isinstance(infile, dict)):
1331 listcatfiles = infile;
1332 else:
1333 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
1334 infile = RemoveWindowsPath(infile);
1335 listcatfiles = ReadInFileBySizeWithContentToArray(infile, listonly, skipchecksum, formatspecs);
1336 if(not listcatfiles):
1337 return False;
1338 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
1339 lenlist = len(listcatfiles['ffilelist']);
1340 lcfi = 0;
1341 lcfx = int(listcatfiles['fnumfiles']);
1342 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
1343 lcfx = int(lenlist);
1344 else:
1345 lcfx = int(listcatfiles['fnumfiles']);
1346 while(lcfi < lcfx):
1347 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
1348 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
1349 catarray['filetoid'].update(filetoidarray);
1350 catarray['idtofile'].update(idtofilearray);
1351 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
1352 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
1353 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
1354 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
1355 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
1356 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
1357 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
1358 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
1359 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
1360 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1361 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1362 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
1363 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
1364 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
1365 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
1366 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
1367 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1368 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1369 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
1370 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
1371 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
1372 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1373 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1374 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
1375 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
1376 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
1377 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
1378 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1379 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1380 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1381 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1382 lcfi = lcfi + 1;
1383 return catarray;
1384
1385 def AppendNullByte(indata, delimiter=__file_format_dict__['format_delimiter']):
1386 outdata = str(indata) + delimiter;
1387 return outdata;
1388
1389 def AppendNullBytes(indata=[], delimiter=__file_format_dict__['format_delimiter']):
1390 outdata = "";
1391 inum = 0;
1392 il = len(indata);
1393 while(inum < il):
1394 outdata = outdata + AppendNullByte(indata[inum], delimiter);
1395 inum = inum + 1;
1396 return outdata;
1397
1398 def AppendFileHeader(fp, numfiles, checksumtype="crc32", formatspecs=__file_format_dict__):
1399 formatspecs = FormatSpecsListToDict(formatspecs);
1400 delimiter = formatspecs['format_delimiter'];
1401 catver = formatspecs['format_ver'];
1402 fileheaderver = str(int(catver.replace(".", "")));
1403 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
1404 fp.write(fileheader.encode('UTF-8'));
1405 fnumfiles = format(int(numfiles), 'x').lower();
1406 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs['format_delimiter']);
1407 catfileheadercshex = GetFileChecksum(fileheader + fnumfilesa, checksumtype, True, formatspecs);
1408 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
1409 fp.write(fnumfilesa.encode('UTF-8'));
1410 try:
1411 fp.flush();
1412 os.fsync(fp.fileno());
1413 except io.UnsupportedOperation:
1414 pass;
1415 except AttributeError:
1416 pass;
1417 except OSError as e:
1418 pass;
1419 return fp;
1420
1421 def MakeEmptyFilePointer(fp, checksumtype="crc32", formatspecs=__file_format_dict__):
1422 formatspecs = FormatSpecsListToDict(formatspecs);
1423 AppendFileHeader(fp, 0, checksumtype, formatspecs);
1424 return fp;
1425
1426 def MakeEmptyFile(outfile, compression="auto", compressionlevel=None, checksumtype="crc32", formatspecs=__file_format_dict__, returnfp=False):
1427 formatspecs = FormatSpecsListToDict(formatspecs);
1428 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1429 if(os.path.exists(outfile)):
1430 try:
1431 os.unlink(outfile);
1432 except OSError as e:
1433 pass;
1434 if(outfile=="-"):
1435 verbose = False;
1436 catfpfp = BytesIO();
1437 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1438 catfp = outfile;
1439 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1440 catfp = BytesIO();
1441 else:
1442 fbasename = os.path.splitext(outfile)[0];
1443 fextname = os.path.splitext(outfile)[1];
1444 catfp = CompressOpenFile(outfile, True, compressionlevel);
1445 catfp = AppendFileHeader(catfp, 0, checksumtype, formatspecs);
1446 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1447 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1448 try:
1449 catfp.flush();
1450 os.fsync(catfp.fileno());
1451 except io.UnsupportedOperation:
1452 pass;
1453 except AttributeError:
1454 pass;
1455 except OSError as e:
1456 pass;
1457 if(outfile=="-"):
1458 catfp.seek(0, 0);
1459 if(hasattr(sys.stdout, "buffer")):
1460 shutil.copyfileobj(catfp, sys.stdout.buffer);
1461 else:
1462 shutil.copyfileobj(catfp, sys.stdout);
1463 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1464 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1465 catfp.seek(0, 0);
1466 upload_file_to_internet_file(catfp, outfile);
1467 if(returnfp):
1468 catfp.seek(0, 0);
1469 return catfp;
1470 else:
1471 catfp.close();
1472 return True;
1473
1474 def AppendFileHeaderWithContent(fp, filevalues=[], extradata=[], filecontent="", checksumtype="crc32", formatspecs=__file_format_dict__):
1475 formatspecs = FormatSpecsListToDict(formatspecs);
1476 extrafields = format(len(extradata), 'x').lower();
1477 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
1478 if(len(extradata)>0):
1479 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
1480 extrasizelen = format(len(extrasizestr), 'x').lower();
1481 catoutlen = len(filevalues) + len(extradata) + 5;
1482 catoutlenhex = format(catoutlen, 'x').lower();
1483 catoutlist = filevalues;
1484 catoutlist.insert(0, catoutlenhex);
1485 catoutlist.append(extrasizelen);
1486 catoutlist.append(extrafields);
1487 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
1488 if(len(extradata)>0):
1489 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
1490 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
1491 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
1492 catfilecontentcshex = GetFileChecksum(filecontent, checksumtype, False, formatspecs);
1493 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
1494 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
1495 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
1496 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
1497 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
1498 catfileoutstrecd = catfileoutstr.encode('UTF-8');
1499 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
1500 catfileout = catfileoutstrecd + filecontent + nullstrecd;
1501 fp.write(catfileout);
1502 try:
1503 fp.flush();
1504 os.fsync(fp.fileno());
1505 except io.UnsupportedOperation:
1506 pass;
1507 except AttributeError:
1508 pass;
1509 except OSError as e:
1510 pass;
1511 return fp;
1512
1513 def AppendFilesWithContent(infiles, fp, dirlistfromtxt=False, filevalues=[], extradata=[], compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
1514 formatspecs = FormatSpecsListToDict(formatspecs);
1515 advancedlist = formatspecs['use_advanced_list'];
1516 altinode = formatspecs['use_alt_inode'];
1517 if(verbose):
1518 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
1519 if(infiles=="-"):
1520 for line in sys.stdin:
1521 infilelist.append(line.strip());
1522 infilelist = list(filter(None, infilelist));
1523 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
1524 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
1525 return False;
1526 with UncompressFile(infiles, formatspecs, "r") as finfile:
1527 for line in finfile:
1528 infilelist.append(line.strip());
1529 infilelist = list(filter(None, infilelist));
1530 else:
1531 if(isinstance(infiles, (list, tuple, ))):
1532 infilelist = list(filter(None, infiles));
1533 elif(isinstance(infiles, (str, ))):
1534 infilelist = list(filter(None, [infiles]));
1535 if(advancedlist):
1536 GetDirList = ListDirAdvanced(infilelist, followlink, False);
1537 else:
1538 GetDirList = ListDir(infilelist, followlink, False);
1539 if(not GetDirList):
1540 return False;
1541 curinode = 0;
1542 curfid = 0;
1543 inodelist = [];
1544 inodetofile = {};
1545 filetoinode = {};
1546 inodetocatinode = {};
1547 numfiles = int(len(GetDirList));
1548 fnumfiles = format(numfiles, 'x').lower();
1549 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
1550 for curfname in GetDirList:
1551 if(re.findall("^[.|/]", curfname)):
1552 fname = curfname;
1553 else:
1554 fname = "./"+curfname;
1555 if(verbose):
1556 VerbosePrintOut(fname);
1557 if(not followlink or followlink is None):
1558 fstatinfo = os.lstat(fname);
1559 else:
1560 fstatinfo = os.stat(fname);
1561 fpremode = fstatinfo.st_mode;
1562 finode = fstatinfo.st_ino;
1563 flinkcount = fstatinfo.st_nlink;
1564 ftype = 0;
1565 if(stat.S_ISREG(fpremode)):
1566 ftype = 0;
1567 elif(stat.S_ISLNK(fpremode)):
1568 ftype = 2;
1569 elif(stat.S_ISCHR(fpremode)):
1570 ftype = 3;
1571 elif(stat.S_ISBLK(fpremode)):
1572 ftype = 4;
1573 elif(stat.S_ISDIR(fpremode)):
1574 ftype = 5;
1575 elif(stat.S_ISFIFO(fpremode)):
1576 ftype = 6;
1577 elif(stat.S_ISSOCK(fpremode)):
1578 ftype = 8;
1579 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
1580 ftype = 9;
1581 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
1582 ftype = 10;
1583 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
1584 ftype = 111;
1585 else:
1586 ftype = 0;
1587 flinkname = "";
1588 fcurfid = format(int(curfid), 'x').lower();
1589 if(not followlink and finode!=0):
1590 if(ftype!=1):
1591 if(finode in inodelist):
1592 ftype = 1;
1593 flinkname = inodetofile[finode];
1594 if(altinode):
1595 fcurinode = format(int(finode), 'x').lower();
1596 else:
1597 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
1598 if(finode not in inodelist):
1599 inodelist.append(finode);
1600 inodetofile.update({finode: fname});
1601 inodetocatinode.update({finode: curinode});
1602 if(altinode):
1603 fcurinode = format(int(finode), 'x').lower();
1604 else:
1605 fcurinode = format(int(curinode), 'x').lower();
1606 curinode = curinode + 1;
1607 else:
1608 fcurinode = format(int(curinode), 'x').lower();
1609 curinode = curinode + 1;
1610 curfid = curfid + 1;
1611 if(ftype==2):
1612 flinkname = os.readlink(fname);
1613 fdev = fstatinfo.st_dev;
1614 getfdev = GetDevMajorMinor(fdev);
1615 fdev_minor = getfdev[0];
1616 fdev_major = getfdev[1];
1617 frdev = fstatinfo.st_dev;
1618 if(hasattr(fstatinfo, "st_rdev")):
1619 frdev = fstatinfo.st_rdev;
1620 else:
1621 frdev = fstatinfo.st_dev;
1622 getfrdev = GetDevMajorMinor(frdev);
1623 frdev_minor = getfrdev[0];
1624 frdev_major = getfrdev[1];
1625 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
1626 fsize = format(int("0"), 'x').lower();
1627 elif(ftype==0 or ftype==7):
1628 fsize = format(int(fstatinfo.st_size), 'x').lower();
1629 else:
1630 fsize = format(int(fstatinfo.st_size)).lower();
1631 fatime = format(int(fstatinfo.st_atime), 'x').lower();
1632 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
1633 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
1634 if(hasattr(fstatinfo, "st_birthtime")):
1635 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
1636 else:
1637 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
1638 fmode = format(int(fstatinfo.st_mode), 'x').lower();
1639 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
1640 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
1641 fuid = format(int(fstatinfo.st_uid), 'x').lower();
1642 fgid = format(int(fstatinfo.st_gid), 'x').lower();
1643 funame = "";
1644 try:
1645 import pwd;
1646 try:
1647 userinfo = pwd.getpwuid(fstatinfo.st_uid);
1648 funame = userinfo.pw_name;
1649 except KeyError:
1650 funame = "";
1651 except ImportError:
1652 funame = "";
1653 fgname = "";
1654 try:
1655 import grp;
1656 try:
1657 groupinfo = grp.getgrgid(fstatinfo.st_gid);
1658 fgname = groupinfo.gr_name;
1659 except KeyError:
1660 fgname = "";
1661 except ImportError:
1662 fgname = "";
1663 fdev_minor = format(int(fdev_minor), 'x').lower();
1664 fdev_major = format(int(fdev_major), 'x').lower();
1665 frdev_minor = format(int(frdev_minor), 'x').lower();
1666 frdev_major = format(int(frdev_major), 'x').lower();
1667 finode = format(int(finode), 'x').lower();
1668 flinkcount = format(int(flinkcount), 'x').lower();
1669 if(hasattr(fstatinfo, "st_file_attributes")):
1670 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
1671 else:
1672 fwinattributes = format(int(0), 'x').lower();
1673 fcompression = "";
1674 fcsize = format(int(0), 'x').lower();
1675 fcontents = BytesIO();
1676 chunk_size = 1024;
1677 if(ftype==0 or ftype==7):
1678 with open(fname, "rb") as fpc:
1679 shutil.copyfileobj(fpc, fcontents);
1680 if(not compresswholefile):
1681 fcontents.seek(0, 2);
1682 ucfsize = fcontents.tell();
1683 fcontents.seek(0, 0);
1684 if(compression=="auto"):
1685 ilsize = len(compressionlistalt);
1686 ilmin = 0;
1687 ilcsize = [];
1688 while(ilmin < ilsize):
1689 cfcontents = BytesIO();
1690 shutil.copyfileobj(fcontents, cfcontents);
1691 fcontents.seek(0, 0);
1692 cfcontents.seek(0, 0);
1693 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
1694 if(cfcontents):
1695 cfcontents.seek(0, 2);
1696 ilcsize.append(cfcontents.tell());
1697 cfcontents.close();
1698 else:
1699 try:
1700 ilcsize.append(sys.maxint);
1701 except AttributeError:
1702 ilcsize.append(sys.maxsize);
1703 ilmin = ilmin + 1;
1704 ilcmin = ilcsize.index(min(ilcsize));
1705 compression = compressionlistalt[ilcmin];
1706 fcontents.seek(0, 0);
1707 cfcontents = BytesIO();
1708 shutil.copyfileobj(fcontents, cfcontents);
1709 cfcontents.seek(0, 0);
1710 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
1711 cfcontents.seek(0, 2);
1712 cfsize = cfcontents.tell();
1713 if(ucfsize > cfsize):
1714 fcsize = format(int(cfsize), 'x').lower();
1715 fcompression = compression;
1716 fcontents.close();
1717 fcontents = cfcontents;
1718 if(followlink and (ftype==1 or ftype==2)):
1719 flstatinfo = os.stat(flinkname);
1720 with open(flinkname, "rb") as fpc:
1721 shutil.copyfileobj(fpc, fcontents);
1722 if(not compresswholefile):
1723 fcontents.seek(0, 2);
1724 ucfsize = fcontents.tell();
1725 fcontents.seek(0, 0);
1726 if(compression=="auto"):
1727 ilsize = len(compressionlistalt);
1728 ilmin = 0;
1729 ilcsize = [];
1730 while(ilmin < ilsize):
1731 cfcontents = BytesIO();
1732 shutil.copyfileobj(fcontents, cfcontents);
1733 fcontents.seek(0, 0);
1734 cfcontents.seek(0, 0);
1735 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
1736 if(cfcontents):
1737 cfcontents.seek(0, 2);
1738 ilcsize.append(cfcontents.tell());
1739 cfcontents.close();
1740 else:
1741 try:
1742 ilcsize.append(sys.maxint);
1743 except AttributeError:
1744 ilcsize.append(sys.maxsize);
1745 ilmin = ilmin + 1;
1746 ilcmin = ilcsize.index(min(ilcsize));
1747 compression = compressionlistalt[ilcmin];
1748 fcontents.seek(0, 0);
1749 cfcontents = BytesIO();
1750 shutil.copyfileobj(fcontents, cfcontents);
1751 cfcontents.seek(0, 0);
1752 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
1753 cfcontents.seek(0, 2);
1754 cfsize = cfcontents.tell();
1755 if(ucfsize > cfsize):
1756 fcsize = format(int(cfsize), 'x').lower();
1757 fcompression = compression;
1758 fcontents.close();
1759 fcontents = cfcontents;
1760 if(fcompression=="none"):
1761 fcompression = "";
1762 fcontents.seek(0, 0);
1763 ftypehex = format(ftype, 'x').lower();
1764 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
1765 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
1766 if(numfiles>0):
1767 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
1768 fp.seek(0, 0);
1769 return fp;
1770
1771 def AppendListsWithContent(inlist, fp, dirlistfromtxt=False, filevalues=[], extradata=[], compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
1772 formatspecs = FormatSpecsListToDict(formatspecs);
1773 if(verbose):
1774 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
1775 GetDirList = inlist;
1776 if(not GetDirList):
1777 return False;
1778 curinode = 0;
1779 curfid = 0;
1780 inodelist = [];
1781 inodetofile = {};
1782 filetoinode = {};
1783 inodetocatinode = {};
1784 numfiles = int(len(GetDirList));
1785 fnumfiles = format(numfiles, 'x').lower();
1786 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
1787 for curfname in GetDirList:
1788 ftype = format(curfname[0], 'x').lower();
1789 if(re.findall("^[.|/]", curfname[1])):
1790 fname = curfname[1];
1791 else:
1792 fname = "./"+curfname[1];
1793 fbasedir = os.path.dirname(fname);
1794 flinkname = curfname[2];
1795 fsize = format(curfname[3], 'x').lower();
1796 fatime = format(curfname[4], 'x').lower();
1797 fmtime = format(curfname[5], 'x').lower();
1798 fctime = format(curfname[6], 'x').lower();
1799 fbtime = format(curfname[7], 'x').lower();
1800 fmode = format(curfname[8], 'x').lower();
1801 fwinattributes = format(curfname[9], 'x').lower();
1802 fcompression = curfname[10];
1803 fcsize = format(curfname[11], 'x').lower();
1804 fuid = format(curfname[12], 'x').lower();
1805 funame = curfname[13];
1806 fgid = format(curfname[14], 'x').lower();
1807 fgname = curfname[15];
1808 fid = format(curfname[16], 'x').lower();
1809 finode = format(curfname[17], 'x').lower();
1810 flinkcount = format(curfname[18], 'x').lower();
1811 fdev_minor = format(curfname[19], 'x').lower();
1812 fdev_major = format(curfname[20], 'x').lower();
1813 frdev_minor = format(curfname[21], 'x').lower();
1814 frdev_major = format(curfname[22], 'x').lower();
1815 extradata = curfname[23];
1816 fchecksumtype = curfname[24];
1817 fcontents = curfname[25];
1818 catoutlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
1819 fcontents.seek(0, 0);
1820 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
1821 if(numfiles>0):
1822 fp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
1823 return fp;
1824
1825 def AppendInFileWithContent(infile, fp, dirlistfromtxt=False, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
1826 formatspecs = FormatSpecsListToDict(formatspecs);
1827 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, False, formatspecs);
1828 return AppendListsWithContent(inlist, fp, dirlistfromtxt, filevalues, extradata, followlink, checksumtype, formatspecs, verbose);
1829
1830 def AppendFilesWithContentToOutFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
1831 formatspecs = FormatSpecsListToDict(formatspecs);
1832 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1833 if(os.path.exists(outfile)):
1834 try:
1835 os.unlink(outfile);
1836 except OSError as e:
1837 pass;
1838 if(outfile=="-"):
1839 verbose = False;
1840 catfpfp = BytesIO();
1841 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1842 catfp = outfile;
1843 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1844 catfp = BytesIO();
1845 else:
1846 fbasename = os.path.splitext(outfile)[0];
1847 fextname = os.path.splitext(outfile)[1];
1848 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
1849 catfp = AppendFilesWithContent(infiles, catfp, dirlistfromtxt, filevalues, extradata, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose);
1850 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1851 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1852 try:
1853 catfp.flush();
1854 os.fsync(catfp.fileno());
1855 except io.UnsupportedOperation:
1856 pass;
1857 except AttributeError:
1858 pass;
1859 except OSError as e:
1860 pass;
1861 if(outfile=="-"):
1862 catfp.seek(0, 0);
1863 if(hasattr(sys.stdout, "buffer")):
1864 shutil.copyfileobj(catfp, sys.stdout.buffer);
1865 else:
1866 shutil.copyfileobj(catfp, sys.stdout);
1867 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1868 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1869 catfp.seek(0, 0);
1870 upload_file_to_internet_file(catfp, outfile);
1871 if(returnfp):
1872 catfp.seek(0, 0);
1873 return catfp;
1874 else:
1875 catfp.close();
1876 return True;
1877
1878 def AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
1879 formatspecs = FormatSpecsListToDict(formatspecs);
1880 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1881 if(os.path.exists(outfile)):
1882 try:
1883 os.unlink(outfile);
1884 except OSError as e:
1885 pass;
1886 if(outfile=="-"):
1887 verbose = False;
1888 catfpfp = BytesIO();
1889 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1890 catfp = outfile;
1891 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1892 catfp = BytesIO();
1893 else:
1894 fbasename = os.path.splitext(outfile)[0];
1895 fextname = os.path.splitext(outfile)[1];
1896 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
1897 catfp = AppendListsWithContent(inlist, catfp, dirlistfromtxt, filevalues, extradata, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose);
1898 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1899 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1900 try:
1901 catfp.flush();
1902 os.fsync(catfp.fileno());
1903 except io.UnsupportedOperation:
1904 pass;
1905 except AttributeError:
1906 pass;
1907 except OSError as e:
1908 pass;
1909 if(outfile=="-"):
1910 catfp.seek(0, 0);
1911 if(hasattr(sys.stdout, "buffer")):
1912 shutil.copyfileobj(catfp, sys.stdout.buffer);
1913 else:
1914 shutil.copyfileobj(catfp, sys.stdout);
1915 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1916 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1917 catfp.seek(0, 0);
1918 upload_file_to_internet_file(catfp, outfile);
1919 if(returnfp):
1920 catfp.seek(0, 0);
1921 return catfp;
1922 else:
1923 catfp.close();
1924 return True;
1925
1926 def AppendInFileWithContentToOutFile(infile, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
1927 formatspecs = FormatSpecsListToDict(formatspecs);
1928 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, False, formatspecs);
1929 return AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt, compression, compresswholefile, compressionlevel, filevalues, extradata, followlink, checksumtype, formatspecs, verbose, returnfp);
1930
1931 def PrintPermissionString(fchmode, ftype):
1932 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
1933 permissionstr = "";
1934 for fmodval in str(oct(fchmode))[-3:]:
1935 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
1936 if(ftype==0 or ftype==7):
1937 permissionstr = "-" + permissionstr;
1938 if(ftype==1):
1939 permissionstr = "h" + permissionstr;
1940 if(ftype==2):
1941 permissionstr = "l" + permissionstr;
1942 if(ftype==3):
1943 permissionstr = "c" + permissionstr;
1944 if(ftype==4):
1945 permissionstr = "b" + permissionstr;
1946 if(ftype==5):
1947 permissionstr = "d" + permissionstr;
1948 if(ftype==6):
1949 permissionstr = "f" + permissionstr;
1950 if(ftype==8):
1951 permissionstr = "D" + permissionstr;
1952 if(ftype==9):
1953 permissionstr = "p" + permissionstr;
1954 if(ftype==10):
1955 permissionstr = "w" + permissionstr;
1956 try:
1957 permissionoutstr = stat.filemode(fchmode);
1958 except AttributeError:
1959 permissionoutstr = permissionstr;
1960 except KeyError:
1961 permissionoutstr = permissionstr;
1962 return permissionoutstr;
1963
1964 def PrintPermissionStringAlt(fchmode, ftype):
1965 permissions = {
1966 '0': '---', '1': '--x', '2': '-w-', '3': '-wx',
1967 '4': 'r--', '5': 'r-x', '6': 'rw-', '7': 'rwx'
1968 };
1969 # Translate file mode into permission string
1970 permissionstr = ''.join([permissions[i] for i in str(oct(fchmode))[-3:]]);
1971 # Append file type indicator
1972 type_indicators = {
1973 0: '-', 1: 'h', 2: 'l', 3: 'c', 4: 'b',
1974 5: 'd', 6: 'f', 8: 'D', 9: 'p', 10: 'w'
1975 };
1976 file_type = type_indicators.get(ftype, '-');
1977 permissionstr = file_type + permissionstr;
1978 try:
1979 permissionoutstr = stat.filemode(fchmode);
1980 except AttributeError:
1981 permissionoutstr = permissionstr;
1982 return permissionoutstr;
1983
1984 def CompressionSupport():
1985 compression_list = [];
1986 try:
1987 import gzip;
1988 compression_list.append("gz");
1989 compression_list.append("gzip");
1990 except ImportError:
1991 '''return False;'''
1992 try:
1993 import bz2;
1994 compression_list.append("bz2");
1995 compression_list.append("bzip2");
1996 except ImportError:
1997 '''return False;'''
1998 try:
1999 import lz4;
2000 compression_list.append("lz4");
2001 except ImportError:
2002 '''return False;'''
2003 try:
2004 import lzo;
2005 compression_list.append("lzo");
2006 compression_list.append("lzop");
2007 except ImportError:
2008 '''return False;'''
2009 try:
2010 import zstandard;
2011 compression_list.append("zstd");
2012 compression_list.append("zstandard");
2013 except ImportError:
2014 '''return False;'''
2015 try:
2016 import lzma;
2017 compression_list.append("lzma");
2018 compression_list.append("xz");
2019 except ImportError:
2020 '''return False;'''
2021 return compression_list;
2022
2023 def CheckCompressionType(infile, formatspecs=__file_format_dict__, closefp=True):
2024 formatspecs = FormatSpecsListToDict(formatspecs);
2025 if(hasattr(infile, "read") or hasattr(infile, "write")):
2026 catfp = infile;
2027 else:
2028 try:
2029 catfp = open(infile, "rb");
2030 except FileNotFoundError:
2031 return False;
2032 catfp.seek(0, 0);
2033 prefp = catfp.read(2);
2034 filetype = False;
2035 if(prefp==binascii.unhexlify("1f8b")):
2036 filetype = "gzip";
2037 catfp.seek(0, 0);
2038 prefp = catfp.read(3);
2039 if(prefp==binascii.unhexlify("425a68")):
2040 filetype = "bzip2";
2041 if(prefp==binascii.unhexlify("5d0000")):
2042 filetype = "lzma";
2043 catfp.seek(0, 0);
2044 prefp = catfp.read(4);
2045 if(prefp==binascii.unhexlify("28b52ffd")):
2046 filetype = "zstd";
2047 if(prefp==binascii.unhexlify("04224d18")):
2048 filetype = "lz4";
2049 if(prefp==binascii.unhexlify("504B0304")):
2050 filetype = "zipfile";
2051 catfp.seek(0, 0);
2052 prefp = catfp.read(5);
2053 if(prefp==binascii.unhexlify("7573746172")):
2054 filetype = "tarfile";
2055 catfp.seek(0, 0);
2056 prefp = catfp.read(6);
2057 if(prefp==binascii.unhexlify("fd377a585a00")):
2058 filetype = "lzma";
2059 if(prefp==binascii.unhexlify("377abcaf271c")):
2060 filetype = "7zipfile";
2061 catfp.seek(0, 0);
2062 prefp = catfp.read(7);
2063 if(prefp==binascii.unhexlify("526172211a0700")):
2064 filetype = "rarfile";
2065 if(prefp==binascii.unhexlify("43617446696c65")):
2066 filetype = "catfile";
2067 catfp.seek(0, 0);
2068 prefp = catfp.read(8);
2069 if(prefp==binascii.unhexlify("526172211a070100")):
2070 filetype = "rarfile";
2071 catfp.seek(0, 0);
2072 prefp = catfp.read(formatspecs['format_len']);
2073 if(prefp==binascii.unhexlify(formatspecs['format_hex'])):
2074 filetype = formatspecs['format_lower'];
2075 catfp.seek(0, 0);
2076 prefp = catfp.read(9);
2077 if(prefp==binascii.unhexlify("894c5a4f000d0a1a0a")):
2078 filetype = "lzo";
2079 catfp.seek(0, 0);
2080 prefp = catfp.read(10);
2081 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
2082 filetype = "tarfile";
2083 catfp.seek(0, 0);
2084 if(filetype=="gzip" or filetype=="bzip2" or filetype=="lzma" or filetype=="zstd" or filetype=="lz4"):
2085 if(is_tarfile(catfp)):
2086 filetype = "tarfile";
2087 if(not filetype):
2088 if(is_tarfile(catfp)):
2089 filetype = "tarfile";
2090 elif(zipfile.is_zipfile(catfp)):
2091 filetype = "zipfile";
2092 elif(rarfile.is_rarfile(catfp) or rarfile.is_rarfile_sfx(catfp)):
2093 filetype = "rarile";
2094 else:
2095 filetype = False;
2096 catfp.seek(0, 0);
2097 if(closefp):
2098 catfp.close();
2099 return filetype;
2100
2101 def CheckCompressionTypeFromString(instring, formatspecs=__file_format_dict__, closefp=True):
2102 formatspecs = FormatSpecsListToDict(formatspecs);
2103 try:
2104 instringsfile = BytesIO(instring);
2105 except TypeError:
2106 instringsfile = BytesIO(instring.encode("UTF-8"));
2107 return CheckCompressionType(instringsfile, formatspecs, closefp);
2108
2109 def GetCompressionMimeType(infile, formatspecs=__file_format_dict__):
2110 formatspecs = FormatSpecsListToDict(formatspecs);
2111 compresscheck = CheckCompressionType(fp, formatspecs, False);
2112 if(compresscheck=="gzip" or compresscheck=="gz"):
2113 return archivefile_gzip_mimetype;
2114 if(compresscheck=="bzip2" or compresscheck=="bz2"):
2115 return archivefile_bzip2_mimetype;
2116 if(compresscheck=="zstd" or compresscheck=="zstandard"):
2117 return archivefile_zstandard_mimetype;
2118 if(compresscheck=="lz4"):
2119 return archivefile_lz4_mimetype;
2120 if(compresscheck=="lzo" or compresscheck=="lzop"):
2121 return archivefile_lzop_mimetype;
2122 if(compresscheck=="lzma"):
2123 return archivefile_lzma_mimetype;
2124 if(compresscheck=="xz"):
2125 return archivefile_xz_mimetype;
2126 if(compresscheck=="catfile" or compresscheck=="cat" or compresscheck==formatspecs['format_lower']):
2127 return archivefile_cat_mimetype;
2128 if(not compresscheck):
2129 return False;
2130 return False;
2131
2132 def UncompressArchiveFile(fp, formatspecs=__file_format_dict__):
2133 formatspecs = FormatSpecsListToDict(formatspecs);
2134 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
2135 return False;
2136 compresscheck = CheckCompressionType(fp, formatspecs, False);
2137 if(compresscheck=="gzip"):
2138 try:
2139 import gzip;
2140 except ImportError:
2141 return False;
2142 catfp = gzip.GzipFile(fileobj=fp, mode="rb");
2143 if(compresscheck=="bzip2"):
2144 try:
2145 import bz2;
2146 except ImportError:
2147 return False;
2148 catfp = BytesIO();
2149 catfp.write(bz2.decompress(fp.read()));
2150 if(compresscheck=="zstd"):
2151 try:
2152 import zstandard;
2153 except ImportError:
2154 return False;
2155 catfp = BytesIO();
2156 catfp.write(zstandard.decompress(fp.read()));
2157 if(compresscheck=="lz4"):
2158 try:
2159 import lz4.frame;
2160 except ImportError:
2161 return False;
2162 catfp = BytesIO();
2163 catfp.write(lz4.frame.decompress(fp.read()));
2164 if(compresscheck=="lzo" or compresscheck=="lzop"):
2165 try:
2166 import lzo;
2167 except ImportError:
2168 return False;
2169 catfp = BytesIO();
2170 catfp.write(lzo.decompress(fp.read()));
2171 if(compresscheck=="lzma" or compresscheck=="xz"):
2172 try:
2173 import lzma;
2174 except ImportError:
2175 return False;
2176 catfp = BytesIO();
2177 catfp.write(lzma.decompress(fp.read()));
2178 if(compresscheck=="catfile" or compresscheck==formatspecs['format_lower']):
2179 catfp = fp;
2180 if(not compresscheck):
2181 try:
2182 import lzma;
2183 except ImportError:
2184 return False;
2185 catfp = BytesIO();
2186 with fp as fpcontent:
2187 try:
2188 catfp.write(lzma.decompress(fp.read()));
2189 except lzma.LZMAError:
2190 return False;
2191 if(compresscheck!="catfile" or compresscheck!=formatspecs['format_lower']):
2192 fp.close();
2193 return catfp;
2194
2195 create_alias_function("Uncompress", __file_format_name__, "", UncompressArchiveFile);
2196
2197 def UncompressFile(infile, formatspecs=__file_format_dict__, mode="rb"):
2198 formatspecs = FormatSpecsListToDict(formatspecs);
2199 compresscheck = CheckCompressionType(infile, formatspecs, False);
2200 if(sys.version_info[0]==2 and compresscheck):
2201 if(mode=="rt"):
2202 mode = "r";
2203 if(mode=="wt"):
2204 mode = "w";
2205 try:
2206 if(compresscheck=="gzip"):
2207 try:
2208 import gzip;
2209 except ImportError:
2210 return False;
2211 try:
2212 filefp = gzip.open(infile, mode, encoding="UTF-8");
2213 except (ValueError, TypeError) as e:
2214 filefp = gzip.open(infile, mode);
2215 if(compresscheck=="bzip2"):
2216 try:
2217 import bz2;
2218 except ImportError:
2219 return False;
2220 try:
2221 filefp = bz2.open(infile, mode, encoding="UTF-8");
2222 except (ValueError, TypeError) as e:
2223 filefp = bz2.open(infile, mode);
2224 if(compresscheck=="zstd"):
2225 try:
2226 import zstandard;
2227 except ImportError:
2228 return False;
2229 try:
2230 filefp = zstandard.open(infile, mode, encoding="UTF-8");
2231 except (ValueError, TypeError) as e:
2232 filefp = zstandard.open(infile, mode);
2233 if(compresscheck=="lz4"):
2234 try:
2235 import lz4.frame;
2236 except ImportError:
2237 return False;
2238 try:
2239 filefp = lz4.frame.open(infile, mode, encoding="UTF-8");
2240 except (ValueError, TypeError) as e:
2241 filefp = lz4.frame.open(infile, mode);
2242 if(compresscheck=="lzo"):
2243 try:
2244 import lzo;
2245 except ImportError:
2246 return False;
2247 try:
2248 filefp = lzo.open(infile, mode, encoding="UTF-8");
2249 except (ValueError, TypeError) as e:
2250 filefp = lzo.open(infile, mode);
2251 if(compresscheck=="lzma"):
2252 try:
2253 import lzma;
2254 except ImportError:
2255 return False;
2256 try:
2257 filefp = lzma.open(infile, mode, encoding="UTF-8");
2258 except (ValueError, TypeError) as e:
2259 filefp = lzma.open(infile, mode);
2260 if(compresscheck=="catfile" or compresscheck==formatspecs['format_lower']):
2261 try:
2262 filefp = open(infile, mode, encoding="UTF-8");
2263 except (ValueError, TypeError) as e:
2264 filefp = open(infile, mode);
2265 if(not compresscheck):
2266 try:
2267 filefp = open(infile, mode, encoding="UTF-8");
2268 except (ValueError, TypeError) as e:
2269 filefp = open(infile, mode);
2270 except FileNotFoundError:
2271 return False;
2272 return filefp;
2273
2274 def UncompressString(infile):
2275 compresscheck = CheckCompressionTypeFromString(infile, formatspecs, False);
2276 if(compresscheck=="gzip"):
2277 try:
2278 import gzip;
2279 except ImportError:
2280 return False;
2281 fileuz = gzip.decompress(infile);
2282 if(compresscheck=="bzip2"):
2283 try:
2284 import bz2;
2285 except ImportError:
2286 return False;
2287 fileuz = bz2.decompress(infile);
2288 if(compresscheck=="zstd"):
2289 try:
2290 import zstandard;
2291 except ImportError:
2292 return False;
2293 fileuz = zstandard.decompress(infile);
2294 if(compresscheck=="lz4"):
2295 try:
2296 import lz4.frame;
2297 except ImportError:
2298 return False;
2299 fileuz = lz4.frame.decompress(infile);
2300 if(compresscheck=="lzo"):
2301 try:
2302 import lzo;
2303 except ImportError:
2304 return False;
2305 fileuz = lzo.decompress(infile);
2306 if(compresscheck=="lzma"):
2307 try:
2308 import lzma;
2309 except ImportError:
2310 return False;
2311 fileuz = lzma.decompress(infile);
2312 if(not compresscheck):
2313 fileuz = infile;
2314 if(hasattr(fileuz, 'decode')):
2315 fileuz = fileuz.decode("UTF-8");
2316 return fileuz;
2317
2318 def UncompressStringAlt(infile):
2319 filefp = StringIO();
2320 outstring = UncompressString(infile);
2321 filefp.write(outstring);
2322 filefp.seek(0, 0);
2323 return filefp;
2324
2325 def CheckCompressionSubType(infile, formatspecs=__file_format_dict__, closefp=True):
2326 formatspecs = FormatSpecsListToDict(formatspecs);
2327 compresscheck = CheckCompressionType(infile, formatspecs, False);
2328 if(not compresscheck):
2329 fextname = os.path.splitext(infile)[1];
2330 if(fextname==".gz"):
2331 compresscheck = "gzip";
2332 elif(fextname==".bz2"):
2333 compresscheck = "bzip2";
2334 elif(fextname==".zst"):
2335 compresscheck = "zstd";
2336 elif(fextname==".lz4"):
2337 compresscheck = "lz4";
2338 elif(fextname==".lzo" or fextname==".lzop"):
2339 compresscheck = "lzo";
2340 elif(fextname==".lzma" or fextname==".xz"):
2341 compresscheck = "lzma";
2342 else:
2343 return False;
2344 if(compresscheck=="gzip" or compresscheck=="bzip2" or compresscheck=="lzma" or compresscheck=="zstd" or compresscheck=="lz4"):
2345 if(is_tarfile(infile)):
2346 filetype = "tarfile";
2347 if(not compresscheck):
2348 if(is_tarfile(infile)):
2349 return "tarfile";
2350 elif(zipfile.is_zipfile(infile)):
2351 return "zipfile";
2352 elif(rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile)):
2353 return "rarile";
2354 else:
2355 return False;
2356 return False;
2357 if(compresscheck=="catfile"):
2358 return "catfile";
2359 if(compresscheck==formatspecs['format_lower']):
2360 return formatspecs['format_lower'];
2361 if(compresscheck=="tarfile"):
2362 return "tarfile";
2363 if(compresscheck=="zipfile"):
2364 return "zipfile";
2365 if(rarfile_support and compresscheck=="rarfile"):
2366 return "rarfile";
2367 if(py7zr_support and compresscheck=="7zipfile"):
2368 return "7zipfile";
2369 if(hasattr(infile, "read") or hasattr(infile, "write")):
2370 catfp = UncompressArchiveFile(infile, formatspecs['format_lower']);
2371 else:
2372 try:
2373 if(compresscheck=="gzip"):
2374 try:
2375 import gzip;
2376 except ImportError:
2377 return False;
2378 catfp = gzip.GzipFile(infile, "rb");
2379 if(compresscheck=="bzip2"):
2380 try:
2381 import bz2;
2382 except ImportError:
2383 return False;
2384 catfp = bz2.BZ2File(infile, "rb");
2385 if(compresscheck=="lz4"):
2386 try:
2387 import lz4.frame;
2388 except ImportError:
2389 return False;
2390 catfp = lz4.frame.open(infile, "rb");
2391 if(compresscheck=="zstd"):
2392 try:
2393 import zstandard;
2394 except ImportError:
2395 return False;
2396 catfp = zstandard.open(infile, "rb");
2397 if(compresscheck=="lzma" or compresscheck=="xz"):
2398 try:
2399 import lzma;
2400 except ImportError:
2401 return False;
2402 catfp = lzma.open(infile, "rb");
2403 except FileNotFoundError:
2404 return False;
2405 filetype = False;
2406 prefp = catfp.read(5);
2407 if(prefp==binascii.unhexlify("7573746172")):
2408 filetype = "tarfile";
2409 catfp.seek(0, 0);
2410 prefp = catfp.read(7);
2411 if(prefp==binascii.unhexlify("43617446696c65")):
2412 filetype = "catfile";
2413 catfp.seek(0, 0);
2414 prefp = catfp.read(formatspecs['format_len']);
2415 if(prefp==binascii.unhexlify(formatspecs['format_hex'])):
2416 filetype = formatspecs['format_lower'];
2417 catfp.seek(0, 0);
2418 prefp = catfp.read(10);
2419 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
2420 filetype = "tarfile";
2421 catfp.seek(0, 0);
2422 if(closefp):
2423 catfp.close();
2424 return filetype;
2425
2426 def GZipCompress(data, compresslevel=9):
2427 try:
2428 import gzip;
2429 except ImportError:
2430 return False;
2431 tmpfp = tempfile.NamedTemporaryFile("wb", delete=False);
2432 tmpfp.close();
2433 tmpfp = gzip.GzipFile(tmpfp.name, mode="wb", compresslevel=compresslevel);
2434 tmpfp.write(data);
2435 tmpfp.close();
2436 try:
2437 catfp = open(tmpfp.name, "rb");
2438 except FileNotFoundError:
2439 return False;
2440 catdata = catfp.read();
2441 catfp.close();
2442 return catdata;
2443
2444 def CompressArchiveFile(fp, compression="auto", compressionlevel=None, formatspecs=__file_format_dict__):
2445 formatspecs = FormatSpecsListToDict(formatspecs);
2446 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
2447 return False;
2448 fp.seek(0, 0);
2449 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
2450 compression = "auto";
2451 if(compression not in compressionlist and compression is None):
2452 compression = "auto";
2453 if(compression=="gzip"):
2454 try:
2455 import gzip;
2456 except ImportError:
2457 return False;
2458 catfp = BytesIO();
2459 if(compressionlevel is None):
2460 compressionlevel = 9;
2461 else:
2462 compressionlevel = int(compressionlevel);
2463 catfp.write(gzip.compress(fp.read(), compresslevel=compressionlevel));
2464 if(compression=="bzip2"):
2465 try:
2466 import bz2;
2467 except ImportError:
2468 return False;
2469 catfp = BytesIO();
2470 if(compressionlevel is None):
2471 compressionlevel = 9;
2472 else:
2473 compressionlevel = int(compressionlevel);
2474 catfp.write(bz2.compress(fp.read(), compresslevel=compressionlevel));
2475 if(compression=="lz4"):
2476 try:
2477 import lz4.frame;
2478 except ImportError:
2479 return False;
2480 catfp = BytesIO();
2481 if(compressionlevel is None):
2482 compressionlevel = 9;
2483 else:
2484 compressionlevel = int(compressionlevel);
2485 catfp.write(lz4.frame.compress(fp.read(), compression_level=compressionlevel));
2486 if(compression=="lzo" or compression=="lzop"):
2487 try:
2488 import lzo;
2489 except ImportError:
2490 return False;
2491 catfp = BytesIO();
2492 if(compressionlevel is None):
2493 compressionlevel = 9;
2494 else:
2495 compressionlevel = int(compressionlevel);
2496 catfp.write(lzo.compress(fp.read(), compresslevel=compressionlevel));
2497 if(compression=="zstd"):
2498 try:
2499 import zstandard;
2500 except ImportError:
2501 return False;
2502 catfp = BytesIO();
2503 if(compressionlevel is None):
2504 compressionlevel = 10;
2505 else:
2506 compressionlevel = int(compressionlevel);
2507 catfp.write(zstandard.compress(fp.read(), level=compressionlevel));
2508 if(compression=="lzma"):
2509 try:
2510 import lzma;
2511 except ImportError:
2512 return False;
2513 catfp = BytesIO();
2514 if(compressionlevel is None):
2515 compressionlevel = 9;
2516 else:
2517 compressionlevel = int(compressionlevel);
2518 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]));
2519 if(compression=="xz"):
2520 try:
2521 import lzma;
2522 except ImportError:
2523 return False;
2524 catfp = BytesIO();
2525 if(compressionlevel is None):
2526 compressionlevel = 9;
2527 else:
2528 compressionlevel = int(compressionlevel);
2529 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]));
2530 if(compression=="auto" or compression is None):
2531 catfp = fp;
2532 catfp.seek(0, 0);
2533 return catfp;
2534
2535 create_alias_function("Compress", __file_format_name__, "", CompressArchiveFile);
2536
2537 def CompressOpenFile(outfile, compressionenable=True, compressionlevel=None):
2538 if(outfile is None):
2539 return False;
2540 fbasename = os.path.splitext(outfile)[0];
2541 fextname = os.path.splitext(outfile)[1];
2542 if(compressionlevel is None and fextname!=".zst"):
2543 compressionlevel = 9;
2544 elif(compressionlevel is None and fextname==".zst"):
2545 compressionlevel = 10;
2546 else:
2547 compressionlevel = int(compressionlevel);
2548 if(sys.version_info[0]==2):
2549 mode = "w";
2550 else:
2551 mode = "wb";
2552 try:
2553 if(fextname not in outextlistwd or not compressionenable):
2554 try:
2555 outfp = open(outfile, "wb", encoding="UTF-8");
2556 except (ValueError, TypeError) as e:
2557 outfp = open(outfile, "wb");
2558 elif(fextname==".gz"):
2559 try:
2560 import gzip;
2561 except ImportError:
2562 return False;
2563 try:
2564 outfp = gzip.open(outfile, mode, compressionlevel, encoding="UTF-8");
2565 except (ValueError, TypeError) as e:
2566 outfp = gzip.open(outfile, mode, compressionlevel);
2567 elif(fextname==".bz2"):
2568 try:
2569 import bz2;
2570 except ImportError:
2571 return False;
2572 try:
2573 outfp = bz2.open(outfile, mode, compressionlevel, encoding="UTF-8");
2574 except (ValueError, TypeError) as e:
2575 outfp = bz2.open(outfile, mode, compressionlevel);
2576 elif(fextname==".zst"):
2577 try:
2578 import zstandard;
2579 except ImportError:
2580 return False;
2581 try:
2582 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel), encoding="UTF-8");
2583 except (ValueError, TypeError) as e:
2584 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel));
2585 elif(fextname==".xz"):
2586 try:
2587 import lzma;
2588 except ImportError:
2589 return False;
2590 try:
2591 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}], encoding="UTF-8");
2592 except (ValueError, TypeError) as e:
2593 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]);
2594 elif(fextname==".lz4"):
2595 try:
2596 import lz4.frame;
2597 except ImportError:
2598 return False;
2599 try:
2600 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel, encoding="UTF-8");
2601 except (ValueError, TypeError) as e:
2602 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel);
2603 elif(fextname==".lzo"):
2604 try:
2605 import lzo;
2606 except ImportError:
2607 return False;
2608 try:
2609 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel, encoding="UTF-8");
2610 except (ValueError, TypeError) as e:
2611 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel);
2612 elif(fextname==".lzma"):
2613 try:
2614 import lzma;
2615 except ImportError:
2616 return False;
2617 try:
2618 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}], encoding="UTF-8");
2619 except (ValueError, TypeError) as e:
2620 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]);
2621 except FileNotFoundError:
2622 return False;
2623 return outfp;
2624
2625 def GetDevMajorMinor(fdev):
2626 retdev = [];
2627 if(hasattr(os, "minor")):
2628 retdev.append(os.minor(fdev));
2629 else:
2630 retdev.append(0);
2631 if(hasattr(os, "major")):
2632 retdev.append(os.major(fdev));
2633 else:
2634 retdev.append(0);
2635 return retdev;
2636
2637 def CheckSumSupport(checkfor, guaranteed=True):
2638 if(guaranteed):
2639 try:
2640 hash_list = sorted(list(hashlib.algorithms_guaranteed));
2641 except AttributeError:
2642 hash_list = sorted(list(hashlib.algorithms));
2643 else:
2644 try:
2645 hash_list = sorted(list(hashlib.algorithms_available));
2646 except AttributeError:
2647 hash_list = sorted(list(hashlib.algorithms));
2648 checklistout = sorted(hash_list + ['adler32', 'crc16', 'crc16_ansi', 'crc16_ibm', 'crc16_ccitt', 'crc32', 'crc64', 'crc64_ecma', 'crc64_iso', 'none']);
2649 if(checkfor in checklistout):
2650 return True;
2651 else:
2652 return False;
2653
2654 def CheckSumSupportAlt(checkfor, guaranteed=True):
2655 if(guaranteed):
2656 try:
2657 hash_list = sorted(list(hashlib.algorithms_guaranteed));
2658 except AttributeError:
2659 hash_list = sorted(list(hashlib.algorithms));
2660 else:
2661 try:
2662 hash_list = sorted(list(hashlib.algorithms_available));
2663 except AttributeError:
2664 hash_list = sorted(list(hashlib.algorithms));
2665 checklistout = hash_list;
2666 if(checkfor in checklistout):
2667 return True;
2668 else:
2669 return False;
2670
2671 def PackArchiveFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2672 formatspecs = FormatSpecsListToDict(formatspecs);
2673 advancedlist = formatspecs['use_advanced_list'];
2674 altinode = formatspecs['use_alt_inode'];
2675 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2676 outfile = RemoveWindowsPath(outfile);
2677 checksumtype = checksumtype.lower();
2678 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
2679 checksumtype="crc32";
2680 if(checksumtype=="none"):
2681 checksumtype = "";
2682 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
2683 compression = "auto";
2684 if(compression not in compressionlist and compression is None):
2685 compression = "auto";
2686 if(verbose):
2687 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2688 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2689 if(os.path.exists(outfile)):
2690 try:
2691 os.unlink(outfile);
2692 except OSError as e:
2693 pass;
2694 if(outfile=="-"):
2695 verbose = False;
2696 catfp = BytesIO();
2697 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2698 catfp = outfile;
2699 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2700 catfp = BytesIO();
2701 else:
2702 fbasename = os.path.splitext(outfile)[0];
2703 fextname = os.path.splitext(outfile)[1];
2704 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
2705 catver = formatspecs['format_ver'];
2706 fileheaderver = str(int(catver.replace(".", "")));
2707 infilelist = [];
2708 if(infiles=="-"):
2709 for line in sys.stdin:
2710 infilelist.append(line.strip());
2711 infilelist = list(filter(None, infilelist));
2712 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
2713 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
2714 return False;
2715 with UncompressFile(infiles, formatspecs, "r") as finfile:
2716 for line in finfile:
2717 infilelist.append(line.strip());
2718 infilelist = list(filter(None, infilelist));
2719 else:
2720 if(isinstance(infiles, (list, tuple, ))):
2721 infilelist = list(filter(None, infiles));
2722 elif(isinstance(infiles, (str, ))):
2723 infilelist = list(filter(None, [infiles]));
2724 if(advancedlist):
2725 GetDirList = ListDirAdvanced(infilelist, followlink, False);
2726 else:
2727 GetDirList = ListDir(infilelist, followlink, False);
2728 if(not GetDirList):
2729 return False;
2730 curinode = 0;
2731 curfid = 0;
2732 inodelist = [];
2733 inodetofile = {};
2734 filetoinode = {};
2735 inodetocatinode = {};
2736 numfiles = int(len(GetDirList));
2737 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
2738 for curfname in GetDirList:
2739 if(re.findall("^[.|/]", curfname)):
2740 fname = curfname;
2741 else:
2742 fname = "./"+curfname;
2743 if(verbose):
2744 VerbosePrintOut(fname);
2745 if(not followlink or followlink is None):
2746 fstatinfo = os.lstat(fname);
2747 else:
2748 fstatinfo = os.stat(fname);
2749 fpremode = fstatinfo.st_mode;
2750 finode = fstatinfo.st_ino;
2751 flinkcount = fstatinfo.st_nlink;
2752 ftype = 0;
2753 if(stat.S_ISREG(fpremode)):
2754 ftype = 0;
2755 elif(stat.S_ISLNK(fpremode)):
2756 ftype = 2;
2757 elif(stat.S_ISCHR(fpremode)):
2758 ftype = 3;
2759 elif(stat.S_ISBLK(fpremode)):
2760 ftype = 4;
2761 elif(stat.S_ISDIR(fpremode)):
2762 ftype = 5;
2763 elif(stat.S_ISFIFO(fpremode)):
2764 ftype = 6;
2765 elif(stat.S_ISSOCK(fpremode)):
2766 ftype = 8;
2767 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
2768 ftype = 9;
2769 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
2770 ftype = 10;
2771 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
2772 ftype = 111;
2773 else:
2774 ftype = 0;
2775 flinkname = "";
2776 fcurfid = format(int(curfid), 'x').lower();
2777 if(not followlink and finode!=0):
2778 if(ftype!=1):
2779 if(finode in inodelist):
2780 ftype = 1;
2781 flinkname = inodetofile[finode];
2782 if(altinode):
2783 fcurinode = format(int(finode), 'x').lower();
2784 else:
2785 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
2786 if(finode not in inodelist):
2787 inodelist.append(finode);
2788 inodetofile.update({finode: fname});
2789 inodetocatinode.update({finode: curinode});
2790 if(altinode):
2791 fcurinode = format(int(finode), 'x').lower();
2792 else:
2793 fcurinode = format(int(curinode), 'x').lower();
2794 curinode = curinode + 1;
2795 else:
2796 fcurinode = format(int(curinode), 'x').lower();
2797 curinode = curinode + 1;
2798 curfid = curfid + 1;
2799 if(ftype==2):
2800 flinkname = os.readlink(fname);
2801 fdev = fstatinfo.st_dev;
2802 getfdev = GetDevMajorMinor(fdev);
2803 fdev_minor = getfdev[0];
2804 fdev_major = getfdev[1];
2805 frdev = fstatinfo.st_dev;
2806 if(hasattr(fstatinfo, "st_rdev")):
2807 frdev = fstatinfo.st_rdev;
2808 else:
2809 frdev = fstatinfo.st_dev;
2810 getfrdev = GetDevMajorMinor(frdev);
2811 frdev_minor = getfrdev[0];
2812 frdev_major = getfrdev[1];
2813 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
2814 fsize = format(int("0"), 'x').lower();
2815 elif(ftype==0 or ftype==7):
2816 fsize = format(int(fstatinfo.st_size), 'x').lower();
2817 else:
2818 fsize = format(int(fstatinfo.st_size)).lower();
2819 fatime = format(int(fstatinfo.st_atime), 'x').lower();
2820 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
2821 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
2822 if(hasattr(fstatinfo, "st_birthtime")):
2823 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
2824 else:
2825 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
2826 fmode = format(int(fstatinfo.st_mode), 'x').lower();
2827 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
2828 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
2829 fuid = format(int(fstatinfo.st_uid), 'x').lower();
2830 fgid = format(int(fstatinfo.st_gid), 'x').lower();
2831 funame = "";
2832 try:
2833 import pwd;
2834 try:
2835 userinfo = pwd.getpwuid(fstatinfo.st_uid);
2836 funame = userinfo.pw_name;
2837 except KeyError:
2838 funame = "";
2839 except ImportError:
2840 funame = "";
2841 fgname = "";
2842 try:
2843 import grp;
2844 try:
2845 groupinfo = grp.getgrgid(fstatinfo.st_gid);
2846 fgname = groupinfo.gr_name;
2847 except KeyError:
2848 fgname = "";
2849 except ImportError:
2850 fgname = "";
2851 fdev_minor = format(int(fdev_minor), 'x').lower();
2852 fdev_major = format(int(fdev_major), 'x').lower();
2853 frdev_minor = format(int(frdev_minor), 'x').lower();
2854 frdev_major = format(int(frdev_major), 'x').lower();
2855 finode = format(int(finode), 'x').lower();
2856 flinkcount = format(int(flinkcount), 'x').lower();
2857 if(hasattr(fstatinfo, "st_file_attributes")):
2858 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
2859 else:
2860 fwinattributes = format(int(0), 'x').lower();
2861 fcompression = "";
2862 fcsize = format(int(0), 'x').lower();
2863 fcontents = BytesIO();
2864 if(ftype==0 or ftype==7):
2865 with open(fname, "rb") as fpc:
2866 shutil.copyfileobj(fpc, fcontents);
2867 if(not compresswholefile):
2868 fcontents.seek(0, 2);
2869 ucfsize = fcontents.tell();
2870 fcontents.seek(0, 0);
2871 if(compression=="auto"):
2872 ilsize = len(compressionlistalt);
2873 ilmin = 0;
2874 ilcsize = [];
2875 while(ilmin < ilsize):
2876 cfcontents = BytesIO();
2877 shutil.copyfileobj(fcontents, cfcontents);
2878 fcontents.seek(0, 0);
2879 cfcontents.seek(0, 0);
2880 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
2881 if(cfcontents):
2882 cfcontents.seek(0, 2);
2883 ilcsize.append(cfcontents.tell());
2884 cfcontents.close();
2885 else:
2886 try:
2887 ilcsize.append(sys.maxint);
2888 except AttributeError:
2889 ilcsize.append(sys.maxsize);
2890 ilmin = ilmin + 1;
2891 ilcmin = ilcsize.index(min(ilcsize));
2892 compression = compressionlistalt[ilcmin];
2893 fcontents.seek(0, 0);
2894 cfcontents = BytesIO();
2895 shutil.copyfileobj(fcontents, cfcontents);
2896 cfcontents.seek(0, 0);
2897 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
2898 cfcontents.seek(0, 2);
2899 cfsize = cfcontents.tell();
2900 if(ucfsize > cfsize):
2901 fcsize = format(int(cfsize), 'x').lower();
2902 fcompression = compression;
2903 fcontents.close();
2904 fcontents = cfcontents;
2905 if(fcompression=="none"):
2906 fcompression = "";
2907 if(followlink and (ftype==1 or ftype==2)):
2908 flstatinfo = os.stat(flinkname);
2909 with open(flinkname, "rb") as fpc:
2910 shutil.copyfileobj(fpc, fcontents);
2911 if(not compresswholefile):
2912 fcontents.seek(0, 2);
2913 ucfsize = fcontents.tell();
2914 fcontents.seek(0, 0);
2915 if(compression=="auto"):
2916 ilsize = len(compressionlistalt);
2917 ilmin = 0;
2918 ilcsize = [];
2919 while(ilmin < ilsize):
2920 cfcontents = BytesIO();
2921 shutil.copyfileobj(fcontents, cfcontents);
2922 fcontents.seek(0, 0);
2923 cfcontents.seek(0, 0);
2924 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
2925 if(cfcontents):
2926 cfcontents.seek(0, 2);
2927 ilcsize.append(cfcontents.tell());
2928 cfcontents.close();
2929 else:
2930 try:
2931 ilcsize.append(sys.maxint);
2932 except AttributeError:
2933 ilcsize.append(sys.maxsize);
2934 ilmin = ilmin + 1;
2935 ilcmin = ilcsize.index(min(ilcsize));
2936 compression = compressionlistalt[ilcmin];
2937 fcontents.seek(0, 0);
2938 cfcontents = BytesIO();
2939 shutil.copyfileobj(fcontents, cfcontents);
2940 cfcontents.seek(0, 0);
2941 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
2942 cfcontents.seek(0, 2);
2943 cfsize = cfcontents.tell();
2944 if(ucfsize > cfsize):
2945 fcsize = format(int(cfsize), 'x').lower();
2946 fcompression = compression;
2947 fcontents.close();
2948 fcontents = cfcontents;
2949 fcontents.seek(0, 0);
2950 ftypehex = format(ftype, 'x').lower();
2951 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
2952 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
2953 fcontents.close();
2954 if(numfiles>0):
2955 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
2956 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
2957 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2958 try:
2959 catfp.flush();
2960 os.fsync(catfp.fileno());
2961 except io.UnsupportedOperation:
2962 pass;
2963 except AttributeError:
2964 pass;
2965 except OSError as e:
2966 pass;
2967 if(outfile=="-"):
2968 catfp.seek(0, 0);
2969 if(hasattr(sys.stdout, "buffer")):
2970 shutil.copyfileobj(catfp, sys.stdout.buffer);
2971 else:
2972 shutil.copyfileobj(catfp, sys.stdout);
2973 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2974 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2975 catfp.seek(0, 0);
2976 upload_file_to_internet_file(catfp, outfile);
2977 if(returnfp):
2978 catfp.seek(0, 0);
2979 return catfp;
2980 else:
2981 catfp.close();
2982 return True;
2983
2984 create_alias_function("Pack", __file_format_name__, "", PackArchiveFile);
2985
2986 if(hasattr(shutil, "register_archive_format")):
2987 def PackArchiveFileFunc(archive_name, source_dir, **kwargs):
2988 return PackArchiveFile(source_dir, archive_name, False, "auto", True, None, False, "crc32", [], __file_format_dict__['format_delimiter'], False, False);
2989 create_alias_function("Pack", __file_format_name__, "Func", PackArchiveFileFunc);
2990
2991 def PackArchiveFileFromDirList(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2992 formatspecs = FormatSpecsListToDict(formatspecs);
2993 return PackArchiveFile(infiles, outfile, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, returnfp);
2994
2995 create_alias_function("Pack", __file_format_name__, "FromDirList", PackArchiveFileFromDirList);
2996
2997 def PackArchiveFileFromTarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2998 formatspecs = FormatSpecsListToDict(formatspecs);
2999 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3000 outfile = RemoveWindowsPath(outfile);
3001 checksumtype = checksumtype.lower();
3002 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3003 checksumtype="crc32";
3004 if(checksumtype=="none"):
3005 checksumtype = "";
3006 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3007 compression = "auto";
3008 if(compression not in compressionlist and compression is None):
3009 compression = "auto";
3010 if(verbose):
3011 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3012 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3013 if(os.path.exists(outfile)):
3014 try:
3015 os.unlink(outfile);
3016 except OSError as e:
3017 pass;
3018 if(outfile=="-"):
3019 verbose = False;
3020 catfp = BytesIO();
3021 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3022 catfp = outfile;
3023 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3024 catfp = BytesIO();
3025 else:
3026 fbasename = os.path.splitext(outfile)[0];
3027 fextname = os.path.splitext(outfile)[1];
3028 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3029 catver = formatspecs['format_ver'];
3030 fileheaderver = str(int(catver.replace(".", "")));
3031 curinode = 0;
3032 curfid = 0;
3033 inodelist = [];
3034 inodetofile = {};
3035 filetoinode = {};
3036 inodetocatinode = {};
3037 if(infile=="-"):
3038 infile = BytesIO();
3039 if(hasattr(sys.stdin, "buffer")):
3040 shutil.copyfileobj(sys.stdin.buffer, infile);
3041 else:
3042 shutil.copyfileobj(sys.stdin, infile);
3043 infile.seek(0, 0);
3044 if(not infile):
3045 return False;
3046 infile.seek(0, 0);
3047 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3048 infile = download_file_from_internet_file(infile);
3049 infile.seek(0, 0);
3050 if(not infile):
3051 return False;
3052 infile.seek(0, 0);
3053 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
3054 return False;
3055 elif(os.path.exists(infile) and os.path.isfile(infile)):
3056 try:
3057 if(not tarfile.is_tarfile(infile)):
3058 return False;
3059 except AttributeError:
3060 if(not is_tarfile(infile)):
3061 return False;
3062 else:
3063 return False;
3064 try:
3065 if(hasattr(infile, "read") or hasattr(infile, "write")):
3066 tarfp = tarfile.open(fileobj=infile, mode="r");
3067 else:
3068 tarfp = tarfile.open(infile, "r");
3069 except FileNotFoundError:
3070 return False;
3071 numfiles = int(len(tarfp.getmembers()));
3072 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3073 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
3074 if(re.findall("^[.|/]", member.name)):
3075 fname = member.name;
3076 else:
3077 fname = "./"+member.name;
3078 if(verbose):
3079 VerbosePrintOut(fname);
3080 fpremode = member.mode;
3081 ffullmode = member.mode;
3082 flinkcount = 0;
3083 ftype = 0;
3084 if(member.isreg()):
3085 ffullmode = member.mode + stat.S_IFREG;
3086 ftype = 0;
3087 elif(member.isdev()):
3088 ffullmode = member.mode;
3089 ftype = 7;
3090 elif(member.islnk()):
3091 ffullmode = member.mode + stat.S_IFREG;
3092 ftype = 1;
3093 elif(member.issym()):
3094 ffullmode = member.mode + stat.S_IFLNK;
3095 ftype = 2;
3096 elif(member.ischr()):
3097 ffullmode = member.mode + stat.S_IFCHR;
3098 ftype = 3;
3099 elif(member.isblk()):
3100 ffullmode = member.mode + stat.S_IFBLK;
3101 ftype = 4;
3102 elif(member.isdir()):
3103 ffullmode = member.mode + stat.S_IFDIR;
3104 ftype = 5;
3105 elif(member.isfifo()):
3106 ffullmode = member.mode + stat.S_IFIFO;
3107 ftype = 6;
3108 elif(member.issparse()):
3109 ffullmode = member.mode;
3110 ftype = 12;
3111 else:
3112 ffullmode = member.mode;
3113 ftype = 0;
3114 flinkname = "";
3115 fcurfid = format(int(curfid), 'x').lower();
3116 fcurinode = format(int(curfid), 'x').lower();
3117 curfid = curfid + 1;
3118 if(ftype==2):
3119 flinkname = member.linkname;
3120 fdev_minor = format(int(member.devminor), 'x').lower();
3121 fdev_major = format(int(member.devmajor), 'x').lower();
3122 frdev_minor = format(int(member.devminor), 'x').lower();
3123 frdev_major = format(int(member.devmajor), 'x').lower();
3124 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
3125 fsize = format(int("0"), 'x').lower();
3126 elif(ftype==0 or ftype==7):
3127 fsize = format(int(member.size), 'x').lower();
3128 else:
3129 fsize = format(int(member.size), 'x').lower();
3130 fatime = format(int(member.mtime), 'x').lower();
3131 fmtime = format(int(member.mtime), 'x').lower();
3132 fctime = format(int(member.mtime), 'x').lower();
3133 fbtime = format(int(member.mtime), 'x').lower();
3134 fmode = format(int(ffullmode), 'x').lower();
3135 fchmode = format(int(stat.S_IMODE(ffullmode)), 'x').lower();
3136 ftypemod = format(int(stat.S_IFMT(ffullmode)), 'x').lower();
3137 fuid = format(int(member.uid), 'x').lower();
3138 fgid = format(int(member.gid), 'x').lower();
3139 funame = member.uname;
3140 fgname = member.gname;
3141 flinkcount = format(int(flinkcount), 'x').lower();
3142 fwinattributes = format(int(0), 'x').lower();
3143 fcompression = "";
3144 fcsize = format(int(0), 'x').lower();
3145 fcontents = BytesIO();
3146 if(ftype==0 or ftype==7):
3147 with tarfp.extractfile(member) as fpc:
3148 shutil.copyfileobj(fpc, fcontents);
3149 if(not compresswholefile):
3150 fcontents.seek(0, 2);
3151 ucfsize = fcontents.tell();
3152 fcontents.seek(0, 0);
3153 if(compression=="auto"):
3154 ilsize = len(compressionlistalt);
3155 ilmin = 0;
3156 ilcsize = [];
3157 while(ilmin < ilsize):
3158 cfcontents = BytesIO();
3159 shutil.copyfileobj(fcontents, cfcontents);
3160 fcontents.seek(0, 0);
3161 cfcontents.seek(0, 0);
3162 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3163 if(cfcontents):
3164 cfcontents.seek(0, 2);
3165 ilcsize.append(cfcontents.tell());
3166 cfcontents.close();
3167 else:
3168 try:
3169 ilcsize.append(sys.maxint);
3170 except AttributeError:
3171 ilcsize.append(sys.maxsize);
3172 ilmin = ilmin + 1;
3173 ilcmin = ilcsize.index(min(ilcsize));
3174 compression = compressionlistalt[ilcmin];
3175 fcontents.seek(0, 0);
3176 cfcontents = BytesIO();
3177 shutil.copyfileobj(fcontents, cfcontents);
3178 cfcontents.seek(0, 0);
3179 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3180 cfcontents.seek(0, 2);
3181 cfsize = cfcontents.tell();
3182 if(ucfsize > cfsize):
3183 fcsize = format(int(cfsize), 'x').lower();
3184 fcompression = compression;
3185 fcontents.close();
3186 fcontents = cfcontents;
3187 if(fcompression=="none"):
3188 fcompression = "";
3189 fcontents.seek(0, 0);
3190 ftypehex = format(ftype, 'x').lower();
3191 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3192 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3193 fcontents.close();
3194 if(numfiles>0):
3195 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3196 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3197 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3198 try:
3199 catfp.flush();
3200 os.fsync(catfp.fileno());
3201 except io.UnsupportedOperation:
3202 pass;
3203 except AttributeError:
3204 pass;
3205 except OSError as e:
3206 pass;
3207 if(outfile=="-"):
3208 catfp.seek(0, 0);
3209 if(hasattr(sys.stdout, "buffer")):
3210 shutil.copyfileobj(catfp, sys.stdout.buffer);
3211 else:
3212 shutil.copyfileobj(catfp, sys.stdout);
3213 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3214 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3215 catfp.seek(0, 0);
3216 upload_file_to_internet_file(catfp, outfile);
3217 if(returnfp):
3218 catfp.seek(0, 0);
3219 return catfp;
3220 else:
3221 catfp.close();
3222 return True;
3223
3224 create_alias_function("Pack", __file_format_name__, "FromTarFile", PackArchiveFileFromTarFile);
3225
3226 def PackArchiveFileFromZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3227 formatspecs = FormatSpecsListToDict(formatspecs);
3228 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3229 outfile = RemoveWindowsPath(outfile);
3230 checksumtype = checksumtype.lower();
3231 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3232 checksumtype="crc32";
3233 if(checksumtype=="none"):
3234 checksumtype = "";
3235 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3236 compression = "auto";
3237 if(compression not in compressionlist and compression is None):
3238 compression = "auto";
3239 if(verbose):
3240 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3241 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3242 if(os.path.exists(outfile)):
3243 try:
3244 os.unlink(outfile);
3245 except OSError as e:
3246 pass;
3247 if(outfile=="-"):
3248 verbose = False;
3249 catfp = BytesIO();
3250 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3251 catfp = outfile;
3252 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3253 catfp = BytesIO();
3254 else:
3255 fbasename = os.path.splitext(outfile)[0];
3256 fextname = os.path.splitext(outfile)[1];
3257 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3258 catver = formatspecs['format_ver'];
3259 fileheaderver = str(int(catver.replace(".", "")));
3260 curinode = 0;
3261 curfid = 0;
3262 inodelist = [];
3263 inodetofile = {};
3264 filetoinode = {};
3265 inodetocatinode = {};
3266 if(infile=="-"):
3267 infile = BytesIO();
3268 if(hasattr(sys.stdin, "buffer")):
3269 shutil.copyfileobj(sys.stdin.buffer, infile);
3270 else:
3271 shutil.copyfileobj(sys.stdin, infile);
3272 infile.seek(0, 0);
3273 if(not infile):
3274 return False;
3275 infile.seek(0, 0);
3276 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3277 infile = download_file_from_internet_file(infile);
3278 infile.seek(0, 0);
3279 if(not infile):
3280 return False;
3281 infile.seek(0, 0);
3282 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
3283 return False;
3284 else:
3285 return False;
3286 if(not zipfile.is_zipfile(infile)):
3287 return False;
3288 try:
3289 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
3290 except FileNotFoundError:
3291 return False;
3292 ziptest = zipfp.testzip();
3293 if(ziptest):
3294 VerbosePrintOut("Bad file found!");
3295 numfiles = int(len(zipfp.infolist()));
3296 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3297 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
3298 if(re.findall("^[.|/]", member.filename)):
3299 fname = member.filename;
3300 else:
3301 fname = "./"+member.filename;
3302 zipinfo = zipfp.getinfo(member.filename);
3303 if(verbose):
3304 VerbosePrintOut(fname);
3305 if(not member.is_dir()):
3306 fpremode = int(stat.S_IFREG + 438);
3307 elif(member.is_dir()):
3308 fpremode = int(stat.S_IFDIR + 511);
3309 flinkcount = 0;
3310 ftype = 0;
3311 if(not member.is_dir()):
3312 ftype = 0;
3313 elif(member.is_dir()):
3314 ftype = 5;
3315 flinkname = "";
3316 fcurfid = format(int(curfid), 'x').lower();
3317 fcurinode = format(int(curfid), 'x').lower();
3318 curfid = curfid + 1;
3319 fdev_minor = format(int(0), 'x').lower();
3320 fdev_major = format(int(0), 'x').lower();
3321 frdev_minor = format(int(0), 'x').lower();
3322 frdev_major = format(int(0), 'x').lower();
3323 if(ftype==5):
3324 fsize = format(int("0"), 'x').lower();
3325 elif(ftype==0):
3326 fsize = format(int(member.file_size), 'x').lower();
3327 else:
3328 fsize = format(int(member.file_size), 'x').lower();
3329 fatime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3330 fmtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3331 fctime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3332 fbtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3333 if(zipinfo.create_system==0 or zipinfo.create_system==10):
3334 fwinattributes = format(int(zipinfo.external_attr), 'x').lower();
3335 if(not member.is_dir()):
3336 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3337 fchmode = stat.S_IMODE(fmode);
3338 ftypemod = stat.S_IFMT(fmode);
3339 elif(member.is_dir()):
3340 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3341 fchmode = stat.S_IMODE(fmode);
3342 ftypemod = stat.S_IFMT(fmode);
3343 elif(zipinfo.create_system==3):
3344 fwinattributes = format(int(0), 'x').lower();
3345 fmode = format(int(zipinfo.external_attr), 'x').lower();
3346 fchmode = stat.S_IMODE(fmode);
3347 ftypemod = stat.S_IFMT(fmode);
3348 else:
3349 fwinattributes = format(int(0), 'x').lower();
3350 if(not member.is_dir()):
3351 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3352 fchmode = stat.S_IMODE(fmode);
3353 ftypemod = stat.S_IFMT(fmode);
3354 elif(member.is_dir()):
3355 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3356 fchmode = stat.S_IMODE(fmode);
3357 ftypemod = stat.S_IFMT(fmode);
3358 fcompression = "";
3359 fcsize = format(int(0), 'x').lower();
3360 try:
3361 fuid = format(int(os.getuid()), 'x').lower();
3362 except AttributeError:
3363 fuid = format(int(0), 'x').lower();
3364 except KeyError:
3365 fuid = format(int(0), 'x').lower();
3366 try:
3367 fgid = format(int(os.getgid()), 'x').lower();
3368 except AttributeError:
3369 fgid = format(int(0), 'x').lower();
3370 except KeyError:
3371 fgid = format(int(0), 'x').lower();
3372 try:
3373 import pwd;
3374 try:
3375 userinfo = pwd.getpwuid(os.getuid());
3376 funame = userinfo.pw_name;
3377 except KeyError:
3378 funame = "";
3379 except AttributeError:
3380 funame = "";
3381 except ImportError:
3382 funame = "";
3383 fgname = "";
3384 try:
3385 import grp;
3386 try:
3387 groupinfo = grp.getgrgid(os.getgid());
3388 fgname = groupinfo.gr_name;
3389 except KeyError:
3390 fgname = "";
3391 except AttributeError:
3392 fgname = "";
3393 except ImportError:
3394 fgname = "";
3395 fcontents = BytesIO();
3396 if(ftype==0):
3397 fcontents.write(zipfp.read(member.filename));
3398 if(not compresswholefile):
3399 fcontents.seek(0, 2);
3400 ucfsize = fcontents.tell();
3401 fcontents.seek(0, 0);
3402 if(compression=="auto"):
3403 ilsize = len(compressionlistalt);
3404 ilmin = 0;
3405 ilcsize = [];
3406 while(ilmin < ilsize):
3407 cfcontents = BytesIO();
3408 shutil.copyfileobj(fcontents, cfcontents);
3409 fcontents.seek(0, 0);
3410 cfcontents.seek(0, 0);
3411 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3412 cfcontents.seek(0, 2);
3413 ilcsize.append(cfcontents.tell());
3414 cfcontents.close();
3415 ilmin = ilmin + 1;
3416 ilcmin = ilcsize.index(min(ilcsize));
3417 compression = compressionlistalt[ilcmin];
3418 fcontents.seek(0, 0);
3419 cfcontents = BytesIO();
3420 shutil.copyfileobj(fcontents, cfcontents);
3421 cfcontents.seek(0, 0);
3422 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3423 cfcontents.seek(0, 2);
3424 cfsize = cfcontents.tell();
3425 if(ucfsize > cfsize):
3426 fcsize = format(int(cfsize), 'x').lower();
3427 fcompression = compression;
3428 fcontents.close();
3429 fcontents = cfcontents;
3430 if(fcompression=="none"):
3431 fcompression = "";
3432 fcontents.seek(0, 0);
3433 ftypehex = format(ftype, 'x').lower();
3434 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3435 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3436 fcontents.close();
3437 if(numfiles>0):
3438 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3439 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3440 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3441 try:
3442 catfp.flush();
3443 os.fsync(catfp.fileno());
3444 except io.UnsupportedOperation:
3445 pass;
3446 except AttributeError:
3447 pass;
3448 except OSError as e:
3449 pass;
3450 if(outfile=="-"):
3451 catfp.seek(0, 0);
3452 if(hasattr(sys.stdout, "buffer")):
3453 shutil.copyfileobj(catfp, sys.stdout.buffer);
3454 else:
3455 shutil.copyfileobj(catfp, sys.stdout);
3456 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3457 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3458 catfp.seek(0, 0);
3459 upload_file_to_internet_file(catfp, outfile);
3460 if(returnfp):
3461 catfp.seek(0, 0);
3462 return catfp;
3463 else:
3464 catfp.close();
3465 return True;
3466
3467 create_alias_function("Pack", __file_format_name__, "FromZipFile", PackArchiveFileFromZipFile);
3468
3469 if(not rarfile_support):
3470 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3471 return False
3472
3473 if(rarfile_support):
3474 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3475 formatspecs = FormatSpecsListToDict(formatspecs);
3476 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3477 outfile = RemoveWindowsPath(outfile);
3478 checksumtype = checksumtype.lower();
3479 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3480 checksumtype="crc32";
3481 if(checksumtype=="none"):
3482 checksumtype = "";
3483 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3484 compression = "auto";
3485 if(compression not in compressionlist and compression is None):
3486 compression = "auto";
3487 if(verbose):
3488 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3489 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3490 if(os.path.exists(outfile)):
3491 try:
3492 os.unlink(outfile);
3493 except OSError as e:
3494 pass;
3495 if(outfile=="-"):
3496 verbose = False;
3497 catfp = BytesIO();
3498 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3499 catfp = outfile;
3500 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3501 catfp = BytesIO();
3502 else:
3503 fbasename = os.path.splitext(outfile)[0];
3504 fextname = os.path.splitext(outfile)[1];
3505 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3506 catver = formatspecs['format_ver'];
3507 fileheaderver = str(int(catver.replace(".", "")));
3508 curinode = 0;
3509 curfid = 0;
3510 inodelist = [];
3511 inodetofile = {};
3512 filetoinode = {};
3513 inodetocatinode = {};
3514 if(not os.path.exists(infile) or not os.path.isfile(infile)):
3515 return False;
3516 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
3517 return False;
3518 rarfp = rarfile.RarFile(infile, "r");
3519 rartest = rarfp.testrar();
3520 if(rartest):
3521 VerbosePrintOut("Bad file found!");
3522 numfiles = int(len(rarfp.infolist()));
3523 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3524 try:
3525 catfp.flush();
3526 os.fsync(catfp.fileno());
3527 except io.UnsupportedOperation:
3528 pass;
3529 except AttributeError:
3530 pass;
3531 except OSError as e:
3532 pass;
3533 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
3534 is_unix = False;
3535 is_windows = False;
3536 if(member.host_os==rarfile.RAR_OS_UNIX):
3537 is_windows = False;
3538 try:
3539 member.external_attr
3540 is_unix = True;
3541 except AttributeError:
3542 is_unix = False;
3543 elif(member.host_os==rarfile.RAR_OS_WIN32):
3544 is_unix = False;
3545 try:
3546 member.external_attr
3547 is_windows = True;
3548 except AttributeError:
3549 is_windows = False;
3550 else:
3551 is_unix = False;
3552 is_windows = False;
3553 if(re.findall("^[.|/]", member.filename)):
3554 fname = member.filename;
3555 else:
3556 fname = "./"+member.filename;
3557 rarinfo = rarfp.getinfo(member.filename);
3558 if(verbose):
3559 VerbosePrintOut(fname);
3560 if(is_unix and member.external_attr !=0):
3561 fpremode = int(member.external_attr);
3562 elif(member.is_file()):
3563 fpremode = int(stat.S_IFREG + 438);
3564 elif(member.is_symlink()):
3565 fpremode = int(stat.S_IFLNK + 438);
3566 elif(member.is_dir()):
3567 fpremode = int(stat.S_IFDIR + 511);
3568 if(is_windows and member.external_attr !=0):
3569 fwinattributes = format(int(member.external_attr), 'x').lower();
3570 else:
3571 fwinattributes = format(int(0), 'x').lower();
3572 fcompression = "";
3573 fcsize = format(int(0), 'x').lower();
3574 flinkcount = 0;
3575 ftype = 0;
3576 if(member.is_file()):
3577 ftype = 0;
3578 elif(member.is_symlink()):
3579 ftype = 2;
3580 elif(member.is_dir()):
3581 ftype = 5;
3582 flinkname = "";
3583 if(ftype==2):
3584 flinkname = rarfp.read(member.filename).decode("UTF-8");
3585 fcurfid = format(int(curfid), 'x').lower();
3586 fcurinode = format(int(curfid), 'x').lower();
3587 curfid = curfid + 1;
3588 fdev_minor = format(int(0), 'x').lower();
3589 fdev_major = format(int(0), 'x').lower();
3590 frdev_minor = format(int(0), 'x').lower();
3591 frdev_major = format(int(0), 'x').lower();
3592 if(ftype==5):
3593 fsize = format(int("0"), 'x').lower();
3594 elif(ftype==0):
3595 fsize = format(int(member.file_size), 'x').lower();
3596 else:
3597 fsize = format(int(member.file_size), 'x').lower();
3598 try:
3599 if(member.atime):
3600 fatime = format(int(member.atime.timestamp()), 'x').lower();
3601 else:
3602 fatime = format(int(member.mtime.timestamp()), 'x').lower();
3603 except AttributeError:
3604 fatime = format(int(member.mtime.timestamp()), 'x').lower();
3605 fmtime = format(int(member.mtime.timestamp()), 'x').lower();
3606 try:
3607 if(member.ctime):
3608 fctime = format(int(member.ctime.timestamp()), 'x').lower();
3609 else:
3610 fctime = format(int(member.mtime.timestamp()), 'x').lower();
3611 except AttributeError:
3612 fctime = format(int(member.mtime.timestamp()), 'x').lower();
3613 fbtime = format(int(member.mtime.timestamp()), 'x').lower();
3614 if(is_unix and member.external_attr !=0):
3615 fmode = format(int(member.external_attr), 'x').lower();
3616 fchmode = format(int(stat.S_IMODE(member.external_attr)), 'x').lower();
3617 ftypemod = format(int(stat.S_IFMT(member.external_attr)), 'x').lower();
3618 elif(member.is_file()):
3619 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3620 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3621 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3622 elif(member.is_symlink()):
3623 fmode = format(int(stat.S_IFLNK + 438), 'x').lower();
3624 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3625 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3626 elif(member.is_dir()):
3627 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3628 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
3629 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
3630 try:
3631 fuid = format(int(os.getuid()), 'x').lower();
3632 except AttributeError:
3633 fuid = format(int(0), 'x').lower();
3634 except KeyError:
3635 fuid = format(int(0), 'x').lower();
3636 try:
3637 fgid = format(int(os.getgid()), 'x').lower();
3638 except AttributeError:
3639 fgid = format(int(0), 'x').lower();
3640 except KeyError:
3641 fgid = format(int(0), 'x').lower();
3642 try:
3643 import pwd;
3644 try:
3645 userinfo = pwd.getpwuid(os.getuid());
3646 funame = userinfo.pw_name;
3647 except KeyError:
3648 funame = "";
3649 except AttributeError:
3650 funame = "";
3651 except ImportError:
3652 funame = "";
3653 fgname = "";
3654 try:
3655 import grp;
3656 try:
3657 groupinfo = grp.getgrgid(os.getgid());
3658 fgname = groupinfo.gr_name;
3659 except KeyError:
3660 fgname = "";
3661 except AttributeError:
3662 fgname = "";
3663 except ImportError:
3664 fgname = "";
3665 fcontents = BytesIO();
3666 if(ftype==0):
3667 fcontents.write(rarfp.read(member.filename));
3668 if(not compresswholefile):
3669 fcontents.seek(0, 2);
3670 ucfsize = fcontents.tell();
3671 fcontents.seek(0, 0);
3672 if(compression=="auto"):
3673 ilsize = len(compressionlistalt);
3674 ilmin = 0;
3675 ilcsize = [];
3676 while(ilmin < ilsize):
3677 cfcontents = BytesIO();
3678 shutil.copyfileobj(fcontents, cfcontents);
3679 fcontents.seek(0, 0);
3680 cfcontents.seek(0, 0);
3681 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3682 if(cfcontents):
3683 cfcontents.seek(0, 2);
3684 ilcsize.append(cfcontents.tell());
3685 cfcontents.close();
3686 else:
3687 try:
3688 ilcsize.append(sys.maxint);
3689 except AttributeError:
3690 ilcsize.append(sys.maxsize);
3691 ilmin = ilmin + 1;
3692 ilcmin = ilcsize.index(min(ilcsize));
3693 compression = compressionlistalt[ilcmin];
3694 fcontents.seek(0, 0);
3695 cfcontents = BytesIO();
3696 shutil.copyfileobj(fcontents, cfcontents);
3697 cfcontents.seek(0, 0);
3698 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3699 cfcontents.seek(0, 2);
3700 cfsize = cfcontents.tell();
3701 if(ucfsize > cfsize):
3702 fcsize = format(int(cfsize), 'x').lower();
3703 fcompression = compression;
3704 fcontents.close();
3705 fcontents = cfcontents;
3706 if(fcompression=="none"):
3707 fcompression = "";
3708 fcontents.seek(0, 0);
3709 ftypehex = format(ftype, 'x').lower();
3710 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3711 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3712 fcontents.close();
3713 if(numfiles>0):
3714 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3715 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3716 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
3717 try:
3718 catfp.flush();
3719 os.fsync(catfp.fileno());
3720 except io.UnsupportedOperation:
3721 pass;
3722 except AttributeError:
3723 pass;
3724 except OSError as e:
3725 pass;
3726 if(outfile=="-"):
3727 catfp.seek(0, 0)
3728 if(hasattr(sys.stdout, "buffer")):
3729 shutil.copyfileobj(catfp, sys.stdout.buffer);
3730 else:
3731 shutil.copyfileobj(catfp, sys.stdout);
3732 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3733 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3734 catfp.seek(0, 0);
3735 upload_file_to_internet_file(catfp, outfile);
3736 if(returnfp):
3737 catfp.seek(0, 0)
3738 return catfp
3739 else:
3740 catfp.close()
3741 return True;
3742
3743 create_alias_function("Pack", __file_format_name__, "FromRarFile", PackArchiveFileFromRarFile);
3744
3745 if(not py7zr_support):
3746 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3747 return False
3748
3749 if(py7zr_support):
3750 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3751 formatspecs = FormatSpecsListToDict(formatspecs);
3752 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3753 outfile = RemoveWindowsPath(outfile);
3754 checksumtype = checksumtype.lower();
3755 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3756 checksumtype="crc32";
3757 if(checksumtype=="none"):
3758 checksumtype = "";
3759 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3760 compression = "auto";
3761 if(compression not in compressionlist and compression is None):
3762 compression = "auto";
3763 if(verbose):
3764 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3765 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3766 if(os.path.exists(outfile)):
3767 try:
3768 os.unlink(outfile);
3769 except OSError as e:
3770 pass;
3771 if(outfile=="-"):
3772 verbose = False;
3773 catfp = BytesIO();
3774 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3775 catfp = outfile;
3776 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3777 catfp = BytesIO();
3778 else:
3779 fbasename = os.path.splitext(outfile)[0];
3780 fextname = os.path.splitext(outfile)[1];
3781 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3782 catver = formatspecs['format_ver'];
3783 fileheaderver = str(int(catver.replace(".", "")));
3784 curinode = 0;
3785 curfid = 0;
3786 inodelist = [];
3787 inodetofile = {};
3788 filetoinode = {};
3789 inodetocatinode = {};
3790 if(not os.path.exists(infile) or not os.path.isfile(infile)):
3791 return False;
3792 szpfp = py7zr.SevenZipFile(infile, mode="r");
3793 file_content = szpfp.readall();
3794 #sztest = szpfp.testzip();
3795 sztestalt = szpfp.test();
3796 if(sztestalt):
3797 VerbosePrintOut("Bad file found!");
3798 numfiles = int(len(szpfp.list()));
3799 AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3800 for member in sorted(szpfp.list(), key=lambda x: x.filename):
3801 if(re.findall("^[.|/]", member.filename)):
3802 fname = member.filename;
3803 else:
3804 fname = "./"+member.filename;
3805 if(verbose):
3806 VerbosePrintOut(fname);
3807 if(not member.is_directory):
3808 fpremode = int(stat.S_IFREG + 438);
3809 elif(member.is_directory):
3810 fpremode = int(stat.S_IFDIR + 511);
3811 fwinattributes = format(int(0), 'x').lower();
3812 fcompression = "";
3813 fcsize = format(int(0), 'x').lower();
3814 flinkcount = 0;
3815 ftype = 0;
3816 if(member.is_directory):
3817 ftype = 5;
3818 else:
3819 ftype = 0;
3820 flinkname = "";
3821 fcurfid = format(int(curfid), 'x').lower();
3822 fcurinode = format(int(curfid), 'x').lower();
3823 curfid = curfid + 1;
3824 fdev_minor = format(int(0), 'x').lower();
3825 fdev_major = format(int(0), 'x').lower();
3826 frdev_minor = format(int(0), 'x').lower();
3827 frdev_major = format(int(0), 'x').lower();
3828 if(ftype==5):
3829 fsize = format(int("0"), 'x').lower();
3830 fatime = format(int(member.creationtime.timestamp()), 'x').lower();
3831 fmtime = format(int(member.creationtime.timestamp()), 'x').lower();
3832 fctime = format(int(member.creationtime.timestamp()), 'x').lower();
3833 fbtime = format(int(member.creationtime.timestamp()), 'x').lower();
3834 if(member.is_directory):
3835 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3836 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
3837 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
3838 else:
3839 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3840 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3841 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3842 try:
3843 fuid = format(int(os.getuid()), 'x').lower();
3844 except AttributeError:
3845 fuid = format(int(0), 'x').lower();
3846 except KeyError:
3847 fuid = format(int(0), 'x').lower();
3848 try:
3849 fgid = format(int(os.getgid()), 'x').lower();
3850 except AttributeError:
3851 fgid = format(int(0), 'x').lower();
3852 except KeyError:
3853 fgid = format(int(0), 'x').lower();
3854 try:
3855 import pwd;
3856 try:
3857 userinfo = pwd.getpwuid(os.getuid());
3858 funame = userinfo.pw_name;
3859 except KeyError:
3860 funame = "";
3861 except AttributeError:
3862 funame = "";
3863 except ImportError:
3864 funame = "";
3865 fgname = "";
3866 try:
3867 import grp;
3868 try:
3869 groupinfo = grp.getgrgid(os.getgid());
3870 fgname = groupinfo.gr_name;
3871 except KeyError:
3872 fgname = "";
3873 except AttributeError:
3874 fgname = "";
3875 except ImportError:
3876 fgname = "";
3877 fcontents = BytesIO();
3878 if(ftype==0):
3879 fcontents.write(file_content[member.filename].read());
3880 fsize = format(fcontents.tell(), 'x').lower();
3881 file_content[member.filename].close();
3882 if(not compresswholefile):
3883 fcontents.seek(0, 2);
3884 ucfsize = fcontents.tell();
3885 fcontents.seek(0, 0);
3886 if(compression=="auto"):
3887 ilsize = len(compressionlistalt);
3888 ilmin = 0;
3889 ilcsize = [];
3890 while(ilmin < ilsize):
3891 cfcontents = BytesIO();
3892 shutil.copyfileobj(fcontents, cfcontents);
3893 fcontents.seek(0, 0);
3894 cfcontents.seek(0, 0);
3895 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3896 if(cfcontents):
3897 cfcontents.seek(0, 2);
3898 ilcsize.append(cfcontents.tell());
3899 cfcontents.close();
3900 else:
3901 try:
3902 ilcsize.append(sys.maxint);
3903 except AttributeError:
3904 ilcsize.append(sys.maxsize);
3905 ilmin = ilmin + 1;
3906 ilcmin = ilcsize.index(min(ilcsize));
3907 compression = compressionlistalt[ilcmin];
3908 fcontents.seek(0, 0);
3909 cfcontents = BytesIO();
3910 shutil.copyfileobj(fcontents, cfcontents);
3911 cfcontents.seek(0, 0);
3912 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3913 cfcontents.seek(0, 2);
3914 cfsize = cfcontents.tell();
3915 if(ucfsize > cfsize):
3916 fcsize = format(int(cfsize), 'x').lower();
3917 fcompression = compression;
3918 fcontents.close();
3919 fcontents = cfcontents;
3920 if(fcompression=="none"):
3921 fcompression = "";
3922 fcontents.seek(0, 0);
3923 ftypehex = format(ftype, 'x').lower();
3924 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3925 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3926 fcontents.close();
3927 if(numfiles>0):
3928 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3929 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3930 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
3931 try:
3932 catfp.flush();
3933 os.fsync(catfp.fileno());
3934 except io.UnsupportedOperation:
3935 pass;
3936 except AttributeError:
3937 pass;
3938 except OSError as e:
3939 pass;
3940 if(outfile=="-"):
3941 catfp.seek(0, 0);
3942 if(hasattr(sys.stdout, "buffer")):
3943 shutil.copyfileobj(catfp, sys.stdout.buffer);
3944 else:
3945 shutil.copyfileobj(catfp, sys.stdout);
3946 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3947 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3948 catfp.seek(0, 0);
3949 upload_file_to_internet_file(catfp, outfile);
3950 if(returnfp):
3951 catfp.seek(0, 0)
3952 return catfp
3953 else:
3954 catfp.close()
3955 return True;
3956
3957 create_alias_function("Pack", __file_format_name__, "FromSevenZipFile", PackArchiveFileFromSevenZipFile);
3958
3959 def PackArchiveFileFromInFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3960 formatspecs = FormatSpecsListToDict(formatspecs);
3961 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
3962 if(verbose):
3963 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3964 if(checkcompressfile=="tarfile"):
3965 return PackArchiveFileFromTarFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3966 elif(checkcompressfile=="zipfile"):
3967 return PackArchiveFileFromZipFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3968 elif(checkcompressfile=="catfile"):
3969 return RePackArchiveFile(infile, outfile, compression, compresswholefile, compressionlevel, False, 0, 0, checksumtype, False, extradata, formatspecs, verbose, returnfp);
3970 elif(rarfile_support and checkcompressfile=="rarfile"):
3971 return PackArchiveFileFromRarFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3972 elif(py7zr_support and checkcompressfile=="7zipfile"):
3973 return PackArchiveFileFromSevenZipFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3974 else:
3975 return False;
3976 return False;
3977
3978 create_alias_function("Pack", __file_format_name__, "FromInFile", PackArchiveFileFromInFile);
3979
3980 def ArchiveFileSeekToFileNum(infile, seekto=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
3981 formatspecs = FormatSpecsListToDict(formatspecs);
3982 if(hasattr(infile, "read") or hasattr(infile, "write")):
3983 catfp = infile;
3984 catfp.seek(0, 0);
3985 catfp = UncompressArchiveFile(catfp, formatspecs);
3986 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
3987 if(checkcompressfile=="tarfile"):
3988 return TarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3989 if(checkcompressfile=="zipfile"):
3990 return ZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3991 if(rarfile_support and checkcompressfile=="rarfile"):
3992 return RarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3993 if(py7zr_support and checkcompressfile=="7zipfile"):
3994 return SevenZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3995 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
3996 return False;
3997 if(not catfp):
3998 return False;
3999 catfp.seek(0, 0);
4000 elif(infile=="-"):
4001 catfp = BytesIO();
4002 if(hasattr(sys.stdin, "buffer")):
4003 shutil.copyfileobj(sys.stdin.buffer, catfp);
4004 else:
4005 shutil.copyfileobj(sys.stdin, catfp);
4006 catfp.seek(0, 0);
4007 catfp = UncompressArchiveFile(catfp, formatspecs);
4008 if(not catfp):
4009 return False;
4010 catfp.seek(0, 0);
4011 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4012 catfp = download_file_from_internet_file(infile);
4013 catfp.seek(0, 0);
4014 catfp = UncompressArchiveFile(catfp, formatspecs);
4015 if(not catfp):
4016 return False;
4017 catfp.seek(0, 0);
4018 else:
4019 infile = RemoveWindowsPath(infile);
4020 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4021 if(checkcompressfile=="tarfile"):
4022 return TarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4023 if(checkcompressfile=="zipfile"):
4024 return ZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4025 if(rarfile_support and checkcompressfile=="rarfile"):
4026 return RarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4027 if(py7zr_support and checkcompressfile=="7zipfile"):
4028 return SevenZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4029 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4030 return False;
4031 compresscheck = CheckCompressionType(infile, formatspecs, True);
4032 if(not compresscheck):
4033 fextname = os.path.splitext(infile)[1];
4034 if(fextname==".gz"):
4035 compresscheck = "gzip";
4036 elif(fextname==".bz2"):
4037 compresscheck = "bzip2";
4038 elif(fextname==".zst"):
4039 compresscheck = "zstd";
4040 elif(fextname==".lz4" or fextname==".clz4"):
4041 compresscheck = "lz4";
4042 elif(fextname==".lzo" or fextname==".lzop"):
4043 compresscheck = "lzo";
4044 elif(fextname==".lzma" or fextname==".xz"):
4045 compresscheck = "lzma";
4046 else:
4047 return False;
4048 if(not compresscheck):
4049 return False;
4050 catfp = UncompressFile(infile, formatspecs, "rb");
4051 '''
4052 try:
4053 catfp.seek(0, 2);
4054 except OSError:
4055 SeekToEndOfFile(catfp);
4056 except ValueError:
4057 SeekToEndOfFile(catfp);
4058 CatSize = catfp.tell();
4059 CatSizeEnd = CatSize;
4060 '''
4061 try:
4062 catfp.seek(0, 0);
4063 except OSError:
4064 return False;
4065 except ValueError:
4066 return False;
4067 curloc = catfp.tell();
4068 if(curloc>0):
4069 catfp.seek(0, 0);
4070 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4071 if(curloc>0):
4072 catfp.seek(curloc, 0);
4073 catstring = catheader[0];
4074 catversion = re.findall(r"([\d]+)$", catstring);
4075 fprenumfiles = catheader[1];
4076 fnumfiles = int(fprenumfiles, 16);
4077 fprechecksumtype = catheader[2];
4078 fprechecksum = catheader[3];
4079 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4080 fnumfileshex = format(int(fnumfiles), 'x').lower();
4081 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4082 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4083 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4084 fheadtell = len(fileheader);
4085 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4086 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
4087 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4088 return False;
4089 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
4090 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4091 if(seekto>=fnumfiles):
4092 seekto = fnumfiles - 1;
4093 if(seekto<0):
4094 seekto = 0;
4095 if(seekto>=0):
4096 il = -1;
4097 while(il < seekto):
4098 prefhstart = catfp.tell();
4099 if(formatspecs['new_style']):
4100 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4101 else:
4102 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4103 if(len(preheaderdata)==0):
4104 break;
4105 prefheadsize = int(preheaderdata[0], 16);
4106 prefnumfields = int(preheaderdata[1], 16);
4107 preftype = int(preheaderdata[2], 16);
4108 if(re.findall("^[.|/]", preheaderdata[3])):
4109 prefname = preheaderdata[3];
4110 else:
4111 prefname = "./"+preheaderdata[3];
4112 prefbasedir = os.path.dirname(prefname);
4113 preflinkname = preheaderdata[4];
4114 prefsize = int(preheaderdata[5], 16);
4115 prefatime = int(preheaderdata[6], 16);
4116 prefmtime = int(preheaderdata[7], 16);
4117 prefctime = int(preheaderdata[8], 16);
4118 prefbtime = int(preheaderdata[9], 16);
4119 prefmode = int(preheaderdata[10], 16);
4120 prefchmode = stat.S_IMODE(prefmode);
4121 preftypemod = stat.S_IFMT(prefmode);
4122 prefwinattributes = int(preheaderdata[11], 16);
4123 prefcompression = preheaderdata[12];
4124 prefcsize = int(preheaderdata[13], 16);
4125 prefuid = int(preheaderdata[14], 16);
4126 prefuname = preheaderdata[15];
4127 prefgid = int(preheaderdata[16], 16);
4128 prefgname = preheaderdata[17];
4129 fid = int(preheaderdata[18], 16);
4130 finode = int(preheaderdata[19], 16);
4131 flinkcount = int(preheaderdata[20], 16);
4132 prefdev_minor = int(preheaderdata[21], 16);
4133 prefdev_major = int(preheaderdata[22], 16);
4134 prefrdev_minor = int(preheaderdata[23], 16);
4135 prefrdev_major = int(preheaderdata[24], 16);
4136 prefextrasize = int(preheaderdata[25], 16);
4137 prefextrafields = int(preheaderdata[26], 16);
4138 extrafieldslist = [];
4139 extrastart = 27;
4140 extraend = extrastart + prefextrafields;
4141 extrafieldslist = [];
4142 if(extrastart<extraend):
4143 extrafieldslist.append(preheaderdata[extrastart]);
4144 extrastart = extrastart + 1;
4145 prefchecksumtype = preheaderdata[extrastart].lower();
4146 prefcs = preheaderdata[extrastart + 1].lower();
4147 prefccs = preheaderdata[extrastart + 2].lower();
4148 hc = 0;
4149 hcmax = len(preheaderdata) - 2;
4150 hout = "";
4151 while(hc<hcmax):
4152 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs['format_delimiter']);
4153 hc = hc + 1;
4154 prenewfcs = GetFileChecksum(hout, preheaderdata[-3].lower(), True, formatspecs);
4155 if(prefcs!=prenewfcs and not skipchecksum):
4156 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4157 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4158 return False;
4159 valid_archive = False;
4160 invalid_archive = True;
4161 prefhend = catfp.tell() - 1;
4162 prefcontentstart = catfp.tell();
4163 prefcontents = "";
4164 pyhascontents = False;
4165 if(prefsize>0):
4166 if(prefcompression):
4167 prefcontents = catfp.read(prefsize);
4168 else:
4169 prefcontents = catfp.read(prefcsize);
4170 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4171 pyhascontents = True;
4172 if(prefccs!=prenewfccs and not skipchecksum):
4173 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4174 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4175 return False;
4176 catfp.seek(1, 1);
4177 il = il + 1;
4178 catfp.seek(seekstart, 0);
4179 fileidnum = il;
4180 catfheadsize = int(preheaderdata[0], 16);
4181 catfnumfields = int(preheaderdata[1], 16);
4182 catftype = int(preheaderdata[2], 16);
4183 if(re.findall("^[.|/]", preheaderdata[3])):
4184 catfname = preheaderdata[3];
4185 else:
4186 catfname = "./"+preheaderdata[3];
4187 catflinkname = preheaderdata[4];
4188 catfsize = int(preheaderdata[5], 16);
4189 catfbasedir = os.path.dirname(catfname);
4190 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
4191 if(returnfp):
4192 catlist.update({'catfp': catfp});
4193 else:
4194 catfp.close();
4195 return catlist;
4196
4197 create_alias_function("", __file_format_name__, "SeekToFileNum", ArchiveFileSeekToFileNum);
4198
4199 def ArchiveFileSeekToFileName(infile, seekfile=None, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4200 formatspecs = FormatSpecsListToDict(formatspecs);
4201 if(hasattr(infile, "read") or hasattr(infile, "write")):
4202 catfp = infile;
4203 catfp.seek(0, 0);
4204 catfp = UncompressArchiveFile(catfp, formatspecs);
4205 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4206 if(checkcompressfile=="tarfile"):
4207 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4208 if(checkcompressfile=="zipfile"):
4209 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4210 if(rarfile_support and checkcompressfile=="rarfile"):
4211 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4212 if(py7zr_support and checkcompressfile=="7zipfile"):
4213 return SevenZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4214 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4215 return False;
4216 if(not catfp):
4217 return False;
4218 catfp.seek(0, 0);
4219 elif(infile=="-"):
4220 catfp = BytesIO();
4221 if(hasattr(sys.stdin, "buffer")):
4222 shutil.copyfileobj(sys.stdin.buffer, catfp);
4223 else:
4224 shutil.copyfileobj(sys.stdin, catfp);
4225 catfp.seek(0, 0);
4226 catfp = UncompressArchiveFile(catfp, formatspecs);
4227 if(not catfp):
4228 return False;
4229 catfp.seek(0, 0);
4230 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4231 catfp = download_file_from_internet_file(infile);
4232 catfp = UncompressArchiveFile(catfp, formatspecs);
4233 catfp.seek(0, 0);
4234 if(not catfp):
4235 return False;
4236 catfp.seek(0, 0);
4237 else:
4238 infile = RemoveWindowsPath(infile);
4239 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4240 if(checkcompressfile=="tarfile"):
4241 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4242 if(checkcompressfile=="zipfile"):
4243 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4244 if(rarfile_support and checkcompressfile=="rarfile"):
4245 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4246 if(py7zr_support and checkcompressfile=="7zipfile"):
4247 return SevenZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4248 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4249 return False;
4250 compresscheck = CheckCompressionType(infile, formatspecs, True);
4251 if(not compresscheck):
4252 fextname = os.path.splitext(infile)[1];
4253 if(fextname==".gz"):
4254 compresscheck = "gzip";
4255 elif(fextname==".bz2"):
4256 compresscheck = "bzip2";
4257 elif(fextname==".zst"):
4258 compresscheck = "zstd";
4259 elif(fextname==".lz4" or fextname==".clz4"):
4260 compresscheck = "lz4";
4261 elif(fextname==".lzo" or fextname==".lzop"):
4262 compresscheck = "lzo";
4263 elif(fextname==".lzma" or fextname==".xz"):
4264 compresscheck = "lzma";
4265 else:
4266 return False;
4267 if(not compresscheck):
4268 return False;
4269 catfp = UncompressFile(infile, formatspecs, "rb");
4270 '''
4271 try:
4272 catfp.seek(0, 2);
4273 except OSError:
4274 SeekToEndOfFile(catfp);
4275 except ValueError:
4276 SeekToEndOfFile(catfp);
4277 CatSize = catfp.tell();
4278 CatSizeEnd = CatSize;
4279 '''
4280 try:
4281 catfp.seek(0, 0);
4282 except OSError:
4283 return False;
4284 except ValueError:
4285 return False;
4286 curloc = catfp.tell();
4287 if(curloc>0):
4288 catfp.seek(0, 0);
4289 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4290 if(curloc>0):
4291 catfp.seek(curloc, 0);
4292 catstring = catheader[0];
4293 catversion = re.findall(r"([\d]+)$", catstring);
4294 fprenumfiles = catheader[1];
4295 fnumfiles = int(fprenumfiles, 16);
4296 fprechecksumtype = catheader[2];
4297 fprechecksum = catheader[3];
4298 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4299 fnumfileshex = format(int(fnumfiles), 'x').lower();
4300 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4301 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4302 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4303 fheadtell = len(fileheader);
4304 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4305 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
4306 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4307 return False;
4308 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
4309 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4310 seekto = fnumfiles - 1
4311 filefound = False;
4312 if(seekto>=0):
4313 il = -1;
4314 while(il < seekto):
4315 prefhstart = catfp.tell();
4316 if(formatspecs['new_style']):
4317 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4318 else:
4319 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4320 if(len(preheaderdata)==0):
4321 break;
4322 prefheadsize = int(preheaderdata[0], 16);
4323 prefnumfields = int(preheaderdata[1], 16);
4324 preftype = int(preheaderdata[2], 16);
4325 if(re.findall("^[.|/]", preheaderdata[3])):
4326 prefname = preheaderdata[3];
4327 else:
4328 prefname = "./"+preheaderdata[3];
4329 prefbasedir = os.path.dirname(prefname);
4330 preflinkname = preheaderdata[4];
4331 prefsize = int(preheaderdata[5], 16);
4332 prefatime = int(preheaderdata[6], 16);
4333 prefmtime = int(preheaderdata[7], 16);
4334 prefctime = int(preheaderdata[8], 16);
4335 prefbtime = int(preheaderdata[9], 16);
4336 prefmode = int(preheaderdata[10], 16);
4337 prefchmode = stat.S_IMODE(prefmode);
4338 preftypemod = stat.S_IFMT(prefmode);
4339 prefwinattributes = int(preheaderdata[11], 16);
4340 prefcompression = preheaderdata[12];
4341 prefcsize = int(preheaderdata[13], 16);
4342 prefuid = int(preheaderdata[14], 16);
4343 prefuname = preheaderdata[15];
4344 prefgid = int(preheaderdata[16], 16);
4345 prefgname = preheaderdata[17];
4346 fid = int(preheaderdata[18], 16);
4347 finode = int(preheaderdata[19], 16);
4348 flinkcount = int(preheaderdata[20], 16);
4349 prefdev_minor = int(preheaderdata[21], 16);
4350 prefdev_major = int(preheaderdata[22], 16);
4351 prefrdev_minor = int(preheaderdata[23], 16);
4352 prefrdev_major = int(preheaderdata[24], 16);
4353 prefextrasize = int(preheaderdata[25], 16);
4354 prefextrafields = int(preheaderdata[26], 16);
4355 extrafieldslist = [];
4356 extrastart = 27;
4357 extraend = extrastart + prefextrafields;
4358 extrafieldslist = [];
4359 if(extrastart<extraend):
4360 extrafieldslist.append(preheaderdata[extrastart]);
4361 extrastart = extrastart + 1;
4362 prefchecksumtype = preheaderdata[extrastart].lower();
4363 prefcs = preheaderdata[extrastart + 1].lower();
4364 prefccs = preheaderdata[extrastart + 2].lower();
4365 hc = 0;
4366 hcmax = len(preheaderdata) - 2;
4367 hout = "";
4368 while(hc<hcmax):
4369 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs['format_delimiter']);
4370 hc = hc + 1;
4371 prenewfcs = GetFileChecksum(hout, preheaderdata[-3].lower(), True, formatspecs);
4372 if(prefcs!=prenewfcs and not skipchecksum):
4373 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4374 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4375 return False;
4376 valid_archive = False;
4377 invalid_archive = True;
4378 prefhend = catfp.tell() - 1;
4379 prefcontentstart = catfp.tell();
4380 prefcontents = "";
4381 pyhascontents = False;
4382 if(prefsize>0):
4383 if(prefcompression):
4384 prefcontents = catfp.read(prefsize);
4385 else:
4386 prefcontents = catfp.read(prefcsize);
4387 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4388 pyhascontents = True;
4389 if(prefccs!=prenewfccs and not skipchecksum):
4390 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4391 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4392 return False;
4393 catfp.seek(1, 1);
4394 il = il + 1;
4395 filefound = False;
4396 prefname = preheaderdata[2];
4397 if(re.findall("^[.|/]", preheaderdata[2])):
4398 prefname = preheaderdata[2];
4399 else:
4400 prefname = "./"+preheaderdata[2];
4401 if(prefname==seekfile):
4402 filefound = True;
4403 break;
4404 catfp.seek(seekstart, 0);
4405 fileidnum = il;
4406 catfheadsize = int(preheaderdata[0], 16);
4407 catfnumfields = int(preheaderdata[1], 16);
4408 catftype = int(preheaderdata[2], 16);
4409 if(re.findall("^[.|/]", preheaderdata[3])):
4410 catfname = preheaderdata[3];
4411 else:
4412 catfname = "./"+preheaderdata[3];
4413 catflinkname = preheaderdata[4];
4414 catfsize = int(preheaderdata[5], 16);
4415 catfbasedir = os.path.dirname(catfname);
4416 if(filefound):
4417 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
4418 else:
4419 return False;
4420 if(returnfp):
4421 catlist.update({'catfp': catfp});
4422 else:
4423 catfp.close();
4424 return catlist;
4425
4426 create_alias_function("", __file_format_name__, "SeekToFileName", ArchiveFileSeekToFileName);
4427
4428 def ArchiveFileValidate(infile, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
4429 formatspecs = FormatSpecsListToDict(formatspecs);
4430 if(verbose):
4431 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
4432 if(hasattr(infile, "read") or hasattr(infile, "write")):
4433 catfp = infile;
4434 catfp.seek(0, 0);
4435 catfp = UncompressArchiveFile(catfp, formatspecs);
4436 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4437 if(checkcompressfile=="tarfile"):
4438 return TarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4439 if(checkcompressfile=="zipfile"):
4440 return ZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4441 if(rarfile_support and checkcompressfile=="rarfile"):
4442 return RarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4443 if(py7zr_support and checkcompressfile=="7zipfile"):
4444 return SevenZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4445 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4446 return False;
4447 if(not catfp):
4448 return False;
4449 catfp.seek(0, 0);
4450 elif(infile=="-"):
4451 catfp = BytesIO();
4452 if(hasattr(sys.stdin, "buffer")):
4453 shutil.copyfileobj(sys.stdin.buffer, catfp);
4454 else:
4455 shutil.copyfileobj(sys.stdin, catfp);
4456 catfp.seek(0, 0);
4457 catfp = UncompressArchiveFile(catfp, formatspecs);
4458 if(not catfp):
4459 return False;
4460 catfp.seek(0, 0);
4461 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4462 catfp = download_file_from_internet_file(infile);
4463 catfp = UncompressArchiveFile(catfp, formatspecs);
4464 catfp.seek(0, 0);
4465 if(not catfp):
4466 return False;
4467 catfp.seek(0, 0);
4468 else:
4469 infile = RemoveWindowsPath(infile);
4470 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4471 if(checkcompressfile=="tarfile"):
4472 return TarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4473 if(checkcompressfile=="zipfile"):
4474 return ZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4475 if(rarfile_support and checkcompressfile=="rarfile"):
4476 return RarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4477 if(py7zr_support and checkcompressfile=="7zipfile"):
4478 return SevenZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4479 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4480 return False;
4481 compresscheck = CheckCompressionType(infile, formatspecs, True);
4482 if(not compresscheck):
4483 fextname = os.path.splitext(infile)[1];
4484 if(fextname==".gz"):
4485 compresscheck = "gzip";
4486 elif(fextname==".bz2"):
4487 compresscheck = "bzip2";
4488 elif(fextname==".zst"):
4489 compresscheck = "zstd";
4490 elif(fextname==".lz4" or fextname==".clz4"):
4491 compresscheck = "lz4";
4492 elif(fextname==".lzo" or fextname==".lzop"):
4493 compresscheck = "lzo";
4494 elif(fextname==".lzma" or fextname==".xz"):
4495 compresscheck = "lzma";
4496 else:
4497 return False;
4498 if(not compresscheck):
4499 return False;
4500 catfp = UncompressFile(infile, formatspecs, "rb");
4501 '''
4502 try:
4503 catfp.seek(0, 2);
4504 except OSError:
4505 SeekToEndOfFile(catfp);
4506 except ValueError:
4507 SeekToEndOfFile(catfp);
4508 CatSize = catfp.tell();
4509 CatSizeEnd = CatSize;
4510 '''
4511 try:
4512 catfp.seek(0, 0);
4513 except OSError:
4514 return False;
4515 except ValueError:
4516 return False;
4517 curloc = catfp.tell();
4518 if(curloc>0):
4519 catfp.seek(0, 0);
4520 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4521 if(curloc>0):
4522 catfp.seek(curloc, 0);
4523 catstring = catheader[0];
4524 catversion = re.findall(r"([\d]+)$", catstring);
4525 fprenumfiles = catheader[1];
4526 fnumfiles = int(fprenumfiles, 16);
4527 fprechecksumtype = catheader[2];
4528 fprechecksum = catheader[3];
4529 il = 0;
4530 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4531 fnumfileshex = format(int(fnumfiles), 'x').lower();
4532 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4533 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4534 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4535 valid_archive = True;
4536 invalid_archive = False;
4537 if(verbose):
4538 VerbosePrintOut(infile);
4539 VerbosePrintOut("Number of Records " + str(fnumfiles));
4540 if(fprechecksum==catfileheadercshex):
4541 if(verbose):
4542 VerbosePrintOut("File Header Checksum Passed at offset " + str(0));
4543 VerbosePrintOut("'" + str(fprechecksum) + "' == " + "'" + str(catfileheadercshex) + "'");
4544 else:
4545 if(verbose):
4546 VerbosePrintOut("File Header Checksum Failed at offset " + str(0));
4547 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4548 valid_archive = False;
4549 invalid_archive = True;
4550 if(verbose):
4551 VerbosePrintOut("");
4552 while(il<fnumfiles):
4553 catfhstart = catfp.tell();
4554 if(formatspecs['new_style']):
4555 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4556 else:
4557 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4558 if(len(catheaderdata)==0):
4559 break;
4560 catfheadsize = int(catheaderdata[0], 16);
4561 catfnumfields = int(catheaderdata[1], 16);
4562 catftype = int(catheaderdata[2], 16);
4563 if(re.findall("^[.|/]", catheaderdata[3])):
4564 catfname = catheaderdata[3];
4565 else:
4566 catfname = "./"+catheaderdata[3];
4567 catfbasedir = os.path.dirname(catfname);
4568 catflinkname = catheaderdata[4];
4569 catfsize = int(catheaderdata[5], 16);
4570 catfatime = int(catheaderdata[6], 16);
4571 catfmtime = int(catheaderdata[7], 16);
4572 catfctime = int(catheaderdata[8], 16);
4573 catfbtime = int(catheaderdata[9], 16);
4574 catfmode = int(catheaderdata[10], 16);
4575 catfchmode = stat.S_IMODE(catfmode);
4576 catftypemod = stat.S_IFMT(catfmode);
4577 catfwinattributes = int(catheaderdata[11], 16);
4578 catfcompression = catheaderdata[12];
4579 catfcsize = int(catheaderdata[13], 16);
4580 catfuid = int(catheaderdata[14], 16);
4581 catfuname = catheaderdata[15];
4582 catfgid = int(catheaderdata[16], 16);
4583 catfgname = catheaderdata[17];
4584 fid = int(catheaderdata[18], 16);
4585 finode = int(catheaderdata[19], 16);
4586 flinkcount = int(catheaderdata[20], 16);
4587 catfdev_minor = int(catheaderdata[21], 16);
4588 catfdev_major = int(catheaderdata[22], 16);
4589 catfrdev_minor = int(catheaderdata[23], 16);
4590 catfrdev_major = int(catheaderdata[24], 16);
4591 catfextrasize = int(catheaderdata[25], 16);
4592 catfextrafields = int(catheaderdata[26], 16);
4593 extrafieldslist = [];
4594 extrastart = 27;
4595 extraend = extrastart + catfextrafields;
4596 extrafieldslist = [];
4597 if(extrastart<extraend):
4598 extrafieldslist.append(catheaderdata[extrastart]);
4599 extrastart = extrastart + 1;
4600 catfchecksumtype = catheaderdata[extrastart].lower();
4601 catfcs = catheaderdata[extrastart + 1].lower();
4602 catfccs = catheaderdata[extrastart + 2].lower();
4603 hc = 0;
4604 hcmax = len(catheaderdata) - 2;
4605 hout = "";
4606 while(hc<hcmax):
4607 hout = hout + AppendNullByte(catheaderdata[hc], formatspecs['format_delimiter']);
4608 hc = hc + 1;
4609 catnewfcs = GetFileChecksum(hout, catheaderdata[-3].lower(), True, formatspecs);
4610 if(verbose):
4611 VerbosePrintOut(catfname);
4612 VerbosePrintOut("Record Number " + str(il) + "; File ID " + str(fid) + "; iNode Number " + str(finode));
4613 if(catfcs==catnewfcs):
4614 if(verbose):
4615 VerbosePrintOut("File Header Checksum Passed at offset " + str(catfhstart));
4616 VerbosePrintOut("'" + str(catfcs) + "' == " + "'" + str(catnewfcs) + "'");
4617 else:
4618 if(verbose):
4619 VerbosePrintOut("File Header Checksum Failed at offset " + str(catfhstart));
4620 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
4621 valid_archive = False;
4622 invalid_archive = True;
4623 catfhend = catfp.tell() - 1;
4624 catfcontentstart = catfp.tell();
4625 catfcontents = "";
4626 pyhascontents = False;
4627 if(catfsize>0):
4628 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4629 catfcontents = catfp.read(catfsize);
4630 else:
4631 catfcontents = catfp.read(catfcsize);
4632 catnewfccs = GetFileChecksum(catfcontents, catheaderdata[-3].lower(), False, formatspecs);
4633 pyhascontents = True;
4634 if(catfccs==catnewfccs):
4635 if(verbose):
4636 VerbosePrintOut("File Content Checksum Passed at offset " + str(catfcontentstart));
4637 VerbosePrintOut("'" + str(catfccs) + "' == " + "'" + str(catnewfccs) + "'");
4638 else:
4639 if(verbose):
4640 VerbosePrintOut("File Content Checksum Failed at offset " + str(catfcontentstart));
4641 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
4642 valid_archive = False;
4643 invalid_archive = True;
4644 if(verbose):
4645 VerbosePrintOut("");
4646 catfp.seek(1, 1);
4647 il = il + 1;
4648 if(valid_archive):
4649 if(returnfp):
4650 return catfp;
4651 else:
4652 catfp.close();
4653 return True;
4654 else:
4655 catfp.close();
4656 return False;
4657
4658 create_alias_function("", __file_format_name__, "Validate", ArchiveFileValidate);
4659
4660 def ArchiveFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4661 formatspecs = FormatSpecsListToDict(formatspecs);
4662 if(hasattr(infile, "read") or hasattr(infile, "write")):
4663 catfp = infile;
4664 catfp.seek(0, 0);
4665 catfp = UncompressArchiveFile(catfp, formatspecs);
4666 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4667 if(checkcompressfile=="tarfile"):
4668 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4669 if(checkcompressfile=="zipfile"):
4670 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4671 if(rarfile_support and checkcompressfile=="rarfile"):
4672 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4673 if(py7zr_support and checkcompressfile=="7zipfile"):
4674 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4675 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4676 return False;
4677 if(not catfp):
4678 return False;
4679 catfp.seek(0, 0);
4680 elif(infile=="-"):
4681 catfp = BytesIO();
4682 if(hasattr(sys.stdin, "buffer")):
4683 shutil.copyfileobj(sys.stdin.buffer, catfp);
4684 else:
4685 shutil.copyfileobj(sys.stdin, catfp);
4686 catfp.seek(0, 0);
4687 catfp = UncompressArchiveFile(catfp, formatspecs);
4688 if(not catfp):
4689 return False;
4690 catfp.seek(0, 0);
4691 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4692 catfp = download_file_from_internet_file(infile);
4693 catfp = UncompressArchiveFile(catfp, formatspecs);
4694 catfp.seek(0, 0);
4695 if(not catfp):
4696 return False;
4697 catfp.seek(0, 0);
4698 else:
4699 infile = RemoveWindowsPath(infile);
4700 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4701 if(checkcompressfile=="tarfile"):
4702 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4703 if(checkcompressfile=="zipfile"):
4704 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4705 if(rarfile_support and checkcompressfile=="rarfile"):
4706 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4707 if(py7zr_support and checkcompressfile=="7zipfile"):
4708 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4709 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4710 return False;
4711 compresscheck = CheckCompressionType(infile, formatspecs, True);
4712 if(not compresscheck):
4713 fextname = os.path.splitext(infile)[1];
4714 if(fextname==".gz"):
4715 compresscheck = "gzip";
4716 elif(fextname==".bz2"):
4717 compresscheck = "bzip2";
4718 elif(fextname==".zst"):
4719 compresscheck = "zstd";
4720 elif(fextname==".lz4" or fextname==".clz4"):
4721 compresscheck = "lz4";
4722 elif(fextname==".lzo" or fextname==".lzop"):
4723 compresscheck = "lzo";
4724 elif(fextname==".lzma" or fextname==".xz"):
4725 compresscheck = "lzma";
4726 else:
4727 return False;
4728 if(not compresscheck):
4729 return False;
4730 catfp = UncompressFile(infile, formatspecs, "rb");
4731 '''
4732 try:
4733 catfp.seek(0, 2);
4734 except OSError:
4735 SeekToEndOfFile(catfp);
4736 except ValueError:
4737 SeekToEndOfFile(catfp);
4738 CatSize = catfp.tell();
4739 CatSizeEnd = CatSize;
4740 '''
4741 try:
4742 catfp.seek(0, 0);
4743 except OSError:
4744 return False;
4745 except ValueError:
4746 return False;
4747 curloc = catfp.tell();
4748 if(curloc>0):
4749 catfp.seek(0, 0);
4750 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4751 if(curloc>0):
4752 catfp.seek(curloc, 0);
4753 catstring = catheader[0];
4754 catversion = re.findall(r"([\d]+)$", catstring);
4755 fprenumfiles = catheader[1];
4756 fnumfiles = int(fprenumfiles, 16);
4757 fprechecksumtype = catheader[2];
4758 fprechecksum = catheader[3];
4759 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4760 fnumfileshex = format(int(fnumfiles), 'x').lower();
4761 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4762 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4763 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4764 fheadtell = len(fileheader);
4765 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4766 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
4767 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4768 return False;
4769 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
4770 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4771 if(seekstart<0 and seekstart>fnumfiles):
4772 seekstart = 0;
4773 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
4774 seekend = fnumfiles;
4775 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
4776 seekend = fnumfiles - abs(seekend);
4777 if(seekstart>0):
4778 il = 0;
4779 while(il < seekstart):
4780 prefhstart = catfp.tell();
4781 if(formatspecs['new_style']):
4782 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4783 else:
4784 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4785 if(len(preheaderdata)==0):
4786 break;
4787 prefheadsize = int(preheaderdata[0], 16);
4788 prefnumfields = int(preheaderdata[1], 16);
4789 if(re.findall("^[.|/]", preheaderdata[3])):
4790 prefname = preheaderdata[3];
4791 else:
4792 prefname = "./"+preheaderdata[3];
4793 prefsize = int(preheaderdata[5], 16);
4794 prefcompression = preheaderdata[12];
4795 prefcsize = int(preheaderdata[13], 16);
4796 prefextrasize = int(preheaderdata[24], 16);
4797 prefextrafields = int(preheaderdata[25], 16);
4798 extrafieldslist = [];
4799 extrastart = 26;
4800 extraend = extrastart + prefextrafields;
4801 extrafieldslist = [];
4802 if(extrastart<extraend):
4803 extrafieldslist.append(preheaderdata[extrastart]);
4804 extrastart = extrastart + 1;
4805 prefchecksumtype = preheaderdata[extrastart].lower();
4806 prefcs = preheaderdata[extrastart + 1].lower();
4807 prefccs = preheaderdata[extrastart + 2].lower();
4808 hc = 0;
4809 hcmax = len(preheaderdata) - 2;
4810 hout = "";
4811 while(hc<hcmax):
4812 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs['format_delimiter']);
4813 hc = hc + 1;
4814 prenewfcs = GetFileChecksum(hout, preheaderdata[-3].lower(), True, formatspecs);
4815 if(prefcs!=prenewfcs and not skipchecksum):
4816 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4817 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4818 return False;
4819 valid_archive = False;
4820 invalid_archive = True;
4821 prefhend = catfp.tell() - 1;
4822 prefcontentstart = catfp.tell();
4823 prefcontents = "";
4824 pyhascontents = False;
4825 if(prefsize>0):
4826 if(prefcompression=="none" or prefcompression=="" or prefcompression=="auto"):
4827 prefcontents = catfp.read(prefsize);
4828 else:
4829 prefcontents = catfp.read(prefcsize);
4830 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4831 pyhascontents = True;
4832 if(prefccs!=prenewfccs and not skipchecksum):
4833 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4834 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4835 return False;
4836 catfp.seek(1, 1);
4837 il = il + 1;
4838 fileidnum = seekstart;
4839 realidnum = 0;
4840 while(fileidnum<seekend):
4841 catfhstart = catfp.tell();
4842 if(formatspecs['new_style']):
4843 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4844 else:
4845 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4846 if(len(catheaderdata)==0):
4847 break;
4848 catfheadsize = int(catheaderdata[0], 16);
4849 catfnumfields = int(catheaderdata[1], 16);
4850 catftype = int(catheaderdata[2], 16);
4851 if(re.findall("^[.|/]", catheaderdata[3])):
4852 catfname = catheaderdata[3];
4853 else:
4854 catfname = "./"+catheaderdata[3];
4855 catfbasedir = os.path.dirname(catfname);
4856 catflinkname = catheaderdata[4];
4857 catfsize = int(catheaderdata[5], 16);
4858 catfatime = int(catheaderdata[6], 16);
4859 catfmtime = int(catheaderdata[7], 16);
4860 catfctime = int(catheaderdata[8], 16);
4861 catfbtime = int(catheaderdata[9], 16);
4862 catfmode = int(catheaderdata[10], 16);
4863 catfchmode = stat.S_IMODE(catfmode);
4864 catftypemod = stat.S_IFMT(catfmode);
4865 catfwinattributes = int(catheaderdata[11], 16);
4866 catfcompression = catheaderdata[12];
4867 catfcsize = int(catheaderdata[13], 16);
4868 catfuid = int(catheaderdata[14], 16);
4869 catfuname = catheaderdata[15];
4870 catfgid = int(catheaderdata[16], 16);
4871 catfgname = catheaderdata[17];
4872 catfid = int(catheaderdata[18], 16);
4873 catfinode = int(catheaderdata[19], 16);
4874 catflinkcount = int(catheaderdata[20], 16);
4875 catfdev_minor = int(catheaderdata[21], 16);
4876 catfdev_major = int(catheaderdata[22], 16);
4877 catfrdev_minor = int(catheaderdata[23], 16);
4878 catfrdev_major = int(catheaderdata[24], 16);
4879 catfextrasize = int(catheaderdata[25], 16);
4880 catfextrafields = int(catheaderdata[26], 16);
4881 extrafieldslist = [];
4882 extrastart = 27;
4883 extraend = extrastart + catfextrafields;
4884 extrafieldslist = [];
4885 if(extrastart<extraend):
4886 extrafieldslist.append(catheaderdata[extrastart]);
4887 extrastart = extrastart + 1;
4888 catfchecksumtype = catheaderdata[extrastart].lower();
4889 catfcs = catheaderdata[extrastart + 1].lower();
4890 catfccs = catheaderdata[extrastart + 2].lower();
4891 hc = 0;
4892 hcmax = len(catheaderdata) - 2;
4893 hout = "";
4894 while(hc<hcmax):
4895 hout = hout + AppendNullByte(catheaderdata[hc], formatspecs['format_delimiter']);
4896 hc = hc + 1;
4897 catnewfcs = GetFileChecksum(hout, catheaderdata[-3].lower(), True, formatspecs);
4898 if(catfcs!=catnewfcs and not skipchecksum):
4899 VerbosePrintOut("File Header Checksum Error with file " + catfname + " at offset " + str(catfhstart));
4900 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
4901 return False;
4902 catfhend = catfp.tell() - 1;
4903 catfcontentstart = catfp.tell();
4904 catfcontents = BytesIO();
4905 pyhascontents = False;
4906 if(catfsize>0 and not listonly):
4907 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4908 catfcontents.write(catfp.read(catfsize));
4909 else:
4910 catfcontents.write(catfp.read(catfcsize));
4911 catfcontents.seek(0, 0);
4912 catnewfccs = GetFileChecksum(catfcontents.read(), catheaderdata[-3].lower(), False, formatspecs);
4913 pyhascontents = True;
4914 if(catfccs!=catnewfccs and skipchecksum):
4915 VerbosePrintOut("File Content Checksum Error with file " + catfname + " at offset " + str(catfcontentstart));
4916 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
4917 return False;
4918 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4919 pass;
4920 else:
4921 catfcontents.seek(0, 0);
4922 catfcontents = UncompressArchiveFile(catfcontents, formatspecs);
4923 if(catfsize>0 and listonly):
4924 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4925 catfp.seek(catfsize, 1);
4926 else:
4927 catfp.seek(catfcsize, 1);
4928 pyhascontents = False;
4929 catfp.seek(1, 1);
4930 catfcontentend = catfp.tell() - 1;
4931 catfcontents.seek(0, 0);
4932 catlist['ffilelist'].update({realidnum: {'fid': realidnum, 'fidalt': fileidnum, 'fheadersize': catfheadsize, 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize, 'fatime': catfatime, 'fmtime': catfmtime, 'fctime': catfctime, 'fbtime': catfbtime, 'fmode': catfmode, 'fchmode': catfchmode, 'ftypemod': catftypemod, 'fwinattributes': catfwinattributes, 'fcompression': catfcompression, 'fcsize': catfcsize, 'fuid': catfuid, 'funame': catfuname, 'fgid': catfgid, 'fgname': catfgname, 'finode': catfinode, 'flinkcount': catflinkcount, 'fminor': catfdev_minor, 'fmajor': catfdev_major, 'frminor': catfrdev_minor, 'frmajor': catfrdev_major, 'fchecksumtype': catfchecksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': catfextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': catfcs, 'fcontentchecksum': catfccs, 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': catfcontents} });
4933 fileidnum = fileidnum + 1;
4934 realidnum = realidnum + 1;
4935 if(returnfp):
4936 catlist.update({'catfp': catfp});
4937 else:
4938 catfp.close();
4939 return catlist;
4940
4941 create_alias_function("", __file_format_name__, "ToArray", ArchiveFileToArray);
4942
4943 def ArchiveFileStringToArray(catstr, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4944 formatspecs = FormatSpecsListToDict(formatspecs);
4945 catfp = BytesIO(catstr);
4946 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4947 return listcatfiles;
4948
4949 create_alias_function("", __file_format_name__, "StringToArray", ArchiveFileStringToArray);
4950
4951 def TarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4952 formatspecs = FormatSpecsListToDict(formatspecs);
4953 catfp = BytesIO();
4954 catfp = PackArchiveFileFromTarFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
4955 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4956 return listcatfiles;
4957
4958 def ZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4959 formatspecs = FormatSpecsListToDict(formatspecs);
4960 catfp = BytesIO();
4961 catfp = PackArchiveFileFromZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
4962 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4963 return listcatfiles;
4964
4965 if(not rarfile_support):
4966 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4967 return False;
4968
4969 if(rarfile_support):
4970 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4971 formatspecs = FormatSpecsListToDict(formatspecs);
4972 catfp = BytesIO();
4973 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
4974 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4975 return listcatfiles;
4976
4977 if(not py7zr_support):
4978 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4979 return False;
4980
4981 if(py7zr_support):
4982 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4983 formatspecs = FormatSpecsListToDict(formatspecs);
4984 catfp = BytesIO();
4985 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
4986 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4987 return listcatfiles;
4988
4989 def InFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4990 formatspecs = FormatSpecsListToDict(formatspecs);
4991 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4992 if(checkcompressfile=="tarfile"):
4993 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4994 elif(checkcompressfile=="zipfile"):
4995 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4996 elif(checkcompressfile=="catfile"):
4997 return ArchiveFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4998 elif(rarfile_support and checkcompressfile=="rarfile"):
4999 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5000 elif(py7zr_support and checkcompressfile=="7zipfile"):
5001 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5002 else:
5003 return False;
5004 return False;
5005
5006 def ListDirToArrayAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5007 formatspecs = FormatSpecsListToDict(formatspecs);
5008 catver = formatspecs['format_ver'];
5009 fileheaderver = str(int(catver.replace(".", "")));
5010 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5011 advancedlist = formatspecs['use_advanced_list'];
5012 altinode = formatspecs['use_alt_inode'];
5013 infilelist = [];
5014 if(infiles=="-"):
5015 for line in sys.stdin:
5016 infilelist.append(line.strip());
5017 infilelist = list(filter(None, infilelist));
5018 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
5019 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
5020 return False;
5021 with UncompressFile(infiles, formatspecs, "r") as finfile:
5022 for line in finfile:
5023 infilelist.append(line.strip());
5024 infilelist = list(filter(None, infilelist));
5025 else:
5026 if(isinstance(infiles, (list, tuple, ))):
5027 infilelist = list(filter(None, infiles));
5028 elif(isinstance(infiles, (str, ))):
5029 infilelist = list(filter(None, [infiles]));
5030 if(advancedlist):
5031 GetDirList = ListDirAdvanced(infilelist, followlink, False);
5032 else:
5033 GetDirList = ListDir(infilelist, followlink, False);
5034 if(not GetDirList):
5035 return False;
5036 curinode = 0;
5037 curfid = 0;
5038 inodelist = [];
5039 inodetofile = {};
5040 filetoinode = {};
5041 inodetocatinode = {};
5042 fileidnum = 0;
5043 fnumfiles = int(len(GetDirList));
5044 catver = formatspecs['format_ver'];
5045 fileheaderver = str(int(catver.replace(".", "")));
5046 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5047 fnumfileshex = format(int(fnumfiles), 'x').lower();
5048 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5049 catversion = re.findall(r"([\d]+)$", fileheader);
5050 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
5051 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
5052 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5053 fheadtell = len(fileheader);
5054 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5055 for curfname in GetDirList:
5056 if(re.findall("^[.|/]", curfname)):
5057 fname = curfname;
5058 else:
5059 fname = "./"+curfname;
5060 if(verbose):
5061 VerbosePrintOut(fname);
5062 if(not followlink or followlink is None):
5063 fstatinfo = os.lstat(fname);
5064 else:
5065 fstatinfo = os.stat(fname);
5066 fpremode = fstatinfo.st_mode;
5067 finode = fstatinfo.st_ino;
5068 flinkcount = fstatinfo.st_nlink;
5069 ftype = 0;
5070 if(stat.S_ISREG(fpremode)):
5071 ftype = 0;
5072 elif(stat.S_ISLNK(fpremode)):
5073 ftype = 2;
5074 elif(stat.S_ISCHR(fpremode)):
5075 ftype = 3;
5076 elif(stat.S_ISBLK(fpremode)):
5077 ftype = 4;
5078 elif(stat.S_ISDIR(fpremode)):
5079 ftype = 5;
5080 elif(stat.S_ISFIFO(fpremode)):
5081 ftype = 6;
5082 elif(stat.S_ISSOCK(fpremode)):
5083 ftype = 8;
5084 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
5085 ftype = 9;
5086 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
5087 ftype = 10;
5088 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
5089 ftype = 11;
5090 else:
5091 ftype = 0;
5092 flinkname = "";
5093 fbasedir = os.path.dirname(fname);
5094 fcurfid = curfid;
5095 if(not followlink and finode!=0):
5096 if(ftype!=1):
5097 if(finode in inodelist):
5098 ftype = 1;
5099 flinkname = inodetofile[finode];
5100 if(altinode):
5101 fcurinode = finode;
5102 else:
5103 fcurinode = inodetocatinode[finode];
5104 if(finode not in inodelist):
5105 inodelist.append(finode);
5106 inodetofile.update({finode: fname});
5107 inodetocatinode.update({finode: curinode});
5108 if(altinode):
5109 fcurinode = finode;
5110 else:
5111 fcurinode = curinode;
5112 curinode = curinode + 1;
5113 else:
5114 fcurinode = curinode;
5115 curinode = curinode + 1;
5116 curfid = curfid + 1;
5117 if(ftype==2):
5118 flinkname = os.readlink(fname);
5119 fdev = fstatinfo.st_dev;
5120 getfdev = GetDevMajorMinor(fdev);
5121 fdev_minor = getfdev[0];
5122 fdev_major = getfdev[1];
5123 frdev = fstatinfo.st_dev;
5124 if(hasattr(fstatinfo, "st_rdev")):
5125 frdev = fstatinfo.st_rdev;
5126 else:
5127 frdev = fstatinfo.st_dev;
5128 getfrdev = GetDevMajorMinor(frdev);
5129 frdev_minor = getfrdev[0];
5130 frdev_major = getfrdev[1];
5131 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
5132 fsize = "0";
5133 if(ftype==0 or ftype==7):
5134 fsize = fstatinfo.st_size;
5135 fatime = fstatinfo.st_atime;
5136 fmtime = fstatinfo.st_mtime;
5137 fctime = fstatinfo.st_ctime;
5138 if(hasattr(fstatinfo, "st_birthtime")):
5139 fbtime = fstatinfo.st_birthtime;
5140 else:
5141 fbtime = fstatinfo.st_ctime;
5142 fmode = fstatinfo.st_mode;
5143 fchmode = stat.S_IMODE(fstatinfo.st_mode);
5144 ftypemod = stat.S_IFMT(fstatinfo.st_mode);
5145 fuid = fstatinfo.st_uid;
5146 fgid = fstatinfo.st_gid;
5147 funame = "";
5148 try:
5149 import pwd;
5150 try:
5151 userinfo = pwd.getpwuid(fstatinfo.st_uid);
5152 funame = userinfo.pw_name;
5153 except KeyError:
5154 funame = "";
5155 except ImportError:
5156 funame = "";
5157 fgname = "";
5158 try:
5159 import grp;
5160 try:
5161 groupinfo = grp.getgrgid(fstatinfo.st_gid);
5162 fgname = groupinfo.gr_name;
5163 except KeyError:
5164 fgname = "";
5165 except ImportError:
5166 fgname = "";
5167 fdev_minor = fdev_minor;
5168 fdev_major = fdev_major;
5169 frdev_minor = frdev_minor;
5170 frdev_major = frdev_major;
5171 flinkcount = flinkcount;
5172 if(hasattr(fstatinfo, "st_file_attributes")):
5173 fwinattributes = fstatinfo.st_file_attributes;
5174 else:
5175 fwinattributes = 0;
5176 fcompression = "";
5177 fcsize = 0;
5178 fcontents = BytesIO();
5179 if(ftype==0 or ftype==7):
5180 with open(fname, "rb") as fpc:
5181 shutil.copyfileobj(fpc, fcontents);
5182 if(followlink and (ftype==1 or ftype==2)):
5183 flstatinfo = os.stat(flinkname);
5184 with open(flinkname, "rb") as fpc:
5185 shutil.copyfileobj(fpc, fcontents);
5186 fcontents.seek(0, 0);
5187 ftypehex = format(ftype, 'x').lower();
5188 extrafields = len(extradata);
5189 extrafieldslist = extradata;
5190 catfextrafields = extrafields;
5191 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5192 if(len(extradata)>0):
5193 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5194 extrasizelen = len(extrasizestr);
5195 extrasizelenhex = format(extrasizelen, 'x').lower();
5196 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5197 catoutlen = len(catoutlist) + len(extradata) + 3;
5198 catoutlenhex = format(catoutlen, 'x').lower();
5199 catoutlist.insert(0, catoutlenhex);
5200 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5201 if(len(extradata)>0):
5202 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5203 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
5204 catfnumfields = catoutlen;
5205 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5206 fcontents.seek(0, 0);
5207 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5208 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5209 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5210 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5211 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5212 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5213 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5214 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5215 fheadtell += len(catfileoutstr) + 1;
5216 catfcontentend = fheadtell - 1;
5217 fcontents.seek(0, 0);
5218 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5219 pyhascontents = False;
5220 if(int(fsize)>0 and not listonly):
5221 pyhascontents = True;
5222 if(int(fsize)>0 and listonly):
5223 fcontents = BytesIO();
5224 pyhascontents = False;
5225 fcontents.seek(0, 0);
5226 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5227 fileidnum = fileidnum + 1;
5228 return catlist;
5229
5230 def TarFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5231 formatspecs = FormatSpecsListToDict(formatspecs);
5232 curinode = 0;
5233 curfid = 0;
5234 inodelist = [];
5235 inodetofile = {};
5236 filetoinode = {};
5237 inodetocatinode = {};
5238 fileidnum = 0;
5239 if(infile=="-"):
5240 infile = BytesIO();
5241 if(hasattr(sys.stdin, "buffer")):
5242 shutil.copyfileobj(sys.stdin.buffer, infile);
5243 else:
5244 shutil.copyfileobj(sys.stdin, infile);
5245 infile.seek(0, 0);
5246 if(not infile):
5247 return False;
5248 infile.seek(0, 0);
5249 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
5250 infile = download_file_from_internet_file(infile);
5251 infile.seek(0, 0);
5252 if(not infile):
5253 return False;
5254 infile.seek(0, 0);
5255 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
5256 return False;
5257 elif(os.path.exists(infile) and os.path.isfile(infile)):
5258 try:
5259 if(not tarfile.is_tarfile(infile)):
5260 return False;
5261 except AttributeError:
5262 if(not is_tarfile(infile)):
5263 return False;
5264 else:
5265 return False;
5266 try:
5267 if(hasattr(infile, "read") or hasattr(infile, "write")):
5268 tarfp = tarfile.open(fileobj=infile, mode="r");
5269 else:
5270 tarfp = tarfile.open(infile, "r");
5271 except FileNotFoundError:
5272 return False;
5273 fnumfiles = int(len(tarfp.getmembers()));
5274 catver = formatspecs['format_ver'];
5275 fileheaderver = str(int(catver.replace(".", "")));
5276 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5277 fnumfileshex = format(int(fnumfiles), 'x').lower();
5278 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5279 catversion = re.findall(r"([\d]+)$", fileheader);
5280 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
5281 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
5282 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5283 fheadtell = len(fileheader);
5284 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5285 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
5286 if(re.findall("^[.|/]", member.name)):
5287 fname = member.name;
5288 else:
5289 fname = "./"+member.name;
5290 if(verbose):
5291 VerbosePrintOut(fname);
5292 fpremode = member.mode;
5293 ffullmode = member.mode;
5294 flinkcount = 0;
5295 ftype = 0;
5296 if(member.isreg()):
5297 ffullmode = member.mode + stat.S_IFREG;
5298 ftype = 0;
5299 elif(member.isdev()):
5300 ffullmode = member.mode;
5301 ftype = 7;
5302 elif(member.islnk()):
5303 ffullmode = member.mode + stat.S_IFREG;
5304 ftype = 1;
5305 elif(member.issym()):
5306 ffullmode = member.mode + stat.S_IFLNK;
5307 ftype = 2;
5308 elif(member.ischr()):
5309 ffullmode = member.mode + stat.S_IFCHR;
5310 ftype = 3;
5311 elif(member.isblk()):
5312 ffullmode = member.mode + stat.S_IFBLK;
5313 ftype = 4;
5314 elif(member.isdir()):
5315 ffullmode = member.mode + stat.S_IFDIR;
5316 ftype = 5;
5317 elif(member.isfifo()):
5318 ffullmode = member.mode + stat.S_IFIFO;
5319 ftype = 6;
5320 elif(member.issparse()):
5321 ffullmode = member.mode;
5322 ftype = 12;
5323 else:
5324 ffullmode = member.mode;
5325 ftype = 0;
5326 flinkname = "";
5327 fbasedir = os.path.dirname(fname);
5328 fcurfid = curfid;
5329 fcurinode = curfid;
5330 finode = fcurinode;
5331 curfid = curfid + 1;
5332 if(ftype==2):
5333 flinkname = member.linkname;
5334 fdev_minor = member.devminor;
5335 fdev_major = member.devmajor;
5336 frdev_minor = member.devminor;
5337 frdev_major = member.devmajor;
5338 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
5339 fsize = "0";
5340 elif(ftype==0 or ftype==7):
5341 fsize = member.size;
5342 else:
5343 fsize = member.size;
5344 fatime = member.mtime;
5345 fmtime = member.mtime;
5346 fctime = member.mtime;
5347 fbtime = member.mtime;
5348 fmode = ffullmode;
5349 fchmode = stat.S_IMODE(ffullmode);
5350 ftypemod = stat.S_IFMT(ffullmode);
5351 fuid = member.uid;
5352 fgid = member.gid;
5353 funame = member.uname;
5354 fgname = member.gname;
5355 flinkcount = flinkcount;
5356 fwinattributes = int(0);
5357 fcompression = "";
5358 fcsize = 0;
5359 fcontents = BytesIO();
5360 if(ftype==0 or ftype==7):
5361 with tarfp.extractfile(member) as fpc:
5362 shutil.copyfileobj(fpc, fcontents);
5363 fcontents.seek(0, 0);
5364 ftypehex = format(ftype, 'x').lower();
5365 extrafields = len(extradata);
5366 extrafieldslist = extradata;
5367 catfextrafields = extrafields;
5368 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5369 if(len(extradata)>0):
5370 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5371 extrasizelen = len(extrasizestr);
5372 extrasizelenhex = format(extrasizelen, 'x').lower();
5373 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5374 catoutlen = len(catoutlist) + len(extradata) + 3;
5375 catoutlenhex = format(catoutlen, 'x').lower();
5376 catoutlist.insert(0, catoutlenhex);
5377 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5378 if(len(extradata)>0):
5379 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5380 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
5381 catfnumfields = catoutlen;
5382 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5383 fcontents.seek(0, 0);
5384 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5385 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5386 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5387 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5388 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5389 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5390 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5391 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5392 fheadtell += len(catfileoutstr) + 1;
5393 catfcontentend = fheadtell - 1;
5394 fcontents.seek(0, 0);
5395 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5396 pyhascontents = False;
5397 if(int(fsize)>0 and not listonly):
5398 pyhascontents = True;
5399 if(int(fsize)>0 and listonly):
5400 fcontents = BytesIO();
5401 pyhascontents = False;
5402 fcontents.seek(0, 0);
5403 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5404 fileidnum = fileidnum + 1;
5405 return catlist;
5406
5407 def ZipFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5408 formatspecs = FormatSpecsListToDict(formatspecs);
5409 curinode = 0;
5410 curfid = 0;
5411 inodelist = [];
5412 inodetofile = {};
5413 filetoinode = {};
5414 inodetocatinode = {};
5415 fileidnum = 0;
5416 if(infile=="-"):
5417 infile = BytesIO();
5418 if(hasattr(sys.stdin, "buffer")):
5419 shutil.copyfileobj(sys.stdin.buffer, infile);
5420 else:
5421 shutil.copyfileobj(sys.stdin, infile);
5422 infile.seek(0, 0);
5423 if(not infile):
5424 return False;
5425 infile.seek(0, 0);
5426 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
5427 infile = download_file_from_internet_file(infile);
5428 infile.seek(0, 0);
5429 if(not infile):
5430 return False;
5431 infile.seek(0, 0);
5432 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
5433 return False;
5434 else:
5435 return False;
5436 if(not zipfile.is_zipfile(infile)):
5437 return False;
5438 try:
5439 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
5440 except FileNotFoundError:
5441 return False;
5442 ziptest = zipfp.testzip();
5443 if(ziptest):
5444 VerbosePrintOut("Bad file found!");
5445 fnumfiles = int(len(zipfp.infolist()));
5446 catver = formatspecs['format_ver'];
5447 fileheaderver = str(int(catver.replace(".", "")));
5448 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5449 catversion = re.findall(r"([\d]+)$", fileheader);
5450 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
5451 fnumfileshex = format(int(fnumfiles), 'x').lower();
5452 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5453 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
5454 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5455 fheadtell = len(fileheader);
5456 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5457 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
5458 if(re.findall("^[.|/]", member.filename)):
5459 fname = member.filename;
5460 else:
5461 fname = "./"+member.filename;
5462 zipinfo = zipfp.getinfo(member.filename);
5463 if(verbose):
5464 VerbosePrintOut(fname);
5465 if(not member.is_dir()):
5466 fpremode = stat.S_IFREG + 438;
5467 elif(member.is_dir()):
5468 fpremode = stat.S_IFDIR + 511;
5469 flinkcount = 0;
5470 ftype = 0;
5471 if(not member.is_dir()):
5472 ftype = 0;
5473 elif(member.is_dir()):
5474 ftype = 5;
5475 flinkname = "";
5476 fbasedir = os.path.dirname(fname);
5477 fcurfid = curfid;
5478 fcurinode = curfid;
5479 finode = fcurinode;
5480 curfid = curfid + 1;
5481 fdev_minor = 0;
5482 fdev_major = 0;
5483 frdev_minor = 0;
5484 frdev_major = 0;
5485 if(ftype==5):
5486 fsize = "0";
5487 elif(ftype==0):
5488 fsize = member.file_size;
5489 else:
5490 fsize = member.file_size;
5491 fatime = time.mktime(member.date_time + (0, 0, -1));
5492 fmtime = time.mktime(member.date_time + (0, 0, -1));
5493 fctime = time.mktime(member.date_time + (0, 0, -1));
5494 fbtime = time.mktime(member.date_time + (0, 0, -1));
5495 if(zipinfo.create_system==0 or zipinfo.create_system==10):
5496 fwinattributes = int(zipinfo.external_attr);
5497 if(not member.is_dir()):
5498 fmode = int(stat.S_IFREG + 438);
5499 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
5500 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
5501 elif(member.is_dir()):
5502 fmode = int(stat.S_IFDIR + 511);
5503 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
5504 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
5505 elif(zipinfo.create_system==3):
5506 fwinattributes = int(0);
5507 fmode = int(zipinfo.external_attr);
5508 else:
5509 fwinattributes = int(0);
5510 if(not member.is_dir()):
5511 fmode = int(stat.S_IFREG + 438);
5512 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
5513 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
5514 elif(member.is_dir()):
5515 fmode = int(stat.S_IFDIR + 511);
5516 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
5517 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
5518 fcompression = "";
5519 fcsize = 0;
5520 try:
5521 fuid = os.getuid();
5522 except AttributeError:
5523 fuid = 0;
5524 except KeyError:
5525 fuid = 0;
5526 try:
5527 fgid = os.getgid();
5528 except AttributeError:
5529 fgid = 0;
5530 except KeyError:
5531 fgid = 0;
5532 try:
5533 import pwd;
5534 try:
5535 userinfo = pwd.getpwuid(os.getuid());
5536 funame = userinfo.pw_name;
5537 except KeyError:
5538 funame = "";
5539 except AttributeError:
5540 funame = "";
5541 except ImportError:
5542 funame = "";
5543 fgname = "";
5544 try:
5545 import grp;
5546 try:
5547 groupinfo = grp.getgrgid(os.getgid());
5548 fgname = groupinfo.gr_name;
5549 except KeyError:
5550 fgname = "";
5551 except AttributeError:
5552 fgname = "";
5553 except ImportError:
5554 fgname = "";
5555 fcontents = BytesIO();
5556 if(ftype==0):
5557 fcontents.write(zipfp.read(member.filename));
5558 fcontents.seek(0, 0);
5559 ftypehex = format(ftype, 'x').lower();
5560 extrafields = len(extradata);
5561 extrafieldslist = extradata;
5562 catfextrafields = extrafields;
5563 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5564 if(len(extradata)>0):
5565 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5566 extrasizelen = len(extrasizestr);
5567 extrasizelenhex = format(extrasizelen, 'x').lower();
5568 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5569 catoutlen = len(catoutlist) + len(extradata) + 3;
5570 catoutlenhex = format(catoutlen, 'x').lower();
5571 catoutlist.insert(0, catoutlenhex);
5572 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5573 if(len(extradata)>0):
5574 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5575 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
5576 catfnumfields = catoutlen;
5577 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5578 fcontents.seek(0, 0);
5579 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5580 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5581 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5582 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5583 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5584 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5585 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5586 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5587 fheadtell += len(catfileoutstr) + 1;
5588 catfcontentend = fheadtell - 1;
5589 fcontents.seek(0, 0);
5590 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5591 pyhascontents = False;
5592 if(int(fsize)>0 and not listonly):
5593 pyhascontents = True;
5594 if(int(fsize)>0 and listonly):
5595 fcontents = BytesIO();
5596 pyhascontents = False;
5597 fcontents.seek(0, 0);
5598 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5599 fileidnum = fileidnum + 1;
5600 return catlist;
5601
5602 if(not rarfile_support):
5603 def RarFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5604 return False;
5605
5606 if(rarfile_support):
5607 def RarFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5608 formatspecs = FormatSpecsListToDict(formatspecs);
5609 curinode = 0;
5610 curfid = 0;
5611 inodelist = [];
5612 inodetofile = {};
5613 filetoinode = {};
5614 inodetocatinode = {};
5615 fileidnum = 0;
5616 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
5617 return False;
5618 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
5619 return False;
5620 rarfp = rarfile.RarFile(infile, "r");
5621 rartest = rarfp.testrar();
5622 if(rartest):
5623 VerbosePrintOut("Bad file found!");
5624 fnumfiles = int(len(rarfp.infolist()));
5625 catver = formatspecs['format_ver'];
5626 fileheaderver = str(int(catver.replace(".", "")));
5627 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5628 catversion = re.findall(r"([\d]+)$", fileheader);
5629 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
5630 fnumfileshex = format(int(fnumfiles), 'x').lower();
5631 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5632 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
5633 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5634 fheadtell = len(fileheader);
5635 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5636 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
5637 is_unix = False;
5638 is_windows = False;
5639 if(member.host_os==rarfile.RAR_OS_UNIX):
5640 is_windows = False;
5641 try:
5642 member.external_attr
5643 is_unix = True;
5644 except AttributeError:
5645 is_unix = False;
5646 elif(member.host_os==rarfile.RAR_OS_WIN32):
5647 is_unix = False;
5648 try:
5649 member.external_attr
5650 is_windows = True;
5651 except AttributeError:
5652 is_windows = False;
5653 else:
5654 is_unix = False;
5655 is_windows = False;
5656 if(re.findall("^[.|/]", member.filename)):
5657 fname = member.filename;
5658 else:
5659 fname = "./"+member.filename;
5660 rarinfo = rarfp.getinfo(member.filename);
5661 if(verbose):
5662 VerbosePrintOut(fname);
5663 if(is_unix and member.external_attr !=0):
5664 fpremode = int(member.external_attr);
5665 elif(member.is_file()):
5666 fpremode = stat.S_IFREG + 438;
5667 elif(member.is_symlink()):
5668 fpremode = stat.S_IFLNK + 438;
5669 elif(member.is_dir()):
5670 fpremode = stat.S_IFDIR + 511;
5671 if(is_windows and member.external_attr !=0):
5672 fwinattributes = int(member.external_attr);
5673 else:
5674 fwinattributes = int(0);
5675 fcompression = "";
5676 fcsize = 0;
5677 flinkcount = 0;
5678 ftype = 0;
5679 if(member.is_file()):
5680 ftype = 0;
5681 elif(member.is_symlink()):
5682 ftype = 2;
5683 elif(member.is_dir()):
5684 ftype = 5;
5685 flinkname = "";
5686 if(ftype==2):
5687 flinkname = rarfp.read(member.filename).decode("UTF-8");
5688 fbasedir = os.path.dirname(fname);
5689 fcurfid = curfid;
5690 fcurinode = curfid;
5691 finode = fcurinode;
5692 curfid = curfid + 1;
5693 fdev_minor = 0;
5694 fdev_major = 0;
5695 frdev_minor = 0;
5696 frdev_major = 0;
5697 if(ftype==5):
5698 fsize = "0";
5699 if(ftype==0):
5700 fsize = member.file_size;
5701 try:
5702 if(member.atime):
5703 fatime = int(member.atime.timestamp());
5704 else:
5705 fatime = int(member.mtime.timestamp());
5706 except AttributeError:
5707 fatime = int(member.mtime.timestamp());
5708 fmtime = int(member.mtime.timestamp());
5709 try:
5710 if(member.ctime):
5711 fctime = int(member.ctime.timestamp());
5712 else:
5713 fctime = int(member.mtime.timestamp());
5714 except AttributeError:
5715 fctime = int(member.mtime.timestamp());
5716 fbtime = int(member.mtime.timestamp());
5717 if(is_unix and member.external_attr !=0):
5718 fmode = int(member.external_attr);
5719 fchmode = int(stat.S_IMODE(member.external_attr));
5720 ftypemod = int(stat.S_IFMT(member.external_attr));
5721 elif(member.is_file()):
5722 fmode = int(stat.S_IFREG + 438)
5723 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5724 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5725 elif(member.is_symlink()):
5726 fmode = int(stat.S_IFLNK + 438)
5727 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5728 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5729 elif(member.is_dir()):
5730 fmode = int(stat.S_IFDIR + 511)
5731 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
5732 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
5733 try:
5734 fuid = os.getuid();
5735 except AttributeError:
5736 fuid = 0;
5737 except KeyError:
5738 fuid = 0;
5739 try:
5740 fgid = os.getgid();
5741 except AttributeError:
5742 fgid = 0;
5743 except KeyError:
5744 fgid = 0;
5745 try:
5746 import pwd;
5747 try:
5748 userinfo = pwd.getpwuid(os.getuid());
5749 funame = userinfo.pw_name;
5750 except KeyError:
5751 funame = "";
5752 except AttributeError:
5753 funame = "";
5754 except ImportError:
5755 funame = "";
5756 fgname = "";
5757 try:
5758 import grp;
5759 try:
5760 groupinfo = grp.getgrgid(os.getgid());
5761 fgname = groupinfo.gr_name;
5762 except KeyError:
5763 fgname = "";
5764 except AttributeError:
5765 fgname = "";
5766 except ImportError:
5767 fgname = "";
5768 fcontents = BytesIO();
5769 if(ftype==0):
5770 fcontents.write(rarfp.read(member.filename));
5771 fcontents.seek(0, 0);
5772 ftypehex = format(ftype, 'x').lower();
5773 extrafields = len(extradata);
5774 extrafieldslist = extradata;
5775 catfextrafields = extrafields;
5776 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5777 if(len(extradata)>0):
5778 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5779 extrasizelen = len(extrasizestr);
5780 extrasizelenhex = format(extrasizelen, 'x').lower();
5781 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5782 catoutlen = len(catoutlist) + len(extradata) + 3;
5783 catoutlenhex = format(catoutlen, 'x').lower();
5784 catoutlist.insert(0, catoutlenhex);
5785 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5786 if(len(extradata)>0):
5787 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5788 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
5789 catfnumfields = 24 + catfextrafields;
5790 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5791 fcontents.seek(0, 0);
5792 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5793 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5794 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5795 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5796 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5797 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5798 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5799 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5800 fheadtell += len(catfileoutstr) + 1;
5801 catfcontentend = fheadtell - 1;
5802 fcontents.seek(0, 0);
5803 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5804 pyhascontents = False;
5805 if(int(fsize)>0 and not listonly):
5806 pyhascontents = True;
5807 if(int(fsize)>0 and listonly):
5808 fcontents = BytesIO();
5809 pyhascontents = False;
5810 fcontents.seek(0, 0);
5811 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5812 fileidnum = fileidnum + 1;
5813 return catlist;
5814
5815 if(not py7zr_support):
5816 def SevenZipFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5817 return False;
5818
5819 if(py7zr_support):
5820 def SevenZipFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5821 formatspecs = FormatSpecsListToDict(formatspecs);
5822 curinode = 0;
5823 curfid = 0;
5824 inodelist = [];
5825 inodetofile = {};
5826 filetoinode = {};
5827 inodetocatinode = {};
5828 fileidnum = 0;
5829 szpfp = py7zr.SevenZipFile(infile, mode="r");
5830 file_content = szpfp.readall();
5831 #sztest = szpfp.testzip();
5832 sztestalt = szpfp.test();
5833 if(sztestalt):
5834 VerbosePrintOut("Bad file found!");
5835 numfiles = int(len(szpfp.list()));
5836 catver = formatspecs['format_ver'];
5837 fileheaderver = str(int(catver.replace(".", "")));
5838 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5839 catversion = re.findall(r"([\d]+)$", fileheader);
5840 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
5841 fnumfileshex = format(int(fnumfiles), 'x').lower();
5842 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5843 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
5844 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5845 fheadtell = len(fileheader);
5846 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5847 for member in sorted(szpfp.list(), key=lambda x: x.filename):
5848 if(re.findall("^[.|/]", member.filename)):
5849 fname = member.filename;
5850 else:
5851 fname = "./"+member.filename;
5852 if(not member.is_directory):
5853 fpremode = int(stat.S_IFREG + 438);
5854 elif(member.is_directory):
5855 fpremode = int(stat.S_IFDIR + 511);
5856 fwinattributes = int(0);
5857 fcompression = "";
5858 fcsize = 0;
5859 flinkcount = 0;
5860 ftype = 0;
5861 if(member.is_directory):
5862 ftype = 5;
5863 else:
5864 ftype = 0;
5865 flinkname = "";
5866 fbasedir = os.path.dirname(fname);
5867 fcurfid = curfid;
5868 fcurinode = curfid;
5869 finode = fcurinode;
5870 curfid = curfid + 1;
5871 fdev_minor = 0;
5872 fdev_major = 0;
5873 frdev_minor = 0;
5874 frdev_major = 0;
5875 if(ftype==5):
5876 fsize = "0";
5877 fatime = int(member.creationtime.timestamp());
5878 fmtime = int(member.creationtime.timestamp());
5879 fctime = int(member.creationtime.timestamp());
5880 fbtime = int(member.creationtime.timestamp());
5881 if(member.is_directory):
5882 fmode = int(stat.S_IFDIR + 511)
5883 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
5884 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
5885 else:
5886 fmode = int(stat.S_IFLNK + 438)
5887 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5888 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5889 try:
5890 fuid = os.getuid();
5891 except AttributeError:
5892 fuid = 0;
5893 except KeyError:
5894 fuid = 0;
5895 try:
5896 fgid = os.getgid();
5897 except AttributeError:
5898 fgid = 0;
5899 except KeyError:
5900 fgid = 0;
5901 try:
5902 import pwd;
5903 try:
5904 userinfo = pwd.getpwuid(os.getuid());
5905 funame = userinfo.pw_name;
5906 except KeyError:
5907 funame = "";
5908 except AttributeError:
5909 funame = "";
5910 except ImportError:
5911 funame = "";
5912 fgname = "";
5913 try:
5914 import grp;
5915 try:
5916 groupinfo = grp.getgrgid(os.getgid());
5917 fgname = groupinfo.gr_name;
5918 except KeyError:
5919 fgname = "";
5920 except AttributeError:
5921 fgname = "";
5922 except ImportError:
5923 fgname = "";
5924 fcontents = BytesIO();
5925 if(ftype==0):
5926 fcontents.write(file_content[member.filename].read());
5927 fsize = format(fcontents.tell(), 'x').lower();
5928 fileop.close();
5929 fcontents.seek(0, 0);
5930 ftypehex = format(ftype, 'x').lower();
5931 extrafields = len(extradata);
5932 extrafieldslist = extradata;
5933 catfextrafields = extrafields;
5934 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5935 if(len(extradata)>0):
5936 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5937 extrasizelen = len(extrasizestr);
5938 extrasizelenhex = format(extrasizelen, 'x').lower();
5939 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5940 catoutlen = len(catoutlist) + len(extradata) + 3;
5941 catoutlenhex = format(catoutlen, 'x').lower();
5942 catoutlist.insert(0, catoutlenhex);
5943 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5944 if(len(extradata)>0):
5945 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5946 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
5947 catfnumfields = 24 + catfextrafields;
5948 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5949 fcontents.seek(0, 0);
5950 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5951 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5952 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5953 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5954 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5955 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5956 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5957 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5958 fheadtell += len(catfileoutstr) + 1;
5959 catfcontentend = fheadtell - 1;
5960 fcontents.seek(0, 0);
5961 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5962 pyhascontents = False;
5963 if(int(fsize)>0 and not listonly):
5964 pyhascontents = True;
5965 if(int(fsize)>0 and listonly):
5966 fcontents = BytesIO();
5967 pyhascontents = False;
5968 fcontents.seek(0, 0);
5969 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5970 fileidnum = fileidnum + 1;
5971 return catlist;
5972
5973 def InFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5974 formatspecs = FormatSpecsListToDict(formatspecs);
5975 checkcompressfile = CheckCompressionSubType(infiles, formatspecs, True);
5976 if(checkcompressfile=="tarfile"):
5977 return TarFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5978 elif(checkcompressfile=="zipfile"):
5979 return ZipFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5980 elif(checkcompressfile=="catfile"):
5981 return ArchiveFileToArray(infile, 0, 0, listonly, False, formatspecs, False);
5982 elif(rarfile_support and checkcompressfile=="rarfile"):
5983 return RarFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5984 elif(py7zr_support and checkcompressfile=="7zipfile"):
5985 return SevenZipFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5986 else:
5987 return False;
5988 return False;
5989
5990 def ListDirToArray(infiles, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, listonly=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
5991 formatspecs = FormatSpecsListToDict(formatspecs);
5992 outarray = BytesIO();
5993 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
5994 listcatfiles = ArchiveFileToArray(outarray, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5995 return listcatfiles;
5996
5997 def ArchiveFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5998 formatspecs = FormatSpecsListToDict(formatspecs);
5999 if(isinstance(infile, dict)):
6000 listcatfiles = infile;
6001 else:
6002 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6003 infile = RemoveWindowsPath(infile);
6004 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
6005 if(not listcatfiles):
6006 return False;
6007 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6008 if(returnfp):
6009 catarray.update({'catfp': listcatfiles['catfp']});
6010 lenlist = len(listcatfiles['ffilelist']);
6011 lcfi = 0;
6012 lcfx = int(listcatfiles['fnumfiles']);
6013 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6014 lcfx = int(lenlist);
6015 else:
6016 lcfx = int(listcatfiles['fnumfiles']);
6017 while(lcfi < lcfx):
6018 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6019 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6020 catarray['filetoid'].update(filetoidarray);
6021 catarray['idtofile'].update(idtofilearray);
6022 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6023 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6024 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6025 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6026 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6027 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6028 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6029 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6030 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6031 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6032 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6033 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6034 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6035 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6036 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6037 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6038 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6039 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6040 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6041 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6042 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6043 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6044 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6045 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6046 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6047 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6048 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6049 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6050 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6051 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6052 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6053 lcfi = lcfi + 1;
6054 return catarray;
6055
6056 create_alias_function("", __file_format_name__, "ToArrayIndex", ArchiveFileToArrayIndex);
6057
6058 def ListDirToArrayIndexAlt(infiles, dirlistfromtxt=False, followlink=False, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6059 formatspecs = FormatSpecsListToDict(formatspecs);
6060 listcatfiles = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, listonly, checksumtype, extradata, formatspecs, verbose);
6061 if(not listcatfiles):
6062 return False;
6063 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6064 lenlist = len(listcatfiles['ffilelist']);
6065 lcfi = 0;
6066 lcfx = int(listcatfiles['fnumfiles']);
6067 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6068 lcfx = int(lenlist);
6069 else:
6070 lcfx = int(listcatfiles['fnumfiles']);
6071 while(lcfi < lcfx):
6072 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6073 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6074 catarray['filetoid'].update(filetoidarray);
6075 catarray['idtofile'].update(idtofilearray);
6076 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6077 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6078 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6079 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6080 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6081 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6082 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6083 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6084 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6085 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6086 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6087 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6088 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6089 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6090 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6091 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6092 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6093 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6094 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6095 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6096 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6097 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6098 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6099 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6100 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6101 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6102 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6103 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6104 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6105 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6106 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6107 lcfi = lcfi + 1;
6108 return catarray;
6109
6110 def TarFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6111 formatspecs = FormatSpecsListToDict(formatspecs);
6112 listcatfiles = TarFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
6113 if(not listcatfiles):
6114 return False;
6115 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6116 lenlist = len(listcatfiles['ffilelist']);
6117 lcfi = 0;
6118 lcfx = int(listcatfiles['fnumfiles']);
6119 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6120 lcfx = int(lenlist);
6121 else:
6122 lcfx = int(listcatfiles['fnumfiles']);
6123 while(lcfi < lcfx):
6124 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6125 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6126 catarray['filetoid'].update(filetoidarray);
6127 catarray['idtofile'].update(idtofilearray);
6128 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6129 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6130 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6131 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6132 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6133 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6134 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6135 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6136 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6137 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6138 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6139 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6140 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6141 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6142 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6143 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6144 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6145 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6146 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6147 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6148 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6149 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6150 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6151 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6152 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6153 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6154 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6155 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6156 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6157 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6158 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6159 lcfi = lcfi + 1;
6160 return catarray;
6161
6162 def ZipFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6163 formatspecs = FormatSpecsListToDict(formatspecs);
6164 listcatfiles = ZipFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
6165 if(not listcatfiles):
6166 return False;
6167 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6168 lenlist = len(listcatfiles['ffilelist']);
6169 lcfi = 0;
6170 lcfx = int(listcatfiles['fnumfiles']);
6171 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6172 lcfx = int(lenlist);
6173 else:
6174 lcfx = int(listcatfiles['fnumfiles']);
6175 while(lcfi < lcfx):
6176 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6177 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6178 catarray['filetoid'].update(filetoidarray);
6179 catarray['idtofile'].update(idtofilearray);
6180 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6181 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6182 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6183 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6184 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6185 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6186 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6187 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6188 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6189 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6190 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6191 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6192 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6193 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6194 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6195 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6196 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6197 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6198 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6199 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6200 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6201 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6202 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6203 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6204 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6205 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6206 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6207 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6208 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6209 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6210 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6211 lcfi = lcfi + 1;
6212 return catarray;
6213
6214 if(not rarfile_support):
6215 def RarFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6216 return False;
6217
6218 if(rarfile_support):
6219 def RarFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6220 formatspecs = FormatSpecsListToDict(formatspecs);
6221 listcatfiles = RarFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
6222 if(not listcatfiles):
6223 return False;
6224 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6225 lenlist = len(listcatfiles['ffilelist']);
6226 lcfi = 0;
6227 lcfx = int(listcatfiles['fnumfiles']);
6228 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6229 lcfx = int(lenlist);
6230 else:
6231 lcfx = int(listcatfiles['fnumfiles']);
6232 while(lcfi < lcfx):
6233 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6234 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6235 catarray['filetoid'].update(filetoidarray);
6236 catarray['idtofile'].update(idtofilearray);
6237 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6238 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6239 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6240 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6241 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6242 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6243 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6244 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6245 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6246 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6247 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6248 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6249 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6250 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6251 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6252 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6253 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6254 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6255 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6256 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6257 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6258 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6259 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6260 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6261 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6262 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6263 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6264 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6265 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6266 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6267 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6268 lcfi = lcfi + 1;
6269 return catarray;
6270
6271 if(not py7zr_support):
6272 def SevenZipFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6273 return False;
6274
6275 if(py7zr_support):
6276 def SevenZipFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6277 formatspecs = FormatSpecsListToDict(formatspecs);
6278 listcatfiles = SevenZipFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
6279 if(not listcatfiles):
6280 return False;
6281 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6282 lenlist = len(listcatfiles['ffilelist']);
6283 lcfi = 0;
6284 lcfx = int(listcatfiles['fnumfiles']);
6285 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6286 lcfx = int(lenlist);
6287 else:
6288 lcfx = int(listcatfiles['fnumfiles']);
6289 while(lcfi < lcfx):
6290 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6291 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6292 catarray['filetoid'].update(filetoidarray);
6293 catarray['idtofile'].update(idtofilearray);
6294 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6295 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6296 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6297 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6298 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6299 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6300 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6301 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6302 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6303 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6304 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6305 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6306 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6307 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6308 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6309 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6310 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6311 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6312 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6313 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6314 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6315 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6316 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6317 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6318 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6319 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6320 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6321 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6322 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6323 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6324 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6325 lcfi = lcfi + 1;
6326 return catarray;
6327
6328 def ArchiveFileStringToArrayIndex(catstr, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6329 formatspecs = FormatSpecsListToDict(formatspecs);
6330 catfp = BytesIO(catstr);
6331 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
6332 return listcatfiles;
6333
6334 create_alias_function("", __file_format_name__, "StringToArrayIndex", ArchiveFileStringToArrayIndex);
6335
6336 def TarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6337 formatspecs = FormatSpecsListToDict(formatspecs);
6338 catfp = BytesIO();
6339 catfp = PackArchiveFileFromTarFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
6340 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
6341 return listcatfiles;
6342
6343 def ZipFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6344 formatspecs = FormatSpecsListToDict(formatspecs);
6345 catfp = BytesIO();
6346 catfp = PackArchiveFileFromZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
6347 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
6348 return listcatfiles;
6349
6350 if(not rarfile_support):
6351 def RarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6352 return False;
6353
6354 if(rarfile_support):
6355 def RarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6356 formatspecs = FormatSpecsListToDict(formatspecs);
6357 catfp = BytesIO();
6358 catfp = PackArchiveFileFromRarFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
6359 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
6360 return listcatfiles;
6361
6362 if(not py7zr_support):
6363 def SevenZipFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6364 return False;
6365
6366 if(py7zr_support):
6367 def SevenZipFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6368 formatspecs = FormatSpecsListToDict(formatspecs);
6369 catfp = BytesIO();
6370 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
6371 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
6372 return listcatfiles;
6373
6374 def ListDirToArrayIndex(infiles, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, listonly=False, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6375 formatspecs = FormatSpecsListToDict(formatspecs);
6376 outarray = BytesIO();
6377 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose, True);
6378 listcatfiles = ArchiveFileToArrayIndex(outarray, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp)
6379 return listcatfiles;
6380
6381 def RePackArchiveFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6382 formatspecs = FormatSpecsListToDict(formatspecs);
6383 if(isinstance(infile, dict)):
6384 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6385 listcatfiles = prelistcatfiles['list'];
6386 else:
6387 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6388 infile = RemoveWindowsPath(infile);
6389 if(followlink):
6390 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6391 listcatfiles = prelistcatfiles['list'];
6392 else:
6393 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6394 if(outfile!="-" and not hasattr(infile, "read") and not hasattr(outfile, "write")):
6395 outfile = RemoveWindowsPath(outfile);
6396 checksumtype = checksumtype.lower();
6397 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
6398 checksumtype="crc32";
6399 if(checksumtype=="none"):
6400 checksumtype = "";
6401 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
6402 compression = "auto";
6403 if(compression not in compressionlist and compression is None):
6404 compression = "auto";
6405 if(verbose):
6406 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6407 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
6408 if(os.path.exists(outfile)):
6409 try:
6410 os.unlink(outfile);
6411 except OSError as e:
6412 pass;
6413 if(not listcatfiles):
6414 return False;
6415 if(outfile=="-"):
6416 verbose = False;
6417 catfp = BytesIO();
6418 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
6419 catfp = outfile;
6420 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
6421 catfp = BytesIO();
6422 else:
6423 fbasename = os.path.splitext(outfile)[0];
6424 fextname = os.path.splitext(outfile)[1];
6425 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
6426 catver = formatspecs['format_ver'];
6427 fileheaderver = str(int(catver.replace(".", "")));
6428 lenlist = len(listcatfiles['ffilelist']);
6429 fnumfiles = int(listcatfiles['fnumfiles']);
6430 if(lenlist>fnumfiles or lenlist<fnumfiles):
6431 fnumfiles = lenlist;
6432 AppendFileHeader(catfp, fnumfiles, checksumtype, formatspecs);
6433 lenlist = len(listcatfiles['ffilelist']);
6434 fnumfiles = int(listcatfiles['fnumfiles']);
6435 lcfi = 0;
6436 lcfx = int(listcatfiles['fnumfiles']);
6437 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6438 lcfx = int(lenlist);
6439 else:
6440 lcfx = int(listcatfiles['fnumfiles']);
6441 curinode = 0;
6442 curfid = 0;
6443 inodelist = [];
6444 inodetofile = {};
6445 filetoinode = {};
6446 reallcfi = 0;
6447 while(lcfi < lcfx):
6448 if(re.findall("^[.|/]", listcatfiles['ffilelist'][reallcfi]['fname'])):
6449 fname = listcatfiles['ffilelist'][reallcfi]['fname'];
6450 else:
6451 fname = "./"+listcatfiles['ffilelist'][reallcfi]['fname'];
6452 if(verbose):
6453 VerbosePrintOut(fname);
6454 fheadersize = format(int(listcatfiles['ffilelist'][reallcfi]['fheadersize']), 'x').lower();
6455 fsize = format(int(listcatfiles['ffilelist'][reallcfi]['fsize']), 'x').lower();
6456 flinkname = listcatfiles['ffilelist'][reallcfi]['flinkname'];
6457 fatime = format(int(listcatfiles['ffilelist'][reallcfi]['fatime']), 'x').lower();
6458 fmtime = format(int(listcatfiles['ffilelist'][reallcfi]['fmtime']), 'x').lower();
6459 fctime = format(int(listcatfiles['ffilelist'][reallcfi]['fctime']), 'x').lower();
6460 fbtime = format(int(listcatfiles['ffilelist'][reallcfi]['fbtime']), 'x').lower();
6461 fmode = format(int(listcatfiles['ffilelist'][reallcfi]['fmode']), 'x').lower();
6462 fchmode = format(int(listcatfiles['ffilelist'][reallcfi]['fchmode']), 'x').lower();
6463 fuid = format(int(listcatfiles['ffilelist'][reallcfi]['fuid']), 'x').lower();
6464 funame = listcatfiles['ffilelist'][reallcfi]['funame'];
6465 fgid = format(int(listcatfiles['ffilelist'][reallcfi]['fgid']), 'x').lower();
6466 fgname = listcatfiles['ffilelist'][reallcfi]['fgname'];
6467 finode = format(int(listcatfiles['ffilelist'][reallcfi]['finode']), 'x').lower();
6468 flinkcount = format(int(listcatfiles['ffilelist'][reallcfi]['flinkcount']), 'x').lower();
6469 fwinattributes = format(int(listcatfiles['ffilelist'][reallcfi]['fwinattributes']), 'x').lower();
6470 fcompression = listcatfiles['ffilelist'][reallcfi]['fcompression'];
6471 fcsize = format(int(listcatfiles['ffilelist'][reallcfi]['fcsize']), 'x').lower();
6472 fdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['fminor']), 'x').lower();
6473 fdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['fmajor']), 'x').lower();
6474 frdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['frminor']), 'x').lower();
6475 frdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['frmajor']), 'x').lower();
6476 if(len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>listcatfiles['ffilelist'][reallcfi]['fextrafields'] and len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>0):
6477 listcatfiles['ffilelist'][reallcfi]['fextrafields'] = len(listcatfiles['ffilelist'][reallcfi]['fextralist']);
6478 if(not followlink and len(extradata)<0):
6479 extradata = listcatfiles['ffilelist'][reallcfi]['fextralist'];
6480 fcontents = listcatfiles['ffilelist'][reallcfi]['fcontents'];
6481 fcompression = "";
6482 fcsize = format(int(0), 'x').lower();
6483 if(not compresswholefile):
6484 fcontents.seek(0, 2);
6485 ucfsize = fcontents.tell();
6486 fcontents.seek(0, 0);
6487 if(compression=="auto"):
6488 ilsize = len(compressionlistalt);
6489 ilmin = 0;
6490 ilcsize = [];
6491 while(ilmin < ilsize):
6492 cfcontents = BytesIO();
6493 shutil.copyfileobj(fcontents, cfcontents);
6494 fcontents.seek(0, 0);
6495 cfcontents.seek(0, 0);
6496 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
6497 if(cfcontents):
6498 cfcontents.seek(0, 2);
6499 ilcsize.append(cfcontents.tell());
6500 cfcontents.close();
6501 else:
6502 try:
6503 ilcsize.append(sys.maxint);
6504 except AttributeError:
6505 ilcsize.append(sys.maxsize);
6506 ilmin = ilmin + 1;
6507 ilcmin = ilcsize.index(min(ilcsize));
6508 compression = compressionlistalt[ilcmin];
6509 fcontents.seek(0, 0);
6510 cfcontents = BytesIO();
6511 shutil.copyfileobj(fcontents, cfcontents);
6512 cfcontents.seek(0, 0);
6513 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
6514 cfcontents.seek(0, 2);
6515 cfsize = cfcontents.tell();
6516 if(ucfsize > cfsize):
6517 fcsize = format(int(cfsize), 'x').lower();
6518 fcompression = compression;
6519 fcontents.close();
6520 fcontents = cfcontents;
6521 if(followlink):
6522 if(listcatfiles['ffilelist'][reallcfi]['ftype']==1 or listcatfiles['ffilelist'][reallcfi]['ftype']==2):
6523 getflinkpath = listcatfiles['ffilelist'][reallcfi]['flinkname'];
6524 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6525 flinkinfo = listcatfiles['ffilelist'][flinkid];
6526 fheadersize = format(int(flinkinfo['fheadersize']), 'x').lower();
6527 fsize = format(int(flinkinfo['fsize']), 'x').lower();
6528 flinkname = flinkinfo['flinkname'];
6529 fatime = format(int(flinkinfo['fatime']), 'x').lower();
6530 fmtime = format(int(flinkinfo['fmtime']), 'x').lower();
6531 fctime = format(int(flinkinfo['fctime']), 'x').lower();
6532 fbtime = format(int(flinkinfo['fbtime']), 'x').lower();
6533 fmode = format(int(flinkinfo['fmode']), 'x').lower();
6534 fchmode = format(int(flinkinfo['fchmode']), 'x').lower();
6535 fuid = format(int(flinkinfo['fuid']), 'x').lower();
6536 funame = flinkinfo['funame'];
6537 fgid = format(int(flinkinfo['fgid']), 'x').lower();
6538 fgname = flinkinfo['fgname'];
6539 finode = format(int(flinkinfo['finode']), 'x').lower();
6540 flinkcount = format(int(flinkinfo['flinkcount']), 'x').lower();
6541 fwinattributes = format(int(flinkinfo['fwinattributes']), 'x').lower();
6542 fcompression = flinkinfo['fcompression'];
6543 fcsize = format(int(flinkinfo['fcsize']), 'x').lower();
6544 fdev_minor = format(int(flinkinfo['fminor']), 'x').lower();
6545 fdev_major = format(int(flinkinfo['fmajor']), 'x').lower();
6546 frdev_minor = format(int(flinkinfo['frminor']), 'x').lower();
6547 frdev_major = format(int(flinkinfo['frmajor']), 'x').lower();
6548 if(len(flinkinfo['fextralist'])>flinkinfo['fextrafields'] and len(flinkinfo['fextralist'])>0):
6549 flinkinfo['fextrafields'] = len(flinkinfo['fextralist']);
6550 if(len(extradata)<0):
6551 extradata = flinkinfo['fextralist'];
6552 fcontents = flinkinfo['fcontents'];
6553 ftypehex = format(flinkinfo['ftype'], 'x').lower();
6554 else:
6555 ftypehex = format(listcatfiles['ffilelist'][reallcfi]['ftype'], 'x').lower();
6556 fcurfid = format(curfid, 'x').lower();
6557 if(not followlink and finode!=0):
6558 if(listcatfiles['ffilelist'][reallcfi]['ftype']!=1):
6559 fcurinode = format(int(curinode), 'x').lower();
6560 inodetofile.update({curinode: fname});
6561 filetoinode.update({fname: curinode});
6562 curinode = curinode + 1;
6563 else:
6564 fcurinode = format(int(filetoinode[flinkname]), 'x').lower();
6565 else:
6566 fcurinode = format(int(curinode), 'x').lower();
6567 curinode = curinode + 1;
6568 curfid = curfid + 1;
6569 if(fcompression=="none"):
6570 fcompression = "";
6571 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
6572 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
6573 fcontents.close();
6574 lcfi = lcfi + 1;
6575 reallcfi = reallcfi + 1;
6576 if(lcfx>0):
6577 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
6578 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
6579 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
6580 try:
6581 catfp.flush();
6582 os.fsync(catfp.fileno());
6583 except io.UnsupportedOperation:
6584 pass;
6585 except AttributeError:
6586 pass;
6587 except OSError as e:
6588 pass;
6589 if(outfile=="-"):
6590 catfp.seek(0, 0);
6591 if(hasattr(sys.stdout, "buffer")):
6592 shutil.copyfileobj(catfp, sys.stdout.buffer);
6593 else:
6594 shutil.copyfileobj(catfp, sys.stdout);
6595 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
6596 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
6597 catfp.seek(0, 0);
6598 upload_file_to_internet_file(catfp, outfile);
6599 if(returnfp):
6600 catfp.seek(0, 0);
6601 return catfp;
6602 else:
6603 catfp.close();
6604 return True;
6605
6606 create_alias_function("RePack", __file_format_name__, "", RePackArchiveFile);
6607
6608 def RePackArchiveFileFromString(catstr, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6609 formatspecs = FormatSpecsListToDict(formatspecs);
6610 catfp = BytesIO(catstr);
6611 listcatfiles = RePackArchiveFile(catfp, compression, compresswholefile, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6612 return listcatfiles;
6613
6614 create_alias_function("RePack", __file_format_name__, "FromString", RePackArchiveFileFromString);
6615
6616 def PackArchiveFileFromListDir(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6617 formatspecs = FormatSpecsListToDict(formatspecs);
6618 outarray = BytesIO();
6619 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
6620 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6621 return listcatfiles;
6622
6623 create_alias_function("Pack", __file_format_name__, "FromListDir", PackArchiveFileFromListDir);
6624
6625 def UnPackArchiveFile(infile, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, preservepermissions=True, preservetime=True, verbose=False, returnfp=False):
6626 formatspecs = FormatSpecsListToDict(formatspecs);
6627 if(outdir is not None):
6628 outdir = RemoveWindowsPath(outdir);
6629 if(verbose):
6630 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6631 if(isinstance(infile, dict)):
6632 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6633 listcatfiles = prelistcatfiles['list'];
6634 else:
6635 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6636 infile = RemoveWindowsPath(infile);
6637 if(followlink):
6638 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6639 listcatfiles = prelistcatfiles['list'];
6640 else:
6641 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6642 if(not listcatfiles):
6643 return False;
6644 lenlist = len(listcatfiles['ffilelist']);
6645 fnumfiles = int(listcatfiles['fnumfiles']);
6646 lcfi = 0;
6647 lcfx = int(listcatfiles['fnumfiles']);
6648 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6649 lcfx = int(lenlist);
6650 else:
6651 lcfx = int(listcatfiles['fnumfiles']);
6652 while(lcfi<lcfx):
6653 funame = "";
6654 try:
6655 import pwd;
6656 try:
6657 userinfo = pwd.getpwuid(listcatfiles['ffilelist'][lcfi]['fuid']);
6658 funame = userinfo.pw_name;
6659 except KeyError:
6660 funame = "";
6661 except ImportError:
6662 funame = "";
6663 fgname = "";
6664 try:
6665 import grp;
6666 try:
6667 groupinfo = grp.getgrgid(listcatfiles['ffilelist'][lcfi]['fgid']);
6668 fgname = groupinfo.gr_name;
6669 except KeyError:
6670 fgname = "";
6671 except ImportError:
6672 fgname = "";
6673 if(verbose):
6674 VerbosePrintOut(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6675 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6676 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6677 listcatfiles['ffilelist'][lcfi]['fcontents'].seek(0, 0);
6678 shutil.copyfileobj(listcatfiles['ffilelist'][lcfi]['fcontents'], fpc);
6679 try:
6680 fpc.flush();
6681 os.fsync(fpc.fileno());
6682 except io.UnsupportedOperation:
6683 pass;
6684 except AttributeError:
6685 pass;
6686 except OSError as e:
6687 pass;
6688 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
6689 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
6690 if(preservepermissions):
6691 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6692 if(preservetime):
6693 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
6694 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6695 if(followlink):
6696 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
6697 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6698 flinkinfo = listcatfiles['ffilelist'][flinkid];
6699 funame = "";
6700 try:
6701 import pwd;
6702 try:
6703 userinfo = pwd.getpwuid(flinkinfo['fuid']);
6704 funame = userinfo.pw_name;
6705 except KeyError:
6706 funame = "";
6707 except ImportError:
6708 funame = "";
6709 fgname = "";
6710 try:
6711 import grp;
6712 try:
6713 groupinfo = grp.getgrgid(flinkinfo['fgid']);
6714 fgname = groupinfo.gr_name;
6715 except KeyError:
6716 fgname = "";
6717 except ImportError:
6718 fgname = "";
6719 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
6720 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6721 flinkinfo['fcontents'].seek(0, 0);
6722 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
6723 try:
6724 fpc.flush();
6725 os.fsync(fpc.fileno());
6726 except io.UnsupportedOperation:
6727 pass;
6728 except AttributeError:
6729 pass;
6730 except OSError as e:
6731 pass;
6732 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6733 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6734 if(preservepermissions):
6735 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6736 if(preservetime):
6737 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6738 if(flinkinfo['ftype']==1):
6739 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6740 if(flinkinfo['ftype']==2):
6741 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6742 if(flinkinfo['ftype']==5):
6743 if(preservepermissions):
6744 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6745 else:
6746 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6747 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6748 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6749 if(preservepermissions):
6750 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6751 if(preservetime):
6752 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6753 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
6754 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6755 else:
6756 os.link(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6757 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6758 if(followlink):
6759 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
6760 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6761 flinkinfo = listcatfiles['ffilelist'][flinkid];
6762 funame = "";
6763 try:
6764 import pwd;
6765 try:
6766 userinfo = pwd.getpwuid(flinkinfo['fuid']);
6767 funame = userinfo.pw_name;
6768 except KeyError:
6769 funame = "";
6770 except ImportError:
6771 funame = "";
6772 fgname = "";
6773 try:
6774 import grp;
6775 try:
6776 groupinfo = grp.getgrgid(flinkinfo['fgid']);
6777 fgname = groupinfo.gr_name;
6778 except KeyError:
6779 fgname = "";
6780 except ImportError:
6781 fgname = "";
6782 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
6783 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6784 flinkinfo['fcontents'].seek(0, 0);
6785 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
6786 try:
6787 fpc.flush();
6788 os.fsync(fpc.fileno());
6789 except io.UnsupportedOperation:
6790 pass;
6791 except AttributeError:
6792 pass;
6793 except OSError as e:
6794 pass;
6795 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6796 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6797 if(preservepermissions):
6798 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6799 if(preservetime):
6800 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6801 if(flinkinfo['ftype']==1):
6802 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6803 if(flinkinfo['ftype']==2):
6804 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6805 if(flinkinfo['ftype']==5):
6806 if(preservepermissions):
6807 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6808 else:
6809 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6810 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6811 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6812 if(preservepermissions):
6813 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6814 if(preservetime):
6815 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6816 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
6817 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6818 else:
6819 os.symlink(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6820 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6821 if(preservepermissions):
6822 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6823 else:
6824 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6825 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
6826 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
6827 if(preservepermissions):
6828 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6829 if(preservetime):
6830 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
6831 if(listcatfiles['ffilelist'][lcfi]['ftype']==6 and hasattr(os, "mkfifo")):
6832 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6833 lcfi = lcfi + 1;
6834 if(returnfp):
6835 return listcatfiles['ffilelist']['catfp'];
6836 else:
6837 return True;
6838
6839 create_alias_function("UnPack", __file_format_name__, "", UnPackArchiveFile);
6840
6841 if(hasattr(shutil, "register_unpack_format")):
6842 def UnPackArchiveFileFunc(archive_name, extract_dir=None, **kwargs):
6843 return UnPackArchiveFile(archive_name, extract_dir, False, 0, 0, False, __file_format_dict__['format_delimiter'], False, False);
6844 create_alias_function("UnPack", __file_format_name__, "Func", UnPackArchiveFileFunc);
6845
6846 def UnPackArchiveFileString(catstr, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6847 formatspecs = FormatSpecsListToDict(formatspecs);
6848 catfp = BytesIO(catstr);
6849 listcatfiles = UnPackArchiveFile(catfp, outdir, followlink, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6850 return listcatfiles;
6851
6852 create_alias_function("UnPack", __file_format_name__, "String", UnPackArchiveFileString);
6853
6854 def ArchiveFileListFiles(infile, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6855 formatspecs = FormatSpecsListToDict(formatspecs);
6856 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6857 if(isinstance(infile, dict)):
6858 listcatfiles = infile;
6859 else:
6860 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6861 infile = RemoveWindowsPath(infile);
6862 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, True, skipchecksum, formatspecs, returnfp);
6863 if(not listcatfiles):
6864 return False;
6865 lenlist = len(listcatfiles['ffilelist']);
6866 fnumfiles = int(listcatfiles['fnumfiles']);
6867 lcfi = 0;
6868 lcfx = int(listcatfiles['fnumfiles']);
6869 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6870 lcfx = int(lenlist);
6871 else:
6872 lcfx = int(listcatfiles['fnumfiles']);
6873 returnval = {};
6874 while(lcfi<lcfx):
6875 returnval.update({lcfi: listcatfiles['ffilelist'][lcfi]['fname']});
6876 if(not verbose):
6877 VerbosePrintOut(listcatfiles['ffilelist'][lcfi]['fname']);
6878 if(verbose):
6879 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6880 printfname = listcatfiles['ffilelist'][lcfi]['fname'];
6881 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6882 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " link to " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6883 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6884 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " -> " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6885 fuprint = listcatfiles['ffilelist'][lcfi]['funame'];
6886 if(len(fuprint)<=0):
6887 fuprint = listcatfiles['ffilelist'][lcfi]['fuid'];
6888 fgprint = listcatfiles['ffilelist'][lcfi]['fgname'];
6889 if(len(fgprint)<=0):
6890 fgprint = listcatfiles['ffilelist'][lcfi]['fgid'];
6891 VerbosePrintOut(PrintPermissionString(listcatfiles['ffilelist'][lcfi]['fmode'], listcatfiles['ffilelist'][lcfi]['ftype']) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(listcatfiles['ffilelist'][lcfi]['fsize']).rjust(15) + " " + datetime.datetime.utcfromtimestamp(listcatfiles['ffilelist'][lcfi]['fmtime']).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6892 lcfi = lcfi + 1;
6893 if(returnfp):
6894 return listcatfiles['catfp'];
6895 else:
6896 return True;
6897
6898 create_alias_function("", __file_format_name__, "ListFiles", ArchiveFileListFiles);
6899
6900 def ArchiveFileStringListFiles(catstr, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6901 formatspecs = FormatSpecsListToDict(formatspecs);
6902 catfp = BytesIO(catstr);
6903 listcatfiles = ArchiveFileListFiles(catstr, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6904 return listcatfiles;
6905
6906 create_alias_function("", __file_format_name__, "StringListFiles", ArchiveFileStringListFiles);
6907
6908 def TarFileListFiles(infile, verbose=False, returnfp=False):
6909 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6910 if(infile=="-"):
6911 infile = BytesIO();
6912 if(hasattr(sys.stdin, "buffer")):
6913 shutil.copyfileobj(sys.stdin.buffer, infile);
6914 else:
6915 shutil.copyfileobj(sys.stdin, infile);
6916 infile.seek(0, 0);
6917 if(not infile):
6918 return False;
6919 infile.seek(0, 0);
6920 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
6921 infile = download_file_from_internet_file(infile);
6922 infile.seek(0, 0);
6923 if(not infile):
6924 return False;
6925 infile.seek(0, 0);
6926 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
6927 return False;
6928 elif(os.path.exists(infile) and os.path.isfile(infile)):
6929 try:
6930 if(not tarfile.is_tarfile(infile)):
6931 return False;
6932 except AttributeError:
6933 if(not is_tarfile(infile)):
6934 return False;
6935 else:
6936 return False;
6937 try:
6938 if(hasattr(infile, "read") or hasattr(infile, "write")):
6939 tarfp = tarfile.open(fileobj=infile, mode="r");
6940 else:
6941 tarfp = tarfile.open(infile, "r");
6942 except FileNotFoundError:
6943 return False;
6944 lcfi = 0
6945 returnval = {};
6946 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
6947 returnval.update({lcfi: member.name});
6948 fpremode = member.mode;
6949 ffullmode = member.mode;
6950 flinkcount = 0;
6951 ftype = 0;
6952 if(member.isreg()):
6953 ffullmode = member.mode + stat.S_IFREG;
6954 ftype = 0;
6955 elif(member.isdev()):
6956 ffullmode = member.mode;
6957 ftype = 7;
6958 elif(member.islnk()):
6959 ffullmode = member.mode + stat.S_IFREG;
6960 ftype = 1;
6961 elif(member.issym()):
6962 ffullmode = member.mode + stat.S_IFLNK;
6963 ftype = 2;
6964 elif(member.ischr()):
6965 ffullmode = member.mode + stat.S_IFCHR;
6966 ftype = 3;
6967 elif(member.isblk()):
6968 ffullmode = member.mode + stat.S_IFBLK;
6969 ftype = 4;
6970 elif(member.isdir()):
6971 ffullmode = member.mode + stat.S_IFDIR;
6972 ftype = 5;
6973 elif(member.isfifo()):
6974 ffullmode = member.mode + stat.S_IFIFO;
6975 ftype = 6;
6976 elif(member.issparse()):
6977 ffullmode = member.mode;
6978 ftype = 12;
6979 if(not verbose):
6980 VerbosePrintOut(member.name);
6981 elif(verbose):
6982 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6983 printfname = member.name;
6984 if(member.islnk()):
6985 printfname = member.name + " link to " + member.linkname;
6986 elif(member.issym()):
6987 printfname = member.name + " -> " + member.linkname;
6988 fuprint = member.uname;
6989 if(len(fuprint)<=0):
6990 fuprint = member.uid;
6991 fgprint = member.gname;
6992 if(len(fgprint)<=0):
6993 fgprint = member.gid;
6994 VerbosePrintOut(PrintPermissionString(ffullmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(member.mtime).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6995 lcfi = lcfi + 1;
6996 if(returnfp):
6997 return listcatfiles['catfp'];
6998 else:
6999 return True;
7000
7001 def ZipFileListFiles(infile, verbose=False, returnfp=False):
7002 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7003 if(infile=="-"):
7004 infile = BytesIO();
7005 if(hasattr(sys.stdin, "buffer")):
7006 shutil.copyfileobj(sys.stdin.buffer, infile);
7007 else:
7008 shutil.copyfileobj(sys.stdin, infile);
7009 infile.seek(0, 0);
7010 if(not infile):
7011 return False;
7012 infile.seek(0, 0);
7013 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
7014 infile = download_file_from_internet_file(infile);
7015 infile.seek(0, 0);
7016 if(not infile):
7017 return False;
7018 infile.seek(0, 0);
7019 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
7020 return False;
7021 else:
7022 return False;
7023 if(not zipfile.is_zipfile(infile)):
7024 return False;
7025 try:
7026 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
7027 except FileNotFoundError:
7028 return False;
7029 lcfi = 0;
7030 returnval = {};
7031 ziptest = zipfp.testzip();
7032 if(ziptest):
7033 VerbosePrintOut("Bad file found!");
7034 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
7035 if(zipinfo.create_system==0 or zipinfo.create_system==10):
7036 fwinattributes = int(zipinfo.external_attr);
7037 if(not member.is_dir()):
7038 fmode = int(stat.S_IFREG + 438);
7039 fchmode = int(stat.S_IMODE(fmode));
7040 ftypemod = int(stat.S_IFMT(fmode));
7041 elif(member.is_dir()):
7042 fmode = int(stat.S_IFDIR + 511);
7043 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7044 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7045 elif(zipinfo.create_system==3):
7046 fwinattributes =int(0);
7047 fmode = int(zipinfo.external_attr);
7048 fchmode = int(stat.S_IMODE(fmode));
7049 ftypemod = int(stat.S_IFMT(fmode));
7050 else:
7051 fwinattributes = int(0);
7052 if(not member.is_dir()):
7053 fmode = int(stat.S_IFREG + 438);
7054 fchmode = int(stat.S_IMODE(fmode));
7055 ftypemod = int(stat.S_IFMT(fmode));
7056 elif(member.is_dir()):
7057 fmode = int(stat.S_IFDIR + 511);
7058 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7059 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7060 returnval.update({lcfi: member.filename});
7061 if(not verbose):
7062 VerbosePrintOut(member.filename);
7063 if(verbose):
7064 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7065 permissionstr = "";
7066 for fmodval in str(oct(fmode))[-3:]:
7067 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
7068 if(not member.is_dir()):
7069 ftype = 0;
7070 permissionstr = "-" + permissionstr;
7071 elif(member.is_dir()):
7072 ftype = 5;
7073 permissionstr = "d" + permissionstr;
7074 printfname = member.filename;
7075 try:
7076 fuid = int(os.getuid());
7077 except AttributeError:
7078 fuid = int(0);
7079 except KeyError:
7080 fuid = int(0);
7081 try:
7082 fgid = int(os.getgid());
7083 except AttributeError:
7084 fgid = int(0);
7085 except KeyError:
7086 fgid = int(0);
7087 try:
7088 import pwd;
7089 try:
7090 userinfo = pwd.getpwuid(os.getuid());
7091 funame = userinfo.pw_name;
7092 except KeyError:
7093 funame = "";
7094 except AttributeError:
7095 funame = "";
7096 except ImportError:
7097 funame = "";
7098 fgname = "";
7099 try:
7100 import grp;
7101 try:
7102 groupinfo = grp.getgrgid(os.getgid());
7103 fgname = groupinfo.gr_name;
7104 except KeyError:
7105 fgname = "";
7106 except AttributeError:
7107 fgname = "";
7108 except ImportError:
7109 fgname = "";
7110 fuprint = funame;
7111 if(len(fuprint)<=0):
7112 fuprint = str(fuid);
7113 fgprint = fgname;
7114 if(len(fgprint)<=0):
7115 fgprint = str(fgid);
7116 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(int(time.mktime(member.date_time + (0, 0, -1)))).strftime('%Y-%m-%d %H:%M') + " " + printfname));
7117 lcfi = lcfi + 1;
7118 if(returnfp):
7119 return listcatfiles['catfp'];
7120 else:
7121 return True;
7122
7123 if(not rarfile_support):
7124 def RarFileListFiles(infile, verbose=False, returnfp=False):
7125 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7126 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7127 return False;
7128
7129 if(rarfile_support):
7130 def RarFileListFiles(infile, verbose=False, returnfp=False):
7131 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7132 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7133 return False;
7134 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
7135 return False;
7136 lcfi = 0;
7137 returnval = {};
7138 rarfp = rarfile.RarFile(infile, "r");
7139 rartest = rarfp.testrar();
7140 if(rartest):
7141 VerbosePrintOut("Bad file found!");
7142 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
7143 is_unix = False;
7144 is_windows = False;
7145 if(member.host_os==rarfile.RAR_OS_UNIX):
7146 is_windows = False;
7147 try:
7148 member.external_attr
7149 is_unix = True;
7150 except AttributeError:
7151 is_unix = False;
7152 elif(member.host_os==rarfile.RAR_OS_WIN32):
7153 is_unix = False;
7154 try:
7155 member.external_attr
7156 is_windows = True;
7157 except AttributeError:
7158 is_windows = False;
7159 else:
7160 is_unix = False;
7161 is_windows = False;
7162 if(is_unix and member.external_attr !=0):
7163 fpremode = int(member.external_attr);
7164 elif(member.is_file()):
7165 fpremode = int(stat.S_IFREG + 438);
7166 elif(member.is_symlink()):
7167 fpremode = int(stat.S_IFLNK + 438);
7168 elif(member.is_dir()):
7169 fpremode = int(stat.S_IFDIR + 511);
7170 if(is_windows and member.external_attr !=0):
7171 fwinattributes = int(member.external_attr);
7172 else:
7173 fwinattributes = int(0);
7174 if(is_unix and member.external_attr !=0):
7175 fmode = int(member.external_attr);
7176 fchmode = int(stat.S_IMODE(member.external_attr));
7177 ftypemod = int(stat.S_IFMT(member.external_attr));
7178 elif(member.is_file()):
7179 fmode = int(stat.S_IFREG + 438);
7180 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
7181 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
7182 elif(member.is_symlink()):
7183 fmode = int(stat.S_IFLNK + 438);
7184 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
7185 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
7186 elif(member.is_dir()):
7187 fmode = int(stat.S_IFDIR + 511);
7188 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7189 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7190 returnval.update({lcfi: member.filename});
7191 if(not verbose):
7192 VerbosePrintOut(member.filename);
7193 if(verbose):
7194 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7195 permissionstr = "";
7196 for fmodval in str(oct(fmode))[-3:]:
7197 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
7198 if(member.is_file()):
7199 ftype = 0;
7200 permissionstr = "-" + permissionstr;
7201 printfname = member.filename;
7202 elif(member.is_symlink()):
7203 ftype = 2;
7204 permissionstr = "l" + permissionstr;
7205 printfname = member.name + " -> " + member.read().decode("UTF-8");
7206 elif(member.is_dir()):
7207 ftype = 5;
7208 permissionstr = "d" + permissionstr;
7209 printfname = member.filename;
7210 try:
7211 fuid = int(os.getuid());
7212 except AttributeError:
7213 fuid = int(0);
7214 except KeyError:
7215 fuid = int(0);
7216 try:
7217 fgid = int(os.getgid());
7218 except AttributeError:
7219 fgid = int(0);
7220 except KeyError:
7221 fgid = int(0);
7222 try:
7223 import pwd;
7224 try:
7225 userinfo = pwd.getpwuid(os.getuid());
7226 funame = userinfo.pw_name;
7227 except KeyError:
7228 funame = "";
7229 except AttributeError:
7230 funame = "";
7231 except ImportError:
7232 funame = "";
7233 fgname = "";
7234 try:
7235 import grp;
7236 try:
7237 groupinfo = grp.getgrgid(os.getgid());
7238 fgname = groupinfo.gr_name;
7239 except KeyError:
7240 fgname = "";
7241 except AttributeError:
7242 fgname = "";
7243 except ImportError:
7244 fgname = "";
7245 fuprint = funame;
7246 if(len(fuprint)<=0):
7247 fuprint = str(fuid);
7248 fgprint = fgname;
7249 if(len(fgprint)<=0):
7250 fgprint = str(fgid);
7251 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + member.mtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
7252 lcfi = lcfi + 1;
7253 if(returnfp):
7254 return listcatfiles['catfp'];
7255 else:
7256 return True;
7257
7258 if(not py7zr_support):
7259 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
7260 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7261 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7262 return False;
7263
7264 if(py7zr_support):
7265 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
7266 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7267 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7268 return False;
7269 lcfi = 0;
7270 returnval = {};
7271 szpfp = py7zr.SevenZipFile(infile, mode="r");
7272 file_content = szpfp.readall();
7273 #sztest = szpfp.testzip();
7274 sztestalt = szpfp.test();
7275 if(sztestalt):
7276 VerbosePrintOut("Bad file found!");
7277 for member in sorted(szpfp.list(), key=lambda x: x.filename):
7278 if(re.findall("^[.|/]", member.filename)):
7279 fname = member.filename;
7280 else:
7281 fname = "./"+member.filename;
7282 if(not member.is_directory):
7283 fpremode = int(stat.S_IFREG + 438);
7284 elif(member.is_directory):
7285 fpremode = int(stat.S_IFDIR + 511);
7286 fwinattributes = int(0);
7287 if(member.is_directory):
7288 fmode = int(stat.S_IFDIR + 511);
7289 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7290 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7291 else:
7292 fmode = int(stat.S_IFLNK + 438);
7293 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
7294 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
7295 returnval.update({lcfi: member.filename});
7296 if(not verbose):
7297 VerbosePrintOut(member.filename);
7298 if(verbose):
7299 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7300 permissionstr = "";
7301 for fmodval in str(oct(fmode))[-3:]:
7302 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
7303 fsize = int("0");
7304 if(not member.is_directory):
7305 ftype = 0;
7306 permissionstr = "-" + permissionstr;
7307 printfname = member.filename;
7308 elif(member.is_directory):
7309 ftype = 5;
7310 permissionstr = "d" + permissionstr;
7311 printfname = member.filename;
7312 if(ftype==0):
7313 fsize = len(file_content[member.filename].read());
7314 file_content[member.filename].close();
7315 try:
7316 fuid = int(os.getuid());
7317 except AttributeError:
7318 fuid = int(0);
7319 except KeyError:
7320 fuid = int(0);
7321 try:
7322 fgid = int(os.getgid());
7323 except AttributeError:
7324 fgid = int(0);
7325 except KeyError:
7326 fgid = int(0);
7327 try:
7328 import pwd;
7329 try:
7330 userinfo = pwd.getpwuid(os.getuid());
7331 funame = userinfo.pw_name;
7332 except KeyError:
7333 funame = "";
7334 except AttributeError:
7335 funame = "";
7336 except ImportError:
7337 funame = "";
7338 fgname = "";
7339 try:
7340 import grp;
7341 try:
7342 groupinfo = grp.getgrgid(os.getgid());
7343 fgname = groupinfo.gr_name;
7344 except KeyError:
7345 fgname = "";
7346 except AttributeError:
7347 fgname = "";
7348 except ImportError:
7349 fgname = "";
7350 fuprint = funame;
7351 if(len(fuprint)<=0):
7352 fuprint = str(fuid);
7353 fgprint = fgname;
7354 if(len(fgprint)<=0):
7355 fgprint = str(fgid);
7356 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(fsize).rjust(15) + " " + member.creationtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
7357 lcfi = lcfi + 1;
7358 if(returnfp):
7359 return listcatfiles['catfp'];
7360 else:
7361 return True;
7362
7363 def InFileListFiles(infile, verbose=False, formatspecs=__file_format_dict__, returnfp=False):
7364 formatspecs = FormatSpecsListToDict(formatspecs);
7365 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7366 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
7367 if(checkcompressfile=="tarfile"):
7368 return TarFileListFiles(infile, verbose, returnfp);
7369 elif(checkcompressfile=="zipfile"):
7370 return ZipFileListFiles(infile, verbose, returnfp);
7371 elif(checkcompressfile=="catfile"):
7372 return ArchiveFileListFiles(infile, 0, 0, False, formatspecs, verbose, returnfp);
7373 elif(rarfile_support and checkcompressfile=="rarfile"):
7374 return RarFileListFiles(infile, verbose, returnfp);
7375 elif(py7zr_support and checkcompressfile=="7zipfile"):
7376 return SevenZipFileListFiles(infile, verbose, returnfp);
7377 else:
7378 return False;
7379 return False;
7380
7381 def ListDirListFiles(infiles, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7382 formatspecs = FormatSpecsListToDict(formatspecs);
7383 outarray = BytesIO();
7384 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, False, True);
7385 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
7386 return listcatfiles;
7387
7388 def ListDirListFilesAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7389 formatspecs = FormatSpecsListToDict(formatspecs);
7390 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, listonly, checksumtype, formatspecs, verbose);
7391 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
7392 return listcatfiles;
7393
7394 def PackArchiveFileFromListDirAlt(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7395 formatspecs = FormatSpecsListToDict(formatspecs);
7396 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, False, checksumtype, extradata, formatspecs, False);
7397 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, followlink, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
7398 return listcatfiles;
7399
7400 create_alias_function("Pack", __file_format_name__, "FromListDirAlt", PackArchiveFileFromListDirAlt);
7401
7402 def PackArchiveFileFromTarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7403 formatspecs = FormatSpecsListToDict(formatspecs);
7404 outarray = TarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7405 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7406 return listcatfiles;
7407
7408 create_alias_function("Pack", __file_format_name__, "FromTarFileAlt", PackArchiveFileFromTarFileAlt);
7409
7410 def PackArchiveFileFromZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7411 formatspecs = FormatSpecsListToDict(formatspecs);
7412 outarray = ZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7413 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7414 return listcatfiles;
7415
7416 create_alias_function("Pack", __file_format_name__, "FromZipFileAlt", PackArchiveFileFromZipFileAlt);
7417
7418 if(not rarfile_support):
7419 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7420 return False;
7421
7422 if(rarfile_support):
7423 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7424 formatspecs = FormatSpecsListToDict(formatspecs);
7425 outarray = RarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7426 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7427 return listcatfiles;
7428
7429 create_alias_function("Pack", __file_format_name__, "FromRarFileAlt", PackArchiveFileFromRarFileAlt);
7430
7431 if(not py7zr_support):
7432 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7433 return False;
7434
7435 if(py7zr_support):
7436 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7437 formatspecs = FormatSpecsListToDict(formatspecs);
7438 outarray = SevenZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7439 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7440 return listcatfiles;
7441
7442 create_alias_function("Pack", __file_format_name__, "FromSevenZipFileAlt", PackArchiveFileFromSevenZipFileAlt);
7443
7444 def download_file_from_ftp_file(url):
7445 urlparts = urlparse(url);
7446 file_name = os.path.basename(urlparts.path);
7447 file_dir = os.path.dirname(urlparts.path);
7448 if(urlparts.username is not None):
7449 ftp_username = urlparts.username;
7450 else:
7451 ftp_username = "anonymous";
7452 if(urlparts.password is not None):
7453 ftp_password = urlparts.password;
7454 elif(urlparts.password is None and urlparts.username=="anonymous"):
7455 ftp_password = "anonymous";
7456 else:
7457 ftp_password = "";
7458 if(urlparts.scheme=="ftp"):
7459 ftp = FTP();
7460 elif(urlparts.scheme=="ftps" and ftpssl):
7461 ftp = FTP_TLS();
7462 else:
7463 return False;
7464 if(urlparts.scheme=="sftp"):
7465 if(__use_pysftp__):
7466 return download_file_from_pysftp_file(url);
7467 else:
7468 return download_file_from_sftp_file(url);
7469 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7470 return download_file_from_http_file(url);
7471 ftp_port = urlparts.port;
7472 if(urlparts.port is None):
7473 ftp_port = 21;
7474 try:
7475 ftp.connect(urlparts.hostname, ftp_port);
7476 except socket.gaierror:
7477 log.info("Error With URL "+url);
7478 return False;
7479 except socket.timeout:
7480 log.info("Error With URL "+url);
7481 return False;
7482 ftp.login(urlparts.username, urlparts.password);
7483 if(urlparts.scheme=="ftps"):
7484 ftp.prot_p();
7485 ftpfile = BytesIO();
7486 ftp.retrbinary("RETR "+urlparts.path, ftpfile.write);
7487 #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
7488 ftp.close();
7489 ftpfile.seek(0, 0);
7490 return ftpfile;
7491
7492 def download_file_from_ftp_string(url):
7493 ftpfile = download_file_from_ftp_file(url);
7494 return ftpfile.read();
7495
7496 def upload_file_to_ftp_file(ftpfile, url):
7497 urlparts = urlparse(url);
7498 file_name = os.path.basename(urlparts.path);
7499 file_dir = os.path.dirname(urlparts.path);
7500 if(urlparts.username is not None):
7501 ftp_username = urlparts.username;
7502 else:
7503 ftp_username = "anonymous";
7504 if(urlparts.password is not None):
7505 ftp_password = urlparts.password;
7506 elif(urlparts.password is None and urlparts.username=="anonymous"):
7507 ftp_password = "anonymous";
7508 else:
7509 ftp_password = "";
7510 if(urlparts.scheme=="ftp"):
7511 ftp = FTP();
7512 elif(urlparts.scheme=="ftps" and ftpssl):
7513 ftp = FTP_TLS();
7514 else:
7515 return False;
7516 if(urlparts.scheme=="sftp"):
7517 if(__use_pysftp__):
7518 return upload_file_to_pysftp_file(url);
7519 else:
7520 return upload_file_to_sftp_file(url);
7521 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7522 return False;
7523 ftp_port = urlparts.port;
7524 if(urlparts.port is None):
7525 ftp_port = 21;
7526 try:
7527 ftp.connect(urlparts.hostname, ftp_port);
7528 except socket.gaierror:
7529 log.info("Error With URL "+url);
7530 return False;
7531 except socket.timeout:
7532 log.info("Error With URL "+url);
7533 return False;
7534 ftp.login(urlparts.username, urlparts.password);
7535 if(urlparts.scheme=="ftps"):
7536 ftp.prot_p();
7537 ftp.storbinary("STOR "+urlparts.path, ftpfile);
7538 ftp.close();
7539 ftpfile.seek(0, 0);
7540 return ftpfile;
7541
7542 def upload_file_to_ftp_string(ftpstring, url):
7543 ftpfileo = BytesIO(ftpstring);
7544 ftpfile = upload_file_to_ftp_file(ftpfileo, url);
7545 ftpfileo.close();
7546 return ftpfile;
7547
7548 def download_file_from_http_file(url, headers=geturls_headers_pycatfile_python_alt):
7549 # Parse the URL to extract username and password if present
7550 urlparts = urlparse(url);
7551 username = urlparts.username;
7552 password = urlparts.password;
7553 # Rebuild the URL without the username and password
7554 netloc = urlparts.hostname;
7555 if(urlparts.scheme=="sftp"):
7556 if(__use_pysftp__):
7557 return download_file_from_pysftp_file(url);
7558 else:
7559 return download_file_from_sftp_file(url);
7560 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7561 return download_file_from_ftp_file(url);
7562 if urlparts.port:
7563 netloc += ':' + str(urlparts.port);
7564 rebuilt_url = urlunparse((urlparts.scheme, netloc, urlparts.path, urlparts.params, urlparts.query, urlparts.fragment));
7565 # Create a temporary file object
7566 httpfile = BytesIO();
7567 if haverequests:
7568 # Use the requests library if available
7569 if username and password:
7570 response = requests.get(rebuilt_url, headers=headers, auth=(username, password), stream=True);
7571 else:
7572 response = requests.get(rebuilt_url, headers=headers, stream=True);
7573 response.raw.decode_content = True
7574 shutil.copyfileobj(response.raw, httpfile);
7575 else:
7576 # Build a Request object for urllib
7577 request = Request(rebuilt_url, headers=headers);
7578 # Create an opener object for handling URLs
7579 if username and password:
7580 # Create a password manager
7581 password_mgr = HTTPPasswordMgrWithDefaultRealm();
7582 # Add the username and password
7583 password_mgr.add_password(None, rebuilt_url, username, password);
7584 # Create an authentication handler using the password manager
7585 auth_handler = HTTPBasicAuthHandler(password_mgr);
7586 # Build the opener with the authentication handler
7587 opener = build_opener(auth_handler);
7588 else:
7589 opener = build_opener();
7590 with opener.open(request) as response:
7591 shutil.copyfileobj(response, httpfile);
7592 # Reset file pointer to the start
7593 httpfile.seek(0, 0);
7594 # Return the temporary file object
7595 return httpfile;
7596
7597 def download_file_from_http_string(url, headers=geturls_headers_pycatfile_python_alt):
7598 httpfile = download_file_from_http_file(url, headers);
7599 return ftpfile.read();
7600
7601 if(haveparamiko):
7602 def download_file_from_sftp_file(url):
7603 urlparts = urlparse(url);
7604 file_name = os.path.basename(urlparts.path);
7605 file_dir = os.path.dirname(urlparts.path);
7606 sftp_port = urlparts.port;
7607 if(urlparts.port is None):
7608 sftp_port = 22;
7609 else:
7610 sftp_port = urlparts.port;
7611 if(urlparts.username is not None):
7612 sftp_username = urlparts.username;
7613 else:
7614 sftp_username = "anonymous";
7615 if(urlparts.password is not None):
7616 sftp_password = urlparts.password;
7617 elif(urlparts.password is None and urlparts.username=="anonymous"):
7618 sftp_password = "anonymous";
7619 else:
7620 sftp_password = "";
7621 if(urlparts.scheme=="ftp"):
7622 return download_file_from_ftp_file(url);
7623 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7624 return download_file_from_http_file(url);
7625 if(urlparts.scheme!="sftp"):
7626 return False;
7627 ssh = paramiko.SSHClient();
7628 ssh.load_system_host_keys();
7629 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
7630 try:
7631 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7632 except paramiko.ssh_exception.SSHException:
7633 return False;
7634 except socket.gaierror:
7635 log.info("Error With URL "+url);
7636 return False;
7637 except socket.timeout:
7638 log.info("Error With URL "+url);
7639 return False;
7640 sftp = ssh.open_sftp();
7641 sftpfile = BytesIO();
7642 sftp.getfo(urlparts.path, sftpfile);
7643 sftp.close();
7644 ssh.close();
7645 sftpfile.seek(0, 0);
7646 return sftpfile;
7647 else:
7648 def download_file_from_sftp_file(url):
7649 return False;
7650
7651 if(haveparamiko):
7652 def download_file_from_sftp_string(url):
7653 sftpfile = download_file_from_sftp_file(url);
7654 return sftpfile.read();
7655 else:
7656 def download_file_from_ftp_string(url):
7657 return False;
7658
7659 if(haveparamiko):
7660 def upload_file_to_sftp_file(sftpfile, url):
7661 urlparts = urlparse(url);
7662 file_name = os.path.basename(urlparts.path);
7663 file_dir = os.path.dirname(urlparts.path);
7664 sftp_port = urlparts.port;
7665 if(urlparts.port is None):
7666 sftp_port = 22;
7667 else:
7668 sftp_port = urlparts.port;
7669 if(urlparts.username is not None):
7670 sftp_username = urlparts.username;
7671 else:
7672 sftp_username = "anonymous";
7673 if(urlparts.password is not None):
7674 sftp_password = urlparts.password;
7675 elif(urlparts.password is None and urlparts.username=="anonymous"):
7676 sftp_password = "anonymous";
7677 else:
7678 sftp_password = "";
7679 if(urlparts.scheme=="ftp"):
7680 return upload_file_to_ftp_file(url);
7681 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7682 return False;
7683 if(urlparts.scheme!="sftp"):
7684 return False;
7685 ssh = paramiko.SSHClient();
7686 ssh.load_system_host_keys();
7687 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
7688 try:
7689 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7690 except paramiko.ssh_exception.SSHException:
7691 return False;
7692 except socket.gaierror:
7693 log.info("Error With URL "+url);
7694 return False;
7695 except socket.timeout:
7696 log.info("Error With URL "+url);
7697 return False;
7698 sftp = ssh.open_sftp();
7699 sftp.putfo(sftpfile, urlparts.path);
7700 sftp.close();
7701 ssh.close();
7702 sftpfile.seek(0, 0);
7703 return sftpfile;
7704 else:
7705 def upload_file_to_sftp_file(sftpfile, url):
7706 return False;
7707
7708 if(haveparamiko):
7709 def upload_file_to_sftp_string(sftpstring, url):
7710 sftpfileo = BytesIO(sftpstring);
7711 sftpfile = upload_file_to_sftp_files(ftpfileo, url);
7712 sftpfileo.close();
7713 return sftpfile;
7714 else:
7715 def upload_file_to_sftp_string(url):
7716 return False;
7717
7718 if(havepysftp):
7719 def download_file_from_pysftp_file(url):
7720 urlparts = urlparse(url);
7721 file_name = os.path.basename(urlparts.path);
7722 file_dir = os.path.dirname(urlparts.path);
7723 sftp_port = urlparts.port;
7724 if(urlparts.port is None):
7725 sftp_port = 22;
7726 else:
7727 sftp_port = urlparts.port;
7728 if(urlparts.username is not None):
7729 sftp_username = urlparts.username;
7730 else:
7731 sftp_username = "anonymous";
7732 if(urlparts.password is not None):
7733 sftp_password = urlparts.password;
7734 elif(urlparts.password is None and urlparts.username=="anonymous"):
7735 sftp_password = "anonymous";
7736 else:
7737 sftp_password = "";
7738 if(urlparts.scheme=="ftp"):
7739 return download_file_from_ftp_file(url);
7740 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7741 return download_file_from_http_file(url);
7742 if(urlparts.scheme!="sftp"):
7743 return False;
7744 try:
7745 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7746 except paramiko.ssh_exception.SSHException:
7747 return False;
7748 except socket.gaierror:
7749 log.info("Error With URL "+url);
7750 return False;
7751 except socket.timeout:
7752 log.info("Error With URL "+url);
7753 return False;
7754 sftp = ssh.open_sftp();
7755 sftpfile = BytesIO();
7756 sftp.getfo(urlparts.path, sftpfile);
7757 sftp.close();
7758 ssh.close();
7759 sftpfile.seek(0, 0);
7760 return sftpfile;
7761 else:
7762 def download_file_from_pysftp_file(url):
7763 return False;
7764
7765 if(havepysftp):
7766 def download_file_from_pysftp_string(url):
7767 sftpfile = download_file_from_pysftp_file(url);
7768 return sftpfile.read();
7769 else:
7770 def download_file_from_ftp_string(url):
7771 return False;
7772
7773 if(havepysftp):
7774 def upload_file_to_pysftp_file(sftpfile, url):
7775 urlparts = urlparse(url);
7776 file_name = os.path.basename(urlparts.path);
7777 file_dir = os.path.dirname(urlparts.path);
7778 sftp_port = urlparts.port;
7779 if(urlparts.port is None):
7780 sftp_port = 22;
7781 else:
7782 sftp_port = urlparts.port;
7783 if(urlparts.username is not None):
7784 sftp_username = urlparts.username;
7785 else:
7786 sftp_username = "anonymous";
7787 if(urlparts.password is not None):
7788 sftp_password = urlparts.password;
7789 elif(urlparts.password is None and urlparts.username=="anonymous"):
7790 sftp_password = "anonymous";
7791 else:
7792 sftp_password = "";
7793 if(urlparts.scheme=="ftp"):
7794 return upload_file_to_ftp_file(url);
7795 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7796 return False;
7797 if(urlparts.scheme!="sftp"):
7798 return False;
7799 try:
7800 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7801 except paramiko.ssh_exception.SSHException:
7802 return False;
7803 except socket.gaierror:
7804 log.info("Error With URL "+url);
7805 return False;
7806 except socket.timeout:
7807 log.info("Error With URL "+url);
7808 return False;
7809 sftp = ssh.open_sftp();
7810 sftp.putfo(sftpfile, urlparts.path);
7811 sftp.close();
7812 ssh.close();
7813 sftpfile.seek(0, 0);
7814 return sftpfile;
7815 else:
7816 def upload_file_to_pysftp_file(sftpfile, url):
7817 return False;
7818
7819 if(havepysftp):
7820 def upload_file_to_pysftp_string(sftpstring, url):
7821 sftpfileo = BytesIO(sftpstring);
7822 sftpfile = upload_file_to_pysftp_files(ftpfileo, url);
7823 sftpfileo.close();
7824 return sftpfile;
7825 else:
7826 def upload_file_to_pysftp_string(url):
7827 return False;
7828
7829 def download_file_from_internet_file(url, headers=geturls_headers_pycatfile_python_alt):
7830 urlparts = urlparse(url);
7831 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7832 return download_file_from_http_file(url, headers);
7833 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7834 return download_file_from_ftp_file(url);
7835 elif(urlparts.scheme=="sftp"):
7836 if(__use_pysftp__ and havepysftp):
7837 return download_file_from_pysftp_file(url);
7838 else:
7839 return download_file_from_sftp_file(url);
7840 else:
7841 return False;
7842 return False;
7843
7844 def download_file_from_internet_uncompress_file(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_dict__):
7845 formatspecs = FormatSpecsListToDict(formatspecs);
7846 fp = download_file_from_internet_file(url);
7847 fp = UncompressArchiveFile(fp, formatspecs);
7848 fp.seek(0, 0);
7849 if(not fp):
7850 return False;
7851 return fp;
7852
7853 def download_file_from_internet_string(url, headers=geturls_headers_pycatfile_python_alt):
7854 urlparts = urlparse(url);
7855 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7856 return download_file_from_http_string(url, headers);
7857 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7858 return download_file_from_ftp_string(url);
7859 elif(urlparts.scheme=="sftp"):
7860 if(__use_pysftp__ and havepysftp):
7861 return download_file_from_pysftp_string(url);
7862 else:
7863 return download_file_from_sftp_string(url);
7864 else:
7865 return False;
7866 return False;
7867
7868 def download_file_from_internet_uncompress_string(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_dict__):
7869 formatspecs = FormatSpecsListToDict(formatspecs);
7870 fp = download_file_from_internet_string(url);
7871 fp = UncompressArchiveFile(fp, formatspecs);
7872 fp.seek(0, 0);
7873 if(not fp):
7874 return False;
7875 return fp;
7876
7877 def upload_file_to_internet_file(ifp, url):
7878 urlparts = urlparse(url);
7879 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7880 return False;
7881 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7882 return upload_file_to_ftp_file(ifp, url);
7883 elif(urlparts.scheme=="sftp"):
7884 if(__use_pysftp__ and havepysftp):
7885 return upload_file_to_pysftp_file(ifp, url);
7886 else:
7887 return upload_file_to_sftp_file(ifp, url);
7888 else:
7889 return False;
7890 return False;
7891
7892 def upload_file_to_internet_compress_file(ifp, url, formatspecs=__file_format_dict__):
7893 formatspecs = FormatSpecsListToDict(formatspecs);
7894 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
7895 if(not catfileout):
7896 return False;
7897 catfp.seek(0, 0);
7898 upload_file_to_internet_file(catfp, outfile);
7899 return True;
7900
7901 def upload_file_to_internet_string(ifp, url):
7902 urlparts = urlparse(url);
7903 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7904 return False;
7905 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7906 return upload_file_to_ftp_string(ifp, url);
7907 elif(urlparts.scheme=="sftp"):
7908 if(__use_pysftp__ and havepysftp):
7909 return upload_file_to_pysftp_string(ifp, url);
7910 else:
7911 return upload_file_to_sftp_string(ifp, url);
7912 else:
7913 return False;
7914 return False;
7915
7916 def upload_file_to_internet_compress_string(ifp, url, formatspecs=__file_format_dict__):
7917 formatspecs = FormatSpecsListToDict(formatspecs);
7918 catfp = CompressArchiveFile(BytesIO(ifp), compression, compressionlevel, formatspecs);
7919 if(not catfileout):
7920 return False;
7921 catfp.seek(0, 0);
7922 upload_file_to_internet_file(catfp, outfile);
7923 return True;
7924
7925 try:
7926 if(hasattr(shutil, "register_archive_format")):
7927 # Register the packing format
7928 shutil.register_archive_format(__file_format_name__, PackArchiveFileFunc, description='Pack concatenated files');
7929 except shutil.RegistryError:
7930 pass;
7931
7932 try:
7933 if(hasattr(shutil, "register_unpack_format")):
7934 # Register the unpacking format
7935 shutil.register_unpack_format(__file_format_name__, archivefile_extensions, UnPackArchiveFileFunc, description='UnPack concatenated files');
7936 except shutil.RegistryError:
7937 pass;
A tar like file format name catfile after unix cat command (concatenate files).