?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Add files via upload
[PyCatFile.git] / pycatfile.py
blobc8b711f9bb3e3e16a5dca61a11b89f9fde0f996f
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 '''
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the Revised BSD License.
7
8 This program is distributed in the hope that it will be useful,
9 but WITHOUT ANY WARRANTY; without even the implied warranty of
10 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 Revised BSD License for more details.
12
13 Copyright 2018-2024 Cool Dude 2k - http://idb.berlios.de/
14 Copyright 2018-2024 Game Maker 2k - http://intdb.sourceforge.net/
15 Copyright 2018-2024 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
16
17 $FileInfo: pycatfile.py - Last Update: 5/12/2024 Ver. 0.11.0 RC 1 - Author: cooldude2k $
18 '''
19
20 from __future__ import absolute_import, division, print_function, unicode_literals;
21 import io, os, re, sys, time, stat, zlib, base64, shutil, socket, hashlib, datetime, logging, binascii, tempfile, zipfile, platform;
22 ftpssl = True;
23 try:
24 ftpssl = True;
25 from ftplib import FTP, FTP_TLS;
26 except ImportError:
27 ftpssl = False;
28 from ftplib import FTP;
29 if(sys.version[0]=="2"):
30 from urlparse import urlparse, urlunparse;
31 elif(sys.version[0]>="3"):
32 from urllib.parse import urlunparse;
33 from urllib.parse import urlparse;
34
35 if os.name == 'nt': # Only modify if on Windows
36 if sys.version[0] == "2":
37 import codecs;
38 sys.stdout = codecs.getwriter('utf-8')(sys.stdout);
39 sys.stderr = codecs.getwriter('utf-8')(sys.stderr);
40 else:
41 import io;
42 sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace', line_buffering=True);
43 sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', errors='replace', line_buffering=True);
44
45 hashlib_guaranteed = False;
46 os.environ["PYTHONIOENCODING"] = "UTF-8";
47 os.environ["LC_CTYPE"] = "UTF-8";
48 try:
49 reload(sys);
50 try:
51 sys.setdefaultencoding('UTF-8');
52 except NameError:
53 pass;
54 except AttributeError:
55 pass;
56 except NameError:
57 pass;
58 except AttributeError:
59 pass;
60
61 try:
62 from zlib import crc32;
63 except ImportError:
64 from binascii import crc32;
65
66 if(sys.version_info[0]==2):
67 FileNotFoundError = IOError;
68
69 rarfile_support = False;
70 try:
71 import rarfile;
72 rarfile_support = True;
73 except ImportError:
74 rarfile_support = False;
75
76 py7zr_support = False;
77 try:
78 import py7zr;
79 py7zr_support = True;
80 except ImportError:
81 py7zr_support = False;
82
83 try:
84 from xtarfile import is_tarfile;
85 except ImportError:
86 try:
87 from safetar import is_tarfile;
88 except ImportError:
89 from tarfile import is_tarfile;
90
91 try:
92 import xtarfile as tarfile;
93 except ImportError:
94 try:
95 import safetar as tarfile;
96 except ImportError:
97 import tarfile;
98
99 haveparamiko = False;
100 try:
101 import paramiko;
102 haveparamiko = True;
103 except ImportError:
104 haveparamiko = False;
105
106 havepysftp = False;
107 try:
108 import pysftp;
109 havepysftp = True;
110 except ImportError:
111 havepysftp = False;
112
113 haverequests = False;
114 try:
115 import requests;
116 haverequests = True;
117 except ImportError:
118 haverequests = False;
119
120 if(haverequests):
121 import urllib3;
122 logging.getLogger("urllib3").setLevel(logging.WARNING);
123
124 try:
125 # Python 3 imports
126 from urllib.request import Request, build_opener, HTTPBasicAuthHandler;
127 from urllib.parse import urlparse;
128 except ImportError:
129 # Python 2 imports
130 from urllib2 import Request, build_opener, HTTPBasicAuthHandler;
131 from urlparse import urlparse;
132
133 if(sys.version[0]=="2"):
134 try:
135 from io import StringIO, BytesIO;
136 except ImportError:
137 try:
138 from cStringIO import StringIO;
139 from cStringIO import StringIO as BytesIO;
140 except ImportError:
141 from StringIO import StringIO;
142 from StringIO import StringIO as BytesIO;
143 elif(sys.version[0]>="3"):
144 from io import StringIO, BytesIO;
145 else:
146 teststringio = 0;
147 if(teststringio<=0):
148 try:
149 from cStringIO import StringIO as BytesIO;
150 teststringio = 1;
151 except ImportError:
152 teststringio = 0;
153 if(teststringio<=0):
154 try:
155 from StringIO import StringIO as BytesIO;
156 teststringio = 2;
157 except ImportError:
158 teststringio = 0;
159 if(teststringio<=0):
160 try:
161 from io import BytesIO;
162 teststringio = 3;
163 except ImportError:
164 teststringio = 0;
165
166 __use_pysftp__ = False;
167 if(not havepysftp):
168 __use_pysftp__ = False;
169 __file_format_name__ = "CatFile";
170 __program_name__ = "Py"+__file_format_name__;
171 __file_format_lower__ = __file_format_name__.lower();
172 __file_format_magic__ = __file_format_name__;
173 __file_format_len__ = len(__file_format_magic__);
174 __file_format_hex__ = binascii.hexlify(__file_format_magic__.encode("UTF-8")).decode("UTF-8");
175 __file_format_delimiter__ = "\x00";
176 __file_format_ver__ = "001";
177 __use_new_style__ = True;
178 __use_advanced_list__ = True;
179 __use_alt_inode__ = False;
180 __file_format_list__ = [__file_format_name__, __file_format_magic__, __file_format_lower__, __file_format_len__, __file_format_hex__, __file_format_delimiter__, __file_format_ver__, __use_new_style__, __use_advanced_list__, __use_alt_inode__];
181 __file_format_dict__ = {'format_name': __file_format_name__, 'format_magic': __file_format_magic__, 'format_lower': __file_format_lower__, 'format_len': __file_format_len__, 'format_hex': __file_format_hex__, 'format_delimiter': __file_format_delimiter__, 'format_ver': __file_format_ver__, 'new_style': __use_new_style__, 'use_advanced_list': __use_advanced_list__, 'use_alt_inode': __use_alt_inode__};
182 __project__ = __program_name__;
183 __project_url__ = "https://github.com/GameMaker2k/PyCatFile";
184 __version_info__ = (0, 11, 0, "RC 1", 1);
185 __version_date_info__ = (2024, 5, 12, "RC 1", 1);
186 __version_date__ = str(__version_date_info__[0]) + "." + str(__version_date_info__[1]).zfill(2) + "." + str(__version_date_info__[2]).zfill(2);
187 __revision__ = __version_info__[3];
188 __revision_id__ = "$Id$";
189 if(__version_info__[4] is not None):
190 __version_date_plusrc__ = __version_date__ + "-" + str(__version_date_info__[4]);
191 if(__version_info__[4] is None):
192 __version_date_plusrc__ = __version_date__;
193 if(__version_info__[3] is not None):
194 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]) + " " + str(__version_info__[3]);
195 if(__version_info__[3] is None):
196 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]);
197
198 PyBitness = platform.architecture();
199 if(PyBitness=="32bit" or PyBitness=="32"):
200 PyBitness = "32";
201 elif(PyBitness=="64bit" or PyBitness=="64"):
202 PyBitness = "64";
203 else:
204 PyBitness = "32";
205
206 geturls_ua_pycatfile_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
207 if(platform.python_implementation()!=""):
208 py_implementation = platform.python_implementation();
209 if(platform.python_implementation()==""):
210 py_implementation = "Python";
211 geturls_ua_pycatfile_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=py_implementation, pyver=platform.python_version(), proname=__project__, prover=__version__);
212 geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
213 geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
214 geturls_headers_pycatfile_python = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
215 geturls_headers_pycatfile_python_alt = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python_alt, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
216 geturls_headers_googlebot_google = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
217 geturls_headers_googlebot_google_old = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
218
219 compressionlist = ['auto', 'gzip', 'bzip2', 'zstd', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
220 compressionlistalt = ['gzip', 'bzip2', 'zstd', 'lz4', 'lzo', 'lzma', 'xz'];
221 outextlist = ['gz', 'bz2', 'zst', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
222 outextlistwd = ['.gz', '.bz2', '.zst', '.lz4', '.lzo', '.lzop', '.lzma', '.xz'];
223
224 tarfile_mimetype = "application/tar";
225 tarfile_tar_mimetype = tarfile_mimetype;
226 zipfile_mimetype = "application/zip";
227 zipfile_zip_mimetype = zipfile_mimetype;
228 rarfile_mimetype = "application/rar";
229 rarfile_rar_mimetype = rarfile_mimetype;
230 archivefile_mimetype = "application/x-"+__file_format_dict__['format_lower']+"";
231 archivefile_cat_mimetype = archivefile_mimetype;
232 archivefile_gzip_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+gzip";
233 archivefile_gz_mimetype = archivefile_gzip_mimetype;
234 archivefile_bzip2_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+bzip2";
235 archivefile_bz2_mimetype = archivefile_bzip2_mimetype;
236 archivefile_lz4_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lz4";
237 archivefile_lzop_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lzop";
238 archivefile_lzo_mimetype = archivefile_lzop_mimetype;
239 archivefile_zstandard_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+zstandard";
240 archivefile_zstd_mimetype = archivefile_zstandard_mimetype;
241 archivefile_lzma_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lzma";
242 archivefile_xz_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+xz";
243 archivefile_extensions = ['.cat', '.cat.gz', '.cat.bz2', '.cat.zst', '.cat.lz4', '.cat.lzo', '.cat.lzop', '.cat.lzma', '.cat.xz'];
244
245 if __name__ == "__main__":
246 import subprocess;
247 curscrpath = os.path.dirname(sys.argv[0]);
248 if(curscrpath==""):
249 curscrpath = ".";
250 if(os.sep=="\\"):
251 curscrpath = curscrpath.replace(os.sep, "/");
252 curscrpath = curscrpath + "/";
253 scrfile = curscrpath + "catfile.py";
254 if(os.path.exists(scrfile) and os.path.isfile(scrfile)):
255 scrcmd = subprocess.Popen([sys.executable, scrfile] + sys.argv[1:]);
256 scrcmd.wait();
257
258 def VerbosePrintOut(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
259 if(not dbgenable):
260 return True;
261 log_functions = {
262 "print": print,
263 "log": logging.info,
264 "warning": logging.warning,
265 "error": logging.error,
266 "critical": logging.critical,
267 "exception": logging.exception,
268 "logalt": lambda x: logging.log(dgblevel, x),
269 "debug": logging.debug
270 };
271 log_function = log_functions.get(outtype);
272 if(log_function):
273 log_function(dbgtxt);
274 return True;
275 return False;
276
277 def VerbosePrintOutReturn(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
278 VerbosePrintOut(dbgtxt, outtype, dbgenable, dgblevel);
279 return dbgtxt;
280
281 def RemoveWindowsPath(dpath):
282 if(dpath is None):
283 dpath = "";
284 if(os.sep!="/"):
285 dpath = dpath.replace(os.path.sep, "/");
286 dpath = dpath.rstrip("/");
287 if(dpath=="." or dpath==".."):
288 dpath = dpath + "/";
289 return dpath;
290
291 def NormalizeRelativePath(inpath):
292 inpath = RemoveWindowsPath(inpath);
293 if(os.path.isabs(inpath)):
294 outpath = inpath;
295 else:
296 if(inpath.startswith("./") or inpath.startswith("../")):
297 outpath = inpath;
298 else:
299 outpath = "./" + inpath;
300 return outpath;
301
302 def PrependPath(base_dir, child_path):
303 # Check if base_dir is None or empty, if so, return child_path as is
304 if not base_dir:
305 return child_path;
306 # Ensure base_dir ends with exactly one slash
307 if not base_dir.endswith('/'):
308 base_dir += '/';
309 # Check if child_path starts with ./ or ../ (indicating a relative path)
310 if child_path.startswith('./') or child_path.startswith('../'):
311 # For relative paths, we don't alter the child_path
312 return base_dir + child_path;
313 else:
314 # For non-relative paths, ensure there's no starting slash on child_path to avoid double slashes
315 return base_dir + child_path.lstrip('/');
316
317 def ListDir(dirpath, followlink=False, duplicates=False):
318 if(isinstance(dirpath, (list, tuple, ))):
319 dirpath = list(filter(None, dirpath));
320 elif(isinstance(dirpath, (str, ))):
321 dirpath = list(filter(None, [dirpath]));
322 retlist = [];
323 for mydirfile in dirpath:
324 if(not os.path.exists(mydirfile)):
325 return False;
326 mydirfile = NormalizeRelativePath(mydirfile);
327 if(os.path.exists(mydirfile) and os.path.islink(mydirfile)):
328 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile));
329 if(os.path.exists(mydirfile) and os.path.isdir(mydirfile)):
330 for root, dirs, filenames in os.walk(mydirfile):
331 dpath = root;
332 dpath = RemoveWindowsPath(dpath);
333 if(dpath not in retlist and not duplicates):
334 retlist.append(dpath);
335 if(duplicates):
336 retlist.append(dpath);
337 for file in filenames:
338 fpath = os.path.join(root, file);
339 fpath = RemoveWindowsPath(fpath);
340 if(fpath not in retlist and not duplicates):
341 retlist.append(fpath);
342 if(duplicates):
343 retlist.append(fpath);
344 else:
345 retlist.append(RemoveWindowsPath(mydirfile));
346 return retlist;
347
348 def ListDirAdvanced(dirpath, followlink=False, duplicates=False):
349 if isinstance(dirpath, (list, tuple)):
350 dirpath = list(filter(None, dirpath));
351 elif isinstance(dirpath, str):
352 dirpath = list(filter(None, [dirpath]));
353 retlist = []
354 for mydirfile in dirpath:
355 if not os.path.exists(mydirfile):
356 return False;
357 mydirfile = NormalizeRelativePath(mydirfile);
358 if os.path.exists(mydirfile) and os.path.islink(mydirfile) and followlink:
359 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile))
360 if os.path.exists(mydirfile) and os.path.isdir(mydirfile):
361 for root, dirs, filenames in os.walk(mydirfile):
362 # Sort dirs and filenames alphabetically in place
363 dirs.sort(key=lambda x: x.lower());
364 filenames.sort(key=lambda x: x.lower());
365 dpath = RemoveWindowsPath(root);
366 if not duplicates and dpath not in retlist:
367 retlist.append(dpath);
368 elif duplicates:
369 retlist.append(dpath);
370 for file in filenames:
371 fpath = os.path.join(root, file);
372 fpath = RemoveWindowsPath(fpath);
373 if not duplicates and fpath not in retlist:
374 retlist.append(fpath);
375 elif duplicates:
376 retlist.append(fpath);
377 else:
378 retlist.append(RemoveWindowsPath(mydirfile));
379 return retlist;
380
381 def create_alias_function(prefix, base_name, suffix, target_function):
382 # Define a new function that wraps the target function
383 def alias_function(*args, **kwargs):
384 return target_function(*args, **kwargs);
385
386 # Create the function name by combining the prefix, base name, and the suffix
387 function_name = "{}{}{}".format(prefix, base_name, suffix);
388
389 # Add the new function to the global namespace
390 globals()[function_name] = alias_function;
391
392 def create_alias_function_alt(prefix, base_name, suffix, target_function):
393 # Create the function name by combining the prefix, base name, and the suffix
394 # Use the format method for string formatting, compatible with Python 2 and 3
395 function_name = "{}{}{}".format(prefix, base_name, suffix)
396 # Add the new function (alias of the target_function) to the global namespace
397 # This line is compatible as-is with both Python 2 and 3
398 globals()[function_name] = target_function
399
400 def FormatSpecsListToDict(formatspecs=__file_format_list__):
401 if(isinstance(formatspecs, (list, tuple, ))):
402 return {'format_name': formatspecs[0], 'format_magic': formatspecs[1], 'format_lower': formatspecs[2], 'format_len': formatspecs[3], 'format_hex': formatspecs[4], 'format_delimiter': formatspecs[5], 'format_ver': formatspecs[6], 'new_style': formatspecs[7], 'use_advanced_list': formatspecs[8], 'use_alt_inode': formatspecs[9]};
403 elif(isinstance(formatspecs, (dict, ))):
404 return formatspecs;
405 else:
406 return __file_format_dict__;
407 return __file_format_dict__;
408
409 # initial_value can be 0xFFFF or 0x0000
410 def crc16_ansi(msg, initial_value=0xFFFF):
411 # CRC-16-IBM / CRC-16-ANSI polynomial and initial value
412 poly = 0x8005; # Polynomial for CRC-16-IBM / CRC-16-ANSI
413 crc = initial_value; # Initial value
414 for b in msg:
415 crc ^= b << 8; # XOR byte into CRC top byte
416 for _ in range(8): # Process each bit
417 if crc & 0x8000: # If the top bit is set
418 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
419 else:
420 crc = crc << 1; # Just shift left
421 crc &= 0xFFFF; # Ensure CRC remains 16-bit
422 return crc;
423
424 # initial_value can be 0xFFFF or 0x0000
425 def crc16_ibm(msg, initial_value=0xFFFF):
426 return crc16_ansi(msg, initial_value);
427
428 # initial_value is 0xFFFF
429 def crc16(msg):
430 return crc16_ansi(msg, 0xFFFF);
431
432 # initial_value can be 0xFFFF, 0x1D0F or 0x0000
433 def crc16_ccitt(msg, initial_value=0xFFFF):
434 # CRC-16-CCITT polynomial
435 poly = 0x1021; # Polynomial for CRC-16-CCITT
436 # Use the specified initial value
437 crc = initial_value;
438 for b in msg:
439 crc ^= b << 8; # XOR byte into CRC top byte
440 for _ in range(8): # Process each bit
441 if crc & 0x8000: # If the top bit is set
442 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
443 else:
444 crc = crc << 1; # Just shift left
445 crc &= 0xFFFF; # Ensure CRC remains 16-bit
446 return crc;
447
448 # initial_value can be 0x42F0E1EBA9EA3693 or 0x0000000000000000
449 def crc64_ecma(msg, initial_value=0x0000000000000000):
450 # CRC-64-ECMA polynomial and initial value
451 poly = 0x42F0E1EBA9EA3693;
452 crc = initial_value; # Initial value for CRC-64-ECMA
453 for b in msg:
454 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
455 for _ in range(8): # Process each bit
456 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
457 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
458 else:
459 crc <<= 1; # Just shift left if the MSB is 0
460 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
461 return crc;
462
463 # initial_value can be 0x000000000000001B or 0xFFFFFFFFFFFFFFFF
464 def crc64_iso(msg, initial_value=0xFFFFFFFFFFFFFFFF):
465 # CRC-64-ISO polynomial and initial value
466 poly = 0x000000000000001B;
467 crc = initial_value; # Common initial value for CRC-64-ISO
468 for b in msg:
469 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
470 for _ in range(8): # Process each bit
471 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
472 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
473 else:
474 crc <<= 1; # Just shift left if the MSB is 0
475 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
476 return crc;
477
478 def GetDataFromArray(data, path, default=None):
479 element = data;
480 try:
481 for key in path:
482 element = element[key];
483 return element;
484 except (KeyError, TypeError, IndexError):
485 return default;
486
487 def GetDataFromArrayAlt(structure, path, default=None):
488 element = structure;
489 for key in path:
490 if isinstance(element, dict) and key in element:
491 element = element[key];
492 elif isinstance(element, list) and isinstance(key, int) and -len(element) <= key < len(element):
493 element = element[key];
494 else:
495 return default;
496 return element;
497
498 def GetHeaderChecksum(inlist=[], checksumtype="crc32", encodedata=True, formatspecs=__file_format_dict__):
499 formatspecs = FormatSpecsListToDict(formatspecs);
500 if isinstance(inlist, list):
501 fileheader = AppendNullBytes(inlist, formatspecs['format_delimiter']);
502 else:
503 fileheader = AppendNullByte(inlist, formatspecs['format_delimiter']);
504 if(encodedata):
505 fileheader = fileheader.encode('UTF-8');
506 if(checksumtype=="none" or checksumtype==""):
507 catfileheadercshex = format(0, 'x').lower();
508 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
509 catfileheadercshex = format(crc16(fileheader) & 0xffff, '04x').lower();
510 elif(checksumtype=="crc16_ccitt"):
511 catfileheadercshex = format(crc16_ccitt(fileheader) & 0xffff, '04x').lower();
512 elif(checksumtype=="adler32"):
513 catfileheadercshex = format(zlib.adler32(fileheader) & 0xffffffff, '08x').lower();
514 elif(checksumtype=="crc32"):
515 catfileheadercshex = format(crc32(fileheader) & 0xffffffff, '08x').lower();
516 elif(checksumtype=="crc64_ecma"):
517 catfileheadercshex = format(crc64_ecma(fileheader) & 0xffffffffffffffff, '016x').lower();
518 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
519 catfileheadercshex = format(crc64_iso(fileheader) & 0xffffffffffffffff, '016x').lower();
520 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
521 checksumoutstr = hashlib.new(checksumtype);
522 checksumoutstr.update(fileheader);
523 catfileheadercshex = checksumoutstr.hexdigest().lower();
524 else:
525 catfileheadercshex = format(0, 'x').lower();
526 return catfileheadercshex;
527
528 def GetFileChecksum(instr, checksumtype="crc32", encodedata=True, formatspecs=__file_format_dict__):
529 formatspecs = FormatSpecsListToDict(formatspecs);
530 if(encodedata):
531 instr = instr.encode('UTF-8');
532 if(checksumtype=="none" or checksumtype==""):
533 catinstrcshex = format(0, 'x').lower();
534 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
535 catinstrcshex = format(crc16(instr) & 0xffff, '04x').lower();
536 elif(checksumtype=="crc16_ccitt"):
537 catinstrcshex = format(crc16_ccitt(instr) & 0xffff, '04x').lower();
538 elif(checksumtype=="adler32"):
539 catinstrcshex = format(zlib.adler32(instr) & 0xffffffff, '08x').lower();
540 elif(checksumtype=="crc32"):
541 catinstrcshex = format(crc32(instr) & 0xffffffff, '08x').lower();
542 elif(checksumtype=="crc64_ecma"):
543 catinstrcshex = format(crc64_ecma(instr) & 0xffffffffffffffff, '016x').lower();
544 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
545 catinstrcshex = format(crc64_iso(instr) & 0xffffffffffffffff, '016x').lower();
546 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
547 checksumoutstr = hashlib.new(checksumtype);
548 checksumoutstr.update(instr);
549 catinstrcshex = checksumoutstr.hexdigest().lower();
550 else:
551 catinstrcshex = format(0, 'x').lower();
552 return catinstrcshex;
553
554 def ValidateHeaderChecksum(inlist=[], checksumtype="crc32", inchecksum="0", formatspecs=__file_format_dict__):
555 formatspecs = FormatSpecsListToDict(formatspecs);
556 catfileheadercshex = GetHeaderChecksum(inlist, checksumtype, True, formatspecs);
557 inchecksum = inchecksum.lower();
558 catfileheadercshex = catfileheadercshex.lower();
559 if(inchecksum==catfileheadercshex):
560 return True;
561 else:
562 return False;
563
564 def ValidateFileChecksum(infile, checksumtype="crc32", inchecksum="0", formatspecs=__file_format_dict__):
565 formatspecs = FormatSpecsListToDict(formatspecs);
566 catinfilecshex = GetFileChecksum(infile, checksumtype, True, formatspecs);
567 inchecksum = inchecksum.lower();
568 catinfilecshex = catinfilecshex.lower();
569 if(inchecksum==catinfilecshex):
570 return True;
571 else:
572 return False;
573
574 def ReadTillNullByteOld(fp, delimiter=__file_format_dict__['format_delimiter']):
575 curbyte = b"";
576 curfullbyte = b"";
577 nullbyte = delimiter.encode("UTF-8");
578 while(True):
579 curbyte = fp.read(1);
580 if(curbyte==nullbyte or not curbyte):
581 break;
582 curfullbyte = curfullbyte + curbyte;
583 return curfullbyte.decode('UTF-8');
584
585 def ReadUntilNullByteOld(fp, delimiter=__file_format_dict__['format_delimiter']):
586 return ReadTillNullByteOld(fp, delimiter);
587
588 def ReadTillNullByteAlt(fp, delimiter=__file_format_dict__['format_delimiter'], chunk_size=1024, max_read=1024000):
589 delimiter = delimiter.encode('UTF-8') # Ensure the delimiter is in bytes
590 buffer = bytearray();
591 total_read = 0;
592 delimiter_length = len(delimiter);
593 while True:
594 chunk = fp.read(chunk_size)
595 if not chunk:
596 # End of file reached without finding the delimiter
597 break;
598 buffer.extend(chunk);
599 total_read += len(chunk);
600 if delimiter in buffer:
601 # Delimiter found, calculate where to reset the file pointer
602 index = buffer.find(delimiter);
603 # Calculate how many extra bytes were read after the delimiter
604 extra_bytes_read = len(buffer) - (index + delimiter_length);
605 # Move the file pointer back to just after the delimiter
606 fp.seek(-extra_bytes_read, 1);
607 buffer = buffer[:index];
608 break;
609 if total_read >= max_read:
610 # Stop reading if max limit is reached to prevent excessive memory usage
611 raise MemoryError("Maximum read limit reached without finding the delimiter.");
612 # Check for incomplete UTF-8 sequences at the end of the buffer
613 if len(buffer) > 1 and 128 <= buffer[-1] < 192:
614 # This suggests that the last byte might be the start of a multi-byte character
615 # Try to read one more byte to complete the character
616 extra_byte = fp.read(1);
617 if extra_byte:
618 buffer.extend(extra_byte);
619 else:
620 # No more data available
621 break;
622 try:
623 return buffer.decode('UTF-8', errors='replace');
624 except UnicodeDecodeError:
625 return buffer.decode('UTF-8', errors='replace');
626
627 def ReadUntilNullByteAlt(fp, delimiter=__file_format_dict__['format_delimiter'], chunk_size=1024, max_read=1024000):
628 return ReadTillNullByteAlt(fp, delimiter, chunk_size, max_read);
629
630 def ReadTillNullByte(fp, delimiter=__file_format_dict__['format_delimiter'], max_read=1024000):
631 curfullbyte = bytearray();
632 nullbyte = delimiter.encode("UTF-8");
633 total_read = 0; # Track the total number of bytes read
634 while True:
635 curbyte = fp.read(1);
636 if curbyte == nullbyte or not curbyte:
637 break;
638 curfullbyte.extend(curbyte);
639 total_read += 1;
640 if total_read >= max_read:
641 raise MemoryError("Maximum read limit reached without finding the delimiter.");
642 # Decode the full byte array to string once out of the loop
643 try:
644 return curfullbyte.decode('UTF-8');
645 except UnicodeDecodeError:
646 # Handle potential partial UTF-8 characters
647 for i in range(1, 4):
648 try:
649 return curfullbyte[:-i].decode('UTF-8');
650 except UnicodeDecodeError:
651 continue;
652 raise; # Re-raise if decoding fails even after trimming
653
654 def ReadUntilNullByte(fp, delimiter=__file_format_dict__['format_delimiter'], max_read=1024000):
655 return ReadTillNullByte(fp, delimiter, max_read);
656
657 def ReadTillNullByteByNum(fp, delimiter=__file_format_dict__['format_delimiter'], num_delimiters=1, chunk_size=1024, max_read=1024000):
658 delimiter = delimiter.encode('UTF-8'); # Ensure the delimiter is in bytes
659 buffer = bytearray();
660 total_read = 0;
661 delimiter_length = len(delimiter);
662 results = [];
663 while len(results) < num_delimiters:
664 chunk = fp.read(chunk_size);
665 if not chunk:
666 # End of file reached; decode whatever is collected if it's the last needed part
667 if len(buffer) > 0:
668 results.append(buffer.decode('UTF-8', errors='replace'));
669 break;
670 buffer.extend(chunk)
671 total_read += len(chunk);
672 # Check if we have found the delimiter
673 while delimiter in buffer:
674 index = buffer.find(delimiter);
675 # Decode the section before the delimiter
676 results.append(buffer[:index].decode('UTF-8', errors='replace'));
677 # Remove the processed part from the buffer
678 buffer = buffer[index + delimiter_length:];
679 if len(results) == num_delimiters:
680 # If reached the required number of delimiters, adjust the file pointer and stop
681 fp.seek(-len(buffer), 1);
682 return results;
683 if total_read >= max_read:
684 # Stop reading if max limit is reached to prevent excessive memory usage
685 raise MemoryError("Maximum read limit reached without finding the delimiter.");
686 # Check for incomplete UTF-8 sequences at the end of the buffer
687 if len(buffer) > 1 and 128 <= buffer[-1] < 192:
688 # This suggests that the last byte might be the start of a multi-byte character
689 # Try to read one more byte to complete the character
690 extra_byte = fp.read(1);
691 if extra_byte:
692 buffer.extend(extra_byte);
693 else:
694 # No more data available
695 break;
696 # Process remaining buffer if less than the required number of delimiters were found
697 if len(buffer) > 0 and len(results) < num_delimiters:
698 results.append(buffer.decode('UTF-8', errors='replace'));
699 return results;
700
701 def ReadUntilNullByteByNum(fp, delimiter=__file_format_dict__['format_delimiter'], num_delimiters=1, chunk_size=1024, max_read=1024000):
702 return ReadTillNullByteByNum(fp, delimiter, num_delimiters, chunk_size, max_read);
703
704 def SeekToEndOfFile(fp):
705 lasttell = 0;
706 while(True):
707 fp.seek(1, 1);
708 if(lasttell==fp.tell()):
709 break;
710 lasttell = fp.tell();
711 return True;
712
713 def ReadFileHeaderData(fp, rounds=0, delimiter=__file_format_dict__['format_delimiter']):
714 rocount = 0;
715 roend = int(rounds);
716 HeaderOut = [];
717 while(rocount<roend):
718 HeaderOut.append(ReadTillNullByte(fp, delimiter));
719 rocount = rocount + 1;
720 return HeaderOut;
721
722 def ReadFileHeaderDataBySize(fp, delimiter=__file_format_dict__['format_delimiter']):
723 headerpresize = ReadTillNullByte(fp, delimiter);
724 headersize = int(headerpresize, 16);
725 if(headersize<=0):
726 return [];
727 headercontent = str(fp.read(headersize).decode('UTF-8')).split(delimiter);
728 fp.seek(1, 1);
729 rocount = 0;
730 roend = int(len(headercontent));
731 HeaderOut = [headerpresize];
732 while(rocount<roend):
733 HeaderOut.append(headercontent[rocount]);
734 rocount = rocount + 1;
735 return HeaderOut;
736
737 def ReadFileHeaderDataWoSize(fp, delimiter=__file_format_dict__['format_delimiter']):
738 preheaderdata = ReadFileHeaderData(fp, 2, delimiter);
739 headersize = int(preheaderdata[0], 16);
740 headernumfields = int(preheaderdata[1], 16);
741 if(headersize<=0 or headernumfields<=0):
742 return [];
743 headerdata = ReadTillNullByteByNum(fp, delimiter, headernumfields);
744 #headerdata = ReadFileHeaderData(fp, headernumfields, delimiter);
745 HeaderOut = preheaderdata + headerdata;
746 return HeaderOut;
747
748 def ReadFileHeaderDataBySizeWithContent(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
749 formatspecs = FormatSpecsListToDict(formatspecs);
750 delimiter = formatspecs['format_delimiter'];
751 fheaderstart = fp.tell();
752 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
753 if(len(HeaderOut)==0):
754 return False;
755 if(re.findall("^[.|/]", HeaderOut[3])):
756 fname = HeaderOut[3];
757 else:
758 fname = "./"+HeaderOut[3];
759 fchecksumtype = HeaderOut[-3].lower();
760 fcs = HeaderOut[-2].lower();
761 fccs = HeaderOut[-1].lower();
762 fsize = int(HeaderOut[5], 16);
763 fcompression = HeaderOut[12];
764 fcsize = int(HeaderOut[13], 16);
765 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
766 if(fcs!=newfcs and not skipchecksum):
767 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
768 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
769 return False;
770 fhend = fp.tell() - 1;
771 fcontentstart = fp.tell();
772 fcontents = BytesIO();
773 if(fsize>0 and not listonly):
774 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
775 fcontents.write(fp.read(fsize));
776 else:
777 fcontents.write(fp.read(fcsize));
778 elif(fsize>0 and listonly):
779 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
780 fp.seek(fsize, 1);
781 else:
782 fp.seek(fcsize, 1);
783 fcontents.seek(0, 0);
784 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
785 if(fccs!=newfccs and not skipchecksum and not listonly):
786 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
787 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
788 return False;
789 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
790 pass;
791 else:
792 fcontents.seek(0, 0);
793 fcontents = UncompressArchiveFile(fcontents, formatspecs);
794 fp.seek(1, 1);
795 fcontentend = fp.tell() - 1;
796 HeaderOut.append(fcontents);
797 return HeaderOut;
798
799 def ReadFileHeaderDataBySizeWithContentToArray(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
800 formatspecs = FormatSpecsListToDict(formatspecs);
801 delimiter = formatspecs['format_delimiter'];
802 fheaderstart = fp.tell();
803 if(formatspecs['new_style']):
804 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
805 else:
806 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
807 if(len(HeaderOut)==0):
808 return False;
809 fheadsize = int(HeaderOut[0], 16);
810 fnumfields = int(HeaderOut[1], 16);
811 ftype = int(HeaderOut[2], 16);
812 if(re.findall("^[.|/]", HeaderOut[3])):
813 fname = HeaderOut[3];
814 else:
815 fname = "./"+HeaderOut[3];
816 fbasedir = os.path.dirname(fname);
817 flinkname = HeaderOut[4];
818 fsize = int(HeaderOut[5], 16);
819 fatime = int(HeaderOut[6], 16);
820 fmtime = int(HeaderOut[7], 16);
821 fctime = int(HeaderOut[8], 16);
822 fbtime = int(HeaderOut[9], 16);
823 fmode = int(HeaderOut[10], 16);
824 fchmode = stat.S_IMODE(fmode);
825 ftypemod = stat.S_IFMT(fmode);
826 fwinattributes = int(HeaderOut[11], 16);
827 fcompression = HeaderOut[12];
828 fcsize = int(HeaderOut[13], 16);
829 fuid = int(HeaderOut[14], 16);
830 funame = HeaderOut[15];
831 fgid = int(HeaderOut[16], 16);
832 fgname = HeaderOut[17];
833 fid = int(HeaderOut[18], 16);
834 finode = int(HeaderOut[19], 16);
835 flinkcount = int(HeaderOut[20], 16);
836 fdev_minor = int(HeaderOut[21], 16);
837 fdev_major = int(HeaderOut[22], 16);
838 frdev_minor = int(HeaderOut[23], 16);
839 frdev_major = int(HeaderOut[24], 16);
840 fextrasize = int(HeaderOut[25], 16);
841 fextrafields = int(HeaderOut[26], 16);
842 extrafieldslist = [];
843 extrastart = 27;
844 extraend = extrastart + fextrafields;
845 extrafieldslist = [];
846 if(extrastart<extraend):
847 extrafieldslist.append(HeaderOut[extrastart]);
848 extrastart = extrastart + 1;
849 fchecksumtype = HeaderOut[extrastart].lower();
850 fcs = HeaderOut[extrastart + 1].lower();
851 fccs = HeaderOut[extrastart + 2].lower();
852 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
853 if(fcs!=newfcs and not skipchecksum):
854 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
855 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
856 return False;
857 fhend = fp.tell() - 1;
858 fcontentstart = fp.tell();
859 fcontents = BytesIO();
860 pyhascontents = False;
861 if(fsize>0 and not listonly):
862 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
863 fcontents.write(fp.read(fsize));
864 else:
865 fcontents.write(fp.read(fcsize));
866 pyhascontents = True;
867 elif(fsize>0 and listonly):
868 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
869 fp.seek(fsize, 1);
870 else:
871 fp.seek(fcsize, 1);
872 pyhascontents = False;
873 fcontents.seek(0, 0);
874 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
875 if(fccs!=newfccs and not skipchecksum and not listonly):
876 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
877 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
878 return False;
879 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
880 pass;
881 else:
882 fcontents.seek(0, 0);
883 fcontents = UncompressArchiveFile(fcontents, formatspecs);
884 fp.seek(1, 1);
885 fcontentend = fp.tell() - 1;
886 fcontents.seek(0, 0);
887 catlist = {'fheadersize': fheadsize, 'fhstart': fheaderstart, 'fhend': fhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': fchecksumtype, 'fnumfields': fnumfields + 2, 'frawheader': HeaderOut, 'fextrafields': fextrafields, 'fextrafieldsize': fextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': fcs, 'fcontentchecksum': fccs, 'fhascontents': pyhascontents, 'fcontentstart': fcontentstart, 'fcontentend': fcontentend, 'fcontents': fcontents};
888 return catlist;
889
890 def ReadFileHeaderDataBySizeWithContentToList(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
891 formatspecs = FormatSpecsListToDict(formatspecs);
892 delimiter = formatspecs['format_delimiter'];
893 fheaderstart = fp.tell();
894 if(formatspecs['new_style']):
895 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
896 else:
897 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
898 if(len(HeaderOut)==0):
899 return False;
900 fheadsize = int(HeaderOut[0], 16);
901 fnumfields = int(HeaderOut[1], 16);
902 ftype = int(HeaderOut[2], 16);
903 if(re.findall("^[.|/]", HeaderOut[3])):
904 fname = HeaderOut[3];
905 else:
906 fname = "./"+HeaderOut[3];
907 fbasedir = os.path.dirname(fname);
908 flinkname = HeaderOut[4];
909 fsize = int(HeaderOut[5], 16);
910 fatime = int(HeaderOut[6], 16);
911 fmtime = int(HeaderOut[7], 16);
912 fctime = int(HeaderOut[8], 16);
913 fbtime = int(HeaderOut[9], 16);
914 fmode = int(HeaderOut[10], 16);
915 fchmode = stat.S_IMODE(fmode);
916 ftypemod = stat.S_IFMT(fmode);
917 fwinattributes = int(HeaderOut[11], 16);
918 fcompression = HeaderOut[12];
919 fcsize = int(HeaderOut[13], 16);
920 fuid = int(HeaderOut[14], 16);
921 funame = HeaderOut[15];
922 fgid = int(HeaderOut[16], 16);
923 fgname = HeaderOut[17];
924 fid = int(HeaderOut[18], 16);
925 finode = int(HeaderOut[19], 16);
926 flinkcount = int(HeaderOut[20], 16);
927 fdev_minor = int(HeaderOut[21], 16);
928 fdev_major = int(HeaderOut[22], 16);
929 frdev_minor = int(HeaderOut[23], 16);
930 frdev_major = int(HeaderOut[24], 16);
931 fextrasize = int(HeaderOut[25], 16);
932 fextrafields = int(HeaderOut[26], 16);
933 extrafieldslist = [];
934 extrastart = 27;
935 extraend = extrastart + fextrafields;
936 extrafieldslist = [];
937 if(extrastart<extraend):
938 extrafieldslist.append(HeaderOut[extrastart]);
939 extrastart = extrastart + 1;
940 fchecksumtype = HeaderOut[extrastart].lower();
941 fcs = HeaderOut[extrastart + 1].lower();
942 fccs = HeaderOut[extrastart + 2].lower();
943 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
944 if(fcs!=newfcs and not skipchecksum):
945 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
946 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
947 return False;
948 fhend = fp.tell() - 1;
949 fcontentstart = fp.tell();
950 fcontents = BytesIO();
951 pyhascontents = False;
952 if(fsize>0 and not listonly):
953 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
954 fcontents.write(fp.read(fsize));
955 else:
956 fcontents.write(fp.read(fcsize));
957 pyhascontents = True;
958 elif(fsize>0 and listonly):
959 if(fcompression=="none" or fcompression=="" or fcompression=="atuo"):
960 fp.seek(fsize, 1);
961 else:
962 fp.seek(fcsize, 1);
963 pyhascontents = False;
964 fcontents.seek(0, 0);
965 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
966 if(fccs!=newfccs and not skipchecksum and not listonly):
967 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
968 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
969 return False;
970 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
971 pass;
972 else:
973 fcontents.seek(0, 0);
974 fcontents = UncompressArchiveFile(fcontents, formatspecs);
975 fp.seek(1, 1);
976 fcontentend = fp.tell() - 1;
977 catlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, extrafieldslist, fchecksumtype, fcontents];
978 return catlist;
979
980 def ReadFileDataBySizeWithContent(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
981 formatspecs = FormatSpecsListToDict(formatspecs);
982 delimiter = formatspecs['format_delimiter'];
983 curloc = fp.tell();
984 if(curloc>0):
985 fp.seek(0, 0);
986 catheader = ReadFileHeaderData(fp, 4, delimiter);
987 if(curloc>0):
988 fp.seek(curloc, 0);
989 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
990 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
991 if(not headercheck and not skipchecksum):
992 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
993 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
994 return False;
995 fnumfiles = int(catheader[1], 16);
996 countnum = 0;
997 flist = [];
998 while(countnum < fnumfiles):
999 HeaderOut = ReadFileHeaderDataBySizeWithContent(fp, listonly, skipchecksum, formatspecs)
1000 if(len(HeaderOut)==0):
1001 break;
1002 flist.append(HeaderOut);
1003 countnum = countnum + 1;
1004 return flist;
1005
1006 def ReadFileDataBySizeWithContentToArray(fp, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
1007 formatspecs = FormatSpecsListToDict(formatspecs);
1008 delimiter = formatspecs['format_delimiter'];
1009 curloc = fp.tell();
1010 if(curloc>0):
1011 fp.seek(0, 0);
1012 catheader = ReadFileHeaderData(fp, 4, delimiter);
1013 if(curloc>0):
1014 fp.seek(curloc, 0);
1015 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1016 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1017 if(not headercheck and not skipchecksum):
1018 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1019 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1020 return False;
1021 catstring = catheader[0];
1022 catversion = re.findall(r"([\d]+)", catstring);
1023 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
1024 fprenumfiles = catheader[1];
1025 fnumfiles = int(fprenumfiles, 16);
1026 fprechecksumtype = catheader[2];
1027 fprechecksum = catheader[3];
1028 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
1029 if(seekstart<0 and seekstart>fnumfiles):
1030 seekstart = 0;
1031 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
1032 seekend = fnumfiles;
1033 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
1034 seekend = fnumfiles - abs(seekend);
1035 if(seekstart>0):
1036 il = 0;
1037 while(il < seekstart):
1038 prefhstart = fp.tell();
1039 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs['format_delimiter']);
1040 if(len(preheaderdata)==0):
1041 break;
1042 prefsize = int(preheaderdata[5], 16);
1043 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-3].lower(), True, formatspecs);
1044 prefcs = preheaderdata[-2];
1045 if(prefcs!=prenewfcs and not skipchecksum):
1046 VVerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
1047 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
1048 return False;
1049 valid_archive = False;
1050 invalid_archive = True;
1051 prefhend = fp.tell() - 1;
1052 prefcontentstart = fp.tell();
1053 prefcontents = BytesIO();
1054 pyhascontents = False;
1055 if(prefsize>0):
1056 prefcontents.write(fp.read(prefsize));
1057 prefcontents.seek(0, 0);
1058 prenewfccs = GetFileChecksum(prefcontents.read(), preheaderdata[-3].lower(), False, formatspecs);
1059 prefccs = preheaderdata[-1];
1060 pyhascontents = True;
1061 if(prefccs!=prenewfccs and not skipchecksum):
1062 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
1063 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
1064 return False;
1065 fp.seek(1, 1);
1066 il = il + 1;
1067 realidnum = 0;
1068 countnum = seekstart;
1069 while(countnum < seekend):
1070 catlist['ffilelist'].update({realidnum: {'fid': realidnum, 'fidalt': realidnum}});
1071 HeaderOut = ReadFileHeaderDataBySizeWithContentToArray(fp, listonly, skipchecksum, formatspecs);
1072 if(len(HeaderOut)==0):
1073 break;
1074 catlist['ffilelist'][realidnum].update(HeaderOut);
1075 countnum = countnum + 1;
1076 realidnum = realidnum + 1;
1077 return catlist;
1078
1079 def ReadFileDataBySizeWithContentToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
1080 formatspecs = FormatSpecsListToDict(formatspecs);
1081 if(isinstance(infile, dict)):
1082 listcatfiles = infile;
1083 else:
1084 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
1085 return False;
1086 listcatfiles = ReadFileDataBySizeWithContentToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs);
1087 if(not listcatfiles):
1088 return False;
1089 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
1090 lenlist = len(listcatfiles['ffilelist']);
1091 lcfi = 0;
1092 lcfx = int(listcatfiles['fnumfiles']);
1093 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
1094 lcfx = int(lenlist);
1095 else:
1096 lcfx = int(listcatfiles['fnumfiles']);
1097 while(lcfi < lcfx):
1098 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
1099 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
1100 catarray['filetoid'].update(filetoidarray);
1101 catarray['idtofile'].update(idtofilearray);
1102 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
1103 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
1104 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
1105 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
1106 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
1107 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
1108 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
1109 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
1110 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
1111 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1112 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1113 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
1114 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
1115 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
1116 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
1117 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
1118 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1119 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1120 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
1121 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
1122 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
1123 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1124 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1125 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
1126 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
1127 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
1128 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
1129 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1130 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1131 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1132 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1133 lcfi = lcfi + 1;
1134 return catarray;
1135
1136 def ReadFileDataBySizeWithContentToList(fp, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
1137 formatspecs = FormatSpecsListToDict(formatspecs);
1138 delimiter = formatspecs['format_delimiter'];
1139 curloc = fp.tell();
1140 if(curloc>0):
1141 fp.seek(0, 0);
1142 catheader = ReadFileHeaderData(fp, 4, delimiter);
1143 if(curloc>0):
1144 fp.seek(curloc, 0);
1145 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1146 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1147 if(not headercheck and not skipchecksum):
1148 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1149 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1150 return False;
1151 catstring = catheader[0];
1152 catversion = re.findall(r"([\d]+)", catstring);
1153 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
1154 fprenumfiles = catheader[1];
1155 fnumfiles = int(fprenumfiles, 16);
1156 fprechecksumtype = catheader[2];
1157 fprechecksum = catheader[3];
1158 catlist = [];
1159 if(seekstart<0 and seekstart>fnumfiles):
1160 seekstart = 0;
1161 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
1162 seekend = fnumfiles;
1163 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
1164 seekend = fnumfiles - abs(seekend);
1165 if(seekstart>0):
1166 il = 0;
1167 while(il < seekstart):
1168 prefhstart = fp.tell();
1169 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs['format_delimiter']);
1170 if(len(preheaderdata)==0):
1171 break;
1172 prefsize = int(preheaderdata[5], 16);
1173 prefcompression = preheaderdata[12];
1174 prefcsize = int(preheaderdata[13], 16);
1175 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-3].lower(), True, formatspecs);
1176 prefcs = preheaderdata[-2];
1177 if(prefcs!=prenewfcs and not skipchecksum):
1178 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
1179 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
1180 return False;
1181 valid_archive = False;
1182 invalid_archive = True;
1183 prefhend = fp.tell() - 1;
1184 prefcontentstart = fp.tell();
1185 prefcontents = "";
1186 pyhascontents = False;
1187 if(prefsize>0):
1188 if(prefcompression=="none" or prefcompression=="" or prefcompression=="auto"):
1189 prefcontents = catfp.read(prefsize);
1190 else:
1191 prefcontents = catfp.read(prefcsize);
1192 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
1193 prefccs = preheaderdata[-1];
1194 pyhascontents = True;
1195 if(prefccs!=prenewfccs and not skipchecksum):
1196 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
1197 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
1198 return False;
1199 fp.seek(1, 1);
1200 il = il + 1;
1201 realidnum = 0;
1202 countnum = seekstart;
1203 while(countnum < seekend):
1204 HeaderOut = ReadFileHeaderDataBySizeWithContentToList(fp, listonly, skipchecksum, formatspecs);
1205 if(len(HeaderOut)==0):
1206 break;
1207 catlist.append(HeaderOut);
1208 countnum = countnum + 1;
1209 realidnum = realidnum + 1;
1210 return catlist;
1211
1212 def ReadInFileBySizeWithContentToArray(infile, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
1213 formatspecs = FormatSpecsListToDict(formatspecs);
1214 delimiter = formatspecs['format_delimiter'];
1215 if(hasattr(infile, "read") or hasattr(infile, "write")):
1216 fp = infile;
1217 fp.seek(0, 0);
1218 fp = UncompressArchiveFile(fp, formatspecs);
1219 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1220 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1221 return False;
1222 if(not fp):
1223 return False;
1224 fp.seek(0, 0);
1225 elif(infile=="-"):
1226 fp = BytesIO();
1227 if(hasattr(sys.stdin, "buffer")):
1228 shutil.copyfileobj(sys.stdin.buffer, fp);
1229 else:
1230 shutil.copyfileobj(sys.stdin, fp);
1231 fp.seek(0, 0);
1232 fp = UncompressArchiveFile(fp, formatspecs);
1233 if(not fp):
1234 return False;
1235 fp.seek(0, 0);
1236 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
1237 fp = download_file_from_internet_file(infile);
1238 fp = UncompressArchiveFile(fp, formatspecs);
1239 fp.seek(0, 0);
1240 if(not fp):
1241 return False;
1242 fp.seek(0, 0);
1243 else:
1244 infile = RemoveWindowsPath(infile);
1245 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
1246 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1247 return False;
1248 compresscheck = CheckCompressionType(infile, formatspecs, True);
1249 if(not compresscheck):
1250 fextname = os.path.splitext(infile)[1];
1251 if(fextname==".gz"):
1252 compresscheck = "gzip";
1253 elif(fextname==".bz2"):
1254 compresscheck = "bzip2";
1255 elif(fextname==".zst"):
1256 compresscheck = "zstd";
1257 elif(fextname==".lz4" or fextname==".clz4"):
1258 compresscheck = "lz4";
1259 elif(fextname==".lzo" or fextname==".lzop"):
1260 compresscheck = "lzo";
1261 elif(fextname==".lzma" or fextname==".xz"):
1262 compresscheck = "lzma";
1263 else:
1264 return False;
1265 if(not compresscheck):
1266 return False;
1267 fp = UncompressFile(infile, formatspecs, "rb");
1268 return ReadFileDataBySizeWithContentToArray(fp, listonly, skipchecksum, formatspecs);
1269
1270 def ReadInFileBySizeWithContentToList(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
1271 formatspecs = FormatSpecsListToDict(formatspecs);
1272 delimiter = formatspecs['format_delimiter'];
1273 if(hasattr(infile, "read") or hasattr(infile, "write")):
1274 fp = infile;
1275 fp.seek(0, 0);
1276 fp = UncompressArchiveFile(fp, formatspecs);
1277 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1278 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1279 return False;
1280 if(not fp):
1281 return False;
1282 fp.seek(0, 0);
1283 elif(infile=="-"):
1284 fp = BytesIO();
1285 if(hasattr(sys.stdin, "buffer")):
1286 shutil.copyfileobj(sys.stdin.buffer, fp);
1287 else:
1288 shutil.copyfileobj(sys.stdin, fp);
1289 fp.seek(0, 0);
1290 fp = UncompressArchiveFile(fp, formatspecs);
1291 if(not fp):
1292 return False;
1293 fp.seek(0, 0);
1294 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
1295 fp = download_file_from_internet_file(infile);
1296 fp = UncompressArchiveFile(fp, formatspecs);
1297 fp.seek(0, 0);
1298 if(not fp):
1299 return False;
1300 fp.seek(0, 0);
1301 else:
1302 infile = RemoveWindowsPath(infile);
1303 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
1304 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1305 return False;
1306 compresscheck = CheckCompressionType(infile, formatspecs, True);
1307 if(not compresscheck):
1308 fextname = os.path.splitext(infile)[1];
1309 if(fextname==".gz"):
1310 compresscheck = "gzip";
1311 elif(fextname==".bz2"):
1312 compresscheck = "bzip2";
1313 elif(fextname==".zst"):
1314 compresscheck = "zstd";
1315 elif(fextname==".lz4" or fextname==".clz4"):
1316 compresscheck = "lz4";
1317 elif(fextname==".lzo" or fextname==".lzop"):
1318 compresscheck = "lzo";
1319 elif(fextname==".lzma" or fextname==".xz"):
1320 compresscheck = "lzma";
1321 else:
1322 return False;
1323 if(not compresscheck):
1324 return False;
1325 fp = UncompressFile(infile, formatspecs, "rb");
1326 return ReadFileDataBySizeWithContentToList(fp, seekstart, seekend, listonly, skipchecksum, formatspecs);
1327
1328 def ReadInFileBySizeWithContentToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
1329 formatspecs = FormatSpecsListToDict(formatspecs);
1330 if(isinstance(infile, dict)):
1331 listcatfiles = infile;
1332 else:
1333 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
1334 infile = RemoveWindowsPath(infile);
1335 listcatfiles = ReadInFileBySizeWithContentToArray(infile, listonly, skipchecksum, formatspecs);
1336 if(not listcatfiles):
1337 return False;
1338 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
1339 lenlist = len(listcatfiles['ffilelist']);
1340 lcfi = 0;
1341 lcfx = int(listcatfiles['fnumfiles']);
1342 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
1343 lcfx = int(lenlist);
1344 else:
1345 lcfx = int(listcatfiles['fnumfiles']);
1346 while(lcfi < lcfx):
1347 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
1348 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
1349 catarray['filetoid'].update(filetoidarray);
1350 catarray['idtofile'].update(idtofilearray);
1351 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
1352 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
1353 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
1354 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
1355 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
1356 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
1357 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
1358 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
1359 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
1360 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1361 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1362 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
1363 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
1364 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
1365 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
1366 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
1367 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1368 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1369 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
1370 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
1371 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
1372 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1373 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1374 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
1375 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
1376 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
1377 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
1378 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1379 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1380 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1381 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1382 lcfi = lcfi + 1;
1383 return catarray;
1384
1385 def AppendNullByte(indata, delimiter=__file_format_dict__['format_delimiter']):
1386 outdata = str(indata) + delimiter;
1387 return outdata;
1388
1389 def AppendNullBytes(indata=[], delimiter=__file_format_dict__['format_delimiter']):
1390 outdata = "";
1391 inum = 0;
1392 il = len(indata);
1393 while(inum < il):
1394 outdata = outdata + AppendNullByte(indata[inum], delimiter);
1395 inum = inum + 1;
1396 return outdata;
1397
1398 def AppendFileHeader(fp, numfiles, checksumtype="crc32", formatspecs=__file_format_dict__):
1399 formatspecs = FormatSpecsListToDict(formatspecs);
1400 delimiter = formatspecs['format_delimiter'];
1401 catver = formatspecs['format_ver'];
1402 fileheaderver = str(int(catver.replace(".", "")));
1403 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
1404 fp.write(fileheader.encode('UTF-8'));
1405 fnumfiles = format(int(numfiles), 'x').lower();
1406 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs['format_delimiter']);
1407 catfileheadercshex = GetFileChecksum(fileheader + fnumfilesa, checksumtype, True, formatspecs);
1408 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
1409 fp.write(fnumfilesa.encode('UTF-8'));
1410 try:
1411 fp.flush();
1412 os.fsync(fp.fileno());
1413 except io.UnsupportedOperation:
1414 pass;
1415 except AttributeError:
1416 pass;
1417 except OSError as e:
1418 pass;
1419 return fp;
1420
1421 def MakeEmptyFilePointer(fp, checksumtype="crc32", formatspecs=__file_format_dict__):
1422 formatspecs = FormatSpecsListToDict(formatspecs);
1423 AppendFileHeader(fp, 0, checksumtype, formatspecs);
1424 return fp;
1425
1426 def MakeEmptyFile(outfile, compression="auto", compressionlevel=None, checksumtype="crc32", formatspecs=__file_format_dict__, returnfp=False):
1427 formatspecs = FormatSpecsListToDict(formatspecs);
1428 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1429 if(os.path.exists(outfile)):
1430 try:
1431 os.unlink(outfile);
1432 except OSError as e:
1433 pass;
1434 if(outfile=="-"):
1435 verbose = False;
1436 catfpfp = BytesIO();
1437 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1438 catfp = outfile;
1439 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1440 catfp = BytesIO();
1441 else:
1442 fbasename = os.path.splitext(outfile)[0];
1443 fextname = os.path.splitext(outfile)[1];
1444 catfp = CompressOpenFile(outfile, True, compressionlevel);
1445 catfp = AppendFileHeader(catfp, 0, checksumtype, formatspecs);
1446 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1447 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1448 try:
1449 catfp.flush();
1450 os.fsync(catfp.fileno());
1451 except io.UnsupportedOperation:
1452 pass;
1453 except AttributeError:
1454 pass;
1455 except OSError as e:
1456 pass;
1457 if(outfile=="-"):
1458 catfp.seek(0, 0);
1459 if(hasattr(sys.stdout, "buffer")):
1460 shutil.copyfileobj(catfp, sys.stdout.buffer);
1461 else:
1462 shutil.copyfileobj(catfp, sys.stdout);
1463 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1464 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1465 catfp.seek(0, 0);
1466 upload_file_to_internet_file(catfp, outfile);
1467 if(returnfp):
1468 catfp.seek(0, 0);
1469 return catfp;
1470 else:
1471 catfp.close();
1472 return True;
1473
1474 def AppendFileHeaderWithContent(fp, filevalues=[], extradata=[], filecontent="", checksumtype="crc32", formatspecs=__file_format_dict__):
1475 formatspecs = FormatSpecsListToDict(formatspecs);
1476 extrafields = format(len(extradata), 'x').lower();
1477 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
1478 if(len(extradata)>0):
1479 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
1480 extrasizelen = format(len(extrasizestr), 'x').lower();
1481 catoutlen = len(filevalues) + len(extradata) + 5;
1482 catoutlenhex = format(catoutlen, 'x').lower();
1483 catoutlist = filevalues;
1484 catoutlist.insert(0, catoutlenhex);
1485 catoutlist.append(extrasizelen);
1486 catoutlist.append(extrafields);
1487 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
1488 if(len(extradata)>0):
1489 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
1490 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
1491 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
1492 catfilecontentcshex = GetFileChecksum(filecontent, checksumtype, False, formatspecs);
1493 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
1494 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
1495 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
1496 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
1497 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
1498 catfileoutstrecd = catfileoutstr.encode('UTF-8');
1499 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
1500 catfileout = catfileoutstrecd + filecontent + nullstrecd;
1501 fp.write(catfileout);
1502 try:
1503 fp.flush();
1504 os.fsync(fp.fileno());
1505 except io.UnsupportedOperation:
1506 pass;
1507 except AttributeError:
1508 pass;
1509 except OSError as e:
1510 pass;
1511 return fp;
1512
1513 def AppendFilesWithContent(infiles, fp, dirlistfromtxt=False, filevalues=[], extradata=[], compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
1514 formatspecs = FormatSpecsListToDict(formatspecs);
1515 advancedlist = formatspecs['use_advanced_list'];
1516 altinode = formatspecs['use_alt_inode'];
1517 if(verbose):
1518 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
1519 if(infiles=="-"):
1520 for line in sys.stdin:
1521 infilelist.append(line.strip());
1522 infilelist = list(filter(None, infilelist));
1523 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
1524 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
1525 return False;
1526 with UncompressFile(infiles, formatspecs, "r") as finfile:
1527 for line in finfile:
1528 infilelist.append(line.strip());
1529 infilelist = list(filter(None, infilelist));
1530 else:
1531 if(isinstance(infiles, (list, tuple, ))):
1532 infilelist = list(filter(None, infiles));
1533 elif(isinstance(infiles, (str, ))):
1534 infilelist = list(filter(None, [infiles]));
1535 if(advancedlist):
1536 GetDirList = ListDirAdvanced(infilelist, followlink, False);
1537 else:
1538 GetDirList = ListDir(infilelist, followlink, False);
1539 if(not GetDirList):
1540 return False;
1541 curinode = 0;
1542 curfid = 0;
1543 inodelist = [];
1544 inodetofile = {};
1545 filetoinode = {};
1546 inodetocatinode = {};
1547 numfiles = int(len(GetDirList));
1548 fnumfiles = format(numfiles, 'x').lower();
1549 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
1550 for curfname in GetDirList:
1551 if(re.findall("^[.|/]", curfname)):
1552 fname = curfname;
1553 else:
1554 fname = "./"+curfname;
1555 if(verbose):
1556 VerbosePrintOut(fname);
1557 if(not followlink or followlink is None):
1558 fstatinfo = os.lstat(fname);
1559 else:
1560 fstatinfo = os.stat(fname);
1561 fpremode = fstatinfo.st_mode;
1562 finode = fstatinfo.st_ino;
1563 flinkcount = fstatinfo.st_nlink;
1564 ftype = 0;
1565 if(stat.S_ISREG(fpremode)):
1566 ftype = 0;
1567 elif(stat.S_ISLNK(fpremode)):
1568 ftype = 2;
1569 elif(stat.S_ISCHR(fpremode)):
1570 ftype = 3;
1571 elif(stat.S_ISBLK(fpremode)):
1572 ftype = 4;
1573 elif(stat.S_ISDIR(fpremode)):
1574 ftype = 5;
1575 elif(stat.S_ISFIFO(fpremode)):
1576 ftype = 6;
1577 elif(stat.S_ISSOCK(fpremode)):
1578 ftype = 8;
1579 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
1580 ftype = 9;
1581 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
1582 ftype = 10;
1583 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
1584 ftype = 111;
1585 else:
1586 ftype = 0;
1587 flinkname = "";
1588 fcurfid = format(int(curfid), 'x').lower();
1589 if(not followlink and finode!=0):
1590 if(ftype!=1):
1591 if(finode in inodelist):
1592 ftype = 1;
1593 flinkname = inodetofile[finode];
1594 if(altinode):
1595 fcurinode = format(int(finode), 'x').lower();
1596 else:
1597 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
1598 if(finode not in inodelist):
1599 inodelist.append(finode);
1600 inodetofile.update({finode: fname});
1601 inodetocatinode.update({finode: curinode});
1602 if(altinode):
1603 fcurinode = format(int(finode), 'x').lower();
1604 else:
1605 fcurinode = format(int(curinode), 'x').lower();
1606 curinode = curinode + 1;
1607 else:
1608 fcurinode = format(int(curinode), 'x').lower();
1609 curinode = curinode + 1;
1610 curfid = curfid + 1;
1611 if(ftype==2):
1612 flinkname = os.readlink(fname);
1613 fdev = fstatinfo.st_dev;
1614 getfdev = GetDevMajorMinor(fdev);
1615 fdev_minor = getfdev[0];
1616 fdev_major = getfdev[1];
1617 frdev = fstatinfo.st_dev;
1618 if(hasattr(fstatinfo, "st_rdev")):
1619 frdev = fstatinfo.st_rdev;
1620 else:
1621 frdev = fstatinfo.st_dev;
1622 getfrdev = GetDevMajorMinor(frdev);
1623 frdev_minor = getfrdev[0];
1624 frdev_major = getfrdev[1];
1625 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
1626 fsize = format(int("0"), 'x').lower();
1627 elif(ftype==0 or ftype==7):
1628 fsize = format(int(fstatinfo.st_size), 'x').lower();
1629 else:
1630 fsize = format(int(fstatinfo.st_size)).lower();
1631 fatime = format(int(fstatinfo.st_atime), 'x').lower();
1632 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
1633 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
1634 if(hasattr(fstatinfo, "st_birthtime")):
1635 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
1636 else:
1637 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
1638 fmode = format(int(fstatinfo.st_mode), 'x').lower();
1639 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
1640 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
1641 fuid = format(int(fstatinfo.st_uid), 'x').lower();
1642 fgid = format(int(fstatinfo.st_gid), 'x').lower();
1643 funame = "";
1644 try:
1645 import pwd;
1646 try:
1647 userinfo = pwd.getpwuid(fstatinfo.st_uid);
1648 funame = userinfo.pw_name;
1649 except KeyError:
1650 funame = "";
1651 except ImportError:
1652 funame = "";
1653 fgname = "";
1654 try:
1655 import grp;
1656 try:
1657 groupinfo = grp.getgrgid(fstatinfo.st_gid);
1658 fgname = groupinfo.gr_name;
1659 except KeyError:
1660 fgname = "";
1661 except ImportError:
1662 fgname = "";
1663 fdev_minor = format(int(fdev_minor), 'x').lower();
1664 fdev_major = format(int(fdev_major), 'x').lower();
1665 frdev_minor = format(int(frdev_minor), 'x').lower();
1666 frdev_major = format(int(frdev_major), 'x').lower();
1667 finode = format(int(finode), 'x').lower();
1668 flinkcount = format(int(flinkcount), 'x').lower();
1669 if(hasattr(fstatinfo, "st_file_attributes")):
1670 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
1671 else:
1672 fwinattributes = format(int(0), 'x').lower();
1673 fcompression = "";
1674 fcsize = format(int(0), 'x').lower();
1675 fcontents = BytesIO();
1676 chunk_size = 1024;
1677 if(ftype==0 or ftype==7):
1678 with open(fname, "rb") as fpc:
1679 shutil.copyfileobj(fpc, fcontents);
1680 if(not compresswholefile):
1681 fcontents.seek(0, 2);
1682 ucfsize = fcontents.tell();
1683 fcontents.seek(0, 0);
1684 if(compression=="auto"):
1685 ilsize = len(compressionlistalt);
1686 ilmin = 0;
1687 ilcsize = [];
1688 while(ilmin < ilsize):
1689 cfcontents = BytesIO();
1690 shutil.copyfileobj(fcontents, cfcontents);
1691 fcontents.seek(0, 0);
1692 cfcontents.seek(0, 0);
1693 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
1694 if(cfcontents):
1695 cfcontents.seek(0, 2);
1696 ilcsize.append(cfcontents.tell());
1697 cfcontents.close();
1698 else:
1699 try:
1700 ilcsize.append(sys.maxint);
1701 except AttributeError:
1702 ilcsize.append(sys.maxsize);
1703 ilmin = ilmin + 1;
1704 ilcmin = ilcsize.index(min(ilcsize));
1705 compression = compressionlistalt[ilcmin];
1706 fcontents.seek(0, 0);
1707 cfcontents = BytesIO();
1708 shutil.copyfileobj(fcontents, cfcontents);
1709 cfcontents.seek(0, 0);
1710 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
1711 cfcontents.seek(0, 2);
1712 cfsize = cfcontents.tell();
1713 if(ucfsize > cfsize):
1714 fcsize = format(int(cfsize), 'x').lower();
1715 fcompression = compression;
1716 fcontents.close();
1717 fcontents = cfcontents;
1718 if(followlink and (ftype==1 or ftype==2)):
1719 flstatinfo = os.stat(flinkname);
1720 with open(flinkname, "rb") as fpc:
1721 shutil.copyfileobj(fpc, fcontents);
1722 if(not compresswholefile):
1723 fcontents.seek(0, 2);
1724 ucfsize = fcontents.tell();
1725 fcontents.seek(0, 0);
1726 if(compression=="auto"):
1727 ilsize = len(compressionlistalt);
1728 ilmin = 0;
1729 ilcsize = [];
1730 while(ilmin < ilsize):
1731 cfcontents = BytesIO();
1732 shutil.copyfileobj(fcontents, cfcontents);
1733 fcontents.seek(0, 0);
1734 cfcontents.seek(0, 0);
1735 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
1736 if(cfcontents):
1737 cfcontents.seek(0, 2);
1738 ilcsize.append(cfcontents.tell());
1739 cfcontents.close();
1740 else:
1741 try:
1742 ilcsize.append(sys.maxint);
1743 except AttributeError:
1744 ilcsize.append(sys.maxsize);
1745 ilmin = ilmin + 1;
1746 ilcmin = ilcsize.index(min(ilcsize));
1747 compression = compressionlistalt[ilcmin];
1748 fcontents.seek(0, 0);
1749 cfcontents = BytesIO();
1750 shutil.copyfileobj(fcontents, cfcontents);
1751 cfcontents.seek(0, 0);
1752 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
1753 cfcontents.seek(0, 2);
1754 cfsize = cfcontents.tell();
1755 if(ucfsize > cfsize):
1756 fcsize = format(int(cfsize), 'x').lower();
1757 fcompression = compression;
1758 fcontents.close();
1759 fcontents = cfcontents;
1760 if(fcompression=="none"):
1761 fcompression = "";
1762 fcontents.seek(0, 0);
1763 ftypehex = format(ftype, 'x').lower();
1764 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
1765 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
1766 if(numfiles>0):
1767 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
1768 fp.seek(0, 0);
1769 return fp;
1770
1771 def AppendListsWithContent(inlist, fp, dirlistfromtxt=False, filevalues=[], extradata=[], compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
1772 formatspecs = FormatSpecsListToDict(formatspecs);
1773 if(verbose):
1774 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
1775 GetDirList = inlist;
1776 if(not GetDirList):
1777 return False;
1778 curinode = 0;
1779 curfid = 0;
1780 inodelist = [];
1781 inodetofile = {};
1782 filetoinode = {};
1783 inodetocatinode = {};
1784 numfiles = int(len(GetDirList));
1785 fnumfiles = format(numfiles, 'x').lower();
1786 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
1787 for curfname in GetDirList:
1788 ftype = format(curfname[0], 'x').lower();
1789 if(re.findall("^[.|/]", curfname[1])):
1790 fname = curfname[1];
1791 else:
1792 fname = "./"+curfname[1];
1793 fbasedir = os.path.dirname(fname);
1794 flinkname = curfname[2];
1795 fsize = format(curfname[3], 'x').lower();
1796 fatime = format(curfname[4], 'x').lower();
1797 fmtime = format(curfname[5], 'x').lower();
1798 fctime = format(curfname[6], 'x').lower();
1799 fbtime = format(curfname[7], 'x').lower();
1800 fmode = format(curfname[8], 'x').lower();
1801 fwinattributes = format(curfname[9], 'x').lower();
1802 fcompression = curfname[10];
1803 fcsize = format(curfname[11], 'x').lower();
1804 fuid = format(curfname[12], 'x').lower();
1805 funame = curfname[13];
1806 fgid = format(curfname[14], 'x').lower();
1807 fgname = curfname[15];
1808 fid = format(curfname[16], 'x').lower();
1809 finode = format(curfname[17], 'x').lower();
1810 flinkcount = format(curfname[18], 'x').lower();
1811 fdev_minor = format(curfname[19], 'x').lower();
1812 fdev_major = format(curfname[20], 'x').lower();
1813 frdev_minor = format(curfname[21], 'x').lower();
1814 frdev_major = format(curfname[22], 'x').lower();
1815 extradata = curfname[23];
1816 fchecksumtype = curfname[24];
1817 fcontents = curfname[25];
1818 catoutlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
1819 fcontents.seek(0, 0);
1820 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
1821 if(numfiles>0):
1822 fp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
1823 return fp;
1824
1825 def AppendInFileWithContent(infile, fp, dirlistfromtxt=False, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
1826 formatspecs = FormatSpecsListToDict(formatspecs);
1827 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, False, formatspecs);
1828 return AppendListsWithContent(inlist, fp, dirlistfromtxt, filevalues, extradata, followlink, checksumtype, formatspecs, verbose);
1829
1830 def AppendFilesWithContentToOutFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
1831 formatspecs = FormatSpecsListToDict(formatspecs);
1832 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1833 if(os.path.exists(outfile)):
1834 try:
1835 os.unlink(outfile);
1836 except OSError as e:
1837 pass;
1838 if(outfile=="-"):
1839 verbose = False;
1840 catfpfp = BytesIO();
1841 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1842 catfp = outfile;
1843 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1844 catfp = BytesIO();
1845 else:
1846 fbasename = os.path.splitext(outfile)[0];
1847 fextname = os.path.splitext(outfile)[1];
1848 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
1849 catfp = AppendFilesWithContent(infiles, catfp, dirlistfromtxt, filevalues, extradata, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose);
1850 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1851 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1852 try:
1853 catfp.flush();
1854 os.fsync(catfp.fileno());
1855 except io.UnsupportedOperation:
1856 pass;
1857 except AttributeError:
1858 pass;
1859 except OSError as e:
1860 pass;
1861 if(outfile=="-"):
1862 catfp.seek(0, 0);
1863 if(hasattr(sys.stdout, "buffer")):
1864 shutil.copyfileobj(catfp, sys.stdout.buffer);
1865 else:
1866 shutil.copyfileobj(catfp, sys.stdout);
1867 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1868 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1869 catfp.seek(0, 0);
1870 upload_file_to_internet_file(catfp, outfile);
1871 if(returnfp):
1872 catfp.seek(0, 0);
1873 return catfp;
1874 else:
1875 catfp.close();
1876 return True;
1877
1878 def AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
1879 formatspecs = FormatSpecsListToDict(formatspecs);
1880 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1881 if(os.path.exists(outfile)):
1882 try:
1883 os.unlink(outfile);
1884 except OSError as e:
1885 pass;
1886 if(outfile=="-"):
1887 verbose = False;
1888 catfpfp = BytesIO();
1889 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1890 catfp = outfile;
1891 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1892 catfp = BytesIO();
1893 else:
1894 fbasename = os.path.splitext(outfile)[0];
1895 fextname = os.path.splitext(outfile)[1];
1896 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
1897 catfp = AppendListsWithContent(inlist, catfp, dirlistfromtxt, filevalues, extradata, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose);
1898 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1899 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1900 try:
1901 catfp.flush();
1902 os.fsync(catfp.fileno());
1903 except io.UnsupportedOperation:
1904 pass;
1905 except AttributeError:
1906 pass;
1907 except OSError as e:
1908 pass;
1909 if(outfile=="-"):
1910 catfp.seek(0, 0);
1911 if(hasattr(sys.stdout, "buffer")):
1912 shutil.copyfileobj(catfp, sys.stdout.buffer);
1913 else:
1914 shutil.copyfileobj(catfp, sys.stdout);
1915 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1916 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1917 catfp.seek(0, 0);
1918 upload_file_to_internet_file(catfp, outfile);
1919 if(returnfp):
1920 catfp.seek(0, 0);
1921 return catfp;
1922 else:
1923 catfp.close();
1924 return True;
1925
1926 def AppendInFileWithContentToOutFile(infile, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
1927 formatspecs = FormatSpecsListToDict(formatspecs);
1928 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, False, formatspecs);
1929 return AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt, compression, compresswholefile, compressionlevel, filevalues, extradata, followlink, checksumtype, formatspecs, verbose, returnfp);
1930
1931 def PrintPermissionString(fchmode, ftype):
1932 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
1933 permissionstr = "";
1934 for fmodval in str(oct(fchmode))[-3:]:
1935 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
1936 if(ftype==0 or ftype==7):
1937 permissionstr = "-" + permissionstr;
1938 if(ftype==1):
1939 permissionstr = "h" + permissionstr;
1940 if(ftype==2):
1941 permissionstr = "l" + permissionstr;
1942 if(ftype==3):
1943 permissionstr = "c" + permissionstr;
1944 if(ftype==4):
1945 permissionstr = "b" + permissionstr;
1946 if(ftype==5):
1947 permissionstr = "d" + permissionstr;
1948 if(ftype==6):
1949 permissionstr = "f" + permissionstr;
1950 if(ftype==8):
1951 permissionstr = "D" + permissionstr;
1952 if(ftype==9):
1953 permissionstr = "p" + permissionstr;
1954 if(ftype==10):
1955 permissionstr = "w" + permissionstr;
1956 try:
1957 permissionoutstr = stat.filemode(fchmode);
1958 except AttributeError:
1959 permissionoutstr = permissionstr;
1960 except KeyError:
1961 permissionoutstr = permissionstr;
1962 return permissionoutstr;
1963
1964 def PrintPermissionStringAlt(fchmode, ftype):
1965 permissions = {
1966 '0': '---', '1': '--x', '2': '-w-', '3': '-wx',
1967 '4': 'r--', '5': 'r-x', '6': 'rw-', '7': 'rwx'
1968 };
1969 # Translate file mode into permission string
1970 permissionstr = ''.join([permissions[i] for i in str(oct(fchmode))[-3:]]);
1971 # Append file type indicator
1972 type_indicators = {
1973 0: '-', 1: 'h', 2: 'l', 3: 'c', 4: 'b',
1974 5: 'd', 6: 'f', 8: 'D', 9: 'p', 10: 'w'
1975 };
1976 file_type = type_indicators.get(ftype, '-');
1977 permissionstr = file_type + permissionstr;
1978 try:
1979 permissionoutstr = stat.filemode(fchmode);
1980 except AttributeError:
1981 permissionoutstr = permissionstr;
1982 return permissionoutstr;
1983
1984 def CompressionSupport():
1985 compression_list = [];
1986 try:
1987 import gzip;
1988 compression_list.append("gz");
1989 compression_list.append("gzip");
1990 except ImportError:
1991 '''return False;'''
1992 try:
1993 import bz2;
1994 compression_list.append("bz2");
1995 compression_list.append("bzip2");
1996 except ImportError:
1997 '''return False;'''
1998 try:
1999 import lz4;
2000 compression_list.append("lz4");
2001 except ImportError:
2002 '''return False;'''
2003 try:
2004 import lzo;
2005 compression_list.append("lzo");
2006 compression_list.append("lzop");
2007 except ImportError:
2008 '''return False;'''
2009 try:
2010 import zstandard;
2011 compression_list.append("zstd");
2012 compression_list.append("zstandard");
2013 except ImportError:
2014 '''return False;'''
2015 try:
2016 import lzma;
2017 compression_list.append("lzma");
2018 compression_list.append("xz");
2019 except ImportError:
2020 '''return False;'''
2021 return compression_list;
2022
2023 def CheckCompressionType(infile, formatspecs=__file_format_dict__, closefp=True):
2024 formatspecs = FormatSpecsListToDict(formatspecs);
2025 if(hasattr(infile, "read") or hasattr(infile, "write")):
2026 catfp = infile;
2027 else:
2028 try:
2029 catfp = open(infile, "rb");
2030 except FileNotFoundError:
2031 return False;
2032 catfp.seek(0, 0);
2033 prefp = catfp.read(2);
2034 filetype = False;
2035 if(prefp==binascii.unhexlify("1f8b")):
2036 filetype = "gzip";
2037 catfp.seek(0, 0);
2038 prefp = catfp.read(3);
2039 if(prefp==binascii.unhexlify("425a68")):
2040 filetype = "bzip2";
2041 if(prefp==binascii.unhexlify("5d0000")):
2042 filetype = "lzma";
2043 catfp.seek(0, 0);
2044 prefp = catfp.read(4);
2045 if(prefp==binascii.unhexlify("28b52ffd")):
2046 filetype = "zstd";
2047 if(prefp==binascii.unhexlify("04224d18")):
2048 filetype = "lz4";
2049 if(prefp==binascii.unhexlify("504B0304")):
2050 filetype = "zipfile";
2051 catfp.seek(0, 0);
2052 prefp = catfp.read(5);
2053 if(prefp==binascii.unhexlify("7573746172")):
2054 filetype = "tarfile";
2055 catfp.seek(0, 0);
2056 prefp = catfp.read(6);
2057 if(prefp==binascii.unhexlify("fd377a585a00")):
2058 filetype = "lzma";
2059 if(prefp==binascii.unhexlify("377abcaf271c")):
2060 filetype = "7zipfile";
2061 catfp.seek(0, 0);
2062 prefp = catfp.read(7);
2063 if(prefp==binascii.unhexlify("526172211a0700")):
2064 filetype = "rarfile";
2065 if(prefp==binascii.unhexlify("43617446696c65")):
2066 filetype = "catfile";
2067 catfp.seek(0, 0);
2068 prefp = catfp.read(8);
2069 if(prefp==binascii.unhexlify("526172211a070100")):
2070 filetype = "rarfile";
2071 catfp.seek(0, 0);
2072 prefp = catfp.read(formatspecs['format_len']);
2073 if(prefp==binascii.unhexlify(formatspecs['format_hex'])):
2074 filetype = formatspecs['format_lower'];
2075 catfp.seek(0, 0);
2076 prefp = catfp.read(9);
2077 if(prefp==binascii.unhexlify("894c5a4f000d0a1a0a")):
2078 filetype = "lzo";
2079 catfp.seek(0, 0);
2080 prefp = catfp.read(10);
2081 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
2082 filetype = "tarfile";
2083 catfp.seek(0, 0);
2084 if(filetype=="gzip" or filetype=="bzip2" or filetype=="lzma" or filetype=="zstd" or filetype=="lz4"):
2085 if(is_tarfile(catfp)):
2086 filetype = "tarfile";
2087 if(not filetype):
2088 if(is_tarfile(catfp)):
2089 filetype = "tarfile";
2090 elif(zipfile.is_zipfile(catfp)):
2091 filetype = "zipfile";
2092 elif(rarfile.is_rarfile(catfp) or rarfile.is_rarfile_sfx(catfp)):
2093 filetype = "rarile";
2094 else:
2095 filetype = False;
2096 catfp.seek(0, 0);
2097 if(closefp):
2098 catfp.close();
2099 return filetype;
2100
2101 def CheckCompressionTypeFromString(instring, formatspecs=__file_format_dict__, closefp=True):
2102 formatspecs = FormatSpecsListToDict(formatspecs);
2103 try:
2104 instringsfile = BytesIO(instring);
2105 except TypeError:
2106 instringsfile = BytesIO(instring.encode("UTF-8"));
2107 return CheckCompressionType(instringsfile, formatspecs, closefp);
2108
2109 def GetCompressionMimeType(infile, formatspecs=__file_format_dict__):
2110 formatspecs = FormatSpecsListToDict(formatspecs);
2111 compresscheck = CheckCompressionType(fp, formatspecs, False);
2112 if(compresscheck=="gzip" or compresscheck=="gz"):
2113 return archivefile_gzip_mimetype;
2114 if(compresscheck=="bzip2" or compresscheck=="bz2"):
2115 return archivefile_bzip2_mimetype;
2116 if(compresscheck=="zstd" or compresscheck=="zstandard"):
2117 return archivefile_zstandard_mimetype;
2118 if(compresscheck=="lz4"):
2119 return archivefile_lz4_mimetype;
2120 if(compresscheck=="lzo" or compresscheck=="lzop"):
2121 return archivefile_lzop_mimetype;
2122 if(compresscheck=="lzma"):
2123 return archivefile_lzma_mimetype;
2124 if(compresscheck=="xz"):
2125 return archivefile_xz_mimetype;
2126 if(compresscheck=="catfile" or compresscheck=="cat" or compresscheck==formatspecs['format_lower']):
2127 return archivefile_cat_mimetype;
2128 if(not compresscheck):
2129 return False;
2130 return False;
2131
2132 def UncompressArchiveFile(fp, formatspecs=__file_format_dict__):
2133 formatspecs = FormatSpecsListToDict(formatspecs);
2134 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
2135 return False;
2136 compresscheck = CheckCompressionType(fp, formatspecs, False);
2137 if(compresscheck=="gzip"):
2138 try:
2139 import gzip;
2140 except ImportError:
2141 return False;
2142 catfp = gzip.GzipFile(fileobj=fp, mode="rb");
2143 if(compresscheck=="bzip2"):
2144 try:
2145 import bz2;
2146 except ImportError:
2147 return False;
2148 catfp = BytesIO();
2149 catfp.write(bz2.decompress(fp.read()));
2150 if(compresscheck=="zstd"):
2151 try:
2152 import zstandard;
2153 except ImportError:
2154 return False;
2155 catfp = BytesIO();
2156 catfp.write(zstandard.decompress(fp.read()));
2157 if(compresscheck=="lz4"):
2158 try:
2159 import lz4.frame;
2160 except ImportError:
2161 return False;
2162 catfp = BytesIO();
2163 catfp.write(lz4.frame.decompress(fp.read()));
2164 if(compresscheck=="lzo" or compresscheck=="lzop"):
2165 try:
2166 import lzo;
2167 except ImportError:
2168 return False;
2169 catfp = BytesIO();
2170 catfp.write(lzo.decompress(fp.read()));
2171 if(compresscheck=="lzma" or compresscheck=="xz"):
2172 try:
2173 import lzma;
2174 except ImportError:
2175 return False;
2176 catfp = BytesIO();
2177 catfp.write(lzma.decompress(fp.read()));
2178 if(compresscheck=="catfile" or compresscheck==formatspecs['format_lower']):
2179 catfp = fp;
2180 if(not compresscheck):
2181 try:
2182 import lzma;
2183 except ImportError:
2184 return False;
2185 catfp = BytesIO();
2186 with fp as fpcontent:
2187 try:
2188 catfp.write(lzma.decompress(fp.read()));
2189 except lzma.LZMAError:
2190 return False;
2191 if(compresscheck!="catfile" or compresscheck!=formatspecs['format_lower']):
2192 fp.close();
2193 return catfp;
2194
2195 create_alias_function("Uncompress", __file_format_name__, "", UncompressArchiveFile);
2196
2197 def UncompressFile(infile, formatspecs=__file_format_dict__, mode="rb"):
2198 formatspecs = FormatSpecsListToDict(formatspecs);
2199 compresscheck = CheckCompressionType(infile, formatspecs, False);
2200 if(sys.version_info[0]==2 and compresscheck):
2201 if(mode=="rt"):
2202 mode = "r";
2203 if(mode=="wt"):
2204 mode = "w";
2205 try:
2206 if(compresscheck=="gzip"):
2207 try:
2208 import gzip;
2209 except ImportError:
2210 return False;
2211 try:
2212 filefp = gzip.open(infile, mode, encoding="UTF-8");
2213 except (ValueError, TypeError) as e:
2214 filefp = gzip.open(infile, mode);
2215 if(compresscheck=="bzip2"):
2216 try:
2217 import bz2;
2218 except ImportError:
2219 return False;
2220 try:
2221 filefp = bz2.open(infile, mode, encoding="UTF-8");
2222 except (ValueError, TypeError) as e:
2223 filefp = bz2.open(infile, mode);
2224 if(compresscheck=="zstd"):
2225 try:
2226 import zstandard;
2227 except ImportError:
2228 return False;
2229 try:
2230 filefp = zstandard.open(infile, mode, encoding="UTF-8");
2231 except (ValueError, TypeError) as e:
2232 filefp = zstandard.open(infile, mode);
2233 if(compresscheck=="lz4"):
2234 try:
2235 import lz4.frame;
2236 except ImportError:
2237 return False;
2238 try:
2239 filefp = lz4.frame.open(infile, mode, encoding="UTF-8");
2240 except (ValueError, TypeError) as e:
2241 filefp = lz4.frame.open(infile, mode);
2242 if(compresscheck=="lzo"):
2243 try:
2244 import lzo;
2245 except ImportError:
2246 return False;
2247 try:
2248 filefp = lzo.open(infile, mode, encoding="UTF-8");
2249 except (ValueError, TypeError) as e:
2250 filefp = lzo.open(infile, mode);
2251 if(compresscheck=="lzma"):
2252 try:
2253 import lzma;
2254 except ImportError:
2255 return False;
2256 try:
2257 filefp = lzma.open(infile, mode, encoding="UTF-8");
2258 except (ValueError, TypeError) as e:
2259 filefp = lzma.open(infile, mode);
2260 if(compresscheck=="catfile" or compresscheck==formatspecs['format_lower']):
2261 try:
2262 filefp = open(infile, mode, encoding="UTF-8");
2263 except (ValueError, TypeError) as e:
2264 filefp = open(infile, mode);
2265 if(not compresscheck):
2266 try:
2267 filefp = open(infile, mode, encoding="UTF-8");
2268 except (ValueError, TypeError) as e:
2269 filefp = open(infile, mode);
2270 except FileNotFoundError:
2271 return False;
2272 return filefp;
2273
2274 def UncompressString(infile):
2275 compresscheck = CheckCompressionTypeFromString(infile, formatspecs, False);
2276 if(compresscheck=="gzip"):
2277 try:
2278 import gzip;
2279 except ImportError:
2280 return False;
2281 fileuz = gzip.decompress(infile);
2282 if(compresscheck=="bzip2"):
2283 try:
2284 import bz2;
2285 except ImportError:
2286 return False;
2287 fileuz = bz2.decompress(infile);
2288 if(compresscheck=="zstd"):
2289 try:
2290 import zstandard;
2291 except ImportError:
2292 return False;
2293 fileuz = zstandard.decompress(infile);
2294 if(compresscheck=="lz4"):
2295 try:
2296 import lz4.frame;
2297 except ImportError:
2298 return False;
2299 fileuz = lz4.frame.decompress(infile);
2300 if(compresscheck=="lzo"):
2301 try:
2302 import lzo;
2303 except ImportError:
2304 return False;
2305 fileuz = lzo.decompress(infile);
2306 if(compresscheck=="lzma"):
2307 try:
2308 import lzma;
2309 except ImportError:
2310 return False;
2311 fileuz = lzma.decompress(infile);
2312 if(not compresscheck):
2313 fileuz = infile;
2314 if(hasattr(fileuz, 'decode')):
2315 fileuz = fileuz.decode("UTF-8");
2316 return fileuz;
2317
2318 def UncompressStringAlt(infile):
2319 filefp = StringIO();
2320 outstring = UncompressString(infile);
2321 filefp.write(outstring);
2322 filefp.seek(0, 0);
2323 return filefp;
2324
2325 def CheckCompressionSubType(infile, formatspecs=__file_format_dict__, closefp=True):
2326 formatspecs = FormatSpecsListToDict(formatspecs);
2327 compresscheck = CheckCompressionType(infile, formatspecs, False);
2328 if(not compresscheck):
2329 fextname = os.path.splitext(infile)[1];
2330 if(fextname==".gz"):
2331 compresscheck = "gzip";
2332 elif(fextname==".bz2"):
2333 compresscheck = "bzip2";
2334 elif(fextname==".zst"):
2335 compresscheck = "zstd";
2336 elif(fextname==".lz4"):
2337 compresscheck = "lz4";
2338 elif(fextname==".lzo" or fextname==".lzop"):
2339 compresscheck = "lzo";
2340 elif(fextname==".lzma" or fextname==".xz"):
2341 compresscheck = "lzma";
2342 else:
2343 return False;
2344 if(compresscheck=="gzip" or compresscheck=="bzip2" or compresscheck=="lzma" or compresscheck=="zstd" or compresscheck=="lz4"):
2345 if(is_tarfile(infile)):
2346 filetype = "tarfile";
2347 if(not compresscheck):
2348 if(is_tarfile(infile)):
2349 return "tarfile";
2350 elif(zipfile.is_zipfile(infile)):
2351 return "zipfile";
2352 elif(rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile)):
2353 return "rarile";
2354 else:
2355 return False;
2356 return False;
2357 if(compresscheck=="catfile"):
2358 return "catfile";
2359 if(compresscheck==formatspecs['format_lower']):
2360 return formatspecs['format_lower'];
2361 if(compresscheck=="tarfile"):
2362 return "tarfile";
2363 if(compresscheck=="zipfile"):
2364 return "zipfile";
2365 if(rarfile_support and compresscheck=="rarfile"):
2366 return "rarfile";
2367 if(py7zr_support and compresscheck=="7zipfile"):
2368 return "7zipfile";
2369 if(hasattr(infile, "read") or hasattr(infile, "write")):
2370 catfp = UncompressArchiveFile(infile, formatspecs['format_lower']);
2371 else:
2372 try:
2373 if(compresscheck=="gzip"):
2374 try:
2375 import gzip;
2376 except ImportError:
2377 return False;
2378 catfp = gzip.GzipFile(infile, "rb");
2379 if(compresscheck=="bzip2"):
2380 try:
2381 import bz2;
2382 except ImportError:
2383 return False;
2384 catfp = bz2.BZ2File(infile, "rb");
2385 if(compresscheck=="lz4"):
2386 try:
2387 import lz4.frame;
2388 except ImportError:
2389 return False;
2390 catfp = lz4.frame.open(infile, "rb");
2391 if(compresscheck=="zstd"):
2392 try:
2393 import zstandard;
2394 except ImportError:
2395 return False;
2396 catfp = zstandard.open(infile, "rb");
2397 if(compresscheck=="lzma" or compresscheck=="xz"):
2398 try:
2399 import lzma;
2400 except ImportError:
2401 return False;
2402 catfp = lzma.open(infile, "rb");
2403 except FileNotFoundError:
2404 return False;
2405 filetype = False;
2406 prefp = catfp.read(5);
2407 if(prefp==binascii.unhexlify("7573746172")):
2408 filetype = "tarfile";
2409 catfp.seek(0, 0);
2410 prefp = catfp.read(7);
2411 if(prefp==binascii.unhexlify("43617446696c65")):
2412 filetype = "catfile";
2413 catfp.seek(0, 0);
2414 prefp = catfp.read(formatspecs['format_len']);
2415 if(prefp==binascii.unhexlify(formatspecs['format_hex'])):
2416 filetype = formatspecs['format_lower'];
2417 catfp.seek(0, 0);
2418 prefp = catfp.read(10);
2419 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
2420 filetype = "tarfile";
2421 catfp.seek(0, 0);
2422 if(closefp):
2423 catfp.close();
2424 return filetype;
2425
2426 def GZipCompress(data, compresslevel=9):
2427 try:
2428 import gzip;
2429 except ImportError:
2430 return False;
2431 tmpfp = tempfile.NamedTemporaryFile("wb", delete=False);
2432 tmpfp.close();
2433 tmpfp = gzip.GzipFile(tmpfp.name, mode="wb", compresslevel=compresslevel);
2434 tmpfp.write(data);
2435 tmpfp.close();
2436 try:
2437 catfp = open(tmpfp.name, "rb");
2438 except FileNotFoundError:
2439 return False;
2440 catdata = catfp.read();
2441 catfp.close();
2442 return catdata;
2443
2444 def CompressArchiveFile(fp, compression="auto", compressionlevel=None, formatspecs=__file_format_dict__):
2445 formatspecs = FormatSpecsListToDict(formatspecs);
2446 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
2447 return False;
2448 fp.seek(0, 0);
2449 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
2450 compression = "auto";
2451 if(compression not in compressionlist and compression is None):
2452 compression = "auto";
2453 if(compression=="gzip"):
2454 try:
2455 import gzip;
2456 except ImportError:
2457 return False;
2458 catfp = BytesIO();
2459 if(compressionlevel is None):
2460 compressionlevel = 9;
2461 else:
2462 compressionlevel = int(compressionlevel);
2463 catfp.write(gzip.compress(fp.read(), compresslevel=compressionlevel));
2464 if(compression=="bzip2"):
2465 try:
2466 import bz2;
2467 except ImportError:
2468 return False;
2469 catfp = BytesIO();
2470 if(compressionlevel is None):
2471 compressionlevel = 9;
2472 else:
2473 compressionlevel = int(compressionlevel);
2474 catfp.write(bz2.compress(fp.read(), compresslevel=compressionlevel));
2475 if(compression=="lz4"):
2476 try:
2477 import lz4.frame;
2478 except ImportError:
2479 return False;
2480 catfp = BytesIO();
2481 if(compressionlevel is None):
2482 compressionlevel = 9;
2483 else:
2484 compressionlevel = int(compressionlevel);
2485 catfp.write(lz4.frame.compress(fp.read(), compression_level=compressionlevel));
2486 if(compression=="lzo" or compression=="lzop"):
2487 try:
2488 import lzo;
2489 except ImportError:
2490 return False;
2491 catfp = BytesIO();
2492 if(compressionlevel is None):
2493 compressionlevel = 9;
2494 else:
2495 compressionlevel = int(compressionlevel);
2496 catfp.write(lzo.compress(fp.read(), compresslevel=compressionlevel));
2497 if(compression=="zstd"):
2498 try:
2499 import zstandard;
2500 except ImportError:
2501 return False;
2502 catfp = BytesIO();
2503 if(compressionlevel is None):
2504 compressionlevel = 10;
2505 else:
2506 compressionlevel = int(compressionlevel);
2507 catfp.write(zstandard.compress(fp.read(), level=compressionlevel));
2508 if(compression=="lzma"):
2509 try:
2510 import lzma;
2511 except ImportError:
2512 return False;
2513 catfp = BytesIO();
2514 if(compressionlevel is None):
2515 compressionlevel = 9;
2516 else:
2517 compressionlevel = int(compressionlevel);
2518 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]));
2519 if(compression=="xz"):
2520 try:
2521 import lzma;
2522 except ImportError:
2523 return False;
2524 catfp = BytesIO();
2525 if(compressionlevel is None):
2526 compressionlevel = 9;
2527 else:
2528 compressionlevel = int(compressionlevel);
2529 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]));
2530 if(compression=="auto" or compression is None):
2531 catfp = fp;
2532 catfp.seek(0, 0);
2533 return catfp;
2534
2535 create_alias_function("Compress", __file_format_name__, "", CompressArchiveFile);
2536
2537 def CompressOpenFile(outfile, compressionenable=True, compressionlevel=None):
2538 if(outfile is None):
2539 return False;
2540 fbasename = os.path.splitext(outfile)[0];
2541 fextname = os.path.splitext(outfile)[1];
2542 if(compressionlevel is None and fextname!=".zst"):
2543 compressionlevel = 9;
2544 elif(compressionlevel is None and fextname==".zst"):
2545 compressionlevel = 10;
2546 else:
2547 compressionlevel = int(compressionlevel);
2548 if(sys.version_info[0]==2):
2549 mode = "w";
2550 else:
2551 mode = "wb";
2552 try:
2553 if(fextname not in outextlistwd or not compressionenable):
2554 try:
2555 outfp = open(outfile, "wb", encoding="UTF-8");
2556 except (ValueError, TypeError) as e:
2557 outfp = open(outfile, "wb");
2558 elif(fextname==".gz"):
2559 try:
2560 import gzip;
2561 except ImportError:
2562 return False;
2563 try:
2564 outfp = gzip.open(outfile, mode, compressionlevel, encoding="UTF-8");
2565 except (ValueError, TypeError) as e:
2566 outfp = gzip.open(outfile, mode, compressionlevel);
2567 elif(fextname==".bz2"):
2568 try:
2569 import bz2;
2570 except ImportError:
2571 return False;
2572 try:
2573 outfp = bz2.open(outfile, mode, compressionlevel, encoding="UTF-8");
2574 except (ValueError, TypeError) as e:
2575 outfp = bz2.open(outfile, mode, compressionlevel);
2576 elif(fextname==".zst"):
2577 try:
2578 import zstandard;
2579 except ImportError:
2580 return False;
2581 try:
2582 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel), encoding="UTF-8");
2583 except (ValueError, TypeError) as e:
2584 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel));
2585 elif(fextname==".xz"):
2586 try:
2587 import lzma;
2588 except ImportError:
2589 return False;
2590 try:
2591 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}], encoding="UTF-8");
2592 except (ValueError, TypeError) as e:
2593 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]);
2594 elif(fextname==".lz4"):
2595 try:
2596 import lz4.frame;
2597 except ImportError:
2598 return False;
2599 try:
2600 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel, encoding="UTF-8");
2601 except (ValueError, TypeError) as e:
2602 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel);
2603 elif(fextname==".lzo"):
2604 try:
2605 import lzo;
2606 except ImportError:
2607 return False;
2608 try:
2609 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel, encoding="UTF-8");
2610 except (ValueError, TypeError) as e:
2611 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel);
2612 elif(fextname==".lzma"):
2613 try:
2614 import lzma;
2615 except ImportError:
2616 return False;
2617 try:
2618 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}], encoding="UTF-8");
2619 except (ValueError, TypeError) as e:
2620 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]);
2621 except FileNotFoundError:
2622 return False;
2623 return outfp;
2624
2625 def GetDevMajorMinor(fdev):
2626 retdev = [];
2627 if(hasattr(os, "minor")):
2628 retdev.append(os.minor(fdev));
2629 else:
2630 retdev.append(0);
2631 if(hasattr(os, "major")):
2632 retdev.append(os.major(fdev));
2633 else:
2634 retdev.append(0);
2635 return retdev;
2636
2637 def CheckSumSupport(checkfor, guaranteed=True):
2638 if(guaranteed):
2639 try:
2640 hash_list = sorted(list(hashlib.algorithms_guaranteed));
2641 except AttributeError:
2642 hash_list = sorted(list(hashlib.algorithms));
2643 else:
2644 try:
2645 hash_list = sorted(list(hashlib.algorithms_available));
2646 except AttributeError:
2647 hash_list = sorted(list(hashlib.algorithms));
2648 checklistout = sorted(hash_list + ['adler32', 'crc16', 'crc16_ansi', 'crc16_ibm', 'crc16_ccitt', 'crc32', 'crc64', 'crc64_ecma', 'crc64_iso', 'none']);
2649 if(checkfor in checklistout):
2650 return True;
2651 else:
2652 return False;
2653
2654 def CheckSumSupportAlt(checkfor, guaranteed=True):
2655 if(guaranteed):
2656 try:
2657 hash_list = sorted(list(hashlib.algorithms_guaranteed));
2658 except AttributeError:
2659 hash_list = sorted(list(hashlib.algorithms));
2660 else:
2661 try:
2662 hash_list = sorted(list(hashlib.algorithms_available));
2663 except AttributeError:
2664 hash_list = sorted(list(hashlib.algorithms));
2665 checklistout = hash_list;
2666 if(checkfor in checklistout):
2667 return True;
2668 else:
2669 return False;
2670
2671 def PackArchiveFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2672 formatspecs = FormatSpecsListToDict(formatspecs);
2673 advancedlist = formatspecs['use_advanced_list'];
2674 altinode = formatspecs['use_alt_inode'];
2675 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2676 outfile = RemoveWindowsPath(outfile);
2677 checksumtype = checksumtype.lower();
2678 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
2679 checksumtype="crc32";
2680 if(checksumtype=="none"):
2681 checksumtype = "";
2682 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
2683 compression = "auto";
2684 if(compression not in compressionlist and compression is None):
2685 compression = "auto";
2686 if(verbose):
2687 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2688 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2689 if(os.path.exists(outfile)):
2690 try:
2691 os.unlink(outfile);
2692 except OSError as e:
2693 pass;
2694 if(outfile=="-"):
2695 verbose = False;
2696 catfp = BytesIO();
2697 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2698 catfp = outfile;
2699 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2700 catfp = BytesIO();
2701 else:
2702 fbasename = os.path.splitext(outfile)[0];
2703 fextname = os.path.splitext(outfile)[1];
2704 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
2705 catver = formatspecs['format_ver'];
2706 fileheaderver = str(int(catver.replace(".", "")));
2707 infilelist = [];
2708 if(infiles=="-"):
2709 for line in sys.stdin:
2710 infilelist.append(line.strip());
2711 infilelist = list(filter(None, infilelist));
2712 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
2713 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
2714 return False;
2715 with UncompressFile(infiles, formatspecs, "r") as finfile:
2716 for line in finfile:
2717 infilelist.append(line.strip());
2718 infilelist = list(filter(None, infilelist));
2719 else:
2720 if(isinstance(infiles, (list, tuple, ))):
2721 infilelist = list(filter(None, infiles));
2722 elif(isinstance(infiles, (str, ))):
2723 infilelist = list(filter(None, [infiles]));
2724 if(advancedlist):
2725 GetDirList = ListDirAdvanced(infilelist, followlink, False);
2726 else:
2727 GetDirList = ListDir(infilelist, followlink, False);
2728 if(not GetDirList):
2729 return False;
2730 curinode = 0;
2731 curfid = 0;
2732 inodelist = [];
2733 inodetofile = {};
2734 filetoinode = {};
2735 inodetocatinode = {};
2736 numfiles = int(len(GetDirList));
2737 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
2738 for curfname in GetDirList:
2739 if(re.findall("^[.|/]", curfname)):
2740 fname = curfname;
2741 else:
2742 fname = "./"+curfname;
2743 if(verbose):
2744 VerbosePrintOut(fname);
2745 if(not followlink or followlink is None):
2746 fstatinfo = os.lstat(fname);
2747 else:
2748 fstatinfo = os.stat(fname);
2749 fpremode = fstatinfo.st_mode;
2750 finode = fstatinfo.st_ino;
2751 flinkcount = fstatinfo.st_nlink;
2752 ftype = 0;
2753 if(stat.S_ISREG(fpremode)):
2754 ftype = 0;
2755 elif(stat.S_ISLNK(fpremode)):
2756 ftype = 2;
2757 elif(stat.S_ISCHR(fpremode)):
2758 ftype = 3;
2759 elif(stat.S_ISBLK(fpremode)):
2760 ftype = 4;
2761 elif(stat.S_ISDIR(fpremode)):
2762 ftype = 5;
2763 elif(stat.S_ISFIFO(fpremode)):
2764 ftype = 6;
2765 elif(stat.S_ISSOCK(fpremode)):
2766 ftype = 8;
2767 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
2768 ftype = 9;
2769 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
2770 ftype = 10;
2771 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
2772 ftype = 111;
2773 else:
2774 ftype = 0;
2775 flinkname = "";
2776 fcurfid = format(int(curfid), 'x').lower();
2777 if(not followlink and finode!=0):
2778 if(ftype!=1):
2779 if(finode in inodelist):
2780 ftype = 1;
2781 flinkname = inodetofile[finode];
2782 if(altinode):
2783 fcurinode = format(int(finode), 'x').lower();
2784 else:
2785 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
2786 if(finode not in inodelist):
2787 inodelist.append(finode);
2788 inodetofile.update({finode: fname});
2789 inodetocatinode.update({finode: curinode});
2790 if(altinode):
2791 fcurinode = format(int(finode), 'x').lower();
2792 else:
2793 fcurinode = format(int(curinode), 'x').lower();
2794 curinode = curinode + 1;
2795 else:
2796 fcurinode = format(int(curinode), 'x').lower();
2797 curinode = curinode + 1;
2798 curfid = curfid + 1;
2799 if(ftype==2):
2800 flinkname = os.readlink(fname);
2801 fdev = fstatinfo.st_dev;
2802 getfdev = GetDevMajorMinor(fdev);
2803 fdev_minor = getfdev[0];
2804 fdev_major = getfdev[1];
2805 frdev = fstatinfo.st_dev;
2806 if(hasattr(fstatinfo, "st_rdev")):
2807 frdev = fstatinfo.st_rdev;
2808 else:
2809 frdev = fstatinfo.st_dev;
2810 getfrdev = GetDevMajorMinor(frdev);
2811 frdev_minor = getfrdev[0];
2812 frdev_major = getfrdev[1];
2813 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
2814 fsize = format(int("0"), 'x').lower();
2815 elif(ftype==0 or ftype==7):
2816 fsize = format(int(fstatinfo.st_size), 'x').lower();
2817 else:
2818 fsize = format(int(fstatinfo.st_size)).lower();
2819 fatime = format(int(fstatinfo.st_atime), 'x').lower();
2820 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
2821 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
2822 if(hasattr(fstatinfo, "st_birthtime")):
2823 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
2824 else:
2825 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
2826 fmode = format(int(fstatinfo.st_mode), 'x').lower();
2827 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
2828 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
2829 fuid = format(int(fstatinfo.st_uid), 'x').lower();
2830 fgid = format(int(fstatinfo.st_gid), 'x').lower();
2831 funame = "";
2832 try:
2833 import pwd;
2834 try:
2835 userinfo = pwd.getpwuid(fstatinfo.st_uid);
2836 funame = userinfo.pw_name;
2837 except KeyError:
2838 funame = "";
2839 except ImportError:
2840 funame = "";
2841 fgname = "";
2842 try:
2843 import grp;
2844 try:
2845 groupinfo = grp.getgrgid(fstatinfo.st_gid);
2846 fgname = groupinfo.gr_name;
2847 except KeyError:
2848 fgname = "";
2849 except ImportError:
2850 fgname = "";
2851 fdev_minor = format(int(fdev_minor), 'x').lower();
2852 fdev_major = format(int(fdev_major), 'x').lower();
2853 frdev_minor = format(int(frdev_minor), 'x').lower();
2854 frdev_major = format(int(frdev_major), 'x').lower();
2855 finode = format(int(finode), 'x').lower();
2856 flinkcount = format(int(flinkcount), 'x').lower();
2857 if(hasattr(fstatinfo, "st_file_attributes")):
2858 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
2859 else:
2860 fwinattributes = format(int(0), 'x').lower();
2861 fcompression = "";
2862 fcsize = format(int(0), 'x').lower();
2863 fcontents = BytesIO();
2864 if(ftype==0 or ftype==7):
2865 with open(fname, "rb") as fpc:
2866 shutil.copyfileobj(fpc, fcontents);
2867 if(not compresswholefile):
2868 fcontents.seek(0, 2);
2869 ucfsize = fcontents.tell();
2870 fcontents.seek(0, 0);
2871 if(compression=="auto"):
2872 ilsize = len(compressionlistalt);
2873 ilmin = 0;
2874 ilcsize = [];
2875 while(ilmin < ilsize):
2876 cfcontents = BytesIO();
2877 shutil.copyfileobj(fcontents, cfcontents);
2878 fcontents.seek(0, 0);
2879 cfcontents.seek(0, 0);
2880 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
2881 if(cfcontents):
2882 cfcontents.seek(0, 2);
2883 ilcsize.append(cfcontents.tell());
2884 cfcontents.close();
2885 else:
2886 try:
2887 ilcsize.append(sys.maxint);
2888 except AttributeError:
2889 ilcsize.append(sys.maxsize);
2890 ilmin = ilmin + 1;
2891 ilcmin = ilcsize.index(min(ilcsize));
2892 compression = compressionlistalt[ilcmin];
2893 fcontents.seek(0, 0);
2894 cfcontents = BytesIO();
2895 shutil.copyfileobj(fcontents, cfcontents);
2896 cfcontents.seek(0, 0);
2897 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
2898 cfcontents.seek(0, 2);
2899 cfsize = cfcontents.tell();
2900 if(ucfsize > cfsize):
2901 fcsize = format(int(cfsize), 'x').lower();
2902 fcompression = compression;
2903 fcontents.close();
2904 fcontents = cfcontents;
2905 if(fcompression=="none"):
2906 fcompression = "";
2907 if(followlink and (ftype==1 or ftype==2)):
2908 flstatinfo = os.stat(flinkname);
2909 with open(flinkname, "rb") as fpc:
2910 shutil.copyfileobj(fpc, fcontents);
2911 if(not compresswholefile):
2912 fcontents.seek(0, 2);
2913 ucfsize = fcontents.tell();
2914 fcontents.seek(0, 0);
2915 if(compression=="auto"):
2916 ilsize = len(compressionlistalt);
2917 ilmin = 0;
2918 ilcsize = [];
2919 while(ilmin < ilsize):
2920 cfcontents = BytesIO();
2921 shutil.copyfileobj(fcontents, cfcontents);
2922 fcontents.seek(0, 0);
2923 cfcontents.seek(0, 0);
2924 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
2925 if(cfcontents):
2926 cfcontents.seek(0, 2);
2927 ilcsize.append(cfcontents.tell());
2928 cfcontents.close();
2929 else:
2930 try:
2931 ilcsize.append(sys.maxint);
2932 except AttributeError:
2933 ilcsize.append(sys.maxsize);
2934 ilmin = ilmin + 1;
2935 ilcmin = ilcsize.index(min(ilcsize));
2936 compression = compressionlistalt[ilcmin];
2937 fcontents.seek(0, 0);
2938 cfcontents = BytesIO();
2939 shutil.copyfileobj(fcontents, cfcontents);
2940 cfcontents.seek(0, 0);
2941 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
2942 cfcontents.seek(0, 2);
2943 cfsize = cfcontents.tell();
2944 if(ucfsize > cfsize):
2945 fcsize = format(int(cfsize), 'x').lower();
2946 fcompression = compression;
2947 fcontents.close();
2948 fcontents = cfcontents;
2949 fcontents.seek(0, 0);
2950 ftypehex = format(ftype, 'x').lower();
2951 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
2952 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
2953 fcontents.close();
2954 if(numfiles>0):
2955 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
2956 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
2957 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2958 try:
2959 catfp.flush();
2960 os.fsync(catfp.fileno());
2961 except io.UnsupportedOperation:
2962 pass;
2963 except AttributeError:
2964 pass;
2965 except OSError as e:
2966 pass;
2967 if(outfile=="-"):
2968 catfp.seek(0, 0);
2969 if(hasattr(sys.stdout, "buffer")):
2970 shutil.copyfileobj(catfp, sys.stdout.buffer);
2971 else:
2972 shutil.copyfileobj(catfp, sys.stdout);
2973 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2974 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2975 catfp.seek(0, 0);
2976 upload_file_to_internet_file(catfp, outfile);
2977 if(returnfp):
2978 catfp.seek(0, 0);
2979 return catfp;
2980 else:
2981 catfp.close();
2982 return True;
2983
2984 create_alias_function("Pack", __file_format_name__, "", PackArchiveFile);
2985
2986 if(hasattr(shutil, "register_archive_format")):
2987 def PackArchiveFileFunc(archive_name, source_dir, **kwargs):
2988 return PackArchiveFile(source_dir, archive_name, False, "auto", True, None, False, "crc32", [], __file_format_dict__['format_delimiter'], False, False);
2989 create_alias_function("Pack", __file_format_name__, "Func", PackArchiveFileFunc);
2990
2991 def PackArchiveFileFromDirList(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2992 formatspecs = FormatSpecsListToDict(formatspecs);
2993 return PackArchiveFile(infiles, outfile, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, returnfp);
2994
2995 create_alias_function("Pack", __file_format_name__, "FromDirList", PackArchiveFileFromDirList);
2996
2997 def PackArchiveFileFromTarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2998 formatspecs = FormatSpecsListToDict(formatspecs);
2999 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3000 outfile = RemoveWindowsPath(outfile);
3001 checksumtype = checksumtype.lower();
3002 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3003 checksumtype="crc32";
3004 if(checksumtype=="none"):
3005 checksumtype = "";
3006 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3007 compression = "auto";
3008 if(compression not in compressionlist and compression is None):
3009 compression = "auto";
3010 if(verbose):
3011 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3012 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3013 if(os.path.exists(outfile)):
3014 try:
3015 os.unlink(outfile);
3016 except OSError as e:
3017 pass;
3018 if(outfile=="-"):
3019 verbose = False;
3020 catfp = BytesIO();
3021 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3022 catfp = outfile;
3023 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3024 catfp = BytesIO();
3025 else:
3026 fbasename = os.path.splitext(outfile)[0];
3027 fextname = os.path.splitext(outfile)[1];
3028 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3029 catver = formatspecs['format_ver'];
3030 fileheaderver = str(int(catver.replace(".", "")));
3031 curinode = 0;
3032 curfid = 0;
3033 inodelist = [];
3034 inodetofile = {};
3035 filetoinode = {};
3036 inodetocatinode = {};
3037 if(infile=="-"):
3038 infile = BytesIO();
3039 if(hasattr(sys.stdin, "buffer")):
3040 shutil.copyfileobj(sys.stdin.buffer, infile);
3041 else:
3042 shutil.copyfileobj(sys.stdin, infile);
3043 infile.seek(0, 0);
3044 if(not infile):
3045 return False;
3046 infile.seek(0, 0);
3047 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3048 infile = download_file_from_internet_file(infile);
3049 infile.seek(0, 0);
3050 if(not infile):
3051 return False;
3052 infile.seek(0, 0);
3053 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
3054 return False;
3055 elif(os.path.exists(infile) and os.path.isfile(infile)):
3056 try:
3057 if(not tarfile.is_tarfile(infile)):
3058 return False;
3059 except AttributeError:
3060 if(not is_tarfile(infile)):
3061 return False;
3062 else:
3063 return False;
3064 try:
3065 if(hasattr(infile, "read") or hasattr(infile, "write")):
3066 tarfp = tarfile.open(fileobj=infile, mode="r");
3067 else:
3068 tarfp = tarfile.open(infile, "r");
3069 except FileNotFoundError:
3070 return False;
3071 numfiles = int(len(tarfp.getmembers()));
3072 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3073 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
3074 if(re.findall("^[.|/]", member.name)):
3075 fname = member.name;
3076 else:
3077 fname = "./"+member.name;
3078 if(verbose):
3079 VerbosePrintOut(fname);
3080 fpremode = member.mode;
3081 ffullmode = member.mode;
3082 flinkcount = 0;
3083 ftype = 0;
3084 if(member.isreg()):
3085 ffullmode = member.mode + stat.S_IFREG;
3086 ftype = 0;
3087 elif(member.isdev()):
3088 ffullmode = member.mode;
3089 ftype = 7;
3090 elif(member.islnk()):
3091 ffullmode = member.mode + stat.S_IFREG;
3092 ftype = 1;
3093 elif(member.issym()):
3094 ffullmode = member.mode + stat.S_IFLNK;
3095 ftype = 2;
3096 elif(member.ischr()):
3097 ffullmode = member.mode + stat.S_IFCHR;
3098 ftype = 3;
3099 elif(member.isblk()):
3100 ffullmode = member.mode + stat.S_IFBLK;
3101 ftype = 4;
3102 elif(member.isdir()):
3103 ffullmode = member.mode + stat.S_IFDIR;
3104 ftype = 5;
3105 elif(member.isfifo()):
3106 ffullmode = member.mode + stat.S_IFIFO;
3107 ftype = 6;
3108 elif(member.issparse()):
3109 ffullmode = member.mode;
3110 ftype = 12;
3111 else:
3112 ffullmode = member.mode;
3113 ftype = 0;
3114 flinkname = "";
3115 fcurfid = format(int(curfid), 'x').lower();
3116 fcurinode = format(int(curfid), 'x').lower();
3117 curfid = curfid + 1;
3118 if(ftype==2):
3119 flinkname = member.linkname;
3120 fdev_minor = format(int(member.devminor), 'x').lower();
3121 fdev_major = format(int(member.devmajor), 'x').lower();
3122 frdev_minor = format(int(member.devminor), 'x').lower();
3123 frdev_major = format(int(member.devmajor), 'x').lower();
3124 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
3125 fsize = format(int("0"), 'x').lower();
3126 elif(ftype==0 or ftype==7):
3127 fsize = format(int(member.size), 'x').lower();
3128 else:
3129 fsize = format(int(member.size), 'x').lower();
3130 fatime = format(int(member.mtime), 'x').lower();
3131 fmtime = format(int(member.mtime), 'x').lower();
3132 fctime = format(int(member.mtime), 'x').lower();
3133 fbtime = format(int(member.mtime), 'x').lower();
3134 fmode = format(int(ffullmode), 'x').lower();
3135 fchmode = format(int(stat.S_IMODE(ffullmode)), 'x').lower();
3136 ftypemod = format(int(stat.S_IFMT(ffullmode)), 'x').lower();
3137 fuid = format(int(member.uid), 'x').lower();
3138 fgid = format(int(member.gid), 'x').lower();
3139 funame = member.uname;
3140 fgname = member.gname;
3141 flinkcount = format(int(flinkcount), 'x').lower();
3142 fwinattributes = format(int(0), 'x').lower();
3143 fcompression = "";
3144 fcsize = format(int(0), 'x').lower();
3145 fcontents = BytesIO();
3146 if(ftype==0 or ftype==7):
3147 with tarfp.extractfile(member) as fpc:
3148 shutil.copyfileobj(fpc, fcontents);
3149 if(not compresswholefile):
3150 fcontents.seek(0, 2);
3151 ucfsize = fcontents.tell();
3152 fcontents.seek(0, 0);
3153 if(compression=="auto"):
3154 ilsize = len(compressionlistalt);
3155 ilmin = 0;
3156 ilcsize = [];
3157 while(ilmin < ilsize):
3158 cfcontents = BytesIO();
3159 shutil.copyfileobj(fcontents, cfcontents);
3160 fcontents.seek(0, 0);
3161 cfcontents.seek(0, 0);
3162 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3163 if(cfcontents):
3164 cfcontents.seek(0, 2);
3165 ilcsize.append(cfcontents.tell());
3166 cfcontents.close();
3167 else:
3168 try:
3169 ilcsize.append(sys.maxint);
3170 except AttributeError:
3171 ilcsize.append(sys.maxsize);
3172 ilmin = ilmin + 1;
3173 ilcmin = ilcsize.index(min(ilcsize));
3174 compression = compressionlistalt[ilcmin];
3175 fcontents.seek(0, 0);
3176 cfcontents = BytesIO();
3177 shutil.copyfileobj(fcontents, cfcontents);
3178 cfcontents.seek(0, 0);
3179 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3180 cfcontents.seek(0, 2);
3181 cfsize = cfcontents.tell();
3182 if(ucfsize > cfsize):
3183 fcsize = format(int(cfsize), 'x').lower();
3184 fcompression = compression;
3185 fcontents.close();
3186 fcontents = cfcontents;
3187 if(fcompression=="none"):
3188 fcompression = "";
3189 fcontents.seek(0, 0);
3190 ftypehex = format(ftype, 'x').lower();
3191 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3192 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3193 fcontents.close();
3194 if(numfiles>0):
3195 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3196 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3197 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3198 try:
3199 catfp.flush();
3200 os.fsync(catfp.fileno());
3201 except io.UnsupportedOperation:
3202 pass;
3203 except AttributeError:
3204 pass;
3205 except OSError as e:
3206 pass;
3207 if(outfile=="-"):
3208 catfp.seek(0, 0);
3209 if(hasattr(sys.stdout, "buffer")):
3210 shutil.copyfileobj(catfp, sys.stdout.buffer);
3211 else:
3212 shutil.copyfileobj(catfp, sys.stdout);
3213 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3214 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3215 catfp.seek(0, 0);
3216 upload_file_to_internet_file(catfp, outfile);
3217 if(returnfp):
3218 catfp.seek(0, 0);
3219 return catfp;
3220 else:
3221 catfp.close();
3222 return True;
3223
3224 create_alias_function("Pack", __file_format_name__, "FromTarFile", PackArchiveFileFromTarFile);
3225
3226 def PackArchiveFileFromZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3227 formatspecs = FormatSpecsListToDict(formatspecs);
3228 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3229 outfile = RemoveWindowsPath(outfile);
3230 checksumtype = checksumtype.lower();
3231 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3232 checksumtype="crc32";
3233 if(checksumtype=="none"):
3234 checksumtype = "";
3235 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3236 compression = "auto";
3237 if(compression not in compressionlist and compression is None):
3238 compression = "auto";
3239 if(verbose):
3240 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3241 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3242 if(os.path.exists(outfile)):
3243 try:
3244 os.unlink(outfile);
3245 except OSError as e:
3246 pass;
3247 if(outfile=="-"):
3248 verbose = False;
3249 catfp = BytesIO();
3250 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3251 catfp = outfile;
3252 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3253 catfp = BytesIO();
3254 else:
3255 fbasename = os.path.splitext(outfile)[0];
3256 fextname = os.path.splitext(outfile)[1];
3257 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3258 catver = formatspecs['format_ver'];
3259 fileheaderver = str(int(catver.replace(".", "")));
3260 curinode = 0;
3261 curfid = 0;
3262 inodelist = [];
3263 inodetofile = {};
3264 filetoinode = {};
3265 inodetocatinode = {};
3266 if(infile=="-"):
3267 infile = BytesIO();
3268 if(hasattr(sys.stdin, "buffer")):
3269 shutil.copyfileobj(sys.stdin.buffer, infile);
3270 else:
3271 shutil.copyfileobj(sys.stdin, infile);
3272 infile.seek(0, 0);
3273 if(not infile):
3274 return False;
3275 infile.seek(0, 0);
3276 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3277 infile = download_file_from_internet_file(infile);
3278 infile.seek(0, 0);
3279 if(not infile):
3280 return False;
3281 infile.seek(0, 0);
3282 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
3283 return False;
3284 else:
3285 return False;
3286 if(not zipfile.is_zipfile(infile)):
3287 return False;
3288 try:
3289 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
3290 except FileNotFoundError:
3291 return False;
3292 ziptest = zipfp.testzip();
3293 if(ziptest):
3294 VerbosePrintOut("Bad file found!");
3295 numfiles = int(len(zipfp.infolist()));
3296 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3297 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
3298 if(re.findall("^[.|/]", member.filename)):
3299 fname = member.filename;
3300 else:
3301 fname = "./"+member.filename;
3302 zipinfo = zipfp.getinfo(member.filename);
3303 if(verbose):
3304 VerbosePrintOut(fname);
3305 if(not member.is_dir()):
3306 fpremode = int(stat.S_IFREG + 438);
3307 elif(member.is_dir()):
3308 fpremode = int(stat.S_IFDIR + 511);
3309 flinkcount = 0;
3310 ftype = 0;
3311 if(not member.is_dir()):
3312 ftype = 0;
3313 elif(member.is_dir()):
3314 ftype = 5;
3315 flinkname = "";
3316 fcurfid = format(int(curfid), 'x').lower();
3317 fcurinode = format(int(curfid), 'x').lower();
3318 curfid = curfid + 1;
3319 fdev_minor = format(int(0), 'x').lower();
3320 fdev_major = format(int(0), 'x').lower();
3321 frdev_minor = format(int(0), 'x').lower();
3322 frdev_major = format(int(0), 'x').lower();
3323 if(ftype==5):
3324 fsize = format(int("0"), 'x').lower();
3325 elif(ftype==0):
3326 fsize = format(int(member.file_size), 'x').lower();
3327 else:
3328 fsize = format(int(member.file_size), 'x').lower();
3329 fatime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3330 fmtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3331 fctime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3332 fbtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3333 if(zipinfo.create_system==0 or zipinfo.create_system==10):
3334 fwinattributes = format(int(zipinfo.external_attr), 'x').lower();
3335 if(not member.is_dir()):
3336 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3337 fchmode = stat.S_IMODE(fmode);
3338 ftypemod = stat.S_IFMT(fmode);
3339 elif(member.is_dir()):
3340 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3341 fchmode = stat.S_IMODE(fmode);
3342 ftypemod = stat.S_IFMT(fmode);
3343 elif(zipinfo.create_system==3):
3344 fwinattributes = format(int(0), 'x').lower();
3345 fmode = format(int(zipinfo.external_attr), 'x').lower();
3346 fchmode = stat.S_IMODE(fmode);
3347 ftypemod = stat.S_IFMT(fmode);
3348 else:
3349 fwinattributes = format(int(0), 'x').lower();
3350 if(not member.is_dir()):
3351 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3352 fchmode = stat.S_IMODE(fmode);
3353 ftypemod = stat.S_IFMT(fmode);
3354 elif(member.is_dir()):
3355 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3356 fchmode = stat.S_IMODE(fmode);
3357 ftypemod = stat.S_IFMT(fmode);
3358 fcompression = "";
3359 fcsize = format(int(0), 'x').lower();
3360 try:
3361 fuid = format(int(os.getuid()), 'x').lower();
3362 except AttributeError:
3363 fuid = format(int(0), 'x').lower();
3364 except KeyError:
3365 fuid = format(int(0), 'x').lower();
3366 try:
3367 fgid = format(int(os.getgid()), 'x').lower();
3368 except AttributeError:
3369 fgid = format(int(0), 'x').lower();
3370 except KeyError:
3371 fgid = format(int(0), 'x').lower();
3372 try:
3373 import pwd;
3374 try:
3375 userinfo = pwd.getpwuid(os.getuid());
3376 funame = userinfo.pw_name;
3377 except KeyError:
3378 funame = "";
3379 except AttributeError:
3380 funame = "";
3381 except ImportError:
3382 funame = "";
3383 fgname = "";
3384 try:
3385 import grp;
3386 try:
3387 groupinfo = grp.getgrgid(os.getgid());
3388 fgname = groupinfo.gr_name;
3389 except KeyError:
3390 fgname = "";
3391 except AttributeError:
3392 fgname = "";
3393 except ImportError:
3394 fgname = "";
3395 fcontents = BytesIO();
3396 if(ftype==0):
3397 fcontents.write(zipfp.read(member.filename));
3398 if(not compresswholefile):
3399 fcontents.seek(0, 2);
3400 ucfsize = fcontents.tell();
3401 fcontents.seek(0, 0);
3402 if(compression=="auto"):
3403 ilsize = len(compressionlistalt);
3404 ilmin = 0;
3405 ilcsize = [];
3406 while(ilmin < ilsize):
3407 cfcontents = BytesIO();
3408 shutil.copyfileobj(fcontents, cfcontents);
3409 fcontents.seek(0, 0);
3410 cfcontents.seek(0, 0);
3411 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3412 cfcontents.seek(0, 2);
3413 ilcsize.append(cfcontents.tell());
3414 cfcontents.close();
3415 ilmin = ilmin + 1;
3416 ilcmin = ilcsize.index(min(ilcsize));
3417 compression = compressionlistalt[ilcmin];
3418 fcontents.seek(0, 0);
3419 cfcontents = BytesIO();
3420 shutil.copyfileobj(fcontents, cfcontents);
3421 cfcontents.seek(0, 0);
3422 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3423 cfcontents.seek(0, 2);
3424 cfsize = cfcontents.tell();
3425 if(ucfsize > cfsize):
3426 fcsize = format(int(cfsize), 'x').lower();
3427 fcompression = compression;
3428 fcontents.close();
3429 fcontents = cfcontents;
3430 if(fcompression=="none"):
3431 fcompression = "";
3432 fcontents.seek(0, 0);
3433 ftypehex = format(ftype, 'x').lower();
3434 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3435 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3436 fcontents.close();
3437 if(numfiles>0):
3438 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3439 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3440 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3441 try:
3442 catfp.flush();
3443 os.fsync(catfp.fileno());
3444 except io.UnsupportedOperation:
3445 pass;
3446 except AttributeError:
3447 pass;
3448 except OSError as e:
3449 pass;
3450 if(outfile=="-"):
3451 catfp.seek(0, 0);
3452 if(hasattr(sys.stdout, "buffer")):
3453 shutil.copyfileobj(catfp, sys.stdout.buffer);
3454 else:
3455 shutil.copyfileobj(catfp, sys.stdout);
3456 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3457 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3458 catfp.seek(0, 0);
3459 upload_file_to_internet_file(catfp, outfile);
3460 if(returnfp):
3461 catfp.seek(0, 0);
3462 return catfp;
3463 else:
3464 catfp.close();
3465 return True;
3466
3467 create_alias_function("Pack", __file_format_name__, "FromZipFile", PackArchiveFileFromZipFile);
3468
3469 if(not rarfile_support):
3470 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3471 return False
3472
3473 if(rarfile_support):
3474 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3475 formatspecs = FormatSpecsListToDict(formatspecs);
3476 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3477 outfile = RemoveWindowsPath(outfile);
3478 checksumtype = checksumtype.lower();
3479 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3480 checksumtype="crc32";
3481 if(checksumtype=="none"):
3482 checksumtype = "";
3483 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3484 compression = "auto";
3485 if(compression not in compressionlist and compression is None):
3486 compression = "auto";
3487 if(verbose):
3488 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3489 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3490 if(os.path.exists(outfile)):
3491 try:
3492 os.unlink(outfile);
3493 except OSError as e:
3494 pass;
3495 if(outfile=="-"):
3496 verbose = False;
3497 catfp = BytesIO();
3498 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3499 catfp = outfile;
3500 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3501 catfp = BytesIO();
3502 else:
3503 fbasename = os.path.splitext(outfile)[0];
3504 fextname = os.path.splitext(outfile)[1];
3505 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3506 catver = formatspecs['format_ver'];
3507 fileheaderver = str(int(catver.replace(".", "")));
3508 curinode = 0;
3509 curfid = 0;
3510 inodelist = [];
3511 inodetofile = {};
3512 filetoinode = {};
3513 inodetocatinode = {};
3514 if(not os.path.exists(infile) or not os.path.isfile(infile)):
3515 return False;
3516 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
3517 return False;
3518 rarfp = rarfile.RarFile(infile, "r");
3519 rartest = rarfp.testrar();
3520 if(rartest):
3521 VerbosePrintOut("Bad file found!");
3522 numfiles = int(len(rarfp.infolist()));
3523 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3524 try:
3525 catfp.flush();
3526 os.fsync(catfp.fileno());
3527 except io.UnsupportedOperation:
3528 pass;
3529 except AttributeError:
3530 pass;
3531 except OSError as e:
3532 pass;
3533 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
3534 is_unix = False;
3535 is_windows = False;
3536 if(member.host_os==rarfile.RAR_OS_UNIX):
3537 is_windows = False;
3538 try:
3539 member.external_attr
3540 is_unix = True;
3541 except AttributeError:
3542 is_unix = False;
3543 elif(member.host_os==rarfile.RAR_OS_WIN32):
3544 is_unix = False;
3545 try:
3546 member.external_attr
3547 is_windows = True;
3548 except AttributeError:
3549 is_windows = False;
3550 else:
3551 is_unix = False;
3552 is_windows = False;
3553 if(re.findall("^[.|/]", member.filename)):
3554 fname = member.filename;
3555 else:
3556 fname = "./"+member.filename;
3557 rarinfo = rarfp.getinfo(member.filename);
3558 if(verbose):
3559 VerbosePrintOut(fname);
3560 if(is_unix and member.external_attr !=0):
3561 fpremode = int(member.external_attr);
3562 elif(member.is_file()):
3563 fpremode = int(stat.S_IFREG + 438);
3564 elif(member.is_symlink()):
3565 fpremode = int(stat.S_IFLNK + 438);
3566 elif(member.is_dir()):
3567 fpremode = int(stat.S_IFDIR + 511);
3568 if(is_windows and member.external_attr !=0):
3569 fwinattributes = format(int(member.external_attr), 'x').lower();
3570 else:
3571 fwinattributes = format(int(0), 'x').lower();
3572 fcompression = "";
3573 fcsize = format(int(0), 'x').lower();
3574 flinkcount = 0;
3575 ftype = 0;
3576 if(member.is_file()):
3577 ftype = 0;
3578 elif(member.is_symlink()):
3579 ftype = 2;
3580 elif(member.is_dir()):
3581 ftype = 5;
3582 flinkname = "";
3583 if(ftype==2):
3584 flinkname = rarfp.read(member.filename).decode("UTF-8");
3585 fcurfid = format(int(curfid), 'x').lower();
3586 fcurinode = format(int(curfid), 'x').lower();
3587 curfid = curfid + 1;
3588 fdev_minor = format(int(0), 'x').lower();
3589 fdev_major = format(int(0), 'x').lower();
3590 frdev_minor = format(int(0), 'x').lower();
3591 frdev_major = format(int(0), 'x').lower();
3592 if(ftype==5):
3593 fsize = format(int("0"), 'x').lower();
3594 elif(ftype==0):
3595 fsize = format(int(member.file_size), 'x').lower();
3596 else:
3597 fsize = format(int(member.file_size), 'x').lower();
3598 try:
3599 if(member.atime):
3600 fatime = format(int(member.atime.timestamp()), 'x').lower();
3601 else:
3602 fatime = format(int(member.mtime.timestamp()), 'x').lower();
3603 except AttributeError:
3604 fatime = format(int(member.mtime.timestamp()), 'x').lower();
3605 fmtime = format(int(member.mtime.timestamp()), 'x').lower();
3606 try:
3607 if(member.ctime):
3608 fctime = format(int(member.ctime.timestamp()), 'x').lower();
3609 else:
3610 fctime = format(int(member.mtime.timestamp()), 'x').lower();
3611 except AttributeError:
3612 fctime = format(int(member.mtime.timestamp()), 'x').lower();
3613 fbtime = format(int(member.mtime.timestamp()), 'x').lower();
3614 if(is_unix and member.external_attr !=0):
3615 fmode = format(int(member.external_attr), 'x').lower();
3616 fchmode = format(int(stat.S_IMODE(member.external_attr)), 'x').lower();
3617 ftypemod = format(int(stat.S_IFMT(member.external_attr)), 'x').lower();
3618 elif(member.is_file()):
3619 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3620 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3621 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3622 elif(member.is_symlink()):
3623 fmode = format(int(stat.S_IFLNK + 438), 'x').lower();
3624 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3625 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3626 elif(member.is_dir()):
3627 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3628 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
3629 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
3630 try:
3631 fuid = format(int(os.getuid()), 'x').lower();
3632 except AttributeError:
3633 fuid = format(int(0), 'x').lower();
3634 except KeyError:
3635 fuid = format(int(0), 'x').lower();
3636 try:
3637 fgid = format(int(os.getgid()), 'x').lower();
3638 except AttributeError:
3639 fgid = format(int(0), 'x').lower();
3640 except KeyError:
3641 fgid = format(int(0), 'x').lower();
3642 try:
3643 import pwd;
3644 try:
3645 userinfo = pwd.getpwuid(os.getuid());
3646 funame = userinfo.pw_name;
3647 except KeyError:
3648 funame = "";
3649 except AttributeError:
3650 funame = "";
3651 except ImportError:
3652 funame = "";
3653 fgname = "";
3654 try:
3655 import grp;
3656 try:
3657 groupinfo = grp.getgrgid(os.getgid());
3658 fgname = groupinfo.gr_name;
3659 except KeyError:
3660 fgname = "";
3661 except AttributeError:
3662 fgname = "";
3663 except ImportError:
3664 fgname = "";
3665 fcontents = BytesIO();
3666 if(ftype==0):
3667 fcontents.write(rarfp.read(member.filename));
3668 if(not compresswholefile):
3669 fcontents.seek(0, 2);
3670 ucfsize = fcontents.tell();
3671 fcontents.seek(0, 0);
3672 if(compression=="auto"):
3673 ilsize = len(compressionlistalt);
3674 ilmin = 0;
3675 ilcsize = [];
3676 while(ilmin < ilsize):
3677 cfcontents = BytesIO();
3678 shutil.copyfileobj(fcontents, cfcontents);
3679 fcontents.seek(0, 0);
3680 cfcontents.seek(0, 0);
3681 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3682 if(cfcontents):
3683 cfcontents.seek(0, 2);
3684 ilcsize.append(cfcontents.tell());
3685 cfcontents.close();
3686 else:
3687 try:
3688 ilcsize.append(sys.maxint);
3689 except AttributeError:
3690 ilcsize.append(sys.maxsize);
3691 ilmin = ilmin + 1;
3692 ilcmin = ilcsize.index(min(ilcsize));
3693 compression = compressionlistalt[ilcmin];
3694 fcontents.seek(0, 0);
3695 cfcontents = BytesIO();
3696 shutil.copyfileobj(fcontents, cfcontents);
3697 cfcontents.seek(0, 0);
3698 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3699 cfcontents.seek(0, 2);
3700 cfsize = cfcontents.tell();
3701 if(ucfsize > cfsize):
3702 fcsize = format(int(cfsize), 'x').lower();
3703 fcompression = compression;
3704 fcontents.close();
3705 fcontents = cfcontents;
3706 if(fcompression=="none"):
3707 fcompression = "";
3708 fcontents.seek(0, 0);
3709 ftypehex = format(ftype, 'x').lower();
3710 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3711 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3712 fcontents.close();
3713 if(numfiles>0):
3714 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3715 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3716 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
3717 try:
3718 catfp.flush();
3719 os.fsync(catfp.fileno());
3720 except io.UnsupportedOperation:
3721 pass;
3722 except AttributeError:
3723 pass;
3724 except OSError as e:
3725 pass;
3726 if(outfile=="-"):
3727 catfp.seek(0, 0)
3728 if(hasattr(sys.stdout, "buffer")):
3729 shutil.copyfileobj(catfp, sys.stdout.buffer);
3730 else:
3731 shutil.copyfileobj(catfp, sys.stdout);
3732 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3733 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3734 catfp.seek(0, 0);
3735 upload_file_to_internet_file(catfp, outfile);
3736 if(returnfp):
3737 catfp.seek(0, 0)
3738 return catfp
3739 else:
3740 catfp.close()
3741 return True;
3742
3743 create_alias_function("Pack", __file_format_name__, "FromRarFile", PackArchiveFileFromRarFile);
3744
3745 if(not py7zr_support):
3746 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3747 return False
3748
3749 if(py7zr_support):
3750 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3751 formatspecs = FormatSpecsListToDict(formatspecs);
3752 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3753 outfile = RemoveWindowsPath(outfile);
3754 checksumtype = checksumtype.lower();
3755 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3756 checksumtype="crc32";
3757 if(checksumtype=="none"):
3758 checksumtype = "";
3759 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3760 compression = "auto";
3761 if(compression not in compressionlist and compression is None):
3762 compression = "auto";
3763 if(verbose):
3764 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3765 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3766 if(os.path.exists(outfile)):
3767 try:
3768 os.unlink(outfile);
3769 except OSError as e:
3770 pass;
3771 if(outfile=="-"):
3772 verbose = False;
3773 catfp = BytesIO();
3774 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3775 catfp = outfile;
3776 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3777 catfp = BytesIO();
3778 else:
3779 fbasename = os.path.splitext(outfile)[0];
3780 fextname = os.path.splitext(outfile)[1];
3781 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3782 catver = formatspecs['format_ver'];
3783 fileheaderver = str(int(catver.replace(".", "")));
3784 curinode = 0;
3785 curfid = 0;
3786 inodelist = [];
3787 inodetofile = {};
3788 filetoinode = {};
3789 inodetocatinode = {};
3790 if(not os.path.exists(infile) or not os.path.isfile(infile)):
3791 return False;
3792 szpfp = py7zr.SevenZipFile(infile, mode="r");
3793 file_content = szpfp.readall();
3794 #sztest = szpfp.testzip();
3795 sztestalt = szpfp.test();
3796 if(sztestalt):
3797 VerbosePrintOut("Bad file found!");
3798 numfiles = int(len(szpfp.list()));
3799 AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3800 for member in sorted(szpfp.list(), key=lambda x: x.filename):
3801 if(re.findall("^[.|/]", member.filename)):
3802 fname = member.filename;
3803 else:
3804 fname = "./"+member.filename;
3805 if(verbose):
3806 VerbosePrintOut(fname);
3807 if(not member.is_directory):
3808 fpremode = int(stat.S_IFREG + 438);
3809 elif(member.is_directory):
3810 fpremode = int(stat.S_IFDIR + 511);
3811 fwinattributes = format(int(0), 'x').lower();
3812 fcompression = "";
3813 fcsize = format(int(0), 'x').lower();
3814 flinkcount = 0;
3815 ftype = 0;
3816 if(member.is_directory):
3817 ftype = 5;
3818 else:
3819 ftype = 0;
3820 flinkname = "";
3821 fcurfid = format(int(curfid), 'x').lower();
3822 fcurinode = format(int(curfid), 'x').lower();
3823 curfid = curfid + 1;
3824 fdev_minor = format(int(0), 'x').lower();
3825 fdev_major = format(int(0), 'x').lower();
3826 frdev_minor = format(int(0), 'x').lower();
3827 frdev_major = format(int(0), 'x').lower();
3828 if(ftype==5):
3829 fsize = format(int("0"), 'x').lower();
3830 fatime = format(int(member.creationtime.timestamp()), 'x').lower();
3831 fmtime = format(int(member.creationtime.timestamp()), 'x').lower();
3832 fctime = format(int(member.creationtime.timestamp()), 'x').lower();
3833 fbtime = format(int(member.creationtime.timestamp()), 'x').lower();
3834 if(member.is_directory):
3835 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3836 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
3837 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
3838 else:
3839 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3840 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3841 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3842 try:
3843 fuid = format(int(os.getuid()), 'x').lower();
3844 except AttributeError:
3845 fuid = format(int(0), 'x').lower();
3846 except KeyError:
3847 fuid = format(int(0), 'x').lower();
3848 try:
3849 fgid = format(int(os.getgid()), 'x').lower();
3850 except AttributeError:
3851 fgid = format(int(0), 'x').lower();
3852 except KeyError:
3853 fgid = format(int(0), 'x').lower();
3854 try:
3855 import pwd;
3856 try:
3857 userinfo = pwd.getpwuid(os.getuid());
3858 funame = userinfo.pw_name;
3859 except KeyError:
3860 funame = "";
3861 except AttributeError:
3862 funame = "";
3863 except ImportError:
3864 funame = "";
3865 fgname = "";
3866 try:
3867 import grp;
3868 try:
3869 groupinfo = grp.getgrgid(os.getgid());
3870 fgname = groupinfo.gr_name;
3871 except KeyError:
3872 fgname = "";
3873 except AttributeError:
3874 fgname = "";
3875 except ImportError:
3876 fgname = "";
3877 fcontents = BytesIO();
3878 if(ftype==0):
3879 fcontents.write(file_content[member.filename].read());
3880 fsize = format(fcontents.tell(), 'x').lower();
3881 file_content[member.filename].close();
3882 if(not compresswholefile):
3883 fcontents.seek(0, 2);
3884 ucfsize = fcontents.tell();
3885 fcontents.seek(0, 0);
3886 if(compression=="auto"):
3887 ilsize = len(compressionlistalt);
3888 ilmin = 0;
3889 ilcsize = [];
3890 while(ilmin < ilsize):
3891 cfcontents = BytesIO();
3892 shutil.copyfileobj(fcontents, cfcontents);
3893 fcontents.seek(0, 0);
3894 cfcontents.seek(0, 0);
3895 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3896 if(cfcontents):
3897 cfcontents.seek(0, 2);
3898 ilcsize.append(cfcontents.tell());
3899 cfcontents.close();
3900 else:
3901 try:
3902 ilcsize.append(sys.maxint);
3903 except AttributeError:
3904 ilcsize.append(sys.maxsize);
3905 ilmin = ilmin + 1;
3906 ilcmin = ilcsize.index(min(ilcsize));
3907 compression = compressionlistalt[ilcmin];
3908 fcontents.seek(0, 0);
3909 cfcontents = BytesIO();
3910 shutil.copyfileobj(fcontents, cfcontents);
3911 cfcontents.seek(0, 0);
3912 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3913 cfcontents.seek(0, 2);
3914 cfsize = cfcontents.tell();
3915 if(ucfsize > cfsize):
3916 fcsize = format(int(cfsize), 'x').lower();
3917 fcompression = compression;
3918 fcontents.close();
3919 fcontents = cfcontents;
3920 if(fcompression=="none"):
3921 fcompression = "";
3922 fcontents.seek(0, 0);
3923 ftypehex = format(ftype, 'x').lower();
3924 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3925 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3926 fcontents.close();
3927 if(numfiles>0):
3928 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3929 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3930 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
3931 try:
3932 catfp.flush();
3933 os.fsync(catfp.fileno());
3934 except io.UnsupportedOperation:
3935 pass;
3936 except AttributeError:
3937 pass;
3938 except OSError as e:
3939 pass;
3940 if(outfile=="-"):
3941 catfp.seek(0, 0);
3942 if(hasattr(sys.stdout, "buffer")):
3943 shutil.copyfileobj(catfp, sys.stdout.buffer);
3944 else:
3945 shutil.copyfileobj(catfp, sys.stdout);
3946 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3947 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3948 catfp.seek(0, 0);
3949 upload_file_to_internet_file(catfp, outfile);
3950 if(returnfp):
3951 catfp.seek(0, 0)
3952 return catfp
3953 else:
3954 catfp.close()
3955 return True;
3956
3957 create_alias_function("Pack", __file_format_name__, "FromSevenZipFile", PackArchiveFileFromSevenZipFile);
3958
3959 def PackArchiveFileFromInFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3960 formatspecs = FormatSpecsListToDict(formatspecs);
3961 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
3962 if(verbose):
3963 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3964 if(checkcompressfile=="tarfile"):
3965 return PackArchiveFileFromTarFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3966 elif(checkcompressfile=="zipfile"):
3967 return PackArchiveFileFromZipFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3968 elif(checkcompressfile=="catfile"):
3969 return RePackArchiveFile(infile, outfile, compression, compresswholefile, compressionlevel, False, 0, 0, checksumtype, False, extradata, formatspecs, verbose, returnfp);
3970 elif(rarfile_support and checkcompressfile=="rarfile"):
3971 return PackArchiveFileFromRarFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3972 elif(py7zr_support and checkcompressfile=="7zipfile"):
3973 return PackArchiveFileFromSevenZipFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3974 else:
3975 return False;
3976 return False;
3977
3978 create_alias_function("Pack", __file_format_name__, "FromInFile", PackArchiveFileFromInFile);
3979
3980 def ArchiveFileSeekToFileNum(infile, seekto=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
3981 formatspecs = FormatSpecsListToDict(formatspecs);
3982 if(hasattr(infile, "read") or hasattr(infile, "write")):
3983 catfp = infile;
3984 catfp.seek(0, 0);
3985 catfp = UncompressArchiveFile(catfp, formatspecs);
3986 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
3987 if(checkcompressfile=="tarfile"):
3988 return TarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3989 if(checkcompressfile=="zipfile"):
3990 return ZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3991 if(rarfile_support and checkcompressfile=="rarfile"):
3992 return RarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3993 if(py7zr_support and checkcompressfile=="7zipfile"):
3994 return SevenZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3995 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
3996 return False;
3997 if(not catfp):
3998 return False;
3999 catfp.seek(0, 0);
4000 elif(infile=="-"):
4001 catfp = BytesIO();
4002 if(hasattr(sys.stdin, "buffer")):
4003 shutil.copyfileobj(sys.stdin.buffer, catfp);
4004 else:
4005 shutil.copyfileobj(sys.stdin, catfp);
4006 catfp.seek(0, 0);
4007 catfp = UncompressArchiveFile(catfp, formatspecs);
4008 if(not catfp):
4009 return False;
4010 catfp.seek(0, 0);
4011 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4012 catfp = download_file_from_internet_file(infile);
4013 catfp.seek(0, 0);
4014 catfp = UncompressArchiveFile(catfp, formatspecs);
4015 if(not catfp):
4016 return False;
4017 catfp.seek(0, 0);
4018 else:
4019 infile = RemoveWindowsPath(infile);
4020 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4021 if(checkcompressfile=="tarfile"):
4022 return TarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4023 if(checkcompressfile=="zipfile"):
4024 return ZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4025 if(rarfile_support and checkcompressfile=="rarfile"):
4026 return RarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4027 if(py7zr_support and checkcompressfile=="7zipfile"):
4028 return SevenZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4029 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4030 return False;
4031 compresscheck = CheckCompressionType(infile, formatspecs, True);
4032 if(not compresscheck):
4033 fextname = os.path.splitext(infile)[1];
4034 if(fextname==".gz"):
4035 compresscheck = "gzip";
4036 elif(fextname==".bz2"):
4037 compresscheck = "bzip2";
4038 elif(fextname==".zst"):
4039 compresscheck = "zstd";
4040 elif(fextname==".lz4" or fextname==".clz4"):
4041 compresscheck = "lz4";
4042 elif(fextname==".lzo" or fextname==".lzop"):
4043 compresscheck = "lzo";
4044 elif(fextname==".lzma" or fextname==".xz"):
4045 compresscheck = "lzma";
4046 else:
4047 return False;
4048 if(not compresscheck):
4049 return False;
4050 catfp = UncompressFile(infile, formatspecs, "rb");
4051 '''
4052 try:
4053 catfp.seek(0, 2);
4054 except OSError:
4055 SeekToEndOfFile(catfp);
4056 except ValueError:
4057 SeekToEndOfFile(catfp);
4058 CatSize = catfp.tell();
4059 CatSizeEnd = CatSize;
4060 '''
4061 try:
4062 catfp.seek(0, 0);
4063 except OSError:
4064 return False;
4065 except ValueError:
4066 return False;
4067 curloc = catfp.tell();
4068 if(curloc>0):
4069 catfp.seek(0, 0);
4070 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4071 if(curloc>0):
4072 catfp.seek(curloc, 0);
4073 catstring = catheader[0];
4074 catversion = re.findall(r"([\d]+)", catstring);
4075 fprenumfiles = catheader[1];
4076 fnumfiles = int(fprenumfiles, 16);
4077 fprechecksumtype = catheader[2];
4078 fprechecksum = catheader[3];
4079 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4080 fnumfileshex = format(int(fnumfiles), 'x').lower();
4081 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4082 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4083 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4084 fheadtell = len(fileheader);
4085 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4086 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
4087 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4088 return False;
4089 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
4090 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4091 if(seekto>=fnumfiles):
4092 seekto = fnumfiles - 1;
4093 if(seekto<0):
4094 seekto = 0;
4095 if(seekto>=0):
4096 il = -1;
4097 while(il < seekto):
4098 prefhstart = catfp.tell();
4099 if(formatspecs['new_style']):
4100 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4101 else:
4102 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4103 if(len(preheaderdata)==0):
4104 break;
4105 prefheadsize = int(preheaderdata[0], 16);
4106 prefnumfields = int(preheaderdata[1], 16);
4107 preftype = int(preheaderdata[2], 16);
4108 if(re.findall("^[.|/]", preheaderdata[3])):
4109 prefname = preheaderdata[3];
4110 else:
4111 prefname = "./"+preheaderdata[3];
4112 prefbasedir = os.path.dirname(prefname);
4113 preflinkname = preheaderdata[4];
4114 prefsize = int(preheaderdata[5], 16);
4115 prefatime = int(preheaderdata[6], 16);
4116 prefmtime = int(preheaderdata[7], 16);
4117 prefctime = int(preheaderdata[8], 16);
4118 prefbtime = int(preheaderdata[9], 16);
4119 prefmode = int(preheaderdata[10], 16);
4120 prefchmode = stat.S_IMODE(prefmode);
4121 preftypemod = stat.S_IFMT(prefmode);
4122 prefwinattributes = int(preheaderdata[11], 16);
4123 prefcompression = preheaderdata[12];
4124 prefcsize = int(preheaderdata[13], 16);
4125 prefuid = int(preheaderdata[14], 16);
4126 prefuname = preheaderdata[15];
4127 prefgid = int(preheaderdata[16], 16);
4128 prefgname = preheaderdata[17];
4129 fid = int(preheaderdata[18], 16);
4130 finode = int(preheaderdata[19], 16);
4131 flinkcount = int(preheaderdata[20], 16);
4132 prefdev_minor = int(preheaderdata[21], 16);
4133 prefdev_major = int(preheaderdata[22], 16);
4134 prefrdev_minor = int(preheaderdata[23], 16);
4135 prefrdev_major = int(preheaderdata[24], 16);
4136 prefextrasize = int(preheaderdata[25], 16);
4137 prefextrafields = int(preheaderdata[26], 16);
4138 extrafieldslist = [];
4139 extrastart = 27;
4140 extraend = extrastart + prefextrafields;
4141 extrafieldslist = [];
4142 if(extrastart<extraend):
4143 extrafieldslist.append(preheaderdata[extrastart]);
4144 extrastart = extrastart + 1;
4145 prefchecksumtype = preheaderdata[extrastart].lower();
4146 prefcs = preheaderdata[extrastart + 1].lower();
4147 prefccs = preheaderdata[extrastart + 2].lower();
4148 hc = 0;
4149 hcmax = len(preheaderdata) - 2;
4150 hout = "";
4151 while(hc<hcmax):
4152 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs['format_delimiter']);
4153 hc = hc + 1;
4154 prenewfcs = GetFileChecksum(hout, preheaderdata[-3].lower(), True, formatspecs);
4155 if(prefcs!=prenewfcs and not skipchecksum):
4156 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4157 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4158 return False;
4159 valid_archive = False;
4160 invalid_archive = True;
4161 prefhend = catfp.tell() - 1;
4162 prefcontentstart = catfp.tell();
4163 prefcontents = "";
4164 pyhascontents = False;
4165 if(prefsize>0):
4166 if(prefcompression):
4167 prefcontents = catfp.read(prefsize);
4168 else:
4169 prefcontents = catfp.read(prefcsize);
4170 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4171 pyhascontents = True;
4172 if(prefccs!=prenewfccs and not skipchecksum):
4173 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4174 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4175 return False;
4176 catfp.seek(1, 1);
4177 il = il + 1;
4178 catfp.seek(seekstart, 0);
4179 fileidnum = il;
4180 catfheadsize = int(preheaderdata[0], 16);
4181 catfnumfields = int(preheaderdata[1], 16);
4182 catftype = int(preheaderdata[2], 16);
4183 if(re.findall("^[.|/]", preheaderdata[3])):
4184 catfname = preheaderdata[3];
4185 else:
4186 catfname = "./"+preheaderdata[3];
4187 catflinkname = preheaderdata[4];
4188 catfsize = int(preheaderdata[5], 16);
4189 catfbasedir = os.path.dirname(catfname);
4190 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
4191 if(returnfp):
4192 catlist.update({'catfp': catfp});
4193 else:
4194 catfp.close();
4195 return catlist;
4196
4197 create_alias_function("", __file_format_name__, "SeekToFileNum", ArchiveFileSeekToFileNum);
4198
4199 def ArchiveFileSeekToFileName(infile, seekfile=None, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4200 formatspecs = FormatSpecsListToDict(formatspecs);
4201 if(hasattr(infile, "read") or hasattr(infile, "write")):
4202 catfp = infile;
4203 catfp.seek(0, 0);
4204 catfp = UncompressArchiveFile(catfp, formatspecs);
4205 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4206 if(checkcompressfile=="tarfile"):
4207 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4208 if(checkcompressfile=="zipfile"):
4209 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4210 if(rarfile_support and checkcompressfile=="rarfile"):
4211 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4212 if(py7zr_support and checkcompressfile=="7zipfile"):
4213 return SevenZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4214 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4215 return False;
4216 if(not catfp):
4217 return False;
4218 catfp.seek(0, 0);
4219 elif(infile=="-"):
4220 catfp = BytesIO();
4221 if(hasattr(sys.stdin, "buffer")):
4222 shutil.copyfileobj(sys.stdin.buffer, catfp);
4223 else:
4224 shutil.copyfileobj(sys.stdin, catfp);
4225 catfp.seek(0, 0);
4226 catfp = UncompressArchiveFile(catfp, formatspecs);
4227 if(not catfp):
4228 return False;
4229 catfp.seek(0, 0);
4230 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4231 catfp = download_file_from_internet_file(infile);
4232 catfp = UncompressArchiveFile(catfp, formatspecs);
4233 catfp.seek(0, 0);
4234 if(not catfp):
4235 return False;
4236 catfp.seek(0, 0);
4237 else:
4238 infile = RemoveWindowsPath(infile);
4239 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4240 if(checkcompressfile=="tarfile"):
4241 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4242 if(checkcompressfile=="zipfile"):
4243 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4244 if(rarfile_support and checkcompressfile=="rarfile"):
4245 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4246 if(py7zr_support and checkcompressfile=="7zipfile"):
4247 return SevenZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4248 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4249 return False;
4250 compresscheck = CheckCompressionType(infile, formatspecs, True);
4251 if(not compresscheck):
4252 fextname = os.path.splitext(infile)[1];
4253 if(fextname==".gz"):
4254 compresscheck = "gzip";
4255 elif(fextname==".bz2"):
4256 compresscheck = "bzip2";
4257 elif(fextname==".zst"):
4258 compresscheck = "zstd";
4259 elif(fextname==".lz4" or fextname==".clz4"):
4260 compresscheck = "lz4";
4261 elif(fextname==".lzo" or fextname==".lzop"):
4262 compresscheck = "lzo";
4263 elif(fextname==".lzma" or fextname==".xz"):
4264 compresscheck = "lzma";
4265 else:
4266 return False;
4267 if(not compresscheck):
4268 return False;
4269 catfp = UncompressFile(infile, formatspecs, "rb");
4270 '''
4271 try:
4272 catfp.seek(0, 2);
4273 except OSError:
4274 SeekToEndOfFile(catfp);
4275 except ValueError:
4276 SeekToEndOfFile(catfp);
4277 CatSize = catfp.tell();
4278 CatSizeEnd = CatSize;
4279 '''
4280 try:
4281 catfp.seek(0, 0);
4282 except OSError:
4283 return False;
4284 except ValueError:
4285 return False;
4286 curloc = catfp.tell();
4287 if(curloc>0):
4288 catfp.seek(0, 0);
4289 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4290 if(curloc>0):
4291 catfp.seek(curloc, 0);
4292 catstring = catheader[0];
4293 catversion = re.findall(r"([\d]+)", catstring);
4294 fprenumfiles = catheader[1];
4295 fnumfiles = int(fprenumfiles, 16);
4296 fprechecksumtype = catheader[2];
4297 fprechecksum = catheader[3];
4298 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4299 fnumfileshex = format(int(fnumfiles), 'x').lower();
4300 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4301 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4302 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4303 fheadtell = len(fileheader);
4304 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4305 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
4306 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4307 return False;
4308 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
4309 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4310 seekto = fnumfiles - 1
4311 filefound = False;
4312 if(seekto>=0):
4313 il = -1;
4314 while(il < seekto):
4315 prefhstart = catfp.tell();
4316 if(formatspecs['new_style']):
4317 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4318 else:
4319 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4320 if(len(preheaderdata)==0):
4321 break;
4322 prefheadsize = int(preheaderdata[0], 16);
4323 prefnumfields = int(preheaderdata[1], 16);
4324 preftype = int(preheaderdata[2], 16);
4325 if(re.findall("^[.|/]", preheaderdata[3])):
4326 prefname = preheaderdata[3];
4327 else:
4328 prefname = "./"+preheaderdata[3];
4329 prefbasedir = os.path.dirname(prefname);
4330 preflinkname = preheaderdata[4];
4331 prefsize = int(preheaderdata[5], 16);
4332 prefatime = int(preheaderdata[6], 16);
4333 prefmtime = int(preheaderdata[7], 16);
4334 prefctime = int(preheaderdata[8], 16);
4335 prefbtime = int(preheaderdata[9], 16);
4336 prefmode = int(preheaderdata[10], 16);
4337 prefchmode = stat.S_IMODE(prefmode);
4338 preftypemod = stat.S_IFMT(prefmode);
4339 prefwinattributes = int(preheaderdata[11], 16);
4340 prefcompression = preheaderdata[12];
4341 prefcsize = int(preheaderdata[13], 16);
4342 prefuid = int(preheaderdata[14], 16);
4343 prefuname = preheaderdata[15];
4344 prefgid = int(preheaderdata[16], 16);
4345 prefgname = preheaderdata[17];
4346 fid = int(preheaderdata[18], 16);
4347 finode = int(preheaderdata[19], 16);
4348 flinkcount = int(preheaderdata[20], 16);
4349 prefdev_minor = int(preheaderdata[21], 16);
4350 prefdev_major = int(preheaderdata[22], 16);
4351 prefrdev_minor = int(preheaderdata[23], 16);
4352 prefrdev_major = int(preheaderdata[24], 16);
4353 prefextrasize = int(preheaderdata[25], 16);
4354 prefextrafields = int(preheaderdata[26], 16);
4355 extrafieldslist = [];
4356 extrastart = 27;
4357 extraend = extrastart + prefextrafields;
4358 extrafieldslist = [];
4359 if(extrastart<extraend):
4360 extrafieldslist.append(preheaderdata[extrastart]);
4361 extrastart = extrastart + 1;
4362 prefchecksumtype = preheaderdata[extrastart].lower();
4363 prefcs = preheaderdata[extrastart + 1].lower();
4364 prefccs = preheaderdata[extrastart + 2].lower();
4365 hc = 0;
4366 hcmax = len(preheaderdata) - 2;
4367 hout = "";
4368 while(hc<hcmax):
4369 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs['format_delimiter']);
4370 hc = hc + 1;
4371 prenewfcs = GetFileChecksum(hout, preheaderdata[-3].lower(), True, formatspecs);
4372 if(prefcs!=prenewfcs and not skipchecksum):
4373 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4374 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4375 return False;
4376 valid_archive = False;
4377 invalid_archive = True;
4378 prefhend = catfp.tell() - 1;
4379 prefcontentstart = catfp.tell();
4380 prefcontents = "";
4381 pyhascontents = False;
4382 if(prefsize>0):
4383 if(prefcompression):
4384 prefcontents = catfp.read(prefsize);
4385 else:
4386 prefcontents = catfp.read(prefcsize);
4387 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4388 pyhascontents = True;
4389 if(prefccs!=prenewfccs and not skipchecksum):
4390 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4391 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4392 return False;
4393 catfp.seek(1, 1);
4394 il = il + 1;
4395 filefound = False;
4396 prefname = preheaderdata[2];
4397 if(re.findall("^[.|/]", preheaderdata[2])):
4398 prefname = preheaderdata[2];
4399 else:
4400 prefname = "./"+preheaderdata[2];
4401 if(prefname==seekfile):
4402 filefound = True;
4403 break;
4404 catfp.seek(seekstart, 0);
4405 fileidnum = il;
4406 catfheadsize = int(preheaderdata[0], 16);
4407 catfnumfields = int(preheaderdata[1], 16);
4408 catftype = int(preheaderdata[2], 16);
4409 if(re.findall("^[.|/]", preheaderdata[3])):
4410 catfname = preheaderdata[3];
4411 else:
4412 catfname = "./"+preheaderdata[3];
4413 catflinkname = preheaderdata[4];
4414 catfsize = int(preheaderdata[5], 16);
4415 catfbasedir = os.path.dirname(catfname);
4416 if(filefound):
4417 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
4418 else:
4419 return False;
4420 if(returnfp):
4421 catlist.update({'catfp': catfp});
4422 else:
4423 catfp.close();
4424 return catlist;
4425
4426 create_alias_function("", __file_format_name__, "SeekToFileName", ArchiveFileSeekToFileName);
4427
4428 def ArchiveFileValidate(infile, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
4429 formatspecs = FormatSpecsListToDict(formatspecs);
4430 if(verbose):
4431 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
4432 if(hasattr(infile, "read") or hasattr(infile, "write")):
4433 catfp = infile;
4434 catfp.seek(0, 0);
4435 catfp = UncompressArchiveFile(catfp, formatspecs);
4436 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4437 if(checkcompressfile=="tarfile"):
4438 return TarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4439 if(checkcompressfile=="zipfile"):
4440 return ZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4441 if(rarfile_support and checkcompressfile=="rarfile"):
4442 return RarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4443 if(py7zr_support and checkcompressfile=="7zipfile"):
4444 return SevenZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4445 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4446 return False;
4447 if(not catfp):
4448 return False;
4449 catfp.seek(0, 0);
4450 elif(infile=="-"):
4451 catfp = BytesIO();
4452 if(hasattr(sys.stdin, "buffer")):
4453 shutil.copyfileobj(sys.stdin.buffer, catfp);
4454 else:
4455 shutil.copyfileobj(sys.stdin, catfp);
4456 catfp.seek(0, 0);
4457 catfp = UncompressArchiveFile(catfp, formatspecs);
4458 if(not catfp):
4459 return False;
4460 catfp.seek(0, 0);
4461 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4462 catfp = download_file_from_internet_file(infile);
4463 catfp = UncompressArchiveFile(catfp, formatspecs);
4464 catfp.seek(0, 0);
4465 if(not catfp):
4466 return False;
4467 catfp.seek(0, 0);
4468 else:
4469 infile = RemoveWindowsPath(infile);
4470 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4471 if(checkcompressfile=="tarfile"):
4472 return TarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4473 if(checkcompressfile=="zipfile"):
4474 return ZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4475 if(rarfile_support and checkcompressfile=="rarfile"):
4476 return RarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4477 if(py7zr_support and checkcompressfile=="7zipfile"):
4478 return SevenZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4479 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4480 return False;
4481 compresscheck = CheckCompressionType(infile, formatspecs, True);
4482 if(not compresscheck):
4483 fextname = os.path.splitext(infile)[1];
4484 if(fextname==".gz"):
4485 compresscheck = "gzip";
4486 elif(fextname==".bz2"):
4487 compresscheck = "bzip2";
4488 elif(fextname==".zst"):
4489 compresscheck = "zstd";
4490 elif(fextname==".lz4" or fextname==".clz4"):
4491 compresscheck = "lz4";
4492 elif(fextname==".lzo" or fextname==".lzop"):
4493 compresscheck = "lzo";
4494 elif(fextname==".lzma" or fextname==".xz"):
4495 compresscheck = "lzma";
4496 else:
4497 return False;
4498 if(not compresscheck):
4499 return False;
4500 catfp = UncompressFile(infile, formatspecs, "rb");
4501 '''
4502 try:
4503 catfp.seek(0, 2);
4504 except OSError:
4505 SeekToEndOfFile(catfp);
4506 except ValueError:
4507 SeekToEndOfFile(catfp);
4508 CatSize = catfp.tell();
4509 CatSizeEnd = CatSize;
4510 '''
4511 try:
4512 catfp.seek(0, 0);
4513 except OSError:
4514 return False;
4515 except ValueError:
4516 return False;
4517 curloc = catfp.tell();
4518 if(curloc>0):
4519 catfp.seek(0, 0);
4520 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4521 if(curloc>0):
4522 catfp.seek(curloc, 0);
4523 catstring = catheader[0];
4524 catversion = re.findall(r"([\d]+)", catstring);
4525 fprenumfiles = catheader[1];
4526 fnumfiles = int(fprenumfiles, 16);
4527 fprechecksumtype = catheader[2];
4528 fprechecksum = catheader[3];
4529 il = 0;
4530 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4531 fnumfileshex = format(int(fnumfiles), 'x').lower();
4532 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4533 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4534 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4535 valid_archive = True;
4536 invalid_archive = False;
4537 if(verbose):
4538 VerbosePrintOut(infile);
4539 VerbosePrintOut("Number of Records " + str(fnumfiles));
4540 if(fprechecksum==catfileheadercshex):
4541 if(verbose):
4542 VerbosePrintOut("File Header Checksum Passed at offset " + str(0));
4543 VerbosePrintOut("'" + str(fprechecksum) + "' == " + "'" + str(catfileheadercshex) + "'");
4544 else:
4545 if(verbose):
4546 VerbosePrintOut("File Header Checksum Failed at offset " + str(0));
4547 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4548 valid_archive = False;
4549 invalid_archive = True;
4550 if(verbose):
4551 VerbosePrintOut("");
4552 while(il<fnumfiles):
4553 catfhstart = catfp.tell();
4554 if(formatspecs['new_style']):
4555 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4556 else:
4557 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4558 if(len(catheaderdata)==0):
4559 break;
4560 catfheadsize = int(catheaderdata[0], 16);
4561 catfnumfields = int(catheaderdata[1], 16);
4562 catftype = int(catheaderdata[2], 16);
4563 if(re.findall("^[.|/]", catheaderdata[3])):
4564 catfname = catheaderdata[3];
4565 else:
4566 catfname = "./"+catheaderdata[3];
4567 catfbasedir = os.path.dirname(catfname);
4568 catflinkname = catheaderdata[4];
4569 catfsize = int(catheaderdata[5], 16);
4570 catfatime = int(catheaderdata[6], 16);
4571 catfmtime = int(catheaderdata[7], 16);
4572 catfctime = int(catheaderdata[8], 16);
4573 catfbtime = int(catheaderdata[9], 16);
4574 catfmode = int(catheaderdata[10], 16);
4575 catfchmode = stat.S_IMODE(catfmode);
4576 catftypemod = stat.S_IFMT(catfmode);
4577 catfwinattributes = int(catheaderdata[11], 16);
4578 catfcompression = catheaderdata[12];
4579 catfcsize = int(catheaderdata[13], 16);
4580 catfuid = int(catheaderdata[14], 16);
4581 catfuname = catheaderdata[15];
4582 catfgid = int(catheaderdata[16], 16);
4583 catfgname = catheaderdata[17];
4584 fid = int(catheaderdata[18], 16);
4585 finode = int(catheaderdata[19], 16);
4586 flinkcount = int(catheaderdata[20], 16);
4587 catfdev_minor = int(catheaderdata[21], 16);
4588 catfdev_major = int(catheaderdata[22], 16);
4589 catfrdev_minor = int(catheaderdata[23], 16);
4590 catfrdev_major = int(catheaderdata[24], 16);
4591 catfextrasize = int(catheaderdata[25], 16);
4592 catfextrafields = int(catheaderdata[26], 16);
4593 extrafieldslist = [];
4594 extrastart = 27;
4595 extraend = extrastart + catfextrafields;
4596 extrafieldslist = [];
4597 if(extrastart<extraend):
4598 extrafieldslist.append(catheaderdata[extrastart]);
4599 extrastart = extrastart + 1;
4600 catfchecksumtype = catheaderdata[extrastart].lower();
4601 catfcs = catheaderdata[extrastart + 1].lower();
4602 catfccs = catheaderdata[extrastart + 2].lower();
4603 hc = 0;
4604 hcmax = len(catheaderdata) - 2;
4605 hout = "";
4606 while(hc<hcmax):
4607 hout = hout + AppendNullByte(catheaderdata[hc], formatspecs['format_delimiter']);
4608 hc = hc + 1;
4609 catnewfcs = GetFileChecksum(hout, catheaderdata[-3].lower(), True, formatspecs);
4610 if(verbose):
4611 VerbosePrintOut(catfname);
4612 VerbosePrintOut("Record Number " + str(il) + "; File ID " + str(fid) + "; iNode Number " + str(finode));
4613 if(catfcs==catnewfcs):
4614 if(verbose):
4615 VerbosePrintOut("File Header Checksum Passed at offset " + str(catfhstart));
4616 VerbosePrintOut("'" + str(catfcs) + "' == " + "'" + str(catnewfcs) + "'");
4617 else:
4618 if(verbose):
4619 VerbosePrintOut("File Header Checksum Failed at offset " + str(catfhstart));
4620 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
4621 valid_archive = False;
4622 invalid_archive = True;
4623 catfhend = catfp.tell() - 1;
4624 catfcontentstart = catfp.tell();
4625 catfcontents = "";
4626 pyhascontents = False;
4627 if(catfsize>0):
4628 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4629 catfcontents = catfp.read(catfsize);
4630 else:
4631 catfcontents = catfp.read(catfcsize);
4632 catnewfccs = GetFileChecksum(catfcontents, catheaderdata[-3].lower(), False, formatspecs);
4633 pyhascontents = True;
4634 if(catfccs==catnewfccs):
4635 if(verbose):
4636 VerbosePrintOut("File Content Checksum Passed at offset " + str(catfcontentstart));
4637 VerbosePrintOut("'" + str(catfccs) + "' == " + "'" + str(catnewfccs) + "'");
4638 else:
4639 if(verbose):
4640 VerbosePrintOut("File Content Checksum Failed at offset " + str(catfcontentstart));
4641 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
4642 valid_archive = False;
4643 invalid_archive = True;
4644 if(verbose):
4645 VerbosePrintOut("");
4646 catfp.seek(1, 1);
4647 il = il + 1;
4648 if(valid_archive):
4649 if(returnfp):
4650 return catfp;
4651 else:
4652 catfp.close();
4653 return True;
4654 else:
4655 catfp.close();
4656 return False;
4657
4658 create_alias_function("", __file_format_name__, "Validate", ArchiveFileValidate);
4659
4660 def ArchiveFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4661 formatspecs = FormatSpecsListToDict(formatspecs);
4662 if(hasattr(infile, "read") or hasattr(infile, "write")):
4663 catfp = infile;
4664 catfp.seek(0, 0);
4665 catfp = UncompressArchiveFile(catfp, formatspecs);
4666 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4667 if(checkcompressfile=="tarfile"):
4668 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4669 if(checkcompressfile=="zipfile"):
4670 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4671 if(rarfile_support and checkcompressfile=="rarfile"):
4672 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4673 if(py7zr_support and checkcompressfile=="7zipfile"):
4674 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4675 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4676 return False;
4677 if(not catfp):
4678 return False;
4679 catfp.seek(0, 0);
4680 elif(infile=="-"):
4681 catfp = BytesIO();
4682 if(hasattr(sys.stdin, "buffer")):
4683 shutil.copyfileobj(sys.stdin.buffer, catfp);
4684 else:
4685 shutil.copyfileobj(sys.stdin, catfp);
4686 catfp.seek(0, 0);
4687 catfp = UncompressArchiveFile(catfp, formatspecs);
4688 if(not catfp):
4689 return False;
4690 catfp.seek(0, 0);
4691 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4692 catfp = download_file_from_internet_file(infile);
4693 catfp = UncompressArchiveFile(catfp, formatspecs);
4694 catfp.seek(0, 0);
4695 if(not catfp):
4696 return False;
4697 catfp.seek(0, 0);
4698 else:
4699 infile = RemoveWindowsPath(infile);
4700 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4701 if(checkcompressfile=="tarfile"):
4702 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4703 if(checkcompressfile=="zipfile"):
4704 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4705 if(rarfile_support and checkcompressfile=="rarfile"):
4706 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4707 if(py7zr_support and checkcompressfile=="7zipfile"):
4708 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4709 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4710 return False;
4711 compresscheck = CheckCompressionType(infile, formatspecs, True);
4712 if(not compresscheck):
4713 fextname = os.path.splitext(infile)[1];
4714 if(fextname==".gz"):
4715 compresscheck = "gzip";
4716 elif(fextname==".bz2"):
4717 compresscheck = "bzip2";
4718 elif(fextname==".zst"):
4719 compresscheck = "zstd";
4720 elif(fextname==".lz4" or fextname==".clz4"):
4721 compresscheck = "lz4";
4722 elif(fextname==".lzo" or fextname==".lzop"):
4723 compresscheck = "lzo";
4724 elif(fextname==".lzma" or fextname==".xz"):
4725 compresscheck = "lzma";
4726 else:
4727 return False;
4728 if(not compresscheck):
4729 return False;
4730 catfp = UncompressFile(infile, formatspecs, "rb");
4731 '''
4732 try:
4733 catfp.seek(0, 2);
4734 except OSError:
4735 SeekToEndOfFile(catfp);
4736 except ValueError:
4737 SeekToEndOfFile(catfp);
4738 CatSize = catfp.tell();
4739 CatSizeEnd = CatSize;
4740 '''
4741 try:
4742 catfp.seek(0, 0);
4743 except OSError:
4744 return False;
4745 except ValueError:
4746 return False;
4747 curloc = catfp.tell();
4748 if(curloc>0):
4749 catfp.seek(0, 0);
4750 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4751 if(curloc>0):
4752 catfp.seek(curloc, 0);
4753 catstring = catheader[0];
4754 catversion = re.findall(r"([\d]+)", catstring);
4755 fprenumfiles = catheader[1];
4756 fnumfiles = int(fprenumfiles, 16);
4757 fprechecksumtype = catheader[2];
4758 fprechecksum = catheader[3];
4759 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4760 fnumfileshex = format(int(fnumfiles), 'x').lower();
4761 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4762 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4763 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4764 fheadtell = len(fileheader);
4765 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4766 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
4767 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4768 return False;
4769 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
4770 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4771 if(seekstart<0 and seekstart>fnumfiles):
4772 seekstart = 0;
4773 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
4774 seekend = fnumfiles;
4775 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
4776 seekend = fnumfiles - abs(seekend);
4777 if(seekstart>0):
4778 il = 0;
4779 while(il < seekstart):
4780 prefhstart = catfp.tell();
4781 if(formatspecs['new_style']):
4782 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4783 else:
4784 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4785 if(len(preheaderdata)==0):
4786 break;
4787 prefheadsize = int(preheaderdata[0], 16);
4788 prefnumfields = int(preheaderdata[1], 16);
4789 if(re.findall("^[.|/]", preheaderdata[3])):
4790 prefname = preheaderdata[3];
4791 else:
4792 prefname = "./"+preheaderdata[3];
4793 prefsize = int(preheaderdata[5], 16);
4794 prefcompression = preheaderdata[12];
4795 prefcsize = int(preheaderdata[13], 16);
4796 prefextrasize = int(preheaderdata[24], 16);
4797 prefextrafields = int(preheaderdata[25], 16);
4798 extrafieldslist = [];
4799 extrastart = 26;
4800 extraend = extrastart + prefextrafields;
4801 extrafieldslist = [];
4802 if(extrastart<extraend):
4803 extrafieldslist.append(preheaderdata[extrastart]);
4804 extrastart = extrastart + 1;
4805 prefchecksumtype = preheaderdata[extrastart].lower();
4806 prefcs = preheaderdata[extrastart + 1].lower();
4807 prefccs = preheaderdata[extrastart + 2].lower();
4808 hc = 0;
4809 hcmax = len(preheaderdata) - 2;
4810 hout = "";
4811 while(hc<hcmax):
4812 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs['format_delimiter']);
4813 hc = hc + 1;
4814 prenewfcs = GetFileChecksum(hout, preheaderdata[-3].lower(), True, formatspecs);
4815 if(prefcs!=prenewfcs and not skipchecksum):
4816 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4817 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4818 return False;
4819 valid_archive = False;
4820 invalid_archive = True;
4821 prefhend = catfp.tell() - 1;
4822 prefcontentstart = catfp.tell();
4823 prefcontents = "";
4824 pyhascontents = False;
4825 if(prefsize>0):
4826 if(prefcompression=="none" or prefcompression=="" or prefcompression=="auto"):
4827 prefcontents = catfp.read(prefsize);
4828 else:
4829 prefcontents = catfp.read(prefcsize);
4830 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4831 pyhascontents = True;
4832 if(prefccs!=prenewfccs and not skipchecksum):
4833 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4834 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4835 return False;
4836 catfp.seek(1, 1);
4837 il = il + 1;
4838 fileidnum = seekstart;
4839 realidnum = 0;
4840 while(fileidnum<seekend):
4841 catfhstart = catfp.tell();
4842 if(formatspecs['new_style']):
4843 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4844 else:
4845 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4846 if(len(catheaderdata)==0):
4847 break;
4848 catfheadsize = int(catheaderdata[0], 16);
4849 catfnumfields = int(catheaderdata[1], 16);
4850 catftype = int(catheaderdata[2], 16);
4851 if(re.findall("^[.|/]", catheaderdata[3])):
4852 catfname = catheaderdata[3];
4853 else:
4854 catfname = "./"+catheaderdata[3];
4855 catfbasedir = os.path.dirname(catfname);
4856 catflinkname = catheaderdata[4];
4857 catfsize = int(catheaderdata[5], 16);
4858 catfatime = int(catheaderdata[6], 16);
4859 catfmtime = int(catheaderdata[7], 16);
4860 catfctime = int(catheaderdata[8], 16);
4861 catfbtime = int(catheaderdata[9], 16);
4862 catfmode = int(catheaderdata[10], 16);
4863 catfchmode = stat.S_IMODE(catfmode);
4864 catftypemod = stat.S_IFMT(catfmode);
4865 catfwinattributes = int(catheaderdata[11], 16);
4866 catfcompression = catheaderdata[12];
4867 catfcsize = int(catheaderdata[13], 16);
4868 catfuid = int(catheaderdata[14], 16);
4869 catfuname = catheaderdata[15];
4870 catfgid = int(catheaderdata[16], 16);
4871 catfgname = catheaderdata[17];
4872 catfid = int(catheaderdata[18], 16);
4873 catfinode = int(catheaderdata[19], 16);
4874 catflinkcount = int(catheaderdata[20], 16);
4875 catfdev_minor = int(catheaderdata[21], 16);
4876 catfdev_major = int(catheaderdata[22], 16);
4877 catfrdev_minor = int(catheaderdata[23], 16);
4878 catfrdev_major = int(catheaderdata[24], 16);
4879 catfextrasize = int(catheaderdata[25], 16);
4880 catfextrafields = int(catheaderdata[26], 16);
4881 extrafieldslist = [];
4882 extrastart = 27;
4883 extraend = extrastart + catfextrafields;
4884 extrafieldslist = [];
4885 if(extrastart<extraend):
4886 extrafieldslist.append(catheaderdata[extrastart]);
4887 extrastart = extrastart + 1;
4888 catfchecksumtype = catheaderdata[extrastart].lower();
4889 catfcs = catheaderdata[extrastart + 1].lower();
4890 catfccs = catheaderdata[extrastart + 2].lower();
4891 hc = 0;
4892 hcmax = len(catheaderdata) - 2;
4893 hout = "";
4894 while(hc<hcmax):
4895 hout = hout + AppendNullByte(catheaderdata[hc], formatspecs['format_delimiter']);
4896 hc = hc + 1;
4897 catnewfcs = GetFileChecksum(hout, catheaderdata[-3].lower(), True, formatspecs);
4898 if(catfcs!=catnewfcs and not skipchecksum):
4899 VerbosePrintOut("File Header Checksum Error with file " + catfname + " at offset " + str(catfhstart));
4900 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
4901 return False;
4902 catfhend = catfp.tell() - 1;
4903 catfcontentstart = catfp.tell();
4904 catfcontents = BytesIO();
4905 pyhascontents = False;
4906 if(catfsize>0 and not listonly):
4907 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4908 catfcontents.write(catfp.read(catfsize));
4909 else:
4910 catfcontents.write(catfp.read(catfcsize));
4911 catfcontents.seek(0, 0);
4912 catnewfccs = GetFileChecksum(catfcontents.read(), catheaderdata[-3].lower(), False, formatspecs);
4913 pyhascontents = True;
4914 if(catfccs!=catnewfccs and skipchecksum):
4915 VerbosePrintOut("File Content Checksum Error with file " + catfname + " at offset " + str(catfcontentstart));
4916 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
4917 return False;
4918 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4919 pass;
4920 else:
4921 catfcontents.seek(0, 0);
4922 catfcontents = UncompressArchiveFile(catfcontents, formatspecs);
4923 if(catfsize>0 and listonly):
4924 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4925 catfp.seek(catfsize, 1);
4926 else:
4927 catfp.seek(catfcsize, 1);
4928 pyhascontents = False;
4929 catfp.seek(1, 1);
4930 catfcontentend = catfp.tell() - 1;
4931 catfcontents.seek(0, 0);
4932 catlist['ffilelist'].update({realidnum: {'fid': realidnum, 'fidalt': fileidnum, 'fheadersize': catfheadsize, 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize, 'fatime': catfatime, 'fmtime': catfmtime, 'fctime': catfctime, 'fbtime': catfbtime, 'fmode': catfmode, 'fchmode': catfchmode, 'ftypemod': catftypemod, 'fwinattributes': catfwinattributes, 'fcompression': catfcompression, 'fcsize': catfcsize, 'fuid': catfuid, 'funame': catfuname, 'fgid': catfgid, 'fgname': catfgname, 'finode': catfinode, 'flinkcount': catflinkcount, 'fminor': catfdev_minor, 'fmajor': catfdev_major, 'frminor': catfrdev_minor, 'frmajor': catfrdev_major, 'fchecksumtype': catfchecksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': catfextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': catfcs, 'fcontentchecksum': catfccs, 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': catfcontents} });
4933 fileidnum = fileidnum + 1;
4934 realidnum = realidnum + 1;
4935 if(returnfp):
4936 catlist.update({'catfp': catfp});
4937 else:
4938 catfp.close();
4939 return catlist;
4940
4941 create_alias_function("", __file_format_name__, "ToArray", ArchiveFileToArray);
4942
4943 def ArchiveFileStringToArray(catstr, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4944 formatspecs = FormatSpecsListToDict(formatspecs);
4945 catfp = BytesIO(catstr);
4946 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4947 return listcatfiles;
4948
4949 create_alias_function("", __file_format_name__, "StringToArray", ArchiveFileStringToArray);
4950
4951 def TarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4952 formatspecs = FormatSpecsListToDict(formatspecs);
4953 catfp = BytesIO();
4954 catfp = PackArchiveFileFromTarFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
4955 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4956 return listcatfiles;
4957
4958 def ZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4959 formatspecs = FormatSpecsListToDict(formatspecs);
4960 catfp = BytesIO();
4961 catfp = PackArchiveFileFromZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
4962 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4963 return listcatfiles;
4964
4965 if(not rarfile_support):
4966 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4967 return False;
4968
4969 if(rarfile_support):
4970 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4971 formatspecs = FormatSpecsListToDict(formatspecs);
4972 catfp = BytesIO();
4973 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
4974 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4975 return listcatfiles;
4976
4977 if(not py7zr_support):
4978 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4979 return False;
4980
4981 if(py7zr_support):
4982 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4983 formatspecs = FormatSpecsListToDict(formatspecs);
4984 catfp = BytesIO();
4985 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
4986 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4987 return listcatfiles;
4988
4989 def InFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4990 formatspecs = FormatSpecsListToDict(formatspecs);
4991 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4992 if(checkcompressfile=="tarfile"):
4993 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4994 elif(checkcompressfile=="zipfile"):
4995 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4996 elif(checkcompressfile=="catfile"):
4997 return ArchiveFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4998 elif(rarfile_support and checkcompressfile=="rarfile"):
4999 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5000 elif(py7zr_support and checkcompressfile=="7zipfile"):
5001 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5002 else:
5003 return False;
5004 return False;
5005
5006 def ListDirToArrayAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5007 formatspecs = FormatSpecsListToDict(formatspecs);
5008 catver = formatspecs['format_ver'];
5009 fileheaderver = str(int(catver.replace(".", "")));
5010 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5011 advancedlist = formatspecs['use_advanced_list'];
5012 altinode = formatspecs['use_alt_inode'];
5013 infilelist = [];
5014 if(infiles=="-"):
5015 for line in sys.stdin:
5016 infilelist.append(line.strip());
5017 infilelist = list(filter(None, infilelist));
5018 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
5019 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
5020 return False;
5021 with UncompressFile(infiles, formatspecs, "r") as finfile:
5022 for line in finfile:
5023 infilelist.append(line.strip());
5024 infilelist = list(filter(None, infilelist));
5025 else:
5026 if(isinstance(infiles, (list, tuple, ))):
5027 infilelist = list(filter(None, infiles));
5028 elif(isinstance(infiles, (str, ))):
5029 infilelist = list(filter(None, [infiles]));
5030 if(advancedlist):
5031 GetDirList = ListDirAdvanced(infilelist, followlink, False);
5032 else:
5033 GetDirList = ListDir(infilelist, followlink, False);
5034 if(not GetDirList):
5035 return False;
5036 curinode = 0;
5037 curfid = 0;
5038 inodelist = [];
5039 inodetofile = {};
5040 filetoinode = {};
5041 inodetocatinode = {};
5042 fileidnum = 0;
5043 fnumfiles = int(len(GetDirList));
5044 catver = formatspecs['format_ver'];
5045 fileheaderver = str(int(catver.replace(".", "")));
5046 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5047 fnumfileshex = format(int(fnumfiles), 'x').lower();
5048 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5049 catversion = re.findall(r"([\d]+)", fileheader);
5050 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5051 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5052 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5053 fheadtell = len(fileheader);
5054 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5055 for curfname in GetDirList:
5056 catfhstart = fheadtell;
5057 if(re.findall("^[.|/]", curfname)):
5058 fname = curfname;
5059 else:
5060 fname = "./"+curfname;
5061 if(verbose):
5062 VerbosePrintOut(fname);
5063 if(not followlink or followlink is None):
5064 fstatinfo = os.lstat(fname);
5065 else:
5066 fstatinfo = os.stat(fname);
5067 fpremode = fstatinfo.st_mode;
5068 finode = fstatinfo.st_ino;
5069 flinkcount = fstatinfo.st_nlink;
5070 ftype = 0;
5071 if(stat.S_ISREG(fpremode)):
5072 ftype = 0;
5073 elif(stat.S_ISLNK(fpremode)):
5074 ftype = 2;
5075 elif(stat.S_ISCHR(fpremode)):
5076 ftype = 3;
5077 elif(stat.S_ISBLK(fpremode)):
5078 ftype = 4;
5079 elif(stat.S_ISDIR(fpremode)):
5080 ftype = 5;
5081 elif(stat.S_ISFIFO(fpremode)):
5082 ftype = 6;
5083 elif(stat.S_ISSOCK(fpremode)):
5084 ftype = 8;
5085 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
5086 ftype = 9;
5087 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
5088 ftype = 10;
5089 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
5090 ftype = 11;
5091 else:
5092 ftype = 0;
5093 flinkname = "";
5094 fbasedir = os.path.dirname(fname);
5095 fcurfid = curfid;
5096 if(not followlink and finode!=0):
5097 if(ftype!=1):
5098 if(finode in inodelist):
5099 ftype = 1;
5100 flinkname = inodetofile[finode];
5101 if(altinode):
5102 fcurinode = finode;
5103 else:
5104 fcurinode = inodetocatinode[finode];
5105 if(finode not in inodelist):
5106 inodelist.append(finode);
5107 inodetofile.update({finode: fname});
5108 inodetocatinode.update({finode: curinode});
5109 if(altinode):
5110 fcurinode = finode;
5111 else:
5112 fcurinode = curinode;
5113 curinode = curinode + 1;
5114 else:
5115 fcurinode = curinode;
5116 curinode = curinode + 1;
5117 curfid = curfid + 1;
5118 if(ftype==2):
5119 flinkname = os.readlink(fname);
5120 fdev = fstatinfo.st_dev;
5121 getfdev = GetDevMajorMinor(fdev);
5122 fdev_minor = getfdev[0];
5123 fdev_major = getfdev[1];
5124 frdev = fstatinfo.st_dev;
5125 if(hasattr(fstatinfo, "st_rdev")):
5126 frdev = fstatinfo.st_rdev;
5127 else:
5128 frdev = fstatinfo.st_dev;
5129 getfrdev = GetDevMajorMinor(frdev);
5130 frdev_minor = getfrdev[0];
5131 frdev_major = getfrdev[1];
5132 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
5133 fsize = "0";
5134 if(ftype==0 or ftype==7):
5135 fsize = fstatinfo.st_size;
5136 fatime = fstatinfo.st_atime;
5137 fmtime = fstatinfo.st_mtime;
5138 fctime = fstatinfo.st_ctime;
5139 if(hasattr(fstatinfo, "st_birthtime")):
5140 fbtime = fstatinfo.st_birthtime;
5141 else:
5142 fbtime = fstatinfo.st_ctime;
5143 fmode = fstatinfo.st_mode;
5144 fchmode = stat.S_IMODE(fstatinfo.st_mode);
5145 ftypemod = stat.S_IFMT(fstatinfo.st_mode);
5146 fuid = fstatinfo.st_uid;
5147 fgid = fstatinfo.st_gid;
5148 funame = "";
5149 try:
5150 import pwd;
5151 try:
5152 userinfo = pwd.getpwuid(fstatinfo.st_uid);
5153 funame = userinfo.pw_name;
5154 except KeyError:
5155 funame = "";
5156 except ImportError:
5157 funame = "";
5158 fgname = "";
5159 try:
5160 import grp;
5161 try:
5162 groupinfo = grp.getgrgid(fstatinfo.st_gid);
5163 fgname = groupinfo.gr_name;
5164 except KeyError:
5165 fgname = "";
5166 except ImportError:
5167 fgname = "";
5168 fdev_minor = fdev_minor;
5169 fdev_major = fdev_major;
5170 frdev_minor = frdev_minor;
5171 frdev_major = frdev_major;
5172 flinkcount = flinkcount;
5173 if(hasattr(fstatinfo, "st_file_attributes")):
5174 fwinattributes = fstatinfo.st_file_attributes;
5175 else:
5176 fwinattributes = 0;
5177 fcompression = "";
5178 fcsize = 0;
5179 fcontents = BytesIO();
5180 if(ftype==0 or ftype==7):
5181 with open(fname, "rb") as fpc:
5182 shutil.copyfileobj(fpc, fcontents);
5183 if(followlink and (ftype==1 or ftype==2)):
5184 flstatinfo = os.stat(flinkname);
5185 with open(flinkname, "rb") as fpc:
5186 shutil.copyfileobj(fpc, fcontents);
5187 fcontents.seek(0, 0);
5188 ftypehex = format(ftype, 'x').lower();
5189 extrafields = len(extradata);
5190 extrafieldslist = extradata;
5191 catfextrafields = extrafields;
5192 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5193 if(len(extradata)>0):
5194 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5195 extrasizelen = len(extrasizestr);
5196 extrasizelenhex = format(extrasizelen, 'x').lower();
5197 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5198 catoutlen = len(catoutlist) + len(extradata) + 3;
5199 catoutlenhex = format(catoutlen, 'x').lower();
5200 catoutlist.insert(0, catoutlenhex);
5201 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5202 catheaderdata = catoutlist;
5203 if(len(extradata)>0):
5204 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5205 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
5206 catfnumfields = catoutlen;
5207 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5208 fcontents.seek(0, 0);
5209 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5210 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5211 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5212 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5213 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5214 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5215 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5216 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5217 catfcontentstart = fheadtell;
5218 fheadtell += len(catfileoutstr) + 1;
5219 catfcontentend = fheadtell - 1;
5220 catfhend = catfcontentend;
5221 fcontents.seek(0, 0);
5222 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5223 pyhascontents = False;
5224 if(int(fsize)>0 and not listonly):
5225 pyhascontents = True;
5226 if(int(fsize)>0 and listonly):
5227 fcontents = BytesIO();
5228 pyhascontents = False;
5229 fcontents.seek(0, 0);
5230 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5231 fileidnum = fileidnum + 1;
5232 return catlist;
5233
5234 def TarFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5235 formatspecs = FormatSpecsListToDict(formatspecs);
5236 curinode = 0;
5237 curfid = 0;
5238 inodelist = [];
5239 inodetofile = {};
5240 filetoinode = {};
5241 inodetocatinode = {};
5242 fileidnum = 0;
5243 if(infile=="-"):
5244 infile = BytesIO();
5245 if(hasattr(sys.stdin, "buffer")):
5246 shutil.copyfileobj(sys.stdin.buffer, infile);
5247 else:
5248 shutil.copyfileobj(sys.stdin, infile);
5249 infile.seek(0, 0);
5250 if(not infile):
5251 return False;
5252 infile.seek(0, 0);
5253 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
5254 infile = download_file_from_internet_file(infile);
5255 infile.seek(0, 0);
5256 if(not infile):
5257 return False;
5258 infile.seek(0, 0);
5259 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
5260 return False;
5261 elif(os.path.exists(infile) and os.path.isfile(infile)):
5262 try:
5263 if(not tarfile.is_tarfile(infile)):
5264 return False;
5265 except AttributeError:
5266 if(not is_tarfile(infile)):
5267 return False;
5268 else:
5269 return False;
5270 try:
5271 if(hasattr(infile, "read") or hasattr(infile, "write")):
5272 tarfp = tarfile.open(fileobj=infile, mode="r");
5273 else:
5274 tarfp = tarfile.open(infile, "r");
5275 except FileNotFoundError:
5276 return False;
5277 fnumfiles = int(len(tarfp.getmembers()));
5278 catver = formatspecs['format_ver'];
5279 fileheaderver = str(int(catver.replace(".", "")));
5280 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5281 fnumfileshex = format(int(fnumfiles), 'x').lower();
5282 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5283 catversion = re.findall(r"([\d]+)", fileheader);
5284 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5285 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5286 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5287 fheadtell = len(fileheader);
5288 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5289 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
5290 catfhstart = fheadtell;
5291 if(re.findall("^[.|/]", member.name)):
5292 fname = member.name;
5293 else:
5294 fname = "./"+member.name;
5295 if(verbose):
5296 VerbosePrintOut(fname);
5297 fpremode = member.mode;
5298 ffullmode = member.mode;
5299 flinkcount = 0;
5300 ftype = 0;
5301 if(member.isreg()):
5302 ffullmode = member.mode + stat.S_IFREG;
5303 ftype = 0;
5304 elif(member.isdev()):
5305 ffullmode = member.mode;
5306 ftype = 7;
5307 elif(member.islnk()):
5308 ffullmode = member.mode + stat.S_IFREG;
5309 ftype = 1;
5310 elif(member.issym()):
5311 ffullmode = member.mode + stat.S_IFLNK;
5312 ftype = 2;
5313 elif(member.ischr()):
5314 ffullmode = member.mode + stat.S_IFCHR;
5315 ftype = 3;
5316 elif(member.isblk()):
5317 ffullmode = member.mode + stat.S_IFBLK;
5318 ftype = 4;
5319 elif(member.isdir()):
5320 ffullmode = member.mode + stat.S_IFDIR;
5321 ftype = 5;
5322 elif(member.isfifo()):
5323 ffullmode = member.mode + stat.S_IFIFO;
5324 ftype = 6;
5325 elif(member.issparse()):
5326 ffullmode = member.mode;
5327 ftype = 12;
5328 else:
5329 ffullmode = member.mode;
5330 ftype = 0;
5331 flinkname = "";
5332 fbasedir = os.path.dirname(fname);
5333 fcurfid = curfid;
5334 fcurinode = curfid;
5335 finode = fcurinode;
5336 curfid = curfid + 1;
5337 if(ftype==2):
5338 flinkname = member.linkname;
5339 fdev_minor = member.devminor;
5340 fdev_major = member.devmajor;
5341 frdev_minor = member.devminor;
5342 frdev_major = member.devmajor;
5343 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
5344 fsize = "0";
5345 elif(ftype==0 or ftype==7):
5346 fsize = member.size;
5347 else:
5348 fsize = member.size;
5349 fatime = member.mtime;
5350 fmtime = member.mtime;
5351 fctime = member.mtime;
5352 fbtime = member.mtime;
5353 fmode = ffullmode;
5354 fchmode = stat.S_IMODE(ffullmode);
5355 ftypemod = stat.S_IFMT(ffullmode);
5356 fuid = member.uid;
5357 fgid = member.gid;
5358 funame = member.uname;
5359 fgname = member.gname;
5360 flinkcount = flinkcount;
5361 fwinattributes = int(0);
5362 fcompression = "";
5363 fcsize = 0;
5364 fcontents = BytesIO();
5365 if(ftype==0 or ftype==7):
5366 with tarfp.extractfile(member) as fpc:
5367 shutil.copyfileobj(fpc, fcontents);
5368 fcontents.seek(0, 0);
5369 ftypehex = format(ftype, 'x').lower();
5370 extrafields = len(extradata);
5371 extrafieldslist = extradata;
5372 catfextrafields = extrafields;
5373 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5374 if(len(extradata)>0):
5375 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5376 extrasizelen = len(extrasizestr);
5377 extrasizelenhex = format(extrasizelen, 'x').lower();
5378 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5379 catoutlen = len(catoutlist) + len(extradata) + 3;
5380 catoutlenhex = format(catoutlen, 'x').lower();
5381 catoutlist.insert(0, catoutlenhex);
5382 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5383 catheaderdata = catoutlist;
5384 if(len(extradata)>0):
5385 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5386 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
5387 catfnumfields = catoutlen;
5388 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5389 fcontents.seek(0, 0);
5390 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5391 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5392 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5393 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5394 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5395 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5396 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5397 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5398 catfcontentstart = fheadtell;
5399 fheadtell += len(catfileoutstr) + 1;
5400 catfcontentend = fheadtell - 1;
5401 catfhend = catfcontentend;
5402 fcontents.seek(0, 0);
5403 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5404 pyhascontents = False;
5405 if(int(fsize)>0 and not listonly):
5406 pyhascontents = True;
5407 if(int(fsize)>0 and listonly):
5408 fcontents = BytesIO();
5409 pyhascontents = False;
5410 fcontents.seek(0, 0);
5411 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5412 fileidnum = fileidnum + 1;
5413 return catlist;
5414
5415 def ZipFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5416 formatspecs = FormatSpecsListToDict(formatspecs);
5417 curinode = 0;
5418 curfid = 0;
5419 inodelist = [];
5420 inodetofile = {};
5421 filetoinode = {};
5422 inodetocatinode = {};
5423 fileidnum = 0;
5424 if(infile=="-"):
5425 infile = BytesIO();
5426 if(hasattr(sys.stdin, "buffer")):
5427 shutil.copyfileobj(sys.stdin.buffer, infile);
5428 else:
5429 shutil.copyfileobj(sys.stdin, infile);
5430 infile.seek(0, 0);
5431 if(not infile):
5432 return False;
5433 infile.seek(0, 0);
5434 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
5435 infile = download_file_from_internet_file(infile);
5436 infile.seek(0, 0);
5437 if(not infile):
5438 return False;
5439 infile.seek(0, 0);
5440 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
5441 return False;
5442 else:
5443 return False;
5444 if(not zipfile.is_zipfile(infile)):
5445 return False;
5446 try:
5447 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
5448 except FileNotFoundError:
5449 return False;
5450 ziptest = zipfp.testzip();
5451 if(ziptest):
5452 VerbosePrintOut("Bad file found!");
5453 fnumfiles = int(len(zipfp.infolist()));
5454 catver = formatspecs['format_ver'];
5455 fileheaderver = str(int(catver.replace(".", "")));
5456 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5457 catversion = re.findall(r"([\d]+)", fileheader);
5458 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5459 fnumfileshex = format(int(fnumfiles), 'x').lower();
5460 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5461 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5462 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5463 fheadtell = len(fileheader);
5464 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5465 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
5466 catfhstart = fheadtell;
5467 if(re.findall("^[.|/]", member.filename)):
5468 fname = member.filename;
5469 else:
5470 fname = "./"+member.filename;
5471 zipinfo = zipfp.getinfo(member.filename);
5472 if(verbose):
5473 VerbosePrintOut(fname);
5474 if(not member.is_dir()):
5475 fpremode = stat.S_IFREG + 438;
5476 elif(member.is_dir()):
5477 fpremode = stat.S_IFDIR + 511;
5478 flinkcount = 0;
5479 ftype = 0;
5480 if(not member.is_dir()):
5481 ftype = 0;
5482 elif(member.is_dir()):
5483 ftype = 5;
5484 flinkname = "";
5485 fbasedir = os.path.dirname(fname);
5486 fcurfid = curfid;
5487 fcurinode = curfid;
5488 finode = fcurinode;
5489 curfid = curfid + 1;
5490 fdev_minor = 0;
5491 fdev_major = 0;
5492 frdev_minor = 0;
5493 frdev_major = 0;
5494 if(ftype==5):
5495 fsize = "0";
5496 elif(ftype==0):
5497 fsize = member.file_size;
5498 else:
5499 fsize = member.file_size;
5500 fatime = time.mktime(member.date_time + (0, 0, -1));
5501 fmtime = time.mktime(member.date_time + (0, 0, -1));
5502 fctime = time.mktime(member.date_time + (0, 0, -1));
5503 fbtime = time.mktime(member.date_time + (0, 0, -1));
5504 if(zipinfo.create_system==0 or zipinfo.create_system==10):
5505 fwinattributes = int(zipinfo.external_attr);
5506 if(not member.is_dir()):
5507 fmode = int(stat.S_IFREG + 438);
5508 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
5509 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
5510 elif(member.is_dir()):
5511 fmode = int(stat.S_IFDIR + 511);
5512 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
5513 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
5514 elif(zipinfo.create_system==3):
5515 fwinattributes = int(0);
5516 fmode = int(zipinfo.external_attr);
5517 else:
5518 fwinattributes = int(0);
5519 if(not member.is_dir()):
5520 fmode = int(stat.S_IFREG + 438);
5521 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
5522 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
5523 elif(member.is_dir()):
5524 fmode = int(stat.S_IFDIR + 511);
5525 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
5526 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
5527 fcompression = "";
5528 fcsize = 0;
5529 try:
5530 fuid = os.getuid();
5531 except AttributeError:
5532 fuid = 0;
5533 except KeyError:
5534 fuid = 0;
5535 try:
5536 fgid = os.getgid();
5537 except AttributeError:
5538 fgid = 0;
5539 except KeyError:
5540 fgid = 0;
5541 try:
5542 import pwd;
5543 try:
5544 userinfo = pwd.getpwuid(os.getuid());
5545 funame = userinfo.pw_name;
5546 except KeyError:
5547 funame = "";
5548 except AttributeError:
5549 funame = "";
5550 except ImportError:
5551 funame = "";
5552 fgname = "";
5553 try:
5554 import grp;
5555 try:
5556 groupinfo = grp.getgrgid(os.getgid());
5557 fgname = groupinfo.gr_name;
5558 except KeyError:
5559 fgname = "";
5560 except AttributeError:
5561 fgname = "";
5562 except ImportError:
5563 fgname = "";
5564 fcontents = BytesIO();
5565 if(ftype==0):
5566 fcontents.write(zipfp.read(member.filename));
5567 fcontents.seek(0, 0);
5568 ftypehex = format(ftype, 'x').lower();
5569 extrafields = len(extradata);
5570 extrafieldslist = extradata;
5571 catfextrafields = extrafields;
5572 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5573 if(len(extradata)>0):
5574 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5575 extrasizelen = len(extrasizestr);
5576 extrasizelenhex = format(extrasizelen, 'x').lower();
5577 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5578 catoutlen = len(catoutlist) + len(extradata) + 3;
5579 catoutlenhex = format(catoutlen, 'x').lower();
5580 catoutlist.insert(0, catoutlenhex);
5581 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5582 catheaderdata = catoutlist;
5583 if(len(extradata)>0):
5584 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5585 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
5586 catfnumfields = catoutlen;
5587 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5588 fcontents.seek(0, 0);
5589 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5590 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5591 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5592 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5593 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5594 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5595 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5596 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5597 catfcontentstart = fheadtell;
5598 fheadtell += len(catfileoutstr) + 1;
5599 catfcontentend = fheadtell - 1;
5600 catfhend = catfcontentend;
5601 fcontents.seek(0, 0);
5602 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5603 pyhascontents = False;
5604 if(int(fsize)>0 and not listonly):
5605 pyhascontents = True;
5606 if(int(fsize)>0 and listonly):
5607 fcontents = BytesIO();
5608 pyhascontents = False;
5609 fcontents.seek(0, 0);
5610 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5611 fileidnum = fileidnum + 1;
5612 return catlist;
5613
5614 if(not rarfile_support):
5615 def RarFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5616 return False;
5617
5618 if(rarfile_support):
5619 def RarFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5620 formatspecs = FormatSpecsListToDict(formatspecs);
5621 curinode = 0;
5622 curfid = 0;
5623 inodelist = [];
5624 inodetofile = {};
5625 filetoinode = {};
5626 inodetocatinode = {};
5627 fileidnum = 0;
5628 if(not os.path.exists(infile,) or not os.path.isfile(infile,)):
5629 return False;
5630 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
5631 return False;
5632 rarfp = rarfile.RarFile(infile, "r");
5633 rartest = rarfp.testrar();
5634 if(rartest):
5635 VerbosePrintOut("Bad file found!");
5636 fnumfiles = int(len(rarfp.infolist()));
5637 catver = formatspecs['format_ver'];
5638 fileheaderver = str(int(catver.replace(".", "")));
5639 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5640 catversion = re.findall(r"([\d]+)", fileheader);
5641 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5642 fnumfileshex = format(int(fnumfiles), 'x').lower();
5643 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5644 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5645 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5646 fheadtell = len(fileheader);
5647 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5648 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
5649 catfhstart = fheadtell;
5650 is_unix = False;
5651 is_windows = False;
5652 if(member.host_os==rarfile.RAR_OS_UNIX):
5653 is_windows = False;
5654 try:
5655 member.external_attr
5656 is_unix = True;
5657 except AttributeError:
5658 is_unix = False;
5659 elif(member.host_os==rarfile.RAR_OS_WIN32):
5660 is_unix = False;
5661 try:
5662 member.external_attr
5663 is_windows = True;
5664 except AttributeError:
5665 is_windows = False;
5666 else:
5667 is_unix = False;
5668 is_windows = False;
5669 if(re.findall("^[.|/]", member.filename)):
5670 fname = member.filename;
5671 else:
5672 fname = "./"+member.filename;
5673 rarinfo = rarfp.getinfo(member.filename);
5674 if(verbose):
5675 VerbosePrintOut(fname);
5676 if(is_unix and member.external_attr !=0):
5677 fpremode = int(member.external_attr);
5678 elif(member.is_file()):
5679 fpremode = stat.S_IFREG + 438;
5680 elif(member.is_symlink()):
5681 fpremode = stat.S_IFLNK + 438;
5682 elif(member.is_dir()):
5683 fpremode = stat.S_IFDIR + 511;
5684 if(is_windows and member.external_attr !=0):
5685 fwinattributes = int(member.external_attr);
5686 else:
5687 fwinattributes = int(0);
5688 fcompression = "";
5689 fcsize = 0;
5690 flinkcount = 0;
5691 ftype = 0;
5692 if(member.is_file()):
5693 ftype = 0;
5694 elif(member.is_symlink()):
5695 ftype = 2;
5696 elif(member.is_dir()):
5697 ftype = 5;
5698 flinkname = "";
5699 if(ftype==2):
5700 flinkname = rarfp.read(member.filename).decode("UTF-8");
5701 fbasedir = os.path.dirname(fname);
5702 fcurfid = curfid;
5703 fcurinode = curfid;
5704 finode = fcurinode;
5705 curfid = curfid + 1;
5706 fdev_minor = 0;
5707 fdev_major = 0;
5708 frdev_minor = 0;
5709 frdev_major = 0;
5710 if(ftype==5):
5711 fsize = "0";
5712 if(ftype==0):
5713 fsize = member.file_size;
5714 try:
5715 if(member.atime):
5716 fatime = int(member.atime.timestamp());
5717 else:
5718 fatime = int(member.mtime.timestamp());
5719 except AttributeError:
5720 fatime = int(member.mtime.timestamp());
5721 fmtime = int(member.mtime.timestamp());
5722 try:
5723 if(member.ctime):
5724 fctime = int(member.ctime.timestamp());
5725 else:
5726 fctime = int(member.mtime.timestamp());
5727 except AttributeError:
5728 fctime = int(member.mtime.timestamp());
5729 fbtime = int(member.mtime.timestamp());
5730 if(is_unix and member.external_attr !=0):
5731 fmode = int(member.external_attr);
5732 fchmode = int(stat.S_IMODE(member.external_attr));
5733 ftypemod = int(stat.S_IFMT(member.external_attr));
5734 elif(member.is_file()):
5735 fmode = int(stat.S_IFREG + 438)
5736 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5737 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5738 elif(member.is_symlink()):
5739 fmode = int(stat.S_IFLNK + 438)
5740 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5741 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5742 elif(member.is_dir()):
5743 fmode = int(stat.S_IFDIR + 511)
5744 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
5745 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
5746 try:
5747 fuid = os.getuid();
5748 except AttributeError:
5749 fuid = 0;
5750 except KeyError:
5751 fuid = 0;
5752 try:
5753 fgid = os.getgid();
5754 except AttributeError:
5755 fgid = 0;
5756 except KeyError:
5757 fgid = 0;
5758 try:
5759 import pwd;
5760 try:
5761 userinfo = pwd.getpwuid(os.getuid());
5762 funame = userinfo.pw_name;
5763 except KeyError:
5764 funame = "";
5765 except AttributeError:
5766 funame = "";
5767 except ImportError:
5768 funame = "";
5769 fgname = "";
5770 try:
5771 import grp;
5772 try:
5773 groupinfo = grp.getgrgid(os.getgid());
5774 fgname = groupinfo.gr_name;
5775 except KeyError:
5776 fgname = "";
5777 except AttributeError:
5778 fgname = "";
5779 except ImportError:
5780 fgname = "";
5781 fcontents = BytesIO();
5782 if(ftype==0):
5783 fcontents.write(rarfp.read(member.filename));
5784 fcontents.seek(0, 0);
5785 ftypehex = format(ftype, 'x').lower();
5786 extrafields = len(extradata);
5787 extrafieldslist = extradata;
5788 catfextrafields = extrafields;
5789 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5790 if(len(extradata)>0):
5791 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5792 extrasizelen = len(extrasizestr);
5793 extrasizelenhex = format(extrasizelen, 'x').lower();
5794 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5795 catoutlen = len(catoutlist) + len(extradata) + 3;
5796 catoutlenhex = format(catoutlen, 'x').lower();
5797 catoutlist.insert(0, catoutlenhex);
5798 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5799 if(len(extradata)>0):
5800 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5801 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
5802 catfnumfields = 24 + catfextrafields;
5803 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5804 fcontents.seek(0, 0);
5805 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5806 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5807 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5808 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5809 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5810 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5811 catheaderdata = catoutlist;
5812 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5813 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5814 catfcontentstart = fheadtell;
5815 fheadtell += len(catfileoutstr) + 1;
5816 catfcontentend = fheadtell - 1;
5817 catfhend = catfcontentend;
5818 fcontents.seek(0, 0);
5819 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5820 pyhascontents = False;
5821 if(int(fsize)>0 and not listonly):
5822 pyhascontents = True;
5823 if(int(fsize)>0 and listonly):
5824 fcontents = BytesIO();
5825 pyhascontents = False;
5826 fcontents.seek(0, 0);
5827 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5828 fileidnum = fileidnum + 1;
5829 return catlist;
5830
5831 if(not py7zr_support):
5832 def SevenZipFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5833 return False;
5834
5835 if(py7zr_support):
5836 def SevenZipFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5837 formatspecs = FormatSpecsListToDict(formatspecs);
5838 curinode = 0;
5839 curfid = 0;
5840 inodelist = [];
5841 inodetofile = {};
5842 filetoinode = {};
5843 inodetocatinode = {};
5844 fileidnum = 0;
5845 szpfp = py7zr.SevenZipFile(infile, mode="r");
5846 file_content = szpfp.readall();
5847 #sztest = szpfp.testzip();
5848 sztestalt = szpfp.test();
5849 if(sztestalt):
5850 VerbosePrintOut("Bad file found!");
5851 numfiles = int(len(szpfp.list()));
5852 catver = formatspecs['format_ver'];
5853 fileheaderver = str(int(catver.replace(".", "")));
5854 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5855 catversion = re.findall(r"([\d]+)", fileheader);
5856 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5857 fnumfileshex = format(int(fnumfiles), 'x').lower();
5858 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5859 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5860 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5861 fheadtell = len(fileheader);
5862 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5863 for member in sorted(szpfp.list(), key=lambda x: x.filename):
5864 catfhstart = fheadtell;
5865 if(re.findall("^[.|/]", member.filename)):
5866 fname = member.filename;
5867 else:
5868 fname = "./"+member.filename;
5869 if(not member.is_directory):
5870 fpremode = int(stat.S_IFREG + 438);
5871 elif(member.is_directory):
5872 fpremode = int(stat.S_IFDIR + 511);
5873 fwinattributes = int(0);
5874 fcompression = "";
5875 fcsize = 0;
5876 flinkcount = 0;
5877 ftype = 0;
5878 if(member.is_directory):
5879 ftype = 5;
5880 else:
5881 ftype = 0;
5882 flinkname = "";
5883 fbasedir = os.path.dirname(fname);
5884 fcurfid = curfid;
5885 fcurinode = curfid;
5886 finode = fcurinode;
5887 curfid = curfid + 1;
5888 fdev_minor = 0;
5889 fdev_major = 0;
5890 frdev_minor = 0;
5891 frdev_major = 0;
5892 if(ftype==5):
5893 fsize = "0";
5894 fatime = int(member.creationtime.timestamp());
5895 fmtime = int(member.creationtime.timestamp());
5896 fctime = int(member.creationtime.timestamp());
5897 fbtime = int(member.creationtime.timestamp());
5898 if(member.is_directory):
5899 fmode = int(stat.S_IFDIR + 511)
5900 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
5901 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
5902 else:
5903 fmode = int(stat.S_IFLNK + 438)
5904 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5905 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5906 try:
5907 fuid = os.getuid();
5908 except AttributeError:
5909 fuid = 0;
5910 except KeyError:
5911 fuid = 0;
5912 try:
5913 fgid = os.getgid();
5914 except AttributeError:
5915 fgid = 0;
5916 except KeyError:
5917 fgid = 0;
5918 try:
5919 import pwd;
5920 try:
5921 userinfo = pwd.getpwuid(os.getuid());
5922 funame = userinfo.pw_name;
5923 except KeyError:
5924 funame = "";
5925 except AttributeError:
5926 funame = "";
5927 except ImportError:
5928 funame = "";
5929 fgname = "";
5930 try:
5931 import grp;
5932 try:
5933 groupinfo = grp.getgrgid(os.getgid());
5934 fgname = groupinfo.gr_name;
5935 except KeyError:
5936 fgname = "";
5937 except AttributeError:
5938 fgname = "";
5939 except ImportError:
5940 fgname = "";
5941 fcontents = BytesIO();
5942 if(ftype==0):
5943 fcontents.write(file_content[member.filename].read());
5944 fsize = format(fcontents.tell(), 'x').lower();
5945 fileop.close();
5946 fcontents.seek(0, 0);
5947 ftypehex = format(ftype, 'x').lower();
5948 extrafields = len(extradata);
5949 extrafieldslist = extradata;
5950 catfextrafields = extrafields;
5951 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5952 if(len(extradata)>0):
5953 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5954 extrasizelen = len(extrasizestr);
5955 extrasizelenhex = format(extrasizelen, 'x').lower();
5956 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5957 catoutlen = len(catoutlist) + len(extradata) + 3;
5958 catoutlenhex = format(catoutlen, 'x').lower();
5959 catoutlist.insert(0, catoutlenhex);
5960 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5961 catheaderdata = catoutlist;
5962 if(len(extradata)>0):
5963 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5964 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
5965 catfnumfields = 24 + catfextrafields;
5966 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5967 fcontents.seek(0, 0);
5968 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5969 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5970 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5971 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5972 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5973 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5974 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5975 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5976 catfcontentstart = fheadtell;
5977 fheadtell += len(catfileoutstr) + 1;
5978 catfcontentend = fheadtell - 1;
5979 catfhend = catfcontentend;
5980 fcontents.seek(0, 0);
5981 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5982 pyhascontents = False;
5983 if(int(fsize)>0 and not listonly):
5984 pyhascontents = True;
5985 if(int(fsize)>0 and listonly):
5986 fcontents = BytesIO();
5987 pyhascontents = False;
5988 fcontents.seek(0, 0);
5989 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5990 fileidnum = fileidnum + 1;
5991 return catlist;
5992
5993 def InFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5994 formatspecs = FormatSpecsListToDict(formatspecs);
5995 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
5996 if(checkcompressfile=="tarfile"):
5997 return TarFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
5998 elif(checkcompressfile=="zipfile"):
5999 return ZipFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6000 elif(checkcompressfile=="catfile"):
6001 return ArchiveFileToArray(infile, 0, 0, listonly, False, formatspecs, False);
6002 elif(rarfile_support and checkcompressfile=="rarfile"):
6003 return RarFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6004 elif(py7zr_support and checkcompressfile=="7zipfile"):
6005 return SevenZipFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6006 else:
6007 return False;
6008 return False;
6009
6010 def ListDirToArray(infiles, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, listonly=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6011 formatspecs = FormatSpecsListToDict(formatspecs);
6012 outarray = BytesIO();
6013 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
6014 listcatfiles = ArchiveFileToArray(outarray, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
6015 return listcatfiles;
6016
6017 def ArchiveFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6018 formatspecs = FormatSpecsListToDict(formatspecs);
6019 if(isinstance(infile, dict)):
6020 listcatfiles = infile;
6021 else:
6022 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6023 infile = RemoveWindowsPath(infile);
6024 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
6025 if(not listcatfiles):
6026 return False;
6027 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6028 if(returnfp):
6029 catarray.update({'catfp': listcatfiles['catfp']});
6030 lenlist = len(listcatfiles['ffilelist']);
6031 lcfi = 0;
6032 lcfx = int(listcatfiles['fnumfiles']);
6033 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6034 lcfx = int(lenlist);
6035 else:
6036 lcfx = int(listcatfiles['fnumfiles']);
6037 while(lcfi < lcfx):
6038 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6039 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6040 catarray['filetoid'].update(filetoidarray);
6041 catarray['idtofile'].update(idtofilearray);
6042 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6043 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6044 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6045 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6046 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6047 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6048 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6049 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6050 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6051 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6052 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6053 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6054 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6055 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6056 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6057 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6058 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6059 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6060 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6061 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6062 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6063 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6064 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6065 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6066 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6067 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6068 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6069 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6070 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6071 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6072 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6073 lcfi = lcfi + 1;
6074 return catarray;
6075
6076 create_alias_function("", __file_format_name__, "ToArrayIndex", ArchiveFileToArrayIndex);
6077
6078 def ListDirToArrayIndexAlt(infiles, dirlistfromtxt=False, followlink=False, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6079 formatspecs = FormatSpecsListToDict(formatspecs);
6080 listcatfiles = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, listonly, checksumtype, extradata, formatspecs, verbose);
6081 if(not listcatfiles):
6082 return False;
6083 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6084 lenlist = len(listcatfiles['ffilelist']);
6085 lcfi = 0;
6086 lcfx = int(listcatfiles['fnumfiles']);
6087 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6088 lcfx = int(lenlist);
6089 else:
6090 lcfx = int(listcatfiles['fnumfiles']);
6091 while(lcfi < lcfx):
6092 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6093 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6094 catarray['filetoid'].update(filetoidarray);
6095 catarray['idtofile'].update(idtofilearray);
6096 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6097 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6098 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6099 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6100 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6101 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6102 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6103 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6104 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6105 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6106 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6107 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6108 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6109 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6110 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6111 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6112 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6113 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6114 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6115 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6116 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6117 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6118 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6119 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6120 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6121 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6122 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6123 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6124 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6125 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6126 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6127 lcfi = lcfi + 1;
6128 return catarray;
6129
6130 def TarFileToArrayIndexAlt(infile, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6131 formatspecs = FormatSpecsListToDict(formatspecs);
6132 listcatfiles = TarFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6133 if(not listcatfiles):
6134 return False;
6135 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6136 lenlist = len(listcatfiles['ffilelist']);
6137 lcfi = 0;
6138 lcfx = int(listcatfiles['fnumfiles']);
6139 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6140 lcfx = int(lenlist);
6141 else:
6142 lcfx = int(listcatfiles['fnumfiles']);
6143 while(lcfi < lcfx):
6144 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6145 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6146 catarray['filetoid'].update(filetoidarray);
6147 catarray['idtofile'].update(idtofilearray);
6148 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6149 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6150 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6151 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6152 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6153 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6154 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6155 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6156 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6157 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6158 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6159 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6160 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6161 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6162 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6163 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6164 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6165 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6166 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6167 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6168 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6169 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6170 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6171 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6172 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6173 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6174 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6175 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6176 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6177 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6178 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6179 lcfi = lcfi + 1;
6180 return catarray;
6181
6182 def ZipFileToArrayIndexAlt(infile, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6183 formatspecs = FormatSpecsListToDict(formatspecs);
6184 listcatfiles = ZipFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6185 if(not listcatfiles):
6186 return False;
6187 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6188 lenlist = len(listcatfiles['ffilelist']);
6189 lcfi = 0;
6190 lcfx = int(listcatfiles['fnumfiles']);
6191 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6192 lcfx = int(lenlist);
6193 else:
6194 lcfx = int(listcatfiles['fnumfiles']);
6195 while(lcfi < lcfx):
6196 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6197 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6198 catarray['filetoid'].update(filetoidarray);
6199 catarray['idtofile'].update(idtofilearray);
6200 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6201 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6202 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6203 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6204 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6205 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6206 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6207 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6208 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6209 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6210 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6211 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6212 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6213 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6214 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6215 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6216 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6217 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6218 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6219 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6220 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6221 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6222 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6223 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6224 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6225 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6226 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6227 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6228 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6229 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6230 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6231 lcfi = lcfi + 1;
6232 return catarray;
6233
6234 if(not rarfile_support):
6235 def RarFileToArrayIndexAlt(infile, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6236 return False;
6237
6238 if(rarfile_support):
6239 def RarFileToArrayIndexAlt(infile, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6240 formatspecs = FormatSpecsListToDict(formatspecs);
6241 listcatfiles = RarFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6242 if(not listcatfiles):
6243 return False;
6244 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6245 lenlist = len(listcatfiles['ffilelist']);
6246 lcfi = 0;
6247 lcfx = int(listcatfiles['fnumfiles']);
6248 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6249 lcfx = int(lenlist);
6250 else:
6251 lcfx = int(listcatfiles['fnumfiles']);
6252 while(lcfi < lcfx):
6253 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6254 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6255 catarray['filetoid'].update(filetoidarray);
6256 catarray['idtofile'].update(idtofilearray);
6257 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6258 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6259 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6260 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6261 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6262 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6263 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6264 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6265 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6266 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6267 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6268 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6269 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6270 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6271 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6272 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6273 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6274 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6275 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6276 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6277 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6278 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6279 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6280 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6281 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6282 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6283 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6284 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6285 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6286 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6287 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6288 lcfi = lcfi + 1;
6289 return catarray;
6290
6291 if(not py7zr_support):
6292 def SevenZipFileToArrayIndexAlt(infile, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6293 return False;
6294
6295 if(py7zr_support):
6296 def SevenZipFileToArrayIndexAlt(infile, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6297 formatspecs = FormatSpecsListToDict(formatspecs);
6298 listcatfiles = SevenZipFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6299 if(not listcatfiles):
6300 return False;
6301 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6302 lenlist = len(listcatfiles['ffilelist']);
6303 lcfi = 0;
6304 lcfx = int(listcatfiles['fnumfiles']);
6305 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6306 lcfx = int(lenlist);
6307 else:
6308 lcfx = int(listcatfiles['fnumfiles']);
6309 while(lcfi < lcfx):
6310 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6311 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6312 catarray['filetoid'].update(filetoidarray);
6313 catarray['idtofile'].update(idtofilearray);
6314 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6315 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6316 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6317 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6318 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6319 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6320 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6321 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6322 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6323 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6324 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6325 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6326 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6327 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6328 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6329 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6330 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6331 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6332 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6333 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6334 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6335 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6336 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6337 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6338 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6339 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6340 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6341 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6342 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6343 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6344 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6345 lcfi = lcfi + 1;
6346 return catarray;
6347
6348 def ArchiveFileStringToArrayIndex(catstr, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6349 formatspecs = FormatSpecsListToDict(formatspecs);
6350 catfp = BytesIO(catstr);
6351 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
6352 return listcatfiles;
6353
6354 create_alias_function("", __file_format_name__, "StringToArrayIndex", ArchiveFileStringToArrayIndex);
6355
6356 def TarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6357 formatspecs = FormatSpecsListToDict(formatspecs);
6358 catfp = BytesIO();
6359 catfp = PackArchiveFileFromTarFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
6360 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
6361 return listcatfiles;
6362
6363 def ZipFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6364 formatspecs = FormatSpecsListToDict(formatspecs);
6365 catfp = BytesIO();
6366 catfp = PackArchiveFileFromZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
6367 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
6368 return listcatfiles;
6369
6370 if(not rarfile_support):
6371 def RarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6372 return False;
6373
6374 if(rarfile_support):
6375 def RarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6376 formatspecs = FormatSpecsListToDict(formatspecs);
6377 catfp = BytesIO();
6378 catfp = PackArchiveFileFromRarFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
6379 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
6380 return listcatfiles;
6381
6382 if(not py7zr_support):
6383 def SevenZipFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6384 return False;
6385
6386 if(py7zr_support):
6387 def SevenZipFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6388 formatspecs = FormatSpecsListToDict(formatspecs);
6389 catfp = BytesIO();
6390 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
6391 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
6392 return listcatfiles;
6393
6394 def ListDirToArrayIndex(infiles, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, listonly=False, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6395 formatspecs = FormatSpecsListToDict(formatspecs);
6396 outarray = BytesIO();
6397 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose, True);
6398 listcatfiles = ArchiveFileToArrayIndex(outarray, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp)
6399 return listcatfiles;
6400
6401 def RePackArchiveFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6402 formatspecs = FormatSpecsListToDict(formatspecs);
6403 if(isinstance(infile, dict)):
6404 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6405 listcatfiles = prelistcatfiles['list'];
6406 else:
6407 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6408 infile = RemoveWindowsPath(infile);
6409 if(followlink):
6410 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6411 listcatfiles = prelistcatfiles['list'];
6412 else:
6413 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6414 if(outfile!="-" and not hasattr(infile, "read") and not hasattr(outfile, "write")):
6415 outfile = RemoveWindowsPath(outfile);
6416 checksumtype = checksumtype.lower();
6417 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
6418 checksumtype="crc32";
6419 if(checksumtype=="none"):
6420 checksumtype = "";
6421 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
6422 compression = "auto";
6423 if(compression not in compressionlist and compression is None):
6424 compression = "auto";
6425 if(verbose):
6426 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6427 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
6428 if(os.path.exists(outfile)):
6429 try:
6430 os.unlink(outfile);
6431 except OSError as e:
6432 pass;
6433 if(not listcatfiles):
6434 return False;
6435 if(outfile=="-"):
6436 verbose = False;
6437 catfp = BytesIO();
6438 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
6439 catfp = outfile;
6440 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
6441 catfp = BytesIO();
6442 else:
6443 fbasename = os.path.splitext(outfile)[0];
6444 fextname = os.path.splitext(outfile)[1];
6445 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
6446 catver = formatspecs['format_ver'];
6447 fileheaderver = str(int(catver.replace(".", "")));
6448 lenlist = len(listcatfiles['ffilelist']);
6449 fnumfiles = int(listcatfiles['fnumfiles']);
6450 if(lenlist>fnumfiles or lenlist<fnumfiles):
6451 fnumfiles = lenlist;
6452 AppendFileHeader(catfp, fnumfiles, checksumtype, formatspecs);
6453 lenlist = len(listcatfiles['ffilelist']);
6454 fnumfiles = int(listcatfiles['fnumfiles']);
6455 lcfi = 0;
6456 lcfx = int(listcatfiles['fnumfiles']);
6457 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6458 lcfx = int(lenlist);
6459 else:
6460 lcfx = int(listcatfiles['fnumfiles']);
6461 curinode = 0;
6462 curfid = 0;
6463 inodelist = [];
6464 inodetofile = {};
6465 filetoinode = {};
6466 reallcfi = 0;
6467 while(lcfi < lcfx):
6468 if(re.findall("^[.|/]", listcatfiles['ffilelist'][reallcfi]['fname'])):
6469 fname = listcatfiles['ffilelist'][reallcfi]['fname'];
6470 else:
6471 fname = "./"+listcatfiles['ffilelist'][reallcfi]['fname'];
6472 if(verbose):
6473 VerbosePrintOut(fname);
6474 fheadersize = format(int(listcatfiles['ffilelist'][reallcfi]['fheadersize']), 'x').lower();
6475 fsize = format(int(listcatfiles['ffilelist'][reallcfi]['fsize']), 'x').lower();
6476 flinkname = listcatfiles['ffilelist'][reallcfi]['flinkname'];
6477 fatime = format(int(listcatfiles['ffilelist'][reallcfi]['fatime']), 'x').lower();
6478 fmtime = format(int(listcatfiles['ffilelist'][reallcfi]['fmtime']), 'x').lower();
6479 fctime = format(int(listcatfiles['ffilelist'][reallcfi]['fctime']), 'x').lower();
6480 fbtime = format(int(listcatfiles['ffilelist'][reallcfi]['fbtime']), 'x').lower();
6481 fmode = format(int(listcatfiles['ffilelist'][reallcfi]['fmode']), 'x').lower();
6482 fchmode = format(int(listcatfiles['ffilelist'][reallcfi]['fchmode']), 'x').lower();
6483 fuid = format(int(listcatfiles['ffilelist'][reallcfi]['fuid']), 'x').lower();
6484 funame = listcatfiles['ffilelist'][reallcfi]['funame'];
6485 fgid = format(int(listcatfiles['ffilelist'][reallcfi]['fgid']), 'x').lower();
6486 fgname = listcatfiles['ffilelist'][reallcfi]['fgname'];
6487 finode = format(int(listcatfiles['ffilelist'][reallcfi]['finode']), 'x').lower();
6488 flinkcount = format(int(listcatfiles['ffilelist'][reallcfi]['flinkcount']), 'x').lower();
6489 fwinattributes = format(int(listcatfiles['ffilelist'][reallcfi]['fwinattributes']), 'x').lower();
6490 fcompression = listcatfiles['ffilelist'][reallcfi]['fcompression'];
6491 fcsize = format(int(listcatfiles['ffilelist'][reallcfi]['fcsize']), 'x').lower();
6492 fdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['fminor']), 'x').lower();
6493 fdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['fmajor']), 'x').lower();
6494 frdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['frminor']), 'x').lower();
6495 frdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['frmajor']), 'x').lower();
6496 if(len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>listcatfiles['ffilelist'][reallcfi]['fextrafields'] and len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>0):
6497 listcatfiles['ffilelist'][reallcfi]['fextrafields'] = len(listcatfiles['ffilelist'][reallcfi]['fextralist']);
6498 if(not followlink and len(extradata)<0):
6499 extradata = listcatfiles['ffilelist'][reallcfi]['fextralist'];
6500 fcontents = listcatfiles['ffilelist'][reallcfi]['fcontents'];
6501 fcompression = "";
6502 fcsize = format(int(0), 'x').lower();
6503 if(not compresswholefile):
6504 fcontents.seek(0, 2);
6505 ucfsize = fcontents.tell();
6506 fcontents.seek(0, 0);
6507 if(compression=="auto"):
6508 ilsize = len(compressionlistalt);
6509 ilmin = 0;
6510 ilcsize = [];
6511 while(ilmin < ilsize):
6512 cfcontents = BytesIO();
6513 shutil.copyfileobj(fcontents, cfcontents);
6514 fcontents.seek(0, 0);
6515 cfcontents.seek(0, 0);
6516 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
6517 if(cfcontents):
6518 cfcontents.seek(0, 2);
6519 ilcsize.append(cfcontents.tell());
6520 cfcontents.close();
6521 else:
6522 try:
6523 ilcsize.append(sys.maxint);
6524 except AttributeError:
6525 ilcsize.append(sys.maxsize);
6526 ilmin = ilmin + 1;
6527 ilcmin = ilcsize.index(min(ilcsize));
6528 compression = compressionlistalt[ilcmin];
6529 fcontents.seek(0, 0);
6530 cfcontents = BytesIO();
6531 shutil.copyfileobj(fcontents, cfcontents);
6532 cfcontents.seek(0, 0);
6533 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
6534 cfcontents.seek(0, 2);
6535 cfsize = cfcontents.tell();
6536 if(ucfsize > cfsize):
6537 fcsize = format(int(cfsize), 'x').lower();
6538 fcompression = compression;
6539 fcontents.close();
6540 fcontents = cfcontents;
6541 if(followlink):
6542 if(listcatfiles['ffilelist'][reallcfi]['ftype']==1 or listcatfiles['ffilelist'][reallcfi]['ftype']==2):
6543 getflinkpath = listcatfiles['ffilelist'][reallcfi]['flinkname'];
6544 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6545 flinkinfo = listcatfiles['ffilelist'][flinkid];
6546 fheadersize = format(int(flinkinfo['fheadersize']), 'x').lower();
6547 fsize = format(int(flinkinfo['fsize']), 'x').lower();
6548 flinkname = flinkinfo['flinkname'];
6549 fatime = format(int(flinkinfo['fatime']), 'x').lower();
6550 fmtime = format(int(flinkinfo['fmtime']), 'x').lower();
6551 fctime = format(int(flinkinfo['fctime']), 'x').lower();
6552 fbtime = format(int(flinkinfo['fbtime']), 'x').lower();
6553 fmode = format(int(flinkinfo['fmode']), 'x').lower();
6554 fchmode = format(int(flinkinfo['fchmode']), 'x').lower();
6555 fuid = format(int(flinkinfo['fuid']), 'x').lower();
6556 funame = flinkinfo['funame'];
6557 fgid = format(int(flinkinfo['fgid']), 'x').lower();
6558 fgname = flinkinfo['fgname'];
6559 finode = format(int(flinkinfo['finode']), 'x').lower();
6560 flinkcount = format(int(flinkinfo['flinkcount']), 'x').lower();
6561 fwinattributes = format(int(flinkinfo['fwinattributes']), 'x').lower();
6562 fcompression = flinkinfo['fcompression'];
6563 fcsize = format(int(flinkinfo['fcsize']), 'x').lower();
6564 fdev_minor = format(int(flinkinfo['fminor']), 'x').lower();
6565 fdev_major = format(int(flinkinfo['fmajor']), 'x').lower();
6566 frdev_minor = format(int(flinkinfo['frminor']), 'x').lower();
6567 frdev_major = format(int(flinkinfo['frmajor']), 'x').lower();
6568 if(len(flinkinfo['fextralist'])>flinkinfo['fextrafields'] and len(flinkinfo['fextralist'])>0):
6569 flinkinfo['fextrafields'] = len(flinkinfo['fextralist']);
6570 if(len(extradata)<0):
6571 extradata = flinkinfo['fextralist'];
6572 fcontents = flinkinfo['fcontents'];
6573 ftypehex = format(flinkinfo['ftype'], 'x').lower();
6574 else:
6575 ftypehex = format(listcatfiles['ffilelist'][reallcfi]['ftype'], 'x').lower();
6576 fcurfid = format(curfid, 'x').lower();
6577 if(not followlink and finode!=0):
6578 if(listcatfiles['ffilelist'][reallcfi]['ftype']!=1):
6579 fcurinode = format(int(curinode), 'x').lower();
6580 inodetofile.update({curinode: fname});
6581 filetoinode.update({fname: curinode});
6582 curinode = curinode + 1;
6583 else:
6584 fcurinode = format(int(filetoinode[flinkname]), 'x').lower();
6585 else:
6586 fcurinode = format(int(curinode), 'x').lower();
6587 curinode = curinode + 1;
6588 curfid = curfid + 1;
6589 if(fcompression=="none"):
6590 fcompression = "";
6591 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
6592 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
6593 fcontents.close();
6594 lcfi = lcfi + 1;
6595 reallcfi = reallcfi + 1;
6596 if(lcfx>0):
6597 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
6598 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
6599 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
6600 try:
6601 catfp.flush();
6602 os.fsync(catfp.fileno());
6603 except io.UnsupportedOperation:
6604 pass;
6605 except AttributeError:
6606 pass;
6607 except OSError as e:
6608 pass;
6609 if(outfile=="-"):
6610 catfp.seek(0, 0);
6611 if(hasattr(sys.stdout, "buffer")):
6612 shutil.copyfileobj(catfp, sys.stdout.buffer);
6613 else:
6614 shutil.copyfileobj(catfp, sys.stdout);
6615 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
6616 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
6617 catfp.seek(0, 0);
6618 upload_file_to_internet_file(catfp, outfile);
6619 if(returnfp):
6620 catfp.seek(0, 0);
6621 return catfp;
6622 else:
6623 catfp.close();
6624 return True;
6625
6626 create_alias_function("RePack", __file_format_name__, "", RePackArchiveFile);
6627
6628 def RePackArchiveFileFromString(catstr, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6629 formatspecs = FormatSpecsListToDict(formatspecs);
6630 catfp = BytesIO(catstr);
6631 listcatfiles = RePackArchiveFile(catfp, compression, compresswholefile, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6632 return listcatfiles;
6633
6634 create_alias_function("RePack", __file_format_name__, "FromString", RePackArchiveFileFromString);
6635
6636 def PackArchiveFileFromListDir(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6637 formatspecs = FormatSpecsListToDict(formatspecs);
6638 outarray = BytesIO();
6639 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
6640 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6641 return listcatfiles;
6642
6643 create_alias_function("Pack", __file_format_name__, "FromListDir", PackArchiveFileFromListDir);
6644
6645 def UnPackArchiveFile(infile, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, preservepermissions=True, preservetime=True, verbose=False, returnfp=False):
6646 formatspecs = FormatSpecsListToDict(formatspecs);
6647 if(outdir is not None):
6648 outdir = RemoveWindowsPath(outdir);
6649 if(verbose):
6650 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6651 if(isinstance(infile, dict)):
6652 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6653 listcatfiles = prelistcatfiles['list'];
6654 else:
6655 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6656 infile = RemoveWindowsPath(infile);
6657 if(followlink):
6658 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6659 listcatfiles = prelistcatfiles['list'];
6660 else:
6661 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6662 if(not listcatfiles):
6663 return False;
6664 lenlist = len(listcatfiles['ffilelist']);
6665 fnumfiles = int(listcatfiles['fnumfiles']);
6666 lcfi = 0;
6667 lcfx = int(listcatfiles['fnumfiles']);
6668 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6669 lcfx = int(lenlist);
6670 else:
6671 lcfx = int(listcatfiles['fnumfiles']);
6672 while(lcfi<lcfx):
6673 funame = "";
6674 try:
6675 import pwd;
6676 try:
6677 userinfo = pwd.getpwuid(listcatfiles['ffilelist'][lcfi]['fuid']);
6678 funame = userinfo.pw_name;
6679 except KeyError:
6680 funame = "";
6681 except ImportError:
6682 funame = "";
6683 fgname = "";
6684 try:
6685 import grp;
6686 try:
6687 groupinfo = grp.getgrgid(listcatfiles['ffilelist'][lcfi]['fgid']);
6688 fgname = groupinfo.gr_name;
6689 except KeyError:
6690 fgname = "";
6691 except ImportError:
6692 fgname = "";
6693 if(verbose):
6694 VerbosePrintOut(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6695 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6696 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6697 listcatfiles['ffilelist'][lcfi]['fcontents'].seek(0, 0);
6698 shutil.copyfileobj(listcatfiles['ffilelist'][lcfi]['fcontents'], fpc);
6699 try:
6700 fpc.flush();
6701 os.fsync(fpc.fileno());
6702 except io.UnsupportedOperation:
6703 pass;
6704 except AttributeError:
6705 pass;
6706 except OSError as e:
6707 pass;
6708 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
6709 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
6710 if(preservepermissions):
6711 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6712 if(preservetime):
6713 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
6714 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6715 if(followlink):
6716 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
6717 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6718 flinkinfo = listcatfiles['ffilelist'][flinkid];
6719 funame = "";
6720 try:
6721 import pwd;
6722 try:
6723 userinfo = pwd.getpwuid(flinkinfo['fuid']);
6724 funame = userinfo.pw_name;
6725 except KeyError:
6726 funame = "";
6727 except ImportError:
6728 funame = "";
6729 fgname = "";
6730 try:
6731 import grp;
6732 try:
6733 groupinfo = grp.getgrgid(flinkinfo['fgid']);
6734 fgname = groupinfo.gr_name;
6735 except KeyError:
6736 fgname = "";
6737 except ImportError:
6738 fgname = "";
6739 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
6740 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6741 flinkinfo['fcontents'].seek(0, 0);
6742 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
6743 try:
6744 fpc.flush();
6745 os.fsync(fpc.fileno());
6746 except io.UnsupportedOperation:
6747 pass;
6748 except AttributeError:
6749 pass;
6750 except OSError as e:
6751 pass;
6752 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6753 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6754 if(preservepermissions):
6755 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6756 if(preservetime):
6757 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6758 if(flinkinfo['ftype']==1):
6759 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6760 if(flinkinfo['ftype']==2):
6761 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6762 if(flinkinfo['ftype']==5):
6763 if(preservepermissions):
6764 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6765 else:
6766 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6767 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6768 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6769 if(preservepermissions):
6770 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6771 if(preservetime):
6772 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6773 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
6774 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6775 else:
6776 os.link(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6777 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6778 if(followlink):
6779 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
6780 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6781 flinkinfo = listcatfiles['ffilelist'][flinkid];
6782 funame = "";
6783 try:
6784 import pwd;
6785 try:
6786 userinfo = pwd.getpwuid(flinkinfo['fuid']);
6787 funame = userinfo.pw_name;
6788 except KeyError:
6789 funame = "";
6790 except ImportError:
6791 funame = "";
6792 fgname = "";
6793 try:
6794 import grp;
6795 try:
6796 groupinfo = grp.getgrgid(flinkinfo['fgid']);
6797 fgname = groupinfo.gr_name;
6798 except KeyError:
6799 fgname = "";
6800 except ImportError:
6801 fgname = "";
6802 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
6803 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6804 flinkinfo['fcontents'].seek(0, 0);
6805 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
6806 try:
6807 fpc.flush();
6808 os.fsync(fpc.fileno());
6809 except io.UnsupportedOperation:
6810 pass;
6811 except AttributeError:
6812 pass;
6813 except OSError as e:
6814 pass;
6815 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6816 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6817 if(preservepermissions):
6818 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6819 if(preservetime):
6820 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6821 if(flinkinfo['ftype']==1):
6822 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6823 if(flinkinfo['ftype']==2):
6824 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6825 if(flinkinfo['ftype']==5):
6826 if(preservepermissions):
6827 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6828 else:
6829 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6830 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6831 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6832 if(preservepermissions):
6833 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6834 if(preservetime):
6835 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6836 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
6837 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6838 else:
6839 os.symlink(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6840 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6841 if(preservepermissions):
6842 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6843 else:
6844 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6845 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
6846 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
6847 if(preservepermissions):
6848 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6849 if(preservetime):
6850 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
6851 if(listcatfiles['ffilelist'][lcfi]['ftype']==6 and hasattr(os, "mkfifo")):
6852 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6853 lcfi = lcfi + 1;
6854 if(returnfp):
6855 return listcatfiles['ffilelist']['catfp'];
6856 else:
6857 return True;
6858
6859 create_alias_function("UnPack", __file_format_name__, "", UnPackArchiveFile);
6860
6861 if(hasattr(shutil, "register_unpack_format")):
6862 def UnPackArchiveFileFunc(archive_name, extract_dir=None, **kwargs):
6863 return UnPackArchiveFile(archive_name, extract_dir, False, 0, 0, False, __file_format_dict__['format_delimiter'], False, False);
6864 create_alias_function("UnPack", __file_format_name__, "Func", UnPackArchiveFileFunc);
6865
6866 def UnPackArchiveFileString(catstr, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6867 formatspecs = FormatSpecsListToDict(formatspecs);
6868 catfp = BytesIO(catstr);
6869 listcatfiles = UnPackArchiveFile(catfp, outdir, followlink, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6870 return listcatfiles;
6871
6872 create_alias_function("UnPack", __file_format_name__, "String", UnPackArchiveFileString);
6873
6874 def ArchiveFileListFiles(infile, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6875 formatspecs = FormatSpecsListToDict(formatspecs);
6876 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6877 if(isinstance(infile, dict)):
6878 listcatfiles = infile;
6879 else:
6880 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6881 infile = RemoveWindowsPath(infile);
6882 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, True, skipchecksum, formatspecs, returnfp);
6883 if(not listcatfiles):
6884 return False;
6885 lenlist = len(listcatfiles['ffilelist']);
6886 fnumfiles = int(listcatfiles['fnumfiles']);
6887 lcfi = 0;
6888 lcfx = int(listcatfiles['fnumfiles']);
6889 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6890 lcfx = int(lenlist);
6891 else:
6892 lcfx = int(listcatfiles['fnumfiles']);
6893 returnval = {};
6894 while(lcfi<lcfx):
6895 returnval.update({lcfi: listcatfiles['ffilelist'][lcfi]['fname']});
6896 if(not verbose):
6897 VerbosePrintOut(listcatfiles['ffilelist'][lcfi]['fname']);
6898 if(verbose):
6899 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6900 printfname = listcatfiles['ffilelist'][lcfi]['fname'];
6901 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6902 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " link to " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6903 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6904 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " -> " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6905 fuprint = listcatfiles['ffilelist'][lcfi]['funame'];
6906 if(len(fuprint)<=0):
6907 fuprint = listcatfiles['ffilelist'][lcfi]['fuid'];
6908 fgprint = listcatfiles['ffilelist'][lcfi]['fgname'];
6909 if(len(fgprint)<=0):
6910 fgprint = listcatfiles['ffilelist'][lcfi]['fgid'];
6911 VerbosePrintOut(PrintPermissionString(listcatfiles['ffilelist'][lcfi]['fmode'], listcatfiles['ffilelist'][lcfi]['ftype']) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(listcatfiles['ffilelist'][lcfi]['fsize']).rjust(15) + " " + datetime.datetime.utcfromtimestamp(listcatfiles['ffilelist'][lcfi]['fmtime']).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6912 lcfi = lcfi + 1;
6913 if(returnfp):
6914 return listcatfiles['catfp'];
6915 else:
6916 return True;
6917
6918 create_alias_function("", __file_format_name__, "ListFiles", ArchiveFileListFiles);
6919
6920 def ArchiveFileStringListFiles(catstr, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6921 formatspecs = FormatSpecsListToDict(formatspecs);
6922 catfp = BytesIO(catstr);
6923 listcatfiles = ArchiveFileListFiles(catstr, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6924 return listcatfiles;
6925
6926 create_alias_function("", __file_format_name__, "StringListFiles", ArchiveFileStringListFiles);
6927
6928 def TarFileListFiles(infile, verbose=False, returnfp=False):
6929 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6930 if(infile=="-"):
6931 infile = BytesIO();
6932 if(hasattr(sys.stdin, "buffer")):
6933 shutil.copyfileobj(sys.stdin.buffer, infile);
6934 else:
6935 shutil.copyfileobj(sys.stdin, infile);
6936 infile.seek(0, 0);
6937 if(not infile):
6938 return False;
6939 infile.seek(0, 0);
6940 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
6941 infile = download_file_from_internet_file(infile);
6942 infile.seek(0, 0);
6943 if(not infile):
6944 return False;
6945 infile.seek(0, 0);
6946 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
6947 return False;
6948 elif(os.path.exists(infile) and os.path.isfile(infile)):
6949 try:
6950 if(not tarfile.is_tarfile(infile)):
6951 return False;
6952 except AttributeError:
6953 if(not is_tarfile(infile)):
6954 return False;
6955 else:
6956 return False;
6957 try:
6958 if(hasattr(infile, "read") or hasattr(infile, "write")):
6959 tarfp = tarfile.open(fileobj=infile, mode="r");
6960 else:
6961 tarfp = tarfile.open(infile, "r");
6962 except FileNotFoundError:
6963 return False;
6964 lcfi = 0
6965 returnval = {};
6966 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
6967 returnval.update({lcfi: member.name});
6968 fpremode = member.mode;
6969 ffullmode = member.mode;
6970 flinkcount = 0;
6971 ftype = 0;
6972 if(member.isreg()):
6973 ffullmode = member.mode + stat.S_IFREG;
6974 ftype = 0;
6975 elif(member.isdev()):
6976 ffullmode = member.mode;
6977 ftype = 7;
6978 elif(member.islnk()):
6979 ffullmode = member.mode + stat.S_IFREG;
6980 ftype = 1;
6981 elif(member.issym()):
6982 ffullmode = member.mode + stat.S_IFLNK;
6983 ftype = 2;
6984 elif(member.ischr()):
6985 ffullmode = member.mode + stat.S_IFCHR;
6986 ftype = 3;
6987 elif(member.isblk()):
6988 ffullmode = member.mode + stat.S_IFBLK;
6989 ftype = 4;
6990 elif(member.isdir()):
6991 ffullmode = member.mode + stat.S_IFDIR;
6992 ftype = 5;
6993 elif(member.isfifo()):
6994 ffullmode = member.mode + stat.S_IFIFO;
6995 ftype = 6;
6996 elif(member.issparse()):
6997 ffullmode = member.mode;
6998 ftype = 12;
6999 if(not verbose):
7000 VerbosePrintOut(member.name);
7001 elif(verbose):
7002 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7003 printfname = member.name;
7004 if(member.islnk()):
7005 printfname = member.name + " link to " + member.linkname;
7006 elif(member.issym()):
7007 printfname = member.name + " -> " + member.linkname;
7008 fuprint = member.uname;
7009 if(len(fuprint)<=0):
7010 fuprint = member.uid;
7011 fgprint = member.gname;
7012 if(len(fgprint)<=0):
7013 fgprint = member.gid;
7014 VerbosePrintOut(PrintPermissionString(ffullmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(member.mtime).strftime('%Y-%m-%d %H:%M') + " " + printfname));
7015 lcfi = lcfi + 1;
7016 if(returnfp):
7017 return listcatfiles['catfp'];
7018 else:
7019 return True;
7020
7021 def ZipFileListFiles(infile, verbose=False, returnfp=False):
7022 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7023 if(infile=="-"):
7024 infile = BytesIO();
7025 if(hasattr(sys.stdin, "buffer")):
7026 shutil.copyfileobj(sys.stdin.buffer, infile);
7027 else:
7028 shutil.copyfileobj(sys.stdin, infile);
7029 infile.seek(0, 0);
7030 if(not infile):
7031 return False;
7032 infile.seek(0, 0);
7033 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
7034 infile = download_file_from_internet_file(infile);
7035 infile.seek(0, 0);
7036 if(not infile):
7037 return False;
7038 infile.seek(0, 0);
7039 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
7040 return False;
7041 else:
7042 return False;
7043 if(not zipfile.is_zipfile(infile)):
7044 return False;
7045 try:
7046 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
7047 except FileNotFoundError:
7048 return False;
7049 lcfi = 0;
7050 returnval = {};
7051 ziptest = zipfp.testzip();
7052 if(ziptest):
7053 VerbosePrintOut("Bad file found!");
7054 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
7055 if(zipinfo.create_system==0 or zipinfo.create_system==10):
7056 fwinattributes = int(zipinfo.external_attr);
7057 if(not member.is_dir()):
7058 fmode = int(stat.S_IFREG + 438);
7059 fchmode = int(stat.S_IMODE(fmode));
7060 ftypemod = int(stat.S_IFMT(fmode));
7061 elif(member.is_dir()):
7062 fmode = int(stat.S_IFDIR + 511);
7063 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7064 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7065 elif(zipinfo.create_system==3):
7066 fwinattributes =int(0);
7067 fmode = int(zipinfo.external_attr);
7068 fchmode = int(stat.S_IMODE(fmode));
7069 ftypemod = int(stat.S_IFMT(fmode));
7070 else:
7071 fwinattributes = int(0);
7072 if(not member.is_dir()):
7073 fmode = int(stat.S_IFREG + 438);
7074 fchmode = int(stat.S_IMODE(fmode));
7075 ftypemod = int(stat.S_IFMT(fmode));
7076 elif(member.is_dir()):
7077 fmode = int(stat.S_IFDIR + 511);
7078 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7079 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7080 returnval.update({lcfi: member.filename});
7081 if(not verbose):
7082 VerbosePrintOut(member.filename);
7083 if(verbose):
7084 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7085 permissionstr = "";
7086 for fmodval in str(oct(fmode))[-3:]:
7087 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
7088 if(not member.is_dir()):
7089 ftype = 0;
7090 permissionstr = "-" + permissionstr;
7091 elif(member.is_dir()):
7092 ftype = 5;
7093 permissionstr = "d" + permissionstr;
7094 printfname = member.filename;
7095 try:
7096 fuid = int(os.getuid());
7097 except AttributeError:
7098 fuid = int(0);
7099 except KeyError:
7100 fuid = int(0);
7101 try:
7102 fgid = int(os.getgid());
7103 except AttributeError:
7104 fgid = int(0);
7105 except KeyError:
7106 fgid = int(0);
7107 try:
7108 import pwd;
7109 try:
7110 userinfo = pwd.getpwuid(os.getuid());
7111 funame = userinfo.pw_name;
7112 except KeyError:
7113 funame = "";
7114 except AttributeError:
7115 funame = "";
7116 except ImportError:
7117 funame = "";
7118 fgname = "";
7119 try:
7120 import grp;
7121 try:
7122 groupinfo = grp.getgrgid(os.getgid());
7123 fgname = groupinfo.gr_name;
7124 except KeyError:
7125 fgname = "";
7126 except AttributeError:
7127 fgname = "";
7128 except ImportError:
7129 fgname = "";
7130 fuprint = funame;
7131 if(len(fuprint)<=0):
7132 fuprint = str(fuid);
7133 fgprint = fgname;
7134 if(len(fgprint)<=0):
7135 fgprint = str(fgid);
7136 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(int(time.mktime(member.date_time + (0, 0, -1)))).strftime('%Y-%m-%d %H:%M') + " " + printfname));
7137 lcfi = lcfi + 1;
7138 if(returnfp):
7139 return listcatfiles['catfp'];
7140 else:
7141 return True;
7142
7143 if(not rarfile_support):
7144 def RarFileListFiles(infile, verbose=False, returnfp=False):
7145 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7146 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7147 return False;
7148
7149 if(rarfile_support):
7150 def RarFileListFiles(infile, verbose=False, returnfp=False):
7151 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7152 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7153 return False;
7154 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
7155 return False;
7156 lcfi = 0;
7157 returnval = {};
7158 rarfp = rarfile.RarFile(infile, "r");
7159 rartest = rarfp.testrar();
7160 if(rartest):
7161 VerbosePrintOut("Bad file found!");
7162 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
7163 is_unix = False;
7164 is_windows = False;
7165 if(member.host_os==rarfile.RAR_OS_UNIX):
7166 is_windows = False;
7167 try:
7168 member.external_attr
7169 is_unix = True;
7170 except AttributeError:
7171 is_unix = False;
7172 elif(member.host_os==rarfile.RAR_OS_WIN32):
7173 is_unix = False;
7174 try:
7175 member.external_attr
7176 is_windows = True;
7177 except AttributeError:
7178 is_windows = False;
7179 else:
7180 is_unix = False;
7181 is_windows = False;
7182 if(is_unix and member.external_attr !=0):
7183 fpremode = int(member.external_attr);
7184 elif(member.is_file()):
7185 fpremode = int(stat.S_IFREG + 438);
7186 elif(member.is_symlink()):
7187 fpremode = int(stat.S_IFLNK + 438);
7188 elif(member.is_dir()):
7189 fpremode = int(stat.S_IFDIR + 511);
7190 if(is_windows and member.external_attr !=0):
7191 fwinattributes = int(member.external_attr);
7192 else:
7193 fwinattributes = int(0);
7194 if(is_unix and member.external_attr !=0):
7195 fmode = int(member.external_attr);
7196 fchmode = int(stat.S_IMODE(member.external_attr));
7197 ftypemod = int(stat.S_IFMT(member.external_attr));
7198 elif(member.is_file()):
7199 fmode = int(stat.S_IFREG + 438);
7200 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
7201 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
7202 elif(member.is_symlink()):
7203 fmode = int(stat.S_IFLNK + 438);
7204 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
7205 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
7206 elif(member.is_dir()):
7207 fmode = int(stat.S_IFDIR + 511);
7208 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7209 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7210 returnval.update({lcfi: member.filename});
7211 if(not verbose):
7212 VerbosePrintOut(member.filename);
7213 if(verbose):
7214 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7215 permissionstr = "";
7216 for fmodval in str(oct(fmode))[-3:]:
7217 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
7218 if(member.is_file()):
7219 ftype = 0;
7220 permissionstr = "-" + permissionstr;
7221 printfname = member.filename;
7222 elif(member.is_symlink()):
7223 ftype = 2;
7224 permissionstr = "l" + permissionstr;
7225 printfname = member.name + " -> " + member.read().decode("UTF-8");
7226 elif(member.is_dir()):
7227 ftype = 5;
7228 permissionstr = "d" + permissionstr;
7229 printfname = member.filename;
7230 try:
7231 fuid = int(os.getuid());
7232 except AttributeError:
7233 fuid = int(0);
7234 except KeyError:
7235 fuid = int(0);
7236 try:
7237 fgid = int(os.getgid());
7238 except AttributeError:
7239 fgid = int(0);
7240 except KeyError:
7241 fgid = int(0);
7242 try:
7243 import pwd;
7244 try:
7245 userinfo = pwd.getpwuid(os.getuid());
7246 funame = userinfo.pw_name;
7247 except KeyError:
7248 funame = "";
7249 except AttributeError:
7250 funame = "";
7251 except ImportError:
7252 funame = "";
7253 fgname = "";
7254 try:
7255 import grp;
7256 try:
7257 groupinfo = grp.getgrgid(os.getgid());
7258 fgname = groupinfo.gr_name;
7259 except KeyError:
7260 fgname = "";
7261 except AttributeError:
7262 fgname = "";
7263 except ImportError:
7264 fgname = "";
7265 fuprint = funame;
7266 if(len(fuprint)<=0):
7267 fuprint = str(fuid);
7268 fgprint = fgname;
7269 if(len(fgprint)<=0):
7270 fgprint = str(fgid);
7271 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + member.mtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
7272 lcfi = lcfi + 1;
7273 if(returnfp):
7274 return listcatfiles['catfp'];
7275 else:
7276 return True;
7277
7278 if(not py7zr_support):
7279 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
7280 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7281 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7282 return False;
7283
7284 if(py7zr_support):
7285 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
7286 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7287 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7288 return False;
7289 lcfi = 0;
7290 returnval = {};
7291 szpfp = py7zr.SevenZipFile(infile, mode="r");
7292 file_content = szpfp.readall();
7293 #sztest = szpfp.testzip();
7294 sztestalt = szpfp.test();
7295 if(sztestalt):
7296 VerbosePrintOut("Bad file found!");
7297 for member in sorted(szpfp.list(), key=lambda x: x.filename):
7298 if(re.findall("^[.|/]", member.filename)):
7299 fname = member.filename;
7300 else:
7301 fname = "./"+member.filename;
7302 if(not member.is_directory):
7303 fpremode = int(stat.S_IFREG + 438);
7304 elif(member.is_directory):
7305 fpremode = int(stat.S_IFDIR + 511);
7306 fwinattributes = int(0);
7307 if(member.is_directory):
7308 fmode = int(stat.S_IFDIR + 511);
7309 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7310 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7311 else:
7312 fmode = int(stat.S_IFLNK + 438);
7313 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
7314 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
7315 returnval.update({lcfi: member.filename});
7316 if(not verbose):
7317 VerbosePrintOut(member.filename);
7318 if(verbose):
7319 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7320 permissionstr = "";
7321 for fmodval in str(oct(fmode))[-3:]:
7322 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
7323 fsize = int("0");
7324 if(not member.is_directory):
7325 ftype = 0;
7326 permissionstr = "-" + permissionstr;
7327 printfname = member.filename;
7328 elif(member.is_directory):
7329 ftype = 5;
7330 permissionstr = "d" + permissionstr;
7331 printfname = member.filename;
7332 if(ftype==0):
7333 fsize = len(file_content[member.filename].read());
7334 file_content[member.filename].close();
7335 try:
7336 fuid = int(os.getuid());
7337 except AttributeError:
7338 fuid = int(0);
7339 except KeyError:
7340 fuid = int(0);
7341 try:
7342 fgid = int(os.getgid());
7343 except AttributeError:
7344 fgid = int(0);
7345 except KeyError:
7346 fgid = int(0);
7347 try:
7348 import pwd;
7349 try:
7350 userinfo = pwd.getpwuid(os.getuid());
7351 funame = userinfo.pw_name;
7352 except KeyError:
7353 funame = "";
7354 except AttributeError:
7355 funame = "";
7356 except ImportError:
7357 funame = "";
7358 fgname = "";
7359 try:
7360 import grp;
7361 try:
7362 groupinfo = grp.getgrgid(os.getgid());
7363 fgname = groupinfo.gr_name;
7364 except KeyError:
7365 fgname = "";
7366 except AttributeError:
7367 fgname = "";
7368 except ImportError:
7369 fgname = "";
7370 fuprint = funame;
7371 if(len(fuprint)<=0):
7372 fuprint = str(fuid);
7373 fgprint = fgname;
7374 if(len(fgprint)<=0):
7375 fgprint = str(fgid);
7376 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(fsize).rjust(15) + " " + member.creationtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
7377 lcfi = lcfi + 1;
7378 if(returnfp):
7379 return listcatfiles['catfp'];
7380 else:
7381 return True;
7382
7383 def InFileListFiles(infile, verbose=False, formatspecs=__file_format_dict__, returnfp=False):
7384 formatspecs = FormatSpecsListToDict(formatspecs);
7385 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7386 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
7387 if(checkcompressfile=="tarfile"):
7388 return TarFileListFiles(infile, verbose, returnfp);
7389 elif(checkcompressfile=="zipfile"):
7390 return ZipFileListFiles(infile, verbose, returnfp);
7391 elif(checkcompressfile=="catfile"):
7392 return ArchiveFileListFiles(infile, 0, 0, False, formatspecs, verbose, returnfp);
7393 elif(rarfile_support and checkcompressfile=="rarfile"):
7394 return RarFileListFiles(infile, verbose, returnfp);
7395 elif(py7zr_support and checkcompressfile=="7zipfile"):
7396 return SevenZipFileListFiles(infile, verbose, returnfp);
7397 else:
7398 return False;
7399 return False;
7400
7401 def ListDirListFiles(infiles, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7402 formatspecs = FormatSpecsListToDict(formatspecs);
7403 outarray = BytesIO();
7404 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, False, True);
7405 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
7406 return listcatfiles;
7407
7408 def ListDirListFilesAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7409 formatspecs = FormatSpecsListToDict(formatspecs);
7410 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, listonly, checksumtype, formatspecs, verbose);
7411 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
7412 return listcatfiles;
7413
7414 def PackArchiveFileFromListDirAlt(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7415 formatspecs = FormatSpecsListToDict(formatspecs);
7416 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, False, checksumtype, extradata, formatspecs, False);
7417 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, followlink, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
7418 return listcatfiles;
7419
7420 create_alias_function("Pack", __file_format_name__, "FromListDirAlt", PackArchiveFileFromListDirAlt);
7421
7422 def PackArchiveFileFromTarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7423 formatspecs = FormatSpecsListToDict(formatspecs);
7424 outarray = TarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7425 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7426 return listcatfiles;
7427
7428 create_alias_function("Pack", __file_format_name__, "FromTarFileAlt", PackArchiveFileFromTarFileAlt);
7429
7430 def PackArchiveFileFromZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7431 formatspecs = FormatSpecsListToDict(formatspecs);
7432 outarray = ZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7433 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7434 return listcatfiles;
7435
7436 create_alias_function("Pack", __file_format_name__, "FromZipFileAlt", PackArchiveFileFromZipFileAlt);
7437
7438 if(not rarfile_support):
7439 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7440 return False;
7441
7442 if(rarfile_support):
7443 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7444 formatspecs = FormatSpecsListToDict(formatspecs);
7445 outarray = RarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7446 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7447 return listcatfiles;
7448
7449 create_alias_function("Pack", __file_format_name__, "FromRarFileAlt", PackArchiveFileFromRarFileAlt);
7450
7451 if(not py7zr_support):
7452 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7453 return False;
7454
7455 if(py7zr_support):
7456 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7457 formatspecs = FormatSpecsListToDict(formatspecs);
7458 outarray = SevenZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7459 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7460 return listcatfiles;
7461
7462 create_alias_function("Pack", __file_format_name__, "FromSevenZipFileAlt", PackArchiveFileFromSevenZipFileAlt);
7463
7464 def download_file_from_ftp_file(url):
7465 urlparts = urlparse(url);
7466 file_name = os.path.basename(urlparts.path);
7467 file_dir = os.path.dirname(urlparts.path);
7468 if(urlparts.username is not None):
7469 ftp_username = urlparts.username;
7470 else:
7471 ftp_username = "anonymous";
7472 if(urlparts.password is not None):
7473 ftp_password = urlparts.password;
7474 elif(urlparts.password is None and urlparts.username=="anonymous"):
7475 ftp_password = "anonymous";
7476 else:
7477 ftp_password = "";
7478 if(urlparts.scheme=="ftp"):
7479 ftp = FTP();
7480 elif(urlparts.scheme=="ftps" and ftpssl):
7481 ftp = FTP_TLS();
7482 else:
7483 return False;
7484 if(urlparts.scheme=="sftp"):
7485 if(__use_pysftp__):
7486 return download_file_from_pysftp_file(url);
7487 else:
7488 return download_file_from_sftp_file(url);
7489 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7490 return download_file_from_http_file(url);
7491 ftp_port = urlparts.port;
7492 if(urlparts.port is None):
7493 ftp_port = 21;
7494 try:
7495 ftp.connect(urlparts.hostname, ftp_port);
7496 except socket.gaierror:
7497 log.info("Error With URL "+url);
7498 return False;
7499 except socket.timeout:
7500 log.info("Error With URL "+url);
7501 return False;
7502 ftp.login(urlparts.username, urlparts.password);
7503 if(urlparts.scheme=="ftps"):
7504 ftp.prot_p();
7505 ftpfile = BytesIO();
7506 ftp.retrbinary("RETR "+urlparts.path, ftpfile.write);
7507 #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
7508 ftp.close();
7509 ftpfile.seek(0, 0);
7510 return ftpfile;
7511
7512 def download_file_from_ftp_string(url):
7513 ftpfile = download_file_from_ftp_file(url);
7514 return ftpfile.read();
7515
7516 def upload_file_to_ftp_file(ftpfile, url):
7517 urlparts = urlparse(url);
7518 file_name = os.path.basename(urlparts.path);
7519 file_dir = os.path.dirname(urlparts.path);
7520 if(urlparts.username is not None):
7521 ftp_username = urlparts.username;
7522 else:
7523 ftp_username = "anonymous";
7524 if(urlparts.password is not None):
7525 ftp_password = urlparts.password;
7526 elif(urlparts.password is None and urlparts.username=="anonymous"):
7527 ftp_password = "anonymous";
7528 else:
7529 ftp_password = "";
7530 if(urlparts.scheme=="ftp"):
7531 ftp = FTP();
7532 elif(urlparts.scheme=="ftps" and ftpssl):
7533 ftp = FTP_TLS();
7534 else:
7535 return False;
7536 if(urlparts.scheme=="sftp"):
7537 if(__use_pysftp__):
7538 return upload_file_to_pysftp_file(url);
7539 else:
7540 return upload_file_to_sftp_file(url);
7541 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7542 return False;
7543 ftp_port = urlparts.port;
7544 if(urlparts.port is None):
7545 ftp_port = 21;
7546 try:
7547 ftp.connect(urlparts.hostname, ftp_port);
7548 except socket.gaierror:
7549 log.info("Error With URL "+url);
7550 return False;
7551 except socket.timeout:
7552 log.info("Error With URL "+url);
7553 return False;
7554 ftp.login(urlparts.username, urlparts.password);
7555 if(urlparts.scheme=="ftps"):
7556 ftp.prot_p();
7557 ftp.storbinary("STOR "+urlparts.path, ftpfile);
7558 ftp.close();
7559 ftpfile.seek(0, 0);
7560 return ftpfile;
7561
7562 def upload_file_to_ftp_string(ftpstring, url):
7563 ftpfileo = BytesIO(ftpstring);
7564 ftpfile = upload_file_to_ftp_file(ftpfileo, url);
7565 ftpfileo.close();
7566 return ftpfile;
7567
7568 def download_file_from_http_file(url, headers=geturls_headers_pycatfile_python_alt):
7569 # Parse the URL to extract username and password if present
7570 urlparts = urlparse(url);
7571 username = urlparts.username;
7572 password = urlparts.password;
7573 # Rebuild the URL without the username and password
7574 netloc = urlparts.hostname;
7575 if(urlparts.scheme=="sftp"):
7576 if(__use_pysftp__):
7577 return download_file_from_pysftp_file(url);
7578 else:
7579 return download_file_from_sftp_file(url);
7580 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7581 return download_file_from_ftp_file(url);
7582 if urlparts.port:
7583 netloc += ':' + str(urlparts.port);
7584 rebuilt_url = urlunparse((urlparts.scheme, netloc, urlparts.path, urlparts.params, urlparts.query, urlparts.fragment));
7585 # Create a temporary file object
7586 httpfile = BytesIO();
7587 if haverequests:
7588 # Use the requests library if available
7589 if username and password:
7590 response = requests.get(rebuilt_url, headers=headers, auth=(username, password), stream=True);
7591 else:
7592 response = requests.get(rebuilt_url, headers=headers, stream=True);
7593 response.raw.decode_content = True
7594 shutil.copyfileobj(response.raw, httpfile);
7595 else:
7596 # Build a Request object for urllib
7597 request = Request(rebuilt_url, headers=headers);
7598 # Create an opener object for handling URLs
7599 if username and password:
7600 # Create a password manager
7601 password_mgr = HTTPPasswordMgrWithDefaultRealm();
7602 # Add the username and password
7603 password_mgr.add_password(None, rebuilt_url, username, password);
7604 # Create an authentication handler using the password manager
7605 auth_handler = HTTPBasicAuthHandler(password_mgr);
7606 # Build the opener with the authentication handler
7607 opener = build_opener(auth_handler);
7608 else:
7609 opener = build_opener();
7610 with opener.open(request) as response:
7611 shutil.copyfileobj(response, httpfile);
7612 # Reset file pointer to the start
7613 httpfile.seek(0, 0);
7614 # Return the temporary file object
7615 return httpfile;
7616
7617 def download_file_from_http_string(url, headers=geturls_headers_pycatfile_python_alt):
7618 httpfile = download_file_from_http_file(url, headers);
7619 return ftpfile.read();
7620
7621 if(haveparamiko):
7622 def download_file_from_sftp_file(url):
7623 urlparts = urlparse(url);
7624 file_name = os.path.basename(urlparts.path);
7625 file_dir = os.path.dirname(urlparts.path);
7626 sftp_port = urlparts.port;
7627 if(urlparts.port is None):
7628 sftp_port = 22;
7629 else:
7630 sftp_port = urlparts.port;
7631 if(urlparts.username is not None):
7632 sftp_username = urlparts.username;
7633 else:
7634 sftp_username = "anonymous";
7635 if(urlparts.password is not None):
7636 sftp_password = urlparts.password;
7637 elif(urlparts.password is None and urlparts.username=="anonymous"):
7638 sftp_password = "anonymous";
7639 else:
7640 sftp_password = "";
7641 if(urlparts.scheme=="ftp"):
7642 return download_file_from_ftp_file(url);
7643 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7644 return download_file_from_http_file(url);
7645 if(urlparts.scheme!="sftp"):
7646 return False;
7647 ssh = paramiko.SSHClient();
7648 ssh.load_system_host_keys();
7649 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
7650 try:
7651 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7652 except paramiko.ssh_exception.SSHException:
7653 return False;
7654 except socket.gaierror:
7655 log.info("Error With URL "+url);
7656 return False;
7657 except socket.timeout:
7658 log.info("Error With URL "+url);
7659 return False;
7660 sftp = ssh.open_sftp();
7661 sftpfile = BytesIO();
7662 sftp.getfo(urlparts.path, sftpfile);
7663 sftp.close();
7664 ssh.close();
7665 sftpfile.seek(0, 0);
7666 return sftpfile;
7667 else:
7668 def download_file_from_sftp_file(url):
7669 return False;
7670
7671 if(haveparamiko):
7672 def download_file_from_sftp_string(url):
7673 sftpfile = download_file_from_sftp_file(url);
7674 return sftpfile.read();
7675 else:
7676 def download_file_from_ftp_string(url):
7677 return False;
7678
7679 if(haveparamiko):
7680 def upload_file_to_sftp_file(sftpfile, url):
7681 urlparts = urlparse(url);
7682 file_name = os.path.basename(urlparts.path);
7683 file_dir = os.path.dirname(urlparts.path);
7684 sftp_port = urlparts.port;
7685 if(urlparts.port is None):
7686 sftp_port = 22;
7687 else:
7688 sftp_port = urlparts.port;
7689 if(urlparts.username is not None):
7690 sftp_username = urlparts.username;
7691 else:
7692 sftp_username = "anonymous";
7693 if(urlparts.password is not None):
7694 sftp_password = urlparts.password;
7695 elif(urlparts.password is None and urlparts.username=="anonymous"):
7696 sftp_password = "anonymous";
7697 else:
7698 sftp_password = "";
7699 if(urlparts.scheme=="ftp"):
7700 return upload_file_to_ftp_file(url);
7701 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7702 return False;
7703 if(urlparts.scheme!="sftp"):
7704 return False;
7705 ssh = paramiko.SSHClient();
7706 ssh.load_system_host_keys();
7707 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
7708 try:
7709 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7710 except paramiko.ssh_exception.SSHException:
7711 return False;
7712 except socket.gaierror:
7713 log.info("Error With URL "+url);
7714 return False;
7715 except socket.timeout:
7716 log.info("Error With URL "+url);
7717 return False;
7718 sftp = ssh.open_sftp();
7719 sftp.putfo(sftpfile, urlparts.path);
7720 sftp.close();
7721 ssh.close();
7722 sftpfile.seek(0, 0);
7723 return sftpfile;
7724 else:
7725 def upload_file_to_sftp_file(sftpfile, url):
7726 return False;
7727
7728 if(haveparamiko):
7729 def upload_file_to_sftp_string(sftpstring, url):
7730 sftpfileo = BytesIO(sftpstring);
7731 sftpfile = upload_file_to_sftp_files(ftpfileo, url);
7732 sftpfileo.close();
7733 return sftpfile;
7734 else:
7735 def upload_file_to_sftp_string(url):
7736 return False;
7737
7738 if(havepysftp):
7739 def download_file_from_pysftp_file(url):
7740 urlparts = urlparse(url);
7741 file_name = os.path.basename(urlparts.path);
7742 file_dir = os.path.dirname(urlparts.path);
7743 sftp_port = urlparts.port;
7744 if(urlparts.port is None):
7745 sftp_port = 22;
7746 else:
7747 sftp_port = urlparts.port;
7748 if(urlparts.username is not None):
7749 sftp_username = urlparts.username;
7750 else:
7751 sftp_username = "anonymous";
7752 if(urlparts.password is not None):
7753 sftp_password = urlparts.password;
7754 elif(urlparts.password is None and urlparts.username=="anonymous"):
7755 sftp_password = "anonymous";
7756 else:
7757 sftp_password = "";
7758 if(urlparts.scheme=="ftp"):
7759 return download_file_from_ftp_file(url);
7760 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7761 return download_file_from_http_file(url);
7762 if(urlparts.scheme!="sftp"):
7763 return False;
7764 try:
7765 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7766 except paramiko.ssh_exception.SSHException:
7767 return False;
7768 except socket.gaierror:
7769 log.info("Error With URL "+url);
7770 return False;
7771 except socket.timeout:
7772 log.info("Error With URL "+url);
7773 return False;
7774 sftp = ssh.open_sftp();
7775 sftpfile = BytesIO();
7776 sftp.getfo(urlparts.path, sftpfile);
7777 sftp.close();
7778 ssh.close();
7779 sftpfile.seek(0, 0);
7780 return sftpfile;
7781 else:
7782 def download_file_from_pysftp_file(url):
7783 return False;
7784
7785 if(havepysftp):
7786 def download_file_from_pysftp_string(url):
7787 sftpfile = download_file_from_pysftp_file(url);
7788 return sftpfile.read();
7789 else:
7790 def download_file_from_ftp_string(url):
7791 return False;
7792
7793 if(havepysftp):
7794 def upload_file_to_pysftp_file(sftpfile, url):
7795 urlparts = urlparse(url);
7796 file_name = os.path.basename(urlparts.path);
7797 file_dir = os.path.dirname(urlparts.path);
7798 sftp_port = urlparts.port;
7799 if(urlparts.port is None):
7800 sftp_port = 22;
7801 else:
7802 sftp_port = urlparts.port;
7803 if(urlparts.username is not None):
7804 sftp_username = urlparts.username;
7805 else:
7806 sftp_username = "anonymous";
7807 if(urlparts.password is not None):
7808 sftp_password = urlparts.password;
7809 elif(urlparts.password is None and urlparts.username=="anonymous"):
7810 sftp_password = "anonymous";
7811 else:
7812 sftp_password = "";
7813 if(urlparts.scheme=="ftp"):
7814 return upload_file_to_ftp_file(url);
7815 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7816 return False;
7817 if(urlparts.scheme!="sftp"):
7818 return False;
7819 try:
7820 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7821 except paramiko.ssh_exception.SSHException:
7822 return False;
7823 except socket.gaierror:
7824 log.info("Error With URL "+url);
7825 return False;
7826 except socket.timeout:
7827 log.info("Error With URL "+url);
7828 return False;
7829 sftp = ssh.open_sftp();
7830 sftp.putfo(sftpfile, urlparts.path);
7831 sftp.close();
7832 ssh.close();
7833 sftpfile.seek(0, 0);
7834 return sftpfile;
7835 else:
7836 def upload_file_to_pysftp_file(sftpfile, url):
7837 return False;
7838
7839 if(havepysftp):
7840 def upload_file_to_pysftp_string(sftpstring, url):
7841 sftpfileo = BytesIO(sftpstring);
7842 sftpfile = upload_file_to_pysftp_files(ftpfileo, url);
7843 sftpfileo.close();
7844 return sftpfile;
7845 else:
7846 def upload_file_to_pysftp_string(url):
7847 return False;
7848
7849 def download_file_from_internet_file(url, headers=geturls_headers_pycatfile_python_alt):
7850 urlparts = urlparse(url);
7851 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7852 return download_file_from_http_file(url, headers);
7853 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7854 return download_file_from_ftp_file(url);
7855 elif(urlparts.scheme=="sftp"):
7856 if(__use_pysftp__ and havepysftp):
7857 return download_file_from_pysftp_file(url);
7858 else:
7859 return download_file_from_sftp_file(url);
7860 else:
7861 return False;
7862 return False;
7863
7864 def download_file_from_internet_uncompress_file(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_dict__):
7865 formatspecs = FormatSpecsListToDict(formatspecs);
7866 fp = download_file_from_internet_file(url);
7867 fp = UncompressArchiveFile(fp, formatspecs);
7868 fp.seek(0, 0);
7869 if(not fp):
7870 return False;
7871 return fp;
7872
7873 def download_file_from_internet_string(url, headers=geturls_headers_pycatfile_python_alt):
7874 urlparts = urlparse(url);
7875 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7876 return download_file_from_http_string(url, headers);
7877 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7878 return download_file_from_ftp_string(url);
7879 elif(urlparts.scheme=="sftp"):
7880 if(__use_pysftp__ and havepysftp):
7881 return download_file_from_pysftp_string(url);
7882 else:
7883 return download_file_from_sftp_string(url);
7884 else:
7885 return False;
7886 return False;
7887
7888 def download_file_from_internet_uncompress_string(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_dict__):
7889 formatspecs = FormatSpecsListToDict(formatspecs);
7890 fp = download_file_from_internet_string(url);
7891 fp = UncompressArchiveFile(fp, formatspecs);
7892 fp.seek(0, 0);
7893 if(not fp):
7894 return False;
7895 return fp;
7896
7897 def upload_file_to_internet_file(ifp, url):
7898 urlparts = urlparse(url);
7899 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7900 return False;
7901 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7902 return upload_file_to_ftp_file(ifp, url);
7903 elif(urlparts.scheme=="sftp"):
7904 if(__use_pysftp__ and havepysftp):
7905 return upload_file_to_pysftp_file(ifp, url);
7906 else:
7907 return upload_file_to_sftp_file(ifp, url);
7908 else:
7909 return False;
7910 return False;
7911
7912 def upload_file_to_internet_compress_file(ifp, url, formatspecs=__file_format_dict__):
7913 formatspecs = FormatSpecsListToDict(formatspecs);
7914 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
7915 if(not catfileout):
7916 return False;
7917 catfp.seek(0, 0);
7918 upload_file_to_internet_file(catfp, outfile);
7919 return True;
7920
7921 def upload_file_to_internet_string(ifp, url):
7922 urlparts = urlparse(url);
7923 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7924 return False;
7925 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7926 return upload_file_to_ftp_string(ifp, url);
7927 elif(urlparts.scheme=="sftp"):
7928 if(__use_pysftp__ and havepysftp):
7929 return upload_file_to_pysftp_string(ifp, url);
7930 else:
7931 return upload_file_to_sftp_string(ifp, url);
7932 else:
7933 return False;
7934 return False;
7935
7936 def upload_file_to_internet_compress_string(ifp, url, formatspecs=__file_format_dict__):
7937 formatspecs = FormatSpecsListToDict(formatspecs);
7938 catfp = CompressArchiveFile(BytesIO(ifp), compression, compressionlevel, formatspecs);
7939 if(not catfileout):
7940 return False;
7941 catfp.seek(0, 0);
7942 upload_file_to_internet_file(catfp, outfile);
7943 return True;
7944
7945 try:
7946 if(hasattr(shutil, "register_archive_format")):
7947 # Register the packing format
7948 shutil.register_archive_format(__file_format_name__, PackArchiveFileFunc, description='Pack concatenated files');
7949 except shutil.RegistryError:
7950 pass;
7951
7952 try:
7953 if(hasattr(shutil, "register_unpack_format")):
7954 # Register the unpacking format
7955 shutil.register_unpack_format(__file_format_name__, archivefile_extensions, UnPackArchiveFileFunc, description='UnPack concatenated files');
7956 except shutil.RegistryError:
7957 pass;
A tar like file format name catfile after unix cat command (concatenate files).