?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Add files via upload
[PyCatFile.git] / pycatfile.py
blob0434bfde72e24ccfbc076b94bfe0cb19dd4a51c7
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 '''
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the Revised BSD License.
7
8 This program is distributed in the hope that it will be useful,
9 but WITHOUT ANY WARRANTY; without even the implied warranty of
10 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 Revised BSD License for more details.
12
13 Copyright 2018-2024 Cool Dude 2k - http://idb.berlios.de/
14 Copyright 2018-2024 Game Maker 2k - http://intdb.sourceforge.net/
15 Copyright 2018-2024 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
16
17 $FileInfo: pycatfile.py - Last Update: 5/15/2024 Ver. 0.11.4 RC 1 - Author: cooldude2k $
18 '''
19
20 from __future__ import absolute_import, division, print_function, unicode_literals;
21 import io, os, re, sys, time, stat, zlib, base64, shutil, socket, hashlib, datetime, logging, binascii, zipfile, platform;
22 try:
23 from backports import tempfile;
24 except ImportError:
25 import tempfile;
26 ftpssl = True;
27 try:
28 ftpssl = True;
29 from ftplib import FTP, FTP_TLS;
30 except ImportError:
31 ftpssl = False;
32 from ftplib import FTP;
33
34 try:
35 # Try Python 3 imports
36 from urllib.parse import urlparse, urlunparse;
37 except ImportError:
38 # Fall back to Python 2 imports
39 from urlparse import urlparse, urlunparse;
40
41 if os.name == 'nt': # Only modify if on Windows
42 if sys.version[0] == "2":
43 import codecs;
44 sys.stdout = codecs.getwriter('utf-8')(sys.stdout);
45 sys.stderr = codecs.getwriter('utf-8')(sys.stderr);
46 else:
47 import io;
48 sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace', line_buffering=True);
49 sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', errors='replace', line_buffering=True);
50
51 hashlib_guaranteed = False;
52 os.environ["PYTHONIOENCODING"] = "UTF-8";
53 os.environ["LC_CTYPE"] = "UTF-8";
54 try:
55 reload(sys);
56 try:
57 sys.setdefaultencoding('UTF-8');
58 except NameError:
59 pass;
60 except AttributeError:
61 pass;
62 except NameError:
63 pass;
64 except AttributeError:
65 pass;
66
67 try:
68 from zlib import crc32;
69 except ImportError:
70 from binascii import crc32;
71
72 if(sys.version_info[0]==2):
73 FileNotFoundError = IOError;
74
75 rarfile_support = False;
76 try:
77 import rarfile;
78 rarfile_support = True;
79 except ImportError:
80 rarfile_support = False;
81
82 py7zr_support = False;
83 try:
84 import py7zr;
85 py7zr_support = True;
86 except ImportError:
87 py7zr_support = False;
88
89 try:
90 from xtarfile import is_tarfile;
91 except ImportError:
92 try:
93 from safetar import is_tarfile;
94 except ImportError:
95 from tarfile import is_tarfile;
96
97 try:
98 import xtarfile as tarfile;
99 except ImportError:
100 try:
101 import safetar as tarfile;
102 except ImportError:
103 import tarfile;
104
105 haveparamiko = False;
106 try:
107 import paramiko;
108 haveparamiko = True;
109 except ImportError:
110 haveparamiko = False;
111
112 havepysftp = False;
113 try:
114 import pysftp;
115 havepysftp = True;
116 except ImportError:
117 havepysftp = False;
118
119 haverequests = False;
120 try:
121 import requests;
122 haverequests = True;
123 except ImportError:
124 haverequests = False;
125
126 if(haverequests):
127 import urllib3;
128 logging.getLogger("urllib3").setLevel(logging.WARNING);
129
130 try:
131 # Python 3 imports
132 from urllib.request import Request, build_opener, HTTPBasicAuthHandler;
133 from urllib.parse import urlparse;
134 except ImportError:
135 # Python 2 imports
136 from urllib2 import Request, build_opener, HTTPBasicAuthHandler;
137 from urlparse import urlparse;
138
139 try:
140 from io import StringIO, BytesIO;
141 except ImportError:
142 try:
143 from cStringIO import StringIO;
144 from cStringIO import StringIO as BytesIO;
145 except ImportError:
146 from StringIO import StringIO;
147 from StringIO import StringIO as BytesIO;
148
149 __use_pysftp__ = False;
150 if(not havepysftp):
151 __use_pysftp__ = False;
152 __file_format_name__ = "CatFile";
153 __program_name__ = "Py"+__file_format_name__;
154 __file_format_lower__ = __file_format_name__.lower();
155 __file_format_magic__ = __file_format_name__;
156 __file_format_len__ = len(__file_format_magic__);
157 __file_format_hex__ = binascii.hexlify(__file_format_magic__.encode("UTF-8")).decode("UTF-8");
158 __file_format_delimiter__ = "\x00";
159 __file_format_ver__ = "001";
160 __use_new_style__ = True;
161 __use_advanced_list__ = True;
162 __use_alt_inode__ = False;
163 __file_format_extension__ = ".cat";
164 __file_format_list__ = [__file_format_name__, __file_format_magic__, __file_format_lower__, __file_format_len__, __file_format_hex__, __file_format_delimiter__, __file_format_ver__, __use_new_style__, __use_advanced_list__, __use_alt_inode__];
165 __file_format_dict__ = {'format_name': __file_format_name__, 'format_magic': __file_format_magic__, 'format_lower': __file_format_lower__, 'format_len': __file_format_len__, 'format_hex': __file_format_hex__, 'format_delimiter': __file_format_delimiter__, 'format_ver': __file_format_ver__, 'new_style': __use_new_style__, 'use_advanced_list': __use_advanced_list__, 'use_alt_inode': __use_alt_inode__};
166 __project__ = __program_name__;
167 __project_url__ = "https://github.com/GameMaker2k/PyCatFile";
168 __version_info__ = (0, 11, 4, "RC 1", 1);
169 __version_date_info__ = (2024, 5, 17, "RC 1", 1);
170 __version_date__ = str(__version_date_info__[0]) + "." + str(__version_date_info__[1]).zfill(2) + "." + str(__version_date_info__[2]).zfill(2);
171 __revision__ = __version_info__[3];
172 __revision_id__ = "$Id$";
173 if(__version_info__[4] is not None):
174 __version_date_plusrc__ = __version_date__ + "-" + str(__version_date_info__[4]);
175 if(__version_info__[4] is None):
176 __version_date_plusrc__ = __version_date__;
177 if(__version_info__[3] is not None):
178 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]) + " " + str(__version_info__[3]);
179 if(__version_info__[3] is None):
180 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]);
181
182 PyBitness = platform.architecture();
183 if(PyBitness=="32bit" or PyBitness=="32"):
184 PyBitness = "32";
185 elif(PyBitness=="64bit" or PyBitness=="64"):
186 PyBitness = "64";
187 else:
188 PyBitness = "32";
189
190 geturls_ua_pycatfile_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
191 if(platform.python_implementation()!=""):
192 py_implementation = platform.python_implementation();
193 if(platform.python_implementation()==""):
194 py_implementation = "Python";
195 geturls_ua_pycatfile_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=py_implementation, pyver=platform.python_version(), proname=__project__, prover=__version__);
196 geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
197 geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
198 geturls_headers_pycatfile_python = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
199 geturls_headers_pycatfile_python_alt = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python_alt, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
200 geturls_headers_googlebot_google = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
201 geturls_headers_googlebot_google_old = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
202
203 def CompressionSupport():
204 compression_list = [];
205 try:
206 import gzip;
207 compression_list.append("gz");
208 compression_list.append("gzip");
209 except ImportError:
210 '''return False;'''
211 try:
212 import bz2;
213 compression_list.append("bz2");
214 compression_list.append("bzip2");
215 except ImportError:
216 '''return False;'''
217 try:
218 import lz4;
219 compression_list.append("lz4");
220 except ImportError:
221 '''return False;'''
222 try:
223 import lzo;
224 compression_list.append("lzo");
225 compression_list.append("lzop");
226 except ImportError:
227 '''return False;'''
228 try:
229 import zstandard;
230 compression_list.append("zstd");
231 compression_list.append("zstandard");
232 except ImportError:
233 '''return False;'''
234 try:
235 import lzma;
236 compression_list.append("lzma");
237 compression_list.append("xz");
238 except ImportError:
239 '''return False;'''
240 try:
241 from backports import lzma;
242 compression_list.append("lzma");
243 compression_list.append("xz");
244 except ImportError:
245 '''return False;'''
246 return compression_list;
247
248 compressionsupport = CompressionSupport();
249 compressionlist = ['auto'];
250 compressionlistalt = [];
251 outextlist = [];
252 outextlistwd = [];
253 if('gzip' in compressionsupport):
254 compressionlist.append('gzip');
255 compressionlistalt.append('gzip');
256 outextlist.append('gz');
257 outextlistwd.append('.gz');
258 if('bzip2' in compressionsupport):
259 compressionlist.append('bzip2');
260 compressionlistalt.append('bzip2');
261 outextlist.append('bz2');
262 outextlistwd.append('.bz2');
263 if('zstd' in compressionsupport):
264 compressionlist.append('zstd');
265 compressionlistalt.append('zstd');
266 outextlist.append('zst');
267 outextlistwd.append('.zst');
268 if('lz4' in compressionsupport):
269 compressionlist.append('lz4');
270 compressionlistalt.append('lz4');
271 outextlist.append('lz4');
272 outextlistwd.append('.lz4');
273 if('lzo' in compressionsupport):
274 compressionlist.append('lzo');
275 compressionlistalt.append('lzo');
276 outextlist.append('lzo');
277 outextlistwd.append('.lzo');
278 if('lzop' in compressionsupport):
279 compressionlist.append('lzop');
280 compressionlistalt.append('lzop');
281 outextlist.append('lzop');
282 outextlistwd.append('.lzop');
283 if('lzma' in compressionsupport):
284 compressionlist.append('lzma');
285 compressionlistalt.append('lzma');
286 outextlist.append('lzma');
287 outextlistwd.append('.lzma');
288 if('xz' in compressionsupport):
289 compressionlist.append('xz');
290 compressionlistalt.append('xz');
291 outextlist.append('xz');
292 outextlistwd.append('.xz');
293
294 tarfile_mimetype = "application/tar";
295 tarfile_tar_mimetype = tarfile_mimetype;
296 zipfile_mimetype = "application/zip";
297 zipfile_zip_mimetype = zipfile_mimetype;
298 rarfile_mimetype = "application/rar";
299 rarfile_rar_mimetype = rarfile_mimetype;
300 archivefile_mimetype = "application/x-"+__file_format_dict__['format_lower']+"";
301 archivefile_cat_mimetype = archivefile_mimetype;
302 archivefile_gzip_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+gzip";
303 archivefile_gz_mimetype = archivefile_gzip_mimetype;
304 archivefile_bzip2_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+bzip2";
305 archivefile_bz2_mimetype = archivefile_bzip2_mimetype;
306 archivefile_lz4_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lz4";
307 archivefile_lzop_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lzop";
308 archivefile_lzo_mimetype = archivefile_lzop_mimetype;
309 archivefile_zstandard_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+zstandard";
310 archivefile_zstd_mimetype = archivefile_zstandard_mimetype;
311 archivefile_lzma_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lzma";
312 archivefile_xz_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+xz";
313 archivefile_extensions = [__file_format_extension__, __file_format_extension__+".gz", __file_format_extension__+".bz2", __file_format_extension__+".zst", __file_format_extension__+".lz4", __file_format_extension__+".lzo", __file_format_extension__+".lzop", __file_format_extension__+".lzma", __file_format_extension__+".xz"];
314
315 if __name__ == "__main__":
316 import subprocess;
317 curscrpath = os.path.dirname(sys.argv[0]);
318 if(curscrpath==""):
319 curscrpath = ".";
320 if(os.sep=="\\"):
321 curscrpath = curscrpath.replace(os.sep, "/");
322 curscrpath = curscrpath + "/";
323 scrfile = curscrpath + "catfile.py";
324 if(os.path.exists(scrfile) and os.path.isfile(scrfile)):
325 scrcmd = subprocess.Popen([sys.executable, scrfile] + sys.argv[1:]);
326 scrcmd.wait();
327
328 def VerbosePrintOut(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
329 if(not dbgenable):
330 return True;
331 log_functions = {
332 "print": print,
333 "log": logging.info,
334 "warning": logging.warning,
335 "error": logging.error,
336 "critical": logging.critical,
337 "exception": logging.exception,
338 "logalt": lambda x: logging.log(dgblevel, x),
339 "debug": logging.debug
340 };
341 log_function = log_functions.get(outtype);
342 if(log_function):
343 log_function(dbgtxt);
344 return True;
345 return False;
346
347 def VerbosePrintOutReturn(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
348 VerbosePrintOut(dbgtxt, outtype, dbgenable, dgblevel);
349 return dbgtxt;
350
351 def RemoveWindowsPath(dpath):
352 if(dpath is None):
353 dpath = "";
354 if(os.sep!="/"):
355 dpath = dpath.replace(os.path.sep, "/");
356 dpath = dpath.rstrip("/");
357 if(dpath=="." or dpath==".."):
358 dpath = dpath + "/";
359 return dpath;
360
361 def NormalizeRelativePath(inpath):
362 inpath = RemoveWindowsPath(inpath);
363 if(os.path.isabs(inpath)):
364 outpath = inpath;
365 else:
366 if(inpath.startswith("./") or inpath.startswith("../")):
367 outpath = inpath;
368 else:
369 outpath = "./" + inpath;
370 return outpath;
371
372 def PrependPath(base_dir, child_path):
373 # Check if base_dir is None or empty, if so, return child_path as is
374 if not base_dir:
375 return child_path;
376 # Ensure base_dir ends with exactly one slash
377 if not base_dir.endswith('/'):
378 base_dir += '/';
379 # Check if child_path starts with ./ or ../ (indicating a relative path)
380 if child_path.startswith('./') or child_path.startswith('../'):
381 # For relative paths, we don't alter the child_path
382 return base_dir + child_path;
383 else:
384 # For non-relative paths, ensure there's no starting slash on child_path to avoid double slashes
385 return base_dir + child_path.lstrip('/');
386
387 def ListDir(dirpath, followlink=False, duplicates=False):
388 if(isinstance(dirpath, (list, tuple, ))):
389 dirpath = list(filter(None, dirpath));
390 elif(isinstance(dirpath, (str, ))):
391 dirpath = list(filter(None, [dirpath]));
392 retlist = [];
393 for mydirfile in dirpath:
394 if(not os.path.exists(mydirfile)):
395 return False;
396 mydirfile = NormalizeRelativePath(mydirfile);
397 if(os.path.exists(mydirfile) and os.path.islink(mydirfile)):
398 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile));
399 if(os.path.exists(mydirfile) and os.path.isdir(mydirfile)):
400 for root, dirs, filenames in os.walk(mydirfile):
401 dpath = root;
402 dpath = RemoveWindowsPath(dpath);
403 if(dpath not in retlist and not duplicates):
404 retlist.append(dpath);
405 if(duplicates):
406 retlist.append(dpath);
407 for file in filenames:
408 fpath = os.path.join(root, file);
409 fpath = RemoveWindowsPath(fpath);
410 if(fpath not in retlist and not duplicates):
411 retlist.append(fpath);
412 if(duplicates):
413 retlist.append(fpath);
414 else:
415 retlist.append(RemoveWindowsPath(mydirfile));
416 return retlist;
417
418 def ListDirAdvanced(dirpath, followlink=False, duplicates=False):
419 if isinstance(dirpath, (list, tuple)):
420 dirpath = list(filter(None, dirpath));
421 elif isinstance(dirpath, str):
422 dirpath = list(filter(None, [dirpath]));
423 retlist = []
424 for mydirfile in dirpath:
425 if not os.path.exists(mydirfile):
426 return False;
427 mydirfile = NormalizeRelativePath(mydirfile);
428 if os.path.exists(mydirfile) and os.path.islink(mydirfile) and followlink:
429 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile))
430 if os.path.exists(mydirfile) and os.path.isdir(mydirfile):
431 for root, dirs, filenames in os.walk(mydirfile):
432 # Sort dirs and filenames alphabetically in place
433 dirs.sort(key=lambda x: x.lower());
434 filenames.sort(key=lambda x: x.lower());
435 dpath = RemoveWindowsPath(root);
436 if not duplicates and dpath not in retlist:
437 retlist.append(dpath);
438 elif duplicates:
439 retlist.append(dpath);
440 for file in filenames:
441 fpath = os.path.join(root, file);
442 fpath = RemoveWindowsPath(fpath);
443 if not duplicates and fpath not in retlist:
444 retlist.append(fpath);
445 elif duplicates:
446 retlist.append(fpath);
447 else:
448 retlist.append(RemoveWindowsPath(mydirfile));
449 return retlist;
450
451 def create_alias_function(prefix, base_name, suffix, target_function):
452 # Define a new function that wraps the target function
453 def alias_function(*args, **kwargs):
454 return target_function(*args, **kwargs);
455
456 # Create the function name by combining the prefix, base name, and the suffix
457 function_name = "{}{}{}".format(prefix, base_name, suffix);
458
459 # Add the new function to the global namespace
460 globals()[function_name] = alias_function;
461
462 def create_alias_function_alt(prefix, base_name, suffix, target_function):
463 # Create the function name by combining the prefix, base name, and the suffix
464 # Use the format method for string formatting, compatible with Python 2 and 3
465 function_name = "{}{}{}".format(prefix, base_name, suffix)
466 # Add the new function (alias of the target_function) to the global namespace
467 # This line is compatible as-is with both Python 2 and 3
468 globals()[function_name] = target_function
469
470 def FormatSpecsListToDict(formatspecs=__file_format_list__):
471 if(isinstance(formatspecs, (list, tuple, ))):
472 return {'format_name': formatspecs[0], 'format_magic': formatspecs[1], 'format_lower': formatspecs[2], 'format_len': formatspecs[3], 'format_hex': formatspecs[4], 'format_delimiter': formatspecs[5], 'format_ver': formatspecs[6], 'new_style': formatspecs[7], 'use_advanced_list': formatspecs[8], 'use_alt_inode': formatspecs[9]};
473 elif(isinstance(formatspecs, (dict, ))):
474 return formatspecs;
475 else:
476 return __file_format_dict__;
477 return __file_format_dict__;
478
479 def TarFileCheck(infile):
480 try:
481 if is_tarfile(infile):
482 return True;
483 else:
484 return False;
485 except TypeError:
486 try:
487 # Check if the input is a file object
488 if hasattr(infile, 'read'):
489 # Save the current file position
490 current_position = infile.tell();
491 # Attempt to open the file object as a tar file
492 tar = tarfile.open(fileobj=infile);
493 tar.close();
494 # Restore the file position
495 infile.seek(current_position);
496 return True;
497 else:
498 # Assume it's a filename
499 tar = tarfile.open(name=infile);
500 tar.close();
501 return True;
502 except tarfile.TarError:
503 return False;
504
505 # initial_value can be 0xFFFF or 0x0000
506 def crc16_ansi(msg, initial_value=0xFFFF):
507 # CRC-16-IBM / CRC-16-ANSI polynomial and initial value
508 poly = 0x8005; # Polynomial for CRC-16-IBM / CRC-16-ANSI
509 crc = initial_value; # Initial value
510 for b in msg:
511 crc ^= b << 8; # XOR byte into CRC top byte
512 for _ in range(8): # Process each bit
513 if crc & 0x8000: # If the top bit is set
514 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
515 else:
516 crc = crc << 1; # Just shift left
517 crc &= 0xFFFF; # Ensure CRC remains 16-bit
518 return crc;
519
520 # initial_value can be 0xFFFF or 0x0000
521 def crc16_ibm(msg, initial_value=0xFFFF):
522 return crc16_ansi(msg, initial_value);
523
524 # initial_value is 0xFFFF
525 def crc16(msg):
526 return crc16_ansi(msg, 0xFFFF);
527
528 # initial_value can be 0xFFFF, 0x1D0F or 0x0000
529 def crc16_ccitt(msg, initial_value=0xFFFF):
530 # CRC-16-CCITT polynomial
531 poly = 0x1021; # Polynomial for CRC-16-CCITT
532 # Use the specified initial value
533 crc = initial_value;
534 for b in msg:
535 crc ^= b << 8; # XOR byte into CRC top byte
536 for _ in range(8): # Process each bit
537 if crc & 0x8000: # If the top bit is set
538 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
539 else:
540 crc = crc << 1; # Just shift left
541 crc &= 0xFFFF; # Ensure CRC remains 16-bit
542 return crc;
543
544 # initial_value can be 0x42F0E1EBA9EA3693 or 0x0000000000000000
545 def crc64_ecma(msg, initial_value=0x0000000000000000):
546 # CRC-64-ECMA polynomial and initial value
547 poly = 0x42F0E1EBA9EA3693;
548 crc = initial_value; # Initial value for CRC-64-ECMA
549 for b in msg:
550 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
551 for _ in range(8): # Process each bit
552 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
553 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
554 else:
555 crc <<= 1; # Just shift left if the MSB is 0
556 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
557 return crc;
558
559 # initial_value can be 0x000000000000001B or 0xFFFFFFFFFFFFFFFF
560 def crc64_iso(msg, initial_value=0xFFFFFFFFFFFFFFFF):
561 # CRC-64-ISO polynomial and initial value
562 poly = 0x000000000000001B;
563 crc = initial_value; # Common initial value for CRC-64-ISO
564 for b in msg:
565 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
566 for _ in range(8): # Process each bit
567 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
568 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
569 else:
570 crc <<= 1; # Just shift left if the MSB is 0
571 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
572 return crc;
573
574 def GetDataFromArray(data, path, default=None):
575 element = data;
576 try:
577 for key in path:
578 element = element[key];
579 return element;
580 except (KeyError, TypeError, IndexError):
581 return default;
582
583 def GetDataFromArrayAlt(structure, path, default=None):
584 element = structure;
585 for key in path:
586 if isinstance(element, dict) and key in element:
587 element = element[key];
588 elif isinstance(element, list) and isinstance(key, int) and -len(element) <= key < len(element):
589 element = element[key];
590 else:
591 return default;
592 return element;
593
594 def GetHeaderChecksum(inlist=[], checksumtype="crc32", encodedata=True, formatspecs=__file_format_dict__):
595 formatspecs = FormatSpecsListToDict(formatspecs);
596 fileheader = AppendNullBytes(inlist, formatspecs['format_delimiter']) if isinstance(inlist, list) else AppendNullByte(inlist, formatspecs['format_delimiter']);
597 if encodedata:
598 fileheader = fileheader.encode('UTF-8');
599 checksum_methods = {
600 "crc16": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
601 "crc16_ansi": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
602 "crc16_ibm": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
603 "crc16_ccitt": lambda data: format(crc16_ccitt(data) & 0xffff, '04x').lower(),
604 "adler32": lambda data: format(zlib.adler32(data) & 0xffffffff, '08x').lower(),
605 "crc32": lambda data: format(crc32(data) & 0xffffffff, '08x').lower(),
606 "crc64_ecma": lambda data: format(crc64_ecma(data) & 0xffffffffffffffff, '016x').lower(),
607 "crc64": lambda data: format(crc64_iso(data) & 0xffffffffffffffff, '016x').lower(),
608 "crc64_iso": lambda data: format(crc64_iso(data) & 0xffffffffffffffff, '016x').lower(),
609 };
610 if checksumtype in checksum_methods:
611 return checksum_methods[checksumtype](fileheader);
612 elif CheckSumSupportAlt(checksumtype, hashlib_guaranteed):
613 checksumoutstr = hashlib.new(checksumtype);
614 checksumoutstr.update(fileheader);
615 return checksumoutstr.hexdigest().lower();
616 return format(0, 'x').lower();
617
618 def GetFileChecksum(instr, checksumtype="crc32", encodedata=True, formatspecs=__file_format_dict__):
619 formatspecs = FormatSpecsListToDict(formatspecs);
620 if encodedata:
621 instr = instr.encode('UTF-8');
622 checksum_methods = {
623 "crc16": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
624 "crc16_ansi": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
625 "crc16_ibm": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
626 "crc16_ccitt": lambda data: format(crc16_ccitt(data) & 0xffff, '04x').lower(),
627 "adler32": lambda data: format(zlib.adler32(data) & 0xffffffff, '08x').lower(),
628 "crc32": lambda data: format(crc32(data) & 0xffffffff, '08x').lower(),
629 "crc64_ecma": lambda data: format(crc64_ecma(data) & 0xffffffffffffffff, '016x').lower(),
630 "crc64": lambda data: format(crc64_iso(data) & 0xffffffffffffffff, '016x').lower(),
631 "crc64_iso": lambda data: format(crc64_iso(data) & 0xffffffffffffffff, '016x').lower(),
632 };
633 if checksumtype in checksum_methods:
634 return checksum_methods[checksumtype](instr);
635 elif CheckSumSupportAlt(checksumtype, hashlib_guaranteed):
636 checksumoutstr = hashlib.new(checksumtype);
637 checksumoutstr.update(instr);
638 return checksumoutstr.hexdigest().lower();
639 return format(0, 'x').lower();
640
641 def ValidateHeaderChecksum(inlist=[], checksumtype="crc32", inchecksum="0", formatspecs=__file_format_dict__):
642 formatspecs = FormatSpecsListToDict(formatspecs);
643 catfileheadercshex = GetHeaderChecksum(inlist, checksumtype, True, formatspecs).lower();
644 return inchecksum.lower() == catfileheadercshex;
645
646 def ValidateFileChecksum(infile, checksumtype="crc32", inchecksum="0", formatspecs=__file_format_dict__):
647 formatspecs = FormatSpecsListToDict(formatspecs);
648 catinfilecshex = GetFileChecksum(infile, checksumtype, True, formatspecs).lower();
649 return inchecksum.lower() == catinfilecshex;
650
651 def ReadTillNullByteOld(fp, delimiter=__file_format_dict__['format_delimiter']):
652 curbyte = b"";
653 curfullbyte = b"";
654 nullbyte = delimiter.encode("UTF-8");
655 while(True):
656 curbyte = fp.read(1);
657 if(curbyte==nullbyte or not curbyte):
658 break;
659 curfullbyte = curfullbyte + curbyte;
660 return curfullbyte.decode('UTF-8');
661
662 def ReadUntilNullByteOld(fp, delimiter=__file_format_dict__['format_delimiter']):
663 return ReadTillNullByteOld(fp, delimiter);
664
665 def ReadTillNullByteAlt(fp, delimiter=__file_format_dict__['format_delimiter'], chunk_size=1024, max_read=1024000):
666 delimiter = delimiter.encode('UTF-8') # Ensure the delimiter is in bytes
667 buffer = bytearray();
668 total_read = 0;
669 delimiter_length = len(delimiter);
670 while True:
671 chunk = fp.read(chunk_size)
672 if not chunk:
673 # End of file reached without finding the delimiter
674 break;
675 buffer.extend(chunk);
676 total_read += len(chunk);
677 if delimiter in buffer:
678 # Delimiter found, calculate where to reset the file pointer
679 index = buffer.find(delimiter);
680 # Calculate how many extra bytes were read after the delimiter
681 extra_bytes_read = len(buffer) - (index + delimiter_length);
682 # Move the file pointer back to just after the delimiter
683 fp.seek(-extra_bytes_read, 1);
684 buffer = buffer[:index];
685 break;
686 if total_read >= max_read:
687 # Stop reading if max limit is reached to prevent excessive memory usage
688 raise MemoryError("Maximum read limit reached without finding the delimiter.");
689 # Check for incomplete UTF-8 sequences at the end of the buffer
690 if len(buffer) > 1 and 128 <= buffer[-1] < 192:
691 # This suggests that the last byte might be the start of a multi-byte character
692 # Try to read one more byte to complete the character
693 extra_byte = fp.read(1);
694 if extra_byte:
695 buffer.extend(extra_byte);
696 else:
697 # No more data available
698 break;
699 try:
700 return buffer.decode('UTF-8', errors='replace');
701 except UnicodeDecodeError:
702 return buffer.decode('UTF-8', errors='replace');
703
704 def ReadUntilNullByteAlt(fp, delimiter=__file_format_dict__['format_delimiter'], chunk_size=1024, max_read=1024000):
705 return ReadTillNullByteAlt(fp, delimiter, chunk_size, max_read);
706
707 def ReadTillNullByte(fp, delimiter=__file_format_dict__['format_delimiter'], max_read=1024000):
708 curfullbyte = bytearray();
709 nullbyte = delimiter.encode("UTF-8");
710 total_read = 0; # Track the total number of bytes read
711 while True:
712 curbyte = fp.read(1);
713 if curbyte == nullbyte or not curbyte:
714 break;
715 curfullbyte.extend(curbyte);
716 total_read += 1;
717 if total_read >= max_read:
718 raise MemoryError("Maximum read limit reached without finding the delimiter.");
719 # Decode the full byte array to string once out of the loop
720 try:
721 return curfullbyte.decode('UTF-8');
722 except UnicodeDecodeError:
723 # Handle potential partial UTF-8 characters
724 for i in range(1, 4):
725 try:
726 return curfullbyte[:-i].decode('UTF-8');
727 except UnicodeDecodeError:
728 continue;
729 raise; # Re-raise if decoding fails even after trimming
730
731 def ReadUntilNullByte(fp, delimiter=__file_format_dict__['format_delimiter'], max_read=1024000):
732 return ReadTillNullByte(fp, delimiter, max_read);
733
734 def ReadTillNullByteByNum(fp, delimiter=__file_format_dict__['format_delimiter'], num_delimiters=1, chunk_size=1024, max_read=1024000):
735 delimiter = delimiter.encode('UTF-8'); # Ensure the delimiter is in bytes
736 buffer = bytearray();
737 total_read = 0;
738 delimiter_length = len(delimiter);
739 results = [];
740 while len(results) < num_delimiters:
741 chunk = fp.read(chunk_size);
742 if not chunk:
743 # End of file reached; decode whatever is collected if it's the last needed part
744 if len(buffer) > 0:
745 results.append(buffer.decode('UTF-8', errors='replace'));
746 break;
747 buffer.extend(chunk)
748 total_read += len(chunk);
749 # Check if we have found the delimiter
750 while delimiter in buffer:
751 index = buffer.find(delimiter);
752 # Decode the section before the delimiter
753 results.append(buffer[:index].decode('UTF-8', errors='replace'));
754 # Remove the processed part from the buffer
755 buffer = buffer[index + delimiter_length:];
756 if len(results) == num_delimiters:
757 # If reached the required number of delimiters, adjust the file pointer and stop
758 fp.seek(-len(buffer), 1);
759 return results;
760 if total_read >= max_read:
761 # Stop reading if max limit is reached to prevent excessive memory usage
762 raise MemoryError("Maximum read limit reached without finding the delimiter.");
763 # Check for incomplete UTF-8 sequences at the end of the buffer
764 if len(buffer) > 1 and 128 <= buffer[-1] < 192:
765 # This suggests that the last byte might be the start of a multi-byte character
766 # Try to read one more byte to complete the character
767 extra_byte = fp.read(1);
768 if extra_byte:
769 buffer.extend(extra_byte);
770 else:
771 # No more data available
772 break;
773 # Process remaining buffer if less than the required number of delimiters were found
774 if len(buffer) > 0 and len(results) < num_delimiters:
775 results.append(buffer.decode('UTF-8', errors='replace'));
776 return results;
777
778 def ReadUntilNullByteByNum(fp, delimiter=__file_format_dict__['format_delimiter'], num_delimiters=1, chunk_size=1024, max_read=1024000):
779 return ReadTillNullByteByNum(fp, delimiter, num_delimiters, chunk_size, max_read);
780
781 def SeekToEndOfFile(fp):
782 lasttell = 0;
783 while(True):
784 fp.seek(1, 1);
785 if(lasttell==fp.tell()):
786 break;
787 lasttell = fp.tell();
788 return True;
789
790 def ReadFileHeaderData(fp, rounds=0, delimiter=__file_format_dict__['format_delimiter']):
791 rocount = 0;
792 roend = int(rounds);
793 HeaderOut = [];
794 while(rocount<roend):
795 HeaderOut.append(ReadTillNullByte(fp, delimiter));
796 rocount = rocount + 1;
797 return HeaderOut;
798
799 def ReadFileHeaderDataBySize(fp, delimiter=__file_format_dict__['format_delimiter']):
800 headerpresize = ReadTillNullByte(fp, delimiter);
801 headersize = int(headerpresize, 16);
802 if(headersize<=0):
803 return [];
804 headercontent = str(fp.read(headersize).decode('UTF-8')).split(delimiter);
805 fp.seek(1, 1);
806 rocount = 0;
807 roend = int(len(headercontent));
808 HeaderOut = [headerpresize];
809 while(rocount<roend):
810 HeaderOut.append(headercontent[rocount]);
811 rocount = rocount + 1;
812 return HeaderOut;
813
814 def ReadFileHeaderDataWoSize(fp, delimiter=__file_format_dict__['format_delimiter']):
815 preheaderdata = ReadFileHeaderData(fp, 2, delimiter);
816 headersize = int(preheaderdata[0], 16);
817 headernumfields = int(preheaderdata[1], 16);
818 if(headersize<=0 or headernumfields<=0):
819 return [];
820 headerdata = ReadTillNullByteByNum(fp, delimiter, headernumfields);
821 #headerdata = ReadFileHeaderData(fp, headernumfields, delimiter);
822 HeaderOut = preheaderdata + headerdata;
823 return HeaderOut;
824
825 def ReadFileHeaderDataBySizeWithContent(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
826 formatspecs = FormatSpecsListToDict(formatspecs);
827 delimiter = formatspecs['format_delimiter'];
828 fheaderstart = fp.tell();
829 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
830 if(len(HeaderOut)==0):
831 return False;
832 if(re.findall("^[.|/]", HeaderOut[3])):
833 fname = HeaderOut[3];
834 else:
835 fname = "./"+HeaderOut[3];
836 fchecksumtype = HeaderOut[-3].lower();
837 fcs = HeaderOut[-2].lower();
838 fccs = HeaderOut[-1].lower();
839 fsize = int(HeaderOut[5], 16);
840 fcompression = HeaderOut[12];
841 fcsize = int(HeaderOut[13], 16);
842 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
843 if(fcs!=newfcs and not skipchecksum):
844 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
845 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
846 return False;
847 fhend = fp.tell() - 1;
848 fcontentstart = fp.tell();
849 fcontents = BytesIO();
850 if(fsize>0 and not listonly):
851 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
852 fcontents.write(fp.read(fsize));
853 else:
854 fcontents.write(fp.read(fcsize));
855 elif(fsize>0 and listonly):
856 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
857 fp.seek(fsize, 1);
858 else:
859 fp.seek(fcsize, 1);
860 fcontents.seek(0, 0);
861 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
862 if(fccs!=newfccs and not skipchecksum and not listonly):
863 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
864 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
865 return False;
866 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
867 pass;
868 else:
869 fcontents.seek(0, 0);
870 if(uncompress):
871 fcontents = UncompressArchiveFile(fcontents, formatspecs);
872 fp.seek(1, 1);
873 fcontentend = fp.tell() - 1;
874 HeaderOut.append(fcontents);
875 return HeaderOut;
876
877 def ReadFileHeaderDataBySizeWithContentToArray(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
878 formatspecs = FormatSpecsListToDict(formatspecs);
879 delimiter = formatspecs['format_delimiter'];
880 fheaderstart = fp.tell();
881 if(formatspecs['new_style']):
882 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
883 else:
884 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
885 if(len(HeaderOut)==0):
886 return False;
887 fheadsize = int(HeaderOut[0], 16);
888 fnumfields = int(HeaderOut[1], 16);
889 ftype = int(HeaderOut[2], 16);
890 if(re.findall("^[.|/]", HeaderOut[3])):
891 fname = HeaderOut[3];
892 else:
893 fname = "./"+HeaderOut[3];
894 fbasedir = os.path.dirname(fname);
895 flinkname = HeaderOut[4];
896 fsize = int(HeaderOut[5], 16);
897 fatime = int(HeaderOut[6], 16);
898 fmtime = int(HeaderOut[7], 16);
899 fctime = int(HeaderOut[8], 16);
900 fbtime = int(HeaderOut[9], 16);
901 fmode = int(HeaderOut[10], 16);
902 fchmode = stat.S_IMODE(fmode);
903 ftypemod = stat.S_IFMT(fmode);
904 fwinattributes = int(HeaderOut[11], 16);
905 fcompression = HeaderOut[12];
906 fcsize = int(HeaderOut[13], 16);
907 fuid = int(HeaderOut[14], 16);
908 funame = HeaderOut[15];
909 fgid = int(HeaderOut[16], 16);
910 fgname = HeaderOut[17];
911 fid = int(HeaderOut[18], 16);
912 finode = int(HeaderOut[19], 16);
913 flinkcount = int(HeaderOut[20], 16);
914 fdev_minor = int(HeaderOut[21], 16);
915 fdev_major = int(HeaderOut[22], 16);
916 frdev_minor = int(HeaderOut[23], 16);
917 frdev_major = int(HeaderOut[24], 16);
918 fextrasize = int(HeaderOut[25], 16);
919 fextrafields = int(HeaderOut[26], 16);
920 extrafieldslist = [];
921 extrastart = 27;
922 extraend = extrastart + fextrafields;
923 extrafieldslist = [];
924 if(extrastart<extraend):
925 extrafieldslist.append(HeaderOut[extrastart]);
926 extrastart = extrastart + 1;
927 fchecksumtype = HeaderOut[extrastart].lower();
928 fcs = HeaderOut[extrastart + 1].lower();
929 fccs = HeaderOut[extrastart + 2].lower();
930 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
931 if(fcs!=newfcs and not skipchecksum):
932 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
933 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
934 return False;
935 fhend = fp.tell() - 1;
936 fcontentstart = fp.tell();
937 fcontents = BytesIO();
938 pyhascontents = False;
939 if(fsize>0 and not listonly):
940 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
941 fcontents.write(fp.read(fsize));
942 else:
943 fcontents.write(fp.read(fcsize));
944 pyhascontents = True;
945 elif(fsize>0 and listonly):
946 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
947 fp.seek(fsize, 1);
948 else:
949 fp.seek(fcsize, 1);
950 pyhascontents = False;
951 fcontents.seek(0, 0);
952 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
953 if(fccs!=newfccs and not skipchecksum and not listonly):
954 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
955 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
956 return False;
957 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
958 pass;
959 else:
960 fcontents.seek(0, 0);
961 if(uncompress):
962 fcontents = UncompressArchiveFile(fcontents, formatspecs);
963 fcontents.seek(0, 0);
964 fccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
965 fp.seek(1, 1);
966 fcontentend = fp.tell() - 1;
967 fcontents.seek(0, 0);
968 catlist = {'fheadersize': fheadsize, 'fhstart': fheaderstart, 'fhend': fhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': fchecksumtype, 'fnumfields': fnumfields + 2, 'frawheader': HeaderOut, 'fextrafields': fextrafields, 'fextrafieldsize': fextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': fcs, 'fcontentchecksum': fccs, 'fhascontents': pyhascontents, 'fcontentstart': fcontentstart, 'fcontentend': fcontentend, 'fcontents': fcontents};
969 return catlist;
970
971 def ReadFileHeaderDataBySizeWithContentToList(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
972 formatspecs = FormatSpecsListToDict(formatspecs);
973 delimiter = formatspecs['format_delimiter'];
974 fheaderstart = fp.tell();
975 if(formatspecs['new_style']):
976 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
977 else:
978 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
979 if(len(HeaderOut)==0):
980 return False;
981 fheadsize = int(HeaderOut[0], 16);
982 fnumfields = int(HeaderOut[1], 16);
983 ftype = int(HeaderOut[2], 16);
984 if(re.findall("^[.|/]", HeaderOut[3])):
985 fname = HeaderOut[3];
986 else:
987 fname = "./"+HeaderOut[3];
988 fbasedir = os.path.dirname(fname);
989 flinkname = HeaderOut[4];
990 fsize = int(HeaderOut[5], 16);
991 fatime = int(HeaderOut[6], 16);
992 fmtime = int(HeaderOut[7], 16);
993 fctime = int(HeaderOut[8], 16);
994 fbtime = int(HeaderOut[9], 16);
995 fmode = int(HeaderOut[10], 16);
996 fchmode = stat.S_IMODE(fmode);
997 ftypemod = stat.S_IFMT(fmode);
998 fwinattributes = int(HeaderOut[11], 16);
999 fcompression = HeaderOut[12];
1000 fcsize = int(HeaderOut[13], 16);
1001 fuid = int(HeaderOut[14], 16);
1002 funame = HeaderOut[15];
1003 fgid = int(HeaderOut[16], 16);
1004 fgname = HeaderOut[17];
1005 fid = int(HeaderOut[18], 16);
1006 finode = int(HeaderOut[19], 16);
1007 flinkcount = int(HeaderOut[20], 16);
1008 fdev_minor = int(HeaderOut[21], 16);
1009 fdev_major = int(HeaderOut[22], 16);
1010 frdev_minor = int(HeaderOut[23], 16);
1011 frdev_major = int(HeaderOut[24], 16);
1012 fextrasize = int(HeaderOut[25], 16);
1013 fextrafields = int(HeaderOut[26], 16);
1014 extrafieldslist = [];
1015 extrastart = 27;
1016 extraend = extrastart + fextrafields;
1017 extrafieldslist = [];
1018 if(extrastart<extraend):
1019 extrafieldslist.append(HeaderOut[extrastart]);
1020 extrastart = extrastart + 1;
1021 fchecksumtype = HeaderOut[extrastart].lower();
1022 fcs = HeaderOut[extrastart + 1].lower();
1023 fccs = HeaderOut[extrastart + 2].lower();
1024 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
1025 if(fcs!=newfcs and not skipchecksum):
1026 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
1027 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
1028 return False;
1029 fhend = fp.tell() - 1;
1030 fcontentstart = fp.tell();
1031 fcontents = BytesIO();
1032 pyhascontents = False;
1033 if(fsize>0 and not listonly):
1034 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1035 fcontents.write(fp.read(fsize));
1036 else:
1037 fcontents.write(fp.read(fcsize));
1038 pyhascontents = True;
1039 elif(fsize>0 and listonly):
1040 if(fcompression=="none" or fcompression=="" or fcompression=="atuo"):
1041 fp.seek(fsize, 1);
1042 else:
1043 fp.seek(fcsize, 1);
1044 pyhascontents = False;
1045 fcontents.seek(0, 0);
1046 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
1047 if(fccs!=newfccs and not skipchecksum and not listonly):
1048 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
1049 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
1050 return False;
1051 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1052 pass;
1053 else:
1054 fcontents.seek(0, 0);
1055 if(uncompress):
1056 fcontents = UncompressArchiveFile(fcontents, formatspecs);
1057 fcontents.seek(0, 0);
1058 fp.seek(1, 1);
1059 fcontentend = fp.tell() - 1;
1060 catlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, extrafieldslist, fchecksumtype, fcontents];
1061 return catlist;
1062
1063 def ReadFileDataBySizeWithContent(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1064 formatspecs = FormatSpecsListToDict(formatspecs);
1065 delimiter = formatspecs['format_delimiter'];
1066 curloc = fp.tell();
1067 if(curloc>0):
1068 fp.seek(0, 0);
1069 catheader = ReadFileHeaderData(fp, 4, delimiter);
1070 if(curloc>0):
1071 fp.seek(curloc, 0);
1072 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1073 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1074 if(not headercheck and not skipchecksum):
1075 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1076 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1077 return False;
1078 fnumfiles = int(catheader[1], 16);
1079 countnum = 0;
1080 flist = [];
1081 while(countnum < fnumfiles):
1082 HeaderOut = ReadFileHeaderDataBySizeWithContent(fp, listonly, uncompress, skipchecksum, formatspecs)
1083 if(len(HeaderOut)==0):
1084 break;
1085 flist.append(HeaderOut);
1086 countnum = countnum + 1;
1087 return flist;
1088
1089 def ReadFileDataBySizeWithContentToArray(fp, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1090 formatspecs = FormatSpecsListToDict(formatspecs);
1091 delimiter = formatspecs['format_delimiter'];
1092 curloc = fp.tell();
1093 if(curloc>0):
1094 fp.seek(0, 0);
1095 catheader = ReadFileHeaderData(fp, 4, delimiter);
1096 if(curloc>0):
1097 fp.seek(curloc, 0);
1098 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1099 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1100 if(not headercheck and not skipchecksum):
1101 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1102 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1103 return False;
1104 catstring = catheader[0];
1105 catversion = re.findall(r"([\d]+)", catstring);
1106 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
1107 fprenumfiles = catheader[1];
1108 fnumfiles = int(fprenumfiles, 16);
1109 fprechecksumtype = catheader[2];
1110 fprechecksum = catheader[3];
1111 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': []};
1112 if(seekstart<0 and seekstart>fnumfiles):
1113 seekstart = 0;
1114 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
1115 seekend = fnumfiles;
1116 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
1117 seekend = fnumfiles - abs(seekend);
1118 if(seekstart>0):
1119 il = 0;
1120 while(il < seekstart):
1121 prefhstart = fp.tell();
1122 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs['format_delimiter']);
1123 if(len(preheaderdata)==0):
1124 break;
1125 prefsize = int(preheaderdata[5], 16);
1126 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-3].lower(), True, formatspecs);
1127 prefcs = preheaderdata[-2];
1128 if(prefcs!=prenewfcs and not skipchecksum):
1129 VVerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
1130 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
1131 return False;
1132 valid_archive = False;
1133 invalid_archive = True;
1134 prefhend = fp.tell() - 1;
1135 prefcontentstart = fp.tell();
1136 prefcontents = BytesIO();
1137 pyhascontents = False;
1138 if(prefsize>0):
1139 prefcontents.write(fp.read(prefsize));
1140 prefcontents.seek(0, 0);
1141 prenewfccs = GetFileChecksum(prefcontents.read(), preheaderdata[-3].lower(), False, formatspecs);
1142 prefccs = preheaderdata[-1];
1143 pyhascontents = True;
1144 if(prefccs!=prenewfccs and not skipchecksum):
1145 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
1146 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
1147 return False;
1148 fp.seek(1, 1);
1149 il = il + 1;
1150 realidnum = 0;
1151 countnum = seekstart;
1152 while(countnum < seekend):
1153 HeaderOut = ReadFileHeaderDataBySizeWithContentToArray(fp, listonly, uncompress, skipchecksum, formatspecs);
1154 if(len(HeaderOut)==0):
1155 break;
1156 HeaderOut.update({'fid': realidnum, 'fidalt': realidnum});
1157 catlist['ffilelist'].append(HeaderOut);
1158 countnum = countnum + 1;
1159 realidnum = realidnum + 1;
1160 return catlist;
1161
1162 def ReadFileDataBySizeWithContentToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1163 formatspecs = FormatSpecsListToDict(formatspecs);
1164 if(isinstance(infile, dict)):
1165 listcatfiles = infile;
1166 else:
1167 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
1168 return False;
1169 listcatfiles = ReadFileDataBySizeWithContentToArray(infile, seekstart, seekend, listonly, uncompress, skipchecksum, formatspecs);
1170 if(not listcatfiles):
1171 return False;
1172 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
1173 lenlist = len(listcatfiles['ffilelist']);
1174 lcfi = 0;
1175 lcfx = int(listcatfiles['fnumfiles']);
1176 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
1177 lcfx = int(lenlist);
1178 else:
1179 lcfx = int(listcatfiles['fnumfiles']);
1180 while(lcfi < lcfx):
1181 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
1182 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
1183 catarray['filetoid'].update(filetoidarray);
1184 catarray['idtofile'].update(idtofilearray);
1185 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
1186 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
1187 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
1188 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
1189 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
1190 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
1191 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
1192 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
1193 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
1194 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1195 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1196 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
1197 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
1198 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
1199 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
1200 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
1201 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1202 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1203 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
1204 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
1205 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
1206 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1207 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1208 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
1209 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
1210 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
1211 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
1212 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1213 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1214 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1215 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1216 lcfi = lcfi + 1;
1217 return catarray;
1218
1219 def ReadFileDataBySizeWithContentToList(fp, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1220 formatspecs = FormatSpecsListToDict(formatspecs);
1221 delimiter = formatspecs['format_delimiter'];
1222 curloc = fp.tell();
1223 if(curloc>0):
1224 fp.seek(0, 0);
1225 catheader = ReadFileHeaderData(fp, 4, delimiter);
1226 if(curloc>0):
1227 fp.seek(curloc, 0);
1228 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1229 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1230 if(not headercheck and not skipchecksum):
1231 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1232 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1233 return False;
1234 catstring = catheader[0];
1235 catversion = re.findall(r"([\d]+)", catstring);
1236 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
1237 fprenumfiles = catheader[1];
1238 fnumfiles = int(fprenumfiles, 16);
1239 fprechecksumtype = catheader[2];
1240 fprechecksum = catheader[3];
1241 catlist = [];
1242 if(seekstart<0 and seekstart>fnumfiles):
1243 seekstart = 0;
1244 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
1245 seekend = fnumfiles;
1246 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
1247 seekend = fnumfiles - abs(seekend);
1248 if(seekstart>0):
1249 il = 0;
1250 while(il < seekstart):
1251 prefhstart = fp.tell();
1252 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs['format_delimiter']);
1253 if(len(preheaderdata)==0):
1254 break;
1255 prefsize = int(preheaderdata[5], 16);
1256 prefcompression = preheaderdata[12];
1257 prefcsize = int(preheaderdata[13], 16);
1258 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-3].lower(), True, formatspecs);
1259 prefcs = preheaderdata[-2];
1260 if(prefcs!=prenewfcs and not skipchecksum):
1261 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
1262 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
1263 return False;
1264 valid_archive = False;
1265 invalid_archive = True;
1266 prefhend = fp.tell() - 1;
1267 prefcontentstart = fp.tell();
1268 prefcontents = "";
1269 pyhascontents = False;
1270 if(prefsize>0):
1271 if(prefcompression=="none" or prefcompression=="" or prefcompression=="auto"):
1272 prefcontents = catfp.read(prefsize);
1273 else:
1274 prefcontents = catfp.read(prefcsize);
1275 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
1276 prefccs = preheaderdata[-1];
1277 pyhascontents = True;
1278 if(prefccs!=prenewfccs and not skipchecksum):
1279 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
1280 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
1281 return False;
1282 fp.seek(1, 1);
1283 il = il + 1;
1284 realidnum = 0;
1285 countnum = seekstart;
1286 while(countnum < seekend):
1287 HeaderOut = ReadFileHeaderDataBySizeWithContentToList(fp, listonly, uncompress, skipchecksum, formatspecs);
1288 if(len(HeaderOut)==0):
1289 break;
1290 catlist.append(HeaderOut);
1291 countnum = countnum + 1;
1292 realidnum = realidnum + 1;
1293 return catlist;
1294
1295 def ReadInFileBySizeWithContentToArray(infile, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1296 formatspecs = FormatSpecsListToDict(formatspecs);
1297 delimiter = formatspecs['format_delimiter'];
1298 if(hasattr(infile, "read") or hasattr(infile, "write")):
1299 fp = infile;
1300 fp.seek(0, 0);
1301 fp = UncompressArchiveFile(fp, formatspecs);
1302 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1303 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1304 return False;
1305 if(not fp):
1306 return False;
1307 fp.seek(0, 0);
1308 elif(infile=="-"):
1309 fp = BytesIO();
1310 if(hasattr(sys.stdin, "buffer")):
1311 shutil.copyfileobj(sys.stdin.buffer, fp);
1312 else:
1313 shutil.copyfileobj(sys.stdin, fp);
1314 fp.seek(0, 0);
1315 fp = UncompressArchiveFile(fp, formatspecs);
1316 if(not fp):
1317 return False;
1318 fp.seek(0, 0);
1319 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
1320 fp = download_file_from_internet_file(infile);
1321 fp = UncompressArchiveFile(fp, formatspecs);
1322 fp.seek(0, 0);
1323 if(not fp):
1324 return False;
1325 fp.seek(0, 0);
1326 else:
1327 infile = RemoveWindowsPath(infile);
1328 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
1329 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1330 return False;
1331 compresscheck = CheckCompressionType(infile, formatspecs, True);
1332 if(not compresscheck):
1333 fextname = os.path.splitext(infile)[1];
1334 if(fextname==".gz"):
1335 compresscheck = "gzip";
1336 elif(fextname==".bz2"):
1337 compresscheck = "bzip2";
1338 elif(fextname==".zst"):
1339 compresscheck = "zstd";
1340 elif(fextname==".lz4" or fextname==".clz4"):
1341 compresscheck = "lz4";
1342 elif(fextname==".lzo" or fextname==".lzop"):
1343 compresscheck = "lzo";
1344 elif(fextname==".lzma" or fextname==".xz"):
1345 compresscheck = "lzma";
1346 else:
1347 return False;
1348 if(not compresscheck):
1349 return False;
1350 fp = UncompressFile(infile, formatspecs, "rb");
1351 return ReadFileDataBySizeWithContentToArray(fp, listonly, uncompress, skipchecksum, formatspecs);
1352
1353 def ReadInFileBySizeWithContentToList(infile, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1354 formatspecs = FormatSpecsListToDict(formatspecs);
1355 delimiter = formatspecs['format_delimiter'];
1356 if(hasattr(infile, "read") or hasattr(infile, "write")):
1357 fp = infile;
1358 fp.seek(0, 0);
1359 fp = UncompressArchiveFile(fp, formatspecs);
1360 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1361 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1362 return False;
1363 if(not fp):
1364 return False;
1365 fp.seek(0, 0);
1366 elif(infile=="-"):
1367 fp = BytesIO();
1368 if(hasattr(sys.stdin, "buffer")):
1369 shutil.copyfileobj(sys.stdin.buffer, fp);
1370 else:
1371 shutil.copyfileobj(sys.stdin, fp);
1372 fp.seek(0, 0);
1373 fp = UncompressArchiveFile(fp, formatspecs);
1374 if(not fp):
1375 return False;
1376 fp.seek(0, 0);
1377 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
1378 fp = download_file_from_internet_file(infile);
1379 fp = UncompressArchiveFile(fp, formatspecs);
1380 fp.seek(0, 0);
1381 if(not fp):
1382 return False;
1383 fp.seek(0, 0);
1384 else:
1385 infile = RemoveWindowsPath(infile);
1386 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
1387 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1388 return False;
1389 compresscheck = CheckCompressionType(infile, formatspecs, True);
1390 if(not compresscheck):
1391 fextname = os.path.splitext(infile)[1];
1392 if(fextname==".gz"):
1393 compresscheck = "gzip";
1394 elif(fextname==".bz2"):
1395 compresscheck = "bzip2";
1396 elif(fextname==".zst"):
1397 compresscheck = "zstd";
1398 elif(fextname==".lz4" or fextname==".clz4"):
1399 compresscheck = "lz4";
1400 elif(fextname==".lzo" or fextname==".lzop"):
1401 compresscheck = "lzo";
1402 elif(fextname==".lzma" or fextname==".xz"):
1403 compresscheck = "lzma";
1404 else:
1405 return False;
1406 if(not compresscheck):
1407 return False;
1408 fp = UncompressFile(infile, formatspecs, "rb");
1409 return ReadFileDataBySizeWithContentToList(fp, seekstart, seekend, listonly, uncompress, skipchecksum, formatspecs);
1410
1411 def ReadInFileBySizeWithContentToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
1412 formatspecs = FormatSpecsListToDict(formatspecs);
1413 if(isinstance(infile, dict)):
1414 listcatfiles = infile;
1415 else:
1416 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
1417 infile = RemoveWindowsPath(infile);
1418 listcatfiles = ReadInFileBySizeWithContentToArray(infile, listonly, skipchecksum, formatspecs);
1419 if(not listcatfiles):
1420 return False;
1421 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
1422 lenlist = len(listcatfiles['ffilelist']);
1423 lcfi = 0;
1424 lcfx = int(listcatfiles['fnumfiles']);
1425 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
1426 lcfx = int(lenlist);
1427 else:
1428 lcfx = int(listcatfiles['fnumfiles']);
1429 while(lcfi < lcfx):
1430 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
1431 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
1432 catarray['filetoid'].update(filetoidarray);
1433 catarray['idtofile'].update(idtofilearray);
1434 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
1435 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
1436 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
1437 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
1438 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
1439 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
1440 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
1441 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
1442 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
1443 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1444 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1445 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
1446 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
1447 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
1448 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
1449 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
1450 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1451 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1452 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
1453 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
1454 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
1455 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1456 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1457 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
1458 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
1459 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
1460 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
1461 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1462 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1463 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1464 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1465 lcfi = lcfi + 1;
1466 return catarray;
1467
1468 def AppendNullByte(indata, delimiter=__file_format_dict__['format_delimiter']):
1469 outdata = str(indata) + delimiter;
1470 return outdata;
1471
1472 def AppendNullBytes(indata=[], delimiter=__file_format_dict__['format_delimiter']):
1473 outdata = "";
1474 inum = 0;
1475 il = len(indata);
1476 while(inum < il):
1477 outdata = outdata + AppendNullByte(indata[inum], delimiter);
1478 inum = inum + 1;
1479 return outdata;
1480
1481 def AppendFileHeader(fp, numfiles, checksumtype="crc32", formatspecs=__file_format_dict__):
1482 formatspecs = FormatSpecsListToDict(formatspecs);
1483 delimiter = formatspecs['format_delimiter'];
1484 catver = formatspecs['format_ver'];
1485 fileheaderver = str(int(catver.replace(".", "")));
1486 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
1487 fp.write(fileheader.encode('UTF-8'));
1488 fnumfiles = format(int(numfiles), 'x').lower();
1489 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs['format_delimiter']);
1490 catfileheadercshex = GetFileChecksum(fileheader + fnumfilesa, checksumtype, True, formatspecs);
1491 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
1492 fp.write(fnumfilesa.encode('UTF-8'));
1493 try:
1494 fp.flush();
1495 os.fsync(fp.fileno());
1496 except io.UnsupportedOperation:
1497 pass;
1498 except AttributeError:
1499 pass;
1500 except OSError as e:
1501 pass;
1502 return fp;
1503
1504 def MakeEmptyFilePointer(fp, checksumtype="crc32", formatspecs=__file_format_dict__):
1505 formatspecs = FormatSpecsListToDict(formatspecs);
1506 AppendFileHeader(fp, 0, checksumtype, formatspecs);
1507 return fp;
1508
1509 def MakeEmptyFile(outfile, compression="auto", compressionlevel=None, checksumtype="crc32", formatspecs=__file_format_dict__, returnfp=False):
1510 formatspecs = FormatSpecsListToDict(formatspecs);
1511 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1512 if(os.path.exists(outfile)):
1513 try:
1514 os.unlink(outfile);
1515 except OSError as e:
1516 pass;
1517 if(outfile=="-"):
1518 verbose = False;
1519 catfpfp = BytesIO();
1520 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1521 catfp = outfile;
1522 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1523 catfp = BytesIO();
1524 else:
1525 fbasename = os.path.splitext(outfile)[0];
1526 fextname = os.path.splitext(outfile)[1];
1527 catfp = CompressOpenFile(outfile, True, compressionlevel);
1528 catfp = AppendFileHeader(catfp, 0, checksumtype, formatspecs);
1529 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1530 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1531 try:
1532 catfp.flush();
1533 os.fsync(catfp.fileno());
1534 except io.UnsupportedOperation:
1535 pass;
1536 except AttributeError:
1537 pass;
1538 except OSError as e:
1539 pass;
1540 if(outfile=="-"):
1541 catfp.seek(0, 0);
1542 if(hasattr(sys.stdout, "buffer")):
1543 shutil.copyfileobj(catfp, sys.stdout.buffer);
1544 else:
1545 shutil.copyfileobj(catfp, sys.stdout);
1546 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1547 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1548 catfp.seek(0, 0);
1549 upload_file_to_internet_file(catfp, outfile);
1550 if(returnfp):
1551 catfp.seek(0, 0);
1552 return catfp;
1553 else:
1554 catfp.close();
1555 return True;
1556
1557 def AppendFileHeaderWithContent(fp, filevalues=[], extradata=[], filecontent="", checksumtype="crc32", formatspecs=__file_format_dict__):
1558 formatspecs = FormatSpecsListToDict(formatspecs);
1559 extrafields = format(len(extradata), 'x').lower();
1560 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
1561 if(len(extradata)>0):
1562 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
1563 extrasizelen = format(len(extrasizestr), 'x').lower();
1564 catoutlen = len(filevalues) + len(extradata) + 5;
1565 catoutlenhex = format(catoutlen, 'x').lower();
1566 catoutlist = filevalues;
1567 catoutlist.insert(0, catoutlenhex);
1568 catoutlist.append(extrasizelen);
1569 catoutlist.append(extrafields);
1570 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
1571 if(len(extradata)>0):
1572 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
1573 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
1574 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
1575 catfilecontentcshex = GetFileChecksum(filecontent, checksumtype, False, formatspecs);
1576 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
1577 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
1578 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
1579 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
1580 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
1581 catfileoutstrecd = catfileoutstr.encode('UTF-8');
1582 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
1583 catfileout = catfileoutstrecd + filecontent + nullstrecd;
1584 fp.write(catfileout);
1585 try:
1586 fp.flush();
1587 os.fsync(fp.fileno());
1588 except io.UnsupportedOperation:
1589 pass;
1590 except AttributeError:
1591 pass;
1592 except OSError as e:
1593 pass;
1594 return fp;
1595
1596 def AppendFilesWithContent(infiles, fp, dirlistfromtxt=False, filevalues=[], extradata=[], compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
1597 formatspecs = FormatSpecsListToDict(formatspecs);
1598 advancedlist = formatspecs['use_advanced_list'];
1599 altinode = formatspecs['use_alt_inode'];
1600 if(verbose):
1601 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
1602 if(infiles=="-"):
1603 for line in sys.stdin:
1604 infilelist.append(line.strip());
1605 infilelist = list(filter(None, infilelist));
1606 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
1607 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
1608 return False;
1609 with UncompressFile(infiles, formatspecs, "r") as finfile:
1610 for line in finfile:
1611 infilelist.append(line.strip());
1612 infilelist = list(filter(None, infilelist));
1613 else:
1614 if(isinstance(infiles, (list, tuple, ))):
1615 infilelist = list(filter(None, infiles));
1616 elif(isinstance(infiles, (str, ))):
1617 infilelist = list(filter(None, [infiles]));
1618 if(advancedlist):
1619 GetDirList = ListDirAdvanced(infilelist, followlink, False);
1620 else:
1621 GetDirList = ListDir(infilelist, followlink, False);
1622 if(not GetDirList):
1623 return False;
1624 curinode = 0;
1625 curfid = 0;
1626 inodelist = [];
1627 inodetofile = {};
1628 filetoinode = {};
1629 inodetocatinode = {};
1630 numfiles = int(len(GetDirList));
1631 fnumfiles = format(numfiles, 'x').lower();
1632 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
1633 for curfname in GetDirList:
1634 if(re.findall("^[.|/]", curfname)):
1635 fname = curfname;
1636 else:
1637 fname = "./"+curfname;
1638 if(verbose):
1639 VerbosePrintOut(fname);
1640 if(not followlink or followlink is None):
1641 fstatinfo = os.lstat(fname);
1642 else:
1643 fstatinfo = os.stat(fname);
1644 fpremode = fstatinfo.st_mode;
1645 finode = fstatinfo.st_ino;
1646 flinkcount = fstatinfo.st_nlink;
1647 ftype = 0;
1648 if(stat.S_ISREG(fpremode)):
1649 ftype = 0;
1650 elif(stat.S_ISLNK(fpremode)):
1651 ftype = 2;
1652 elif(stat.S_ISCHR(fpremode)):
1653 ftype = 3;
1654 elif(stat.S_ISBLK(fpremode)):
1655 ftype = 4;
1656 elif(stat.S_ISDIR(fpremode)):
1657 ftype = 5;
1658 elif(stat.S_ISFIFO(fpremode)):
1659 ftype = 6;
1660 elif(stat.S_ISSOCK(fpremode)):
1661 ftype = 8;
1662 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
1663 ftype = 9;
1664 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
1665 ftype = 10;
1666 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
1667 ftype = 111;
1668 else:
1669 ftype = 0;
1670 flinkname = "";
1671 fcurfid = format(int(curfid), 'x').lower();
1672 if(not followlink and finode!=0):
1673 if(ftype!=1):
1674 if(finode in inodelist):
1675 ftype = 1;
1676 flinkname = inodetofile[finode];
1677 if(altinode):
1678 fcurinode = format(int(finode), 'x').lower();
1679 else:
1680 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
1681 if(finode not in inodelist):
1682 inodelist.append(finode);
1683 inodetofile.update({finode: fname});
1684 inodetocatinode.update({finode: curinode});
1685 if(altinode):
1686 fcurinode = format(int(finode), 'x').lower();
1687 else:
1688 fcurinode = format(int(curinode), 'x').lower();
1689 curinode = curinode + 1;
1690 else:
1691 fcurinode = format(int(curinode), 'x').lower();
1692 curinode = curinode + 1;
1693 curfid = curfid + 1;
1694 if(ftype==2):
1695 flinkname = os.readlink(fname);
1696 fdev = fstatinfo.st_dev;
1697 getfdev = GetDevMajorMinor(fdev);
1698 fdev_minor = getfdev[0];
1699 fdev_major = getfdev[1];
1700 frdev = fstatinfo.st_dev;
1701 if(hasattr(fstatinfo, "st_rdev")):
1702 frdev = fstatinfo.st_rdev;
1703 else:
1704 frdev = fstatinfo.st_dev;
1705 getfrdev = GetDevMajorMinor(frdev);
1706 frdev_minor = getfrdev[0];
1707 frdev_major = getfrdev[1];
1708 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
1709 fsize = format(int("0"), 'x').lower();
1710 elif(ftype==0 or ftype==7):
1711 fsize = format(int(fstatinfo.st_size), 'x').lower();
1712 else:
1713 fsize = format(int(fstatinfo.st_size)).lower();
1714 fatime = format(int(fstatinfo.st_atime), 'x').lower();
1715 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
1716 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
1717 if(hasattr(fstatinfo, "st_birthtime")):
1718 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
1719 else:
1720 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
1721 fmode = format(int(fstatinfo.st_mode), 'x').lower();
1722 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
1723 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
1724 fuid = format(int(fstatinfo.st_uid), 'x').lower();
1725 fgid = format(int(fstatinfo.st_gid), 'x').lower();
1726 funame = "";
1727 try:
1728 import pwd;
1729 try:
1730 userinfo = pwd.getpwuid(fstatinfo.st_uid);
1731 funame = userinfo.pw_name;
1732 except KeyError:
1733 funame = "";
1734 except ImportError:
1735 funame = "";
1736 fgname = "";
1737 try:
1738 import grp;
1739 try:
1740 groupinfo = grp.getgrgid(fstatinfo.st_gid);
1741 fgname = groupinfo.gr_name;
1742 except KeyError:
1743 fgname = "";
1744 except ImportError:
1745 fgname = "";
1746 fdev_minor = format(int(fdev_minor), 'x').lower();
1747 fdev_major = format(int(fdev_major), 'x').lower();
1748 frdev_minor = format(int(frdev_minor), 'x').lower();
1749 frdev_major = format(int(frdev_major), 'x').lower();
1750 finode = format(int(finode), 'x').lower();
1751 flinkcount = format(int(flinkcount), 'x').lower();
1752 if(hasattr(fstatinfo, "st_file_attributes")):
1753 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
1754 else:
1755 fwinattributes = format(int(0), 'x').lower();
1756 fcompression = "";
1757 fcsize = format(int(0), 'x').lower();
1758 fcontents = BytesIO();
1759 chunk_size = 1024;
1760 if(ftype==0 or ftype==7):
1761 with open(fname, "rb") as fpc:
1762 shutil.copyfileobj(fpc, fcontents);
1763 if(not compresswholefile):
1764 fcontents.seek(0, 2);
1765 ucfsize = fcontents.tell();
1766 fcontents.seek(0, 0);
1767 if(compression=="auto"):
1768 ilsize = len(compressionlistalt);
1769 ilmin = 0;
1770 ilcsize = [];
1771 while(ilmin < ilsize):
1772 cfcontents = BytesIO();
1773 shutil.copyfileobj(fcontents, cfcontents);
1774 fcontents.seek(0, 0);
1775 cfcontents.seek(0, 0);
1776 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
1777 if(cfcontents):
1778 cfcontents.seek(0, 2);
1779 ilcsize.append(cfcontents.tell());
1780 cfcontents.close();
1781 else:
1782 try:
1783 ilcsize.append(sys.maxint);
1784 except AttributeError:
1785 ilcsize.append(sys.maxsize);
1786 ilmin = ilmin + 1;
1787 ilcmin = ilcsize.index(min(ilcsize));
1788 compression = compressionlistalt[ilcmin];
1789 fcontents.seek(0, 0);
1790 cfcontents = BytesIO();
1791 shutil.copyfileobj(fcontents, cfcontents);
1792 cfcontents.seek(0, 0);
1793 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
1794 cfcontents.seek(0, 2);
1795 cfsize = cfcontents.tell();
1796 if(ucfsize > cfsize):
1797 fcsize = format(int(cfsize), 'x').lower();
1798 fcompression = compression;
1799 fcontents.close();
1800 fcontents = cfcontents;
1801 if(followlink and (ftype==1 or ftype==2)):
1802 flstatinfo = os.stat(flinkname);
1803 with open(flinkname, "rb") as fpc:
1804 shutil.copyfileobj(fpc, fcontents);
1805 if(not compresswholefile):
1806 fcontents.seek(0, 2);
1807 ucfsize = fcontents.tell();
1808 fcontents.seek(0, 0);
1809 if(compression=="auto"):
1810 ilsize = len(compressionlistalt);
1811 ilmin = 0;
1812 ilcsize = [];
1813 while(ilmin < ilsize):
1814 cfcontents = BytesIO();
1815 shutil.copyfileobj(fcontents, cfcontents);
1816 fcontents.seek(0, 0);
1817 cfcontents.seek(0, 0);
1818 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
1819 if(cfcontents):
1820 cfcontents.seek(0, 2);
1821 ilcsize.append(cfcontents.tell());
1822 cfcontents.close();
1823 else:
1824 try:
1825 ilcsize.append(sys.maxint);
1826 except AttributeError:
1827 ilcsize.append(sys.maxsize);
1828 ilmin = ilmin + 1;
1829 ilcmin = ilcsize.index(min(ilcsize));
1830 compression = compressionlistalt[ilcmin];
1831 fcontents.seek(0, 0);
1832 cfcontents = BytesIO();
1833 shutil.copyfileobj(fcontents, cfcontents);
1834 cfcontents.seek(0, 0);
1835 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
1836 cfcontents.seek(0, 2);
1837 cfsize = cfcontents.tell();
1838 if(ucfsize > cfsize):
1839 fcsize = format(int(cfsize), 'x').lower();
1840 fcompression = compression;
1841 fcontents.close();
1842 fcontents = cfcontents;
1843 if(fcompression=="none"):
1844 fcompression = "";
1845 fcontents.seek(0, 0);
1846 ftypehex = format(ftype, 'x').lower();
1847 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
1848 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
1849 if(numfiles>0):
1850 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
1851 fp.seek(0, 0);
1852 return fp;
1853
1854 def AppendListsWithContent(inlist, fp, dirlistfromtxt=False, filevalues=[], extradata=[], compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
1855 formatspecs = FormatSpecsListToDict(formatspecs);
1856 if(verbose):
1857 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
1858 GetDirList = inlist;
1859 if(not GetDirList):
1860 return False;
1861 curinode = 0;
1862 curfid = 0;
1863 inodelist = [];
1864 inodetofile = {};
1865 filetoinode = {};
1866 inodetocatinode = {};
1867 numfiles = int(len(GetDirList));
1868 fnumfiles = format(numfiles, 'x').lower();
1869 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
1870 for curfname in GetDirList:
1871 ftype = format(curfname[0], 'x').lower();
1872 if(re.findall("^[.|/]", curfname[1])):
1873 fname = curfname[1];
1874 else:
1875 fname = "./"+curfname[1];
1876 fbasedir = os.path.dirname(fname);
1877 flinkname = curfname[2];
1878 fsize = format(curfname[3], 'x').lower();
1879 fatime = format(curfname[4], 'x').lower();
1880 fmtime = format(curfname[5], 'x').lower();
1881 fctime = format(curfname[6], 'x').lower();
1882 fbtime = format(curfname[7], 'x').lower();
1883 fmode = format(curfname[8], 'x').lower();
1884 fwinattributes = format(curfname[9], 'x').lower();
1885 fcompression = curfname[10];
1886 fcsize = format(curfname[11], 'x').lower();
1887 fuid = format(curfname[12], 'x').lower();
1888 funame = curfname[13];
1889 fgid = format(curfname[14], 'x').lower();
1890 fgname = curfname[15];
1891 fid = format(curfname[16], 'x').lower();
1892 finode = format(curfname[17], 'x').lower();
1893 flinkcount = format(curfname[18], 'x').lower();
1894 fdev_minor = format(curfname[19], 'x').lower();
1895 fdev_major = format(curfname[20], 'x').lower();
1896 frdev_minor = format(curfname[21], 'x').lower();
1897 frdev_major = format(curfname[22], 'x').lower();
1898 extradata = curfname[23];
1899 fchecksumtype = curfname[24];
1900 fcontents = curfname[25];
1901 catoutlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
1902 fcontents.seek(0, 0);
1903 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
1904 if(numfiles>0):
1905 fp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
1906 return fp;
1907
1908 def AppendInFileWithContent(infile, fp, dirlistfromtxt=False, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
1909 formatspecs = FormatSpecsListToDict(formatspecs);
1910 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, True, False, formatspecs);
1911 return AppendListsWithContent(inlist, fp, dirlistfromtxt, filevalues, extradata, followlink, checksumtype, formatspecs, verbose);
1912
1913 def AppendFilesWithContentToOutFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
1914 formatspecs = FormatSpecsListToDict(formatspecs);
1915 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1916 if(os.path.exists(outfile)):
1917 try:
1918 os.unlink(outfile);
1919 except OSError as e:
1920 pass;
1921 if(outfile=="-"):
1922 verbose = False;
1923 catfpfp = BytesIO();
1924 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1925 catfp = outfile;
1926 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1927 catfp = BytesIO();
1928 else:
1929 fbasename = os.path.splitext(outfile)[0];
1930 fextname = os.path.splitext(outfile)[1];
1931 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
1932 catfp = AppendFilesWithContent(infiles, catfp, dirlistfromtxt, filevalues, extradata, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose);
1933 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1934 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1935 try:
1936 catfp.flush();
1937 os.fsync(catfp.fileno());
1938 except io.UnsupportedOperation:
1939 pass;
1940 except AttributeError:
1941 pass;
1942 except OSError as e:
1943 pass;
1944 if(outfile=="-"):
1945 catfp.seek(0, 0);
1946 if(hasattr(sys.stdout, "buffer")):
1947 shutil.copyfileobj(catfp, sys.stdout.buffer);
1948 else:
1949 shutil.copyfileobj(catfp, sys.stdout);
1950 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1951 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1952 catfp.seek(0, 0);
1953 upload_file_to_internet_file(catfp, outfile);
1954 if(returnfp):
1955 catfp.seek(0, 0);
1956 return catfp;
1957 else:
1958 catfp.close();
1959 return True;
1960
1961 def AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
1962 formatspecs = FormatSpecsListToDict(formatspecs);
1963 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1964 if(os.path.exists(outfile)):
1965 try:
1966 os.unlink(outfile);
1967 except OSError as e:
1968 pass;
1969 if(outfile=="-"):
1970 verbose = False;
1971 catfpfp = BytesIO();
1972 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1973 catfp = outfile;
1974 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1975 catfp = BytesIO();
1976 else:
1977 fbasename = os.path.splitext(outfile)[0];
1978 fextname = os.path.splitext(outfile)[1];
1979 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
1980 catfp = AppendListsWithContent(inlist, catfp, dirlistfromtxt, filevalues, extradata, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose);
1981 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1982 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1983 try:
1984 catfp.flush();
1985 os.fsync(catfp.fileno());
1986 except io.UnsupportedOperation:
1987 pass;
1988 except AttributeError:
1989 pass;
1990 except OSError as e:
1991 pass;
1992 if(outfile=="-"):
1993 catfp.seek(0, 0);
1994 if(hasattr(sys.stdout, "buffer")):
1995 shutil.copyfileobj(catfp, sys.stdout.buffer);
1996 else:
1997 shutil.copyfileobj(catfp, sys.stdout);
1998 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1999 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2000 catfp.seek(0, 0);
2001 upload_file_to_internet_file(catfp, outfile);
2002 if(returnfp):
2003 catfp.seek(0, 0);
2004 return catfp;
2005 else:
2006 catfp.close();
2007 return True;
2008
2009 def AppendInFileWithContentToOutFile(infile, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2010 formatspecs = FormatSpecsListToDict(formatspecs);
2011 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, True, False, formatspecs);
2012 return AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt, compression, compresswholefile, compressionlevel, filevalues, extradata, followlink, checksumtype, formatspecs, verbose, returnfp);
2013
2014 def PrintPermissionString(fchmode, ftype):
2015 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
2016 permissionstr = "";
2017 for fmodval in str(oct(fchmode))[-3:]:
2018 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
2019 if(ftype==0 or ftype==7):
2020 permissionstr = "-" + permissionstr;
2021 if(ftype==1):
2022 permissionstr = "h" + permissionstr;
2023 if(ftype==2):
2024 permissionstr = "l" + permissionstr;
2025 if(ftype==3):
2026 permissionstr = "c" + permissionstr;
2027 if(ftype==4):
2028 permissionstr = "b" + permissionstr;
2029 if(ftype==5):
2030 permissionstr = "d" + permissionstr;
2031 if(ftype==6):
2032 permissionstr = "f" + permissionstr;
2033 if(ftype==8):
2034 permissionstr = "D" + permissionstr;
2035 if(ftype==9):
2036 permissionstr = "p" + permissionstr;
2037 if(ftype==10):
2038 permissionstr = "w" + permissionstr;
2039 try:
2040 permissionoutstr = stat.filemode(fchmode);
2041 except AttributeError:
2042 permissionoutstr = permissionstr;
2043 except KeyError:
2044 permissionoutstr = permissionstr;
2045 return permissionoutstr;
2046
2047 def PrintPermissionStringAlt(fchmode, ftype):
2048 permissions = {
2049 '0': '---', '1': '--x', '2': '-w-', '3': '-wx',
2050 '4': 'r--', '5': 'r-x', '6': 'rw-', '7': 'rwx'
2051 };
2052 # Translate file mode into permission string
2053 permissionstr = ''.join([permissions[i] for i in str(oct(fchmode))[-3:]]);
2054 # Append file type indicator
2055 type_indicators = {
2056 0: '-', 1: 'h', 2: 'l', 3: 'c', 4: 'b',
2057 5: 'd', 6: 'f', 8: 'D', 9: 'p', 10: 'w'
2058 };
2059 file_type = type_indicators.get(ftype, '-');
2060 permissionstr = file_type + permissionstr;
2061 try:
2062 permissionoutstr = stat.filemode(fchmode);
2063 except AttributeError:
2064 permissionoutstr = permissionstr;
2065 return permissionoutstr;
2066
2067 def CheckCompressionType(infile, formatspecs=__file_format_dict__, closefp=True):
2068 formatspecs = FormatSpecsListToDict(formatspecs);
2069 if(hasattr(infile, "read") or hasattr(infile, "write")):
2070 catfp = infile;
2071 else:
2072 try:
2073 catfp = open(infile, "rb");
2074 except FileNotFoundError:
2075 return False;
2076 catfp.seek(0, 0);
2077 prefp = catfp.read(2);
2078 filetype = False;
2079 if(prefp==binascii.unhexlify("1f8b")):
2080 filetype = "gzip";
2081 catfp.seek(0, 0);
2082 prefp = catfp.read(3);
2083 if(prefp==binascii.unhexlify("425a68")):
2084 filetype = "bzip2";
2085 if(prefp==binascii.unhexlify("5d0000")):
2086 filetype = "lzma";
2087 catfp.seek(0, 0);
2088 prefp = catfp.read(4);
2089 if(prefp==binascii.unhexlify("28b52ffd")):
2090 filetype = "zstd";
2091 if(prefp==binascii.unhexlify("04224d18")):
2092 filetype = "lz4";
2093 if(prefp==binascii.unhexlify("504B0304")):
2094 filetype = "zipfile";
2095 catfp.seek(0, 0);
2096 prefp = catfp.read(5);
2097 if(prefp==binascii.unhexlify("7573746172")):
2098 filetype = "tarfile";
2099 catfp.seek(0, 0);
2100 prefp = catfp.read(6);
2101 if(prefp==binascii.unhexlify("fd377a585a00")):
2102 filetype = "lzma";
2103 if(prefp==binascii.unhexlify("377abcaf271c")):
2104 filetype = "7zipfile";
2105 catfp.seek(0, 0);
2106 prefp = catfp.read(7);
2107 if(prefp==binascii.unhexlify("526172211a0700")):
2108 filetype = "rarfile";
2109 if(prefp==binascii.unhexlify("43617446696c65")):
2110 filetype = "catfile";
2111 catfp.seek(0, 0);
2112 prefp = catfp.read(8);
2113 if(prefp==binascii.unhexlify("526172211a070100")):
2114 filetype = "rarfile";
2115 catfp.seek(0, 0);
2116 prefp = catfp.read(formatspecs['format_len']);
2117 if(prefp==binascii.unhexlify(formatspecs['format_hex'])):
2118 filetype = formatspecs['format_lower'];
2119 catfp.seek(0, 0);
2120 prefp = catfp.read(9);
2121 if(prefp==binascii.unhexlify("894c5a4f000d0a1a0a")):
2122 filetype = "lzo";
2123 catfp.seek(0, 0);
2124 prefp = catfp.read(10);
2125 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
2126 filetype = "tarfile";
2127 catfp.seek(0, 0);
2128 if(filetype=="gzip" or filetype=="bzip2" or filetype=="lzma" or filetype=="zstd" or filetype=="lz4"):
2129 if(TarFileCheck(catfp)):
2130 filetype = "tarfile";
2131 if(not filetype):
2132 if(TarFileCheck(catfp)):
2133 filetype = "tarfile";
2134 elif(zipfile.is_zipfile(catfp)):
2135 filetype = "zipfile";
2136 elif(rarfile.is_rarfile(catfp) or rarfile.is_rarfile_sfx(catfp)):
2137 filetype = "rarile";
2138 else:
2139 filetype = False;
2140 catfp.seek(0, 0);
2141 if(closefp):
2142 catfp.close();
2143 return filetype;
2144
2145 def CheckCompressionTypeFromString(instring, formatspecs=__file_format_dict__, closefp=True):
2146 formatspecs = FormatSpecsListToDict(formatspecs);
2147 try:
2148 instringsfile = BytesIO(instring);
2149 except TypeError:
2150 instringsfile = BytesIO(instring.encode("UTF-8"));
2151 return CheckCompressionType(instringsfile, formatspecs, closefp);
2152
2153 def GetCompressionMimeType(infile, formatspecs=__file_format_dict__):
2154 formatspecs = FormatSpecsListToDict(formatspecs);
2155 compresscheck = CheckCompressionType(fp, formatspecs, False);
2156 if(compresscheck=="gzip" or compresscheck=="gz"):
2157 return archivefile_gzip_mimetype;
2158 if(compresscheck=="bzip2" or compresscheck=="bz2"):
2159 return archivefile_bzip2_mimetype;
2160 if(compresscheck=="zstd" or compresscheck=="zstandard"):
2161 return archivefile_zstandard_mimetype;
2162 if(compresscheck=="lz4"):
2163 return archivefile_lz4_mimetype;
2164 if(compresscheck=="lzo" or compresscheck=="lzop"):
2165 return archivefile_lzop_mimetype;
2166 if(compresscheck=="lzma"):
2167 return archivefile_lzma_mimetype;
2168 if(compresscheck=="xz"):
2169 return archivefile_xz_mimetype;
2170 if(compresscheck=="catfile" or compresscheck=="cat" or compresscheck==formatspecs['format_lower']):
2171 return archivefile_cat_mimetype;
2172 if(not compresscheck):
2173 return False;
2174 return False;
2175
2176 def UncompressArchiveFile(fp, formatspecs=__file_format_dict__):
2177 formatspecs = FormatSpecsListToDict(formatspecs);
2178 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
2179 return False;
2180 compresscheck = CheckCompressionType(fp, formatspecs, False);
2181 if(compresscheck=="gzip"):
2182 try:
2183 import gzip;
2184 except ImportError:
2185 return False;
2186 catfp = gzip.GzipFile(fileobj=fp, mode="rb");
2187 if(compresscheck=="bzip2"):
2188 try:
2189 import bz2;
2190 except ImportError:
2191 return False;
2192 catfp = BytesIO();
2193 catfp.write(bz2.decompress(fp.read()));
2194 if(compresscheck=="zstd"):
2195 try:
2196 import zstandard;
2197 except ImportError:
2198 return False;
2199 catfp = BytesIO();
2200 catfp.write(zstandard.decompress(fp.read()));
2201 if(compresscheck=="lz4"):
2202 try:
2203 import lz4.frame;
2204 except ImportError:
2205 return False;
2206 catfp = BytesIO();
2207 catfp.write(lz4.frame.decompress(fp.read()));
2208 if(compresscheck=="lzo" or compresscheck=="lzop"):
2209 try:
2210 import lzo;
2211 except ImportError:
2212 return False;
2213 catfp = BytesIO();
2214 catfp.write(lzo.decompress(fp.read()));
2215 if(compresscheck=="lzma" or compresscheck=="xz"):
2216 try:
2217 import lzma;
2218 except ImportError:
2219 try:
2220 from backports import lzma
2221 except ImportError:
2222 return False;
2223 catfp = BytesIO();
2224 catfp.write(lzma.decompress(fp.read()));
2225 if(compresscheck=="catfile" or compresscheck==formatspecs['format_lower']):
2226 catfp = fp;
2227 if(not compresscheck):
2228 try:
2229 import lzma;
2230 except ImportError:
2231 try:
2232 from backports import lzma
2233 except ImportError:
2234 return False;
2235 catfp = BytesIO();
2236 with fp as fpcontent:
2237 try:
2238 catfp.write(lzma.decompress(fp.read()));
2239 except lzma.LZMAError:
2240 return False;
2241 if(compresscheck!="catfile" or compresscheck!=formatspecs['format_lower']):
2242 fp.close();
2243 return catfp;
2244
2245 create_alias_function("Uncompress", __file_format_name__, "", UncompressArchiveFile);
2246
2247 def UncompressFile(infile, formatspecs=__file_format_dict__, mode="rb"):
2248 formatspecs = FormatSpecsListToDict(formatspecs);
2249 compresscheck = CheckCompressionType(infile, formatspecs, False);
2250 if(sys.version_info[0]==2 and compresscheck):
2251 if(mode=="rt"):
2252 mode = "r";
2253 if(mode=="wt"):
2254 mode = "w";
2255 try:
2256 if(compresscheck=="gzip"):
2257 try:
2258 import gzip;
2259 except ImportError:
2260 return False;
2261 try:
2262 filefp = gzip.open(infile, mode, encoding="UTF-8");
2263 except (ValueError, TypeError) as e:
2264 filefp = gzip.open(infile, mode);
2265 if(compresscheck=="bzip2"):
2266 try:
2267 import bz2;
2268 except ImportError:
2269 return False;
2270 try:
2271 filefp = bz2.open(infile, mode, encoding="UTF-8");
2272 except (ValueError, TypeError) as e:
2273 filefp = bz2.open(infile, mode);
2274 if(compresscheck=="zstd"):
2275 try:
2276 import zstandard;
2277 except ImportError:
2278 return False;
2279 try:
2280 filefp = zstandard.open(infile, mode, encoding="UTF-8");
2281 except (ValueError, TypeError) as e:
2282 filefp = zstandard.open(infile, mode);
2283 if(compresscheck=="lz4"):
2284 try:
2285 import lz4.frame;
2286 except ImportError:
2287 return False;
2288 try:
2289 filefp = lz4.frame.open(infile, mode, encoding="UTF-8");
2290 except (ValueError, TypeError) as e:
2291 filefp = lz4.frame.open(infile, mode);
2292 if(compresscheck=="lzo"):
2293 try:
2294 import lzo;
2295 except ImportError:
2296 return False;
2297 try:
2298 filefp = lzo.open(infile, mode, encoding="UTF-8");
2299 except (ValueError, TypeError) as e:
2300 filefp = lzo.open(infile, mode);
2301 if(compresscheck=="lzma"):
2302 try:
2303 import lzma;
2304 except ImportError:
2305 try:
2306 from backports import lzma
2307 except ImportError:
2308 return False;
2309 try:
2310 filefp = lzma.open(infile, mode, encoding="UTF-8");
2311 except (ValueError, TypeError) as e:
2312 filefp = lzma.open(infile, mode);
2313 if(compresscheck=="catfile" or compresscheck==formatspecs['format_lower']):
2314 try:
2315 filefp = open(infile, mode, encoding="UTF-8");
2316 except (ValueError, TypeError) as e:
2317 filefp = open(infile, mode);
2318 if(not compresscheck):
2319 try:
2320 filefp = open(infile, mode, encoding="UTF-8");
2321 except (ValueError, TypeError) as e:
2322 filefp = open(infile, mode);
2323 except FileNotFoundError:
2324 return False;
2325 return filefp;
2326
2327 def UncompressString(infile):
2328 compresscheck = CheckCompressionTypeFromString(infile, formatspecs, False);
2329 if(compresscheck=="gzip"):
2330 try:
2331 import gzip;
2332 except ImportError:
2333 return False;
2334 fileuz = gzip.decompress(infile);
2335 if(compresscheck=="bzip2"):
2336 try:
2337 import bz2;
2338 except ImportError:
2339 return False;
2340 fileuz = bz2.decompress(infile);
2341 if(compresscheck=="zstd"):
2342 try:
2343 import zstandard;
2344 except ImportError:
2345 return False;
2346 fileuz = zstandard.decompress(infile);
2347 if(compresscheck=="lz4"):
2348 try:
2349 import lz4.frame;
2350 except ImportError:
2351 return False;
2352 fileuz = lz4.frame.decompress(infile);
2353 if(compresscheck=="lzo"):
2354 try:
2355 import lzo;
2356 except ImportError:
2357 return False;
2358 fileuz = lzo.decompress(infile);
2359 if(compresscheck=="lzma"):
2360 try:
2361 import lzma;
2362 except ImportError:
2363 try:
2364 from backports import lzma
2365 except ImportError:
2366 return False;
2367 fileuz = lzma.decompress(infile);
2368 if(not compresscheck):
2369 fileuz = infile;
2370 if(hasattr(fileuz, 'decode')):
2371 fileuz = fileuz.decode("UTF-8");
2372 return fileuz;
2373
2374 def UncompressStringAlt(infile):
2375 filefp = StringIO();
2376 outstring = UncompressString(infile);
2377 filefp.write(outstring);
2378 filefp.seek(0, 0);
2379 return filefp;
2380
2381 def CheckCompressionSubType(infile, formatspecs=__file_format_dict__, closefp=True):
2382 formatspecs = FormatSpecsListToDict(formatspecs);
2383 compresscheck = CheckCompressionType(infile, formatspecs, False);
2384 if(not compresscheck):
2385 fextname = os.path.splitext(infile)[1];
2386 if(fextname==".gz"):
2387 compresscheck = "gzip";
2388 elif(fextname==".bz2"):
2389 compresscheck = "bzip2";
2390 elif(fextname==".zst"):
2391 compresscheck = "zstd";
2392 elif(fextname==".lz4"):
2393 compresscheck = "lz4";
2394 elif(fextname==".lzo" or fextname==".lzop"):
2395 compresscheck = "lzo";
2396 elif(fextname==".lzma" or fextname==".xz"):
2397 compresscheck = "lzma";
2398 else:
2399 return False;
2400 if(compresscheck=="gzip" or compresscheck=="bzip2" or compresscheck=="lzma" or compresscheck=="zstd" or compresscheck=="lz4"):
2401 if(TarFileCheck(infile)):
2402 filetype = "tarfile";
2403 if(not compresscheck):
2404 if(TarFileCheck(infile)):
2405 return "tarfile";
2406 elif(zipfile.is_zipfile(infile)):
2407 return "zipfile";
2408 elif(rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile)):
2409 return "rarile";
2410 else:
2411 return False;
2412 return False;
2413 if(compresscheck=="catfile"):
2414 return "catfile";
2415 if(compresscheck==formatspecs['format_lower']):
2416 return formatspecs['format_lower'];
2417 if(compresscheck=="tarfile"):
2418 return "tarfile";
2419 if(compresscheck=="zipfile"):
2420 return "zipfile";
2421 if(rarfile_support and compresscheck=="rarfile"):
2422 return "rarfile";
2423 if(py7zr_support and compresscheck=="7zipfile"):
2424 return "7zipfile";
2425 if(hasattr(infile, "read") or hasattr(infile, "write")):
2426 catfp = UncompressArchiveFile(infile, formatspecs['format_lower']);
2427 else:
2428 try:
2429 if(compresscheck=="gzip"):
2430 try:
2431 import gzip;
2432 except ImportError:
2433 return False;
2434 catfp = gzip.GzipFile(infile, "rb");
2435 if(compresscheck=="bzip2"):
2436 try:
2437 import bz2;
2438 except ImportError:
2439 return False;
2440 catfp = bz2.BZ2File(infile, "rb");
2441 if(compresscheck=="lz4"):
2442 try:
2443 import lz4.frame;
2444 except ImportError:
2445 return False;
2446 catfp = lz4.frame.open(infile, "rb");
2447 if(compresscheck=="zstd"):
2448 try:
2449 import zstandard;
2450 except ImportError:
2451 return False;
2452 catfp = zstandard.open(infile, "rb");
2453 if(compresscheck=="lzma" or compresscheck=="xz"):
2454 try:
2455 import lzma;
2456 except ImportError:
2457 try:
2458 from backports import lzma
2459 except ImportError:
2460 return False;
2461 catfp = lzma.open(infile, "rb");
2462 except FileNotFoundError:
2463 return False;
2464 filetype = False;
2465 prefp = catfp.read(5);
2466 if(prefp==binascii.unhexlify("7573746172")):
2467 filetype = "tarfile";
2468 catfp.seek(0, 0);
2469 prefp = catfp.read(7);
2470 if(prefp==binascii.unhexlify("43617446696c65")):
2471 filetype = "catfile";
2472 catfp.seek(0, 0);
2473 prefp = catfp.read(formatspecs['format_len']);
2474 if(prefp==binascii.unhexlify(formatspecs['format_hex'])):
2475 filetype = formatspecs['format_lower'];
2476 catfp.seek(0, 0);
2477 prefp = catfp.read(10);
2478 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
2479 filetype = "tarfile";
2480 catfp.seek(0, 0);
2481 if(closefp):
2482 catfp.close();
2483 return filetype;
2484
2485 def GZipCompress(data, compresslevel=9):
2486 try:
2487 import gzip;
2488 except ImportError:
2489 return False;
2490 tmpfp = tempfile.NamedTemporaryFile("wb", delete=False);
2491 tmpfp.close();
2492 tmpfp = gzip.GzipFile(tmpfp.name, mode="wb", compresslevel=compresslevel);
2493 tmpfp.write(data);
2494 tmpfp.close();
2495 try:
2496 catfp = open(tmpfp.name, "rb");
2497 except FileNotFoundError:
2498 return False;
2499 catdata = catfp.read();
2500 catfp.close();
2501 return catdata;
2502
2503 def CompressArchiveFile(fp, compression="auto", compressionlevel=None, formatspecs=__file_format_dict__):
2504 formatspecs = FormatSpecsListToDict(formatspecs);
2505 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
2506 return False;
2507 fp.seek(0, 0);
2508 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
2509 compression = "auto";
2510 if(compression not in compressionlist and compression is None):
2511 compression = "auto";
2512 if(compression=="gzip"):
2513 try:
2514 import gzip;
2515 except ImportError:
2516 return False;
2517 catfp = BytesIO();
2518 if(compressionlevel is None):
2519 compressionlevel = 9;
2520 else:
2521 compressionlevel = int(compressionlevel);
2522 catfp.write(gzip.compress(fp.read(), compresslevel=compressionlevel));
2523 if(compression=="bzip2"):
2524 try:
2525 import bz2;
2526 except ImportError:
2527 return False;
2528 catfp = BytesIO();
2529 if(compressionlevel is None):
2530 compressionlevel = 9;
2531 else:
2532 compressionlevel = int(compressionlevel);
2533 catfp.write(bz2.compress(fp.read(), compresslevel=compressionlevel));
2534 if(compression=="lz4"):
2535 try:
2536 import lz4.frame;
2537 except ImportError:
2538 return False;
2539 catfp = BytesIO();
2540 if(compressionlevel is None):
2541 compressionlevel = 9;
2542 else:
2543 compressionlevel = int(compressionlevel);
2544 catfp.write(lz4.frame.compress(fp.read(), compression_level=compressionlevel));
2545 if(compression=="lzo" or compression=="lzop"):
2546 try:
2547 import lzo;
2548 except ImportError:
2549 return False;
2550 catfp = BytesIO();
2551 if(compressionlevel is None):
2552 compressionlevel = 9;
2553 else:
2554 compressionlevel = int(compressionlevel);
2555 catfp.write(lzo.compress(fp.read(), compresslevel=compressionlevel));
2556 if(compression=="zstd"):
2557 try:
2558 import zstandard;
2559 except ImportError:
2560 return False;
2561 catfp = BytesIO();
2562 if(compressionlevel is None):
2563 compressionlevel = 10;
2564 else:
2565 compressionlevel = int(compressionlevel);
2566 catfp.write(zstandard.compress(fp.read(), level=compressionlevel));
2567 if(compression=="lzma"):
2568 try:
2569 import lzma;
2570 except ImportError:
2571 try:
2572 from backports import lzma
2573 except ImportError:
2574 return False;
2575 catfp = BytesIO();
2576 if(compressionlevel is None):
2577 compressionlevel = 9;
2578 else:
2579 compressionlevel = int(compressionlevel);
2580 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]));
2581 if(compression=="xz"):
2582 try:
2583 import lzma;
2584 except ImportError:
2585 try:
2586 from backports import lzma
2587 except ImportError:
2588 return False;
2589 catfp = BytesIO();
2590 if(compressionlevel is None):
2591 compressionlevel = 9;
2592 else:
2593 compressionlevel = int(compressionlevel);
2594 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]));
2595 if(compression=="auto" or compression is None):
2596 catfp = fp;
2597 catfp.seek(0, 0);
2598 return catfp;
2599
2600 create_alias_function("Compress", __file_format_name__, "", CompressArchiveFile);
2601
2602 def CompressOpenFile(outfile, compressionenable=True, compressionlevel=None):
2603 if(outfile is None):
2604 return False;
2605 fbasename = os.path.splitext(outfile)[0];
2606 fextname = os.path.splitext(outfile)[1];
2607 if(compressionlevel is None and fextname!=".zst"):
2608 compressionlevel = 9;
2609 elif(compressionlevel is None and fextname==".zst"):
2610 compressionlevel = 10;
2611 else:
2612 compressionlevel = int(compressionlevel);
2613 if(sys.version_info[0]==2):
2614 mode = "w";
2615 else:
2616 mode = "wb";
2617 try:
2618 if(fextname not in outextlistwd or not compressionenable):
2619 try:
2620 outfp = open(outfile, "wb", encoding="UTF-8");
2621 except (ValueError, TypeError) as e:
2622 outfp = open(outfile, "wb");
2623 elif(fextname==".gz"):
2624 try:
2625 import gzip;
2626 except ImportError:
2627 return False;
2628 try:
2629 outfp = gzip.open(outfile, mode, compressionlevel, encoding="UTF-8");
2630 except (ValueError, TypeError) as e:
2631 outfp = gzip.open(outfile, mode, compressionlevel);
2632 elif(fextname==".bz2"):
2633 try:
2634 import bz2;
2635 except ImportError:
2636 return False;
2637 try:
2638 outfp = bz2.open(outfile, mode, compressionlevel, encoding="UTF-8");
2639 except (ValueError, TypeError) as e:
2640 outfp = bz2.open(outfile, mode, compressionlevel);
2641 elif(fextname==".zst"):
2642 try:
2643 import zstandard;
2644 except ImportError:
2645 return False;
2646 try:
2647 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel), encoding="UTF-8");
2648 except (ValueError, TypeError) as e:
2649 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel));
2650 elif(fextname==".xz"):
2651 try:
2652 import lzma;
2653 except ImportError:
2654 try:
2655 from backports import lzma
2656 except ImportError:
2657 return False;
2658 try:
2659 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}], encoding="UTF-8");
2660 except (ValueError, TypeError) as e:
2661 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]);
2662 elif(fextname==".lz4"):
2663 try:
2664 import lz4.frame;
2665 except ImportError:
2666 return False;
2667 try:
2668 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel, encoding="UTF-8");
2669 except (ValueError, TypeError) as e:
2670 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel);
2671 elif(fextname==".lzo"):
2672 try:
2673 import lzo;
2674 except ImportError:
2675 return False;
2676 try:
2677 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel, encoding="UTF-8");
2678 except (ValueError, TypeError) as e:
2679 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel);
2680 elif(fextname==".lzma"):
2681 try:
2682 import lzma;
2683 except ImportError:
2684 try:
2685 from backports import lzma
2686 except ImportError:
2687 return False;
2688 try:
2689 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}], encoding="UTF-8");
2690 except (ValueError, TypeError) as e:
2691 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]);
2692 except FileNotFoundError:
2693 return False;
2694 return outfp;
2695
2696 def GetDevMajorMinor(fdev):
2697 retdev = [];
2698 if(hasattr(os, "minor")):
2699 retdev.append(os.minor(fdev));
2700 else:
2701 retdev.append(0);
2702 if(hasattr(os, "major")):
2703 retdev.append(os.major(fdev));
2704 else:
2705 retdev.append(0);
2706 return retdev;
2707
2708 def CheckSumSupport(checkfor, guaranteed=True):
2709 if(guaranteed):
2710 try:
2711 hash_list = sorted(list(hashlib.algorithms_guaranteed));
2712 except AttributeError:
2713 hash_list = sorted(list(hashlib.algorithms));
2714 else:
2715 try:
2716 hash_list = sorted(list(hashlib.algorithms_available));
2717 except AttributeError:
2718 hash_list = sorted(list(hashlib.algorithms));
2719 checklistout = sorted(hash_list + ['adler32', 'crc16', 'crc16_ansi', 'crc16_ibm', 'crc16_ccitt', 'crc32', 'crc64', 'crc64_ecma', 'crc64_iso', 'none']);
2720 if(checkfor in checklistout):
2721 return True;
2722 else:
2723 return False;
2724
2725 def CheckSumSupportAlt(checkfor, guaranteed=True):
2726 if(guaranteed):
2727 try:
2728 hash_list = sorted(list(hashlib.algorithms_guaranteed));
2729 except AttributeError:
2730 hash_list = sorted(list(hashlib.algorithms));
2731 else:
2732 try:
2733 hash_list = sorted(list(hashlib.algorithms_available));
2734 except AttributeError:
2735 hash_list = sorted(list(hashlib.algorithms));
2736 checklistout = hash_list;
2737 if(checkfor in checklistout):
2738 return True;
2739 else:
2740 return False;
2741
2742 def PackArchiveFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2743 formatspecs = FormatSpecsListToDict(formatspecs);
2744 advancedlist = formatspecs['use_advanced_list'];
2745 altinode = formatspecs['use_alt_inode'];
2746 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2747 outfile = RemoveWindowsPath(outfile);
2748 checksumtype = checksumtype.lower();
2749 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
2750 checksumtype="crc32";
2751 if(checksumtype=="none"):
2752 checksumtype = "";
2753 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
2754 compression = "auto";
2755 if(compression not in compressionlist and compression is None):
2756 compression = "auto";
2757 if(verbose):
2758 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2759 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2760 if(os.path.exists(outfile)):
2761 try:
2762 os.unlink(outfile);
2763 except OSError as e:
2764 pass;
2765 if(outfile=="-"):
2766 verbose = False;
2767 catfp = BytesIO();
2768 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2769 catfp = outfile;
2770 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2771 catfp = BytesIO();
2772 else:
2773 fbasename = os.path.splitext(outfile)[0];
2774 fextname = os.path.splitext(outfile)[1];
2775 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
2776 catver = formatspecs['format_ver'];
2777 fileheaderver = str(int(catver.replace(".", "")));
2778 infilelist = [];
2779 if(infiles=="-"):
2780 for line in sys.stdin:
2781 infilelist.append(line.strip());
2782 infilelist = list(filter(None, infilelist));
2783 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
2784 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
2785 return False;
2786 with UncompressFile(infiles, formatspecs, "r") as finfile:
2787 for line in finfile:
2788 infilelist.append(line.strip());
2789 infilelist = list(filter(None, infilelist));
2790 else:
2791 if(isinstance(infiles, (list, tuple, ))):
2792 infilelist = list(filter(None, infiles));
2793 elif(isinstance(infiles, (str, ))):
2794 infilelist = list(filter(None, [infiles]));
2795 if(advancedlist):
2796 GetDirList = ListDirAdvanced(infilelist, followlink, False);
2797 else:
2798 GetDirList = ListDir(infilelist, followlink, False);
2799 if(not GetDirList):
2800 return False;
2801 curinode = 0;
2802 curfid = 0;
2803 inodelist = [];
2804 inodetofile = {};
2805 filetoinode = {};
2806 inodetocatinode = {};
2807 numfiles = int(len(GetDirList));
2808 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
2809 for curfname in GetDirList:
2810 if(re.findall("^[.|/]", curfname)):
2811 fname = curfname;
2812 else:
2813 fname = "./"+curfname;
2814 if(verbose):
2815 VerbosePrintOut(fname);
2816 if(not followlink or followlink is None):
2817 fstatinfo = os.lstat(fname);
2818 else:
2819 fstatinfo = os.stat(fname);
2820 fpremode = fstatinfo.st_mode;
2821 finode = fstatinfo.st_ino;
2822 flinkcount = fstatinfo.st_nlink;
2823 ftype = 0;
2824 if(stat.S_ISREG(fpremode)):
2825 ftype = 0;
2826 elif(stat.S_ISLNK(fpremode)):
2827 ftype = 2;
2828 elif(stat.S_ISCHR(fpremode)):
2829 ftype = 3;
2830 elif(stat.S_ISBLK(fpremode)):
2831 ftype = 4;
2832 elif(stat.S_ISDIR(fpremode)):
2833 ftype = 5;
2834 elif(stat.S_ISFIFO(fpremode)):
2835 ftype = 6;
2836 elif(stat.S_ISSOCK(fpremode)):
2837 ftype = 8;
2838 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
2839 ftype = 9;
2840 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
2841 ftype = 10;
2842 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
2843 ftype = 111;
2844 else:
2845 ftype = 0;
2846 flinkname = "";
2847 fcurfid = format(int(curfid), 'x').lower();
2848 if(not followlink and finode!=0):
2849 if(ftype!=1):
2850 if(finode in inodelist):
2851 ftype = 1;
2852 flinkname = inodetofile[finode];
2853 if(altinode):
2854 fcurinode = format(int(finode), 'x').lower();
2855 else:
2856 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
2857 if(finode not in inodelist):
2858 inodelist.append(finode);
2859 inodetofile.update({finode: fname});
2860 inodetocatinode.update({finode: curinode});
2861 if(altinode):
2862 fcurinode = format(int(finode), 'x').lower();
2863 else:
2864 fcurinode = format(int(curinode), 'x').lower();
2865 curinode = curinode + 1;
2866 else:
2867 fcurinode = format(int(curinode), 'x').lower();
2868 curinode = curinode + 1;
2869 curfid = curfid + 1;
2870 if(ftype==2):
2871 flinkname = os.readlink(fname);
2872 fdev = fstatinfo.st_dev;
2873 getfdev = GetDevMajorMinor(fdev);
2874 fdev_minor = getfdev[0];
2875 fdev_major = getfdev[1];
2876 frdev = fstatinfo.st_dev;
2877 if(hasattr(fstatinfo, "st_rdev")):
2878 frdev = fstatinfo.st_rdev;
2879 else:
2880 frdev = fstatinfo.st_dev;
2881 getfrdev = GetDevMajorMinor(frdev);
2882 frdev_minor = getfrdev[0];
2883 frdev_major = getfrdev[1];
2884 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
2885 fsize = format(int("0"), 'x').lower();
2886 elif(ftype==0 or ftype==7):
2887 fsize = format(int(fstatinfo.st_size), 'x').lower();
2888 else:
2889 fsize = format(int(fstatinfo.st_size)).lower();
2890 fatime = format(int(fstatinfo.st_atime), 'x').lower();
2891 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
2892 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
2893 if(hasattr(fstatinfo, "st_birthtime")):
2894 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
2895 else:
2896 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
2897 fmode = format(int(fstatinfo.st_mode), 'x').lower();
2898 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
2899 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
2900 fuid = format(int(fstatinfo.st_uid), 'x').lower();
2901 fgid = format(int(fstatinfo.st_gid), 'x').lower();
2902 funame = "";
2903 try:
2904 import pwd;
2905 try:
2906 userinfo = pwd.getpwuid(fstatinfo.st_uid);
2907 funame = userinfo.pw_name;
2908 except KeyError:
2909 funame = "";
2910 except ImportError:
2911 funame = "";
2912 fgname = "";
2913 try:
2914 import grp;
2915 try:
2916 groupinfo = grp.getgrgid(fstatinfo.st_gid);
2917 fgname = groupinfo.gr_name;
2918 except KeyError:
2919 fgname = "";
2920 except ImportError:
2921 fgname = "";
2922 fdev_minor = format(int(fdev_minor), 'x').lower();
2923 fdev_major = format(int(fdev_major), 'x').lower();
2924 frdev_minor = format(int(frdev_minor), 'x').lower();
2925 frdev_major = format(int(frdev_major), 'x').lower();
2926 finode = format(int(finode), 'x').lower();
2927 flinkcount = format(int(flinkcount), 'x').lower();
2928 if(hasattr(fstatinfo, "st_file_attributes")):
2929 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
2930 else:
2931 fwinattributes = format(int(0), 'x').lower();
2932 fcompression = "";
2933 fcsize = format(int(0), 'x').lower();
2934 fcontents = BytesIO();
2935 if(ftype==0 or ftype==7):
2936 with open(fname, "rb") as fpc:
2937 shutil.copyfileobj(fpc, fcontents);
2938 if(not compresswholefile):
2939 fcontents.seek(0, 2);
2940 ucfsize = fcontents.tell();
2941 fcontents.seek(0, 0);
2942 if(compression=="auto"):
2943 ilsize = len(compressionlistalt);
2944 ilmin = 0;
2945 ilcsize = [];
2946 while(ilmin < ilsize):
2947 cfcontents = BytesIO();
2948 shutil.copyfileobj(fcontents, cfcontents);
2949 fcontents.seek(0, 0);
2950 cfcontents.seek(0, 0);
2951 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
2952 if(cfcontents):
2953 cfcontents.seek(0, 2);
2954 ilcsize.append(cfcontents.tell());
2955 cfcontents.close();
2956 else:
2957 try:
2958 ilcsize.append(sys.maxint);
2959 except AttributeError:
2960 ilcsize.append(sys.maxsize);
2961 ilmin = ilmin + 1;
2962 ilcmin = ilcsize.index(min(ilcsize));
2963 compression = compressionlistalt[ilcmin];
2964 fcontents.seek(0, 0);
2965 cfcontents = BytesIO();
2966 shutil.copyfileobj(fcontents, cfcontents);
2967 cfcontents.seek(0, 0);
2968 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
2969 cfcontents.seek(0, 2);
2970 cfsize = cfcontents.tell();
2971 if(ucfsize > cfsize):
2972 fcsize = format(int(cfsize), 'x').lower();
2973 fcompression = compression;
2974 fcontents.close();
2975 fcontents = cfcontents;
2976 if(fcompression=="none"):
2977 fcompression = "";
2978 if(followlink and (ftype==1 or ftype==2)):
2979 flstatinfo = os.stat(flinkname);
2980 with open(flinkname, "rb") as fpc:
2981 shutil.copyfileobj(fpc, fcontents);
2982 if(not compresswholefile):
2983 fcontents.seek(0, 2);
2984 ucfsize = fcontents.tell();
2985 fcontents.seek(0, 0);
2986 if(compression=="auto"):
2987 ilsize = len(compressionlistalt);
2988 ilmin = 0;
2989 ilcsize = [];
2990 while(ilmin < ilsize):
2991 cfcontents = BytesIO();
2992 shutil.copyfileobj(fcontents, cfcontents);
2993 fcontents.seek(0, 0);
2994 cfcontents.seek(0, 0);
2995 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
2996 if(cfcontents):
2997 cfcontents.seek(0, 2);
2998 ilcsize.append(cfcontents.tell());
2999 cfcontents.close();
3000 else:
3001 try:
3002 ilcsize.append(sys.maxint);
3003 except AttributeError:
3004 ilcsize.append(sys.maxsize);
3005 ilmin = ilmin + 1;
3006 ilcmin = ilcsize.index(min(ilcsize));
3007 compression = compressionlistalt[ilcmin];
3008 fcontents.seek(0, 0);
3009 cfcontents = BytesIO();
3010 shutil.copyfileobj(fcontents, cfcontents);
3011 cfcontents.seek(0, 0);
3012 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3013 cfcontents.seek(0, 2);
3014 cfsize = cfcontents.tell();
3015 if(ucfsize > cfsize):
3016 fcsize = format(int(cfsize), 'x').lower();
3017 fcompression = compression;
3018 fcontents.close();
3019 fcontents = cfcontents;
3020 fcontents.seek(0, 0);
3021 ftypehex = format(ftype, 'x').lower();
3022 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3023 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3024 fcontents.close();
3025 if(numfiles>0):
3026 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3027 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3028 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3029 try:
3030 catfp.flush();
3031 os.fsync(catfp.fileno());
3032 except io.UnsupportedOperation:
3033 pass;
3034 except AttributeError:
3035 pass;
3036 except OSError as e:
3037 pass;
3038 if(outfile=="-"):
3039 catfp.seek(0, 0);
3040 if(hasattr(sys.stdout, "buffer")):
3041 shutil.copyfileobj(catfp, sys.stdout.buffer);
3042 else:
3043 shutil.copyfileobj(catfp, sys.stdout);
3044 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3045 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3046 catfp.seek(0, 0);
3047 upload_file_to_internet_file(catfp, outfile);
3048 if(returnfp):
3049 catfp.seek(0, 0);
3050 return catfp;
3051 else:
3052 catfp.close();
3053 return True;
3054
3055 create_alias_function("Pack", __file_format_name__, "", PackArchiveFile);
3056
3057 if(hasattr(shutil, "register_archive_format")):
3058 def PackArchiveFileFunc(archive_name, source_dir, **kwargs):
3059 return PackArchiveFile(source_dir, archive_name, False, "auto", True, None, False, "crc32", [], __file_format_dict__['format_delimiter'], False, False);
3060 create_alias_function("Pack", __file_format_name__, "Func", PackArchiveFileFunc);
3061
3062 def PackArchiveFileFromDirList(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3063 formatspecs = FormatSpecsListToDict(formatspecs);
3064 return PackArchiveFile(infiles, outfile, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, returnfp);
3065
3066 create_alias_function("Pack", __file_format_name__, "FromDirList", PackArchiveFileFromDirList);
3067
3068 def PackArchiveFileFromTarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3069 formatspecs = FormatSpecsListToDict(formatspecs);
3070 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3071 outfile = RemoveWindowsPath(outfile);
3072 checksumtype = checksumtype.lower();
3073 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3074 checksumtype="crc32";
3075 if(checksumtype=="none"):
3076 checksumtype = "";
3077 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3078 compression = "auto";
3079 if(compression not in compressionlist and compression is None):
3080 compression = "auto";
3081 if(verbose):
3082 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3083 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3084 if(os.path.exists(outfile)):
3085 try:
3086 os.unlink(outfile);
3087 except OSError as e:
3088 pass;
3089 if(outfile=="-"):
3090 verbose = False;
3091 catfp = BytesIO();
3092 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3093 catfp = outfile;
3094 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3095 catfp = BytesIO();
3096 else:
3097 fbasename = os.path.splitext(outfile)[0];
3098 fextname = os.path.splitext(outfile)[1];
3099 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3100 catver = formatspecs['format_ver'];
3101 fileheaderver = str(int(catver.replace(".", "")));
3102 curinode = 0;
3103 curfid = 0;
3104 inodelist = [];
3105 inodetofile = {};
3106 filetoinode = {};
3107 inodetocatinode = {};
3108 if(infile=="-"):
3109 infile = BytesIO();
3110 if(hasattr(sys.stdin, "buffer")):
3111 shutil.copyfileobj(sys.stdin.buffer, infile);
3112 else:
3113 shutil.copyfileobj(sys.stdin, infile);
3114 infile.seek(0, 0);
3115 if(not infile):
3116 return False;
3117 infile.seek(0, 0);
3118 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3119 infile = download_file_from_internet_file(infile);
3120 infile.seek(0, 0);
3121 if(not infile):
3122 return False;
3123 infile.seek(0, 0);
3124 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
3125 return False;
3126 elif(os.path.exists(infile) and os.path.isfile(infile)):
3127 try:
3128 if(not tarfile.TarFileCheck(infile)):
3129 return False;
3130 except AttributeError:
3131 if(not TarFileCheck(infile)):
3132 return False;
3133 else:
3134 return False;
3135 try:
3136 if(hasattr(infile, "read") or hasattr(infile, "write")):
3137 tarfp = tarfile.open(fileobj=infile, mode="r");
3138 else:
3139 tarfp = tarfile.open(infile, "r");
3140 except FileNotFoundError:
3141 return False;
3142 numfiles = int(len(tarfp.getmembers()));
3143 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3144 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
3145 if(re.findall("^[.|/]", member.name)):
3146 fname = member.name;
3147 else:
3148 fname = "./"+member.name;
3149 if(verbose):
3150 VerbosePrintOut(fname);
3151 fpremode = member.mode;
3152 ffullmode = member.mode;
3153 flinkcount = 0;
3154 ftype = 0;
3155 if(member.isreg()):
3156 ffullmode = member.mode + stat.S_IFREG;
3157 ftype = 0;
3158 elif(member.isdev()):
3159 ffullmode = member.mode;
3160 ftype = 7;
3161 elif(member.islnk()):
3162 ffullmode = member.mode + stat.S_IFREG;
3163 ftype = 1;
3164 elif(member.issym()):
3165 ffullmode = member.mode + stat.S_IFLNK;
3166 ftype = 2;
3167 elif(member.ischr()):
3168 ffullmode = member.mode + stat.S_IFCHR;
3169 ftype = 3;
3170 elif(member.isblk()):
3171 ffullmode = member.mode + stat.S_IFBLK;
3172 ftype = 4;
3173 elif(member.isdir()):
3174 ffullmode = member.mode + stat.S_IFDIR;
3175 ftype = 5;
3176 elif(member.isfifo()):
3177 ffullmode = member.mode + stat.S_IFIFO;
3178 ftype = 6;
3179 elif(member.issparse()):
3180 ffullmode = member.mode;
3181 ftype = 12;
3182 else:
3183 ffullmode = member.mode;
3184 ftype = 0;
3185 flinkname = "";
3186 fcurfid = format(int(curfid), 'x').lower();
3187 fcurinode = format(int(curfid), 'x').lower();
3188 curfid = curfid + 1;
3189 if(ftype==2):
3190 flinkname = member.linkname;
3191 fdev_minor = format(int(member.devminor), 'x').lower();
3192 fdev_major = format(int(member.devmajor), 'x').lower();
3193 frdev_minor = format(int(member.devminor), 'x').lower();
3194 frdev_major = format(int(member.devmajor), 'x').lower();
3195 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
3196 fsize = format(int("0"), 'x').lower();
3197 elif(ftype==0 or ftype==7):
3198 fsize = format(int(member.size), 'x').lower();
3199 else:
3200 fsize = format(int(member.size), 'x').lower();
3201 fatime = format(int(member.mtime), 'x').lower();
3202 fmtime = format(int(member.mtime), 'x').lower();
3203 fctime = format(int(member.mtime), 'x').lower();
3204 fbtime = format(int(member.mtime), 'x').lower();
3205 fmode = format(int(ffullmode), 'x').lower();
3206 fchmode = format(int(stat.S_IMODE(ffullmode)), 'x').lower();
3207 ftypemod = format(int(stat.S_IFMT(ffullmode)), 'x').lower();
3208 fuid = format(int(member.uid), 'x').lower();
3209 fgid = format(int(member.gid), 'x').lower();
3210 funame = member.uname;
3211 fgname = member.gname;
3212 flinkcount = format(int(flinkcount), 'x').lower();
3213 fwinattributes = format(int(0), 'x').lower();
3214 fcompression = "";
3215 fcsize = format(int(0), 'x').lower();
3216 fcontents = BytesIO();
3217 if(ftype==0 or ftype==7):
3218 with tarfp.extractfile(member) as fpc:
3219 shutil.copyfileobj(fpc, fcontents);
3220 if(not compresswholefile):
3221 fcontents.seek(0, 2);
3222 ucfsize = fcontents.tell();
3223 fcontents.seek(0, 0);
3224 if(compression=="auto"):
3225 ilsize = len(compressionlistalt);
3226 ilmin = 0;
3227 ilcsize = [];
3228 while(ilmin < ilsize):
3229 cfcontents = BytesIO();
3230 shutil.copyfileobj(fcontents, cfcontents);
3231 fcontents.seek(0, 0);
3232 cfcontents.seek(0, 0);
3233 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3234 if(cfcontents):
3235 cfcontents.seek(0, 2);
3236 ilcsize.append(cfcontents.tell());
3237 cfcontents.close();
3238 else:
3239 try:
3240 ilcsize.append(sys.maxint);
3241 except AttributeError:
3242 ilcsize.append(sys.maxsize);
3243 ilmin = ilmin + 1;
3244 ilcmin = ilcsize.index(min(ilcsize));
3245 compression = compressionlistalt[ilcmin];
3246 fcontents.seek(0, 0);
3247 cfcontents = BytesIO();
3248 shutil.copyfileobj(fcontents, cfcontents);
3249 cfcontents.seek(0, 0);
3250 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3251 cfcontents.seek(0, 2);
3252 cfsize = cfcontents.tell();
3253 if(ucfsize > cfsize):
3254 fcsize = format(int(cfsize), 'x').lower();
3255 fcompression = compression;
3256 fcontents.close();
3257 fcontents = cfcontents;
3258 if(fcompression=="none"):
3259 fcompression = "";
3260 fcontents.seek(0, 0);
3261 ftypehex = format(ftype, 'x').lower();
3262 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3263 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3264 fcontents.close();
3265 if(numfiles>0):
3266 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3267 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3268 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3269 try:
3270 catfp.flush();
3271 os.fsync(catfp.fileno());
3272 except io.UnsupportedOperation:
3273 pass;
3274 except AttributeError:
3275 pass;
3276 except OSError as e:
3277 pass;
3278 if(outfile=="-"):
3279 catfp.seek(0, 0);
3280 if(hasattr(sys.stdout, "buffer")):
3281 shutil.copyfileobj(catfp, sys.stdout.buffer);
3282 else:
3283 shutil.copyfileobj(catfp, sys.stdout);
3284 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3285 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3286 catfp.seek(0, 0);
3287 upload_file_to_internet_file(catfp, outfile);
3288 if(returnfp):
3289 catfp.seek(0, 0);
3290 return catfp;
3291 else:
3292 catfp.close();
3293 return True;
3294
3295 create_alias_function("Pack", __file_format_name__, "FromTarFile", PackArchiveFileFromTarFile);
3296
3297 def PackArchiveFileFromZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3298 formatspecs = FormatSpecsListToDict(formatspecs);
3299 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3300 outfile = RemoveWindowsPath(outfile);
3301 checksumtype = checksumtype.lower();
3302 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3303 checksumtype="crc32";
3304 if(checksumtype=="none"):
3305 checksumtype = "";
3306 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3307 compression = "auto";
3308 if(compression not in compressionlist and compression is None):
3309 compression = "auto";
3310 if(verbose):
3311 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3312 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3313 if(os.path.exists(outfile)):
3314 try:
3315 os.unlink(outfile);
3316 except OSError as e:
3317 pass;
3318 if(outfile=="-"):
3319 verbose = False;
3320 catfp = BytesIO();
3321 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3322 catfp = outfile;
3323 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3324 catfp = BytesIO();
3325 else:
3326 fbasename = os.path.splitext(outfile)[0];
3327 fextname = os.path.splitext(outfile)[1];
3328 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3329 catver = formatspecs['format_ver'];
3330 fileheaderver = str(int(catver.replace(".", "")));
3331 curinode = 0;
3332 curfid = 0;
3333 inodelist = [];
3334 inodetofile = {};
3335 filetoinode = {};
3336 inodetocatinode = {};
3337 if(infile=="-"):
3338 infile = BytesIO();
3339 if(hasattr(sys.stdin, "buffer")):
3340 shutil.copyfileobj(sys.stdin.buffer, infile);
3341 else:
3342 shutil.copyfileobj(sys.stdin, infile);
3343 infile.seek(0, 0);
3344 if(not infile):
3345 return False;
3346 infile.seek(0, 0);
3347 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3348 infile = download_file_from_internet_file(infile);
3349 infile.seek(0, 0);
3350 if(not infile):
3351 return False;
3352 infile.seek(0, 0);
3353 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
3354 return False;
3355 else:
3356 return False;
3357 if(not zipfile.is_zipfile(infile)):
3358 return False;
3359 try:
3360 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
3361 except FileNotFoundError:
3362 return False;
3363 ziptest = zipfp.testzip();
3364 if(ziptest):
3365 VerbosePrintOut("Bad file found!");
3366 numfiles = int(len(zipfp.infolist()));
3367 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3368 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
3369 if(re.findall("^[.|/]", member.filename)):
3370 fname = member.filename;
3371 else:
3372 fname = "./"+member.filename;
3373 zipinfo = zipfp.getinfo(member.filename);
3374 if(verbose):
3375 VerbosePrintOut(fname);
3376 if(not member.is_dir()):
3377 fpremode = int(stat.S_IFREG + 438);
3378 elif(member.is_dir()):
3379 fpremode = int(stat.S_IFDIR + 511);
3380 flinkcount = 0;
3381 ftype = 0;
3382 if(not member.is_dir()):
3383 ftype = 0;
3384 elif(member.is_dir()):
3385 ftype = 5;
3386 flinkname = "";
3387 fcurfid = format(int(curfid), 'x').lower();
3388 fcurinode = format(int(curfid), 'x').lower();
3389 curfid = curfid + 1;
3390 fdev_minor = format(int(0), 'x').lower();
3391 fdev_major = format(int(0), 'x').lower();
3392 frdev_minor = format(int(0), 'x').lower();
3393 frdev_major = format(int(0), 'x').lower();
3394 if(ftype==5):
3395 fsize = format(int("0"), 'x').lower();
3396 elif(ftype==0):
3397 fsize = format(int(member.file_size), 'x').lower();
3398 else:
3399 fsize = format(int(member.file_size), 'x').lower();
3400 fatime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3401 fmtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3402 fctime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3403 fbtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3404 if(zipinfo.create_system==0 or zipinfo.create_system==10):
3405 fwinattributes = format(int(zipinfo.external_attr), 'x').lower();
3406 if(not member.is_dir()):
3407 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3408 fchmode = stat.S_IMODE(fmode);
3409 ftypemod = stat.S_IFMT(fmode);
3410 elif(member.is_dir()):
3411 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3412 fchmode = stat.S_IMODE(fmode);
3413 ftypemod = stat.S_IFMT(fmode);
3414 elif(zipinfo.create_system==3):
3415 fwinattributes = format(int(0), 'x').lower();
3416 fmode = format(int(zipinfo.external_attr), 'x').lower();
3417 fchmode = stat.S_IMODE(fmode);
3418 ftypemod = stat.S_IFMT(fmode);
3419 else:
3420 fwinattributes = format(int(0), 'x').lower();
3421 if(not member.is_dir()):
3422 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3423 fchmode = stat.S_IMODE(fmode);
3424 ftypemod = stat.S_IFMT(fmode);
3425 elif(member.is_dir()):
3426 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3427 fchmode = stat.S_IMODE(fmode);
3428 ftypemod = stat.S_IFMT(fmode);
3429 fcompression = "";
3430 fcsize = format(int(0), 'x').lower();
3431 try:
3432 fuid = format(int(os.getuid()), 'x').lower();
3433 except AttributeError:
3434 fuid = format(int(0), 'x').lower();
3435 except KeyError:
3436 fuid = format(int(0), 'x').lower();
3437 try:
3438 fgid = format(int(os.getgid()), 'x').lower();
3439 except AttributeError:
3440 fgid = format(int(0), 'x').lower();
3441 except KeyError:
3442 fgid = format(int(0), 'x').lower();
3443 try:
3444 import pwd;
3445 try:
3446 userinfo = pwd.getpwuid(os.getuid());
3447 funame = userinfo.pw_name;
3448 except KeyError:
3449 funame = "";
3450 except AttributeError:
3451 funame = "";
3452 except ImportError:
3453 funame = "";
3454 fgname = "";
3455 try:
3456 import grp;
3457 try:
3458 groupinfo = grp.getgrgid(os.getgid());
3459 fgname = groupinfo.gr_name;
3460 except KeyError:
3461 fgname = "";
3462 except AttributeError:
3463 fgname = "";
3464 except ImportError:
3465 fgname = "";
3466 fcontents = BytesIO();
3467 if(ftype==0):
3468 fcontents.write(zipfp.read(member.filename));
3469 if(not compresswholefile):
3470 fcontents.seek(0, 2);
3471 ucfsize = fcontents.tell();
3472 fcontents.seek(0, 0);
3473 if(compression=="auto"):
3474 ilsize = len(compressionlistalt);
3475 ilmin = 0;
3476 ilcsize = [];
3477 while(ilmin < ilsize):
3478 cfcontents = BytesIO();
3479 shutil.copyfileobj(fcontents, cfcontents);
3480 fcontents.seek(0, 0);
3481 cfcontents.seek(0, 0);
3482 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3483 cfcontents.seek(0, 2);
3484 ilcsize.append(cfcontents.tell());
3485 cfcontents.close();
3486 ilmin = ilmin + 1;
3487 ilcmin = ilcsize.index(min(ilcsize));
3488 compression = compressionlistalt[ilcmin];
3489 fcontents.seek(0, 0);
3490 cfcontents = BytesIO();
3491 shutil.copyfileobj(fcontents, cfcontents);
3492 cfcontents.seek(0, 0);
3493 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3494 cfcontents.seek(0, 2);
3495 cfsize = cfcontents.tell();
3496 if(ucfsize > cfsize):
3497 fcsize = format(int(cfsize), 'x').lower();
3498 fcompression = compression;
3499 fcontents.close();
3500 fcontents = cfcontents;
3501 if(fcompression=="none"):
3502 fcompression = "";
3503 fcontents.seek(0, 0);
3504 ftypehex = format(ftype, 'x').lower();
3505 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3506 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3507 fcontents.close();
3508 if(numfiles>0):
3509 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3510 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3511 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3512 try:
3513 catfp.flush();
3514 os.fsync(catfp.fileno());
3515 except io.UnsupportedOperation:
3516 pass;
3517 except AttributeError:
3518 pass;
3519 except OSError as e:
3520 pass;
3521 if(outfile=="-"):
3522 catfp.seek(0, 0);
3523 if(hasattr(sys.stdout, "buffer")):
3524 shutil.copyfileobj(catfp, sys.stdout.buffer);
3525 else:
3526 shutil.copyfileobj(catfp, sys.stdout);
3527 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3528 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3529 catfp.seek(0, 0);
3530 upload_file_to_internet_file(catfp, outfile);
3531 if(returnfp):
3532 catfp.seek(0, 0);
3533 return catfp;
3534 else:
3535 catfp.close();
3536 return True;
3537
3538 create_alias_function("Pack", __file_format_name__, "FromZipFile", PackArchiveFileFromZipFile);
3539
3540 if(not rarfile_support):
3541 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3542 return False
3543
3544 if(rarfile_support):
3545 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3546 formatspecs = FormatSpecsListToDict(formatspecs);
3547 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3548 outfile = RemoveWindowsPath(outfile);
3549 checksumtype = checksumtype.lower();
3550 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3551 checksumtype="crc32";
3552 if(checksumtype=="none"):
3553 checksumtype = "";
3554 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3555 compression = "auto";
3556 if(compression not in compressionlist and compression is None):
3557 compression = "auto";
3558 if(verbose):
3559 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3560 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3561 if(os.path.exists(outfile)):
3562 try:
3563 os.unlink(outfile);
3564 except OSError as e:
3565 pass;
3566 if(outfile=="-"):
3567 verbose = False;
3568 catfp = BytesIO();
3569 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3570 catfp = outfile;
3571 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3572 catfp = BytesIO();
3573 else:
3574 fbasename = os.path.splitext(outfile)[0];
3575 fextname = os.path.splitext(outfile)[1];
3576 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3577 catver = formatspecs['format_ver'];
3578 fileheaderver = str(int(catver.replace(".", "")));
3579 curinode = 0;
3580 curfid = 0;
3581 inodelist = [];
3582 inodetofile = {};
3583 filetoinode = {};
3584 inodetocatinode = {};
3585 if(not os.path.exists(infile) or not os.path.isfile(infile)):
3586 return False;
3587 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
3588 return False;
3589 rarfp = rarfile.RarFile(infile, "r");
3590 rartest = rarfp.testrar();
3591 if(rartest):
3592 VerbosePrintOut("Bad file found!");
3593 numfiles = int(len(rarfp.infolist()));
3594 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3595 try:
3596 catfp.flush();
3597 os.fsync(catfp.fileno());
3598 except io.UnsupportedOperation:
3599 pass;
3600 except AttributeError:
3601 pass;
3602 except OSError as e:
3603 pass;
3604 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
3605 is_unix = False;
3606 is_windows = False;
3607 if(member.host_os==rarfile.RAR_OS_UNIX):
3608 is_windows = False;
3609 try:
3610 member.external_attr
3611 is_unix = True;
3612 except AttributeError:
3613 is_unix = False;
3614 elif(member.host_os==rarfile.RAR_OS_WIN32):
3615 is_unix = False;
3616 try:
3617 member.external_attr
3618 is_windows = True;
3619 except AttributeError:
3620 is_windows = False;
3621 else:
3622 is_unix = False;
3623 is_windows = False;
3624 if(re.findall("^[.|/]", member.filename)):
3625 fname = member.filename;
3626 else:
3627 fname = "./"+member.filename;
3628 rarinfo = rarfp.getinfo(member.filename);
3629 if(verbose):
3630 VerbosePrintOut(fname);
3631 if(is_unix and member.external_attr !=0):
3632 fpremode = int(member.external_attr);
3633 elif(member.is_file()):
3634 fpremode = int(stat.S_IFREG + 438);
3635 elif(member.is_symlink()):
3636 fpremode = int(stat.S_IFLNK + 438);
3637 elif(member.is_dir()):
3638 fpremode = int(stat.S_IFDIR + 511);
3639 if(is_windows and member.external_attr !=0):
3640 fwinattributes = format(int(member.external_attr), 'x').lower();
3641 else:
3642 fwinattributes = format(int(0), 'x').lower();
3643 fcompression = "";
3644 fcsize = format(int(0), 'x').lower();
3645 flinkcount = 0;
3646 ftype = 0;
3647 if(member.is_file()):
3648 ftype = 0;
3649 elif(member.is_symlink()):
3650 ftype = 2;
3651 elif(member.is_dir()):
3652 ftype = 5;
3653 flinkname = "";
3654 if(ftype==2):
3655 flinkname = rarfp.read(member.filename).decode("UTF-8");
3656 fcurfid = format(int(curfid), 'x').lower();
3657 fcurinode = format(int(curfid), 'x').lower();
3658 curfid = curfid + 1;
3659 fdev_minor = format(int(0), 'x').lower();
3660 fdev_major = format(int(0), 'x').lower();
3661 frdev_minor = format(int(0), 'x').lower();
3662 frdev_major = format(int(0), 'x').lower();
3663 if(ftype==5):
3664 fsize = format(int("0"), 'x').lower();
3665 elif(ftype==0):
3666 fsize = format(int(member.file_size), 'x').lower();
3667 else:
3668 fsize = format(int(member.file_size), 'x').lower();
3669 try:
3670 if(member.atime):
3671 fatime = format(int(member.atime.timestamp()), 'x').lower();
3672 else:
3673 fatime = format(int(member.mtime.timestamp()), 'x').lower();
3674 except AttributeError:
3675 fatime = format(int(member.mtime.timestamp()), 'x').lower();
3676 fmtime = format(int(member.mtime.timestamp()), 'x').lower();
3677 try:
3678 if(member.ctime):
3679 fctime = format(int(member.ctime.timestamp()), 'x').lower();
3680 else:
3681 fctime = format(int(member.mtime.timestamp()), 'x').lower();
3682 except AttributeError:
3683 fctime = format(int(member.mtime.timestamp()), 'x').lower();
3684 fbtime = format(int(member.mtime.timestamp()), 'x').lower();
3685 if(is_unix and member.external_attr !=0):
3686 fmode = format(int(member.external_attr), 'x').lower();
3687 fchmode = format(int(stat.S_IMODE(member.external_attr)), 'x').lower();
3688 ftypemod = format(int(stat.S_IFMT(member.external_attr)), 'x').lower();
3689 elif(member.is_file()):
3690 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3691 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3692 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3693 elif(member.is_symlink()):
3694 fmode = format(int(stat.S_IFLNK + 438), 'x').lower();
3695 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3696 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3697 elif(member.is_dir()):
3698 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3699 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
3700 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
3701 try:
3702 fuid = format(int(os.getuid()), 'x').lower();
3703 except AttributeError:
3704 fuid = format(int(0), 'x').lower();
3705 except KeyError:
3706 fuid = format(int(0), 'x').lower();
3707 try:
3708 fgid = format(int(os.getgid()), 'x').lower();
3709 except AttributeError:
3710 fgid = format(int(0), 'x').lower();
3711 except KeyError:
3712 fgid = format(int(0), 'x').lower();
3713 try:
3714 import pwd;
3715 try:
3716 userinfo = pwd.getpwuid(os.getuid());
3717 funame = userinfo.pw_name;
3718 except KeyError:
3719 funame = "";
3720 except AttributeError:
3721 funame = "";
3722 except ImportError:
3723 funame = "";
3724 fgname = "";
3725 try:
3726 import grp;
3727 try:
3728 groupinfo = grp.getgrgid(os.getgid());
3729 fgname = groupinfo.gr_name;
3730 except KeyError:
3731 fgname = "";
3732 except AttributeError:
3733 fgname = "";
3734 except ImportError:
3735 fgname = "";
3736 fcontents = BytesIO();
3737 if(ftype==0):
3738 fcontents.write(rarfp.read(member.filename));
3739 if(not compresswholefile):
3740 fcontents.seek(0, 2);
3741 ucfsize = fcontents.tell();
3742 fcontents.seek(0, 0);
3743 if(compression=="auto"):
3744 ilsize = len(compressionlistalt);
3745 ilmin = 0;
3746 ilcsize = [];
3747 while(ilmin < ilsize):
3748 cfcontents = BytesIO();
3749 shutil.copyfileobj(fcontents, cfcontents);
3750 fcontents.seek(0, 0);
3751 cfcontents.seek(0, 0);
3752 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3753 if(cfcontents):
3754 cfcontents.seek(0, 2);
3755 ilcsize.append(cfcontents.tell());
3756 cfcontents.close();
3757 else:
3758 try:
3759 ilcsize.append(sys.maxint);
3760 except AttributeError:
3761 ilcsize.append(sys.maxsize);
3762 ilmin = ilmin + 1;
3763 ilcmin = ilcsize.index(min(ilcsize));
3764 compression = compressionlistalt[ilcmin];
3765 fcontents.seek(0, 0);
3766 cfcontents = BytesIO();
3767 shutil.copyfileobj(fcontents, cfcontents);
3768 cfcontents.seek(0, 0);
3769 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3770 cfcontents.seek(0, 2);
3771 cfsize = cfcontents.tell();
3772 if(ucfsize > cfsize):
3773 fcsize = format(int(cfsize), 'x').lower();
3774 fcompression = compression;
3775 fcontents.close();
3776 fcontents = cfcontents;
3777 if(fcompression=="none"):
3778 fcompression = "";
3779 fcontents.seek(0, 0);
3780 ftypehex = format(ftype, 'x').lower();
3781 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3782 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3783 fcontents.close();
3784 if(numfiles>0):
3785 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3786 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3787 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
3788 try:
3789 catfp.flush();
3790 os.fsync(catfp.fileno());
3791 except io.UnsupportedOperation:
3792 pass;
3793 except AttributeError:
3794 pass;
3795 except OSError as e:
3796 pass;
3797 if(outfile=="-"):
3798 catfp.seek(0, 0)
3799 if(hasattr(sys.stdout, "buffer")):
3800 shutil.copyfileobj(catfp, sys.stdout.buffer);
3801 else:
3802 shutil.copyfileobj(catfp, sys.stdout);
3803 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3804 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3805 catfp.seek(0, 0);
3806 upload_file_to_internet_file(catfp, outfile);
3807 if(returnfp):
3808 catfp.seek(0, 0)
3809 return catfp
3810 else:
3811 catfp.close()
3812 return True;
3813
3814 create_alias_function("Pack", __file_format_name__, "FromRarFile", PackArchiveFileFromRarFile);
3815
3816 if(not py7zr_support):
3817 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3818 return False
3819
3820 if(py7zr_support):
3821 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3822 formatspecs = FormatSpecsListToDict(formatspecs);
3823 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3824 outfile = RemoveWindowsPath(outfile);
3825 checksumtype = checksumtype.lower();
3826 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3827 checksumtype="crc32";
3828 if(checksumtype=="none"):
3829 checksumtype = "";
3830 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3831 compression = "auto";
3832 if(compression not in compressionlist and compression is None):
3833 compression = "auto";
3834 if(verbose):
3835 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3836 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3837 if(os.path.exists(outfile)):
3838 try:
3839 os.unlink(outfile);
3840 except OSError as e:
3841 pass;
3842 if(outfile=="-"):
3843 verbose = False;
3844 catfp = BytesIO();
3845 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3846 catfp = outfile;
3847 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3848 catfp = BytesIO();
3849 else:
3850 fbasename = os.path.splitext(outfile)[0];
3851 fextname = os.path.splitext(outfile)[1];
3852 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3853 catver = formatspecs['format_ver'];
3854 fileheaderver = str(int(catver.replace(".", "")));
3855 curinode = 0;
3856 curfid = 0;
3857 inodelist = [];
3858 inodetofile = {};
3859 filetoinode = {};
3860 inodetocatinode = {};
3861 if(not os.path.exists(infile) or not os.path.isfile(infile)):
3862 return False;
3863 szpfp = py7zr.SevenZipFile(infile, mode="r");
3864 file_content = szpfp.readall();
3865 #sztest = szpfp.testzip();
3866 sztestalt = szpfp.test();
3867 if(sztestalt):
3868 VerbosePrintOut("Bad file found!");
3869 numfiles = int(len(szpfp.list()));
3870 AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3871 for member in sorted(szpfp.list(), key=lambda x: x.filename):
3872 if(re.findall("^[.|/]", member.filename)):
3873 fname = member.filename;
3874 else:
3875 fname = "./"+member.filename;
3876 if(verbose):
3877 VerbosePrintOut(fname);
3878 if(not member.is_directory):
3879 fpremode = int(stat.S_IFREG + 438);
3880 elif(member.is_directory):
3881 fpremode = int(stat.S_IFDIR + 511);
3882 fwinattributes = format(int(0), 'x').lower();
3883 fcompression = "";
3884 fcsize = format(int(0), 'x').lower();
3885 flinkcount = 0;
3886 ftype = 0;
3887 if(member.is_directory):
3888 ftype = 5;
3889 else:
3890 ftype = 0;
3891 flinkname = "";
3892 fcurfid = format(int(curfid), 'x').lower();
3893 fcurinode = format(int(curfid), 'x').lower();
3894 curfid = curfid + 1;
3895 fdev_minor = format(int(0), 'x').lower();
3896 fdev_major = format(int(0), 'x').lower();
3897 frdev_minor = format(int(0), 'x').lower();
3898 frdev_major = format(int(0), 'x').lower();
3899 if(ftype==5):
3900 fsize = format(int("0"), 'x').lower();
3901 fatime = format(int(member.creationtime.timestamp()), 'x').lower();
3902 fmtime = format(int(member.creationtime.timestamp()), 'x').lower();
3903 fctime = format(int(member.creationtime.timestamp()), 'x').lower();
3904 fbtime = format(int(member.creationtime.timestamp()), 'x').lower();
3905 if(member.is_directory):
3906 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3907 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
3908 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
3909 else:
3910 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3911 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3912 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3913 try:
3914 fuid = format(int(os.getuid()), 'x').lower();
3915 except AttributeError:
3916 fuid = format(int(0), 'x').lower();
3917 except KeyError:
3918 fuid = format(int(0), 'x').lower();
3919 try:
3920 fgid = format(int(os.getgid()), 'x').lower();
3921 except AttributeError:
3922 fgid = format(int(0), 'x').lower();
3923 except KeyError:
3924 fgid = format(int(0), 'x').lower();
3925 try:
3926 import pwd;
3927 try:
3928 userinfo = pwd.getpwuid(os.getuid());
3929 funame = userinfo.pw_name;
3930 except KeyError:
3931 funame = "";
3932 except AttributeError:
3933 funame = "";
3934 except ImportError:
3935 funame = "";
3936 fgname = "";
3937 try:
3938 import grp;
3939 try:
3940 groupinfo = grp.getgrgid(os.getgid());
3941 fgname = groupinfo.gr_name;
3942 except KeyError:
3943 fgname = "";
3944 except AttributeError:
3945 fgname = "";
3946 except ImportError:
3947 fgname = "";
3948 fcontents = BytesIO();
3949 if(ftype==0):
3950 fcontents.write(file_content[member.filename].read());
3951 fsize = format(fcontents.tell(), 'x').lower();
3952 file_content[member.filename].close();
3953 if(not compresswholefile):
3954 fcontents.seek(0, 2);
3955 ucfsize = fcontents.tell();
3956 fcontents.seek(0, 0);
3957 if(compression=="auto"):
3958 ilsize = len(compressionlistalt);
3959 ilmin = 0;
3960 ilcsize = [];
3961 while(ilmin < ilsize):
3962 cfcontents = BytesIO();
3963 shutil.copyfileobj(fcontents, cfcontents);
3964 fcontents.seek(0, 0);
3965 cfcontents.seek(0, 0);
3966 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3967 if(cfcontents):
3968 cfcontents.seek(0, 2);
3969 ilcsize.append(cfcontents.tell());
3970 cfcontents.close();
3971 else:
3972 try:
3973 ilcsize.append(sys.maxint);
3974 except AttributeError:
3975 ilcsize.append(sys.maxsize);
3976 ilmin = ilmin + 1;
3977 ilcmin = ilcsize.index(min(ilcsize));
3978 compression = compressionlistalt[ilcmin];
3979 fcontents.seek(0, 0);
3980 cfcontents = BytesIO();
3981 shutil.copyfileobj(fcontents, cfcontents);
3982 cfcontents.seek(0, 0);
3983 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3984 cfcontents.seek(0, 2);
3985 cfsize = cfcontents.tell();
3986 if(ucfsize > cfsize):
3987 fcsize = format(int(cfsize), 'x').lower();
3988 fcompression = compression;
3989 fcontents.close();
3990 fcontents = cfcontents;
3991 if(fcompression=="none"):
3992 fcompression = "";
3993 fcontents.seek(0, 0);
3994 ftypehex = format(ftype, 'x').lower();
3995 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3996 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3997 fcontents.close();
3998 if(numfiles>0):
3999 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
4000 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
4001 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
4002 try:
4003 catfp.flush();
4004 os.fsync(catfp.fileno());
4005 except io.UnsupportedOperation:
4006 pass;
4007 except AttributeError:
4008 pass;
4009 except OSError as e:
4010 pass;
4011 if(outfile=="-"):
4012 catfp.seek(0, 0);
4013 if(hasattr(sys.stdout, "buffer")):
4014 shutil.copyfileobj(catfp, sys.stdout.buffer);
4015 else:
4016 shutil.copyfileobj(catfp, sys.stdout);
4017 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
4018 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
4019 catfp.seek(0, 0);
4020 upload_file_to_internet_file(catfp, outfile);
4021 if(returnfp):
4022 catfp.seek(0, 0)
4023 return catfp
4024 else:
4025 catfp.close()
4026 return True;
4027
4028 create_alias_function("Pack", __file_format_name__, "FromSevenZipFile", PackArchiveFileFromSevenZipFile);
4029
4030 def PackArchiveFileFromInFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
4031 formatspecs = FormatSpecsListToDict(formatspecs);
4032 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4033 if(verbose):
4034 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
4035 if(checkcompressfile=="tarfile"):
4036 return PackArchiveFileFromTarFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4037 elif(checkcompressfile=="zipfile"):
4038 return PackArchiveFileFromZipFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4039 elif(checkcompressfile=="catfile"):
4040 return RePackArchiveFile(infile, outfile, compression, compresswholefile, compressionlevel, False, 0, 0, checksumtype, False, extradata, formatspecs, verbose, returnfp);
4041 elif(rarfile_support and checkcompressfile=="rarfile"):
4042 return PackArchiveFileFromRarFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4043 elif(py7zr_support and checkcompressfile=="7zipfile"):
4044 return PackArchiveFileFromSevenZipFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4045 else:
4046 return False;
4047 return False;
4048
4049 create_alias_function("Pack", __file_format_name__, "FromInFile", PackArchiveFileFromInFile);
4050
4051 def ArchiveFileSeekToFileNum(infile, seekto=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4052 formatspecs = FormatSpecsListToDict(formatspecs);
4053 if(hasattr(infile, "read") or hasattr(infile, "write")):
4054 catfp = infile;
4055 catfp.seek(0, 0);
4056 catfp = UncompressArchiveFile(catfp, formatspecs);
4057 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4058 if(checkcompressfile=="tarfile"):
4059 return TarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4060 if(checkcompressfile=="zipfile"):
4061 return ZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4062 if(rarfile_support and checkcompressfile=="rarfile"):
4063 return RarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4064 if(py7zr_support and checkcompressfile=="7zipfile"):
4065 return SevenZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4066 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4067 return False;
4068 if(not catfp):
4069 return False;
4070 catfp.seek(0, 0);
4071 elif(infile=="-"):
4072 catfp = BytesIO();
4073 if(hasattr(sys.stdin, "buffer")):
4074 shutil.copyfileobj(sys.stdin.buffer, catfp);
4075 else:
4076 shutil.copyfileobj(sys.stdin, catfp);
4077 catfp.seek(0, 0);
4078 catfp = UncompressArchiveFile(catfp, formatspecs);
4079 if(not catfp):
4080 return False;
4081 catfp.seek(0, 0);
4082 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4083 catfp = download_file_from_internet_file(infile);
4084 catfp.seek(0, 0);
4085 catfp = UncompressArchiveFile(catfp, formatspecs);
4086 if(not catfp):
4087 return False;
4088 catfp.seek(0, 0);
4089 else:
4090 infile = RemoveWindowsPath(infile);
4091 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4092 if(checkcompressfile=="tarfile"):
4093 return TarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4094 if(checkcompressfile=="zipfile"):
4095 return ZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4096 if(rarfile_support and checkcompressfile=="rarfile"):
4097 return RarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4098 if(py7zr_support and checkcompressfile=="7zipfile"):
4099 return SevenZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4100 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4101 return False;
4102 compresscheck = CheckCompressionType(infile, formatspecs, True);
4103 if(not compresscheck):
4104 fextname = os.path.splitext(infile)[1];
4105 if(fextname==".gz"):
4106 compresscheck = "gzip";
4107 elif(fextname==".bz2"):
4108 compresscheck = "bzip2";
4109 elif(fextname==".zst"):
4110 compresscheck = "zstd";
4111 elif(fextname==".lz4" or fextname==".clz4"):
4112 compresscheck = "lz4";
4113 elif(fextname==".lzo" or fextname==".lzop"):
4114 compresscheck = "lzo";
4115 elif(fextname==".lzma" or fextname==".xz"):
4116 compresscheck = "lzma";
4117 else:
4118 return False;
4119 if(not compresscheck):
4120 return False;
4121 catfp = UncompressFile(infile, formatspecs, "rb");
4122 '''
4123 try:
4124 catfp.seek(0, 2);
4125 except OSError:
4126 SeekToEndOfFile(catfp);
4127 except ValueError:
4128 SeekToEndOfFile(catfp);
4129 CatSize = catfp.tell();
4130 CatSizeEnd = CatSize;
4131 '''
4132 try:
4133 catfp.seek(0, 0);
4134 except OSError:
4135 return False;
4136 except ValueError:
4137 return False;
4138 curloc = catfp.tell();
4139 if(curloc>0):
4140 catfp.seek(0, 0);
4141 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4142 if(curloc>0):
4143 catfp.seek(curloc, 0);
4144 catstring = catheader[0];
4145 catversion = re.findall(r"([\d]+)", catstring);
4146 fprenumfiles = catheader[1];
4147 fnumfiles = int(fprenumfiles, 16);
4148 fprechecksumtype = catheader[2];
4149 fprechecksum = catheader[3];
4150 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4151 fnumfileshex = format(int(fnumfiles), 'x').lower();
4152 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4153 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4154 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4155 fheadtell = len(fileheader);
4156 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4157 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
4158 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4159 return False;
4160 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
4161 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4162 if(seekto>=fnumfiles):
4163 seekto = fnumfiles - 1;
4164 if(seekto<0):
4165 seekto = 0;
4166 if(seekto>=0):
4167 il = -1;
4168 while(il < seekto):
4169 prefhstart = catfp.tell();
4170 if(formatspecs['new_style']):
4171 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4172 else:
4173 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4174 if(len(preheaderdata)==0):
4175 break;
4176 prefheadsize = int(preheaderdata[0], 16);
4177 prefnumfields = int(preheaderdata[1], 16);
4178 preftype = int(preheaderdata[2], 16);
4179 if(re.findall("^[.|/]", preheaderdata[3])):
4180 prefname = preheaderdata[3];
4181 else:
4182 prefname = "./"+preheaderdata[3];
4183 prefbasedir = os.path.dirname(prefname);
4184 preflinkname = preheaderdata[4];
4185 prefsize = int(preheaderdata[5], 16);
4186 prefatime = int(preheaderdata[6], 16);
4187 prefmtime = int(preheaderdata[7], 16);
4188 prefctime = int(preheaderdata[8], 16);
4189 prefbtime = int(preheaderdata[9], 16);
4190 prefmode = int(preheaderdata[10], 16);
4191 prefchmode = stat.S_IMODE(prefmode);
4192 preftypemod = stat.S_IFMT(prefmode);
4193 prefwinattributes = int(preheaderdata[11], 16);
4194 prefcompression = preheaderdata[12];
4195 prefcsize = int(preheaderdata[13], 16);
4196 prefuid = int(preheaderdata[14], 16);
4197 prefuname = preheaderdata[15];
4198 prefgid = int(preheaderdata[16], 16);
4199 prefgname = preheaderdata[17];
4200 fid = int(preheaderdata[18], 16);
4201 finode = int(preheaderdata[19], 16);
4202 flinkcount = int(preheaderdata[20], 16);
4203 prefdev_minor = int(preheaderdata[21], 16);
4204 prefdev_major = int(preheaderdata[22], 16);
4205 prefrdev_minor = int(preheaderdata[23], 16);
4206 prefrdev_major = int(preheaderdata[24], 16);
4207 prefextrasize = int(preheaderdata[25], 16);
4208 prefextrafields = int(preheaderdata[26], 16);
4209 extrafieldslist = [];
4210 extrastart = 27;
4211 extraend = extrastart + prefextrafields;
4212 extrafieldslist = [];
4213 if(extrastart<extraend):
4214 extrafieldslist.append(preheaderdata[extrastart]);
4215 extrastart = extrastart + 1;
4216 prefchecksumtype = preheaderdata[extrastart].lower();
4217 prefcs = preheaderdata[extrastart + 1].lower();
4218 prefccs = preheaderdata[extrastart + 2].lower();
4219 hc = 0;
4220 hcmax = len(preheaderdata) - 2;
4221 hout = "";
4222 while(hc<hcmax):
4223 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs['format_delimiter']);
4224 hc = hc + 1;
4225 prenewfcs = GetFileChecksum(hout, preheaderdata[-3].lower(), True, formatspecs);
4226 if(prefcs!=prenewfcs and not skipchecksum):
4227 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4228 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4229 return False;
4230 valid_archive = False;
4231 invalid_archive = True;
4232 prefhend = catfp.tell() - 1;
4233 prefcontentstart = catfp.tell();
4234 prefcontents = "";
4235 pyhascontents = False;
4236 if(prefsize>0):
4237 if(prefcompression):
4238 prefcontents = catfp.read(prefsize);
4239 else:
4240 prefcontents = catfp.read(prefcsize);
4241 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4242 pyhascontents = True;
4243 if(prefccs!=prenewfccs and not skipchecksum):
4244 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4245 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4246 return False;
4247 catfp.seek(1, 1);
4248 il = il + 1;
4249 catfp.seek(seekstart, 0);
4250 fileidnum = il;
4251 catfheadsize = int(preheaderdata[0], 16);
4252 catfnumfields = int(preheaderdata[1], 16);
4253 catftype = int(preheaderdata[2], 16);
4254 if(re.findall("^[.|/]", preheaderdata[3])):
4255 catfname = preheaderdata[3];
4256 else:
4257 catfname = "./"+preheaderdata[3];
4258 catflinkname = preheaderdata[4];
4259 catfsize = int(preheaderdata[5], 16);
4260 catfbasedir = os.path.dirname(catfname);
4261 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
4262 if(returnfp):
4263 catlist.update({'catfp': catfp});
4264 else:
4265 catfp.close();
4266 return catlist;
4267
4268 create_alias_function("", __file_format_name__, "SeekToFileNum", ArchiveFileSeekToFileNum);
4269
4270 def ArchiveFileSeekToFileName(infile, seekfile=None, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4271 formatspecs = FormatSpecsListToDict(formatspecs);
4272 if(hasattr(infile, "read") or hasattr(infile, "write")):
4273 catfp = infile;
4274 catfp.seek(0, 0);
4275 catfp = UncompressArchiveFile(catfp, formatspecs);
4276 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4277 if(checkcompressfile=="tarfile"):
4278 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4279 if(checkcompressfile=="zipfile"):
4280 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4281 if(rarfile_support and checkcompressfile=="rarfile"):
4282 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4283 if(py7zr_support and checkcompressfile=="7zipfile"):
4284 return SevenZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4285 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4286 return False;
4287 if(not catfp):
4288 return False;
4289 catfp.seek(0, 0);
4290 elif(infile=="-"):
4291 catfp = BytesIO();
4292 if(hasattr(sys.stdin, "buffer")):
4293 shutil.copyfileobj(sys.stdin.buffer, catfp);
4294 else:
4295 shutil.copyfileobj(sys.stdin, catfp);
4296 catfp.seek(0, 0);
4297 catfp = UncompressArchiveFile(catfp, formatspecs);
4298 if(not catfp):
4299 return False;
4300 catfp.seek(0, 0);
4301 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4302 catfp = download_file_from_internet_file(infile);
4303 catfp = UncompressArchiveFile(catfp, formatspecs);
4304 catfp.seek(0, 0);
4305 if(not catfp):
4306 return False;
4307 catfp.seek(0, 0);
4308 else:
4309 infile = RemoveWindowsPath(infile);
4310 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4311 if(checkcompressfile=="tarfile"):
4312 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4313 if(checkcompressfile=="zipfile"):
4314 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4315 if(rarfile_support and checkcompressfile=="rarfile"):
4316 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4317 if(py7zr_support and checkcompressfile=="7zipfile"):
4318 return SevenZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4319 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4320 return False;
4321 compresscheck = CheckCompressionType(infile, formatspecs, True);
4322 if(not compresscheck):
4323 fextname = os.path.splitext(infile)[1];
4324 if(fextname==".gz"):
4325 compresscheck = "gzip";
4326 elif(fextname==".bz2"):
4327 compresscheck = "bzip2";
4328 elif(fextname==".zst"):
4329 compresscheck = "zstd";
4330 elif(fextname==".lz4" or fextname==".clz4"):
4331 compresscheck = "lz4";
4332 elif(fextname==".lzo" or fextname==".lzop"):
4333 compresscheck = "lzo";
4334 elif(fextname==".lzma" or fextname==".xz"):
4335 compresscheck = "lzma";
4336 else:
4337 return False;
4338 if(not compresscheck):
4339 return False;
4340 catfp = UncompressFile(infile, formatspecs, "rb");
4341 '''
4342 try:
4343 catfp.seek(0, 2);
4344 except OSError:
4345 SeekToEndOfFile(catfp);
4346 except ValueError:
4347 SeekToEndOfFile(catfp);
4348 CatSize = catfp.tell();
4349 CatSizeEnd = CatSize;
4350 '''
4351 try:
4352 catfp.seek(0, 0);
4353 except OSError:
4354 return False;
4355 except ValueError:
4356 return False;
4357 curloc = catfp.tell();
4358 if(curloc>0):
4359 catfp.seek(0, 0);
4360 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4361 if(curloc>0):
4362 catfp.seek(curloc, 0);
4363 catstring = catheader[0];
4364 catversion = re.findall(r"([\d]+)", catstring);
4365 fprenumfiles = catheader[1];
4366 fnumfiles = int(fprenumfiles, 16);
4367 fprechecksumtype = catheader[2];
4368 fprechecksum = catheader[3];
4369 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4370 fnumfileshex = format(int(fnumfiles), 'x').lower();
4371 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4372 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4373 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4374 fheadtell = len(fileheader);
4375 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4376 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
4377 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4378 return False;
4379 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
4380 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4381 seekto = fnumfiles - 1
4382 filefound = False;
4383 if(seekto>=0):
4384 il = -1;
4385 while(il < seekto):
4386 prefhstart = catfp.tell();
4387 if(formatspecs['new_style']):
4388 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4389 else:
4390 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4391 if(len(preheaderdata)==0):
4392 break;
4393 prefheadsize = int(preheaderdata[0], 16);
4394 prefnumfields = int(preheaderdata[1], 16);
4395 preftype = int(preheaderdata[2], 16);
4396 if(re.findall("^[.|/]", preheaderdata[3])):
4397 prefname = preheaderdata[3];
4398 else:
4399 prefname = "./"+preheaderdata[3];
4400 prefbasedir = os.path.dirname(prefname);
4401 preflinkname = preheaderdata[4];
4402 prefsize = int(preheaderdata[5], 16);
4403 prefatime = int(preheaderdata[6], 16);
4404 prefmtime = int(preheaderdata[7], 16);
4405 prefctime = int(preheaderdata[8], 16);
4406 prefbtime = int(preheaderdata[9], 16);
4407 prefmode = int(preheaderdata[10], 16);
4408 prefchmode = stat.S_IMODE(prefmode);
4409 preftypemod = stat.S_IFMT(prefmode);
4410 prefwinattributes = int(preheaderdata[11], 16);
4411 prefcompression = preheaderdata[12];
4412 prefcsize = int(preheaderdata[13], 16);
4413 prefuid = int(preheaderdata[14], 16);
4414 prefuname = preheaderdata[15];
4415 prefgid = int(preheaderdata[16], 16);
4416 prefgname = preheaderdata[17];
4417 fid = int(preheaderdata[18], 16);
4418 finode = int(preheaderdata[19], 16);
4419 flinkcount = int(preheaderdata[20], 16);
4420 prefdev_minor = int(preheaderdata[21], 16);
4421 prefdev_major = int(preheaderdata[22], 16);
4422 prefrdev_minor = int(preheaderdata[23], 16);
4423 prefrdev_major = int(preheaderdata[24], 16);
4424 prefextrasize = int(preheaderdata[25], 16);
4425 prefextrafields = int(preheaderdata[26], 16);
4426 extrafieldslist = [];
4427 extrastart = 27;
4428 extraend = extrastart + prefextrafields;
4429 extrafieldslist = [];
4430 if(extrastart<extraend):
4431 extrafieldslist.append(preheaderdata[extrastart]);
4432 extrastart = extrastart + 1;
4433 prefchecksumtype = preheaderdata[extrastart].lower();
4434 prefcs = preheaderdata[extrastart + 1].lower();
4435 prefccs = preheaderdata[extrastart + 2].lower();
4436 hc = 0;
4437 hcmax = len(preheaderdata) - 2;
4438 hout = "";
4439 while(hc<hcmax):
4440 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs['format_delimiter']);
4441 hc = hc + 1;
4442 prenewfcs = GetFileChecksum(hout, preheaderdata[-3].lower(), True, formatspecs);
4443 if(prefcs!=prenewfcs and not skipchecksum):
4444 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4445 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4446 return False;
4447 valid_archive = False;
4448 invalid_archive = True;
4449 prefhend = catfp.tell() - 1;
4450 prefcontentstart = catfp.tell();
4451 prefcontents = "";
4452 pyhascontents = False;
4453 if(prefsize>0):
4454 if(prefcompression):
4455 prefcontents = catfp.read(prefsize);
4456 else:
4457 prefcontents = catfp.read(prefcsize);
4458 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4459 pyhascontents = True;
4460 if(prefccs!=prenewfccs and not skipchecksum):
4461 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4462 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4463 return False;
4464 catfp.seek(1, 1);
4465 il = il + 1;
4466 filefound = False;
4467 prefname = preheaderdata[2];
4468 if(re.findall("^[.|/]", preheaderdata[2])):
4469 prefname = preheaderdata[2];
4470 else:
4471 prefname = "./"+preheaderdata[2];
4472 if(prefname==seekfile):
4473 filefound = True;
4474 break;
4475 catfp.seek(seekstart, 0);
4476 fileidnum = il;
4477 catfheadsize = int(preheaderdata[0], 16);
4478 catfnumfields = int(preheaderdata[1], 16);
4479 catftype = int(preheaderdata[2], 16);
4480 if(re.findall("^[.|/]", preheaderdata[3])):
4481 catfname = preheaderdata[3];
4482 else:
4483 catfname = "./"+preheaderdata[3];
4484 catflinkname = preheaderdata[4];
4485 catfsize = int(preheaderdata[5], 16);
4486 catfbasedir = os.path.dirname(catfname);
4487 if(filefound):
4488 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
4489 else:
4490 return False;
4491 if(returnfp):
4492 catlist.update({'catfp': catfp});
4493 else:
4494 catfp.close();
4495 return catlist;
4496
4497 create_alias_function("", __file_format_name__, "SeekToFileName", ArchiveFileSeekToFileName);
4498
4499 def ArchiveFileValidate(infile, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
4500 formatspecs = FormatSpecsListToDict(formatspecs);
4501 if(verbose):
4502 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
4503 if(hasattr(infile, "read") or hasattr(infile, "write")):
4504 catfp = infile;
4505 catfp.seek(0, 0);
4506 catfp = UncompressArchiveFile(catfp, formatspecs);
4507 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4508 if(checkcompressfile=="tarfile"):
4509 return TarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4510 if(checkcompressfile=="zipfile"):
4511 return ZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4512 if(rarfile_support and checkcompressfile=="rarfile"):
4513 return RarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4514 if(py7zr_support and checkcompressfile=="7zipfile"):
4515 return SevenZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4516 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4517 return False;
4518 if(not catfp):
4519 return False;
4520 catfp.seek(0, 0);
4521 elif(infile=="-"):
4522 catfp = BytesIO();
4523 if(hasattr(sys.stdin, "buffer")):
4524 shutil.copyfileobj(sys.stdin.buffer, catfp);
4525 else:
4526 shutil.copyfileobj(sys.stdin, catfp);
4527 catfp.seek(0, 0);
4528 catfp = UncompressArchiveFile(catfp, formatspecs);
4529 if(not catfp):
4530 return False;
4531 catfp.seek(0, 0);
4532 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4533 catfp = download_file_from_internet_file(infile);
4534 catfp = UncompressArchiveFile(catfp, formatspecs);
4535 catfp.seek(0, 0);
4536 if(not catfp):
4537 return False;
4538 catfp.seek(0, 0);
4539 else:
4540 infile = RemoveWindowsPath(infile);
4541 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4542 if(checkcompressfile=="tarfile"):
4543 return TarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4544 if(checkcompressfile=="zipfile"):
4545 return ZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4546 if(rarfile_support and checkcompressfile=="rarfile"):
4547 return RarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4548 if(py7zr_support and checkcompressfile=="7zipfile"):
4549 return SevenZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4550 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4551 return False;
4552 compresscheck = CheckCompressionType(infile, formatspecs, True);
4553 if(not compresscheck):
4554 fextname = os.path.splitext(infile)[1];
4555 if(fextname==".gz"):
4556 compresscheck = "gzip";
4557 elif(fextname==".bz2"):
4558 compresscheck = "bzip2";
4559 elif(fextname==".zst"):
4560 compresscheck = "zstd";
4561 elif(fextname==".lz4" or fextname==".clz4"):
4562 compresscheck = "lz4";
4563 elif(fextname==".lzo" or fextname==".lzop"):
4564 compresscheck = "lzo";
4565 elif(fextname==".lzma" or fextname==".xz"):
4566 compresscheck = "lzma";
4567 else:
4568 return False;
4569 if(not compresscheck):
4570 return False;
4571 catfp = UncompressFile(infile, formatspecs, "rb");
4572 '''
4573 try:
4574 catfp.seek(0, 2);
4575 except OSError:
4576 SeekToEndOfFile(catfp);
4577 except ValueError:
4578 SeekToEndOfFile(catfp);
4579 CatSize = catfp.tell();
4580 CatSizeEnd = CatSize;
4581 '''
4582 try:
4583 catfp.seek(0, 0);
4584 except OSError:
4585 return False;
4586 except ValueError:
4587 return False;
4588 curloc = catfp.tell();
4589 if(curloc>0):
4590 catfp.seek(0, 0);
4591 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4592 if(curloc>0):
4593 catfp.seek(curloc, 0);
4594 catstring = catheader[0];
4595 catversion = re.findall(r"([\d]+)", catstring);
4596 fprenumfiles = catheader[1];
4597 fnumfiles = int(fprenumfiles, 16);
4598 fprechecksumtype = catheader[2];
4599 fprechecksum = catheader[3];
4600 il = 0;
4601 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4602 fnumfileshex = format(int(fnumfiles), 'x').lower();
4603 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4604 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4605 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4606 valid_archive = True;
4607 invalid_archive = False;
4608 if(verbose):
4609 VerbosePrintOut(infile);
4610 VerbosePrintOut("Number of Records " + str(fnumfiles));
4611 if(fprechecksum==catfileheadercshex):
4612 if(verbose):
4613 VerbosePrintOut("File Header Checksum Passed at offset " + str(0));
4614 VerbosePrintOut("'" + str(fprechecksum) + "' == " + "'" + str(catfileheadercshex) + "'");
4615 else:
4616 if(verbose):
4617 VerbosePrintOut("File Header Checksum Failed at offset " + str(0));
4618 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4619 valid_archive = False;
4620 invalid_archive = True;
4621 if(verbose):
4622 VerbosePrintOut("");
4623 while(il<fnumfiles):
4624 catfhstart = catfp.tell();
4625 if(formatspecs['new_style']):
4626 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4627 else:
4628 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4629 if(len(catheaderdata)==0):
4630 break;
4631 catfheadsize = int(catheaderdata[0], 16);
4632 catfnumfields = int(catheaderdata[1], 16);
4633 catftype = int(catheaderdata[2], 16);
4634 if(re.findall("^[.|/]", catheaderdata[3])):
4635 catfname = catheaderdata[3];
4636 else:
4637 catfname = "./"+catheaderdata[3];
4638 catfbasedir = os.path.dirname(catfname);
4639 catflinkname = catheaderdata[4];
4640 catfsize = int(catheaderdata[5], 16);
4641 catfatime = int(catheaderdata[6], 16);
4642 catfmtime = int(catheaderdata[7], 16);
4643 catfctime = int(catheaderdata[8], 16);
4644 catfbtime = int(catheaderdata[9], 16);
4645 catfmode = int(catheaderdata[10], 16);
4646 catfchmode = stat.S_IMODE(catfmode);
4647 catftypemod = stat.S_IFMT(catfmode);
4648 catfwinattributes = int(catheaderdata[11], 16);
4649 catfcompression = catheaderdata[12];
4650 catfcsize = int(catheaderdata[13], 16);
4651 catfuid = int(catheaderdata[14], 16);
4652 catfuname = catheaderdata[15];
4653 catfgid = int(catheaderdata[16], 16);
4654 catfgname = catheaderdata[17];
4655 fid = int(catheaderdata[18], 16);
4656 finode = int(catheaderdata[19], 16);
4657 flinkcount = int(catheaderdata[20], 16);
4658 catfdev_minor = int(catheaderdata[21], 16);
4659 catfdev_major = int(catheaderdata[22], 16);
4660 catfrdev_minor = int(catheaderdata[23], 16);
4661 catfrdev_major = int(catheaderdata[24], 16);
4662 catfextrasize = int(catheaderdata[25], 16);
4663 catfextrafields = int(catheaderdata[26], 16);
4664 extrafieldslist = [];
4665 extrastart = 27;
4666 extraend = extrastart + catfextrafields;
4667 extrafieldslist = [];
4668 if(extrastart<extraend):
4669 extrafieldslist.append(catheaderdata[extrastart]);
4670 extrastart = extrastart + 1;
4671 catfchecksumtype = catheaderdata[extrastart].lower();
4672 catfcs = catheaderdata[extrastart + 1].lower();
4673 catfccs = catheaderdata[extrastart + 2].lower();
4674 hc = 0;
4675 hcmax = len(catheaderdata) - 2;
4676 hout = "";
4677 while(hc<hcmax):
4678 hout = hout + AppendNullByte(catheaderdata[hc], formatspecs['format_delimiter']);
4679 hc = hc + 1;
4680 catnewfcs = GetFileChecksum(hout, catheaderdata[-3].lower(), True, formatspecs);
4681 if(verbose):
4682 VerbosePrintOut(catfname);
4683 VerbosePrintOut("Record Number " + str(il) + "; File ID " + str(fid) + "; iNode Number " + str(finode));
4684 if(catfcs==catnewfcs):
4685 if(verbose):
4686 VerbosePrintOut("File Header Checksum Passed at offset " + str(catfhstart));
4687 VerbosePrintOut("'" + str(catfcs) + "' == " + "'" + str(catnewfcs) + "'");
4688 else:
4689 if(verbose):
4690 VerbosePrintOut("File Header Checksum Failed at offset " + str(catfhstart));
4691 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
4692 valid_archive = False;
4693 invalid_archive = True;
4694 catfhend = catfp.tell() - 1;
4695 catfcontentstart = catfp.tell();
4696 catfcontents = "";
4697 pyhascontents = False;
4698 if(catfsize>0):
4699 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4700 catfcontents = catfp.read(catfsize);
4701 else:
4702 catfcontents = catfp.read(catfcsize);
4703 catnewfccs = GetFileChecksum(catfcontents, catheaderdata[-3].lower(), False, formatspecs);
4704 pyhascontents = True;
4705 if(catfccs==catnewfccs):
4706 if(verbose):
4707 VerbosePrintOut("File Content Checksum Passed at offset " + str(catfcontentstart));
4708 VerbosePrintOut("'" + str(catfccs) + "' == " + "'" + str(catnewfccs) + "'");
4709 else:
4710 if(verbose):
4711 VerbosePrintOut("File Content Checksum Failed at offset " + str(catfcontentstart));
4712 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
4713 valid_archive = False;
4714 invalid_archive = True;
4715 if(verbose):
4716 VerbosePrintOut("");
4717 catfp.seek(1, 1);
4718 il = il + 1;
4719 if(valid_archive):
4720 if(returnfp):
4721 return catfp;
4722 else:
4723 catfp.close();
4724 return True;
4725 else:
4726 catfp.close();
4727 return False;
4728
4729 create_alias_function("", __file_format_name__, "Validate", ArchiveFileValidate);
4730
4731 def ArchiveFileToArray(infile, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4732 formatspecs = FormatSpecsListToDict(formatspecs);
4733 if(hasattr(infile, "read") or hasattr(infile, "write")):
4734 catfp = infile;
4735 catfp.seek(0, 0);
4736 catfp = UncompressArchiveFile(catfp, formatspecs);
4737 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4738 if(checkcompressfile=="tarfile"):
4739 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4740 if(checkcompressfile=="zipfile"):
4741 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4742 if(rarfile_support and checkcompressfile=="rarfile"):
4743 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4744 if(py7zr_support and checkcompressfile=="7zipfile"):
4745 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4746 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4747 return False;
4748 if(not catfp):
4749 return False;
4750 catfp.seek(0, 0);
4751 elif(infile=="-"):
4752 catfp = BytesIO();
4753 if(hasattr(sys.stdin, "buffer")):
4754 shutil.copyfileobj(sys.stdin.buffer, catfp);
4755 else:
4756 shutil.copyfileobj(sys.stdin, catfp);
4757 catfp.seek(0, 0);
4758 catfp = UncompressArchiveFile(catfp, formatspecs);
4759 if(not catfp):
4760 return False;
4761 catfp.seek(0, 0);
4762 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4763 catfp = download_file_from_internet_file(infile);
4764 catfp = UncompressArchiveFile(catfp, formatspecs);
4765 catfp.seek(0, 0);
4766 if(not catfp):
4767 return False;
4768 catfp.seek(0, 0);
4769 else:
4770 infile = RemoveWindowsPath(infile);
4771 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4772 if(checkcompressfile=="tarfile"):
4773 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4774 if(checkcompressfile=="zipfile"):
4775 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4776 if(rarfile_support and checkcompressfile=="rarfile"):
4777 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4778 if(py7zr_support and checkcompressfile=="7zipfile"):
4779 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4780 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4781 return False;
4782 compresscheck = CheckCompressionType(infile, formatspecs, True);
4783 if(not compresscheck):
4784 fextname = os.path.splitext(infile)[1];
4785 if(fextname==".gz"):
4786 compresscheck = "gzip";
4787 elif(fextname==".bz2"):
4788 compresscheck = "bzip2";
4789 elif(fextname==".zst"):
4790 compresscheck = "zstd";
4791 elif(fextname==".lz4" or fextname==".clz4"):
4792 compresscheck = "lz4";
4793 elif(fextname==".lzo" or fextname==".lzop"):
4794 compresscheck = "lzo";
4795 elif(fextname==".lzma" or fextname==".xz"):
4796 compresscheck = "lzma";
4797 else:
4798 return False;
4799 if(not compresscheck):
4800 return False;
4801 catfp = UncompressFile(infile, formatspecs, "rb");
4802 '''
4803 try:
4804 catfp.seek(0, 2);
4805 except OSError:
4806 SeekToEndOfFile(catfp);
4807 except ValueError:
4808 SeekToEndOfFile(catfp);
4809 CatSize = catfp.tell();
4810 CatSizeEnd = CatSize;
4811 '''
4812 try:
4813 catfp.seek(0, 0);
4814 except OSError:
4815 return False;
4816 except ValueError:
4817 return False;
4818 curloc = catfp.tell();
4819 if(curloc>0):
4820 catfp.seek(0, 0);
4821 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4822 if(curloc>0):
4823 catfp.seek(curloc, 0);
4824 catstring = catheader[0];
4825 catversion = re.findall(r"([\d]+)", catstring);
4826 fprenumfiles = catheader[1];
4827 fnumfiles = int(fprenumfiles, 16);
4828 fprechecksumtype = catheader[2];
4829 fprechecksum = catheader[3];
4830 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4831 fnumfileshex = format(int(fnumfiles), 'x').lower();
4832 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4833 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4834 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4835 fheadtell = len(fileheader);
4836 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4837 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
4838 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4839 return False;
4840 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
4841 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': []};
4842 if(seekstart<0 and seekstart>fnumfiles):
4843 seekstart = 0;
4844 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
4845 seekend = fnumfiles;
4846 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
4847 seekend = fnumfiles - abs(seekend);
4848 if(seekstart>0):
4849 il = 0;
4850 while(il < seekstart):
4851 prefhstart = catfp.tell();
4852 if(formatspecs['new_style']):
4853 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4854 else:
4855 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4856 if(len(preheaderdata)==0):
4857 break;
4858 prefheadsize = int(preheaderdata[0], 16);
4859 prefnumfields = int(preheaderdata[1], 16);
4860 if(re.findall("^[.|/]", preheaderdata[3])):
4861 prefname = preheaderdata[3];
4862 else:
4863 prefname = "./"+preheaderdata[3];
4864 prefsize = int(preheaderdata[5], 16);
4865 prefcompression = preheaderdata[12];
4866 prefcsize = int(preheaderdata[13], 16);
4867 prefextrasize = int(preheaderdata[24], 16);
4868 prefextrafields = int(preheaderdata[25], 16);
4869 extrafieldslist = [];
4870 extrastart = 26;
4871 extraend = extrastart + prefextrafields;
4872 extrafieldslist = [];
4873 if(extrastart<extraend):
4874 extrafieldslist.append(preheaderdata[extrastart]);
4875 extrastart = extrastart + 1;
4876 prefchecksumtype = preheaderdata[extrastart].lower();
4877 prefcs = preheaderdata[extrastart + 1].lower();
4878 prefccs = preheaderdata[extrastart + 2].lower();
4879 hc = 0;
4880 hcmax = len(preheaderdata) - 2;
4881 hout = "";
4882 while(hc<hcmax):
4883 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs['format_delimiter']);
4884 hc = hc + 1;
4885 prenewfcs = GetFileChecksum(hout, preheaderdata[-3].lower(), True, formatspecs);
4886 if(prefcs!=prenewfcs and not skipchecksum):
4887 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4888 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4889 return False;
4890 valid_archive = False;
4891 invalid_archive = True;
4892 prefhend = catfp.tell() - 1;
4893 prefcontentstart = catfp.tell();
4894 prefcontents = "";
4895 pyhascontents = False;
4896 if(prefsize>0):
4897 if(prefcompression=="none" or prefcompression=="" or prefcompression=="auto"):
4898 prefcontents = catfp.read(prefsize);
4899 else:
4900 prefcontents = catfp.read(prefcsize);
4901 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4902 pyhascontents = True;
4903 if(prefccs!=prenewfccs and not skipchecksum):
4904 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4905 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4906 return False;
4907 catfp.seek(1, 1);
4908 il = il + 1;
4909 fileidnum = seekstart;
4910 realidnum = 0;
4911 while(fileidnum<seekend):
4912 catfhstart = catfp.tell();
4913 if(formatspecs['new_style']):
4914 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4915 else:
4916 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4917 if(len(catheaderdata)==0):
4918 break;
4919 catfheadsize = int(catheaderdata[0], 16);
4920 catfnumfields = int(catheaderdata[1], 16);
4921 catftype = int(catheaderdata[2], 16);
4922 if(re.findall("^[.|/]", catheaderdata[3])):
4923 catfname = catheaderdata[3];
4924 else:
4925 catfname = "./"+catheaderdata[3];
4926 catfbasedir = os.path.dirname(catfname);
4927 catflinkname = catheaderdata[4];
4928 catfsize = int(catheaderdata[5], 16);
4929 catfatime = int(catheaderdata[6], 16);
4930 catfmtime = int(catheaderdata[7], 16);
4931 catfctime = int(catheaderdata[8], 16);
4932 catfbtime = int(catheaderdata[9], 16);
4933 catfmode = int(catheaderdata[10], 16);
4934 catfchmode = stat.S_IMODE(catfmode);
4935 catftypemod = stat.S_IFMT(catfmode);
4936 catfwinattributes = int(catheaderdata[11], 16);
4937 catfcompression = catheaderdata[12];
4938 catfcsize = int(catheaderdata[13], 16);
4939 catfuid = int(catheaderdata[14], 16);
4940 catfuname = catheaderdata[15];
4941 catfgid = int(catheaderdata[16], 16);
4942 catfgname = catheaderdata[17];
4943 catfid = int(catheaderdata[18], 16);
4944 catfinode = int(catheaderdata[19], 16);
4945 catflinkcount = int(catheaderdata[20], 16);
4946 catfdev_minor = int(catheaderdata[21], 16);
4947 catfdev_major = int(catheaderdata[22], 16);
4948 catfrdev_minor = int(catheaderdata[23], 16);
4949 catfrdev_major = int(catheaderdata[24], 16);
4950 catfextrasize = int(catheaderdata[25], 16);
4951 catfextrafields = int(catheaderdata[26], 16);
4952 extrafieldslist = [];
4953 extrastart = 27;
4954 extraend = extrastart + catfextrafields;
4955 extrafieldslist = [];
4956 if(extrastart<extraend):
4957 extrafieldslist.append(catheaderdata[extrastart]);
4958 extrastart = extrastart + 1;
4959 catfchecksumtype = catheaderdata[extrastart].lower();
4960 catfcs = catheaderdata[extrastart + 1].lower();
4961 catfccs = catheaderdata[extrastart + 2].lower();
4962 hc = 0;
4963 hcmax = len(catheaderdata) - 2;
4964 hout = "";
4965 while(hc<hcmax):
4966 hout = hout + AppendNullByte(catheaderdata[hc], formatspecs['format_delimiter']);
4967 hc = hc + 1;
4968 catnewfcs = GetFileChecksum(hout, catheaderdata[-3].lower(), True, formatspecs);
4969 if(catfcs!=catnewfcs and not skipchecksum):
4970 VerbosePrintOut("File Header Checksum Error with file " + catfname + " at offset " + str(catfhstart));
4971 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
4972 return False;
4973 catfhend = catfp.tell() - 1;
4974 catfcontentstart = catfp.tell();
4975 catfcontents = BytesIO();
4976 pyhascontents = False;
4977 if(catfsize>0 and not listonly):
4978 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4979 catfcontents.write(catfp.read(catfsize));
4980 else:
4981 catfcontents.write(catfp.read(catfcsize));
4982 catfcontents.seek(0, 0);
4983 catnewfccs = GetFileChecksum(catfcontents.read(), catheaderdata[-3].lower(), False, formatspecs);
4984 pyhascontents = True;
4985 if(catfccs!=catnewfccs and skipchecksum):
4986 VerbosePrintOut("File Content Checksum Error with file " + catfname + " at offset " + str(catfcontentstart));
4987 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
4988 return False;
4989 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4990 pass;
4991 else:
4992 catfcontents.seek(0, 0);
4993 if(uncompress):
4994 catfcontents = UncompressArchiveFile(catfcontents, formatspecs);
4995 catfcontents.seek(0, 0);
4996 catfccs = GetFileChecksum(catfcontents.read(), catheaderdata[-3].lower(), False, formatspecs);
4997 if(catfsize>0 and listonly):
4998 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4999 catfp.seek(catfsize, 1);
5000 else:
5001 catfp.seek(catfcsize, 1);
5002 pyhascontents = False;
5003 catfp.seek(1, 1);
5004 catfcontentend = catfp.tell() - 1;
5005 catfcontents.seek(0, 0);
5006 catlist['ffilelist'].append({'fid': realidnum, 'fidalt': fileidnum, 'fheadersize': catfheadsize, 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize, 'fatime': catfatime, 'fmtime': catfmtime, 'fctime': catfctime, 'fbtime': catfbtime, 'fmode': catfmode, 'fchmode': catfchmode, 'ftypemod': catftypemod, 'fwinattributes': catfwinattributes, 'fcompression': catfcompression, 'fcsize': catfcsize, 'fuid': catfuid, 'funame': catfuname, 'fgid': catfgid, 'fgname': catfgname, 'finode': catfinode, 'flinkcount': catflinkcount, 'fminor': catfdev_minor, 'fmajor': catfdev_major, 'frminor': catfrdev_minor, 'frmajor': catfrdev_major, 'fchecksumtype': catfchecksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': catfextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': catfcs, 'fcontentchecksum': catfccs, 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': catfcontents});
5007 fileidnum = fileidnum + 1;
5008 realidnum = realidnum + 1;
5009 if(returnfp):
5010 catlist.update({'catfp': catfp});
5011 else:
5012 catfp.close();
5013 return catlist;
5014
5015 create_alias_function("", __file_format_name__, "ToArray", ArchiveFileToArray);
5016
5017 def ArchiveFileStringToArray(catstr, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5018 formatspecs = FormatSpecsListToDict(formatspecs);
5019 catfp = BytesIO(catstr);
5020 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5021 return listcatfiles;
5022
5023 create_alias_function("", __file_format_name__, "StringToArray", ArchiveFileStringToArray);
5024
5025 def TarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5026 formatspecs = FormatSpecsListToDict(formatspecs);
5027 catfp = BytesIO();
5028 catfp = PackArchiveFileFromTarFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5029 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5030 return listcatfiles;
5031
5032 def ZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5033 formatspecs = FormatSpecsListToDict(formatspecs);
5034 catfp = BytesIO();
5035 catfp = PackArchiveFileFromZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5036 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5037 return listcatfiles;
5038
5039 if(not rarfile_support):
5040 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5041 return False;
5042
5043 if(rarfile_support):
5044 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5045 formatspecs = FormatSpecsListToDict(formatspecs);
5046 catfp = BytesIO();
5047 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5048 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5049 return listcatfiles;
5050
5051 if(not py7zr_support):
5052 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5053 return False;
5054
5055 if(py7zr_support):
5056 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5057 formatspecs = FormatSpecsListToDict(formatspecs);
5058 catfp = BytesIO();
5059 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5060 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5061 return listcatfiles;
5062
5063 def InFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5064 formatspecs = FormatSpecsListToDict(formatspecs);
5065 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
5066 if(checkcompressfile=="tarfile"):
5067 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5068 elif(checkcompressfile=="zipfile"):
5069 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5070 elif(checkcompressfile=="catfile"):
5071 return ArchiveFileToArray(infile, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5072 elif(rarfile_support and checkcompressfile=="rarfile"):
5073 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5074 elif(py7zr_support and checkcompressfile=="7zipfile"):
5075 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5076 else:
5077 return False;
5078 return False;
5079
5080 def ListDirToArrayAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5081 formatspecs = FormatSpecsListToDict(formatspecs);
5082 catver = formatspecs['format_ver'];
5083 fileheaderver = str(int(catver.replace(".", "")));
5084 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5085 advancedlist = formatspecs['use_advanced_list'];
5086 altinode = formatspecs['use_alt_inode'];
5087 infilelist = [];
5088 if(infiles=="-"):
5089 for line in sys.stdin:
5090 infilelist.append(line.strip());
5091 infilelist = list(filter(None, infilelist));
5092 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
5093 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
5094 return False;
5095 with UncompressFile(infiles, formatspecs, "r") as finfile:
5096 for line in finfile:
5097 infilelist.append(line.strip());
5098 infilelist = list(filter(None, infilelist));
5099 else:
5100 if(isinstance(infiles, (list, tuple, ))):
5101 infilelist = list(filter(None, infiles));
5102 elif(isinstance(infiles, (str, ))):
5103 infilelist = list(filter(None, [infiles]));
5104 if(advancedlist):
5105 GetDirList = ListDirAdvanced(infilelist, followlink, False);
5106 else:
5107 GetDirList = ListDir(infilelist, followlink, False);
5108 if(not GetDirList):
5109 return False;
5110 curinode = 0;
5111 curfid = 0;
5112 inodelist = [];
5113 inodetofile = {};
5114 filetoinode = {};
5115 inodetocatinode = {};
5116 fileidnum = 0;
5117 fnumfiles = int(len(GetDirList));
5118 catver = formatspecs['format_ver'];
5119 fileheaderver = str(int(catver.replace(".", "")));
5120 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5121 fnumfileshex = format(int(fnumfiles), 'x').lower();
5122 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5123 catversion = re.findall(r"([\d]+)", fileheader);
5124 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5125 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5126 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5127 fheadtell = len(fileheader);
5128 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
5129 for curfname in GetDirList:
5130 catfhstart = fheadtell;
5131 if(re.findall("^[.|/]", curfname)):
5132 fname = curfname;
5133 else:
5134 fname = "./"+curfname;
5135 if(verbose):
5136 VerbosePrintOut(fname);
5137 if(not followlink or followlink is None):
5138 fstatinfo = os.lstat(fname);
5139 else:
5140 fstatinfo = os.stat(fname);
5141 fpremode = fstatinfo.st_mode;
5142 finode = fstatinfo.st_ino;
5143 flinkcount = fstatinfo.st_nlink;
5144 ftype = 0;
5145 if(stat.S_ISREG(fpremode)):
5146 ftype = 0;
5147 elif(stat.S_ISLNK(fpremode)):
5148 ftype = 2;
5149 elif(stat.S_ISCHR(fpremode)):
5150 ftype = 3;
5151 elif(stat.S_ISBLK(fpremode)):
5152 ftype = 4;
5153 elif(stat.S_ISDIR(fpremode)):
5154 ftype = 5;
5155 elif(stat.S_ISFIFO(fpremode)):
5156 ftype = 6;
5157 elif(stat.S_ISSOCK(fpremode)):
5158 ftype = 8;
5159 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
5160 ftype = 9;
5161 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
5162 ftype = 10;
5163 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
5164 ftype = 11;
5165 else:
5166 ftype = 0;
5167 flinkname = "";
5168 fbasedir = os.path.dirname(fname);
5169 fcurfid = curfid;
5170 if(not followlink and finode!=0):
5171 if(ftype!=1):
5172 if(finode in inodelist):
5173 ftype = 1;
5174 flinkname = inodetofile[finode];
5175 if(altinode):
5176 fcurinode = finode;
5177 else:
5178 fcurinode = inodetocatinode[finode];
5179 if(finode not in inodelist):
5180 inodelist.append(finode);
5181 inodetofile.update({finode: fname});
5182 inodetocatinode.update({finode: curinode});
5183 if(altinode):
5184 fcurinode = finode;
5185 else:
5186 fcurinode = curinode;
5187 curinode = curinode + 1;
5188 else:
5189 fcurinode = curinode;
5190 curinode = curinode + 1;
5191 curfid = curfid + 1;
5192 if(ftype==2):
5193 flinkname = os.readlink(fname);
5194 fdev = fstatinfo.st_dev;
5195 getfdev = GetDevMajorMinor(fdev);
5196 fdev_minor = getfdev[0];
5197 fdev_major = getfdev[1];
5198 frdev = fstatinfo.st_dev;
5199 if(hasattr(fstatinfo, "st_rdev")):
5200 frdev = fstatinfo.st_rdev;
5201 else:
5202 frdev = fstatinfo.st_dev;
5203 getfrdev = GetDevMajorMinor(frdev);
5204 frdev_minor = getfrdev[0];
5205 frdev_major = getfrdev[1];
5206 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
5207 fsize = "0";
5208 if(ftype==0 or ftype==7):
5209 fsize = fstatinfo.st_size;
5210 fatime = fstatinfo.st_atime;
5211 fmtime = fstatinfo.st_mtime;
5212 fctime = fstatinfo.st_ctime;
5213 if(hasattr(fstatinfo, "st_birthtime")):
5214 fbtime = fstatinfo.st_birthtime;
5215 else:
5216 fbtime = fstatinfo.st_ctime;
5217 fmode = fstatinfo.st_mode;
5218 fchmode = stat.S_IMODE(fstatinfo.st_mode);
5219 ftypemod = stat.S_IFMT(fstatinfo.st_mode);
5220 fuid = fstatinfo.st_uid;
5221 fgid = fstatinfo.st_gid;
5222 funame = "";
5223 try:
5224 import pwd;
5225 try:
5226 userinfo = pwd.getpwuid(fstatinfo.st_uid);
5227 funame = userinfo.pw_name;
5228 except KeyError:
5229 funame = "";
5230 except ImportError:
5231 funame = "";
5232 fgname = "";
5233 try:
5234 import grp;
5235 try:
5236 groupinfo = grp.getgrgid(fstatinfo.st_gid);
5237 fgname = groupinfo.gr_name;
5238 except KeyError:
5239 fgname = "";
5240 except ImportError:
5241 fgname = "";
5242 fdev_minor = fdev_minor;
5243 fdev_major = fdev_major;
5244 frdev_minor = frdev_minor;
5245 frdev_major = frdev_major;
5246 flinkcount = flinkcount;
5247 if(hasattr(fstatinfo, "st_file_attributes")):
5248 fwinattributes = fstatinfo.st_file_attributes;
5249 else:
5250 fwinattributes = 0;
5251 fcompression = "";
5252 fcsize = 0;
5253 fcontents = BytesIO();
5254 if(ftype==0 or ftype==7):
5255 with open(fname, "rb") as fpc:
5256 shutil.copyfileobj(fpc, fcontents);
5257 if(followlink and (ftype==1 or ftype==2)):
5258 flstatinfo = os.stat(flinkname);
5259 with open(flinkname, "rb") as fpc:
5260 shutil.copyfileobj(fpc, fcontents);
5261 fcontents.seek(0, 0);
5262 ftypehex = format(ftype, 'x').lower();
5263 extrafields = len(extradata);
5264 extrafieldslist = extradata;
5265 catfextrafields = extrafields;
5266 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5267 if(len(extradata)>0):
5268 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5269 extrasizelen = len(extrasizestr);
5270 extrasizelenhex = format(extrasizelen, 'x').lower();
5271 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5272 catoutlen = len(catoutlist) + len(extradata) + 3;
5273 catoutlenhex = format(catoutlen, 'x').lower();
5274 catoutlist.insert(0, catoutlenhex);
5275 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5276 catheaderdata = catoutlist;
5277 if(len(extradata)>0):
5278 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5279 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
5280 catfnumfields = catoutlen;
5281 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5282 fcontents.seek(0, 0);
5283 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5284 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5285 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5286 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5287 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5288 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5289 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5290 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5291 catfcontentstart = fheadtell;
5292 fheadtell += len(catfileoutstr) + 1;
5293 catfcontentend = fheadtell - 1;
5294 catfhend = catfcontentend;
5295 fcontents.seek(0, 0);
5296 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5297 pyhascontents = False;
5298 if(int(fsize)>0 and not listonly):
5299 pyhascontents = True;
5300 if(int(fsize)>0 and listonly):
5301 fcontents = BytesIO();
5302 pyhascontents = False;
5303 fcontents.seek(0, 0);
5304 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents});
5305 fileidnum = fileidnum + 1;
5306 return catlist;
5307
5308 def TarFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5309 formatspecs = FormatSpecsListToDict(formatspecs);
5310 curinode = 0;
5311 curfid = 0;
5312 inodelist = [];
5313 inodetofile = {};
5314 filetoinode = {};
5315 inodetocatinode = {};
5316 fileidnum = 0;
5317 if(infile=="-"):
5318 infile = BytesIO();
5319 if(hasattr(sys.stdin, "buffer")):
5320 shutil.copyfileobj(sys.stdin.buffer, infile);
5321 else:
5322 shutil.copyfileobj(sys.stdin, infile);
5323 infile.seek(0, 0);
5324 if(not infile):
5325 return False;
5326 infile.seek(0, 0);
5327 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
5328 infile = download_file_from_internet_file(infile);
5329 infile.seek(0, 0);
5330 if(not infile):
5331 return False;
5332 infile.seek(0, 0);
5333 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
5334 return False;
5335 elif(os.path.exists(infile) and os.path.isfile(infile)):
5336 try:
5337 if(not tarfile.TarFileCheck(infile)):
5338 return False;
5339 except AttributeError:
5340 if(not TarFileCheck(infile)):
5341 return False;
5342 else:
5343 return False;
5344 try:
5345 if(hasattr(infile, "read") or hasattr(infile, "write")):
5346 tarfp = tarfile.open(fileobj=infile, mode="r");
5347 else:
5348 tarfp = tarfile.open(infile, "r");
5349 except FileNotFoundError:
5350 return False;
5351 fnumfiles = int(len(tarfp.getmembers()));
5352 catver = formatspecs['format_ver'];
5353 fileheaderver = str(int(catver.replace(".", "")));
5354 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5355 fnumfileshex = format(int(fnumfiles), 'x').lower();
5356 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5357 catversion = re.findall(r"([\d]+)", fileheader);
5358 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5359 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5360 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5361 fheadtell = len(fileheader);
5362 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
5363 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
5364 catfhstart = fheadtell;
5365 if(re.findall("^[.|/]", member.name)):
5366 fname = member.name;
5367 else:
5368 fname = "./"+member.name;
5369 if(verbose):
5370 VerbosePrintOut(fname);
5371 fpremode = member.mode;
5372 ffullmode = member.mode;
5373 flinkcount = 0;
5374 ftype = 0;
5375 if(member.isreg()):
5376 ffullmode = member.mode + stat.S_IFREG;
5377 ftype = 0;
5378 elif(member.isdev()):
5379 ffullmode = member.mode;
5380 ftype = 7;
5381 elif(member.islnk()):
5382 ffullmode = member.mode + stat.S_IFREG;
5383 ftype = 1;
5384 elif(member.issym()):
5385 ffullmode = member.mode + stat.S_IFLNK;
5386 ftype = 2;
5387 elif(member.ischr()):
5388 ffullmode = member.mode + stat.S_IFCHR;
5389 ftype = 3;
5390 elif(member.isblk()):
5391 ffullmode = member.mode + stat.S_IFBLK;
5392 ftype = 4;
5393 elif(member.isdir()):
5394 ffullmode = member.mode + stat.S_IFDIR;
5395 ftype = 5;
5396 elif(member.isfifo()):
5397 ffullmode = member.mode + stat.S_IFIFO;
5398 ftype = 6;
5399 elif(member.issparse()):
5400 ffullmode = member.mode;
5401 ftype = 12;
5402 else:
5403 ffullmode = member.mode;
5404 ftype = 0;
5405 flinkname = "";
5406 fbasedir = os.path.dirname(fname);
5407 fcurfid = curfid;
5408 fcurinode = curfid;
5409 finode = fcurinode;
5410 curfid = curfid + 1;
5411 if(ftype==2):
5412 flinkname = member.linkname;
5413 fdev_minor = member.devminor;
5414 fdev_major = member.devmajor;
5415 frdev_minor = member.devminor;
5416 frdev_major = member.devmajor;
5417 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
5418 fsize = "0";
5419 elif(ftype==0 or ftype==7):
5420 fsize = member.size;
5421 else:
5422 fsize = member.size;
5423 fatime = member.mtime;
5424 fmtime = member.mtime;
5425 fctime = member.mtime;
5426 fbtime = member.mtime;
5427 fmode = ffullmode;
5428 fchmode = stat.S_IMODE(ffullmode);
5429 ftypemod = stat.S_IFMT(ffullmode);
5430 fuid = member.uid;
5431 fgid = member.gid;
5432 funame = member.uname;
5433 fgname = member.gname;
5434 flinkcount = flinkcount;
5435 fwinattributes = int(0);
5436 fcompression = "";
5437 fcsize = 0;
5438 fcontents = BytesIO();
5439 if(ftype==0 or ftype==7):
5440 with tarfp.extractfile(member) as fpc:
5441 shutil.copyfileobj(fpc, fcontents);
5442 fcontents.seek(0, 0);
5443 ftypehex = format(ftype, 'x').lower();
5444 extrafields = len(extradata);
5445 extrafieldslist = extradata;
5446 catfextrafields = extrafields;
5447 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5448 if(len(extradata)>0):
5449 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5450 extrasizelen = len(extrasizestr);
5451 extrasizelenhex = format(extrasizelen, 'x').lower();
5452 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5453 catoutlen = len(catoutlist) + len(extradata) + 3;
5454 catoutlenhex = format(catoutlen, 'x').lower();
5455 catoutlist.insert(0, catoutlenhex);
5456 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5457 catheaderdata = catoutlist;
5458 if(len(extradata)>0):
5459 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5460 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
5461 catfnumfields = catoutlen;
5462 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5463 fcontents.seek(0, 0);
5464 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5465 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5466 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5467 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5468 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5469 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5470 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5471 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5472 catfcontentstart = fheadtell;
5473 fheadtell += len(catfileoutstr) + 1;
5474 catfcontentend = fheadtell - 1;
5475 catfhend = catfcontentend;
5476 fcontents.seek(0, 0);
5477 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5478 pyhascontents = False;
5479 if(int(fsize)>0 and not listonly):
5480 pyhascontents = True;
5481 if(int(fsize)>0 and listonly):
5482 fcontents = BytesIO();
5483 pyhascontents = False;
5484 fcontents.seek(0, 0);
5485 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents});
5486 fileidnum = fileidnum + 1;
5487 return catlist;
5488
5489 def ZipFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5490 formatspecs = FormatSpecsListToDict(formatspecs);
5491 curinode = 0;
5492 curfid = 0;
5493 inodelist = [];
5494 inodetofile = {};
5495 filetoinode = {};
5496 inodetocatinode = {};
5497 fileidnum = 0;
5498 if(infile=="-"):
5499 infile = BytesIO();
5500 if(hasattr(sys.stdin, "buffer")):
5501 shutil.copyfileobj(sys.stdin.buffer, infile);
5502 else:
5503 shutil.copyfileobj(sys.stdin, infile);
5504 infile.seek(0, 0);
5505 if(not infile):
5506 return False;
5507 infile.seek(0, 0);
5508 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
5509 infile = download_file_from_internet_file(infile);
5510 infile.seek(0, 0);
5511 if(not infile):
5512 return False;
5513 infile.seek(0, 0);
5514 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
5515 return False;
5516 else:
5517 return False;
5518 if(not zipfile.is_zipfile(infile)):
5519 return False;
5520 try:
5521 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
5522 except FileNotFoundError:
5523 return False;
5524 ziptest = zipfp.testzip();
5525 if(ziptest):
5526 VerbosePrintOut("Bad file found!");
5527 fnumfiles = int(len(zipfp.infolist()));
5528 catver = formatspecs['format_ver'];
5529 fileheaderver = str(int(catver.replace(".", "")));
5530 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5531 catversion = re.findall(r"([\d]+)", fileheader);
5532 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5533 fnumfileshex = format(int(fnumfiles), 'x').lower();
5534 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5535 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5536 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5537 fheadtell = len(fileheader);
5538 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
5539 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
5540 catfhstart = fheadtell;
5541 if(re.findall("^[.|/]", member.filename)):
5542 fname = member.filename;
5543 else:
5544 fname = "./"+member.filename;
5545 zipinfo = zipfp.getinfo(member.filename);
5546 if(verbose):
5547 VerbosePrintOut(fname);
5548 if(not member.is_dir()):
5549 fpremode = stat.S_IFREG + 438;
5550 elif(member.is_dir()):
5551 fpremode = stat.S_IFDIR + 511;
5552 flinkcount = 0;
5553 ftype = 0;
5554 if(not member.is_dir()):
5555 ftype = 0;
5556 elif(member.is_dir()):
5557 ftype = 5;
5558 flinkname = "";
5559 fbasedir = os.path.dirname(fname);
5560 fcurfid = curfid;
5561 fcurinode = curfid;
5562 finode = fcurinode;
5563 curfid = curfid + 1;
5564 fdev_minor = 0;
5565 fdev_major = 0;
5566 frdev_minor = 0;
5567 frdev_major = 0;
5568 if(ftype==5):
5569 fsize = "0";
5570 elif(ftype==0):
5571 fsize = member.file_size;
5572 else:
5573 fsize = member.file_size;
5574 fatime = time.mktime(member.date_time + (0, 0, -1));
5575 fmtime = time.mktime(member.date_time + (0, 0, -1));
5576 fctime = time.mktime(member.date_time + (0, 0, -1));
5577 fbtime = time.mktime(member.date_time + (0, 0, -1));
5578 if(zipinfo.create_system==0 or zipinfo.create_system==10):
5579 fwinattributes = int(zipinfo.external_attr);
5580 if(not member.is_dir()):
5581 fmode = int(stat.S_IFREG + 438);
5582 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
5583 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
5584 elif(member.is_dir()):
5585 fmode = int(stat.S_IFDIR + 511);
5586 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
5587 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
5588 elif(zipinfo.create_system==3):
5589 fwinattributes = int(0);
5590 fmode = int(zipinfo.external_attr);
5591 else:
5592 fwinattributes = int(0);
5593 if(not member.is_dir()):
5594 fmode = int(stat.S_IFREG + 438);
5595 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
5596 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
5597 elif(member.is_dir()):
5598 fmode = int(stat.S_IFDIR + 511);
5599 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
5600 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
5601 fcompression = "";
5602 fcsize = 0;
5603 try:
5604 fuid = os.getuid();
5605 except AttributeError:
5606 fuid = 0;
5607 except KeyError:
5608 fuid = 0;
5609 try:
5610 fgid = os.getgid();
5611 except AttributeError:
5612 fgid = 0;
5613 except KeyError:
5614 fgid = 0;
5615 try:
5616 import pwd;
5617 try:
5618 userinfo = pwd.getpwuid(os.getuid());
5619 funame = userinfo.pw_name;
5620 except KeyError:
5621 funame = "";
5622 except AttributeError:
5623 funame = "";
5624 except ImportError:
5625 funame = "";
5626 fgname = "";
5627 try:
5628 import grp;
5629 try:
5630 groupinfo = grp.getgrgid(os.getgid());
5631 fgname = groupinfo.gr_name;
5632 except KeyError:
5633 fgname = "";
5634 except AttributeError:
5635 fgname = "";
5636 except ImportError:
5637 fgname = "";
5638 fcontents = BytesIO();
5639 if(ftype==0):
5640 fcontents.write(zipfp.read(member.filename));
5641 fcontents.seek(0, 0);
5642 ftypehex = format(ftype, 'x').lower();
5643 extrafields = len(extradata);
5644 extrafieldslist = extradata;
5645 catfextrafields = extrafields;
5646 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5647 if(len(extradata)>0):
5648 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5649 extrasizelen = len(extrasizestr);
5650 extrasizelenhex = format(extrasizelen, 'x').lower();
5651 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5652 catoutlen = len(catoutlist) + len(extradata) + 3;
5653 catoutlenhex = format(catoutlen, 'x').lower();
5654 catoutlist.insert(0, catoutlenhex);
5655 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5656 catheaderdata = catoutlist;
5657 if(len(extradata)>0):
5658 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5659 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
5660 catfnumfields = catoutlen;
5661 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5662 fcontents.seek(0, 0);
5663 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5664 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5665 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5666 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5667 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5668 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5669 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5670 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5671 catfcontentstart = fheadtell;
5672 fheadtell += len(catfileoutstr) + 1;
5673 catfcontentend = fheadtell - 1;
5674 catfhend = catfcontentend;
5675 fcontents.seek(0, 0);
5676 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5677 pyhascontents = False;
5678 if(int(fsize)>0 and not listonly):
5679 pyhascontents = True;
5680 if(int(fsize)>0 and listonly):
5681 fcontents = BytesIO();
5682 pyhascontents = False;
5683 fcontents.seek(0, 0);
5684 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents});
5685 fileidnum = fileidnum + 1;
5686 return catlist;
5687
5688 if(not rarfile_support):
5689 def RarFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5690 return False;
5691
5692 if(rarfile_support):
5693 def RarFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5694 formatspecs = FormatSpecsListToDict(formatspecs);
5695 curinode = 0;
5696 curfid = 0;
5697 inodelist = [];
5698 inodetofile = {};
5699 filetoinode = {};
5700 inodetocatinode = {};
5701 fileidnum = 0;
5702 if(not os.path.exists(infile,) or not os.path.isfile(infile,)):
5703 return False;
5704 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
5705 return False;
5706 rarfp = rarfile.RarFile(infile, "r");
5707 rartest = rarfp.testrar();
5708 if(rartest):
5709 VerbosePrintOut("Bad file found!");
5710 fnumfiles = int(len(rarfp.infolist()));
5711 catver = formatspecs['format_ver'];
5712 fileheaderver = str(int(catver.replace(".", "")));
5713 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5714 catversion = re.findall(r"([\d]+)", fileheader);
5715 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5716 fnumfileshex = format(int(fnumfiles), 'x').lower();
5717 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5718 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5719 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5720 fheadtell = len(fileheader);
5721 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
5722 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
5723 catfhstart = fheadtell;
5724 is_unix = False;
5725 is_windows = False;
5726 if(member.host_os==rarfile.RAR_OS_UNIX):
5727 is_windows = False;
5728 try:
5729 member.external_attr
5730 is_unix = True;
5731 except AttributeError:
5732 is_unix = False;
5733 elif(member.host_os==rarfile.RAR_OS_WIN32):
5734 is_unix = False;
5735 try:
5736 member.external_attr
5737 is_windows = True;
5738 except AttributeError:
5739 is_windows = False;
5740 else:
5741 is_unix = False;
5742 is_windows = False;
5743 if(re.findall("^[.|/]", member.filename)):
5744 fname = member.filename;
5745 else:
5746 fname = "./"+member.filename;
5747 rarinfo = rarfp.getinfo(member.filename);
5748 if(verbose):
5749 VerbosePrintOut(fname);
5750 if(is_unix and member.external_attr !=0):
5751 fpremode = int(member.external_attr);
5752 elif(member.is_file()):
5753 fpremode = stat.S_IFREG + 438;
5754 elif(member.is_symlink()):
5755 fpremode = stat.S_IFLNK + 438;
5756 elif(member.is_dir()):
5757 fpremode = stat.S_IFDIR + 511;
5758 if(is_windows and member.external_attr !=0):
5759 fwinattributes = int(member.external_attr);
5760 else:
5761 fwinattributes = int(0);
5762 fcompression = "";
5763 fcsize = 0;
5764 flinkcount = 0;
5765 ftype = 0;
5766 if(member.is_file()):
5767 ftype = 0;
5768 elif(member.is_symlink()):
5769 ftype = 2;
5770 elif(member.is_dir()):
5771 ftype = 5;
5772 flinkname = "";
5773 if(ftype==2):
5774 flinkname = rarfp.read(member.filename).decode("UTF-8");
5775 fbasedir = os.path.dirname(fname);
5776 fcurfid = curfid;
5777 fcurinode = curfid;
5778 finode = fcurinode;
5779 curfid = curfid + 1;
5780 fdev_minor = 0;
5781 fdev_major = 0;
5782 frdev_minor = 0;
5783 frdev_major = 0;
5784 if(ftype==5):
5785 fsize = "0";
5786 if(ftype==0):
5787 fsize = member.file_size;
5788 try:
5789 if(member.atime):
5790 fatime = int(member.atime.timestamp());
5791 else:
5792 fatime = int(member.mtime.timestamp());
5793 except AttributeError:
5794 fatime = int(member.mtime.timestamp());
5795 fmtime = int(member.mtime.timestamp());
5796 try:
5797 if(member.ctime):
5798 fctime = int(member.ctime.timestamp());
5799 else:
5800 fctime = int(member.mtime.timestamp());
5801 except AttributeError:
5802 fctime = int(member.mtime.timestamp());
5803 fbtime = int(member.mtime.timestamp());
5804 if(is_unix and member.external_attr !=0):
5805 fmode = int(member.external_attr);
5806 fchmode = int(stat.S_IMODE(member.external_attr));
5807 ftypemod = int(stat.S_IFMT(member.external_attr));
5808 elif(member.is_file()):
5809 fmode = int(stat.S_IFREG + 438)
5810 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5811 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5812 elif(member.is_symlink()):
5813 fmode = int(stat.S_IFLNK + 438)
5814 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5815 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5816 elif(member.is_dir()):
5817 fmode = int(stat.S_IFDIR + 511)
5818 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
5819 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
5820 try:
5821 fuid = os.getuid();
5822 except AttributeError:
5823 fuid = 0;
5824 except KeyError:
5825 fuid = 0;
5826 try:
5827 fgid = os.getgid();
5828 except AttributeError:
5829 fgid = 0;
5830 except KeyError:
5831 fgid = 0;
5832 try:
5833 import pwd;
5834 try:
5835 userinfo = pwd.getpwuid(os.getuid());
5836 funame = userinfo.pw_name;
5837 except KeyError:
5838 funame = "";
5839 except AttributeError:
5840 funame = "";
5841 except ImportError:
5842 funame = "";
5843 fgname = "";
5844 try:
5845 import grp;
5846 try:
5847 groupinfo = grp.getgrgid(os.getgid());
5848 fgname = groupinfo.gr_name;
5849 except KeyError:
5850 fgname = "";
5851 except AttributeError:
5852 fgname = "";
5853 except ImportError:
5854 fgname = "";
5855 fcontents = BytesIO();
5856 if(ftype==0):
5857 fcontents.write(rarfp.read(member.filename));
5858 fcontents.seek(0, 0);
5859 ftypehex = format(ftype, 'x').lower();
5860 extrafields = len(extradata);
5861 extrafieldslist = extradata;
5862 catfextrafields = extrafields;
5863 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5864 if(len(extradata)>0):
5865 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5866 extrasizelen = len(extrasizestr);
5867 extrasizelenhex = format(extrasizelen, 'x').lower();
5868 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5869 catoutlen = len(catoutlist) + len(extradata) + 3;
5870 catoutlenhex = format(catoutlen, 'x').lower();
5871 catoutlist.insert(0, catoutlenhex);
5872 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5873 if(len(extradata)>0):
5874 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5875 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
5876 catfnumfields = 24 + catfextrafields;
5877 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5878 fcontents.seek(0, 0);
5879 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5880 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5881 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5882 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5883 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5884 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5885 catheaderdata = catoutlist;
5886 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5887 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5888 catfcontentstart = fheadtell;
5889 fheadtell += len(catfileoutstr) + 1;
5890 catfcontentend = fheadtell - 1;
5891 catfhend = catfcontentend;
5892 fcontents.seek(0, 0);
5893 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5894 pyhascontents = False;
5895 if(int(fsize)>0 and not listonly):
5896 pyhascontents = True;
5897 if(int(fsize)>0 and listonly):
5898 fcontents = BytesIO();
5899 pyhascontents = False;
5900 fcontents.seek(0, 0);
5901 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents});
5902 fileidnum = fileidnum + 1;
5903 return catlist;
5904
5905 if(not py7zr_support):
5906 def SevenZipFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5907 return False;
5908
5909 if(py7zr_support):
5910 def SevenZipFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5911 formatspecs = FormatSpecsListToDict(formatspecs);
5912 curinode = 0;
5913 curfid = 0;
5914 inodelist = [];
5915 inodetofile = {};
5916 filetoinode = {};
5917 inodetocatinode = {};
5918 fileidnum = 0;
5919 szpfp = py7zr.SevenZipFile(infile, mode="r");
5920 file_content = szpfp.readall();
5921 #sztest = szpfp.testzip();
5922 sztestalt = szpfp.test();
5923 if(sztestalt):
5924 VerbosePrintOut("Bad file found!");
5925 numfiles = int(len(szpfp.list()));
5926 catver = formatspecs['format_ver'];
5927 fileheaderver = str(int(catver.replace(".", "")));
5928 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5929 catversion = re.findall(r"([\d]+)", fileheader);
5930 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5931 fnumfileshex = format(int(fnumfiles), 'x').lower();
5932 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5933 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5934 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5935 fheadtell = len(fileheader);
5936 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
5937 for member in sorted(szpfp.list(), key=lambda x: x.filename):
5938 catfhstart = fheadtell;
5939 if(re.findall("^[.|/]", member.filename)):
5940 fname = member.filename;
5941 else:
5942 fname = "./"+member.filename;
5943 if(not member.is_directory):
5944 fpremode = int(stat.S_IFREG + 438);
5945 elif(member.is_directory):
5946 fpremode = int(stat.S_IFDIR + 511);
5947 fwinattributes = int(0);
5948 fcompression = "";
5949 fcsize = 0;
5950 flinkcount = 0;
5951 ftype = 0;
5952 if(member.is_directory):
5953 ftype = 5;
5954 else:
5955 ftype = 0;
5956 flinkname = "";
5957 fbasedir = os.path.dirname(fname);
5958 fcurfid = curfid;
5959 fcurinode = curfid;
5960 finode = fcurinode;
5961 curfid = curfid + 1;
5962 fdev_minor = 0;
5963 fdev_major = 0;
5964 frdev_minor = 0;
5965 frdev_major = 0;
5966 if(ftype==5):
5967 fsize = "0";
5968 fatime = int(member.creationtime.timestamp());
5969 fmtime = int(member.creationtime.timestamp());
5970 fctime = int(member.creationtime.timestamp());
5971 fbtime = int(member.creationtime.timestamp());
5972 if(member.is_directory):
5973 fmode = int(stat.S_IFDIR + 511)
5974 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
5975 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
5976 else:
5977 fmode = int(stat.S_IFLNK + 438)
5978 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5979 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5980 try:
5981 fuid = os.getuid();
5982 except AttributeError:
5983 fuid = 0;
5984 except KeyError:
5985 fuid = 0;
5986 try:
5987 fgid = os.getgid();
5988 except AttributeError:
5989 fgid = 0;
5990 except KeyError:
5991 fgid = 0;
5992 try:
5993 import pwd;
5994 try:
5995 userinfo = pwd.getpwuid(os.getuid());
5996 funame = userinfo.pw_name;
5997 except KeyError:
5998 funame = "";
5999 except AttributeError:
6000 funame = "";
6001 except ImportError:
6002 funame = "";
6003 fgname = "";
6004 try:
6005 import grp;
6006 try:
6007 groupinfo = grp.getgrgid(os.getgid());
6008 fgname = groupinfo.gr_name;
6009 except KeyError:
6010 fgname = "";
6011 except AttributeError:
6012 fgname = "";
6013 except ImportError:
6014 fgname = "";
6015 fcontents = BytesIO();
6016 if(ftype==0):
6017 fcontents.write(file_content[member.filename].read());
6018 fsize = format(fcontents.tell(), 'x').lower();
6019 fileop.close();
6020 fcontents.seek(0, 0);
6021 ftypehex = format(ftype, 'x').lower();
6022 extrafields = len(extradata);
6023 extrafieldslist = extradata;
6024 catfextrafields = extrafields;
6025 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
6026 if(len(extradata)>0):
6027 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
6028 extrasizelen = len(extrasizestr);
6029 extrasizelenhex = format(extrasizelen, 'x').lower();
6030 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
6031 catoutlen = len(catoutlist) + len(extradata) + 3;
6032 catoutlenhex = format(catoutlen, 'x').lower();
6033 catoutlist.insert(0, catoutlenhex);
6034 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
6035 catheaderdata = catoutlist;
6036 if(len(extradata)>0):
6037 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
6038 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
6039 catfnumfields = 24 + catfextrafields;
6040 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
6041 fcontents.seek(0, 0);
6042 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
6043 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
6044 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
6045 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
6046 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
6047 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
6048 catfileoutstrecd = catfileoutstr.encode('UTF-8');
6049 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
6050 catfcontentstart = fheadtell;
6051 fheadtell += len(catfileoutstr) + 1;
6052 catfcontentend = fheadtell - 1;
6053 catfhend = catfcontentend;
6054 fcontents.seek(0, 0);
6055 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
6056 pyhascontents = False;
6057 if(int(fsize)>0 and not listonly):
6058 pyhascontents = True;
6059 if(int(fsize)>0 and listonly):
6060 fcontents = BytesIO();
6061 pyhascontents = False;
6062 fcontents.seek(0, 0);
6063 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents});
6064 fileidnum = fileidnum + 1;
6065 return catlist;
6066
6067 def InFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6068 formatspecs = FormatSpecsListToDict(formatspecs);
6069 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
6070 if(checkcompressfile=="tarfile"):
6071 return TarFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6072 elif(checkcompressfile=="zipfile"):
6073 return ZipFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6074 elif(checkcompressfile=="catfile"):
6075 return ArchiveFileToArray(infile, 0, 0, listonly, True, False, formatspecs, False);
6076 elif(rarfile_support and checkcompressfile=="rarfile"):
6077 return RarFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6078 elif(py7zr_support and checkcompressfile=="7zipfile"):
6079 return SevenZipFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6080 else:
6081 return False;
6082 return False;
6083
6084 def ListDirToArray(infiles, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, listonly=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6085 formatspecs = FormatSpecsListToDict(formatspecs);
6086 outarray = BytesIO();
6087 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
6088 listcatfiles = ArchiveFileToArray(outarray, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
6089 return listcatfiles;
6090
6091 def ArchiveFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6092 formatspecs = FormatSpecsListToDict(formatspecs);
6093 if(isinstance(infile, dict)):
6094 listcatfiles = infile;
6095 else:
6096 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6097 infile = RemoveWindowsPath(infile);
6098 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, listonly, uncompress, skipchecksum, formatspecs, returnfp);
6099 if(not listcatfiles):
6100 return False;
6101 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6102 if(returnfp):
6103 catarray.update({'catfp': listcatfiles['catfp']});
6104 lenlist = len(listcatfiles['ffilelist']);
6105 lcfi = 0;
6106 lcfx = int(listcatfiles['fnumfiles']);
6107 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6108 lcfx = int(lenlist);
6109 else:
6110 lcfx = int(listcatfiles['fnumfiles']);
6111 while(lcfi < lcfx):
6112 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6113 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6114 catarray['filetoid'].update(filetoidarray);
6115 catarray['idtofile'].update(idtofilearray);
6116 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6117 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6118 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6119 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6120 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6121 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6122 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6123 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6124 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6125 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6126 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6127 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6128 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6129 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6130 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6131 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6132 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6133 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6134 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6135 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6136 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6137 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6138 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6139 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6140 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6141 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6142 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6143 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6144 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6145 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6146 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6147 lcfi = lcfi + 1;
6148 return catarray;
6149
6150 create_alias_function("", __file_format_name__, "ToArrayIndex", ArchiveFileToArrayIndex);
6151
6152 def ListDirToArrayIndexAlt(infiles, dirlistfromtxt=False, followlink=False, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6153 formatspecs = FormatSpecsListToDict(formatspecs);
6154 listcatfiles = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, listonly, checksumtype, extradata, formatspecs, verbose);
6155 if(not listcatfiles):
6156 return False;
6157 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6158 lenlist = len(listcatfiles['ffilelist']);
6159 lcfi = 0;
6160 lcfx = int(listcatfiles['fnumfiles']);
6161 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6162 lcfx = int(lenlist);
6163 else:
6164 lcfx = int(listcatfiles['fnumfiles']);
6165 while(lcfi < lcfx):
6166 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6167 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6168 catarray['filetoid'].update(filetoidarray);
6169 catarray['idtofile'].update(idtofilearray);
6170 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6171 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6172 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6173 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6174 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6175 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6176 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6177 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6178 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6179 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6180 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6181 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6182 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6183 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6184 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6185 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6186 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6187 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6188 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6189 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6190 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6191 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6192 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6193 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6194 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6195 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6196 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6197 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6198 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6199 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6200 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6201 lcfi = lcfi + 1;
6202 return catarray;
6203
6204 def TarFileToArrayIndexAlt(infile, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6205 formatspecs = FormatSpecsListToDict(formatspecs);
6206 listcatfiles = TarFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6207 if(not listcatfiles):
6208 return False;
6209 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6210 lenlist = len(listcatfiles['ffilelist']);
6211 lcfi = 0;
6212 lcfx = int(listcatfiles['fnumfiles']);
6213 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6214 lcfx = int(lenlist);
6215 else:
6216 lcfx = int(listcatfiles['fnumfiles']);
6217 while(lcfi < lcfx):
6218 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6219 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6220 catarray['filetoid'].update(filetoidarray);
6221 catarray['idtofile'].update(idtofilearray);
6222 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6223 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6224 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6225 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6226 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6227 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6228 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6229 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6230 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6231 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6232 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6233 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6234 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6235 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6236 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6237 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6238 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6239 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6240 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6241 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6242 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6243 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6244 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6245 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6246 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6247 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6248 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6249 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6250 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6251 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6252 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6253 lcfi = lcfi + 1;
6254 return catarray;
6255
6256 def ZipFileToArrayIndexAlt(infile, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6257 formatspecs = FormatSpecsListToDict(formatspecs);
6258 listcatfiles = ZipFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6259 if(not listcatfiles):
6260 return False;
6261 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6262 lenlist = len(listcatfiles['ffilelist']);
6263 lcfi = 0;
6264 lcfx = int(listcatfiles['fnumfiles']);
6265 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6266 lcfx = int(lenlist);
6267 else:
6268 lcfx = int(listcatfiles['fnumfiles']);
6269 while(lcfi < lcfx):
6270 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6271 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6272 catarray['filetoid'].update(filetoidarray);
6273 catarray['idtofile'].update(idtofilearray);
6274 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6275 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6276 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6277 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6278 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6279 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6280 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6281 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6282 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6283 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6284 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6285 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6286 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6287 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6288 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6289 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6290 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6291 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6292 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6293 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6294 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6295 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6296 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6297 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6298 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6299 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6300 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6301 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6302 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6303 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6304 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6305 lcfi = lcfi + 1;
6306 return catarray;
6307
6308 if(not rarfile_support):
6309 def RarFileToArrayIndexAlt(infile, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6310 return False;
6311
6312 if(rarfile_support):
6313 def RarFileToArrayIndexAlt(infile, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6314 formatspecs = FormatSpecsListToDict(formatspecs);
6315 listcatfiles = RarFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6316 if(not listcatfiles):
6317 return False;
6318 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6319 lenlist = len(listcatfiles['ffilelist']);
6320 lcfi = 0;
6321 lcfx = int(listcatfiles['fnumfiles']);
6322 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6323 lcfx = int(lenlist);
6324 else:
6325 lcfx = int(listcatfiles['fnumfiles']);
6326 while(lcfi < lcfx):
6327 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6328 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6329 catarray['filetoid'].update(filetoidarray);
6330 catarray['idtofile'].update(idtofilearray);
6331 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6332 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6333 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6334 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6335 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6336 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6337 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6338 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6339 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6340 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6341 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6342 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6343 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6344 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6345 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6346 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6347 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6348 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6349 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6350 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6351 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6352 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6353 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6354 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6355 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6356 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6357 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6358 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6359 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6360 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6361 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6362 lcfi = lcfi + 1;
6363 return catarray;
6364
6365 if(not py7zr_support):
6366 def SevenZipFileToArrayIndexAlt(infile, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6367 return False;
6368
6369 if(py7zr_support):
6370 def SevenZipFileToArrayIndexAlt(infile, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6371 formatspecs = FormatSpecsListToDict(formatspecs);
6372 listcatfiles = SevenZipFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6373 if(not listcatfiles):
6374 return False;
6375 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6376 lenlist = len(listcatfiles['ffilelist']);
6377 lcfi = 0;
6378 lcfx = int(listcatfiles['fnumfiles']);
6379 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6380 lcfx = int(lenlist);
6381 else:
6382 lcfx = int(listcatfiles['fnumfiles']);
6383 while(lcfi < lcfx):
6384 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6385 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6386 catarray['filetoid'].update(filetoidarray);
6387 catarray['idtofile'].update(idtofilearray);
6388 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6389 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6390 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6391 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6392 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6393 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6394 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6395 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6396 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6397 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6398 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6399 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6400 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6401 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6402 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6403 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6404 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6405 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6406 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6407 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6408 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6409 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6410 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6411 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6412 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6413 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6414 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6415 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6416 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6417 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6418 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6419 lcfi = lcfi + 1;
6420 return catarray;
6421
6422 def ArchiveFileStringToArrayIndex(catstr, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6423 formatspecs = FormatSpecsListToDict(formatspecs);
6424 catfp = BytesIO(catstr);
6425 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, uncompress, skipchecksum, formatspecs, returnfp);
6426 return listcatfiles;
6427
6428 create_alias_function("", __file_format_name__, "StringToArrayIndex", ArchiveFileStringToArrayIndex);
6429
6430 def TarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6431 formatspecs = FormatSpecsListToDict(formatspecs);
6432 catfp = BytesIO();
6433 catfp = PackArchiveFileFromTarFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
6434 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
6435 return listcatfiles;
6436
6437 def ZipFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6438 formatspecs = FormatSpecsListToDict(formatspecs);
6439 catfp = BytesIO();
6440 catfp = PackArchiveFileFromZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
6441 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
6442 return listcatfiles;
6443
6444 if(not rarfile_support):
6445 def RarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6446 return False;
6447
6448 if(rarfile_support):
6449 def RarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6450 formatspecs = FormatSpecsListToDict(formatspecs);
6451 catfp = BytesIO();
6452 catfp = PackArchiveFileFromRarFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
6453 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
6454 return listcatfiles;
6455
6456 if(not py7zr_support):
6457 def SevenZipFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6458 return False;
6459
6460 if(py7zr_support):
6461 def SevenZipFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6462 formatspecs = FormatSpecsListToDict(formatspecs);
6463 catfp = BytesIO();
6464 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
6465 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
6466 return listcatfiles;
6467
6468 def ListDirToArrayIndex(infiles, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, listonly=False, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6469 formatspecs = FormatSpecsListToDict(formatspecs);
6470 outarray = BytesIO();
6471 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose, True);
6472 listcatfiles = ArchiveFileToArrayIndex(outarray, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp)
6473 return listcatfiles;
6474
6475 def RePackArchiveFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6476 formatspecs = FormatSpecsListToDict(formatspecs);
6477 if(isinstance(infile, dict)):
6478 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, True, skipchecksum, formatspecs, returnfp);
6479 listcatfiles = prelistcatfiles['list'];
6480 else:
6481 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6482 infile = RemoveWindowsPath(infile);
6483 if(followlink):
6484 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, True, skipchecksum, formatspecs, returnfp);
6485 listcatfiles = prelistcatfiles['list'];
6486 else:
6487 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, True, skipchecksum, formatspecs, returnfp);
6488 if(outfile!="-" and not hasattr(infile, "read") and not hasattr(outfile, "write")):
6489 outfile = RemoveWindowsPath(outfile);
6490 checksumtype = checksumtype.lower();
6491 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
6492 checksumtype="crc32";
6493 if(checksumtype=="none"):
6494 checksumtype = "";
6495 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
6496 compression = "auto";
6497 if(compression not in compressionlist and compression is None):
6498 compression = "auto";
6499 if(verbose):
6500 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6501 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
6502 if(os.path.exists(outfile)):
6503 try:
6504 os.unlink(outfile);
6505 except OSError as e:
6506 pass;
6507 if(not listcatfiles):
6508 return False;
6509 if(outfile=="-"):
6510 verbose = False;
6511 catfp = BytesIO();
6512 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
6513 catfp = outfile;
6514 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
6515 catfp = BytesIO();
6516 else:
6517 fbasename = os.path.splitext(outfile)[0];
6518 fextname = os.path.splitext(outfile)[1];
6519 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
6520 catver = formatspecs['format_ver'];
6521 fileheaderver = str(int(catver.replace(".", "")));
6522 lenlist = len(listcatfiles['ffilelist']);
6523 fnumfiles = int(listcatfiles['fnumfiles']);
6524 if(lenlist>fnumfiles or lenlist<fnumfiles):
6525 fnumfiles = lenlist;
6526 AppendFileHeader(catfp, fnumfiles, checksumtype, formatspecs);
6527 lenlist = len(listcatfiles['ffilelist']);
6528 fnumfiles = int(listcatfiles['fnumfiles']);
6529 lcfi = 0;
6530 lcfx = int(listcatfiles['fnumfiles']);
6531 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6532 lcfx = int(lenlist);
6533 else:
6534 lcfx = int(listcatfiles['fnumfiles']);
6535 curinode = 0;
6536 curfid = 0;
6537 inodelist = [];
6538 inodetofile = {};
6539 filetoinode = {};
6540 reallcfi = 0;
6541 while(lcfi < lcfx):
6542 if(re.findall("^[.|/]", listcatfiles['ffilelist'][reallcfi]['fname'])):
6543 fname = listcatfiles['ffilelist'][reallcfi]['fname'];
6544 else:
6545 fname = "./"+listcatfiles['ffilelist'][reallcfi]['fname'];
6546 if(verbose):
6547 VerbosePrintOut(fname);
6548 fheadersize = format(int(listcatfiles['ffilelist'][reallcfi]['fheadersize']), 'x').lower();
6549 fsize = format(int(listcatfiles['ffilelist'][reallcfi]['fsize']), 'x').lower();
6550 flinkname = listcatfiles['ffilelist'][reallcfi]['flinkname'];
6551 fatime = format(int(listcatfiles['ffilelist'][reallcfi]['fatime']), 'x').lower();
6552 fmtime = format(int(listcatfiles['ffilelist'][reallcfi]['fmtime']), 'x').lower();
6553 fctime = format(int(listcatfiles['ffilelist'][reallcfi]['fctime']), 'x').lower();
6554 fbtime = format(int(listcatfiles['ffilelist'][reallcfi]['fbtime']), 'x').lower();
6555 fmode = format(int(listcatfiles['ffilelist'][reallcfi]['fmode']), 'x').lower();
6556 fchmode = format(int(listcatfiles['ffilelist'][reallcfi]['fchmode']), 'x').lower();
6557 fuid = format(int(listcatfiles['ffilelist'][reallcfi]['fuid']), 'x').lower();
6558 funame = listcatfiles['ffilelist'][reallcfi]['funame'];
6559 fgid = format(int(listcatfiles['ffilelist'][reallcfi]['fgid']), 'x').lower();
6560 fgname = listcatfiles['ffilelist'][reallcfi]['fgname'];
6561 finode = format(int(listcatfiles['ffilelist'][reallcfi]['finode']), 'x').lower();
6562 flinkcount = format(int(listcatfiles['ffilelist'][reallcfi]['flinkcount']), 'x').lower();
6563 fwinattributes = format(int(listcatfiles['ffilelist'][reallcfi]['fwinattributes']), 'x').lower();
6564 fcompression = listcatfiles['ffilelist'][reallcfi]['fcompression'];
6565 fcsize = format(int(listcatfiles['ffilelist'][reallcfi]['fcsize']), 'x').lower();
6566 fdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['fminor']), 'x').lower();
6567 fdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['fmajor']), 'x').lower();
6568 frdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['frminor']), 'x').lower();
6569 frdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['frmajor']), 'x').lower();
6570 if(len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>listcatfiles['ffilelist'][reallcfi]['fextrafields'] and len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>0):
6571 listcatfiles['ffilelist'][reallcfi]['fextrafields'] = len(listcatfiles['ffilelist'][reallcfi]['fextralist']);
6572 if(not followlink and len(extradata)<0):
6573 extradata = listcatfiles['ffilelist'][reallcfi]['fextralist'];
6574 fcontents = listcatfiles['ffilelist'][reallcfi]['fcontents'];
6575 fcompression = "";
6576 fcsize = format(int(0), 'x').lower();
6577 if(not compresswholefile):
6578 fcontents.seek(0, 2);
6579 ucfsize = fcontents.tell();
6580 fcontents.seek(0, 0);
6581 if(compression=="auto"):
6582 ilsize = len(compressionlistalt);
6583 ilmin = 0;
6584 ilcsize = [];
6585 while(ilmin < ilsize):
6586 cfcontents = BytesIO();
6587 shutil.copyfileobj(fcontents, cfcontents);
6588 fcontents.seek(0, 0);
6589 cfcontents.seek(0, 0);
6590 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
6591 if(cfcontents):
6592 cfcontents.seek(0, 2);
6593 ilcsize.append(cfcontents.tell());
6594 cfcontents.close();
6595 else:
6596 try:
6597 ilcsize.append(sys.maxint);
6598 except AttributeError:
6599 ilcsize.append(sys.maxsize);
6600 ilmin = ilmin + 1;
6601 ilcmin = ilcsize.index(min(ilcsize));
6602 compression = compressionlistalt[ilcmin];
6603 fcontents.seek(0, 0);
6604 cfcontents = BytesIO();
6605 shutil.copyfileobj(fcontents, cfcontents);
6606 cfcontents.seek(0, 0);
6607 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
6608 cfcontents.seek(0, 2);
6609 cfsize = cfcontents.tell();
6610 if(ucfsize > cfsize):
6611 fcsize = format(int(cfsize), 'x').lower();
6612 fcompression = compression;
6613 fcontents.close();
6614 fcontents = cfcontents;
6615 if(followlink):
6616 if(listcatfiles['ffilelist'][reallcfi]['ftype']==1 or listcatfiles['ffilelist'][reallcfi]['ftype']==2):
6617 getflinkpath = listcatfiles['ffilelist'][reallcfi]['flinkname'];
6618 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6619 flinkinfo = listcatfiles['ffilelist'][flinkid];
6620 fheadersize = format(int(flinkinfo['fheadersize']), 'x').lower();
6621 fsize = format(int(flinkinfo['fsize']), 'x').lower();
6622 flinkname = flinkinfo['flinkname'];
6623 fatime = format(int(flinkinfo['fatime']), 'x').lower();
6624 fmtime = format(int(flinkinfo['fmtime']), 'x').lower();
6625 fctime = format(int(flinkinfo['fctime']), 'x').lower();
6626 fbtime = format(int(flinkinfo['fbtime']), 'x').lower();
6627 fmode = format(int(flinkinfo['fmode']), 'x').lower();
6628 fchmode = format(int(flinkinfo['fchmode']), 'x').lower();
6629 fuid = format(int(flinkinfo['fuid']), 'x').lower();
6630 funame = flinkinfo['funame'];
6631 fgid = format(int(flinkinfo['fgid']), 'x').lower();
6632 fgname = flinkinfo['fgname'];
6633 finode = format(int(flinkinfo['finode']), 'x').lower();
6634 flinkcount = format(int(flinkinfo['flinkcount']), 'x').lower();
6635 fwinattributes = format(int(flinkinfo['fwinattributes']), 'x').lower();
6636 fcompression = flinkinfo['fcompression'];
6637 fcsize = format(int(flinkinfo['fcsize']), 'x').lower();
6638 fdev_minor = format(int(flinkinfo['fminor']), 'x').lower();
6639 fdev_major = format(int(flinkinfo['fmajor']), 'x').lower();
6640 frdev_minor = format(int(flinkinfo['frminor']), 'x').lower();
6641 frdev_major = format(int(flinkinfo['frmajor']), 'x').lower();
6642 if(len(flinkinfo['fextralist'])>flinkinfo['fextrafields'] and len(flinkinfo['fextralist'])>0):
6643 flinkinfo['fextrafields'] = len(flinkinfo['fextralist']);
6644 if(len(extradata)<0):
6645 extradata = flinkinfo['fextralist'];
6646 fcontents = flinkinfo['fcontents'];
6647 ftypehex = format(flinkinfo['ftype'], 'x').lower();
6648 else:
6649 ftypehex = format(listcatfiles['ffilelist'][reallcfi]['ftype'], 'x').lower();
6650 fcurfid = format(curfid, 'x').lower();
6651 if(not followlink and finode!=0):
6652 if(listcatfiles['ffilelist'][reallcfi]['ftype']!=1):
6653 fcurinode = format(int(curinode), 'x').lower();
6654 inodetofile.update({curinode: fname});
6655 filetoinode.update({fname: curinode});
6656 curinode = curinode + 1;
6657 else:
6658 fcurinode = format(int(filetoinode[flinkname]), 'x').lower();
6659 else:
6660 fcurinode = format(int(curinode), 'x').lower();
6661 curinode = curinode + 1;
6662 curfid = curfid + 1;
6663 if(fcompression=="none"):
6664 fcompression = "";
6665 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
6666 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
6667 fcontents.close();
6668 lcfi = lcfi + 1;
6669 reallcfi = reallcfi + 1;
6670 if(lcfx>0):
6671 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
6672 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
6673 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
6674 try:
6675 catfp.flush();
6676 os.fsync(catfp.fileno());
6677 except io.UnsupportedOperation:
6678 pass;
6679 except AttributeError:
6680 pass;
6681 except OSError as e:
6682 pass;
6683 if(outfile=="-"):
6684 catfp.seek(0, 0);
6685 if(hasattr(sys.stdout, "buffer")):
6686 shutil.copyfileobj(catfp, sys.stdout.buffer);
6687 else:
6688 shutil.copyfileobj(catfp, sys.stdout);
6689 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
6690 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
6691 catfp.seek(0, 0);
6692 upload_file_to_internet_file(catfp, outfile);
6693 if(returnfp):
6694 catfp.seek(0, 0);
6695 return catfp;
6696 else:
6697 catfp.close();
6698 return True;
6699
6700 create_alias_function("RePack", __file_format_name__, "", RePackArchiveFile);
6701
6702 def RePackArchiveFileFromString(catstr, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6703 formatspecs = FormatSpecsListToDict(formatspecs);
6704 catfp = BytesIO(catstr);
6705 listcatfiles = RePackArchiveFile(catfp, compression, compresswholefile, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6706 return listcatfiles;
6707
6708 create_alias_function("RePack", __file_format_name__, "FromString", RePackArchiveFileFromString);
6709
6710 def PackArchiveFileFromListDir(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6711 formatspecs = FormatSpecsListToDict(formatspecs);
6712 outarray = BytesIO();
6713 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
6714 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6715 return listcatfiles;
6716
6717 create_alias_function("Pack", __file_format_name__, "FromListDir", PackArchiveFileFromListDir);
6718
6719 def UnPackArchiveFile(infile, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, preservepermissions=True, preservetime=True, verbose=False, returnfp=False):
6720 formatspecs = FormatSpecsListToDict(formatspecs);
6721 if(outdir is not None):
6722 outdir = RemoveWindowsPath(outdir);
6723 if(verbose):
6724 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6725 if(isinstance(infile, dict)):
6726 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, True, skipchecksum, formatspecs, returnfp);
6727 listcatfiles = prelistcatfiles['list'];
6728 else:
6729 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6730 infile = RemoveWindowsPath(infile);
6731 if(followlink):
6732 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, True, skipchecksum, formatspecs, returnfp);
6733 listcatfiles = prelistcatfiles['list'];
6734 else:
6735 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, True, skipchecksum, formatspecs, returnfp);
6736 if(not listcatfiles):
6737 return False;
6738 lenlist = len(listcatfiles['ffilelist']);
6739 fnumfiles = int(listcatfiles['fnumfiles']);
6740 lcfi = 0;
6741 lcfx = int(listcatfiles['fnumfiles']);
6742 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6743 lcfx = int(lenlist);
6744 else:
6745 lcfx = int(listcatfiles['fnumfiles']);
6746 while(lcfi<lcfx):
6747 funame = "";
6748 try:
6749 import pwd;
6750 try:
6751 userinfo = pwd.getpwuid(listcatfiles['ffilelist'][lcfi]['fuid']);
6752 funame = userinfo.pw_name;
6753 except KeyError:
6754 funame = "";
6755 except ImportError:
6756 funame = "";
6757 fgname = "";
6758 try:
6759 import grp;
6760 try:
6761 groupinfo = grp.getgrgid(listcatfiles['ffilelist'][lcfi]['fgid']);
6762 fgname = groupinfo.gr_name;
6763 except KeyError:
6764 fgname = "";
6765 except ImportError:
6766 fgname = "";
6767 if(verbose):
6768 VerbosePrintOut(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6769 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6770 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6771 listcatfiles['ffilelist'][lcfi]['fcontents'].seek(0, 0);
6772 shutil.copyfileobj(listcatfiles['ffilelist'][lcfi]['fcontents'], fpc);
6773 try:
6774 fpc.flush();
6775 os.fsync(fpc.fileno());
6776 except io.UnsupportedOperation:
6777 pass;
6778 except AttributeError:
6779 pass;
6780 except OSError as e:
6781 pass;
6782 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
6783 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
6784 if(preservepermissions):
6785 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6786 if(preservetime):
6787 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
6788 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6789 if(followlink):
6790 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
6791 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6792 flinkinfo = listcatfiles['ffilelist'][flinkid];
6793 funame = "";
6794 try:
6795 import pwd;
6796 try:
6797 userinfo = pwd.getpwuid(flinkinfo['fuid']);
6798 funame = userinfo.pw_name;
6799 except KeyError:
6800 funame = "";
6801 except ImportError:
6802 funame = "";
6803 fgname = "";
6804 try:
6805 import grp;
6806 try:
6807 groupinfo = grp.getgrgid(flinkinfo['fgid']);
6808 fgname = groupinfo.gr_name;
6809 except KeyError:
6810 fgname = "";
6811 except ImportError:
6812 fgname = "";
6813 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
6814 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6815 flinkinfo['fcontents'].seek(0, 0);
6816 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
6817 try:
6818 fpc.flush();
6819 os.fsync(fpc.fileno());
6820 except io.UnsupportedOperation:
6821 pass;
6822 except AttributeError:
6823 pass;
6824 except OSError as e:
6825 pass;
6826 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6827 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6828 if(preservepermissions):
6829 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6830 if(preservetime):
6831 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6832 if(flinkinfo['ftype']==1):
6833 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6834 if(flinkinfo['ftype']==2):
6835 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6836 if(flinkinfo['ftype']==5):
6837 if(preservepermissions):
6838 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6839 else:
6840 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6841 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6842 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6843 if(preservepermissions):
6844 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6845 if(preservetime):
6846 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6847 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
6848 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6849 else:
6850 os.link(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6851 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6852 if(followlink):
6853 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
6854 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6855 flinkinfo = listcatfiles['ffilelist'][flinkid];
6856 funame = "";
6857 try:
6858 import pwd;
6859 try:
6860 userinfo = pwd.getpwuid(flinkinfo['fuid']);
6861 funame = userinfo.pw_name;
6862 except KeyError:
6863 funame = "";
6864 except ImportError:
6865 funame = "";
6866 fgname = "";
6867 try:
6868 import grp;
6869 try:
6870 groupinfo = grp.getgrgid(flinkinfo['fgid']);
6871 fgname = groupinfo.gr_name;
6872 except KeyError:
6873 fgname = "";
6874 except ImportError:
6875 fgname = "";
6876 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
6877 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6878 flinkinfo['fcontents'].seek(0, 0);
6879 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
6880 try:
6881 fpc.flush();
6882 os.fsync(fpc.fileno());
6883 except io.UnsupportedOperation:
6884 pass;
6885 except AttributeError:
6886 pass;
6887 except OSError as e:
6888 pass;
6889 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6890 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6891 if(preservepermissions):
6892 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6893 if(preservetime):
6894 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6895 if(flinkinfo['ftype']==1):
6896 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6897 if(flinkinfo['ftype']==2):
6898 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6899 if(flinkinfo['ftype']==5):
6900 if(preservepermissions):
6901 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6902 else:
6903 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6904 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6905 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6906 if(preservepermissions):
6907 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6908 if(preservetime):
6909 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6910 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
6911 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6912 else:
6913 os.symlink(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6914 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6915 if(preservepermissions):
6916 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6917 else:
6918 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6919 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
6920 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
6921 if(preservepermissions):
6922 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6923 if(preservetime):
6924 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
6925 if(listcatfiles['ffilelist'][lcfi]['ftype']==6 and hasattr(os, "mkfifo")):
6926 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6927 lcfi = lcfi + 1;
6928 if(returnfp):
6929 return listcatfiles['ffilelist']['catfp'];
6930 else:
6931 return True;
6932
6933 create_alias_function("UnPack", __file_format_name__, "", UnPackArchiveFile);
6934
6935 if(hasattr(shutil, "register_unpack_format")):
6936 def UnPackArchiveFileFunc(archive_name, extract_dir=None, **kwargs):
6937 return UnPackArchiveFile(archive_name, extract_dir, False, 0, 0, False, __file_format_dict__['format_delimiter'], False, False);
6938 create_alias_function("UnPack", __file_format_name__, "Func", UnPackArchiveFileFunc);
6939
6940 def UnPackArchiveFileString(catstr, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6941 formatspecs = FormatSpecsListToDict(formatspecs);
6942 catfp = BytesIO(catstr);
6943 listcatfiles = UnPackArchiveFile(catfp, outdir, followlink, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6944 return listcatfiles;
6945
6946 create_alias_function("UnPack", __file_format_name__, "String", UnPackArchiveFileString);
6947
6948 def ArchiveFileListFiles(infile, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6949 formatspecs = FormatSpecsListToDict(formatspecs);
6950 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6951 if(isinstance(infile, dict)):
6952 listcatfiles = infile;
6953 else:
6954 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6955 infile = RemoveWindowsPath(infile);
6956 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, True, False, skipchecksum, formatspecs, returnfp);
6957 if(not listcatfiles):
6958 return False;
6959 lenlist = len(listcatfiles['ffilelist']);
6960 fnumfiles = int(listcatfiles['fnumfiles']);
6961 lcfi = 0;
6962 lcfx = int(listcatfiles['fnumfiles']);
6963 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6964 lcfx = int(lenlist);
6965 else:
6966 lcfx = int(listcatfiles['fnumfiles']);
6967 returnval = {};
6968 while(lcfi<lcfx):
6969 returnval.update({lcfi: listcatfiles['ffilelist'][lcfi]['fname']});
6970 if(not verbose):
6971 VerbosePrintOut(listcatfiles['ffilelist'][lcfi]['fname']);
6972 if(verbose):
6973 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6974 printfname = listcatfiles['ffilelist'][lcfi]['fname'];
6975 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6976 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " link to " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6977 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6978 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " -> " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6979 fuprint = listcatfiles['ffilelist'][lcfi]['funame'];
6980 if(len(fuprint)<=0):
6981 fuprint = listcatfiles['ffilelist'][lcfi]['fuid'];
6982 fgprint = listcatfiles['ffilelist'][lcfi]['fgname'];
6983 if(len(fgprint)<=0):
6984 fgprint = listcatfiles['ffilelist'][lcfi]['fgid'];
6985 VerbosePrintOut(PrintPermissionString(listcatfiles['ffilelist'][lcfi]['fmode'], listcatfiles['ffilelist'][lcfi]['ftype']) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(listcatfiles['ffilelist'][lcfi]['fsize']).rjust(15) + " " + datetime.datetime.utcfromtimestamp(listcatfiles['ffilelist'][lcfi]['fmtime']).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6986 lcfi = lcfi + 1;
6987 if(returnfp):
6988 return listcatfiles['catfp'];
6989 else:
6990 return True;
6991
6992 create_alias_function("", __file_format_name__, "ListFiles", ArchiveFileListFiles);
6993
6994 def ArchiveFileStringListFiles(catstr, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6995 formatspecs = FormatSpecsListToDict(formatspecs);
6996 catfp = BytesIO(catstr);
6997 listcatfiles = ArchiveFileListFiles(catstr, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6998 return listcatfiles;
6999
7000 create_alias_function("", __file_format_name__, "StringListFiles", ArchiveFileStringListFiles);
7001
7002 def TarFileListFiles(infile, verbose=False, returnfp=False):
7003 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7004 if(infile=="-"):
7005 infile = BytesIO();
7006 if(hasattr(sys.stdin, "buffer")):
7007 shutil.copyfileobj(sys.stdin.buffer, infile);
7008 else:
7009 shutil.copyfileobj(sys.stdin, infile);
7010 infile.seek(0, 0);
7011 if(not infile):
7012 return False;
7013 infile.seek(0, 0);
7014 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
7015 infile = download_file_from_internet_file(infile);
7016 infile.seek(0, 0);
7017 if(not infile):
7018 return False;
7019 infile.seek(0, 0);
7020 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
7021 return False;
7022 elif(os.path.exists(infile) and os.path.isfile(infile)):
7023 try:
7024 if(not tarfile.TarFileCheck(infile)):
7025 return False;
7026 except AttributeError:
7027 if(not TarFileCheck(infile)):
7028 return False;
7029 else:
7030 return False;
7031 try:
7032 if(hasattr(infile, "read") or hasattr(infile, "write")):
7033 tarfp = tarfile.open(fileobj=infile, mode="r");
7034 else:
7035 tarfp = tarfile.open(infile, "r");
7036 except FileNotFoundError:
7037 return False;
7038 lcfi = 0
7039 returnval = {};
7040 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
7041 returnval.update({lcfi: member.name});
7042 fpremode = member.mode;
7043 ffullmode = member.mode;
7044 flinkcount = 0;
7045 ftype = 0;
7046 if(member.isreg()):
7047 ffullmode = member.mode + stat.S_IFREG;
7048 ftype = 0;
7049 elif(member.isdev()):
7050 ffullmode = member.mode;
7051 ftype = 7;
7052 elif(member.islnk()):
7053 ffullmode = member.mode + stat.S_IFREG;
7054 ftype = 1;
7055 elif(member.issym()):
7056 ffullmode = member.mode + stat.S_IFLNK;
7057 ftype = 2;
7058 elif(member.ischr()):
7059 ffullmode = member.mode + stat.S_IFCHR;
7060 ftype = 3;
7061 elif(member.isblk()):
7062 ffullmode = member.mode + stat.S_IFBLK;
7063 ftype = 4;
7064 elif(member.isdir()):
7065 ffullmode = member.mode + stat.S_IFDIR;
7066 ftype = 5;
7067 elif(member.isfifo()):
7068 ffullmode = member.mode + stat.S_IFIFO;
7069 ftype = 6;
7070 elif(member.issparse()):
7071 ffullmode = member.mode;
7072 ftype = 12;
7073 if(not verbose):
7074 VerbosePrintOut(member.name);
7075 elif(verbose):
7076 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7077 printfname = member.name;
7078 if(member.islnk()):
7079 printfname = member.name + " link to " + member.linkname;
7080 elif(member.issym()):
7081 printfname = member.name + " -> " + member.linkname;
7082 fuprint = member.uname;
7083 if(len(fuprint)<=0):
7084 fuprint = member.uid;
7085 fgprint = member.gname;
7086 if(len(fgprint)<=0):
7087 fgprint = member.gid;
7088 VerbosePrintOut(PrintPermissionString(ffullmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(member.mtime).strftime('%Y-%m-%d %H:%M') + " " + printfname));
7089 lcfi = lcfi + 1;
7090 if(returnfp):
7091 return listcatfiles['catfp'];
7092 else:
7093 return True;
7094
7095 def ZipFileListFiles(infile, verbose=False, returnfp=False):
7096 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7097 if(infile=="-"):
7098 infile = BytesIO();
7099 if(hasattr(sys.stdin, "buffer")):
7100 shutil.copyfileobj(sys.stdin.buffer, infile);
7101 else:
7102 shutil.copyfileobj(sys.stdin, infile);
7103 infile.seek(0, 0);
7104 if(not infile):
7105 return False;
7106 infile.seek(0, 0);
7107 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
7108 infile = download_file_from_internet_file(infile);
7109 infile.seek(0, 0);
7110 if(not infile):
7111 return False;
7112 infile.seek(0, 0);
7113 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
7114 return False;
7115 else:
7116 return False;
7117 if(not zipfile.is_zipfile(infile)):
7118 return False;
7119 try:
7120 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
7121 except FileNotFoundError:
7122 return False;
7123 lcfi = 0;
7124 returnval = {};
7125 ziptest = zipfp.testzip();
7126 if(ziptest):
7127 VerbosePrintOut("Bad file found!");
7128 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
7129 if(zipinfo.create_system==0 or zipinfo.create_system==10):
7130 fwinattributes = int(zipinfo.external_attr);
7131 if(not member.is_dir()):
7132 fmode = int(stat.S_IFREG + 438);
7133 fchmode = int(stat.S_IMODE(fmode));
7134 ftypemod = int(stat.S_IFMT(fmode));
7135 elif(member.is_dir()):
7136 fmode = int(stat.S_IFDIR + 511);
7137 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7138 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7139 elif(zipinfo.create_system==3):
7140 fwinattributes =int(0);
7141 fmode = int(zipinfo.external_attr);
7142 fchmode = int(stat.S_IMODE(fmode));
7143 ftypemod = int(stat.S_IFMT(fmode));
7144 else:
7145 fwinattributes = int(0);
7146 if(not member.is_dir()):
7147 fmode = int(stat.S_IFREG + 438);
7148 fchmode = int(stat.S_IMODE(fmode));
7149 ftypemod = int(stat.S_IFMT(fmode));
7150 elif(member.is_dir()):
7151 fmode = int(stat.S_IFDIR + 511);
7152 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7153 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7154 returnval.update({lcfi: member.filename});
7155 if(not verbose):
7156 VerbosePrintOut(member.filename);
7157 if(verbose):
7158 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7159 permissionstr = "";
7160 for fmodval in str(oct(fmode))[-3:]:
7161 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
7162 if(not member.is_dir()):
7163 ftype = 0;
7164 permissionstr = "-" + permissionstr;
7165 elif(member.is_dir()):
7166 ftype = 5;
7167 permissionstr = "d" + permissionstr;
7168 printfname = member.filename;
7169 try:
7170 fuid = int(os.getuid());
7171 except AttributeError:
7172 fuid = int(0);
7173 except KeyError:
7174 fuid = int(0);
7175 try:
7176 fgid = int(os.getgid());
7177 except AttributeError:
7178 fgid = int(0);
7179 except KeyError:
7180 fgid = int(0);
7181 try:
7182 import pwd;
7183 try:
7184 userinfo = pwd.getpwuid(os.getuid());
7185 funame = userinfo.pw_name;
7186 except KeyError:
7187 funame = "";
7188 except AttributeError:
7189 funame = "";
7190 except ImportError:
7191 funame = "";
7192 fgname = "";
7193 try:
7194 import grp;
7195 try:
7196 groupinfo = grp.getgrgid(os.getgid());
7197 fgname = groupinfo.gr_name;
7198 except KeyError:
7199 fgname = "";
7200 except AttributeError:
7201 fgname = "";
7202 except ImportError:
7203 fgname = "";
7204 fuprint = funame;
7205 if(len(fuprint)<=0):
7206 fuprint = str(fuid);
7207 fgprint = fgname;
7208 if(len(fgprint)<=0):
7209 fgprint = str(fgid);
7210 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(int(time.mktime(member.date_time + (0, 0, -1)))).strftime('%Y-%m-%d %H:%M') + " " + printfname));
7211 lcfi = lcfi + 1;
7212 if(returnfp):
7213 return listcatfiles['catfp'];
7214 else:
7215 return True;
7216
7217 if(not rarfile_support):
7218 def RarFileListFiles(infile, verbose=False, returnfp=False):
7219 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7220 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7221 return False;
7222
7223 if(rarfile_support):
7224 def RarFileListFiles(infile, verbose=False, returnfp=False):
7225 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7226 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7227 return False;
7228 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
7229 return False;
7230 lcfi = 0;
7231 returnval = {};
7232 rarfp = rarfile.RarFile(infile, "r");
7233 rartest = rarfp.testrar();
7234 if(rartest):
7235 VerbosePrintOut("Bad file found!");
7236 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
7237 is_unix = False;
7238 is_windows = False;
7239 if(member.host_os==rarfile.RAR_OS_UNIX):
7240 is_windows = False;
7241 try:
7242 member.external_attr
7243 is_unix = True;
7244 except AttributeError:
7245 is_unix = False;
7246 elif(member.host_os==rarfile.RAR_OS_WIN32):
7247 is_unix = False;
7248 try:
7249 member.external_attr
7250 is_windows = True;
7251 except AttributeError:
7252 is_windows = False;
7253 else:
7254 is_unix = False;
7255 is_windows = False;
7256 if(is_unix and member.external_attr !=0):
7257 fpremode = int(member.external_attr);
7258 elif(member.is_file()):
7259 fpremode = int(stat.S_IFREG + 438);
7260 elif(member.is_symlink()):
7261 fpremode = int(stat.S_IFLNK + 438);
7262 elif(member.is_dir()):
7263 fpremode = int(stat.S_IFDIR + 511);
7264 if(is_windows and member.external_attr !=0):
7265 fwinattributes = int(member.external_attr);
7266 else:
7267 fwinattributes = int(0);
7268 if(is_unix and member.external_attr !=0):
7269 fmode = int(member.external_attr);
7270 fchmode = int(stat.S_IMODE(member.external_attr));
7271 ftypemod = int(stat.S_IFMT(member.external_attr));
7272 elif(member.is_file()):
7273 fmode = int(stat.S_IFREG + 438);
7274 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
7275 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
7276 elif(member.is_symlink()):
7277 fmode = int(stat.S_IFLNK + 438);
7278 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
7279 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
7280 elif(member.is_dir()):
7281 fmode = int(stat.S_IFDIR + 511);
7282 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7283 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7284 returnval.update({lcfi: member.filename});
7285 if(not verbose):
7286 VerbosePrintOut(member.filename);
7287 if(verbose):
7288 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7289 permissionstr = "";
7290 for fmodval in str(oct(fmode))[-3:]:
7291 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
7292 if(member.is_file()):
7293 ftype = 0;
7294 permissionstr = "-" + permissionstr;
7295 printfname = member.filename;
7296 elif(member.is_symlink()):
7297 ftype = 2;
7298 permissionstr = "l" + permissionstr;
7299 printfname = member.name + " -> " + member.read().decode("UTF-8");
7300 elif(member.is_dir()):
7301 ftype = 5;
7302 permissionstr = "d" + permissionstr;
7303 printfname = member.filename;
7304 try:
7305 fuid = int(os.getuid());
7306 except AttributeError:
7307 fuid = int(0);
7308 except KeyError:
7309 fuid = int(0);
7310 try:
7311 fgid = int(os.getgid());
7312 except AttributeError:
7313 fgid = int(0);
7314 except KeyError:
7315 fgid = int(0);
7316 try:
7317 import pwd;
7318 try:
7319 userinfo = pwd.getpwuid(os.getuid());
7320 funame = userinfo.pw_name;
7321 except KeyError:
7322 funame = "";
7323 except AttributeError:
7324 funame = "";
7325 except ImportError:
7326 funame = "";
7327 fgname = "";
7328 try:
7329 import grp;
7330 try:
7331 groupinfo = grp.getgrgid(os.getgid());
7332 fgname = groupinfo.gr_name;
7333 except KeyError:
7334 fgname = "";
7335 except AttributeError:
7336 fgname = "";
7337 except ImportError:
7338 fgname = "";
7339 fuprint = funame;
7340 if(len(fuprint)<=0):
7341 fuprint = str(fuid);
7342 fgprint = fgname;
7343 if(len(fgprint)<=0):
7344 fgprint = str(fgid);
7345 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + member.mtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
7346 lcfi = lcfi + 1;
7347 if(returnfp):
7348 return listcatfiles['catfp'];
7349 else:
7350 return True;
7351
7352 if(not py7zr_support):
7353 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
7354 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7355 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7356 return False;
7357
7358 if(py7zr_support):
7359 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
7360 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7361 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7362 return False;
7363 lcfi = 0;
7364 returnval = {};
7365 szpfp = py7zr.SevenZipFile(infile, mode="r");
7366 file_content = szpfp.readall();
7367 #sztest = szpfp.testzip();
7368 sztestalt = szpfp.test();
7369 if(sztestalt):
7370 VerbosePrintOut("Bad file found!");
7371 for member in sorted(szpfp.list(), key=lambda x: x.filename):
7372 if(re.findall("^[.|/]", member.filename)):
7373 fname = member.filename;
7374 else:
7375 fname = "./"+member.filename;
7376 if(not member.is_directory):
7377 fpremode = int(stat.S_IFREG + 438);
7378 elif(member.is_directory):
7379 fpremode = int(stat.S_IFDIR + 511);
7380 fwinattributes = int(0);
7381 if(member.is_directory):
7382 fmode = int(stat.S_IFDIR + 511);
7383 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7384 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7385 else:
7386 fmode = int(stat.S_IFLNK + 438);
7387 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
7388 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
7389 returnval.update({lcfi: member.filename});
7390 if(not verbose):
7391 VerbosePrintOut(member.filename);
7392 if(verbose):
7393 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7394 permissionstr = "";
7395 for fmodval in str(oct(fmode))[-3:]:
7396 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
7397 fsize = int("0");
7398 if(not member.is_directory):
7399 ftype = 0;
7400 permissionstr = "-" + permissionstr;
7401 printfname = member.filename;
7402 elif(member.is_directory):
7403 ftype = 5;
7404 permissionstr = "d" + permissionstr;
7405 printfname = member.filename;
7406 if(ftype==0):
7407 fsize = len(file_content[member.filename].read());
7408 file_content[member.filename].close();
7409 try:
7410 fuid = int(os.getuid());
7411 except AttributeError:
7412 fuid = int(0);
7413 except KeyError:
7414 fuid = int(0);
7415 try:
7416 fgid = int(os.getgid());
7417 except AttributeError:
7418 fgid = int(0);
7419 except KeyError:
7420 fgid = int(0);
7421 try:
7422 import pwd;
7423 try:
7424 userinfo = pwd.getpwuid(os.getuid());
7425 funame = userinfo.pw_name;
7426 except KeyError:
7427 funame = "";
7428 except AttributeError:
7429 funame = "";
7430 except ImportError:
7431 funame = "";
7432 fgname = "";
7433 try:
7434 import grp;
7435 try:
7436 groupinfo = grp.getgrgid(os.getgid());
7437 fgname = groupinfo.gr_name;
7438 except KeyError:
7439 fgname = "";
7440 except AttributeError:
7441 fgname = "";
7442 except ImportError:
7443 fgname = "";
7444 fuprint = funame;
7445 if(len(fuprint)<=0):
7446 fuprint = str(fuid);
7447 fgprint = fgname;
7448 if(len(fgprint)<=0):
7449 fgprint = str(fgid);
7450 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(fsize).rjust(15) + " " + member.creationtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
7451 lcfi = lcfi + 1;
7452 if(returnfp):
7453 return listcatfiles['catfp'];
7454 else:
7455 return True;
7456
7457 def InFileListFiles(infile, verbose=False, formatspecs=__file_format_dict__, returnfp=False):
7458 formatspecs = FormatSpecsListToDict(formatspecs);
7459 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7460 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
7461 if(checkcompressfile=="tarfile"):
7462 return TarFileListFiles(infile, verbose, returnfp);
7463 elif(checkcompressfile=="zipfile"):
7464 return ZipFileListFiles(infile, verbose, returnfp);
7465 elif(checkcompressfile=="catfile"):
7466 return ArchiveFileListFiles(infile, 0, 0, False, formatspecs, verbose, returnfp);
7467 elif(rarfile_support and checkcompressfile=="rarfile"):
7468 return RarFileListFiles(infile, verbose, returnfp);
7469 elif(py7zr_support and checkcompressfile=="7zipfile"):
7470 return SevenZipFileListFiles(infile, verbose, returnfp);
7471 else:
7472 return False;
7473 return False;
7474
7475 def ListDirListFiles(infiles, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7476 formatspecs = FormatSpecsListToDict(formatspecs);
7477 outarray = BytesIO();
7478 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, False, True);
7479 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
7480 return listcatfiles;
7481
7482 def ListDirListFilesAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7483 formatspecs = FormatSpecsListToDict(formatspecs);
7484 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, listonly, checksumtype, formatspecs, verbose);
7485 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
7486 return listcatfiles;
7487
7488 def PackArchiveFileFromListDirAlt(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7489 formatspecs = FormatSpecsListToDict(formatspecs);
7490 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, False, checksumtype, extradata, formatspecs, False);
7491 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, followlink, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
7492 return listcatfiles;
7493
7494 create_alias_function("Pack", __file_format_name__, "FromListDirAlt", PackArchiveFileFromListDirAlt);
7495
7496 def PackArchiveFileFromTarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7497 formatspecs = FormatSpecsListToDict(formatspecs);
7498 outarray = TarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7499 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7500 return listcatfiles;
7501
7502 create_alias_function("Pack", __file_format_name__, "FromTarFileAlt", PackArchiveFileFromTarFileAlt);
7503
7504 def PackArchiveFileFromZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7505 formatspecs = FormatSpecsListToDict(formatspecs);
7506 outarray = ZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7507 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7508 return listcatfiles;
7509
7510 create_alias_function("Pack", __file_format_name__, "FromZipFileAlt", PackArchiveFileFromZipFileAlt);
7511
7512 if(not rarfile_support):
7513 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7514 return False;
7515
7516 if(rarfile_support):
7517 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7518 formatspecs = FormatSpecsListToDict(formatspecs);
7519 outarray = RarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7520 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7521 return listcatfiles;
7522
7523 create_alias_function("Pack", __file_format_name__, "FromRarFileAlt", PackArchiveFileFromRarFileAlt);
7524
7525 if(not py7zr_support):
7526 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7527 return False;
7528
7529 if(py7zr_support):
7530 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7531 formatspecs = FormatSpecsListToDict(formatspecs);
7532 outarray = SevenZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7533 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7534 return listcatfiles;
7535
7536 create_alias_function("Pack", __file_format_name__, "FromSevenZipFileAlt", PackArchiveFileFromSevenZipFileAlt);
7537
7538 def download_file_from_ftp_file(url):
7539 urlparts = urlparse(url);
7540 file_name = os.path.basename(urlparts.path);
7541 file_dir = os.path.dirname(urlparts.path);
7542 if(urlparts.username is not None):
7543 ftp_username = urlparts.username;
7544 else:
7545 ftp_username = "anonymous";
7546 if(urlparts.password is not None):
7547 ftp_password = urlparts.password;
7548 elif(urlparts.password is None and urlparts.username=="anonymous"):
7549 ftp_password = "anonymous";
7550 else:
7551 ftp_password = "";
7552 if(urlparts.scheme=="ftp"):
7553 ftp = FTP();
7554 elif(urlparts.scheme=="ftps" and ftpssl):
7555 ftp = FTP_TLS();
7556 else:
7557 return False;
7558 if(urlparts.scheme=="sftp"):
7559 if(__use_pysftp__):
7560 return download_file_from_pysftp_file(url);
7561 else:
7562 return download_file_from_sftp_file(url);
7563 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7564 return download_file_from_http_file(url);
7565 ftp_port = urlparts.port;
7566 if(urlparts.port is None):
7567 ftp_port = 21;
7568 try:
7569 ftp.connect(urlparts.hostname, ftp_port);
7570 except socket.gaierror:
7571 log.info("Error With URL "+url);
7572 return False;
7573 except socket.timeout:
7574 log.info("Error With URL "+url);
7575 return False;
7576 ftp.login(urlparts.username, urlparts.password);
7577 if(urlparts.scheme=="ftps"):
7578 ftp.prot_p();
7579 ftpfile = BytesIO();
7580 ftp.retrbinary("RETR "+urlparts.path, ftpfile.write);
7581 #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
7582 ftp.close();
7583 ftpfile.seek(0, 0);
7584 return ftpfile;
7585
7586 def download_file_from_ftp_string(url):
7587 ftpfile = download_file_from_ftp_file(url);
7588 return ftpfile.read();
7589
7590 def upload_file_to_ftp_file(ftpfile, url):
7591 urlparts = urlparse(url);
7592 file_name = os.path.basename(urlparts.path);
7593 file_dir = os.path.dirname(urlparts.path);
7594 if(urlparts.username is not None):
7595 ftp_username = urlparts.username;
7596 else:
7597 ftp_username = "anonymous";
7598 if(urlparts.password is not None):
7599 ftp_password = urlparts.password;
7600 elif(urlparts.password is None and urlparts.username=="anonymous"):
7601 ftp_password = "anonymous";
7602 else:
7603 ftp_password = "";
7604 if(urlparts.scheme=="ftp"):
7605 ftp = FTP();
7606 elif(urlparts.scheme=="ftps" and ftpssl):
7607 ftp = FTP_TLS();
7608 else:
7609 return False;
7610 if(urlparts.scheme=="sftp"):
7611 if(__use_pysftp__):
7612 return upload_file_to_pysftp_file(url);
7613 else:
7614 return upload_file_to_sftp_file(url);
7615 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7616 return False;
7617 ftp_port = urlparts.port;
7618 if(urlparts.port is None):
7619 ftp_port = 21;
7620 try:
7621 ftp.connect(urlparts.hostname, ftp_port);
7622 except socket.gaierror:
7623 log.info("Error With URL "+url);
7624 return False;
7625 except socket.timeout:
7626 log.info("Error With URL "+url);
7627 return False;
7628 ftp.login(urlparts.username, urlparts.password);
7629 if(urlparts.scheme=="ftps"):
7630 ftp.prot_p();
7631 ftp.storbinary("STOR "+urlparts.path, ftpfile);
7632 ftp.close();
7633 ftpfile.seek(0, 0);
7634 return ftpfile;
7635
7636 def upload_file_to_ftp_string(ftpstring, url):
7637 ftpfileo = BytesIO(ftpstring);
7638 ftpfile = upload_file_to_ftp_file(ftpfileo, url);
7639 ftpfileo.close();
7640 return ftpfile;
7641
7642 def download_file_from_http_file(url, headers=geturls_headers_pycatfile_python_alt):
7643 # Parse the URL to extract username and password if present
7644 urlparts = urlparse(url);
7645 username = urlparts.username;
7646 password = urlparts.password;
7647 # Rebuild the URL without the username and password
7648 netloc = urlparts.hostname;
7649 if(urlparts.scheme=="sftp"):
7650 if(__use_pysftp__):
7651 return download_file_from_pysftp_file(url);
7652 else:
7653 return download_file_from_sftp_file(url);
7654 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7655 return download_file_from_ftp_file(url);
7656 if urlparts.port:
7657 netloc += ':' + str(urlparts.port);
7658 rebuilt_url = urlunparse((urlparts.scheme, netloc, urlparts.path, urlparts.params, urlparts.query, urlparts.fragment));
7659 # Create a temporary file object
7660 httpfile = BytesIO();
7661 if haverequests:
7662 # Use the requests library if available
7663 if username and password:
7664 response = requests.get(rebuilt_url, headers=headers, auth=(username, password), stream=True);
7665 else:
7666 response = requests.get(rebuilt_url, headers=headers, stream=True);
7667 response.raw.decode_content = True
7668 shutil.copyfileobj(response.raw, httpfile);
7669 else:
7670 # Build a Request object for urllib
7671 request = Request(rebuilt_url, headers=headers);
7672 # Create an opener object for handling URLs
7673 if username and password:
7674 # Create a password manager
7675 password_mgr = HTTPPasswordMgrWithDefaultRealm();
7676 # Add the username and password
7677 password_mgr.add_password(None, rebuilt_url, username, password);
7678 # Create an authentication handler using the password manager
7679 auth_handler = HTTPBasicAuthHandler(password_mgr);
7680 # Build the opener with the authentication handler
7681 opener = build_opener(auth_handler);
7682 else:
7683 opener = build_opener();
7684 with opener.open(request) as response:
7685 shutil.copyfileobj(response, httpfile);
7686 # Reset file pointer to the start
7687 httpfile.seek(0, 0);
7688 # Return the temporary file object
7689 return httpfile;
7690
7691 def download_file_from_http_string(url, headers=geturls_headers_pycatfile_python_alt):
7692 httpfile = download_file_from_http_file(url, headers);
7693 return ftpfile.read();
7694
7695 if(haveparamiko):
7696 def download_file_from_sftp_file(url):
7697 urlparts = urlparse(url);
7698 file_name = os.path.basename(urlparts.path);
7699 file_dir = os.path.dirname(urlparts.path);
7700 sftp_port = urlparts.port;
7701 if(urlparts.port is None):
7702 sftp_port = 22;
7703 else:
7704 sftp_port = urlparts.port;
7705 if(urlparts.username is not None):
7706 sftp_username = urlparts.username;
7707 else:
7708 sftp_username = "anonymous";
7709 if(urlparts.password is not None):
7710 sftp_password = urlparts.password;
7711 elif(urlparts.password is None and urlparts.username=="anonymous"):
7712 sftp_password = "anonymous";
7713 else:
7714 sftp_password = "";
7715 if(urlparts.scheme=="ftp"):
7716 return download_file_from_ftp_file(url);
7717 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7718 return download_file_from_http_file(url);
7719 if(urlparts.scheme!="sftp"):
7720 return False;
7721 ssh = paramiko.SSHClient();
7722 ssh.load_system_host_keys();
7723 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
7724 try:
7725 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7726 except paramiko.ssh_exception.SSHException:
7727 return False;
7728 except socket.gaierror:
7729 log.info("Error With URL "+url);
7730 return False;
7731 except socket.timeout:
7732 log.info("Error With URL "+url);
7733 return False;
7734 sftp = ssh.open_sftp();
7735 sftpfile = BytesIO();
7736 sftp.getfo(urlparts.path, sftpfile);
7737 sftp.close();
7738 ssh.close();
7739 sftpfile.seek(0, 0);
7740 return sftpfile;
7741 else:
7742 def download_file_from_sftp_file(url):
7743 return False;
7744
7745 if(haveparamiko):
7746 def download_file_from_sftp_string(url):
7747 sftpfile = download_file_from_sftp_file(url);
7748 return sftpfile.read();
7749 else:
7750 def download_file_from_ftp_string(url):
7751 return False;
7752
7753 if(haveparamiko):
7754 def upload_file_to_sftp_file(sftpfile, url):
7755 urlparts = urlparse(url);
7756 file_name = os.path.basename(urlparts.path);
7757 file_dir = os.path.dirname(urlparts.path);
7758 sftp_port = urlparts.port;
7759 if(urlparts.port is None):
7760 sftp_port = 22;
7761 else:
7762 sftp_port = urlparts.port;
7763 if(urlparts.username is not None):
7764 sftp_username = urlparts.username;
7765 else:
7766 sftp_username = "anonymous";
7767 if(urlparts.password is not None):
7768 sftp_password = urlparts.password;
7769 elif(urlparts.password is None and urlparts.username=="anonymous"):
7770 sftp_password = "anonymous";
7771 else:
7772 sftp_password = "";
7773 if(urlparts.scheme=="ftp"):
7774 return upload_file_to_ftp_file(url);
7775 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7776 return False;
7777 if(urlparts.scheme!="sftp"):
7778 return False;
7779 ssh = paramiko.SSHClient();
7780 ssh.load_system_host_keys();
7781 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
7782 try:
7783 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7784 except paramiko.ssh_exception.SSHException:
7785 return False;
7786 except socket.gaierror:
7787 log.info("Error With URL "+url);
7788 return False;
7789 except socket.timeout:
7790 log.info("Error With URL "+url);
7791 return False;
7792 sftp = ssh.open_sftp();
7793 sftp.putfo(sftpfile, urlparts.path);
7794 sftp.close();
7795 ssh.close();
7796 sftpfile.seek(0, 0);
7797 return sftpfile;
7798 else:
7799 def upload_file_to_sftp_file(sftpfile, url):
7800 return False;
7801
7802 if(haveparamiko):
7803 def upload_file_to_sftp_string(sftpstring, url):
7804 sftpfileo = BytesIO(sftpstring);
7805 sftpfile = upload_file_to_sftp_files(ftpfileo, url);
7806 sftpfileo.close();
7807 return sftpfile;
7808 else:
7809 def upload_file_to_sftp_string(url):
7810 return False;
7811
7812 if(havepysftp):
7813 def download_file_from_pysftp_file(url):
7814 urlparts = urlparse(url);
7815 file_name = os.path.basename(urlparts.path);
7816 file_dir = os.path.dirname(urlparts.path);
7817 sftp_port = urlparts.port;
7818 if(urlparts.port is None):
7819 sftp_port = 22;
7820 else:
7821 sftp_port = urlparts.port;
7822 if(urlparts.username is not None):
7823 sftp_username = urlparts.username;
7824 else:
7825 sftp_username = "anonymous";
7826 if(urlparts.password is not None):
7827 sftp_password = urlparts.password;
7828 elif(urlparts.password is None and urlparts.username=="anonymous"):
7829 sftp_password = "anonymous";
7830 else:
7831 sftp_password = "";
7832 if(urlparts.scheme=="ftp"):
7833 return download_file_from_ftp_file(url);
7834 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7835 return download_file_from_http_file(url);
7836 if(urlparts.scheme!="sftp"):
7837 return False;
7838 try:
7839 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7840 except paramiko.ssh_exception.SSHException:
7841 return False;
7842 except socket.gaierror:
7843 log.info("Error With URL "+url);
7844 return False;
7845 except socket.timeout:
7846 log.info("Error With URL "+url);
7847 return False;
7848 sftp = ssh.open_sftp();
7849 sftpfile = BytesIO();
7850 sftp.getfo(urlparts.path, sftpfile);
7851 sftp.close();
7852 ssh.close();
7853 sftpfile.seek(0, 0);
7854 return sftpfile;
7855 else:
7856 def download_file_from_pysftp_file(url):
7857 return False;
7858
7859 if(havepysftp):
7860 def download_file_from_pysftp_string(url):
7861 sftpfile = download_file_from_pysftp_file(url);
7862 return sftpfile.read();
7863 else:
7864 def download_file_from_ftp_string(url):
7865 return False;
7866
7867 if(havepysftp):
7868 def upload_file_to_pysftp_file(sftpfile, url):
7869 urlparts = urlparse(url);
7870 file_name = os.path.basename(urlparts.path);
7871 file_dir = os.path.dirname(urlparts.path);
7872 sftp_port = urlparts.port;
7873 if(urlparts.port is None):
7874 sftp_port = 22;
7875 else:
7876 sftp_port = urlparts.port;
7877 if(urlparts.username is not None):
7878 sftp_username = urlparts.username;
7879 else:
7880 sftp_username = "anonymous";
7881 if(urlparts.password is not None):
7882 sftp_password = urlparts.password;
7883 elif(urlparts.password is None and urlparts.username=="anonymous"):
7884 sftp_password = "anonymous";
7885 else:
7886 sftp_password = "";
7887 if(urlparts.scheme=="ftp"):
7888 return upload_file_to_ftp_file(url);
7889 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7890 return False;
7891 if(urlparts.scheme!="sftp"):
7892 return False;
7893 try:
7894 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7895 except paramiko.ssh_exception.SSHException:
7896 return False;
7897 except socket.gaierror:
7898 log.info("Error With URL "+url);
7899 return False;
7900 except socket.timeout:
7901 log.info("Error With URL "+url);
7902 return False;
7903 sftp = ssh.open_sftp();
7904 sftp.putfo(sftpfile, urlparts.path);
7905 sftp.close();
7906 ssh.close();
7907 sftpfile.seek(0, 0);
7908 return sftpfile;
7909 else:
7910 def upload_file_to_pysftp_file(sftpfile, url):
7911 return False;
7912
7913 if(havepysftp):
7914 def upload_file_to_pysftp_string(sftpstring, url):
7915 sftpfileo = BytesIO(sftpstring);
7916 sftpfile = upload_file_to_pysftp_files(ftpfileo, url);
7917 sftpfileo.close();
7918 return sftpfile;
7919 else:
7920 def upload_file_to_pysftp_string(url):
7921 return False;
7922
7923 def download_file_from_internet_file(url, headers=geturls_headers_pycatfile_python_alt):
7924 urlparts = urlparse(url);
7925 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7926 return download_file_from_http_file(url, headers);
7927 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7928 return download_file_from_ftp_file(url);
7929 elif(urlparts.scheme=="sftp"):
7930 if(__use_pysftp__ and havepysftp):
7931 return download_file_from_pysftp_file(url);
7932 else:
7933 return download_file_from_sftp_file(url);
7934 else:
7935 return False;
7936 return False;
7937
7938 def download_file_from_internet_uncompress_file(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_dict__):
7939 formatspecs = FormatSpecsListToDict(formatspecs);
7940 fp = download_file_from_internet_file(url);
7941 fp = UncompressArchiveFile(fp, formatspecs);
7942 fp.seek(0, 0);
7943 if(not fp):
7944 return False;
7945 return fp;
7946
7947 def download_file_from_internet_string(url, headers=geturls_headers_pycatfile_python_alt):
7948 urlparts = urlparse(url);
7949 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7950 return download_file_from_http_string(url, headers);
7951 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7952 return download_file_from_ftp_string(url);
7953 elif(urlparts.scheme=="sftp"):
7954 if(__use_pysftp__ and havepysftp):
7955 return download_file_from_pysftp_string(url);
7956 else:
7957 return download_file_from_sftp_string(url);
7958 else:
7959 return False;
7960 return False;
7961
7962 def download_file_from_internet_uncompress_string(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_dict__):
7963 formatspecs = FormatSpecsListToDict(formatspecs);
7964 fp = download_file_from_internet_string(url);
7965 fp = UncompressArchiveFile(fp, formatspecs);
7966 fp.seek(0, 0);
7967 if(not fp):
7968 return False;
7969 return fp;
7970
7971 def upload_file_to_internet_file(ifp, url):
7972 urlparts = urlparse(url);
7973 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7974 return False;
7975 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7976 return upload_file_to_ftp_file(ifp, url);
7977 elif(urlparts.scheme=="sftp"):
7978 if(__use_pysftp__ and havepysftp):
7979 return upload_file_to_pysftp_file(ifp, url);
7980 else:
7981 return upload_file_to_sftp_file(ifp, url);
7982 else:
7983 return False;
7984 return False;
7985
7986 def upload_file_to_internet_compress_file(ifp, url, formatspecs=__file_format_dict__):
7987 formatspecs = FormatSpecsListToDict(formatspecs);
7988 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
7989 if(not catfileout):
7990 return False;
7991 catfp.seek(0, 0);
7992 upload_file_to_internet_file(catfp, outfile);
7993 return True;
7994
7995 def upload_file_to_internet_string(ifp, url):
7996 urlparts = urlparse(url);
7997 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7998 return False;
7999 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
8000 return upload_file_to_ftp_string(ifp, url);
8001 elif(urlparts.scheme=="sftp"):
8002 if(__use_pysftp__ and havepysftp):
8003 return upload_file_to_pysftp_string(ifp, url);
8004 else:
8005 return upload_file_to_sftp_string(ifp, url);
8006 else:
8007 return False;
8008 return False;
8009
8010 def upload_file_to_internet_compress_string(ifp, url, formatspecs=__file_format_dict__):
8011 formatspecs = FormatSpecsListToDict(formatspecs);
8012 catfp = CompressArchiveFile(BytesIO(ifp), compression, compressionlevel, formatspecs);
8013 if(not catfileout):
8014 return False;
8015 catfp.seek(0, 0);
8016 upload_file_to_internet_file(catfp, outfile);
8017 return True;
8018
8019 try:
8020 if(hasattr(shutil, "register_archive_format")):
8021 # Register the packing format
8022 shutil.register_archive_format(__file_format_name__, PackArchiveFileFunc, description='Pack concatenated files');
8023 except shutil.RegistryError:
8024 pass;
8025
8026 try:
8027 if(hasattr(shutil, "register_unpack_format")):
8028 # Register the unpacking format
8029 shutil.register_unpack_format(__file_format_name__, archivefile_extensions, UnPackArchiveFileFunc, description='UnPack concatenated files');
8030 except shutil.RegistryError:
8031 pass;
A tar like file format name catfile after unix cat command (concatenate files).