?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Add files via upload
[PyCatFile.git] / pycatfile.py
blobf727cbae598a98b14e1bda872e0f1924532610e8
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 '''
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the Revised BSD License.
7
8 This program is distributed in the hope that it will be useful,
9 but WITHOUT ANY WARRANTY; without even the implied warranty of
10 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 Revised BSD License for more details.
12
13 Copyright 2018-2024 Cool Dude 2k - http://idb.berlios.de/
14 Copyright 2018-2024 Game Maker 2k - http://intdb.sourceforge.net/
15 Copyright 2018-2024 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
16
17 $FileInfo: pycatfile.py - Last Update: 5/15/2024 Ver. 0.11.4 RC 1 - Author: cooldude2k $
18 '''
19
20 from __future__ import absolute_import, division, print_function, unicode_literals;
21 import io, os, re, sys, time, stat, zlib, base64, shutil, socket, hashlib, datetime, logging, binascii, zipfile, platform;
22 try:
23 from backports import tempfile;
24 except ImportError:
25 import tempfile;
26 ftpssl = True;
27 try:
28 ftpssl = True;
29 from ftplib import FTP, FTP_TLS;
30 except ImportError:
31 ftpssl = False;
32 from ftplib import FTP;
33
34 try:
35 # Try Python 3 imports
36 from urllib.parse import urlparse, urlunparse;
37 except ImportError:
38 # Fall back to Python 2 imports
39 from urlparse import urlparse, urlunparse;
40
41 if os.name == 'nt': # Only modify if on Windows
42 if sys.version[0] == "2":
43 import codecs;
44 sys.stdout = codecs.getwriter('utf-8')(sys.stdout);
45 sys.stderr = codecs.getwriter('utf-8')(sys.stderr);
46 else:
47 import io;
48 sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace', line_buffering=True);
49 sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', errors='replace', line_buffering=True);
50
51 hashlib_guaranteed = False;
52 os.environ["PYTHONIOENCODING"] = "UTF-8";
53 os.environ["LC_CTYPE"] = "UTF-8";
54 try:
55 reload(sys);
56 try:
57 sys.setdefaultencoding('UTF-8');
58 except NameError:
59 pass;
60 except AttributeError:
61 pass;
62 except NameError:
63 pass;
64 except AttributeError:
65 pass;
66
67 try:
68 from zlib import crc32;
69 except ImportError:
70 from binascii import crc32;
71
72 if(sys.version_info[0]==2):
73 FileNotFoundError = IOError;
74
75 rarfile_support = False;
76 try:
77 import rarfile;
78 rarfile_support = True;
79 except ImportError:
80 rarfile_support = False;
81
82 py7zr_support = False;
83 try:
84 import py7zr;
85 py7zr_support = True;
86 except ImportError:
87 py7zr_support = False;
88
89 try:
90 from xtarfile import is_tarfile;
91 except ImportError:
92 try:
93 from safetar import is_tarfile;
94 except ImportError:
95 from tarfile import is_tarfile;
96
97 try:
98 import xtarfile as tarfile;
99 except ImportError:
100 try:
101 import safetar as tarfile;
102 except ImportError:
103 import tarfile;
104
105 haveparamiko = False;
106 try:
107 import paramiko;
108 haveparamiko = True;
109 except ImportError:
110 haveparamiko = False;
111
112 havepysftp = False;
113 try:
114 import pysftp;
115 havepysftp = True;
116 except ImportError:
117 havepysftp = False;
118
119 haverequests = False;
120 try:
121 import requests;
122 haverequests = True;
123 except ImportError:
124 haverequests = False;
125
126 if(haverequests):
127 import urllib3;
128 logging.getLogger("urllib3").setLevel(logging.WARNING);
129
130 try:
131 # Python 3 imports
132 from urllib.request import Request, build_opener, HTTPBasicAuthHandler;
133 from urllib.parse import urlparse;
134 except ImportError:
135 # Python 2 imports
136 from urllib2 import Request, build_opener, HTTPBasicAuthHandler;
137 from urlparse import urlparse;
138
139 try:
140 from io import StringIO, BytesIO;
141 except ImportError:
142 try:
143 from cStringIO import StringIO, StringIO as BytesIO;
144 except ImportError:
145 from StringIO import StringIO, StringIO as BytesIO;
146
147 __use_pysftp__ = False;
148 if(not havepysftp):
149 __use_pysftp__ = False;
150 __file_format_name__ = "CatFile";
151 __program_name__ = "Py"+__file_format_name__;
152 __file_format_lower__ = __file_format_name__.lower();
153 __file_format_magic__ = __file_format_name__;
154 __file_format_len__ = len(__file_format_magic__);
155 __file_format_hex__ = binascii.hexlify(__file_format_magic__.encode("UTF-8")).decode("UTF-8");
156 __file_format_delimiter__ = "\x00";
157 __file_format_ver__ = "001";
158 __use_new_style__ = True;
159 __use_advanced_list__ = True;
160 __use_alt_inode__ = False;
161 __file_format_extension__ = ".cat";
162 __file_format_list__ = [__file_format_name__, __file_format_magic__, __file_format_lower__, __file_format_len__, __file_format_hex__, __file_format_delimiter__, __file_format_ver__, __use_new_style__, __use_advanced_list__, __use_alt_inode__];
163 __file_format_dict__ = {'format_name': __file_format_name__, 'format_magic': __file_format_magic__, 'format_lower': __file_format_lower__, 'format_len': __file_format_len__, 'format_hex': __file_format_hex__, 'format_delimiter': __file_format_delimiter__, 'format_ver': __file_format_ver__, 'new_style': __use_new_style__, 'use_advanced_list': __use_advanced_list__, 'use_alt_inode': __use_alt_inode__};
164 __project__ = __program_name__;
165 __project_url__ = "https://github.com/GameMaker2k/PyCatFile";
166 __version_info__ = (0, 11, 4, "RC 1", 1);
167 __version_date_info__ = (2024, 5, 17, "RC 1", 1);
168 __version_date__ = str(__version_date_info__[0]) + "." + str(__version_date_info__[1]).zfill(2) + "." + str(__version_date_info__[2]).zfill(2);
169 __revision__ = __version_info__[3];
170 __revision_id__ = "$Id$";
171 if(__version_info__[4] is not None):
172 __version_date_plusrc__ = __version_date__ + "-" + str(__version_date_info__[4]);
173 if(__version_info__[4] is None):
174 __version_date_plusrc__ = __version_date__;
175 if(__version_info__[3] is not None):
176 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]) + " " + str(__version_info__[3]);
177 if(__version_info__[3] is None):
178 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]);
179
180 PyBitness = platform.architecture();
181 if(PyBitness=="32bit" or PyBitness=="32"):
182 PyBitness = "32";
183 elif(PyBitness=="64bit" or PyBitness=="64"):
184 PyBitness = "64";
185 else:
186 PyBitness = "32";
187
188 geturls_ua_pycatfile_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
189 if(platform.python_implementation()!=""):
190 py_implementation = platform.python_implementation();
191 if(platform.python_implementation()==""):
192 py_implementation = "Python";
193 geturls_ua_pycatfile_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=py_implementation, pyver=platform.python_version(), proname=__project__, prover=__version__);
194 geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
195 geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
196 geturls_headers_pycatfile_python = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
197 geturls_headers_pycatfile_python_alt = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python_alt, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
198 geturls_headers_googlebot_google = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
199 geturls_headers_googlebot_google_old = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
200
201 def CompressionSupport():
202 compression_list = [];
203 try:
204 import gzip;
205 compression_list.append("gz");
206 compression_list.append("gzip");
207 except ImportError:
208 '''return False;'''
209 try:
210 import bz2;
211 compression_list.append("bz2");
212 compression_list.append("bzip2");
213 except ImportError:
214 '''return False;'''
215 try:
216 import lz4;
217 compression_list.append("lz4");
218 except ImportError:
219 '''return False;'''
220 try:
221 import lzo;
222 compression_list.append("lzo");
223 compression_list.append("lzop");
224 except ImportError:
225 '''return False;'''
226 try:
227 import zstandard;
228 compression_list.append("zstd");
229 compression_list.append("zstandard");
230 except ImportError:
231 '''return False;'''
232 try:
233 import lzma;
234 compression_list.append("lzma");
235 compression_list.append("xz");
236 except ImportError:
237 '''return False;'''
238 try:
239 from backports import lzma;
240 compression_list.append("lzma");
241 compression_list.append("xz");
242 except ImportError:
243 '''return False;'''
244 return compression_list;
245
246 compressionsupport = CompressionSupport();
247 compressionlist = ['auto'];
248 compressionlistalt = [];
249 outextlist = [];
250 outextlistwd = [];
251 if('gzip' in compressionsupport):
252 compressionlist.append('gzip');
253 compressionlistalt.append('gzip');
254 outextlist.append('gz');
255 outextlistwd.append('.gz');
256 if('bzip2' in compressionsupport):
257 compressionlist.append('bzip2');
258 compressionlistalt.append('bzip2');
259 outextlist.append('bz2');
260 outextlistwd.append('.bz2');
261 if('zstd' in compressionsupport):
262 compressionlist.append('zstd');
263 compressionlistalt.append('zstd');
264 outextlist.append('zst');
265 outextlistwd.append('.zst');
266 if('lz4' in compressionsupport):
267 compressionlist.append('lz4');
268 compressionlistalt.append('lz4');
269 outextlist.append('lz4');
270 outextlistwd.append('.lz4');
271 if('lzo' in compressionsupport):
272 compressionlist.append('lzo');
273 compressionlistalt.append('lzo');
274 outextlist.append('lzo');
275 outextlistwd.append('.lzo');
276 if('lzop' in compressionsupport):
277 compressionlist.append('lzop');
278 compressionlistalt.append('lzop');
279 outextlist.append('lzop');
280 outextlistwd.append('.lzop');
281 if('lzma' in compressionsupport):
282 compressionlist.append('lzma');
283 compressionlistalt.append('lzma');
284 outextlist.append('lzma');
285 outextlistwd.append('.lzma');
286 if('xz' in compressionsupport):
287 compressionlist.append('xz');
288 compressionlistalt.append('xz');
289 outextlist.append('xz');
290 outextlistwd.append('.xz');
291
292 tarfile_mimetype = "application/tar";
293 tarfile_tar_mimetype = tarfile_mimetype;
294 zipfile_mimetype = "application/zip";
295 zipfile_zip_mimetype = zipfile_mimetype;
296 rarfile_mimetype = "application/rar";
297 rarfile_rar_mimetype = rarfile_mimetype;
298 archivefile_mimetype = "application/x-"+__file_format_dict__['format_lower']+"";
299 archivefile_cat_mimetype = archivefile_mimetype;
300 archivefile_gzip_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+gzip";
301 archivefile_gz_mimetype = archivefile_gzip_mimetype;
302 archivefile_bzip2_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+bzip2";
303 archivefile_bz2_mimetype = archivefile_bzip2_mimetype;
304 archivefile_lz4_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lz4";
305 archivefile_lzop_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lzop";
306 archivefile_lzo_mimetype = archivefile_lzop_mimetype;
307 archivefile_zstandard_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+zstandard";
308 archivefile_zstd_mimetype = archivefile_zstandard_mimetype;
309 archivefile_lzma_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lzma";
310 archivefile_xz_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+xz";
311 archivefile_extensions = [__file_format_extension__, __file_format_extension__+".gz", __file_format_extension__+".bz2", __file_format_extension__+".zst", __file_format_extension__+".lz4", __file_format_extension__+".lzo", __file_format_extension__+".lzop", __file_format_extension__+".lzma", __file_format_extension__+".xz"];
312
313 if __name__ == "__main__":
314 import subprocess;
315 curscrpath = os.path.dirname(sys.argv[0]);
316 if(curscrpath==""):
317 curscrpath = ".";
318 if(os.sep=="\\"):
319 curscrpath = curscrpath.replace(os.sep, "/");
320 curscrpath = curscrpath + "/";
321 scrfile = curscrpath + "catfile.py";
322 if(os.path.exists(scrfile) and os.path.isfile(scrfile)):
323 scrcmd = subprocess.Popen([sys.executable, scrfile] + sys.argv[1:]);
324 scrcmd.wait();
325
326 def VerbosePrintOut(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
327 if(not dbgenable):
328 return True;
329 log_functions = {
330 "print": print,
331 "log": logging.info,
332 "warning": logging.warning,
333 "error": logging.error,
334 "critical": logging.critical,
335 "exception": logging.exception,
336 "logalt": lambda x: logging.log(dgblevel, x),
337 "debug": logging.debug
338 };
339 log_function = log_functions.get(outtype);
340 if(log_function):
341 log_function(dbgtxt);
342 return True;
343 return False;
344
345 def VerbosePrintOutReturn(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
346 VerbosePrintOut(dbgtxt, outtype, dbgenable, dgblevel);
347 return dbgtxt;
348
349 def RemoveWindowsPath(dpath):
350 if(dpath is None):
351 dpath = "";
352 if(os.sep!="/"):
353 dpath = dpath.replace(os.path.sep, "/");
354 dpath = dpath.rstrip("/");
355 if(dpath=="." or dpath==".."):
356 dpath = dpath + "/";
357 return dpath;
358
359 def NormalizeRelativePath(inpath):
360 inpath = RemoveWindowsPath(inpath);
361 if(os.path.isabs(inpath)):
362 outpath = inpath;
363 else:
364 if(inpath.startswith("./") or inpath.startswith("../")):
365 outpath = inpath;
366 else:
367 outpath = "./" + inpath;
368 return outpath;
369
370 def PrependPath(base_dir, child_path):
371 # Check if base_dir is None or empty, if so, return child_path as is
372 if not base_dir:
373 return child_path;
374 # Ensure base_dir ends with exactly one slash
375 if not base_dir.endswith('/'):
376 base_dir += '/';
377 # Check if child_path starts with ./ or ../ (indicating a relative path)
378 if child_path.startswith('./') or child_path.startswith('../'):
379 # For relative paths, we don't alter the child_path
380 return base_dir + child_path;
381 else:
382 # For non-relative paths, ensure there's no starting slash on child_path to avoid double slashes
383 return base_dir + child_path.lstrip('/');
384
385 def ListDir(dirpath, followlink=False, duplicates=False):
386 if(isinstance(dirpath, (list, tuple, ))):
387 dirpath = list(filter(None, dirpath));
388 elif(isinstance(dirpath, (str, ))):
389 dirpath = list(filter(None, [dirpath]));
390 retlist = [];
391 for mydirfile in dirpath:
392 if(not os.path.exists(mydirfile)):
393 return False;
394 mydirfile = NormalizeRelativePath(mydirfile);
395 if(os.path.exists(mydirfile) and os.path.islink(mydirfile)):
396 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile));
397 if(os.path.exists(mydirfile) and os.path.isdir(mydirfile)):
398 for root, dirs, filenames in os.walk(mydirfile):
399 dpath = root;
400 dpath = RemoveWindowsPath(dpath);
401 if(dpath not in retlist and not duplicates):
402 retlist.append(dpath);
403 if(duplicates):
404 retlist.append(dpath);
405 for file in filenames:
406 fpath = os.path.join(root, file);
407 fpath = RemoveWindowsPath(fpath);
408 if(fpath not in retlist and not duplicates):
409 retlist.append(fpath);
410 if(duplicates):
411 retlist.append(fpath);
412 else:
413 retlist.append(RemoveWindowsPath(mydirfile));
414 return retlist;
415
416 def ListDirAdvanced(dirpath, followlink=False, duplicates=False):
417 if isinstance(dirpath, (list, tuple)):
418 dirpath = list(filter(None, dirpath));
419 elif isinstance(dirpath, str):
420 dirpath = list(filter(None, [dirpath]));
421 retlist = []
422 for mydirfile in dirpath:
423 if not os.path.exists(mydirfile):
424 return False;
425 mydirfile = NormalizeRelativePath(mydirfile);
426 if os.path.exists(mydirfile) and os.path.islink(mydirfile) and followlink:
427 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile))
428 if os.path.exists(mydirfile) and os.path.isdir(mydirfile):
429 for root, dirs, filenames in os.walk(mydirfile):
430 # Sort dirs and filenames alphabetically in place
431 dirs.sort(key=lambda x: x.lower());
432 filenames.sort(key=lambda x: x.lower());
433 dpath = RemoveWindowsPath(root);
434 if not duplicates and dpath not in retlist:
435 retlist.append(dpath);
436 elif duplicates:
437 retlist.append(dpath);
438 for file in filenames:
439 fpath = os.path.join(root, file);
440 fpath = RemoveWindowsPath(fpath);
441 if not duplicates and fpath not in retlist:
442 retlist.append(fpath);
443 elif duplicates:
444 retlist.append(fpath);
445 else:
446 retlist.append(RemoveWindowsPath(mydirfile));
447 return retlist;
448
449 def create_alias_function(prefix, base_name, suffix, target_function):
450 # Define a new function that wraps the target function
451 def alias_function(*args, **kwargs):
452 return target_function(*args, **kwargs);
453
454 # Create the function name by combining the prefix, base name, and the suffix
455 function_name = "{}{}{}".format(prefix, base_name, suffix);
456
457 # Add the new function to the global namespace
458 globals()[function_name] = alias_function;
459
460 def create_alias_function_alt(prefix, base_name, suffix, target_function):
461 # Create the function name by combining the prefix, base name, and the suffix
462 # Use the format method for string formatting, compatible with Python 2 and 3
463 function_name = "{}{}{}".format(prefix, base_name, suffix)
464 # Add the new function (alias of the target_function) to the global namespace
465 # This line is compatible as-is with both Python 2 and 3
466 globals()[function_name] = target_function
467
468 def FormatSpecsListToDict(formatspecs=__file_format_list__):
469 if(isinstance(formatspecs, (list, tuple, ))):
470 return {'format_name': formatspecs[0], 'format_magic': formatspecs[1], 'format_lower': formatspecs[2], 'format_len': formatspecs[3], 'format_hex': formatspecs[4], 'format_delimiter': formatspecs[5], 'format_ver': formatspecs[6], 'new_style': formatspecs[7], 'use_advanced_list': formatspecs[8], 'use_alt_inode': formatspecs[9]};
471 elif(isinstance(formatspecs, (dict, ))):
472 return formatspecs;
473 else:
474 return __file_format_dict__;
475 return __file_format_dict__;
476
477 def TarFileCheck(infile):
478 try:
479 if tarfile.is_tarfile(infile):
480 return True;
481 else:
482 return False;
483 except TypeError:
484 try:
485 # Check if the input is a file object
486 if hasattr(infile, 'read'):
487 # Save the current file position
488 current_position = infile.tell();
489 # Attempt to open the file object as a tar file
490 tar = tarfile.open(fileobj=infile);
491 tar.close();
492 # Restore the file position
493 infile.seek(current_position);
494 return True;
495 else:
496 # Assume it's a filename
497 tar = tarfile.open(name=infile);
498 tar.close();
499 return True;
500 except tarfile.TarError:
501 return False;
502
503 # initial_value can be 0xFFFF or 0x0000
504 def crc16_ansi(msg, initial_value=0xFFFF):
505 # CRC-16-IBM / CRC-16-ANSI polynomial and initial value
506 poly = 0x8005; # Polynomial for CRC-16-IBM / CRC-16-ANSI
507 crc = initial_value; # Initial value
508 for b in msg:
509 crc ^= b << 8; # XOR byte into CRC top byte
510 for _ in range(8): # Process each bit
511 if crc & 0x8000: # If the top bit is set
512 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
513 else:
514 crc = crc << 1; # Just shift left
515 crc &= 0xFFFF; # Ensure CRC remains 16-bit
516 return crc;
517
518 # initial_value can be 0xFFFF or 0x0000
519 def crc16_ibm(msg, initial_value=0xFFFF):
520 return crc16_ansi(msg, initial_value);
521
522 # initial_value is 0xFFFF
523 def crc16(msg):
524 return crc16_ansi(msg, 0xFFFF);
525
526 # initial_value can be 0xFFFF, 0x1D0F or 0x0000
527 def crc16_ccitt(msg, initial_value=0xFFFF):
528 # CRC-16-CCITT polynomial
529 poly = 0x1021; # Polynomial for CRC-16-CCITT
530 # Use the specified initial value
531 crc = initial_value;
532 for b in msg:
533 crc ^= b << 8; # XOR byte into CRC top byte
534 for _ in range(8): # Process each bit
535 if crc & 0x8000: # If the top bit is set
536 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
537 else:
538 crc = crc << 1; # Just shift left
539 crc &= 0xFFFF; # Ensure CRC remains 16-bit
540 return crc;
541
542 # initial_value can be 0x42F0E1EBA9EA3693 or 0x0000000000000000
543 def crc64_ecma(msg, initial_value=0x0000000000000000):
544 # CRC-64-ECMA polynomial and initial value
545 poly = 0x42F0E1EBA9EA3693;
546 crc = initial_value; # Initial value for CRC-64-ECMA
547 for b in msg:
548 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
549 for _ in range(8): # Process each bit
550 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
551 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
552 else:
553 crc <<= 1; # Just shift left if the MSB is 0
554 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
555 return crc;
556
557 # initial_value can be 0x000000000000001B or 0xFFFFFFFFFFFFFFFF
558 def crc64_iso(msg, initial_value=0xFFFFFFFFFFFFFFFF):
559 # CRC-64-ISO polynomial and initial value
560 poly = 0x000000000000001B;
561 crc = initial_value; # Common initial value for CRC-64-ISO
562 for b in msg:
563 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
564 for _ in range(8): # Process each bit
565 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
566 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
567 else:
568 crc <<= 1; # Just shift left if the MSB is 0
569 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
570 return crc;
571
572 def GetDataFromArray(data, path, default=None):
573 element = data;
574 try:
575 for key in path:
576 element = element[key];
577 return element;
578 except (KeyError, TypeError, IndexError):
579 return default;
580
581 def GetDataFromArrayAlt(structure, path, default=None):
582 element = structure;
583 for key in path:
584 if isinstance(element, dict) and key in element:
585 element = element[key];
586 elif isinstance(element, list) and isinstance(key, int) and -len(element) <= key < len(element):
587 element = element[key];
588 else:
589 return default;
590 return element;
591
592 def GetHeaderChecksum(inlist=[], checksumtype="crc32", encodedata=True, formatspecs=__file_format_dict__):
593 formatspecs = FormatSpecsListToDict(formatspecs);
594 if isinstance(inlist, list):
595 fileheader = AppendNullBytes(inlist, formatspecs['format_delimiter']);
596 else:
597 fileheader = AppendNullByte(inlist, formatspecs['format_delimiter']);
598 if(encodedata):
599 fileheader = fileheader.encode('UTF-8');
600 if(checksumtype=="none" or checksumtype==""):
601 catfileheadercshex = format(0, 'x').lower();
602 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
603 catfileheadercshex = format(crc16(fileheader) & 0xffff, '04x').lower();
604 elif(checksumtype=="crc16_ccitt"):
605 catfileheadercshex = format(crc16_ccitt(fileheader) & 0xffff, '04x').lower();
606 elif(checksumtype=="adler32"):
607 catfileheadercshex = format(zlib.adler32(fileheader) & 0xffffffff, '08x').lower();
608 elif(checksumtype=="crc32"):
609 catfileheadercshex = format(crc32(fileheader) & 0xffffffff, '08x').lower();
610 elif(checksumtype=="crc64_ecma"):
611 catfileheadercshex = format(crc64_ecma(fileheader) & 0xffffffffffffffff, '016x').lower();
612 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
613 catfileheadercshex = format(crc64_iso(fileheader) & 0xffffffffffffffff, '016x').lower();
614 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
615 checksumoutstr = hashlib.new(checksumtype);
616 checksumoutstr.update(fileheader);
617 catfileheadercshex = checksumoutstr.hexdigest().lower();
618 else:
619 catfileheadercshex = format(0, 'x').lower();
620 return catfileheadercshex;
621
622 def GetFileChecksum(instr, checksumtype="crc32", encodedata=True, formatspecs=__file_format_dict__):
623 formatspecs = FormatSpecsListToDict(formatspecs);
624 if(encodedata):
625 instr = instr.encode('UTF-8');
626 if(checksumtype=="none" or checksumtype==""):
627 catinstrcshex = format(0, 'x').lower();
628 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
629 catinstrcshex = format(crc16(instr) & 0xffff, '04x').lower();
630 elif(checksumtype=="crc16_ccitt"):
631 catinstrcshex = format(crc16_ccitt(instr) & 0xffff, '04x').lower();
632 elif(checksumtype=="adler32"):
633 catinstrcshex = format(zlib.adler32(instr) & 0xffffffff, '08x').lower();
634 elif(checksumtype=="crc32"):
635 catinstrcshex = format(crc32(instr) & 0xffffffff, '08x').lower();
636 elif(checksumtype=="crc64_ecma"):
637 catinstrcshex = format(crc64_ecma(instr) & 0xffffffffffffffff, '016x').lower();
638 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
639 catinstrcshex = format(crc64_iso(instr) & 0xffffffffffffffff, '016x').lower();
640 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
641 checksumoutstr = hashlib.new(checksumtype);
642 checksumoutstr.update(instr);
643 catinstrcshex = checksumoutstr.hexdigest().lower();
644 else:
645 catinstrcshex = format(0, 'x').lower();
646 return catinstrcshex;
647
648 def ValidateHeaderChecksum(inlist=[], checksumtype="crc32", inchecksum="0", formatspecs=__file_format_dict__):
649 formatspecs = FormatSpecsListToDict(formatspecs);
650 catfileheadercshex = GetHeaderChecksum(inlist, checksumtype, True, formatspecs);
651 inchecksum = inchecksum.lower();
652 catfileheadercshex = catfileheadercshex.lower();
653 if(inchecksum==catfileheadercshex):
654 return True;
655 else:
656 return False;
657
658 def ValidateFileChecksum(infile, checksumtype="crc32", inchecksum="0", formatspecs=__file_format_dict__):
659 formatspecs = FormatSpecsListToDict(formatspecs);
660 catinfilecshex = GetFileChecksum(infile, checksumtype, True, formatspecs);
661 inchecksum = inchecksum.lower();
662 catinfilecshex = catinfilecshex.lower();
663 if(inchecksum==catinfilecshex):
664 return True;
665 else:
666 return False;
667
668 def ReadTillNullByteOld(fp, delimiter=__file_format_dict__['format_delimiter']):
669 curbyte = b"";
670 curfullbyte = b"";
671 nullbyte = delimiter.encode("UTF-8");
672 while(True):
673 curbyte = fp.read(1);
674 if(curbyte==nullbyte or not curbyte):
675 break;
676 curfullbyte = curfullbyte + curbyte;
677 return curfullbyte.decode('UTF-8');
678
679 def ReadUntilNullByteOld(fp, delimiter=__file_format_dict__['format_delimiter']):
680 return ReadTillNullByteOld(fp, delimiter);
681
682 def ReadTillNullByteAlt(fp, delimiter=__file_format_dict__['format_delimiter'], chunk_size=1024, max_read=1024000):
683 delimiter = delimiter.encode('UTF-8') # Ensure the delimiter is in bytes
684 buffer = bytearray();
685 total_read = 0;
686 delimiter_length = len(delimiter);
687 while True:
688 chunk = fp.read(chunk_size)
689 if not chunk:
690 # End of file reached without finding the delimiter
691 break;
692 buffer.extend(chunk);
693 total_read += len(chunk);
694 if delimiter in buffer:
695 # Delimiter found, calculate where to reset the file pointer
696 index = buffer.find(delimiter);
697 # Calculate how many extra bytes were read after the delimiter
698 extra_bytes_read = len(buffer) - (index + delimiter_length);
699 # Move the file pointer back to just after the delimiter
700 fp.seek(-extra_bytes_read, 1);
701 buffer = buffer[:index];
702 break;
703 if total_read >= max_read:
704 # Stop reading if max limit is reached to prevent excessive memory usage
705 raise MemoryError("Maximum read limit reached without finding the delimiter.");
706 # Check for incomplete UTF-8 sequences at the end of the buffer
707 if len(buffer) > 1 and 128 <= buffer[-1] < 192:
708 # This suggests that the last byte might be the start of a multi-byte character
709 # Try to read one more byte to complete the character
710 extra_byte = fp.read(1);
711 if extra_byte:
712 buffer.extend(extra_byte);
713 else:
714 # No more data available
715 break;
716 try:
717 return buffer.decode('UTF-8', errors='replace');
718 except UnicodeDecodeError:
719 return buffer.decode('UTF-8', errors='replace');
720
721 def ReadUntilNullByteAlt(fp, delimiter=__file_format_dict__['format_delimiter'], chunk_size=1024, max_read=1024000):
722 return ReadTillNullByteAlt(fp, delimiter, chunk_size, max_read);
723
724 def ReadTillNullByte(fp, delimiter=__file_format_dict__['format_delimiter'], max_read=1024000):
725 curfullbyte = bytearray();
726 nullbyte = delimiter.encode("UTF-8");
727 total_read = 0; # Track the total number of bytes read
728 while True:
729 curbyte = fp.read(1);
730 if curbyte == nullbyte or not curbyte:
731 break;
732 curfullbyte.extend(curbyte);
733 total_read += 1;
734 if total_read >= max_read:
735 raise MemoryError("Maximum read limit reached without finding the delimiter.");
736 # Decode the full byte array to string once out of the loop
737 try:
738 return curfullbyte.decode('UTF-8');
739 except UnicodeDecodeError:
740 # Handle potential partial UTF-8 characters
741 for i in range(1, 4):
742 try:
743 return curfullbyte[:-i].decode('UTF-8');
744 except UnicodeDecodeError:
745 continue;
746 raise; # Re-raise if decoding fails even after trimming
747
748 def ReadUntilNullByte(fp, delimiter=__file_format_dict__['format_delimiter'], max_read=1024000):
749 return ReadTillNullByte(fp, delimiter, max_read);
750
751 def ReadTillNullByteByNum(fp, delimiter=__file_format_dict__['format_delimiter'], num_delimiters=1, chunk_size=1024, max_read=1024000):
752 delimiter = delimiter.encode('UTF-8'); # Ensure the delimiter is in bytes
753 buffer = bytearray();
754 total_read = 0;
755 delimiter_length = len(delimiter);
756 results = [];
757 while len(results) < num_delimiters:
758 chunk = fp.read(chunk_size);
759 if not chunk:
760 # End of file reached; decode whatever is collected if it's the last needed part
761 if len(buffer) > 0:
762 results.append(buffer.decode('UTF-8', errors='replace'));
763 break;
764 buffer.extend(chunk)
765 total_read += len(chunk);
766 # Check if we have found the delimiter
767 while delimiter in buffer:
768 index = buffer.find(delimiter);
769 # Decode the section before the delimiter
770 results.append(buffer[:index].decode('UTF-8', errors='replace'));
771 # Remove the processed part from the buffer
772 buffer = buffer[index + delimiter_length:];
773 if len(results) == num_delimiters:
774 # If reached the required number of delimiters, adjust the file pointer and stop
775 fp.seek(-len(buffer), 1);
776 return results;
777 if total_read >= max_read:
778 # Stop reading if max limit is reached to prevent excessive memory usage
779 raise MemoryError("Maximum read limit reached without finding the delimiter.");
780 # Check for incomplete UTF-8 sequences at the end of the buffer
781 if len(buffer) > 1 and 128 <= buffer[-1] < 192:
782 # This suggests that the last byte might be the start of a multi-byte character
783 # Try to read one more byte to complete the character
784 extra_byte = fp.read(1);
785 if extra_byte:
786 buffer.extend(extra_byte);
787 else:
788 # No more data available
789 break;
790 # Process remaining buffer if less than the required number of delimiters were found
791 if len(buffer) > 0 and len(results) < num_delimiters:
792 results.append(buffer.decode('UTF-8', errors='replace'));
793 return results;
794
795 def ReadUntilNullByteByNum(fp, delimiter=__file_format_dict__['format_delimiter'], num_delimiters=1, chunk_size=1024, max_read=1024000):
796 return ReadTillNullByteByNum(fp, delimiter, num_delimiters, chunk_size, max_read);
797
798 def SeekToEndOfFile(fp):
799 lasttell = 0;
800 while(True):
801 fp.seek(1, 1);
802 if(lasttell==fp.tell()):
803 break;
804 lasttell = fp.tell();
805 return True;
806
807 def ReadFileHeaderData(fp, rounds=0, delimiter=__file_format_dict__['format_delimiter']):
808 rocount = 0;
809 roend = int(rounds);
810 HeaderOut = [];
811 while(rocount<roend):
812 HeaderOut.append(ReadTillNullByte(fp, delimiter));
813 rocount = rocount + 1;
814 return HeaderOut;
815
816 def ReadFileHeaderDataBySize(fp, delimiter=__file_format_dict__['format_delimiter']):
817 headerpresize = ReadTillNullByte(fp, delimiter);
818 headersize = int(headerpresize, 16);
819 if(headersize<=0):
820 return [];
821 headercontent = str(fp.read(headersize).decode('UTF-8')).split(delimiter);
822 fp.seek(1, 1);
823 rocount = 0;
824 roend = int(len(headercontent));
825 HeaderOut = [headerpresize];
826 while(rocount<roend):
827 HeaderOut.append(headercontent[rocount]);
828 rocount = rocount + 1;
829 return HeaderOut;
830
831 def ReadFileHeaderDataWoSize(fp, delimiter=__file_format_dict__['format_delimiter']):
832 preheaderdata = ReadFileHeaderData(fp, 2, delimiter);
833 headersize = int(preheaderdata[0], 16);
834 headernumfields = int(preheaderdata[1], 16);
835 if(headersize<=0 or headernumfields<=0):
836 return [];
837 headerdata = ReadTillNullByteByNum(fp, delimiter, headernumfields);
838 #headerdata = ReadFileHeaderData(fp, headernumfields, delimiter);
839 HeaderOut = preheaderdata + headerdata;
840 return HeaderOut;
841
842 def ReadFileHeaderDataBySizeWithContent(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
843 formatspecs = FormatSpecsListToDict(formatspecs);
844 delimiter = formatspecs['format_delimiter'];
845 fheaderstart = fp.tell();
846 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
847 if(len(HeaderOut)==0):
848 return False;
849 if(re.findall("^[.|/]", HeaderOut[3])):
850 fname = HeaderOut[3];
851 else:
852 fname = "./"+HeaderOut[3];
853 fchecksumtype = HeaderOut[-3].lower();
854 fcs = HeaderOut[-2].lower();
855 fccs = HeaderOut[-1].lower();
856 fsize = int(HeaderOut[5], 16);
857 fcompression = HeaderOut[12];
858 fcsize = int(HeaderOut[13], 16);
859 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
860 if(fcs!=newfcs and not skipchecksum):
861 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
862 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
863 return False;
864 fhend = fp.tell() - 1;
865 fcontentstart = fp.tell();
866 fcontents = BytesIO();
867 if(fsize>0 and not listonly):
868 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
869 fcontents.write(fp.read(fsize));
870 else:
871 fcontents.write(fp.read(fcsize));
872 elif(fsize>0 and listonly):
873 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
874 fp.seek(fsize, 1);
875 else:
876 fp.seek(fcsize, 1);
877 fcontents.seek(0, 0);
878 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
879 if(fccs!=newfccs and not skipchecksum and not listonly):
880 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
881 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
882 return False;
883 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
884 pass;
885 else:
886 fcontents.seek(0, 0);
887 if(uncompress):
888 fcontents = UncompressArchiveFile(fcontents, formatspecs);
889 fp.seek(1, 1);
890 fcontentend = fp.tell() - 1;
891 HeaderOut.append(fcontents);
892 return HeaderOut;
893
894 def ReadFileHeaderDataBySizeWithContentToArray(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
895 formatspecs = FormatSpecsListToDict(formatspecs);
896 delimiter = formatspecs['format_delimiter'];
897 fheaderstart = fp.tell();
898 if(formatspecs['new_style']):
899 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
900 else:
901 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
902 if(len(HeaderOut)==0):
903 return False;
904 fheadsize = int(HeaderOut[0], 16);
905 fnumfields = int(HeaderOut[1], 16);
906 ftype = int(HeaderOut[2], 16);
907 if(re.findall("^[.|/]", HeaderOut[3])):
908 fname = HeaderOut[3];
909 else:
910 fname = "./"+HeaderOut[3];
911 fbasedir = os.path.dirname(fname);
912 flinkname = HeaderOut[4];
913 fsize = int(HeaderOut[5], 16);
914 fatime = int(HeaderOut[6], 16);
915 fmtime = int(HeaderOut[7], 16);
916 fctime = int(HeaderOut[8], 16);
917 fbtime = int(HeaderOut[9], 16);
918 fmode = int(HeaderOut[10], 16);
919 fchmode = stat.S_IMODE(fmode);
920 ftypemod = stat.S_IFMT(fmode);
921 fwinattributes = int(HeaderOut[11], 16);
922 fcompression = HeaderOut[12];
923 fcsize = int(HeaderOut[13], 16);
924 fuid = int(HeaderOut[14], 16);
925 funame = HeaderOut[15];
926 fgid = int(HeaderOut[16], 16);
927 fgname = HeaderOut[17];
928 fid = int(HeaderOut[18], 16);
929 finode = int(HeaderOut[19], 16);
930 flinkcount = int(HeaderOut[20], 16);
931 fdev_minor = int(HeaderOut[21], 16);
932 fdev_major = int(HeaderOut[22], 16);
933 frdev_minor = int(HeaderOut[23], 16);
934 frdev_major = int(HeaderOut[24], 16);
935 fextrasize = int(HeaderOut[25], 16);
936 fextrafields = int(HeaderOut[26], 16);
937 extrafieldslist = [];
938 extrastart = 27;
939 extraend = extrastart + fextrafields;
940 extrafieldslist = [];
941 if(extrastart<extraend):
942 extrafieldslist.append(HeaderOut[extrastart]);
943 extrastart = extrastart + 1;
944 fchecksumtype = HeaderOut[extrastart].lower();
945 fcs = HeaderOut[extrastart + 1].lower();
946 fccs = HeaderOut[extrastart + 2].lower();
947 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
948 if(fcs!=newfcs and not skipchecksum):
949 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
950 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
951 return False;
952 fhend = fp.tell() - 1;
953 fcontentstart = fp.tell();
954 fcontents = BytesIO();
955 pyhascontents = False;
956 if(fsize>0 and not listonly):
957 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
958 fcontents.write(fp.read(fsize));
959 else:
960 fcontents.write(fp.read(fcsize));
961 pyhascontents = True;
962 elif(fsize>0 and listonly):
963 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
964 fp.seek(fsize, 1);
965 else:
966 fp.seek(fcsize, 1);
967 pyhascontents = False;
968 fcontents.seek(0, 0);
969 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
970 if(fccs!=newfccs and not skipchecksum and not listonly):
971 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
972 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
973 return False;
974 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
975 pass;
976 else:
977 fcontents.seek(0, 0);
978 if(uncompress):
979 fcontents = UncompressArchiveFile(fcontents, formatspecs);
980 fcontents.seek(0, 0);
981 fccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
982 fp.seek(1, 1);
983 fcontentend = fp.tell() - 1;
984 fcontents.seek(0, 0);
985 catlist = {'fheadersize': fheadsize, 'fhstart': fheaderstart, 'fhend': fhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': fchecksumtype, 'fnumfields': fnumfields + 2, 'frawheader': HeaderOut, 'fextrafields': fextrafields, 'fextrafieldsize': fextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': fcs, 'fcontentchecksum': fccs, 'fhascontents': pyhascontents, 'fcontentstart': fcontentstart, 'fcontentend': fcontentend, 'fcontents': fcontents};
986 return catlist;
987
988 def ReadFileHeaderDataBySizeWithContentToList(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
989 formatspecs = FormatSpecsListToDict(formatspecs);
990 delimiter = formatspecs['format_delimiter'];
991 fheaderstart = fp.tell();
992 if(formatspecs['new_style']):
993 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
994 else:
995 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
996 if(len(HeaderOut)==0):
997 return False;
998 fheadsize = int(HeaderOut[0], 16);
999 fnumfields = int(HeaderOut[1], 16);
1000 ftype = int(HeaderOut[2], 16);
1001 if(re.findall("^[.|/]", HeaderOut[3])):
1002 fname = HeaderOut[3];
1003 else:
1004 fname = "./"+HeaderOut[3];
1005 fbasedir = os.path.dirname(fname);
1006 flinkname = HeaderOut[4];
1007 fsize = int(HeaderOut[5], 16);
1008 fatime = int(HeaderOut[6], 16);
1009 fmtime = int(HeaderOut[7], 16);
1010 fctime = int(HeaderOut[8], 16);
1011 fbtime = int(HeaderOut[9], 16);
1012 fmode = int(HeaderOut[10], 16);
1013 fchmode = stat.S_IMODE(fmode);
1014 ftypemod = stat.S_IFMT(fmode);
1015 fwinattributes = int(HeaderOut[11], 16);
1016 fcompression = HeaderOut[12];
1017 fcsize = int(HeaderOut[13], 16);
1018 fuid = int(HeaderOut[14], 16);
1019 funame = HeaderOut[15];
1020 fgid = int(HeaderOut[16], 16);
1021 fgname = HeaderOut[17];
1022 fid = int(HeaderOut[18], 16);
1023 finode = int(HeaderOut[19], 16);
1024 flinkcount = int(HeaderOut[20], 16);
1025 fdev_minor = int(HeaderOut[21], 16);
1026 fdev_major = int(HeaderOut[22], 16);
1027 frdev_minor = int(HeaderOut[23], 16);
1028 frdev_major = int(HeaderOut[24], 16);
1029 fextrasize = int(HeaderOut[25], 16);
1030 fextrafields = int(HeaderOut[26], 16);
1031 extrafieldslist = [];
1032 extrastart = 27;
1033 extraend = extrastart + fextrafields;
1034 extrafieldslist = [];
1035 if(extrastart<extraend):
1036 extrafieldslist.append(HeaderOut[extrastart]);
1037 extrastart = extrastart + 1;
1038 fchecksumtype = HeaderOut[extrastart].lower();
1039 fcs = HeaderOut[extrastart + 1].lower();
1040 fccs = HeaderOut[extrastart + 2].lower();
1041 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
1042 if(fcs!=newfcs and not skipchecksum):
1043 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
1044 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
1045 return False;
1046 fhend = fp.tell() - 1;
1047 fcontentstart = fp.tell();
1048 fcontents = BytesIO();
1049 pyhascontents = False;
1050 if(fsize>0 and not listonly):
1051 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1052 fcontents.write(fp.read(fsize));
1053 else:
1054 fcontents.write(fp.read(fcsize));
1055 pyhascontents = True;
1056 elif(fsize>0 and listonly):
1057 if(fcompression=="none" or fcompression=="" or fcompression=="atuo"):
1058 fp.seek(fsize, 1);
1059 else:
1060 fp.seek(fcsize, 1);
1061 pyhascontents = False;
1062 fcontents.seek(0, 0);
1063 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
1064 if(fccs!=newfccs and not skipchecksum and not listonly):
1065 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
1066 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
1067 return False;
1068 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1069 pass;
1070 else:
1071 fcontents.seek(0, 0);
1072 if(uncompress):
1073 fcontents = UncompressArchiveFile(fcontents, formatspecs);
1074 fcontents.seek(0, 0);
1075 fp.seek(1, 1);
1076 fcontentend = fp.tell() - 1;
1077 catlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, extrafieldslist, fchecksumtype, fcontents];
1078 return catlist;
1079
1080 def ReadFileDataBySizeWithContent(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1081 formatspecs = FormatSpecsListToDict(formatspecs);
1082 delimiter = formatspecs['format_delimiter'];
1083 curloc = fp.tell();
1084 if(curloc>0):
1085 fp.seek(0, 0);
1086 catheader = ReadFileHeaderData(fp, 4, delimiter);
1087 if(curloc>0):
1088 fp.seek(curloc, 0);
1089 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1090 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1091 if(not headercheck and not skipchecksum):
1092 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1093 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1094 return False;
1095 fnumfiles = int(catheader[1], 16);
1096 countnum = 0;
1097 flist = [];
1098 while(countnum < fnumfiles):
1099 HeaderOut = ReadFileHeaderDataBySizeWithContent(fp, listonly, uncompress, skipchecksum, formatspecs)
1100 if(len(HeaderOut)==0):
1101 break;
1102 flist.append(HeaderOut);
1103 countnum = countnum + 1;
1104 return flist;
1105
1106 def ReadFileDataBySizeWithContentToArray(fp, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1107 formatspecs = FormatSpecsListToDict(formatspecs);
1108 delimiter = formatspecs['format_delimiter'];
1109 curloc = fp.tell();
1110 if(curloc>0):
1111 fp.seek(0, 0);
1112 catheader = ReadFileHeaderData(fp, 4, delimiter);
1113 if(curloc>0):
1114 fp.seek(curloc, 0);
1115 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1116 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1117 if(not headercheck and not skipchecksum):
1118 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1119 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1120 return False;
1121 catstring = catheader[0];
1122 catversion = re.findall(r"([\d]+)", catstring);
1123 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
1124 fprenumfiles = catheader[1];
1125 fnumfiles = int(fprenumfiles, 16);
1126 fprechecksumtype = catheader[2];
1127 fprechecksum = catheader[3];
1128 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
1129 if(seekstart<0 and seekstart>fnumfiles):
1130 seekstart = 0;
1131 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
1132 seekend = fnumfiles;
1133 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
1134 seekend = fnumfiles - abs(seekend);
1135 if(seekstart>0):
1136 il = 0;
1137 while(il < seekstart):
1138 prefhstart = fp.tell();
1139 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs['format_delimiter']);
1140 if(len(preheaderdata)==0):
1141 break;
1142 prefsize = int(preheaderdata[5], 16);
1143 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-3].lower(), True, formatspecs);
1144 prefcs = preheaderdata[-2];
1145 if(prefcs!=prenewfcs and not skipchecksum):
1146 VVerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
1147 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
1148 return False;
1149 valid_archive = False;
1150 invalid_archive = True;
1151 prefhend = fp.tell() - 1;
1152 prefcontentstart = fp.tell();
1153 prefcontents = BytesIO();
1154 pyhascontents = False;
1155 if(prefsize>0):
1156 prefcontents.write(fp.read(prefsize));
1157 prefcontents.seek(0, 0);
1158 prenewfccs = GetFileChecksum(prefcontents.read(), preheaderdata[-3].lower(), False, formatspecs);
1159 prefccs = preheaderdata[-1];
1160 pyhascontents = True;
1161 if(prefccs!=prenewfccs and not skipchecksum):
1162 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
1163 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
1164 return False;
1165 fp.seek(1, 1);
1166 il = il + 1;
1167 realidnum = 0;
1168 countnum = seekstart;
1169 while(countnum < seekend):
1170 catlist['ffilelist'].update({realidnum: {'fid': realidnum, 'fidalt': realidnum}});
1171 HeaderOut = ReadFileHeaderDataBySizeWithContentToArray(fp, listonly, uncompress, skipchecksum, formatspecs);
1172 if(len(HeaderOut)==0):
1173 break;
1174 catlist['ffilelist'][realidnum].update(HeaderOut);
1175 countnum = countnum + 1;
1176 realidnum = realidnum + 1;
1177 return catlist;
1178
1179 def ReadFileDataBySizeWithContentToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1180 formatspecs = FormatSpecsListToDict(formatspecs);
1181 if(isinstance(infile, dict)):
1182 listcatfiles = infile;
1183 else:
1184 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
1185 return False;
1186 listcatfiles = ReadFileDataBySizeWithContentToArray(infile, seekstart, seekend, listonly, uncompress, skipchecksum, formatspecs);
1187 if(not listcatfiles):
1188 return False;
1189 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
1190 lenlist = len(listcatfiles['ffilelist']);
1191 lcfi = 0;
1192 lcfx = int(listcatfiles['fnumfiles']);
1193 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
1194 lcfx = int(lenlist);
1195 else:
1196 lcfx = int(listcatfiles['fnumfiles']);
1197 while(lcfi < lcfx):
1198 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
1199 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
1200 catarray['filetoid'].update(filetoidarray);
1201 catarray['idtofile'].update(idtofilearray);
1202 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
1203 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
1204 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
1205 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
1206 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
1207 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
1208 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
1209 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
1210 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
1211 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1212 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1213 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
1214 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
1215 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
1216 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
1217 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
1218 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1219 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1220 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
1221 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
1222 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
1223 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1224 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1225 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
1226 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
1227 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
1228 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
1229 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1230 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1231 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1232 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1233 lcfi = lcfi + 1;
1234 return catarray;
1235
1236 def ReadFileDataBySizeWithContentToList(fp, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1237 formatspecs = FormatSpecsListToDict(formatspecs);
1238 delimiter = formatspecs['format_delimiter'];
1239 curloc = fp.tell();
1240 if(curloc>0):
1241 fp.seek(0, 0);
1242 catheader = ReadFileHeaderData(fp, 4, delimiter);
1243 if(curloc>0):
1244 fp.seek(curloc, 0);
1245 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1246 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1247 if(not headercheck and not skipchecksum):
1248 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1249 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1250 return False;
1251 catstring = catheader[0];
1252 catversion = re.findall(r"([\d]+)", catstring);
1253 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
1254 fprenumfiles = catheader[1];
1255 fnumfiles = int(fprenumfiles, 16);
1256 fprechecksumtype = catheader[2];
1257 fprechecksum = catheader[3];
1258 catlist = [];
1259 if(seekstart<0 and seekstart>fnumfiles):
1260 seekstart = 0;
1261 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
1262 seekend = fnumfiles;
1263 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
1264 seekend = fnumfiles - abs(seekend);
1265 if(seekstart>0):
1266 il = 0;
1267 while(il < seekstart):
1268 prefhstart = fp.tell();
1269 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs['format_delimiter']);
1270 if(len(preheaderdata)==0):
1271 break;
1272 prefsize = int(preheaderdata[5], 16);
1273 prefcompression = preheaderdata[12];
1274 prefcsize = int(preheaderdata[13], 16);
1275 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-3].lower(), True, formatspecs);
1276 prefcs = preheaderdata[-2];
1277 if(prefcs!=prenewfcs and not skipchecksum):
1278 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
1279 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
1280 return False;
1281 valid_archive = False;
1282 invalid_archive = True;
1283 prefhend = fp.tell() - 1;
1284 prefcontentstart = fp.tell();
1285 prefcontents = "";
1286 pyhascontents = False;
1287 if(prefsize>0):
1288 if(prefcompression=="none" or prefcompression=="" or prefcompression=="auto"):
1289 prefcontents = catfp.read(prefsize);
1290 else:
1291 prefcontents = catfp.read(prefcsize);
1292 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
1293 prefccs = preheaderdata[-1];
1294 pyhascontents = True;
1295 if(prefccs!=prenewfccs and not skipchecksum):
1296 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
1297 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
1298 return False;
1299 fp.seek(1, 1);
1300 il = il + 1;
1301 realidnum = 0;
1302 countnum = seekstart;
1303 while(countnum < seekend):
1304 HeaderOut = ReadFileHeaderDataBySizeWithContentToList(fp, listonly, uncompress, skipchecksum, formatspecs);
1305 if(len(HeaderOut)==0):
1306 break;
1307 catlist.append(HeaderOut);
1308 countnum = countnum + 1;
1309 realidnum = realidnum + 1;
1310 return catlist;
1311
1312 def ReadInFileBySizeWithContentToArray(infile, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1313 formatspecs = FormatSpecsListToDict(formatspecs);
1314 delimiter = formatspecs['format_delimiter'];
1315 if(hasattr(infile, "read") or hasattr(infile, "write")):
1316 fp = infile;
1317 fp.seek(0, 0);
1318 fp = UncompressArchiveFile(fp, formatspecs);
1319 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1320 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1321 return False;
1322 if(not fp):
1323 return False;
1324 fp.seek(0, 0);
1325 elif(infile=="-"):
1326 fp = BytesIO();
1327 if(hasattr(sys.stdin, "buffer")):
1328 shutil.copyfileobj(sys.stdin.buffer, fp);
1329 else:
1330 shutil.copyfileobj(sys.stdin, fp);
1331 fp.seek(0, 0);
1332 fp = UncompressArchiveFile(fp, formatspecs);
1333 if(not fp):
1334 return False;
1335 fp.seek(0, 0);
1336 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
1337 fp = download_file_from_internet_file(infile);
1338 fp = UncompressArchiveFile(fp, formatspecs);
1339 fp.seek(0, 0);
1340 if(not fp):
1341 return False;
1342 fp.seek(0, 0);
1343 else:
1344 infile = RemoveWindowsPath(infile);
1345 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
1346 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1347 return False;
1348 compresscheck = CheckCompressionType(infile, formatspecs, True);
1349 if(not compresscheck):
1350 fextname = os.path.splitext(infile)[1];
1351 if(fextname==".gz"):
1352 compresscheck = "gzip";
1353 elif(fextname==".bz2"):
1354 compresscheck = "bzip2";
1355 elif(fextname==".zst"):
1356 compresscheck = "zstd";
1357 elif(fextname==".lz4" or fextname==".clz4"):
1358 compresscheck = "lz4";
1359 elif(fextname==".lzo" or fextname==".lzop"):
1360 compresscheck = "lzo";
1361 elif(fextname==".lzma" or fextname==".xz"):
1362 compresscheck = "lzma";
1363 else:
1364 return False;
1365 if(not compresscheck):
1366 return False;
1367 fp = UncompressFile(infile, formatspecs, "rb");
1368 return ReadFileDataBySizeWithContentToArray(fp, listonly, uncompress, skipchecksum, formatspecs);
1369
1370 def ReadInFileBySizeWithContentToList(infile, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1371 formatspecs = FormatSpecsListToDict(formatspecs);
1372 delimiter = formatspecs['format_delimiter'];
1373 if(hasattr(infile, "read") or hasattr(infile, "write")):
1374 fp = infile;
1375 fp.seek(0, 0);
1376 fp = UncompressArchiveFile(fp, formatspecs);
1377 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1378 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1379 return False;
1380 if(not fp):
1381 return False;
1382 fp.seek(0, 0);
1383 elif(infile=="-"):
1384 fp = BytesIO();
1385 if(hasattr(sys.stdin, "buffer")):
1386 shutil.copyfileobj(sys.stdin.buffer, fp);
1387 else:
1388 shutil.copyfileobj(sys.stdin, fp);
1389 fp.seek(0, 0);
1390 fp = UncompressArchiveFile(fp, formatspecs);
1391 if(not fp):
1392 return False;
1393 fp.seek(0, 0);
1394 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
1395 fp = download_file_from_internet_file(infile);
1396 fp = UncompressArchiveFile(fp, formatspecs);
1397 fp.seek(0, 0);
1398 if(not fp):
1399 return False;
1400 fp.seek(0, 0);
1401 else:
1402 infile = RemoveWindowsPath(infile);
1403 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
1404 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1405 return False;
1406 compresscheck = CheckCompressionType(infile, formatspecs, True);
1407 if(not compresscheck):
1408 fextname = os.path.splitext(infile)[1];
1409 if(fextname==".gz"):
1410 compresscheck = "gzip";
1411 elif(fextname==".bz2"):
1412 compresscheck = "bzip2";
1413 elif(fextname==".zst"):
1414 compresscheck = "zstd";
1415 elif(fextname==".lz4" or fextname==".clz4"):
1416 compresscheck = "lz4";
1417 elif(fextname==".lzo" or fextname==".lzop"):
1418 compresscheck = "lzo";
1419 elif(fextname==".lzma" or fextname==".xz"):
1420 compresscheck = "lzma";
1421 else:
1422 return False;
1423 if(not compresscheck):
1424 return False;
1425 fp = UncompressFile(infile, formatspecs, "rb");
1426 return ReadFileDataBySizeWithContentToList(fp, seekstart, seekend, listonly, uncompress, skipchecksum, formatspecs);
1427
1428 def ReadInFileBySizeWithContentToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
1429 formatspecs = FormatSpecsListToDict(formatspecs);
1430 if(isinstance(infile, dict)):
1431 listcatfiles = infile;
1432 else:
1433 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
1434 infile = RemoveWindowsPath(infile);
1435 listcatfiles = ReadInFileBySizeWithContentToArray(infile, listonly, skipchecksum, formatspecs);
1436 if(not listcatfiles):
1437 return False;
1438 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
1439 lenlist = len(listcatfiles['ffilelist']);
1440 lcfi = 0;
1441 lcfx = int(listcatfiles['fnumfiles']);
1442 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
1443 lcfx = int(lenlist);
1444 else:
1445 lcfx = int(listcatfiles['fnumfiles']);
1446 while(lcfi < lcfx):
1447 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
1448 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
1449 catarray['filetoid'].update(filetoidarray);
1450 catarray['idtofile'].update(idtofilearray);
1451 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
1452 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
1453 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
1454 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
1455 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
1456 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
1457 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
1458 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
1459 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
1460 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1461 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1462 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
1463 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
1464 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
1465 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
1466 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
1467 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1468 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1469 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
1470 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
1471 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
1472 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1473 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1474 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
1475 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
1476 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
1477 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
1478 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1479 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1480 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1481 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1482 lcfi = lcfi + 1;
1483 return catarray;
1484
1485 def AppendNullByte(indata, delimiter=__file_format_dict__['format_delimiter']):
1486 outdata = str(indata) + delimiter;
1487 return outdata;
1488
1489 def AppendNullBytes(indata=[], delimiter=__file_format_dict__['format_delimiter']):
1490 outdata = "";
1491 inum = 0;
1492 il = len(indata);
1493 while(inum < il):
1494 outdata = outdata + AppendNullByte(indata[inum], delimiter);
1495 inum = inum + 1;
1496 return outdata;
1497
1498 def AppendFileHeader(fp, numfiles, checksumtype="crc32", formatspecs=__file_format_dict__):
1499 formatspecs = FormatSpecsListToDict(formatspecs);
1500 delimiter = formatspecs['format_delimiter'];
1501 catver = formatspecs['format_ver'];
1502 fileheaderver = str(int(catver.replace(".", "")));
1503 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
1504 fp.write(fileheader.encode('UTF-8'));
1505 fnumfiles = format(int(numfiles), 'x').lower();
1506 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs['format_delimiter']);
1507 catfileheadercshex = GetFileChecksum(fileheader + fnumfilesa, checksumtype, True, formatspecs);
1508 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
1509 fp.write(fnumfilesa.encode('UTF-8'));
1510 try:
1511 fp.flush();
1512 os.fsync(fp.fileno());
1513 except io.UnsupportedOperation:
1514 pass;
1515 except AttributeError:
1516 pass;
1517 except OSError as e:
1518 pass;
1519 return fp;
1520
1521 def MakeEmptyFilePointer(fp, checksumtype="crc32", formatspecs=__file_format_dict__):
1522 formatspecs = FormatSpecsListToDict(formatspecs);
1523 AppendFileHeader(fp, 0, checksumtype, formatspecs);
1524 return fp;
1525
1526 def MakeEmptyFile(outfile, compression="auto", compressionlevel=None, checksumtype="crc32", formatspecs=__file_format_dict__, returnfp=False):
1527 formatspecs = FormatSpecsListToDict(formatspecs);
1528 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1529 if(os.path.exists(outfile)):
1530 try:
1531 os.unlink(outfile);
1532 except OSError as e:
1533 pass;
1534 if(outfile=="-"):
1535 verbose = False;
1536 catfpfp = BytesIO();
1537 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1538 catfp = outfile;
1539 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1540 catfp = BytesIO();
1541 else:
1542 fbasename = os.path.splitext(outfile)[0];
1543 fextname = os.path.splitext(outfile)[1];
1544 catfp = CompressOpenFile(outfile, True, compressionlevel);
1545 catfp = AppendFileHeader(catfp, 0, checksumtype, formatspecs);
1546 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1547 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1548 try:
1549 catfp.flush();
1550 os.fsync(catfp.fileno());
1551 except io.UnsupportedOperation:
1552 pass;
1553 except AttributeError:
1554 pass;
1555 except OSError as e:
1556 pass;
1557 if(outfile=="-"):
1558 catfp.seek(0, 0);
1559 if(hasattr(sys.stdout, "buffer")):
1560 shutil.copyfileobj(catfp, sys.stdout.buffer);
1561 else:
1562 shutil.copyfileobj(catfp, sys.stdout);
1563 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1564 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1565 catfp.seek(0, 0);
1566 upload_file_to_internet_file(catfp, outfile);
1567 if(returnfp):
1568 catfp.seek(0, 0);
1569 return catfp;
1570 else:
1571 catfp.close();
1572 return True;
1573
1574 def AppendFileHeaderWithContent(fp, filevalues=[], extradata=[], filecontent="", checksumtype="crc32", formatspecs=__file_format_dict__):
1575 formatspecs = FormatSpecsListToDict(formatspecs);
1576 extrafields = format(len(extradata), 'x').lower();
1577 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
1578 if(len(extradata)>0):
1579 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
1580 extrasizelen = format(len(extrasizestr), 'x').lower();
1581 catoutlen = len(filevalues) + len(extradata) + 5;
1582 catoutlenhex = format(catoutlen, 'x').lower();
1583 catoutlist = filevalues;
1584 catoutlist.insert(0, catoutlenhex);
1585 catoutlist.append(extrasizelen);
1586 catoutlist.append(extrafields);
1587 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
1588 if(len(extradata)>0):
1589 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
1590 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
1591 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
1592 catfilecontentcshex = GetFileChecksum(filecontent, checksumtype, False, formatspecs);
1593 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
1594 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
1595 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
1596 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
1597 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
1598 catfileoutstrecd = catfileoutstr.encode('UTF-8');
1599 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
1600 catfileout = catfileoutstrecd + filecontent + nullstrecd;
1601 fp.write(catfileout);
1602 try:
1603 fp.flush();
1604 os.fsync(fp.fileno());
1605 except io.UnsupportedOperation:
1606 pass;
1607 except AttributeError:
1608 pass;
1609 except OSError as e:
1610 pass;
1611 return fp;
1612
1613 def AppendFilesWithContent(infiles, fp, dirlistfromtxt=False, filevalues=[], extradata=[], compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
1614 formatspecs = FormatSpecsListToDict(formatspecs);
1615 advancedlist = formatspecs['use_advanced_list'];
1616 altinode = formatspecs['use_alt_inode'];
1617 if(verbose):
1618 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
1619 if(infiles=="-"):
1620 for line in sys.stdin:
1621 infilelist.append(line.strip());
1622 infilelist = list(filter(None, infilelist));
1623 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
1624 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
1625 return False;
1626 with UncompressFile(infiles, formatspecs, "r") as finfile:
1627 for line in finfile:
1628 infilelist.append(line.strip());
1629 infilelist = list(filter(None, infilelist));
1630 else:
1631 if(isinstance(infiles, (list, tuple, ))):
1632 infilelist = list(filter(None, infiles));
1633 elif(isinstance(infiles, (str, ))):
1634 infilelist = list(filter(None, [infiles]));
1635 if(advancedlist):
1636 GetDirList = ListDirAdvanced(infilelist, followlink, False);
1637 else:
1638 GetDirList = ListDir(infilelist, followlink, False);
1639 if(not GetDirList):
1640 return False;
1641 curinode = 0;
1642 curfid = 0;
1643 inodelist = [];
1644 inodetofile = {};
1645 filetoinode = {};
1646 inodetocatinode = {};
1647 numfiles = int(len(GetDirList));
1648 fnumfiles = format(numfiles, 'x').lower();
1649 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
1650 for curfname in GetDirList:
1651 if(re.findall("^[.|/]", curfname)):
1652 fname = curfname;
1653 else:
1654 fname = "./"+curfname;
1655 if(verbose):
1656 VerbosePrintOut(fname);
1657 if(not followlink or followlink is None):
1658 fstatinfo = os.lstat(fname);
1659 else:
1660 fstatinfo = os.stat(fname);
1661 fpremode = fstatinfo.st_mode;
1662 finode = fstatinfo.st_ino;
1663 flinkcount = fstatinfo.st_nlink;
1664 ftype = 0;
1665 if(stat.S_ISREG(fpremode)):
1666 ftype = 0;
1667 elif(stat.S_ISLNK(fpremode)):
1668 ftype = 2;
1669 elif(stat.S_ISCHR(fpremode)):
1670 ftype = 3;
1671 elif(stat.S_ISBLK(fpremode)):
1672 ftype = 4;
1673 elif(stat.S_ISDIR(fpremode)):
1674 ftype = 5;
1675 elif(stat.S_ISFIFO(fpremode)):
1676 ftype = 6;
1677 elif(stat.S_ISSOCK(fpremode)):
1678 ftype = 8;
1679 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
1680 ftype = 9;
1681 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
1682 ftype = 10;
1683 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
1684 ftype = 111;
1685 else:
1686 ftype = 0;
1687 flinkname = "";
1688 fcurfid = format(int(curfid), 'x').lower();
1689 if(not followlink and finode!=0):
1690 if(ftype!=1):
1691 if(finode in inodelist):
1692 ftype = 1;
1693 flinkname = inodetofile[finode];
1694 if(altinode):
1695 fcurinode = format(int(finode), 'x').lower();
1696 else:
1697 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
1698 if(finode not in inodelist):
1699 inodelist.append(finode);
1700 inodetofile.update({finode: fname});
1701 inodetocatinode.update({finode: curinode});
1702 if(altinode):
1703 fcurinode = format(int(finode), 'x').lower();
1704 else:
1705 fcurinode = format(int(curinode), 'x').lower();
1706 curinode = curinode + 1;
1707 else:
1708 fcurinode = format(int(curinode), 'x').lower();
1709 curinode = curinode + 1;
1710 curfid = curfid + 1;
1711 if(ftype==2):
1712 flinkname = os.readlink(fname);
1713 fdev = fstatinfo.st_dev;
1714 getfdev = GetDevMajorMinor(fdev);
1715 fdev_minor = getfdev[0];
1716 fdev_major = getfdev[1];
1717 frdev = fstatinfo.st_dev;
1718 if(hasattr(fstatinfo, "st_rdev")):
1719 frdev = fstatinfo.st_rdev;
1720 else:
1721 frdev = fstatinfo.st_dev;
1722 getfrdev = GetDevMajorMinor(frdev);
1723 frdev_minor = getfrdev[0];
1724 frdev_major = getfrdev[1];
1725 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
1726 fsize = format(int("0"), 'x').lower();
1727 elif(ftype==0 or ftype==7):
1728 fsize = format(int(fstatinfo.st_size), 'x').lower();
1729 else:
1730 fsize = format(int(fstatinfo.st_size)).lower();
1731 fatime = format(int(fstatinfo.st_atime), 'x').lower();
1732 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
1733 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
1734 if(hasattr(fstatinfo, "st_birthtime")):
1735 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
1736 else:
1737 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
1738 fmode = format(int(fstatinfo.st_mode), 'x').lower();
1739 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
1740 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
1741 fuid = format(int(fstatinfo.st_uid), 'x').lower();
1742 fgid = format(int(fstatinfo.st_gid), 'x').lower();
1743 funame = "";
1744 try:
1745 import pwd;
1746 try:
1747 userinfo = pwd.getpwuid(fstatinfo.st_uid);
1748 funame = userinfo.pw_name;
1749 except KeyError:
1750 funame = "";
1751 except ImportError:
1752 funame = "";
1753 fgname = "";
1754 try:
1755 import grp;
1756 try:
1757 groupinfo = grp.getgrgid(fstatinfo.st_gid);
1758 fgname = groupinfo.gr_name;
1759 except KeyError:
1760 fgname = "";
1761 except ImportError:
1762 fgname = "";
1763 fdev_minor = format(int(fdev_minor), 'x').lower();
1764 fdev_major = format(int(fdev_major), 'x').lower();
1765 frdev_minor = format(int(frdev_minor), 'x').lower();
1766 frdev_major = format(int(frdev_major), 'x').lower();
1767 finode = format(int(finode), 'x').lower();
1768 flinkcount = format(int(flinkcount), 'x').lower();
1769 if(hasattr(fstatinfo, "st_file_attributes")):
1770 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
1771 else:
1772 fwinattributes = format(int(0), 'x').lower();
1773 fcompression = "";
1774 fcsize = format(int(0), 'x').lower();
1775 fcontents = BytesIO();
1776 chunk_size = 1024;
1777 if(ftype==0 or ftype==7):
1778 with open(fname, "rb") as fpc:
1779 shutil.copyfileobj(fpc, fcontents);
1780 if(not compresswholefile):
1781 fcontents.seek(0, 2);
1782 ucfsize = fcontents.tell();
1783 fcontents.seek(0, 0);
1784 if(compression=="auto"):
1785 ilsize = len(compressionlistalt);
1786 ilmin = 0;
1787 ilcsize = [];
1788 while(ilmin < ilsize):
1789 cfcontents = BytesIO();
1790 shutil.copyfileobj(fcontents, cfcontents);
1791 fcontents.seek(0, 0);
1792 cfcontents.seek(0, 0);
1793 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
1794 if(cfcontents):
1795 cfcontents.seek(0, 2);
1796 ilcsize.append(cfcontents.tell());
1797 cfcontents.close();
1798 else:
1799 try:
1800 ilcsize.append(sys.maxint);
1801 except AttributeError:
1802 ilcsize.append(sys.maxsize);
1803 ilmin = ilmin + 1;
1804 ilcmin = ilcsize.index(min(ilcsize));
1805 compression = compressionlistalt[ilcmin];
1806 fcontents.seek(0, 0);
1807 cfcontents = BytesIO();
1808 shutil.copyfileobj(fcontents, cfcontents);
1809 cfcontents.seek(0, 0);
1810 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
1811 cfcontents.seek(0, 2);
1812 cfsize = cfcontents.tell();
1813 if(ucfsize > cfsize):
1814 fcsize = format(int(cfsize), 'x').lower();
1815 fcompression = compression;
1816 fcontents.close();
1817 fcontents = cfcontents;
1818 if(followlink and (ftype==1 or ftype==2)):
1819 flstatinfo = os.stat(flinkname);
1820 with open(flinkname, "rb") as fpc:
1821 shutil.copyfileobj(fpc, fcontents);
1822 if(not compresswholefile):
1823 fcontents.seek(0, 2);
1824 ucfsize = fcontents.tell();
1825 fcontents.seek(0, 0);
1826 if(compression=="auto"):
1827 ilsize = len(compressionlistalt);
1828 ilmin = 0;
1829 ilcsize = [];
1830 while(ilmin < ilsize):
1831 cfcontents = BytesIO();
1832 shutil.copyfileobj(fcontents, cfcontents);
1833 fcontents.seek(0, 0);
1834 cfcontents.seek(0, 0);
1835 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
1836 if(cfcontents):
1837 cfcontents.seek(0, 2);
1838 ilcsize.append(cfcontents.tell());
1839 cfcontents.close();
1840 else:
1841 try:
1842 ilcsize.append(sys.maxint);
1843 except AttributeError:
1844 ilcsize.append(sys.maxsize);
1845 ilmin = ilmin + 1;
1846 ilcmin = ilcsize.index(min(ilcsize));
1847 compression = compressionlistalt[ilcmin];
1848 fcontents.seek(0, 0);
1849 cfcontents = BytesIO();
1850 shutil.copyfileobj(fcontents, cfcontents);
1851 cfcontents.seek(0, 0);
1852 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
1853 cfcontents.seek(0, 2);
1854 cfsize = cfcontents.tell();
1855 if(ucfsize > cfsize):
1856 fcsize = format(int(cfsize), 'x').lower();
1857 fcompression = compression;
1858 fcontents.close();
1859 fcontents = cfcontents;
1860 if(fcompression=="none"):
1861 fcompression = "";
1862 fcontents.seek(0, 0);
1863 ftypehex = format(ftype, 'x').lower();
1864 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
1865 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
1866 if(numfiles>0):
1867 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
1868 fp.seek(0, 0);
1869 return fp;
1870
1871 def AppendListsWithContent(inlist, fp, dirlistfromtxt=False, filevalues=[], extradata=[], compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
1872 formatspecs = FormatSpecsListToDict(formatspecs);
1873 if(verbose):
1874 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
1875 GetDirList = inlist;
1876 if(not GetDirList):
1877 return False;
1878 curinode = 0;
1879 curfid = 0;
1880 inodelist = [];
1881 inodetofile = {};
1882 filetoinode = {};
1883 inodetocatinode = {};
1884 numfiles = int(len(GetDirList));
1885 fnumfiles = format(numfiles, 'x').lower();
1886 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
1887 for curfname in GetDirList:
1888 ftype = format(curfname[0], 'x').lower();
1889 if(re.findall("^[.|/]", curfname[1])):
1890 fname = curfname[1];
1891 else:
1892 fname = "./"+curfname[1];
1893 fbasedir = os.path.dirname(fname);
1894 flinkname = curfname[2];
1895 fsize = format(curfname[3], 'x').lower();
1896 fatime = format(curfname[4], 'x').lower();
1897 fmtime = format(curfname[5], 'x').lower();
1898 fctime = format(curfname[6], 'x').lower();
1899 fbtime = format(curfname[7], 'x').lower();
1900 fmode = format(curfname[8], 'x').lower();
1901 fwinattributes = format(curfname[9], 'x').lower();
1902 fcompression = curfname[10];
1903 fcsize = format(curfname[11], 'x').lower();
1904 fuid = format(curfname[12], 'x').lower();
1905 funame = curfname[13];
1906 fgid = format(curfname[14], 'x').lower();
1907 fgname = curfname[15];
1908 fid = format(curfname[16], 'x').lower();
1909 finode = format(curfname[17], 'x').lower();
1910 flinkcount = format(curfname[18], 'x').lower();
1911 fdev_minor = format(curfname[19], 'x').lower();
1912 fdev_major = format(curfname[20], 'x').lower();
1913 frdev_minor = format(curfname[21], 'x').lower();
1914 frdev_major = format(curfname[22], 'x').lower();
1915 extradata = curfname[23];
1916 fchecksumtype = curfname[24];
1917 fcontents = curfname[25];
1918 catoutlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
1919 fcontents.seek(0, 0);
1920 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
1921 if(numfiles>0):
1922 fp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
1923 return fp;
1924
1925 def AppendInFileWithContent(infile, fp, dirlistfromtxt=False, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
1926 formatspecs = FormatSpecsListToDict(formatspecs);
1927 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, True, False, formatspecs);
1928 return AppendListsWithContent(inlist, fp, dirlistfromtxt, filevalues, extradata, followlink, checksumtype, formatspecs, verbose);
1929
1930 def AppendFilesWithContentToOutFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
1931 formatspecs = FormatSpecsListToDict(formatspecs);
1932 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1933 if(os.path.exists(outfile)):
1934 try:
1935 os.unlink(outfile);
1936 except OSError as e:
1937 pass;
1938 if(outfile=="-"):
1939 verbose = False;
1940 catfpfp = BytesIO();
1941 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1942 catfp = outfile;
1943 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1944 catfp = BytesIO();
1945 else:
1946 fbasename = os.path.splitext(outfile)[0];
1947 fextname = os.path.splitext(outfile)[1];
1948 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
1949 catfp = AppendFilesWithContent(infiles, catfp, dirlistfromtxt, filevalues, extradata, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose);
1950 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1951 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1952 try:
1953 catfp.flush();
1954 os.fsync(catfp.fileno());
1955 except io.UnsupportedOperation:
1956 pass;
1957 except AttributeError:
1958 pass;
1959 except OSError as e:
1960 pass;
1961 if(outfile=="-"):
1962 catfp.seek(0, 0);
1963 if(hasattr(sys.stdout, "buffer")):
1964 shutil.copyfileobj(catfp, sys.stdout.buffer);
1965 else:
1966 shutil.copyfileobj(catfp, sys.stdout);
1967 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1968 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1969 catfp.seek(0, 0);
1970 upload_file_to_internet_file(catfp, outfile);
1971 if(returnfp):
1972 catfp.seek(0, 0);
1973 return catfp;
1974 else:
1975 catfp.close();
1976 return True;
1977
1978 def AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
1979 formatspecs = FormatSpecsListToDict(formatspecs);
1980 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1981 if(os.path.exists(outfile)):
1982 try:
1983 os.unlink(outfile);
1984 except OSError as e:
1985 pass;
1986 if(outfile=="-"):
1987 verbose = False;
1988 catfpfp = BytesIO();
1989 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1990 catfp = outfile;
1991 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1992 catfp = BytesIO();
1993 else:
1994 fbasename = os.path.splitext(outfile)[0];
1995 fextname = os.path.splitext(outfile)[1];
1996 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
1997 catfp = AppendListsWithContent(inlist, catfp, dirlistfromtxt, filevalues, extradata, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose);
1998 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1999 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2000 try:
2001 catfp.flush();
2002 os.fsync(catfp.fileno());
2003 except io.UnsupportedOperation:
2004 pass;
2005 except AttributeError:
2006 pass;
2007 except OSError as e:
2008 pass;
2009 if(outfile=="-"):
2010 catfp.seek(0, 0);
2011 if(hasattr(sys.stdout, "buffer")):
2012 shutil.copyfileobj(catfp, sys.stdout.buffer);
2013 else:
2014 shutil.copyfileobj(catfp, sys.stdout);
2015 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2016 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2017 catfp.seek(0, 0);
2018 upload_file_to_internet_file(catfp, outfile);
2019 if(returnfp):
2020 catfp.seek(0, 0);
2021 return catfp;
2022 else:
2023 catfp.close();
2024 return True;
2025
2026 def AppendInFileWithContentToOutFile(infile, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2027 formatspecs = FormatSpecsListToDict(formatspecs);
2028 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, True, False, formatspecs);
2029 return AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt, compression, compresswholefile, compressionlevel, filevalues, extradata, followlink, checksumtype, formatspecs, verbose, returnfp);
2030
2031 def PrintPermissionString(fchmode, ftype):
2032 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
2033 permissionstr = "";
2034 for fmodval in str(oct(fchmode))[-3:]:
2035 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
2036 if(ftype==0 or ftype==7):
2037 permissionstr = "-" + permissionstr;
2038 if(ftype==1):
2039 permissionstr = "h" + permissionstr;
2040 if(ftype==2):
2041 permissionstr = "l" + permissionstr;
2042 if(ftype==3):
2043 permissionstr = "c" + permissionstr;
2044 if(ftype==4):
2045 permissionstr = "b" + permissionstr;
2046 if(ftype==5):
2047 permissionstr = "d" + permissionstr;
2048 if(ftype==6):
2049 permissionstr = "f" + permissionstr;
2050 if(ftype==8):
2051 permissionstr = "D" + permissionstr;
2052 if(ftype==9):
2053 permissionstr = "p" + permissionstr;
2054 if(ftype==10):
2055 permissionstr = "w" + permissionstr;
2056 try:
2057 permissionoutstr = stat.filemode(fchmode);
2058 except AttributeError:
2059 permissionoutstr = permissionstr;
2060 except KeyError:
2061 permissionoutstr = permissionstr;
2062 return permissionoutstr;
2063
2064 def PrintPermissionStringAlt(fchmode, ftype):
2065 permissions = {
2066 '0': '---', '1': '--x', '2': '-w-', '3': '-wx',
2067 '4': 'r--', '5': 'r-x', '6': 'rw-', '7': 'rwx'
2068 };
2069 # Translate file mode into permission string
2070 permissionstr = ''.join([permissions[i] for i in str(oct(fchmode))[-3:]]);
2071 # Append file type indicator
2072 type_indicators = {
2073 0: '-', 1: 'h', 2: 'l', 3: 'c', 4: 'b',
2074 5: 'd', 6: 'f', 8: 'D', 9: 'p', 10: 'w'
2075 };
2076 file_type = type_indicators.get(ftype, '-');
2077 permissionstr = file_type + permissionstr;
2078 try:
2079 permissionoutstr = stat.filemode(fchmode);
2080 except AttributeError:
2081 permissionoutstr = permissionstr;
2082 return permissionoutstr;
2083
2084 def CheckCompressionType(infile, formatspecs=__file_format_dict__, closefp=True):
2085 formatspecs = FormatSpecsListToDict(formatspecs);
2086 if(hasattr(infile, "read") or hasattr(infile, "write")):
2087 catfp = infile;
2088 else:
2089 try:
2090 catfp = open(infile, "rb");
2091 except FileNotFoundError:
2092 return False;
2093 catfp.seek(0, 0);
2094 prefp = catfp.read(2);
2095 filetype = False;
2096 if(prefp==binascii.unhexlify("1f8b")):
2097 filetype = "gzip";
2098 catfp.seek(0, 0);
2099 prefp = catfp.read(3);
2100 if(prefp==binascii.unhexlify("425a68")):
2101 filetype = "bzip2";
2102 if(prefp==binascii.unhexlify("5d0000")):
2103 filetype = "lzma";
2104 catfp.seek(0, 0);
2105 prefp = catfp.read(4);
2106 if(prefp==binascii.unhexlify("28b52ffd")):
2107 filetype = "zstd";
2108 if(prefp==binascii.unhexlify("04224d18")):
2109 filetype = "lz4";
2110 if(prefp==binascii.unhexlify("504B0304")):
2111 filetype = "zipfile";
2112 catfp.seek(0, 0);
2113 prefp = catfp.read(5);
2114 if(prefp==binascii.unhexlify("7573746172")):
2115 filetype = "tarfile";
2116 catfp.seek(0, 0);
2117 prefp = catfp.read(6);
2118 if(prefp==binascii.unhexlify("fd377a585a00")):
2119 filetype = "lzma";
2120 if(prefp==binascii.unhexlify("377abcaf271c")):
2121 filetype = "7zipfile";
2122 catfp.seek(0, 0);
2123 prefp = catfp.read(7);
2124 if(prefp==binascii.unhexlify("526172211a0700")):
2125 filetype = "rarfile";
2126 if(prefp==binascii.unhexlify("43617446696c65")):
2127 filetype = "catfile";
2128 catfp.seek(0, 0);
2129 prefp = catfp.read(8);
2130 if(prefp==binascii.unhexlify("526172211a070100")):
2131 filetype = "rarfile";
2132 catfp.seek(0, 0);
2133 prefp = catfp.read(formatspecs['format_len']);
2134 if(prefp==binascii.unhexlify(formatspecs['format_hex'])):
2135 filetype = formatspecs['format_lower'];
2136 catfp.seek(0, 0);
2137 prefp = catfp.read(9);
2138 if(prefp==binascii.unhexlify("894c5a4f000d0a1a0a")):
2139 filetype = "lzo";
2140 catfp.seek(0, 0);
2141 prefp = catfp.read(10);
2142 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
2143 filetype = "tarfile";
2144 catfp.seek(0, 0);
2145 if(filetype=="gzip" or filetype=="bzip2" or filetype=="lzma" or filetype=="zstd" or filetype=="lz4"):
2146 if(TarFileCheck(catfp)):
2147 filetype = "tarfile";
2148 if(not filetype):
2149 if(TarFileCheck(catfp)):
2150 filetype = "tarfile";
2151 elif(zipfile.is_zipfile(catfp)):
2152 filetype = "zipfile";
2153 elif(rarfile.is_rarfile(catfp) or rarfile.is_rarfile_sfx(catfp)):
2154 filetype = "rarile";
2155 else:
2156 filetype = False;
2157 catfp.seek(0, 0);
2158 if(closefp):
2159 catfp.close();
2160 return filetype;
2161
2162 def CheckCompressionTypeFromString(instring, formatspecs=__file_format_dict__, closefp=True):
2163 formatspecs = FormatSpecsListToDict(formatspecs);
2164 try:
2165 instringsfile = BytesIO(instring);
2166 except TypeError:
2167 instringsfile = BytesIO(instring.encode("UTF-8"));
2168 return CheckCompressionType(instringsfile, formatspecs, closefp);
2169
2170 def GetCompressionMimeType(infile, formatspecs=__file_format_dict__):
2171 formatspecs = FormatSpecsListToDict(formatspecs);
2172 compresscheck = CheckCompressionType(fp, formatspecs, False);
2173 if(compresscheck=="gzip" or compresscheck=="gz"):
2174 return archivefile_gzip_mimetype;
2175 if(compresscheck=="bzip2" or compresscheck=="bz2"):
2176 return archivefile_bzip2_mimetype;
2177 if(compresscheck=="zstd" or compresscheck=="zstandard"):
2178 return archivefile_zstandard_mimetype;
2179 if(compresscheck=="lz4"):
2180 return archivefile_lz4_mimetype;
2181 if(compresscheck=="lzo" or compresscheck=="lzop"):
2182 return archivefile_lzop_mimetype;
2183 if(compresscheck=="lzma"):
2184 return archivefile_lzma_mimetype;
2185 if(compresscheck=="xz"):
2186 return archivefile_xz_mimetype;
2187 if(compresscheck=="catfile" or compresscheck=="cat" or compresscheck==formatspecs['format_lower']):
2188 return archivefile_cat_mimetype;
2189 if(not compresscheck):
2190 return False;
2191 return False;
2192
2193 def UncompressArchiveFile(fp, formatspecs=__file_format_dict__):
2194 formatspecs = FormatSpecsListToDict(formatspecs);
2195 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
2196 return False;
2197 compresscheck = CheckCompressionType(fp, formatspecs, False);
2198 if(compresscheck=="gzip"):
2199 try:
2200 import gzip;
2201 except ImportError:
2202 return False;
2203 catfp = gzip.GzipFile(fileobj=fp, mode="rb");
2204 if(compresscheck=="bzip2"):
2205 try:
2206 import bz2;
2207 except ImportError:
2208 return False;
2209 catfp = BytesIO();
2210 catfp.write(bz2.decompress(fp.read()));
2211 if(compresscheck=="zstd"):
2212 try:
2213 import zstandard;
2214 except ImportError:
2215 return False;
2216 catfp = BytesIO();
2217 catfp.write(zstandard.decompress(fp.read()));
2218 if(compresscheck=="lz4"):
2219 try:
2220 import lz4.frame;
2221 except ImportError:
2222 return False;
2223 catfp = BytesIO();
2224 catfp.write(lz4.frame.decompress(fp.read()));
2225 if(compresscheck=="lzo" or compresscheck=="lzop"):
2226 try:
2227 import lzo;
2228 except ImportError:
2229 return False;
2230 catfp = BytesIO();
2231 catfp.write(lzo.decompress(fp.read()));
2232 if(compresscheck=="lzma" or compresscheck=="xz"):
2233 try:
2234 import lzma;
2235 except ImportError:
2236 try:
2237 from backports import lzma
2238 except ImportError:
2239 return False;
2240 catfp = BytesIO();
2241 catfp.write(lzma.decompress(fp.read()));
2242 if(compresscheck=="catfile" or compresscheck==formatspecs['format_lower']):
2243 catfp = fp;
2244 if(not compresscheck):
2245 try:
2246 import lzma;
2247 except ImportError:
2248 try:
2249 from backports import lzma
2250 except ImportError:
2251 return False;
2252 catfp = BytesIO();
2253 with fp as fpcontent:
2254 try:
2255 catfp.write(lzma.decompress(fp.read()));
2256 except lzma.LZMAError:
2257 return False;
2258 if(compresscheck!="catfile" or compresscheck!=formatspecs['format_lower']):
2259 fp.close();
2260 return catfp;
2261
2262 create_alias_function("Uncompress", __file_format_name__, "", UncompressArchiveFile);
2263
2264 def UncompressFile(infile, formatspecs=__file_format_dict__, mode="rb"):
2265 formatspecs = FormatSpecsListToDict(formatspecs);
2266 compresscheck = CheckCompressionType(infile, formatspecs, False);
2267 if(sys.version_info[0]==2 and compresscheck):
2268 if(mode=="rt"):
2269 mode = "r";
2270 if(mode=="wt"):
2271 mode = "w";
2272 try:
2273 if(compresscheck=="gzip"):
2274 try:
2275 import gzip;
2276 except ImportError:
2277 return False;
2278 try:
2279 filefp = gzip.open(infile, mode, encoding="UTF-8");
2280 except (ValueError, TypeError) as e:
2281 filefp = gzip.open(infile, mode);
2282 if(compresscheck=="bzip2"):
2283 try:
2284 import bz2;
2285 except ImportError:
2286 return False;
2287 try:
2288 filefp = bz2.open(infile, mode, encoding="UTF-8");
2289 except (ValueError, TypeError) as e:
2290 filefp = bz2.open(infile, mode);
2291 if(compresscheck=="zstd"):
2292 try:
2293 import zstandard;
2294 except ImportError:
2295 return False;
2296 try:
2297 filefp = zstandard.open(infile, mode, encoding="UTF-8");
2298 except (ValueError, TypeError) as e:
2299 filefp = zstandard.open(infile, mode);
2300 if(compresscheck=="lz4"):
2301 try:
2302 import lz4.frame;
2303 except ImportError:
2304 return False;
2305 try:
2306 filefp = lz4.frame.open(infile, mode, encoding="UTF-8");
2307 except (ValueError, TypeError) as e:
2308 filefp = lz4.frame.open(infile, mode);
2309 if(compresscheck=="lzo"):
2310 try:
2311 import lzo;
2312 except ImportError:
2313 return False;
2314 try:
2315 filefp = lzo.open(infile, mode, encoding="UTF-8");
2316 except (ValueError, TypeError) as e:
2317 filefp = lzo.open(infile, mode);
2318 if(compresscheck=="lzma"):
2319 try:
2320 import lzma;
2321 except ImportError:
2322 try:
2323 from backports import lzma
2324 except ImportError:
2325 return False;
2326 try:
2327 filefp = lzma.open(infile, mode, encoding="UTF-8");
2328 except (ValueError, TypeError) as e:
2329 filefp = lzma.open(infile, mode);
2330 if(compresscheck=="catfile" or compresscheck==formatspecs['format_lower']):
2331 try:
2332 filefp = open(infile, mode, encoding="UTF-8");
2333 except (ValueError, TypeError) as e:
2334 filefp = open(infile, mode);
2335 if(not compresscheck):
2336 try:
2337 filefp = open(infile, mode, encoding="UTF-8");
2338 except (ValueError, TypeError) as e:
2339 filefp = open(infile, mode);
2340 except FileNotFoundError:
2341 return False;
2342 return filefp;
2343
2344 def UncompressString(infile):
2345 compresscheck = CheckCompressionTypeFromString(infile, formatspecs, False);
2346 if(compresscheck=="gzip"):
2347 try:
2348 import gzip;
2349 except ImportError:
2350 return False;
2351 fileuz = gzip.decompress(infile);
2352 if(compresscheck=="bzip2"):
2353 try:
2354 import bz2;
2355 except ImportError:
2356 return False;
2357 fileuz = bz2.decompress(infile);
2358 if(compresscheck=="zstd"):
2359 try:
2360 import zstandard;
2361 except ImportError:
2362 return False;
2363 fileuz = zstandard.decompress(infile);
2364 if(compresscheck=="lz4"):
2365 try:
2366 import lz4.frame;
2367 except ImportError:
2368 return False;
2369 fileuz = lz4.frame.decompress(infile);
2370 if(compresscheck=="lzo"):
2371 try:
2372 import lzo;
2373 except ImportError:
2374 return False;
2375 fileuz = lzo.decompress(infile);
2376 if(compresscheck=="lzma"):
2377 try:
2378 import lzma;
2379 except ImportError:
2380 try:
2381 from backports import lzma
2382 except ImportError:
2383 return False;
2384 fileuz = lzma.decompress(infile);
2385 if(not compresscheck):
2386 fileuz = infile;
2387 if(hasattr(fileuz, 'decode')):
2388 fileuz = fileuz.decode("UTF-8");
2389 return fileuz;
2390
2391 def UncompressStringAlt(infile):
2392 filefp = StringIO();
2393 outstring = UncompressString(infile);
2394 filefp.write(outstring);
2395 filefp.seek(0, 0);
2396 return filefp;
2397
2398 def CheckCompressionSubType(infile, formatspecs=__file_format_dict__, closefp=True):
2399 formatspecs = FormatSpecsListToDict(formatspecs);
2400 compresscheck = CheckCompressionType(infile, formatspecs, False);
2401 if(not compresscheck):
2402 fextname = os.path.splitext(infile)[1];
2403 if(fextname==".gz"):
2404 compresscheck = "gzip";
2405 elif(fextname==".bz2"):
2406 compresscheck = "bzip2";
2407 elif(fextname==".zst"):
2408 compresscheck = "zstd";
2409 elif(fextname==".lz4"):
2410 compresscheck = "lz4";
2411 elif(fextname==".lzo" or fextname==".lzop"):
2412 compresscheck = "lzo";
2413 elif(fextname==".lzma" or fextname==".xz"):
2414 compresscheck = "lzma";
2415 else:
2416 return False;
2417 if(compresscheck=="gzip" or compresscheck=="bzip2" or compresscheck=="lzma" or compresscheck=="zstd" or compresscheck=="lz4"):
2418 if(TarFileCheck(infile)):
2419 filetype = "tarfile";
2420 if(not compresscheck):
2421 if(TarFileCheck(infile)):
2422 return "tarfile";
2423 elif(zipfile.is_zipfile(infile)):
2424 return "zipfile";
2425 elif(rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile)):
2426 return "rarile";
2427 else:
2428 return False;
2429 return False;
2430 if(compresscheck=="catfile"):
2431 return "catfile";
2432 if(compresscheck==formatspecs['format_lower']):
2433 return formatspecs['format_lower'];
2434 if(compresscheck=="tarfile"):
2435 return "tarfile";
2436 if(compresscheck=="zipfile"):
2437 return "zipfile";
2438 if(rarfile_support and compresscheck=="rarfile"):
2439 return "rarfile";
2440 if(py7zr_support and compresscheck=="7zipfile"):
2441 return "7zipfile";
2442 if(hasattr(infile, "read") or hasattr(infile, "write")):
2443 catfp = UncompressArchiveFile(infile, formatspecs['format_lower']);
2444 else:
2445 try:
2446 if(compresscheck=="gzip"):
2447 try:
2448 import gzip;
2449 except ImportError:
2450 return False;
2451 catfp = gzip.GzipFile(infile, "rb");
2452 if(compresscheck=="bzip2"):
2453 try:
2454 import bz2;
2455 except ImportError:
2456 return False;
2457 catfp = bz2.BZ2File(infile, "rb");
2458 if(compresscheck=="lz4"):
2459 try:
2460 import lz4.frame;
2461 except ImportError:
2462 return False;
2463 catfp = lz4.frame.open(infile, "rb");
2464 if(compresscheck=="zstd"):
2465 try:
2466 import zstandard;
2467 except ImportError:
2468 return False;
2469 catfp = zstandard.open(infile, "rb");
2470 if(compresscheck=="lzma" or compresscheck=="xz"):
2471 try:
2472 import lzma;
2473 except ImportError:
2474 try:
2475 from backports import lzma
2476 except ImportError:
2477 return False;
2478 catfp = lzma.open(infile, "rb");
2479 except FileNotFoundError:
2480 return False;
2481 filetype = False;
2482 prefp = catfp.read(5);
2483 if(prefp==binascii.unhexlify("7573746172")):
2484 filetype = "tarfile";
2485 catfp.seek(0, 0);
2486 prefp = catfp.read(7);
2487 if(prefp==binascii.unhexlify("43617446696c65")):
2488 filetype = "catfile";
2489 catfp.seek(0, 0);
2490 prefp = catfp.read(formatspecs['format_len']);
2491 if(prefp==binascii.unhexlify(formatspecs['format_hex'])):
2492 filetype = formatspecs['format_lower'];
2493 catfp.seek(0, 0);
2494 prefp = catfp.read(10);
2495 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
2496 filetype = "tarfile";
2497 catfp.seek(0, 0);
2498 if(closefp):
2499 catfp.close();
2500 return filetype;
2501
2502 def GZipCompress(data, compresslevel=9):
2503 try:
2504 import gzip;
2505 except ImportError:
2506 return False;
2507 tmpfp = tempfile.NamedTemporaryFile("wb", delete=False);
2508 tmpfp.close();
2509 tmpfp = gzip.GzipFile(tmpfp.name, mode="wb", compresslevel=compresslevel);
2510 tmpfp.write(data);
2511 tmpfp.close();
2512 try:
2513 catfp = open(tmpfp.name, "rb");
2514 except FileNotFoundError:
2515 return False;
2516 catdata = catfp.read();
2517 catfp.close();
2518 return catdata;
2519
2520 def CompressArchiveFile(fp, compression="auto", compressionlevel=None, formatspecs=__file_format_dict__):
2521 formatspecs = FormatSpecsListToDict(formatspecs);
2522 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
2523 return False;
2524 fp.seek(0, 0);
2525 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
2526 compression = "auto";
2527 if(compression not in compressionlist and compression is None):
2528 compression = "auto";
2529 if(compression=="gzip"):
2530 try:
2531 import gzip;
2532 except ImportError:
2533 return False;
2534 catfp = BytesIO();
2535 if(compressionlevel is None):
2536 compressionlevel = 9;
2537 else:
2538 compressionlevel = int(compressionlevel);
2539 catfp.write(gzip.compress(fp.read(), compresslevel=compressionlevel));
2540 if(compression=="bzip2"):
2541 try:
2542 import bz2;
2543 except ImportError:
2544 return False;
2545 catfp = BytesIO();
2546 if(compressionlevel is None):
2547 compressionlevel = 9;
2548 else:
2549 compressionlevel = int(compressionlevel);
2550 catfp.write(bz2.compress(fp.read(), compresslevel=compressionlevel));
2551 if(compression=="lz4"):
2552 try:
2553 import lz4.frame;
2554 except ImportError:
2555 return False;
2556 catfp = BytesIO();
2557 if(compressionlevel is None):
2558 compressionlevel = 9;
2559 else:
2560 compressionlevel = int(compressionlevel);
2561 catfp.write(lz4.frame.compress(fp.read(), compression_level=compressionlevel));
2562 if(compression=="lzo" or compression=="lzop"):
2563 try:
2564 import lzo;
2565 except ImportError:
2566 return False;
2567 catfp = BytesIO();
2568 if(compressionlevel is None):
2569 compressionlevel = 9;
2570 else:
2571 compressionlevel = int(compressionlevel);
2572 catfp.write(lzo.compress(fp.read(), compresslevel=compressionlevel));
2573 if(compression=="zstd"):
2574 try:
2575 import zstandard;
2576 except ImportError:
2577 return False;
2578 catfp = BytesIO();
2579 if(compressionlevel is None):
2580 compressionlevel = 10;
2581 else:
2582 compressionlevel = int(compressionlevel);
2583 catfp.write(zstandard.compress(fp.read(), level=compressionlevel));
2584 if(compression=="lzma"):
2585 try:
2586 import lzma;
2587 except ImportError:
2588 try:
2589 from backports import lzma
2590 except ImportError:
2591 return False;
2592 catfp = BytesIO();
2593 if(compressionlevel is None):
2594 compressionlevel = 9;
2595 else:
2596 compressionlevel = int(compressionlevel);
2597 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]));
2598 if(compression=="xz"):
2599 try:
2600 import lzma;
2601 except ImportError:
2602 try:
2603 from backports import lzma
2604 except ImportError:
2605 return False;
2606 catfp = BytesIO();
2607 if(compressionlevel is None):
2608 compressionlevel = 9;
2609 else:
2610 compressionlevel = int(compressionlevel);
2611 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]));
2612 if(compression=="auto" or compression is None):
2613 catfp = fp;
2614 catfp.seek(0, 0);
2615 return catfp;
2616
2617 create_alias_function("Compress", __file_format_name__, "", CompressArchiveFile);
2618
2619 def CompressOpenFile(outfile, compressionenable=True, compressionlevel=None):
2620 if(outfile is None):
2621 return False;
2622 fbasename = os.path.splitext(outfile)[0];
2623 fextname = os.path.splitext(outfile)[1];
2624 if(compressionlevel is None and fextname!=".zst"):
2625 compressionlevel = 9;
2626 elif(compressionlevel is None and fextname==".zst"):
2627 compressionlevel = 10;
2628 else:
2629 compressionlevel = int(compressionlevel);
2630 if(sys.version_info[0]==2):
2631 mode = "w";
2632 else:
2633 mode = "wb";
2634 try:
2635 if(fextname not in outextlistwd or not compressionenable):
2636 try:
2637 outfp = open(outfile, "wb", encoding="UTF-8");
2638 except (ValueError, TypeError) as e:
2639 outfp = open(outfile, "wb");
2640 elif(fextname==".gz"):
2641 try:
2642 import gzip;
2643 except ImportError:
2644 return False;
2645 try:
2646 outfp = gzip.open(outfile, mode, compressionlevel, encoding="UTF-8");
2647 except (ValueError, TypeError) as e:
2648 outfp = gzip.open(outfile, mode, compressionlevel);
2649 elif(fextname==".bz2"):
2650 try:
2651 import bz2;
2652 except ImportError:
2653 return False;
2654 try:
2655 outfp = bz2.open(outfile, mode, compressionlevel, encoding="UTF-8");
2656 except (ValueError, TypeError) as e:
2657 outfp = bz2.open(outfile, mode, compressionlevel);
2658 elif(fextname==".zst"):
2659 try:
2660 import zstandard;
2661 except ImportError:
2662 return False;
2663 try:
2664 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel), encoding="UTF-8");
2665 except (ValueError, TypeError) as e:
2666 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel));
2667 elif(fextname==".xz"):
2668 try:
2669 import lzma;
2670 except ImportError:
2671 try:
2672 from backports import lzma
2673 except ImportError:
2674 return False;
2675 try:
2676 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}], encoding="UTF-8");
2677 except (ValueError, TypeError) as e:
2678 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]);
2679 elif(fextname==".lz4"):
2680 try:
2681 import lz4.frame;
2682 except ImportError:
2683 return False;
2684 try:
2685 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel, encoding="UTF-8");
2686 except (ValueError, TypeError) as e:
2687 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel);
2688 elif(fextname==".lzo"):
2689 try:
2690 import lzo;
2691 except ImportError:
2692 return False;
2693 try:
2694 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel, encoding="UTF-8");
2695 except (ValueError, TypeError) as e:
2696 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel);
2697 elif(fextname==".lzma"):
2698 try:
2699 import lzma;
2700 except ImportError:
2701 try:
2702 from backports import lzma
2703 except ImportError:
2704 return False;
2705 try:
2706 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}], encoding="UTF-8");
2707 except (ValueError, TypeError) as e:
2708 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]);
2709 except FileNotFoundError:
2710 return False;
2711 return outfp;
2712
2713 def GetDevMajorMinor(fdev):
2714 retdev = [];
2715 if(hasattr(os, "minor")):
2716 retdev.append(os.minor(fdev));
2717 else:
2718 retdev.append(0);
2719 if(hasattr(os, "major")):
2720 retdev.append(os.major(fdev));
2721 else:
2722 retdev.append(0);
2723 return retdev;
2724
2725 def CheckSumSupport(checkfor, guaranteed=True):
2726 if(guaranteed):
2727 try:
2728 hash_list = sorted(list(hashlib.algorithms_guaranteed));
2729 except AttributeError:
2730 hash_list = sorted(list(hashlib.algorithms));
2731 else:
2732 try:
2733 hash_list = sorted(list(hashlib.algorithms_available));
2734 except AttributeError:
2735 hash_list = sorted(list(hashlib.algorithms));
2736 checklistout = sorted(hash_list + ['adler32', 'crc16', 'crc16_ansi', 'crc16_ibm', 'crc16_ccitt', 'crc32', 'crc64', 'crc64_ecma', 'crc64_iso', 'none']);
2737 if(checkfor in checklistout):
2738 return True;
2739 else:
2740 return False;
2741
2742 def CheckSumSupportAlt(checkfor, guaranteed=True):
2743 if(guaranteed):
2744 try:
2745 hash_list = sorted(list(hashlib.algorithms_guaranteed));
2746 except AttributeError:
2747 hash_list = sorted(list(hashlib.algorithms));
2748 else:
2749 try:
2750 hash_list = sorted(list(hashlib.algorithms_available));
2751 except AttributeError:
2752 hash_list = sorted(list(hashlib.algorithms));
2753 checklistout = hash_list;
2754 if(checkfor in checklistout):
2755 return True;
2756 else:
2757 return False;
2758
2759 def PackArchiveFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2760 formatspecs = FormatSpecsListToDict(formatspecs);
2761 advancedlist = formatspecs['use_advanced_list'];
2762 altinode = formatspecs['use_alt_inode'];
2763 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2764 outfile = RemoveWindowsPath(outfile);
2765 checksumtype = checksumtype.lower();
2766 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
2767 checksumtype="crc32";
2768 if(checksumtype=="none"):
2769 checksumtype = "";
2770 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
2771 compression = "auto";
2772 if(compression not in compressionlist and compression is None):
2773 compression = "auto";
2774 if(verbose):
2775 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2776 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2777 if(os.path.exists(outfile)):
2778 try:
2779 os.unlink(outfile);
2780 except OSError as e:
2781 pass;
2782 if(outfile=="-"):
2783 verbose = False;
2784 catfp = BytesIO();
2785 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2786 catfp = outfile;
2787 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2788 catfp = BytesIO();
2789 else:
2790 fbasename = os.path.splitext(outfile)[0];
2791 fextname = os.path.splitext(outfile)[1];
2792 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
2793 catver = formatspecs['format_ver'];
2794 fileheaderver = str(int(catver.replace(".", "")));
2795 infilelist = [];
2796 if(infiles=="-"):
2797 for line in sys.stdin:
2798 infilelist.append(line.strip());
2799 infilelist = list(filter(None, infilelist));
2800 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
2801 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
2802 return False;
2803 with UncompressFile(infiles, formatspecs, "r") as finfile:
2804 for line in finfile:
2805 infilelist.append(line.strip());
2806 infilelist = list(filter(None, infilelist));
2807 else:
2808 if(isinstance(infiles, (list, tuple, ))):
2809 infilelist = list(filter(None, infiles));
2810 elif(isinstance(infiles, (str, ))):
2811 infilelist = list(filter(None, [infiles]));
2812 if(advancedlist):
2813 GetDirList = ListDirAdvanced(infilelist, followlink, False);
2814 else:
2815 GetDirList = ListDir(infilelist, followlink, False);
2816 if(not GetDirList):
2817 return False;
2818 curinode = 0;
2819 curfid = 0;
2820 inodelist = [];
2821 inodetofile = {};
2822 filetoinode = {};
2823 inodetocatinode = {};
2824 numfiles = int(len(GetDirList));
2825 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
2826 for curfname in GetDirList:
2827 if(re.findall("^[.|/]", curfname)):
2828 fname = curfname;
2829 else:
2830 fname = "./"+curfname;
2831 if(verbose):
2832 VerbosePrintOut(fname);
2833 if(not followlink or followlink is None):
2834 fstatinfo = os.lstat(fname);
2835 else:
2836 fstatinfo = os.stat(fname);
2837 fpremode = fstatinfo.st_mode;
2838 finode = fstatinfo.st_ino;
2839 flinkcount = fstatinfo.st_nlink;
2840 ftype = 0;
2841 if(stat.S_ISREG(fpremode)):
2842 ftype = 0;
2843 elif(stat.S_ISLNK(fpremode)):
2844 ftype = 2;
2845 elif(stat.S_ISCHR(fpremode)):
2846 ftype = 3;
2847 elif(stat.S_ISBLK(fpremode)):
2848 ftype = 4;
2849 elif(stat.S_ISDIR(fpremode)):
2850 ftype = 5;
2851 elif(stat.S_ISFIFO(fpremode)):
2852 ftype = 6;
2853 elif(stat.S_ISSOCK(fpremode)):
2854 ftype = 8;
2855 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
2856 ftype = 9;
2857 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
2858 ftype = 10;
2859 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
2860 ftype = 111;
2861 else:
2862 ftype = 0;
2863 flinkname = "";
2864 fcurfid = format(int(curfid), 'x').lower();
2865 if(not followlink and finode!=0):
2866 if(ftype!=1):
2867 if(finode in inodelist):
2868 ftype = 1;
2869 flinkname = inodetofile[finode];
2870 if(altinode):
2871 fcurinode = format(int(finode), 'x').lower();
2872 else:
2873 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
2874 if(finode not in inodelist):
2875 inodelist.append(finode);
2876 inodetofile.update({finode: fname});
2877 inodetocatinode.update({finode: curinode});
2878 if(altinode):
2879 fcurinode = format(int(finode), 'x').lower();
2880 else:
2881 fcurinode = format(int(curinode), 'x').lower();
2882 curinode = curinode + 1;
2883 else:
2884 fcurinode = format(int(curinode), 'x').lower();
2885 curinode = curinode + 1;
2886 curfid = curfid + 1;
2887 if(ftype==2):
2888 flinkname = os.readlink(fname);
2889 fdev = fstatinfo.st_dev;
2890 getfdev = GetDevMajorMinor(fdev);
2891 fdev_minor = getfdev[0];
2892 fdev_major = getfdev[1];
2893 frdev = fstatinfo.st_dev;
2894 if(hasattr(fstatinfo, "st_rdev")):
2895 frdev = fstatinfo.st_rdev;
2896 else:
2897 frdev = fstatinfo.st_dev;
2898 getfrdev = GetDevMajorMinor(frdev);
2899 frdev_minor = getfrdev[0];
2900 frdev_major = getfrdev[1];
2901 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
2902 fsize = format(int("0"), 'x').lower();
2903 elif(ftype==0 or ftype==7):
2904 fsize = format(int(fstatinfo.st_size), 'x').lower();
2905 else:
2906 fsize = format(int(fstatinfo.st_size)).lower();
2907 fatime = format(int(fstatinfo.st_atime), 'x').lower();
2908 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
2909 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
2910 if(hasattr(fstatinfo, "st_birthtime")):
2911 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
2912 else:
2913 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
2914 fmode = format(int(fstatinfo.st_mode), 'x').lower();
2915 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
2916 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
2917 fuid = format(int(fstatinfo.st_uid), 'x').lower();
2918 fgid = format(int(fstatinfo.st_gid), 'x').lower();
2919 funame = "";
2920 try:
2921 import pwd;
2922 try:
2923 userinfo = pwd.getpwuid(fstatinfo.st_uid);
2924 funame = userinfo.pw_name;
2925 except KeyError:
2926 funame = "";
2927 except ImportError:
2928 funame = "";
2929 fgname = "";
2930 try:
2931 import grp;
2932 try:
2933 groupinfo = grp.getgrgid(fstatinfo.st_gid);
2934 fgname = groupinfo.gr_name;
2935 except KeyError:
2936 fgname = "";
2937 except ImportError:
2938 fgname = "";
2939 fdev_minor = format(int(fdev_minor), 'x').lower();
2940 fdev_major = format(int(fdev_major), 'x').lower();
2941 frdev_minor = format(int(frdev_minor), 'x').lower();
2942 frdev_major = format(int(frdev_major), 'x').lower();
2943 finode = format(int(finode), 'x').lower();
2944 flinkcount = format(int(flinkcount), 'x').lower();
2945 if(hasattr(fstatinfo, "st_file_attributes")):
2946 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
2947 else:
2948 fwinattributes = format(int(0), 'x').lower();
2949 fcompression = "";
2950 fcsize = format(int(0), 'x').lower();
2951 fcontents = BytesIO();
2952 if(ftype==0 or ftype==7):
2953 with open(fname, "rb") as fpc:
2954 shutil.copyfileobj(fpc, fcontents);
2955 if(not compresswholefile):
2956 fcontents.seek(0, 2);
2957 ucfsize = fcontents.tell();
2958 fcontents.seek(0, 0);
2959 if(compression=="auto"):
2960 ilsize = len(compressionlistalt);
2961 ilmin = 0;
2962 ilcsize = [];
2963 while(ilmin < ilsize):
2964 cfcontents = BytesIO();
2965 shutil.copyfileobj(fcontents, cfcontents);
2966 fcontents.seek(0, 0);
2967 cfcontents.seek(0, 0);
2968 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
2969 if(cfcontents):
2970 cfcontents.seek(0, 2);
2971 ilcsize.append(cfcontents.tell());
2972 cfcontents.close();
2973 else:
2974 try:
2975 ilcsize.append(sys.maxint);
2976 except AttributeError:
2977 ilcsize.append(sys.maxsize);
2978 ilmin = ilmin + 1;
2979 ilcmin = ilcsize.index(min(ilcsize));
2980 compression = compressionlistalt[ilcmin];
2981 fcontents.seek(0, 0);
2982 cfcontents = BytesIO();
2983 shutil.copyfileobj(fcontents, cfcontents);
2984 cfcontents.seek(0, 0);
2985 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
2986 cfcontents.seek(0, 2);
2987 cfsize = cfcontents.tell();
2988 if(ucfsize > cfsize):
2989 fcsize = format(int(cfsize), 'x').lower();
2990 fcompression = compression;
2991 fcontents.close();
2992 fcontents = cfcontents;
2993 if(fcompression=="none"):
2994 fcompression = "";
2995 if(followlink and (ftype==1 or ftype==2)):
2996 flstatinfo = os.stat(flinkname);
2997 with open(flinkname, "rb") as fpc:
2998 shutil.copyfileobj(fpc, fcontents);
2999 if(not compresswholefile):
3000 fcontents.seek(0, 2);
3001 ucfsize = fcontents.tell();
3002 fcontents.seek(0, 0);
3003 if(compression=="auto"):
3004 ilsize = len(compressionlistalt);
3005 ilmin = 0;
3006 ilcsize = [];
3007 while(ilmin < ilsize):
3008 cfcontents = BytesIO();
3009 shutil.copyfileobj(fcontents, cfcontents);
3010 fcontents.seek(0, 0);
3011 cfcontents.seek(0, 0);
3012 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3013 if(cfcontents):
3014 cfcontents.seek(0, 2);
3015 ilcsize.append(cfcontents.tell());
3016 cfcontents.close();
3017 else:
3018 try:
3019 ilcsize.append(sys.maxint);
3020 except AttributeError:
3021 ilcsize.append(sys.maxsize);
3022 ilmin = ilmin + 1;
3023 ilcmin = ilcsize.index(min(ilcsize));
3024 compression = compressionlistalt[ilcmin];
3025 fcontents.seek(0, 0);
3026 cfcontents = BytesIO();
3027 shutil.copyfileobj(fcontents, cfcontents);
3028 cfcontents.seek(0, 0);
3029 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3030 cfcontents.seek(0, 2);
3031 cfsize = cfcontents.tell();
3032 if(ucfsize > cfsize):
3033 fcsize = format(int(cfsize), 'x').lower();
3034 fcompression = compression;
3035 fcontents.close();
3036 fcontents = cfcontents;
3037 fcontents.seek(0, 0);
3038 ftypehex = format(ftype, 'x').lower();
3039 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3040 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3041 fcontents.close();
3042 if(numfiles>0):
3043 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3044 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3045 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3046 try:
3047 catfp.flush();
3048 os.fsync(catfp.fileno());
3049 except io.UnsupportedOperation:
3050 pass;
3051 except AttributeError:
3052 pass;
3053 except OSError as e:
3054 pass;
3055 if(outfile=="-"):
3056 catfp.seek(0, 0);
3057 if(hasattr(sys.stdout, "buffer")):
3058 shutil.copyfileobj(catfp, sys.stdout.buffer);
3059 else:
3060 shutil.copyfileobj(catfp, sys.stdout);
3061 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3062 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3063 catfp.seek(0, 0);
3064 upload_file_to_internet_file(catfp, outfile);
3065 if(returnfp):
3066 catfp.seek(0, 0);
3067 return catfp;
3068 else:
3069 catfp.close();
3070 return True;
3071
3072 create_alias_function("Pack", __file_format_name__, "", PackArchiveFile);
3073
3074 if(hasattr(shutil, "register_archive_format")):
3075 def PackArchiveFileFunc(archive_name, source_dir, **kwargs):
3076 return PackArchiveFile(source_dir, archive_name, False, "auto", True, None, False, "crc32", [], __file_format_dict__['format_delimiter'], False, False);
3077 create_alias_function("Pack", __file_format_name__, "Func", PackArchiveFileFunc);
3078
3079 def PackArchiveFileFromDirList(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3080 formatspecs = FormatSpecsListToDict(formatspecs);
3081 return PackArchiveFile(infiles, outfile, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, returnfp);
3082
3083 create_alias_function("Pack", __file_format_name__, "FromDirList", PackArchiveFileFromDirList);
3084
3085 def PackArchiveFileFromTarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3086 formatspecs = FormatSpecsListToDict(formatspecs);
3087 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3088 outfile = RemoveWindowsPath(outfile);
3089 checksumtype = checksumtype.lower();
3090 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3091 checksumtype="crc32";
3092 if(checksumtype=="none"):
3093 checksumtype = "";
3094 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3095 compression = "auto";
3096 if(compression not in compressionlist and compression is None):
3097 compression = "auto";
3098 if(verbose):
3099 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3100 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3101 if(os.path.exists(outfile)):
3102 try:
3103 os.unlink(outfile);
3104 except OSError as e:
3105 pass;
3106 if(outfile=="-"):
3107 verbose = False;
3108 catfp = BytesIO();
3109 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3110 catfp = outfile;
3111 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3112 catfp = BytesIO();
3113 else:
3114 fbasename = os.path.splitext(outfile)[0];
3115 fextname = os.path.splitext(outfile)[1];
3116 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3117 catver = formatspecs['format_ver'];
3118 fileheaderver = str(int(catver.replace(".", "")));
3119 curinode = 0;
3120 curfid = 0;
3121 inodelist = [];
3122 inodetofile = {};
3123 filetoinode = {};
3124 inodetocatinode = {};
3125 if(infile=="-"):
3126 infile = BytesIO();
3127 if(hasattr(sys.stdin, "buffer")):
3128 shutil.copyfileobj(sys.stdin.buffer, infile);
3129 else:
3130 shutil.copyfileobj(sys.stdin, infile);
3131 infile.seek(0, 0);
3132 if(not infile):
3133 return False;
3134 infile.seek(0, 0);
3135 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3136 infile = download_file_from_internet_file(infile);
3137 infile.seek(0, 0);
3138 if(not infile):
3139 return False;
3140 infile.seek(0, 0);
3141 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
3142 return False;
3143 elif(os.path.exists(infile) and os.path.isfile(infile)):
3144 try:
3145 if(not tarfile.TarFileCheck(infile)):
3146 return False;
3147 except AttributeError:
3148 if(not TarFileCheck(infile)):
3149 return False;
3150 else:
3151 return False;
3152 try:
3153 if(hasattr(infile, "read") or hasattr(infile, "write")):
3154 tarfp = tarfile.open(fileobj=infile, mode="r");
3155 else:
3156 tarfp = tarfile.open(infile, "r");
3157 except FileNotFoundError:
3158 return False;
3159 numfiles = int(len(tarfp.getmembers()));
3160 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3161 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
3162 if(re.findall("^[.|/]", member.name)):
3163 fname = member.name;
3164 else:
3165 fname = "./"+member.name;
3166 if(verbose):
3167 VerbosePrintOut(fname);
3168 fpremode = member.mode;
3169 ffullmode = member.mode;
3170 flinkcount = 0;
3171 ftype = 0;
3172 if(member.isreg()):
3173 ffullmode = member.mode + stat.S_IFREG;
3174 ftype = 0;
3175 elif(member.isdev()):
3176 ffullmode = member.mode;
3177 ftype = 7;
3178 elif(member.islnk()):
3179 ffullmode = member.mode + stat.S_IFREG;
3180 ftype = 1;
3181 elif(member.issym()):
3182 ffullmode = member.mode + stat.S_IFLNK;
3183 ftype = 2;
3184 elif(member.ischr()):
3185 ffullmode = member.mode + stat.S_IFCHR;
3186 ftype = 3;
3187 elif(member.isblk()):
3188 ffullmode = member.mode + stat.S_IFBLK;
3189 ftype = 4;
3190 elif(member.isdir()):
3191 ffullmode = member.mode + stat.S_IFDIR;
3192 ftype = 5;
3193 elif(member.isfifo()):
3194 ffullmode = member.mode + stat.S_IFIFO;
3195 ftype = 6;
3196 elif(member.issparse()):
3197 ffullmode = member.mode;
3198 ftype = 12;
3199 else:
3200 ffullmode = member.mode;
3201 ftype = 0;
3202 flinkname = "";
3203 fcurfid = format(int(curfid), 'x').lower();
3204 fcurinode = format(int(curfid), 'x').lower();
3205 curfid = curfid + 1;
3206 if(ftype==2):
3207 flinkname = member.linkname;
3208 fdev_minor = format(int(member.devminor), 'x').lower();
3209 fdev_major = format(int(member.devmajor), 'x').lower();
3210 frdev_minor = format(int(member.devminor), 'x').lower();
3211 frdev_major = format(int(member.devmajor), 'x').lower();
3212 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
3213 fsize = format(int("0"), 'x').lower();
3214 elif(ftype==0 or ftype==7):
3215 fsize = format(int(member.size), 'x').lower();
3216 else:
3217 fsize = format(int(member.size), 'x').lower();
3218 fatime = format(int(member.mtime), 'x').lower();
3219 fmtime = format(int(member.mtime), 'x').lower();
3220 fctime = format(int(member.mtime), 'x').lower();
3221 fbtime = format(int(member.mtime), 'x').lower();
3222 fmode = format(int(ffullmode), 'x').lower();
3223 fchmode = format(int(stat.S_IMODE(ffullmode)), 'x').lower();
3224 ftypemod = format(int(stat.S_IFMT(ffullmode)), 'x').lower();
3225 fuid = format(int(member.uid), 'x').lower();
3226 fgid = format(int(member.gid), 'x').lower();
3227 funame = member.uname;
3228 fgname = member.gname;
3229 flinkcount = format(int(flinkcount), 'x').lower();
3230 fwinattributes = format(int(0), 'x').lower();
3231 fcompression = "";
3232 fcsize = format(int(0), 'x').lower();
3233 fcontents = BytesIO();
3234 if(ftype==0 or ftype==7):
3235 with tarfp.extractfile(member) as fpc:
3236 shutil.copyfileobj(fpc, fcontents);
3237 if(not compresswholefile):
3238 fcontents.seek(0, 2);
3239 ucfsize = fcontents.tell();
3240 fcontents.seek(0, 0);
3241 if(compression=="auto"):
3242 ilsize = len(compressionlistalt);
3243 ilmin = 0;
3244 ilcsize = [];
3245 while(ilmin < ilsize):
3246 cfcontents = BytesIO();
3247 shutil.copyfileobj(fcontents, cfcontents);
3248 fcontents.seek(0, 0);
3249 cfcontents.seek(0, 0);
3250 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3251 if(cfcontents):
3252 cfcontents.seek(0, 2);
3253 ilcsize.append(cfcontents.tell());
3254 cfcontents.close();
3255 else:
3256 try:
3257 ilcsize.append(sys.maxint);
3258 except AttributeError:
3259 ilcsize.append(sys.maxsize);
3260 ilmin = ilmin + 1;
3261 ilcmin = ilcsize.index(min(ilcsize));
3262 compression = compressionlistalt[ilcmin];
3263 fcontents.seek(0, 0);
3264 cfcontents = BytesIO();
3265 shutil.copyfileobj(fcontents, cfcontents);
3266 cfcontents.seek(0, 0);
3267 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3268 cfcontents.seek(0, 2);
3269 cfsize = cfcontents.tell();
3270 if(ucfsize > cfsize):
3271 fcsize = format(int(cfsize), 'x').lower();
3272 fcompression = compression;
3273 fcontents.close();
3274 fcontents = cfcontents;
3275 if(fcompression=="none"):
3276 fcompression = "";
3277 fcontents.seek(0, 0);
3278 ftypehex = format(ftype, 'x').lower();
3279 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3280 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3281 fcontents.close();
3282 if(numfiles>0):
3283 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3284 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3285 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3286 try:
3287 catfp.flush();
3288 os.fsync(catfp.fileno());
3289 except io.UnsupportedOperation:
3290 pass;
3291 except AttributeError:
3292 pass;
3293 except OSError as e:
3294 pass;
3295 if(outfile=="-"):
3296 catfp.seek(0, 0);
3297 if(hasattr(sys.stdout, "buffer")):
3298 shutil.copyfileobj(catfp, sys.stdout.buffer);
3299 else:
3300 shutil.copyfileobj(catfp, sys.stdout);
3301 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3302 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3303 catfp.seek(0, 0);
3304 upload_file_to_internet_file(catfp, outfile);
3305 if(returnfp):
3306 catfp.seek(0, 0);
3307 return catfp;
3308 else:
3309 catfp.close();
3310 return True;
3311
3312 create_alias_function("Pack", __file_format_name__, "FromTarFile", PackArchiveFileFromTarFile);
3313
3314 def PackArchiveFileFromZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3315 formatspecs = FormatSpecsListToDict(formatspecs);
3316 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3317 outfile = RemoveWindowsPath(outfile);
3318 checksumtype = checksumtype.lower();
3319 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3320 checksumtype="crc32";
3321 if(checksumtype=="none"):
3322 checksumtype = "";
3323 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3324 compression = "auto";
3325 if(compression not in compressionlist and compression is None):
3326 compression = "auto";
3327 if(verbose):
3328 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3329 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3330 if(os.path.exists(outfile)):
3331 try:
3332 os.unlink(outfile);
3333 except OSError as e:
3334 pass;
3335 if(outfile=="-"):
3336 verbose = False;
3337 catfp = BytesIO();
3338 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3339 catfp = outfile;
3340 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3341 catfp = BytesIO();
3342 else:
3343 fbasename = os.path.splitext(outfile)[0];
3344 fextname = os.path.splitext(outfile)[1];
3345 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3346 catver = formatspecs['format_ver'];
3347 fileheaderver = str(int(catver.replace(".", "")));
3348 curinode = 0;
3349 curfid = 0;
3350 inodelist = [];
3351 inodetofile = {};
3352 filetoinode = {};
3353 inodetocatinode = {};
3354 if(infile=="-"):
3355 infile = BytesIO();
3356 if(hasattr(sys.stdin, "buffer")):
3357 shutil.copyfileobj(sys.stdin.buffer, infile);
3358 else:
3359 shutil.copyfileobj(sys.stdin, infile);
3360 infile.seek(0, 0);
3361 if(not infile):
3362 return False;
3363 infile.seek(0, 0);
3364 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3365 infile = download_file_from_internet_file(infile);
3366 infile.seek(0, 0);
3367 if(not infile):
3368 return False;
3369 infile.seek(0, 0);
3370 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
3371 return False;
3372 else:
3373 return False;
3374 if(not zipfile.is_zipfile(infile)):
3375 return False;
3376 try:
3377 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
3378 except FileNotFoundError:
3379 return False;
3380 ziptest = zipfp.testzip();
3381 if(ziptest):
3382 VerbosePrintOut("Bad file found!");
3383 numfiles = int(len(zipfp.infolist()));
3384 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3385 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
3386 if(re.findall("^[.|/]", member.filename)):
3387 fname = member.filename;
3388 else:
3389 fname = "./"+member.filename;
3390 zipinfo = zipfp.getinfo(member.filename);
3391 if(verbose):
3392 VerbosePrintOut(fname);
3393 if(not member.is_dir()):
3394 fpremode = int(stat.S_IFREG + 438);
3395 elif(member.is_dir()):
3396 fpremode = int(stat.S_IFDIR + 511);
3397 flinkcount = 0;
3398 ftype = 0;
3399 if(not member.is_dir()):
3400 ftype = 0;
3401 elif(member.is_dir()):
3402 ftype = 5;
3403 flinkname = "";
3404 fcurfid = format(int(curfid), 'x').lower();
3405 fcurinode = format(int(curfid), 'x').lower();
3406 curfid = curfid + 1;
3407 fdev_minor = format(int(0), 'x').lower();
3408 fdev_major = format(int(0), 'x').lower();
3409 frdev_minor = format(int(0), 'x').lower();
3410 frdev_major = format(int(0), 'x').lower();
3411 if(ftype==5):
3412 fsize = format(int("0"), 'x').lower();
3413 elif(ftype==0):
3414 fsize = format(int(member.file_size), 'x').lower();
3415 else:
3416 fsize = format(int(member.file_size), 'x').lower();
3417 fatime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3418 fmtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3419 fctime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3420 fbtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3421 if(zipinfo.create_system==0 or zipinfo.create_system==10):
3422 fwinattributes = format(int(zipinfo.external_attr), 'x').lower();
3423 if(not member.is_dir()):
3424 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3425 fchmode = stat.S_IMODE(fmode);
3426 ftypemod = stat.S_IFMT(fmode);
3427 elif(member.is_dir()):
3428 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3429 fchmode = stat.S_IMODE(fmode);
3430 ftypemod = stat.S_IFMT(fmode);
3431 elif(zipinfo.create_system==3):
3432 fwinattributes = format(int(0), 'x').lower();
3433 fmode = format(int(zipinfo.external_attr), 'x').lower();
3434 fchmode = stat.S_IMODE(fmode);
3435 ftypemod = stat.S_IFMT(fmode);
3436 else:
3437 fwinattributes = format(int(0), 'x').lower();
3438 if(not member.is_dir()):
3439 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3440 fchmode = stat.S_IMODE(fmode);
3441 ftypemod = stat.S_IFMT(fmode);
3442 elif(member.is_dir()):
3443 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3444 fchmode = stat.S_IMODE(fmode);
3445 ftypemod = stat.S_IFMT(fmode);
3446 fcompression = "";
3447 fcsize = format(int(0), 'x').lower();
3448 try:
3449 fuid = format(int(os.getuid()), 'x').lower();
3450 except AttributeError:
3451 fuid = format(int(0), 'x').lower();
3452 except KeyError:
3453 fuid = format(int(0), 'x').lower();
3454 try:
3455 fgid = format(int(os.getgid()), 'x').lower();
3456 except AttributeError:
3457 fgid = format(int(0), 'x').lower();
3458 except KeyError:
3459 fgid = format(int(0), 'x').lower();
3460 try:
3461 import pwd;
3462 try:
3463 userinfo = pwd.getpwuid(os.getuid());
3464 funame = userinfo.pw_name;
3465 except KeyError:
3466 funame = "";
3467 except AttributeError:
3468 funame = "";
3469 except ImportError:
3470 funame = "";
3471 fgname = "";
3472 try:
3473 import grp;
3474 try:
3475 groupinfo = grp.getgrgid(os.getgid());
3476 fgname = groupinfo.gr_name;
3477 except KeyError:
3478 fgname = "";
3479 except AttributeError:
3480 fgname = "";
3481 except ImportError:
3482 fgname = "";
3483 fcontents = BytesIO();
3484 if(ftype==0):
3485 fcontents.write(zipfp.read(member.filename));
3486 if(not compresswholefile):
3487 fcontents.seek(0, 2);
3488 ucfsize = fcontents.tell();
3489 fcontents.seek(0, 0);
3490 if(compression=="auto"):
3491 ilsize = len(compressionlistalt);
3492 ilmin = 0;
3493 ilcsize = [];
3494 while(ilmin < ilsize):
3495 cfcontents = BytesIO();
3496 shutil.copyfileobj(fcontents, cfcontents);
3497 fcontents.seek(0, 0);
3498 cfcontents.seek(0, 0);
3499 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3500 cfcontents.seek(0, 2);
3501 ilcsize.append(cfcontents.tell());
3502 cfcontents.close();
3503 ilmin = ilmin + 1;
3504 ilcmin = ilcsize.index(min(ilcsize));
3505 compression = compressionlistalt[ilcmin];
3506 fcontents.seek(0, 0);
3507 cfcontents = BytesIO();
3508 shutil.copyfileobj(fcontents, cfcontents);
3509 cfcontents.seek(0, 0);
3510 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3511 cfcontents.seek(0, 2);
3512 cfsize = cfcontents.tell();
3513 if(ucfsize > cfsize):
3514 fcsize = format(int(cfsize), 'x').lower();
3515 fcompression = compression;
3516 fcontents.close();
3517 fcontents = cfcontents;
3518 if(fcompression=="none"):
3519 fcompression = "";
3520 fcontents.seek(0, 0);
3521 ftypehex = format(ftype, 'x').lower();
3522 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3523 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3524 fcontents.close();
3525 if(numfiles>0):
3526 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3527 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3528 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3529 try:
3530 catfp.flush();
3531 os.fsync(catfp.fileno());
3532 except io.UnsupportedOperation:
3533 pass;
3534 except AttributeError:
3535 pass;
3536 except OSError as e:
3537 pass;
3538 if(outfile=="-"):
3539 catfp.seek(0, 0);
3540 if(hasattr(sys.stdout, "buffer")):
3541 shutil.copyfileobj(catfp, sys.stdout.buffer);
3542 else:
3543 shutil.copyfileobj(catfp, sys.stdout);
3544 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3545 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3546 catfp.seek(0, 0);
3547 upload_file_to_internet_file(catfp, outfile);
3548 if(returnfp):
3549 catfp.seek(0, 0);
3550 return catfp;
3551 else:
3552 catfp.close();
3553 return True;
3554
3555 create_alias_function("Pack", __file_format_name__, "FromZipFile", PackArchiveFileFromZipFile);
3556
3557 if(not rarfile_support):
3558 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3559 return False
3560
3561 if(rarfile_support):
3562 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3563 formatspecs = FormatSpecsListToDict(formatspecs);
3564 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3565 outfile = RemoveWindowsPath(outfile);
3566 checksumtype = checksumtype.lower();
3567 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3568 checksumtype="crc32";
3569 if(checksumtype=="none"):
3570 checksumtype = "";
3571 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3572 compression = "auto";
3573 if(compression not in compressionlist and compression is None):
3574 compression = "auto";
3575 if(verbose):
3576 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3577 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3578 if(os.path.exists(outfile)):
3579 try:
3580 os.unlink(outfile);
3581 except OSError as e:
3582 pass;
3583 if(outfile=="-"):
3584 verbose = False;
3585 catfp = BytesIO();
3586 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3587 catfp = outfile;
3588 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3589 catfp = BytesIO();
3590 else:
3591 fbasename = os.path.splitext(outfile)[0];
3592 fextname = os.path.splitext(outfile)[1];
3593 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3594 catver = formatspecs['format_ver'];
3595 fileheaderver = str(int(catver.replace(".", "")));
3596 curinode = 0;
3597 curfid = 0;
3598 inodelist = [];
3599 inodetofile = {};
3600 filetoinode = {};
3601 inodetocatinode = {};
3602 if(not os.path.exists(infile) or not os.path.isfile(infile)):
3603 return False;
3604 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
3605 return False;
3606 rarfp = rarfile.RarFile(infile, "r");
3607 rartest = rarfp.testrar();
3608 if(rartest):
3609 VerbosePrintOut("Bad file found!");
3610 numfiles = int(len(rarfp.infolist()));
3611 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3612 try:
3613 catfp.flush();
3614 os.fsync(catfp.fileno());
3615 except io.UnsupportedOperation:
3616 pass;
3617 except AttributeError:
3618 pass;
3619 except OSError as e:
3620 pass;
3621 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
3622 is_unix = False;
3623 is_windows = False;
3624 if(member.host_os==rarfile.RAR_OS_UNIX):
3625 is_windows = False;
3626 try:
3627 member.external_attr
3628 is_unix = True;
3629 except AttributeError:
3630 is_unix = False;
3631 elif(member.host_os==rarfile.RAR_OS_WIN32):
3632 is_unix = False;
3633 try:
3634 member.external_attr
3635 is_windows = True;
3636 except AttributeError:
3637 is_windows = False;
3638 else:
3639 is_unix = False;
3640 is_windows = False;
3641 if(re.findall("^[.|/]", member.filename)):
3642 fname = member.filename;
3643 else:
3644 fname = "./"+member.filename;
3645 rarinfo = rarfp.getinfo(member.filename);
3646 if(verbose):
3647 VerbosePrintOut(fname);
3648 if(is_unix and member.external_attr !=0):
3649 fpremode = int(member.external_attr);
3650 elif(member.is_file()):
3651 fpremode = int(stat.S_IFREG + 438);
3652 elif(member.is_symlink()):
3653 fpremode = int(stat.S_IFLNK + 438);
3654 elif(member.is_dir()):
3655 fpremode = int(stat.S_IFDIR + 511);
3656 if(is_windows and member.external_attr !=0):
3657 fwinattributes = format(int(member.external_attr), 'x').lower();
3658 else:
3659 fwinattributes = format(int(0), 'x').lower();
3660 fcompression = "";
3661 fcsize = format(int(0), 'x').lower();
3662 flinkcount = 0;
3663 ftype = 0;
3664 if(member.is_file()):
3665 ftype = 0;
3666 elif(member.is_symlink()):
3667 ftype = 2;
3668 elif(member.is_dir()):
3669 ftype = 5;
3670 flinkname = "";
3671 if(ftype==2):
3672 flinkname = rarfp.read(member.filename).decode("UTF-8");
3673 fcurfid = format(int(curfid), 'x').lower();
3674 fcurinode = format(int(curfid), 'x').lower();
3675 curfid = curfid + 1;
3676 fdev_minor = format(int(0), 'x').lower();
3677 fdev_major = format(int(0), 'x').lower();
3678 frdev_minor = format(int(0), 'x').lower();
3679 frdev_major = format(int(0), 'x').lower();
3680 if(ftype==5):
3681 fsize = format(int("0"), 'x').lower();
3682 elif(ftype==0):
3683 fsize = format(int(member.file_size), 'x').lower();
3684 else:
3685 fsize = format(int(member.file_size), 'x').lower();
3686 try:
3687 if(member.atime):
3688 fatime = format(int(member.atime.timestamp()), 'x').lower();
3689 else:
3690 fatime = format(int(member.mtime.timestamp()), 'x').lower();
3691 except AttributeError:
3692 fatime = format(int(member.mtime.timestamp()), 'x').lower();
3693 fmtime = format(int(member.mtime.timestamp()), 'x').lower();
3694 try:
3695 if(member.ctime):
3696 fctime = format(int(member.ctime.timestamp()), 'x').lower();
3697 else:
3698 fctime = format(int(member.mtime.timestamp()), 'x').lower();
3699 except AttributeError:
3700 fctime = format(int(member.mtime.timestamp()), 'x').lower();
3701 fbtime = format(int(member.mtime.timestamp()), 'x').lower();
3702 if(is_unix and member.external_attr !=0):
3703 fmode = format(int(member.external_attr), 'x').lower();
3704 fchmode = format(int(stat.S_IMODE(member.external_attr)), 'x').lower();
3705 ftypemod = format(int(stat.S_IFMT(member.external_attr)), 'x').lower();
3706 elif(member.is_file()):
3707 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3708 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3709 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3710 elif(member.is_symlink()):
3711 fmode = format(int(stat.S_IFLNK + 438), 'x').lower();
3712 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3713 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3714 elif(member.is_dir()):
3715 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3716 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
3717 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
3718 try:
3719 fuid = format(int(os.getuid()), 'x').lower();
3720 except AttributeError:
3721 fuid = format(int(0), 'x').lower();
3722 except KeyError:
3723 fuid = format(int(0), 'x').lower();
3724 try:
3725 fgid = format(int(os.getgid()), 'x').lower();
3726 except AttributeError:
3727 fgid = format(int(0), 'x').lower();
3728 except KeyError:
3729 fgid = format(int(0), 'x').lower();
3730 try:
3731 import pwd;
3732 try:
3733 userinfo = pwd.getpwuid(os.getuid());
3734 funame = userinfo.pw_name;
3735 except KeyError:
3736 funame = "";
3737 except AttributeError:
3738 funame = "";
3739 except ImportError:
3740 funame = "";
3741 fgname = "";
3742 try:
3743 import grp;
3744 try:
3745 groupinfo = grp.getgrgid(os.getgid());
3746 fgname = groupinfo.gr_name;
3747 except KeyError:
3748 fgname = "";
3749 except AttributeError:
3750 fgname = "";
3751 except ImportError:
3752 fgname = "";
3753 fcontents = BytesIO();
3754 if(ftype==0):
3755 fcontents.write(rarfp.read(member.filename));
3756 if(not compresswholefile):
3757 fcontents.seek(0, 2);
3758 ucfsize = fcontents.tell();
3759 fcontents.seek(0, 0);
3760 if(compression=="auto"):
3761 ilsize = len(compressionlistalt);
3762 ilmin = 0;
3763 ilcsize = [];
3764 while(ilmin < ilsize):
3765 cfcontents = BytesIO();
3766 shutil.copyfileobj(fcontents, cfcontents);
3767 fcontents.seek(0, 0);
3768 cfcontents.seek(0, 0);
3769 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3770 if(cfcontents):
3771 cfcontents.seek(0, 2);
3772 ilcsize.append(cfcontents.tell());
3773 cfcontents.close();
3774 else:
3775 try:
3776 ilcsize.append(sys.maxint);
3777 except AttributeError:
3778 ilcsize.append(sys.maxsize);
3779 ilmin = ilmin + 1;
3780 ilcmin = ilcsize.index(min(ilcsize));
3781 compression = compressionlistalt[ilcmin];
3782 fcontents.seek(0, 0);
3783 cfcontents = BytesIO();
3784 shutil.copyfileobj(fcontents, cfcontents);
3785 cfcontents.seek(0, 0);
3786 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3787 cfcontents.seek(0, 2);
3788 cfsize = cfcontents.tell();
3789 if(ucfsize > cfsize):
3790 fcsize = format(int(cfsize), 'x').lower();
3791 fcompression = compression;
3792 fcontents.close();
3793 fcontents = cfcontents;
3794 if(fcompression=="none"):
3795 fcompression = "";
3796 fcontents.seek(0, 0);
3797 ftypehex = format(ftype, 'x').lower();
3798 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3799 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3800 fcontents.close();
3801 if(numfiles>0):
3802 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3803 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3804 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
3805 try:
3806 catfp.flush();
3807 os.fsync(catfp.fileno());
3808 except io.UnsupportedOperation:
3809 pass;
3810 except AttributeError:
3811 pass;
3812 except OSError as e:
3813 pass;
3814 if(outfile=="-"):
3815 catfp.seek(0, 0)
3816 if(hasattr(sys.stdout, "buffer")):
3817 shutil.copyfileobj(catfp, sys.stdout.buffer);
3818 else:
3819 shutil.copyfileobj(catfp, sys.stdout);
3820 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3821 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3822 catfp.seek(0, 0);
3823 upload_file_to_internet_file(catfp, outfile);
3824 if(returnfp):
3825 catfp.seek(0, 0)
3826 return catfp
3827 else:
3828 catfp.close()
3829 return True;
3830
3831 create_alias_function("Pack", __file_format_name__, "FromRarFile", PackArchiveFileFromRarFile);
3832
3833 if(not py7zr_support):
3834 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3835 return False
3836
3837 if(py7zr_support):
3838 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3839 formatspecs = FormatSpecsListToDict(formatspecs);
3840 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3841 outfile = RemoveWindowsPath(outfile);
3842 checksumtype = checksumtype.lower();
3843 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3844 checksumtype="crc32";
3845 if(checksumtype=="none"):
3846 checksumtype = "";
3847 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3848 compression = "auto";
3849 if(compression not in compressionlist and compression is None):
3850 compression = "auto";
3851 if(verbose):
3852 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3853 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3854 if(os.path.exists(outfile)):
3855 try:
3856 os.unlink(outfile);
3857 except OSError as e:
3858 pass;
3859 if(outfile=="-"):
3860 verbose = False;
3861 catfp = BytesIO();
3862 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3863 catfp = outfile;
3864 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3865 catfp = BytesIO();
3866 else:
3867 fbasename = os.path.splitext(outfile)[0];
3868 fextname = os.path.splitext(outfile)[1];
3869 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3870 catver = formatspecs['format_ver'];
3871 fileheaderver = str(int(catver.replace(".", "")));
3872 curinode = 0;
3873 curfid = 0;
3874 inodelist = [];
3875 inodetofile = {};
3876 filetoinode = {};
3877 inodetocatinode = {};
3878 if(not os.path.exists(infile) or not os.path.isfile(infile)):
3879 return False;
3880 szpfp = py7zr.SevenZipFile(infile, mode="r");
3881 file_content = szpfp.readall();
3882 #sztest = szpfp.testzip();
3883 sztestalt = szpfp.test();
3884 if(sztestalt):
3885 VerbosePrintOut("Bad file found!");
3886 numfiles = int(len(szpfp.list()));
3887 AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3888 for member in sorted(szpfp.list(), key=lambda x: x.filename):
3889 if(re.findall("^[.|/]", member.filename)):
3890 fname = member.filename;
3891 else:
3892 fname = "./"+member.filename;
3893 if(verbose):
3894 VerbosePrintOut(fname);
3895 if(not member.is_directory):
3896 fpremode = int(stat.S_IFREG + 438);
3897 elif(member.is_directory):
3898 fpremode = int(stat.S_IFDIR + 511);
3899 fwinattributes = format(int(0), 'x').lower();
3900 fcompression = "";
3901 fcsize = format(int(0), 'x').lower();
3902 flinkcount = 0;
3903 ftype = 0;
3904 if(member.is_directory):
3905 ftype = 5;
3906 else:
3907 ftype = 0;
3908 flinkname = "";
3909 fcurfid = format(int(curfid), 'x').lower();
3910 fcurinode = format(int(curfid), 'x').lower();
3911 curfid = curfid + 1;
3912 fdev_minor = format(int(0), 'x').lower();
3913 fdev_major = format(int(0), 'x').lower();
3914 frdev_minor = format(int(0), 'x').lower();
3915 frdev_major = format(int(0), 'x').lower();
3916 if(ftype==5):
3917 fsize = format(int("0"), 'x').lower();
3918 fatime = format(int(member.creationtime.timestamp()), 'x').lower();
3919 fmtime = format(int(member.creationtime.timestamp()), 'x').lower();
3920 fctime = format(int(member.creationtime.timestamp()), 'x').lower();
3921 fbtime = format(int(member.creationtime.timestamp()), 'x').lower();
3922 if(member.is_directory):
3923 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3924 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
3925 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
3926 else:
3927 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3928 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3929 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3930 try:
3931 fuid = format(int(os.getuid()), 'x').lower();
3932 except AttributeError:
3933 fuid = format(int(0), 'x').lower();
3934 except KeyError:
3935 fuid = format(int(0), 'x').lower();
3936 try:
3937 fgid = format(int(os.getgid()), 'x').lower();
3938 except AttributeError:
3939 fgid = format(int(0), 'x').lower();
3940 except KeyError:
3941 fgid = format(int(0), 'x').lower();
3942 try:
3943 import pwd;
3944 try:
3945 userinfo = pwd.getpwuid(os.getuid());
3946 funame = userinfo.pw_name;
3947 except KeyError:
3948 funame = "";
3949 except AttributeError:
3950 funame = "";
3951 except ImportError:
3952 funame = "";
3953 fgname = "";
3954 try:
3955 import grp;
3956 try:
3957 groupinfo = grp.getgrgid(os.getgid());
3958 fgname = groupinfo.gr_name;
3959 except KeyError:
3960 fgname = "";
3961 except AttributeError:
3962 fgname = "";
3963 except ImportError:
3964 fgname = "";
3965 fcontents = BytesIO();
3966 if(ftype==0):
3967 fcontents.write(file_content[member.filename].read());
3968 fsize = format(fcontents.tell(), 'x').lower();
3969 file_content[member.filename].close();
3970 if(not compresswholefile):
3971 fcontents.seek(0, 2);
3972 ucfsize = fcontents.tell();
3973 fcontents.seek(0, 0);
3974 if(compression=="auto"):
3975 ilsize = len(compressionlistalt);
3976 ilmin = 0;
3977 ilcsize = [];
3978 while(ilmin < ilsize):
3979 cfcontents = BytesIO();
3980 shutil.copyfileobj(fcontents, cfcontents);
3981 fcontents.seek(0, 0);
3982 cfcontents.seek(0, 0);
3983 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3984 if(cfcontents):
3985 cfcontents.seek(0, 2);
3986 ilcsize.append(cfcontents.tell());
3987 cfcontents.close();
3988 else:
3989 try:
3990 ilcsize.append(sys.maxint);
3991 except AttributeError:
3992 ilcsize.append(sys.maxsize);
3993 ilmin = ilmin + 1;
3994 ilcmin = ilcsize.index(min(ilcsize));
3995 compression = compressionlistalt[ilcmin];
3996 fcontents.seek(0, 0);
3997 cfcontents = BytesIO();
3998 shutil.copyfileobj(fcontents, cfcontents);
3999 cfcontents.seek(0, 0);
4000 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
4001 cfcontents.seek(0, 2);
4002 cfsize = cfcontents.tell();
4003 if(ucfsize > cfsize):
4004 fcsize = format(int(cfsize), 'x').lower();
4005 fcompression = compression;
4006 fcontents.close();
4007 fcontents = cfcontents;
4008 if(fcompression=="none"):
4009 fcompression = "";
4010 fcontents.seek(0, 0);
4011 ftypehex = format(ftype, 'x').lower();
4012 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
4013 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
4014 fcontents.close();
4015 if(numfiles>0):
4016 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
4017 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
4018 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
4019 try:
4020 catfp.flush();
4021 os.fsync(catfp.fileno());
4022 except io.UnsupportedOperation:
4023 pass;
4024 except AttributeError:
4025 pass;
4026 except OSError as e:
4027 pass;
4028 if(outfile=="-"):
4029 catfp.seek(0, 0);
4030 if(hasattr(sys.stdout, "buffer")):
4031 shutil.copyfileobj(catfp, sys.stdout.buffer);
4032 else:
4033 shutil.copyfileobj(catfp, sys.stdout);
4034 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
4035 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
4036 catfp.seek(0, 0);
4037 upload_file_to_internet_file(catfp, outfile);
4038 if(returnfp):
4039 catfp.seek(0, 0)
4040 return catfp
4041 else:
4042 catfp.close()
4043 return True;
4044
4045 create_alias_function("Pack", __file_format_name__, "FromSevenZipFile", PackArchiveFileFromSevenZipFile);
4046
4047 def PackArchiveFileFromInFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
4048 formatspecs = FormatSpecsListToDict(formatspecs);
4049 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4050 if(verbose):
4051 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
4052 if(checkcompressfile=="tarfile"):
4053 return PackArchiveFileFromTarFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4054 elif(checkcompressfile=="zipfile"):
4055 return PackArchiveFileFromZipFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4056 elif(checkcompressfile=="catfile"):
4057 return RePackArchiveFile(infile, outfile, compression, compresswholefile, compressionlevel, False, 0, 0, checksumtype, False, extradata, formatspecs, verbose, returnfp);
4058 elif(rarfile_support and checkcompressfile=="rarfile"):
4059 return PackArchiveFileFromRarFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4060 elif(py7zr_support and checkcompressfile=="7zipfile"):
4061 return PackArchiveFileFromSevenZipFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4062 else:
4063 return False;
4064 return False;
4065
4066 create_alias_function("Pack", __file_format_name__, "FromInFile", PackArchiveFileFromInFile);
4067
4068 def ArchiveFileSeekToFileNum(infile, seekto=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4069 formatspecs = FormatSpecsListToDict(formatspecs);
4070 if(hasattr(infile, "read") or hasattr(infile, "write")):
4071 catfp = infile;
4072 catfp.seek(0, 0);
4073 catfp = UncompressArchiveFile(catfp, formatspecs);
4074 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4075 if(checkcompressfile=="tarfile"):
4076 return TarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4077 if(checkcompressfile=="zipfile"):
4078 return ZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4079 if(rarfile_support and checkcompressfile=="rarfile"):
4080 return RarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4081 if(py7zr_support and checkcompressfile=="7zipfile"):
4082 return SevenZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4083 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4084 return False;
4085 if(not catfp):
4086 return False;
4087 catfp.seek(0, 0);
4088 elif(infile=="-"):
4089 catfp = BytesIO();
4090 if(hasattr(sys.stdin, "buffer")):
4091 shutil.copyfileobj(sys.stdin.buffer, catfp);
4092 else:
4093 shutil.copyfileobj(sys.stdin, catfp);
4094 catfp.seek(0, 0);
4095 catfp = UncompressArchiveFile(catfp, formatspecs);
4096 if(not catfp):
4097 return False;
4098 catfp.seek(0, 0);
4099 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4100 catfp = download_file_from_internet_file(infile);
4101 catfp.seek(0, 0);
4102 catfp = UncompressArchiveFile(catfp, formatspecs);
4103 if(not catfp):
4104 return False;
4105 catfp.seek(0, 0);
4106 else:
4107 infile = RemoveWindowsPath(infile);
4108 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4109 if(checkcompressfile=="tarfile"):
4110 return TarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4111 if(checkcompressfile=="zipfile"):
4112 return ZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4113 if(rarfile_support and checkcompressfile=="rarfile"):
4114 return RarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4115 if(py7zr_support and checkcompressfile=="7zipfile"):
4116 return SevenZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4117 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4118 return False;
4119 compresscheck = CheckCompressionType(infile, formatspecs, True);
4120 if(not compresscheck):
4121 fextname = os.path.splitext(infile)[1];
4122 if(fextname==".gz"):
4123 compresscheck = "gzip";
4124 elif(fextname==".bz2"):
4125 compresscheck = "bzip2";
4126 elif(fextname==".zst"):
4127 compresscheck = "zstd";
4128 elif(fextname==".lz4" or fextname==".clz4"):
4129 compresscheck = "lz4";
4130 elif(fextname==".lzo" or fextname==".lzop"):
4131 compresscheck = "lzo";
4132 elif(fextname==".lzma" or fextname==".xz"):
4133 compresscheck = "lzma";
4134 else:
4135 return False;
4136 if(not compresscheck):
4137 return False;
4138 catfp = UncompressFile(infile, formatspecs, "rb");
4139 '''
4140 try:
4141 catfp.seek(0, 2);
4142 except OSError:
4143 SeekToEndOfFile(catfp);
4144 except ValueError:
4145 SeekToEndOfFile(catfp);
4146 CatSize = catfp.tell();
4147 CatSizeEnd = CatSize;
4148 '''
4149 try:
4150 catfp.seek(0, 0);
4151 except OSError:
4152 return False;
4153 except ValueError:
4154 return False;
4155 curloc = catfp.tell();
4156 if(curloc>0):
4157 catfp.seek(0, 0);
4158 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4159 if(curloc>0):
4160 catfp.seek(curloc, 0);
4161 catstring = catheader[0];
4162 catversion = re.findall(r"([\d]+)", catstring);
4163 fprenumfiles = catheader[1];
4164 fnumfiles = int(fprenumfiles, 16);
4165 fprechecksumtype = catheader[2];
4166 fprechecksum = catheader[3];
4167 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4168 fnumfileshex = format(int(fnumfiles), 'x').lower();
4169 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4170 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4171 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4172 fheadtell = len(fileheader);
4173 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4174 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
4175 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4176 return False;
4177 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
4178 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4179 if(seekto>=fnumfiles):
4180 seekto = fnumfiles - 1;
4181 if(seekto<0):
4182 seekto = 0;
4183 if(seekto>=0):
4184 il = -1;
4185 while(il < seekto):
4186 prefhstart = catfp.tell();
4187 if(formatspecs['new_style']):
4188 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4189 else:
4190 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4191 if(len(preheaderdata)==0):
4192 break;
4193 prefheadsize = int(preheaderdata[0], 16);
4194 prefnumfields = int(preheaderdata[1], 16);
4195 preftype = int(preheaderdata[2], 16);
4196 if(re.findall("^[.|/]", preheaderdata[3])):
4197 prefname = preheaderdata[3];
4198 else:
4199 prefname = "./"+preheaderdata[3];
4200 prefbasedir = os.path.dirname(prefname);
4201 preflinkname = preheaderdata[4];
4202 prefsize = int(preheaderdata[5], 16);
4203 prefatime = int(preheaderdata[6], 16);
4204 prefmtime = int(preheaderdata[7], 16);
4205 prefctime = int(preheaderdata[8], 16);
4206 prefbtime = int(preheaderdata[9], 16);
4207 prefmode = int(preheaderdata[10], 16);
4208 prefchmode = stat.S_IMODE(prefmode);
4209 preftypemod = stat.S_IFMT(prefmode);
4210 prefwinattributes = int(preheaderdata[11], 16);
4211 prefcompression = preheaderdata[12];
4212 prefcsize = int(preheaderdata[13], 16);
4213 prefuid = int(preheaderdata[14], 16);
4214 prefuname = preheaderdata[15];
4215 prefgid = int(preheaderdata[16], 16);
4216 prefgname = preheaderdata[17];
4217 fid = int(preheaderdata[18], 16);
4218 finode = int(preheaderdata[19], 16);
4219 flinkcount = int(preheaderdata[20], 16);
4220 prefdev_minor = int(preheaderdata[21], 16);
4221 prefdev_major = int(preheaderdata[22], 16);
4222 prefrdev_minor = int(preheaderdata[23], 16);
4223 prefrdev_major = int(preheaderdata[24], 16);
4224 prefextrasize = int(preheaderdata[25], 16);
4225 prefextrafields = int(preheaderdata[26], 16);
4226 extrafieldslist = [];
4227 extrastart = 27;
4228 extraend = extrastart + prefextrafields;
4229 extrafieldslist = [];
4230 if(extrastart<extraend):
4231 extrafieldslist.append(preheaderdata[extrastart]);
4232 extrastart = extrastart + 1;
4233 prefchecksumtype = preheaderdata[extrastart].lower();
4234 prefcs = preheaderdata[extrastart + 1].lower();
4235 prefccs = preheaderdata[extrastart + 2].lower();
4236 hc = 0;
4237 hcmax = len(preheaderdata) - 2;
4238 hout = "";
4239 while(hc<hcmax):
4240 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs['format_delimiter']);
4241 hc = hc + 1;
4242 prenewfcs = GetFileChecksum(hout, preheaderdata[-3].lower(), True, formatspecs);
4243 if(prefcs!=prenewfcs and not skipchecksum):
4244 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4245 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4246 return False;
4247 valid_archive = False;
4248 invalid_archive = True;
4249 prefhend = catfp.tell() - 1;
4250 prefcontentstart = catfp.tell();
4251 prefcontents = "";
4252 pyhascontents = False;
4253 if(prefsize>0):
4254 if(prefcompression):
4255 prefcontents = catfp.read(prefsize);
4256 else:
4257 prefcontents = catfp.read(prefcsize);
4258 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4259 pyhascontents = True;
4260 if(prefccs!=prenewfccs and not skipchecksum):
4261 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4262 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4263 return False;
4264 catfp.seek(1, 1);
4265 il = il + 1;
4266 catfp.seek(seekstart, 0);
4267 fileidnum = il;
4268 catfheadsize = int(preheaderdata[0], 16);
4269 catfnumfields = int(preheaderdata[1], 16);
4270 catftype = int(preheaderdata[2], 16);
4271 if(re.findall("^[.|/]", preheaderdata[3])):
4272 catfname = preheaderdata[3];
4273 else:
4274 catfname = "./"+preheaderdata[3];
4275 catflinkname = preheaderdata[4];
4276 catfsize = int(preheaderdata[5], 16);
4277 catfbasedir = os.path.dirname(catfname);
4278 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
4279 if(returnfp):
4280 catlist.update({'catfp': catfp});
4281 else:
4282 catfp.close();
4283 return catlist;
4284
4285 create_alias_function("", __file_format_name__, "SeekToFileNum", ArchiveFileSeekToFileNum);
4286
4287 def ArchiveFileSeekToFileName(infile, seekfile=None, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4288 formatspecs = FormatSpecsListToDict(formatspecs);
4289 if(hasattr(infile, "read") or hasattr(infile, "write")):
4290 catfp = infile;
4291 catfp.seek(0, 0);
4292 catfp = UncompressArchiveFile(catfp, formatspecs);
4293 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4294 if(checkcompressfile=="tarfile"):
4295 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4296 if(checkcompressfile=="zipfile"):
4297 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4298 if(rarfile_support and checkcompressfile=="rarfile"):
4299 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4300 if(py7zr_support and checkcompressfile=="7zipfile"):
4301 return SevenZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4302 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4303 return False;
4304 if(not catfp):
4305 return False;
4306 catfp.seek(0, 0);
4307 elif(infile=="-"):
4308 catfp = BytesIO();
4309 if(hasattr(sys.stdin, "buffer")):
4310 shutil.copyfileobj(sys.stdin.buffer, catfp);
4311 else:
4312 shutil.copyfileobj(sys.stdin, catfp);
4313 catfp.seek(0, 0);
4314 catfp = UncompressArchiveFile(catfp, formatspecs);
4315 if(not catfp):
4316 return False;
4317 catfp.seek(0, 0);
4318 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4319 catfp = download_file_from_internet_file(infile);
4320 catfp = UncompressArchiveFile(catfp, formatspecs);
4321 catfp.seek(0, 0);
4322 if(not catfp):
4323 return False;
4324 catfp.seek(0, 0);
4325 else:
4326 infile = RemoveWindowsPath(infile);
4327 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4328 if(checkcompressfile=="tarfile"):
4329 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4330 if(checkcompressfile=="zipfile"):
4331 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4332 if(rarfile_support and checkcompressfile=="rarfile"):
4333 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4334 if(py7zr_support and checkcompressfile=="7zipfile"):
4335 return SevenZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4336 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4337 return False;
4338 compresscheck = CheckCompressionType(infile, formatspecs, True);
4339 if(not compresscheck):
4340 fextname = os.path.splitext(infile)[1];
4341 if(fextname==".gz"):
4342 compresscheck = "gzip";
4343 elif(fextname==".bz2"):
4344 compresscheck = "bzip2";
4345 elif(fextname==".zst"):
4346 compresscheck = "zstd";
4347 elif(fextname==".lz4" or fextname==".clz4"):
4348 compresscheck = "lz4";
4349 elif(fextname==".lzo" or fextname==".lzop"):
4350 compresscheck = "lzo";
4351 elif(fextname==".lzma" or fextname==".xz"):
4352 compresscheck = "lzma";
4353 else:
4354 return False;
4355 if(not compresscheck):
4356 return False;
4357 catfp = UncompressFile(infile, formatspecs, "rb");
4358 '''
4359 try:
4360 catfp.seek(0, 2);
4361 except OSError:
4362 SeekToEndOfFile(catfp);
4363 except ValueError:
4364 SeekToEndOfFile(catfp);
4365 CatSize = catfp.tell();
4366 CatSizeEnd = CatSize;
4367 '''
4368 try:
4369 catfp.seek(0, 0);
4370 except OSError:
4371 return False;
4372 except ValueError:
4373 return False;
4374 curloc = catfp.tell();
4375 if(curloc>0):
4376 catfp.seek(0, 0);
4377 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4378 if(curloc>0):
4379 catfp.seek(curloc, 0);
4380 catstring = catheader[0];
4381 catversion = re.findall(r"([\d]+)", catstring);
4382 fprenumfiles = catheader[1];
4383 fnumfiles = int(fprenumfiles, 16);
4384 fprechecksumtype = catheader[2];
4385 fprechecksum = catheader[3];
4386 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4387 fnumfileshex = format(int(fnumfiles), 'x').lower();
4388 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4389 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4390 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4391 fheadtell = len(fileheader);
4392 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4393 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
4394 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4395 return False;
4396 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
4397 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4398 seekto = fnumfiles - 1
4399 filefound = False;
4400 if(seekto>=0):
4401 il = -1;
4402 while(il < seekto):
4403 prefhstart = catfp.tell();
4404 if(formatspecs['new_style']):
4405 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4406 else:
4407 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4408 if(len(preheaderdata)==0):
4409 break;
4410 prefheadsize = int(preheaderdata[0], 16);
4411 prefnumfields = int(preheaderdata[1], 16);
4412 preftype = int(preheaderdata[2], 16);
4413 if(re.findall("^[.|/]", preheaderdata[3])):
4414 prefname = preheaderdata[3];
4415 else:
4416 prefname = "./"+preheaderdata[3];
4417 prefbasedir = os.path.dirname(prefname);
4418 preflinkname = preheaderdata[4];
4419 prefsize = int(preheaderdata[5], 16);
4420 prefatime = int(preheaderdata[6], 16);
4421 prefmtime = int(preheaderdata[7], 16);
4422 prefctime = int(preheaderdata[8], 16);
4423 prefbtime = int(preheaderdata[9], 16);
4424 prefmode = int(preheaderdata[10], 16);
4425 prefchmode = stat.S_IMODE(prefmode);
4426 preftypemod = stat.S_IFMT(prefmode);
4427 prefwinattributes = int(preheaderdata[11], 16);
4428 prefcompression = preheaderdata[12];
4429 prefcsize = int(preheaderdata[13], 16);
4430 prefuid = int(preheaderdata[14], 16);
4431 prefuname = preheaderdata[15];
4432 prefgid = int(preheaderdata[16], 16);
4433 prefgname = preheaderdata[17];
4434 fid = int(preheaderdata[18], 16);
4435 finode = int(preheaderdata[19], 16);
4436 flinkcount = int(preheaderdata[20], 16);
4437 prefdev_minor = int(preheaderdata[21], 16);
4438 prefdev_major = int(preheaderdata[22], 16);
4439 prefrdev_minor = int(preheaderdata[23], 16);
4440 prefrdev_major = int(preheaderdata[24], 16);
4441 prefextrasize = int(preheaderdata[25], 16);
4442 prefextrafields = int(preheaderdata[26], 16);
4443 extrafieldslist = [];
4444 extrastart = 27;
4445 extraend = extrastart + prefextrafields;
4446 extrafieldslist = [];
4447 if(extrastart<extraend):
4448 extrafieldslist.append(preheaderdata[extrastart]);
4449 extrastart = extrastart + 1;
4450 prefchecksumtype = preheaderdata[extrastart].lower();
4451 prefcs = preheaderdata[extrastart + 1].lower();
4452 prefccs = preheaderdata[extrastart + 2].lower();
4453 hc = 0;
4454 hcmax = len(preheaderdata) - 2;
4455 hout = "";
4456 while(hc<hcmax):
4457 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs['format_delimiter']);
4458 hc = hc + 1;
4459 prenewfcs = GetFileChecksum(hout, preheaderdata[-3].lower(), True, formatspecs);
4460 if(prefcs!=prenewfcs and not skipchecksum):
4461 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4462 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4463 return False;
4464 valid_archive = False;
4465 invalid_archive = True;
4466 prefhend = catfp.tell() - 1;
4467 prefcontentstart = catfp.tell();
4468 prefcontents = "";
4469 pyhascontents = False;
4470 if(prefsize>0):
4471 if(prefcompression):
4472 prefcontents = catfp.read(prefsize);
4473 else:
4474 prefcontents = catfp.read(prefcsize);
4475 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4476 pyhascontents = True;
4477 if(prefccs!=prenewfccs and not skipchecksum):
4478 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4479 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4480 return False;
4481 catfp.seek(1, 1);
4482 il = il + 1;
4483 filefound = False;
4484 prefname = preheaderdata[2];
4485 if(re.findall("^[.|/]", preheaderdata[2])):
4486 prefname = preheaderdata[2];
4487 else:
4488 prefname = "./"+preheaderdata[2];
4489 if(prefname==seekfile):
4490 filefound = True;
4491 break;
4492 catfp.seek(seekstart, 0);
4493 fileidnum = il;
4494 catfheadsize = int(preheaderdata[0], 16);
4495 catfnumfields = int(preheaderdata[1], 16);
4496 catftype = int(preheaderdata[2], 16);
4497 if(re.findall("^[.|/]", preheaderdata[3])):
4498 catfname = preheaderdata[3];
4499 else:
4500 catfname = "./"+preheaderdata[3];
4501 catflinkname = preheaderdata[4];
4502 catfsize = int(preheaderdata[5], 16);
4503 catfbasedir = os.path.dirname(catfname);
4504 if(filefound):
4505 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
4506 else:
4507 return False;
4508 if(returnfp):
4509 catlist.update({'catfp': catfp});
4510 else:
4511 catfp.close();
4512 return catlist;
4513
4514 create_alias_function("", __file_format_name__, "SeekToFileName", ArchiveFileSeekToFileName);
4515
4516 def ArchiveFileValidate(infile, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
4517 formatspecs = FormatSpecsListToDict(formatspecs);
4518 if(verbose):
4519 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
4520 if(hasattr(infile, "read") or hasattr(infile, "write")):
4521 catfp = infile;
4522 catfp.seek(0, 0);
4523 catfp = UncompressArchiveFile(catfp, formatspecs);
4524 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4525 if(checkcompressfile=="tarfile"):
4526 return TarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4527 if(checkcompressfile=="zipfile"):
4528 return ZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4529 if(rarfile_support and checkcompressfile=="rarfile"):
4530 return RarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4531 if(py7zr_support and checkcompressfile=="7zipfile"):
4532 return SevenZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4533 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4534 return False;
4535 if(not catfp):
4536 return False;
4537 catfp.seek(0, 0);
4538 elif(infile=="-"):
4539 catfp = BytesIO();
4540 if(hasattr(sys.stdin, "buffer")):
4541 shutil.copyfileobj(sys.stdin.buffer, catfp);
4542 else:
4543 shutil.copyfileobj(sys.stdin, catfp);
4544 catfp.seek(0, 0);
4545 catfp = UncompressArchiveFile(catfp, formatspecs);
4546 if(not catfp):
4547 return False;
4548 catfp.seek(0, 0);
4549 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4550 catfp = download_file_from_internet_file(infile);
4551 catfp = UncompressArchiveFile(catfp, formatspecs);
4552 catfp.seek(0, 0);
4553 if(not catfp):
4554 return False;
4555 catfp.seek(0, 0);
4556 else:
4557 infile = RemoveWindowsPath(infile);
4558 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4559 if(checkcompressfile=="tarfile"):
4560 return TarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4561 if(checkcompressfile=="zipfile"):
4562 return ZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4563 if(rarfile_support and checkcompressfile=="rarfile"):
4564 return RarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4565 if(py7zr_support and checkcompressfile=="7zipfile"):
4566 return SevenZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4567 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4568 return False;
4569 compresscheck = CheckCompressionType(infile, formatspecs, True);
4570 if(not compresscheck):
4571 fextname = os.path.splitext(infile)[1];
4572 if(fextname==".gz"):
4573 compresscheck = "gzip";
4574 elif(fextname==".bz2"):
4575 compresscheck = "bzip2";
4576 elif(fextname==".zst"):
4577 compresscheck = "zstd";
4578 elif(fextname==".lz4" or fextname==".clz4"):
4579 compresscheck = "lz4";
4580 elif(fextname==".lzo" or fextname==".lzop"):
4581 compresscheck = "lzo";
4582 elif(fextname==".lzma" or fextname==".xz"):
4583 compresscheck = "lzma";
4584 else:
4585 return False;
4586 if(not compresscheck):
4587 return False;
4588 catfp = UncompressFile(infile, formatspecs, "rb");
4589 '''
4590 try:
4591 catfp.seek(0, 2);
4592 except OSError:
4593 SeekToEndOfFile(catfp);
4594 except ValueError:
4595 SeekToEndOfFile(catfp);
4596 CatSize = catfp.tell();
4597 CatSizeEnd = CatSize;
4598 '''
4599 try:
4600 catfp.seek(0, 0);
4601 except OSError:
4602 return False;
4603 except ValueError:
4604 return False;
4605 curloc = catfp.tell();
4606 if(curloc>0):
4607 catfp.seek(0, 0);
4608 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4609 if(curloc>0):
4610 catfp.seek(curloc, 0);
4611 catstring = catheader[0];
4612 catversion = re.findall(r"([\d]+)", catstring);
4613 fprenumfiles = catheader[1];
4614 fnumfiles = int(fprenumfiles, 16);
4615 fprechecksumtype = catheader[2];
4616 fprechecksum = catheader[3];
4617 il = 0;
4618 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4619 fnumfileshex = format(int(fnumfiles), 'x').lower();
4620 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4621 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4622 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4623 valid_archive = True;
4624 invalid_archive = False;
4625 if(verbose):
4626 VerbosePrintOut(infile);
4627 VerbosePrintOut("Number of Records " + str(fnumfiles));
4628 if(fprechecksum==catfileheadercshex):
4629 if(verbose):
4630 VerbosePrintOut("File Header Checksum Passed at offset " + str(0));
4631 VerbosePrintOut("'" + str(fprechecksum) + "' == " + "'" + str(catfileheadercshex) + "'");
4632 else:
4633 if(verbose):
4634 VerbosePrintOut("File Header Checksum Failed at offset " + str(0));
4635 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4636 valid_archive = False;
4637 invalid_archive = True;
4638 if(verbose):
4639 VerbosePrintOut("");
4640 while(il<fnumfiles):
4641 catfhstart = catfp.tell();
4642 if(formatspecs['new_style']):
4643 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4644 else:
4645 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4646 if(len(catheaderdata)==0):
4647 break;
4648 catfheadsize = int(catheaderdata[0], 16);
4649 catfnumfields = int(catheaderdata[1], 16);
4650 catftype = int(catheaderdata[2], 16);
4651 if(re.findall("^[.|/]", catheaderdata[3])):
4652 catfname = catheaderdata[3];
4653 else:
4654 catfname = "./"+catheaderdata[3];
4655 catfbasedir = os.path.dirname(catfname);
4656 catflinkname = catheaderdata[4];
4657 catfsize = int(catheaderdata[5], 16);
4658 catfatime = int(catheaderdata[6], 16);
4659 catfmtime = int(catheaderdata[7], 16);
4660 catfctime = int(catheaderdata[8], 16);
4661 catfbtime = int(catheaderdata[9], 16);
4662 catfmode = int(catheaderdata[10], 16);
4663 catfchmode = stat.S_IMODE(catfmode);
4664 catftypemod = stat.S_IFMT(catfmode);
4665 catfwinattributes = int(catheaderdata[11], 16);
4666 catfcompression = catheaderdata[12];
4667 catfcsize = int(catheaderdata[13], 16);
4668 catfuid = int(catheaderdata[14], 16);
4669 catfuname = catheaderdata[15];
4670 catfgid = int(catheaderdata[16], 16);
4671 catfgname = catheaderdata[17];
4672 fid = int(catheaderdata[18], 16);
4673 finode = int(catheaderdata[19], 16);
4674 flinkcount = int(catheaderdata[20], 16);
4675 catfdev_minor = int(catheaderdata[21], 16);
4676 catfdev_major = int(catheaderdata[22], 16);
4677 catfrdev_minor = int(catheaderdata[23], 16);
4678 catfrdev_major = int(catheaderdata[24], 16);
4679 catfextrasize = int(catheaderdata[25], 16);
4680 catfextrafields = int(catheaderdata[26], 16);
4681 extrafieldslist = [];
4682 extrastart = 27;
4683 extraend = extrastart + catfextrafields;
4684 extrafieldslist = [];
4685 if(extrastart<extraend):
4686 extrafieldslist.append(catheaderdata[extrastart]);
4687 extrastart = extrastart + 1;
4688 catfchecksumtype = catheaderdata[extrastart].lower();
4689 catfcs = catheaderdata[extrastart + 1].lower();
4690 catfccs = catheaderdata[extrastart + 2].lower();
4691 hc = 0;
4692 hcmax = len(catheaderdata) - 2;
4693 hout = "";
4694 while(hc<hcmax):
4695 hout = hout + AppendNullByte(catheaderdata[hc], formatspecs['format_delimiter']);
4696 hc = hc + 1;
4697 catnewfcs = GetFileChecksum(hout, catheaderdata[-3].lower(), True, formatspecs);
4698 if(verbose):
4699 VerbosePrintOut(catfname);
4700 VerbosePrintOut("Record Number " + str(il) + "; File ID " + str(fid) + "; iNode Number " + str(finode));
4701 if(catfcs==catnewfcs):
4702 if(verbose):
4703 VerbosePrintOut("File Header Checksum Passed at offset " + str(catfhstart));
4704 VerbosePrintOut("'" + str(catfcs) + "' == " + "'" + str(catnewfcs) + "'");
4705 else:
4706 if(verbose):
4707 VerbosePrintOut("File Header Checksum Failed at offset " + str(catfhstart));
4708 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
4709 valid_archive = False;
4710 invalid_archive = True;
4711 catfhend = catfp.tell() - 1;
4712 catfcontentstart = catfp.tell();
4713 catfcontents = "";
4714 pyhascontents = False;
4715 if(catfsize>0):
4716 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4717 catfcontents = catfp.read(catfsize);
4718 else:
4719 catfcontents = catfp.read(catfcsize);
4720 catnewfccs = GetFileChecksum(catfcontents, catheaderdata[-3].lower(), False, formatspecs);
4721 pyhascontents = True;
4722 if(catfccs==catnewfccs):
4723 if(verbose):
4724 VerbosePrintOut("File Content Checksum Passed at offset " + str(catfcontentstart));
4725 VerbosePrintOut("'" + str(catfccs) + "' == " + "'" + str(catnewfccs) + "'");
4726 else:
4727 if(verbose):
4728 VerbosePrintOut("File Content Checksum Failed at offset " + str(catfcontentstart));
4729 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
4730 valid_archive = False;
4731 invalid_archive = True;
4732 if(verbose):
4733 VerbosePrintOut("");
4734 catfp.seek(1, 1);
4735 il = il + 1;
4736 if(valid_archive):
4737 if(returnfp):
4738 return catfp;
4739 else:
4740 catfp.close();
4741 return True;
4742 else:
4743 catfp.close();
4744 return False;
4745
4746 create_alias_function("", __file_format_name__, "Validate", ArchiveFileValidate);
4747
4748 def ArchiveFileToArray(infile, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4749 formatspecs = FormatSpecsListToDict(formatspecs);
4750 if(hasattr(infile, "read") or hasattr(infile, "write")):
4751 catfp = infile;
4752 catfp.seek(0, 0);
4753 catfp = UncompressArchiveFile(catfp, formatspecs);
4754 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4755 if(checkcompressfile=="tarfile"):
4756 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4757 if(checkcompressfile=="zipfile"):
4758 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4759 if(rarfile_support and checkcompressfile=="rarfile"):
4760 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4761 if(py7zr_support and checkcompressfile=="7zipfile"):
4762 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4763 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4764 return False;
4765 if(not catfp):
4766 return False;
4767 catfp.seek(0, 0);
4768 elif(infile=="-"):
4769 catfp = BytesIO();
4770 if(hasattr(sys.stdin, "buffer")):
4771 shutil.copyfileobj(sys.stdin.buffer, catfp);
4772 else:
4773 shutil.copyfileobj(sys.stdin, catfp);
4774 catfp.seek(0, 0);
4775 catfp = UncompressArchiveFile(catfp, formatspecs);
4776 if(not catfp):
4777 return False;
4778 catfp.seek(0, 0);
4779 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4780 catfp = download_file_from_internet_file(infile);
4781 catfp = UncompressArchiveFile(catfp, formatspecs);
4782 catfp.seek(0, 0);
4783 if(not catfp):
4784 return False;
4785 catfp.seek(0, 0);
4786 else:
4787 infile = RemoveWindowsPath(infile);
4788 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4789 if(checkcompressfile=="tarfile"):
4790 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4791 if(checkcompressfile=="zipfile"):
4792 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4793 if(rarfile_support and checkcompressfile=="rarfile"):
4794 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4795 if(py7zr_support and checkcompressfile=="7zipfile"):
4796 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4797 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4798 return False;
4799 compresscheck = CheckCompressionType(infile, formatspecs, True);
4800 if(not compresscheck):
4801 fextname = os.path.splitext(infile)[1];
4802 if(fextname==".gz"):
4803 compresscheck = "gzip";
4804 elif(fextname==".bz2"):
4805 compresscheck = "bzip2";
4806 elif(fextname==".zst"):
4807 compresscheck = "zstd";
4808 elif(fextname==".lz4" or fextname==".clz4"):
4809 compresscheck = "lz4";
4810 elif(fextname==".lzo" or fextname==".lzop"):
4811 compresscheck = "lzo";
4812 elif(fextname==".lzma" or fextname==".xz"):
4813 compresscheck = "lzma";
4814 else:
4815 return False;
4816 if(not compresscheck):
4817 return False;
4818 catfp = UncompressFile(infile, formatspecs, "rb");
4819 '''
4820 try:
4821 catfp.seek(0, 2);
4822 except OSError:
4823 SeekToEndOfFile(catfp);
4824 except ValueError:
4825 SeekToEndOfFile(catfp);
4826 CatSize = catfp.tell();
4827 CatSizeEnd = CatSize;
4828 '''
4829 try:
4830 catfp.seek(0, 0);
4831 except OSError:
4832 return False;
4833 except ValueError:
4834 return False;
4835 curloc = catfp.tell();
4836 if(curloc>0):
4837 catfp.seek(0, 0);
4838 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4839 if(curloc>0):
4840 catfp.seek(curloc, 0);
4841 catstring = catheader[0];
4842 catversion = re.findall(r"([\d]+)", catstring);
4843 fprenumfiles = catheader[1];
4844 fnumfiles = int(fprenumfiles, 16);
4845 fprechecksumtype = catheader[2];
4846 fprechecksum = catheader[3];
4847 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4848 fnumfileshex = format(int(fnumfiles), 'x').lower();
4849 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4850 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4851 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4852 fheadtell = len(fileheader);
4853 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4854 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
4855 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4856 return False;
4857 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
4858 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4859 if(seekstart<0 and seekstart>fnumfiles):
4860 seekstart = 0;
4861 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
4862 seekend = fnumfiles;
4863 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
4864 seekend = fnumfiles - abs(seekend);
4865 if(seekstart>0):
4866 il = 0;
4867 while(il < seekstart):
4868 prefhstart = catfp.tell();
4869 if(formatspecs['new_style']):
4870 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4871 else:
4872 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4873 if(len(preheaderdata)==0):
4874 break;
4875 prefheadsize = int(preheaderdata[0], 16);
4876 prefnumfields = int(preheaderdata[1], 16);
4877 if(re.findall("^[.|/]", preheaderdata[3])):
4878 prefname = preheaderdata[3];
4879 else:
4880 prefname = "./"+preheaderdata[3];
4881 prefsize = int(preheaderdata[5], 16);
4882 prefcompression = preheaderdata[12];
4883 prefcsize = int(preheaderdata[13], 16);
4884 prefextrasize = int(preheaderdata[24], 16);
4885 prefextrafields = int(preheaderdata[25], 16);
4886 extrafieldslist = [];
4887 extrastart = 26;
4888 extraend = extrastart + prefextrafields;
4889 extrafieldslist = [];
4890 if(extrastart<extraend):
4891 extrafieldslist.append(preheaderdata[extrastart]);
4892 extrastart = extrastart + 1;
4893 prefchecksumtype = preheaderdata[extrastart].lower();
4894 prefcs = preheaderdata[extrastart + 1].lower();
4895 prefccs = preheaderdata[extrastart + 2].lower();
4896 hc = 0;
4897 hcmax = len(preheaderdata) - 2;
4898 hout = "";
4899 while(hc<hcmax):
4900 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs['format_delimiter']);
4901 hc = hc + 1;
4902 prenewfcs = GetFileChecksum(hout, preheaderdata[-3].lower(), True, formatspecs);
4903 if(prefcs!=prenewfcs and not skipchecksum):
4904 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4905 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4906 return False;
4907 valid_archive = False;
4908 invalid_archive = True;
4909 prefhend = catfp.tell() - 1;
4910 prefcontentstart = catfp.tell();
4911 prefcontents = "";
4912 pyhascontents = False;
4913 if(prefsize>0):
4914 if(prefcompression=="none" or prefcompression=="" or prefcompression=="auto"):
4915 prefcontents = catfp.read(prefsize);
4916 else:
4917 prefcontents = catfp.read(prefcsize);
4918 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4919 pyhascontents = True;
4920 if(prefccs!=prenewfccs and not skipchecksum):
4921 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4922 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4923 return False;
4924 catfp.seek(1, 1);
4925 il = il + 1;
4926 fileidnum = seekstart;
4927 realidnum = 0;
4928 while(fileidnum<seekend):
4929 catfhstart = catfp.tell();
4930 if(formatspecs['new_style']):
4931 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4932 else:
4933 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4934 if(len(catheaderdata)==0):
4935 break;
4936 catfheadsize = int(catheaderdata[0], 16);
4937 catfnumfields = int(catheaderdata[1], 16);
4938 catftype = int(catheaderdata[2], 16);
4939 if(re.findall("^[.|/]", catheaderdata[3])):
4940 catfname = catheaderdata[3];
4941 else:
4942 catfname = "./"+catheaderdata[3];
4943 catfbasedir = os.path.dirname(catfname);
4944 catflinkname = catheaderdata[4];
4945 catfsize = int(catheaderdata[5], 16);
4946 catfatime = int(catheaderdata[6], 16);
4947 catfmtime = int(catheaderdata[7], 16);
4948 catfctime = int(catheaderdata[8], 16);
4949 catfbtime = int(catheaderdata[9], 16);
4950 catfmode = int(catheaderdata[10], 16);
4951 catfchmode = stat.S_IMODE(catfmode);
4952 catftypemod = stat.S_IFMT(catfmode);
4953 catfwinattributes = int(catheaderdata[11], 16);
4954 catfcompression = catheaderdata[12];
4955 catfcsize = int(catheaderdata[13], 16);
4956 catfuid = int(catheaderdata[14], 16);
4957 catfuname = catheaderdata[15];
4958 catfgid = int(catheaderdata[16], 16);
4959 catfgname = catheaderdata[17];
4960 catfid = int(catheaderdata[18], 16);
4961 catfinode = int(catheaderdata[19], 16);
4962 catflinkcount = int(catheaderdata[20], 16);
4963 catfdev_minor = int(catheaderdata[21], 16);
4964 catfdev_major = int(catheaderdata[22], 16);
4965 catfrdev_minor = int(catheaderdata[23], 16);
4966 catfrdev_major = int(catheaderdata[24], 16);
4967 catfextrasize = int(catheaderdata[25], 16);
4968 catfextrafields = int(catheaderdata[26], 16);
4969 extrafieldslist = [];
4970 extrastart = 27;
4971 extraend = extrastart + catfextrafields;
4972 extrafieldslist = [];
4973 if(extrastart<extraend):
4974 extrafieldslist.append(catheaderdata[extrastart]);
4975 extrastart = extrastart + 1;
4976 catfchecksumtype = catheaderdata[extrastart].lower();
4977 catfcs = catheaderdata[extrastart + 1].lower();
4978 catfccs = catheaderdata[extrastart + 2].lower();
4979 hc = 0;
4980 hcmax = len(catheaderdata) - 2;
4981 hout = "";
4982 while(hc<hcmax):
4983 hout = hout + AppendNullByte(catheaderdata[hc], formatspecs['format_delimiter']);
4984 hc = hc + 1;
4985 catnewfcs = GetFileChecksum(hout, catheaderdata[-3].lower(), True, formatspecs);
4986 if(catfcs!=catnewfcs and not skipchecksum):
4987 VerbosePrintOut("File Header Checksum Error with file " + catfname + " at offset " + str(catfhstart));
4988 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
4989 return False;
4990 catfhend = catfp.tell() - 1;
4991 catfcontentstart = catfp.tell();
4992 catfcontents = BytesIO();
4993 pyhascontents = False;
4994 if(catfsize>0 and not listonly):
4995 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4996 catfcontents.write(catfp.read(catfsize));
4997 else:
4998 catfcontents.write(catfp.read(catfcsize));
4999 catfcontents.seek(0, 0);
5000 catnewfccs = GetFileChecksum(catfcontents.read(), catheaderdata[-3].lower(), False, formatspecs);
5001 pyhascontents = True;
5002 if(catfccs!=catnewfccs and skipchecksum):
5003 VerbosePrintOut("File Content Checksum Error with file " + catfname + " at offset " + str(catfcontentstart));
5004 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
5005 return False;
5006 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
5007 pass;
5008 else:
5009 catfcontents.seek(0, 0);
5010 if(uncompress):
5011 catfcontents = UncompressArchiveFile(catfcontents, formatspecs);
5012 catfcontents.seek(0, 0);
5013 catfccs = GetFileChecksum(catfcontents.read(), catheaderdata[-3].lower(), False, formatspecs);
5014 if(catfsize>0 and listonly):
5015 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
5016 catfp.seek(catfsize, 1);
5017 else:
5018 catfp.seek(catfcsize, 1);
5019 pyhascontents = False;
5020 catfp.seek(1, 1);
5021 catfcontentend = catfp.tell() - 1;
5022 catfcontents.seek(0, 0);
5023 catlist['ffilelist'].update({realidnum: {'fid': realidnum, 'fidalt': fileidnum, 'fheadersize': catfheadsize, 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize, 'fatime': catfatime, 'fmtime': catfmtime, 'fctime': catfctime, 'fbtime': catfbtime, 'fmode': catfmode, 'fchmode': catfchmode, 'ftypemod': catftypemod, 'fwinattributes': catfwinattributes, 'fcompression': catfcompression, 'fcsize': catfcsize, 'fuid': catfuid, 'funame': catfuname, 'fgid': catfgid, 'fgname': catfgname, 'finode': catfinode, 'flinkcount': catflinkcount, 'fminor': catfdev_minor, 'fmajor': catfdev_major, 'frminor': catfrdev_minor, 'frmajor': catfrdev_major, 'fchecksumtype': catfchecksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': catfextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': catfcs, 'fcontentchecksum': catfccs, 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': catfcontents} });
5024 fileidnum = fileidnum + 1;
5025 realidnum = realidnum + 1;
5026 if(returnfp):
5027 catlist.update({'catfp': catfp});
5028 else:
5029 catfp.close();
5030 return catlist;
5031
5032 create_alias_function("", __file_format_name__, "ToArray", ArchiveFileToArray);
5033
5034 def ArchiveFileStringToArray(catstr, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5035 formatspecs = FormatSpecsListToDict(formatspecs);
5036 catfp = BytesIO(catstr);
5037 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5038 return listcatfiles;
5039
5040 create_alias_function("", __file_format_name__, "StringToArray", ArchiveFileStringToArray);
5041
5042 def TarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5043 formatspecs = FormatSpecsListToDict(formatspecs);
5044 catfp = BytesIO();
5045 catfp = PackArchiveFileFromTarFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5046 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5047 return listcatfiles;
5048
5049 def ZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5050 formatspecs = FormatSpecsListToDict(formatspecs);
5051 catfp = BytesIO();
5052 catfp = PackArchiveFileFromZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5053 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5054 return listcatfiles;
5055
5056 if(not rarfile_support):
5057 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5058 return False;
5059
5060 if(rarfile_support):
5061 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5062 formatspecs = FormatSpecsListToDict(formatspecs);
5063 catfp = BytesIO();
5064 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5065 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5066 return listcatfiles;
5067
5068 if(not py7zr_support):
5069 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5070 return False;
5071
5072 if(py7zr_support):
5073 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5074 formatspecs = FormatSpecsListToDict(formatspecs);
5075 catfp = BytesIO();
5076 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5077 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5078 return listcatfiles;
5079
5080 def InFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5081 formatspecs = FormatSpecsListToDict(formatspecs);
5082 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
5083 if(checkcompressfile=="tarfile"):
5084 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5085 elif(checkcompressfile=="zipfile"):
5086 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5087 elif(checkcompressfile=="catfile"):
5088 return ArchiveFileToArray(infile, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5089 elif(rarfile_support and checkcompressfile=="rarfile"):
5090 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5091 elif(py7zr_support and checkcompressfile=="7zipfile"):
5092 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5093 else:
5094 return False;
5095 return False;
5096
5097 def ListDirToArrayAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5098 formatspecs = FormatSpecsListToDict(formatspecs);
5099 catver = formatspecs['format_ver'];
5100 fileheaderver = str(int(catver.replace(".", "")));
5101 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5102 advancedlist = formatspecs['use_advanced_list'];
5103 altinode = formatspecs['use_alt_inode'];
5104 infilelist = [];
5105 if(infiles=="-"):
5106 for line in sys.stdin:
5107 infilelist.append(line.strip());
5108 infilelist = list(filter(None, infilelist));
5109 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
5110 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
5111 return False;
5112 with UncompressFile(infiles, formatspecs, "r") as finfile:
5113 for line in finfile:
5114 infilelist.append(line.strip());
5115 infilelist = list(filter(None, infilelist));
5116 else:
5117 if(isinstance(infiles, (list, tuple, ))):
5118 infilelist = list(filter(None, infiles));
5119 elif(isinstance(infiles, (str, ))):
5120 infilelist = list(filter(None, [infiles]));
5121 if(advancedlist):
5122 GetDirList = ListDirAdvanced(infilelist, followlink, False);
5123 else:
5124 GetDirList = ListDir(infilelist, followlink, False);
5125 if(not GetDirList):
5126 return False;
5127 curinode = 0;
5128 curfid = 0;
5129 inodelist = [];
5130 inodetofile = {};
5131 filetoinode = {};
5132 inodetocatinode = {};
5133 fileidnum = 0;
5134 fnumfiles = int(len(GetDirList));
5135 catver = formatspecs['format_ver'];
5136 fileheaderver = str(int(catver.replace(".", "")));
5137 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5138 fnumfileshex = format(int(fnumfiles), 'x').lower();
5139 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5140 catversion = re.findall(r"([\d]+)", fileheader);
5141 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5142 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5143 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5144 fheadtell = len(fileheader);
5145 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5146 for curfname in GetDirList:
5147 catfhstart = fheadtell;
5148 if(re.findall("^[.|/]", curfname)):
5149 fname = curfname;
5150 else:
5151 fname = "./"+curfname;
5152 if(verbose):
5153 VerbosePrintOut(fname);
5154 if(not followlink or followlink is None):
5155 fstatinfo = os.lstat(fname);
5156 else:
5157 fstatinfo = os.stat(fname);
5158 fpremode = fstatinfo.st_mode;
5159 finode = fstatinfo.st_ino;
5160 flinkcount = fstatinfo.st_nlink;
5161 ftype = 0;
5162 if(stat.S_ISREG(fpremode)):
5163 ftype = 0;
5164 elif(stat.S_ISLNK(fpremode)):
5165 ftype = 2;
5166 elif(stat.S_ISCHR(fpremode)):
5167 ftype = 3;
5168 elif(stat.S_ISBLK(fpremode)):
5169 ftype = 4;
5170 elif(stat.S_ISDIR(fpremode)):
5171 ftype = 5;
5172 elif(stat.S_ISFIFO(fpremode)):
5173 ftype = 6;
5174 elif(stat.S_ISSOCK(fpremode)):
5175 ftype = 8;
5176 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
5177 ftype = 9;
5178 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
5179 ftype = 10;
5180 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
5181 ftype = 11;
5182 else:
5183 ftype = 0;
5184 flinkname = "";
5185 fbasedir = os.path.dirname(fname);
5186 fcurfid = curfid;
5187 if(not followlink and finode!=0):
5188 if(ftype!=1):
5189 if(finode in inodelist):
5190 ftype = 1;
5191 flinkname = inodetofile[finode];
5192 if(altinode):
5193 fcurinode = finode;
5194 else:
5195 fcurinode = inodetocatinode[finode];
5196 if(finode not in inodelist):
5197 inodelist.append(finode);
5198 inodetofile.update({finode: fname});
5199 inodetocatinode.update({finode: curinode});
5200 if(altinode):
5201 fcurinode = finode;
5202 else:
5203 fcurinode = curinode;
5204 curinode = curinode + 1;
5205 else:
5206 fcurinode = curinode;
5207 curinode = curinode + 1;
5208 curfid = curfid + 1;
5209 if(ftype==2):
5210 flinkname = os.readlink(fname);
5211 fdev = fstatinfo.st_dev;
5212 getfdev = GetDevMajorMinor(fdev);
5213 fdev_minor = getfdev[0];
5214 fdev_major = getfdev[1];
5215 frdev = fstatinfo.st_dev;
5216 if(hasattr(fstatinfo, "st_rdev")):
5217 frdev = fstatinfo.st_rdev;
5218 else:
5219 frdev = fstatinfo.st_dev;
5220 getfrdev = GetDevMajorMinor(frdev);
5221 frdev_minor = getfrdev[0];
5222 frdev_major = getfrdev[1];
5223 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
5224 fsize = "0";
5225 if(ftype==0 or ftype==7):
5226 fsize = fstatinfo.st_size;
5227 fatime = fstatinfo.st_atime;
5228 fmtime = fstatinfo.st_mtime;
5229 fctime = fstatinfo.st_ctime;
5230 if(hasattr(fstatinfo, "st_birthtime")):
5231 fbtime = fstatinfo.st_birthtime;
5232 else:
5233 fbtime = fstatinfo.st_ctime;
5234 fmode = fstatinfo.st_mode;
5235 fchmode = stat.S_IMODE(fstatinfo.st_mode);
5236 ftypemod = stat.S_IFMT(fstatinfo.st_mode);
5237 fuid = fstatinfo.st_uid;
5238 fgid = fstatinfo.st_gid;
5239 funame = "";
5240 try:
5241 import pwd;
5242 try:
5243 userinfo = pwd.getpwuid(fstatinfo.st_uid);
5244 funame = userinfo.pw_name;
5245 except KeyError:
5246 funame = "";
5247 except ImportError:
5248 funame = "";
5249 fgname = "";
5250 try:
5251 import grp;
5252 try:
5253 groupinfo = grp.getgrgid(fstatinfo.st_gid);
5254 fgname = groupinfo.gr_name;
5255 except KeyError:
5256 fgname = "";
5257 except ImportError:
5258 fgname = "";
5259 fdev_minor = fdev_minor;
5260 fdev_major = fdev_major;
5261 frdev_minor = frdev_minor;
5262 frdev_major = frdev_major;
5263 flinkcount = flinkcount;
5264 if(hasattr(fstatinfo, "st_file_attributes")):
5265 fwinattributes = fstatinfo.st_file_attributes;
5266 else:
5267 fwinattributes = 0;
5268 fcompression = "";
5269 fcsize = 0;
5270 fcontents = BytesIO();
5271 if(ftype==0 or ftype==7):
5272 with open(fname, "rb") as fpc:
5273 shutil.copyfileobj(fpc, fcontents);
5274 if(followlink and (ftype==1 or ftype==2)):
5275 flstatinfo = os.stat(flinkname);
5276 with open(flinkname, "rb") as fpc:
5277 shutil.copyfileobj(fpc, fcontents);
5278 fcontents.seek(0, 0);
5279 ftypehex = format(ftype, 'x').lower();
5280 extrafields = len(extradata);
5281 extrafieldslist = extradata;
5282 catfextrafields = extrafields;
5283 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5284 if(len(extradata)>0):
5285 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5286 extrasizelen = len(extrasizestr);
5287 extrasizelenhex = format(extrasizelen, 'x').lower();
5288 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5289 catoutlen = len(catoutlist) + len(extradata) + 3;
5290 catoutlenhex = format(catoutlen, 'x').lower();
5291 catoutlist.insert(0, catoutlenhex);
5292 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5293 catheaderdata = catoutlist;
5294 if(len(extradata)>0):
5295 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5296 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
5297 catfnumfields = catoutlen;
5298 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5299 fcontents.seek(0, 0);
5300 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5301 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5302 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5303 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5304 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5305 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5306 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5307 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5308 catfcontentstart = fheadtell;
5309 fheadtell += len(catfileoutstr) + 1;
5310 catfcontentend = fheadtell - 1;
5311 catfhend = catfcontentend;
5312 fcontents.seek(0, 0);
5313 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5314 pyhascontents = False;
5315 if(int(fsize)>0 and not listonly):
5316 pyhascontents = True;
5317 if(int(fsize)>0 and listonly):
5318 fcontents = BytesIO();
5319 pyhascontents = False;
5320 fcontents.seek(0, 0);
5321 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5322 fileidnum = fileidnum + 1;
5323 return catlist;
5324
5325 def TarFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5326 formatspecs = FormatSpecsListToDict(formatspecs);
5327 curinode = 0;
5328 curfid = 0;
5329 inodelist = [];
5330 inodetofile = {};
5331 filetoinode = {};
5332 inodetocatinode = {};
5333 fileidnum = 0;
5334 if(infile=="-"):
5335 infile = BytesIO();
5336 if(hasattr(sys.stdin, "buffer")):
5337 shutil.copyfileobj(sys.stdin.buffer, infile);
5338 else:
5339 shutil.copyfileobj(sys.stdin, infile);
5340 infile.seek(0, 0);
5341 if(not infile):
5342 return False;
5343 infile.seek(0, 0);
5344 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
5345 infile = download_file_from_internet_file(infile);
5346 infile.seek(0, 0);
5347 if(not infile):
5348 return False;
5349 infile.seek(0, 0);
5350 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
5351 return False;
5352 elif(os.path.exists(infile) and os.path.isfile(infile)):
5353 try:
5354 if(not tarfile.TarFileCheck(infile)):
5355 return False;
5356 except AttributeError:
5357 if(not TarFileCheck(infile)):
5358 return False;
5359 else:
5360 return False;
5361 try:
5362 if(hasattr(infile, "read") or hasattr(infile, "write")):
5363 tarfp = tarfile.open(fileobj=infile, mode="r");
5364 else:
5365 tarfp = tarfile.open(infile, "r");
5366 except FileNotFoundError:
5367 return False;
5368 fnumfiles = int(len(tarfp.getmembers()));
5369 catver = formatspecs['format_ver'];
5370 fileheaderver = str(int(catver.replace(".", "")));
5371 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5372 fnumfileshex = format(int(fnumfiles), 'x').lower();
5373 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5374 catversion = re.findall(r"([\d]+)", fileheader);
5375 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5376 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5377 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5378 fheadtell = len(fileheader);
5379 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5380 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
5381 catfhstart = fheadtell;
5382 if(re.findall("^[.|/]", member.name)):
5383 fname = member.name;
5384 else:
5385 fname = "./"+member.name;
5386 if(verbose):
5387 VerbosePrintOut(fname);
5388 fpremode = member.mode;
5389 ffullmode = member.mode;
5390 flinkcount = 0;
5391 ftype = 0;
5392 if(member.isreg()):
5393 ffullmode = member.mode + stat.S_IFREG;
5394 ftype = 0;
5395 elif(member.isdev()):
5396 ffullmode = member.mode;
5397 ftype = 7;
5398 elif(member.islnk()):
5399 ffullmode = member.mode + stat.S_IFREG;
5400 ftype = 1;
5401 elif(member.issym()):
5402 ffullmode = member.mode + stat.S_IFLNK;
5403 ftype = 2;
5404 elif(member.ischr()):
5405 ffullmode = member.mode + stat.S_IFCHR;
5406 ftype = 3;
5407 elif(member.isblk()):
5408 ffullmode = member.mode + stat.S_IFBLK;
5409 ftype = 4;
5410 elif(member.isdir()):
5411 ffullmode = member.mode + stat.S_IFDIR;
5412 ftype = 5;
5413 elif(member.isfifo()):
5414 ffullmode = member.mode + stat.S_IFIFO;
5415 ftype = 6;
5416 elif(member.issparse()):
5417 ffullmode = member.mode;
5418 ftype = 12;
5419 else:
5420 ffullmode = member.mode;
5421 ftype = 0;
5422 flinkname = "";
5423 fbasedir = os.path.dirname(fname);
5424 fcurfid = curfid;
5425 fcurinode = curfid;
5426 finode = fcurinode;
5427 curfid = curfid + 1;
5428 if(ftype==2):
5429 flinkname = member.linkname;
5430 fdev_minor = member.devminor;
5431 fdev_major = member.devmajor;
5432 frdev_minor = member.devminor;
5433 frdev_major = member.devmajor;
5434 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
5435 fsize = "0";
5436 elif(ftype==0 or ftype==7):
5437 fsize = member.size;
5438 else:
5439 fsize = member.size;
5440 fatime = member.mtime;
5441 fmtime = member.mtime;
5442 fctime = member.mtime;
5443 fbtime = member.mtime;
5444 fmode = ffullmode;
5445 fchmode = stat.S_IMODE(ffullmode);
5446 ftypemod = stat.S_IFMT(ffullmode);
5447 fuid = member.uid;
5448 fgid = member.gid;
5449 funame = member.uname;
5450 fgname = member.gname;
5451 flinkcount = flinkcount;
5452 fwinattributes = int(0);
5453 fcompression = "";
5454 fcsize = 0;
5455 fcontents = BytesIO();
5456 if(ftype==0 or ftype==7):
5457 with tarfp.extractfile(member) as fpc:
5458 shutil.copyfileobj(fpc, fcontents);
5459 fcontents.seek(0, 0);
5460 ftypehex = format(ftype, 'x').lower();
5461 extrafields = len(extradata);
5462 extrafieldslist = extradata;
5463 catfextrafields = extrafields;
5464 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5465 if(len(extradata)>0):
5466 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5467 extrasizelen = len(extrasizestr);
5468 extrasizelenhex = format(extrasizelen, 'x').lower();
5469 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5470 catoutlen = len(catoutlist) + len(extradata) + 3;
5471 catoutlenhex = format(catoutlen, 'x').lower();
5472 catoutlist.insert(0, catoutlenhex);
5473 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5474 catheaderdata = catoutlist;
5475 if(len(extradata)>0):
5476 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5477 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
5478 catfnumfields = catoutlen;
5479 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5480 fcontents.seek(0, 0);
5481 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5482 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5483 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5484 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5485 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5486 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5487 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5488 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5489 catfcontentstart = fheadtell;
5490 fheadtell += len(catfileoutstr) + 1;
5491 catfcontentend = fheadtell - 1;
5492 catfhend = catfcontentend;
5493 fcontents.seek(0, 0);
5494 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5495 pyhascontents = False;
5496 if(int(fsize)>0 and not listonly):
5497 pyhascontents = True;
5498 if(int(fsize)>0 and listonly):
5499 fcontents = BytesIO();
5500 pyhascontents = False;
5501 fcontents.seek(0, 0);
5502 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5503 fileidnum = fileidnum + 1;
5504 return catlist;
5505
5506 def ZipFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5507 formatspecs = FormatSpecsListToDict(formatspecs);
5508 curinode = 0;
5509 curfid = 0;
5510 inodelist = [];
5511 inodetofile = {};
5512 filetoinode = {};
5513 inodetocatinode = {};
5514 fileidnum = 0;
5515 if(infile=="-"):
5516 infile = BytesIO();
5517 if(hasattr(sys.stdin, "buffer")):
5518 shutil.copyfileobj(sys.stdin.buffer, infile);
5519 else:
5520 shutil.copyfileobj(sys.stdin, infile);
5521 infile.seek(0, 0);
5522 if(not infile):
5523 return False;
5524 infile.seek(0, 0);
5525 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
5526 infile = download_file_from_internet_file(infile);
5527 infile.seek(0, 0);
5528 if(not infile):
5529 return False;
5530 infile.seek(0, 0);
5531 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
5532 return False;
5533 else:
5534 return False;
5535 if(not zipfile.is_zipfile(infile)):
5536 return False;
5537 try:
5538 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
5539 except FileNotFoundError:
5540 return False;
5541 ziptest = zipfp.testzip();
5542 if(ziptest):
5543 VerbosePrintOut("Bad file found!");
5544 fnumfiles = int(len(zipfp.infolist()));
5545 catver = formatspecs['format_ver'];
5546 fileheaderver = str(int(catver.replace(".", "")));
5547 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5548 catversion = re.findall(r"([\d]+)", fileheader);
5549 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5550 fnumfileshex = format(int(fnumfiles), 'x').lower();
5551 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5552 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5553 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5554 fheadtell = len(fileheader);
5555 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5556 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
5557 catfhstart = fheadtell;
5558 if(re.findall("^[.|/]", member.filename)):
5559 fname = member.filename;
5560 else:
5561 fname = "./"+member.filename;
5562 zipinfo = zipfp.getinfo(member.filename);
5563 if(verbose):
5564 VerbosePrintOut(fname);
5565 if(not member.is_dir()):
5566 fpremode = stat.S_IFREG + 438;
5567 elif(member.is_dir()):
5568 fpremode = stat.S_IFDIR + 511;
5569 flinkcount = 0;
5570 ftype = 0;
5571 if(not member.is_dir()):
5572 ftype = 0;
5573 elif(member.is_dir()):
5574 ftype = 5;
5575 flinkname = "";
5576 fbasedir = os.path.dirname(fname);
5577 fcurfid = curfid;
5578 fcurinode = curfid;
5579 finode = fcurinode;
5580 curfid = curfid + 1;
5581 fdev_minor = 0;
5582 fdev_major = 0;
5583 frdev_minor = 0;
5584 frdev_major = 0;
5585 if(ftype==5):
5586 fsize = "0";
5587 elif(ftype==0):
5588 fsize = member.file_size;
5589 else:
5590 fsize = member.file_size;
5591 fatime = time.mktime(member.date_time + (0, 0, -1));
5592 fmtime = time.mktime(member.date_time + (0, 0, -1));
5593 fctime = time.mktime(member.date_time + (0, 0, -1));
5594 fbtime = time.mktime(member.date_time + (0, 0, -1));
5595 if(zipinfo.create_system==0 or zipinfo.create_system==10):
5596 fwinattributes = int(zipinfo.external_attr);
5597 if(not member.is_dir()):
5598 fmode = int(stat.S_IFREG + 438);
5599 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
5600 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
5601 elif(member.is_dir()):
5602 fmode = int(stat.S_IFDIR + 511);
5603 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
5604 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
5605 elif(zipinfo.create_system==3):
5606 fwinattributes = int(0);
5607 fmode = int(zipinfo.external_attr);
5608 else:
5609 fwinattributes = int(0);
5610 if(not member.is_dir()):
5611 fmode = int(stat.S_IFREG + 438);
5612 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
5613 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
5614 elif(member.is_dir()):
5615 fmode = int(stat.S_IFDIR + 511);
5616 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
5617 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
5618 fcompression = "";
5619 fcsize = 0;
5620 try:
5621 fuid = os.getuid();
5622 except AttributeError:
5623 fuid = 0;
5624 except KeyError:
5625 fuid = 0;
5626 try:
5627 fgid = os.getgid();
5628 except AttributeError:
5629 fgid = 0;
5630 except KeyError:
5631 fgid = 0;
5632 try:
5633 import pwd;
5634 try:
5635 userinfo = pwd.getpwuid(os.getuid());
5636 funame = userinfo.pw_name;
5637 except KeyError:
5638 funame = "";
5639 except AttributeError:
5640 funame = "";
5641 except ImportError:
5642 funame = "";
5643 fgname = "";
5644 try:
5645 import grp;
5646 try:
5647 groupinfo = grp.getgrgid(os.getgid());
5648 fgname = groupinfo.gr_name;
5649 except KeyError:
5650 fgname = "";
5651 except AttributeError:
5652 fgname = "";
5653 except ImportError:
5654 fgname = "";
5655 fcontents = BytesIO();
5656 if(ftype==0):
5657 fcontents.write(zipfp.read(member.filename));
5658 fcontents.seek(0, 0);
5659 ftypehex = format(ftype, 'x').lower();
5660 extrafields = len(extradata);
5661 extrafieldslist = extradata;
5662 catfextrafields = extrafields;
5663 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5664 if(len(extradata)>0):
5665 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5666 extrasizelen = len(extrasizestr);
5667 extrasizelenhex = format(extrasizelen, 'x').lower();
5668 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5669 catoutlen = len(catoutlist) + len(extradata) + 3;
5670 catoutlenhex = format(catoutlen, 'x').lower();
5671 catoutlist.insert(0, catoutlenhex);
5672 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5673 catheaderdata = catoutlist;
5674 if(len(extradata)>0):
5675 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5676 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
5677 catfnumfields = catoutlen;
5678 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5679 fcontents.seek(0, 0);
5680 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5681 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5682 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5683 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5684 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5685 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5686 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5687 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5688 catfcontentstart = fheadtell;
5689 fheadtell += len(catfileoutstr) + 1;
5690 catfcontentend = fheadtell - 1;
5691 catfhend = catfcontentend;
5692 fcontents.seek(0, 0);
5693 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5694 pyhascontents = False;
5695 if(int(fsize)>0 and not listonly):
5696 pyhascontents = True;
5697 if(int(fsize)>0 and listonly):
5698 fcontents = BytesIO();
5699 pyhascontents = False;
5700 fcontents.seek(0, 0);
5701 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5702 fileidnum = fileidnum + 1;
5703 return catlist;
5704
5705 if(not rarfile_support):
5706 def RarFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5707 return False;
5708
5709 if(rarfile_support):
5710 def RarFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5711 formatspecs = FormatSpecsListToDict(formatspecs);
5712 curinode = 0;
5713 curfid = 0;
5714 inodelist = [];
5715 inodetofile = {};
5716 filetoinode = {};
5717 inodetocatinode = {};
5718 fileidnum = 0;
5719 if(not os.path.exists(infile,) or not os.path.isfile(infile,)):
5720 return False;
5721 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
5722 return False;
5723 rarfp = rarfile.RarFile(infile, "r");
5724 rartest = rarfp.testrar();
5725 if(rartest):
5726 VerbosePrintOut("Bad file found!");
5727 fnumfiles = int(len(rarfp.infolist()));
5728 catver = formatspecs['format_ver'];
5729 fileheaderver = str(int(catver.replace(".", "")));
5730 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5731 catversion = re.findall(r"([\d]+)", fileheader);
5732 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5733 fnumfileshex = format(int(fnumfiles), 'x').lower();
5734 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5735 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5736 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5737 fheadtell = len(fileheader);
5738 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5739 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
5740 catfhstart = fheadtell;
5741 is_unix = False;
5742 is_windows = False;
5743 if(member.host_os==rarfile.RAR_OS_UNIX):
5744 is_windows = False;
5745 try:
5746 member.external_attr
5747 is_unix = True;
5748 except AttributeError:
5749 is_unix = False;
5750 elif(member.host_os==rarfile.RAR_OS_WIN32):
5751 is_unix = False;
5752 try:
5753 member.external_attr
5754 is_windows = True;
5755 except AttributeError:
5756 is_windows = False;
5757 else:
5758 is_unix = False;
5759 is_windows = False;
5760 if(re.findall("^[.|/]", member.filename)):
5761 fname = member.filename;
5762 else:
5763 fname = "./"+member.filename;
5764 rarinfo = rarfp.getinfo(member.filename);
5765 if(verbose):
5766 VerbosePrintOut(fname);
5767 if(is_unix and member.external_attr !=0):
5768 fpremode = int(member.external_attr);
5769 elif(member.is_file()):
5770 fpremode = stat.S_IFREG + 438;
5771 elif(member.is_symlink()):
5772 fpremode = stat.S_IFLNK + 438;
5773 elif(member.is_dir()):
5774 fpremode = stat.S_IFDIR + 511;
5775 if(is_windows and member.external_attr !=0):
5776 fwinattributes = int(member.external_attr);
5777 else:
5778 fwinattributes = int(0);
5779 fcompression = "";
5780 fcsize = 0;
5781 flinkcount = 0;
5782 ftype = 0;
5783 if(member.is_file()):
5784 ftype = 0;
5785 elif(member.is_symlink()):
5786 ftype = 2;
5787 elif(member.is_dir()):
5788 ftype = 5;
5789 flinkname = "";
5790 if(ftype==2):
5791 flinkname = rarfp.read(member.filename).decode("UTF-8");
5792 fbasedir = os.path.dirname(fname);
5793 fcurfid = curfid;
5794 fcurinode = curfid;
5795 finode = fcurinode;
5796 curfid = curfid + 1;
5797 fdev_minor = 0;
5798 fdev_major = 0;
5799 frdev_minor = 0;
5800 frdev_major = 0;
5801 if(ftype==5):
5802 fsize = "0";
5803 if(ftype==0):
5804 fsize = member.file_size;
5805 try:
5806 if(member.atime):
5807 fatime = int(member.atime.timestamp());
5808 else:
5809 fatime = int(member.mtime.timestamp());
5810 except AttributeError:
5811 fatime = int(member.mtime.timestamp());
5812 fmtime = int(member.mtime.timestamp());
5813 try:
5814 if(member.ctime):
5815 fctime = int(member.ctime.timestamp());
5816 else:
5817 fctime = int(member.mtime.timestamp());
5818 except AttributeError:
5819 fctime = int(member.mtime.timestamp());
5820 fbtime = int(member.mtime.timestamp());
5821 if(is_unix and member.external_attr !=0):
5822 fmode = int(member.external_attr);
5823 fchmode = int(stat.S_IMODE(member.external_attr));
5824 ftypemod = int(stat.S_IFMT(member.external_attr));
5825 elif(member.is_file()):
5826 fmode = int(stat.S_IFREG + 438)
5827 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5828 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5829 elif(member.is_symlink()):
5830 fmode = int(stat.S_IFLNK + 438)
5831 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5832 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5833 elif(member.is_dir()):
5834 fmode = int(stat.S_IFDIR + 511)
5835 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
5836 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
5837 try:
5838 fuid = os.getuid();
5839 except AttributeError:
5840 fuid = 0;
5841 except KeyError:
5842 fuid = 0;
5843 try:
5844 fgid = os.getgid();
5845 except AttributeError:
5846 fgid = 0;
5847 except KeyError:
5848 fgid = 0;
5849 try:
5850 import pwd;
5851 try:
5852 userinfo = pwd.getpwuid(os.getuid());
5853 funame = userinfo.pw_name;
5854 except KeyError:
5855 funame = "";
5856 except AttributeError:
5857 funame = "";
5858 except ImportError:
5859 funame = "";
5860 fgname = "";
5861 try:
5862 import grp;
5863 try:
5864 groupinfo = grp.getgrgid(os.getgid());
5865 fgname = groupinfo.gr_name;
5866 except KeyError:
5867 fgname = "";
5868 except AttributeError:
5869 fgname = "";
5870 except ImportError:
5871 fgname = "";
5872 fcontents = BytesIO();
5873 if(ftype==0):
5874 fcontents.write(rarfp.read(member.filename));
5875 fcontents.seek(0, 0);
5876 ftypehex = format(ftype, 'x').lower();
5877 extrafields = len(extradata);
5878 extrafieldslist = extradata;
5879 catfextrafields = extrafields;
5880 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5881 if(len(extradata)>0):
5882 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5883 extrasizelen = len(extrasizestr);
5884 extrasizelenhex = format(extrasizelen, 'x').lower();
5885 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5886 catoutlen = len(catoutlist) + len(extradata) + 3;
5887 catoutlenhex = format(catoutlen, 'x').lower();
5888 catoutlist.insert(0, catoutlenhex);
5889 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5890 if(len(extradata)>0):
5891 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5892 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
5893 catfnumfields = 24 + catfextrafields;
5894 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5895 fcontents.seek(0, 0);
5896 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5897 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5898 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5899 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5900 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5901 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5902 catheaderdata = catoutlist;
5903 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5904 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5905 catfcontentstart = fheadtell;
5906 fheadtell += len(catfileoutstr) + 1;
5907 catfcontentend = fheadtell - 1;
5908 catfhend = catfcontentend;
5909 fcontents.seek(0, 0);
5910 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5911 pyhascontents = False;
5912 if(int(fsize)>0 and not listonly):
5913 pyhascontents = True;
5914 if(int(fsize)>0 and listonly):
5915 fcontents = BytesIO();
5916 pyhascontents = False;
5917 fcontents.seek(0, 0);
5918 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5919 fileidnum = fileidnum + 1;
5920 return catlist;
5921
5922 if(not py7zr_support):
5923 def SevenZipFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5924 return False;
5925
5926 if(py7zr_support):
5927 def SevenZipFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5928 formatspecs = FormatSpecsListToDict(formatspecs);
5929 curinode = 0;
5930 curfid = 0;
5931 inodelist = [];
5932 inodetofile = {};
5933 filetoinode = {};
5934 inodetocatinode = {};
5935 fileidnum = 0;
5936 szpfp = py7zr.SevenZipFile(infile, mode="r");
5937 file_content = szpfp.readall();
5938 #sztest = szpfp.testzip();
5939 sztestalt = szpfp.test();
5940 if(sztestalt):
5941 VerbosePrintOut("Bad file found!");
5942 numfiles = int(len(szpfp.list()));
5943 catver = formatspecs['format_ver'];
5944 fileheaderver = str(int(catver.replace(".", "")));
5945 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5946 catversion = re.findall(r"([\d]+)", fileheader);
5947 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5948 fnumfileshex = format(int(fnumfiles), 'x').lower();
5949 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5950 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5951 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5952 fheadtell = len(fileheader);
5953 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5954 for member in sorted(szpfp.list(), key=lambda x: x.filename):
5955 catfhstart = fheadtell;
5956 if(re.findall("^[.|/]", member.filename)):
5957 fname = member.filename;
5958 else:
5959 fname = "./"+member.filename;
5960 if(not member.is_directory):
5961 fpremode = int(stat.S_IFREG + 438);
5962 elif(member.is_directory):
5963 fpremode = int(stat.S_IFDIR + 511);
5964 fwinattributes = int(0);
5965 fcompression = "";
5966 fcsize = 0;
5967 flinkcount = 0;
5968 ftype = 0;
5969 if(member.is_directory):
5970 ftype = 5;
5971 else:
5972 ftype = 0;
5973 flinkname = "";
5974 fbasedir = os.path.dirname(fname);
5975 fcurfid = curfid;
5976 fcurinode = curfid;
5977 finode = fcurinode;
5978 curfid = curfid + 1;
5979 fdev_minor = 0;
5980 fdev_major = 0;
5981 frdev_minor = 0;
5982 frdev_major = 0;
5983 if(ftype==5):
5984 fsize = "0";
5985 fatime = int(member.creationtime.timestamp());
5986 fmtime = int(member.creationtime.timestamp());
5987 fctime = int(member.creationtime.timestamp());
5988 fbtime = int(member.creationtime.timestamp());
5989 if(member.is_directory):
5990 fmode = int(stat.S_IFDIR + 511)
5991 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
5992 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
5993 else:
5994 fmode = int(stat.S_IFLNK + 438)
5995 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5996 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5997 try:
5998 fuid = os.getuid();
5999 except AttributeError:
6000 fuid = 0;
6001 except KeyError:
6002 fuid = 0;
6003 try:
6004 fgid = os.getgid();
6005 except AttributeError:
6006 fgid = 0;
6007 except KeyError:
6008 fgid = 0;
6009 try:
6010 import pwd;
6011 try:
6012 userinfo = pwd.getpwuid(os.getuid());
6013 funame = userinfo.pw_name;
6014 except KeyError:
6015 funame = "";
6016 except AttributeError:
6017 funame = "";
6018 except ImportError:
6019 funame = "";
6020 fgname = "";
6021 try:
6022 import grp;
6023 try:
6024 groupinfo = grp.getgrgid(os.getgid());
6025 fgname = groupinfo.gr_name;
6026 except KeyError:
6027 fgname = "";
6028 except AttributeError:
6029 fgname = "";
6030 except ImportError:
6031 fgname = "";
6032 fcontents = BytesIO();
6033 if(ftype==0):
6034 fcontents.write(file_content[member.filename].read());
6035 fsize = format(fcontents.tell(), 'x').lower();
6036 fileop.close();
6037 fcontents.seek(0, 0);
6038 ftypehex = format(ftype, 'x').lower();
6039 extrafields = len(extradata);
6040 extrafieldslist = extradata;
6041 catfextrafields = extrafields;
6042 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
6043 if(len(extradata)>0):
6044 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
6045 extrasizelen = len(extrasizestr);
6046 extrasizelenhex = format(extrasizelen, 'x').lower();
6047 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
6048 catoutlen = len(catoutlist) + len(extradata) + 3;
6049 catoutlenhex = format(catoutlen, 'x').lower();
6050 catoutlist.insert(0, catoutlenhex);
6051 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
6052 catheaderdata = catoutlist;
6053 if(len(extradata)>0):
6054 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
6055 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
6056 catfnumfields = 24 + catfextrafields;
6057 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
6058 fcontents.seek(0, 0);
6059 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
6060 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
6061 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
6062 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
6063 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
6064 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
6065 catfileoutstrecd = catfileoutstr.encode('UTF-8');
6066 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
6067 catfcontentstart = fheadtell;
6068 fheadtell += len(catfileoutstr) + 1;
6069 catfcontentend = fheadtell - 1;
6070 catfhend = catfcontentend;
6071 fcontents.seek(0, 0);
6072 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
6073 pyhascontents = False;
6074 if(int(fsize)>0 and not listonly):
6075 pyhascontents = True;
6076 if(int(fsize)>0 and listonly):
6077 fcontents = BytesIO();
6078 pyhascontents = False;
6079 fcontents.seek(0, 0);
6080 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
6081 fileidnum = fileidnum + 1;
6082 return catlist;
6083
6084 def InFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6085 formatspecs = FormatSpecsListToDict(formatspecs);
6086 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
6087 if(checkcompressfile=="tarfile"):
6088 return TarFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6089 elif(checkcompressfile=="zipfile"):
6090 return ZipFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6091 elif(checkcompressfile=="catfile"):
6092 return ArchiveFileToArray(infile, 0, 0, listonly, True, False, formatspecs, False);
6093 elif(rarfile_support and checkcompressfile=="rarfile"):
6094 return RarFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6095 elif(py7zr_support and checkcompressfile=="7zipfile"):
6096 return SevenZipFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6097 else:
6098 return False;
6099 return False;
6100
6101 def ListDirToArray(infiles, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, listonly=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6102 formatspecs = FormatSpecsListToDict(formatspecs);
6103 outarray = BytesIO();
6104 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
6105 listcatfiles = ArchiveFileToArray(outarray, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
6106 return listcatfiles;
6107
6108 def ArchiveFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6109 formatspecs = FormatSpecsListToDict(formatspecs);
6110 if(isinstance(infile, dict)):
6111 listcatfiles = infile;
6112 else:
6113 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6114 infile = RemoveWindowsPath(infile);
6115 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, listonly, uncompress, skipchecksum, formatspecs, returnfp);
6116 if(not listcatfiles):
6117 return False;
6118 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6119 if(returnfp):
6120 catarray.update({'catfp': listcatfiles['catfp']});
6121 lenlist = len(listcatfiles['ffilelist']);
6122 lcfi = 0;
6123 lcfx = int(listcatfiles['fnumfiles']);
6124 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6125 lcfx = int(lenlist);
6126 else:
6127 lcfx = int(listcatfiles['fnumfiles']);
6128 while(lcfi < lcfx):
6129 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6130 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6131 catarray['filetoid'].update(filetoidarray);
6132 catarray['idtofile'].update(idtofilearray);
6133 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6134 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6135 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6136 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6137 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6138 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6139 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6140 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6141 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6142 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6143 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6144 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6145 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6146 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6147 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6148 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6149 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6150 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6151 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6152 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6153 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6154 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6155 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6156 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6157 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6158 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6159 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6160 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6161 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6162 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6163 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6164 lcfi = lcfi + 1;
6165 return catarray;
6166
6167 create_alias_function("", __file_format_name__, "ToArrayIndex", ArchiveFileToArrayIndex);
6168
6169 def ListDirToArrayIndexAlt(infiles, dirlistfromtxt=False, followlink=False, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6170 formatspecs = FormatSpecsListToDict(formatspecs);
6171 listcatfiles = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, listonly, checksumtype, extradata, formatspecs, verbose);
6172 if(not listcatfiles):
6173 return False;
6174 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6175 lenlist = len(listcatfiles['ffilelist']);
6176 lcfi = 0;
6177 lcfx = int(listcatfiles['fnumfiles']);
6178 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6179 lcfx = int(lenlist);
6180 else:
6181 lcfx = int(listcatfiles['fnumfiles']);
6182 while(lcfi < lcfx):
6183 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6184 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6185 catarray['filetoid'].update(filetoidarray);
6186 catarray['idtofile'].update(idtofilearray);
6187 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6188 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6189 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6190 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6191 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6192 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6193 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6194 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6195 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6196 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6197 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6198 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6199 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6200 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6201 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6202 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6203 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6204 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6205 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6206 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6207 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6208 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6209 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6210 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6211 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6212 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6213 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6214 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6215 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6216 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6217 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6218 lcfi = lcfi + 1;
6219 return catarray;
6220
6221 def TarFileToArrayIndexAlt(infile, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6222 formatspecs = FormatSpecsListToDict(formatspecs);
6223 listcatfiles = TarFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6224 if(not listcatfiles):
6225 return False;
6226 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6227 lenlist = len(listcatfiles['ffilelist']);
6228 lcfi = 0;
6229 lcfx = int(listcatfiles['fnumfiles']);
6230 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6231 lcfx = int(lenlist);
6232 else:
6233 lcfx = int(listcatfiles['fnumfiles']);
6234 while(lcfi < lcfx):
6235 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6236 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6237 catarray['filetoid'].update(filetoidarray);
6238 catarray['idtofile'].update(idtofilearray);
6239 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6240 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6241 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6242 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6243 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6244 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6245 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6246 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6247 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6248 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6249 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6250 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6251 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6252 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6253 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6254 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6255 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6256 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6257 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6258 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6259 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6260 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6261 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6262 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6263 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6264 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6265 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6266 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6267 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6268 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6269 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6270 lcfi = lcfi + 1;
6271 return catarray;
6272
6273 def ZipFileToArrayIndexAlt(infile, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6274 formatspecs = FormatSpecsListToDict(formatspecs);
6275 listcatfiles = ZipFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6276 if(not listcatfiles):
6277 return False;
6278 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6279 lenlist = len(listcatfiles['ffilelist']);
6280 lcfi = 0;
6281 lcfx = int(listcatfiles['fnumfiles']);
6282 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6283 lcfx = int(lenlist);
6284 else:
6285 lcfx = int(listcatfiles['fnumfiles']);
6286 while(lcfi < lcfx):
6287 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6288 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6289 catarray['filetoid'].update(filetoidarray);
6290 catarray['idtofile'].update(idtofilearray);
6291 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6292 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6293 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6294 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6295 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6296 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6297 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6298 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6299 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6300 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6301 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6302 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6303 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6304 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6305 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6306 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6307 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6308 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6309 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6310 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6311 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6312 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6313 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6314 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6315 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6316 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6317 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6318 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6319 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6320 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6321 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6322 lcfi = lcfi + 1;
6323 return catarray;
6324
6325 if(not rarfile_support):
6326 def RarFileToArrayIndexAlt(infile, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6327 return False;
6328
6329 if(rarfile_support):
6330 def RarFileToArrayIndexAlt(infile, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6331 formatspecs = FormatSpecsListToDict(formatspecs);
6332 listcatfiles = RarFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6333 if(not listcatfiles):
6334 return False;
6335 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6336 lenlist = len(listcatfiles['ffilelist']);
6337 lcfi = 0;
6338 lcfx = int(listcatfiles['fnumfiles']);
6339 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6340 lcfx = int(lenlist);
6341 else:
6342 lcfx = int(listcatfiles['fnumfiles']);
6343 while(lcfi < lcfx):
6344 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6345 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6346 catarray['filetoid'].update(filetoidarray);
6347 catarray['idtofile'].update(idtofilearray);
6348 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6349 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6350 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6351 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6352 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6353 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6354 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6355 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6356 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6357 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6358 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6359 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6360 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6361 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6362 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6363 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6364 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6365 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6366 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6367 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6368 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6369 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6370 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6371 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6372 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6373 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6374 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6375 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6376 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6377 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6378 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6379 lcfi = lcfi + 1;
6380 return catarray;
6381
6382 if(not py7zr_support):
6383 def SevenZipFileToArrayIndexAlt(infile, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6384 return False;
6385
6386 if(py7zr_support):
6387 def SevenZipFileToArrayIndexAlt(infile, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6388 formatspecs = FormatSpecsListToDict(formatspecs);
6389 listcatfiles = SevenZipFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6390 if(not listcatfiles):
6391 return False;
6392 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6393 lenlist = len(listcatfiles['ffilelist']);
6394 lcfi = 0;
6395 lcfx = int(listcatfiles['fnumfiles']);
6396 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6397 lcfx = int(lenlist);
6398 else:
6399 lcfx = int(listcatfiles['fnumfiles']);
6400 while(lcfi < lcfx):
6401 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6402 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6403 catarray['filetoid'].update(filetoidarray);
6404 catarray['idtofile'].update(idtofilearray);
6405 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6406 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6407 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6408 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6409 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6410 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6411 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6412 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6413 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6414 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6415 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6416 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6417 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6418 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6419 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6420 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6421 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6422 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6423 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6424 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6425 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6426 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6427 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6428 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6429 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6430 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6431 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6432 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6433 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6434 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6435 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6436 lcfi = lcfi + 1;
6437 return catarray;
6438
6439 def ArchiveFileStringToArrayIndex(catstr, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6440 formatspecs = FormatSpecsListToDict(formatspecs);
6441 catfp = BytesIO(catstr);
6442 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, uncompress, skipchecksum, formatspecs, returnfp);
6443 return listcatfiles;
6444
6445 create_alias_function("", __file_format_name__, "StringToArrayIndex", ArchiveFileStringToArrayIndex);
6446
6447 def TarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6448 formatspecs = FormatSpecsListToDict(formatspecs);
6449 catfp = BytesIO();
6450 catfp = PackArchiveFileFromTarFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
6451 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
6452 return listcatfiles;
6453
6454 def ZipFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6455 formatspecs = FormatSpecsListToDict(formatspecs);
6456 catfp = BytesIO();
6457 catfp = PackArchiveFileFromZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
6458 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
6459 return listcatfiles;
6460
6461 if(not rarfile_support):
6462 def RarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6463 return False;
6464
6465 if(rarfile_support):
6466 def RarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6467 formatspecs = FormatSpecsListToDict(formatspecs);
6468 catfp = BytesIO();
6469 catfp = PackArchiveFileFromRarFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
6470 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
6471 return listcatfiles;
6472
6473 if(not py7zr_support):
6474 def SevenZipFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6475 return False;
6476
6477 if(py7zr_support):
6478 def SevenZipFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6479 formatspecs = FormatSpecsListToDict(formatspecs);
6480 catfp = BytesIO();
6481 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
6482 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
6483 return listcatfiles;
6484
6485 def ListDirToArrayIndex(infiles, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, listonly=False, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6486 formatspecs = FormatSpecsListToDict(formatspecs);
6487 outarray = BytesIO();
6488 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose, True);
6489 listcatfiles = ArchiveFileToArrayIndex(outarray, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp)
6490 return listcatfiles;
6491
6492 def RePackArchiveFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6493 formatspecs = FormatSpecsListToDict(formatspecs);
6494 if(isinstance(infile, dict)):
6495 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, True, skipchecksum, formatspecs, returnfp);
6496 listcatfiles = prelistcatfiles['list'];
6497 else:
6498 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6499 infile = RemoveWindowsPath(infile);
6500 if(followlink):
6501 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, True, skipchecksum, formatspecs, returnfp);
6502 listcatfiles = prelistcatfiles['list'];
6503 else:
6504 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, True, skipchecksum, formatspecs, returnfp);
6505 if(outfile!="-" and not hasattr(infile, "read") and not hasattr(outfile, "write")):
6506 outfile = RemoveWindowsPath(outfile);
6507 checksumtype = checksumtype.lower();
6508 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
6509 checksumtype="crc32";
6510 if(checksumtype=="none"):
6511 checksumtype = "";
6512 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
6513 compression = "auto";
6514 if(compression not in compressionlist and compression is None):
6515 compression = "auto";
6516 if(verbose):
6517 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6518 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
6519 if(os.path.exists(outfile)):
6520 try:
6521 os.unlink(outfile);
6522 except OSError as e:
6523 pass;
6524 if(not listcatfiles):
6525 return False;
6526 if(outfile=="-"):
6527 verbose = False;
6528 catfp = BytesIO();
6529 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
6530 catfp = outfile;
6531 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
6532 catfp = BytesIO();
6533 else:
6534 fbasename = os.path.splitext(outfile)[0];
6535 fextname = os.path.splitext(outfile)[1];
6536 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
6537 catver = formatspecs['format_ver'];
6538 fileheaderver = str(int(catver.replace(".", "")));
6539 lenlist = len(listcatfiles['ffilelist']);
6540 fnumfiles = int(listcatfiles['fnumfiles']);
6541 if(lenlist>fnumfiles or lenlist<fnumfiles):
6542 fnumfiles = lenlist;
6543 AppendFileHeader(catfp, fnumfiles, checksumtype, formatspecs);
6544 lenlist = len(listcatfiles['ffilelist']);
6545 fnumfiles = int(listcatfiles['fnumfiles']);
6546 lcfi = 0;
6547 lcfx = int(listcatfiles['fnumfiles']);
6548 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6549 lcfx = int(lenlist);
6550 else:
6551 lcfx = int(listcatfiles['fnumfiles']);
6552 curinode = 0;
6553 curfid = 0;
6554 inodelist = [];
6555 inodetofile = {};
6556 filetoinode = {};
6557 reallcfi = 0;
6558 while(lcfi < lcfx):
6559 if(re.findall("^[.|/]", listcatfiles['ffilelist'][reallcfi]['fname'])):
6560 fname = listcatfiles['ffilelist'][reallcfi]['fname'];
6561 else:
6562 fname = "./"+listcatfiles['ffilelist'][reallcfi]['fname'];
6563 if(verbose):
6564 VerbosePrintOut(fname);
6565 fheadersize = format(int(listcatfiles['ffilelist'][reallcfi]['fheadersize']), 'x').lower();
6566 fsize = format(int(listcatfiles['ffilelist'][reallcfi]['fsize']), 'x').lower();
6567 flinkname = listcatfiles['ffilelist'][reallcfi]['flinkname'];
6568 fatime = format(int(listcatfiles['ffilelist'][reallcfi]['fatime']), 'x').lower();
6569 fmtime = format(int(listcatfiles['ffilelist'][reallcfi]['fmtime']), 'x').lower();
6570 fctime = format(int(listcatfiles['ffilelist'][reallcfi]['fctime']), 'x').lower();
6571 fbtime = format(int(listcatfiles['ffilelist'][reallcfi]['fbtime']), 'x').lower();
6572 fmode = format(int(listcatfiles['ffilelist'][reallcfi]['fmode']), 'x').lower();
6573 fchmode = format(int(listcatfiles['ffilelist'][reallcfi]['fchmode']), 'x').lower();
6574 fuid = format(int(listcatfiles['ffilelist'][reallcfi]['fuid']), 'x').lower();
6575 funame = listcatfiles['ffilelist'][reallcfi]['funame'];
6576 fgid = format(int(listcatfiles['ffilelist'][reallcfi]['fgid']), 'x').lower();
6577 fgname = listcatfiles['ffilelist'][reallcfi]['fgname'];
6578 finode = format(int(listcatfiles['ffilelist'][reallcfi]['finode']), 'x').lower();
6579 flinkcount = format(int(listcatfiles['ffilelist'][reallcfi]['flinkcount']), 'x').lower();
6580 fwinattributes = format(int(listcatfiles['ffilelist'][reallcfi]['fwinattributes']), 'x').lower();
6581 fcompression = listcatfiles['ffilelist'][reallcfi]['fcompression'];
6582 fcsize = format(int(listcatfiles['ffilelist'][reallcfi]['fcsize']), 'x').lower();
6583 fdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['fminor']), 'x').lower();
6584 fdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['fmajor']), 'x').lower();
6585 frdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['frminor']), 'x').lower();
6586 frdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['frmajor']), 'x').lower();
6587 if(len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>listcatfiles['ffilelist'][reallcfi]['fextrafields'] and len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>0):
6588 listcatfiles['ffilelist'][reallcfi]['fextrafields'] = len(listcatfiles['ffilelist'][reallcfi]['fextralist']);
6589 if(not followlink and len(extradata)<0):
6590 extradata = listcatfiles['ffilelist'][reallcfi]['fextralist'];
6591 fcontents = listcatfiles['ffilelist'][reallcfi]['fcontents'];
6592 fcompression = "";
6593 fcsize = format(int(0), 'x').lower();
6594 if(not compresswholefile):
6595 fcontents.seek(0, 2);
6596 ucfsize = fcontents.tell();
6597 fcontents.seek(0, 0);
6598 if(compression=="auto"):
6599 ilsize = len(compressionlistalt);
6600 ilmin = 0;
6601 ilcsize = [];
6602 while(ilmin < ilsize):
6603 cfcontents = BytesIO();
6604 shutil.copyfileobj(fcontents, cfcontents);
6605 fcontents.seek(0, 0);
6606 cfcontents.seek(0, 0);
6607 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
6608 if(cfcontents):
6609 cfcontents.seek(0, 2);
6610 ilcsize.append(cfcontents.tell());
6611 cfcontents.close();
6612 else:
6613 try:
6614 ilcsize.append(sys.maxint);
6615 except AttributeError:
6616 ilcsize.append(sys.maxsize);
6617 ilmin = ilmin + 1;
6618 ilcmin = ilcsize.index(min(ilcsize));
6619 compression = compressionlistalt[ilcmin];
6620 fcontents.seek(0, 0);
6621 cfcontents = BytesIO();
6622 shutil.copyfileobj(fcontents, cfcontents);
6623 cfcontents.seek(0, 0);
6624 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
6625 cfcontents.seek(0, 2);
6626 cfsize = cfcontents.tell();
6627 if(ucfsize > cfsize):
6628 fcsize = format(int(cfsize), 'x').lower();
6629 fcompression = compression;
6630 fcontents.close();
6631 fcontents = cfcontents;
6632 if(followlink):
6633 if(listcatfiles['ffilelist'][reallcfi]['ftype']==1 or listcatfiles['ffilelist'][reallcfi]['ftype']==2):
6634 getflinkpath = listcatfiles['ffilelist'][reallcfi]['flinkname'];
6635 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6636 flinkinfo = listcatfiles['ffilelist'][flinkid];
6637 fheadersize = format(int(flinkinfo['fheadersize']), 'x').lower();
6638 fsize = format(int(flinkinfo['fsize']), 'x').lower();
6639 flinkname = flinkinfo['flinkname'];
6640 fatime = format(int(flinkinfo['fatime']), 'x').lower();
6641 fmtime = format(int(flinkinfo['fmtime']), 'x').lower();
6642 fctime = format(int(flinkinfo['fctime']), 'x').lower();
6643 fbtime = format(int(flinkinfo['fbtime']), 'x').lower();
6644 fmode = format(int(flinkinfo['fmode']), 'x').lower();
6645 fchmode = format(int(flinkinfo['fchmode']), 'x').lower();
6646 fuid = format(int(flinkinfo['fuid']), 'x').lower();
6647 funame = flinkinfo['funame'];
6648 fgid = format(int(flinkinfo['fgid']), 'x').lower();
6649 fgname = flinkinfo['fgname'];
6650 finode = format(int(flinkinfo['finode']), 'x').lower();
6651 flinkcount = format(int(flinkinfo['flinkcount']), 'x').lower();
6652 fwinattributes = format(int(flinkinfo['fwinattributes']), 'x').lower();
6653 fcompression = flinkinfo['fcompression'];
6654 fcsize = format(int(flinkinfo['fcsize']), 'x').lower();
6655 fdev_minor = format(int(flinkinfo['fminor']), 'x').lower();
6656 fdev_major = format(int(flinkinfo['fmajor']), 'x').lower();
6657 frdev_minor = format(int(flinkinfo['frminor']), 'x').lower();
6658 frdev_major = format(int(flinkinfo['frmajor']), 'x').lower();
6659 if(len(flinkinfo['fextralist'])>flinkinfo['fextrafields'] and len(flinkinfo['fextralist'])>0):
6660 flinkinfo['fextrafields'] = len(flinkinfo['fextralist']);
6661 if(len(extradata)<0):
6662 extradata = flinkinfo['fextralist'];
6663 fcontents = flinkinfo['fcontents'];
6664 ftypehex = format(flinkinfo['ftype'], 'x').lower();
6665 else:
6666 ftypehex = format(listcatfiles['ffilelist'][reallcfi]['ftype'], 'x').lower();
6667 fcurfid = format(curfid, 'x').lower();
6668 if(not followlink and finode!=0):
6669 if(listcatfiles['ffilelist'][reallcfi]['ftype']!=1):
6670 fcurinode = format(int(curinode), 'x').lower();
6671 inodetofile.update({curinode: fname});
6672 filetoinode.update({fname: curinode});
6673 curinode = curinode + 1;
6674 else:
6675 fcurinode = format(int(filetoinode[flinkname]), 'x').lower();
6676 else:
6677 fcurinode = format(int(curinode), 'x').lower();
6678 curinode = curinode + 1;
6679 curfid = curfid + 1;
6680 if(fcompression=="none"):
6681 fcompression = "";
6682 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
6683 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
6684 fcontents.close();
6685 lcfi = lcfi + 1;
6686 reallcfi = reallcfi + 1;
6687 if(lcfx>0):
6688 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
6689 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
6690 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
6691 try:
6692 catfp.flush();
6693 os.fsync(catfp.fileno());
6694 except io.UnsupportedOperation:
6695 pass;
6696 except AttributeError:
6697 pass;
6698 except OSError as e:
6699 pass;
6700 if(outfile=="-"):
6701 catfp.seek(0, 0);
6702 if(hasattr(sys.stdout, "buffer")):
6703 shutil.copyfileobj(catfp, sys.stdout.buffer);
6704 else:
6705 shutil.copyfileobj(catfp, sys.stdout);
6706 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
6707 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
6708 catfp.seek(0, 0);
6709 upload_file_to_internet_file(catfp, outfile);
6710 if(returnfp):
6711 catfp.seek(0, 0);
6712 return catfp;
6713 else:
6714 catfp.close();
6715 return True;
6716
6717 create_alias_function("RePack", __file_format_name__, "", RePackArchiveFile);
6718
6719 def RePackArchiveFileFromString(catstr, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6720 formatspecs = FormatSpecsListToDict(formatspecs);
6721 catfp = BytesIO(catstr);
6722 listcatfiles = RePackArchiveFile(catfp, compression, compresswholefile, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6723 return listcatfiles;
6724
6725 create_alias_function("RePack", __file_format_name__, "FromString", RePackArchiveFileFromString);
6726
6727 def PackArchiveFileFromListDir(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6728 formatspecs = FormatSpecsListToDict(formatspecs);
6729 outarray = BytesIO();
6730 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
6731 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6732 return listcatfiles;
6733
6734 create_alias_function("Pack", __file_format_name__, "FromListDir", PackArchiveFileFromListDir);
6735
6736 def UnPackArchiveFile(infile, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, preservepermissions=True, preservetime=True, verbose=False, returnfp=False):
6737 formatspecs = FormatSpecsListToDict(formatspecs);
6738 if(outdir is not None):
6739 outdir = RemoveWindowsPath(outdir);
6740 if(verbose):
6741 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6742 if(isinstance(infile, dict)):
6743 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, True, skipchecksum, formatspecs, returnfp);
6744 listcatfiles = prelistcatfiles['list'];
6745 else:
6746 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6747 infile = RemoveWindowsPath(infile);
6748 if(followlink):
6749 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, True, skipchecksum, formatspecs, returnfp);
6750 listcatfiles = prelistcatfiles['list'];
6751 else:
6752 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, True, skipchecksum, formatspecs, returnfp);
6753 if(not listcatfiles):
6754 return False;
6755 lenlist = len(listcatfiles['ffilelist']);
6756 fnumfiles = int(listcatfiles['fnumfiles']);
6757 lcfi = 0;
6758 lcfx = int(listcatfiles['fnumfiles']);
6759 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6760 lcfx = int(lenlist);
6761 else:
6762 lcfx = int(listcatfiles['fnumfiles']);
6763 while(lcfi<lcfx):
6764 funame = "";
6765 try:
6766 import pwd;
6767 try:
6768 userinfo = pwd.getpwuid(listcatfiles['ffilelist'][lcfi]['fuid']);
6769 funame = userinfo.pw_name;
6770 except KeyError:
6771 funame = "";
6772 except ImportError:
6773 funame = "";
6774 fgname = "";
6775 try:
6776 import grp;
6777 try:
6778 groupinfo = grp.getgrgid(listcatfiles['ffilelist'][lcfi]['fgid']);
6779 fgname = groupinfo.gr_name;
6780 except KeyError:
6781 fgname = "";
6782 except ImportError:
6783 fgname = "";
6784 if(verbose):
6785 VerbosePrintOut(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6786 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6787 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6788 listcatfiles['ffilelist'][lcfi]['fcontents'].seek(0, 0);
6789 shutil.copyfileobj(listcatfiles['ffilelist'][lcfi]['fcontents'], fpc);
6790 try:
6791 fpc.flush();
6792 os.fsync(fpc.fileno());
6793 except io.UnsupportedOperation:
6794 pass;
6795 except AttributeError:
6796 pass;
6797 except OSError as e:
6798 pass;
6799 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
6800 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
6801 if(preservepermissions):
6802 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6803 if(preservetime):
6804 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
6805 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6806 if(followlink):
6807 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
6808 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6809 flinkinfo = listcatfiles['ffilelist'][flinkid];
6810 funame = "";
6811 try:
6812 import pwd;
6813 try:
6814 userinfo = pwd.getpwuid(flinkinfo['fuid']);
6815 funame = userinfo.pw_name;
6816 except KeyError:
6817 funame = "";
6818 except ImportError:
6819 funame = "";
6820 fgname = "";
6821 try:
6822 import grp;
6823 try:
6824 groupinfo = grp.getgrgid(flinkinfo['fgid']);
6825 fgname = groupinfo.gr_name;
6826 except KeyError:
6827 fgname = "";
6828 except ImportError:
6829 fgname = "";
6830 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
6831 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6832 flinkinfo['fcontents'].seek(0, 0);
6833 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
6834 try:
6835 fpc.flush();
6836 os.fsync(fpc.fileno());
6837 except io.UnsupportedOperation:
6838 pass;
6839 except AttributeError:
6840 pass;
6841 except OSError as e:
6842 pass;
6843 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6844 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6845 if(preservepermissions):
6846 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6847 if(preservetime):
6848 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6849 if(flinkinfo['ftype']==1):
6850 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6851 if(flinkinfo['ftype']==2):
6852 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6853 if(flinkinfo['ftype']==5):
6854 if(preservepermissions):
6855 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6856 else:
6857 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6858 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6859 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6860 if(preservepermissions):
6861 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6862 if(preservetime):
6863 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6864 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
6865 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6866 else:
6867 os.link(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6868 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6869 if(followlink):
6870 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
6871 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6872 flinkinfo = listcatfiles['ffilelist'][flinkid];
6873 funame = "";
6874 try:
6875 import pwd;
6876 try:
6877 userinfo = pwd.getpwuid(flinkinfo['fuid']);
6878 funame = userinfo.pw_name;
6879 except KeyError:
6880 funame = "";
6881 except ImportError:
6882 funame = "";
6883 fgname = "";
6884 try:
6885 import grp;
6886 try:
6887 groupinfo = grp.getgrgid(flinkinfo['fgid']);
6888 fgname = groupinfo.gr_name;
6889 except KeyError:
6890 fgname = "";
6891 except ImportError:
6892 fgname = "";
6893 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
6894 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6895 flinkinfo['fcontents'].seek(0, 0);
6896 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
6897 try:
6898 fpc.flush();
6899 os.fsync(fpc.fileno());
6900 except io.UnsupportedOperation:
6901 pass;
6902 except AttributeError:
6903 pass;
6904 except OSError as e:
6905 pass;
6906 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6907 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6908 if(preservepermissions):
6909 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6910 if(preservetime):
6911 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6912 if(flinkinfo['ftype']==1):
6913 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6914 if(flinkinfo['ftype']==2):
6915 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6916 if(flinkinfo['ftype']==5):
6917 if(preservepermissions):
6918 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6919 else:
6920 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6921 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6922 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6923 if(preservepermissions):
6924 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6925 if(preservetime):
6926 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6927 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
6928 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6929 else:
6930 os.symlink(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6931 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6932 if(preservepermissions):
6933 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6934 else:
6935 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6936 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
6937 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
6938 if(preservepermissions):
6939 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6940 if(preservetime):
6941 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
6942 if(listcatfiles['ffilelist'][lcfi]['ftype']==6 and hasattr(os, "mkfifo")):
6943 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6944 lcfi = lcfi + 1;
6945 if(returnfp):
6946 return listcatfiles['ffilelist']['catfp'];
6947 else:
6948 return True;
6949
6950 create_alias_function("UnPack", __file_format_name__, "", UnPackArchiveFile);
6951
6952 if(hasattr(shutil, "register_unpack_format")):
6953 def UnPackArchiveFileFunc(archive_name, extract_dir=None, **kwargs):
6954 return UnPackArchiveFile(archive_name, extract_dir, False, 0, 0, False, __file_format_dict__['format_delimiter'], False, False);
6955 create_alias_function("UnPack", __file_format_name__, "Func", UnPackArchiveFileFunc);
6956
6957 def UnPackArchiveFileString(catstr, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6958 formatspecs = FormatSpecsListToDict(formatspecs);
6959 catfp = BytesIO(catstr);
6960 listcatfiles = UnPackArchiveFile(catfp, outdir, followlink, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6961 return listcatfiles;
6962
6963 create_alias_function("UnPack", __file_format_name__, "String", UnPackArchiveFileString);
6964
6965 def ArchiveFileListFiles(infile, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6966 formatspecs = FormatSpecsListToDict(formatspecs);
6967 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6968 if(isinstance(infile, dict)):
6969 listcatfiles = infile;
6970 else:
6971 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6972 infile = RemoveWindowsPath(infile);
6973 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, True, False, skipchecksum, formatspecs, returnfp);
6974 if(not listcatfiles):
6975 return False;
6976 lenlist = len(listcatfiles['ffilelist']);
6977 fnumfiles = int(listcatfiles['fnumfiles']);
6978 lcfi = 0;
6979 lcfx = int(listcatfiles['fnumfiles']);
6980 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6981 lcfx = int(lenlist);
6982 else:
6983 lcfx = int(listcatfiles['fnumfiles']);
6984 returnval = {};
6985 while(lcfi<lcfx):
6986 returnval.update({lcfi: listcatfiles['ffilelist'][lcfi]['fname']});
6987 if(not verbose):
6988 VerbosePrintOut(listcatfiles['ffilelist'][lcfi]['fname']);
6989 if(verbose):
6990 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6991 printfname = listcatfiles['ffilelist'][lcfi]['fname'];
6992 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6993 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " link to " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6994 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6995 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " -> " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6996 fuprint = listcatfiles['ffilelist'][lcfi]['funame'];
6997 if(len(fuprint)<=0):
6998 fuprint = listcatfiles['ffilelist'][lcfi]['fuid'];
6999 fgprint = listcatfiles['ffilelist'][lcfi]['fgname'];
7000 if(len(fgprint)<=0):
7001 fgprint = listcatfiles['ffilelist'][lcfi]['fgid'];
7002 VerbosePrintOut(PrintPermissionString(listcatfiles['ffilelist'][lcfi]['fmode'], listcatfiles['ffilelist'][lcfi]['ftype']) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(listcatfiles['ffilelist'][lcfi]['fsize']).rjust(15) + " " + datetime.datetime.utcfromtimestamp(listcatfiles['ffilelist'][lcfi]['fmtime']).strftime('%Y-%m-%d %H:%M') + " " + printfname));
7003 lcfi = lcfi + 1;
7004 if(returnfp):
7005 return listcatfiles['catfp'];
7006 else:
7007 return True;
7008
7009 create_alias_function("", __file_format_name__, "ListFiles", ArchiveFileListFiles);
7010
7011 def ArchiveFileStringListFiles(catstr, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7012 formatspecs = FormatSpecsListToDict(formatspecs);
7013 catfp = BytesIO(catstr);
7014 listcatfiles = ArchiveFileListFiles(catstr, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
7015 return listcatfiles;
7016
7017 create_alias_function("", __file_format_name__, "StringListFiles", ArchiveFileStringListFiles);
7018
7019 def TarFileListFiles(infile, verbose=False, returnfp=False):
7020 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7021 if(infile=="-"):
7022 infile = BytesIO();
7023 if(hasattr(sys.stdin, "buffer")):
7024 shutil.copyfileobj(sys.stdin.buffer, infile);
7025 else:
7026 shutil.copyfileobj(sys.stdin, infile);
7027 infile.seek(0, 0);
7028 if(not infile):
7029 return False;
7030 infile.seek(0, 0);
7031 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
7032 infile = download_file_from_internet_file(infile);
7033 infile.seek(0, 0);
7034 if(not infile):
7035 return False;
7036 infile.seek(0, 0);
7037 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
7038 return False;
7039 elif(os.path.exists(infile) and os.path.isfile(infile)):
7040 try:
7041 if(not tarfile.TarFileCheck(infile)):
7042 return False;
7043 except AttributeError:
7044 if(not TarFileCheck(infile)):
7045 return False;
7046 else:
7047 return False;
7048 try:
7049 if(hasattr(infile, "read") or hasattr(infile, "write")):
7050 tarfp = tarfile.open(fileobj=infile, mode="r");
7051 else:
7052 tarfp = tarfile.open(infile, "r");
7053 except FileNotFoundError:
7054 return False;
7055 lcfi = 0
7056 returnval = {};
7057 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
7058 returnval.update({lcfi: member.name});
7059 fpremode = member.mode;
7060 ffullmode = member.mode;
7061 flinkcount = 0;
7062 ftype = 0;
7063 if(member.isreg()):
7064 ffullmode = member.mode + stat.S_IFREG;
7065 ftype = 0;
7066 elif(member.isdev()):
7067 ffullmode = member.mode;
7068 ftype = 7;
7069 elif(member.islnk()):
7070 ffullmode = member.mode + stat.S_IFREG;
7071 ftype = 1;
7072 elif(member.issym()):
7073 ffullmode = member.mode + stat.S_IFLNK;
7074 ftype = 2;
7075 elif(member.ischr()):
7076 ffullmode = member.mode + stat.S_IFCHR;
7077 ftype = 3;
7078 elif(member.isblk()):
7079 ffullmode = member.mode + stat.S_IFBLK;
7080 ftype = 4;
7081 elif(member.isdir()):
7082 ffullmode = member.mode + stat.S_IFDIR;
7083 ftype = 5;
7084 elif(member.isfifo()):
7085 ffullmode = member.mode + stat.S_IFIFO;
7086 ftype = 6;
7087 elif(member.issparse()):
7088 ffullmode = member.mode;
7089 ftype = 12;
7090 if(not verbose):
7091 VerbosePrintOut(member.name);
7092 elif(verbose):
7093 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7094 printfname = member.name;
7095 if(member.islnk()):
7096 printfname = member.name + " link to " + member.linkname;
7097 elif(member.issym()):
7098 printfname = member.name + " -> " + member.linkname;
7099 fuprint = member.uname;
7100 if(len(fuprint)<=0):
7101 fuprint = member.uid;
7102 fgprint = member.gname;
7103 if(len(fgprint)<=0):
7104 fgprint = member.gid;
7105 VerbosePrintOut(PrintPermissionString(ffullmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(member.mtime).strftime('%Y-%m-%d %H:%M') + " " + printfname));
7106 lcfi = lcfi + 1;
7107 if(returnfp):
7108 return listcatfiles['catfp'];
7109 else:
7110 return True;
7111
7112 def ZipFileListFiles(infile, verbose=False, returnfp=False):
7113 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7114 if(infile=="-"):
7115 infile = BytesIO();
7116 if(hasattr(sys.stdin, "buffer")):
7117 shutil.copyfileobj(sys.stdin.buffer, infile);
7118 else:
7119 shutil.copyfileobj(sys.stdin, infile);
7120 infile.seek(0, 0);
7121 if(not infile):
7122 return False;
7123 infile.seek(0, 0);
7124 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
7125 infile = download_file_from_internet_file(infile);
7126 infile.seek(0, 0);
7127 if(not infile):
7128 return False;
7129 infile.seek(0, 0);
7130 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
7131 return False;
7132 else:
7133 return False;
7134 if(not zipfile.is_zipfile(infile)):
7135 return False;
7136 try:
7137 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
7138 except FileNotFoundError:
7139 return False;
7140 lcfi = 0;
7141 returnval = {};
7142 ziptest = zipfp.testzip();
7143 if(ziptest):
7144 VerbosePrintOut("Bad file found!");
7145 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
7146 if(zipinfo.create_system==0 or zipinfo.create_system==10):
7147 fwinattributes = int(zipinfo.external_attr);
7148 if(not member.is_dir()):
7149 fmode = int(stat.S_IFREG + 438);
7150 fchmode = int(stat.S_IMODE(fmode));
7151 ftypemod = int(stat.S_IFMT(fmode));
7152 elif(member.is_dir()):
7153 fmode = int(stat.S_IFDIR + 511);
7154 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7155 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7156 elif(zipinfo.create_system==3):
7157 fwinattributes =int(0);
7158 fmode = int(zipinfo.external_attr);
7159 fchmode = int(stat.S_IMODE(fmode));
7160 ftypemod = int(stat.S_IFMT(fmode));
7161 else:
7162 fwinattributes = int(0);
7163 if(not member.is_dir()):
7164 fmode = int(stat.S_IFREG + 438);
7165 fchmode = int(stat.S_IMODE(fmode));
7166 ftypemod = int(stat.S_IFMT(fmode));
7167 elif(member.is_dir()):
7168 fmode = int(stat.S_IFDIR + 511);
7169 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7170 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7171 returnval.update({lcfi: member.filename});
7172 if(not verbose):
7173 VerbosePrintOut(member.filename);
7174 if(verbose):
7175 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7176 permissionstr = "";
7177 for fmodval in str(oct(fmode))[-3:]:
7178 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
7179 if(not member.is_dir()):
7180 ftype = 0;
7181 permissionstr = "-" + permissionstr;
7182 elif(member.is_dir()):
7183 ftype = 5;
7184 permissionstr = "d" + permissionstr;
7185 printfname = member.filename;
7186 try:
7187 fuid = int(os.getuid());
7188 except AttributeError:
7189 fuid = int(0);
7190 except KeyError:
7191 fuid = int(0);
7192 try:
7193 fgid = int(os.getgid());
7194 except AttributeError:
7195 fgid = int(0);
7196 except KeyError:
7197 fgid = int(0);
7198 try:
7199 import pwd;
7200 try:
7201 userinfo = pwd.getpwuid(os.getuid());
7202 funame = userinfo.pw_name;
7203 except KeyError:
7204 funame = "";
7205 except AttributeError:
7206 funame = "";
7207 except ImportError:
7208 funame = "";
7209 fgname = "";
7210 try:
7211 import grp;
7212 try:
7213 groupinfo = grp.getgrgid(os.getgid());
7214 fgname = groupinfo.gr_name;
7215 except KeyError:
7216 fgname = "";
7217 except AttributeError:
7218 fgname = "";
7219 except ImportError:
7220 fgname = "";
7221 fuprint = funame;
7222 if(len(fuprint)<=0):
7223 fuprint = str(fuid);
7224 fgprint = fgname;
7225 if(len(fgprint)<=0):
7226 fgprint = str(fgid);
7227 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(int(time.mktime(member.date_time + (0, 0, -1)))).strftime('%Y-%m-%d %H:%M') + " " + printfname));
7228 lcfi = lcfi + 1;
7229 if(returnfp):
7230 return listcatfiles['catfp'];
7231 else:
7232 return True;
7233
7234 if(not rarfile_support):
7235 def RarFileListFiles(infile, verbose=False, returnfp=False):
7236 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7237 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7238 return False;
7239
7240 if(rarfile_support):
7241 def RarFileListFiles(infile, verbose=False, returnfp=False):
7242 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7243 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7244 return False;
7245 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
7246 return False;
7247 lcfi = 0;
7248 returnval = {};
7249 rarfp = rarfile.RarFile(infile, "r");
7250 rartest = rarfp.testrar();
7251 if(rartest):
7252 VerbosePrintOut("Bad file found!");
7253 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
7254 is_unix = False;
7255 is_windows = False;
7256 if(member.host_os==rarfile.RAR_OS_UNIX):
7257 is_windows = False;
7258 try:
7259 member.external_attr
7260 is_unix = True;
7261 except AttributeError:
7262 is_unix = False;
7263 elif(member.host_os==rarfile.RAR_OS_WIN32):
7264 is_unix = False;
7265 try:
7266 member.external_attr
7267 is_windows = True;
7268 except AttributeError:
7269 is_windows = False;
7270 else:
7271 is_unix = False;
7272 is_windows = False;
7273 if(is_unix and member.external_attr !=0):
7274 fpremode = int(member.external_attr);
7275 elif(member.is_file()):
7276 fpremode = int(stat.S_IFREG + 438);
7277 elif(member.is_symlink()):
7278 fpremode = int(stat.S_IFLNK + 438);
7279 elif(member.is_dir()):
7280 fpremode = int(stat.S_IFDIR + 511);
7281 if(is_windows and member.external_attr !=0):
7282 fwinattributes = int(member.external_attr);
7283 else:
7284 fwinattributes = int(0);
7285 if(is_unix and member.external_attr !=0):
7286 fmode = int(member.external_attr);
7287 fchmode = int(stat.S_IMODE(member.external_attr));
7288 ftypemod = int(stat.S_IFMT(member.external_attr));
7289 elif(member.is_file()):
7290 fmode = int(stat.S_IFREG + 438);
7291 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
7292 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
7293 elif(member.is_symlink()):
7294 fmode = int(stat.S_IFLNK + 438);
7295 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
7296 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
7297 elif(member.is_dir()):
7298 fmode = int(stat.S_IFDIR + 511);
7299 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7300 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7301 returnval.update({lcfi: member.filename});
7302 if(not verbose):
7303 VerbosePrintOut(member.filename);
7304 if(verbose):
7305 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7306 permissionstr = "";
7307 for fmodval in str(oct(fmode))[-3:]:
7308 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
7309 if(member.is_file()):
7310 ftype = 0;
7311 permissionstr = "-" + permissionstr;
7312 printfname = member.filename;
7313 elif(member.is_symlink()):
7314 ftype = 2;
7315 permissionstr = "l" + permissionstr;
7316 printfname = member.name + " -> " + member.read().decode("UTF-8");
7317 elif(member.is_dir()):
7318 ftype = 5;
7319 permissionstr = "d" + permissionstr;
7320 printfname = member.filename;
7321 try:
7322 fuid = int(os.getuid());
7323 except AttributeError:
7324 fuid = int(0);
7325 except KeyError:
7326 fuid = int(0);
7327 try:
7328 fgid = int(os.getgid());
7329 except AttributeError:
7330 fgid = int(0);
7331 except KeyError:
7332 fgid = int(0);
7333 try:
7334 import pwd;
7335 try:
7336 userinfo = pwd.getpwuid(os.getuid());
7337 funame = userinfo.pw_name;
7338 except KeyError:
7339 funame = "";
7340 except AttributeError:
7341 funame = "";
7342 except ImportError:
7343 funame = "";
7344 fgname = "";
7345 try:
7346 import grp;
7347 try:
7348 groupinfo = grp.getgrgid(os.getgid());
7349 fgname = groupinfo.gr_name;
7350 except KeyError:
7351 fgname = "";
7352 except AttributeError:
7353 fgname = "";
7354 except ImportError:
7355 fgname = "";
7356 fuprint = funame;
7357 if(len(fuprint)<=0):
7358 fuprint = str(fuid);
7359 fgprint = fgname;
7360 if(len(fgprint)<=0):
7361 fgprint = str(fgid);
7362 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + member.mtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
7363 lcfi = lcfi + 1;
7364 if(returnfp):
7365 return listcatfiles['catfp'];
7366 else:
7367 return True;
7368
7369 if(not py7zr_support):
7370 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
7371 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7372 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7373 return False;
7374
7375 if(py7zr_support):
7376 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
7377 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7378 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7379 return False;
7380 lcfi = 0;
7381 returnval = {};
7382 szpfp = py7zr.SevenZipFile(infile, mode="r");
7383 file_content = szpfp.readall();
7384 #sztest = szpfp.testzip();
7385 sztestalt = szpfp.test();
7386 if(sztestalt):
7387 VerbosePrintOut("Bad file found!");
7388 for member in sorted(szpfp.list(), key=lambda x: x.filename):
7389 if(re.findall("^[.|/]", member.filename)):
7390 fname = member.filename;
7391 else:
7392 fname = "./"+member.filename;
7393 if(not member.is_directory):
7394 fpremode = int(stat.S_IFREG + 438);
7395 elif(member.is_directory):
7396 fpremode = int(stat.S_IFDIR + 511);
7397 fwinattributes = int(0);
7398 if(member.is_directory):
7399 fmode = int(stat.S_IFDIR + 511);
7400 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7401 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7402 else:
7403 fmode = int(stat.S_IFLNK + 438);
7404 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
7405 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
7406 returnval.update({lcfi: member.filename});
7407 if(not verbose):
7408 VerbosePrintOut(member.filename);
7409 if(verbose):
7410 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7411 permissionstr = "";
7412 for fmodval in str(oct(fmode))[-3:]:
7413 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
7414 fsize = int("0");
7415 if(not member.is_directory):
7416 ftype = 0;
7417 permissionstr = "-" + permissionstr;
7418 printfname = member.filename;
7419 elif(member.is_directory):
7420 ftype = 5;
7421 permissionstr = "d" + permissionstr;
7422 printfname = member.filename;
7423 if(ftype==0):
7424 fsize = len(file_content[member.filename].read());
7425 file_content[member.filename].close();
7426 try:
7427 fuid = int(os.getuid());
7428 except AttributeError:
7429 fuid = int(0);
7430 except KeyError:
7431 fuid = int(0);
7432 try:
7433 fgid = int(os.getgid());
7434 except AttributeError:
7435 fgid = int(0);
7436 except KeyError:
7437 fgid = int(0);
7438 try:
7439 import pwd;
7440 try:
7441 userinfo = pwd.getpwuid(os.getuid());
7442 funame = userinfo.pw_name;
7443 except KeyError:
7444 funame = "";
7445 except AttributeError:
7446 funame = "";
7447 except ImportError:
7448 funame = "";
7449 fgname = "";
7450 try:
7451 import grp;
7452 try:
7453 groupinfo = grp.getgrgid(os.getgid());
7454 fgname = groupinfo.gr_name;
7455 except KeyError:
7456 fgname = "";
7457 except AttributeError:
7458 fgname = "";
7459 except ImportError:
7460 fgname = "";
7461 fuprint = funame;
7462 if(len(fuprint)<=0):
7463 fuprint = str(fuid);
7464 fgprint = fgname;
7465 if(len(fgprint)<=0):
7466 fgprint = str(fgid);
7467 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(fsize).rjust(15) + " " + member.creationtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
7468 lcfi = lcfi + 1;
7469 if(returnfp):
7470 return listcatfiles['catfp'];
7471 else:
7472 return True;
7473
7474 def InFileListFiles(infile, verbose=False, formatspecs=__file_format_dict__, returnfp=False):
7475 formatspecs = FormatSpecsListToDict(formatspecs);
7476 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7477 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
7478 if(checkcompressfile=="tarfile"):
7479 return TarFileListFiles(infile, verbose, returnfp);
7480 elif(checkcompressfile=="zipfile"):
7481 return ZipFileListFiles(infile, verbose, returnfp);
7482 elif(checkcompressfile=="catfile"):
7483 return ArchiveFileListFiles(infile, 0, 0, False, formatspecs, verbose, returnfp);
7484 elif(rarfile_support and checkcompressfile=="rarfile"):
7485 return RarFileListFiles(infile, verbose, returnfp);
7486 elif(py7zr_support and checkcompressfile=="7zipfile"):
7487 return SevenZipFileListFiles(infile, verbose, returnfp);
7488 else:
7489 return False;
7490 return False;
7491
7492 def ListDirListFiles(infiles, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7493 formatspecs = FormatSpecsListToDict(formatspecs);
7494 outarray = BytesIO();
7495 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, False, True);
7496 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
7497 return listcatfiles;
7498
7499 def ListDirListFilesAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7500 formatspecs = FormatSpecsListToDict(formatspecs);
7501 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, listonly, checksumtype, formatspecs, verbose);
7502 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
7503 return listcatfiles;
7504
7505 def PackArchiveFileFromListDirAlt(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7506 formatspecs = FormatSpecsListToDict(formatspecs);
7507 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, False, checksumtype, extradata, formatspecs, False);
7508 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, followlink, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
7509 return listcatfiles;
7510
7511 create_alias_function("Pack", __file_format_name__, "FromListDirAlt", PackArchiveFileFromListDirAlt);
7512
7513 def PackArchiveFileFromTarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7514 formatspecs = FormatSpecsListToDict(formatspecs);
7515 outarray = TarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7516 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7517 return listcatfiles;
7518
7519 create_alias_function("Pack", __file_format_name__, "FromTarFileAlt", PackArchiveFileFromTarFileAlt);
7520
7521 def PackArchiveFileFromZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7522 formatspecs = FormatSpecsListToDict(formatspecs);
7523 outarray = ZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7524 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7525 return listcatfiles;
7526
7527 create_alias_function("Pack", __file_format_name__, "FromZipFileAlt", PackArchiveFileFromZipFileAlt);
7528
7529 if(not rarfile_support):
7530 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7531 return False;
7532
7533 if(rarfile_support):
7534 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7535 formatspecs = FormatSpecsListToDict(formatspecs);
7536 outarray = RarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7537 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7538 return listcatfiles;
7539
7540 create_alias_function("Pack", __file_format_name__, "FromRarFileAlt", PackArchiveFileFromRarFileAlt);
7541
7542 if(not py7zr_support):
7543 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7544 return False;
7545
7546 if(py7zr_support):
7547 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7548 formatspecs = FormatSpecsListToDict(formatspecs);
7549 outarray = SevenZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7550 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7551 return listcatfiles;
7552
7553 create_alias_function("Pack", __file_format_name__, "FromSevenZipFileAlt", PackArchiveFileFromSevenZipFileAlt);
7554
7555 def download_file_from_ftp_file(url):
7556 urlparts = urlparse(url);
7557 file_name = os.path.basename(urlparts.path);
7558 file_dir = os.path.dirname(urlparts.path);
7559 if(urlparts.username is not None):
7560 ftp_username = urlparts.username;
7561 else:
7562 ftp_username = "anonymous";
7563 if(urlparts.password is not None):
7564 ftp_password = urlparts.password;
7565 elif(urlparts.password is None and urlparts.username=="anonymous"):
7566 ftp_password = "anonymous";
7567 else:
7568 ftp_password = "";
7569 if(urlparts.scheme=="ftp"):
7570 ftp = FTP();
7571 elif(urlparts.scheme=="ftps" and ftpssl):
7572 ftp = FTP_TLS();
7573 else:
7574 return False;
7575 if(urlparts.scheme=="sftp"):
7576 if(__use_pysftp__):
7577 return download_file_from_pysftp_file(url);
7578 else:
7579 return download_file_from_sftp_file(url);
7580 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7581 return download_file_from_http_file(url);
7582 ftp_port = urlparts.port;
7583 if(urlparts.port is None):
7584 ftp_port = 21;
7585 try:
7586 ftp.connect(urlparts.hostname, ftp_port);
7587 except socket.gaierror:
7588 log.info("Error With URL "+url);
7589 return False;
7590 except socket.timeout:
7591 log.info("Error With URL "+url);
7592 return False;
7593 ftp.login(urlparts.username, urlparts.password);
7594 if(urlparts.scheme=="ftps"):
7595 ftp.prot_p();
7596 ftpfile = BytesIO();
7597 ftp.retrbinary("RETR "+urlparts.path, ftpfile.write);
7598 #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
7599 ftp.close();
7600 ftpfile.seek(0, 0);
7601 return ftpfile;
7602
7603 def download_file_from_ftp_string(url):
7604 ftpfile = download_file_from_ftp_file(url);
7605 return ftpfile.read();
7606
7607 def upload_file_to_ftp_file(ftpfile, url):
7608 urlparts = urlparse(url);
7609 file_name = os.path.basename(urlparts.path);
7610 file_dir = os.path.dirname(urlparts.path);
7611 if(urlparts.username is not None):
7612 ftp_username = urlparts.username;
7613 else:
7614 ftp_username = "anonymous";
7615 if(urlparts.password is not None):
7616 ftp_password = urlparts.password;
7617 elif(urlparts.password is None and urlparts.username=="anonymous"):
7618 ftp_password = "anonymous";
7619 else:
7620 ftp_password = "";
7621 if(urlparts.scheme=="ftp"):
7622 ftp = FTP();
7623 elif(urlparts.scheme=="ftps" and ftpssl):
7624 ftp = FTP_TLS();
7625 else:
7626 return False;
7627 if(urlparts.scheme=="sftp"):
7628 if(__use_pysftp__):
7629 return upload_file_to_pysftp_file(url);
7630 else:
7631 return upload_file_to_sftp_file(url);
7632 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7633 return False;
7634 ftp_port = urlparts.port;
7635 if(urlparts.port is None):
7636 ftp_port = 21;
7637 try:
7638 ftp.connect(urlparts.hostname, ftp_port);
7639 except socket.gaierror:
7640 log.info("Error With URL "+url);
7641 return False;
7642 except socket.timeout:
7643 log.info("Error With URL "+url);
7644 return False;
7645 ftp.login(urlparts.username, urlparts.password);
7646 if(urlparts.scheme=="ftps"):
7647 ftp.prot_p();
7648 ftp.storbinary("STOR "+urlparts.path, ftpfile);
7649 ftp.close();
7650 ftpfile.seek(0, 0);
7651 return ftpfile;
7652
7653 def upload_file_to_ftp_string(ftpstring, url):
7654 ftpfileo = BytesIO(ftpstring);
7655 ftpfile = upload_file_to_ftp_file(ftpfileo, url);
7656 ftpfileo.close();
7657 return ftpfile;
7658
7659 def download_file_from_http_file(url, headers=geturls_headers_pycatfile_python_alt):
7660 # Parse the URL to extract username and password if present
7661 urlparts = urlparse(url);
7662 username = urlparts.username;
7663 password = urlparts.password;
7664 # Rebuild the URL without the username and password
7665 netloc = urlparts.hostname;
7666 if(urlparts.scheme=="sftp"):
7667 if(__use_pysftp__):
7668 return download_file_from_pysftp_file(url);
7669 else:
7670 return download_file_from_sftp_file(url);
7671 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7672 return download_file_from_ftp_file(url);
7673 if urlparts.port:
7674 netloc += ':' + str(urlparts.port);
7675 rebuilt_url = urlunparse((urlparts.scheme, netloc, urlparts.path, urlparts.params, urlparts.query, urlparts.fragment));
7676 # Create a temporary file object
7677 httpfile = BytesIO();
7678 if haverequests:
7679 # Use the requests library if available
7680 if username and password:
7681 response = requests.get(rebuilt_url, headers=headers, auth=(username, password), stream=True);
7682 else:
7683 response = requests.get(rebuilt_url, headers=headers, stream=True);
7684 response.raw.decode_content = True
7685 shutil.copyfileobj(response.raw, httpfile);
7686 else:
7687 # Build a Request object for urllib
7688 request = Request(rebuilt_url, headers=headers);
7689 # Create an opener object for handling URLs
7690 if username and password:
7691 # Create a password manager
7692 password_mgr = HTTPPasswordMgrWithDefaultRealm();
7693 # Add the username and password
7694 password_mgr.add_password(None, rebuilt_url, username, password);
7695 # Create an authentication handler using the password manager
7696 auth_handler = HTTPBasicAuthHandler(password_mgr);
7697 # Build the opener with the authentication handler
7698 opener = build_opener(auth_handler);
7699 else:
7700 opener = build_opener();
7701 with opener.open(request) as response:
7702 shutil.copyfileobj(response, httpfile);
7703 # Reset file pointer to the start
7704 httpfile.seek(0, 0);
7705 # Return the temporary file object
7706 return httpfile;
7707
7708 def download_file_from_http_string(url, headers=geturls_headers_pycatfile_python_alt):
7709 httpfile = download_file_from_http_file(url, headers);
7710 return ftpfile.read();
7711
7712 if(haveparamiko):
7713 def download_file_from_sftp_file(url):
7714 urlparts = urlparse(url);
7715 file_name = os.path.basename(urlparts.path);
7716 file_dir = os.path.dirname(urlparts.path);
7717 sftp_port = urlparts.port;
7718 if(urlparts.port is None):
7719 sftp_port = 22;
7720 else:
7721 sftp_port = urlparts.port;
7722 if(urlparts.username is not None):
7723 sftp_username = urlparts.username;
7724 else:
7725 sftp_username = "anonymous";
7726 if(urlparts.password is not None):
7727 sftp_password = urlparts.password;
7728 elif(urlparts.password is None and urlparts.username=="anonymous"):
7729 sftp_password = "anonymous";
7730 else:
7731 sftp_password = "";
7732 if(urlparts.scheme=="ftp"):
7733 return download_file_from_ftp_file(url);
7734 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7735 return download_file_from_http_file(url);
7736 if(urlparts.scheme!="sftp"):
7737 return False;
7738 ssh = paramiko.SSHClient();
7739 ssh.load_system_host_keys();
7740 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
7741 try:
7742 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7743 except paramiko.ssh_exception.SSHException:
7744 return False;
7745 except socket.gaierror:
7746 log.info("Error With URL "+url);
7747 return False;
7748 except socket.timeout:
7749 log.info("Error With URL "+url);
7750 return False;
7751 sftp = ssh.open_sftp();
7752 sftpfile = BytesIO();
7753 sftp.getfo(urlparts.path, sftpfile);
7754 sftp.close();
7755 ssh.close();
7756 sftpfile.seek(0, 0);
7757 return sftpfile;
7758 else:
7759 def download_file_from_sftp_file(url):
7760 return False;
7761
7762 if(haveparamiko):
7763 def download_file_from_sftp_string(url):
7764 sftpfile = download_file_from_sftp_file(url);
7765 return sftpfile.read();
7766 else:
7767 def download_file_from_ftp_string(url):
7768 return False;
7769
7770 if(haveparamiko):
7771 def upload_file_to_sftp_file(sftpfile, url):
7772 urlparts = urlparse(url);
7773 file_name = os.path.basename(urlparts.path);
7774 file_dir = os.path.dirname(urlparts.path);
7775 sftp_port = urlparts.port;
7776 if(urlparts.port is None):
7777 sftp_port = 22;
7778 else:
7779 sftp_port = urlparts.port;
7780 if(urlparts.username is not None):
7781 sftp_username = urlparts.username;
7782 else:
7783 sftp_username = "anonymous";
7784 if(urlparts.password is not None):
7785 sftp_password = urlparts.password;
7786 elif(urlparts.password is None and urlparts.username=="anonymous"):
7787 sftp_password = "anonymous";
7788 else:
7789 sftp_password = "";
7790 if(urlparts.scheme=="ftp"):
7791 return upload_file_to_ftp_file(url);
7792 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7793 return False;
7794 if(urlparts.scheme!="sftp"):
7795 return False;
7796 ssh = paramiko.SSHClient();
7797 ssh.load_system_host_keys();
7798 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
7799 try:
7800 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7801 except paramiko.ssh_exception.SSHException:
7802 return False;
7803 except socket.gaierror:
7804 log.info("Error With URL "+url);
7805 return False;
7806 except socket.timeout:
7807 log.info("Error With URL "+url);
7808 return False;
7809 sftp = ssh.open_sftp();
7810 sftp.putfo(sftpfile, urlparts.path);
7811 sftp.close();
7812 ssh.close();
7813 sftpfile.seek(0, 0);
7814 return sftpfile;
7815 else:
7816 def upload_file_to_sftp_file(sftpfile, url):
7817 return False;
7818
7819 if(haveparamiko):
7820 def upload_file_to_sftp_string(sftpstring, url):
7821 sftpfileo = BytesIO(sftpstring);
7822 sftpfile = upload_file_to_sftp_files(ftpfileo, url);
7823 sftpfileo.close();
7824 return sftpfile;
7825 else:
7826 def upload_file_to_sftp_string(url):
7827 return False;
7828
7829 if(havepysftp):
7830 def download_file_from_pysftp_file(url):
7831 urlparts = urlparse(url);
7832 file_name = os.path.basename(urlparts.path);
7833 file_dir = os.path.dirname(urlparts.path);
7834 sftp_port = urlparts.port;
7835 if(urlparts.port is None):
7836 sftp_port = 22;
7837 else:
7838 sftp_port = urlparts.port;
7839 if(urlparts.username is not None):
7840 sftp_username = urlparts.username;
7841 else:
7842 sftp_username = "anonymous";
7843 if(urlparts.password is not None):
7844 sftp_password = urlparts.password;
7845 elif(urlparts.password is None and urlparts.username=="anonymous"):
7846 sftp_password = "anonymous";
7847 else:
7848 sftp_password = "";
7849 if(urlparts.scheme=="ftp"):
7850 return download_file_from_ftp_file(url);
7851 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7852 return download_file_from_http_file(url);
7853 if(urlparts.scheme!="sftp"):
7854 return False;
7855 try:
7856 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7857 except paramiko.ssh_exception.SSHException:
7858 return False;
7859 except socket.gaierror:
7860 log.info("Error With URL "+url);
7861 return False;
7862 except socket.timeout:
7863 log.info("Error With URL "+url);
7864 return False;
7865 sftp = ssh.open_sftp();
7866 sftpfile = BytesIO();
7867 sftp.getfo(urlparts.path, sftpfile);
7868 sftp.close();
7869 ssh.close();
7870 sftpfile.seek(0, 0);
7871 return sftpfile;
7872 else:
7873 def download_file_from_pysftp_file(url):
7874 return False;
7875
7876 if(havepysftp):
7877 def download_file_from_pysftp_string(url):
7878 sftpfile = download_file_from_pysftp_file(url);
7879 return sftpfile.read();
7880 else:
7881 def download_file_from_ftp_string(url):
7882 return False;
7883
7884 if(havepysftp):
7885 def upload_file_to_pysftp_file(sftpfile, url):
7886 urlparts = urlparse(url);
7887 file_name = os.path.basename(urlparts.path);
7888 file_dir = os.path.dirname(urlparts.path);
7889 sftp_port = urlparts.port;
7890 if(urlparts.port is None):
7891 sftp_port = 22;
7892 else:
7893 sftp_port = urlparts.port;
7894 if(urlparts.username is not None):
7895 sftp_username = urlparts.username;
7896 else:
7897 sftp_username = "anonymous";
7898 if(urlparts.password is not None):
7899 sftp_password = urlparts.password;
7900 elif(urlparts.password is None and urlparts.username=="anonymous"):
7901 sftp_password = "anonymous";
7902 else:
7903 sftp_password = "";
7904 if(urlparts.scheme=="ftp"):
7905 return upload_file_to_ftp_file(url);
7906 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7907 return False;
7908 if(urlparts.scheme!="sftp"):
7909 return False;
7910 try:
7911 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7912 except paramiko.ssh_exception.SSHException:
7913 return False;
7914 except socket.gaierror:
7915 log.info("Error With URL "+url);
7916 return False;
7917 except socket.timeout:
7918 log.info("Error With URL "+url);
7919 return False;
7920 sftp = ssh.open_sftp();
7921 sftp.putfo(sftpfile, urlparts.path);
7922 sftp.close();
7923 ssh.close();
7924 sftpfile.seek(0, 0);
7925 return sftpfile;
7926 else:
7927 def upload_file_to_pysftp_file(sftpfile, url):
7928 return False;
7929
7930 if(havepysftp):
7931 def upload_file_to_pysftp_string(sftpstring, url):
7932 sftpfileo = BytesIO(sftpstring);
7933 sftpfile = upload_file_to_pysftp_files(ftpfileo, url);
7934 sftpfileo.close();
7935 return sftpfile;
7936 else:
7937 def upload_file_to_pysftp_string(url):
7938 return False;
7939
7940 def download_file_from_internet_file(url, headers=geturls_headers_pycatfile_python_alt):
7941 urlparts = urlparse(url);
7942 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7943 return download_file_from_http_file(url, headers);
7944 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7945 return download_file_from_ftp_file(url);
7946 elif(urlparts.scheme=="sftp"):
7947 if(__use_pysftp__ and havepysftp):
7948 return download_file_from_pysftp_file(url);
7949 else:
7950 return download_file_from_sftp_file(url);
7951 else:
7952 return False;
7953 return False;
7954
7955 def download_file_from_internet_uncompress_file(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_dict__):
7956 formatspecs = FormatSpecsListToDict(formatspecs);
7957 fp = download_file_from_internet_file(url);
7958 fp = UncompressArchiveFile(fp, formatspecs);
7959 fp.seek(0, 0);
7960 if(not fp):
7961 return False;
7962 return fp;
7963
7964 def download_file_from_internet_string(url, headers=geturls_headers_pycatfile_python_alt):
7965 urlparts = urlparse(url);
7966 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7967 return download_file_from_http_string(url, headers);
7968 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7969 return download_file_from_ftp_string(url);
7970 elif(urlparts.scheme=="sftp"):
7971 if(__use_pysftp__ and havepysftp):
7972 return download_file_from_pysftp_string(url);
7973 else:
7974 return download_file_from_sftp_string(url);
7975 else:
7976 return False;
7977 return False;
7978
7979 def download_file_from_internet_uncompress_string(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_dict__):
7980 formatspecs = FormatSpecsListToDict(formatspecs);
7981 fp = download_file_from_internet_string(url);
7982 fp = UncompressArchiveFile(fp, formatspecs);
7983 fp.seek(0, 0);
7984 if(not fp):
7985 return False;
7986 return fp;
7987
7988 def upload_file_to_internet_file(ifp, url):
7989 urlparts = urlparse(url);
7990 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7991 return False;
7992 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7993 return upload_file_to_ftp_file(ifp, url);
7994 elif(urlparts.scheme=="sftp"):
7995 if(__use_pysftp__ and havepysftp):
7996 return upload_file_to_pysftp_file(ifp, url);
7997 else:
7998 return upload_file_to_sftp_file(ifp, url);
7999 else:
8000 return False;
8001 return False;
8002
8003 def upload_file_to_internet_compress_file(ifp, url, formatspecs=__file_format_dict__):
8004 formatspecs = FormatSpecsListToDict(formatspecs);
8005 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
8006 if(not catfileout):
8007 return False;
8008 catfp.seek(0, 0);
8009 upload_file_to_internet_file(catfp, outfile);
8010 return True;
8011
8012 def upload_file_to_internet_string(ifp, url):
8013 urlparts = urlparse(url);
8014 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
8015 return False;
8016 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
8017 return upload_file_to_ftp_string(ifp, url);
8018 elif(urlparts.scheme=="sftp"):
8019 if(__use_pysftp__ and havepysftp):
8020 return upload_file_to_pysftp_string(ifp, url);
8021 else:
8022 return upload_file_to_sftp_string(ifp, url);
8023 else:
8024 return False;
8025 return False;
8026
8027 def upload_file_to_internet_compress_string(ifp, url, formatspecs=__file_format_dict__):
8028 formatspecs = FormatSpecsListToDict(formatspecs);
8029 catfp = CompressArchiveFile(BytesIO(ifp), compression, compressionlevel, formatspecs);
8030 if(not catfileout):
8031 return False;
8032 catfp.seek(0, 0);
8033 upload_file_to_internet_file(catfp, outfile);
8034 return True;
8035
8036 try:
8037 if(hasattr(shutil, "register_archive_format")):
8038 # Register the packing format
8039 shutil.register_archive_format(__file_format_name__, PackArchiveFileFunc, description='Pack concatenated files');
8040 except shutil.RegistryError:
8041 pass;
8042
8043 try:
8044 if(hasattr(shutil, "register_unpack_format")):
8045 # Register the unpacking format
8046 shutil.register_unpack_format(__file_format_name__, archivefile_extensions, UnPackArchiveFileFunc, description='UnPack concatenated files');
8047 except shutil.RegistryError:
8048 pass;
A tar like file format name catfile after unix cat command (concatenate files).