Add files via upload
[PyCatFile.git] / pycatfile.py
blobae04a009b8e42b0b1aad65c619615d392d04508e
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
4 '''
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the Revised BSD License.
8 This program is distributed in the hope that it will be useful,
9 but WITHOUT ANY WARRANTY; without even the implied warranty of
10 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 Revised BSD License for more details.
13 Copyright 2018-2024 Cool Dude 2k - http://idb.berlios.de/
14 Copyright 2018-2024 Game Maker 2k - http://intdb.sourceforge.net/
15 Copyright 2018-2024 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
17 $FileInfo: pycatfile.py - Last Update: 5/31/2024 Ver. 0.12.0 RC 1 - Author: cooldude2k $
18 '''
20 from __future__ import absolute_import, division, print_function, unicode_literals;
21 import io, os, re, sys, time, stat, zlib, base64, shutil, socket, hashlib, datetime, logging, binascii, zipfile, platform;
22 try:
23 from backports import tempfile;
24 except ImportError:
25 import tempfile;
26 # FTP Support
27 ftpssl = True;
28 try:
29 from ftplib import FTP, FTP_TLS;
30 except ImportError:
31 ftpssl = False;
32 from ftplib import FTP;
34 try:
35 import simplejson as json;
36 except ImportError:
37 import json;
39 # URL Parsing
40 try:
41 from urllib.parse import urlparse, urlunparse;
42 except ImportError:
43 from urlparse import urlparse, urlunparse;
45 # Windows-specific setup
46 if os.name == 'nt':
47 if sys.version_info[0] == 2:
48 import codecs;
49 sys.stdout = codecs.getwriter('utf-8')(sys.stdout);
50 sys.stderr = codecs.getwriter('utf-8')(sys.stderr);
51 else:
52 sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace', line_buffering=True);
53 sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', errors='replace', line_buffering=True);
55 hashlib_guaranteed = False;
56 # Environment setup
57 os.environ["PYTHONIOENCODING"] = "UTF-8";
58 os.environ["LC_CTYPE"] = "UTF-8";
60 # Reload sys to set default encoding to UTF-8 (Python 2 only)
61 if sys.version_info[0] == 2:
62 try:
63 reload(sys);
64 sys.setdefaultencoding('UTF-8');
65 except (NameError, AttributeError):
66 pass;
68 # CRC32 import
69 try:
70 from zlib import crc32;
71 except ImportError:
72 from binascii import crc32;
74 # Define FileNotFoundError for Python 2
75 try:
76 FileNotFoundError;
77 except NameError:
78 FileNotFoundError = IOError;
80 # RAR file support
81 rarfile_support = False;
82 try:
83 import rarfile;
84 rarfile_support = True;
85 except ImportError:
86 pass;
88 # 7z file support
89 py7zr_support = False;
90 try:
91 import py7zr;
92 py7zr_support = True;
93 except ImportError:
94 pass;
96 # TAR file checking
97 try:
98 from xtarfile import is_tarfile;
99 except ImportError:
100 try:
101 from safetar import is_tarfile;
102 except ImportError:
103 from tarfile import is_tarfile;
105 # TAR file module
106 try:
107 import xtarfile as tarfile;
108 except ImportError:
109 try:
110 import safetar as tarfile;
111 except ImportError:
112 import tarfile;
114 # Paramiko support
115 haveparamiko = False;
116 try:
117 import paramiko;
118 haveparamiko = True;
119 except ImportError:
120 pass;
122 # PySFTP support
123 havepysftp = False;
124 try:
125 import pysftp;
126 havepysftp = True;
127 except ImportError:
128 pass;
130 # Requests support
131 haverequests = False;
132 try:
133 import requests;
134 haverequests = True;
135 import urllib3;
136 logging.getLogger("urllib3").setLevel(logging.WARNING);
137 except ImportError:
138 pass;
140 # HTTP and URL parsing
141 try:
142 from urllib.request import Request, build_opener, HTTPBasicAuthHandler;
143 from urllib.parse import urlparse;
144 except ImportError:
145 from urllib2 import Request, build_opener, HTTPBasicAuthHandler;
146 from urlparse import urlparse;
148 # StringIO and BytesIO
149 try:
150 from io import StringIO, BytesIO;
151 except ImportError:
152 try:
153 from cStringIO import StringIO;
154 from cStringIO import StringIO as BytesIO;
155 except ImportError:
156 from StringIO import StringIO;
157 from StringIO import StringIO as BytesIO;
159 __use_pysftp__ = False;
160 if(not havepysftp):
161 __use_pysftp__ = False;
162 __file_format_name__ = "CatFile";
163 __program_name__ = "Py"+__file_format_name__;
164 __file_format_lower__ = __file_format_name__.lower();
165 __file_format_magic__ = __file_format_name__;
166 __file_format_len__ = len(__file_format_magic__);
167 __file_format_hex__ = binascii.hexlify(__file_format_magic__.encode("UTF-8")).decode("UTF-8");
168 __file_format_delimiter__ = "\x00";
169 __file_format_ver__ = "001";
170 __use_new_style__ = True;
171 __use_advanced_list__ = True;
172 __use_alt_inode__ = False;
173 __file_format_extension__ = ".cat";
175 __file_format_name__ = "FastArchive";
176 __program_name__ = "Py" + __file_format_name__;
177 __file_format_lower__ = __file_format_name__.lower();
178 __file_format_magic__ = "FstArch";
179 __file_format_len__ = len(__file_format_magic__);
180 __file_format_hex__ = binascii.hexlify(__file_format_magic__.encode("UTF-8")).decode("UTF-8");
181 __file_format_delimiter__ = "\x1F"; # Using a non-printable ASCII character as delimiter
182 __file_format_ver__ = "001";
183 __use_new_style__ = True;
184 __use_advanced_list__ = False;
185 __use_alt_inode__ = False;
186 __file_format_extension__ = ".fast";
188 __file_format_list__ = [__file_format_name__, __file_format_magic__, __file_format_lower__, __file_format_len__, __file_format_hex__, __file_format_delimiter__, __file_format_ver__, __use_new_style__, __use_advanced_list__, __use_alt_inode__];
189 __file_format_dict__ = {'format_name': __file_format_name__, 'format_magic': __file_format_magic__, 'format_lower': __file_format_lower__, 'format_len': __file_format_len__, 'format_hex': __file_format_hex__, 'format_delimiter': __file_format_delimiter__, 'format_ver': __file_format_ver__, 'new_style': __use_new_style__, 'use_advanced_list': __use_advanced_list__, 'use_alt_inode': __use_alt_inode__};
190 __project__ = __program_name__;
191 __project_url__ = "https://github.com/GameMaker2k/PyCatFile";
192 __version_info__ = (0, 12, 0, "RC 1", 1);
193 __version_date_info__ = (2024, 5, 31, "RC 1", 1);
194 __version_date__ = str(__version_date_info__[0]) + "." + str(__version_date_info__[1]).zfill(2) + "." + str(__version_date_info__[2]).zfill(2);
195 __revision__ = __version_info__[3];
196 __revision_id__ = "$Id$";
197 if(__version_info__[4] is not None):
198 __version_date_plusrc__ = __version_date__ + "-" + str(__version_date_info__[4]);
199 if(__version_info__[4] is None):
200 __version_date_plusrc__ = __version_date__;
201 if(__version_info__[3] is not None):
202 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]) + " " + str(__version_info__[3]);
203 if(__version_info__[3] is None):
204 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]);
206 PyBitness = platform.architecture();
207 if(PyBitness=="32bit" or PyBitness=="32"):
208 PyBitness = "32";
209 elif(PyBitness=="64bit" or PyBitness=="64"):
210 PyBitness = "64";
211 else:
212 PyBitness = "32";
214 geturls_ua_pycatfile_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
215 if(platform.python_implementation()!=""):
216 py_implementation = platform.python_implementation();
217 if(platform.python_implementation()==""):
218 py_implementation = "Python";
219 geturls_ua_pycatfile_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=py_implementation, pyver=platform.python_version(), proname=__project__, prover=__version__);
220 geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
221 geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
222 geturls_headers_pycatfile_python = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
223 geturls_headers_pycatfile_python_alt = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python_alt, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
224 geturls_headers_googlebot_google = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
225 geturls_headers_googlebot_google_old = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
227 def CompressionSupport():
228 compression_list = [];
229 try:
230 import gzip;
231 compression_list.append("gz");
232 compression_list.append("gzip");
233 except ImportError:
234 '''return False;'''
235 try:
236 import bz2;
237 compression_list.append("bz2");
238 compression_list.append("bzip2");
239 except ImportError:
240 '''return False;'''
241 try:
242 import lz4;
243 compression_list.append("lz4");
244 except ImportError:
245 '''return False;'''
246 try:
247 import lzo;
248 compression_list.append("lzo");
249 compression_list.append("lzop");
250 except ImportError:
251 '''return False;'''
252 try:
253 import zstandard;
254 compression_list.append("zstd");
255 compression_list.append("zstandard");
256 except ImportError:
257 '''return False;'''
258 try:
259 import lzma;
260 compression_list.append("lzma");
261 compression_list.append("xz");
262 except ImportError:
263 '''return False;'''
264 try:
265 from backports import lzma;
266 compression_list.append("lzma");
267 compression_list.append("xz");
268 except ImportError:
269 '''return False;'''
270 return compression_list;
272 compressionsupport = CompressionSupport();
273 compressionlist = ['auto'];
274 compressionlistalt = [];
275 outextlist = [];
276 outextlistwd = [];
277 if('gzip' in compressionsupport):
278 compressionlist.append('gzip');
279 compressionlistalt.append('gzip');
280 outextlist.append('gz');
281 outextlistwd.append('.gz');
282 if('bzip2' in compressionsupport):
283 compressionlist.append('bzip2');
284 compressionlistalt.append('bzip2');
285 outextlist.append('bz2');
286 outextlistwd.append('.bz2');
287 if('zstd' in compressionsupport):
288 compressionlist.append('zstd');
289 compressionlistalt.append('zstd');
290 outextlist.append('zst');
291 outextlistwd.append('.zst');
292 if('lz4' in compressionsupport):
293 compressionlist.append('lz4');
294 compressionlistalt.append('lz4');
295 outextlist.append('lz4');
296 outextlistwd.append('.lz4');
297 if('lzo' in compressionsupport):
298 compressionlist.append('lzo');
299 compressionlistalt.append('lzo');
300 outextlist.append('lzo');
301 outextlistwd.append('.lzo');
302 if('lzop' in compressionsupport):
303 compressionlist.append('lzop');
304 compressionlistalt.append('lzop');
305 outextlist.append('lzop');
306 outextlistwd.append('.lzop');
307 if('lzma' in compressionsupport):
308 compressionlist.append('lzma');
309 compressionlistalt.append('lzma');
310 outextlist.append('lzma');
311 outextlistwd.append('.lzma');
312 if('xz' in compressionsupport):
313 compressionlist.append('xz');
314 compressionlistalt.append('xz');
315 outextlist.append('xz');
316 outextlistwd.append('.xz');
318 tarfile_mimetype = "application/tar";
319 tarfile_tar_mimetype = tarfile_mimetype;
320 zipfile_mimetype = "application/zip";
321 zipfile_zip_mimetype = zipfile_mimetype;
322 rarfile_mimetype = "application/rar";
323 rarfile_rar_mimetype = rarfile_mimetype;
324 archivefile_mimetype = "application/x-"+__file_format_dict__['format_lower']+"";
325 archivefile_cat_mimetype = archivefile_mimetype;
326 archivefile_gzip_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+gzip";
327 archivefile_gz_mimetype = archivefile_gzip_mimetype;
328 archivefile_bzip2_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+bzip2";
329 archivefile_bz2_mimetype = archivefile_bzip2_mimetype;
330 archivefile_lz4_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lz4";
331 archivefile_lzop_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lzop";
332 archivefile_lzo_mimetype = archivefile_lzop_mimetype;
333 archivefile_zstandard_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+zstandard";
334 archivefile_zstd_mimetype = archivefile_zstandard_mimetype;
335 archivefile_lzma_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lzma";
336 archivefile_xz_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+xz";
337 archivefile_extensions = [__file_format_extension__, __file_format_extension__+".gz", __file_format_extension__+".bz2", __file_format_extension__+".zst", __file_format_extension__+".lz4", __file_format_extension__+".lzo", __file_format_extension__+".lzop", __file_format_extension__+".lzma", __file_format_extension__+".xz"];
339 if __name__ == "__main__":
340 import subprocess;
341 curscrpath = os.path.dirname(sys.argv[0]);
342 if(curscrpath==""):
343 curscrpath = ".";
344 if(os.sep=="\\"):
345 curscrpath = curscrpath.replace(os.sep, "/");
346 curscrpath = curscrpath + "/";
347 scrfile = curscrpath + "catfile.py";
348 if(os.path.exists(scrfile) and os.path.isfile(scrfile)):
349 scrcmd = subprocess.Popen([sys.executable, scrfile] + sys.argv[1:]);
350 scrcmd.wait();
352 def VerbosePrintOut(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
353 if(not dbgenable):
354 return True;
355 log_functions = {
356 "print": print,
357 "log": logging.info,
358 "warning": logging.warning,
359 "error": logging.error,
360 "critical": logging.critical,
361 "exception": logging.exception,
362 "logalt": lambda x: logging.log(dgblevel, x),
363 "debug": logging.debug
365 log_function = log_functions.get(outtype);
366 if(log_function):
367 log_function(dbgtxt);
368 return True;
369 return False;
371 def VerbosePrintOutReturn(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
372 VerbosePrintOut(dbgtxt, outtype, dbgenable, dgblevel);
373 return dbgtxt;
375 def RemoveWindowsPath(dpath):
376 if(dpath is None):
377 dpath = "";
378 if(os.sep!="/"):
379 dpath = dpath.replace(os.path.sep, "/");
380 dpath = dpath.rstrip("/");
381 if(dpath=="." or dpath==".."):
382 dpath = dpath + "/";
383 return dpath;
385 def NormalizeRelativePath(inpath):
386 inpath = RemoveWindowsPath(inpath);
387 if(os.path.isabs(inpath)):
388 outpath = inpath;
389 else:
390 if(inpath.startswith("./") or inpath.startswith("../")):
391 outpath = inpath;
392 else:
393 outpath = "./" + inpath;
394 return outpath;
396 def PrependPath(base_dir, child_path):
397 # Check if base_dir is None or empty, if so, return child_path as is
398 if not base_dir:
399 return child_path;
400 # Ensure base_dir ends with exactly one slash
401 if not base_dir.endswith('/'):
402 base_dir += '/';
403 # Check if child_path starts with ./ or ../ (indicating a relative path)
404 if child_path.startswith('./') or child_path.startswith('../'):
405 # For relative paths, we don't alter the child_path
406 return base_dir + child_path;
407 else:
408 # For non-relative paths, ensure there's no starting slash on child_path to avoid double slashes
409 return base_dir + child_path.lstrip('/');
411 def ListDir(dirpath, followlink=False, duplicates=False):
412 if isinstance(dirpath, (list, tuple, )):
413 dirpath = list(filter(None, dirpath));
414 elif isinstance(dirpath, str):
415 dirpath = list(filter(None, [dirpath]));
416 retlist = [];
417 fs_encoding = sys.getfilesystemencoding();
418 for mydirfile in dirpath:
419 if not os.path.exists(mydirfile):
420 return False;
421 mydirfile = NormalizeRelativePath(mydirfile);
422 if os.path.exists(mydirfile) and os.path.islink(mydirfile):
423 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile));
424 if os.path.exists(mydirfile) and os.path.isdir(mydirfile):
425 for root, dirs, filenames in os.walk(mydirfile):
426 dpath = root;
427 dpath = RemoveWindowsPath(dpath);
428 if fs_encoding != 'utf-8':
429 dpath = dpath.encode(fs_encoding).decode('utf-8');
430 if dpath not in retlist and not duplicates:
431 retlist.append(dpath);
432 if duplicates:
433 retlist.append(dpath);
434 for file in filenames:
435 fpath = os.path.join(root, file);
436 fpath = RemoveWindowsPath(fpath);
437 if fs_encoding != 'utf-8':
438 fpath = fpath.encode(fs_encoding).decode('utf-8');
439 if fpath not in retlist and not duplicates:
440 retlist.append(fpath);
441 if duplicates:
442 retlist.append(fpath);
443 else:
444 path = RemoveWindowsPath(mydirfile);
445 if fs_encoding != 'utf-8':
446 path = path.encode(fs_encoding).decode('utf-8');
447 retlist.append(path);
448 return retlist;
450 def ListDirAdvanced(dirpath, followlink=False, duplicates=False):
451 if isinstance(dirpath, (list, tuple, )):
452 dirpath = list(filter(None, dirpath));
453 elif isinstance(dirpath, str):
454 dirpath = list(filter(None, [dirpath]));
455 retlist = [];
456 fs_encoding = sys.getfilesystemencoding();
457 for mydirfile in dirpath:
458 if not os.path.exists(mydirfile):
459 return False;
460 mydirfile = NormalizeRelativePath(mydirfile);
461 if os.path.exists(mydirfile) and os.path.islink(mydirfile) and followlink:
462 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile));
463 if os.path.exists(mydirfile) and os.path.isdir(mydirfile):
464 for root, dirs, filenames in os.walk(mydirfile):
465 # Sort dirs and filenames alphabetically in place
466 dirs.sort(key=lambda x: x.lower());
467 filenames.sort(key=lambda x: x.lower());
468 dpath = RemoveWindowsPath(root);
469 if fs_encoding != 'utf-8':
470 dpath = dpath.encode(fs_encoding).decode('utf-8');
471 if not duplicates and dpath not in retlist:
472 retlist.append(dpath);
473 elif duplicates:
474 retlist.append(dpath);
475 for file in filenames:
476 fpath = os.path.join(root, file);
477 fpath = RemoveWindowsPath(fpath);
478 if fs_encoding != 'utf-8':
479 fpath = fpath.encode(fs_encoding).decode('utf-8');
480 if not duplicates and fpath not in retlist:
481 retlist.append(fpath);
482 elif duplicates:
483 retlist.append(fpath);
484 else:
485 path = RemoveWindowsPath(mydirfile);
486 if fs_encoding != 'utf-8':
487 path = path.encode(fs_encoding).decode('utf-8');
488 retlist.append(path);
489 return retlist;
491 def create_alias_function(prefix, base_name, suffix, target_function):
492 # Define a new function that wraps the target function
493 def alias_function(*args, **kwargs):
494 return target_function(*args, **kwargs);
495 # Create the function name by combining the prefix, base name, and the suffix
496 function_name = "{}{}{}".format(prefix, base_name, suffix);
497 # Add the new function to the global namespace
498 globals()[function_name] = alias_function;
500 def create_alias_function_alt(prefix, base_name, suffix, target_function):
501 # Create the function name by combining the prefix, base name, and the suffix
502 # Use the format method for string formatting, compatible with Python 2 and 3
503 function_name = "{}{}{}".format(prefix, base_name, suffix)
504 # Add the new function (alias of the target_function) to the global namespace
505 # This line is compatible as-is with both Python 2 and 3
506 globals()[function_name] = target_function
508 def FormatSpecsListToDict(formatspecs=__file_format_list__):
509 if(isinstance(formatspecs, (list, tuple, ))):
510 return {'format_name': formatspecs[0], 'format_magic': formatspecs[1], 'format_lower': formatspecs[2], 'format_len': formatspecs[3], 'format_hex': formatspecs[4], 'format_delimiter': formatspecs[5], 'format_ver': formatspecs[6], 'new_style': formatspecs[7], 'use_advanced_list': formatspecs[8], 'use_alt_inode': formatspecs[9]};
511 elif(isinstance(formatspecs, (dict, ))):
512 return formatspecs;
513 else:
514 return __file_format_dict__;
515 return __file_format_dict__;
517 def TarFileCheck(infile):
518 try:
519 if is_tarfile(infile):
520 return True;
521 else:
522 return False;
523 except TypeError:
524 try:
525 # Check if the input is a file object
526 if hasattr(infile, 'read'):
527 # Save the current file position
528 current_position = infile.tell();
529 # Attempt to open the file object as a tar file
530 tar = tarfile.open(fileobj=infile);
531 tar.close();
532 # Restore the file position
533 infile.seek(current_position);
534 return True;
535 else:
536 # Assume it's a filename
537 tar = tarfile.open(name=infile);
538 tar.close();
539 return True;
540 except tarfile.TarError:
541 return False;
543 # initial_value can be 0xFFFF or 0x0000
544 def crc16_ansi(msg, initial_value=0xFFFF):
545 # CRC-16-IBM / CRC-16-ANSI polynomial and initial value
546 poly = 0x8005; # Polynomial for CRC-16-IBM / CRC-16-ANSI
547 crc = initial_value; # Initial value
548 for b in msg:
549 crc ^= b << 8; # XOR byte into CRC top byte
550 for _ in range(8): # Process each bit
551 if crc & 0x8000: # If the top bit is set
552 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
553 else:
554 crc = crc << 1; # Just shift left
555 crc &= 0xFFFF; # Ensure CRC remains 16-bit
556 return crc;
558 # initial_value can be 0xFFFF or 0x0000
559 def crc16_ibm(msg, initial_value=0xFFFF):
560 return crc16_ansi(msg, initial_value);
562 # initial_value is 0xFFFF
563 def crc16(msg):
564 return crc16_ansi(msg, 0xFFFF);
566 # initial_value can be 0xFFFF, 0x1D0F or 0x0000
567 def crc16_ccitt(msg, initial_value=0xFFFF):
568 # CRC-16-CCITT polynomial
569 poly = 0x1021; # Polynomial for CRC-16-CCITT
570 # Use the specified initial value
571 crc = initial_value;
572 for b in msg:
573 crc ^= b << 8; # XOR byte into CRC top byte
574 for _ in range(8): # Process each bit
575 if crc & 0x8000: # If the top bit is set
576 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
577 else:
578 crc = crc << 1; # Just shift left
579 crc &= 0xFFFF; # Ensure CRC remains 16-bit
580 return crc;
582 # initial_value can be 0x42F0E1EBA9EA3693 or 0x0000000000000000
583 def crc64_ecma(msg, initial_value=0x0000000000000000):
584 # CRC-64-ECMA polynomial and initial value
585 poly = 0x42F0E1EBA9EA3693;
586 crc = initial_value; # Initial value for CRC-64-ECMA
587 for b in msg:
588 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
589 for _ in range(8): # Process each bit
590 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
591 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
592 else:
593 crc <<= 1; # Just shift left if the MSB is 0
594 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
595 return crc;
597 # initial_value can be 0x000000000000001B or 0xFFFFFFFFFFFFFFFF
598 def crc64_iso(msg, initial_value=0xFFFFFFFFFFFFFFFF):
599 # CRC-64-ISO polynomial and initial value
600 poly = 0x000000000000001B;
601 crc = initial_value; # Common initial value for CRC-64-ISO
602 for b in msg:
603 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
604 for _ in range(8): # Process each bit
605 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
606 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
607 else:
608 crc <<= 1; # Just shift left if the MSB is 0
609 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
610 return crc;
612 def GetDataFromArray(data, path, default=None):
613 element = data;
614 try:
615 for key in path:
616 element = element[key];
617 return element;
618 except (KeyError, TypeError, IndexError):
619 return default;
621 def GetDataFromArrayAlt(structure, path, default=None):
622 element = structure;
623 for key in path:
624 if isinstance(element, dict) and key in element:
625 element = element[key];
626 elif isinstance(element, list) and isinstance(key, int) and -len(element) <= key < len(element):
627 element = element[key];
628 else:
629 return default;
630 return element;
632 def GetHeaderChecksum(inlist=[], checksumtype="crc32", encodedata=True, formatspecs=__file_format_dict__):
633 formatspecs = FormatSpecsListToDict(formatspecs);
634 fileheader = AppendNullBytes(inlist, formatspecs['format_delimiter']) if isinstance(inlist, list) else AppendNullByte(inlist, formatspecs['format_delimiter']);
635 if encodedata:
636 fileheader = fileheader.encode('UTF-8');
637 checksum_methods = {
638 "crc16": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
639 "crc16_ansi": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
640 "crc16_ibm": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
641 "crc16_ccitt": lambda data: format(crc16_ccitt(data) & 0xffff, '04x').lower(),
642 "adler32": lambda data: format(zlib.adler32(data) & 0xffffffff, '08x').lower(),
643 "crc32": lambda data: format(crc32(data) & 0xffffffff, '08x').lower(),
644 "crc64_ecma": lambda data: format(crc64_ecma(data) & 0xffffffffffffffff, '016x').lower(),
645 "crc64": lambda data: format(crc64_iso(data) & 0xffffffffffffffff, '016x').lower(),
646 "crc64_iso": lambda data: format(crc64_iso(data) & 0xffffffffffffffff, '016x').lower(),
648 if checksumtype in checksum_methods:
649 return checksum_methods[checksumtype](fileheader);
650 elif CheckSumSupportAlt(checksumtype, hashlib_guaranteed):
651 checksumoutstr = hashlib.new(checksumtype);
652 checksumoutstr.update(fileheader);
653 return checksumoutstr.hexdigest().lower();
654 return format(0, 'x').lower();
656 def GetFileChecksum(instr, checksumtype="crc32", encodedata=True, formatspecs=__file_format_dict__):
657 formatspecs = FormatSpecsListToDict(formatspecs);
658 if encodedata:
659 instr = instr.encode('UTF-8');
660 checksum_methods = {
661 "crc16": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
662 "crc16_ansi": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
663 "crc16_ibm": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
664 "crc16_ccitt": lambda data: format(crc16_ccitt(data) & 0xffff, '04x').lower(),
665 "adler32": lambda data: format(zlib.adler32(data) & 0xffffffff, '08x').lower(),
666 "crc32": lambda data: format(crc32(data) & 0xffffffff, '08x').lower(),
667 "crc64_ecma": lambda data: format(crc64_ecma(data) & 0xffffffffffffffff, '016x').lower(),
668 "crc64": lambda data: format(crc64_iso(data) & 0xffffffffffffffff, '016x').lower(),
669 "crc64_iso": lambda data: format(crc64_iso(data) & 0xffffffffffffffff, '016x').lower(),
671 if checksumtype in checksum_methods:
672 return checksum_methods[checksumtype](instr);
673 elif CheckSumSupportAlt(checksumtype, hashlib_guaranteed):
674 checksumoutstr = hashlib.new(checksumtype);
675 checksumoutstr.update(instr);
676 return checksumoutstr.hexdigest().lower();
677 return format(0, 'x').lower();
679 def ValidateHeaderChecksum(inlist=[], checksumtype="crc32", inchecksum="0", formatspecs=__file_format_dict__):
680 formatspecs = FormatSpecsListToDict(formatspecs);
681 catfileheadercshex = GetHeaderChecksum(inlist, checksumtype, True, formatspecs).lower();
682 return inchecksum.lower() == catfileheadercshex;
684 def ValidateFileChecksum(infile, checksumtype="crc32", inchecksum="0", formatspecs=__file_format_dict__):
685 formatspecs = FormatSpecsListToDict(formatspecs);
686 catinfilecshex = GetFileChecksum(infile, checksumtype, True, formatspecs).lower();
687 return inchecksum.lower() == catinfilecshex;
689 def ReadTillNullByteOld(fp, delimiter=__file_format_dict__['format_delimiter']):
690 curbyte = b"";
691 curfullbyte = b"";
692 nullbyte = delimiter.encode("UTF-8");
693 while(True):
694 curbyte = fp.read(1);
695 if(curbyte==nullbyte or not curbyte):
696 break;
697 curfullbyte = curfullbyte + curbyte;
698 return curfullbyte.decode('UTF-8');
700 def ReadUntilNullByteOld(fp, delimiter=__file_format_dict__['format_delimiter']):
701 return ReadTillNullByteOld(fp, delimiter);
703 def ReadTillNullByteAlt(fp, delimiter=__file_format_dict__['format_delimiter'], chunk_size=1024, max_read=1024000):
704 delimiter = delimiter.encode('UTF-8') # Ensure the delimiter is in bytes
705 buffer = bytearray();
706 total_read = 0;
707 delimiter_length = len(delimiter);
708 while True:
709 chunk = fp.read(chunk_size)
710 if not chunk:
711 # End of file reached without finding the delimiter
712 break;
713 buffer.extend(chunk);
714 total_read += len(chunk);
715 if delimiter in buffer:
716 # Delimiter found, calculate where to reset the file pointer
717 index = buffer.find(delimiter);
718 # Calculate how many extra bytes were read after the delimiter
719 extra_bytes_read = len(buffer) - (index + delimiter_length);
720 # Move the file pointer back to just after the delimiter
721 fp.seek(-extra_bytes_read, 1);
722 buffer = buffer[:index];
723 break;
724 if total_read >= max_read:
725 # Stop reading if max limit is reached to prevent excessive memory usage
726 raise MemoryError("Maximum read limit reached without finding the delimiter.");
727 # Check for incomplete UTF-8 sequences at the end of the buffer
728 if len(buffer) > 1 and 128 <= buffer[-1] < 192:
729 # This suggests that the last byte might be the start of a multi-byte character
730 # Try to read one more byte to complete the character
731 extra_byte = fp.read(1);
732 if extra_byte:
733 buffer.extend(extra_byte);
734 else:
735 # No more data available
736 break;
737 try:
738 return buffer.decode('UTF-8', errors='replace');
739 except UnicodeDecodeError:
740 return buffer.decode('UTF-8', errors='replace');
742 def ReadUntilNullByteAlt(fp, delimiter=__file_format_dict__['format_delimiter'], chunk_size=1024, max_read=1024000):
743 return ReadTillNullByteAlt(fp, delimiter, chunk_size, max_read);
745 def ReadTillNullByte(fp, delimiter=__file_format_dict__['format_delimiter'], max_read=1024000):
746 curfullbyte = bytearray();
747 nullbyte = delimiter.encode("UTF-8");
748 total_read = 0; # Track the total number of bytes read
749 while True:
750 curbyte = fp.read(1);
751 if curbyte == nullbyte or not curbyte:
752 break;
753 curfullbyte.extend(curbyte);
754 total_read += 1;
755 if total_read >= max_read:
756 raise MemoryError("Maximum read limit reached without finding the delimiter.");
757 # Decode the full byte array to string once out of the loop
758 try:
759 return curfullbyte.decode('UTF-8');
760 except UnicodeDecodeError:
761 # Handle potential partial UTF-8 characters
762 for i in range(1, 4):
763 try:
764 return curfullbyte[:-i].decode('UTF-8');
765 except UnicodeDecodeError:
766 continue;
767 raise; # Re-raise if decoding fails even after trimming
769 def ReadUntilNullByte(fp, delimiter=__file_format_dict__['format_delimiter'], max_read=1024000):
770 return ReadTillNullByte(fp, delimiter, max_read);
772 def ReadTillNullByteByNum(fp, delimiter=__file_format_dict__['format_delimiter'], num_delimiters=1, chunk_size=1024, max_read=1024000):
773 delimiter = delimiter.encode('UTF-8'); # Ensure the delimiter is in bytes
774 buffer = bytearray();
775 total_read = 0;
776 delimiter_length = len(delimiter);
777 results = [];
778 while len(results) < num_delimiters:
779 chunk = fp.read(chunk_size);
780 if not chunk:
781 # End of file reached; decode whatever is collected if it's the last needed part
782 if len(buffer) > 0:
783 results.append(buffer.decode('UTF-8', errors='replace'));
784 break;
785 buffer.extend(chunk)
786 total_read += len(chunk);
787 # Check if we have found the delimiter
788 while delimiter in buffer:
789 index = buffer.find(delimiter);
790 # Decode the section before the delimiter
791 results.append(buffer[:index].decode('UTF-8', errors='replace'));
792 # Remove the processed part from the buffer
793 buffer = buffer[index + delimiter_length:];
794 if len(results) == num_delimiters:
795 # If reached the required number of delimiters, adjust the file pointer and stop
796 fp.seek(-len(buffer), 1);
797 return results;
798 if total_read >= max_read:
799 # Stop reading if max limit is reached to prevent excessive memory usage
800 raise MemoryError("Maximum read limit reached without finding the delimiter.");
801 # Check for incomplete UTF-8 sequences at the end of the buffer
802 if len(buffer) > 1 and 128 <= buffer[-1] < 192:
803 # This suggests that the last byte might be the start of a multi-byte character
804 # Try to read one more byte to complete the character
805 extra_byte = fp.read(1);
806 if extra_byte:
807 buffer.extend(extra_byte);
808 else:
809 # No more data available
810 break;
811 # Process remaining buffer if less than the required number of delimiters were found
812 if len(buffer) > 0 and len(results) < num_delimiters:
813 results.append(buffer.decode('UTF-8', errors='replace'));
814 return results;
816 def ReadUntilNullByteByNum(fp, delimiter=__file_format_dict__['format_delimiter'], num_delimiters=1, chunk_size=1024, max_read=1024000):
817 return ReadTillNullByteByNum(fp, delimiter, num_delimiters, chunk_size, max_read);
819 def SeekToEndOfFile(fp):
820 lasttell = 0;
821 while(True):
822 fp.seek(1, 1);
823 if(lasttell==fp.tell()):
824 break;
825 lasttell = fp.tell();
826 return True;
828 def ReadFileHeaderData(fp, rounds=0, delimiter=__file_format_dict__['format_delimiter']):
829 rocount = 0;
830 roend = int(rounds);
831 HeaderOut = [];
832 while(rocount<roend):
833 HeaderOut.append(ReadTillNullByte(fp, delimiter));
834 rocount = rocount + 1;
835 return HeaderOut;
837 def ReadFileHeaderDataBySize(fp, delimiter=__file_format_dict__['format_delimiter']):
838 headerpresize = ReadTillNullByte(fp, delimiter);
839 headersize = int(headerpresize, 16);
840 if(headersize<=0):
841 return [];
842 headercontent = str(fp.read(headersize).decode('UTF-8')).split(delimiter);
843 fp.seek(1, 1);
844 rocount = 0;
845 roend = int(len(headercontent));
846 HeaderOut = [headerpresize];
847 while(rocount<roend):
848 HeaderOut.append(headercontent[rocount]);
849 rocount = rocount + 1;
850 return HeaderOut;
852 def ReadFileHeaderDataWoSize(fp, delimiter=__file_format_dict__['format_delimiter']):
853 preheaderdata = ReadFileHeaderData(fp, 2, delimiter);
854 headersize = int(preheaderdata[0], 16);
855 headernumfields = int(preheaderdata[1], 16);
856 if(headersize<=0 or headernumfields<=0):
857 return [];
858 headerdata = ReadTillNullByteByNum(fp, delimiter, headernumfields);
859 #headerdata = ReadFileHeaderData(fp, headernumfields, delimiter);
860 HeaderOut = preheaderdata + headerdata;
861 return HeaderOut;
863 def ReadFileHeaderDataBySizeWithContent(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
864 formatspecs = FormatSpecsListToDict(formatspecs);
865 delimiter = formatspecs['format_delimiter'];
866 fheaderstart = fp.tell();
867 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
868 if(len(HeaderOut)==0):
869 return False;
870 if(re.findall(r"^[.|/]", HeaderOut[3])):
871 fname = HeaderOut[3];
872 else:
873 fname = "./"+HeaderOut[3];
874 fcs = HeaderOut[-2].lower();
875 fccs = HeaderOut[-1].lower();
876 fsize = int(HeaderOut[5], 16);
877 fcompression = HeaderOut[12];
878 fcsize = int(HeaderOut[13], 16);
879 fseeknextfile = HeaderOut[25];
880 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-4].lower(), True, formatspecs);
881 if(fcs!=newfcs and not skipchecksum):
882 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
883 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
884 return False;
885 fhend = fp.tell() - 1;
886 fcontentstart = fp.tell();
887 fcontents = BytesIO();
888 if(fsize>0 and not listonly):
889 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
890 fcontents.write(fp.read(fsize));
891 else:
892 fcontents.write(fp.read(fcsize));
893 elif(fsize>0 and listonly):
894 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
895 fp.seek(fsize, 1);
896 else:
897 fp.seek(fcsize, 1);
898 fcontents.seek(0, 0);
899 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
900 if(fccs!=newfccs and not skipchecksum and not listonly):
901 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
902 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
903 return False;
904 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
905 pass;
906 else:
907 fcontents.seek(0, 0);
908 if(uncompress):
909 fcontents = UncompressArchiveFile(fcontents, formatspecs);
910 fcontentend = fp.tell();
911 if(re.findall(r"^\+([0-9]+)", fseeknextfile)):
912 fseeknextasnum = int(fseeknextfile.replace("+", ""));
913 if(abs(fseeknextasnum)==0):
914 pass;
915 fp.seek(fseeknextasnum, 1);
916 elif(re.findall(r"^\-([0-9]+)", fseeknextfile)):
917 fseeknextasnum = int(fseeknextfile);
918 if(abs(fseeknextasnum)==0):
919 pass;
920 fp.seek(fseeknextasnum, 1);
921 elif(re.findall(r"^([0-9]+)", fseeknextfile)):
922 fseeknextasnum = int(fseeknextfile);
923 if(abs(fseeknextasnum)==0):
924 pass;
925 fp.seek(fseeknextasnum, 0);
926 else:
927 return False;
928 HeaderOut.append(fcontents);
929 return HeaderOut;
931 def ReadFileHeaderDataBySizeWithContentToArray(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
932 formatspecs = FormatSpecsListToDict(formatspecs);
933 delimiter = formatspecs['format_delimiter'];
934 fheaderstart = fp.tell();
935 if(formatspecs['new_style']):
936 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
937 else:
938 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
939 if(len(HeaderOut)==0):
940 return False;
941 fheadsize = int(HeaderOut[0], 16);
942 fnumfields = int(HeaderOut[1], 16);
943 ftype = int(HeaderOut[2], 16);
944 if(re.findall(r"^[.|/]", HeaderOut[3])):
945 fname = HeaderOut[3];
946 else:
947 fname = "./"+HeaderOut[3];
948 fbasedir = os.path.dirname(fname);
949 flinkname = HeaderOut[4];
950 fsize = int(HeaderOut[5], 16);
951 fatime = int(HeaderOut[6], 16);
952 fmtime = int(HeaderOut[7], 16);
953 fctime = int(HeaderOut[8], 16);
954 fbtime = int(HeaderOut[9], 16);
955 fmode = int(HeaderOut[10], 16);
956 fchmode = stat.S_IMODE(fmode);
957 ftypemod = stat.S_IFMT(fmode);
958 fwinattributes = int(HeaderOut[11], 16);
959 fcompression = HeaderOut[12];
960 fcsize = int(HeaderOut[13], 16);
961 fuid = int(HeaderOut[14], 16);
962 funame = HeaderOut[15];
963 fgid = int(HeaderOut[16], 16);
964 fgname = HeaderOut[17];
965 fid = int(HeaderOut[18], 16);
966 finode = int(HeaderOut[19], 16);
967 flinkcount = int(HeaderOut[20], 16);
968 fdev_minor = int(HeaderOut[21], 16);
969 fdev_major = int(HeaderOut[22], 16);
970 frdev_minor = int(HeaderOut[23], 16);
971 frdev_major = int(HeaderOut[24], 16);
972 fseeknextfile = HeaderOut[25];
973 fextrasize = int(HeaderOut[26], 16);
974 fextrafields = int(HeaderOut[27], 16);
975 extrafieldslist = [];
976 extrastart = 28
977 extraend = extrastart + fextrafields;
978 extrafieldslist = [];
979 if(extrastart<extraend):
980 extrafieldslist.append(HeaderOut[extrastart]);
981 extrastart = extrastart + 1;
982 fcs = HeaderOut[-2].lower();
983 fccs = HeaderOut[-1].lower();
984 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-4].lower(), True, formatspecs);
985 if(fcs!=newfcs and not skipchecksum):
986 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
987 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
988 return False;
989 fhend = fp.tell() - 1;
990 fcontentstart = fp.tell();
991 fcontents = BytesIO();
992 pyhascontents = False;
993 if(fsize>0 and not listonly):
994 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
995 fcontents.write(fp.read(fsize));
996 else:
997 fcontents.write(fp.read(fcsize));
998 pyhascontents = True;
999 elif(fsize>0 and listonly):
1000 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1001 fp.seek(fsize, 1);
1002 else:
1003 fp.seek(fcsize, 1);
1004 pyhascontents = False;
1005 fcontents.seek(0, 0);
1006 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
1007 if(fccs!=newfccs and not skipchecksum and not listonly):
1008 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
1009 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
1010 return False;
1011 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1012 pass;
1013 else:
1014 fcontents.seek(0, 0);
1015 if(uncompress):
1016 fcontents = UncompressArchiveFile(fcontents, formatspecs);
1017 fcontents.seek(0, 0);
1018 fccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
1019 fcontentend = fp.tell() - 1;
1020 if(re.findall(r"^\+([0-9]+)", fseeknextfile)):
1021 fseeknextasnum = int(fseeknextfile.replace("+", ""));
1022 if(abs(fseeknextasnum)==0):
1023 pass;
1024 fp.seek(fseeknextasnum, 1);
1025 elif(re.findall(r"^\-([0-9]+)", fseeknextfile)):
1026 fseeknextasnum = int(fseeknextfile);
1027 if(abs(fseeknextasnum)==0):
1028 pass;
1029 fp.seek(fseeknextasnum, 1);
1030 elif(re.findall(r"^([0-9]+)", fseeknextfile)):
1031 fseeknextasnum = int(fseeknextfile);
1032 if(abs(fseeknextasnum)==0):
1033 pass;
1034 fp.seek(fseeknextasnum, 0);
1035 else:
1036 return False;
1037 fcontents.seek(0, 0);
1038 catlist = {'fheadersize': fheadsize, 'fhstart': fheaderstart, 'fhend': fhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': fseeknextfile, 'fheaderchecksumtype': HeaderOut[-4], 'fcontentchecksumtype': HeaderOut[-3], 'fnumfields': fnumfields + 2, 'frawheader': HeaderOut, 'fextrafields': fextrafields, 'fextrafieldsize': fextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': fcs, 'fcontentchecksum': fccs, 'fhascontents': pyhascontents, 'fcontentstart': fcontentstart, 'fcontentend': fcontentend, 'fcontents': fcontents};
1039 return catlist;
1041 def ReadFileHeaderDataBySizeWithContentToList(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1042 formatspecs = FormatSpecsListToDict(formatspecs);
1043 delimiter = formatspecs['format_delimiter'];
1044 fheaderstart = fp.tell();
1045 if(formatspecs['new_style']):
1046 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
1047 else:
1048 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
1049 if(len(HeaderOut)==0):
1050 return False;
1051 fheadsize = int(HeaderOut[0], 16);
1052 fnumfields = int(HeaderOut[1], 16);
1053 ftype = int(HeaderOut[2], 16);
1054 if(re.findall(r"^[.|/]", HeaderOut[3])):
1055 fname = HeaderOut[3];
1056 else:
1057 fname = "./"+HeaderOut[3];
1058 fbasedir = os.path.dirname(fname);
1059 flinkname = HeaderOut[4];
1060 fsize = int(HeaderOut[5], 16);
1061 fatime = int(HeaderOut[6], 16);
1062 fmtime = int(HeaderOut[7], 16);
1063 fctime = int(HeaderOut[8], 16);
1064 fbtime = int(HeaderOut[9], 16);
1065 fmode = int(HeaderOut[10], 16);
1066 fchmode = stat.S_IMODE(fmode);
1067 ftypemod = stat.S_IFMT(fmode);
1068 fwinattributes = int(HeaderOut[11], 16);
1069 fcompression = HeaderOut[12];
1070 fcsize = int(HeaderOut[13], 16);
1071 fuid = int(HeaderOut[14], 16);
1072 funame = HeaderOut[15];
1073 fgid = int(HeaderOut[16], 16);
1074 fgname = HeaderOut[17];
1075 fid = int(HeaderOut[18], 16);
1076 finode = int(HeaderOut[19], 16);
1077 flinkcount = int(HeaderOut[20], 16);
1078 fdev_minor = int(HeaderOut[21], 16);
1079 fdev_major = int(HeaderOut[22], 16);
1080 frdev_minor = int(HeaderOut[23], 16);
1081 frdev_major = int(HeaderOut[24], 16);
1082 fseeknextfile = HeaderOut[25];
1083 fextrasize = int(HeaderOut[26], 16);
1084 fextrafields = int(HeaderOut[27], 16);
1085 extrafieldslist = [];
1086 extrastart = 28;
1087 extraend = extrastart + fextrafields;
1088 extrafieldslist = [];
1089 if(extrastart<extraend):
1090 extrafieldslist.append(HeaderOut[extrastart]);
1091 extrastart = extrastart + 1;
1092 fheaderchecksumtype = [extrastart].lower();
1093 fcontentchecksumtype = [extrastart + 1].lower();
1094 fcs = HeaderOut[-2].lower();
1095 fccs = HeaderOut[-1].lower();
1096 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-4].lower(), True, formatspecs);
1097 if(fcs!=newfcs and not skipchecksum):
1098 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
1099 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
1100 return False;
1101 fhend = fp.tell() - 1;
1102 fcontentstart = fp.tell();
1103 fcontents = BytesIO();
1104 pyhascontents = False;
1105 if(fsize>0 and not listonly):
1106 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1107 fcontents.write(fp.read(fsize));
1108 else:
1109 fcontents.write(fp.read(fcsize));
1110 pyhascontents = True;
1111 elif(fsize>0 and listonly):
1112 if(fcompression=="none" or fcompression=="" or fcompression=="atuo"):
1113 fp.seek(fsize, 1);
1114 else:
1115 fp.seek(fcsize, 1);
1116 pyhascontents = False;
1117 fcontents.seek(0, 0);
1118 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
1119 if(fccs!=newfccs and not skipchecksum and not listonly):
1120 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
1121 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
1122 return False;
1123 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1124 pass;
1125 else:
1126 fcontents.seek(0, 0);
1127 if(uncompress):
1128 fcontents = UncompressArchiveFile(fcontents, formatspecs);
1129 fcontents.seek(0, 0);
1130 fcontentend = fp.tell() - 1;
1131 if(re.findall(r"^\+([0-9]+)", fseeknextfile)):
1132 fseeknextasnum = int(fseeknextfile.replace("+", ""));
1133 if(abs(fseeknextasnum)==0):
1134 pass;
1135 fp.seek(fseeknextasnum, 1);
1136 elif(re.findall(r"^\-([0-9]+)", fseeknextfile)):
1137 fseeknextasnum = int(fseeknextfile);
1138 if(abs(fseeknextasnum)==0):
1139 pass;
1140 fp.seek(fseeknextasnum, 1);
1141 elif(re.findall(r"^([0-9]+)", fseeknextfile)):
1142 fseeknextasnum = int(fseeknextfile);
1143 if(abs(fseeknextasnum)==0):
1144 pass;
1145 fp.seek(fseeknextasnum, 0);
1146 else:
1147 return False;
1148 catlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, fseeknextfile, extrafieldslist, fheaderchecksumtype, fcontentchecksumtype, fcontents];
1149 return catlist;
1151 def ReadFileDataBySizeWithContent(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1152 formatspecs = FormatSpecsListToDict(formatspecs);
1153 delimiter = formatspecs['format_delimiter'];
1154 curloc = fp.tell();
1155 if(curloc>0):
1156 fp.seek(0, 0);
1157 catheader = ReadFileHeaderData(fp, 4, delimiter);
1158 if(curloc>0):
1159 fp.seek(curloc, 0);
1160 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1161 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1162 if(not headercheck and not skipchecksum):
1163 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1164 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1165 return False;
1166 fnumfiles = int(catheader[1], 16);
1167 countnum = 0;
1168 flist = [];
1169 while(countnum < fnumfiles):
1170 HeaderOut = ReadFileHeaderDataBySizeWithContent(fp, listonly, uncompress, skipchecksum, formatspecs)
1171 if(len(HeaderOut)==0):
1172 break;
1173 flist.append(HeaderOut);
1174 countnum = countnum + 1;
1175 return flist;
1177 def ReadFileDataBySizeWithContentToArray(fp, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1178 formatspecs = FormatSpecsListToDict(formatspecs);
1179 delimiter = formatspecs['format_delimiter'];
1180 curloc = fp.tell();
1181 if(curloc>0):
1182 fp.seek(0, 0);
1183 catheader = ReadFileHeaderData(fp, 4, delimiter);
1184 if(curloc>0):
1185 fp.seek(curloc, 0);
1186 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1187 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1188 if(not headercheck and not skipchecksum):
1189 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1190 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1191 return False;
1192 catstring = catheader[0];
1193 catversion = re.findall(r"([\d]+)", catstring);
1194 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
1195 fprenumfiles = catheader[1];
1196 fnumfiles = int(fprenumfiles, 16);
1197 fprechecksumtype = catheader[2];
1198 fprechecksum = catheader[3];
1199 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': []};
1200 if(seekstart<0 and seekstart>fnumfiles):
1201 seekstart = 0;
1202 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
1203 seekend = fnumfiles;
1204 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
1205 seekend = fnumfiles - abs(seekend);
1206 if(seekstart>0):
1207 il = 0;
1208 while(il < seekstart):
1209 prefhstart = fp.tell();
1210 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs['format_delimiter']);
1211 if(len(preheaderdata)==0):
1212 break;
1213 prefsize = int(preheaderdata[5], 16);
1214 prefseeknextfile = preheaderdata[25];
1215 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-4].lower(), True, formatspecs);
1216 prefcs = preheaderdata[-2];
1217 if(prefcs!=prenewfcs and not skipchecksum):
1218 VVerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
1219 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
1220 return False;
1221 valid_archive = False;
1222 invalid_archive = True;
1223 prefhend = fp.tell() - 1;
1224 prefcontentstart = fp.tell();
1225 prefcontents = BytesIO();
1226 pyhascontents = False;
1227 if(prefsize>0):
1228 prefcontents.write(fp.read(prefsize));
1229 prefcontents.seek(0, 0);
1230 prenewfccs = GetFileChecksum(prefcontents.read(), preheaderdata[-3].lower(), False, formatspecs);
1231 prefccs = preheaderdata[-1];
1232 pyhascontents = True;
1233 if(prefccs!=prenewfccs and not skipchecksum):
1234 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
1235 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
1236 return False;
1237 if(re.findall(r"^\+([0-9]+)", prefseeknextfile)):
1238 fseeknextasnum = int(prefseeknextfile.replace("+", ""));
1239 if(abs(fseeknextasnum)==0):
1240 pass;
1241 fp.seek(fseeknextasnum, 1);
1242 elif(re.findall(r"^\-([0-9]+)", prefseeknextfile)):
1243 fseeknextasnum = int(prefseeknextfile);
1244 if(abs(fseeknextasnum)==0):
1245 pass;
1246 fp.seek(fseeknextasnum, 1);
1247 elif(re.findall(r"^([0-9]+)", prefseeknextfile)):
1248 fseeknextasnum = int(prefseeknextfile);
1249 if(abs(fseeknextasnum)==0):
1250 pass;
1251 fp.seek(fseeknextasnum, 0);
1252 else:
1253 return False;
1254 il = il + 1;
1255 realidnum = 0;
1256 countnum = seekstart;
1257 while(countnum < seekend):
1258 HeaderOut = ReadFileHeaderDataBySizeWithContentToArray(fp, listonly, uncompress, skipchecksum, formatspecs);
1259 if(len(HeaderOut)==0):
1260 break;
1261 HeaderOut.update({'fid': realidnum, 'fidalt': realidnum});
1262 catlist['ffilelist'].append(HeaderOut);
1263 countnum = countnum + 1;
1264 realidnum = realidnum + 1;
1265 return catlist;
1267 def ReadFileDataBySizeWithContentToList(fp, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1268 formatspecs = FormatSpecsListToDict(formatspecs);
1269 delimiter = formatspecs['format_delimiter'];
1270 curloc = fp.tell();
1271 if(curloc>0):
1272 fp.seek(0, 0);
1273 catheader = ReadFileHeaderData(fp, 4, delimiter);
1274 if(curloc>0):
1275 fp.seek(curloc, 0);
1276 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1277 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1278 if(not headercheck and not skipchecksum):
1279 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1280 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1281 return False;
1282 catstring = catheader[0];
1283 catversion = re.findall(r"([\d]+)", catstring);
1284 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
1285 fprenumfiles = catheader[1];
1286 fnumfiles = int(fprenumfiles, 16);
1287 fprechecksumtype = catheader[2];
1288 fprechecksum = catheader[3];
1289 catlist = [];
1290 if(seekstart<0 and seekstart>fnumfiles):
1291 seekstart = 0;
1292 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
1293 seekend = fnumfiles;
1294 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
1295 seekend = fnumfiles - abs(seekend);
1296 if(seekstart>0):
1297 il = 0;
1298 while(il < seekstart):
1299 prefhstart = fp.tell();
1300 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs['format_delimiter']);
1301 if(len(preheaderdata)==0):
1302 break;
1303 prefsize = int(preheaderdata[5], 16);
1304 prefcompression = preheaderdata[12];
1305 prefcsize = int(preheaderdata[13], 16);
1306 prefseeknextfile = HeaderOut[25];
1307 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-4].lower(), True, formatspecs);
1308 prefcs = preheaderdata[-2];
1309 if(prefcs!=prenewfcs and not skipchecksum):
1310 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
1311 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
1312 return False;
1313 valid_archive = False;
1314 invalid_archive = True;
1315 prefhend = fp.tell() - 1;
1316 prefcontentstart = fp.tell();
1317 prefcontents = "";
1318 pyhascontents = False;
1319 if(prefsize>0):
1320 if(prefcompression=="none" or prefcompression=="" or prefcompression=="auto"):
1321 prefcontents = catfp.read(prefsize);
1322 else:
1323 prefcontents = catfp.read(prefcsize);
1324 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
1325 prefccs = preheaderdata[-1];
1326 pyhascontents = True;
1327 if(prefccs!=prenewfccs and not skipchecksum):
1328 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
1329 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
1330 return False;
1331 if(re.findall(r"^\+([0-9]+)", prefseeknextfile)):
1332 fseeknextasnum = int(prefseeknextfile.replace("+", ""));
1333 if(abs(fseeknextasnum)==0):
1334 pass;
1335 catfp.seek(fseeknextasnum, 1);
1336 elif(re.findall(r"^\-([0-9]+)", prefseeknextfile)):
1337 fseeknextasnum = int(prefseeknextfile);
1338 if(abs(fseeknextasnum)==0):
1339 pass;
1340 catfp.seek(fseeknextasnum, 1);
1341 elif(re.findall(r"^([0-9]+)", prefseeknextfile)):
1342 fseeknextasnum = int(prefseeknextfile);
1343 if(abs(fseeknextasnum)==0):
1344 pass;
1345 catfp.seek(fseeknextasnum, 0);
1346 else:
1347 return False;
1348 il = il + 1;
1349 realidnum = 0;
1350 countnum = seekstart;
1351 while(countnum < seekend):
1352 HeaderOut = ReadFileHeaderDataBySizeWithContentToList(fp, listonly, uncompress, skipchecksum, formatspecs);
1353 if(len(HeaderOut)==0):
1354 break;
1355 catlist.append(HeaderOut);
1356 countnum = countnum + 1;
1357 realidnum = realidnum + 1;
1358 return catlist;
1360 def ReadInFileBySizeWithContentToArray(infile, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1361 formatspecs = FormatSpecsListToDict(formatspecs);
1362 delimiter = formatspecs['format_delimiter'];
1363 if(hasattr(infile, "read") or hasattr(infile, "write")):
1364 fp = infile;
1365 fp.seek(0, 0);
1366 fp = UncompressArchiveFile(fp, formatspecs);
1367 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1368 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1369 return False;
1370 if(not fp):
1371 return False;
1372 fp.seek(0, 0);
1373 elif(infile=="-"):
1374 fp = BytesIO();
1375 if(hasattr(sys.stdin, "buffer")):
1376 shutil.copyfileobj(sys.stdin.buffer, fp);
1377 else:
1378 shutil.copyfileobj(sys.stdin, fp);
1379 fp.seek(0, 0);
1380 fp = UncompressArchiveFile(fp, formatspecs);
1381 if(not fp):
1382 return False;
1383 fp.seek(0, 0);
1384 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
1385 fp = download_file_from_internet_file(infile);
1386 fp = UncompressArchiveFile(fp, formatspecs);
1387 fp.seek(0, 0);
1388 if(not fp):
1389 return False;
1390 fp.seek(0, 0);
1391 else:
1392 infile = RemoveWindowsPath(infile);
1393 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
1394 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1395 return False;
1396 compresscheck = CheckCompressionType(infile, formatspecs, True);
1397 if(not compresscheck):
1398 fextname = os.path.splitext(infile)[1];
1399 if(fextname==".gz"):
1400 compresscheck = "gzip";
1401 elif(fextname==".bz2"):
1402 compresscheck = "bzip2";
1403 elif(fextname==".zst"):
1404 compresscheck = "zstd";
1405 elif(fextname==".lz4" or fextname==".clz4"):
1406 compresscheck = "lz4";
1407 elif(fextname==".lzo" or fextname==".lzop"):
1408 compresscheck = "lzo";
1409 elif(fextname==".lzma" or fextname==".xz"):
1410 compresscheck = "lzma";
1411 else:
1412 return False;
1413 if(not compresscheck):
1414 return False;
1415 fp = UncompressFile(infile, formatspecs, "rb");
1416 return ReadFileDataBySizeWithContentToArray(fp, listonly, uncompress, skipchecksum, formatspecs);
1418 def ReadInFileBySizeWithContentToList(infile, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1419 formatspecs = FormatSpecsListToDict(formatspecs);
1420 delimiter = formatspecs['format_delimiter'];
1421 if(hasattr(infile, "read") or hasattr(infile, "write")):
1422 fp = infile;
1423 fp.seek(0, 0);
1424 fp = UncompressArchiveFile(fp, formatspecs);
1425 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1426 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1427 return False;
1428 if(not fp):
1429 return False;
1430 fp.seek(0, 0);
1431 elif(infile=="-"):
1432 fp = BytesIO();
1433 if(hasattr(sys.stdin, "buffer")):
1434 shutil.copyfileobj(sys.stdin.buffer, fp);
1435 else:
1436 shutil.copyfileobj(sys.stdin, fp);
1437 fp.seek(0, 0);
1438 fp = UncompressArchiveFile(fp, formatspecs);
1439 if(not fp):
1440 return False;
1441 fp.seek(0, 0);
1442 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
1443 fp = download_file_from_internet_file(infile);
1444 fp = UncompressArchiveFile(fp, formatspecs);
1445 fp.seek(0, 0);
1446 if(not fp):
1447 return False;
1448 fp.seek(0, 0);
1449 else:
1450 infile = RemoveWindowsPath(infile);
1451 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
1452 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1453 return False;
1454 compresscheck = CheckCompressionType(infile, formatspecs, True);
1455 if(not compresscheck):
1456 fextname = os.path.splitext(infile)[1];
1457 if(fextname==".gz"):
1458 compresscheck = "gzip";
1459 elif(fextname==".bz2"):
1460 compresscheck = "bzip2";
1461 elif(fextname==".zst"):
1462 compresscheck = "zstd";
1463 elif(fextname==".lz4" or fextname==".clz4"):
1464 compresscheck = "lz4";
1465 elif(fextname==".lzo" or fextname==".lzop"):
1466 compresscheck = "lzo";
1467 elif(fextname==".lzma" or fextname==".xz"):
1468 compresscheck = "lzma";
1469 else:
1470 return False;
1471 if(not compresscheck):
1472 return False;
1473 fp = UncompressFile(infile, formatspecs, "rb");
1474 return ReadFileDataBySizeWithContentToList(fp, seekstart, seekend, listonly, uncompress, skipchecksum, formatspecs);
1476 def AppendNullByte(indata, delimiter=__file_format_dict__['format_delimiter']):
1477 outdata = str(indata) + delimiter;
1478 return outdata;
1480 def AppendNullBytes(indata=[], delimiter=__file_format_dict__['format_delimiter']):
1481 outdata = "";
1482 inum = 0;
1483 il = len(indata);
1484 while(inum < il):
1485 outdata = outdata + AppendNullByte(indata[inum], delimiter);
1486 inum = inum + 1;
1487 return outdata;
1489 def AppendFileHeader(fp, numfiles, checksumtype="crc32", formatspecs=__file_format_dict__):
1490 formatspecs = FormatSpecsListToDict(formatspecs);
1491 delimiter = formatspecs['format_delimiter'];
1492 catver = formatspecs['format_ver'];
1493 fileheaderver = str(int(catver.replace(".", "")));
1494 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
1495 fp.write(fileheader.encode('UTF-8'));
1496 fnumfiles = format(int(numfiles), 'x').lower();
1497 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs['format_delimiter']);
1498 catfileheadercshex = GetFileChecksum(fileheader + fnumfilesa, checksumtype, True, formatspecs);
1499 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
1500 fp.write(fnumfilesa.encode('UTF-8'));
1501 try:
1502 fp.flush();
1503 os.fsync(fp.fileno());
1504 except io.UnsupportedOperation:
1505 pass;
1506 except AttributeError:
1507 pass;
1508 except OSError as e:
1509 pass;
1510 return fp;
1512 def MakeEmptyFilePointer(fp, checksumtype="crc32", formatspecs=__file_format_dict__):
1513 formatspecs = FormatSpecsListToDict(formatspecs);
1514 AppendFileHeader(fp, 0, checksumtype, formatspecs);
1515 return fp;
1517 def MakeEmptyFile(outfile, compression="auto", compressionlevel=None, checksumtype="crc32", formatspecs=__file_format_dict__, returnfp=False):
1518 formatspecs = FormatSpecsListToDict(formatspecs);
1519 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1520 if(os.path.exists(outfile)):
1521 try:
1522 os.unlink(outfile);
1523 except OSError as e:
1524 pass;
1525 if(outfile=="-"):
1526 verbose = False;
1527 catfpfp = BytesIO();
1528 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1529 catfp = outfile;
1530 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1531 catfp = BytesIO();
1532 else:
1533 fbasename = os.path.splitext(outfile)[0];
1534 fextname = os.path.splitext(outfile)[1];
1535 catfp = CompressOpenFile(outfile, True, compressionlevel);
1536 catfp = AppendFileHeader(catfp, 0, checksumtype, formatspecs);
1537 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1538 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1539 try:
1540 catfp.flush();
1541 os.fsync(catfp.fileno());
1542 except io.UnsupportedOperation:
1543 pass;
1544 except AttributeError:
1545 pass;
1546 except OSError as e:
1547 pass;
1548 if(outfile=="-"):
1549 catfp.seek(0, 0);
1550 if(hasattr(sys.stdout, "buffer")):
1551 shutil.copyfileobj(catfp, sys.stdout.buffer);
1552 else:
1553 shutil.copyfileobj(catfp, sys.stdout);
1554 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1555 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1556 catfp.seek(0, 0);
1557 upload_file_to_internet_file(catfp, outfile);
1558 if(returnfp):
1559 catfp.seek(0, 0);
1560 return catfp;
1561 else:
1562 catfp.close();
1563 return True;
1565 def AppendFileHeaderWithContent(fp, filevalues=[], extradata=[], filecontent="", checksumtype="crc32", formatspecs=__file_format_dict__):
1566 formatspecs = FormatSpecsListToDict(formatspecs);
1567 extrafields = format(len(extradata), 'x').lower();
1568 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
1569 if(len(extradata)>0):
1570 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
1571 extrasizelen = format(len(extrasizestr), 'x').lower();
1572 catoutlen = len(filevalues) + len(extradata) + 5;
1573 catoutlenhex = format(catoutlen, 'x').lower();
1574 catoutlist = filevalues;
1575 catoutlist.insert(0, catoutlenhex);
1576 catoutlist.append(extrasizelen);
1577 catoutlist.append(extrafields);
1578 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
1579 if(len(extradata)>0):
1580 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
1581 catfileoutstr = catfileoutstr + AppendNullBytes([checksumtype, checksumtype], formatspecs['format_delimiter']);
1582 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
1583 catfilecontentcshex = GetFileChecksum(filecontent, checksumtype, False, formatspecs);
1584 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
1585 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
1586 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
1587 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
1588 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
1589 catfileoutstrecd = catfileoutstr.encode('UTF-8');
1590 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
1591 catfileout = catfileoutstrecd + filecontent + nullstrecd;
1592 fp.write(catfileout);
1593 try:
1594 fp.flush();
1595 os.fsync(fp.fileno());
1596 except io.UnsupportedOperation:
1597 pass;
1598 except AttributeError:
1599 pass;
1600 except OSError as e:
1601 pass;
1602 return fp;
1604 def AppendFilesWithContent(infiles, fp, dirlistfromtxt=False, filevalues=[], extradata=[], compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
1605 formatspecs = FormatSpecsListToDict(formatspecs);
1606 advancedlist = formatspecs['use_advanced_list'];
1607 altinode = formatspecs['use_alt_inode'];
1608 if(verbose):
1609 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
1610 if(infiles=="-"):
1611 for line in sys.stdin:
1612 infilelist.append(line.strip());
1613 infilelist = list(filter(None, infilelist));
1614 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
1615 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
1616 return False;
1617 with UncompressFile(infiles, formatspecs, "r") as finfile:
1618 for line in finfile:
1619 infilelist.append(line.strip());
1620 infilelist = list(filter(None, infilelist));
1621 else:
1622 if(isinstance(infiles, (list, tuple, ))):
1623 infilelist = list(filter(None, infiles));
1624 elif(isinstance(infiles, (str, ))):
1625 infilelist = list(filter(None, [infiles]));
1626 if(advancedlist):
1627 GetDirList = ListDirAdvanced(infilelist, followlink, False);
1628 else:
1629 GetDirList = ListDir(infilelist, followlink, False);
1630 if(not GetDirList):
1631 return False;
1632 curinode = 0;
1633 curfid = 0;
1634 inodelist = [];
1635 inodetofile = {};
1636 filetoinode = {};
1637 inodetocatinode = {};
1638 numfiles = int(len(GetDirList));
1639 fnumfiles = format(numfiles, 'x').lower();
1640 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
1641 for curfname in GetDirList:
1642 if(re.findall(r"^[.|/]", curfname)):
1643 fname = curfname;
1644 else:
1645 fname = "./"+curfname;
1646 if(verbose):
1647 VerbosePrintOut(fname);
1648 if(not followlink or followlink is None):
1649 fstatinfo = os.lstat(fname);
1650 else:
1651 fstatinfo = os.stat(fname);
1652 fpremode = fstatinfo.st_mode;
1653 finode = fstatinfo.st_ino;
1654 flinkcount = fstatinfo.st_nlink;
1655 ftype = 0;
1656 if(stat.S_ISREG(fpremode)):
1657 ftype = 0;
1658 elif(stat.S_ISLNK(fpremode)):
1659 ftype = 2;
1660 elif(stat.S_ISCHR(fpremode)):
1661 ftype = 3;
1662 elif(stat.S_ISBLK(fpremode)):
1663 ftype = 4;
1664 elif(stat.S_ISDIR(fpremode)):
1665 ftype = 5;
1666 elif(stat.S_ISFIFO(fpremode)):
1667 ftype = 6;
1668 elif(stat.S_ISSOCK(fpremode)):
1669 ftype = 8;
1670 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
1671 ftype = 9;
1672 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
1673 ftype = 10;
1674 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
1675 ftype = 11;
1676 else:
1677 ftype = 0;
1678 flinkname = "";
1679 fcurfid = format(int(curfid), 'x').lower();
1680 if(not followlink and finode!=0):
1681 if(ftype!=1):
1682 if(finode in inodelist):
1683 ftype = 1;
1684 flinkname = inodetofile[finode];
1685 if(altinode):
1686 fcurinode = format(int(finode), 'x').lower();
1687 else:
1688 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
1689 if(finode not in inodelist):
1690 inodelist.append(finode);
1691 inodetofile.update({finode: fname});
1692 inodetocatinode.update({finode: curinode});
1693 if(altinode):
1694 fcurinode = format(int(finode), 'x').lower();
1695 else:
1696 fcurinode = format(int(curinode), 'x').lower();
1697 curinode = curinode + 1;
1698 else:
1699 fcurinode = format(int(curinode), 'x').lower();
1700 curinode = curinode + 1;
1701 curfid = curfid + 1;
1702 if(ftype==2):
1703 flinkname = os.readlink(fname);
1704 fdev = fstatinfo.st_dev;
1705 getfdev = GetDevMajorMinor(fdev);
1706 fdev_minor = getfdev[0];
1707 fdev_major = getfdev[1];
1708 frdev = fstatinfo.st_dev;
1709 if(hasattr(fstatinfo, "st_rdev")):
1710 frdev = fstatinfo.st_rdev;
1711 else:
1712 frdev = fstatinfo.st_dev;
1713 getfrdev = GetDevMajorMinor(frdev);
1714 frdev_minor = getfrdev[0];
1715 frdev_major = getfrdev[1];
1716 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
1717 fsize = format(int("0"), 'x').lower();
1718 elif(ftype==0 or ftype==7):
1719 fsize = format(int(fstatinfo.st_size), 'x').lower();
1720 else:
1721 fsize = format(int(fstatinfo.st_size)).lower();
1722 fatime = format(int(fstatinfo.st_atime), 'x').lower();
1723 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
1724 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
1725 if(hasattr(fstatinfo, "st_birthtime")):
1726 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
1727 else:
1728 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
1729 fmode = format(int(fstatinfo.st_mode), 'x').lower();
1730 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
1731 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
1732 fuid = format(int(fstatinfo.st_uid), 'x').lower();
1733 fgid = format(int(fstatinfo.st_gid), 'x').lower();
1734 funame = "";
1735 try:
1736 import pwd;
1737 try:
1738 userinfo = pwd.getpwuid(fstatinfo.st_uid);
1739 funame = userinfo.pw_name;
1740 except KeyError:
1741 funame = "";
1742 except ImportError:
1743 funame = "";
1744 fgname = "";
1745 try:
1746 import grp;
1747 try:
1748 groupinfo = grp.getgrgid(fstatinfo.st_gid);
1749 fgname = groupinfo.gr_name;
1750 except KeyError:
1751 fgname = "";
1752 except ImportError:
1753 fgname = "";
1754 fdev_minor = format(int(fdev_minor), 'x').lower();
1755 fdev_major = format(int(fdev_major), 'x').lower();
1756 frdev_minor = format(int(frdev_minor), 'x').lower();
1757 frdev_major = format(int(frdev_major), 'x').lower();
1758 finode = format(int(finode), 'x').lower();
1759 flinkcount = format(int(flinkcount), 'x').lower();
1760 if(hasattr(fstatinfo, "st_file_attributes")):
1761 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
1762 else:
1763 fwinattributes = format(int(0), 'x').lower();
1764 fcompression = "";
1765 fcsize = format(int(0), 'x').lower();
1766 fcontents = BytesIO();
1767 chunk_size = 1024;
1768 if(ftype==0 or ftype==7):
1769 with open(fname, "rb") as fpc:
1770 shutil.copyfileobj(fpc, fcontents);
1771 if(not compresswholefile):
1772 fcontents.seek(0, 2);
1773 ucfsize = fcontents.tell();
1774 fcontents.seek(0, 0);
1775 if(compression=="auto"):
1776 ilsize = len(compressionlistalt);
1777 ilmin = 0;
1778 ilcsize = [];
1779 while(ilmin < ilsize):
1780 cfcontents = BytesIO();
1781 shutil.copyfileobj(fcontents, cfcontents);
1782 fcontents.seek(0, 0);
1783 cfcontents.seek(0, 0);
1784 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
1785 if(cfcontents):
1786 cfcontents.seek(0, 2);
1787 ilcsize.append(cfcontents.tell());
1788 cfcontents.close();
1789 else:
1790 try:
1791 ilcsize.append(sys.maxint);
1792 except AttributeError:
1793 ilcsize.append(sys.maxsize);
1794 ilmin = ilmin + 1;
1795 ilcmin = ilcsize.index(min(ilcsize));
1796 compression = compressionlistalt[ilcmin];
1797 fcontents.seek(0, 0);
1798 cfcontents = BytesIO();
1799 shutil.copyfileobj(fcontents, cfcontents);
1800 cfcontents.seek(0, 0);
1801 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
1802 cfcontents.seek(0, 2);
1803 cfsize = cfcontents.tell();
1804 if(ucfsize > cfsize):
1805 fcsize = format(int(cfsize), 'x').lower();
1806 fcompression = compression;
1807 fcontents.close();
1808 fcontents = cfcontents;
1809 if(followlink and (ftype==1 or ftype==2)):
1810 flstatinfo = os.stat(flinkname);
1811 with open(flinkname, "rb") as fpc:
1812 shutil.copyfileobj(fpc, fcontents);
1813 if(not compresswholefile):
1814 fcontents.seek(0, 2);
1815 ucfsize = fcontents.tell();
1816 fcontents.seek(0, 0);
1817 if(compression=="auto"):
1818 ilsize = len(compressionlistalt);
1819 ilmin = 0;
1820 ilcsize = [];
1821 while(ilmin < ilsize):
1822 cfcontents = BytesIO();
1823 shutil.copyfileobj(fcontents, cfcontents);
1824 fcontents.seek(0, 0);
1825 cfcontents.seek(0, 0);
1826 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
1827 if(cfcontents):
1828 cfcontents.seek(0, 2);
1829 ilcsize.append(cfcontents.tell());
1830 cfcontents.close();
1831 else:
1832 try:
1833 ilcsize.append(sys.maxint);
1834 except AttributeError:
1835 ilcsize.append(sys.maxsize);
1836 ilmin = ilmin + 1;
1837 ilcmin = ilcsize.index(min(ilcsize));
1838 compression = compressionlistalt[ilcmin];
1839 fcontents.seek(0, 0);
1840 cfcontents = BytesIO();
1841 shutil.copyfileobj(fcontents, cfcontents);
1842 cfcontents.seek(0, 0);
1843 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
1844 cfcontents.seek(0, 2);
1845 cfsize = cfcontents.tell();
1846 if(ucfsize > cfsize):
1847 fcsize = format(int(cfsize), 'x').lower();
1848 fcompression = compression;
1849 fcontents.close();
1850 fcontents = cfcontents;
1851 if(fcompression=="none"):
1852 fcompression = "";
1853 fcontents.seek(0, 0);
1854 ftypehex = format(ftype, 'x').lower();
1855 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
1856 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
1857 if(numfiles>0):
1858 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
1859 fp.seek(0, 0);
1860 return fp;
1862 def AppendListsWithContent(inlist, fp, dirlistfromtxt=False, filevalues=[], extradata=[], compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
1863 formatspecs = FormatSpecsListToDict(formatspecs);
1864 if(verbose):
1865 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
1866 GetDirList = inlist;
1867 if(not GetDirList):
1868 return False;
1869 curinode = 0;
1870 curfid = 0;
1871 inodelist = [];
1872 inodetofile = {};
1873 filetoinode = {};
1874 inodetocatinode = {};
1875 numfiles = int(len(GetDirList));
1876 fnumfiles = format(numfiles, 'x').lower();
1877 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
1878 for curfname in GetDirList:
1879 ftype = format(curfname[0], 'x').lower();
1880 if(re.findall(r"^[.|/]", curfname[1])):
1881 fname = curfname[1];
1882 else:
1883 fname = "./"+curfname[1];
1884 fbasedir = os.path.dirname(fname);
1885 flinkname = curfname[2];
1886 fsize = format(curfname[3], 'x').lower();
1887 fatime = format(curfname[4], 'x').lower();
1888 fmtime = format(curfname[5], 'x').lower();
1889 fctime = format(curfname[6], 'x').lower();
1890 fbtime = format(curfname[7], 'x').lower();
1891 fmode = format(curfname[8], 'x').lower();
1892 fwinattributes = format(curfname[9], 'x').lower();
1893 fcompression = curfname[10];
1894 fcsize = format(curfname[11], 'x').lower();
1895 fuid = format(curfname[12], 'x').lower();
1896 funame = curfname[13];
1897 fgid = format(curfname[14], 'x').lower();
1898 fgname = curfname[15];
1899 fid = format(curfname[16], 'x').lower();
1900 finode = format(curfname[17], 'x').lower();
1901 flinkcount = format(curfname[18], 'x').lower();
1902 fdev_minor = format(curfname[19], 'x').lower();
1903 fdev_major = format(curfname[20], 'x').lower();
1904 frdev_minor = format(curfname[21], 'x').lower();
1905 frdev_major = format(curfname[22], 'x').lower();
1906 fseeknextfile = curfname[23];
1907 extradata = curfname[24];
1908 fheaderchecksumtype = curfname[25];
1909 fcontentchecksumtype = curfname[26];
1910 fcontents = curfname[27];
1911 catoutlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, fseeknextfile];
1912 fcontents.seek(0, 0);
1913 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
1914 if(numfiles>0):
1915 fp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
1916 return fp;
1918 def AppendInFileWithContent(infile, fp, dirlistfromtxt=False, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
1919 formatspecs = FormatSpecsListToDict(formatspecs);
1920 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, True, False, formatspecs);
1921 return AppendListsWithContent(inlist, fp, dirlistfromtxt, filevalues, extradata, followlink, checksumtype, formatspecs, verbose);
1923 def AppendFilesWithContentToOutFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
1924 formatspecs = FormatSpecsListToDict(formatspecs);
1925 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1926 if(os.path.exists(outfile)):
1927 try:
1928 os.unlink(outfile);
1929 except OSError as e:
1930 pass;
1931 if(outfile=="-"):
1932 verbose = False;
1933 catfpfp = BytesIO();
1934 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1935 catfp = outfile;
1936 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1937 catfp = BytesIO();
1938 else:
1939 fbasename = os.path.splitext(outfile)[0];
1940 fextname = os.path.splitext(outfile)[1];
1941 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
1942 catfp = AppendFilesWithContent(infiles, catfp, dirlistfromtxt, filevalues, extradata, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose);
1943 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1944 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1945 try:
1946 catfp.flush();
1947 os.fsync(catfp.fileno());
1948 except io.UnsupportedOperation:
1949 pass;
1950 except AttributeError:
1951 pass;
1952 except OSError as e:
1953 pass;
1954 if(outfile=="-"):
1955 catfp.seek(0, 0);
1956 if(hasattr(sys.stdout, "buffer")):
1957 shutil.copyfileobj(catfp, sys.stdout.buffer);
1958 else:
1959 shutil.copyfileobj(catfp, sys.stdout);
1960 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1961 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1962 catfp.seek(0, 0);
1963 upload_file_to_internet_file(catfp, outfile);
1964 if(returnfp):
1965 catfp.seek(0, 0);
1966 return catfp;
1967 else:
1968 catfp.close();
1969 return True;
1971 def AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
1972 formatspecs = FormatSpecsListToDict(formatspecs);
1973 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1974 if(os.path.exists(outfile)):
1975 try:
1976 os.unlink(outfile);
1977 except OSError as e:
1978 pass;
1979 if(outfile=="-"):
1980 verbose = False;
1981 catfpfp = BytesIO();
1982 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1983 catfp = outfile;
1984 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1985 catfp = BytesIO();
1986 else:
1987 fbasename = os.path.splitext(outfile)[0];
1988 fextname = os.path.splitext(outfile)[1];
1989 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
1990 catfp = AppendListsWithContent(inlist, catfp, dirlistfromtxt, filevalues, extradata, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose);
1991 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1992 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1993 try:
1994 catfp.flush();
1995 os.fsync(catfp.fileno());
1996 except io.UnsupportedOperation:
1997 pass;
1998 except AttributeError:
1999 pass;
2000 except OSError as e:
2001 pass;
2002 if(outfile=="-"):
2003 catfp.seek(0, 0);
2004 if(hasattr(sys.stdout, "buffer")):
2005 shutil.copyfileobj(catfp, sys.stdout.buffer);
2006 else:
2007 shutil.copyfileobj(catfp, sys.stdout);
2008 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2009 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2010 catfp.seek(0, 0);
2011 upload_file_to_internet_file(catfp, outfile);
2012 if(returnfp):
2013 catfp.seek(0, 0);
2014 return catfp;
2015 else:
2016 catfp.close();
2017 return True;
2019 def AppendInFileWithContentToOutFile(infile, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2020 formatspecs = FormatSpecsListToDict(formatspecs);
2021 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, True, False, formatspecs);
2022 return AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt, compression, compresswholefile, compressionlevel, filevalues, extradata, followlink, checksumtype, formatspecs, verbose, returnfp);
2024 def PrintPermissionString(fchmode, ftype):
2025 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
2026 permissionstr = "";
2027 for fmodval in str(oct(fchmode))[-3:]:
2028 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
2029 if(ftype==0 or ftype==7):
2030 permissionstr = "-" + permissionstr;
2031 if(ftype==1):
2032 permissionstr = "h" + permissionstr;
2033 if(ftype==2):
2034 permissionstr = "l" + permissionstr;
2035 if(ftype==3):
2036 permissionstr = "c" + permissionstr;
2037 if(ftype==4):
2038 permissionstr = "b" + permissionstr;
2039 if(ftype==5):
2040 permissionstr = "d" + permissionstr;
2041 if(ftype==6):
2042 permissionstr = "f" + permissionstr;
2043 if(ftype==8):
2044 permissionstr = "D" + permissionstr;
2045 if(ftype==9):
2046 permissionstr = "p" + permissionstr;
2047 if(ftype==10):
2048 permissionstr = "w" + permissionstr;
2049 try:
2050 permissionoutstr = stat.filemode(fchmode);
2051 except AttributeError:
2052 permissionoutstr = permissionstr;
2053 except KeyError:
2054 permissionoutstr = permissionstr;
2055 return permissionoutstr;
2057 def PrintPermissionStringAlt(fchmode, ftype):
2058 permissions = {
2059 '0': '---', '1': '--x', '2': '-w-', '3': '-wx',
2060 '4': 'r--', '5': 'r-x', '6': 'rw-', '7': 'rwx'
2062 # Translate file mode into permission string
2063 permissionstr = ''.join([permissions[i] for i in str(oct(fchmode))[-3:]]);
2064 # Append file type indicator
2065 type_indicators = {
2066 0: '-', 1: 'h', 2: 'l', 3: 'c', 4: 'b',
2067 5: 'd', 6: 'f', 8: 'D', 9: 'p', 10: 'w'
2069 file_type = type_indicators.get(ftype, '-');
2070 permissionstr = file_type + permissionstr;
2071 try:
2072 permissionoutstr = stat.filemode(fchmode);
2073 except AttributeError:
2074 permissionoutstr = permissionstr;
2075 return permissionoutstr;
2077 def CheckCompressionType(infile, formatspecs=__file_format_dict__, closefp=True):
2078 formatspecs = FormatSpecsListToDict(formatspecs);
2079 if(hasattr(infile, "read") or hasattr(infile, "write")):
2080 catfp = infile;
2081 else:
2082 try:
2083 catfp = open(infile, "rb");
2084 except FileNotFoundError:
2085 return False;
2086 catfp.seek(0, 0);
2087 prefp = catfp.read(2);
2088 filetype = False;
2089 if(prefp==binascii.unhexlify("1f8b")):
2090 filetype = "gzip";
2091 catfp.seek(0, 0);
2092 prefp = catfp.read(3);
2093 if(prefp==binascii.unhexlify("425a68")):
2094 filetype = "bzip2";
2095 if(prefp==binascii.unhexlify("5d0000")):
2096 filetype = "lzma";
2097 catfp.seek(0, 0);
2098 prefp = catfp.read(4);
2099 if(prefp==binascii.unhexlify("28b52ffd")):
2100 filetype = "zstd";
2101 if(prefp==binascii.unhexlify("04224d18")):
2102 filetype = "lz4";
2103 if(prefp==binascii.unhexlify("504B0304")):
2104 filetype = "zipfile";
2105 catfp.seek(0, 0);
2106 prefp = catfp.read(5);
2107 if(prefp==binascii.unhexlify("7573746172")):
2108 filetype = "tarfile";
2109 catfp.seek(0, 0);
2110 prefp = catfp.read(6);
2111 if(prefp==binascii.unhexlify("fd377a585a00")):
2112 filetype = "lzma";
2113 if(prefp==binascii.unhexlify("377abcaf271c")):
2114 filetype = "7zipfile";
2115 catfp.seek(0, 0);
2116 prefp = catfp.read(7);
2117 if(prefp==binascii.unhexlify("526172211a0700")):
2118 filetype = "rarfile";
2119 if(prefp==binascii.unhexlify("43617446696c65")):
2120 filetype = "catfile";
2121 catfp.seek(0, 0);
2122 prefp = catfp.read(8);
2123 if(prefp==binascii.unhexlify("526172211a070100")):
2124 filetype = "rarfile";
2125 catfp.seek(0, 0);
2126 prefp = catfp.read(formatspecs['format_len']);
2127 if(prefp==binascii.unhexlify(formatspecs['format_hex'])):
2128 filetype = formatspecs['format_lower'];
2129 catfp.seek(0, 0);
2130 prefp = catfp.read(9);
2131 if(prefp==binascii.unhexlify("894c5a4f000d0a1a0a")):
2132 filetype = "lzo";
2133 catfp.seek(0, 0);
2134 prefp = catfp.read(10);
2135 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
2136 filetype = "tarfile";
2137 catfp.seek(0, 0);
2138 if(filetype=="gzip" or filetype=="bzip2" or filetype=="lzma" or filetype=="zstd" or filetype=="lz4"):
2139 if(TarFileCheck(catfp)):
2140 filetype = "tarfile";
2141 if(not filetype):
2142 if(TarFileCheck(catfp)):
2143 filetype = "tarfile";
2144 elif(zipfile.is_zipfile(catfp)):
2145 filetype = "zipfile";
2146 elif(rarfile.is_rarfile(catfp) or rarfile.is_rarfile_sfx(catfp)):
2147 filetype = "rarile";
2148 else:
2149 filetype = False;
2150 catfp.seek(0, 0);
2151 if(closefp):
2152 catfp.close();
2153 return filetype;
2155 def CheckCompressionTypeFromString(instring, formatspecs=__file_format_dict__, closefp=True):
2156 formatspecs = FormatSpecsListToDict(formatspecs);
2157 try:
2158 instringsfile = BytesIO(instring);
2159 except TypeError:
2160 instringsfile = BytesIO(instring.encode("UTF-8"));
2161 return CheckCompressionType(instringsfile, formatspecs, closefp);
2163 def GetCompressionMimeType(infile, formatspecs=__file_format_dict__):
2164 formatspecs = FormatSpecsListToDict(formatspecs);
2165 compresscheck = CheckCompressionType(fp, formatspecs, False);
2166 if(compresscheck=="gzip" or compresscheck=="gz"):
2167 return archivefile_gzip_mimetype;
2168 if(compresscheck=="bzip2" or compresscheck=="bz2"):
2169 return archivefile_bzip2_mimetype;
2170 if(compresscheck=="zstd" or compresscheck=="zstandard"):
2171 return archivefile_zstandard_mimetype;
2172 if(compresscheck=="lz4"):
2173 return archivefile_lz4_mimetype;
2174 if(compresscheck=="lzo" or compresscheck=="lzop"):
2175 return archivefile_lzop_mimetype;
2176 if(compresscheck=="lzma"):
2177 return archivefile_lzma_mimetype;
2178 if(compresscheck=="xz"):
2179 return archivefile_xz_mimetype;
2180 if(compresscheck=="catfile" or compresscheck=="cat" or compresscheck==formatspecs['format_lower']):
2181 return archivefile_cat_mimetype;
2182 if(not compresscheck):
2183 return False;
2184 return False;
2186 def UncompressArchiveFile(fp, formatspecs=__file_format_dict__):
2187 formatspecs = FormatSpecsListToDict(formatspecs);
2188 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
2189 return False;
2190 compresscheck = CheckCompressionType(fp, formatspecs, False);
2191 if(compresscheck=="gzip"):
2192 try:
2193 import gzip;
2194 except ImportError:
2195 return False;
2196 catfp = gzip.GzipFile(fileobj=fp, mode="rb");
2197 if(compresscheck=="bzip2"):
2198 try:
2199 import bz2;
2200 except ImportError:
2201 return False;
2202 catfp = BytesIO();
2203 catfp.write(bz2.decompress(fp.read()));
2204 if(compresscheck=="zstd"):
2205 try:
2206 import zstandard;
2207 except ImportError:
2208 return False;
2209 catfp = BytesIO();
2210 catfp.write(zstandard.decompress(fp.read()));
2211 if(compresscheck=="lz4"):
2212 try:
2213 import lz4.frame;
2214 except ImportError:
2215 return False;
2216 catfp = BytesIO();
2217 catfp.write(lz4.frame.decompress(fp.read()));
2218 if(compresscheck=="lzo" or compresscheck=="lzop"):
2219 try:
2220 import lzo;
2221 except ImportError:
2222 return False;
2223 catfp = BytesIO();
2224 catfp.write(lzo.decompress(fp.read()));
2225 if(compresscheck=="lzma" or compresscheck=="xz"):
2226 try:
2227 import lzma;
2228 except ImportError:
2229 try:
2230 from backports import lzma
2231 except ImportError:
2232 return False;
2233 catfp = BytesIO();
2234 catfp.write(lzma.decompress(fp.read()));
2235 if(compresscheck=="catfile" or compresscheck==formatspecs['format_lower']):
2236 catfp = fp;
2237 if(not compresscheck):
2238 try:
2239 import lzma;
2240 except ImportError:
2241 try:
2242 from backports import lzma
2243 except ImportError:
2244 return False;
2245 catfp = BytesIO();
2246 with fp as fpcontent:
2247 try:
2248 catfp.write(lzma.decompress(fp.read()));
2249 except lzma.LZMAError:
2250 return False;
2251 if(compresscheck!="catfile" or compresscheck!=formatspecs['format_lower']):
2252 fp.close();
2253 return catfp;
2255 create_alias_function("Uncompress", __file_format_name__, "", UncompressArchiveFile);
2257 def UncompressFile(infile, formatspecs=__file_format_dict__, mode="rb"):
2258 formatspecs = FormatSpecsListToDict(formatspecs);
2259 compresscheck = CheckCompressionType(infile, formatspecs, False);
2260 if(sys.version_info[0]==2 and compresscheck):
2261 if(mode=="rt"):
2262 mode = "r";
2263 if(mode=="wt"):
2264 mode = "w";
2265 try:
2266 if(compresscheck=="gzip"):
2267 try:
2268 import gzip;
2269 except ImportError:
2270 return False;
2271 try:
2272 filefp = gzip.open(infile, mode, encoding="UTF-8");
2273 except (ValueError, TypeError) as e:
2274 filefp = gzip.open(infile, mode);
2275 if(compresscheck=="bzip2"):
2276 try:
2277 import bz2;
2278 except ImportError:
2279 return False;
2280 try:
2281 filefp = bz2.open(infile, mode, encoding="UTF-8");
2282 except (ValueError, TypeError) as e:
2283 filefp = bz2.open(infile, mode);
2284 if(compresscheck=="zstd"):
2285 try:
2286 import zstandard;
2287 except ImportError:
2288 return False;
2289 try:
2290 filefp = zstandard.open(infile, mode, encoding="UTF-8");
2291 except (ValueError, TypeError) as e:
2292 filefp = zstandard.open(infile, mode);
2293 if(compresscheck=="lz4"):
2294 try:
2295 import lz4.frame;
2296 except ImportError:
2297 return False;
2298 try:
2299 filefp = lz4.frame.open(infile, mode, encoding="UTF-8");
2300 except (ValueError, TypeError) as e:
2301 filefp = lz4.frame.open(infile, mode);
2302 if(compresscheck=="lzo"):
2303 try:
2304 import lzo;
2305 except ImportError:
2306 return False;
2307 try:
2308 filefp = lzo.open(infile, mode, encoding="UTF-8");
2309 except (ValueError, TypeError) as e:
2310 filefp = lzo.open(infile, mode);
2311 if(compresscheck=="lzma"):
2312 try:
2313 import lzma;
2314 except ImportError:
2315 try:
2316 from backports import lzma
2317 except ImportError:
2318 return False;
2319 try:
2320 filefp = lzma.open(infile, mode, encoding="UTF-8");
2321 except (ValueError, TypeError) as e:
2322 filefp = lzma.open(infile, mode);
2323 if(compresscheck=="catfile" or compresscheck==formatspecs['format_lower']):
2324 try:
2325 filefp = open(infile, mode, encoding="UTF-8");
2326 except (ValueError, TypeError) as e:
2327 filefp = open(infile, mode);
2328 if(not compresscheck):
2329 try:
2330 filefp = open(infile, mode, encoding="UTF-8");
2331 except (ValueError, TypeError) as e:
2332 filefp = open(infile, mode);
2333 except FileNotFoundError:
2334 return False;
2335 return filefp;
2337 def UncompressString(infile):
2338 compresscheck = CheckCompressionTypeFromString(infile, formatspecs, False);
2339 if(compresscheck=="gzip"):
2340 try:
2341 import gzip;
2342 except ImportError:
2343 return False;
2344 fileuz = gzip.decompress(infile);
2345 if(compresscheck=="bzip2"):
2346 try:
2347 import bz2;
2348 except ImportError:
2349 return False;
2350 fileuz = bz2.decompress(infile);
2351 if(compresscheck=="zstd"):
2352 try:
2353 import zstandard;
2354 except ImportError:
2355 return False;
2356 fileuz = zstandard.decompress(infile);
2357 if(compresscheck=="lz4"):
2358 try:
2359 import lz4.frame;
2360 except ImportError:
2361 return False;
2362 fileuz = lz4.frame.decompress(infile);
2363 if(compresscheck=="lzo"):
2364 try:
2365 import lzo;
2366 except ImportError:
2367 return False;
2368 fileuz = lzo.decompress(infile);
2369 if(compresscheck=="lzma"):
2370 try:
2371 import lzma;
2372 except ImportError:
2373 try:
2374 from backports import lzma
2375 except ImportError:
2376 return False;
2377 fileuz = lzma.decompress(infile);
2378 if(not compresscheck):
2379 fileuz = infile;
2380 if(hasattr(fileuz, 'decode')):
2381 fileuz = fileuz.decode("UTF-8");
2382 return fileuz;
2384 def UncompressStringAlt(infile):
2385 filefp = StringIO();
2386 outstring = UncompressString(infile);
2387 filefp.write(outstring);
2388 filefp.seek(0, 0);
2389 return filefp;
2391 def CheckCompressionSubType(infile, formatspecs=__file_format_dict__, closefp=True):
2392 formatspecs = FormatSpecsListToDict(formatspecs);
2393 compresscheck = CheckCompressionType(infile, formatspecs, False);
2394 if(not compresscheck):
2395 fextname = os.path.splitext(infile)[1];
2396 if(fextname==".gz"):
2397 compresscheck = "gzip";
2398 elif(fextname==".bz2"):
2399 compresscheck = "bzip2";
2400 elif(fextname==".zst"):
2401 compresscheck = "zstd";
2402 elif(fextname==".lz4"):
2403 compresscheck = "lz4";
2404 elif(fextname==".lzo" or fextname==".lzop"):
2405 compresscheck = "lzo";
2406 elif(fextname==".lzma" or fextname==".xz"):
2407 compresscheck = "lzma";
2408 else:
2409 return False;
2410 if(compresscheck=="gzip" or compresscheck=="bzip2" or compresscheck=="lzma" or compresscheck=="zstd" or compresscheck=="lz4"):
2411 if(TarFileCheck(infile)):
2412 filetype = "tarfile";
2413 if(not compresscheck):
2414 if(TarFileCheck(infile)):
2415 return "tarfile";
2416 elif(zipfile.is_zipfile(infile)):
2417 return "zipfile";
2418 elif(rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile)):
2419 return "rarile";
2420 else:
2421 return False;
2422 return False;
2423 if(compresscheck=="catfile"):
2424 return "catfile";
2425 if(compresscheck==formatspecs['format_lower']):
2426 return formatspecs['format_lower'];
2427 if(compresscheck=="tarfile"):
2428 return "tarfile";
2429 if(compresscheck=="zipfile"):
2430 return "zipfile";
2431 if(rarfile_support and compresscheck=="rarfile"):
2432 return "rarfile";
2433 if(py7zr_support and compresscheck=="7zipfile"):
2434 return "7zipfile";
2435 if(hasattr(infile, "read") or hasattr(infile, "write")):
2436 catfp = UncompressArchiveFile(infile, formatspecs['format_lower']);
2437 else:
2438 try:
2439 if(compresscheck=="gzip"):
2440 try:
2441 import gzip;
2442 except ImportError:
2443 return False;
2444 catfp = gzip.GzipFile(infile, "rb");
2445 if(compresscheck=="bzip2"):
2446 try:
2447 import bz2;
2448 except ImportError:
2449 return False;
2450 catfp = bz2.BZ2File(infile, "rb");
2451 if(compresscheck=="lz4"):
2452 try:
2453 import lz4.frame;
2454 except ImportError:
2455 return False;
2456 catfp = lz4.frame.open(infile, "rb");
2457 if(compresscheck=="zstd"):
2458 try:
2459 import zstandard;
2460 except ImportError:
2461 return False;
2462 catfp = zstandard.open(infile, "rb");
2463 if(compresscheck=="lzma" or compresscheck=="xz"):
2464 try:
2465 import lzma;
2466 except ImportError:
2467 try:
2468 from backports import lzma
2469 except ImportError:
2470 return False;
2471 catfp = lzma.open(infile, "rb");
2472 except FileNotFoundError:
2473 return False;
2474 filetype = False;
2475 prefp = catfp.read(5);
2476 if(prefp==binascii.unhexlify("7573746172")):
2477 filetype = "tarfile";
2478 catfp.seek(0, 0);
2479 prefp = catfp.read(7);
2480 if(prefp==binascii.unhexlify("43617446696c65")):
2481 filetype = "catfile";
2482 catfp.seek(0, 0);
2483 prefp = catfp.read(formatspecs['format_len']);
2484 if(prefp==binascii.unhexlify(formatspecs['format_hex'])):
2485 filetype = formatspecs['format_lower'];
2486 catfp.seek(0, 0);
2487 prefp = catfp.read(10);
2488 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
2489 filetype = "tarfile";
2490 catfp.seek(0, 0);
2491 if(closefp):
2492 catfp.close();
2493 return filetype;
2495 def GZipCompress(data, compresslevel=9):
2496 try:
2497 import gzip;
2498 except ImportError:
2499 return False;
2500 tmpfp = tempfile.NamedTemporaryFile("wb", delete=False);
2501 tmpfp.close();
2502 tmpfp = gzip.GzipFile(tmpfp.name, mode="wb", compresslevel=compresslevel);
2503 tmpfp.write(data);
2504 tmpfp.close();
2505 try:
2506 catfp = open(tmpfp.name, "rb");
2507 except FileNotFoundError:
2508 return False;
2509 catdata = catfp.read();
2510 catfp.close();
2511 return catdata;
2513 def CompressArchiveFile(fp, compression="auto", compressionlevel=None, formatspecs=__file_format_dict__):
2514 formatspecs = FormatSpecsListToDict(formatspecs);
2515 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
2516 return False;
2517 fp.seek(0, 0);
2518 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
2519 compression = "auto";
2520 if(compression not in compressionlist and compression is None):
2521 compression = "auto";
2522 if(compression=="gzip"):
2523 try:
2524 import gzip;
2525 except ImportError:
2526 return False;
2527 catfp = BytesIO();
2528 if(compressionlevel is None):
2529 compressionlevel = 9;
2530 else:
2531 compressionlevel = int(compressionlevel);
2532 catfp.write(gzip.compress(fp.read(), compresslevel=compressionlevel));
2533 if(compression=="bzip2"):
2534 try:
2535 import bz2;
2536 except ImportError:
2537 return False;
2538 catfp = BytesIO();
2539 if(compressionlevel is None):
2540 compressionlevel = 9;
2541 else:
2542 compressionlevel = int(compressionlevel);
2543 catfp.write(bz2.compress(fp.read(), compresslevel=compressionlevel));
2544 if(compression=="lz4"):
2545 try:
2546 import lz4.frame;
2547 except ImportError:
2548 return False;
2549 catfp = BytesIO();
2550 if(compressionlevel is None):
2551 compressionlevel = 9;
2552 else:
2553 compressionlevel = int(compressionlevel);
2554 catfp.write(lz4.frame.compress(fp.read(), compression_level=compressionlevel));
2555 if(compression=="lzo" or compression=="lzop"):
2556 try:
2557 import lzo;
2558 except ImportError:
2559 return False;
2560 catfp = BytesIO();
2561 if(compressionlevel is None):
2562 compressionlevel = 9;
2563 else:
2564 compressionlevel = int(compressionlevel);
2565 catfp.write(lzo.compress(fp.read(), compresslevel=compressionlevel));
2566 if(compression=="zstd"):
2567 try:
2568 import zstandard;
2569 except ImportError:
2570 return False;
2571 catfp = BytesIO();
2572 if(compressionlevel is None):
2573 compressionlevel = 10;
2574 else:
2575 compressionlevel = int(compressionlevel);
2576 catfp.write(zstandard.compress(fp.read(), level=compressionlevel));
2577 if(compression=="lzma"):
2578 try:
2579 import lzma;
2580 except ImportError:
2581 try:
2582 from backports import lzma
2583 except ImportError:
2584 return False;
2585 catfp = BytesIO();
2586 if(compressionlevel is None):
2587 compressionlevel = 9;
2588 else:
2589 compressionlevel = int(compressionlevel);
2590 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]));
2591 if(compression=="xz"):
2592 try:
2593 import lzma;
2594 except ImportError:
2595 try:
2596 from backports import lzma
2597 except ImportError:
2598 return False;
2599 catfp = BytesIO();
2600 if(compressionlevel is None):
2601 compressionlevel = 9;
2602 else:
2603 compressionlevel = int(compressionlevel);
2604 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]));
2605 if(compression=="auto" or compression is None):
2606 catfp = fp;
2607 catfp.seek(0, 0);
2608 return catfp;
2610 create_alias_function("Compress", __file_format_name__, "", CompressArchiveFile);
2612 def CompressOpenFile(outfile, compressionenable=True, compressionlevel=None):
2613 if(outfile is None):
2614 return False;
2615 fbasename = os.path.splitext(outfile)[0];
2616 fextname = os.path.splitext(outfile)[1];
2617 if(compressionlevel is None and fextname!=".zst"):
2618 compressionlevel = 9;
2619 elif(compressionlevel is None and fextname==".zst"):
2620 compressionlevel = 10;
2621 else:
2622 compressionlevel = int(compressionlevel);
2623 if(sys.version_info[0]==2):
2624 mode = "w";
2625 else:
2626 mode = "wb";
2627 try:
2628 if(fextname not in outextlistwd or not compressionenable):
2629 try:
2630 outfp = open(outfile, "wb", encoding="UTF-8");
2631 except (ValueError, TypeError) as e:
2632 outfp = open(outfile, "wb");
2633 elif(fextname==".gz"):
2634 try:
2635 import gzip;
2636 except ImportError:
2637 return False;
2638 try:
2639 outfp = gzip.open(outfile, mode, compressionlevel, encoding="UTF-8");
2640 except (ValueError, TypeError) as e:
2641 outfp = gzip.open(outfile, mode, compressionlevel);
2642 elif(fextname==".bz2"):
2643 try:
2644 import bz2;
2645 except ImportError:
2646 return False;
2647 try:
2648 outfp = bz2.open(outfile, mode, compressionlevel, encoding="UTF-8");
2649 except (ValueError, TypeError) as e:
2650 outfp = bz2.open(outfile, mode, compressionlevel);
2651 elif(fextname==".zst"):
2652 try:
2653 import zstandard;
2654 except ImportError:
2655 return False;
2656 try:
2657 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel), encoding="UTF-8");
2658 except (ValueError, TypeError) as e:
2659 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel));
2660 elif(fextname==".xz"):
2661 try:
2662 import lzma;
2663 except ImportError:
2664 try:
2665 from backports import lzma
2666 except ImportError:
2667 return False;
2668 try:
2669 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}], encoding="UTF-8");
2670 except (ValueError, TypeError) as e:
2671 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]);
2672 elif(fextname==".lz4"):
2673 try:
2674 import lz4.frame;
2675 except ImportError:
2676 return False;
2677 try:
2678 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel, encoding="UTF-8");
2679 except (ValueError, TypeError) as e:
2680 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel);
2681 elif(fextname==".lzo"):
2682 try:
2683 import lzo;
2684 except ImportError:
2685 return False;
2686 try:
2687 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel, encoding="UTF-8");
2688 except (ValueError, TypeError) as e:
2689 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel);
2690 elif(fextname==".lzma"):
2691 try:
2692 import lzma;
2693 except ImportError:
2694 try:
2695 from backports import lzma
2696 except ImportError:
2697 return False;
2698 try:
2699 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}], encoding="UTF-8");
2700 except (ValueError, TypeError) as e:
2701 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]);
2702 except FileNotFoundError:
2703 return False;
2704 return outfp;
2706 def GetDevMajorMinor(fdev):
2707 retdev = [];
2708 if(hasattr(os, "minor")):
2709 retdev.append(os.minor(fdev));
2710 else:
2711 retdev.append(0);
2712 if(hasattr(os, "major")):
2713 retdev.append(os.major(fdev));
2714 else:
2715 retdev.append(0);
2716 return retdev;
2718 def CheckSumSupport(checkfor, guaranteed=True):
2719 if(guaranteed):
2720 try:
2721 hash_list = sorted(list(hashlib.algorithms_guaranteed));
2722 except AttributeError:
2723 hash_list = sorted(list(hashlib.algorithms));
2724 else:
2725 try:
2726 hash_list = sorted(list(hashlib.algorithms_available));
2727 except AttributeError:
2728 hash_list = sorted(list(hashlib.algorithms));
2729 checklistout = sorted(hash_list + ['adler32', 'crc16', 'crc16_ansi', 'crc16_ibm', 'crc16_ccitt', 'crc32', 'crc64', 'crc64_ecma', 'crc64_iso', 'none']);
2730 if(checkfor in checklistout):
2731 return True;
2732 else:
2733 return False;
2735 def CheckSumSupportAlt(checkfor, guaranteed=True):
2736 if(guaranteed):
2737 try:
2738 hash_list = sorted(list(hashlib.algorithms_guaranteed));
2739 except AttributeError:
2740 hash_list = sorted(list(hashlib.algorithms));
2741 else:
2742 try:
2743 hash_list = sorted(list(hashlib.algorithms_available));
2744 except AttributeError:
2745 hash_list = sorted(list(hashlib.algorithms));
2746 checklistout = hash_list;
2747 if(checkfor in checklistout):
2748 return True;
2749 else:
2750 return False;
2752 def PackArchiveFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2753 formatspecs = FormatSpecsListToDict(formatspecs);
2754 advancedlist = formatspecs['use_advanced_list'];
2755 altinode = formatspecs['use_alt_inode'];
2756 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2757 outfile = RemoveWindowsPath(outfile);
2758 checksumtype = checksumtype.lower();
2759 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
2760 checksumtype="crc32";
2761 if(checksumtype=="none"):
2762 checksumtype = "";
2763 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
2764 compression = "auto";
2765 if(compression not in compressionlist and compression is None):
2766 compression = "auto";
2767 if(verbose):
2768 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2769 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2770 if(os.path.exists(outfile)):
2771 try:
2772 os.unlink(outfile);
2773 except OSError as e:
2774 pass;
2775 if(outfile=="-"):
2776 verbose = False;
2777 catfp = BytesIO();
2778 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2779 catfp = outfile;
2780 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2781 catfp = BytesIO();
2782 else:
2783 fbasename = os.path.splitext(outfile)[0];
2784 fextname = os.path.splitext(outfile)[1];
2785 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
2786 catver = formatspecs['format_ver'];
2787 fileheaderver = str(int(catver.replace(".", "")));
2788 infilelist = [];
2789 if(infiles=="-"):
2790 for line in sys.stdin:
2791 infilelist.append(line.strip());
2792 infilelist = list(filter(None, infilelist));
2793 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
2794 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
2795 return False;
2796 with UncompressFile(infiles, formatspecs, "r") as finfile:
2797 for line in finfile:
2798 infilelist.append(line.strip());
2799 infilelist = list(filter(None, infilelist));
2800 else:
2801 if(isinstance(infiles, (list, tuple, ))):
2802 infilelist = list(filter(None, infiles));
2803 elif(isinstance(infiles, (str, ))):
2804 infilelist = list(filter(None, [infiles]));
2805 if(advancedlist):
2806 GetDirList = ListDirAdvanced(infilelist, followlink, False);
2807 else:
2808 GetDirList = ListDir(infilelist, followlink, False);
2809 if(not GetDirList):
2810 return False;
2811 curinode = 0;
2812 curfid = 0;
2813 inodelist = [];
2814 inodetofile = {};
2815 filetoinode = {};
2816 inodetocatinode = {};
2817 numfiles = int(len(GetDirList));
2818 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
2819 for curfname in GetDirList:
2820 if(re.findall(r"^[.|/]", curfname)):
2821 fname = curfname;
2822 else:
2823 fname = "./"+curfname;
2824 if(verbose):
2825 VerbosePrintOut(fname);
2826 if(not followlink or followlink is None):
2827 fstatinfo = os.lstat(fname);
2828 else:
2829 fstatinfo = os.stat(fname);
2830 fpremode = fstatinfo.st_mode;
2831 finode = fstatinfo.st_ino;
2832 flinkcount = fstatinfo.st_nlink;
2833 ftype = 0;
2834 if(stat.S_ISREG(fpremode)):
2835 ftype = 0;
2836 elif(stat.S_ISLNK(fpremode)):
2837 ftype = 2;
2838 elif(stat.S_ISCHR(fpremode)):
2839 ftype = 3;
2840 elif(stat.S_ISBLK(fpremode)):
2841 ftype = 4;
2842 elif(stat.S_ISDIR(fpremode)):
2843 ftype = 5;
2844 elif(stat.S_ISFIFO(fpremode)):
2845 ftype = 6;
2846 elif(stat.S_ISSOCK(fpremode)):
2847 ftype = 8;
2848 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
2849 ftype = 9;
2850 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
2851 ftype = 10;
2852 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
2853 ftype = 11;
2854 else:
2855 ftype = 0;
2856 flinkname = "";
2857 fcurfid = format(int(curfid), 'x').lower();
2858 if(not followlink and finode!=0):
2859 if(ftype!=1):
2860 if(finode in inodelist):
2861 ftype = 1;
2862 flinkname = inodetofile[finode];
2863 if(altinode):
2864 fcurinode = format(int(finode), 'x').lower();
2865 else:
2866 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
2867 if(finode not in inodelist):
2868 inodelist.append(finode);
2869 inodetofile.update({finode: fname});
2870 inodetocatinode.update({finode: curinode});
2871 if(altinode):
2872 fcurinode = format(int(finode), 'x').lower();
2873 else:
2874 fcurinode = format(int(curinode), 'x').lower();
2875 curinode = curinode + 1;
2876 else:
2877 fcurinode = format(int(curinode), 'x').lower();
2878 curinode = curinode + 1;
2879 curfid = curfid + 1;
2880 if(ftype==2):
2881 flinkname = os.readlink(fname);
2882 fdev = fstatinfo.st_dev;
2883 getfdev = GetDevMajorMinor(fdev);
2884 fdev_minor = getfdev[0];
2885 fdev_major = getfdev[1];
2886 frdev = fstatinfo.st_dev;
2887 if(hasattr(fstatinfo, "st_rdev")):
2888 frdev = fstatinfo.st_rdev;
2889 else:
2890 frdev = fstatinfo.st_dev;
2891 getfrdev = GetDevMajorMinor(frdev);
2892 frdev_minor = getfrdev[0];
2893 frdev_major = getfrdev[1];
2894 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
2895 fsize = format(int("0"), 'x').lower();
2896 elif(ftype==0 or ftype==7):
2897 fsize = format(int(fstatinfo.st_size), 'x').lower();
2898 else:
2899 fsize = format(int(fstatinfo.st_size)).lower();
2900 fatime = format(int(fstatinfo.st_atime), 'x').lower();
2901 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
2902 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
2903 if(hasattr(fstatinfo, "st_birthtime")):
2904 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
2905 else:
2906 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
2907 fmode = format(int(fstatinfo.st_mode), 'x').lower();
2908 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
2909 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
2910 fuid = format(int(fstatinfo.st_uid), 'x').lower();
2911 fgid = format(int(fstatinfo.st_gid), 'x').lower();
2912 funame = "";
2913 try:
2914 import pwd;
2915 try:
2916 userinfo = pwd.getpwuid(fstatinfo.st_uid);
2917 funame = userinfo.pw_name;
2918 except KeyError:
2919 funame = "";
2920 except ImportError:
2921 funame = "";
2922 fgname = "";
2923 try:
2924 import grp;
2925 try:
2926 groupinfo = grp.getgrgid(fstatinfo.st_gid);
2927 fgname = groupinfo.gr_name;
2928 except KeyError:
2929 fgname = "";
2930 except ImportError:
2931 fgname = "";
2932 fdev_minor = format(int(fdev_minor), 'x').lower();
2933 fdev_major = format(int(fdev_major), 'x').lower();
2934 frdev_minor = format(int(frdev_minor), 'x').lower();
2935 frdev_major = format(int(frdev_major), 'x').lower();
2936 finode = format(int(finode), 'x').lower();
2937 flinkcount = format(int(flinkcount), 'x').lower();
2938 if(hasattr(fstatinfo, "st_file_attributes")):
2939 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
2940 else:
2941 fwinattributes = format(int(0), 'x').lower();
2942 fcompression = "";
2943 fcsize = format(int(0), 'x').lower();
2944 fcontents = BytesIO();
2945 if(ftype==0 or ftype==7):
2946 with open(fname, "rb") as fpc:
2947 shutil.copyfileobj(fpc, fcontents);
2948 if(not compresswholefile):
2949 fcontents.seek(0, 2);
2950 ucfsize = fcontents.tell();
2951 fcontents.seek(0, 0);
2952 if(compression=="auto"):
2953 ilsize = len(compressionlistalt);
2954 ilmin = 0;
2955 ilcsize = [];
2956 while(ilmin < ilsize):
2957 cfcontents = BytesIO();
2958 shutil.copyfileobj(fcontents, cfcontents);
2959 fcontents.seek(0, 0);
2960 cfcontents.seek(0, 0);
2961 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
2962 if(cfcontents):
2963 cfcontents.seek(0, 2);
2964 ilcsize.append(cfcontents.tell());
2965 cfcontents.close();
2966 else:
2967 try:
2968 ilcsize.append(sys.maxint);
2969 except AttributeError:
2970 ilcsize.append(sys.maxsize);
2971 ilmin = ilmin + 1;
2972 ilcmin = ilcsize.index(min(ilcsize));
2973 compression = compressionlistalt[ilcmin];
2974 fcontents.seek(0, 0);
2975 cfcontents = BytesIO();
2976 shutil.copyfileobj(fcontents, cfcontents);
2977 cfcontents.seek(0, 0);
2978 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
2979 cfcontents.seek(0, 2);
2980 cfsize = cfcontents.tell();
2981 if(ucfsize > cfsize):
2982 fcsize = format(int(cfsize), 'x').lower();
2983 fcompression = compression;
2984 fcontents.close();
2985 fcontents = cfcontents;
2986 if(fcompression=="none"):
2987 fcompression = "";
2988 if(followlink and (ftype==1 or ftype==2)):
2989 flstatinfo = os.stat(flinkname);
2990 with open(flinkname, "rb") as fpc:
2991 shutil.copyfileobj(fpc, fcontents);
2992 if(not compresswholefile):
2993 fcontents.seek(0, 2);
2994 ucfsize = fcontents.tell();
2995 fcontents.seek(0, 0);
2996 if(compression=="auto"):
2997 ilsize = len(compressionlistalt);
2998 ilmin = 0;
2999 ilcsize = [];
3000 while(ilmin < ilsize):
3001 cfcontents = BytesIO();
3002 shutil.copyfileobj(fcontents, cfcontents);
3003 fcontents.seek(0, 0);
3004 cfcontents.seek(0, 0);
3005 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3006 if(cfcontents):
3007 cfcontents.seek(0, 2);
3008 ilcsize.append(cfcontents.tell());
3009 cfcontents.close();
3010 else:
3011 try:
3012 ilcsize.append(sys.maxint);
3013 except AttributeError:
3014 ilcsize.append(sys.maxsize);
3015 ilmin = ilmin + 1;
3016 ilcmin = ilcsize.index(min(ilcsize));
3017 compression = compressionlistalt[ilcmin];
3018 fcontents.seek(0, 0);
3019 cfcontents = BytesIO();
3020 shutil.copyfileobj(fcontents, cfcontents);
3021 cfcontents.seek(0, 0);
3022 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3023 cfcontents.seek(0, 2);
3024 cfsize = cfcontents.tell();
3025 if(ucfsize > cfsize):
3026 fcsize = format(int(cfsize), 'x').lower();
3027 fcompression = compression;
3028 fcontents.close();
3029 fcontents = cfcontents;
3030 fcontents.seek(0, 0);
3031 ftypehex = format(ftype, 'x').lower();
3032 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
3033 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3034 fcontents.close();
3035 if(numfiles>0):
3036 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3037 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3038 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3039 try:
3040 catfp.flush();
3041 os.fsync(catfp.fileno());
3042 except io.UnsupportedOperation:
3043 pass;
3044 except AttributeError:
3045 pass;
3046 except OSError as e:
3047 pass;
3048 if(outfile=="-"):
3049 catfp.seek(0, 0);
3050 if(hasattr(sys.stdout, "buffer")):
3051 shutil.copyfileobj(catfp, sys.stdout.buffer);
3052 else:
3053 shutil.copyfileobj(catfp, sys.stdout);
3054 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3055 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3056 catfp.seek(0, 0);
3057 upload_file_to_internet_file(catfp, outfile);
3058 if(returnfp):
3059 catfp.seek(0, 0);
3060 return catfp;
3061 else:
3062 catfp.close();
3063 return True;
3065 create_alias_function("Pack", __file_format_name__, "", PackArchiveFile);
3067 if(hasattr(shutil, "register_archive_format")):
3068 def PackArchiveFileFunc(archive_name, source_dir, **kwargs):
3069 return PackArchiveFile(source_dir, archive_name, False, "auto", True, None, False, "crc32", [], __file_format_dict__['format_delimiter'], False, False);
3070 create_alias_function("Pack", __file_format_name__, "Func", PackArchiveFileFunc);
3072 def PackArchiveFileFromDirList(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3073 formatspecs = FormatSpecsListToDict(formatspecs);
3074 return PackArchiveFile(infiles, outfile, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, returnfp);
3076 create_alias_function("Pack", __file_format_name__, "FromDirList", PackArchiveFileFromDirList);
3078 def PackArchiveFileFromTarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3079 formatspecs = FormatSpecsListToDict(formatspecs);
3080 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3081 outfile = RemoveWindowsPath(outfile);
3082 checksumtype = checksumtype.lower();
3083 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3084 checksumtype="crc32";
3085 if(checksumtype=="none"):
3086 checksumtype = "";
3087 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3088 compression = "auto";
3089 if(compression not in compressionlist and compression is None):
3090 compression = "auto";
3091 if(verbose):
3092 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3093 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3094 if(os.path.exists(outfile)):
3095 try:
3096 os.unlink(outfile);
3097 except OSError as e:
3098 pass;
3099 if(outfile=="-"):
3100 verbose = False;
3101 catfp = BytesIO();
3102 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3103 catfp = outfile;
3104 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3105 catfp = BytesIO();
3106 else:
3107 fbasename = os.path.splitext(outfile)[0];
3108 fextname = os.path.splitext(outfile)[1];
3109 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3110 catver = formatspecs['format_ver'];
3111 fileheaderver = str(int(catver.replace(".", "")));
3112 curinode = 0;
3113 curfid = 0;
3114 inodelist = [];
3115 inodetofile = {};
3116 filetoinode = {};
3117 inodetocatinode = {};
3118 if(infile=="-"):
3119 infile = BytesIO();
3120 if(hasattr(sys.stdin, "buffer")):
3121 shutil.copyfileobj(sys.stdin.buffer, infile);
3122 else:
3123 shutil.copyfileobj(sys.stdin, infile);
3124 infile.seek(0, 0);
3125 if(not infile):
3126 return False;
3127 infile.seek(0, 0);
3128 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3129 infile = download_file_from_internet_file(infile);
3130 infile.seek(0, 0);
3131 if(not infile):
3132 return False;
3133 infile.seek(0, 0);
3134 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
3135 return False;
3136 elif(os.path.exists(infile) and os.path.isfile(infile)):
3137 try:
3138 if(not tarfile.TarFileCheck(infile)):
3139 return False;
3140 except AttributeError:
3141 if(not TarFileCheck(infile)):
3142 return False;
3143 else:
3144 return False;
3145 try:
3146 if(hasattr(infile, "read") or hasattr(infile, "write")):
3147 tarfp = tarfile.open(fileobj=infile, mode="r");
3148 else:
3149 tarfp = tarfile.open(infile, "r");
3150 except FileNotFoundError:
3151 return False;
3152 numfiles = int(len(tarfp.getmembers()));
3153 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3154 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
3155 if(re.findall(r"^[.|/]", member.name)):
3156 fname = member.name;
3157 else:
3158 fname = "./"+member.name;
3159 if(verbose):
3160 VerbosePrintOut(fname);
3161 fpremode = member.mode;
3162 ffullmode = member.mode;
3163 flinkcount = 0;
3164 ftype = 0;
3165 if(member.isreg()):
3166 ffullmode = member.mode + stat.S_IFREG;
3167 ftype = 0;
3168 elif(member.isdev()):
3169 ffullmode = member.mode;
3170 ftype = 7;
3171 elif(member.islnk()):
3172 ffullmode = member.mode + stat.S_IFREG;
3173 ftype = 1;
3174 elif(member.issym()):
3175 ffullmode = member.mode + stat.S_IFLNK;
3176 ftype = 2;
3177 elif(member.ischr()):
3178 ffullmode = member.mode + stat.S_IFCHR;
3179 ftype = 3;
3180 elif(member.isblk()):
3181 ffullmode = member.mode + stat.S_IFBLK;
3182 ftype = 4;
3183 elif(member.isdir()):
3184 ffullmode = member.mode + stat.S_IFDIR;
3185 ftype = 5;
3186 elif(member.isfifo()):
3187 ffullmode = member.mode + stat.S_IFIFO;
3188 ftype = 6;
3189 elif(member.issparse()):
3190 ffullmode = member.mode;
3191 ftype = 12;
3192 else:
3193 ffullmode = member.mode;
3194 ftype = 0;
3195 flinkname = "";
3196 fcurfid = format(int(curfid), 'x').lower();
3197 fcurinode = format(int(curfid), 'x').lower();
3198 curfid = curfid + 1;
3199 if(ftype==2):
3200 flinkname = member.linkname;
3201 fdev_minor = format(int(member.devminor), 'x').lower();
3202 fdev_major = format(int(member.devmajor), 'x').lower();
3203 frdev_minor = format(int(member.devminor), 'x').lower();
3204 frdev_major = format(int(member.devmajor), 'x').lower();
3205 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
3206 fsize = format(int("0"), 'x').lower();
3207 elif(ftype==0 or ftype==7):
3208 fsize = format(int(member.size), 'x').lower();
3209 else:
3210 fsize = format(int(member.size), 'x').lower();
3211 fatime = format(int(member.mtime), 'x').lower();
3212 fmtime = format(int(member.mtime), 'x').lower();
3213 fctime = format(int(member.mtime), 'x').lower();
3214 fbtime = format(int(member.mtime), 'x').lower();
3215 fmode = format(int(ffullmode), 'x').lower();
3216 fchmode = format(int(stat.S_IMODE(ffullmode)), 'x').lower();
3217 ftypemod = format(int(stat.S_IFMT(ffullmode)), 'x').lower();
3218 fuid = format(int(member.uid), 'x').lower();
3219 fgid = format(int(member.gid), 'x').lower();
3220 funame = member.uname;
3221 fgname = member.gname;
3222 flinkcount = format(int(flinkcount), 'x').lower();
3223 fwinattributes = format(int(0), 'x').lower();
3224 fcompression = "";
3225 fcsize = format(int(0), 'x').lower();
3226 fcontents = BytesIO();
3227 if(ftype==0 or ftype==7):
3228 with tarfp.extractfile(member) as fpc:
3229 shutil.copyfileobj(fpc, fcontents);
3230 if(not compresswholefile):
3231 fcontents.seek(0, 2);
3232 ucfsize = fcontents.tell();
3233 fcontents.seek(0, 0);
3234 if(compression=="auto"):
3235 ilsize = len(compressionlistalt);
3236 ilmin = 0;
3237 ilcsize = [];
3238 while(ilmin < ilsize):
3239 cfcontents = BytesIO();
3240 shutil.copyfileobj(fcontents, cfcontents);
3241 fcontents.seek(0, 0);
3242 cfcontents.seek(0, 0);
3243 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3244 if(cfcontents):
3245 cfcontents.seek(0, 2);
3246 ilcsize.append(cfcontents.tell());
3247 cfcontents.close();
3248 else:
3249 try:
3250 ilcsize.append(sys.maxint);
3251 except AttributeError:
3252 ilcsize.append(sys.maxsize);
3253 ilmin = ilmin + 1;
3254 ilcmin = ilcsize.index(min(ilcsize));
3255 compression = compressionlistalt[ilcmin];
3256 fcontents.seek(0, 0);
3257 cfcontents = BytesIO();
3258 shutil.copyfileobj(fcontents, cfcontents);
3259 cfcontents.seek(0, 0);
3260 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3261 cfcontents.seek(0, 2);
3262 cfsize = cfcontents.tell();
3263 if(ucfsize > cfsize):
3264 fcsize = format(int(cfsize), 'x').lower();
3265 fcompression = compression;
3266 fcontents.close();
3267 fcontents = cfcontents;
3268 if(fcompression=="none"):
3269 fcompression = "";
3270 fcontents.seek(0, 0);
3271 ftypehex = format(ftype, 'x').lower();
3272 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
3273 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3274 fcontents.close();
3275 if(numfiles>0):
3276 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3277 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3278 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3279 try:
3280 catfp.flush();
3281 os.fsync(catfp.fileno());
3282 except io.UnsupportedOperation:
3283 pass;
3284 except AttributeError:
3285 pass;
3286 except OSError as e:
3287 pass;
3288 if(outfile=="-"):
3289 catfp.seek(0, 0);
3290 if(hasattr(sys.stdout, "buffer")):
3291 shutil.copyfileobj(catfp, sys.stdout.buffer);
3292 else:
3293 shutil.copyfileobj(catfp, sys.stdout);
3294 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3295 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3296 catfp.seek(0, 0);
3297 upload_file_to_internet_file(catfp, outfile);
3298 if(returnfp):
3299 catfp.seek(0, 0);
3300 return catfp;
3301 else:
3302 catfp.close();
3303 return True;
3305 create_alias_function("Pack", __file_format_name__, "FromTarFile", PackArchiveFileFromTarFile);
3307 def PackArchiveFileFromZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3308 formatspecs = FormatSpecsListToDict(formatspecs);
3309 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3310 outfile = RemoveWindowsPath(outfile);
3311 checksumtype = checksumtype.lower();
3312 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3313 checksumtype="crc32";
3314 if(checksumtype=="none"):
3315 checksumtype = "";
3316 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3317 compression = "auto";
3318 if(compression not in compressionlist and compression is None):
3319 compression = "auto";
3320 if(verbose):
3321 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3322 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3323 if(os.path.exists(outfile)):
3324 try:
3325 os.unlink(outfile);
3326 except OSError as e:
3327 pass;
3328 if(outfile=="-"):
3329 verbose = False;
3330 catfp = BytesIO();
3331 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3332 catfp = outfile;
3333 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3334 catfp = BytesIO();
3335 else:
3336 fbasename = os.path.splitext(outfile)[0];
3337 fextname = os.path.splitext(outfile)[1];
3338 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3339 catver = formatspecs['format_ver'];
3340 fileheaderver = str(int(catver.replace(".", "")));
3341 curinode = 0;
3342 curfid = 0;
3343 inodelist = [];
3344 inodetofile = {};
3345 filetoinode = {};
3346 inodetocatinode = {};
3347 if(infile=="-"):
3348 infile = BytesIO();
3349 if(hasattr(sys.stdin, "buffer")):
3350 shutil.copyfileobj(sys.stdin.buffer, infile);
3351 else:
3352 shutil.copyfileobj(sys.stdin, infile);
3353 infile.seek(0, 0);
3354 if(not infile):
3355 return False;
3356 infile.seek(0, 0);
3357 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3358 infile = download_file_from_internet_file(infile);
3359 infile.seek(0, 0);
3360 if(not infile):
3361 return False;
3362 infile.seek(0, 0);
3363 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
3364 return False;
3365 else:
3366 return False;
3367 if(not zipfile.is_zipfile(infile)):
3368 return False;
3369 try:
3370 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
3371 except FileNotFoundError:
3372 return False;
3373 ziptest = zipfp.testzip();
3374 if(ziptest):
3375 VerbosePrintOut("Bad file found!");
3376 numfiles = int(len(zipfp.infolist()));
3377 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3378 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
3379 if(re.findall(r"^[.|/]", member.filename)):
3380 fname = member.filename;
3381 else:
3382 fname = "./"+member.filename;
3383 zipinfo = zipfp.getinfo(member.filename);
3384 if(verbose):
3385 VerbosePrintOut(fname);
3386 if(not member.is_dir()):
3387 fpremode = int(stat.S_IFREG + 438);
3388 elif(member.is_dir()):
3389 fpremode = int(stat.S_IFDIR + 511);
3390 flinkcount = 0;
3391 ftype = 0;
3392 if(not member.is_dir()):
3393 ftype = 0;
3394 elif(member.is_dir()):
3395 ftype = 5;
3396 flinkname = "";
3397 fcurfid = format(int(curfid), 'x').lower();
3398 fcurinode = format(int(curfid), 'x').lower();
3399 curfid = curfid + 1;
3400 fdev_minor = format(int(0), 'x').lower();
3401 fdev_major = format(int(0), 'x').lower();
3402 frdev_minor = format(int(0), 'x').lower();
3403 frdev_major = format(int(0), 'x').lower();
3404 if(ftype==5):
3405 fsize = format(int("0"), 'x').lower();
3406 elif(ftype==0):
3407 fsize = format(int(member.file_size), 'x').lower();
3408 else:
3409 fsize = format(int(member.file_size), 'x').lower();
3410 fatime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3411 fmtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3412 fctime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3413 fbtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3414 if(zipinfo.create_system==0 or zipinfo.create_system==10):
3415 fwinattributes = format(int(zipinfo.external_attr), 'x').lower();
3416 if(not member.is_dir()):
3417 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3418 fchmode = stat.S_IMODE(fmode);
3419 ftypemod = stat.S_IFMT(fmode);
3420 elif(member.is_dir()):
3421 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3422 fchmode = stat.S_IMODE(fmode);
3423 ftypemod = stat.S_IFMT(fmode);
3424 elif(zipinfo.create_system==3):
3425 fwinattributes = format(int(0), 'x').lower();
3426 fmode = format(int(zipinfo.external_attr), 'x').lower();
3427 fchmode = stat.S_IMODE(fmode);
3428 ftypemod = stat.S_IFMT(fmode);
3429 else:
3430 fwinattributes = format(int(0), 'x').lower();
3431 if(not member.is_dir()):
3432 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3433 fchmode = stat.S_IMODE(fmode);
3434 ftypemod = stat.S_IFMT(fmode);
3435 elif(member.is_dir()):
3436 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3437 fchmode = stat.S_IMODE(fmode);
3438 ftypemod = stat.S_IFMT(fmode);
3439 fcompression = "";
3440 fcsize = format(int(0), 'x').lower();
3441 try:
3442 fuid = format(int(os.getuid()), 'x').lower();
3443 except AttributeError:
3444 fuid = format(int(0), 'x').lower();
3445 except KeyError:
3446 fuid = format(int(0), 'x').lower();
3447 try:
3448 fgid = format(int(os.getgid()), 'x').lower();
3449 except AttributeError:
3450 fgid = format(int(0), 'x').lower();
3451 except KeyError:
3452 fgid = format(int(0), 'x').lower();
3453 try:
3454 import pwd;
3455 try:
3456 userinfo = pwd.getpwuid(os.getuid());
3457 funame = userinfo.pw_name;
3458 except KeyError:
3459 funame = "";
3460 except AttributeError:
3461 funame = "";
3462 except ImportError:
3463 funame = "";
3464 fgname = "";
3465 try:
3466 import grp;
3467 try:
3468 groupinfo = grp.getgrgid(os.getgid());
3469 fgname = groupinfo.gr_name;
3470 except KeyError:
3471 fgname = "";
3472 except AttributeError:
3473 fgname = "";
3474 except ImportError:
3475 fgname = "";
3476 fcontents = BytesIO();
3477 if(ftype==0):
3478 fcontents.write(zipfp.read(member.filename));
3479 if(not compresswholefile):
3480 fcontents.seek(0, 2);
3481 ucfsize = fcontents.tell();
3482 fcontents.seek(0, 0);
3483 if(compression=="auto"):
3484 ilsize = len(compressionlistalt);
3485 ilmin = 0;
3486 ilcsize = [];
3487 while(ilmin < ilsize):
3488 cfcontents = BytesIO();
3489 shutil.copyfileobj(fcontents, cfcontents);
3490 fcontents.seek(0, 0);
3491 cfcontents.seek(0, 0);
3492 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3493 cfcontents.seek(0, 2);
3494 ilcsize.append(cfcontents.tell());
3495 cfcontents.close();
3496 ilmin = ilmin + 1;
3497 ilcmin = ilcsize.index(min(ilcsize));
3498 compression = compressionlistalt[ilcmin];
3499 fcontents.seek(0, 0);
3500 cfcontents = BytesIO();
3501 shutil.copyfileobj(fcontents, cfcontents);
3502 cfcontents.seek(0, 0);
3503 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3504 cfcontents.seek(0, 2);
3505 cfsize = cfcontents.tell();
3506 if(ucfsize > cfsize):
3507 fcsize = format(int(cfsize), 'x').lower();
3508 fcompression = compression;
3509 fcontents.close();
3510 fcontents = cfcontents;
3511 if(fcompression=="none"):
3512 fcompression = "";
3513 fcontents.seek(0, 0);
3514 ftypehex = format(ftype, 'x').lower();
3515 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
3516 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3517 fcontents.close();
3518 if(numfiles>0):
3519 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3520 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3521 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3522 try:
3523 catfp.flush();
3524 os.fsync(catfp.fileno());
3525 except io.UnsupportedOperation:
3526 pass;
3527 except AttributeError:
3528 pass;
3529 except OSError as e:
3530 pass;
3531 if(outfile=="-"):
3532 catfp.seek(0, 0);
3533 if(hasattr(sys.stdout, "buffer")):
3534 shutil.copyfileobj(catfp, sys.stdout.buffer);
3535 else:
3536 shutil.copyfileobj(catfp, sys.stdout);
3537 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3538 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3539 catfp.seek(0, 0);
3540 upload_file_to_internet_file(catfp, outfile);
3541 if(returnfp):
3542 catfp.seek(0, 0);
3543 return catfp;
3544 else:
3545 catfp.close();
3546 return True;
3548 create_alias_function("Pack", __file_format_name__, "FromZipFile", PackArchiveFileFromZipFile);
3550 if(not rarfile_support):
3551 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3552 return False
3554 if(rarfile_support):
3555 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3556 formatspecs = FormatSpecsListToDict(formatspecs);
3557 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3558 outfile = RemoveWindowsPath(outfile);
3559 checksumtype = checksumtype.lower();
3560 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3561 checksumtype="crc32";
3562 if(checksumtype=="none"):
3563 checksumtype = "";
3564 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3565 compression = "auto";
3566 if(compression not in compressionlist and compression is None):
3567 compression = "auto";
3568 if(verbose):
3569 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3570 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3571 if(os.path.exists(outfile)):
3572 try:
3573 os.unlink(outfile);
3574 except OSError as e:
3575 pass;
3576 if(outfile=="-"):
3577 verbose = False;
3578 catfp = BytesIO();
3579 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3580 catfp = outfile;
3581 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3582 catfp = BytesIO();
3583 else:
3584 fbasename = os.path.splitext(outfile)[0];
3585 fextname = os.path.splitext(outfile)[1];
3586 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3587 catver = formatspecs['format_ver'];
3588 fileheaderver = str(int(catver.replace(".", "")));
3589 curinode = 0;
3590 curfid = 0;
3591 inodelist = [];
3592 inodetofile = {};
3593 filetoinode = {};
3594 inodetocatinode = {};
3595 if(not os.path.exists(infile) or not os.path.isfile(infile)):
3596 return False;
3597 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
3598 return False;
3599 rarfp = rarfile.RarFile(infile, "r");
3600 rartest = rarfp.testrar();
3601 if(rartest):
3602 VerbosePrintOut("Bad file found!");
3603 numfiles = int(len(rarfp.infolist()));
3604 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3605 try:
3606 catfp.flush();
3607 os.fsync(catfp.fileno());
3608 except io.UnsupportedOperation:
3609 pass;
3610 except AttributeError:
3611 pass;
3612 except OSError as e:
3613 pass;
3614 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
3615 is_unix = False;
3616 is_windows = False;
3617 if(member.host_os==rarfile.RAR_OS_UNIX):
3618 is_windows = False;
3619 try:
3620 member.external_attr
3621 is_unix = True;
3622 except AttributeError:
3623 is_unix = False;
3624 elif(member.host_os==rarfile.RAR_OS_WIN32):
3625 is_unix = False;
3626 try:
3627 member.external_attr
3628 is_windows = True;
3629 except AttributeError:
3630 is_windows = False;
3631 else:
3632 is_unix = False;
3633 is_windows = False;
3634 if(re.findall(r"^[.|/]", member.filename)):
3635 fname = member.filename;
3636 else:
3637 fname = "./"+member.filename;
3638 rarinfo = rarfp.getinfo(member.filename);
3639 if(verbose):
3640 VerbosePrintOut(fname);
3641 if(is_unix and member.external_attr !=0):
3642 fpremode = int(member.external_attr);
3643 elif(member.is_file()):
3644 fpremode = int(stat.S_IFREG + 438);
3645 elif(member.is_symlink()):
3646 fpremode = int(stat.S_IFLNK + 438);
3647 elif(member.is_dir()):
3648 fpremode = int(stat.S_IFDIR + 511);
3649 if(is_windows and member.external_attr !=0):
3650 fwinattributes = format(int(member.external_attr), 'x').lower();
3651 else:
3652 fwinattributes = format(int(0), 'x').lower();
3653 fcompression = "";
3654 fcsize = format(int(0), 'x').lower();
3655 flinkcount = 0;
3656 ftype = 0;
3657 if(member.is_file()):
3658 ftype = 0;
3659 elif(member.is_symlink()):
3660 ftype = 2;
3661 elif(member.is_dir()):
3662 ftype = 5;
3663 flinkname = "";
3664 if(ftype==2):
3665 flinkname = rarfp.read(member.filename).decode("UTF-8");
3666 fcurfid = format(int(curfid), 'x').lower();
3667 fcurinode = format(int(curfid), 'x').lower();
3668 curfid = curfid + 1;
3669 fdev_minor = format(int(0), 'x').lower();
3670 fdev_major = format(int(0), 'x').lower();
3671 frdev_minor = format(int(0), 'x').lower();
3672 frdev_major = format(int(0), 'x').lower();
3673 if(ftype==5):
3674 fsize = format(int("0"), 'x').lower();
3675 elif(ftype==0):
3676 fsize = format(int(member.file_size), 'x').lower();
3677 else:
3678 fsize = format(int(member.file_size), 'x').lower();
3679 try:
3680 if(member.atime):
3681 fatime = format(int(member.atime.timestamp()), 'x').lower();
3682 else:
3683 fatime = format(int(member.mtime.timestamp()), 'x').lower();
3684 except AttributeError:
3685 fatime = format(int(member.mtime.timestamp()), 'x').lower();
3686 fmtime = format(int(member.mtime.timestamp()), 'x').lower();
3687 try:
3688 if(member.ctime):
3689 fctime = format(int(member.ctime.timestamp()), 'x').lower();
3690 else:
3691 fctime = format(int(member.mtime.timestamp()), 'x').lower();
3692 except AttributeError:
3693 fctime = format(int(member.mtime.timestamp()), 'x').lower();
3694 fbtime = format(int(member.mtime.timestamp()), 'x').lower();
3695 if(is_unix and member.external_attr !=0):
3696 fmode = format(int(member.external_attr), 'x').lower();
3697 fchmode = format(int(stat.S_IMODE(member.external_attr)), 'x').lower();
3698 ftypemod = format(int(stat.S_IFMT(member.external_attr)), 'x').lower();
3699 elif(member.is_file()):
3700 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3701 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3702 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3703 elif(member.is_symlink()):
3704 fmode = format(int(stat.S_IFLNK + 438), 'x').lower();
3705 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3706 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3707 elif(member.is_dir()):
3708 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3709 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
3710 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
3711 try:
3712 fuid = format(int(os.getuid()), 'x').lower();
3713 except AttributeError:
3714 fuid = format(int(0), 'x').lower();
3715 except KeyError:
3716 fuid = format(int(0), 'x').lower();
3717 try:
3718 fgid = format(int(os.getgid()), 'x').lower();
3719 except AttributeError:
3720 fgid = format(int(0), 'x').lower();
3721 except KeyError:
3722 fgid = format(int(0), 'x').lower();
3723 try:
3724 import pwd;
3725 try:
3726 userinfo = pwd.getpwuid(os.getuid());
3727 funame = userinfo.pw_name;
3728 except KeyError:
3729 funame = "";
3730 except AttributeError:
3731 funame = "";
3732 except ImportError:
3733 funame = "";
3734 fgname = "";
3735 try:
3736 import grp;
3737 try:
3738 groupinfo = grp.getgrgid(os.getgid());
3739 fgname = groupinfo.gr_name;
3740 except KeyError:
3741 fgname = "";
3742 except AttributeError:
3743 fgname = "";
3744 except ImportError:
3745 fgname = "";
3746 fcontents = BytesIO();
3747 if(ftype==0):
3748 fcontents.write(rarfp.read(member.filename));
3749 if(not compresswholefile):
3750 fcontents.seek(0, 2);
3751 ucfsize = fcontents.tell();
3752 fcontents.seek(0, 0);
3753 if(compression=="auto"):
3754 ilsize = len(compressionlistalt);
3755 ilmin = 0;
3756 ilcsize = [];
3757 while(ilmin < ilsize):
3758 cfcontents = BytesIO();
3759 shutil.copyfileobj(fcontents, cfcontents);
3760 fcontents.seek(0, 0);
3761 cfcontents.seek(0, 0);
3762 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3763 if(cfcontents):
3764 cfcontents.seek(0, 2);
3765 ilcsize.append(cfcontents.tell());
3766 cfcontents.close();
3767 else:
3768 try:
3769 ilcsize.append(sys.maxint);
3770 except AttributeError:
3771 ilcsize.append(sys.maxsize);
3772 ilmin = ilmin + 1;
3773 ilcmin = ilcsize.index(min(ilcsize));
3774 compression = compressionlistalt[ilcmin];
3775 fcontents.seek(0, 0);
3776 cfcontents = BytesIO();
3777 shutil.copyfileobj(fcontents, cfcontents);
3778 cfcontents.seek(0, 0);
3779 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3780 cfcontents.seek(0, 2);
3781 cfsize = cfcontents.tell();
3782 if(ucfsize > cfsize):
3783 fcsize = format(int(cfsize), 'x').lower();
3784 fcompression = compression;
3785 fcontents.close();
3786 fcontents = cfcontents;
3787 if(fcompression=="none"):
3788 fcompression = "";
3789 fcontents.seek(0, 0);
3790 ftypehex = format(ftype, 'x').lower();
3791 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
3792 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3793 fcontents.close();
3794 if(numfiles>0):
3795 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3796 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3797 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
3798 try:
3799 catfp.flush();
3800 os.fsync(catfp.fileno());
3801 except io.UnsupportedOperation:
3802 pass;
3803 except AttributeError:
3804 pass;
3805 except OSError as e:
3806 pass;
3807 if(outfile=="-"):
3808 catfp.seek(0, 0)
3809 if(hasattr(sys.stdout, "buffer")):
3810 shutil.copyfileobj(catfp, sys.stdout.buffer);
3811 else:
3812 shutil.copyfileobj(catfp, sys.stdout);
3813 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3814 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3815 catfp.seek(0, 0);
3816 upload_file_to_internet_file(catfp, outfile);
3817 if(returnfp):
3818 catfp.seek(0, 0)
3819 return catfp
3820 else:
3821 catfp.close()
3822 return True;
3824 create_alias_function("Pack", __file_format_name__, "FromRarFile", PackArchiveFileFromRarFile);
3826 if(not py7zr_support):
3827 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3828 return False
3830 if(py7zr_support):
3831 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3832 formatspecs = FormatSpecsListToDict(formatspecs);
3833 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3834 outfile = RemoveWindowsPath(outfile);
3835 checksumtype = checksumtype.lower();
3836 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3837 checksumtype="crc32";
3838 if(checksumtype=="none"):
3839 checksumtype = "";
3840 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3841 compression = "auto";
3842 if(compression not in compressionlist and compression is None):
3843 compression = "auto";
3844 if(verbose):
3845 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3846 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3847 if(os.path.exists(outfile)):
3848 try:
3849 os.unlink(outfile);
3850 except OSError as e:
3851 pass;
3852 if(outfile=="-"):
3853 verbose = False;
3854 catfp = BytesIO();
3855 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3856 catfp = outfile;
3857 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3858 catfp = BytesIO();
3859 else:
3860 fbasename = os.path.splitext(outfile)[0];
3861 fextname = os.path.splitext(outfile)[1];
3862 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3863 catver = formatspecs['format_ver'];
3864 fileheaderver = str(int(catver.replace(".", "")));
3865 curinode = 0;
3866 curfid = 0;
3867 inodelist = [];
3868 inodetofile = {};
3869 filetoinode = {};
3870 inodetocatinode = {};
3871 if(not os.path.exists(infile) or not os.path.isfile(infile)):
3872 return False;
3873 szpfp = py7zr.SevenZipFile(infile, mode="r");
3874 file_content = szpfp.readall();
3875 #sztest = szpfp.testzip();
3876 sztestalt = szpfp.test();
3877 if(sztestalt):
3878 VerbosePrintOut("Bad file found!");
3879 numfiles = int(len(szpfp.list()));
3880 AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3881 for member in sorted(szpfp.list(), key=lambda x: x.filename):
3882 if(re.findall(r"^[.|/]", member.filename)):
3883 fname = member.filename;
3884 else:
3885 fname = "./"+member.filename;
3886 if(verbose):
3887 VerbosePrintOut(fname);
3888 if(not member.is_directory):
3889 fpremode = int(stat.S_IFREG + 438);
3890 elif(member.is_directory):
3891 fpremode = int(stat.S_IFDIR + 511);
3892 fwinattributes = format(int(0), 'x').lower();
3893 fcompression = "";
3894 fcsize = format(int(0), 'x').lower();
3895 flinkcount = 0;
3896 ftype = 0;
3897 if(member.is_directory):
3898 ftype = 5;
3899 else:
3900 ftype = 0;
3901 flinkname = "";
3902 fcurfid = format(int(curfid), 'x').lower();
3903 fcurinode = format(int(curfid), 'x').lower();
3904 curfid = curfid + 1;
3905 fdev_minor = format(int(0), 'x').lower();
3906 fdev_major = format(int(0), 'x').lower();
3907 frdev_minor = format(int(0), 'x').lower();
3908 frdev_major = format(int(0), 'x').lower();
3909 if(ftype==5):
3910 fsize = format(int("0"), 'x').lower();
3911 fatime = format(int(member.creationtime.timestamp()), 'x').lower();
3912 fmtime = format(int(member.creationtime.timestamp()), 'x').lower();
3913 fctime = format(int(member.creationtime.timestamp()), 'x').lower();
3914 fbtime = format(int(member.creationtime.timestamp()), 'x').lower();
3915 if(member.is_directory):
3916 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3917 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
3918 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
3919 else:
3920 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3921 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3922 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3923 try:
3924 fuid = format(int(os.getuid()), 'x').lower();
3925 except AttributeError:
3926 fuid = format(int(0), 'x').lower();
3927 except KeyError:
3928 fuid = format(int(0), 'x').lower();
3929 try:
3930 fgid = format(int(os.getgid()), 'x').lower();
3931 except AttributeError:
3932 fgid = format(int(0), 'x').lower();
3933 except KeyError:
3934 fgid = format(int(0), 'x').lower();
3935 try:
3936 import pwd;
3937 try:
3938 userinfo = pwd.getpwuid(os.getuid());
3939 funame = userinfo.pw_name;
3940 except KeyError:
3941 funame = "";
3942 except AttributeError:
3943 funame = "";
3944 except ImportError:
3945 funame = "";
3946 fgname = "";
3947 try:
3948 import grp;
3949 try:
3950 groupinfo = grp.getgrgid(os.getgid());
3951 fgname = groupinfo.gr_name;
3952 except KeyError:
3953 fgname = "";
3954 except AttributeError:
3955 fgname = "";
3956 except ImportError:
3957 fgname = "";
3958 fcontents = BytesIO();
3959 if(ftype==0):
3960 fcontents.write(file_content[member.filename].read());
3961 fsize = format(fcontents.tell(), 'x').lower();
3962 file_content[member.filename].close();
3963 if(not compresswholefile):
3964 fcontents.seek(0, 2);
3965 ucfsize = fcontents.tell();
3966 fcontents.seek(0, 0);
3967 if(compression=="auto"):
3968 ilsize = len(compressionlistalt);
3969 ilmin = 0;
3970 ilcsize = [];
3971 while(ilmin < ilsize):
3972 cfcontents = BytesIO();
3973 shutil.copyfileobj(fcontents, cfcontents);
3974 fcontents.seek(0, 0);
3975 cfcontents.seek(0, 0);
3976 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3977 if(cfcontents):
3978 cfcontents.seek(0, 2);
3979 ilcsize.append(cfcontents.tell());
3980 cfcontents.close();
3981 else:
3982 try:
3983 ilcsize.append(sys.maxint);
3984 except AttributeError:
3985 ilcsize.append(sys.maxsize);
3986 ilmin = ilmin + 1;
3987 ilcmin = ilcsize.index(min(ilcsize));
3988 compression = compressionlistalt[ilcmin];
3989 fcontents.seek(0, 0);
3990 cfcontents = BytesIO();
3991 shutil.copyfileobj(fcontents, cfcontents);
3992 cfcontents.seek(0, 0);
3993 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3994 cfcontents.seek(0, 2);
3995 cfsize = cfcontents.tell();
3996 if(ucfsize > cfsize):
3997 fcsize = format(int(cfsize), 'x').lower();
3998 fcompression = compression;
3999 fcontents.close();
4000 fcontents = cfcontents;
4001 if(fcompression=="none"):
4002 fcompression = "";
4003 fcontents.seek(0, 0);
4004 ftypehex = format(ftype, 'x').lower();
4005 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
4006 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
4007 fcontents.close();
4008 if(numfiles>0):
4009 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
4010 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
4011 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
4012 try:
4013 catfp.flush();
4014 os.fsync(catfp.fileno());
4015 except io.UnsupportedOperation:
4016 pass;
4017 except AttributeError:
4018 pass;
4019 except OSError as e:
4020 pass;
4021 if(outfile=="-"):
4022 catfp.seek(0, 0);
4023 if(hasattr(sys.stdout, "buffer")):
4024 shutil.copyfileobj(catfp, sys.stdout.buffer);
4025 else:
4026 shutil.copyfileobj(catfp, sys.stdout);
4027 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
4028 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
4029 catfp.seek(0, 0);
4030 upload_file_to_internet_file(catfp, outfile);
4031 if(returnfp):
4032 catfp.seek(0, 0)
4033 return catfp
4034 else:
4035 catfp.close()
4036 return True;
4038 create_alias_function("Pack", __file_format_name__, "FromSevenZipFile", PackArchiveFileFromSevenZipFile);
4040 def PackArchiveFileFromInFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
4041 formatspecs = FormatSpecsListToDict(formatspecs);
4042 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4043 if(verbose):
4044 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
4045 if(checkcompressfile=="tarfile"):
4046 return PackArchiveFileFromTarFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4047 elif(checkcompressfile=="zipfile"):
4048 return PackArchiveFileFromZipFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4049 elif(checkcompressfile=="catfile"):
4050 return RePackArchiveFile(infile, outfile, compression, compresswholefile, compressionlevel, False, 0, 0, checksumtype, False, extradata, formatspecs, verbose, returnfp);
4051 elif(rarfile_support and checkcompressfile=="rarfile"):
4052 return PackArchiveFileFromRarFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4053 elif(py7zr_support and checkcompressfile=="7zipfile"):
4054 return PackArchiveFileFromSevenZipFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
4055 else:
4056 return False;
4057 return False;
4059 create_alias_function("Pack", __file_format_name__, "FromInFile", PackArchiveFileFromInFile);
4061 def ArchiveFileSeekToFileNum(infile, seekto=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4062 formatspecs = FormatSpecsListToDict(formatspecs);
4063 if(hasattr(infile, "read") or hasattr(infile, "write")):
4064 catfp = infile;
4065 catfp.seek(0, 0);
4066 catfp = UncompressArchiveFile(catfp, formatspecs);
4067 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4068 if(checkcompressfile=="tarfile"):
4069 return TarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4070 if(checkcompressfile=="zipfile"):
4071 return ZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4072 if(rarfile_support and checkcompressfile=="rarfile"):
4073 return RarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4074 if(py7zr_support and checkcompressfile=="7zipfile"):
4075 return SevenZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4076 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4077 return False;
4078 if(not catfp):
4079 return False;
4080 catfp.seek(0, 0);
4081 elif(infile=="-"):
4082 catfp = BytesIO();
4083 if(hasattr(sys.stdin, "buffer")):
4084 shutil.copyfileobj(sys.stdin.buffer, catfp);
4085 else:
4086 shutil.copyfileobj(sys.stdin, catfp);
4087 catfp.seek(0, 0);
4088 catfp = UncompressArchiveFile(catfp, formatspecs);
4089 if(not catfp):
4090 return False;
4091 catfp.seek(0, 0);
4092 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4093 catfp = download_file_from_internet_file(infile);
4094 catfp.seek(0, 0);
4095 catfp = UncompressArchiveFile(catfp, formatspecs);
4096 if(not catfp):
4097 return False;
4098 catfp.seek(0, 0);
4099 else:
4100 infile = RemoveWindowsPath(infile);
4101 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4102 if(checkcompressfile=="tarfile"):
4103 return TarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4104 if(checkcompressfile=="zipfile"):
4105 return ZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4106 if(rarfile_support and checkcompressfile=="rarfile"):
4107 return RarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4108 if(py7zr_support and checkcompressfile=="7zipfile"):
4109 return SevenZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
4110 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4111 return False;
4112 compresscheck = CheckCompressionType(infile, formatspecs, True);
4113 if(not compresscheck):
4114 fextname = os.path.splitext(infile)[1];
4115 if(fextname==".gz"):
4116 compresscheck = "gzip";
4117 elif(fextname==".bz2"):
4118 compresscheck = "bzip2";
4119 elif(fextname==".zst"):
4120 compresscheck = "zstd";
4121 elif(fextname==".lz4" or fextname==".clz4"):
4122 compresscheck = "lz4";
4123 elif(fextname==".lzo" or fextname==".lzop"):
4124 compresscheck = "lzo";
4125 elif(fextname==".lzma" or fextname==".xz"):
4126 compresscheck = "lzma";
4127 else:
4128 return False;
4129 if(not compresscheck):
4130 return False;
4131 catfp = UncompressFile(infile, formatspecs, "rb");
4133 try:
4134 catfp.seek(0, 2);
4135 except OSError:
4136 SeekToEndOfFile(catfp);
4137 except ValueError:
4138 SeekToEndOfFile(catfp);
4139 CatSize = catfp.tell();
4140 CatSizeEnd = CatSize;
4142 try:
4143 catfp.seek(0, 0);
4144 except OSError:
4145 return False;
4146 except ValueError:
4147 return False;
4148 curloc = catfp.tell();
4149 if(curloc>0):
4150 catfp.seek(0, 0);
4151 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4152 if(curloc>0):
4153 catfp.seek(curloc, 0);
4154 catstring = catheader[0];
4155 catversion = re.findall(r"([\d]+)", catstring);
4156 fprenumfiles = catheader[1];
4157 fnumfiles = int(fprenumfiles, 16);
4158 fprechecksumtype = catheader[2];
4159 fprechecksum = catheader[3];
4160 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4161 fnumfileshex = format(int(fnumfiles), 'x').lower();
4162 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4163 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4164 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4165 fheadtell = len(fileheader);
4166 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4167 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
4168 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4169 return False;
4170 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
4171 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4172 if(seekto>=fnumfiles):
4173 seekto = fnumfiles - 1;
4174 if(seekto<0):
4175 seekto = 0;
4176 if(seekto>=0):
4177 il = -1;
4178 while(il < seekto):
4179 prefhstart = catfp.tell();
4180 if(formatspecs['new_style']):
4181 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4182 else:
4183 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4184 if(len(preheaderdata)==0):
4185 break;
4186 prefheadsize = int(preheaderdata[0], 16);
4187 prefnumfields = int(preheaderdata[1], 16);
4188 preftype = int(preheaderdata[2], 16);
4189 if(re.findall(r"^[.|/]", preheaderdata[3])):
4190 prefname = preheaderdata[3];
4191 else:
4192 prefname = "./"+preheaderdata[3];
4193 prefbasedir = os.path.dirname(prefname);
4194 preflinkname = preheaderdata[4];
4195 prefsize = int(preheaderdata[5], 16);
4196 prefatime = int(preheaderdata[6], 16);
4197 prefmtime = int(preheaderdata[7], 16);
4198 prefctime = int(preheaderdata[8], 16);
4199 prefbtime = int(preheaderdata[9], 16);
4200 prefmode = int(preheaderdata[10], 16);
4201 prefchmode = stat.S_IMODE(prefmode);
4202 preftypemod = stat.S_IFMT(prefmode);
4203 prefwinattributes = int(preheaderdata[11], 16);
4204 prefcompression = preheaderdata[12];
4205 prefcsize = int(preheaderdata[13], 16);
4206 prefuid = int(preheaderdata[14], 16);
4207 prefuname = preheaderdata[15];
4208 prefgid = int(preheaderdata[16], 16);
4209 prefgname = preheaderdata[17];
4210 fid = int(preheaderdata[18], 16);
4211 finode = int(preheaderdata[19], 16);
4212 flinkcount = int(preheaderdata[20], 16);
4213 prefdev_minor = int(preheaderdata[21], 16);
4214 prefdev_major = int(preheaderdata[22], 16);
4215 prefrdev_minor = int(preheaderdata[23], 16);
4216 prefrdev_major = int(preheaderdata[24], 16);
4217 prefseeknextfile = preheaderdata[25];
4218 prefextrasize = int(preheaderdata[26], 16);
4219 prefextrafields = int(preheaderdata[27], 16);
4220 extrafieldslist = [];
4221 extrastart = 28;
4222 extraend = extrastart + prefextrafields;
4223 extrafieldslist = [];
4224 if(extrastart<extraend):
4225 extrafieldslist.append(preheaderdata[extrastart]);
4226 extrastart = extrastart + 1;
4227 prefcs = preheaderdata[-2].lower();
4228 prenewfcs = preheaderdata[-1].lower();
4229 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-4].lower(), True, formatspecs);
4230 if(prefcs!=prenewfcs and not skipchecksum):
4231 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4232 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4233 return False;
4234 valid_archive = False;
4235 invalid_archive = True;
4236 prefhend = catfp.tell() - 1;
4237 prefcontentstart = catfp.tell();
4238 prefcontents = "";
4239 pyhascontents = False;
4240 if(prefsize>0):
4241 if(prefcompression):
4242 prefcontents = catfp.read(prefsize);
4243 else:
4244 prefcontents = catfp.read(prefcsize);
4245 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4246 pyhascontents = True;
4247 if(prefccs!=prenewfccs and not skipchecksum):
4248 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4249 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4250 return False;
4251 if(re.findall(r"^\+([0-9]+)", prefseeknextfile)):
4252 fseeknextasnum = int(prefseeknextfile.replace("+", ""));
4253 if(abs(fseeknextasnum)==0):
4254 pass;
4255 catfp.seek(fseeknextasnum, 1);
4256 elif(re.findall(r"^\-([0-9]+)", prefseeknextfile)):
4257 fseeknextasnum = int(prefseeknextfile);
4258 if(abs(fseeknextasnum)==0):
4259 pass;
4260 catfp.seek(fseeknextasnum, 1);
4261 elif(re.findall(r"^([0-9]+)", prefseeknextfile)):
4262 fseeknextasnum = int(prefseeknextfile);
4263 if(abs(fseeknextasnum)==0):
4264 pass;
4265 catfp.seek(fseeknextasnum, 0);
4266 else:
4267 return False;
4268 il = il + 1;
4269 catfp.seek(seekstart, 0);
4270 fileidnum = il;
4271 catfheadsize = int(preheaderdata[0], 16);
4272 catfnumfields = int(preheaderdata[1], 16);
4273 catftype = int(preheaderdata[2], 16);
4274 if(re.findall(r"^[.|/]", preheaderdata[3])):
4275 catfname = preheaderdata[3];
4276 else:
4277 catfname = "./"+preheaderdata[3];
4278 catflinkname = preheaderdata[4];
4279 catfsize = int(preheaderdata[5], 16);
4280 catfbasedir = os.path.dirname(catfname);
4281 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
4282 if(returnfp):
4283 catlist.update({'catfp': catfp});
4284 else:
4285 catfp.close();
4286 return catlist;
4288 create_alias_function("", __file_format_name__, "SeekToFileNum", ArchiveFileSeekToFileNum);
4290 def ArchiveFileSeekToFileName(infile, seekfile=None, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4291 formatspecs = FormatSpecsListToDict(formatspecs);
4292 if(hasattr(infile, "read") or hasattr(infile, "write")):
4293 catfp = infile;
4294 catfp.seek(0, 0);
4295 catfp = UncompressArchiveFile(catfp, formatspecs);
4296 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4297 if(checkcompressfile=="tarfile"):
4298 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4299 if(checkcompressfile=="zipfile"):
4300 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4301 if(rarfile_support and checkcompressfile=="rarfile"):
4302 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4303 if(py7zr_support and checkcompressfile=="7zipfile"):
4304 return SevenZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4305 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4306 return False;
4307 if(not catfp):
4308 return False;
4309 catfp.seek(0, 0);
4310 elif(infile=="-"):
4311 catfp = BytesIO();
4312 if(hasattr(sys.stdin, "buffer")):
4313 shutil.copyfileobj(sys.stdin.buffer, catfp);
4314 else:
4315 shutil.copyfileobj(sys.stdin, catfp);
4316 catfp.seek(0, 0);
4317 catfp = UncompressArchiveFile(catfp, formatspecs);
4318 if(not catfp):
4319 return False;
4320 catfp.seek(0, 0);
4321 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4322 catfp = download_file_from_internet_file(infile);
4323 catfp = UncompressArchiveFile(catfp, formatspecs);
4324 catfp.seek(0, 0);
4325 if(not catfp):
4326 return False;
4327 catfp.seek(0, 0);
4328 else:
4329 infile = RemoveWindowsPath(infile);
4330 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4331 if(checkcompressfile=="tarfile"):
4332 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4333 if(checkcompressfile=="zipfile"):
4334 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4335 if(rarfile_support and checkcompressfile=="rarfile"):
4336 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4337 if(py7zr_support and checkcompressfile=="7zipfile"):
4338 return SevenZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4339 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4340 return False;
4341 compresscheck = CheckCompressionType(infile, formatspecs, True);
4342 if(not compresscheck):
4343 fextname = os.path.splitext(infile)[1];
4344 if(fextname==".gz"):
4345 compresscheck = "gzip";
4346 elif(fextname==".bz2"):
4347 compresscheck = "bzip2";
4348 elif(fextname==".zst"):
4349 compresscheck = "zstd";
4350 elif(fextname==".lz4" or fextname==".clz4"):
4351 compresscheck = "lz4";
4352 elif(fextname==".lzo" or fextname==".lzop"):
4353 compresscheck = "lzo";
4354 elif(fextname==".lzma" or fextname==".xz"):
4355 compresscheck = "lzma";
4356 else:
4357 return False;
4358 if(not compresscheck):
4359 return False;
4360 catfp = UncompressFile(infile, formatspecs, "rb");
4362 try:
4363 catfp.seek(0, 2);
4364 except OSError:
4365 SeekToEndOfFile(catfp);
4366 except ValueError:
4367 SeekToEndOfFile(catfp);
4368 CatSize = catfp.tell();
4369 CatSizeEnd = CatSize;
4371 try:
4372 catfp.seek(0, 0);
4373 except OSError:
4374 return False;
4375 except ValueError:
4376 return False;
4377 curloc = catfp.tell();
4378 if(curloc>0):
4379 catfp.seek(0, 0);
4380 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4381 if(curloc>0):
4382 catfp.seek(curloc, 0);
4383 catstring = catheader[0];
4384 catversion = re.findall(r"([\d]+)", catstring);
4385 fprenumfiles = catheader[1];
4386 fnumfiles = int(fprenumfiles, 16);
4387 fprechecksumtype = catheader[2];
4388 fprechecksum = catheader[3];
4389 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4390 fnumfileshex = format(int(fnumfiles), 'x').lower();
4391 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4392 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4393 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4394 fheadtell = len(fileheader);
4395 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4396 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
4397 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4398 return False;
4399 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
4400 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4401 seekto = fnumfiles - 1
4402 filefound = False;
4403 if(seekto>=0):
4404 il = -1;
4405 while(il < seekto):
4406 prefhstart = catfp.tell();
4407 if(formatspecs['new_style']):
4408 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4409 else:
4410 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4411 if(len(preheaderdata)==0):
4412 break;
4413 prefheadsize = int(preheaderdata[0], 16);
4414 prefnumfields = int(preheaderdata[1], 16);
4415 preftype = int(preheaderdata[2], 16);
4416 if(re.findall(r"^[.|/]", preheaderdata[3])):
4417 prefname = preheaderdata[3];
4418 else:
4419 prefname = "./"+preheaderdata[3];
4420 prefbasedir = os.path.dirname(prefname);
4421 preflinkname = preheaderdata[4];
4422 prefsize = int(preheaderdata[5], 16);
4423 prefatime = int(preheaderdata[6], 16);
4424 prefmtime = int(preheaderdata[7], 16);
4425 prefctime = int(preheaderdata[8], 16);
4426 prefbtime = int(preheaderdata[9], 16);
4427 prefmode = int(preheaderdata[10], 16);
4428 prefchmode = stat.S_IMODE(prefmode);
4429 preftypemod = stat.S_IFMT(prefmode);
4430 prefwinattributes = int(preheaderdata[11], 16);
4431 prefcompression = preheaderdata[12];
4432 prefcsize = int(preheaderdata[13], 16);
4433 prefuid = int(preheaderdata[14], 16);
4434 prefuname = preheaderdata[15];
4435 prefgid = int(preheaderdata[16], 16);
4436 prefgname = preheaderdata[17];
4437 fid = int(preheaderdata[18], 16);
4438 finode = int(preheaderdata[19], 16);
4439 flinkcount = int(preheaderdata[20], 16);
4440 prefdev_minor = int(preheaderdata[21], 16);
4441 prefdev_major = int(preheaderdata[22], 16);
4442 prefrdev_minor = int(preheaderdata[23], 16);
4443 prefrdev_major = int(preheaderdata[24], 16);
4444 prefseeknextfile = preheaderdata[25];
4445 prefextrasize = int(preheaderdata[26], 16);
4446 prefextrafields = int(preheaderdata[27], 16);
4447 extrafieldslist = [];
4448 extrastart = 28;
4449 extraend = extrastart + prefextrafields;
4450 extrafieldslist = [];
4451 if(extrastart<extraend):
4452 extrafieldslist.append(preheaderdata[extrastart]);
4453 extrastart = extrastart + 1;
4454 prefcs = preheaderdata[-2].lower();
4455 prenewfcs = preheaderdata[-1].lower();
4456 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-4].lower(), True, formatspecs);
4457 if(prefcs!=prenewfcs and not skipchecksum):
4458 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4459 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4460 return False;
4461 valid_archive = False;
4462 invalid_archive = True;
4463 prefhend = catfp.tell() - 1;
4464 prefcontentstart = catfp.tell();
4465 prefcontents = "";
4466 pyhascontents = False;
4467 if(prefsize>0):
4468 if(prefcompression):
4469 prefcontents = catfp.read(prefsize);
4470 else:
4471 prefcontents = catfp.read(prefcsize);
4472 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4473 pyhascontents = True;
4474 if(prefccs!=prenewfccs and not skipchecksum):
4475 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4476 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4477 return False;
4478 if(re.findall(r"^\+([0-9]+)", prefseeknextfile)):
4479 fseeknextasnum = int(prefseeknextfile.replace("+", ""));
4480 if(abs(fseeknextasnum)==0):
4481 pass;
4482 catfp.seek(fseeknextasnum, 1);
4483 elif(re.findall(r"^\-([0-9]+)", prefseeknextfile)):
4484 fseeknextasnum = int(prefseeknextfile);
4485 if(abs(fseeknextasnum)==0):
4486 pass;
4487 catfp.seek(fseeknextasnum, 1);
4488 elif(re.findall(r"^([0-9]+)", prefseeknextfile)):
4489 fseeknextasnum = int(prefseeknextfile);
4490 if(abs(fseeknextasnum)==0):
4491 pass;
4492 catfp.seek(fseeknextasnum, 0);
4493 else:
4494 return False;
4495 il = il + 1;
4496 filefound = False;
4497 prefname = preheaderdata[2];
4498 if(re.findall(r"^[.|/]", preheaderdata[2])):
4499 prefname = preheaderdata[2];
4500 else:
4501 prefname = "./"+preheaderdata[2];
4502 if(prefname==seekfile):
4503 filefound = True;
4504 break;
4505 catfp.seek(seekstart, 0);
4506 fileidnum = il;
4507 catfheadsize = int(preheaderdata[0], 16);
4508 catfnumfields = int(preheaderdata[1], 16);
4509 catftype = int(preheaderdata[2], 16);
4510 if(re.findall(r"^[.|/]", preheaderdata[3])):
4511 catfname = preheaderdata[3];
4512 else:
4513 catfname = "./"+preheaderdata[3];
4514 catflinkname = preheaderdata[4];
4515 catfsize = int(preheaderdata[5], 16);
4516 catfbasedir = os.path.dirname(catfname);
4517 if(filefound):
4518 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
4519 else:
4520 return False;
4521 if(returnfp):
4522 catlist.update({'catfp': catfp});
4523 else:
4524 catfp.close();
4525 return catlist;
4527 create_alias_function("", __file_format_name__, "SeekToFileName", ArchiveFileSeekToFileName);
4529 def ArchiveFileValidate(infile, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
4530 formatspecs = FormatSpecsListToDict(formatspecs);
4531 if(verbose):
4532 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
4533 if(hasattr(infile, "read") or hasattr(infile, "write")):
4534 catfp = infile;
4535 catfp.seek(0, 0);
4536 catfp = UncompressArchiveFile(catfp, formatspecs);
4537 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4538 if(checkcompressfile=="tarfile"):
4539 return TarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4540 if(checkcompressfile=="zipfile"):
4541 return ZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4542 if(rarfile_support and checkcompressfile=="rarfile"):
4543 return RarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4544 if(py7zr_support and checkcompressfile=="7zipfile"):
4545 return SevenZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4546 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4547 return False;
4548 if(not catfp):
4549 return False;
4550 catfp.seek(0, 0);
4551 elif(infile=="-"):
4552 catfp = BytesIO();
4553 if(hasattr(sys.stdin, "buffer")):
4554 shutil.copyfileobj(sys.stdin.buffer, catfp);
4555 else:
4556 shutil.copyfileobj(sys.stdin, catfp);
4557 catfp.seek(0, 0);
4558 catfp = UncompressArchiveFile(catfp, formatspecs);
4559 if(not catfp):
4560 return False;
4561 catfp.seek(0, 0);
4562 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4563 catfp = download_file_from_internet_file(infile);
4564 catfp = UncompressArchiveFile(catfp, formatspecs);
4565 catfp.seek(0, 0);
4566 if(not catfp):
4567 return False;
4568 catfp.seek(0, 0);
4569 else:
4570 infile = RemoveWindowsPath(infile);
4571 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4572 if(checkcompressfile=="tarfile"):
4573 return TarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4574 if(checkcompressfile=="zipfile"):
4575 return ZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4576 if(rarfile_support and checkcompressfile=="rarfile"):
4577 return RarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4578 if(py7zr_support and checkcompressfile=="7zipfile"):
4579 return SevenZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4580 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4581 return False;
4582 compresscheck = CheckCompressionType(infile, formatspecs, True);
4583 if(not compresscheck):
4584 fextname = os.path.splitext(infile)[1];
4585 if(fextname==".gz"):
4586 compresscheck = "gzip";
4587 elif(fextname==".bz2"):
4588 compresscheck = "bzip2";
4589 elif(fextname==".zst"):
4590 compresscheck = "zstd";
4591 elif(fextname==".lz4" or fextname==".clz4"):
4592 compresscheck = "lz4";
4593 elif(fextname==".lzo" or fextname==".lzop"):
4594 compresscheck = "lzo";
4595 elif(fextname==".lzma" or fextname==".xz"):
4596 compresscheck = "lzma";
4597 else:
4598 return False;
4599 if(not compresscheck):
4600 return False;
4601 catfp = UncompressFile(infile, formatspecs, "rb");
4603 try:
4604 catfp.seek(0, 2);
4605 except OSError:
4606 SeekToEndOfFile(catfp);
4607 except ValueError:
4608 SeekToEndOfFile(catfp);
4609 CatSize = catfp.tell();
4610 CatSizeEnd = CatSize;
4612 try:
4613 catfp.seek(0, 0);
4614 except OSError:
4615 return False;
4616 except ValueError:
4617 return False;
4618 curloc = catfp.tell();
4619 if(curloc>0):
4620 catfp.seek(0, 0);
4621 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4622 if(curloc>0):
4623 catfp.seek(curloc, 0);
4624 catstring = catheader[0];
4625 catversion = re.findall(r"([\d]+)", catstring);
4626 fprenumfiles = catheader[1];
4627 fnumfiles = int(fprenumfiles, 16);
4628 fprechecksumtype = catheader[2];
4629 fprechecksum = catheader[3];
4630 il = 0;
4631 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4632 fnumfileshex = format(int(fnumfiles), 'x').lower();
4633 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4634 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4635 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4636 valid_archive = True;
4637 invalid_archive = False;
4638 if(verbose):
4639 VerbosePrintOut(infile);
4640 VerbosePrintOut("Number of Records " + str(fnumfiles));
4641 if(fprechecksum==catfileheadercshex):
4642 if(verbose):
4643 VerbosePrintOut("File Header Checksum Passed at offset " + str(0));
4644 VerbosePrintOut("'" + str(fprechecksum) + "' == " + "'" + str(catfileheadercshex) + "'");
4645 else:
4646 if(verbose):
4647 VerbosePrintOut("File Header Checksum Failed at offset " + str(0));
4648 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4649 valid_archive = False;
4650 invalid_archive = True;
4651 if(verbose):
4652 VerbosePrintOut("");
4653 while(il<fnumfiles):
4654 catfhstart = catfp.tell();
4655 if(formatspecs['new_style']):
4656 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4657 else:
4658 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4659 if(len(catheaderdata)==0):
4660 break;
4661 catfheadsize = int(catheaderdata[0], 16);
4662 catfnumfields = int(catheaderdata[1], 16);
4663 catftype = int(catheaderdata[2], 16);
4664 if(re.findall(r"^[.|/]", catheaderdata[3])):
4665 catfname = catheaderdata[3];
4666 else:
4667 catfname = "./"+catheaderdata[3];
4668 catfbasedir = os.path.dirname(catfname);
4669 catflinkname = catheaderdata[4];
4670 catfsize = int(catheaderdata[5], 16);
4671 catfatime = int(catheaderdata[6], 16);
4672 catfmtime = int(catheaderdata[7], 16);
4673 catfctime = int(catheaderdata[8], 16);
4674 catfbtime = int(catheaderdata[9], 16);
4675 catfmode = int(catheaderdata[10], 16);
4676 catfchmode = stat.S_IMODE(catfmode);
4677 catftypemod = stat.S_IFMT(catfmode);
4678 catfwinattributes = int(catheaderdata[11], 16);
4679 catfcompression = catheaderdata[12];
4680 catfcsize = int(catheaderdata[13], 16);
4681 catfuid = int(catheaderdata[14], 16);
4682 catfuname = catheaderdata[15];
4683 catfgid = int(catheaderdata[16], 16);
4684 catfgname = catheaderdata[17];
4685 fid = int(catheaderdata[18], 16);
4686 finode = int(catheaderdata[19], 16);
4687 flinkcount = int(catheaderdata[20], 16);
4688 catfdev_minor = int(catheaderdata[21], 16);
4689 catfdev_major = int(catheaderdata[22], 16);
4690 catfrdev_minor = int(catheaderdata[23], 16);
4691 catfrdev_major = int(catheaderdata[24], 16);
4692 catfseeknextfile = catheaderdata[25];
4693 catfextrasize = int(catheaderdata[26], 16);
4694 catfextrafields = int(catheaderdata[27], 16);
4695 extrafieldslist = [];
4696 extrastart = 28;
4697 extraend = extrastart + catfextrafields;
4698 extrafieldslist = [];
4699 if(extrastart<extraend):
4700 extrafieldslist.append(catheaderdata[extrastart]);
4701 extrastart = extrastart + 1;
4702 catfcs = catheaderdata[-2].lower();
4703 catfccs = catheaderdata[-1].lower();
4704 catnewfcs = GetHeaderChecksum(catheaderdata[:-2], catheaderdata[-4].lower(), True, formatspecs);
4705 if(verbose):
4706 VerbosePrintOut(catfname);
4707 VerbosePrintOut("Record Number " + str(il) + "; File ID " + str(fid) + "; iNode Number " + str(finode));
4708 if(catfcs==catnewfcs):
4709 if(verbose):
4710 VerbosePrintOut("File Header Checksum Passed at offset " + str(catfhstart));
4711 VerbosePrintOut("'" + str(catfcs) + "' == " + "'" + str(catnewfcs) + "'");
4712 else:
4713 if(verbose):
4714 VerbosePrintOut("File Header Checksum Failed at offset " + str(catfhstart));
4715 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
4716 valid_archive = False;
4717 invalid_archive = True;
4718 catfhend = catfp.tell() - 1;
4719 catfcontentstart = catfp.tell();
4720 catfcontents = "";
4721 pyhascontents = False;
4722 if(catfsize>0):
4723 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4724 catfcontents = catfp.read(catfsize);
4725 else:
4726 catfcontents = catfp.read(catfcsize);
4727 catnewfccs = GetFileChecksum(catfcontents, catheaderdata[-3].lower(), False, formatspecs);
4728 pyhascontents = True;
4729 if(catfccs==catnewfccs):
4730 if(verbose):
4731 VerbosePrintOut("File Content Checksum Passed at offset " + str(catfcontentstart));
4732 VerbosePrintOut("'" + str(catfccs) + "' == " + "'" + str(catnewfccs) + "'");
4733 else:
4734 if(verbose):
4735 VerbosePrintOut("File Content Checksum Failed at offset " + str(catfcontentstart));
4736 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
4737 valid_archive = False;
4738 invalid_archive = True;
4739 if(verbose):
4740 VerbosePrintOut("");
4741 if(re.findall(r"^\+([0-9]+)", catfseeknextfile)):
4742 fseeknextasnum = int(catfseeknextfile.replace("+", ""));
4743 if(abs(fseeknextasnum)==0):
4744 pass;
4745 catfp.seek(fseeknextasnum, 1);
4746 elif(re.findall(r"^\-([0-9]+)", catfseeknextfile)):
4747 fseeknextasnum = int(catfseeknextfile);
4748 if(abs(fseeknextasnum)==0):
4749 pass;
4750 catfp.seek(fseeknextasnum, 1);
4751 elif(re.findall(r"^([0-9]+)", catfseeknextfile)):
4752 fseeknextasnum = int(catfseeknextfile);
4753 if(abs(fseeknextasnum)==0):
4754 pass;
4755 catfp.seek(fseeknextasnum, 0);
4756 else:
4757 return False;
4758 il = il + 1;
4759 if(valid_archive):
4760 if(returnfp):
4761 return catfp;
4762 else:
4763 catfp.close();
4764 return True;
4765 else:
4766 catfp.close();
4767 return False;
4769 create_alias_function("", __file_format_name__, "Validate", ArchiveFileValidate);
4771 def ArchiveFileToArray(infile, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4772 formatspecs = FormatSpecsListToDict(formatspecs);
4773 if(hasattr(infile, "read") or hasattr(infile, "write")):
4774 catfp = infile;
4775 catfp.seek(0, 0);
4776 catfp = UncompressArchiveFile(catfp, formatspecs);
4777 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4778 if(checkcompressfile=="tarfile"):
4779 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4780 if(checkcompressfile=="zipfile"):
4781 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4782 if(rarfile_support and checkcompressfile=="rarfile"):
4783 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4784 if(py7zr_support and checkcompressfile=="7zipfile"):
4785 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4786 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4787 return False;
4788 if(not catfp):
4789 return False;
4790 catfp.seek(0, 0);
4791 elif(infile=="-"):
4792 catfp = BytesIO();
4793 if(hasattr(sys.stdin, "buffer")):
4794 shutil.copyfileobj(sys.stdin.buffer, catfp);
4795 else:
4796 shutil.copyfileobj(sys.stdin, catfp);
4797 catfp.seek(0, 0);
4798 catfp = UncompressArchiveFile(catfp, formatspecs);
4799 if(not catfp):
4800 return False;
4801 catfp.seek(0, 0);
4802 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4803 catfp = download_file_from_internet_file(infile);
4804 catfp = UncompressArchiveFile(catfp, formatspecs);
4805 catfp.seek(0, 0);
4806 if(not catfp):
4807 return False;
4808 catfp.seek(0, 0);
4809 else:
4810 infile = RemoveWindowsPath(infile);
4811 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4812 if(checkcompressfile=="tarfile"):
4813 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4814 if(checkcompressfile=="zipfile"):
4815 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4816 if(rarfile_support and checkcompressfile=="rarfile"):
4817 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4818 if(py7zr_support and checkcompressfile=="7zipfile"):
4819 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4820 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4821 return False;
4822 compresscheck = CheckCompressionType(infile, formatspecs, True);
4823 if(not compresscheck):
4824 fextname = os.path.splitext(infile)[1];
4825 if(fextname==".gz"):
4826 compresscheck = "gzip";
4827 elif(fextname==".bz2"):
4828 compresscheck = "bzip2";
4829 elif(fextname==".zst"):
4830 compresscheck = "zstd";
4831 elif(fextname==".lz4" or fextname==".clz4"):
4832 compresscheck = "lz4";
4833 elif(fextname==".lzo" or fextname==".lzop"):
4834 compresscheck = "lzo";
4835 elif(fextname==".lzma" or fextname==".xz"):
4836 compresscheck = "lzma";
4837 else:
4838 return False;
4839 if(not compresscheck):
4840 return False;
4841 catfp = UncompressFile(infile, formatspecs, "rb");
4843 try:
4844 catfp.seek(0, 2);
4845 except OSError:
4846 SeekToEndOfFile(catfp);
4847 except ValueError:
4848 SeekToEndOfFile(catfp);
4849 CatSize = catfp.tell();
4850 CatSizeEnd = CatSize;
4852 try:
4853 catfp.seek(0, 0);
4854 except OSError:
4855 return False;
4856 except ValueError:
4857 return False;
4858 curloc = catfp.tell();
4859 if(curloc>0):
4860 catfp.seek(0, 0);
4861 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4862 if(curloc>0):
4863 catfp.seek(curloc, 0);
4864 catstring = catheader[0];
4865 catversion = re.findall(r"([\d]+)", catstring);
4866 fprenumfiles = catheader[1];
4867 fnumfiles = int(fprenumfiles, 16);
4868 fprechecksumtype = catheader[2];
4869 fprechecksum = catheader[3];
4870 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4871 fnumfileshex = format(int(fnumfiles), 'x').lower();
4872 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4873 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4874 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4875 fheadtell = len(fileheader);
4876 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4877 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
4878 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4879 return False;
4880 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
4881 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': []};
4882 if(seekstart<0 and seekstart>fnumfiles):
4883 seekstart = 0;
4884 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
4885 seekend = fnumfiles;
4886 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
4887 seekend = fnumfiles - abs(seekend);
4888 if(seekstart>0):
4889 il = 0;
4890 while(il < seekstart):
4891 prefhstart = catfp.tell();
4892 if(formatspecs['new_style']):
4893 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4894 else:
4895 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4896 if(len(preheaderdata)==0):
4897 break;
4898 prefheadsize = int(preheaderdata[0], 16);
4899 prefnumfields = int(preheaderdata[1], 16);
4900 if(re.findall(r"^[.|/]", preheaderdata[3])):
4901 prefname = preheaderdata[3];
4902 else:
4903 prefname = "./"+preheaderdata[3];
4904 prefsize = int(preheaderdata[5], 16);
4905 prefcompression = preheaderdata[12];
4906 prefcsize = int(preheaderdata[13], 16);
4907 prefseeknextfile = preheaderdata[25];
4908 prefextrasize = int(preheaderdata[26], 16);
4909 prefextrafields = int(preheaderdata[27], 16);
4910 extrafieldslist = [];
4911 extrastart = 28;
4912 extraend = extrastart + prefextrafields;
4913 extrafieldslist = [];
4914 if(extrastart<extraend):
4915 extrafieldslist.append(preheaderdata[extrastart]);
4916 extrastart = extrastart + 1;
4917 prefcs = preheaderdata[-2].lower();
4918 prenewfcs = preheaderdata[-1].lower();
4919 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-4].lower(), True, formatspecs);
4920 if(prefcs!=prenewfcs and not skipchecksum):
4921 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4922 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4923 return False;
4924 valid_archive = False;
4925 invalid_archive = True;
4926 prefhend = catfp.tell() - 1;
4927 prefcontentstart = catfp.tell();
4928 prefcontents = "";
4929 pyhascontents = False;
4930 if(prefsize>0):
4931 if(prefcompression=="none" or prefcompression=="" or prefcompression=="auto"):
4932 prefcontents = catfp.read(prefsize);
4933 else:
4934 prefcontents = catfp.read(prefcsize);
4935 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4936 pyhascontents = True;
4937 if(prefccs!=prenewfccs and not skipchecksum):
4938 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4939 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4940 return False;
4941 if(re.findall(r"^\+([0-9]+)", prefseeknextfile)):
4942 fseeknextasnum = int(prefseeknextfile.replace("+", ""));
4943 if(abs(fseeknextasnum)==0):
4944 pass;
4945 catfp.seek(fseeknextasnum, 1);
4946 elif(re.findall(r"^\-([0-9]+)", prefseeknextfile)):
4947 fseeknextasnum = int(prefseeknextfile);
4948 if(abs(fseeknextasnum)==0):
4949 pass;
4950 catfp.seek(fseeknextasnum, 1);
4951 elif(re.findall(r"^([0-9]+)", prefseeknextfile)):
4952 fseeknextasnum = int(prefseeknextfile);
4953 if(abs(fseeknextasnum)==0):
4954 pass;
4955 catfp.seek(fseeknextasnum, 0);
4956 else:
4957 return False;
4958 il = il + 1;
4959 fileidnum = seekstart;
4960 realidnum = 0;
4961 while(fileidnum<seekend):
4962 catfhstart = catfp.tell();
4963 if(formatspecs['new_style']):
4964 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4965 else:
4966 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4967 if(len(catheaderdata)==0):
4968 break;
4969 catfheadsize = int(catheaderdata[0], 16);
4970 catfnumfields = int(catheaderdata[1], 16);
4971 catftype = int(catheaderdata[2], 16);
4972 if(re.findall(r"^[.|/]", catheaderdata[3])):
4973 catfname = catheaderdata[3];
4974 else:
4975 catfname = "./"+catheaderdata[3];
4976 catfbasedir = os.path.dirname(catfname);
4977 catflinkname = catheaderdata[4];
4978 catfsize = int(catheaderdata[5], 16);
4979 catfatime = int(catheaderdata[6], 16);
4980 catfmtime = int(catheaderdata[7], 16);
4981 catfctime = int(catheaderdata[8], 16);
4982 catfbtime = int(catheaderdata[9], 16);
4983 catfmode = int(catheaderdata[10], 16);
4984 catfchmode = stat.S_IMODE(catfmode);
4985 catftypemod = stat.S_IFMT(catfmode);
4986 catfwinattributes = int(catheaderdata[11], 16);
4987 catfcompression = catheaderdata[12];
4988 catfcsize = int(catheaderdata[13], 16);
4989 catfuid = int(catheaderdata[14], 16);
4990 catfuname = catheaderdata[15];
4991 catfgid = int(catheaderdata[16], 16);
4992 catfgname = catheaderdata[17];
4993 catfid = int(catheaderdata[18], 16);
4994 catfinode = int(catheaderdata[19], 16);
4995 catflinkcount = int(catheaderdata[20], 16);
4996 catfdev_minor = int(catheaderdata[21], 16);
4997 catfdev_major = int(catheaderdata[22], 16);
4998 catfrdev_minor = int(catheaderdata[23], 16);
4999 catfrdev_major = int(catheaderdata[24], 16);
5000 catfseeknextfile = catheaderdata[25];
5001 catfextrasize = int(catheaderdata[26], 16);
5002 catfextrafields = int(catheaderdata[27], 16);
5003 extrafieldslist = [];
5004 extrastart = 28;
5005 extraend = extrastart + catfextrafields;
5006 extrafieldslist = [];
5007 if(extrastart<extraend):
5008 extrafieldslist.append(catheaderdata[extrastart]);
5009 extrastart = extrastart + 1;
5010 catfcs = catheaderdata[-2].lower();
5011 catfccs = catheaderdata[-1].lower();
5012 catnewfcs = GetHeaderChecksum(catheaderdata[:-2], catheaderdata[-4].lower(), True, formatspecs);
5013 if(catfcs!=catnewfcs and not skipchecksum):
5014 VerbosePrintOut("File Header Checksum Error with file " + catfname + " at offset " + str(catfhstart));
5015 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
5016 return False;
5017 catfhend = catfp.tell() - 1;
5018 catfcontentstart = catfp.tell();
5019 catfcontents = BytesIO();
5020 pyhascontents = False;
5021 if(catfsize>0 and not listonly):
5022 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
5023 catfcontents.write(catfp.read(catfsize));
5024 else:
5025 catfcontents.write(catfp.read(catfcsize));
5026 catfcontents.seek(0, 0);
5027 catnewfccs = GetFileChecksum(catfcontents.read(), catheaderdata[-3].lower(), False, formatspecs);
5028 pyhascontents = True;
5029 if(catfccs!=catnewfccs and skipchecksum):
5030 VerbosePrintOut("File Content Checksum Error with file " + catfname + " at offset " + str(catfcontentstart));
5031 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
5032 return False;
5033 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
5034 pass;
5035 else:
5036 catfcontents.seek(0, 0);
5037 if(uncompress):
5038 catfcontents = UncompressArchiveFile(catfcontents, formatspecs);
5039 catfcontents.seek(0, 0);
5040 catfccs = GetFileChecksum(catfcontents.read(), catheaderdata[-3].lower(), False, formatspecs);
5041 if(catfsize>0 and listonly):
5042 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
5043 catfp.seek(catfsize, 1);
5044 else:
5045 catfp.seek(catfcsize, 1);
5046 pyhascontents = False;
5047 catfcontentend = catfp.tell();
5048 if(re.findall(r"^\+([0-9]+)", catfseeknextfile)):
5049 fseeknextasnum = int(catfseeknextfile.replace("+", ""));
5050 if(abs(fseeknextasnum)==0):
5051 pass;
5052 catfp.seek(fseeknextasnum, 1);
5053 elif(re.findall(r"^\-([0-9]+)", catfseeknextfile)):
5054 fseeknextasnum = int(catfseeknextfile);
5055 if(abs(fseeknextasnum)==0):
5056 pass;
5057 catfp.seek(fseeknextasnum, 1);
5058 elif(re.findall(r"^([0-9]+)", catfseeknextfile)):
5059 fseeknextasnum = int(catfseeknextfile);
5060 if(abs(fseeknextasnum)==0):
5061 pass;
5062 catfp.seek(fseeknextasnum, 0);
5063 else:
5064 return False;
5065 catfcontents.seek(0, 0);
5066 catlist['ffilelist'].append({'fid': realidnum, 'fidalt': fileidnum, 'fheadersize': catfheadsize, 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize, 'fatime': catfatime, 'fmtime': catfmtime, 'fctime': catfctime, 'fbtime': catfbtime, 'fmode': catfmode, 'fchmode': catfchmode, 'ftypemod': catftypemod, 'fwinattributes': catfwinattributes, 'fcompression': catfcompression, 'fcsize': catfcsize, 'fuid': catfuid, 'funame': catfuname, 'fgid': catfgid, 'fgname': catfgname, 'finode': catfinode, 'flinkcount': catflinkcount, 'fminor': catfdev_minor, 'fmajor': catfdev_major, 'frminor': catfrdev_minor, 'frmajor': catfrdev_major, 'fseeknextfile': catfseeknextfile, 'fheaderchecksumtype': catheaderdata[-4], 'fcontentchecksumtype': catheaderdata[-3], 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': catfextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': catfcs, 'fcontentchecksum': catfccs, 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': catfcontents});
5067 fileidnum = fileidnum + 1;
5068 realidnum = realidnum + 1;
5069 if(returnfp):
5070 catlist.update({'catfp': catfp});
5071 else:
5072 catfp.close();
5073 return catlist;
5075 create_alias_function("", __file_format_name__, "ToArray", ArchiveFileToArray);
5077 def ArchiveFileStringToArray(catstr, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5078 formatspecs = FormatSpecsListToDict(formatspecs);
5079 catfp = BytesIO(catstr);
5080 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5081 return listcatfiles;
5083 create_alias_function("", __file_format_name__, "StringToArray", ArchiveFileStringToArray);
5085 def TarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5086 formatspecs = FormatSpecsListToDict(formatspecs);
5087 catfp = BytesIO();
5088 catfp = PackArchiveFileFromTarFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5089 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5090 return listcatfiles;
5092 def ZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5093 formatspecs = FormatSpecsListToDict(formatspecs);
5094 catfp = BytesIO();
5095 catfp = PackArchiveFileFromZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5096 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5097 return listcatfiles;
5099 if(not rarfile_support):
5100 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5101 return False;
5103 if(rarfile_support):
5104 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5105 formatspecs = FormatSpecsListToDict(formatspecs);
5106 catfp = BytesIO();
5107 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5108 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5109 return listcatfiles;
5111 if(not py7zr_support):
5112 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5113 return False;
5115 if(py7zr_support):
5116 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5117 formatspecs = FormatSpecsListToDict(formatspecs);
5118 catfp = BytesIO();
5119 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
5120 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5121 return listcatfiles;
5123 def InFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5124 formatspecs = FormatSpecsListToDict(formatspecs);
5125 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
5126 if(checkcompressfile=="tarfile"):
5127 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5128 elif(checkcompressfile=="zipfile"):
5129 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5130 elif(checkcompressfile=="catfile"):
5131 return ArchiveFileToArray(infile, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
5132 elif(rarfile_support and checkcompressfile=="rarfile"):
5133 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5134 elif(py7zr_support and checkcompressfile=="7zipfile"):
5135 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5136 else:
5137 return False;
5138 return False;
5140 def ListDirToArrayAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5141 formatspecs = FormatSpecsListToDict(formatspecs);
5142 catver = formatspecs['format_ver'];
5143 fileheaderver = str(int(catver.replace(".", "")));
5144 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5145 advancedlist = formatspecs['use_advanced_list'];
5146 altinode = formatspecs['use_alt_inode'];
5147 infilelist = [];
5148 if(infiles=="-"):
5149 for line in sys.stdin:
5150 infilelist.append(line.strip());
5151 infilelist = list(filter(None, infilelist));
5152 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
5153 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
5154 return False;
5155 with UncompressFile(infiles, formatspecs, "r") as finfile:
5156 for line in finfile:
5157 infilelist.append(line.strip());
5158 infilelist = list(filter(None, infilelist));
5159 else:
5160 if(isinstance(infiles, (list, tuple, ))):
5161 infilelist = list(filter(None, infiles));
5162 elif(isinstance(infiles, (str, ))):
5163 infilelist = list(filter(None, [infiles]));
5164 if(advancedlist):
5165 GetDirList = ListDirAdvanced(infilelist, followlink, False);
5166 else:
5167 GetDirList = ListDir(infilelist, followlink, False);
5168 if(not GetDirList):
5169 return False;
5170 curinode = 0;
5171 curfid = 0;
5172 inodelist = [];
5173 inodetofile = {};
5174 filetoinode = {};
5175 inodetocatinode = {};
5176 fileidnum = 0;
5177 fnumfiles = int(len(GetDirList));
5178 catver = formatspecs['format_ver'];
5179 fileheaderver = str(int(catver.replace(".", "")));
5180 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5181 fnumfileshex = format(int(fnumfiles), 'x').lower();
5182 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5183 catversion = re.findall(r"([\d]+)", fileheader);
5184 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5185 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5186 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5187 fheadtell = len(fileheader);
5188 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
5189 for curfname in GetDirList:
5190 catfhstart = fheadtell;
5191 if(re.findall(r"^[.|/]", curfname)):
5192 fname = curfname;
5193 else:
5194 fname = "./"+curfname;
5195 if(verbose):
5196 VerbosePrintOut(fname);
5197 if(not followlink or followlink is None):
5198 fstatinfo = os.lstat(fname);
5199 else:
5200 fstatinfo = os.stat(fname);
5201 fpremode = fstatinfo.st_mode;
5202 finode = fstatinfo.st_ino;
5203 flinkcount = fstatinfo.st_nlink;
5204 ftype = 0;
5205 if(stat.S_ISREG(fpremode)):
5206 ftype = 0;
5207 elif(stat.S_ISLNK(fpremode)):
5208 ftype = 2;
5209 elif(stat.S_ISCHR(fpremode)):
5210 ftype = 3;
5211 elif(stat.S_ISBLK(fpremode)):
5212 ftype = 4;
5213 elif(stat.S_ISDIR(fpremode)):
5214 ftype = 5;
5215 elif(stat.S_ISFIFO(fpremode)):
5216 ftype = 6;
5217 elif(stat.S_ISSOCK(fpremode)):
5218 ftype = 8;
5219 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
5220 ftype = 9;
5221 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
5222 ftype = 10;
5223 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
5224 ftype = 11;
5225 else:
5226 ftype = 0;
5227 flinkname = "";
5228 fbasedir = os.path.dirname(fname);
5229 fcurfid = curfid;
5230 if(not followlink and finode!=0):
5231 if(ftype!=1):
5232 if(finode in inodelist):
5233 ftype = 1;
5234 flinkname = inodetofile[finode];
5235 if(altinode):
5236 fcurinode = finode;
5237 else:
5238 fcurinode = inodetocatinode[finode];
5239 if(finode not in inodelist):
5240 inodelist.append(finode);
5241 inodetofile.update({finode: fname});
5242 inodetocatinode.update({finode: curinode});
5243 if(altinode):
5244 fcurinode = finode;
5245 else:
5246 fcurinode = curinode;
5247 curinode = curinode + 1;
5248 else:
5249 fcurinode = curinode;
5250 curinode = curinode + 1;
5251 curfid = curfid + 1;
5252 if(ftype==2):
5253 flinkname = os.readlink(fname);
5254 fdev = fstatinfo.st_dev;
5255 getfdev = GetDevMajorMinor(fdev);
5256 fdev_minor = getfdev[0];
5257 fdev_major = getfdev[1];
5258 frdev = fstatinfo.st_dev;
5259 if(hasattr(fstatinfo, "st_rdev")):
5260 frdev = fstatinfo.st_rdev;
5261 else:
5262 frdev = fstatinfo.st_dev;
5263 getfrdev = GetDevMajorMinor(frdev);
5264 frdev_minor = getfrdev[0];
5265 frdev_major = getfrdev[1];
5266 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
5267 fsize = "0";
5268 if(ftype==0 or ftype==7):
5269 fsize = fstatinfo.st_size;
5270 fatime = fstatinfo.st_atime;
5271 fmtime = fstatinfo.st_mtime;
5272 fctime = fstatinfo.st_ctime;
5273 if(hasattr(fstatinfo, "st_birthtime")):
5274 fbtime = fstatinfo.st_birthtime;
5275 else:
5276 fbtime = fstatinfo.st_ctime;
5277 fmode = fstatinfo.st_mode;
5278 fchmode = stat.S_IMODE(fstatinfo.st_mode);
5279 ftypemod = stat.S_IFMT(fstatinfo.st_mode);
5280 fuid = fstatinfo.st_uid;
5281 fgid = fstatinfo.st_gid;
5282 funame = "";
5283 try:
5284 import pwd;
5285 try:
5286 userinfo = pwd.getpwuid(fstatinfo.st_uid);
5287 funame = userinfo.pw_name;
5288 except KeyError:
5289 funame = "";
5290 except ImportError:
5291 funame = "";
5292 fgname = "";
5293 try:
5294 import grp;
5295 try:
5296 groupinfo = grp.getgrgid(fstatinfo.st_gid);
5297 fgname = groupinfo.gr_name;
5298 except KeyError:
5299 fgname = "";
5300 except ImportError:
5301 fgname = "";
5302 fdev_minor = fdev_minor;
5303 fdev_major = fdev_major;
5304 frdev_minor = frdev_minor;
5305 frdev_major = frdev_major;
5306 flinkcount = flinkcount;
5307 if(hasattr(fstatinfo, "st_file_attributes")):
5308 fwinattributes = fstatinfo.st_file_attributes;
5309 else:
5310 fwinattributes = 0;
5311 fcompression = "";
5312 fcsize = 0;
5313 fcontents = BytesIO();
5314 if(ftype==0 or ftype==7):
5315 with open(fname, "rb") as fpc:
5316 shutil.copyfileobj(fpc, fcontents);
5317 if(followlink and (ftype==1 or ftype==2)):
5318 flstatinfo = os.stat(flinkname);
5319 with open(flinkname, "rb") as fpc:
5320 shutil.copyfileobj(fpc, fcontents);
5321 fcontents.seek(0, 0);
5322 ftypehex = format(ftype, 'x').lower();
5323 extrafields = len(extradata);
5324 extrafieldslist = extradata;
5325 catfextrafields = extrafields;
5326 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5327 if(len(extradata)>0):
5328 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5329 extrasizelen = len(extrasizestr);
5330 extrasizelenhex = format(extrasizelen, 'x').lower();
5331 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), "+1", extrasizelenhex, format(catfextrafields, 'x').lower()];
5332 catoutlen = len(catoutlist) + len(extradata) + 3;
5333 catoutlenhex = format(catoutlen, 'x').lower();
5334 catoutlist.insert(0, catoutlenhex);
5335 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5336 catheaderdata = catoutlist;
5337 if(len(extradata)>0):
5338 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5339 catfileoutstr = catfileoutstr + AppendNullBytes([checksumtype, checksumtype], formatspecs['format_delimiter']);
5340 catfnumfields = catoutlen;
5341 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5342 fcontents.seek(0, 0);
5343 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5344 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5345 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5346 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5347 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5348 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5349 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5350 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5351 catfcontentstart = fheadtell;
5352 fheadtell += len(catfileoutstr) + 1;
5353 catfcontentend = fheadtell - 1;
5354 catfhend = catfcontentend;
5355 fcontents.seek(0, 0);
5356 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5357 pyhascontents = False;
5358 if(int(fsize)>0 and not listonly):
5359 pyhascontents = True;
5360 if(int(fsize)>0 and listonly):
5361 fcontents = BytesIO();
5362 pyhascontents = False;
5363 fcontents.seek(0, 0);
5364 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype, 'fcontentchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents});
5365 fileidnum = fileidnum + 1;
5366 return catlist;
5368 def TarFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5369 formatspecs = FormatSpecsListToDict(formatspecs);
5370 curinode = 0;
5371 curfid = 0;
5372 inodelist = [];
5373 inodetofile = {};
5374 filetoinode = {};
5375 inodetocatinode = {};
5376 fileidnum = 0;
5377 if(infile=="-"):
5378 infile = BytesIO();
5379 if(hasattr(sys.stdin, "buffer")):
5380 shutil.copyfileobj(sys.stdin.buffer, infile);
5381 else:
5382 shutil.copyfileobj(sys.stdin, infile);
5383 infile.seek(0, 0);
5384 if(not infile):
5385 return False;
5386 infile.seek(0, 0);
5387 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
5388 infile = download_file_from_internet_file(infile);
5389 infile.seek(0, 0);
5390 if(not infile):
5391 return False;
5392 infile.seek(0, 0);
5393 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
5394 return False;
5395 elif(os.path.exists(infile) and os.path.isfile(infile)):
5396 try:
5397 if(not tarfile.TarFileCheck(infile)):
5398 return False;
5399 except AttributeError:
5400 if(not TarFileCheck(infile)):
5401 return False;
5402 else:
5403 return False;
5404 try:
5405 if(hasattr(infile, "read") or hasattr(infile, "write")):
5406 tarfp = tarfile.open(fileobj=infile, mode="r");
5407 else:
5408 tarfp = tarfile.open(infile, "r");
5409 except FileNotFoundError:
5410 return False;
5411 fnumfiles = int(len(tarfp.getmembers()));
5412 catver = formatspecs['format_ver'];
5413 fileheaderver = str(int(catver.replace(".", "")));
5414 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5415 fnumfileshex = format(int(fnumfiles), 'x').lower();
5416 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5417 catversion = re.findall(r"([\d]+)", fileheader);
5418 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5419 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5420 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5421 fheadtell = len(fileheader);
5422 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
5423 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
5424 catfhstart = fheadtell;
5425 if(re.findall(r"^[.|/]", member.name)):
5426 fname = member.name;
5427 else:
5428 fname = "./"+member.name;
5429 if(verbose):
5430 VerbosePrintOut(fname);
5431 fpremode = member.mode;
5432 ffullmode = member.mode;
5433 flinkcount = 0;
5434 ftype = 0;
5435 if(member.isreg()):
5436 ffullmode = member.mode + stat.S_IFREG;
5437 ftype = 0;
5438 elif(member.isdev()):
5439 ffullmode = member.mode;
5440 ftype = 7;
5441 elif(member.islnk()):
5442 ffullmode = member.mode + stat.S_IFREG;
5443 ftype = 1;
5444 elif(member.issym()):
5445 ffullmode = member.mode + stat.S_IFLNK;
5446 ftype = 2;
5447 elif(member.ischr()):
5448 ffullmode = member.mode + stat.S_IFCHR;
5449 ftype = 3;
5450 elif(member.isblk()):
5451 ffullmode = member.mode + stat.S_IFBLK;
5452 ftype = 4;
5453 elif(member.isdir()):
5454 ffullmode = member.mode + stat.S_IFDIR;
5455 ftype = 5;
5456 elif(member.isfifo()):
5457 ffullmode = member.mode + stat.S_IFIFO;
5458 ftype = 6;
5459 elif(member.issparse()):
5460 ffullmode = member.mode;
5461 ftype = 12;
5462 else:
5463 ffullmode = member.mode;
5464 ftype = 0;
5465 flinkname = "";
5466 fbasedir = os.path.dirname(fname);
5467 fcurfid = curfid;
5468 fcurinode = curfid;
5469 finode = fcurinode;
5470 curfid = curfid + 1;
5471 if(ftype==2):
5472 flinkname = member.linkname;
5473 fdev_minor = member.devminor;
5474 fdev_major = member.devmajor;
5475 frdev_minor = member.devminor;
5476 frdev_major = member.devmajor;
5477 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
5478 fsize = "0";
5479 elif(ftype==0 or ftype==7):
5480 fsize = member.size;
5481 else:
5482 fsize = member.size;
5483 fatime = member.mtime;
5484 fmtime = member.mtime;
5485 fctime = member.mtime;
5486 fbtime = member.mtime;
5487 fmode = ffullmode;
5488 fchmode = stat.S_IMODE(ffullmode);
5489 ftypemod = stat.S_IFMT(ffullmode);
5490 fuid = member.uid;
5491 fgid = member.gid;
5492 funame = member.uname;
5493 fgname = member.gname;
5494 flinkcount = flinkcount;
5495 fwinattributes = int(0);
5496 fcompression = "";
5497 fcsize = 0;
5498 fcontents = BytesIO();
5499 if(ftype==0 or ftype==7):
5500 with tarfp.extractfile(member) as fpc:
5501 shutil.copyfileobj(fpc, fcontents);
5502 fcontents.seek(0, 0);
5503 ftypehex = format(ftype, 'x').lower();
5504 extrafields = len(extradata);
5505 extrafieldslist = extradata;
5506 catfextrafields = extrafields;
5507 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5508 if(len(extradata)>0):
5509 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5510 extrasizelen = len(extrasizestr);
5511 extrasizelenhex = format(extrasizelen, 'x').lower();
5512 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), "+1", extrasizelenhex, format(catfextrafields, 'x').lower()];
5513 catoutlen = len(catoutlist) + len(extradata) + 3;
5514 catoutlenhex = format(catoutlen, 'x').lower();
5515 catoutlist.insert(0, catoutlenhex);
5516 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5517 catheaderdata = catoutlist;
5518 if(len(extradata)>0):
5519 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5520 catfileoutstr = catfileoutstr + AppendNullBytes([checksumtype, checksumtype], formatspecs['format_delimiter']);
5521 catfnumfields = catoutlen;
5522 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5523 fcontents.seek(0, 0);
5524 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5525 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5526 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5527 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5528 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5529 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5530 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5531 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5532 catfcontentstart = fheadtell;
5533 fheadtell += len(catfileoutstr) + 1;
5534 catfcontentend = fheadtell - 1;
5535 catfhend = catfcontentend;
5536 fcontents.seek(0, 0);
5537 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5538 pyhascontents = False;
5539 if(int(fsize)>0 and not listonly):
5540 pyhascontents = True;
5541 if(int(fsize)>0 and listonly):
5542 fcontents = BytesIO();
5543 pyhascontents = False;
5544 fcontents.seek(0, 0);
5545 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype, 'fcontentchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents});
5546 fileidnum = fileidnum + 1;
5547 return catlist;
5549 def ZipFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5550 formatspecs = FormatSpecsListToDict(formatspecs);
5551 curinode = 0;
5552 curfid = 0;
5553 inodelist = [];
5554 inodetofile = {};
5555 filetoinode = {};
5556 inodetocatinode = {};
5557 fileidnum = 0;
5558 if(infile=="-"):
5559 infile = BytesIO();
5560 if(hasattr(sys.stdin, "buffer")):
5561 shutil.copyfileobj(sys.stdin.buffer, infile);
5562 else:
5563 shutil.copyfileobj(sys.stdin, infile);
5564 infile.seek(0, 0);
5565 if(not infile):
5566 return False;
5567 infile.seek(0, 0);
5568 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
5569 infile = download_file_from_internet_file(infile);
5570 infile.seek(0, 0);
5571 if(not infile):
5572 return False;
5573 infile.seek(0, 0);
5574 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
5575 return False;
5576 else:
5577 return False;
5578 if(not zipfile.is_zipfile(infile)):
5579 return False;
5580 try:
5581 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
5582 except FileNotFoundError:
5583 return False;
5584 ziptest = zipfp.testzip();
5585 if(ziptest):
5586 VerbosePrintOut("Bad file found!");
5587 fnumfiles = int(len(zipfp.infolist()));
5588 catver = formatspecs['format_ver'];
5589 fileheaderver = str(int(catver.replace(".", "")));
5590 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5591 catversion = re.findall(r"([\d]+)", fileheader);
5592 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5593 fnumfileshex = format(int(fnumfiles), 'x').lower();
5594 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5595 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5596 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5597 fheadtell = len(fileheader);
5598 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
5599 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
5600 catfhstart = fheadtell;
5601 if(re.findall(r"^[.|/]", member.filename)):
5602 fname = member.filename;
5603 else:
5604 fname = "./"+member.filename;
5605 zipinfo = zipfp.getinfo(member.filename);
5606 if(verbose):
5607 VerbosePrintOut(fname);
5608 if(not member.is_dir()):
5609 fpremode = stat.S_IFREG + 438;
5610 elif(member.is_dir()):
5611 fpremode = stat.S_IFDIR + 511;
5612 flinkcount = 0;
5613 ftype = 0;
5614 if(not member.is_dir()):
5615 ftype = 0;
5616 elif(member.is_dir()):
5617 ftype = 5;
5618 flinkname = "";
5619 fbasedir = os.path.dirname(fname);
5620 fcurfid = curfid;
5621 fcurinode = curfid;
5622 finode = fcurinode;
5623 curfid = curfid + 1;
5624 fdev_minor = 0;
5625 fdev_major = 0;
5626 frdev_minor = 0;
5627 frdev_major = 0;
5628 if(ftype==5):
5629 fsize = "0";
5630 elif(ftype==0):
5631 fsize = member.file_size;
5632 else:
5633 fsize = member.file_size;
5634 fatime = time.mktime(member.date_time + (0, 0, -1));
5635 fmtime = time.mktime(member.date_time + (0, 0, -1));
5636 fctime = time.mktime(member.date_time + (0, 0, -1));
5637 fbtime = time.mktime(member.date_time + (0, 0, -1));
5638 if(zipinfo.create_system==0 or zipinfo.create_system==10):
5639 fwinattributes = int(zipinfo.external_attr);
5640 if(not member.is_dir()):
5641 fmode = int(stat.S_IFREG + 438);
5642 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
5643 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
5644 elif(member.is_dir()):
5645 fmode = int(stat.S_IFDIR + 511);
5646 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
5647 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
5648 elif(zipinfo.create_system==3):
5649 fwinattributes = int(0);
5650 fmode = int(zipinfo.external_attr);
5651 else:
5652 fwinattributes = int(0);
5653 if(not member.is_dir()):
5654 fmode = int(stat.S_IFREG + 438);
5655 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
5656 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
5657 elif(member.is_dir()):
5658 fmode = int(stat.S_IFDIR + 511);
5659 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
5660 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
5661 fcompression = "";
5662 fcsize = 0;
5663 try:
5664 fuid = os.getuid();
5665 except AttributeError:
5666 fuid = 0;
5667 except KeyError:
5668 fuid = 0;
5669 try:
5670 fgid = os.getgid();
5671 except AttributeError:
5672 fgid = 0;
5673 except KeyError:
5674 fgid = 0;
5675 try:
5676 import pwd;
5677 try:
5678 userinfo = pwd.getpwuid(os.getuid());
5679 funame = userinfo.pw_name;
5680 except KeyError:
5681 funame = "";
5682 except AttributeError:
5683 funame = "";
5684 except ImportError:
5685 funame = "";
5686 fgname = "";
5687 try:
5688 import grp;
5689 try:
5690 groupinfo = grp.getgrgid(os.getgid());
5691 fgname = groupinfo.gr_name;
5692 except KeyError:
5693 fgname = "";
5694 except AttributeError:
5695 fgname = "";
5696 except ImportError:
5697 fgname = "";
5698 fcontents = BytesIO();
5699 if(ftype==0):
5700 fcontents.write(zipfp.read(member.filename));
5701 fcontents.seek(0, 0);
5702 ftypehex = format(ftype, 'x').lower();
5703 extrafields = len(extradata);
5704 extrafieldslist = extradata;
5705 catfextrafields = extrafields;
5706 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5707 if(len(extradata)>0):
5708 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5709 extrasizelen = len(extrasizestr);
5710 extrasizelenhex = format(extrasizelen, 'x').lower();
5711 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), "+1", extrasizelenhex, format(catfextrafields, 'x').lower()];
5712 catoutlen = len(catoutlist) + len(extradata) + 3;
5713 catoutlenhex = format(catoutlen, 'x').lower();
5714 catoutlist.insert(0, catoutlenhex);
5715 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5716 catheaderdata = catoutlist;
5717 if(len(extradata)>0):
5718 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5719 catfileoutstr = catfileoutstr + AppendNullBytes([checksumtype, checksumtype], formatspecs['format_delimiter']);
5720 catfnumfields = catoutlen;
5721 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5722 fcontents.seek(0, 0);
5723 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5724 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5725 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5726 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5727 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5728 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5729 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5730 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5731 catfcontentstart = fheadtell;
5732 fheadtell += len(catfileoutstr) + 1;
5733 catfcontentend = fheadtell - 1;
5734 catfhend = catfcontentend;
5735 fcontents.seek(0, 0);
5736 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5737 pyhascontents = False;
5738 if(int(fsize)>0 and not listonly):
5739 pyhascontents = True;
5740 if(int(fsize)>0 and listonly):
5741 fcontents = BytesIO();
5742 pyhascontents = False;
5743 fcontents.seek(0, 0);
5744 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype, 'fcontentchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents});
5745 fileidnum = fileidnum + 1;
5746 return catlist;
5748 if(not rarfile_support):
5749 def RarFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5750 return False;
5752 if(rarfile_support):
5753 def RarFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5754 formatspecs = FormatSpecsListToDict(formatspecs);
5755 curinode = 0;
5756 curfid = 0;
5757 inodelist = [];
5758 inodetofile = {};
5759 filetoinode = {};
5760 inodetocatinode = {};
5761 fileidnum = 0;
5762 if(not os.path.exists(infile,) or not os.path.isfile(infile,)):
5763 return False;
5764 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
5765 return False;
5766 rarfp = rarfile.RarFile(infile, "r");
5767 rartest = rarfp.testrar();
5768 if(rartest):
5769 VerbosePrintOut("Bad file found!");
5770 fnumfiles = int(len(rarfp.infolist()));
5771 catver = formatspecs['format_ver'];
5772 fileheaderver = str(int(catver.replace(".", "")));
5773 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5774 catversion = re.findall(r"([\d]+)", fileheader);
5775 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5776 fnumfileshex = format(int(fnumfiles), 'x').lower();
5777 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5778 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5779 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5780 fheadtell = len(fileheader);
5781 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
5782 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
5783 catfhstart = fheadtell;
5784 is_unix = False;
5785 is_windows = False;
5786 if(member.host_os==rarfile.RAR_OS_UNIX):
5787 is_windows = False;
5788 try:
5789 member.external_attr
5790 is_unix = True;
5791 except AttributeError:
5792 is_unix = False;
5793 elif(member.host_os==rarfile.RAR_OS_WIN32):
5794 is_unix = False;
5795 try:
5796 member.external_attr
5797 is_windows = True;
5798 except AttributeError:
5799 is_windows = False;
5800 else:
5801 is_unix = False;
5802 is_windows = False;
5803 if(re.findall(r"^[.|/]", member.filename)):
5804 fname = member.filename;
5805 else:
5806 fname = "./"+member.filename;
5807 rarinfo = rarfp.getinfo(member.filename);
5808 if(verbose):
5809 VerbosePrintOut(fname);
5810 if(is_unix and member.external_attr !=0):
5811 fpremode = int(member.external_attr);
5812 elif(member.is_file()):
5813 fpremode = stat.S_IFREG + 438;
5814 elif(member.is_symlink()):
5815 fpremode = stat.S_IFLNK + 438;
5816 elif(member.is_dir()):
5817 fpremode = stat.S_IFDIR + 511;
5818 if(is_windows and member.external_attr !=0):
5819 fwinattributes = int(member.external_attr);
5820 else:
5821 fwinattributes = int(0);
5822 fcompression = "";
5823 fcsize = 0;
5824 flinkcount = 0;
5825 ftype = 0;
5826 if(member.is_file()):
5827 ftype = 0;
5828 elif(member.is_symlink()):
5829 ftype = 2;
5830 elif(member.is_dir()):
5831 ftype = 5;
5832 flinkname = "";
5833 if(ftype==2):
5834 flinkname = rarfp.read(member.filename).decode("UTF-8");
5835 fbasedir = os.path.dirname(fname);
5836 fcurfid = curfid;
5837 fcurinode = curfid;
5838 finode = fcurinode;
5839 curfid = curfid + 1;
5840 fdev_minor = 0;
5841 fdev_major = 0;
5842 frdev_minor = 0;
5843 frdev_major = 0;
5844 if(ftype==5):
5845 fsize = "0";
5846 if(ftype==0):
5847 fsize = member.file_size;
5848 try:
5849 if(member.atime):
5850 fatime = int(member.atime.timestamp());
5851 else:
5852 fatime = int(member.mtime.timestamp());
5853 except AttributeError:
5854 fatime = int(member.mtime.timestamp());
5855 fmtime = int(member.mtime.timestamp());
5856 try:
5857 if(member.ctime):
5858 fctime = int(member.ctime.timestamp());
5859 else:
5860 fctime = int(member.mtime.timestamp());
5861 except AttributeError:
5862 fctime = int(member.mtime.timestamp());
5863 fbtime = int(member.mtime.timestamp());
5864 if(is_unix and member.external_attr !=0):
5865 fmode = int(member.external_attr);
5866 fchmode = int(stat.S_IMODE(member.external_attr));
5867 ftypemod = int(stat.S_IFMT(member.external_attr));
5868 elif(member.is_file()):
5869 fmode = int(stat.S_IFREG + 438)
5870 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5871 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5872 elif(member.is_symlink()):
5873 fmode = int(stat.S_IFLNK + 438)
5874 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5875 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5876 elif(member.is_dir()):
5877 fmode = int(stat.S_IFDIR + 511)
5878 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
5879 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
5880 try:
5881 fuid = os.getuid();
5882 except AttributeError:
5883 fuid = 0;
5884 except KeyError:
5885 fuid = 0;
5886 try:
5887 fgid = os.getgid();
5888 except AttributeError:
5889 fgid = 0;
5890 except KeyError:
5891 fgid = 0;
5892 try:
5893 import pwd;
5894 try:
5895 userinfo = pwd.getpwuid(os.getuid());
5896 funame = userinfo.pw_name;
5897 except KeyError:
5898 funame = "";
5899 except AttributeError:
5900 funame = "";
5901 except ImportError:
5902 funame = "";
5903 fgname = "";
5904 try:
5905 import grp;
5906 try:
5907 groupinfo = grp.getgrgid(os.getgid());
5908 fgname = groupinfo.gr_name;
5909 except KeyError:
5910 fgname = "";
5911 except AttributeError:
5912 fgname = "";
5913 except ImportError:
5914 fgname = "";
5915 fcontents = BytesIO();
5916 if(ftype==0):
5917 fcontents.write(rarfp.read(member.filename));
5918 fcontents.seek(0, 0);
5919 ftypehex = format(ftype, 'x').lower();
5920 extrafields = len(extradata);
5921 extrafieldslist = extradata;
5922 catfextrafields = extrafields;
5923 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5924 if(len(extradata)>0):
5925 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5926 extrasizelen = len(extrasizestr);
5927 extrasizelenhex = format(extrasizelen, 'x').lower();
5928 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), "+1", extrasizelenhex, format(catfextrafields, 'x').lower()];
5929 catoutlen = len(catoutlist) + len(extradata) + 3;
5930 catoutlenhex = format(catoutlen, 'x').lower();
5931 catoutlist.insert(0, catoutlenhex);
5932 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5933 if(len(extradata)>0):
5934 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5935 ccatfileoutstr = catfileoutstr + AppendNullBytes([checksumtype, checksumtype], formatspecs['format_delimiter']);
5936 catfnumfields = 24 + catfextrafields;
5937 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5938 fcontents.seek(0, 0);
5939 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5940 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5941 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5942 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5943 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5944 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5945 catheaderdata = catoutlist;
5946 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5947 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5948 catfcontentstart = fheadtell;
5949 fheadtell += len(catfileoutstr) + 1;
5950 catfcontentend = fheadtell - 1;
5951 catfhend = catfcontentend;
5952 fcontents.seek(0, 0);
5953 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5954 pyhascontents = False;
5955 if(int(fsize)>0 and not listonly):
5956 pyhascontents = True;
5957 if(int(fsize)>0 and listonly):
5958 fcontents = BytesIO();
5959 pyhascontents = False;
5960 fcontents.seek(0, 0);
5961 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype, 'fcontentchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents});
5962 fileidnum = fileidnum + 1;
5963 return catlist;
5965 if(not py7zr_support):
5966 def SevenZipFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5967 return False;
5969 if(py7zr_support):
5970 def SevenZipFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5971 formatspecs = FormatSpecsListToDict(formatspecs);
5972 curinode = 0;
5973 curfid = 0;
5974 inodelist = [];
5975 inodetofile = {};
5976 filetoinode = {};
5977 inodetocatinode = {};
5978 fileidnum = 0;
5979 szpfp = py7zr.SevenZipFile(infile, mode="r");
5980 file_content = szpfp.readall();
5981 #sztest = szpfp.testzip();
5982 sztestalt = szpfp.test();
5983 if(sztestalt):
5984 VerbosePrintOut("Bad file found!");
5985 numfiles = int(len(szpfp.list()));
5986 catver = formatspecs['format_ver'];
5987 fileheaderver = str(int(catver.replace(".", "")));
5988 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5989 catversion = re.findall(r"([\d]+)", fileheader);
5990 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5991 fnumfileshex = format(int(fnumfiles), 'x').lower();
5992 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5993 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5994 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5995 fheadtell = len(fileheader);
5996 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
5997 for member in sorted(szpfp.list(), key=lambda x: x.filename):
5998 catfhstart = fheadtell;
5999 if(re.findall(r"^[.|/]", member.filename)):
6000 fname = member.filename;
6001 else:
6002 fname = "./"+member.filename;
6003 if(not member.is_directory):
6004 fpremode = int(stat.S_IFREG + 438);
6005 elif(member.is_directory):
6006 fpremode = int(stat.S_IFDIR + 511);
6007 fwinattributes = int(0);
6008 fcompression = "";
6009 fcsize = 0;
6010 flinkcount = 0;
6011 ftype = 0;
6012 if(member.is_directory):
6013 ftype = 5;
6014 else:
6015 ftype = 0;
6016 flinkname = "";
6017 fbasedir = os.path.dirname(fname);
6018 fcurfid = curfid;
6019 fcurinode = curfid;
6020 finode = fcurinode;
6021 curfid = curfid + 1;
6022 fdev_minor = 0;
6023 fdev_major = 0;
6024 frdev_minor = 0;
6025 frdev_major = 0;
6026 if(ftype==5):
6027 fsize = "0";
6028 fatime = int(member.creationtime.timestamp());
6029 fmtime = int(member.creationtime.timestamp());
6030 fctime = int(member.creationtime.timestamp());
6031 fbtime = int(member.creationtime.timestamp());
6032 if(member.is_directory):
6033 fmode = int(stat.S_IFDIR + 511)
6034 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
6035 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
6036 else:
6037 fmode = int(stat.S_IFLNK + 438)
6038 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
6039 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
6040 try:
6041 fuid = os.getuid();
6042 except AttributeError:
6043 fuid = 0;
6044 except KeyError:
6045 fuid = 0;
6046 try:
6047 fgid = os.getgid();
6048 except AttributeError:
6049 fgid = 0;
6050 except KeyError:
6051 fgid = 0;
6052 try:
6053 import pwd;
6054 try:
6055 userinfo = pwd.getpwuid(os.getuid());
6056 funame = userinfo.pw_name;
6057 except KeyError:
6058 funame = "";
6059 except AttributeError:
6060 funame = "";
6061 except ImportError:
6062 funame = "";
6063 fgname = "";
6064 try:
6065 import grp;
6066 try:
6067 groupinfo = grp.getgrgid(os.getgid());
6068 fgname = groupinfo.gr_name;
6069 except KeyError:
6070 fgname = "";
6071 except AttributeError:
6072 fgname = "";
6073 except ImportError:
6074 fgname = "";
6075 fcontents = BytesIO();
6076 if(ftype==0):
6077 fcontents.write(file_content[member.filename].read());
6078 fsize = format(fcontents.tell(), 'x').lower();
6079 fileop.close();
6080 fcontents.seek(0, 0);
6081 ftypehex = format(ftype, 'x').lower();
6082 extrafields = len(extradata);
6083 extrafieldslist = extradata;
6084 catfextrafields = extrafields;
6085 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
6086 if(len(extradata)>0):
6087 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
6088 extrasizelen = len(extrasizestr);
6089 extrasizelenhex = format(extrasizelen, 'x').lower();
6090 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), "+1", extrasizelenhex, format(catfextrafields, 'x').lower()];
6091 catoutlen = len(catoutlist) + len(extradata) + 3;
6092 catoutlenhex = format(catoutlen, 'x').lower();
6093 catoutlist.insert(0, catoutlenhex);
6094 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
6095 catheaderdata = catoutlist;
6096 if(len(extradata)>0):
6097 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
6098 catfileoutstr = catfileoutstr + AppendNullBytes([checksumtype, checksumtype], formatspecs['format_delimiter']);
6099 catfnumfields = 24 + catfextrafields;
6100 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
6101 fcontents.seek(0, 0);
6102 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
6103 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
6104 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
6105 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
6106 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
6107 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
6108 catfileoutstrecd = catfileoutstr.encode('UTF-8');
6109 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
6110 catfcontentstart = fheadtell;
6111 fheadtell += len(catfileoutstr) + 1;
6112 catfcontentend = fheadtell - 1;
6113 catfhend = catfcontentend;
6114 fcontents.seek(0, 0);
6115 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
6116 pyhascontents = False;
6117 if(int(fsize)>0 and not listonly):
6118 pyhascontents = True;
6119 if(int(fsize)>0 and listonly):
6120 fcontents = BytesIO();
6121 pyhascontents = False;
6122 fcontents.seek(0, 0);
6123 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype, 'fcontentchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents});
6124 fileidnum = fileidnum + 1;
6125 return catlist;
6127 def InFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6128 formatspecs = FormatSpecsListToDict(formatspecs);
6129 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
6130 if(checkcompressfile=="tarfile"):
6131 return TarFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6132 elif(checkcompressfile=="zipfile"):
6133 return ZipFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6134 elif(checkcompressfile=="catfile"):
6135 return ArchiveFileToArray(infile, 0, 0, listonly, True, False, formatspecs, False);
6136 elif(rarfile_support and checkcompressfile=="rarfile"):
6137 return RarFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6138 elif(py7zr_support and checkcompressfile=="7zipfile"):
6139 return SevenZipFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6140 else:
6141 return False;
6142 return False;
6144 def ListDirToArray(infiles, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, listonly=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6145 formatspecs = FormatSpecsListToDict(formatspecs);
6146 outarray = BytesIO();
6147 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
6148 listcatfiles = ArchiveFileToArray(outarray, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
6149 return listcatfiles;
6151 def ArchiveFileArrayToArrayIndex(inarray, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6152 formatspecs = FormatSpecsListToDict(formatspecs);
6153 if(isinstance(inarray, dict)):
6154 listcatfiles = inarray;
6155 else:
6156 return False;
6157 if(not listcatfiles):
6158 return False;
6159 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6160 if(returnfp):
6161 catarray.update({'catfp': listcatfiles['catfp']});
6162 lenlist = len(listcatfiles['ffilelist']);
6163 lcfi = 0;
6164 lcfx = int(listcatfiles['fnumfiles']);
6165 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6166 lcfx = int(lenlist);
6167 else:
6168 lcfx = int(listcatfiles['fnumfiles']);
6169 while(lcfi < lcfx):
6170 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6171 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6172 catarray['filetoid'].update(filetoidarray);
6173 catarray['idtofile'].update(idtofilearray);
6174 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6175 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6176 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6177 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6178 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6179 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6180 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6181 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6182 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6183 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6184 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6185 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6186 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6187 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6188 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6189 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6190 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6191 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6192 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6193 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6194 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6195 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6196 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6197 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6198 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6199 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6200 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6201 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6202 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6203 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6204 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6205 lcfi = lcfi + 1;
6206 return catarray;
6208 create_alias_function("", __file_format_name__, "ArrayToArrayIndex", ArchiveFileArrayToArrayIndex);
6210 def RePackArchiveFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6211 formatspecs = FormatSpecsListToDict(formatspecs);
6212 if(isinstance(infile, dict)):
6213 listcatfiles = infile;
6214 else:
6215 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6216 infile = RemoveWindowsPath(infile);
6217 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, True, skipchecksum, formatspecs, returnfp);
6218 if(outfile!="-" and not hasattr(infile, "read") and not hasattr(outfile, "write")):
6219 outfile = RemoveWindowsPath(outfile);
6220 checksumtype = checksumtype.lower();
6221 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
6222 checksumtype="crc32";
6223 if(checksumtype=="none"):
6224 checksumtype = "";
6225 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
6226 compression = "auto";
6227 if(compression not in compressionlist and compression is None):
6228 compression = "auto";
6229 if(verbose):
6230 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6231 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
6232 if(os.path.exists(outfile)):
6233 try:
6234 os.unlink(outfile);
6235 except OSError as e:
6236 pass;
6237 if(not listcatfiles):
6238 return False;
6239 if(outfile=="-"):
6240 verbose = False;
6241 catfp = BytesIO();
6242 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
6243 catfp = outfile;
6244 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
6245 catfp = BytesIO();
6246 else:
6247 fbasename = os.path.splitext(outfile)[0];
6248 fextname = os.path.splitext(outfile)[1];
6249 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
6250 catver = formatspecs['format_ver'];
6251 fileheaderver = str(int(catver.replace(".", "")));
6252 lenlist = len(listcatfiles['ffilelist']);
6253 fnumfiles = int(listcatfiles['fnumfiles']);
6254 if(lenlist>fnumfiles or lenlist<fnumfiles):
6255 fnumfiles = lenlist;
6256 AppendFileHeader(catfp, fnumfiles, checksumtype, formatspecs);
6257 lenlist = len(listcatfiles['ffilelist']);
6258 fnumfiles = int(listcatfiles['fnumfiles']);
6259 lcfi = 0;
6260 lcfx = int(listcatfiles['fnumfiles']);
6261 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6262 lcfx = int(lenlist);
6263 else:
6264 lcfx = int(listcatfiles['fnumfiles']);
6265 curinode = 0;
6266 curfid = 0;
6267 inodelist = [];
6268 inodetofile = {};
6269 filetoinode = {};
6270 reallcfi = 0;
6271 while(lcfi < lcfx):
6272 if(re.findall(r"^[.|/]", listcatfiles['ffilelist'][reallcfi]['fname'])):
6273 fname = listcatfiles['ffilelist'][reallcfi]['fname'];
6274 else:
6275 fname = "./"+listcatfiles['ffilelist'][reallcfi]['fname'];
6276 if(verbose):
6277 VerbosePrintOut(fname);
6278 fheadersize = format(int(listcatfiles['ffilelist'][reallcfi]['fheadersize']), 'x').lower();
6279 fsize = format(int(listcatfiles['ffilelist'][reallcfi]['fsize']), 'x').lower();
6280 flinkname = listcatfiles['ffilelist'][reallcfi]['flinkname'];
6281 fatime = format(int(listcatfiles['ffilelist'][reallcfi]['fatime']), 'x').lower();
6282 fmtime = format(int(listcatfiles['ffilelist'][reallcfi]['fmtime']), 'x').lower();
6283 fctime = format(int(listcatfiles['ffilelist'][reallcfi]['fctime']), 'x').lower();
6284 fbtime = format(int(listcatfiles['ffilelist'][reallcfi]['fbtime']), 'x').lower();
6285 fmode = format(int(listcatfiles['ffilelist'][reallcfi]['fmode']), 'x').lower();
6286 fchmode = format(int(listcatfiles['ffilelist'][reallcfi]['fchmode']), 'x').lower();
6287 fuid = format(int(listcatfiles['ffilelist'][reallcfi]['fuid']), 'x').lower();
6288 funame = listcatfiles['ffilelist'][reallcfi]['funame'];
6289 fgid = format(int(listcatfiles['ffilelist'][reallcfi]['fgid']), 'x').lower();
6290 fgname = listcatfiles['ffilelist'][reallcfi]['fgname'];
6291 finode = format(int(listcatfiles['ffilelist'][reallcfi]['finode']), 'x').lower();
6292 flinkcount = format(int(listcatfiles['ffilelist'][reallcfi]['flinkcount']), 'x').lower();
6293 fwinattributes = format(int(listcatfiles['ffilelist'][reallcfi]['fwinattributes']), 'x').lower();
6294 fcompression = listcatfiles['ffilelist'][reallcfi]['fcompression'];
6295 fcsize = format(int(listcatfiles['ffilelist'][reallcfi]['fcsize']), 'x').lower();
6296 fdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['fminor']), 'x').lower();
6297 fdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['fmajor']), 'x').lower();
6298 frdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['frminor']), 'x').lower();
6299 frdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['frmajor']), 'x').lower();
6300 fseeknextfile = listcatfiles['ffilelist'][reallcfi]['fseeknextfile'];
6301 if(len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>listcatfiles['ffilelist'][reallcfi]['fextrafields'] and len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>0):
6302 listcatfiles['ffilelist'][reallcfi]['fextrafields'] = len(listcatfiles['ffilelist'][reallcfi]['fextralist']);
6303 if(not followlink and len(extradata)<0):
6304 extradata = listcatfiles['ffilelist'][reallcfi]['fextralist'];
6305 fcontents = listcatfiles['ffilelist'][reallcfi]['fcontents'];
6306 fcompression = "";
6307 fcsize = format(int(0), 'x').lower();
6308 if(not compresswholefile):
6309 fcontents.seek(0, 2);
6310 ucfsize = fcontents.tell();
6311 fcontents.seek(0, 0);
6312 if(compression=="auto"):
6313 ilsize = len(compressionlistalt);
6314 ilmin = 0;
6315 ilcsize = [];
6316 while(ilmin < ilsize):
6317 cfcontents = BytesIO();
6318 shutil.copyfileobj(fcontents, cfcontents);
6319 fcontents.seek(0, 0);
6320 cfcontents.seek(0, 0);
6321 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
6322 if(cfcontents):
6323 cfcontents.seek(0, 2);
6324 ilcsize.append(cfcontents.tell());
6325 cfcontents.close();
6326 else:
6327 try:
6328 ilcsize.append(sys.maxint);
6329 except AttributeError:
6330 ilcsize.append(sys.maxsize);
6331 ilmin = ilmin + 1;
6332 ilcmin = ilcsize.index(min(ilcsize));
6333 compression = compressionlistalt[ilcmin];
6334 fcontents.seek(0, 0);
6335 cfcontents = BytesIO();
6336 shutil.copyfileobj(fcontents, cfcontents);
6337 cfcontents.seek(0, 0);
6338 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
6339 cfcontents.seek(0, 2);
6340 cfsize = cfcontents.tell();
6341 if(ucfsize > cfsize):
6342 fcsize = format(int(cfsize), 'x').lower();
6343 fcompression = compression;
6344 fcontents.close();
6345 fcontents = cfcontents;
6346 if(followlink):
6347 if(listcatfiles['ffilelist'][reallcfi]['ftype']==1 or listcatfiles['ffilelist'][reallcfi]['ftype']==2):
6348 getflinkpath = listcatfiles['ffilelist'][reallcfi]['flinkname'];
6349 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6350 flinkinfo = listcatfiles['ffilelist'][flinkid];
6351 fheadersize = format(int(flinkinfo['fheadersize']), 'x').lower();
6352 fsize = format(int(flinkinfo['fsize']), 'x').lower();
6353 flinkname = flinkinfo['flinkname'];
6354 fatime = format(int(flinkinfo['fatime']), 'x').lower();
6355 fmtime = format(int(flinkinfo['fmtime']), 'x').lower();
6356 fctime = format(int(flinkinfo['fctime']), 'x').lower();
6357 fbtime = format(int(flinkinfo['fbtime']), 'x').lower();
6358 fmode = format(int(flinkinfo['fmode']), 'x').lower();
6359 fchmode = format(int(flinkinfo['fchmode']), 'x').lower();
6360 fuid = format(int(flinkinfo['fuid']), 'x').lower();
6361 funame = flinkinfo['funame'];
6362 fgid = format(int(flinkinfo['fgid']), 'x').lower();
6363 fgname = flinkinfo['fgname'];
6364 finode = format(int(flinkinfo['finode']), 'x').lower();
6365 flinkcount = format(int(flinkinfo['flinkcount']), 'x').lower();
6366 fwinattributes = format(int(flinkinfo['fwinattributes']), 'x').lower();
6367 fcompression = flinkinfo['fcompression'];
6368 fcsize = format(int(flinkinfo['fcsize']), 'x').lower();
6369 fdev_minor = format(int(flinkinfo['fminor']), 'x').lower();
6370 fdev_major = format(int(flinkinfo['fmajor']), 'x').lower();
6371 frdev_minor = format(int(flinkinfo['frminor']), 'x').lower();
6372 frdev_major = format(int(flinkinfo['frmajor']), 'x').lower();
6373 fseeknextfile = flinkinfo['fseeknextfile'];
6374 if(len(flinkinfo['fextralist'])>flinkinfo['fextrafields'] and len(flinkinfo['fextralist'])>0):
6375 flinkinfo['fextrafields'] = len(flinkinfo['fextralist']);
6376 if(len(extradata)<0):
6377 extradata = flinkinfo['fextralist'];
6378 fcontents = flinkinfo['fcontents'];
6379 ftypehex = format(flinkinfo['ftype'], 'x').lower();
6380 else:
6381 ftypehex = format(listcatfiles['ffilelist'][reallcfi]['ftype'], 'x').lower();
6382 fcurfid = format(curfid, 'x').lower();
6383 if(not followlink and finode!=0):
6384 if(listcatfiles['ffilelist'][reallcfi]['ftype']!=1):
6385 fcurinode = format(int(curinode), 'x').lower();
6386 inodetofile.update({curinode: fname});
6387 filetoinode.update({fname: curinode});
6388 curinode = curinode + 1;
6389 else:
6390 fcurinode = format(int(filetoinode[flinkname]), 'x').lower();
6391 else:
6392 fcurinode = format(int(curinode), 'x').lower();
6393 curinode = curinode + 1;
6394 curfid = curfid + 1;
6395 if(fcompression=="none"):
6396 fcompression = "";
6397 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, fseeknextfile];
6398 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
6399 fcontents.close();
6400 lcfi = lcfi + 1;
6401 reallcfi = reallcfi + 1;
6402 if(lcfx>0):
6403 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
6404 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
6405 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
6406 try:
6407 catfp.flush();
6408 os.fsync(catfp.fileno());
6409 except io.UnsupportedOperation:
6410 pass;
6411 except AttributeError:
6412 pass;
6413 except OSError as e:
6414 pass;
6415 if(outfile=="-"):
6416 catfp.seek(0, 0);
6417 if(hasattr(sys.stdout, "buffer")):
6418 shutil.copyfileobj(catfp, sys.stdout.buffer);
6419 else:
6420 shutil.copyfileobj(catfp, sys.stdout);
6421 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
6422 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
6423 catfp.seek(0, 0);
6424 upload_file_to_internet_file(catfp, outfile);
6425 if(returnfp):
6426 catfp.seek(0, 0);
6427 return catfp;
6428 else:
6429 catfp.close();
6430 return True;
6432 create_alias_function("RePack", __file_format_name__, "", RePackArchiveFile);
6434 def RePackArchiveFileFromString(catstr, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6435 formatspecs = FormatSpecsListToDict(formatspecs);
6436 catfp = BytesIO(catstr);
6437 listcatfiles = RePackArchiveFile(catfp, compression, compresswholefile, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6438 return listcatfiles;
6440 create_alias_function("RePack", __file_format_name__, "FromString", RePackArchiveFileFromString);
6442 def PackArchiveFileFromListDir(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6443 formatspecs = FormatSpecsListToDict(formatspecs);
6444 outarray = BytesIO();
6445 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
6446 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6447 return listcatfiles;
6449 create_alias_function("Pack", __file_format_name__, "FromListDir", PackArchiveFileFromListDir);
6451 def UnPackArchiveFile(infile, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, preservepermissions=True, preservetime=True, verbose=False, returnfp=False):
6452 formatspecs = FormatSpecsListToDict(formatspecs);
6453 if(outdir is not None):
6454 outdir = RemoveWindowsPath(outdir);
6455 if(verbose):
6456 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6457 if(isinstance(infile, dict)):
6458 listcatfiles = infile;
6459 else:
6460 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6461 infile = RemoveWindowsPath(infile);
6462 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, True, skipchecksum, formatspecs, returnfp);
6463 if(not listcatfiles):
6464 return False;
6465 lenlist = len(listcatfiles['ffilelist']);
6466 fnumfiles = int(listcatfiles['fnumfiles']);
6467 lcfi = 0;
6468 lcfx = int(listcatfiles['fnumfiles']);
6469 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6470 lcfx = int(lenlist);
6471 else:
6472 lcfx = int(listcatfiles['fnumfiles']);
6473 while(lcfi<lcfx):
6474 funame = "";
6475 try:
6476 import pwd;
6477 try:
6478 userinfo = pwd.getpwuid(listcatfiles['ffilelist'][lcfi]['fuid']);
6479 funame = userinfo.pw_name;
6480 except KeyError:
6481 funame = "";
6482 except ImportError:
6483 funame = "";
6484 fgname = "";
6485 try:
6486 import grp;
6487 try:
6488 groupinfo = grp.getgrgid(listcatfiles['ffilelist'][lcfi]['fgid']);
6489 fgname = groupinfo.gr_name;
6490 except KeyError:
6491 fgname = "";
6492 except ImportError:
6493 fgname = "";
6494 if(verbose):
6495 VerbosePrintOut(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6496 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6497 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6498 listcatfiles['ffilelist'][lcfi]['fcontents'].seek(0, 0);
6499 shutil.copyfileobj(listcatfiles['ffilelist'][lcfi]['fcontents'], fpc);
6500 try:
6501 fpc.flush();
6502 os.fsync(fpc.fileno());
6503 except io.UnsupportedOperation:
6504 pass;
6505 except AttributeError:
6506 pass;
6507 except OSError as e:
6508 pass;
6509 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
6510 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
6511 if(preservepermissions):
6512 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6513 if(preservetime):
6514 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
6515 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6516 if(followlink):
6517 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
6518 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6519 flinkinfo = listcatfiles['ffilelist'][flinkid];
6520 funame = "";
6521 try:
6522 import pwd;
6523 try:
6524 userinfo = pwd.getpwuid(flinkinfo['fuid']);
6525 funame = userinfo.pw_name;
6526 except KeyError:
6527 funame = "";
6528 except ImportError:
6529 funame = "";
6530 fgname = "";
6531 try:
6532 import grp;
6533 try:
6534 groupinfo = grp.getgrgid(flinkinfo['fgid']);
6535 fgname = groupinfo.gr_name;
6536 except KeyError:
6537 fgname = "";
6538 except ImportError:
6539 fgname = "";
6540 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
6541 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6542 flinkinfo['fcontents'].seek(0, 0);
6543 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
6544 try:
6545 fpc.flush();
6546 os.fsync(fpc.fileno());
6547 except io.UnsupportedOperation:
6548 pass;
6549 except AttributeError:
6550 pass;
6551 except OSError as e:
6552 pass;
6553 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6554 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6555 if(preservepermissions):
6556 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6557 if(preservetime):
6558 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6559 if(flinkinfo['ftype']==1):
6560 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6561 if(flinkinfo['ftype']==2):
6562 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6563 if(flinkinfo['ftype']==5):
6564 if(preservepermissions):
6565 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6566 else:
6567 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6568 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6569 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6570 if(preservepermissions):
6571 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6572 if(preservetime):
6573 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6574 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
6575 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6576 else:
6577 os.link(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6578 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6579 if(followlink):
6580 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
6581 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6582 flinkinfo = listcatfiles['ffilelist'][flinkid];
6583 funame = "";
6584 try:
6585 import pwd;
6586 try:
6587 userinfo = pwd.getpwuid(flinkinfo['fuid']);
6588 funame = userinfo.pw_name;
6589 except KeyError:
6590 funame = "";
6591 except ImportError:
6592 funame = "";
6593 fgname = "";
6594 try:
6595 import grp;
6596 try:
6597 groupinfo = grp.getgrgid(flinkinfo['fgid']);
6598 fgname = groupinfo.gr_name;
6599 except KeyError:
6600 fgname = "";
6601 except ImportError:
6602 fgname = "";
6603 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
6604 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6605 flinkinfo['fcontents'].seek(0, 0);
6606 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
6607 try:
6608 fpc.flush();
6609 os.fsync(fpc.fileno());
6610 except io.UnsupportedOperation:
6611 pass;
6612 except AttributeError:
6613 pass;
6614 except OSError as e:
6615 pass;
6616 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6617 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6618 if(preservepermissions):
6619 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6620 if(preservetime):
6621 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6622 if(flinkinfo['ftype']==1):
6623 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6624 if(flinkinfo['ftype']==2):
6625 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6626 if(flinkinfo['ftype']==5):
6627 if(preservepermissions):
6628 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6629 else:
6630 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6631 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6632 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6633 if(preservepermissions):
6634 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6635 if(preservetime):
6636 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6637 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
6638 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6639 else:
6640 os.symlink(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6641 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6642 if(preservepermissions):
6643 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6644 else:
6645 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6646 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
6647 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
6648 if(preservepermissions):
6649 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6650 if(preservetime):
6651 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
6652 if(listcatfiles['ffilelist'][lcfi]['ftype']==6 and hasattr(os, "mkfifo")):
6653 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6654 lcfi = lcfi + 1;
6655 if(returnfp):
6656 return listcatfiles['ffilelist']['catfp'];
6657 else:
6658 return True;
6660 create_alias_function("UnPack", __file_format_name__, "", UnPackArchiveFile);
6662 if(hasattr(shutil, "register_unpack_format")):
6663 def UnPackArchiveFileFunc(archive_name, extract_dir=None, **kwargs):
6664 return UnPackArchiveFile(archive_name, extract_dir, False, 0, 0, False, __file_format_dict__['format_delimiter'], False, False);
6665 create_alias_function("UnPack", __file_format_name__, "Func", UnPackArchiveFileFunc);
6667 def UnPackArchiveFileString(catstr, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6668 formatspecs = FormatSpecsListToDict(formatspecs);
6669 catfp = BytesIO(catstr);
6670 listcatfiles = UnPackArchiveFile(catfp, outdir, followlink, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6671 return listcatfiles;
6673 create_alias_function("UnPack", __file_format_name__, "String", UnPackArchiveFileString);
6675 def ArchiveFileListFiles(infile, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6676 formatspecs = FormatSpecsListToDict(formatspecs);
6677 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6678 if(isinstance(infile, dict)):
6679 listcatfiles = infile;
6680 else:
6681 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6682 infile = RemoveWindowsPath(infile);
6683 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, True, False, skipchecksum, formatspecs, returnfp);
6684 if(not listcatfiles):
6685 return False;
6686 lenlist = len(listcatfiles['ffilelist']);
6687 fnumfiles = int(listcatfiles['fnumfiles']);
6688 lcfi = 0;
6689 lcfx = int(listcatfiles['fnumfiles']);
6690 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6691 lcfx = int(lenlist);
6692 else:
6693 lcfx = int(listcatfiles['fnumfiles']);
6694 returnval = {};
6695 while(lcfi<lcfx):
6696 returnval.update({lcfi: listcatfiles['ffilelist'][lcfi]['fname']});
6697 if(not verbose):
6698 VerbosePrintOut(listcatfiles['ffilelist'][lcfi]['fname']);
6699 if(verbose):
6700 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6701 printfname = listcatfiles['ffilelist'][lcfi]['fname'];
6702 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6703 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " link to " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6704 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6705 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " -> " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6706 fuprint = listcatfiles['ffilelist'][lcfi]['funame'];
6707 if(len(fuprint)<=0):
6708 fuprint = listcatfiles['ffilelist'][lcfi]['fuid'];
6709 fgprint = listcatfiles['ffilelist'][lcfi]['fgname'];
6710 if(len(fgprint)<=0):
6711 fgprint = listcatfiles['ffilelist'][lcfi]['fgid'];
6712 VerbosePrintOut(PrintPermissionString(listcatfiles['ffilelist'][lcfi]['fmode'], listcatfiles['ffilelist'][lcfi]['ftype']) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(listcatfiles['ffilelist'][lcfi]['fsize']).rjust(15) + " " + datetime.datetime.utcfromtimestamp(listcatfiles['ffilelist'][lcfi]['fmtime']).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6713 lcfi = lcfi + 1;
6714 if(returnfp):
6715 return listcatfiles['catfp'];
6716 else:
6717 return True;
6719 create_alias_function("", __file_format_name__, "ListFiles", ArchiveFileListFiles);
6721 def ArchiveFileStringListFiles(catstr, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6722 formatspecs = FormatSpecsListToDict(formatspecs);
6723 catfp = BytesIO(catstr);
6724 listcatfiles = ArchiveFileListFiles(catstr, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6725 return listcatfiles;
6727 create_alias_function("", __file_format_name__, "StringListFiles", ArchiveFileStringListFiles);
6729 def TarFileListFiles(infile, verbose=False, returnfp=False):
6730 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6731 if(infile=="-"):
6732 infile = BytesIO();
6733 if(hasattr(sys.stdin, "buffer")):
6734 shutil.copyfileobj(sys.stdin.buffer, infile);
6735 else:
6736 shutil.copyfileobj(sys.stdin, infile);
6737 infile.seek(0, 0);
6738 if(not infile):
6739 return False;
6740 infile.seek(0, 0);
6741 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
6742 infile = download_file_from_internet_file(infile);
6743 infile.seek(0, 0);
6744 if(not infile):
6745 return False;
6746 infile.seek(0, 0);
6747 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
6748 return False;
6749 elif(os.path.exists(infile) and os.path.isfile(infile)):
6750 try:
6751 if(not tarfile.TarFileCheck(infile)):
6752 return False;
6753 except AttributeError:
6754 if(not TarFileCheck(infile)):
6755 return False;
6756 else:
6757 return False;
6758 try:
6759 if(hasattr(infile, "read") or hasattr(infile, "write")):
6760 tarfp = tarfile.open(fileobj=infile, mode="r");
6761 else:
6762 tarfp = tarfile.open(infile, "r");
6763 except FileNotFoundError:
6764 return False;
6765 lcfi = 0
6766 returnval = {};
6767 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
6768 returnval.update({lcfi: member.name});
6769 fpremode = member.mode;
6770 ffullmode = member.mode;
6771 flinkcount = 0;
6772 ftype = 0;
6773 if(member.isreg()):
6774 ffullmode = member.mode + stat.S_IFREG;
6775 ftype = 0;
6776 elif(member.isdev()):
6777 ffullmode = member.mode;
6778 ftype = 7;
6779 elif(member.islnk()):
6780 ffullmode = member.mode + stat.S_IFREG;
6781 ftype = 1;
6782 elif(member.issym()):
6783 ffullmode = member.mode + stat.S_IFLNK;
6784 ftype = 2;
6785 elif(member.ischr()):
6786 ffullmode = member.mode + stat.S_IFCHR;
6787 ftype = 3;
6788 elif(member.isblk()):
6789 ffullmode = member.mode + stat.S_IFBLK;
6790 ftype = 4;
6791 elif(member.isdir()):
6792 ffullmode = member.mode + stat.S_IFDIR;
6793 ftype = 5;
6794 elif(member.isfifo()):
6795 ffullmode = member.mode + stat.S_IFIFO;
6796 ftype = 6;
6797 elif(member.issparse()):
6798 ffullmode = member.mode;
6799 ftype = 12;
6800 if(not verbose):
6801 VerbosePrintOut(member.name);
6802 elif(verbose):
6803 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6804 printfname = member.name;
6805 if(member.islnk()):
6806 printfname = member.name + " link to " + member.linkname;
6807 elif(member.issym()):
6808 printfname = member.name + " -> " + member.linkname;
6809 fuprint = member.uname;
6810 if(len(fuprint)<=0):
6811 fuprint = member.uid;
6812 fgprint = member.gname;
6813 if(len(fgprint)<=0):
6814 fgprint = member.gid;
6815 VerbosePrintOut(PrintPermissionString(ffullmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(member.mtime).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6816 lcfi = lcfi + 1;
6817 if(returnfp):
6818 return listcatfiles['catfp'];
6819 else:
6820 return True;
6822 def ZipFileListFiles(infile, verbose=False, returnfp=False):
6823 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6824 if(infile=="-"):
6825 infile = BytesIO();
6826 if(hasattr(sys.stdin, "buffer")):
6827 shutil.copyfileobj(sys.stdin.buffer, infile);
6828 else:
6829 shutil.copyfileobj(sys.stdin, infile);
6830 infile.seek(0, 0);
6831 if(not infile):
6832 return False;
6833 infile.seek(0, 0);
6834 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
6835 infile = download_file_from_internet_file(infile);
6836 infile.seek(0, 0);
6837 if(not infile):
6838 return False;
6839 infile.seek(0, 0);
6840 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
6841 return False;
6842 else:
6843 return False;
6844 if(not zipfile.is_zipfile(infile)):
6845 return False;
6846 try:
6847 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
6848 except FileNotFoundError:
6849 return False;
6850 lcfi = 0;
6851 returnval = {};
6852 ziptest = zipfp.testzip();
6853 if(ziptest):
6854 VerbosePrintOut("Bad file found!");
6855 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
6856 if(zipinfo.create_system==0 or zipinfo.create_system==10):
6857 fwinattributes = int(zipinfo.external_attr);
6858 if(not member.is_dir()):
6859 fmode = int(stat.S_IFREG + 438);
6860 fchmode = int(stat.S_IMODE(fmode));
6861 ftypemod = int(stat.S_IFMT(fmode));
6862 elif(member.is_dir()):
6863 fmode = int(stat.S_IFDIR + 511);
6864 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6865 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6866 elif(zipinfo.create_system==3):
6867 fwinattributes =int(0);
6868 fmode = int(zipinfo.external_attr);
6869 fchmode = int(stat.S_IMODE(fmode));
6870 ftypemod = int(stat.S_IFMT(fmode));
6871 else:
6872 fwinattributes = int(0);
6873 if(not member.is_dir()):
6874 fmode = int(stat.S_IFREG + 438);
6875 fchmode = int(stat.S_IMODE(fmode));
6876 ftypemod = int(stat.S_IFMT(fmode));
6877 elif(member.is_dir()):
6878 fmode = int(stat.S_IFDIR + 511);
6879 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6880 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6881 returnval.update({lcfi: member.filename});
6882 if(not verbose):
6883 VerbosePrintOut(member.filename);
6884 if(verbose):
6885 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6886 permissionstr = "";
6887 for fmodval in str(oct(fmode))[-3:]:
6888 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
6889 if(not member.is_dir()):
6890 ftype = 0;
6891 permissionstr = "-" + permissionstr;
6892 elif(member.is_dir()):
6893 ftype = 5;
6894 permissionstr = "d" + permissionstr;
6895 printfname = member.filename;
6896 try:
6897 fuid = int(os.getuid());
6898 except AttributeError:
6899 fuid = int(0);
6900 except KeyError:
6901 fuid = int(0);
6902 try:
6903 fgid = int(os.getgid());
6904 except AttributeError:
6905 fgid = int(0);
6906 except KeyError:
6907 fgid = int(0);
6908 try:
6909 import pwd;
6910 try:
6911 userinfo = pwd.getpwuid(os.getuid());
6912 funame = userinfo.pw_name;
6913 except KeyError:
6914 funame = "";
6915 except AttributeError:
6916 funame = "";
6917 except ImportError:
6918 funame = "";
6919 fgname = "";
6920 try:
6921 import grp;
6922 try:
6923 groupinfo = grp.getgrgid(os.getgid());
6924 fgname = groupinfo.gr_name;
6925 except KeyError:
6926 fgname = "";
6927 except AttributeError:
6928 fgname = "";
6929 except ImportError:
6930 fgname = "";
6931 fuprint = funame;
6932 if(len(fuprint)<=0):
6933 fuprint = str(fuid);
6934 fgprint = fgname;
6935 if(len(fgprint)<=0):
6936 fgprint = str(fgid);
6937 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(int(time.mktime(member.date_time + (0, 0, -1)))).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6938 lcfi = lcfi + 1;
6939 if(returnfp):
6940 return listcatfiles['catfp'];
6941 else:
6942 return True;
6944 if(not rarfile_support):
6945 def RarFileListFiles(infile, verbose=False, returnfp=False):
6946 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6947 if(not os.path.exists(infile) or not os.path.isfile(infile)):
6948 return False;
6950 if(rarfile_support):
6951 def RarFileListFiles(infile, verbose=False, returnfp=False):
6952 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6953 if(not os.path.exists(infile) or not os.path.isfile(infile)):
6954 return False;
6955 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
6956 return False;
6957 lcfi = 0;
6958 returnval = {};
6959 rarfp = rarfile.RarFile(infile, "r");
6960 rartest = rarfp.testrar();
6961 if(rartest):
6962 VerbosePrintOut("Bad file found!");
6963 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
6964 is_unix = False;
6965 is_windows = False;
6966 if(member.host_os==rarfile.RAR_OS_UNIX):
6967 is_windows = False;
6968 try:
6969 member.external_attr
6970 is_unix = True;
6971 except AttributeError:
6972 is_unix = False;
6973 elif(member.host_os==rarfile.RAR_OS_WIN32):
6974 is_unix = False;
6975 try:
6976 member.external_attr
6977 is_windows = True;
6978 except AttributeError:
6979 is_windows = False;
6980 else:
6981 is_unix = False;
6982 is_windows = False;
6983 if(is_unix and member.external_attr !=0):
6984 fpremode = int(member.external_attr);
6985 elif(member.is_file()):
6986 fpremode = int(stat.S_IFREG + 438);
6987 elif(member.is_symlink()):
6988 fpremode = int(stat.S_IFLNK + 438);
6989 elif(member.is_dir()):
6990 fpremode = int(stat.S_IFDIR + 511);
6991 if(is_windows and member.external_attr !=0):
6992 fwinattributes = int(member.external_attr);
6993 else:
6994 fwinattributes = int(0);
6995 if(is_unix and member.external_attr !=0):
6996 fmode = int(member.external_attr);
6997 fchmode = int(stat.S_IMODE(member.external_attr));
6998 ftypemod = int(stat.S_IFMT(member.external_attr));
6999 elif(member.is_file()):
7000 fmode = int(stat.S_IFREG + 438);
7001 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
7002 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
7003 elif(member.is_symlink()):
7004 fmode = int(stat.S_IFLNK + 438);
7005 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
7006 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
7007 elif(member.is_dir()):
7008 fmode = int(stat.S_IFDIR + 511);
7009 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7010 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7011 returnval.update({lcfi: member.filename});
7012 if(not verbose):
7013 VerbosePrintOut(member.filename);
7014 if(verbose):
7015 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7016 permissionstr = "";
7017 for fmodval in str(oct(fmode))[-3:]:
7018 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
7019 if(member.is_file()):
7020 ftype = 0;
7021 permissionstr = "-" + permissionstr;
7022 printfname = member.filename;
7023 elif(member.is_symlink()):
7024 ftype = 2;
7025 permissionstr = "l" + permissionstr;
7026 printfname = member.name + " -> " + member.read().decode("UTF-8");
7027 elif(member.is_dir()):
7028 ftype = 5;
7029 permissionstr = "d" + permissionstr;
7030 printfname = member.filename;
7031 try:
7032 fuid = int(os.getuid());
7033 except AttributeError:
7034 fuid = int(0);
7035 except KeyError:
7036 fuid = int(0);
7037 try:
7038 fgid = int(os.getgid());
7039 except AttributeError:
7040 fgid = int(0);
7041 except KeyError:
7042 fgid = int(0);
7043 try:
7044 import pwd;
7045 try:
7046 userinfo = pwd.getpwuid(os.getuid());
7047 funame = userinfo.pw_name;
7048 except KeyError:
7049 funame = "";
7050 except AttributeError:
7051 funame = "";
7052 except ImportError:
7053 funame = "";
7054 fgname = "";
7055 try:
7056 import grp;
7057 try:
7058 groupinfo = grp.getgrgid(os.getgid());
7059 fgname = groupinfo.gr_name;
7060 except KeyError:
7061 fgname = "";
7062 except AttributeError:
7063 fgname = "";
7064 except ImportError:
7065 fgname = "";
7066 fuprint = funame;
7067 if(len(fuprint)<=0):
7068 fuprint = str(fuid);
7069 fgprint = fgname;
7070 if(len(fgprint)<=0):
7071 fgprint = str(fgid);
7072 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + member.mtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
7073 lcfi = lcfi + 1;
7074 if(returnfp):
7075 return listcatfiles['catfp'];
7076 else:
7077 return True;
7079 if(not py7zr_support):
7080 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
7081 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7082 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7083 return False;
7085 if(py7zr_support):
7086 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
7087 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7088 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7089 return False;
7090 lcfi = 0;
7091 returnval = {};
7092 szpfp = py7zr.SevenZipFile(infile, mode="r");
7093 file_content = szpfp.readall();
7094 #sztest = szpfp.testzip();
7095 sztestalt = szpfp.test();
7096 if(sztestalt):
7097 VerbosePrintOut("Bad file found!");
7098 for member in sorted(szpfp.list(), key=lambda x: x.filename):
7099 if(re.findall(r"^[.|/]", member.filename)):
7100 fname = member.filename;
7101 else:
7102 fname = "./"+member.filename;
7103 if(not member.is_directory):
7104 fpremode = int(stat.S_IFREG + 438);
7105 elif(member.is_directory):
7106 fpremode = int(stat.S_IFDIR + 511);
7107 fwinattributes = int(0);
7108 if(member.is_directory):
7109 fmode = int(stat.S_IFDIR + 511);
7110 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7111 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7112 else:
7113 fmode = int(stat.S_IFLNK + 438);
7114 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
7115 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
7116 returnval.update({lcfi: member.filename});
7117 if(not verbose):
7118 VerbosePrintOut(member.filename);
7119 if(verbose):
7120 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7121 permissionstr = "";
7122 for fmodval in str(oct(fmode))[-3:]:
7123 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
7124 fsize = int("0");
7125 if(not member.is_directory):
7126 ftype = 0;
7127 permissionstr = "-" + permissionstr;
7128 printfname = member.filename;
7129 elif(member.is_directory):
7130 ftype = 5;
7131 permissionstr = "d" + permissionstr;
7132 printfname = member.filename;
7133 if(ftype==0):
7134 fsize = len(file_content[member.filename].read());
7135 file_content[member.filename].close();
7136 try:
7137 fuid = int(os.getuid());
7138 except AttributeError:
7139 fuid = int(0);
7140 except KeyError:
7141 fuid = int(0);
7142 try:
7143 fgid = int(os.getgid());
7144 except AttributeError:
7145 fgid = int(0);
7146 except KeyError:
7147 fgid = int(0);
7148 try:
7149 import pwd;
7150 try:
7151 userinfo = pwd.getpwuid(os.getuid());
7152 funame = userinfo.pw_name;
7153 except KeyError:
7154 funame = "";
7155 except AttributeError:
7156 funame = "";
7157 except ImportError:
7158 funame = "";
7159 fgname = "";
7160 try:
7161 import grp;
7162 try:
7163 groupinfo = grp.getgrgid(os.getgid());
7164 fgname = groupinfo.gr_name;
7165 except KeyError:
7166 fgname = "";
7167 except AttributeError:
7168 fgname = "";
7169 except ImportError:
7170 fgname = "";
7171 fuprint = funame;
7172 if(len(fuprint)<=0):
7173 fuprint = str(fuid);
7174 fgprint = fgname;
7175 if(len(fgprint)<=0):
7176 fgprint = str(fgid);
7177 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(fsize).rjust(15) + " " + member.creationtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
7178 lcfi = lcfi + 1;
7179 if(returnfp):
7180 return listcatfiles['catfp'];
7181 else:
7182 return True;
7184 def InFileListFiles(infile, verbose=False, formatspecs=__file_format_dict__, returnfp=False):
7185 formatspecs = FormatSpecsListToDict(formatspecs);
7186 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7187 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
7188 if(checkcompressfile=="tarfile"):
7189 return TarFileListFiles(infile, verbose, returnfp);
7190 elif(checkcompressfile=="zipfile"):
7191 return ZipFileListFiles(infile, verbose, returnfp);
7192 elif(checkcompressfile=="catfile"):
7193 return ArchiveFileListFiles(infile, 0, 0, False, formatspecs, verbose, returnfp);
7194 elif(rarfile_support and checkcompressfile=="rarfile"):
7195 return RarFileListFiles(infile, verbose, returnfp);
7196 elif(py7zr_support and checkcompressfile=="7zipfile"):
7197 return SevenZipFileListFiles(infile, verbose, returnfp);
7198 else:
7199 return False;
7200 return False;
7202 def ListDirListFiles(infiles, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7203 formatspecs = FormatSpecsListToDict(formatspecs);
7204 outarray = BytesIO();
7205 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, False, True);
7206 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
7207 return listcatfiles;
7209 def ListDirListFilesAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7210 formatspecs = FormatSpecsListToDict(formatspecs);
7211 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, listonly, checksumtype, formatspecs, verbose);
7212 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
7213 return listcatfiles;
7215 def PackArchiveFileFromListDirAlt(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7216 formatspecs = FormatSpecsListToDict(formatspecs);
7217 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, False, checksumtype, extradata, formatspecs, False);
7218 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, followlink, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
7219 return listcatfiles;
7221 create_alias_function("Pack", __file_format_name__, "FromListDirAlt", PackArchiveFileFromListDirAlt);
7223 def PackArchiveFileFromTarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7224 formatspecs = FormatSpecsListToDict(formatspecs);
7225 outarray = TarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7226 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7227 return listcatfiles;
7229 create_alias_function("Pack", __file_format_name__, "FromTarFileAlt", PackArchiveFileFromTarFileAlt);
7231 def PackArchiveFileFromZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7232 formatspecs = FormatSpecsListToDict(formatspecs);
7233 outarray = ZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7234 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7235 return listcatfiles;
7237 create_alias_function("Pack", __file_format_name__, "FromZipFileAlt", PackArchiveFileFromZipFileAlt);
7239 if(not rarfile_support):
7240 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7241 return False;
7243 if(rarfile_support):
7244 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7245 formatspecs = FormatSpecsListToDict(formatspecs);
7246 outarray = RarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7247 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7248 return listcatfiles;
7250 create_alias_function("Pack", __file_format_name__, "FromRarFileAlt", PackArchiveFileFromRarFileAlt);
7252 if(not py7zr_support):
7253 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7254 return False;
7256 if(py7zr_support):
7257 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7258 formatspecs = FormatSpecsListToDict(formatspecs);
7259 outarray = SevenZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7260 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7261 return listcatfiles;
7263 create_alias_function("Pack", __file_format_name__, "FromSevenZipFileAlt", PackArchiveFileFromSevenZipFileAlt);
7265 def download_file_from_ftp_file(url):
7266 urlparts = urlparse(url);
7267 file_name = os.path.basename(urlparts.path);
7268 file_dir = os.path.dirname(urlparts.path);
7269 if(urlparts.username is not None):
7270 ftp_username = urlparts.username;
7271 else:
7272 ftp_username = "anonymous";
7273 if(urlparts.password is not None):
7274 ftp_password = urlparts.password;
7275 elif(urlparts.password is None and urlparts.username=="anonymous"):
7276 ftp_password = "anonymous";
7277 else:
7278 ftp_password = "";
7279 if(urlparts.scheme=="ftp"):
7280 ftp = FTP();
7281 elif(urlparts.scheme=="ftps" and ftpssl):
7282 ftp = FTP_TLS();
7283 else:
7284 return False;
7285 if(urlparts.scheme=="sftp"):
7286 if(__use_pysftp__):
7287 return download_file_from_pysftp_file(url);
7288 else:
7289 return download_file_from_sftp_file(url);
7290 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7291 return download_file_from_http_file(url);
7292 ftp_port = urlparts.port;
7293 if(urlparts.port is None):
7294 ftp_port = 21;
7295 try:
7296 ftp.connect(urlparts.hostname, ftp_port);
7297 except socket.gaierror:
7298 log.info("Error With URL "+url);
7299 return False;
7300 except socket.timeout:
7301 log.info("Error With URL "+url);
7302 return False;
7303 ftp.login(urlparts.username, urlparts.password);
7304 if(urlparts.scheme=="ftps"):
7305 ftp.prot_p();
7306 ftpfile = BytesIO();
7307 ftp.retrbinary("RETR "+urlparts.path, ftpfile.write);
7308 #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
7309 ftp.close();
7310 ftpfile.seek(0, 0);
7311 return ftpfile;
7313 def download_file_from_ftp_string(url):
7314 ftpfile = download_file_from_ftp_file(url);
7315 return ftpfile.read();
7317 def upload_file_to_ftp_file(ftpfile, url):
7318 urlparts = urlparse(url);
7319 file_name = os.path.basename(urlparts.path);
7320 file_dir = os.path.dirname(urlparts.path);
7321 if(urlparts.username is not None):
7322 ftp_username = urlparts.username;
7323 else:
7324 ftp_username = "anonymous";
7325 if(urlparts.password is not None):
7326 ftp_password = urlparts.password;
7327 elif(urlparts.password is None and urlparts.username=="anonymous"):
7328 ftp_password = "anonymous";
7329 else:
7330 ftp_password = "";
7331 if(urlparts.scheme=="ftp"):
7332 ftp = FTP();
7333 elif(urlparts.scheme=="ftps" and ftpssl):
7334 ftp = FTP_TLS();
7335 else:
7336 return False;
7337 if(urlparts.scheme=="sftp"):
7338 if(__use_pysftp__):
7339 return upload_file_to_pysftp_file(url);
7340 else:
7341 return upload_file_to_sftp_file(url);
7342 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7343 return False;
7344 ftp_port = urlparts.port;
7345 if(urlparts.port is None):
7346 ftp_port = 21;
7347 try:
7348 ftp.connect(urlparts.hostname, ftp_port);
7349 except socket.gaierror:
7350 log.info("Error With URL "+url);
7351 return False;
7352 except socket.timeout:
7353 log.info("Error With URL "+url);
7354 return False;
7355 ftp.login(urlparts.username, urlparts.password);
7356 if(urlparts.scheme=="ftps"):
7357 ftp.prot_p();
7358 ftp.storbinary("STOR "+urlparts.path, ftpfile);
7359 ftp.close();
7360 ftpfile.seek(0, 0);
7361 return ftpfile;
7363 def upload_file_to_ftp_string(ftpstring, url):
7364 ftpfileo = BytesIO(ftpstring);
7365 ftpfile = upload_file_to_ftp_file(ftpfileo, url);
7366 ftpfileo.close();
7367 return ftpfile;
7369 def download_file_from_http_file(url, headers=geturls_headers_pycatfile_python_alt):
7370 # Parse the URL to extract username and password if present
7371 urlparts = urlparse(url);
7372 username = urlparts.username;
7373 password = urlparts.password;
7374 # Rebuild the URL without the username and password
7375 netloc = urlparts.hostname;
7376 if(urlparts.scheme=="sftp"):
7377 if(__use_pysftp__):
7378 return download_file_from_pysftp_file(url);
7379 else:
7380 return download_file_from_sftp_file(url);
7381 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7382 return download_file_from_ftp_file(url);
7383 if urlparts.port:
7384 netloc += ':' + str(urlparts.port);
7385 rebuilt_url = urlunparse((urlparts.scheme, netloc, urlparts.path, urlparts.params, urlparts.query, urlparts.fragment));
7386 # Create a temporary file object
7387 httpfile = BytesIO();
7388 if haverequests:
7389 # Use the requests library if available
7390 if username and password:
7391 response = requests.get(rebuilt_url, headers=headers, auth=(username, password), stream=True);
7392 else:
7393 response = requests.get(rebuilt_url, headers=headers, stream=True);
7394 response.raw.decode_content = True
7395 shutil.copyfileobj(response.raw, httpfile);
7396 else:
7397 # Build a Request object for urllib
7398 request = Request(rebuilt_url, headers=headers);
7399 # Create an opener object for handling URLs
7400 if username and password:
7401 # Create a password manager
7402 password_mgr = HTTPPasswordMgrWithDefaultRealm();
7403 # Add the username and password
7404 password_mgr.add_password(None, rebuilt_url, username, password);
7405 # Create an authentication handler using the password manager
7406 auth_handler = HTTPBasicAuthHandler(password_mgr);
7407 # Build the opener with the authentication handler
7408 opener = build_opener(auth_handler);
7409 else:
7410 opener = build_opener();
7411 with opener.open(request) as response:
7412 shutil.copyfileobj(response, httpfile);
7413 # Reset file pointer to the start
7414 httpfile.seek(0, 0);
7415 # Return the temporary file object
7416 return httpfile;
7418 def download_file_from_http_string(url, headers=geturls_headers_pycatfile_python_alt):
7419 httpfile = download_file_from_http_file(url, headers);
7420 return ftpfile.read();
7422 if(haveparamiko):
7423 def download_file_from_sftp_file(url):
7424 urlparts = urlparse(url);
7425 file_name = os.path.basename(urlparts.path);
7426 file_dir = os.path.dirname(urlparts.path);
7427 sftp_port = urlparts.port;
7428 if(urlparts.port is None):
7429 sftp_port = 22;
7430 else:
7431 sftp_port = urlparts.port;
7432 if(urlparts.username is not None):
7433 sftp_username = urlparts.username;
7434 else:
7435 sftp_username = "anonymous";
7436 if(urlparts.password is not None):
7437 sftp_password = urlparts.password;
7438 elif(urlparts.password is None and urlparts.username=="anonymous"):
7439 sftp_password = "anonymous";
7440 else:
7441 sftp_password = "";
7442 if(urlparts.scheme=="ftp"):
7443 return download_file_from_ftp_file(url);
7444 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7445 return download_file_from_http_file(url);
7446 if(urlparts.scheme!="sftp"):
7447 return False;
7448 ssh = paramiko.SSHClient();
7449 ssh.load_system_host_keys();
7450 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
7451 try:
7452 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7453 except paramiko.ssh_exception.SSHException:
7454 return False;
7455 except socket.gaierror:
7456 log.info("Error With URL "+url);
7457 return False;
7458 except socket.timeout:
7459 log.info("Error With URL "+url);
7460 return False;
7461 sftp = ssh.open_sftp();
7462 sftpfile = BytesIO();
7463 sftp.getfo(urlparts.path, sftpfile);
7464 sftp.close();
7465 ssh.close();
7466 sftpfile.seek(0, 0);
7467 return sftpfile;
7468 else:
7469 def download_file_from_sftp_file(url):
7470 return False;
7472 if(haveparamiko):
7473 def download_file_from_sftp_string(url):
7474 sftpfile = download_file_from_sftp_file(url);
7475 return sftpfile.read();
7476 else:
7477 def download_file_from_ftp_string(url):
7478 return False;
7480 if(haveparamiko):
7481 def upload_file_to_sftp_file(sftpfile, url):
7482 urlparts = urlparse(url);
7483 file_name = os.path.basename(urlparts.path);
7484 file_dir = os.path.dirname(urlparts.path);
7485 sftp_port = urlparts.port;
7486 if(urlparts.port is None):
7487 sftp_port = 22;
7488 else:
7489 sftp_port = urlparts.port;
7490 if(urlparts.username is not None):
7491 sftp_username = urlparts.username;
7492 else:
7493 sftp_username = "anonymous";
7494 if(urlparts.password is not None):
7495 sftp_password = urlparts.password;
7496 elif(urlparts.password is None and urlparts.username=="anonymous"):
7497 sftp_password = "anonymous";
7498 else:
7499 sftp_password = "";
7500 if(urlparts.scheme=="ftp"):
7501 return upload_file_to_ftp_file(url);
7502 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7503 return False;
7504 if(urlparts.scheme!="sftp"):
7505 return False;
7506 ssh = paramiko.SSHClient();
7507 ssh.load_system_host_keys();
7508 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
7509 try:
7510 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7511 except paramiko.ssh_exception.SSHException:
7512 return False;
7513 except socket.gaierror:
7514 log.info("Error With URL "+url);
7515 return False;
7516 except socket.timeout:
7517 log.info("Error With URL "+url);
7518 return False;
7519 sftp = ssh.open_sftp();
7520 sftp.putfo(sftpfile, urlparts.path);
7521 sftp.close();
7522 ssh.close();
7523 sftpfile.seek(0, 0);
7524 return sftpfile;
7525 else:
7526 def upload_file_to_sftp_file(sftpfile, url):
7527 return False;
7529 if(haveparamiko):
7530 def upload_file_to_sftp_string(sftpstring, url):
7531 sftpfileo = BytesIO(sftpstring);
7532 sftpfile = upload_file_to_sftp_files(ftpfileo, url);
7533 sftpfileo.close();
7534 return sftpfile;
7535 else:
7536 def upload_file_to_sftp_string(url):
7537 return False;
7539 if(havepysftp):
7540 def download_file_from_pysftp_file(url):
7541 urlparts = urlparse(url);
7542 file_name = os.path.basename(urlparts.path);
7543 file_dir = os.path.dirname(urlparts.path);
7544 sftp_port = urlparts.port;
7545 if(urlparts.port is None):
7546 sftp_port = 22;
7547 else:
7548 sftp_port = urlparts.port;
7549 if(urlparts.username is not None):
7550 sftp_username = urlparts.username;
7551 else:
7552 sftp_username = "anonymous";
7553 if(urlparts.password is not None):
7554 sftp_password = urlparts.password;
7555 elif(urlparts.password is None and urlparts.username=="anonymous"):
7556 sftp_password = "anonymous";
7557 else:
7558 sftp_password = "";
7559 if(urlparts.scheme=="ftp"):
7560 return download_file_from_ftp_file(url);
7561 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7562 return download_file_from_http_file(url);
7563 if(urlparts.scheme!="sftp"):
7564 return False;
7565 try:
7566 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7567 except paramiko.ssh_exception.SSHException:
7568 return False;
7569 except socket.gaierror:
7570 log.info("Error With URL "+url);
7571 return False;
7572 except socket.timeout:
7573 log.info("Error With URL "+url);
7574 return False;
7575 sftp = ssh.open_sftp();
7576 sftpfile = BytesIO();
7577 sftp.getfo(urlparts.path, sftpfile);
7578 sftp.close();
7579 ssh.close();
7580 sftpfile.seek(0, 0);
7581 return sftpfile;
7582 else:
7583 def download_file_from_pysftp_file(url):
7584 return False;
7586 if(havepysftp):
7587 def download_file_from_pysftp_string(url):
7588 sftpfile = download_file_from_pysftp_file(url);
7589 return sftpfile.read();
7590 else:
7591 def download_file_from_ftp_string(url):
7592 return False;
7594 if(havepysftp):
7595 def upload_file_to_pysftp_file(sftpfile, url):
7596 urlparts = urlparse(url);
7597 file_name = os.path.basename(urlparts.path);
7598 file_dir = os.path.dirname(urlparts.path);
7599 sftp_port = urlparts.port;
7600 if(urlparts.port is None):
7601 sftp_port = 22;
7602 else:
7603 sftp_port = urlparts.port;
7604 if(urlparts.username is not None):
7605 sftp_username = urlparts.username;
7606 else:
7607 sftp_username = "anonymous";
7608 if(urlparts.password is not None):
7609 sftp_password = urlparts.password;
7610 elif(urlparts.password is None and urlparts.username=="anonymous"):
7611 sftp_password = "anonymous";
7612 else:
7613 sftp_password = "";
7614 if(urlparts.scheme=="ftp"):
7615 return upload_file_to_ftp_file(url);
7616 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7617 return False;
7618 if(urlparts.scheme!="sftp"):
7619 return False;
7620 try:
7621 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7622 except paramiko.ssh_exception.SSHException:
7623 return False;
7624 except socket.gaierror:
7625 log.info("Error With URL "+url);
7626 return False;
7627 except socket.timeout:
7628 log.info("Error With URL "+url);
7629 return False;
7630 sftp = ssh.open_sftp();
7631 sftp.putfo(sftpfile, urlparts.path);
7632 sftp.close();
7633 ssh.close();
7634 sftpfile.seek(0, 0);
7635 return sftpfile;
7636 else:
7637 def upload_file_to_pysftp_file(sftpfile, url):
7638 return False;
7640 if(havepysftp):
7641 def upload_file_to_pysftp_string(sftpstring, url):
7642 sftpfileo = BytesIO(sftpstring);
7643 sftpfile = upload_file_to_pysftp_files(ftpfileo, url);
7644 sftpfileo.close();
7645 return sftpfile;
7646 else:
7647 def upload_file_to_pysftp_string(url):
7648 return False;
7650 def download_file_from_internet_file(url, headers=geturls_headers_pycatfile_python_alt):
7651 urlparts = urlparse(url);
7652 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7653 return download_file_from_http_file(url, headers);
7654 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7655 return download_file_from_ftp_file(url);
7656 elif(urlparts.scheme=="sftp"):
7657 if(__use_pysftp__ and havepysftp):
7658 return download_file_from_pysftp_file(url);
7659 else:
7660 return download_file_from_sftp_file(url);
7661 else:
7662 return False;
7663 return False;
7665 def download_file_from_internet_uncompress_file(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_dict__):
7666 formatspecs = FormatSpecsListToDict(formatspecs);
7667 fp = download_file_from_internet_file(url);
7668 fp = UncompressArchiveFile(fp, formatspecs);
7669 fp.seek(0, 0);
7670 if(not fp):
7671 return False;
7672 return fp;
7674 def download_file_from_internet_string(url, headers=geturls_headers_pycatfile_python_alt):
7675 urlparts = urlparse(url);
7676 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7677 return download_file_from_http_string(url, headers);
7678 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7679 return download_file_from_ftp_string(url);
7680 elif(urlparts.scheme=="sftp"):
7681 if(__use_pysftp__ and havepysftp):
7682 return download_file_from_pysftp_string(url);
7683 else:
7684 return download_file_from_sftp_string(url);
7685 else:
7686 return False;
7687 return False;
7689 def download_file_from_internet_uncompress_string(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_dict__):
7690 formatspecs = FormatSpecsListToDict(formatspecs);
7691 fp = download_file_from_internet_string(url);
7692 fp = UncompressArchiveFile(fp, formatspecs);
7693 fp.seek(0, 0);
7694 if(not fp):
7695 return False;
7696 return fp;
7698 def upload_file_to_internet_file(ifp, url):
7699 urlparts = urlparse(url);
7700 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7701 return False;
7702 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7703 return upload_file_to_ftp_file(ifp, url);
7704 elif(urlparts.scheme=="sftp"):
7705 if(__use_pysftp__ and havepysftp):
7706 return upload_file_to_pysftp_file(ifp, url);
7707 else:
7708 return upload_file_to_sftp_file(ifp, url);
7709 else:
7710 return False;
7711 return False;
7713 def upload_file_to_internet_compress_file(ifp, url, formatspecs=__file_format_dict__):
7714 formatspecs = FormatSpecsListToDict(formatspecs);
7715 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
7716 if(not catfileout):
7717 return False;
7718 catfp.seek(0, 0);
7719 upload_file_to_internet_file(catfp, outfile);
7720 return True;
7722 def upload_file_to_internet_string(ifp, url):
7723 urlparts = urlparse(url);
7724 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7725 return False;
7726 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7727 return upload_file_to_ftp_string(ifp, url);
7728 elif(urlparts.scheme=="sftp"):
7729 if(__use_pysftp__ and havepysftp):
7730 return upload_file_to_pysftp_string(ifp, url);
7731 else:
7732 return upload_file_to_sftp_string(ifp, url);
7733 else:
7734 return False;
7735 return False;
7737 def upload_file_to_internet_compress_string(ifp, url, formatspecs=__file_format_dict__):
7738 formatspecs = FormatSpecsListToDict(formatspecs);
7739 catfp = CompressArchiveFile(BytesIO(ifp), compression, compressionlevel, formatspecs);
7740 if(not catfileout):
7741 return False;
7742 catfp.seek(0, 0);
7743 upload_file_to_internet_file(catfp, outfile);
7744 return True;
7746 try:
7747 if(hasattr(shutil, "register_archive_format")):
7748 # Register the packing format
7749 shutil.register_archive_format(__file_format_name__, PackArchiveFileFunc, description='Pack concatenated files');
7750 except shutil.RegistryError:
7751 pass;
7753 try:
7754 if(hasattr(shutil, "register_unpack_format")):
7755 # Register the unpacking format
7756 shutil.register_unpack_format(__file_format_name__, archivefile_extensions, UnPackArchiveFileFunc, description='UnPack concatenated files');
7757 except shutil.RegistryError:
7758 pass;