?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Add files via upload
[PyCatFile.git] / pycatfile.py
blobe2e925c47e7c87bc95659fcb3fdc74488e6ac357
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 '''
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the Revised BSD License.
7
8 This program is distributed in the hope that it will be useful,
9 but WITHOUT ANY WARRANTY; without even the implied warranty of
10 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 Revised BSD License for more details.
12
13 Copyright 2018-2024 Cool Dude 2k - http://idb.berlios.de/
14 Copyright 2018-2024 Game Maker 2k - http://intdb.sourceforge.net/
15 Copyright 2018-2024 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
16
17 $FileInfo: pycatfile.py - Last Update: 6/2/2024 Ver. 0.12.4 RC 1 - Author: cooldude2k $
18 '''
19
20 from __future__ import absolute_import, division, print_function, unicode_literals;
21 import io, os, re, sys, time, stat, zlib, base64, shutil, socket, hashlib, datetime, logging, binascii, zipfile, platform;
22 try:
23 from backports import tempfile;
24 except ImportError:
25 import tempfile;
26 # FTP Support
27 ftpssl = True;
28 try:
29 from ftplib import FTP, FTP_TLS;
30 except ImportError:
31 ftpssl = False;
32 from ftplib import FTP;
33
34 try:
35 import simplejson as json;
36 except ImportError:
37 import json;
38
39 # URL Parsing
40 try:
41 from urllib.parse import urlparse, urlunparse;
42 except ImportError:
43 from urlparse import urlparse, urlunparse;
44
45 # Windows-specific setup
46 if os.name == 'nt':
47 if sys.version_info[0] == 2:
48 import codecs;
49 sys.stdout = codecs.getwriter('utf-8')(sys.stdout);
50 sys.stderr = codecs.getwriter('utf-8')(sys.stderr);
51 else:
52 sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace', line_buffering=True);
53 sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', errors='replace', line_buffering=True);
54
55 hashlib_guaranteed = False;
56 # Environment setup
57 os.environ["PYTHONIOENCODING"] = "UTF-8";
58 os.environ["LC_CTYPE"] = "UTF-8";
59
60 # Reload sys to set default encoding to UTF-8 (Python 2 only)
61 if sys.version_info[0] == 2:
62 try:
63 reload(sys);
64 sys.setdefaultencoding('UTF-8');
65 except (NameError, AttributeError):
66 pass;
67
68 # CRC32 import
69 try:
70 from zlib import crc32;
71 except ImportError:
72 from binascii import crc32;
73
74 # Define FileNotFoundError for Python 2
75 try:
76 FileNotFoundError;
77 except NameError:
78 FileNotFoundError = IOError;
79
80 # RAR file support
81 rarfile_support = False;
82 try:
83 import rarfile;
84 rarfile_support = True;
85 except ImportError:
86 pass;
87
88 # 7z file support
89 py7zr_support = False;
90 try:
91 import py7zr;
92 py7zr_support = True;
93 except ImportError:
94 pass;
95
96 # TAR file checking
97 try:
98 from xtarfile import is_tarfile;
99 except ImportError:
100 try:
101 from safetar import is_tarfile;
102 except ImportError:
103 from tarfile import is_tarfile;
104
105 # TAR file module
106 try:
107 import xtarfile as tarfile;
108 except ImportError:
109 try:
110 import safetar as tarfile;
111 except ImportError:
112 import tarfile;
113
114 # Paramiko support
115 haveparamiko = False;
116 try:
117 import paramiko;
118 haveparamiko = True;
119 except ImportError:
120 pass;
121
122 # PySFTP support
123 havepysftp = False;
124 try:
125 import pysftp;
126 havepysftp = True;
127 except ImportError:
128 pass;
129
130 # Requests support
131 haverequests = False;
132 try:
133 import requests;
134 haverequests = True;
135 import urllib3;
136 logging.getLogger("urllib3").setLevel(logging.WARNING);
137 except ImportError:
138 pass;
139
140 # HTTP and URL parsing
141 try:
142 from urllib.request import Request, build_opener, HTTPBasicAuthHandler;
143 from urllib.parse import urlparse;
144 except ImportError:
145 from urllib2 import Request, build_opener, HTTPBasicAuthHandler;
146 from urlparse import urlparse;
147
148 # StringIO and BytesIO
149 try:
150 from io import StringIO, BytesIO;
151 except ImportError:
152 try:
153 from cStringIO import StringIO;
154 from cStringIO import StringIO as BytesIO;
155 except ImportError:
156 from StringIO import StringIO;
157 from StringIO import StringIO as BytesIO;
158
159 __use_pysftp__ = False;
160 if(not havepysftp):
161 __use_pysftp__ = False;
162 __file_format_name__ = "CatFile";
163 __program_name__ = "Py"+__file_format_name__;
164 __file_format_lower__ = __file_format_name__.lower();
165 __file_format_magic__ = __file_format_name__;
166 __file_format_len__ = len(__file_format_magic__);
167 __file_format_hex__ = binascii.hexlify(__file_format_magic__.encode("UTF-8")).decode("UTF-8");
168 __file_format_delimiter__ = "\x00";
169 __file_format_ver__ = "001";
170 __use_new_style__ = True;
171 __use_advanced_list__ = True;
172 __use_alt_inode__ = False;
173 __file_format_extension__ = ".cat";
174 '''
175 __file_format_name__ = "FastArchive";
176 __program_name__ = "Py" + __file_format_name__;
177 __file_format_lower__ = __file_format_name__.lower();
178 __file_format_magic__ = "FstArch";
179 __file_format_len__ = len(__file_format_magic__);
180 __file_format_hex__ = binascii.hexlify(__file_format_magic__.encode("UTF-8")).decode("UTF-8");
181 __file_format_delimiter__ = "\x1F"; # Using a non-printable ASCII character as delimiter
182 __file_format_ver__ = "001";
183 __use_new_style__ = True;
184 __use_advanced_list__ = False;
185 __use_alt_inode__ = False;
186 __file_format_extension__ = ".fast";
187 '''
188 __file_format_list__ = [__file_format_name__, __file_format_magic__, __file_format_lower__, __file_format_len__, __file_format_hex__, __file_format_delimiter__, __file_format_ver__, __use_new_style__, __use_advanced_list__, __use_alt_inode__];
189 __file_format_dict__ = {'format_name': __file_format_name__, 'format_magic': __file_format_magic__, 'format_lower': __file_format_lower__, 'format_len': __file_format_len__, 'format_hex': __file_format_hex__, 'format_delimiter': __file_format_delimiter__, 'format_ver': __file_format_ver__, 'new_style': __use_new_style__, 'use_advanced_list': __use_advanced_list__, 'use_alt_inode': __use_alt_inode__};
190 __project__ = __program_name__;
191 __project_url__ = "https://github.com/GameMaker2k/PyCatFile";
192 __version_info__ = (0, 12, 4, "RC 1", 1);
193 __version_date_info__ = (2024, 6, 2, "RC 1", 1);
194 __version_date__ = str(__version_date_info__[0]) + "." + str(__version_date_info__[1]).zfill(2) + "." + str(__version_date_info__[2]).zfill(2);
195 __revision__ = __version_info__[3];
196 __revision_id__ = "$Id$";
197 if(__version_info__[4] is not None):
198 __version_date_plusrc__ = __version_date__ + "-" + str(__version_date_info__[4]);
199 if(__version_info__[4] is None):
200 __version_date_plusrc__ = __version_date__;
201 if(__version_info__[3] is not None):
202 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]) + " " + str(__version_info__[3]);
203 if(__version_info__[3] is None):
204 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]);
205
206 PyBitness = platform.architecture();
207 if(PyBitness=="32bit" or PyBitness=="32"):
208 PyBitness = "32";
209 elif(PyBitness=="64bit" or PyBitness=="64"):
210 PyBitness = "64";
211 else:
212 PyBitness = "32";
213
214 geturls_ua_pycatfile_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
215 if(platform.python_implementation()!=""):
216 py_implementation = platform.python_implementation();
217 if(platform.python_implementation()==""):
218 py_implementation = "Python";
219 geturls_ua_pycatfile_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=py_implementation, pyver=platform.python_version(), proname=__project__, prover=__version__);
220 geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
221 geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
222 geturls_headers_pycatfile_python = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
223 geturls_headers_pycatfile_python_alt = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python_alt, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
224 geturls_headers_googlebot_google = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
225 geturls_headers_googlebot_google_old = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
226
227 compressionsupport = [];
228 try:
229 import gzip;
230 compressionsupport.append("gz");
231 compressionsupport.append("gzip");
232 except ImportError:
233 pass;
234 try:
235 import bz2;
236 compressionsupport.append("bz2");
237 compressionsupport.append("bzip2");
238 except ImportError:
239 pass;
240 try:
241 import lz4;
242 compressionsupport.append("lz4");
243 except ImportError:
244 pass;
245 try:
246 import lzo;
247 compressionsupport.append("lzo");
248 compressionsupport.append("lzop");
249 except ImportError:
250 pass;
251 try:
252 import zstandard;
253 compressionsupport.append("zstd");
254 compressionsupport.append("zstandard");
255 except ImportError:
256 pass;
257 try:
258 import lzma;
259 compressionsupport.append("lzma");
260 compressionsupport.append("xz");
261 except ImportError:
262 try:
263 from backports import lzma;
264 compressionsupport.append("lzma");
265 compressionsupport.append("xz");
266 except ImportError:
267 pass;
268
269 compressionlist = ['auto'];
270 compressionlistalt = [];
271 outextlist = [];
272 outextlistwd = [];
273 if('gzip' in compressionsupport):
274 compressionlist.append('gzip');
275 compressionlistalt.append('gzip');
276 outextlist.append('gz');
277 outextlistwd.append('.gz');
278 if('bzip2' in compressionsupport):
279 compressionlist.append('bzip2');
280 compressionlistalt.append('bzip2');
281 outextlist.append('bz2');
282 outextlistwd.append('.bz2');
283 if('zstd' in compressionsupport):
284 compressionlist.append('zstd');
285 compressionlistalt.append('zstd');
286 outextlist.append('zst');
287 outextlistwd.append('.zst');
288 if('lz4' in compressionsupport):
289 compressionlist.append('lz4');
290 compressionlistalt.append('lz4');
291 outextlist.append('lz4');
292 outextlistwd.append('.lz4');
293 if('lzo' in compressionsupport):
294 compressionlist.append('lzo');
295 compressionlistalt.append('lzo');
296 outextlist.append('lzo');
297 outextlistwd.append('.lzo');
298 if('lzop' in compressionsupport):
299 compressionlist.append('lzop');
300 compressionlistalt.append('lzop');
301 outextlist.append('lzop');
302 outextlistwd.append('.lzop');
303 if('lzma' in compressionsupport):
304 compressionlist.append('lzma');
305 compressionlistalt.append('lzma');
306 outextlist.append('lzma');
307 outextlistwd.append('.lzma');
308 if('xz' in compressionsupport):
309 compressionlist.append('xz');
310 compressionlistalt.append('xz');
311 outextlist.append('xz');
312 outextlistwd.append('.xz');
313
314 tarfile_mimetype = "application/tar";
315 tarfile_tar_mimetype = tarfile_mimetype;
316 zipfile_mimetype = "application/zip";
317 zipfile_zip_mimetype = zipfile_mimetype;
318 rarfile_mimetype = "application/rar";
319 rarfile_rar_mimetype = rarfile_mimetype;
320 archivefile_mimetype = "application/x-"+__file_format_dict__['format_lower']+"";
321 archivefile_cat_mimetype = archivefile_mimetype;
322 archivefile_gzip_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+gzip";
323 archivefile_gz_mimetype = archivefile_gzip_mimetype;
324 archivefile_bzip2_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+bzip2";
325 archivefile_bz2_mimetype = archivefile_bzip2_mimetype;
326 archivefile_lz4_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lz4";
327 archivefile_lzop_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lzop";
328 archivefile_lzo_mimetype = archivefile_lzop_mimetype;
329 archivefile_zstandard_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+zstandard";
330 archivefile_zstd_mimetype = archivefile_zstandard_mimetype;
331 archivefile_lzma_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lzma";
332 archivefile_xz_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+xz";
333 archivefile_extensions = [__file_format_extension__, __file_format_extension__+".gz", __file_format_extension__+".bz2", __file_format_extension__+".zst", __file_format_extension__+".lz4", __file_format_extension__+".lzo", __file_format_extension__+".lzop", __file_format_extension__+".lzma", __file_format_extension__+".xz"];
334
335 if __name__ == "__main__":
336 import subprocess;
337 curscrpath = os.path.dirname(sys.argv[0]);
338 if(curscrpath==""):
339 curscrpath = ".";
340 if(os.sep=="\\"):
341 curscrpath = curscrpath.replace(os.sep, "/");
342 curscrpath = curscrpath + "/";
343 scrfile = curscrpath + "catfile.py";
344 if(os.path.exists(scrfile) and os.path.isfile(scrfile)):
345 scrcmd = subprocess.Popen([sys.executable, scrfile] + sys.argv[1:]);
346 scrcmd.wait();
347
348 def VerbosePrintOut(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
349 if(not dbgenable):
350 return True;
351 log_functions = {
352 "print": print,
353 "log": logging.info,
354 "warning": logging.warning,
355 "error": logging.error,
356 "critical": logging.critical,
357 "exception": logging.exception,
358 "logalt": lambda x: logging.log(dgblevel, x),
359 "debug": logging.debug
360 };
361 log_function = log_functions.get(outtype);
362 if(log_function):
363 log_function(dbgtxt);
364 return True;
365 return False;
366
367 def VerbosePrintOutReturn(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
368 VerbosePrintOut(dbgtxt, outtype, dbgenable, dgblevel);
369 return dbgtxt;
370
371 def RemoveWindowsPath(dpath):
372 if(dpath is None):
373 dpath = "";
374 if(os.sep!="/"):
375 dpath = dpath.replace(os.path.sep, "/");
376 dpath = dpath.rstrip("/");
377 if(dpath=="." or dpath==".."):
378 dpath = dpath + "/";
379 return dpath;
380
381 def NormalizeRelativePath(inpath):
382 inpath = RemoveWindowsPath(inpath);
383 if(os.path.isabs(inpath)):
384 outpath = inpath;
385 else:
386 if(inpath.startswith("./") or inpath.startswith("../")):
387 outpath = inpath;
388 else:
389 outpath = "./" + inpath;
390 return outpath;
391
392 def PrependPath(base_dir, child_path):
393 # Check if base_dir is None or empty, if so, return child_path as is
394 if not base_dir:
395 return child_path;
396 # Ensure base_dir ends with exactly one slash
397 if not base_dir.endswith('/'):
398 base_dir += '/';
399 # Check if child_path starts with ./ or ../ (indicating a relative path)
400 if child_path.startswith('./') or child_path.startswith('../'):
401 # For relative paths, we don't alter the child_path
402 return base_dir + child_path;
403 else:
404 # For non-relative paths, ensure there's no starting slash on child_path to avoid double slashes
405 return base_dir + child_path.lstrip('/');
406
407 def ListDir(dirpath, followlink=False, duplicates=False):
408 if isinstance(dirpath, (list, tuple, )):
409 dirpath = list(filter(None, dirpath));
410 elif isinstance(dirpath, str):
411 dirpath = list(filter(None, [dirpath]));
412 retlist = [];
413 fs_encoding = sys.getfilesystemencoding();
414 for mydirfile in dirpath:
415 if not os.path.exists(mydirfile):
416 return False;
417 mydirfile = NormalizeRelativePath(mydirfile);
418 if os.path.exists(mydirfile) and os.path.islink(mydirfile):
419 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile));
420 if os.path.exists(mydirfile) and os.path.isdir(mydirfile):
421 for root, dirs, filenames in os.walk(mydirfile):
422 dpath = root;
423 dpath = RemoveWindowsPath(dpath);
424 if fs_encoding != 'utf-8':
425 dpath = dpath.encode(fs_encoding).decode('utf-8');
426 if dpath not in retlist and not duplicates:
427 retlist.append(dpath);
428 if duplicates:
429 retlist.append(dpath);
430 for file in filenames:
431 fpath = os.path.join(root, file);
432 fpath = RemoveWindowsPath(fpath);
433 if fs_encoding != 'utf-8':
434 fpath = fpath.encode(fs_encoding).decode('utf-8');
435 if fpath not in retlist and not duplicates:
436 retlist.append(fpath);
437 if duplicates:
438 retlist.append(fpath);
439 else:
440 path = RemoveWindowsPath(mydirfile);
441 if fs_encoding != 'utf-8':
442 path = path.encode(fs_encoding).decode('utf-8');
443 retlist.append(path);
444 return retlist;
445
446 def ListDirAdvanced(dirpath, followlink=False, duplicates=False):
447 if isinstance(dirpath, (list, tuple, )):
448 dirpath = list(filter(None, dirpath));
449 elif isinstance(dirpath, str):
450 dirpath = list(filter(None, [dirpath]));
451 retlist = [];
452 fs_encoding = sys.getfilesystemencoding();
453 for mydirfile in dirpath:
454 if not os.path.exists(mydirfile):
455 return False;
456 mydirfile = NormalizeRelativePath(mydirfile);
457 if os.path.exists(mydirfile) and os.path.islink(mydirfile) and followlink:
458 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile));
459 if os.path.exists(mydirfile) and os.path.isdir(mydirfile):
460 for root, dirs, filenames in os.walk(mydirfile):
461 # Sort dirs and filenames alphabetically in place
462 dirs.sort(key=lambda x: x.lower());
463 filenames.sort(key=lambda x: x.lower());
464 dpath = RemoveWindowsPath(root);
465 if fs_encoding != 'utf-8':
466 dpath = dpath.encode(fs_encoding).decode('utf-8');
467 if not duplicates and dpath not in retlist:
468 retlist.append(dpath);
469 elif duplicates:
470 retlist.append(dpath);
471 for file in filenames:
472 fpath = os.path.join(root, file);
473 fpath = RemoveWindowsPath(fpath);
474 if fs_encoding != 'utf-8':
475 fpath = fpath.encode(fs_encoding).decode('utf-8');
476 if not duplicates and fpath not in retlist:
477 retlist.append(fpath);
478 elif duplicates:
479 retlist.append(fpath);
480 else:
481 path = RemoveWindowsPath(mydirfile);
482 if fs_encoding != 'utf-8':
483 path = path.encode(fs_encoding).decode('utf-8');
484 retlist.append(path);
485 return retlist;
486
487 def create_alias_function(prefix, base_name, suffix, target_function):
488 # Define a new function that wraps the target function
489 def alias_function(*args, **kwargs):
490 return target_function(*args, **kwargs);
491 # Create the function name by combining the prefix, base name, and the suffix
492 function_name = "{}{}{}".format(prefix, base_name, suffix);
493 # Add the new function to the global namespace
494 globals()[function_name] = alias_function;
495
496 def create_alias_function_alt(prefix, base_name, suffix, target_function):
497 # Create the function name by combining the prefix, base name, and the suffix
498 # Use the format method for string formatting, compatible with Python 2 and 3
499 function_name = "{}{}{}".format(prefix, base_name, suffix)
500 # Add the new function (alias of the target_function) to the global namespace
501 # This line is compatible as-is with both Python 2 and 3
502 globals()[function_name] = target_function
503
504 def FormatSpecsListToDict(formatspecs=__file_format_list__):
505 if(isinstance(formatspecs, (list, tuple, ))):
506 return {'format_name': formatspecs[0], 'format_magic': formatspecs[1], 'format_lower': formatspecs[2], 'format_len': formatspecs[3], 'format_hex': formatspecs[4], 'format_delimiter': formatspecs[5], 'format_ver': formatspecs[6], 'new_style': formatspecs[7], 'use_advanced_list': formatspecs[8], 'use_alt_inode': formatspecs[9]};
507 elif(isinstance(formatspecs, (dict, ))):
508 return formatspecs;
509 else:
510 return __file_format_dict__;
511 return __file_format_dict__;
512
513 def TarFileCheck(infile):
514 try:
515 if is_tarfile(infile):
516 return True;
517 else:
518 return False;
519 except TypeError:
520 try:
521 # Check if the input is a file object
522 if hasattr(infile, 'read'):
523 # Save the current file position
524 current_position = infile.tell();
525 # Attempt to open the file object as a tar file
526 tar = tarfile.open(fileobj=infile);
527 tar.close();
528 # Restore the file position
529 infile.seek(current_position);
530 return True;
531 else:
532 # Assume it's a filename
533 tar = tarfile.open(name=infile);
534 tar.close();
535 return True;
536 except tarfile.TarError:
537 return False;
538
539 # initial_value can be 0xFFFF or 0x0000
540 def crc16_ansi(msg, initial_value=0xFFFF):
541 # CRC-16-IBM / CRC-16-ANSI polynomial and initial value
542 poly = 0x8005; # Polynomial for CRC-16-IBM / CRC-16-ANSI
543 crc = initial_value; # Initial value
544 for b in msg:
545 crc ^= b << 8; # XOR byte into CRC top byte
546 for _ in range(8): # Process each bit
547 if crc & 0x8000: # If the top bit is set
548 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
549 else:
550 crc = crc << 1; # Just shift left
551 crc &= 0xFFFF; # Ensure CRC remains 16-bit
552 return crc;
553
554 # initial_value can be 0xFFFF or 0x0000
555 def crc16_ibm(msg, initial_value=0xFFFF):
556 return crc16_ansi(msg, initial_value);
557
558 # initial_value is 0xFFFF
559 def crc16(msg):
560 return crc16_ansi(msg, 0xFFFF);
561
562 # initial_value can be 0xFFFF, 0x1D0F or 0x0000
563 def crc16_ccitt(msg, initial_value=0xFFFF):
564 # CRC-16-CCITT polynomial
565 poly = 0x1021; # Polynomial for CRC-16-CCITT
566 # Use the specified initial value
567 crc = initial_value;
568 for b in msg:
569 crc ^= b << 8; # XOR byte into CRC top byte
570 for _ in range(8): # Process each bit
571 if crc & 0x8000: # If the top bit is set
572 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
573 else:
574 crc = crc << 1; # Just shift left
575 crc &= 0xFFFF; # Ensure CRC remains 16-bit
576 return crc;
577
578 # initial_value can be 0x42F0E1EBA9EA3693 or 0x0000000000000000
579 def crc64_ecma(msg, initial_value=0x0000000000000000):
580 # CRC-64-ECMA polynomial and initial value
581 poly = 0x42F0E1EBA9EA3693;
582 crc = initial_value; # Initial value for CRC-64-ECMA
583 for b in msg:
584 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
585 for _ in range(8): # Process each bit
586 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
587 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
588 else:
589 crc <<= 1; # Just shift left if the MSB is 0
590 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
591 return crc;
592
593 # initial_value can be 0x000000000000001B or 0xFFFFFFFFFFFFFFFF
594 def crc64_iso(msg, initial_value=0xFFFFFFFFFFFFFFFF):
595 # CRC-64-ISO polynomial and initial value
596 poly = 0x000000000000001B;
597 crc = initial_value; # Common initial value for CRC-64-ISO
598 for b in msg:
599 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
600 for _ in range(8): # Process each bit
601 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
602 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
603 else:
604 crc <<= 1; # Just shift left if the MSB is 0
605 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
606 return crc;
607
608 def GetDataFromArray(data, path, default=None):
609 element = data;
610 try:
611 for key in path:
612 element = element[key];
613 return element;
614 except (KeyError, TypeError, IndexError):
615 return default;
616
617 def GetDataFromArrayAlt(structure, path, default=None):
618 element = structure;
619 for key in path:
620 if isinstance(element, dict) and key in element:
621 element = element[key];
622 elif isinstance(element, list) and isinstance(key, int) and -len(element) <= key < len(element):
623 element = element[key];
624 else:
625 return default;
626 return element;
627
628 def GetHeaderChecksum(inlist=[], checksumtype="crc32", encodedata=True, formatspecs=__file_format_dict__):
629 formatspecs = FormatSpecsListToDict(formatspecs);
630 fileheader = AppendNullBytes(inlist, formatspecs['format_delimiter']) if isinstance(inlist, list) else AppendNullByte(inlist, formatspecs['format_delimiter']);
631 if encodedata:
632 fileheader = fileheader.encode('UTF-8');
633 checksum_methods = {
634 "crc16": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
635 "crc16_ansi": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
636 "crc16_ibm": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
637 "crc16_ccitt": lambda data: format(crc16_ccitt(data) & 0xffff, '04x').lower(),
638 "adler32": lambda data: format(zlib.adler32(data) & 0xffffffff, '08x').lower(),
639 "crc32": lambda data: format(crc32(data) & 0xffffffff, '08x').lower(),
640 "crc64_ecma": lambda data: format(crc64_ecma(data) & 0xffffffffffffffff, '016x').lower(),
641 "crc64": lambda data: format(crc64_iso(data) & 0xffffffffffffffff, '016x').lower(),
642 "crc64_iso": lambda data: format(crc64_iso(data) & 0xffffffffffffffff, '016x').lower(),
643 };
644 if checksumtype in checksum_methods:
645 return checksum_methods[checksumtype](fileheader);
646 elif CheckSumSupportAlt(checksumtype, hashlib_guaranteed):
647 checksumoutstr = hashlib.new(checksumtype);
648 checksumoutstr.update(fileheader);
649 return checksumoutstr.hexdigest().lower();
650 return format(0, 'x').lower();
651
652 def GetFileChecksum(instr, checksumtype="crc32", encodedata=True, formatspecs=__file_format_dict__):
653 formatspecs = FormatSpecsListToDict(formatspecs);
654 if encodedata:
655 instr = instr.encode('UTF-8');
656 checksum_methods = {
657 "crc16": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
658 "crc16_ansi": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
659 "crc16_ibm": lambda data: format(crc16(data) & 0xffff, '04x').lower(),
660 "crc16_ccitt": lambda data: format(crc16_ccitt(data) & 0xffff, '04x').lower(),
661 "adler32": lambda data: format(zlib.adler32(data) & 0xffffffff, '08x').lower(),
662 "crc32": lambda data: format(crc32(data) & 0xffffffff, '08x').lower(),
663 "crc64_ecma": lambda data: format(crc64_ecma(data) & 0xffffffffffffffff, '016x').lower(),
664 "crc64": lambda data: format(crc64_iso(data) & 0xffffffffffffffff, '016x').lower(),
665 "crc64_iso": lambda data: format(crc64_iso(data) & 0xffffffffffffffff, '016x').lower(),
666 };
667 if checksumtype in checksum_methods:
668 return checksum_methods[checksumtype](instr);
669 elif CheckSumSupportAlt(checksumtype, hashlib_guaranteed):
670 checksumoutstr = hashlib.new(checksumtype);
671 checksumoutstr.update(instr);
672 return checksumoutstr.hexdigest().lower();
673 return format(0, 'x').lower();
674
675 def ValidateHeaderChecksum(inlist=[], checksumtype="crc32", inchecksum="0", formatspecs=__file_format_dict__):
676 formatspecs = FormatSpecsListToDict(formatspecs);
677 catfileheadercshex = GetHeaderChecksum(inlist, checksumtype, True, formatspecs).lower();
678 return inchecksum.lower() == catfileheadercshex;
679
680 def ValidateFileChecksum(infile, checksumtype="crc32", inchecksum="0", formatspecs=__file_format_dict__):
681 formatspecs = FormatSpecsListToDict(formatspecs);
682 catinfilecshex = GetFileChecksum(infile, checksumtype, True, formatspecs).lower();
683 return inchecksum.lower() == catinfilecshex;
684
685 def ReadTillNullByteOld(fp, delimiter=__file_format_dict__['format_delimiter']):
686 curbyte = b"";
687 curfullbyte = b"";
688 nullbyte = delimiter.encode("UTF-8");
689 while(True):
690 curbyte = fp.read(1);
691 if(curbyte==nullbyte or not curbyte):
692 break;
693 curfullbyte = curfullbyte + curbyte;
694 return curfullbyte.decode('UTF-8');
695
696 def ReadUntilNullByteOld(fp, delimiter=__file_format_dict__['format_delimiter']):
697 return ReadTillNullByteOld(fp, delimiter);
698
699 def ReadTillNullByteAlt(fp, delimiter=__file_format_dict__['format_delimiter'], chunk_size=1024, max_read=1024000):
700 delimiter = delimiter.encode('UTF-8') # Ensure the delimiter is in bytes
701 buffer = bytearray();
702 total_read = 0;
703 delimiter_length = len(delimiter);
704 while True:
705 chunk = fp.read(chunk_size)
706 if not chunk:
707 # End of file reached without finding the delimiter
708 break;
709 buffer.extend(chunk);
710 total_read += len(chunk);
711 if delimiter in buffer:
712 # Delimiter found, calculate where to reset the file pointer
713 index = buffer.find(delimiter);
714 # Calculate how many extra bytes were read after the delimiter
715 extra_bytes_read = len(buffer) - (index + delimiter_length);
716 # Move the file pointer back to just after the delimiter
717 fp.seek(-extra_bytes_read, 1);
718 buffer = buffer[:index];
719 break;
720 if total_read >= max_read:
721 # Stop reading if max limit is reached to prevent excessive memory usage
722 raise MemoryError("Maximum read limit reached without finding the delimiter.");
723 # Check for incomplete UTF-8 sequences at the end of the buffer
724 if len(buffer) > 1 and 128 <= buffer[-1] < 192:
725 # This suggests that the last byte might be the start of a multi-byte character
726 # Try to read one more byte to complete the character
727 extra_byte = fp.read(1);
728 if extra_byte:
729 buffer.extend(extra_byte);
730 else:
731 # No more data available
732 break;
733 try:
734 return buffer.decode('UTF-8', errors='replace');
735 except UnicodeDecodeError:
736 return buffer.decode('UTF-8', errors='replace');
737
738 def ReadUntilNullByteAlt(fp, delimiter=__file_format_dict__['format_delimiter'], chunk_size=1024, max_read=1024000):
739 return ReadTillNullByteAlt(fp, delimiter, chunk_size, max_read);
740
741 def ReadTillNullByte(fp, delimiter=__file_format_dict__['format_delimiter'], max_read=1024000):
742 curfullbyte = bytearray();
743 nullbyte = delimiter.encode("UTF-8");
744 total_read = 0; # Track the total number of bytes read
745 while True:
746 curbyte = fp.read(1);
747 if curbyte == nullbyte or not curbyte:
748 break;
749 curfullbyte.extend(curbyte);
750 total_read += 1;
751 if total_read >= max_read:
752 raise MemoryError("Maximum read limit reached without finding the delimiter.");
753 # Decode the full byte array to string once out of the loop
754 try:
755 return curfullbyte.decode('UTF-8');
756 except UnicodeDecodeError:
757 # Handle potential partial UTF-8 characters
758 for i in range(1, 4):
759 try:
760 return curfullbyte[:-i].decode('UTF-8');
761 except UnicodeDecodeError:
762 continue;
763 raise; # Re-raise if decoding fails even after trimming
764
765 def ReadUntilNullByte(fp, delimiter=__file_format_dict__['format_delimiter'], max_read=1024000):
766 return ReadTillNullByte(fp, delimiter, max_read);
767
768 def ReadTillNullByteByNum(fp, delimiter=__file_format_dict__['format_delimiter'], num_delimiters=1, chunk_size=1024, max_read=1024000):
769 delimiter = delimiter.encode('UTF-8'); # Ensure the delimiter is in bytes
770 buffer = bytearray();
771 total_read = 0;
772 delimiter_length = len(delimiter);
773 results = [];
774 while len(results) < num_delimiters:
775 chunk = fp.read(chunk_size);
776 if not chunk:
777 # End of file reached; decode whatever is collected if it's the last needed part
778 if len(buffer) > 0:
779 results.append(buffer.decode('UTF-8', errors='replace'));
780 break;
781 buffer.extend(chunk)
782 total_read += len(chunk);
783 # Check if we have found the delimiter
784 while delimiter in buffer:
785 index = buffer.find(delimiter);
786 # Decode the section before the delimiter
787 results.append(buffer[:index].decode('UTF-8', errors='replace'));
788 # Remove the processed part from the buffer
789 buffer = buffer[index + delimiter_length:];
790 if len(results) == num_delimiters:
791 # If reached the required number of delimiters, adjust the file pointer and stop
792 fp.seek(-len(buffer), 1);
793 return results;
794 if total_read >= max_read:
795 # Stop reading if max limit is reached to prevent excessive memory usage
796 raise MemoryError("Maximum read limit reached without finding the delimiter.");
797 # Check for incomplete UTF-8 sequences at the end of the buffer
798 if len(buffer) > 1 and 128 <= buffer[-1] < 192:
799 # This suggests that the last byte might be the start of a multi-byte character
800 # Try to read one more byte to complete the character
801 extra_byte = fp.read(1);
802 if extra_byte:
803 buffer.extend(extra_byte);
804 else:
805 # No more data available
806 break;
807 # Process remaining buffer if less than the required number of delimiters were found
808 if len(buffer) > 0 and len(results) < num_delimiters:
809 results.append(buffer.decode('UTF-8', errors='replace'));
810 return results;
811
812 def ReadUntilNullByteByNum(fp, delimiter=__file_format_dict__['format_delimiter'], num_delimiters=1, chunk_size=1024, max_read=1024000):
813 return ReadTillNullByteByNum(fp, delimiter, num_delimiters, chunk_size, max_read);
814
815 def SeekToEndOfFile(fp):
816 lasttell = 0;
817 while(True):
818 fp.seek(1, 1);
819 if(lasttell==fp.tell()):
820 break;
821 lasttell = fp.tell();
822 return True;
823
824 def ReadFileHeaderData(fp, rounds=0, delimiter=__file_format_dict__['format_delimiter']):
825 rocount = 0;
826 roend = int(rounds);
827 HeaderOut = [];
828 while(rocount<roend):
829 HeaderOut.append(ReadTillNullByte(fp, delimiter));
830 rocount = rocount + 1;
831 return HeaderOut;
832
833 def ReadFileHeaderDataBySize(fp, delimiter=__file_format_dict__['format_delimiter']):
834 headerpresize = ReadTillNullByte(fp, delimiter);
835 headersize = int(headerpresize, 16);
836 if(headersize<=0):
837 return [];
838 headercontent = str(fp.read(headersize).decode('UTF-8')).split(delimiter);
839 fp.seek(1, 1);
840 rocount = 0;
841 roend = int(len(headercontent));
842 HeaderOut = [headerpresize];
843 while(rocount<roend):
844 HeaderOut.append(headercontent[rocount]);
845 rocount = rocount + 1;
846 return HeaderOut;
847
848 def ReadFileHeaderDataWoSize(fp, delimiter=__file_format_dict__['format_delimiter']):
849 preheaderdata = ReadFileHeaderData(fp, 2, delimiter);
850 headersize = int(preheaderdata[0], 16);
851 headernumfields = int(preheaderdata[1], 16);
852 if(headersize<=0 or headernumfields<=0):
853 return [];
854 headerdata = ReadTillNullByteByNum(fp, delimiter, headernumfields);
855 #headerdata = ReadFileHeaderData(fp, headernumfields, delimiter);
856 HeaderOut = preheaderdata + headerdata;
857 return HeaderOut;
858
859 def ReadFileHeaderDataBySizeWithContent(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
860 formatspecs = FormatSpecsListToDict(formatspecs);
861 delimiter = formatspecs['format_delimiter'];
862 fheaderstart = fp.tell();
863 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
864 if(len(HeaderOut)==0):
865 return False;
866 if(re.findall(r"^[.|/]", HeaderOut[3])):
867 fname = HeaderOut[3];
868 else:
869 fname = "./"+HeaderOut[3];
870 fcs = HeaderOut[-2].lower();
871 fccs = HeaderOut[-1].lower();
872 fsize = int(HeaderOut[5], 16);
873 fcompression = HeaderOut[12];
874 fcsize = int(HeaderOut[13], 16);
875 fseeknextfile = HeaderOut[25];
876 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-4].lower(), True, formatspecs);
877 if(fcs!=newfcs and not skipchecksum):
878 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
879 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
880 return False;
881 fhend = fp.tell() - 1;
882 fcontentstart = fp.tell();
883 fcontents = BytesIO();
884 if(fsize>0 and not listonly):
885 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
886 fcontents.write(fp.read(fsize));
887 else:
888 fcontents.write(fp.read(fcsize));
889 elif(fsize>0 and listonly):
890 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
891 fp.seek(fsize, 1);
892 else:
893 fp.seek(fcsize, 1);
894 fcontents.seek(0, 0);
895 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
896 if(fccs!=newfccs and not skipchecksum and not listonly):
897 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
898 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
899 return False;
900 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
901 pass;
902 else:
903 fcontents.seek(0, 0);
904 if(uncompress):
905 fcontents = UncompressArchiveFile(fcontents, formatspecs);
906 fcontentend = fp.tell();
907 if(re.findall(r"^\+([0-9]+)", fseeknextfile)):
908 fseeknextasnum = int(fseeknextfile.replace("+", ""));
909 if(abs(fseeknextasnum)==0):
910 pass;
911 fp.seek(fseeknextasnum, 1);
912 elif(re.findall(r"^\-([0-9]+)", fseeknextfile)):
913 fseeknextasnum = int(fseeknextfile);
914 if(abs(fseeknextasnum)==0):
915 pass;
916 fp.seek(fseeknextasnum, 1);
917 elif(re.findall(r"^([0-9]+)", fseeknextfile)):
918 fseeknextasnum = int(fseeknextfile);
919 if(abs(fseeknextasnum)==0):
920 pass;
921 fp.seek(fseeknextasnum, 0);
922 else:
923 return False;
924 HeaderOut.append(fcontents);
925 return HeaderOut;
926
927 def ReadFileHeaderDataBySizeWithContentToArray(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
928 formatspecs = FormatSpecsListToDict(formatspecs);
929 delimiter = formatspecs['format_delimiter'];
930 fheaderstart = fp.tell();
931 if(formatspecs['new_style']):
932 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
933 else:
934 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
935 if(len(HeaderOut)==0):
936 return False;
937 fheadsize = int(HeaderOut[0], 16);
938 fnumfields = int(HeaderOut[1], 16);
939 ftype = int(HeaderOut[2], 16);
940 if(re.findall(r"^[.|/]", HeaderOut[3])):
941 fname = HeaderOut[3];
942 else:
943 fname = "./"+HeaderOut[3];
944 fbasedir = os.path.dirname(fname);
945 flinkname = HeaderOut[4];
946 fsize = int(HeaderOut[5], 16);
947 fatime = int(HeaderOut[6], 16);
948 fmtime = int(HeaderOut[7], 16);
949 fctime = int(HeaderOut[8], 16);
950 fbtime = int(HeaderOut[9], 16);
951 fmode = int(HeaderOut[10], 16);
952 fchmode = stat.S_IMODE(fmode);
953 ftypemod = stat.S_IFMT(fmode);
954 fwinattributes = int(HeaderOut[11], 16);
955 fcompression = HeaderOut[12];
956 fcsize = int(HeaderOut[13], 16);
957 fuid = int(HeaderOut[14], 16);
958 funame = HeaderOut[15];
959 fgid = int(HeaderOut[16], 16);
960 fgname = HeaderOut[17];
961 fid = int(HeaderOut[18], 16);
962 finode = int(HeaderOut[19], 16);
963 flinkcount = int(HeaderOut[20], 16);
964 fdev_minor = int(HeaderOut[21], 16);
965 fdev_major = int(HeaderOut[22], 16);
966 frdev_minor = int(HeaderOut[23], 16);
967 frdev_major = int(HeaderOut[24], 16);
968 fseeknextfile = HeaderOut[25];
969 fextrasize = int(HeaderOut[26], 16);
970 fextrafields = int(HeaderOut[27], 16);
971 extrafieldslist = [];
972 extrastart = 28
973 extraend = extrastart + fextrafields;
974 extrafieldslist = [];
975 if(extrastart<extraend):
976 extrafieldslist.append(HeaderOut[extrastart]);
977 extrastart = extrastart + 1;
978 fcs = HeaderOut[-2].lower();
979 fccs = HeaderOut[-1].lower();
980 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-4].lower(), True, formatspecs);
981 if(fcs!=newfcs and not skipchecksum):
982 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
983 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
984 return False;
985 fhend = fp.tell() - 1;
986 fcontentstart = fp.tell();
987 fcontents = BytesIO();
988 pyhascontents = False;
989 if(fsize>0 and not listonly):
990 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
991 fcontents.write(fp.read(fsize));
992 else:
993 fcontents.write(fp.read(fcsize));
994 pyhascontents = True;
995 elif(fsize>0 and listonly):
996 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
997 fp.seek(fsize, 1);
998 else:
999 fp.seek(fcsize, 1);
1000 pyhascontents = False;
1001 fcontents.seek(0, 0);
1002 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
1003 if(fccs!=newfccs and not skipchecksum and not listonly):
1004 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
1005 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
1006 return False;
1007 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1008 pass;
1009 else:
1010 fcontents.seek(0, 0);
1011 if(uncompress):
1012 fcontents = UncompressArchiveFile(fcontents, formatspecs);
1013 fcontents.seek(0, 0);
1014 fccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
1015 fcontentend = fp.tell() - 1;
1016 if(re.findall(r"^\+([0-9]+)", fseeknextfile)):
1017 fseeknextasnum = int(fseeknextfile.replace("+", ""));
1018 if(abs(fseeknextasnum)==0):
1019 pass;
1020 fp.seek(fseeknextasnum, 1);
1021 elif(re.findall(r"^\-([0-9]+)", fseeknextfile)):
1022 fseeknextasnum = int(fseeknextfile);
1023 if(abs(fseeknextasnum)==0):
1024 pass;
1025 fp.seek(fseeknextasnum, 1);
1026 elif(re.findall(r"^([0-9]+)", fseeknextfile)):
1027 fseeknextasnum = int(fseeknextfile);
1028 if(abs(fseeknextasnum)==0):
1029 pass;
1030 fp.seek(fseeknextasnum, 0);
1031 else:
1032 return False;
1033 fcontents.seek(0, 0);
1034 catlist = {'fheadersize': fheadsize, 'fhstart': fheaderstart, 'fhend': fhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': fseeknextfile, 'fheaderchecksumtype': HeaderOut[-4], 'fcontentchecksumtype': HeaderOut[-3], 'fnumfields': fnumfields + 2, 'frawheader': HeaderOut, 'fextrafields': fextrafields, 'fextrafieldsize': fextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': fcs, 'fcontentchecksum': fccs, 'fhascontents': pyhascontents, 'fcontentstart': fcontentstart, 'fcontentend': fcontentend, 'fcontents': fcontents};
1035 return catlist;
1036
1037 def ReadFileHeaderDataBySizeWithContentToList(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1038 formatspecs = FormatSpecsListToDict(formatspecs);
1039 delimiter = formatspecs['format_delimiter'];
1040 fheaderstart = fp.tell();
1041 if(formatspecs['new_style']):
1042 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
1043 else:
1044 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
1045 if(len(HeaderOut)==0):
1046 return False;
1047 fheadsize = int(HeaderOut[0], 16);
1048 fnumfields = int(HeaderOut[1], 16);
1049 ftype = int(HeaderOut[2], 16);
1050 if(re.findall(r"^[.|/]", HeaderOut[3])):
1051 fname = HeaderOut[3];
1052 else:
1053 fname = "./"+HeaderOut[3];
1054 fbasedir = os.path.dirname(fname);
1055 flinkname = HeaderOut[4];
1056 fsize = int(HeaderOut[5], 16);
1057 fatime = int(HeaderOut[6], 16);
1058 fmtime = int(HeaderOut[7], 16);
1059 fctime = int(HeaderOut[8], 16);
1060 fbtime = int(HeaderOut[9], 16);
1061 fmode = int(HeaderOut[10], 16);
1062 fchmode = stat.S_IMODE(fmode);
1063 ftypemod = stat.S_IFMT(fmode);
1064 fwinattributes = int(HeaderOut[11], 16);
1065 fcompression = HeaderOut[12];
1066 fcsize = int(HeaderOut[13], 16);
1067 fuid = int(HeaderOut[14], 16);
1068 funame = HeaderOut[15];
1069 fgid = int(HeaderOut[16], 16);
1070 fgname = HeaderOut[17];
1071 fid = int(HeaderOut[18], 16);
1072 finode = int(HeaderOut[19], 16);
1073 flinkcount = int(HeaderOut[20], 16);
1074 fdev_minor = int(HeaderOut[21], 16);
1075 fdev_major = int(HeaderOut[22], 16);
1076 frdev_minor = int(HeaderOut[23], 16);
1077 frdev_major = int(HeaderOut[24], 16);
1078 fseeknextfile = HeaderOut[25];
1079 fextrasize = int(HeaderOut[26], 16);
1080 fextrafields = int(HeaderOut[27], 16);
1081 extrafieldslist = [];
1082 extrastart = 28;
1083 extraend = extrastart + fextrafields;
1084 extrafieldslist = [];
1085 if(extrastart<extraend):
1086 extrafieldslist.append(HeaderOut[extrastart]);
1087 extrastart = extrastart + 1;
1088 fheaderchecksumtype = HeaderOut[extrastart].lower();
1089 fcontentchecksumtype = HeaderOut[extrastart + 1].lower();
1090 fcs = HeaderOut[-2].lower();
1091 fccs = HeaderOut[-1].lower();
1092 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-4].lower(), True, formatspecs);
1093 if(fcs!=newfcs and not skipchecksum):
1094 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
1095 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
1096 return False;
1097 fhend = fp.tell() - 1;
1098 fcontentstart = fp.tell();
1099 fcontents = BytesIO();
1100 pyhascontents = False;
1101 if(fsize>0 and not listonly):
1102 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1103 fcontents.write(fp.read(fsize));
1104 else:
1105 fcontents.write(fp.read(fcsize));
1106 pyhascontents = True;
1107 elif(fsize>0 and listonly):
1108 if(fcompression=="none" or fcompression=="" or fcompression=="atuo"):
1109 fp.seek(fsize, 1);
1110 else:
1111 fp.seek(fcsize, 1);
1112 pyhascontents = False;
1113 fcontents.seek(0, 0);
1114 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
1115 if(fccs!=newfccs and not skipchecksum and not listonly):
1116 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
1117 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
1118 return False;
1119 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
1120 pass;
1121 else:
1122 fcontents.seek(0, 0);
1123 if(uncompress):
1124 fcontents = UncompressArchiveFile(fcontents, formatspecs);
1125 fcontents.seek(0, 0);
1126 fcontentend = fp.tell() - 1;
1127 if(re.findall(r"^\+([0-9]+)", fseeknextfile)):
1128 fseeknextasnum = int(fseeknextfile.replace("+", ""));
1129 if(abs(fseeknextasnum)==0):
1130 pass;
1131 fp.seek(fseeknextasnum, 1);
1132 elif(re.findall(r"^\-([0-9]+)", fseeknextfile)):
1133 fseeknextasnum = int(fseeknextfile);
1134 if(abs(fseeknextasnum)==0):
1135 pass;
1136 fp.seek(fseeknextasnum, 1);
1137 elif(re.findall(r"^([0-9]+)", fseeknextfile)):
1138 fseeknextasnum = int(fseeknextfile);
1139 if(abs(fseeknextasnum)==0):
1140 pass;
1141 fp.seek(fseeknextasnum, 0);
1142 else:
1143 return False;
1144 catlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, fseeknextfile, extrafieldslist, fheaderchecksumtype, fcontentchecksumtype, fcontents];
1145 return catlist;
1146
1147 def ReadFileDataBySizeWithContent(fp, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1148 formatspecs = FormatSpecsListToDict(formatspecs);
1149 delimiter = formatspecs['format_delimiter'];
1150 curloc = fp.tell();
1151 if(curloc>0):
1152 fp.seek(0, 0);
1153 catheader = ReadFileHeaderData(fp, 4, delimiter);
1154 if(curloc>0):
1155 fp.seek(curloc, 0);
1156 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1157 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1158 if(not headercheck and not skipchecksum):
1159 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1160 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1161 return False;
1162 fnumfiles = int(catheader[1], 16);
1163 countnum = 0;
1164 flist = [];
1165 while(countnum < fnumfiles):
1166 HeaderOut = ReadFileHeaderDataBySizeWithContent(fp, listonly, uncompress, skipchecksum, formatspecs)
1167 if(len(HeaderOut)==0):
1168 break;
1169 flist.append(HeaderOut);
1170 countnum = countnum + 1;
1171 return flist;
1172
1173 def ReadFileDataBySizeWithContentToArray(fp, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1174 formatspecs = FormatSpecsListToDict(formatspecs);
1175 delimiter = formatspecs['format_delimiter'];
1176 curloc = fp.tell();
1177 if(curloc>0):
1178 fp.seek(0, 0);
1179 catheader = ReadFileHeaderData(fp, 4, delimiter);
1180 if(curloc>0):
1181 fp.seek(curloc, 0);
1182 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1183 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1184 if(not headercheck and not skipchecksum):
1185 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1186 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1187 return False;
1188 catstring = catheader[0];
1189 catversion = re.findall(r"([\d]+)", catstring);
1190 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
1191 fprenumfiles = catheader[1];
1192 fnumfiles = int(fprenumfiles, 16);
1193 fprechecksumtype = catheader[2];
1194 fprechecksum = catheader[3];
1195 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': []};
1196 if(seekstart<0 and seekstart>fnumfiles):
1197 seekstart = 0;
1198 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
1199 seekend = fnumfiles;
1200 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
1201 seekend = fnumfiles - abs(seekend);
1202 if(seekstart>0):
1203 il = 0;
1204 while(il < seekstart):
1205 prefhstart = fp.tell();
1206 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs['format_delimiter']);
1207 if(len(preheaderdata)==0):
1208 break;
1209 prefsize = int(preheaderdata[5], 16);
1210 prefseeknextfile = preheaderdata[25];
1211 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-4].lower(), True, formatspecs);
1212 prefcs = preheaderdata[-2];
1213 if(prefcs!=prenewfcs and not skipchecksum):
1214 VVerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
1215 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
1216 return False;
1217 valid_archive = False;
1218 invalid_archive = True;
1219 prefhend = fp.tell() - 1;
1220 prefcontentstart = fp.tell();
1221 prefcontents = BytesIO();
1222 pyhascontents = False;
1223 if(prefsize>0):
1224 prefcontents.write(fp.read(prefsize));
1225 prefcontents.seek(0, 0);
1226 prenewfccs = GetFileChecksum(prefcontents.read(), preheaderdata[-3].lower(), False, formatspecs);
1227 prefccs = preheaderdata[-1];
1228 pyhascontents = True;
1229 if(prefccs!=prenewfccs and not skipchecksum):
1230 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
1231 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
1232 return False;
1233 if(re.findall(r"^\+([0-9]+)", prefseeknextfile)):
1234 fseeknextasnum = int(prefseeknextfile.replace("+", ""));
1235 if(abs(fseeknextasnum)==0):
1236 pass;
1237 fp.seek(fseeknextasnum, 1);
1238 elif(re.findall(r"^\-([0-9]+)", prefseeknextfile)):
1239 fseeknextasnum = int(prefseeknextfile);
1240 if(abs(fseeknextasnum)==0):
1241 pass;
1242 fp.seek(fseeknextasnum, 1);
1243 elif(re.findall(r"^([0-9]+)", prefseeknextfile)):
1244 fseeknextasnum = int(prefseeknextfile);
1245 if(abs(fseeknextasnum)==0):
1246 pass;
1247 fp.seek(fseeknextasnum, 0);
1248 else:
1249 return False;
1250 il = il + 1;
1251 realidnum = 0;
1252 countnum = seekstart;
1253 while(countnum < seekend):
1254 HeaderOut = ReadFileHeaderDataBySizeWithContentToArray(fp, listonly, uncompress, skipchecksum, formatspecs);
1255 if(len(HeaderOut)==0):
1256 break;
1257 HeaderOut.update({'fid': realidnum, 'fidalt': realidnum});
1258 catlist['ffilelist'].append(HeaderOut);
1259 countnum = countnum + 1;
1260 realidnum = realidnum + 1;
1261 return catlist;
1262
1263 def ReadFileDataBySizeWithContentToList(fp, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1264 formatspecs = FormatSpecsListToDict(formatspecs);
1265 delimiter = formatspecs['format_delimiter'];
1266 curloc = fp.tell();
1267 if(curloc>0):
1268 fp.seek(0, 0);
1269 catheader = ReadFileHeaderData(fp, 4, delimiter);
1270 if(curloc>0):
1271 fp.seek(curloc, 0);
1272 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1273 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1274 if(not headercheck and not skipchecksum):
1275 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1276 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1277 return False;
1278 catstring = catheader[0];
1279 catversion = re.findall(r"([\d]+)", catstring);
1280 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
1281 fprenumfiles = catheader[1];
1282 fnumfiles = int(fprenumfiles, 16);
1283 fprechecksumtype = catheader[2];
1284 fprechecksum = catheader[3];
1285 catlist = [];
1286 if(seekstart<0 and seekstart>fnumfiles):
1287 seekstart = 0;
1288 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
1289 seekend = fnumfiles;
1290 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
1291 seekend = fnumfiles - abs(seekend);
1292 if(seekstart>0):
1293 il = 0;
1294 while(il < seekstart):
1295 prefhstart = fp.tell();
1296 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs['format_delimiter']);
1297 if(len(preheaderdata)==0):
1298 break;
1299 prefsize = int(preheaderdata[5], 16);
1300 prefcompression = preheaderdata[12];
1301 prefcsize = int(preheaderdata[13], 16);
1302 prefseeknextfile = HeaderOut[25];
1303 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-4].lower(), True, formatspecs);
1304 prefcs = preheaderdata[-2];
1305 if(prefcs!=prenewfcs and not skipchecksum):
1306 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
1307 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
1308 return False;
1309 valid_archive = False;
1310 invalid_archive = True;
1311 prefhend = fp.tell() - 1;
1312 prefcontentstart = fp.tell();
1313 prefcontents = "";
1314 pyhascontents = False;
1315 if(prefsize>0):
1316 if(prefcompression=="none" or prefcompression=="" or prefcompression=="auto"):
1317 prefcontents = catfp.read(prefsize);
1318 else:
1319 prefcontents = catfp.read(prefcsize);
1320 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
1321 prefccs = preheaderdata[-1];
1322 pyhascontents = True;
1323 if(prefccs!=prenewfccs and not skipchecksum):
1324 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
1325 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
1326 return False;
1327 if(re.findall(r"^\+([0-9]+)", prefseeknextfile)):
1328 fseeknextasnum = int(prefseeknextfile.replace("+", ""));
1329 if(abs(fseeknextasnum)==0):
1330 pass;
1331 catfp.seek(fseeknextasnum, 1);
1332 elif(re.findall(r"^\-([0-9]+)", prefseeknextfile)):
1333 fseeknextasnum = int(prefseeknextfile);
1334 if(abs(fseeknextasnum)==0):
1335 pass;
1336 catfp.seek(fseeknextasnum, 1);
1337 elif(re.findall(r"^([0-9]+)", prefseeknextfile)):
1338 fseeknextasnum = int(prefseeknextfile);
1339 if(abs(fseeknextasnum)==0):
1340 pass;
1341 catfp.seek(fseeknextasnum, 0);
1342 else:
1343 return False;
1344 il = il + 1;
1345 realidnum = 0;
1346 countnum = seekstart;
1347 while(countnum < seekend):
1348 HeaderOut = ReadFileHeaderDataBySizeWithContentToList(fp, listonly, uncompress, skipchecksum, formatspecs);
1349 if(len(HeaderOut)==0):
1350 break;
1351 catlist.append(HeaderOut);
1352 countnum = countnum + 1;
1353 realidnum = realidnum + 1;
1354 return catlist;
1355
1356 def ReadInFileBySizeWithContentToArray(infile, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1357 formatspecs = FormatSpecsListToDict(formatspecs);
1358 delimiter = formatspecs['format_delimiter'];
1359 if(hasattr(infile, "read") or hasattr(infile, "write")):
1360 fp = infile;
1361 fp.seek(0, 0);
1362 fp = UncompressArchiveFile(fp, formatspecs);
1363 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1364 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1365 return False;
1366 if(not fp):
1367 return False;
1368 fp.seek(0, 0);
1369 elif(infile=="-"):
1370 fp = BytesIO();
1371 if(hasattr(sys.stdin, "buffer")):
1372 shutil.copyfileobj(sys.stdin.buffer, fp);
1373 else:
1374 shutil.copyfileobj(sys.stdin, fp);
1375 fp.seek(0, 0);
1376 fp = UncompressArchiveFile(fp, formatspecs);
1377 if(not fp):
1378 return False;
1379 fp.seek(0, 0);
1380 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
1381 fp = download_file_from_internet_file(infile);
1382 fp = UncompressArchiveFile(fp, formatspecs);
1383 fp.seek(0, 0);
1384 if(not fp):
1385 return False;
1386 fp.seek(0, 0);
1387 else:
1388 infile = RemoveWindowsPath(infile);
1389 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
1390 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1391 return False;
1392 compresscheck = CheckCompressionType(infile, formatspecs, True);
1393 if(not compresscheck):
1394 fextname = os.path.splitext(infile)[1];
1395 if(fextname==".gz"):
1396 compresscheck = "gzip";
1397 elif(fextname==".bz2"):
1398 compresscheck = "bzip2";
1399 elif(fextname==".zst"):
1400 compresscheck = "zstd";
1401 elif(fextname==".lz4" or fextname==".clz4"):
1402 compresscheck = "lz4";
1403 elif(fextname==".lzo" or fextname==".lzop"):
1404 compresscheck = "lzo";
1405 elif(fextname==".lzma" or fextname==".xz"):
1406 compresscheck = "lzma";
1407 else:
1408 return False;
1409 if(not compresscheck):
1410 return False;
1411 fp = UncompressFile(infile, formatspecs, "rb");
1412 return ReadFileDataBySizeWithContentToArray(fp, seekstart, seekend, listonly, uncompress, skipchecksum, formatspecs);
1413
1414 def ReadInFileBySizeWithContentToList(infile, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__):
1415 formatspecs = FormatSpecsListToDict(formatspecs);
1416 delimiter = formatspecs['format_delimiter'];
1417 if(hasattr(infile, "read") or hasattr(infile, "write")):
1418 fp = infile;
1419 fp.seek(0, 0);
1420 fp = UncompressArchiveFile(fp, formatspecs);
1421 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1422 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1423 return False;
1424 if(not fp):
1425 return False;
1426 fp.seek(0, 0);
1427 elif(infile=="-"):
1428 fp = BytesIO();
1429 if(hasattr(sys.stdin, "buffer")):
1430 shutil.copyfileobj(sys.stdin.buffer, fp);
1431 else:
1432 shutil.copyfileobj(sys.stdin, fp);
1433 fp.seek(0, 0);
1434 fp = UncompressArchiveFile(fp, formatspecs);
1435 if(not fp):
1436 return False;
1437 fp.seek(0, 0);
1438 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
1439 fp = download_file_from_internet_file(infile);
1440 fp = UncompressArchiveFile(fp, formatspecs);
1441 fp.seek(0, 0);
1442 if(not fp):
1443 return False;
1444 fp.seek(0, 0);
1445 else:
1446 infile = RemoveWindowsPath(infile);
1447 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
1448 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1449 return False;
1450 compresscheck = CheckCompressionType(infile, formatspecs, True);
1451 if(not compresscheck):
1452 fextname = os.path.splitext(infile)[1];
1453 if(fextname==".gz"):
1454 compresscheck = "gzip";
1455 elif(fextname==".bz2"):
1456 compresscheck = "bzip2";
1457 elif(fextname==".zst"):
1458 compresscheck = "zstd";
1459 elif(fextname==".lz4" or fextname==".clz4"):
1460 compresscheck = "lz4";
1461 elif(fextname==".lzo" or fextname==".lzop"):
1462 compresscheck = "lzo";
1463 elif(fextname==".lzma" or fextname==".xz"):
1464 compresscheck = "lzma";
1465 else:
1466 return False;
1467 if(not compresscheck):
1468 return False;
1469 fp = UncompressFile(infile, formatspecs, "rb");
1470 return ReadFileDataBySizeWithContentToList(fp, seekstart, seekend, listonly, uncompress, skipchecksum, formatspecs);
1471
1472 def AppendNullByte(indata, delimiter=__file_format_dict__['format_delimiter']):
1473 outdata = str(indata) + delimiter;
1474 return outdata;
1475
1476 def AppendNullBytes(indata=[], delimiter=__file_format_dict__['format_delimiter']):
1477 outdata = "";
1478 inum = 0;
1479 il = len(indata);
1480 while(inum < il):
1481 outdata = outdata + AppendNullByte(indata[inum], delimiter);
1482 inum = inum + 1;
1483 return outdata;
1484
1485 def AppendFileHeader(fp, numfiles, checksumtype="crc32", formatspecs=__file_format_dict__):
1486 formatspecs = FormatSpecsListToDict(formatspecs);
1487 delimiter = formatspecs['format_delimiter'];
1488 catver = formatspecs['format_ver'];
1489 fileheaderver = str(int(catver.replace(".", "")));
1490 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
1491 fp.write(fileheader.encode('UTF-8'));
1492 fnumfiles = format(int(numfiles), 'x').lower();
1493 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs['format_delimiter']);
1494 catfileheadercshex = GetFileChecksum(fileheader + fnumfilesa, checksumtype, True, formatspecs);
1495 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
1496 fp.write(fnumfilesa.encode('UTF-8'));
1497 try:
1498 fp.flush();
1499 os.fsync(fp.fileno());
1500 except io.UnsupportedOperation:
1501 pass;
1502 except AttributeError:
1503 pass;
1504 except OSError as e:
1505 pass;
1506 return fp;
1507
1508 def MakeEmptyFilePointer(fp, checksumtype="crc32", formatspecs=__file_format_dict__):
1509 formatspecs = FormatSpecsListToDict(formatspecs);
1510 AppendFileHeader(fp, 0, checksumtype, formatspecs);
1511 return fp;
1512
1513 def MakeEmptyFile(outfile, compression="auto", compressionlevel=None, checksumtype="crc32", formatspecs=__file_format_dict__, returnfp=False):
1514 formatspecs = FormatSpecsListToDict(formatspecs);
1515 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1516 if(os.path.exists(outfile)):
1517 try:
1518 os.unlink(outfile);
1519 except OSError as e:
1520 pass;
1521 if(outfile=="-"):
1522 verbose = False;
1523 catfpfp = BytesIO();
1524 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1525 catfp = outfile;
1526 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1527 catfp = BytesIO();
1528 else:
1529 fbasename = os.path.splitext(outfile)[0];
1530 fextname = os.path.splitext(outfile)[1];
1531 if(not compresswholefile and fextname in outextlistwd):
1532 compresswholefile = True;
1533 catfp = CompressOpenFile(outfile, True, compressionlevel);
1534 catfp = AppendFileHeader(catfp, 0, checksumtype, formatspecs);
1535 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1536 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1537 try:
1538 catfp.flush();
1539 os.fsync(catfp.fileno());
1540 except io.UnsupportedOperation:
1541 pass;
1542 except AttributeError:
1543 pass;
1544 except OSError as e:
1545 pass;
1546 if(outfile=="-"):
1547 catfp.seek(0, 0);
1548 if(hasattr(sys.stdout, "buffer")):
1549 shutil.copyfileobj(catfp, sys.stdout.buffer);
1550 else:
1551 shutil.copyfileobj(catfp, sys.stdout);
1552 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1553 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1554 catfp.seek(0, 0);
1555 upload_file_to_internet_file(catfp, outfile);
1556 if(returnfp):
1557 catfp.seek(0, 0);
1558 return catfp;
1559 else:
1560 catfp.close();
1561 return True;
1562
1563 def AppendFileHeaderWithContent(fp, filevalues=[], extradata=[], filecontent="", checksumtype="crc32", formatspecs=__file_format_dict__):
1564 formatspecs = FormatSpecsListToDict(formatspecs);
1565 extrafields = format(len(extradata), 'x').lower();
1566 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
1567 if(len(extradata)>0):
1568 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
1569 extrasizelen = format(len(extrasizestr), 'x').lower();
1570 catoutlen = len(filevalues) + len(extradata) + 5;
1571 catoutlenhex = format(catoutlen, 'x').lower();
1572 catoutlist = filevalues;
1573 catoutlist.insert(0, catoutlenhex);
1574 catoutlist.append(extrasizelen);
1575 catoutlist.append(extrafields);
1576 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
1577 if(len(extradata)>0):
1578 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
1579 if(len(filecontent)==0):
1580 checksumlist = [checksumtype, "none"];
1581 else:
1582 checksumlist = [checksumtype, checksumtype];
1583 catfileoutstr = catfileoutstr + AppendNullBytes(checksumlist, formatspecs['format_delimiter']);
1584 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
1585 if(len(filecontent)==0):
1586 catfilecontentcshex = GetFileChecksum(filecontent, "none", False, formatspecs);
1587 else:
1588 catfilecontentcshex = GetFileChecksum(filecontent, checksumtype, False, formatspecs);
1589 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
1590 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
1591 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
1592 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
1593 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
1594 catfileoutstrecd = catfileoutstr.encode('UTF-8');
1595 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
1596 catfileout = catfileoutstrecd + filecontent + nullstrecd;
1597 fp.write(catfileout);
1598 try:
1599 fp.flush();
1600 os.fsync(fp.fileno());
1601 except io.UnsupportedOperation:
1602 pass;
1603 except AttributeError:
1604 pass;
1605 except OSError as e:
1606 pass;
1607 return fp;
1608
1609 def AppendFilesWithContent(infiles, fp, dirlistfromtxt=False, filevalues=[], extradata=[], compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
1610 formatspecs = FormatSpecsListToDict(formatspecs);
1611 advancedlist = formatspecs['use_advanced_list'];
1612 altinode = formatspecs['use_alt_inode'];
1613 if(verbose):
1614 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
1615 if(infiles=="-"):
1616 for line in sys.stdin:
1617 infilelist.append(line.strip());
1618 infilelist = list(filter(None, infilelist));
1619 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
1620 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
1621 return False;
1622 with UncompressFile(infiles, formatspecs, "r") as finfile:
1623 for line in finfile:
1624 infilelist.append(line.strip());
1625 infilelist = list(filter(None, infilelist));
1626 else:
1627 if(isinstance(infiles, (list, tuple, ))):
1628 infilelist = list(filter(None, infiles));
1629 elif(isinstance(infiles, (str, ))):
1630 infilelist = list(filter(None, [infiles]));
1631 if(advancedlist):
1632 GetDirList = ListDirAdvanced(infilelist, followlink, False);
1633 else:
1634 GetDirList = ListDir(infilelist, followlink, False);
1635 if(not GetDirList):
1636 return False;
1637 curinode = 0;
1638 curfid = 0;
1639 inodelist = [];
1640 inodetofile = {};
1641 filetoinode = {};
1642 inodetocatinode = {};
1643 numfiles = int(len(GetDirList));
1644 fnumfiles = format(numfiles, 'x').lower();
1645 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
1646 for curfname in GetDirList:
1647 if(re.findall(r"^[.|/]", curfname)):
1648 fname = curfname;
1649 else:
1650 fname = "./"+curfname;
1651 if(verbose):
1652 VerbosePrintOut(fname);
1653 if(not followlink or followlink is None):
1654 fstatinfo = os.lstat(fname);
1655 else:
1656 fstatinfo = os.stat(fname);
1657 fpremode = fstatinfo.st_mode;
1658 finode = fstatinfo.st_ino;
1659 flinkcount = fstatinfo.st_nlink;
1660 ftype = 0;
1661 if(stat.S_ISREG(fpremode)):
1662 ftype = 0;
1663 elif(stat.S_ISLNK(fpremode)):
1664 ftype = 2;
1665 elif(stat.S_ISCHR(fpremode)):
1666 ftype = 3;
1667 elif(stat.S_ISBLK(fpremode)):
1668 ftype = 4;
1669 elif(stat.S_ISDIR(fpremode)):
1670 ftype = 5;
1671 elif(stat.S_ISFIFO(fpremode)):
1672 ftype = 6;
1673 elif(stat.S_ISSOCK(fpremode)):
1674 ftype = 8;
1675 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
1676 ftype = 9;
1677 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
1678 ftype = 10;
1679 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
1680 ftype = 11;
1681 else:
1682 ftype = 0;
1683 flinkname = "";
1684 fcurfid = format(int(curfid), 'x').lower();
1685 if(not followlink and finode!=0):
1686 if(ftype!=1):
1687 if(finode in inodelist):
1688 ftype = 1;
1689 flinkname = inodetofile[finode];
1690 if(altinode):
1691 fcurinode = format(int(finode), 'x').lower();
1692 else:
1693 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
1694 if(finode not in inodelist):
1695 inodelist.append(finode);
1696 inodetofile.update({finode: fname});
1697 inodetocatinode.update({finode: curinode});
1698 if(altinode):
1699 fcurinode = format(int(finode), 'x').lower();
1700 else:
1701 fcurinode = format(int(curinode), 'x').lower();
1702 curinode = curinode + 1;
1703 else:
1704 fcurinode = format(int(curinode), 'x').lower();
1705 curinode = curinode + 1;
1706 curfid = curfid + 1;
1707 if(ftype==2):
1708 flinkname = os.readlink(fname);
1709 fdev = fstatinfo.st_dev;
1710 getfdev = GetDevMajorMinor(fdev);
1711 fdev_minor = getfdev[0];
1712 fdev_major = getfdev[1];
1713 frdev = fstatinfo.st_dev;
1714 if(hasattr(fstatinfo, "st_rdev")):
1715 frdev = fstatinfo.st_rdev;
1716 else:
1717 frdev = fstatinfo.st_dev;
1718 getfrdev = GetDevMajorMinor(frdev);
1719 frdev_minor = getfrdev[0];
1720 frdev_major = getfrdev[1];
1721 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
1722 fsize = format(int("0"), 'x').lower();
1723 elif(ftype==0 or ftype==7):
1724 fsize = format(int(fstatinfo.st_size), 'x').lower();
1725 else:
1726 fsize = format(int(fstatinfo.st_size)).lower();
1727 fatime = format(int(fstatinfo.st_atime), 'x').lower();
1728 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
1729 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
1730 if(hasattr(fstatinfo, "st_birthtime")):
1731 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
1732 else:
1733 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
1734 fmode = format(int(fstatinfo.st_mode), 'x').lower();
1735 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
1736 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
1737 fuid = format(int(fstatinfo.st_uid), 'x').lower();
1738 fgid = format(int(fstatinfo.st_gid), 'x').lower();
1739 funame = "";
1740 try:
1741 import pwd;
1742 try:
1743 userinfo = pwd.getpwuid(fstatinfo.st_uid);
1744 funame = userinfo.pw_name;
1745 except KeyError:
1746 funame = "";
1747 except ImportError:
1748 funame = "";
1749 fgname = "";
1750 try:
1751 import grp;
1752 try:
1753 groupinfo = grp.getgrgid(fstatinfo.st_gid);
1754 fgname = groupinfo.gr_name;
1755 except KeyError:
1756 fgname = "";
1757 except ImportError:
1758 fgname = "";
1759 fdev_minor = format(int(fdev_minor), 'x').lower();
1760 fdev_major = format(int(fdev_major), 'x').lower();
1761 frdev_minor = format(int(frdev_minor), 'x').lower();
1762 frdev_major = format(int(frdev_major), 'x').lower();
1763 finode = format(int(finode), 'x').lower();
1764 flinkcount = format(int(flinkcount), 'x').lower();
1765 if(hasattr(fstatinfo, "st_file_attributes")):
1766 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
1767 else:
1768 fwinattributes = format(int(0), 'x').lower();
1769 fcompression = "";
1770 fcsize = format(int(0), 'x').lower();
1771 fcontents = BytesIO();
1772 chunk_size = 1024;
1773 if(ftype==0 or ftype==7):
1774 with open(fname, "rb") as fpc:
1775 shutil.copyfileobj(fpc, fcontents);
1776 if(not compresswholefile):
1777 fcontents.seek(0, 2);
1778 ucfsize = fcontents.tell();
1779 fcontents.seek(0, 0);
1780 if(compression=="auto"):
1781 ilsize = len(compressionlistalt);
1782 ilmin = 0;
1783 ilcsize = [];
1784 while(ilmin < ilsize):
1785 cfcontents = BytesIO();
1786 shutil.copyfileobj(fcontents, cfcontents);
1787 fcontents.seek(0, 0);
1788 cfcontents.seek(0, 0);
1789 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
1790 if(cfcontents):
1791 cfcontents.seek(0, 2);
1792 ilcsize.append(cfcontents.tell());
1793 cfcontents.close();
1794 else:
1795 try:
1796 ilcsize.append(sys.maxint);
1797 except AttributeError:
1798 ilcsize.append(sys.maxsize);
1799 ilmin = ilmin + 1;
1800 ilcmin = ilcsize.index(min(ilcsize));
1801 compression = compressionlistalt[ilcmin];
1802 fcontents.seek(0, 0);
1803 cfcontents = BytesIO();
1804 shutil.copyfileobj(fcontents, cfcontents);
1805 cfcontents.seek(0, 0);
1806 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
1807 cfcontents.seek(0, 2);
1808 cfsize = cfcontents.tell();
1809 if(ucfsize > cfsize):
1810 fcsize = format(int(cfsize), 'x').lower();
1811 fcompression = compression;
1812 fcontents.close();
1813 fcontents = cfcontents;
1814 if(followlink and (ftype==1 or ftype==2)):
1815 flstatinfo = os.stat(flinkname);
1816 with open(flinkname, "rb") as fpc:
1817 shutil.copyfileobj(fpc, fcontents);
1818 if(not compresswholefile):
1819 fcontents.seek(0, 2);
1820 ucfsize = fcontents.tell();
1821 fcontents.seek(0, 0);
1822 if(compression=="auto"):
1823 ilsize = len(compressionlistalt);
1824 ilmin = 0;
1825 ilcsize = [];
1826 while(ilmin < ilsize):
1827 cfcontents = BytesIO();
1828 shutil.copyfileobj(fcontents, cfcontents);
1829 fcontents.seek(0, 0);
1830 cfcontents.seek(0, 0);
1831 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
1832 if(cfcontents):
1833 cfcontents.seek(0, 2);
1834 ilcsize.append(cfcontents.tell());
1835 cfcontents.close();
1836 else:
1837 try:
1838 ilcsize.append(sys.maxint);
1839 except AttributeError:
1840 ilcsize.append(sys.maxsize);
1841 ilmin = ilmin + 1;
1842 ilcmin = ilcsize.index(min(ilcsize));
1843 compression = compressionlistalt[ilcmin];
1844 fcontents.seek(0, 0);
1845 cfcontents = BytesIO();
1846 shutil.copyfileobj(fcontents, cfcontents);
1847 cfcontents.seek(0, 0);
1848 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
1849 cfcontents.seek(0, 2);
1850 cfsize = cfcontents.tell();
1851 if(ucfsize > cfsize):
1852 fcsize = format(int(cfsize), 'x').lower();
1853 fcompression = compression;
1854 fcontents.close();
1855 fcontents = cfcontents;
1856 if(fcompression=="none"):
1857 fcompression = "";
1858 fcontents.seek(0, 0);
1859 ftypehex = format(ftype, 'x').lower();
1860 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
1861 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
1862 if(numfiles>0):
1863 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
1864 fp.seek(0, 0);
1865 return fp;
1866
1867 def AppendListsWithContent(inlist, fp, dirlistfromtxt=False, filevalues=[], extradata=[], compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
1868 formatspecs = FormatSpecsListToDict(formatspecs);
1869 if(verbose):
1870 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
1871 GetDirList = inlist;
1872 if(not GetDirList):
1873 return False;
1874 curinode = 0;
1875 curfid = 0;
1876 inodelist = [];
1877 inodetofile = {};
1878 filetoinode = {};
1879 inodetocatinode = {};
1880 numfiles = int(len(GetDirList));
1881 fnumfiles = format(numfiles, 'x').lower();
1882 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
1883 for curfname in GetDirList:
1884 ftype = format(curfname[0], 'x').lower();
1885 if(re.findall(r"^[.|/]", curfname[1])):
1886 fname = curfname[1];
1887 else:
1888 fname = "./"+curfname[1];
1889 fbasedir = os.path.dirname(fname);
1890 flinkname = curfname[2];
1891 fsize = format(curfname[3], 'x').lower();
1892 fatime = format(curfname[4], 'x').lower();
1893 fmtime = format(curfname[5], 'x').lower();
1894 fctime = format(curfname[6], 'x').lower();
1895 fbtime = format(curfname[7], 'x').lower();
1896 fmode = format(curfname[8], 'x').lower();
1897 fwinattributes = format(curfname[9], 'x').lower();
1898 fcompression = curfname[10];
1899 fcsize = format(curfname[11], 'x').lower();
1900 fuid = format(curfname[12], 'x').lower();
1901 funame = curfname[13];
1902 fgid = format(curfname[14], 'x').lower();
1903 fgname = curfname[15];
1904 fid = format(curfname[16], 'x').lower();
1905 finode = format(curfname[17], 'x').lower();
1906 flinkcount = format(curfname[18], 'x').lower();
1907 fdev_minor = format(curfname[19], 'x').lower();
1908 fdev_major = format(curfname[20], 'x').lower();
1909 frdev_minor = format(curfname[21], 'x').lower();
1910 frdev_major = format(curfname[22], 'x').lower();
1911 fseeknextfile = curfname[23];
1912 extradata = curfname[24];
1913 fheaderchecksumtype = curfname[25];
1914 fcontentchecksumtype = curfname[26];
1915 fcontents = curfname[27];
1916 catoutlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, fseeknextfile];
1917 fcontents.seek(0, 0);
1918 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
1919 if(numfiles>0):
1920 fp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
1921 return fp;
1922
1923 def AppendInFileWithContent(infile, fp, dirlistfromtxt=False, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
1924 formatspecs = FormatSpecsListToDict(formatspecs);
1925 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, True, False, formatspecs);
1926 return AppendListsWithContent(inlist, fp, dirlistfromtxt, filevalues, extradata, followlink, checksumtype, formatspecs, verbose);
1927
1928 def AppendFilesWithContentToOutFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
1929 formatspecs = FormatSpecsListToDict(formatspecs);
1930 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1931 if(os.path.exists(outfile)):
1932 try:
1933 os.unlink(outfile);
1934 except OSError as e:
1935 pass;
1936 if(outfile=="-"):
1937 verbose = False;
1938 catfpfp = BytesIO();
1939 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1940 catfp = outfile;
1941 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1942 catfp = BytesIO();
1943 else:
1944 fbasename = os.path.splitext(outfile)[0];
1945 fextname = os.path.splitext(outfile)[1];
1946 if(not compresswholefile and fextname in outextlistwd):
1947 compresswholefile = True;
1948 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
1949 catfp = AppendFilesWithContent(infiles, catfp, dirlistfromtxt, filevalues, extradata, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose);
1950 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1951 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1952 try:
1953 catfp.flush();
1954 os.fsync(catfp.fileno());
1955 except io.UnsupportedOperation:
1956 pass;
1957 except AttributeError:
1958 pass;
1959 except OSError as e:
1960 pass;
1961 if(outfile=="-"):
1962 catfp.seek(0, 0);
1963 if(hasattr(sys.stdout, "buffer")):
1964 shutil.copyfileobj(catfp, sys.stdout.buffer);
1965 else:
1966 shutil.copyfileobj(catfp, sys.stdout);
1967 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1968 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1969 catfp.seek(0, 0);
1970 upload_file_to_internet_file(catfp, outfile);
1971 if(returnfp):
1972 catfp.seek(0, 0);
1973 return catfp;
1974 else:
1975 catfp.close();
1976 return True;
1977
1978 def AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
1979 formatspecs = FormatSpecsListToDict(formatspecs);
1980 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1981 if(os.path.exists(outfile)):
1982 try:
1983 os.unlink(outfile);
1984 except OSError as e:
1985 pass;
1986 if(outfile=="-"):
1987 verbose = False;
1988 catfpfp = BytesIO();
1989 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1990 catfp = outfile;
1991 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1992 catfp = BytesIO();
1993 else:
1994 fbasename = os.path.splitext(outfile)[0];
1995 fextname = os.path.splitext(outfile)[1];
1996 if(not compresswholefile and fextname in outextlistwd):
1997 compresswholefile = True;
1998 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
1999 catfp = AppendListsWithContent(inlist, catfp, dirlistfromtxt, filevalues, extradata, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose);
2000 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
2001 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2002 try:
2003 catfp.flush();
2004 os.fsync(catfp.fileno());
2005 except io.UnsupportedOperation:
2006 pass;
2007 except AttributeError:
2008 pass;
2009 except OSError as e:
2010 pass;
2011 if(outfile=="-"):
2012 catfp.seek(0, 0);
2013 if(hasattr(sys.stdout, "buffer")):
2014 shutil.copyfileobj(catfp, sys.stdout.buffer);
2015 else:
2016 shutil.copyfileobj(catfp, sys.stdout);
2017 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2018 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2019 catfp.seek(0, 0);
2020 upload_file_to_internet_file(catfp, outfile);
2021 if(returnfp):
2022 catfp.seek(0, 0);
2023 return catfp;
2024 else:
2025 catfp.close();
2026 return True;
2027
2028 def AppendInFileWithContentToOutFile(infile, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2029 formatspecs = FormatSpecsListToDict(formatspecs);
2030 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, True, False, formatspecs);
2031 return AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt, compression, compresswholefile, compressionlevel, filevalues, extradata, followlink, checksumtype, formatspecs, verbose, returnfp);
2032
2033 def PrintPermissionString(fchmode, ftype):
2034 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
2035 permissionstr = "";
2036 for fmodval in str(oct(fchmode))[-3:]:
2037 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
2038 if(ftype==0 or ftype==7):
2039 permissionstr = "-" + permissionstr;
2040 if(ftype==1):
2041 permissionstr = "h" + permissionstr;
2042 if(ftype==2):
2043 permissionstr = "l" + permissionstr;
2044 if(ftype==3):
2045 permissionstr = "c" + permissionstr;
2046 if(ftype==4):
2047 permissionstr = "b" + permissionstr;
2048 if(ftype==5):
2049 permissionstr = "d" + permissionstr;
2050 if(ftype==6):
2051 permissionstr = "f" + permissionstr;
2052 if(ftype==8):
2053 permissionstr = "D" + permissionstr;
2054 if(ftype==9):
2055 permissionstr = "p" + permissionstr;
2056 if(ftype==10):
2057 permissionstr = "w" + permissionstr;
2058 try:
2059 permissionoutstr = stat.filemode(fchmode);
2060 except AttributeError:
2061 permissionoutstr = permissionstr;
2062 except KeyError:
2063 permissionoutstr = permissionstr;
2064 return permissionoutstr;
2065
2066 def PrintPermissionStringAlt(fchmode, ftype):
2067 permissions = {
2068 '0': '---', '1': '--x', '2': '-w-', '3': '-wx',
2069 '4': 'r--', '5': 'r-x', '6': 'rw-', '7': 'rwx'
2070 };
2071 # Translate file mode into permission string
2072 permissionstr = ''.join([permissions[i] for i in str(oct(fchmode))[-3:]]);
2073 # Append file type indicator
2074 type_indicators = {
2075 0: '-', 1: 'h', 2: 'l', 3: 'c', 4: 'b',
2076 5: 'd', 6: 'f', 8: 'D', 9: 'p', 10: 'w'
2077 };
2078 file_type = type_indicators.get(ftype, '-');
2079 permissionstr = file_type + permissionstr;
2080 try:
2081 permissionoutstr = stat.filemode(fchmode);
2082 except AttributeError:
2083 permissionoutstr = permissionstr;
2084 return permissionoutstr;
2085
2086 def CheckCompressionType(infile, formatspecs=__file_format_dict__, closefp=True):
2087 formatspecs = FormatSpecsListToDict(formatspecs);
2088 if(hasattr(infile, "read") or hasattr(infile, "write")):
2089 catfp = infile;
2090 else:
2091 try:
2092 catfp = open(infile, "rb");
2093 except FileNotFoundError:
2094 return False;
2095 catfp.seek(0, 0);
2096 prefp = catfp.read(2);
2097 filetype = False;
2098 if(prefp==binascii.unhexlify("1f8b")):
2099 filetype = "gzip";
2100 catfp.seek(0, 0);
2101 prefp = catfp.read(3);
2102 if(prefp==binascii.unhexlify("425a68")):
2103 filetype = "bzip2";
2104 if(prefp==binascii.unhexlify("5d0000")):
2105 filetype = "lzma";
2106 catfp.seek(0, 0);
2107 prefp = catfp.read(4);
2108 if(prefp==binascii.unhexlify("28b52ffd")):
2109 filetype = "zstd";
2110 if(prefp==binascii.unhexlify("04224d18")):
2111 filetype = "lz4";
2112 if(prefp==binascii.unhexlify("504B0304")):
2113 filetype = "zipfile";
2114 catfp.seek(0, 0);
2115 prefp = catfp.read(5);
2116 if(prefp==binascii.unhexlify("7573746172")):
2117 filetype = "tarfile";
2118 catfp.seek(0, 0);
2119 prefp = catfp.read(6);
2120 if(prefp==binascii.unhexlify("fd377a585a00")):
2121 filetype = "lzma";
2122 if(prefp==binascii.unhexlify("377abcaf271c")):
2123 filetype = "7zipfile";
2124 catfp.seek(0, 0);
2125 prefp = catfp.read(7);
2126 if(prefp==binascii.unhexlify("526172211a0700")):
2127 filetype = "rarfile";
2128 if(prefp==binascii.unhexlify("43617446696c65")):
2129 filetype = "catfile";
2130 catfp.seek(0, 0);
2131 prefp = catfp.read(8);
2132 if(prefp==binascii.unhexlify("526172211a070100")):
2133 filetype = "rarfile";
2134 catfp.seek(0, 0);
2135 prefp = catfp.read(formatspecs['format_len']);
2136 if(prefp==binascii.unhexlify(formatspecs['format_hex'])):
2137 filetype = formatspecs['format_lower'];
2138 catfp.seek(0, 0);
2139 prefp = catfp.read(9);
2140 if(prefp==binascii.unhexlify("894c5a4f000d0a1a0a")):
2141 filetype = "lzo";
2142 catfp.seek(0, 0);
2143 prefp = catfp.read(10);
2144 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
2145 filetype = "tarfile";
2146 catfp.seek(0, 0);
2147 if(filetype=="gzip" or filetype=="bzip2" or filetype=="lzma" or filetype=="zstd" or filetype=="lz4"):
2148 if(TarFileCheck(catfp)):
2149 filetype = "tarfile";
2150 if(not filetype):
2151 if(TarFileCheck(catfp)):
2152 filetype = "tarfile";
2153 elif(zipfile.is_zipfile(catfp)):
2154 filetype = "zipfile";
2155 elif(rarfile.is_rarfile(catfp) or rarfile.is_rarfile_sfx(catfp)):
2156 filetype = "rarile";
2157 else:
2158 filetype = False;
2159 catfp.seek(0, 0);
2160 if(closefp):
2161 catfp.close();
2162 return filetype;
2163
2164 def CheckCompressionTypeFromString(instring, formatspecs=__file_format_dict__, closefp=True):
2165 formatspecs = FormatSpecsListToDict(formatspecs);
2166 try:
2167 instringsfile = BytesIO(instring);
2168 except TypeError:
2169 instringsfile = BytesIO(instring.encode("UTF-8"));
2170 return CheckCompressionType(instringsfile, formatspecs, closefp);
2171
2172 def GetCompressionMimeType(infile, formatspecs=__file_format_dict__):
2173 formatspecs = FormatSpecsListToDict(formatspecs);
2174 compresscheck = CheckCompressionType(fp, formatspecs, False);
2175 if(compresscheck=="gzip" or compresscheck=="gz"):
2176 return archivefile_gzip_mimetype;
2177 if(compresscheck=="bzip2" or compresscheck=="bz2"):
2178 return archivefile_bzip2_mimetype;
2179 if(compresscheck=="zstd" or compresscheck=="zstandard"):
2180 return archivefile_zstandard_mimetype;
2181 if(compresscheck=="lz4"):
2182 return archivefile_lz4_mimetype;
2183 if(compresscheck=="lzo" or compresscheck=="lzop"):
2184 return archivefile_lzop_mimetype;
2185 if(compresscheck=="lzma"):
2186 return archivefile_lzma_mimetype;
2187 if(compresscheck=="xz"):
2188 return archivefile_xz_mimetype;
2189 if(compresscheck=="catfile" or compresscheck=="cat" or compresscheck==formatspecs['format_lower']):
2190 return archivefile_cat_mimetype;
2191 if(not compresscheck):
2192 return False;
2193 return False;
2194
2195 def UncompressArchiveFile(fp, formatspecs=__file_format_dict__):
2196 formatspecs = FormatSpecsListToDict(formatspecs);
2197 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
2198 return False;
2199 compresscheck = CheckCompressionType(fp, formatspecs, False);
2200 if(compresscheck=="gzip" and compresscheck in compressionsupport):
2201 catfp = gzip.GzipFile(fileobj=fp, mode="rb");
2202 if(compresscheck=="bzip2" and compresscheck in compressionsupport):
2203 catfp = BytesIO();
2204 catfp.write(bz2.decompress(fp.read()));
2205 if(compresscheck=="zstd" and compresscheck in compressionsupport):
2206 catfp = BytesIO();
2207 catfp.write(zstandard.decompress(fp.read()));
2208 if(compresscheck=="lz4" and compresscheck in compressionsupport):
2209 catfp = BytesIO();
2210 catfp.write(lz4.frame.decompress(fp.read()));
2211 if((compresscheck=="lzo" or compresscheck=="lzop") and compresscheck in compressionsupport):
2212 catfp = BytesIO();
2213 catfp.write(lzo.decompress(fp.read()));
2214 if((compresscheck=="lzma" or compresscheck=="xz") and compresscheck in compressionsupport):
2215 catfp = BytesIO();
2216 catfp.write(lzma.decompress(fp.read()));
2217 if(compresscheck=="catfile" or compresscheck==formatspecs['format_lower']):
2218 catfp = fp;
2219 if(not compresscheck):
2220 try:
2221 import lzma;
2222 except ImportError:
2223 try:
2224 from backports import lzma
2225 except ImportError:
2226 return False;
2227 catfp = BytesIO();
2228 with fp as fpcontent:
2229 try:
2230 catfp.write(lzma.decompress(fp.read()));
2231 except lzma.LZMAError:
2232 return False;
2233 if(compresscheck!="catfile" or compresscheck!=formatspecs['format_lower']):
2234 fp.close();
2235 return catfp;
2236
2237 create_alias_function("Uncompress", __file_format_name__, "", UncompressArchiveFile);
2238
2239 def UncompressFile(infile, formatspecs=__file_format_dict__, mode="rb"):
2240 formatspecs = FormatSpecsListToDict(formatspecs);
2241 compresscheck = CheckCompressionType(infile, formatspecs, False);
2242 if(sys.version_info[0]==2 and compresscheck):
2243 if(mode=="rt"):
2244 mode = "r";
2245 if(mode=="wt"):
2246 mode = "w";
2247 try:
2248 if(compresscheck=="gzip" and compresscheck in compressionsupport):
2249 try:
2250 filefp = gzip.open(infile, mode, encoding="UTF-8");
2251 except (ValueError, TypeError) as e:
2252 filefp = gzip.open(infile, mode);
2253 if(compresscheck=="bzip2" and compresscheck in compressionsupport):
2254 try:
2255 filefp = bz2.open(infile, mode, encoding="UTF-8");
2256 except (ValueError, TypeError) as e:
2257 filefp = bz2.open(infile, mode);
2258 if(compresscheck=="zstd" and compresscheck in compressionsupport):
2259 try:
2260 filefp = zstandard.open(infile, mode, encoding="UTF-8");
2261 except (ValueError, TypeError) as e:
2262 filefp = zstandard.open(infile, mode);
2263 if(compresscheck=="lz4" and compresscheck in compressionsupport):
2264 try:
2265 filefp = lz4.frame.open(infile, mode, encoding="UTF-8");
2266 except (ValueError, TypeError) as e:
2267 filefp = lz4.frame.open(infile, mode);
2268 if((compresscheck=="lzo" or compresscheck=="lzop") and compresscheck in compressionsupport):
2269 try:
2270 filefp = lzo.open(infile, mode, encoding="UTF-8");
2271 except (ValueError, TypeError) as e:
2272 filefp = lzo.open(infile, mode);
2273 if((compresscheck=="lzma" or compresscheck=="xz") and compresscheck in compressionsupport):
2274 try:
2275 filefp = lzma.open(infile, mode, encoding="UTF-8");
2276 except (ValueError, TypeError) as e:
2277 filefp = lzma.open(infile, mode);
2278 if(compresscheck=="catfile" or compresscheck==formatspecs['format_lower']):
2279 try:
2280 filefp = open(infile, mode, encoding="UTF-8");
2281 except (ValueError, TypeError) as e:
2282 filefp = open(infile, mode);
2283 if(not compresscheck):
2284 try:
2285 filefp = open(infile, mode, encoding="UTF-8");
2286 except (ValueError, TypeError) as e:
2287 filefp = open(infile, mode);
2288 except FileNotFoundError:
2289 return False;
2290 return filefp;
2291
2292 def UncompressString(infile):
2293 compresscheck = CheckCompressionTypeFromString(infile, formatspecs, False);
2294 if(compresscheck=="gzip" and compresscheck in compressionsupport):
2295 fileuz = gzip.decompress(infile);
2296 if(compresscheck=="bzip2" and compresscheck in compressionsupport):
2297 fileuz = bz2.decompress(infile);
2298 if(compresscheck=="zstd" and compresscheck in compressionsupport):
2299 try:
2300 import zstandard;
2301 except ImportError:
2302 return False;
2303 fileuz = zstandard.decompress(infile);
2304 if(compresscheck=="lz4" and compresscheck in compressionsupport):
2305 fileuz = lz4.frame.decompress(infile);
2306 if((compresscheck=="lzo" or compresscheck=="lzop") and compresscheck in compressionsupport):
2307 fileuz = lzo.decompress(infile);
2308 if((compresscheck=="lzma" or compresscheck=="xz") and compresscheck in compressionsupport):
2309 fileuz = lzma.decompress(infile);
2310 if(not compresscheck):
2311 fileuz = infile;
2312 if(hasattr(fileuz, 'decode')):
2313 fileuz = fileuz.decode("UTF-8");
2314 return fileuz;
2315
2316 def UncompressStringAlt(infile):
2317 filefp = StringIO();
2318 outstring = UncompressString(infile);
2319 filefp.write(outstring);
2320 filefp.seek(0, 0);
2321 return filefp;
2322
2323 def CheckCompressionSubType(infile, formatspecs=__file_format_dict__, closefp=True):
2324 formatspecs = FormatSpecsListToDict(formatspecs);
2325 compresscheck = CheckCompressionType(infile, formatspecs, False);
2326 if(not compresscheck):
2327 fextname = os.path.splitext(infile)[1];
2328 if(fextname==".gz"):
2329 compresscheck = "gzip";
2330 elif(fextname==".bz2"):
2331 compresscheck = "bzip2";
2332 elif(fextname==".zst"):
2333 compresscheck = "zstd";
2334 elif(fextname==".lz4"):
2335 compresscheck = "lz4";
2336 elif(fextname==".lzo" or fextname==".lzop"):
2337 compresscheck = "lzo";
2338 elif(fextname==".lzma" or fextname==".xz"):
2339 compresscheck = "lzma";
2340 else:
2341 return False;
2342 if(compresscheck=="gzip" or compresscheck=="bzip2" or compresscheck=="lzma" or compresscheck=="zstd" or compresscheck=="lz4"):
2343 if(TarFileCheck(infile)):
2344 filetype = "tarfile";
2345 if(not compresscheck):
2346 if(TarFileCheck(infile)):
2347 return "tarfile";
2348 elif(zipfile.is_zipfile(infile)):
2349 return "zipfile";
2350 elif(rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile)):
2351 return "rarile";
2352 else:
2353 return False;
2354 return False;
2355 if(compresscheck=="catfile"):
2356 return "catfile";
2357 if(compresscheck==formatspecs['format_lower']):
2358 return formatspecs['format_lower'];
2359 if(compresscheck=="tarfile"):
2360 return "tarfile";
2361 if(compresscheck=="zipfile"):
2362 return "zipfile";
2363 if(rarfile_support and compresscheck=="rarfile"):
2364 return "rarfile";
2365 if(py7zr_support and compresscheck=="7zipfile" and py7zr.is_7zfile(infile)):
2366 return "7zipfile";
2367 if(hasattr(infile, "read") or hasattr(infile, "write")):
2368 catfp = UncompressArchiveFile(infile, formatspecs['format_lower']);
2369 else:
2370 try:
2371 if(compresscheck=="gzip" and compresscheck in compressionsupport):
2372 catfp = gzip.GzipFile(infile, "rb");
2373 if(compresscheck=="bzip2" and compresscheck in compressionsupport):
2374 catfp = bz2.BZ2File(infile, "rb");
2375 if(compresscheck=="lz4" and compresscheck in compressionsupport):
2376 catfp = lz4.frame.open(infile, "rb");
2377 if(compresscheck=="zstd" and compresscheck in compressionsupport):
2378 catfp = zstandard.open(infile, "rb");
2379 if((compresscheck=="lzo" or compresscheck=="lzop") and compresscheck in compressionsupport):
2380 catfp = lzo.open(infile, "rb");
2381 if((compresscheck=="lzma" or compresscheck=="xz") and compresscheck in compressionsupport):
2382 catfp = lzma.open(infile, "rb");
2383 except FileNotFoundError:
2384 return False;
2385 filetype = False;
2386 prefp = catfp.read(5);
2387 if(prefp==binascii.unhexlify("7573746172")):
2388 filetype = "tarfile";
2389 catfp.seek(0, 0);
2390 prefp = catfp.read(7);
2391 if(prefp==binascii.unhexlify("43617446696c65")):
2392 filetype = "catfile";
2393 catfp.seek(0, 0);
2394 prefp = catfp.read(formatspecs['format_len']);
2395 if(prefp==binascii.unhexlify(formatspecs['format_hex'])):
2396 filetype = formatspecs['format_lower'];
2397 catfp.seek(0, 0);
2398 prefp = catfp.read(10);
2399 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
2400 filetype = "tarfile";
2401 catfp.seek(0, 0);
2402 if(closefp):
2403 catfp.close();
2404 return filetype;
2405
2406 def GZipCompress(data, compresslevel=9):
2407 if("gzip" not in compressionsupport):
2408 return False;
2409 tmpfp = tempfile.NamedTemporaryFile("wb", delete=False);
2410 tmpfp.close();
2411 tmpfp = gzip.GzipFile(tmpfp.name, mode="wb", compresslevel=compresslevel);
2412 tmpfp.write(data);
2413 tmpfp.close();
2414 try:
2415 catfp = open(tmpfp.name, "rb");
2416 except FileNotFoundError:
2417 return False;
2418 catdata = catfp.read();
2419 catfp.close();
2420 return catdata;
2421
2422 def CompressArchiveFile(fp, compression="auto", compressionlevel=None, formatspecs=__file_format_dict__):
2423 formatspecs = FormatSpecsListToDict(formatspecs);
2424 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
2425 return False;
2426 fp.seek(0, 0);
2427 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
2428 compression = "auto";
2429 if(compression not in compressionlist and compression is None):
2430 compression = "auto";
2431 if(compression=="gzip" and compression in compressionsupport):
2432 catfp = BytesIO();
2433 if(compressionlevel is None):
2434 compressionlevel = 9;
2435 else:
2436 compressionlevel = int(compressionlevel);
2437 catfp.write(gzip.compress(fp.read(), compresslevel=compressionlevel));
2438 if(compression=="bzip2" and compression in compressionsupport):
2439 catfp = BytesIO();
2440 if(compressionlevel is None):
2441 compressionlevel = 9;
2442 else:
2443 compressionlevel = int(compressionlevel);
2444 catfp.write(bz2.compress(fp.read(), compresslevel=compressionlevel));
2445 if(compression=="lz4" and compression in compressionsupport):
2446 catfp = BytesIO();
2447 if(compressionlevel is None):
2448 compressionlevel = 9;
2449 else:
2450 compressionlevel = int(compressionlevel);
2451 catfp.write(lz4.frame.compress(fp.read(), compression_level=compressionlevel));
2452 if((compression=="lzo" or compression=="lzop") and compression in compressionsupport):
2453 catfp = BytesIO();
2454 if(compressionlevel is None):
2455 compressionlevel = 9;
2456 else:
2457 compressionlevel = int(compressionlevel);
2458 catfp.write(lzo.compress(fp.read(), compresslevel=compressionlevel));
2459 if(compression=="zstd" and compression in compressionsupport):
2460 catfp = BytesIO();
2461 if(compressionlevel is None):
2462 compressionlevel = 10;
2463 else:
2464 compressionlevel = int(compressionlevel);
2465 catfp.write(zstandard.compress(fp.read(), level=compressionlevel));
2466 if(compression=="lzma" and compression in compressionsupport):
2467 catfp = BytesIO();
2468 if(compressionlevel is None):
2469 compressionlevel = 9;
2470 else:
2471 compressionlevel = int(compressionlevel);
2472 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]));
2473 if(compression=="xz" and compression in compressionsupport):
2474 catfp = BytesIO();
2475 if(compressionlevel is None):
2476 compressionlevel = 9;
2477 else:
2478 compressionlevel = int(compressionlevel);
2479 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]));
2480 if(compression=="auto" or compression is None):
2481 catfp = fp;
2482 catfp.seek(0, 0);
2483 return catfp;
2484
2485 create_alias_function("Compress", __file_format_name__, "", CompressArchiveFile);
2486
2487 def CompressOpenFile(outfile, compressionenable=True, compressionlevel=None):
2488 if(outfile is None):
2489 return False;
2490 fbasename = os.path.splitext(outfile)[0];
2491 fextname = os.path.splitext(outfile)[1];
2492 if(compressionlevel is None and fextname!=".zst"):
2493 compressionlevel = 9;
2494 elif(compressionlevel is None and fextname==".zst"):
2495 compressionlevel = 10;
2496 else:
2497 compressionlevel = int(compressionlevel);
2498 if(sys.version_info[0]==2):
2499 mode = "w";
2500 else:
2501 mode = "wb";
2502 try:
2503 if(fextname not in outextlistwd or not compressionenable):
2504 try:
2505 outfp = open(outfile, "wb", encoding="UTF-8");
2506 except (ValueError, TypeError) as e:
2507 outfp = open(outfile, "wb");
2508 elif(fextname==".gz" and "gzip" in compressionsupport):
2509 try:
2510 outfp = gzip.open(outfile, mode, compressionlevel, encoding="UTF-8");
2511 except (ValueError, TypeError) as e:
2512 outfp = gzip.open(outfile, mode, compressionlevel);
2513 elif(fextname==".bz2" and "bzip2" in compressionsupport):
2514 try:
2515 outfp = bz2.open(outfile, mode, compressionlevel, encoding="UTF-8");
2516 except (ValueError, TypeError) as e:
2517 outfp = bz2.open(outfile, mode, compressionlevel);
2518 elif(fextname==".zst" and "zstandard" in compressionsupport):
2519 try:
2520 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel), encoding="UTF-8");
2521 except (ValueError, TypeError) as e:
2522 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel));
2523 elif(fextname==".xz" and "xz" in compressionsupport):
2524 try:
2525 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}], encoding="UTF-8");
2526 except (ValueError, TypeError) as e:
2527 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]);
2528 elif(fextname==".lz4" and "lz4" in compressionsupport):
2529 try:
2530 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel, encoding="UTF-8");
2531 except (ValueError, TypeError) as e:
2532 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel);
2533 elif(fextname==".lzo" and "lzop" in compressionsupport):
2534 try:
2535 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel, encoding="UTF-8");
2536 except (ValueError, TypeError) as e:
2537 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel);
2538 elif(fextname==".lzma" and "lzma" in compressionsupport):
2539 try:
2540 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}], encoding="UTF-8");
2541 except (ValueError, TypeError) as e:
2542 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]);
2543 except FileNotFoundError:
2544 return False;
2545 return outfp;
2546
2547 def GetDevMajorMinor(fdev):
2548 retdev = [];
2549 if(hasattr(os, "minor")):
2550 retdev.append(os.minor(fdev));
2551 else:
2552 retdev.append(0);
2553 if(hasattr(os, "major")):
2554 retdev.append(os.major(fdev));
2555 else:
2556 retdev.append(0);
2557 return retdev;
2558
2559 def CheckSumSupport(checkfor, guaranteed=True):
2560 if(guaranteed):
2561 try:
2562 hash_list = sorted(list(hashlib.algorithms_guaranteed));
2563 except AttributeError:
2564 hash_list = sorted(list(hashlib.algorithms));
2565 else:
2566 try:
2567 hash_list = sorted(list(hashlib.algorithms_available));
2568 except AttributeError:
2569 hash_list = sorted(list(hashlib.algorithms));
2570 checklistout = sorted(hash_list + ['adler32', 'crc16', 'crc16_ansi', 'crc16_ibm', 'crc16_ccitt', 'crc32', 'crc64', 'crc64_ecma', 'crc64_iso', 'none']);
2571 if(checkfor in checklistout):
2572 return True;
2573 else:
2574 return False;
2575
2576 def CheckSumSupportAlt(checkfor, guaranteed=True):
2577 if(guaranteed):
2578 try:
2579 hash_list = sorted(list(hashlib.algorithms_guaranteed));
2580 except AttributeError:
2581 hash_list = sorted(list(hashlib.algorithms));
2582 else:
2583 try:
2584 hash_list = sorted(list(hashlib.algorithms_available));
2585 except AttributeError:
2586 hash_list = sorted(list(hashlib.algorithms));
2587 checklistout = hash_list;
2588 if(checkfor in checklistout):
2589 return True;
2590 else:
2591 return False;
2592
2593 def PackArchiveFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2594 formatspecs = FormatSpecsListToDict(formatspecs);
2595 advancedlist = formatspecs['use_advanced_list'];
2596 altinode = formatspecs['use_alt_inode'];
2597 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2598 outfile = RemoveWindowsPath(outfile);
2599 checksumtype = checksumtype.lower();
2600 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
2601 checksumtype="crc32";
2602 if(checksumtype=="none"):
2603 checksumtype = "";
2604 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
2605 compression = "auto";
2606 if(compression not in compressionlist and compression is None):
2607 compression = "auto";
2608 if(verbose):
2609 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2610 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2611 if(os.path.exists(outfile)):
2612 try:
2613 os.unlink(outfile);
2614 except OSError as e:
2615 pass;
2616 if(outfile=="-"):
2617 verbose = False;
2618 catfp = BytesIO();
2619 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2620 catfp = outfile;
2621 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2622 catfp = BytesIO();
2623 else:
2624 fbasename = os.path.splitext(outfile)[0];
2625 fextname = os.path.splitext(outfile)[1];
2626 if(not compresswholefile and fextname in outextlistwd):
2627 compresswholefile = True;
2628 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
2629 catver = formatspecs['format_ver'];
2630 fileheaderver = str(int(catver.replace(".", "")));
2631 infilelist = [];
2632 if(infiles=="-"):
2633 for line in sys.stdin:
2634 infilelist.append(line.strip());
2635 infilelist = list(filter(None, infilelist));
2636 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
2637 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
2638 return False;
2639 with UncompressFile(infiles, formatspecs, "r") as finfile:
2640 for line in finfile:
2641 infilelist.append(line.strip());
2642 infilelist = list(filter(None, infilelist));
2643 else:
2644 if(isinstance(infiles, (list, tuple, ))):
2645 infilelist = list(filter(None, infiles));
2646 elif(isinstance(infiles, (str, ))):
2647 infilelist = list(filter(None, [infiles]));
2648 if(advancedlist):
2649 GetDirList = ListDirAdvanced(infilelist, followlink, False);
2650 else:
2651 GetDirList = ListDir(infilelist, followlink, False);
2652 if(not GetDirList):
2653 return False;
2654 curinode = 0;
2655 curfid = 0;
2656 inodelist = [];
2657 inodetofile = {};
2658 filetoinode = {};
2659 inodetocatinode = {};
2660 numfiles = int(len(GetDirList));
2661 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
2662 for curfname in GetDirList:
2663 if(re.findall(r"^[.|/]", curfname)):
2664 fname = curfname;
2665 else:
2666 fname = "./"+curfname;
2667 if(verbose):
2668 VerbosePrintOut(fname);
2669 if(not followlink or followlink is None):
2670 fstatinfo = os.lstat(fname);
2671 else:
2672 fstatinfo = os.stat(fname);
2673 fpremode = fstatinfo.st_mode;
2674 finode = fstatinfo.st_ino;
2675 flinkcount = fstatinfo.st_nlink;
2676 ftype = 0;
2677 if(stat.S_ISREG(fpremode)):
2678 ftype = 0;
2679 elif(stat.S_ISLNK(fpremode)):
2680 ftype = 2;
2681 elif(stat.S_ISCHR(fpremode)):
2682 ftype = 3;
2683 elif(stat.S_ISBLK(fpremode)):
2684 ftype = 4;
2685 elif(stat.S_ISDIR(fpremode)):
2686 ftype = 5;
2687 elif(stat.S_ISFIFO(fpremode)):
2688 ftype = 6;
2689 elif(stat.S_ISSOCK(fpremode)):
2690 ftype = 8;
2691 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
2692 ftype = 9;
2693 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
2694 ftype = 10;
2695 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
2696 ftype = 11;
2697 else:
2698 ftype = 0;
2699 flinkname = "";
2700 fcurfid = format(int(curfid), 'x').lower();
2701 if(not followlink and finode!=0):
2702 if(ftype!=1):
2703 if(finode in inodelist):
2704 ftype = 1;
2705 flinkname = inodetofile[finode];
2706 if(altinode):
2707 fcurinode = format(int(finode), 'x').lower();
2708 else:
2709 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
2710 if(finode not in inodelist):
2711 inodelist.append(finode);
2712 inodetofile.update({finode: fname});
2713 inodetocatinode.update({finode: curinode});
2714 if(altinode):
2715 fcurinode = format(int(finode), 'x').lower();
2716 else:
2717 fcurinode = format(int(curinode), 'x').lower();
2718 curinode = curinode + 1;
2719 else:
2720 fcurinode = format(int(curinode), 'x').lower();
2721 curinode = curinode + 1;
2722 curfid = curfid + 1;
2723 if(ftype==2):
2724 flinkname = os.readlink(fname);
2725 fdev = fstatinfo.st_dev;
2726 getfdev = GetDevMajorMinor(fdev);
2727 fdev_minor = getfdev[0];
2728 fdev_major = getfdev[1];
2729 frdev = fstatinfo.st_dev;
2730 if(hasattr(fstatinfo, "st_rdev")):
2731 frdev = fstatinfo.st_rdev;
2732 else:
2733 frdev = fstatinfo.st_dev;
2734 getfrdev = GetDevMajorMinor(frdev);
2735 frdev_minor = getfrdev[0];
2736 frdev_major = getfrdev[1];
2737 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
2738 fsize = format(int("0"), 'x').lower();
2739 elif(ftype==0 or ftype==7):
2740 fsize = format(int(fstatinfo.st_size), 'x').lower();
2741 else:
2742 fsize = format(int(fstatinfo.st_size)).lower();
2743 fatime = format(int(fstatinfo.st_atime), 'x').lower();
2744 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
2745 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
2746 if(hasattr(fstatinfo, "st_birthtime")):
2747 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
2748 else:
2749 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
2750 fmode = format(int(fstatinfo.st_mode), 'x').lower();
2751 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
2752 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
2753 fuid = format(int(fstatinfo.st_uid), 'x').lower();
2754 fgid = format(int(fstatinfo.st_gid), 'x').lower();
2755 funame = "";
2756 try:
2757 import pwd;
2758 try:
2759 userinfo = pwd.getpwuid(fstatinfo.st_uid);
2760 funame = userinfo.pw_name;
2761 except KeyError:
2762 funame = "";
2763 except ImportError:
2764 funame = "";
2765 fgname = "";
2766 try:
2767 import grp;
2768 try:
2769 groupinfo = grp.getgrgid(fstatinfo.st_gid);
2770 fgname = groupinfo.gr_name;
2771 except KeyError:
2772 fgname = "";
2773 except ImportError:
2774 fgname = "";
2775 fdev_minor = format(int(fdev_minor), 'x').lower();
2776 fdev_major = format(int(fdev_major), 'x').lower();
2777 frdev_minor = format(int(frdev_minor), 'x').lower();
2778 frdev_major = format(int(frdev_major), 'x').lower();
2779 finode = format(int(finode), 'x').lower();
2780 flinkcount = format(int(flinkcount), 'x').lower();
2781 if(hasattr(fstatinfo, "st_file_attributes")):
2782 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
2783 else:
2784 fwinattributes = format(int(0), 'x').lower();
2785 fcompression = "";
2786 fcsize = format(int(0), 'x').lower();
2787 fcontents = BytesIO();
2788 if(ftype==0 or ftype==7):
2789 with open(fname, "rb") as fpc:
2790 shutil.copyfileobj(fpc, fcontents);
2791 if(not compresswholefile):
2792 fcontents.seek(0, 2);
2793 ucfsize = fcontents.tell();
2794 fcontents.seek(0, 0);
2795 if(compression=="auto"):
2796 ilsize = len(compressionlistalt);
2797 ilmin = 0;
2798 ilcsize = [];
2799 while(ilmin < ilsize):
2800 cfcontents = BytesIO();
2801 shutil.copyfileobj(fcontents, cfcontents);
2802 fcontents.seek(0, 0);
2803 cfcontents.seek(0, 0);
2804 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
2805 if(cfcontents):
2806 cfcontents.seek(0, 2);
2807 ilcsize.append(cfcontents.tell());
2808 cfcontents.close();
2809 else:
2810 try:
2811 ilcsize.append(sys.maxint);
2812 except AttributeError:
2813 ilcsize.append(sys.maxsize);
2814 ilmin = ilmin + 1;
2815 ilcmin = ilcsize.index(min(ilcsize));
2816 compression = compressionlistalt[ilcmin];
2817 fcontents.seek(0, 0);
2818 cfcontents = BytesIO();
2819 shutil.copyfileobj(fcontents, cfcontents);
2820 cfcontents.seek(0, 0);
2821 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
2822 cfcontents.seek(0, 2);
2823 cfsize = cfcontents.tell();
2824 if(ucfsize > cfsize):
2825 fcsize = format(int(cfsize), 'x').lower();
2826 fcompression = compression;
2827 fcontents.close();
2828 fcontents = cfcontents;
2829 if(fcompression=="none"):
2830 fcompression = "";
2831 if(followlink and (ftype==1 or ftype==2)):
2832 flstatinfo = os.stat(flinkname);
2833 with open(flinkname, "rb") as fpc:
2834 shutil.copyfileobj(fpc, fcontents);
2835 if(not compresswholefile):
2836 fcontents.seek(0, 2);
2837 ucfsize = fcontents.tell();
2838 fcontents.seek(0, 0);
2839 if(compression=="auto"):
2840 ilsize = len(compressionlistalt);
2841 ilmin = 0;
2842 ilcsize = [];
2843 while(ilmin < ilsize):
2844 cfcontents = BytesIO();
2845 shutil.copyfileobj(fcontents, cfcontents);
2846 fcontents.seek(0, 0);
2847 cfcontents.seek(0, 0);
2848 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
2849 if(cfcontents):
2850 cfcontents.seek(0, 2);
2851 ilcsize.append(cfcontents.tell());
2852 cfcontents.close();
2853 else:
2854 try:
2855 ilcsize.append(sys.maxint);
2856 except AttributeError:
2857 ilcsize.append(sys.maxsize);
2858 ilmin = ilmin + 1;
2859 ilcmin = ilcsize.index(min(ilcsize));
2860 compression = compressionlistalt[ilcmin];
2861 fcontents.seek(0, 0);
2862 cfcontents = BytesIO();
2863 shutil.copyfileobj(fcontents, cfcontents);
2864 cfcontents.seek(0, 0);
2865 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
2866 cfcontents.seek(0, 2);
2867 cfsize = cfcontents.tell();
2868 if(ucfsize > cfsize):
2869 fcsize = format(int(cfsize), 'x').lower();
2870 fcompression = compression;
2871 fcontents.close();
2872 fcontents = cfcontents;
2873 fcontents.seek(0, 0);
2874 ftypehex = format(ftype, 'x').lower();
2875 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
2876 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
2877 fcontents.close();
2878 if(numfiles>0):
2879 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
2880 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
2881 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2882 try:
2883 catfp.flush();
2884 os.fsync(catfp.fileno());
2885 except io.UnsupportedOperation:
2886 pass;
2887 except AttributeError:
2888 pass;
2889 except OSError as e:
2890 pass;
2891 if(outfile=="-"):
2892 catfp.seek(0, 0);
2893 if(hasattr(sys.stdout, "buffer")):
2894 shutil.copyfileobj(catfp, sys.stdout.buffer);
2895 else:
2896 shutil.copyfileobj(catfp, sys.stdout);
2897 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2898 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2899 catfp.seek(0, 0);
2900 upload_file_to_internet_file(catfp, outfile);
2901 if(returnfp):
2902 catfp.seek(0, 0);
2903 return catfp;
2904 else:
2905 catfp.close();
2906 return True;
2907
2908 create_alias_function("Pack", __file_format_name__, "", PackArchiveFile);
2909
2910 if(hasattr(shutil, "register_archive_format")):
2911 def PackArchiveFileFunc(archive_name, source_dir, **kwargs):
2912 return PackArchiveFile(source_dir, archive_name, False, "auto", True, None, False, "crc32", [], __file_format_dict__['format_delimiter'], False, False);
2913 create_alias_function("Pack", __file_format_name__, "Func", PackArchiveFileFunc);
2914
2915 def PackArchiveFileFromDirList(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2916 formatspecs = FormatSpecsListToDict(formatspecs);
2917 return PackArchiveFile(infiles, outfile, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, returnfp);
2918
2919 create_alias_function("Pack", __file_format_name__, "FromDirList", PackArchiveFileFromDirList);
2920
2921 def PackArchiveFileFromTarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2922 formatspecs = FormatSpecsListToDict(formatspecs);
2923 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2924 outfile = RemoveWindowsPath(outfile);
2925 checksumtype = checksumtype.lower();
2926 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
2927 checksumtype="crc32";
2928 if(checksumtype=="none"):
2929 checksumtype = "";
2930 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
2931 compression = "auto";
2932 if(compression not in compressionlist and compression is None):
2933 compression = "auto";
2934 if(verbose):
2935 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2936 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2937 if(os.path.exists(outfile)):
2938 try:
2939 os.unlink(outfile);
2940 except OSError as e:
2941 pass;
2942 if(outfile=="-"):
2943 verbose = False;
2944 catfp = BytesIO();
2945 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2946 catfp = outfile;
2947 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2948 catfp = BytesIO();
2949 else:
2950 fbasename = os.path.splitext(outfile)[0];
2951 fextname = os.path.splitext(outfile)[1];
2952 if(not compresswholefile and fextname in outextlistwd):
2953 compresswholefile = True;
2954 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
2955 catver = formatspecs['format_ver'];
2956 fileheaderver = str(int(catver.replace(".", "")));
2957 curinode = 0;
2958 curfid = 0;
2959 inodelist = [];
2960 inodetofile = {};
2961 filetoinode = {};
2962 inodetocatinode = {};
2963 if(infile=="-"):
2964 infile = BytesIO();
2965 if(hasattr(sys.stdin, "buffer")):
2966 shutil.copyfileobj(sys.stdin.buffer, infile);
2967 else:
2968 shutil.copyfileobj(sys.stdin, infile);
2969 infile.seek(0, 0);
2970 if(not infile):
2971 return False;
2972 infile.seek(0, 0);
2973 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
2974 infile = download_file_from_internet_file(infile);
2975 infile.seek(0, 0);
2976 if(not infile):
2977 return False;
2978 infile.seek(0, 0);
2979 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
2980 return False;
2981 elif(os.path.exists(infile) and os.path.isfile(infile)):
2982 try:
2983 if(not tarfile.TarFileCheck(infile)):
2984 return False;
2985 except AttributeError:
2986 if(not TarFileCheck(infile)):
2987 return False;
2988 else:
2989 return False;
2990 try:
2991 if(hasattr(infile, "read") or hasattr(infile, "write")):
2992 tarfp = tarfile.open(fileobj=infile, mode="r");
2993 else:
2994 tarfp = tarfile.open(infile, "r");
2995 except FileNotFoundError:
2996 return False;
2997 numfiles = int(len(tarfp.getmembers()));
2998 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
2999 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
3000 if(re.findall(r"^[.|/]", member.name)):
3001 fname = member.name;
3002 else:
3003 fname = "./"+member.name;
3004 if(verbose):
3005 VerbosePrintOut(fname);
3006 fpremode = member.mode;
3007 ffullmode = member.mode;
3008 flinkcount = 0;
3009 ftype = 0;
3010 if(member.isreg()):
3011 ffullmode = member.mode + stat.S_IFREG;
3012 ftype = 0;
3013 elif(member.isdev()):
3014 ffullmode = member.mode;
3015 ftype = 7;
3016 elif(member.islnk()):
3017 ffullmode = member.mode + stat.S_IFREG;
3018 ftype = 1;
3019 elif(member.issym()):
3020 ffullmode = member.mode + stat.S_IFLNK;
3021 ftype = 2;
3022 elif(member.ischr()):
3023 ffullmode = member.mode + stat.S_IFCHR;
3024 ftype = 3;
3025 elif(member.isblk()):
3026 ffullmode = member.mode + stat.S_IFBLK;
3027 ftype = 4;
3028 elif(member.isdir()):
3029 ffullmode = member.mode + stat.S_IFDIR;
3030 ftype = 5;
3031 elif(member.isfifo()):
3032 ffullmode = member.mode + stat.S_IFIFO;
3033 ftype = 6;
3034 elif(member.issparse()):
3035 ffullmode = member.mode;
3036 ftype = 12;
3037 else:
3038 ffullmode = member.mode;
3039 ftype = 0;
3040 flinkname = "";
3041 fcurfid = format(int(curfid), 'x').lower();
3042 fcurinode = format(int(curfid), 'x').lower();
3043 curfid = curfid + 1;
3044 if(ftype==2):
3045 flinkname = member.linkname;
3046 fdev_minor = format(int(member.devminor), 'x').lower();
3047 fdev_major = format(int(member.devmajor), 'x').lower();
3048 frdev_minor = format(int(member.devminor), 'x').lower();
3049 frdev_major = format(int(member.devmajor), 'x').lower();
3050 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
3051 fsize = format(int("0"), 'x').lower();
3052 elif(ftype==0 or ftype==7):
3053 fsize = format(int(member.size), 'x').lower();
3054 else:
3055 fsize = format(int(member.size), 'x').lower();
3056 fatime = format(int(member.mtime), 'x').lower();
3057 fmtime = format(int(member.mtime), 'x').lower();
3058 fctime = format(int(member.mtime), 'x').lower();
3059 fbtime = format(int(member.mtime), 'x').lower();
3060 fmode = format(int(ffullmode), 'x').lower();
3061 fchmode = format(int(stat.S_IMODE(ffullmode)), 'x').lower();
3062 ftypemod = format(int(stat.S_IFMT(ffullmode)), 'x').lower();
3063 fuid = format(int(member.uid), 'x').lower();
3064 fgid = format(int(member.gid), 'x').lower();
3065 funame = member.uname;
3066 fgname = member.gname;
3067 flinkcount = format(int(flinkcount), 'x').lower();
3068 fwinattributes = format(int(0), 'x').lower();
3069 fcompression = "";
3070 fcsize = format(int(0), 'x').lower();
3071 fcontents = BytesIO();
3072 if(ftype==0 or ftype==7):
3073 with tarfp.extractfile(member) as fpc:
3074 shutil.copyfileobj(fpc, fcontents);
3075 if(not compresswholefile):
3076 fcontents.seek(0, 2);
3077 ucfsize = fcontents.tell();
3078 fcontents.seek(0, 0);
3079 if(compression=="auto"):
3080 ilsize = len(compressionlistalt);
3081 ilmin = 0;
3082 ilcsize = [];
3083 while(ilmin < ilsize):
3084 cfcontents = BytesIO();
3085 shutil.copyfileobj(fcontents, cfcontents);
3086 fcontents.seek(0, 0);
3087 cfcontents.seek(0, 0);
3088 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3089 if(cfcontents):
3090 cfcontents.seek(0, 2);
3091 ilcsize.append(cfcontents.tell());
3092 cfcontents.close();
3093 else:
3094 try:
3095 ilcsize.append(sys.maxint);
3096 except AttributeError:
3097 ilcsize.append(sys.maxsize);
3098 ilmin = ilmin + 1;
3099 ilcmin = ilcsize.index(min(ilcsize));
3100 compression = compressionlistalt[ilcmin];
3101 fcontents.seek(0, 0);
3102 cfcontents = BytesIO();
3103 shutil.copyfileobj(fcontents, cfcontents);
3104 cfcontents.seek(0, 0);
3105 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3106 cfcontents.seek(0, 2);
3107 cfsize = cfcontents.tell();
3108 if(ucfsize > cfsize):
3109 fcsize = format(int(cfsize), 'x').lower();
3110 fcompression = compression;
3111 fcontents.close();
3112 fcontents = cfcontents;
3113 if(fcompression=="none"):
3114 fcompression = "";
3115 fcontents.seek(0, 0);
3116 ftypehex = format(ftype, 'x').lower();
3117 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
3118 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3119 fcontents.close();
3120 if(numfiles>0):
3121 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3122 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3123 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3124 try:
3125 catfp.flush();
3126 os.fsync(catfp.fileno());
3127 except io.UnsupportedOperation:
3128 pass;
3129 except AttributeError:
3130 pass;
3131 except OSError as e:
3132 pass;
3133 if(outfile=="-"):
3134 catfp.seek(0, 0);
3135 if(hasattr(sys.stdout, "buffer")):
3136 shutil.copyfileobj(catfp, sys.stdout.buffer);
3137 else:
3138 shutil.copyfileobj(catfp, sys.stdout);
3139 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3140 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3141 catfp.seek(0, 0);
3142 upload_file_to_internet_file(catfp, outfile);
3143 if(returnfp):
3144 catfp.seek(0, 0);
3145 return catfp;
3146 else:
3147 catfp.close();
3148 return True;
3149
3150 create_alias_function("Pack", __file_format_name__, "FromTarFile", PackArchiveFileFromTarFile);
3151
3152 def PackArchiveFileFromZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3153 formatspecs = FormatSpecsListToDict(formatspecs);
3154 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3155 outfile = RemoveWindowsPath(outfile);
3156 checksumtype = checksumtype.lower();
3157 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3158 checksumtype="crc32";
3159 if(checksumtype=="none"):
3160 checksumtype = "";
3161 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3162 compression = "auto";
3163 if(compression not in compressionlist and compression is None):
3164 compression = "auto";
3165 if(verbose):
3166 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3167 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3168 if(os.path.exists(outfile)):
3169 try:
3170 os.unlink(outfile);
3171 except OSError as e:
3172 pass;
3173 if(outfile=="-"):
3174 verbose = False;
3175 catfp = BytesIO();
3176 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3177 catfp = outfile;
3178 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3179 catfp = BytesIO();
3180 else:
3181 fbasename = os.path.splitext(outfile)[0];
3182 fextname = os.path.splitext(outfile)[1];
3183 if(not compresswholefile and fextname in outextlistwd):
3184 compresswholefile = True;
3185 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3186 catver = formatspecs['format_ver'];
3187 fileheaderver = str(int(catver.replace(".", "")));
3188 curinode = 0;
3189 curfid = 0;
3190 inodelist = [];
3191 inodetofile = {};
3192 filetoinode = {};
3193 inodetocatinode = {};
3194 if(infile=="-"):
3195 infile = BytesIO();
3196 if(hasattr(sys.stdin, "buffer")):
3197 shutil.copyfileobj(sys.stdin.buffer, infile);
3198 else:
3199 shutil.copyfileobj(sys.stdin, infile);
3200 infile.seek(0, 0);
3201 if(not infile):
3202 return False;
3203 infile.seek(0, 0);
3204 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3205 infile = download_file_from_internet_file(infile);
3206 infile.seek(0, 0);
3207 if(not infile):
3208 return False;
3209 infile.seek(0, 0);
3210 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
3211 return False;
3212 else:
3213 return False;
3214 if(not zipfile.is_zipfile(infile)):
3215 return False;
3216 try:
3217 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
3218 except FileNotFoundError:
3219 return False;
3220 ziptest = zipfp.testzip();
3221 if(ziptest):
3222 VerbosePrintOut("Bad file found!");
3223 numfiles = int(len(zipfp.infolist()));
3224 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3225 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
3226 if(re.findall(r"^[.|/]", member.filename)):
3227 fname = member.filename;
3228 else:
3229 fname = "./"+member.filename;
3230 zipinfo = zipfp.getinfo(member.filename);
3231 if(verbose):
3232 VerbosePrintOut(fname);
3233 if(not member.is_dir()):
3234 fpremode = int(stat.S_IFREG + 438);
3235 elif(member.is_dir()):
3236 fpremode = int(stat.S_IFDIR + 511);
3237 flinkcount = 0;
3238 ftype = 0;
3239 if(not member.is_dir()):
3240 ftype = 0;
3241 elif(member.is_dir()):
3242 ftype = 5;
3243 flinkname = "";
3244 fcurfid = format(int(curfid), 'x').lower();
3245 fcurinode = format(int(curfid), 'x').lower();
3246 curfid = curfid + 1;
3247 fdev_minor = format(int(0), 'x').lower();
3248 fdev_major = format(int(0), 'x').lower();
3249 frdev_minor = format(int(0), 'x').lower();
3250 frdev_major = format(int(0), 'x').lower();
3251 if(ftype==5):
3252 fsize = format(int("0"), 'x').lower();
3253 elif(ftype==0):
3254 fsize = format(int(member.file_size), 'x').lower();
3255 else:
3256 fsize = format(int(member.file_size), 'x').lower();
3257 fatime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3258 fmtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3259 fctime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3260 fbtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3261 if(zipinfo.create_system==0 or zipinfo.create_system==10):
3262 fwinattributes = format(int(zipinfo.external_attr), 'x').lower();
3263 if(not member.is_dir()):
3264 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3265 fchmode = stat.S_IMODE(fmode);
3266 ftypemod = stat.S_IFMT(fmode);
3267 elif(member.is_dir()):
3268 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3269 fchmode = stat.S_IMODE(fmode);
3270 ftypemod = stat.S_IFMT(fmode);
3271 elif(zipinfo.create_system==3):
3272 fwinattributes = format(int(0), 'x').lower();
3273 fmode = format(int(zipinfo.external_attr), 'x').lower();
3274 fchmode = stat.S_IMODE(fmode);
3275 ftypemod = stat.S_IFMT(fmode);
3276 else:
3277 fwinattributes = format(int(0), 'x').lower();
3278 if(not member.is_dir()):
3279 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3280 fchmode = stat.S_IMODE(fmode);
3281 ftypemod = stat.S_IFMT(fmode);
3282 elif(member.is_dir()):
3283 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3284 fchmode = stat.S_IMODE(fmode);
3285 ftypemod = stat.S_IFMT(fmode);
3286 fcompression = "";
3287 fcsize = format(int(0), 'x').lower();
3288 try:
3289 fuid = format(int(os.getuid()), 'x').lower();
3290 except AttributeError:
3291 fuid = format(int(0), 'x').lower();
3292 except KeyError:
3293 fuid = format(int(0), 'x').lower();
3294 try:
3295 fgid = format(int(os.getgid()), 'x').lower();
3296 except AttributeError:
3297 fgid = format(int(0), 'x').lower();
3298 except KeyError:
3299 fgid = format(int(0), 'x').lower();
3300 try:
3301 import pwd;
3302 try:
3303 userinfo = pwd.getpwuid(os.getuid());
3304 funame = userinfo.pw_name;
3305 except KeyError:
3306 funame = "";
3307 except AttributeError:
3308 funame = "";
3309 except ImportError:
3310 funame = "";
3311 fgname = "";
3312 try:
3313 import grp;
3314 try:
3315 groupinfo = grp.getgrgid(os.getgid());
3316 fgname = groupinfo.gr_name;
3317 except KeyError:
3318 fgname = "";
3319 except AttributeError:
3320 fgname = "";
3321 except ImportError:
3322 fgname = "";
3323 fcontents = BytesIO();
3324 if(ftype==0):
3325 fcontents.write(zipfp.read(member.filename));
3326 if(not compresswholefile):
3327 fcontents.seek(0, 2);
3328 ucfsize = fcontents.tell();
3329 fcontents.seek(0, 0);
3330 if(compression=="auto"):
3331 ilsize = len(compressionlistalt);
3332 ilmin = 0;
3333 ilcsize = [];
3334 while(ilmin < ilsize):
3335 cfcontents = BytesIO();
3336 shutil.copyfileobj(fcontents, cfcontents);
3337 fcontents.seek(0, 0);
3338 cfcontents.seek(0, 0);
3339 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3340 cfcontents.seek(0, 2);
3341 ilcsize.append(cfcontents.tell());
3342 cfcontents.close();
3343 ilmin = ilmin + 1;
3344 ilcmin = ilcsize.index(min(ilcsize));
3345 compression = compressionlistalt[ilcmin];
3346 fcontents.seek(0, 0);
3347 cfcontents = BytesIO();
3348 shutil.copyfileobj(fcontents, cfcontents);
3349 cfcontents.seek(0, 0);
3350 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3351 cfcontents.seek(0, 2);
3352 cfsize = cfcontents.tell();
3353 if(ucfsize > cfsize):
3354 fcsize = format(int(cfsize), 'x').lower();
3355 fcompression = compression;
3356 fcontents.close();
3357 fcontents = cfcontents;
3358 if(fcompression=="none"):
3359 fcompression = "";
3360 fcontents.seek(0, 0);
3361 ftypehex = format(ftype, 'x').lower();
3362 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
3363 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3364 fcontents.close();
3365 if(numfiles>0):
3366 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3367 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3368 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3369 try:
3370 catfp.flush();
3371 os.fsync(catfp.fileno());
3372 except io.UnsupportedOperation:
3373 pass;
3374 except AttributeError:
3375 pass;
3376 except OSError as e:
3377 pass;
3378 if(outfile=="-"):
3379 catfp.seek(0, 0);
3380 if(hasattr(sys.stdout, "buffer")):
3381 shutil.copyfileobj(catfp, sys.stdout.buffer);
3382 else:
3383 shutil.copyfileobj(catfp, sys.stdout);
3384 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3385 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3386 catfp.seek(0, 0);
3387 upload_file_to_internet_file(catfp, outfile);
3388 if(returnfp):
3389 catfp.seek(0, 0);
3390 return catfp;
3391 else:
3392 catfp.close();
3393 return True;
3394
3395 create_alias_function("Pack", __file_format_name__, "FromZipFile", PackArchiveFileFromZipFile);
3396
3397 if(not rarfile_support):
3398 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3399 return False
3400
3401 if(rarfile_support):
3402 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3403 formatspecs = FormatSpecsListToDict(formatspecs);
3404 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3405 outfile = RemoveWindowsPath(outfile);
3406 checksumtype = checksumtype.lower();
3407 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3408 checksumtype="crc32";
3409 if(checksumtype=="none"):
3410 checksumtype = "";
3411 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3412 compression = "auto";
3413 if(compression not in compressionlist and compression is None):
3414 compression = "auto";
3415 if(verbose):
3416 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3417 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3418 if(os.path.exists(outfile)):
3419 try:
3420 os.unlink(outfile);
3421 except OSError as e:
3422 pass;
3423 if(outfile=="-"):
3424 verbose = False;
3425 catfp = BytesIO();
3426 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3427 catfp = outfile;
3428 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3429 catfp = BytesIO();
3430 else:
3431 fbasename = os.path.splitext(outfile)[0];
3432 fextname = os.path.splitext(outfile)[1];
3433 if(not compresswholefile and fextname in outextlistwd):
3434 compresswholefile = True;
3435 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3436 catver = formatspecs['format_ver'];
3437 fileheaderver = str(int(catver.replace(".", "")));
3438 curinode = 0;
3439 curfid = 0;
3440 inodelist = [];
3441 inodetofile = {};
3442 filetoinode = {};
3443 inodetocatinode = {};
3444 if(not os.path.exists(infile) or not os.path.isfile(infile)):
3445 return False;
3446 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
3447 return False;
3448 rarfp = rarfile.RarFile(infile, "r");
3449 rartest = rarfp.testrar();
3450 if(rartest):
3451 VerbosePrintOut("Bad file found!");
3452 numfiles = int(len(rarfp.infolist()));
3453 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3454 try:
3455 catfp.flush();
3456 os.fsync(catfp.fileno());
3457 except io.UnsupportedOperation:
3458 pass;
3459 except AttributeError:
3460 pass;
3461 except OSError as e:
3462 pass;
3463 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
3464 is_unix = False;
3465 is_windows = False;
3466 if(member.host_os==rarfile.RAR_OS_UNIX):
3467 is_windows = False;
3468 try:
3469 member.external_attr
3470 is_unix = True;
3471 except AttributeError:
3472 is_unix = False;
3473 elif(member.host_os==rarfile.RAR_OS_WIN32):
3474 is_unix = False;
3475 try:
3476 member.external_attr
3477 is_windows = True;
3478 except AttributeError:
3479 is_windows = False;
3480 else:
3481 is_unix = False;
3482 is_windows = False;
3483 if(re.findall(r"^[.|/]", member.filename)):
3484 fname = member.filename;
3485 else:
3486 fname = "./"+member.filename;
3487 rarinfo = rarfp.getinfo(member.filename);
3488 if(verbose):
3489 VerbosePrintOut(fname);
3490 if(is_unix and member.external_attr !=0):
3491 fpremode = int(member.external_attr);
3492 elif(member.is_file()):
3493 fpremode = int(stat.S_IFREG + 438);
3494 elif(member.is_symlink()):
3495 fpremode = int(stat.S_IFLNK + 438);
3496 elif(member.is_dir()):
3497 fpremode = int(stat.S_IFDIR + 511);
3498 if(is_windows and member.external_attr !=0):
3499 fwinattributes = format(int(member.external_attr), 'x').lower();
3500 else:
3501 fwinattributes = format(int(0), 'x').lower();
3502 fcompression = "";
3503 fcsize = format(int(0), 'x').lower();
3504 flinkcount = 0;
3505 ftype = 0;
3506 if(member.is_file()):
3507 ftype = 0;
3508 elif(member.is_symlink()):
3509 ftype = 2;
3510 elif(member.is_dir()):
3511 ftype = 5;
3512 flinkname = "";
3513 if(ftype==2):
3514 flinkname = rarfp.read(member.filename).decode("UTF-8");
3515 fcurfid = format(int(curfid), 'x').lower();
3516 fcurinode = format(int(curfid), 'x').lower();
3517 curfid = curfid + 1;
3518 fdev_minor = format(int(0), 'x').lower();
3519 fdev_major = format(int(0), 'x').lower();
3520 frdev_minor = format(int(0), 'x').lower();
3521 frdev_major = format(int(0), 'x').lower();
3522 if(ftype==5):
3523 fsize = format(int("0"), 'x').lower();
3524 elif(ftype==0):
3525 fsize = format(int(member.file_size), 'x').lower();
3526 else:
3527 fsize = format(int(member.file_size), 'x').lower();
3528 try:
3529 if(member.atime):
3530 fatime = format(int(member.atime.timestamp()), 'x').lower();
3531 else:
3532 fatime = format(int(member.mtime.timestamp()), 'x').lower();
3533 except AttributeError:
3534 fatime = format(int(member.mtime.timestamp()), 'x').lower();
3535 fmtime = format(int(member.mtime.timestamp()), 'x').lower();
3536 try:
3537 if(member.ctime):
3538 fctime = format(int(member.ctime.timestamp()), 'x').lower();
3539 else:
3540 fctime = format(int(member.mtime.timestamp()), 'x').lower();
3541 except AttributeError:
3542 fctime = format(int(member.mtime.timestamp()), 'x').lower();
3543 fbtime = format(int(member.mtime.timestamp()), 'x').lower();
3544 if(is_unix and member.external_attr !=0):
3545 fmode = format(int(member.external_attr), 'x').lower();
3546 fchmode = format(int(stat.S_IMODE(member.external_attr)), 'x').lower();
3547 ftypemod = format(int(stat.S_IFMT(member.external_attr)), 'x').lower();
3548 elif(member.is_file()):
3549 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3550 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3551 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3552 elif(member.is_symlink()):
3553 fmode = format(int(stat.S_IFLNK + 438), 'x').lower();
3554 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3555 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3556 elif(member.is_dir()):
3557 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3558 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
3559 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
3560 try:
3561 fuid = format(int(os.getuid()), 'x').lower();
3562 except AttributeError:
3563 fuid = format(int(0), 'x').lower();
3564 except KeyError:
3565 fuid = format(int(0), 'x').lower();
3566 try:
3567 fgid = format(int(os.getgid()), 'x').lower();
3568 except AttributeError:
3569 fgid = format(int(0), 'x').lower();
3570 except KeyError:
3571 fgid = format(int(0), 'x').lower();
3572 try:
3573 import pwd;
3574 try:
3575 userinfo = pwd.getpwuid(os.getuid());
3576 funame = userinfo.pw_name;
3577 except KeyError:
3578 funame = "";
3579 except AttributeError:
3580 funame = "";
3581 except ImportError:
3582 funame = "";
3583 fgname = "";
3584 try:
3585 import grp;
3586 try:
3587 groupinfo = grp.getgrgid(os.getgid());
3588 fgname = groupinfo.gr_name;
3589 except KeyError:
3590 fgname = "";
3591 except AttributeError:
3592 fgname = "";
3593 except ImportError:
3594 fgname = "";
3595 fcontents = BytesIO();
3596 if(ftype==0):
3597 fcontents.write(rarfp.read(member.filename));
3598 if(not compresswholefile):
3599 fcontents.seek(0, 2);
3600 ucfsize = fcontents.tell();
3601 fcontents.seek(0, 0);
3602 if(compression=="auto"):
3603 ilsize = len(compressionlistalt);
3604 ilmin = 0;
3605 ilcsize = [];
3606 while(ilmin < ilsize):
3607 cfcontents = BytesIO();
3608 shutil.copyfileobj(fcontents, cfcontents);
3609 fcontents.seek(0, 0);
3610 cfcontents.seek(0, 0);
3611 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3612 if(cfcontents):
3613 cfcontents.seek(0, 2);
3614 ilcsize.append(cfcontents.tell());
3615 cfcontents.close();
3616 else:
3617 try:
3618 ilcsize.append(sys.maxint);
3619 except AttributeError:
3620 ilcsize.append(sys.maxsize);
3621 ilmin = ilmin + 1;
3622 ilcmin = ilcsize.index(min(ilcsize));
3623 compression = compressionlistalt[ilcmin];
3624 fcontents.seek(0, 0);
3625 cfcontents = BytesIO();
3626 shutil.copyfileobj(fcontents, cfcontents);
3627 cfcontents.seek(0, 0);
3628 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3629 cfcontents.seek(0, 2);
3630 cfsize = cfcontents.tell();
3631 if(ucfsize > cfsize):
3632 fcsize = format(int(cfsize), 'x').lower();
3633 fcompression = compression;
3634 fcontents.close();
3635 fcontents = cfcontents;
3636 if(fcompression=="none"):
3637 fcompression = "";
3638 fcontents.seek(0, 0);
3639 ftypehex = format(ftype, 'x').lower();
3640 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
3641 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3642 fcontents.close();
3643 if(numfiles>0):
3644 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3645 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3646 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
3647 try:
3648 catfp.flush();
3649 os.fsync(catfp.fileno());
3650 except io.UnsupportedOperation:
3651 pass;
3652 except AttributeError:
3653 pass;
3654 except OSError as e:
3655 pass;
3656 if(outfile=="-"):
3657 catfp.seek(0, 0)
3658 if(hasattr(sys.stdout, "buffer")):
3659 shutil.copyfileobj(catfp, sys.stdout.buffer);
3660 else:
3661 shutil.copyfileobj(catfp, sys.stdout);
3662 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3663 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3664 catfp.seek(0, 0);
3665 upload_file_to_internet_file(catfp, outfile);
3666 if(returnfp):
3667 catfp.seek(0, 0)
3668 return catfp
3669 else:
3670 catfp.close()
3671 return True;
3672
3673 create_alias_function("Pack", __file_format_name__, "FromRarFile", PackArchiveFileFromRarFile);
3674
3675 if(not py7zr_support):
3676 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3677 return False
3678
3679 if(py7zr_support):
3680 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3681 formatspecs = FormatSpecsListToDict(formatspecs);
3682 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3683 outfile = RemoveWindowsPath(outfile);
3684 checksumtype = checksumtype.lower();
3685 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3686 checksumtype="crc32";
3687 if(checksumtype=="none"):
3688 checksumtype = "";
3689 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3690 compression = "auto";
3691 if(compression not in compressionlist and compression is None):
3692 compression = "auto";
3693 if(verbose):
3694 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3695 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3696 if(os.path.exists(outfile)):
3697 try:
3698 os.unlink(outfile);
3699 except OSError as e:
3700 pass;
3701 if(outfile=="-"):
3702 verbose = False;
3703 catfp = BytesIO();
3704 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3705 catfp = outfile;
3706 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3707 catfp = BytesIO();
3708 else:
3709 fbasename = os.path.splitext(outfile)[0];
3710 fextname = os.path.splitext(outfile)[1];
3711 if(not compresswholefile and fextname in outextlistwd):
3712 compresswholefile = True;
3713 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3714 catver = formatspecs['format_ver'];
3715 fileheaderver = str(int(catver.replace(".", "")));
3716 curinode = 0;
3717 curfid = 0;
3718 inodelist = [];
3719 inodetofile = {};
3720 filetoinode = {};
3721 inodetocatinode = {};
3722 if(not os.path.exists(infile) or not os.path.isfile(infile)):
3723 return False;
3724 szpfp = py7zr.SevenZipFile(infile, mode="r");
3725 file_content = szpfp.readall();
3726 #sztest = szpfp.testzip();
3727 sztestalt = szpfp.test();
3728 if(sztestalt):
3729 VerbosePrintOut("Bad file found!");
3730 numfiles = int(len(szpfp.list()));
3731 AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3732 for member in sorted(szpfp.list(), key=lambda x: x.filename):
3733 if(re.findall(r"^[.|/]", member.filename)):
3734 fname = member.filename;
3735 else:
3736 fname = "./"+member.filename;
3737 if(verbose):
3738 VerbosePrintOut(fname);
3739 if(not member.is_directory):
3740 fpremode = int(stat.S_IFREG + 438);
3741 elif(member.is_directory):
3742 fpremode = int(stat.S_IFDIR + 511);
3743 fwinattributes = format(int(0), 'x').lower();
3744 fcompression = "";
3745 fcsize = format(int(0), 'x').lower();
3746 flinkcount = 0;
3747 ftype = 0;
3748 if(member.is_directory):
3749 ftype = 5;
3750 else:
3751 ftype = 0;
3752 flinkname = "";
3753 fcurfid = format(int(curfid), 'x').lower();
3754 fcurinode = format(int(curfid), 'x').lower();
3755 curfid = curfid + 1;
3756 fdev_minor = format(int(0), 'x').lower();
3757 fdev_major = format(int(0), 'x').lower();
3758 frdev_minor = format(int(0), 'x').lower();
3759 frdev_major = format(int(0), 'x').lower();
3760 if(ftype==5):
3761 fsize = format(int("0"), 'x').lower();
3762 fatime = format(int(member.creationtime.timestamp()), 'x').lower();
3763 fmtime = format(int(member.creationtime.timestamp()), 'x').lower();
3764 fctime = format(int(member.creationtime.timestamp()), 'x').lower();
3765 fbtime = format(int(member.creationtime.timestamp()), 'x').lower();
3766 if(member.is_directory):
3767 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3768 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
3769 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
3770 else:
3771 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3772 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3773 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3774 try:
3775 fuid = format(int(os.getuid()), 'x').lower();
3776 except AttributeError:
3777 fuid = format(int(0), 'x').lower();
3778 except KeyError:
3779 fuid = format(int(0), 'x').lower();
3780 try:
3781 fgid = format(int(os.getgid()), 'x').lower();
3782 except AttributeError:
3783 fgid = format(int(0), 'x').lower();
3784 except KeyError:
3785 fgid = format(int(0), 'x').lower();
3786 try:
3787 import pwd;
3788 try:
3789 userinfo = pwd.getpwuid(os.getuid());
3790 funame = userinfo.pw_name;
3791 except KeyError:
3792 funame = "";
3793 except AttributeError:
3794 funame = "";
3795 except ImportError:
3796 funame = "";
3797 fgname = "";
3798 try:
3799 import grp;
3800 try:
3801 groupinfo = grp.getgrgid(os.getgid());
3802 fgname = groupinfo.gr_name;
3803 except KeyError:
3804 fgname = "";
3805 except AttributeError:
3806 fgname = "";
3807 except ImportError:
3808 fgname = "";
3809 fcontents = BytesIO();
3810 if(ftype==0):
3811 fcontents.write(file_content[member.filename].read());
3812 fsize = format(fcontents.tell(), 'x').lower();
3813 file_content[member.filename].close();
3814 if(not compresswholefile):
3815 fcontents.seek(0, 2);
3816 ucfsize = fcontents.tell();
3817 fcontents.seek(0, 0);
3818 if(compression=="auto"):
3819 ilsize = len(compressionlistalt);
3820 ilmin = 0;
3821 ilcsize = [];
3822 while(ilmin < ilsize):
3823 cfcontents = BytesIO();
3824 shutil.copyfileobj(fcontents, cfcontents);
3825 fcontents.seek(0, 0);
3826 cfcontents.seek(0, 0);
3827 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3828 if(cfcontents):
3829 cfcontents.seek(0, 2);
3830 ilcsize.append(cfcontents.tell());
3831 cfcontents.close();
3832 else:
3833 try:
3834 ilcsize.append(sys.maxint);
3835 except AttributeError:
3836 ilcsize.append(sys.maxsize);
3837 ilmin = ilmin + 1;
3838 ilcmin = ilcsize.index(min(ilcsize));
3839 compression = compressionlistalt[ilcmin];
3840 fcontents.seek(0, 0);
3841 cfcontents = BytesIO();
3842 shutil.copyfileobj(fcontents, cfcontents);
3843 cfcontents.seek(0, 0);
3844 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3845 cfcontents.seek(0, 2);
3846 cfsize = cfcontents.tell();
3847 if(ucfsize > cfsize):
3848 fcsize = format(int(cfsize), 'x').lower();
3849 fcompression = compression;
3850 fcontents.close();
3851 fcontents = cfcontents;
3852 if(fcompression=="none"):
3853 fcompression = "";
3854 fcontents.seek(0, 0);
3855 ftypehex = format(ftype, 'x').lower();
3856 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, "+1"];
3857 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3858 fcontents.close();
3859 if(numfiles>0):
3860 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3861 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3862 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
3863 try:
3864 catfp.flush();
3865 os.fsync(catfp.fileno());
3866 except io.UnsupportedOperation:
3867 pass;
3868 except AttributeError:
3869 pass;
3870 except OSError as e:
3871 pass;
3872 if(outfile=="-"):
3873 catfp.seek(0, 0);
3874 if(hasattr(sys.stdout, "buffer")):
3875 shutil.copyfileobj(catfp, sys.stdout.buffer);
3876 else:
3877 shutil.copyfileobj(catfp, sys.stdout);
3878 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3879 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3880 catfp.seek(0, 0);
3881 upload_file_to_internet_file(catfp, outfile);
3882 if(returnfp):
3883 catfp.seek(0, 0)
3884 return catfp
3885 else:
3886 catfp.close()
3887 return True;
3888
3889 create_alias_function("Pack", __file_format_name__, "FromSevenZipFile", PackArchiveFileFromSevenZipFile);
3890
3891 def PackArchiveFileFromInFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3892 formatspecs = FormatSpecsListToDict(formatspecs);
3893 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
3894 if(verbose):
3895 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3896 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
3897 return PackArchiveFileFromTarFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3898 elif(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
3899 return PackArchiveFileFromZipFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3900 elif(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
3901 return PackArchiveFileFromRarFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3902 elif(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
3903 return PackArchiveFileFromSevenZipFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3904 elif(checkcompressfile=="catfile"):
3905 return RePackArchiveFile(infile, outfile, compression, compresswholefile, compressionlevel, False, 0, 0, checksumtype, False, extradata, formatspecs, verbose, returnfp);
3906 else:
3907 return False;
3908 return False;
3909
3910 create_alias_function("Pack", __file_format_name__, "FromInFile", PackArchiveFileFromInFile);
3911
3912 def ArchiveFileSeekToFileNum(infile, seekto=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
3913 formatspecs = FormatSpecsListToDict(formatspecs);
3914 if(hasattr(infile, "read") or hasattr(infile, "write")):
3915 catfp = infile;
3916 catfp.seek(0, 0);
3917 catfp = UncompressArchiveFile(catfp, formatspecs);
3918 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
3919 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
3920 return TarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3921 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
3922 return ZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3923 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
3924 return RarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3925 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
3926 return SevenZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3927 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
3928 return False;
3929 if(not catfp):
3930 return False;
3931 catfp.seek(0, 0);
3932 elif(infile=="-"):
3933 catfp = BytesIO();
3934 if(hasattr(sys.stdin, "buffer")):
3935 shutil.copyfileobj(sys.stdin.buffer, catfp);
3936 else:
3937 shutil.copyfileobj(sys.stdin, catfp);
3938 catfp.seek(0, 0);
3939 catfp = UncompressArchiveFile(catfp, formatspecs);
3940 if(not catfp):
3941 return False;
3942 catfp.seek(0, 0);
3943 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3944 catfp = download_file_from_internet_file(infile);
3945 catfp.seek(0, 0);
3946 catfp = UncompressArchiveFile(catfp, formatspecs);
3947 if(not catfp):
3948 return False;
3949 catfp.seek(0, 0);
3950 else:
3951 infile = RemoveWindowsPath(infile);
3952 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
3953 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
3954 return TarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3955 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
3956 return ZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3957 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
3958 return RarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3959 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
3960 return SevenZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3961 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
3962 return False;
3963 compresscheck = CheckCompressionType(infile, formatspecs, True);
3964 if(not compresscheck):
3965 fextname = os.path.splitext(infile)[1];
3966 if(fextname==".gz"):
3967 compresscheck = "gzip";
3968 elif(fextname==".bz2"):
3969 compresscheck = "bzip2";
3970 elif(fextname==".zst"):
3971 compresscheck = "zstd";
3972 elif(fextname==".lz4" or fextname==".clz4"):
3973 compresscheck = "lz4";
3974 elif(fextname==".lzo" or fextname==".lzop"):
3975 compresscheck = "lzo";
3976 elif(fextname==".lzma" or fextname==".xz"):
3977 compresscheck = "lzma";
3978 else:
3979 return False;
3980 if(not compresscheck):
3981 return False;
3982 catfp = UncompressFile(infile, formatspecs, "rb");
3983 '''
3984 try:
3985 catfp.seek(0, 2);
3986 except OSError:
3987 SeekToEndOfFile(catfp);
3988 except ValueError:
3989 SeekToEndOfFile(catfp);
3990 CatSize = catfp.tell();
3991 CatSizeEnd = CatSize;
3992 '''
3993 try:
3994 catfp.seek(0, 0);
3995 except OSError:
3996 return False;
3997 except ValueError:
3998 return False;
3999 curloc = catfp.tell();
4000 if(curloc>0):
4001 catfp.seek(0, 0);
4002 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4003 if(curloc>0):
4004 catfp.seek(curloc, 0);
4005 catstring = catheader[0];
4006 catversion = re.findall(r"([\d]+)", catstring);
4007 fprenumfiles = catheader[1];
4008 fnumfiles = int(fprenumfiles, 16);
4009 fprechecksumtype = catheader[2];
4010 fprechecksum = catheader[3];
4011 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4012 fnumfileshex = format(int(fnumfiles), 'x').lower();
4013 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4014 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4015 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4016 fheadtell = len(fileheader);
4017 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4018 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
4019 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4020 return False;
4021 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
4022 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4023 if(seekto>=fnumfiles):
4024 seekto = fnumfiles - 1;
4025 if(seekto<0):
4026 seekto = 0;
4027 if(seekto>=0):
4028 il = -1;
4029 while(il < seekto):
4030 prefhstart = catfp.tell();
4031 if(formatspecs['new_style']):
4032 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4033 else:
4034 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4035 if(len(preheaderdata)==0):
4036 break;
4037 prefheadsize = int(preheaderdata[0], 16);
4038 prefnumfields = int(preheaderdata[1], 16);
4039 preftype = int(preheaderdata[2], 16);
4040 if(re.findall(r"^[.|/]", preheaderdata[3])):
4041 prefname = preheaderdata[3];
4042 else:
4043 prefname = "./"+preheaderdata[3];
4044 prefbasedir = os.path.dirname(prefname);
4045 preflinkname = preheaderdata[4];
4046 prefsize = int(preheaderdata[5], 16);
4047 prefatime = int(preheaderdata[6], 16);
4048 prefmtime = int(preheaderdata[7], 16);
4049 prefctime = int(preheaderdata[8], 16);
4050 prefbtime = int(preheaderdata[9], 16);
4051 prefmode = int(preheaderdata[10], 16);
4052 prefchmode = stat.S_IMODE(prefmode);
4053 preftypemod = stat.S_IFMT(prefmode);
4054 prefwinattributes = int(preheaderdata[11], 16);
4055 prefcompression = preheaderdata[12];
4056 prefcsize = int(preheaderdata[13], 16);
4057 prefuid = int(preheaderdata[14], 16);
4058 prefuname = preheaderdata[15];
4059 prefgid = int(preheaderdata[16], 16);
4060 prefgname = preheaderdata[17];
4061 fid = int(preheaderdata[18], 16);
4062 finode = int(preheaderdata[19], 16);
4063 flinkcount = int(preheaderdata[20], 16);
4064 prefdev_minor = int(preheaderdata[21], 16);
4065 prefdev_major = int(preheaderdata[22], 16);
4066 prefrdev_minor = int(preheaderdata[23], 16);
4067 prefrdev_major = int(preheaderdata[24], 16);
4068 prefseeknextfile = preheaderdata[25];
4069 prefextrasize = int(preheaderdata[26], 16);
4070 prefextrafields = int(preheaderdata[27], 16);
4071 extrafieldslist = [];
4072 extrastart = 28;
4073 extraend = extrastart + prefextrafields;
4074 extrafieldslist = [];
4075 if(extrastart<extraend):
4076 extrafieldslist.append(preheaderdata[extrastart]);
4077 extrastart = extrastart + 1;
4078 prefcs = preheaderdata[-2].lower();
4079 prenewfcs = preheaderdata[-1].lower();
4080 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-4].lower(), True, formatspecs);
4081 if(prefcs!=prenewfcs and not skipchecksum):
4082 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4083 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4084 return False;
4085 valid_archive = False;
4086 invalid_archive = True;
4087 prefhend = catfp.tell() - 1;
4088 prefcontentstart = catfp.tell();
4089 prefcontents = "";
4090 pyhascontents = False;
4091 if(prefsize>0):
4092 if(prefcompression):
4093 prefcontents = catfp.read(prefsize);
4094 else:
4095 prefcontents = catfp.read(prefcsize);
4096 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4097 pyhascontents = True;
4098 if(prefccs!=prenewfccs and not skipchecksum):
4099 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4100 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4101 return False;
4102 if(re.findall(r"^\+([0-9]+)", prefseeknextfile)):
4103 fseeknextasnum = int(prefseeknextfile.replace("+", ""));
4104 if(abs(fseeknextasnum)==0):
4105 pass;
4106 catfp.seek(fseeknextasnum, 1);
4107 elif(re.findall(r"^\-([0-9]+)", prefseeknextfile)):
4108 fseeknextasnum = int(prefseeknextfile);
4109 if(abs(fseeknextasnum)==0):
4110 pass;
4111 catfp.seek(fseeknextasnum, 1);
4112 elif(re.findall(r"^([0-9]+)", prefseeknextfile)):
4113 fseeknextasnum = int(prefseeknextfile);
4114 if(abs(fseeknextasnum)==0):
4115 pass;
4116 catfp.seek(fseeknextasnum, 0);
4117 else:
4118 return False;
4119 il = il + 1;
4120 catfp.seek(seekstart, 0);
4121 fileidnum = il;
4122 catfheadsize = int(preheaderdata[0], 16);
4123 catfnumfields = int(preheaderdata[1], 16);
4124 catftype = int(preheaderdata[2], 16);
4125 if(re.findall(r"^[.|/]", preheaderdata[3])):
4126 catfname = preheaderdata[3];
4127 else:
4128 catfname = "./"+preheaderdata[3];
4129 catflinkname = preheaderdata[4];
4130 catfsize = int(preheaderdata[5], 16);
4131 catfbasedir = os.path.dirname(catfname);
4132 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
4133 if(returnfp):
4134 catlist.update({'catfp': catfp});
4135 else:
4136 catfp.close();
4137 return catlist;
4138
4139 create_alias_function("", __file_format_name__, "SeekToFileNum", ArchiveFileSeekToFileNum);
4140
4141 def ArchiveFileSeekToFileName(infile, seekfile=None, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4142 formatspecs = FormatSpecsListToDict(formatspecs);
4143 if(hasattr(infile, "read") or hasattr(infile, "write")):
4144 catfp = infile;
4145 catfp.seek(0, 0);
4146 catfp = UncompressArchiveFile(catfp, formatspecs);
4147 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4148 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
4149 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4150 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
4151 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4152 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
4153 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4154 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
4155 return SevenZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4156 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4157 return False;
4158 if(not catfp):
4159 return False;
4160 catfp.seek(0, 0);
4161 elif(infile=="-"):
4162 catfp = BytesIO();
4163 if(hasattr(sys.stdin, "buffer")):
4164 shutil.copyfileobj(sys.stdin.buffer, catfp);
4165 else:
4166 shutil.copyfileobj(sys.stdin, catfp);
4167 catfp.seek(0, 0);
4168 catfp = UncompressArchiveFile(catfp, formatspecs);
4169 if(not catfp):
4170 return False;
4171 catfp.seek(0, 0);
4172 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4173 catfp = download_file_from_internet_file(infile);
4174 catfp = UncompressArchiveFile(catfp, formatspecs);
4175 catfp.seek(0, 0);
4176 if(not catfp):
4177 return False;
4178 catfp.seek(0, 0);
4179 else:
4180 infile = RemoveWindowsPath(infile);
4181 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4182 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
4183 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4184 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
4185 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4186 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
4187 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4188 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
4189 return SevenZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4190 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4191 return False;
4192 compresscheck = CheckCompressionType(infile, formatspecs, True);
4193 if(not compresscheck):
4194 fextname = os.path.splitext(infile)[1];
4195 if(fextname==".gz"):
4196 compresscheck = "gzip";
4197 elif(fextname==".bz2"):
4198 compresscheck = "bzip2";
4199 elif(fextname==".zst"):
4200 compresscheck = "zstd";
4201 elif(fextname==".lz4" or fextname==".clz4"):
4202 compresscheck = "lz4";
4203 elif(fextname==".lzo" or fextname==".lzop"):
4204 compresscheck = "lzo";
4205 elif(fextname==".lzma" or fextname==".xz"):
4206 compresscheck = "lzma";
4207 else:
4208 return False;
4209 if(not compresscheck):
4210 return False;
4211 catfp = UncompressFile(infile, formatspecs, "rb");
4212 '''
4213 try:
4214 catfp.seek(0, 2);
4215 except OSError:
4216 SeekToEndOfFile(catfp);
4217 except ValueError:
4218 SeekToEndOfFile(catfp);
4219 CatSize = catfp.tell();
4220 CatSizeEnd = CatSize;
4221 '''
4222 try:
4223 catfp.seek(0, 0);
4224 except OSError:
4225 return False;
4226 except ValueError:
4227 return False;
4228 curloc = catfp.tell();
4229 if(curloc>0):
4230 catfp.seek(0, 0);
4231 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4232 if(curloc>0):
4233 catfp.seek(curloc, 0);
4234 catstring = catheader[0];
4235 catversion = re.findall(r"([\d]+)", catstring);
4236 fprenumfiles = catheader[1];
4237 fnumfiles = int(fprenumfiles, 16);
4238 fprechecksumtype = catheader[2];
4239 fprechecksum = catheader[3];
4240 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4241 fnumfileshex = format(int(fnumfiles), 'x').lower();
4242 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4243 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4244 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4245 fheadtell = len(fileheader);
4246 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4247 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
4248 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4249 return False;
4250 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
4251 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4252 seekto = fnumfiles - 1
4253 filefound = False;
4254 if(seekto>=0):
4255 il = -1;
4256 while(il < seekto):
4257 prefhstart = catfp.tell();
4258 if(formatspecs['new_style']):
4259 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4260 else:
4261 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4262 if(len(preheaderdata)==0):
4263 break;
4264 prefheadsize = int(preheaderdata[0], 16);
4265 prefnumfields = int(preheaderdata[1], 16);
4266 preftype = int(preheaderdata[2], 16);
4267 if(re.findall(r"^[.|/]", preheaderdata[3])):
4268 prefname = preheaderdata[3];
4269 else:
4270 prefname = "./"+preheaderdata[3];
4271 prefbasedir = os.path.dirname(prefname);
4272 preflinkname = preheaderdata[4];
4273 prefsize = int(preheaderdata[5], 16);
4274 prefatime = int(preheaderdata[6], 16);
4275 prefmtime = int(preheaderdata[7], 16);
4276 prefctime = int(preheaderdata[8], 16);
4277 prefbtime = int(preheaderdata[9], 16);
4278 prefmode = int(preheaderdata[10], 16);
4279 prefchmode = stat.S_IMODE(prefmode);
4280 preftypemod = stat.S_IFMT(prefmode);
4281 prefwinattributes = int(preheaderdata[11], 16);
4282 prefcompression = preheaderdata[12];
4283 prefcsize = int(preheaderdata[13], 16);
4284 prefuid = int(preheaderdata[14], 16);
4285 prefuname = preheaderdata[15];
4286 prefgid = int(preheaderdata[16], 16);
4287 prefgname = preheaderdata[17];
4288 fid = int(preheaderdata[18], 16);
4289 finode = int(preheaderdata[19], 16);
4290 flinkcount = int(preheaderdata[20], 16);
4291 prefdev_minor = int(preheaderdata[21], 16);
4292 prefdev_major = int(preheaderdata[22], 16);
4293 prefrdev_minor = int(preheaderdata[23], 16);
4294 prefrdev_major = int(preheaderdata[24], 16);
4295 prefseeknextfile = preheaderdata[25];
4296 prefextrasize = int(preheaderdata[26], 16);
4297 prefextrafields = int(preheaderdata[27], 16);
4298 extrafieldslist = [];
4299 extrastart = 28;
4300 extraend = extrastart + prefextrafields;
4301 extrafieldslist = [];
4302 if(extrastart<extraend):
4303 extrafieldslist.append(preheaderdata[extrastart]);
4304 extrastart = extrastart + 1;
4305 prefcs = preheaderdata[-2].lower();
4306 prenewfcs = preheaderdata[-1].lower();
4307 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-4].lower(), True, formatspecs);
4308 if(prefcs!=prenewfcs and not skipchecksum):
4309 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4310 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4311 return False;
4312 valid_archive = False;
4313 invalid_archive = True;
4314 prefhend = catfp.tell() - 1;
4315 prefcontentstart = catfp.tell();
4316 prefcontents = "";
4317 pyhascontents = False;
4318 if(prefsize>0):
4319 if(prefcompression):
4320 prefcontents = catfp.read(prefsize);
4321 else:
4322 prefcontents = catfp.read(prefcsize);
4323 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4324 pyhascontents = True;
4325 if(prefccs!=prenewfccs and not skipchecksum):
4326 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4327 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4328 return False;
4329 if(re.findall(r"^\+([0-9]+)", prefseeknextfile)):
4330 fseeknextasnum = int(prefseeknextfile.replace("+", ""));
4331 if(abs(fseeknextasnum)==0):
4332 pass;
4333 catfp.seek(fseeknextasnum, 1);
4334 elif(re.findall(r"^\-([0-9]+)", prefseeknextfile)):
4335 fseeknextasnum = int(prefseeknextfile);
4336 if(abs(fseeknextasnum)==0):
4337 pass;
4338 catfp.seek(fseeknextasnum, 1);
4339 elif(re.findall(r"^([0-9]+)", prefseeknextfile)):
4340 fseeknextasnum = int(prefseeknextfile);
4341 if(abs(fseeknextasnum)==0):
4342 pass;
4343 catfp.seek(fseeknextasnum, 0);
4344 else:
4345 return False;
4346 il = il + 1;
4347 filefound = False;
4348 prefname = preheaderdata[2];
4349 if(re.findall(r"^[.|/]", preheaderdata[2])):
4350 prefname = preheaderdata[2];
4351 else:
4352 prefname = "./"+preheaderdata[2];
4353 if(prefname==seekfile):
4354 filefound = True;
4355 break;
4356 catfp.seek(seekstart, 0);
4357 fileidnum = il;
4358 catfheadsize = int(preheaderdata[0], 16);
4359 catfnumfields = int(preheaderdata[1], 16);
4360 catftype = int(preheaderdata[2], 16);
4361 if(re.findall(r"^[.|/]", preheaderdata[3])):
4362 catfname = preheaderdata[3];
4363 else:
4364 catfname = "./"+preheaderdata[3];
4365 catflinkname = preheaderdata[4];
4366 catfsize = int(preheaderdata[5], 16);
4367 catfbasedir = os.path.dirname(catfname);
4368 if(filefound):
4369 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
4370 else:
4371 return False;
4372 if(returnfp):
4373 catlist.update({'catfp': catfp});
4374 else:
4375 catfp.close();
4376 return catlist;
4377
4378 create_alias_function("", __file_format_name__, "SeekToFileName", ArchiveFileSeekToFileName);
4379
4380 def ArchiveFileValidate(infile, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
4381 formatspecs = FormatSpecsListToDict(formatspecs);
4382 if(verbose):
4383 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
4384 if(hasattr(infile, "read") or hasattr(infile, "write")):
4385 catfp = infile;
4386 catfp.seek(0, 0);
4387 catfp = UncompressArchiveFile(catfp, formatspecs);
4388 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4389 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
4390 return TarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4391 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
4392 return ZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4393 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
4394 return RarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4395 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
4396 return SevenZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4397 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4398 return False;
4399 if(not catfp):
4400 return False;
4401 catfp.seek(0, 0);
4402 elif(infile=="-"):
4403 catfp = BytesIO();
4404 if(hasattr(sys.stdin, "buffer")):
4405 shutil.copyfileobj(sys.stdin.buffer, catfp);
4406 else:
4407 shutil.copyfileobj(sys.stdin, catfp);
4408 catfp.seek(0, 0);
4409 catfp = UncompressArchiveFile(catfp, formatspecs);
4410 if(not catfp):
4411 return False;
4412 catfp.seek(0, 0);
4413 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4414 catfp = download_file_from_internet_file(infile);
4415 catfp = UncompressArchiveFile(catfp, formatspecs);
4416 catfp.seek(0, 0);
4417 if(not catfp):
4418 return False;
4419 catfp.seek(0, 0);
4420 else:
4421 infile = RemoveWindowsPath(infile);
4422 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4423 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
4424 return TarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4425 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
4426 return ZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4427 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
4428 return RarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4429 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
4430 return SevenZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4431 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4432 return False;
4433 compresscheck = CheckCompressionType(infile, formatspecs, True);
4434 if(not compresscheck):
4435 fextname = os.path.splitext(infile)[1];
4436 if(fextname==".gz"):
4437 compresscheck = "gzip";
4438 elif(fextname==".bz2"):
4439 compresscheck = "bzip2";
4440 elif(fextname==".zst"):
4441 compresscheck = "zstd";
4442 elif(fextname==".lz4" or fextname==".clz4"):
4443 compresscheck = "lz4";
4444 elif(fextname==".lzo" or fextname==".lzop"):
4445 compresscheck = "lzo";
4446 elif(fextname==".lzma" or fextname==".xz"):
4447 compresscheck = "lzma";
4448 else:
4449 return False;
4450 if(not compresscheck):
4451 return False;
4452 catfp = UncompressFile(infile, formatspecs, "rb");
4453 '''
4454 try:
4455 catfp.seek(0, 2);
4456 except OSError:
4457 SeekToEndOfFile(catfp);
4458 except ValueError:
4459 SeekToEndOfFile(catfp);
4460 CatSize = catfp.tell();
4461 CatSizeEnd = CatSize;
4462 '''
4463 try:
4464 catfp.seek(0, 0);
4465 except OSError:
4466 return False;
4467 except ValueError:
4468 return False;
4469 curloc = catfp.tell();
4470 if(curloc>0):
4471 catfp.seek(0, 0);
4472 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4473 if(curloc>0):
4474 catfp.seek(curloc, 0);
4475 catstring = catheader[0];
4476 catversion = re.findall(r"([\d]+)", catstring);
4477 fprenumfiles = catheader[1];
4478 fnumfiles = int(fprenumfiles, 16);
4479 fprechecksumtype = catheader[2];
4480 fprechecksum = catheader[3];
4481 il = 0;
4482 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4483 fnumfileshex = format(int(fnumfiles), 'x').lower();
4484 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4485 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4486 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4487 valid_archive = True;
4488 invalid_archive = False;
4489 if(verbose):
4490 VerbosePrintOut(infile);
4491 VerbosePrintOut("Number of Records " + str(fnumfiles));
4492 if(fprechecksum==catfileheadercshex):
4493 if(verbose):
4494 VerbosePrintOut("File Header Checksum Passed at offset " + str(0));
4495 VerbosePrintOut("'" + str(fprechecksum) + "' == " + "'" + str(catfileheadercshex) + "'");
4496 else:
4497 if(verbose):
4498 VerbosePrintOut("File Header Checksum Failed at offset " + str(0));
4499 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4500 valid_archive = False;
4501 invalid_archive = True;
4502 if(verbose):
4503 VerbosePrintOut("");
4504 while(il<fnumfiles):
4505 catfhstart = catfp.tell();
4506 if(formatspecs['new_style']):
4507 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4508 else:
4509 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4510 if(len(catheaderdata)==0):
4511 break;
4512 catfheadsize = int(catheaderdata[0], 16);
4513 catfnumfields = int(catheaderdata[1], 16);
4514 catftype = int(catheaderdata[2], 16);
4515 if(re.findall(r"^[.|/]", catheaderdata[3])):
4516 catfname = catheaderdata[3];
4517 else:
4518 catfname = "./"+catheaderdata[3];
4519 catfbasedir = os.path.dirname(catfname);
4520 catflinkname = catheaderdata[4];
4521 catfsize = int(catheaderdata[5], 16);
4522 catfatime = int(catheaderdata[6], 16);
4523 catfmtime = int(catheaderdata[7], 16);
4524 catfctime = int(catheaderdata[8], 16);
4525 catfbtime = int(catheaderdata[9], 16);
4526 catfmode = int(catheaderdata[10], 16);
4527 catfchmode = stat.S_IMODE(catfmode);
4528 catftypemod = stat.S_IFMT(catfmode);
4529 catfwinattributes = int(catheaderdata[11], 16);
4530 catfcompression = catheaderdata[12];
4531 catfcsize = int(catheaderdata[13], 16);
4532 catfuid = int(catheaderdata[14], 16);
4533 catfuname = catheaderdata[15];
4534 catfgid = int(catheaderdata[16], 16);
4535 catfgname = catheaderdata[17];
4536 fid = int(catheaderdata[18], 16);
4537 finode = int(catheaderdata[19], 16);
4538 flinkcount = int(catheaderdata[20], 16);
4539 catfdev_minor = int(catheaderdata[21], 16);
4540 catfdev_major = int(catheaderdata[22], 16);
4541 catfrdev_minor = int(catheaderdata[23], 16);
4542 catfrdev_major = int(catheaderdata[24], 16);
4543 catfseeknextfile = catheaderdata[25];
4544 catfextrasize = int(catheaderdata[26], 16);
4545 catfextrafields = int(catheaderdata[27], 16);
4546 extrafieldslist = [];
4547 extrastart = 28;
4548 extraend = extrastart + catfextrafields;
4549 extrafieldslist = [];
4550 if(extrastart<extraend):
4551 extrafieldslist.append(catheaderdata[extrastart]);
4552 extrastart = extrastart + 1;
4553 catfcs = catheaderdata[-2].lower();
4554 catfccs = catheaderdata[-1].lower();
4555 catnewfcs = GetHeaderChecksum(catheaderdata[:-2], catheaderdata[-4].lower(), True, formatspecs);
4556 if(verbose):
4557 VerbosePrintOut(catfname);
4558 VerbosePrintOut("Record Number " + str(il) + "; File ID " + str(fid) + "; iNode Number " + str(finode));
4559 if(catfcs==catnewfcs):
4560 if(verbose):
4561 VerbosePrintOut("File Header Checksum Passed at offset " + str(catfhstart));
4562 VerbosePrintOut("'" + str(catfcs) + "' == " + "'" + str(catnewfcs) + "'");
4563 else:
4564 if(verbose):
4565 VerbosePrintOut("File Header Checksum Failed at offset " + str(catfhstart));
4566 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
4567 valid_archive = False;
4568 invalid_archive = True;
4569 catfhend = catfp.tell() - 1;
4570 catfcontentstart = catfp.tell();
4571 catfcontents = "";
4572 pyhascontents = False;
4573 if(catfsize>0):
4574 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4575 catfcontents = catfp.read(catfsize);
4576 else:
4577 catfcontents = catfp.read(catfcsize);
4578 catnewfccs = GetFileChecksum(catfcontents, catheaderdata[-3].lower(), False, formatspecs);
4579 pyhascontents = True;
4580 if(catfccs==catnewfccs):
4581 if(verbose):
4582 VerbosePrintOut("File Content Checksum Passed at offset " + str(catfcontentstart));
4583 VerbosePrintOut("'" + str(catfccs) + "' == " + "'" + str(catnewfccs) + "'");
4584 else:
4585 if(verbose):
4586 VerbosePrintOut("File Content Checksum Failed at offset " + str(catfcontentstart));
4587 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
4588 valid_archive = False;
4589 invalid_archive = True;
4590 if(verbose):
4591 VerbosePrintOut("");
4592 if(re.findall(r"^\+([0-9]+)", catfseeknextfile)):
4593 fseeknextasnum = int(catfseeknextfile.replace("+", ""));
4594 if(abs(fseeknextasnum)==0):
4595 pass;
4596 catfp.seek(fseeknextasnum, 1);
4597 elif(re.findall(r"^\-([0-9]+)", catfseeknextfile)):
4598 fseeknextasnum = int(catfseeknextfile);
4599 if(abs(fseeknextasnum)==0):
4600 pass;
4601 catfp.seek(fseeknextasnum, 1);
4602 elif(re.findall(r"^([0-9]+)", catfseeknextfile)):
4603 fseeknextasnum = int(catfseeknextfile);
4604 if(abs(fseeknextasnum)==0):
4605 pass;
4606 catfp.seek(fseeknextasnum, 0);
4607 else:
4608 return False;
4609 il = il + 1;
4610 if(valid_archive):
4611 if(returnfp):
4612 return catfp;
4613 else:
4614 catfp.close();
4615 return True;
4616 else:
4617 catfp.close();
4618 return False;
4619
4620 create_alias_function("", __file_format_name__, "Validate", ArchiveFileValidate);
4621
4622 def ArchiveFileToArray(infile, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4623 formatspecs = FormatSpecsListToDict(formatspecs);
4624 if(hasattr(infile, "read") or hasattr(infile, "write")):
4625 catfp = infile;
4626 catfp.seek(0, 0);
4627 catfp = UncompressArchiveFile(catfp, formatspecs);
4628 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4629 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
4630 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4631 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
4632 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4633 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
4634 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4635 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
4636 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4637 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4638 return False;
4639 if(not catfp):
4640 return False;
4641 catfp.seek(0, 0);
4642 elif(infile=="-"):
4643 catfp = BytesIO();
4644 if(hasattr(sys.stdin, "buffer")):
4645 shutil.copyfileobj(sys.stdin.buffer, catfp);
4646 else:
4647 shutil.copyfileobj(sys.stdin, catfp);
4648 catfp.seek(0, 0);
4649 catfp = UncompressArchiveFile(catfp, formatspecs);
4650 if(not catfp):
4651 return False;
4652 catfp.seek(0, 0);
4653 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4654 catfp = download_file_from_internet_file(infile);
4655 catfp = UncompressArchiveFile(catfp, formatspecs);
4656 catfp.seek(0, 0);
4657 if(not catfp):
4658 return False;
4659 catfp.seek(0, 0);
4660 else:
4661 infile = RemoveWindowsPath(infile);
4662 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4663 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
4664 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4665 if(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
4666 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4667 if(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
4668 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4669 if(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
4670 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4671 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4672 return False;
4673 compresscheck = CheckCompressionType(infile, formatspecs, True);
4674 if(not compresscheck):
4675 fextname = os.path.splitext(infile)[1];
4676 if(fextname==".gz"):
4677 compresscheck = "gzip";
4678 elif(fextname==".bz2"):
4679 compresscheck = "bzip2";
4680 elif(fextname==".zst"):
4681 compresscheck = "zstd";
4682 elif(fextname==".lz4" or fextname==".clz4"):
4683 compresscheck = "lz4";
4684 elif(fextname==".lzo" or fextname==".lzop"):
4685 compresscheck = "lzo";
4686 elif(fextname==".lzma" or fextname==".xz"):
4687 compresscheck = "lzma";
4688 else:
4689 return False;
4690 if(not compresscheck):
4691 return False;
4692 catfp = UncompressFile(infile, formatspecs, "rb");
4693 '''
4694 try:
4695 catfp.seek(0, 2);
4696 except OSError:
4697 SeekToEndOfFile(catfp);
4698 except ValueError:
4699 SeekToEndOfFile(catfp);
4700 CatSize = catfp.tell();
4701 CatSizeEnd = CatSize;
4702 '''
4703 try:
4704 catfp.seek(0, 0);
4705 except OSError:
4706 return False;
4707 except ValueError:
4708 return False;
4709 curloc = catfp.tell();
4710 if(curloc>0):
4711 catfp.seek(0, 0);
4712 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4713 if(curloc>0):
4714 catfp.seek(curloc, 0);
4715 catstring = catheader[0];
4716 catversion = re.findall(r"([\d]+)", catstring);
4717 fprenumfiles = catheader[1];
4718 fnumfiles = int(fprenumfiles, 16);
4719 fprechecksumtype = catheader[2];
4720 fprechecksum = catheader[3];
4721 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4722 fnumfileshex = format(int(fnumfiles), 'x').lower();
4723 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4724 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4725 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4726 fheadtell = len(fileheader);
4727 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4728 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
4729 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4730 return False;
4731 catversions = re.search(r'(.*?)(\d+)', catstring).groups();
4732 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': []};
4733 if(seekstart<0 and seekstart>fnumfiles):
4734 seekstart = 0;
4735 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
4736 seekend = fnumfiles;
4737 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
4738 seekend = fnumfiles - abs(seekend);
4739 if(seekstart>0):
4740 il = 0;
4741 while(il < seekstart):
4742 prefhstart = catfp.tell();
4743 if(formatspecs['new_style']):
4744 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4745 else:
4746 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4747 if(len(preheaderdata)==0):
4748 break;
4749 prefheadsize = int(preheaderdata[0], 16);
4750 prefnumfields = int(preheaderdata[1], 16);
4751 if(re.findall(r"^[.|/]", preheaderdata[3])):
4752 prefname = preheaderdata[3];
4753 else:
4754 prefname = "./"+preheaderdata[3];
4755 prefsize = int(preheaderdata[5], 16);
4756 prefcompression = preheaderdata[12];
4757 prefcsize = int(preheaderdata[13], 16);
4758 prefseeknextfile = preheaderdata[25];
4759 prefextrasize = int(preheaderdata[26], 16);
4760 prefextrafields = int(preheaderdata[27], 16);
4761 extrafieldslist = [];
4762 extrastart = 28;
4763 extraend = extrastart + prefextrafields;
4764 extrafieldslist = [];
4765 if(extrastart<extraend):
4766 extrafieldslist.append(preheaderdata[extrastart]);
4767 extrastart = extrastart + 1;
4768 prefcs = preheaderdata[-2].lower();
4769 prenewfcs = preheaderdata[-1].lower();
4770 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-4].lower(), True, formatspecs);
4771 if(prefcs!=prenewfcs and not skipchecksum):
4772 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4773 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4774 return False;
4775 valid_archive = False;
4776 invalid_archive = True;
4777 prefhend = catfp.tell() - 1;
4778 prefcontentstart = catfp.tell();
4779 prefcontents = "";
4780 pyhascontents = False;
4781 if(prefsize>0):
4782 if(prefcompression=="none" or prefcompression=="" or prefcompression=="auto"):
4783 prefcontents = catfp.read(prefsize);
4784 else:
4785 prefcontents = catfp.read(prefcsize);
4786 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4787 pyhascontents = True;
4788 if(prefccs!=prenewfccs and not skipchecksum):
4789 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4790 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4791 return False;
4792 if(re.findall(r"^\+([0-9]+)", prefseeknextfile)):
4793 fseeknextasnum = int(prefseeknextfile.replace("+", ""));
4794 if(abs(fseeknextasnum)==0):
4795 pass;
4796 catfp.seek(fseeknextasnum, 1);
4797 elif(re.findall(r"^\-([0-9]+)", prefseeknextfile)):
4798 fseeknextasnum = int(prefseeknextfile);
4799 if(abs(fseeknextasnum)==0):
4800 pass;
4801 catfp.seek(fseeknextasnum, 1);
4802 elif(re.findall(r"^([0-9]+)", prefseeknextfile)):
4803 fseeknextasnum = int(prefseeknextfile);
4804 if(abs(fseeknextasnum)==0):
4805 pass;
4806 catfp.seek(fseeknextasnum, 0);
4807 else:
4808 return False;
4809 il = il + 1;
4810 fileidnum = seekstart;
4811 realidnum = 0;
4812 while(fileidnum<seekend):
4813 catfhstart = catfp.tell();
4814 if(formatspecs['new_style']):
4815 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4816 else:
4817 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4818 if(len(catheaderdata)==0):
4819 break;
4820 catfheadsize = int(catheaderdata[0], 16);
4821 catfnumfields = int(catheaderdata[1], 16);
4822 catftype = int(catheaderdata[2], 16);
4823 if(re.findall(r"^[.|/]", catheaderdata[3])):
4824 catfname = catheaderdata[3];
4825 else:
4826 catfname = "./"+catheaderdata[3];
4827 catfbasedir = os.path.dirname(catfname);
4828 catflinkname = catheaderdata[4];
4829 catfsize = int(catheaderdata[5], 16);
4830 catfatime = int(catheaderdata[6], 16);
4831 catfmtime = int(catheaderdata[7], 16);
4832 catfctime = int(catheaderdata[8], 16);
4833 catfbtime = int(catheaderdata[9], 16);
4834 catfmode = int(catheaderdata[10], 16);
4835 catfchmode = stat.S_IMODE(catfmode);
4836 catftypemod = stat.S_IFMT(catfmode);
4837 catfwinattributes = int(catheaderdata[11], 16);
4838 catfcompression = catheaderdata[12];
4839 catfcsize = int(catheaderdata[13], 16);
4840 catfuid = int(catheaderdata[14], 16);
4841 catfuname = catheaderdata[15];
4842 catfgid = int(catheaderdata[16], 16);
4843 catfgname = catheaderdata[17];
4844 catfid = int(catheaderdata[18], 16);
4845 catfinode = int(catheaderdata[19], 16);
4846 catflinkcount = int(catheaderdata[20], 16);
4847 catfdev_minor = int(catheaderdata[21], 16);
4848 catfdev_major = int(catheaderdata[22], 16);
4849 catfrdev_minor = int(catheaderdata[23], 16);
4850 catfrdev_major = int(catheaderdata[24], 16);
4851 catfseeknextfile = catheaderdata[25];
4852 catfextrasize = int(catheaderdata[26], 16);
4853 catfextrafields = int(catheaderdata[27], 16);
4854 extrafieldslist = [];
4855 extrastart = 28;
4856 extraend = extrastart + catfextrafields;
4857 extrafieldslist = [];
4858 if(extrastart<extraend):
4859 extrafieldslist.append(catheaderdata[extrastart]);
4860 extrastart = extrastart + 1;
4861 catfcs = catheaderdata[-2].lower();
4862 catfccs = catheaderdata[-1].lower();
4863 catnewfcs = GetHeaderChecksum(catheaderdata[:-2], catheaderdata[-4].lower(), True, formatspecs);
4864 if(catfcs!=catnewfcs and not skipchecksum):
4865 VerbosePrintOut("File Header Checksum Error with file " + catfname + " at offset " + str(catfhstart));
4866 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
4867 return False;
4868 catfhend = catfp.tell() - 1;
4869 catfcontentstart = catfp.tell();
4870 catfcontents = BytesIO();
4871 pyhascontents = False;
4872 if(catfsize>0 and not listonly):
4873 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4874 catfcontents.write(catfp.read(catfsize));
4875 else:
4876 catfcontents.write(catfp.read(catfcsize));
4877 catfcontents.seek(0, 0);
4878 catnewfccs = GetFileChecksum(catfcontents.read(), catheaderdata[-3].lower(), False, formatspecs);
4879 pyhascontents = True;
4880 if(catfccs!=catnewfccs and skipchecksum):
4881 VerbosePrintOut("File Content Checksum Error with file " + catfname + " at offset " + str(catfcontentstart));
4882 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
4883 return False;
4884 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4885 pass;
4886 else:
4887 catfcontents.seek(0, 0);
4888 if(uncompress):
4889 catfcontents = UncompressArchiveFile(catfcontents, formatspecs);
4890 catfcontents.seek(0, 0);
4891 catfccs = GetFileChecksum(catfcontents.read(), catheaderdata[-3].lower(), False, formatspecs);
4892 if(catfsize>0 and listonly):
4893 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4894 catfp.seek(catfsize, 1);
4895 else:
4896 catfp.seek(catfcsize, 1);
4897 pyhascontents = False;
4898 catfcontentend = catfp.tell();
4899 if(re.findall(r"^\+([0-9]+)", catfseeknextfile)):
4900 fseeknextasnum = int(catfseeknextfile.replace("+", ""));
4901 if(abs(fseeknextasnum)==0):
4902 pass;
4903 catfp.seek(fseeknextasnum, 1);
4904 elif(re.findall(r"^\-([0-9]+)", catfseeknextfile)):
4905 fseeknextasnum = int(catfseeknextfile);
4906 if(abs(fseeknextasnum)==0):
4907 pass;
4908 catfp.seek(fseeknextasnum, 1);
4909 elif(re.findall(r"^([0-9]+)", catfseeknextfile)):
4910 fseeknextasnum = int(catfseeknextfile);
4911 if(abs(fseeknextasnum)==0):
4912 pass;
4913 catfp.seek(fseeknextasnum, 0);
4914 else:
4915 return False;
4916 catfcontents.seek(0, 0);
4917 catlist['ffilelist'].append({'fid': realidnum, 'fidalt': fileidnum, 'fheadersize': catfheadsize, 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize, 'fatime': catfatime, 'fmtime': catfmtime, 'fctime': catfctime, 'fbtime': catfbtime, 'fmode': catfmode, 'fchmode': catfchmode, 'ftypemod': catftypemod, 'fwinattributes': catfwinattributes, 'fcompression': catfcompression, 'fcsize': catfcsize, 'fuid': catfuid, 'funame': catfuname, 'fgid': catfgid, 'fgname': catfgname, 'finode': catfinode, 'flinkcount': catflinkcount, 'fminor': catfdev_minor, 'fmajor': catfdev_major, 'frminor': catfrdev_minor, 'frmajor': catfrdev_major, 'fseeknextfile': catfseeknextfile, 'fheaderchecksumtype': catheaderdata[-4], 'fcontentchecksumtype': catheaderdata[-3], 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': catfextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': catfcs, 'fcontentchecksum': catfccs, 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': catfcontents});
4918 fileidnum = fileidnum + 1;
4919 realidnum = realidnum + 1;
4920 if(returnfp):
4921 catlist.update({'catfp': catfp});
4922 else:
4923 catfp.close();
4924 return catlist;
4925
4926 create_alias_function("", __file_format_name__, "ToArray", ArchiveFileToArray);
4927
4928 def ArchiveFileStringToArray(catstr, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4929 formatspecs = FormatSpecsListToDict(formatspecs);
4930 catfp = BytesIO(catstr);
4931 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
4932 return listcatfiles;
4933
4934 create_alias_function("", __file_format_name__, "StringToArray", ArchiveFileStringToArray);
4935
4936 def TarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4937 formatspecs = FormatSpecsListToDict(formatspecs);
4938 catfp = BytesIO();
4939 catfp = PackArchiveFileFromTarFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
4940 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
4941 return listcatfiles;
4942
4943 def ZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4944 formatspecs = FormatSpecsListToDict(formatspecs);
4945 catfp = BytesIO();
4946 catfp = PackArchiveFileFromZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
4947 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
4948 return listcatfiles;
4949
4950 if(not rarfile_support):
4951 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4952 return False;
4953
4954 if(rarfile_support):
4955 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4956 formatspecs = FormatSpecsListToDict(formatspecs);
4957 catfp = BytesIO();
4958 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
4959 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
4960 return listcatfiles;
4961
4962 if(not py7zr_support):
4963 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4964 return False;
4965
4966 if(py7zr_support):
4967 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4968 formatspecs = FormatSpecsListToDict(formatspecs);
4969 catfp = BytesIO();
4970 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", True, None, "crc32", [], formatspecs, False, True);
4971 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
4972 return listcatfiles;
4973
4974 def InFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4975 formatspecs = FormatSpecsListToDict(formatspecs);
4976 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4977 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
4978 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4979 elif(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
4980 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4981 elif(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
4982 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4983 elif(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
4984 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4985 elif(checkcompressfile=="catfile"):
4986 return ArchiveFileToArray(infile, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
4987 else:
4988 return False;
4989 return False;
4990
4991 def ListDirToArrayAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
4992 formatspecs = FormatSpecsListToDict(formatspecs);
4993 catver = formatspecs['format_ver'];
4994 fileheaderver = str(int(catver.replace(".", "")));
4995 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
4996 advancedlist = formatspecs['use_advanced_list'];
4997 altinode = formatspecs['use_alt_inode'];
4998 infilelist = [];
4999 if(infiles=="-"):
5000 for line in sys.stdin:
5001 infilelist.append(line.strip());
5002 infilelist = list(filter(None, infilelist));
5003 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
5004 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
5005 return False;
5006 with UncompressFile(infiles, formatspecs, "r") as finfile:
5007 for line in finfile:
5008 infilelist.append(line.strip());
5009 infilelist = list(filter(None, infilelist));
5010 else:
5011 if(isinstance(infiles, (list, tuple, ))):
5012 infilelist = list(filter(None, infiles));
5013 elif(isinstance(infiles, (str, ))):
5014 infilelist = list(filter(None, [infiles]));
5015 if(advancedlist):
5016 GetDirList = ListDirAdvanced(infilelist, followlink, False);
5017 else:
5018 GetDirList = ListDir(infilelist, followlink, False);
5019 if(not GetDirList):
5020 return False;
5021 curinode = 0;
5022 curfid = 0;
5023 inodelist = [];
5024 inodetofile = {};
5025 filetoinode = {};
5026 inodetocatinode = {};
5027 fileidnum = 0;
5028 fnumfiles = int(len(GetDirList));
5029 catver = formatspecs['format_ver'];
5030 fileheaderver = str(int(catver.replace(".", "")));
5031 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5032 fnumfileshex = format(int(fnumfiles), 'x').lower();
5033 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5034 catversion = re.findall(r"([\d]+)", fileheader);
5035 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5036 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5037 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5038 fheadtell = len(fileheader);
5039 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
5040 for curfname in GetDirList:
5041 catfhstart = fheadtell;
5042 if(re.findall(r"^[.|/]", curfname)):
5043 fname = curfname;
5044 else:
5045 fname = "./"+curfname;
5046 if(verbose):
5047 VerbosePrintOut(fname);
5048 if(not followlink or followlink is None):
5049 fstatinfo = os.lstat(fname);
5050 else:
5051 fstatinfo = os.stat(fname);
5052 fpremode = fstatinfo.st_mode;
5053 finode = fstatinfo.st_ino;
5054 flinkcount = fstatinfo.st_nlink;
5055 ftype = 0;
5056 if(stat.S_ISREG(fpremode)):
5057 ftype = 0;
5058 elif(stat.S_ISLNK(fpremode)):
5059 ftype = 2;
5060 elif(stat.S_ISCHR(fpremode)):
5061 ftype = 3;
5062 elif(stat.S_ISBLK(fpremode)):
5063 ftype = 4;
5064 elif(stat.S_ISDIR(fpremode)):
5065 ftype = 5;
5066 elif(stat.S_ISFIFO(fpremode)):
5067 ftype = 6;
5068 elif(stat.S_ISSOCK(fpremode)):
5069 ftype = 8;
5070 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
5071 ftype = 9;
5072 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
5073 ftype = 10;
5074 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
5075 ftype = 11;
5076 else:
5077 ftype = 0;
5078 flinkname = "";
5079 fbasedir = os.path.dirname(fname);
5080 fcurfid = curfid;
5081 if(not followlink and finode!=0):
5082 if(ftype!=1):
5083 if(finode in inodelist):
5084 ftype = 1;
5085 flinkname = inodetofile[finode];
5086 if(altinode):
5087 fcurinode = finode;
5088 else:
5089 fcurinode = inodetocatinode[finode];
5090 if(finode not in inodelist):
5091 inodelist.append(finode);
5092 inodetofile.update({finode: fname});
5093 inodetocatinode.update({finode: curinode});
5094 if(altinode):
5095 fcurinode = finode;
5096 else:
5097 fcurinode = curinode;
5098 curinode = curinode + 1;
5099 else:
5100 fcurinode = curinode;
5101 curinode = curinode + 1;
5102 curfid = curfid + 1;
5103 if(ftype==2):
5104 flinkname = os.readlink(fname);
5105 fdev = fstatinfo.st_dev;
5106 getfdev = GetDevMajorMinor(fdev);
5107 fdev_minor = getfdev[0];
5108 fdev_major = getfdev[1];
5109 frdev = fstatinfo.st_dev;
5110 if(hasattr(fstatinfo, "st_rdev")):
5111 frdev = fstatinfo.st_rdev;
5112 else:
5113 frdev = fstatinfo.st_dev;
5114 getfrdev = GetDevMajorMinor(frdev);
5115 frdev_minor = getfrdev[0];
5116 frdev_major = getfrdev[1];
5117 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
5118 fsize = "0";
5119 if(ftype==0 or ftype==7):
5120 fsize = fstatinfo.st_size;
5121 fatime = fstatinfo.st_atime;
5122 fmtime = fstatinfo.st_mtime;
5123 fctime = fstatinfo.st_ctime;
5124 if(hasattr(fstatinfo, "st_birthtime")):
5125 fbtime = fstatinfo.st_birthtime;
5126 else:
5127 fbtime = fstatinfo.st_ctime;
5128 fmode = fstatinfo.st_mode;
5129 fchmode = stat.S_IMODE(fstatinfo.st_mode);
5130 ftypemod = stat.S_IFMT(fstatinfo.st_mode);
5131 fuid = fstatinfo.st_uid;
5132 fgid = fstatinfo.st_gid;
5133 funame = "";
5134 try:
5135 import pwd;
5136 try:
5137 userinfo = pwd.getpwuid(fstatinfo.st_uid);
5138 funame = userinfo.pw_name;
5139 except KeyError:
5140 funame = "";
5141 except ImportError:
5142 funame = "";
5143 fgname = "";
5144 try:
5145 import grp;
5146 try:
5147 groupinfo = grp.getgrgid(fstatinfo.st_gid);
5148 fgname = groupinfo.gr_name;
5149 except KeyError:
5150 fgname = "";
5151 except ImportError:
5152 fgname = "";
5153 fdev_minor = fdev_minor;
5154 fdev_major = fdev_major;
5155 frdev_minor = frdev_minor;
5156 frdev_major = frdev_major;
5157 flinkcount = flinkcount;
5158 if(hasattr(fstatinfo, "st_file_attributes")):
5159 fwinattributes = fstatinfo.st_file_attributes;
5160 else:
5161 fwinattributes = 0;
5162 fcompression = "";
5163 fcsize = 0;
5164 fcontents = BytesIO();
5165 if(ftype==0 or ftype==7):
5166 with open(fname, "rb") as fpc:
5167 shutil.copyfileobj(fpc, fcontents);
5168 if(followlink and (ftype==1 or ftype==2)):
5169 flstatinfo = os.stat(flinkname);
5170 with open(flinkname, "rb") as fpc:
5171 shutil.copyfileobj(fpc, fcontents);
5172 fcontents.seek(0, 0);
5173 ftypehex = format(ftype, 'x').lower();
5174 extrafields = len(extradata);
5175 extrafieldslist = extradata;
5176 catfextrafields = extrafields;
5177 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5178 if(len(extradata)>0):
5179 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5180 extrasizelen = len(extrasizestr);
5181 extrasizelenhex = format(extrasizelen, 'x').lower();
5182 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), "+1", extrasizelenhex, format(catfextrafields, 'x').lower()];
5183 catoutlen = len(catoutlist) + len(extradata) + 3;
5184 catoutlenhex = format(catoutlen, 'x').lower();
5185 catoutlist.insert(0, catoutlenhex);
5186 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5187 catheaderdata = catoutlist;
5188 if(len(extradata)>0):
5189 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5190 if(fsize==0):
5191 checksumlist = [checksumtype, "none"];
5192 else:
5193 checksumlist = [checksumtype, checksumtype];
5194 catfileoutstr = catfileoutstr + AppendNullBytes(checksumlist, formatspecs['format_delimiter']);
5195 catfnumfields = catoutlen;
5196 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5197 fcontents.seek(0, 0);
5198 if(fsize==0):
5199 catfilecontentcshex = GetFileChecksum(fcontents.read(), "none", False, formatspecs);
5200 else:
5201 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5202 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5203 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5204 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5205 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5206 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5207 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5208 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5209 catfcontentstart = fheadtell;
5210 fheadtell += len(catfileoutstr) + 1;
5211 catfcontentend = fheadtell - 1;
5212 catfhend = catfcontentend;
5213 fcontents.seek(0, 0);
5214 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5215 pyhascontents = False;
5216 if(int(fsize)>0 and not listonly):
5217 pyhascontents = True;
5218 if(int(fsize)>0 and listonly):
5219 fcontents = BytesIO();
5220 pyhascontents = False;
5221 fcontents.seek(0, 0);
5222 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype, 'fcontentchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents});
5223 fileidnum = fileidnum + 1;
5224 return catlist;
5225
5226 def TarFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5227 formatspecs = FormatSpecsListToDict(formatspecs);
5228 curinode = 0;
5229 curfid = 0;
5230 inodelist = [];
5231 inodetofile = {};
5232 filetoinode = {};
5233 inodetocatinode = {};
5234 fileidnum = 0;
5235 if(infile=="-"):
5236 infile = BytesIO();
5237 if(hasattr(sys.stdin, "buffer")):
5238 shutil.copyfileobj(sys.stdin.buffer, infile);
5239 else:
5240 shutil.copyfileobj(sys.stdin, infile);
5241 infile.seek(0, 0);
5242 if(not infile):
5243 return False;
5244 infile.seek(0, 0);
5245 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
5246 infile = download_file_from_internet_file(infile);
5247 infile.seek(0, 0);
5248 if(not infile):
5249 return False;
5250 infile.seek(0, 0);
5251 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
5252 return False;
5253 elif(os.path.exists(infile) and os.path.isfile(infile)):
5254 try:
5255 if(not tarfile.TarFileCheck(infile)):
5256 return False;
5257 except AttributeError:
5258 if(not TarFileCheck(infile)):
5259 return False;
5260 else:
5261 return False;
5262 try:
5263 if(hasattr(infile, "read") or hasattr(infile, "write")):
5264 tarfp = tarfile.open(fileobj=infile, mode="r");
5265 else:
5266 tarfp = tarfile.open(infile, "r");
5267 except FileNotFoundError:
5268 return False;
5269 fnumfiles = int(len(tarfp.getmembers()));
5270 catver = formatspecs['format_ver'];
5271 fileheaderver = str(int(catver.replace(".", "")));
5272 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5273 fnumfileshex = format(int(fnumfiles), 'x').lower();
5274 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5275 catversion = re.findall(r"([\d]+)", fileheader);
5276 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5277 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5278 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5279 fheadtell = len(fileheader);
5280 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
5281 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
5282 catfhstart = fheadtell;
5283 if(re.findall(r"^[.|/]", member.name)):
5284 fname = member.name;
5285 else:
5286 fname = "./"+member.name;
5287 if(verbose):
5288 VerbosePrintOut(fname);
5289 fpremode = member.mode;
5290 ffullmode = member.mode;
5291 flinkcount = 0;
5292 ftype = 0;
5293 if(member.isreg()):
5294 ffullmode = member.mode + stat.S_IFREG;
5295 ftype = 0;
5296 elif(member.isdev()):
5297 ffullmode = member.mode;
5298 ftype = 7;
5299 elif(member.islnk()):
5300 ffullmode = member.mode + stat.S_IFREG;
5301 ftype = 1;
5302 elif(member.issym()):
5303 ffullmode = member.mode + stat.S_IFLNK;
5304 ftype = 2;
5305 elif(member.ischr()):
5306 ffullmode = member.mode + stat.S_IFCHR;
5307 ftype = 3;
5308 elif(member.isblk()):
5309 ffullmode = member.mode + stat.S_IFBLK;
5310 ftype = 4;
5311 elif(member.isdir()):
5312 ffullmode = member.mode + stat.S_IFDIR;
5313 ftype = 5;
5314 elif(member.isfifo()):
5315 ffullmode = member.mode + stat.S_IFIFO;
5316 ftype = 6;
5317 elif(member.issparse()):
5318 ffullmode = member.mode;
5319 ftype = 12;
5320 else:
5321 ffullmode = member.mode;
5322 ftype = 0;
5323 flinkname = "";
5324 fbasedir = os.path.dirname(fname);
5325 fcurfid = curfid;
5326 fcurinode = curfid;
5327 finode = fcurinode;
5328 curfid = curfid + 1;
5329 if(ftype==2):
5330 flinkname = member.linkname;
5331 fdev_minor = member.devminor;
5332 fdev_major = member.devmajor;
5333 frdev_minor = member.devminor;
5334 frdev_major = member.devmajor;
5335 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
5336 fsize = "0";
5337 elif(ftype==0 or ftype==7):
5338 fsize = member.size;
5339 else:
5340 fsize = member.size;
5341 fatime = member.mtime;
5342 fmtime = member.mtime;
5343 fctime = member.mtime;
5344 fbtime = member.mtime;
5345 fmode = ffullmode;
5346 fchmode = stat.S_IMODE(ffullmode);
5347 ftypemod = stat.S_IFMT(ffullmode);
5348 fuid = member.uid;
5349 fgid = member.gid;
5350 funame = member.uname;
5351 fgname = member.gname;
5352 flinkcount = flinkcount;
5353 fwinattributes = int(0);
5354 fcompression = "";
5355 fcsize = 0;
5356 fcontents = BytesIO();
5357 if(ftype==0 or ftype==7):
5358 with tarfp.extractfile(member) as fpc:
5359 shutil.copyfileobj(fpc, fcontents);
5360 fcontents.seek(0, 0);
5361 ftypehex = format(ftype, 'x').lower();
5362 extrafields = len(extradata);
5363 extrafieldslist = extradata;
5364 catfextrafields = extrafields;
5365 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5366 if(len(extradata)>0):
5367 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5368 extrasizelen = len(extrasizestr);
5369 extrasizelenhex = format(extrasizelen, 'x').lower();
5370 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), "+1", extrasizelenhex, format(catfextrafields, 'x').lower()];
5371 catoutlen = len(catoutlist) + len(extradata) + 3;
5372 catoutlenhex = format(catoutlen, 'x').lower();
5373 catoutlist.insert(0, catoutlenhex);
5374 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5375 catheaderdata = catoutlist;
5376 if(len(extradata)>0):
5377 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5378 if(fsize==0):
5379 checksumlist = [checksumtype, "none"];
5380 else:
5381 checksumlist = [checksumtype, checksumtype];
5382 catfileoutstr = catfileoutstr + AppendNullBytes(checksumlist, formatspecs['format_delimiter']);
5383 catfnumfields = catoutlen;
5384 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5385 fcontents.seek(0, 0);
5386 if(fsize==0):
5387 catfilecontentcshex = GetFileChecksum(fcontents.read(), "none", False, formatspecs);
5388 else:
5389 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5390 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5391 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5392 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5393 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5394 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5395 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5396 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5397 catfcontentstart = fheadtell;
5398 fheadtell += len(catfileoutstr) + 1;
5399 catfcontentend = fheadtell - 1;
5400 catfhend = catfcontentend;
5401 fcontents.seek(0, 0);
5402 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5403 pyhascontents = False;
5404 if(int(fsize)>0 and not listonly):
5405 pyhascontents = True;
5406 if(int(fsize)>0 and listonly):
5407 fcontents = BytesIO();
5408 pyhascontents = False;
5409 fcontents.seek(0, 0);
5410 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype, 'fcontentchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents});
5411 fileidnum = fileidnum + 1;
5412 return catlist;
5413
5414 def ZipFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5415 formatspecs = FormatSpecsListToDict(formatspecs);
5416 curinode = 0;
5417 curfid = 0;
5418 inodelist = [];
5419 inodetofile = {};
5420 filetoinode = {};
5421 inodetocatinode = {};
5422 fileidnum = 0;
5423 if(infile=="-"):
5424 infile = BytesIO();
5425 if(hasattr(sys.stdin, "buffer")):
5426 shutil.copyfileobj(sys.stdin.buffer, infile);
5427 else:
5428 shutil.copyfileobj(sys.stdin, infile);
5429 infile.seek(0, 0);
5430 if(not infile):
5431 return False;
5432 infile.seek(0, 0);
5433 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
5434 infile = download_file_from_internet_file(infile);
5435 infile.seek(0, 0);
5436 if(not infile):
5437 return False;
5438 infile.seek(0, 0);
5439 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
5440 return False;
5441 else:
5442 return False;
5443 if(not zipfile.is_zipfile(infile)):
5444 return False;
5445 try:
5446 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
5447 except FileNotFoundError:
5448 return False;
5449 ziptest = zipfp.testzip();
5450 if(ziptest):
5451 VerbosePrintOut("Bad file found!");
5452 fnumfiles = int(len(zipfp.infolist()));
5453 catver = formatspecs['format_ver'];
5454 fileheaderver = str(int(catver.replace(".", "")));
5455 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5456 catversion = re.findall(r"([\d]+)", fileheader);
5457 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5458 fnumfileshex = format(int(fnumfiles), 'x').lower();
5459 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5460 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5461 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5462 fheadtell = len(fileheader);
5463 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
5464 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
5465 catfhstart = fheadtell;
5466 if(re.findall(r"^[.|/]", member.filename)):
5467 fname = member.filename;
5468 else:
5469 fname = "./"+member.filename;
5470 zipinfo = zipfp.getinfo(member.filename);
5471 if(verbose):
5472 VerbosePrintOut(fname);
5473 if(not member.is_dir()):
5474 fpremode = stat.S_IFREG + 438;
5475 elif(member.is_dir()):
5476 fpremode = stat.S_IFDIR + 511;
5477 flinkcount = 0;
5478 ftype = 0;
5479 if(not member.is_dir()):
5480 ftype = 0;
5481 elif(member.is_dir()):
5482 ftype = 5;
5483 flinkname = "";
5484 fbasedir = os.path.dirname(fname);
5485 fcurfid = curfid;
5486 fcurinode = curfid;
5487 finode = fcurinode;
5488 curfid = curfid + 1;
5489 fdev_minor = 0;
5490 fdev_major = 0;
5491 frdev_minor = 0;
5492 frdev_major = 0;
5493 if(ftype==5):
5494 fsize = "0";
5495 elif(ftype==0):
5496 fsize = member.file_size;
5497 else:
5498 fsize = member.file_size;
5499 fatime = time.mktime(member.date_time + (0, 0, -1));
5500 fmtime = time.mktime(member.date_time + (0, 0, -1));
5501 fctime = time.mktime(member.date_time + (0, 0, -1));
5502 fbtime = time.mktime(member.date_time + (0, 0, -1));
5503 if(zipinfo.create_system==0 or zipinfo.create_system==10):
5504 fwinattributes = int(zipinfo.external_attr);
5505 if(not member.is_dir()):
5506 fmode = int(stat.S_IFREG + 438);
5507 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
5508 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
5509 elif(member.is_dir()):
5510 fmode = int(stat.S_IFDIR + 511);
5511 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
5512 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
5513 elif(zipinfo.create_system==3):
5514 fwinattributes = int(0);
5515 fmode = int(zipinfo.external_attr);
5516 else:
5517 fwinattributes = int(0);
5518 if(not member.is_dir()):
5519 fmode = int(stat.S_IFREG + 438);
5520 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
5521 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
5522 elif(member.is_dir()):
5523 fmode = int(stat.S_IFDIR + 511);
5524 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
5525 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
5526 fcompression = "";
5527 fcsize = 0;
5528 try:
5529 fuid = os.getuid();
5530 except AttributeError:
5531 fuid = 0;
5532 except KeyError:
5533 fuid = 0;
5534 try:
5535 fgid = os.getgid();
5536 except AttributeError:
5537 fgid = 0;
5538 except KeyError:
5539 fgid = 0;
5540 try:
5541 import pwd;
5542 try:
5543 userinfo = pwd.getpwuid(os.getuid());
5544 funame = userinfo.pw_name;
5545 except KeyError:
5546 funame = "";
5547 except AttributeError:
5548 funame = "";
5549 except ImportError:
5550 funame = "";
5551 fgname = "";
5552 try:
5553 import grp;
5554 try:
5555 groupinfo = grp.getgrgid(os.getgid());
5556 fgname = groupinfo.gr_name;
5557 except KeyError:
5558 fgname = "";
5559 except AttributeError:
5560 fgname = "";
5561 except ImportError:
5562 fgname = "";
5563 fcontents = BytesIO();
5564 if(ftype==0):
5565 fcontents.write(zipfp.read(member.filename));
5566 fcontents.seek(0, 0);
5567 ftypehex = format(ftype, 'x').lower();
5568 extrafields = len(extradata);
5569 extrafieldslist = extradata;
5570 catfextrafields = extrafields;
5571 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5572 if(len(extradata)>0):
5573 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5574 extrasizelen = len(extrasizestr);
5575 extrasizelenhex = format(extrasizelen, 'x').lower();
5576 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), "+1", extrasizelenhex, format(catfextrafields, 'x').lower()];
5577 catoutlen = len(catoutlist) + len(extradata) + 3;
5578 catoutlenhex = format(catoutlen, 'x').lower();
5579 catoutlist.insert(0, catoutlenhex);
5580 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5581 catheaderdata = catoutlist;
5582 if(len(extradata)>0):
5583 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5584 if(fsize==0):
5585 checksumlist = [checksumtype, "none"];
5586 else:
5587 checksumlist = [checksumtype, checksumtype];
5588 catfileoutstr = catfileoutstr + AppendNullBytes(checksumlist, formatspecs['format_delimiter']);
5589 catfnumfields = catoutlen;
5590 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5591 fcontents.seek(0, 0);
5592 if(fsize==0):
5593 catfilecontentcshex = GetFileChecksum(fcontents.read(), "none", False, formatspecs);
5594 else:
5595 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5596 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5597 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5598 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5599 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5600 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5601 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5602 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5603 catfcontentstart = fheadtell;
5604 fheadtell += len(catfileoutstr) + 1;
5605 catfcontentend = fheadtell - 1;
5606 catfhend = catfcontentend;
5607 fcontents.seek(0, 0);
5608 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5609 pyhascontents = False;
5610 if(int(fsize)>0 and not listonly):
5611 pyhascontents = True;
5612 if(int(fsize)>0 and listonly):
5613 fcontents = BytesIO();
5614 pyhascontents = False;
5615 fcontents.seek(0, 0);
5616 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype, 'fcontentchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents});
5617 fileidnum = fileidnum + 1;
5618 return catlist;
5619
5620 if(not rarfile_support):
5621 def RarFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5622 return False;
5623
5624 if(rarfile_support):
5625 def RarFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5626 formatspecs = FormatSpecsListToDict(formatspecs);
5627 curinode = 0;
5628 curfid = 0;
5629 inodelist = [];
5630 inodetofile = {};
5631 filetoinode = {};
5632 inodetocatinode = {};
5633 fileidnum = 0;
5634 if(not os.path.exists(infile,) or not os.path.isfile(infile,)):
5635 return False;
5636 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
5637 return False;
5638 rarfp = rarfile.RarFile(infile, "r");
5639 rartest = rarfp.testrar();
5640 if(rartest):
5641 VerbosePrintOut("Bad file found!");
5642 fnumfiles = int(len(rarfp.infolist()));
5643 catver = formatspecs['format_ver'];
5644 fileheaderver = str(int(catver.replace(".", "")));
5645 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5646 catversion = re.findall(r"([\d]+)", fileheader);
5647 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5648 fnumfileshex = format(int(fnumfiles), 'x').lower();
5649 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5650 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5651 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5652 fheadtell = len(fileheader);
5653 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
5654 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
5655 catfhstart = fheadtell;
5656 is_unix = False;
5657 is_windows = False;
5658 if(member.host_os==rarfile.RAR_OS_UNIX):
5659 is_windows = False;
5660 try:
5661 member.external_attr
5662 is_unix = True;
5663 except AttributeError:
5664 is_unix = False;
5665 elif(member.host_os==rarfile.RAR_OS_WIN32):
5666 is_unix = False;
5667 try:
5668 member.external_attr
5669 is_windows = True;
5670 except AttributeError:
5671 is_windows = False;
5672 else:
5673 is_unix = False;
5674 is_windows = False;
5675 if(re.findall(r"^[.|/]", member.filename)):
5676 fname = member.filename;
5677 else:
5678 fname = "./"+member.filename;
5679 rarinfo = rarfp.getinfo(member.filename);
5680 if(verbose):
5681 VerbosePrintOut(fname);
5682 if(is_unix and member.external_attr !=0):
5683 fpremode = int(member.external_attr);
5684 elif(member.is_file()):
5685 fpremode = stat.S_IFREG + 438;
5686 elif(member.is_symlink()):
5687 fpremode = stat.S_IFLNK + 438;
5688 elif(member.is_dir()):
5689 fpremode = stat.S_IFDIR + 511;
5690 if(is_windows and member.external_attr !=0):
5691 fwinattributes = int(member.external_attr);
5692 else:
5693 fwinattributes = int(0);
5694 fcompression = "";
5695 fcsize = 0;
5696 flinkcount = 0;
5697 ftype = 0;
5698 if(member.is_file()):
5699 ftype = 0;
5700 elif(member.is_symlink()):
5701 ftype = 2;
5702 elif(member.is_dir()):
5703 ftype = 5;
5704 flinkname = "";
5705 if(ftype==2):
5706 flinkname = rarfp.read(member.filename).decode("UTF-8");
5707 fbasedir = os.path.dirname(fname);
5708 fcurfid = curfid;
5709 fcurinode = curfid;
5710 finode = fcurinode;
5711 curfid = curfid + 1;
5712 fdev_minor = 0;
5713 fdev_major = 0;
5714 frdev_minor = 0;
5715 frdev_major = 0;
5716 if(ftype==5):
5717 fsize = "0";
5718 if(ftype==0):
5719 fsize = member.file_size;
5720 try:
5721 if(member.atime):
5722 fatime = int(member.atime.timestamp());
5723 else:
5724 fatime = int(member.mtime.timestamp());
5725 except AttributeError:
5726 fatime = int(member.mtime.timestamp());
5727 fmtime = int(member.mtime.timestamp());
5728 try:
5729 if(member.ctime):
5730 fctime = int(member.ctime.timestamp());
5731 else:
5732 fctime = int(member.mtime.timestamp());
5733 except AttributeError:
5734 fctime = int(member.mtime.timestamp());
5735 fbtime = int(member.mtime.timestamp());
5736 if(is_unix and member.external_attr !=0):
5737 fmode = int(member.external_attr);
5738 fchmode = int(stat.S_IMODE(member.external_attr));
5739 ftypemod = int(stat.S_IFMT(member.external_attr));
5740 elif(member.is_file()):
5741 fmode = int(stat.S_IFREG + 438)
5742 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5743 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5744 elif(member.is_symlink()):
5745 fmode = int(stat.S_IFLNK + 438)
5746 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5747 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5748 elif(member.is_dir()):
5749 fmode = int(stat.S_IFDIR + 511)
5750 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
5751 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
5752 try:
5753 fuid = os.getuid();
5754 except AttributeError:
5755 fuid = 0;
5756 except KeyError:
5757 fuid = 0;
5758 try:
5759 fgid = os.getgid();
5760 except AttributeError:
5761 fgid = 0;
5762 except KeyError:
5763 fgid = 0;
5764 try:
5765 import pwd;
5766 try:
5767 userinfo = pwd.getpwuid(os.getuid());
5768 funame = userinfo.pw_name;
5769 except KeyError:
5770 funame = "";
5771 except AttributeError:
5772 funame = "";
5773 except ImportError:
5774 funame = "";
5775 fgname = "";
5776 try:
5777 import grp;
5778 try:
5779 groupinfo = grp.getgrgid(os.getgid());
5780 fgname = groupinfo.gr_name;
5781 except KeyError:
5782 fgname = "";
5783 except AttributeError:
5784 fgname = "";
5785 except ImportError:
5786 fgname = "";
5787 fcontents = BytesIO();
5788 if(ftype==0):
5789 fcontents.write(rarfp.read(member.filename));
5790 fcontents.seek(0, 0);
5791 ftypehex = format(ftype, 'x').lower();
5792 extrafields = len(extradata);
5793 extrafieldslist = extradata;
5794 catfextrafields = extrafields;
5795 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5796 if(len(extradata)>0):
5797 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5798 extrasizelen = len(extrasizestr);
5799 extrasizelenhex = format(extrasizelen, 'x').lower();
5800 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), "+1", extrasizelenhex, format(catfextrafields, 'x').lower()];
5801 catoutlen = len(catoutlist) + len(extradata) + 3;
5802 catoutlenhex = format(catoutlen, 'x').lower();
5803 catoutlist.insert(0, catoutlenhex);
5804 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5805 if(len(extradata)>0):
5806 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5807 if(fsize==0):
5808 checksumlist = [checksumtype, "none"];
5809 else:
5810 checksumlist = [checksumtype, checksumtype];
5811 ccatfileoutstr = catfileoutstr + AppendNullBytes(checksumlist, formatspecs['format_delimiter']);
5812 catfnumfields = 24 + catfextrafields;
5813 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5814 fcontents.seek(0, 0);
5815 if(fsize==0):
5816 catfilecontentcshex = GetFileChecksum(fcontents.read(), "none", False, formatspecs);
5817 else:
5818 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5819 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5820 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5821 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5822 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5823 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5824 catheaderdata = catoutlist;
5825 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5826 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5827 catfcontentstart = fheadtell;
5828 fheadtell += len(catfileoutstr) + 1;
5829 catfcontentend = fheadtell - 1;
5830 catfhend = catfcontentend;
5831 fcontents.seek(0, 0);
5832 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5833 pyhascontents = False;
5834 if(int(fsize)>0 and not listonly):
5835 pyhascontents = True;
5836 if(int(fsize)>0 and listonly):
5837 fcontents = BytesIO();
5838 pyhascontents = False;
5839 fcontents.seek(0, 0);
5840 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype, 'fcontentchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents});
5841 fileidnum = fileidnum + 1;
5842 return catlist;
5843
5844 if(not py7zr_support):
5845 def SevenZipFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5846 return False;
5847
5848 if(py7zr_support):
5849 def SevenZipFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5850 formatspecs = FormatSpecsListToDict(formatspecs);
5851 curinode = 0;
5852 curfid = 0;
5853 inodelist = [];
5854 inodetofile = {};
5855 filetoinode = {};
5856 inodetocatinode = {};
5857 fileidnum = 0;
5858 szpfp = py7zr.SevenZipFile(infile, mode="r");
5859 file_content = szpfp.readall();
5860 #sztest = szpfp.testzip();
5861 sztestalt = szpfp.test();
5862 if(sztestalt):
5863 VerbosePrintOut("Bad file found!");
5864 numfiles = int(len(szpfp.list()));
5865 catver = formatspecs['format_ver'];
5866 fileheaderver = str(int(catver.replace(".", "")));
5867 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5868 catversion = re.findall(r"([\d]+)", fileheader);
5869 catversions = re.search(r'(.*?)(\d+)', fileheader).groups();
5870 fnumfileshex = format(int(fnumfiles), 'x').lower();
5871 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5872 catfileheadercshex = GetFileChecksum(fileheader, checksumtype, True, formatspecs);
5873 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5874 fheadtell = len(fileheader);
5875 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': []};
5876 for member in sorted(szpfp.list(), key=lambda x: x.filename):
5877 catfhstart = fheadtell;
5878 if(re.findall(r"^[.|/]", member.filename)):
5879 fname = member.filename;
5880 else:
5881 fname = "./"+member.filename;
5882 if(not member.is_directory):
5883 fpremode = int(stat.S_IFREG + 438);
5884 elif(member.is_directory):
5885 fpremode = int(stat.S_IFDIR + 511);
5886 fwinattributes = int(0);
5887 fcompression = "";
5888 fcsize = 0;
5889 flinkcount = 0;
5890 ftype = 0;
5891 if(member.is_directory):
5892 ftype = 5;
5893 else:
5894 ftype = 0;
5895 flinkname = "";
5896 fbasedir = os.path.dirname(fname);
5897 fcurfid = curfid;
5898 fcurinode = curfid;
5899 finode = fcurinode;
5900 curfid = curfid + 1;
5901 fdev_minor = 0;
5902 fdev_major = 0;
5903 frdev_minor = 0;
5904 frdev_major = 0;
5905 if(ftype==5):
5906 fsize = "0";
5907 fatime = int(member.creationtime.timestamp());
5908 fmtime = int(member.creationtime.timestamp());
5909 fctime = int(member.creationtime.timestamp());
5910 fbtime = int(member.creationtime.timestamp());
5911 if(member.is_directory):
5912 fmode = int(stat.S_IFDIR + 511)
5913 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
5914 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
5915 else:
5916 fmode = int(stat.S_IFLNK + 438)
5917 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5918 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5919 try:
5920 fuid = os.getuid();
5921 except AttributeError:
5922 fuid = 0;
5923 except KeyError:
5924 fuid = 0;
5925 try:
5926 fgid = os.getgid();
5927 except AttributeError:
5928 fgid = 0;
5929 except KeyError:
5930 fgid = 0;
5931 try:
5932 import pwd;
5933 try:
5934 userinfo = pwd.getpwuid(os.getuid());
5935 funame = userinfo.pw_name;
5936 except KeyError:
5937 funame = "";
5938 except AttributeError:
5939 funame = "";
5940 except ImportError:
5941 funame = "";
5942 fgname = "";
5943 try:
5944 import grp;
5945 try:
5946 groupinfo = grp.getgrgid(os.getgid());
5947 fgname = groupinfo.gr_name;
5948 except KeyError:
5949 fgname = "";
5950 except AttributeError:
5951 fgname = "";
5952 except ImportError:
5953 fgname = "";
5954 fcontents = BytesIO();
5955 if(ftype==0):
5956 fcontents.write(file_content[member.filename].read());
5957 fsize = format(fcontents.tell(), 'x').lower();
5958 fileop.close();
5959 fcontents.seek(0, 0);
5960 ftypehex = format(ftype, 'x').lower();
5961 extrafields = len(extradata);
5962 extrafieldslist = extradata;
5963 catfextrafields = extrafields;
5964 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5965 if(len(extradata)>0):
5966 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5967 extrasizelen = len(extrasizestr);
5968 extrasizelenhex = format(extrasizelen, 'x').lower();
5969 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), "+1", extrasizelenhex, format(catfextrafields, 'x').lower()];
5970 catoutlen = len(catoutlist) + len(extradata) + 3;
5971 catoutlenhex = format(catoutlen, 'x').lower();
5972 catoutlist.insert(0, catoutlenhex);
5973 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5974 catheaderdata = catoutlist;
5975 if(len(extradata)>0):
5976 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5977 if(fsize==0):
5978 checksumlist = [checksumtype, "none"];
5979 else:
5980 checksumlist = [checksumtype, checksumtype];
5981 catfileoutstr = catfileoutstr + AppendNullBytes(checksumlist, formatspecs['format_delimiter']);
5982 catfnumfields = 24 + catfextrafields;
5983 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5984 fcontents.seek(0, 0);
5985 if(fsize==0):
5986 catfilecontentcshex = GetFileChecksum(fcontents.read(), "none", False, formatspecs);
5987 else:
5988 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5989 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5990 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5991 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5992 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5993 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5994 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5995 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5996 catfcontentstart = fheadtell;
5997 fheadtell += len(catfileoutstr) + 1;
5998 catfcontentend = fheadtell - 1;
5999 catfhend = catfcontentend;
6000 fcontents.seek(0, 0);
6001 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
6002 pyhascontents = False;
6003 if(int(fsize)>0 and not listonly):
6004 pyhascontents = True;
6005 if(int(fsize)>0 and listonly):
6006 fcontents = BytesIO();
6007 pyhascontents = False;
6008 fcontents.seek(0, 0);
6009 catlist['ffilelist'].append({'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fseeknextfile': "+1", 'fheaderchecksumtype': checksumtype, 'fcontentchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents});
6010 fileidnum = fileidnum + 1;
6011 return catlist;
6012
6013 def InFileToArrayAlt(infile, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6014 formatspecs = FormatSpecsListToDict(formatspecs);
6015 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
6016 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
6017 return TarFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6018 elif(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
6019 return ZipFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6020 elif(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
6021 return RarFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6022 elif(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
6023 return SevenZipFileToArrayAlt(infile, listonly, checksumtype, extradata, formatspecs, verbose);
6024 elif(checkcompressfile=="catfile"):
6025 return ArchiveFileToArray(infile, 0, 0, listonly, True, False, formatspecs, False);
6026 else:
6027 return False;
6028 return False;
6029
6030 def ListDirToArray(infiles, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, listonly=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6031 formatspecs = FormatSpecsListToDict(formatspecs);
6032 outarray = BytesIO();
6033 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
6034 listcatfiles = ArchiveFileToArray(outarray, seekstart, seekend, listonly, True, skipchecksum, formatspecs, returnfp);
6035 return listcatfiles;
6036
6037 def ArchiveFileArrayToArrayIndex(inarray, seekstart=0, seekend=0, listonly=False, uncompress=True, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6038 formatspecs = FormatSpecsListToDict(formatspecs);
6039 if(isinstance(inarray, dict)):
6040 listcatfiles = inarray;
6041 else:
6042 return False;
6043 if(not listcatfiles):
6044 return False;
6045 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6046 if(returnfp):
6047 catarray.update({'catfp': listcatfiles['catfp']});
6048 lenlist = len(listcatfiles['ffilelist']);
6049 lcfi = 0;
6050 lcfx = int(listcatfiles['fnumfiles']);
6051 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6052 lcfx = int(lenlist);
6053 else:
6054 lcfx = int(listcatfiles['fnumfiles']);
6055 while(lcfi < lcfx):
6056 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6057 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6058 catarray['filetoid'].update(filetoidarray);
6059 catarray['idtofile'].update(idtofilearray);
6060 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6061 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6062 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6063 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6064 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6065 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6066 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6067 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6068 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6069 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6070 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6071 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6072 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6073 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6074 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6075 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6076 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6077 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6078 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6079 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6080 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6081 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6082 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6083 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6084 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6085 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6086 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6087 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6088 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6089 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6090 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6091 lcfi = lcfi + 1;
6092 return catarray;
6093
6094 create_alias_function("", __file_format_name__, "ArrayToArrayIndex", ArchiveFileArrayToArrayIndex);
6095
6096 def RePackArchiveFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6097 formatspecs = FormatSpecsListToDict(formatspecs);
6098 if(isinstance(infile, dict)):
6099 listcatfiles = infile;
6100 else:
6101 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6102 infile = RemoveWindowsPath(infile);
6103 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, True, skipchecksum, formatspecs, returnfp);
6104 if(outfile!="-" and not hasattr(infile, "read") and not hasattr(outfile, "write")):
6105 outfile = RemoveWindowsPath(outfile);
6106 checksumtype = checksumtype.lower();
6107 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
6108 checksumtype="crc32";
6109 if(checksumtype=="none"):
6110 checksumtype = "";
6111 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
6112 compression = "auto";
6113 if(compression not in compressionlist and compression is None):
6114 compression = "auto";
6115 if(verbose):
6116 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6117 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
6118 if(os.path.exists(outfile)):
6119 try:
6120 os.unlink(outfile);
6121 except OSError as e:
6122 pass;
6123 if(not listcatfiles):
6124 return False;
6125 if(outfile=="-"):
6126 verbose = False;
6127 catfp = BytesIO();
6128 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
6129 catfp = outfile;
6130 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
6131 catfp = BytesIO();
6132 else:
6133 fbasename = os.path.splitext(outfile)[0];
6134 fextname = os.path.splitext(outfile)[1];
6135 if(not compresswholefile and fextname in outextlistwd):
6136 compresswholefile = True;
6137 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
6138 catver = formatspecs['format_ver'];
6139 fileheaderver = str(int(catver.replace(".", "")));
6140 lenlist = len(listcatfiles['ffilelist']);
6141 fnumfiles = int(listcatfiles['fnumfiles']);
6142 if(lenlist>fnumfiles or lenlist<fnumfiles):
6143 fnumfiles = lenlist;
6144 AppendFileHeader(catfp, fnumfiles, checksumtype, formatspecs);
6145 lenlist = len(listcatfiles['ffilelist']);
6146 fnumfiles = int(listcatfiles['fnumfiles']);
6147 lcfi = 0;
6148 lcfx = int(listcatfiles['fnumfiles']);
6149 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6150 lcfx = int(lenlist);
6151 else:
6152 lcfx = int(listcatfiles['fnumfiles']);
6153 curinode = 0;
6154 curfid = 0;
6155 inodelist = [];
6156 inodetofile = {};
6157 filetoinode = {};
6158 reallcfi = 0;
6159 while(lcfi < lcfx):
6160 if(re.findall(r"^[.|/]", listcatfiles['ffilelist'][reallcfi]['fname'])):
6161 fname = listcatfiles['ffilelist'][reallcfi]['fname'];
6162 else:
6163 fname = "./"+listcatfiles['ffilelist'][reallcfi]['fname'];
6164 if(verbose):
6165 VerbosePrintOut(fname);
6166 fheadersize = format(int(listcatfiles['ffilelist'][reallcfi]['fheadersize']), 'x').lower();
6167 fsize = format(int(listcatfiles['ffilelist'][reallcfi]['fsize']), 'x').lower();
6168 flinkname = listcatfiles['ffilelist'][reallcfi]['flinkname'];
6169 fatime = format(int(listcatfiles['ffilelist'][reallcfi]['fatime']), 'x').lower();
6170 fmtime = format(int(listcatfiles['ffilelist'][reallcfi]['fmtime']), 'x').lower();
6171 fctime = format(int(listcatfiles['ffilelist'][reallcfi]['fctime']), 'x').lower();
6172 fbtime = format(int(listcatfiles['ffilelist'][reallcfi]['fbtime']), 'x').lower();
6173 fmode = format(int(listcatfiles['ffilelist'][reallcfi]['fmode']), 'x').lower();
6174 fchmode = format(int(listcatfiles['ffilelist'][reallcfi]['fchmode']), 'x').lower();
6175 fuid = format(int(listcatfiles['ffilelist'][reallcfi]['fuid']), 'x').lower();
6176 funame = listcatfiles['ffilelist'][reallcfi]['funame'];
6177 fgid = format(int(listcatfiles['ffilelist'][reallcfi]['fgid']), 'x').lower();
6178 fgname = listcatfiles['ffilelist'][reallcfi]['fgname'];
6179 finode = format(int(listcatfiles['ffilelist'][reallcfi]['finode']), 'x').lower();
6180 flinkcount = format(int(listcatfiles['ffilelist'][reallcfi]['flinkcount']), 'x').lower();
6181 fwinattributes = format(int(listcatfiles['ffilelist'][reallcfi]['fwinattributes']), 'x').lower();
6182 fcompression = listcatfiles['ffilelist'][reallcfi]['fcompression'];
6183 fcsize = format(int(listcatfiles['ffilelist'][reallcfi]['fcsize']), 'x').lower();
6184 fdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['fminor']), 'x').lower();
6185 fdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['fmajor']), 'x').lower();
6186 frdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['frminor']), 'x').lower();
6187 frdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['frmajor']), 'x').lower();
6188 fseeknextfile = listcatfiles['ffilelist'][reallcfi]['fseeknextfile'];
6189 if(len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>listcatfiles['ffilelist'][reallcfi]['fextrafields'] and len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>0):
6190 listcatfiles['ffilelist'][reallcfi]['fextrafields'] = len(listcatfiles['ffilelist'][reallcfi]['fextralist']);
6191 if(not followlink and len(extradata)<0):
6192 extradata = listcatfiles['ffilelist'][reallcfi]['fextralist'];
6193 fcontents = listcatfiles['ffilelist'][reallcfi]['fcontents'];
6194 fcompression = "";
6195 fcsize = format(int(0), 'x').lower();
6196 if(not compresswholefile):
6197 fcontents.seek(0, 2);
6198 ucfsize = fcontents.tell();
6199 fcontents.seek(0, 0);
6200 if(compression=="auto"):
6201 ilsize = len(compressionlistalt);
6202 ilmin = 0;
6203 ilcsize = [];
6204 while(ilmin < ilsize):
6205 cfcontents = BytesIO();
6206 shutil.copyfileobj(fcontents, cfcontents);
6207 fcontents.seek(0, 0);
6208 cfcontents.seek(0, 0);
6209 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
6210 if(cfcontents):
6211 cfcontents.seek(0, 2);
6212 ilcsize.append(cfcontents.tell());
6213 cfcontents.close();
6214 else:
6215 try:
6216 ilcsize.append(sys.maxint);
6217 except AttributeError:
6218 ilcsize.append(sys.maxsize);
6219 ilmin = ilmin + 1;
6220 ilcmin = ilcsize.index(min(ilcsize));
6221 compression = compressionlistalt[ilcmin];
6222 fcontents.seek(0, 0);
6223 cfcontents = BytesIO();
6224 shutil.copyfileobj(fcontents, cfcontents);
6225 cfcontents.seek(0, 0);
6226 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
6227 cfcontents.seek(0, 2);
6228 cfsize = cfcontents.tell();
6229 if(ucfsize > cfsize):
6230 fcsize = format(int(cfsize), 'x').lower();
6231 fcompression = compression;
6232 fcontents.close();
6233 fcontents = cfcontents;
6234 if(followlink):
6235 if(listcatfiles['ffilelist'][reallcfi]['ftype']==1 or listcatfiles['ffilelist'][reallcfi]['ftype']==2):
6236 getflinkpath = listcatfiles['ffilelist'][reallcfi]['flinkname'];
6237 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6238 flinkinfo = listcatfiles['ffilelist'][flinkid];
6239 fheadersize = format(int(flinkinfo['fheadersize']), 'x').lower();
6240 fsize = format(int(flinkinfo['fsize']), 'x').lower();
6241 flinkname = flinkinfo['flinkname'];
6242 fatime = format(int(flinkinfo['fatime']), 'x').lower();
6243 fmtime = format(int(flinkinfo['fmtime']), 'x').lower();
6244 fctime = format(int(flinkinfo['fctime']), 'x').lower();
6245 fbtime = format(int(flinkinfo['fbtime']), 'x').lower();
6246 fmode = format(int(flinkinfo['fmode']), 'x').lower();
6247 fchmode = format(int(flinkinfo['fchmode']), 'x').lower();
6248 fuid = format(int(flinkinfo['fuid']), 'x').lower();
6249 funame = flinkinfo['funame'];
6250 fgid = format(int(flinkinfo['fgid']), 'x').lower();
6251 fgname = flinkinfo['fgname'];
6252 finode = format(int(flinkinfo['finode']), 'x').lower();
6253 flinkcount = format(int(flinkinfo['flinkcount']), 'x').lower();
6254 fwinattributes = format(int(flinkinfo['fwinattributes']), 'x').lower();
6255 fcompression = flinkinfo['fcompression'];
6256 fcsize = format(int(flinkinfo['fcsize']), 'x').lower();
6257 fdev_minor = format(int(flinkinfo['fminor']), 'x').lower();
6258 fdev_major = format(int(flinkinfo['fmajor']), 'x').lower();
6259 frdev_minor = format(int(flinkinfo['frminor']), 'x').lower();
6260 frdev_major = format(int(flinkinfo['frmajor']), 'x').lower();
6261 fseeknextfile = flinkinfo['fseeknextfile'];
6262 if(len(flinkinfo['fextralist'])>flinkinfo['fextrafields'] and len(flinkinfo['fextralist'])>0):
6263 flinkinfo['fextrafields'] = len(flinkinfo['fextralist']);
6264 if(len(extradata)<0):
6265 extradata = flinkinfo['fextralist'];
6266 fcontents = flinkinfo['fcontents'];
6267 ftypehex = format(flinkinfo['ftype'], 'x').lower();
6268 else:
6269 ftypehex = format(listcatfiles['ffilelist'][reallcfi]['ftype'], 'x').lower();
6270 fcurfid = format(curfid, 'x').lower();
6271 if(not followlink and finode!=0):
6272 if(listcatfiles['ffilelist'][reallcfi]['ftype']!=1):
6273 fcurinode = format(int(curinode), 'x').lower();
6274 inodetofile.update({curinode: fname});
6275 filetoinode.update({fname: curinode});
6276 curinode = curinode + 1;
6277 else:
6278 fcurinode = format(int(filetoinode[flinkname]), 'x').lower();
6279 else:
6280 fcurinode = format(int(curinode), 'x').lower();
6281 curinode = curinode + 1;
6282 curfid = curfid + 1;
6283 if(fcompression=="none"):
6284 fcompression = "";
6285 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, fseeknextfile];
6286 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
6287 fcontents.close();
6288 lcfi = lcfi + 1;
6289 reallcfi = reallcfi + 1;
6290 if(lcfx>0):
6291 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
6292 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
6293 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
6294 try:
6295 catfp.flush();
6296 os.fsync(catfp.fileno());
6297 except io.UnsupportedOperation:
6298 pass;
6299 except AttributeError:
6300 pass;
6301 except OSError as e:
6302 pass;
6303 if(outfile=="-"):
6304 catfp.seek(0, 0);
6305 if(hasattr(sys.stdout, "buffer")):
6306 shutil.copyfileobj(catfp, sys.stdout.buffer);
6307 else:
6308 shutil.copyfileobj(catfp, sys.stdout);
6309 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
6310 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
6311 catfp.seek(0, 0);
6312 upload_file_to_internet_file(catfp, outfile);
6313 if(returnfp):
6314 catfp.seek(0, 0);
6315 return catfp;
6316 else:
6317 catfp.close();
6318 return True;
6319
6320 create_alias_function("RePack", __file_format_name__, "", RePackArchiveFile);
6321
6322 def RePackArchiveFileFromString(catstr, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6323 formatspecs = FormatSpecsListToDict(formatspecs);
6324 catfp = BytesIO(catstr);
6325 listcatfiles = RePackArchiveFile(catfp, compression, compresswholefile, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6326 return listcatfiles;
6327
6328 create_alias_function("RePack", __file_format_name__, "FromString", RePackArchiveFileFromString);
6329
6330 def PackArchiveFileFromListDir(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6331 formatspecs = FormatSpecsListToDict(formatspecs);
6332 outarray = BytesIO();
6333 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
6334 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6335 return listcatfiles;
6336
6337 create_alias_function("Pack", __file_format_name__, "FromListDir", PackArchiveFileFromListDir);
6338
6339 def UnPackArchiveFile(infile, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, preservepermissions=True, preservetime=True, verbose=False, returnfp=False):
6340 formatspecs = FormatSpecsListToDict(formatspecs);
6341 if(outdir is not None):
6342 outdir = RemoveWindowsPath(outdir);
6343 if(verbose):
6344 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6345 if(isinstance(infile, dict)):
6346 listcatfiles = infile;
6347 else:
6348 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6349 infile = RemoveWindowsPath(infile);
6350 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, True, skipchecksum, formatspecs, returnfp);
6351 if(not listcatfiles):
6352 return False;
6353 lenlist = len(listcatfiles['ffilelist']);
6354 fnumfiles = int(listcatfiles['fnumfiles']);
6355 lcfi = 0;
6356 lcfx = int(listcatfiles['fnumfiles']);
6357 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6358 lcfx = int(lenlist);
6359 else:
6360 lcfx = int(listcatfiles['fnumfiles']);
6361 while(lcfi<lcfx):
6362 funame = "";
6363 try:
6364 import pwd;
6365 try:
6366 userinfo = pwd.getpwuid(listcatfiles['ffilelist'][lcfi]['fuid']);
6367 funame = userinfo.pw_name;
6368 except KeyError:
6369 funame = "";
6370 except ImportError:
6371 funame = "";
6372 fgname = "";
6373 try:
6374 import grp;
6375 try:
6376 groupinfo = grp.getgrgid(listcatfiles['ffilelist'][lcfi]['fgid']);
6377 fgname = groupinfo.gr_name;
6378 except KeyError:
6379 fgname = "";
6380 except ImportError:
6381 fgname = "";
6382 if(verbose):
6383 VerbosePrintOut(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6384 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6385 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6386 listcatfiles['ffilelist'][lcfi]['fcontents'].seek(0, 0);
6387 shutil.copyfileobj(listcatfiles['ffilelist'][lcfi]['fcontents'], fpc);
6388 try:
6389 fpc.flush();
6390 os.fsync(fpc.fileno());
6391 except io.UnsupportedOperation:
6392 pass;
6393 except AttributeError:
6394 pass;
6395 except OSError as e:
6396 pass;
6397 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
6398 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
6399 if(preservepermissions):
6400 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6401 if(preservetime):
6402 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
6403 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6404 if(followlink):
6405 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
6406 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6407 flinkinfo = listcatfiles['ffilelist'][flinkid];
6408 funame = "";
6409 try:
6410 import pwd;
6411 try:
6412 userinfo = pwd.getpwuid(flinkinfo['fuid']);
6413 funame = userinfo.pw_name;
6414 except KeyError:
6415 funame = "";
6416 except ImportError:
6417 funame = "";
6418 fgname = "";
6419 try:
6420 import grp;
6421 try:
6422 groupinfo = grp.getgrgid(flinkinfo['fgid']);
6423 fgname = groupinfo.gr_name;
6424 except KeyError:
6425 fgname = "";
6426 except ImportError:
6427 fgname = "";
6428 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
6429 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6430 flinkinfo['fcontents'].seek(0, 0);
6431 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
6432 try:
6433 fpc.flush();
6434 os.fsync(fpc.fileno());
6435 except io.UnsupportedOperation:
6436 pass;
6437 except AttributeError:
6438 pass;
6439 except OSError as e:
6440 pass;
6441 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6442 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6443 if(preservepermissions):
6444 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6445 if(preservetime):
6446 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6447 if(flinkinfo['ftype']==1):
6448 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6449 if(flinkinfo['ftype']==2):
6450 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6451 if(flinkinfo['ftype']==5):
6452 if(preservepermissions):
6453 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6454 else:
6455 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6456 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6457 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6458 if(preservepermissions):
6459 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6460 if(preservetime):
6461 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6462 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
6463 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6464 else:
6465 os.link(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6466 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6467 if(followlink):
6468 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
6469 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6470 flinkinfo = listcatfiles['ffilelist'][flinkid];
6471 funame = "";
6472 try:
6473 import pwd;
6474 try:
6475 userinfo = pwd.getpwuid(flinkinfo['fuid']);
6476 funame = userinfo.pw_name;
6477 except KeyError:
6478 funame = "";
6479 except ImportError:
6480 funame = "";
6481 fgname = "";
6482 try:
6483 import grp;
6484 try:
6485 groupinfo = grp.getgrgid(flinkinfo['fgid']);
6486 fgname = groupinfo.gr_name;
6487 except KeyError:
6488 fgname = "";
6489 except ImportError:
6490 fgname = "";
6491 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
6492 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6493 flinkinfo['fcontents'].seek(0, 0);
6494 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
6495 try:
6496 fpc.flush();
6497 os.fsync(fpc.fileno());
6498 except io.UnsupportedOperation:
6499 pass;
6500 except AttributeError:
6501 pass;
6502 except OSError as e:
6503 pass;
6504 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6505 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6506 if(preservepermissions):
6507 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6508 if(preservetime):
6509 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6510 if(flinkinfo['ftype']==1):
6511 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6512 if(flinkinfo['ftype']==2):
6513 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6514 if(flinkinfo['ftype']==5):
6515 if(preservepermissions):
6516 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6517 else:
6518 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6519 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6520 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6521 if(preservepermissions):
6522 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6523 if(preservetime):
6524 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6525 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
6526 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6527 else:
6528 os.symlink(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6529 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6530 if(preservepermissions):
6531 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6532 else:
6533 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6534 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
6535 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
6536 if(preservepermissions):
6537 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6538 if(preservetime):
6539 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
6540 if(listcatfiles['ffilelist'][lcfi]['ftype']==6 and hasattr(os, "mkfifo")):
6541 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6542 lcfi = lcfi + 1;
6543 if(returnfp):
6544 return listcatfiles['ffilelist']['catfp'];
6545 else:
6546 return True;
6547
6548 create_alias_function("UnPack", __file_format_name__, "", UnPackArchiveFile);
6549
6550 if(hasattr(shutil, "register_unpack_format")):
6551 def UnPackArchiveFileFunc(archive_name, extract_dir=None, **kwargs):
6552 return UnPackArchiveFile(archive_name, extract_dir, False, 0, 0, False, __file_format_dict__['format_delimiter'], False, False);
6553 create_alias_function("UnPack", __file_format_name__, "Func", UnPackArchiveFileFunc);
6554
6555 def UnPackArchiveFileString(catstr, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6556 formatspecs = FormatSpecsListToDict(formatspecs);
6557 catfp = BytesIO(catstr);
6558 listcatfiles = UnPackArchiveFile(catfp, outdir, followlink, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6559 return listcatfiles;
6560
6561 create_alias_function("UnPack", __file_format_name__, "String", UnPackArchiveFileString);
6562
6563 def ArchiveFileListFiles(infile, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6564 formatspecs = FormatSpecsListToDict(formatspecs);
6565 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6566 if(isinstance(infile, dict)):
6567 listcatfiles = infile;
6568 else:
6569 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6570 infile = RemoveWindowsPath(infile);
6571 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, True, False, skipchecksum, formatspecs, returnfp);
6572 if(not listcatfiles):
6573 return False;
6574 lenlist = len(listcatfiles['ffilelist']);
6575 fnumfiles = int(listcatfiles['fnumfiles']);
6576 lcfi = 0;
6577 lcfx = int(listcatfiles['fnumfiles']);
6578 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6579 lcfx = int(lenlist);
6580 else:
6581 lcfx = int(listcatfiles['fnumfiles']);
6582 returnval = {};
6583 while(lcfi<lcfx):
6584 returnval.update({lcfi: listcatfiles['ffilelist'][lcfi]['fname']});
6585 if(not verbose):
6586 VerbosePrintOut(listcatfiles['ffilelist'][lcfi]['fname']);
6587 if(verbose):
6588 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6589 printfname = listcatfiles['ffilelist'][lcfi]['fname'];
6590 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6591 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " link to " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6592 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6593 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " -> " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6594 fuprint = listcatfiles['ffilelist'][lcfi]['funame'];
6595 if(len(fuprint)<=0):
6596 fuprint = listcatfiles['ffilelist'][lcfi]['fuid'];
6597 fgprint = listcatfiles['ffilelist'][lcfi]['fgname'];
6598 if(len(fgprint)<=0):
6599 fgprint = listcatfiles['ffilelist'][lcfi]['fgid'];
6600 VerbosePrintOut(PrintPermissionString(listcatfiles['ffilelist'][lcfi]['fmode'], listcatfiles['ffilelist'][lcfi]['ftype']) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(listcatfiles['ffilelist'][lcfi]['fsize']).rjust(15) + " " + datetime.datetime.utcfromtimestamp(listcatfiles['ffilelist'][lcfi]['fmtime']).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6601 lcfi = lcfi + 1;
6602 if(returnfp):
6603 return listcatfiles['catfp'];
6604 else:
6605 return True;
6606
6607 create_alias_function("", __file_format_name__, "ListFiles", ArchiveFileListFiles);
6608
6609 def ArchiveFileStringListFiles(catstr, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6610 formatspecs = FormatSpecsListToDict(formatspecs);
6611 catfp = BytesIO(catstr);
6612 listcatfiles = ArchiveFileListFiles(catstr, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6613 return listcatfiles;
6614
6615 create_alias_function("", __file_format_name__, "StringListFiles", ArchiveFileStringListFiles);
6616
6617 def TarFileListFiles(infile, verbose=False, returnfp=False):
6618 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6619 if(infile=="-"):
6620 infile = BytesIO();
6621 if(hasattr(sys.stdin, "buffer")):
6622 shutil.copyfileobj(sys.stdin.buffer, infile);
6623 else:
6624 shutil.copyfileobj(sys.stdin, infile);
6625 infile.seek(0, 0);
6626 if(not infile):
6627 return False;
6628 infile.seek(0, 0);
6629 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
6630 infile = download_file_from_internet_file(infile);
6631 infile.seek(0, 0);
6632 if(not infile):
6633 return False;
6634 infile.seek(0, 0);
6635 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
6636 return False;
6637 elif(os.path.exists(infile) and os.path.isfile(infile)):
6638 try:
6639 if(not tarfile.TarFileCheck(infile)):
6640 return False;
6641 except AttributeError:
6642 if(not TarFileCheck(infile)):
6643 return False;
6644 else:
6645 return False;
6646 try:
6647 if(hasattr(infile, "read") or hasattr(infile, "write")):
6648 tarfp = tarfile.open(fileobj=infile, mode="r");
6649 else:
6650 tarfp = tarfile.open(infile, "r");
6651 except FileNotFoundError:
6652 return False;
6653 lcfi = 0
6654 returnval = {};
6655 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
6656 returnval.update({lcfi: member.name});
6657 fpremode = member.mode;
6658 ffullmode = member.mode;
6659 flinkcount = 0;
6660 ftype = 0;
6661 if(member.isreg()):
6662 ffullmode = member.mode + stat.S_IFREG;
6663 ftype = 0;
6664 elif(member.isdev()):
6665 ffullmode = member.mode;
6666 ftype = 7;
6667 elif(member.islnk()):
6668 ffullmode = member.mode + stat.S_IFREG;
6669 ftype = 1;
6670 elif(member.issym()):
6671 ffullmode = member.mode + stat.S_IFLNK;
6672 ftype = 2;
6673 elif(member.ischr()):
6674 ffullmode = member.mode + stat.S_IFCHR;
6675 ftype = 3;
6676 elif(member.isblk()):
6677 ffullmode = member.mode + stat.S_IFBLK;
6678 ftype = 4;
6679 elif(member.isdir()):
6680 ffullmode = member.mode + stat.S_IFDIR;
6681 ftype = 5;
6682 elif(member.isfifo()):
6683 ffullmode = member.mode + stat.S_IFIFO;
6684 ftype = 6;
6685 elif(member.issparse()):
6686 ffullmode = member.mode;
6687 ftype = 12;
6688 if(not verbose):
6689 VerbosePrintOut(member.name);
6690 elif(verbose):
6691 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6692 printfname = member.name;
6693 if(member.islnk()):
6694 printfname = member.name + " link to " + member.linkname;
6695 elif(member.issym()):
6696 printfname = member.name + " -> " + member.linkname;
6697 fuprint = member.uname;
6698 if(len(fuprint)<=0):
6699 fuprint = member.uid;
6700 fgprint = member.gname;
6701 if(len(fgprint)<=0):
6702 fgprint = member.gid;
6703 VerbosePrintOut(PrintPermissionString(ffullmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(member.mtime).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6704 lcfi = lcfi + 1;
6705 if(returnfp):
6706 return listcatfiles['catfp'];
6707 else:
6708 return True;
6709
6710 def ZipFileListFiles(infile, verbose=False, returnfp=False):
6711 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6712 if(infile=="-"):
6713 infile = BytesIO();
6714 if(hasattr(sys.stdin, "buffer")):
6715 shutil.copyfileobj(sys.stdin.buffer, infile);
6716 else:
6717 shutil.copyfileobj(sys.stdin, infile);
6718 infile.seek(0, 0);
6719 if(not infile):
6720 return False;
6721 infile.seek(0, 0);
6722 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
6723 infile = download_file_from_internet_file(infile);
6724 infile.seek(0, 0);
6725 if(not infile):
6726 return False;
6727 infile.seek(0, 0);
6728 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
6729 return False;
6730 else:
6731 return False;
6732 if(not zipfile.is_zipfile(infile)):
6733 return False;
6734 try:
6735 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
6736 except FileNotFoundError:
6737 return False;
6738 lcfi = 0;
6739 returnval = {};
6740 ziptest = zipfp.testzip();
6741 if(ziptest):
6742 VerbosePrintOut("Bad file found!");
6743 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
6744 if(zipinfo.create_system==0 or zipinfo.create_system==10):
6745 fwinattributes = int(zipinfo.external_attr);
6746 if(not member.is_dir()):
6747 fmode = int(stat.S_IFREG + 438);
6748 fchmode = int(stat.S_IMODE(fmode));
6749 ftypemod = int(stat.S_IFMT(fmode));
6750 elif(member.is_dir()):
6751 fmode = int(stat.S_IFDIR + 511);
6752 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6753 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6754 elif(zipinfo.create_system==3):
6755 fwinattributes =int(0);
6756 fmode = int(zipinfo.external_attr);
6757 fchmode = int(stat.S_IMODE(fmode));
6758 ftypemod = int(stat.S_IFMT(fmode));
6759 else:
6760 fwinattributes = int(0);
6761 if(not member.is_dir()):
6762 fmode = int(stat.S_IFREG + 438);
6763 fchmode = int(stat.S_IMODE(fmode));
6764 ftypemod = int(stat.S_IFMT(fmode));
6765 elif(member.is_dir()):
6766 fmode = int(stat.S_IFDIR + 511);
6767 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6768 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6769 returnval.update({lcfi: member.filename});
6770 if(not verbose):
6771 VerbosePrintOut(member.filename);
6772 if(verbose):
6773 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6774 permissionstr = "";
6775 for fmodval in str(oct(fmode))[-3:]:
6776 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
6777 if(not member.is_dir()):
6778 ftype = 0;
6779 permissionstr = "-" + permissionstr;
6780 elif(member.is_dir()):
6781 ftype = 5;
6782 permissionstr = "d" + permissionstr;
6783 printfname = member.filename;
6784 try:
6785 fuid = int(os.getuid());
6786 except AttributeError:
6787 fuid = int(0);
6788 except KeyError:
6789 fuid = int(0);
6790 try:
6791 fgid = int(os.getgid());
6792 except AttributeError:
6793 fgid = int(0);
6794 except KeyError:
6795 fgid = int(0);
6796 try:
6797 import pwd;
6798 try:
6799 userinfo = pwd.getpwuid(os.getuid());
6800 funame = userinfo.pw_name;
6801 except KeyError:
6802 funame = "";
6803 except AttributeError:
6804 funame = "";
6805 except ImportError:
6806 funame = "";
6807 fgname = "";
6808 try:
6809 import grp;
6810 try:
6811 groupinfo = grp.getgrgid(os.getgid());
6812 fgname = groupinfo.gr_name;
6813 except KeyError:
6814 fgname = "";
6815 except AttributeError:
6816 fgname = "";
6817 except ImportError:
6818 fgname = "";
6819 fuprint = funame;
6820 if(len(fuprint)<=0):
6821 fuprint = str(fuid);
6822 fgprint = fgname;
6823 if(len(fgprint)<=0):
6824 fgprint = str(fgid);
6825 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(int(time.mktime(member.date_time + (0, 0, -1)))).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6826 lcfi = lcfi + 1;
6827 if(returnfp):
6828 return listcatfiles['catfp'];
6829 else:
6830 return True;
6831
6832 if(not rarfile_support):
6833 def RarFileListFiles(infile, verbose=False, returnfp=False):
6834 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6835 if(not os.path.exists(infile) or not os.path.isfile(infile)):
6836 return False;
6837
6838 if(rarfile_support):
6839 def RarFileListFiles(infile, verbose=False, returnfp=False):
6840 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6841 if(not os.path.exists(infile) or not os.path.isfile(infile)):
6842 return False;
6843 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
6844 return False;
6845 lcfi = 0;
6846 returnval = {};
6847 rarfp = rarfile.RarFile(infile, "r");
6848 rartest = rarfp.testrar();
6849 if(rartest):
6850 VerbosePrintOut("Bad file found!");
6851 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
6852 is_unix = False;
6853 is_windows = False;
6854 if(member.host_os==rarfile.RAR_OS_UNIX):
6855 is_windows = False;
6856 try:
6857 member.external_attr
6858 is_unix = True;
6859 except AttributeError:
6860 is_unix = False;
6861 elif(member.host_os==rarfile.RAR_OS_WIN32):
6862 is_unix = False;
6863 try:
6864 member.external_attr
6865 is_windows = True;
6866 except AttributeError:
6867 is_windows = False;
6868 else:
6869 is_unix = False;
6870 is_windows = False;
6871 if(is_unix and member.external_attr !=0):
6872 fpremode = int(member.external_attr);
6873 elif(member.is_file()):
6874 fpremode = int(stat.S_IFREG + 438);
6875 elif(member.is_symlink()):
6876 fpremode = int(stat.S_IFLNK + 438);
6877 elif(member.is_dir()):
6878 fpremode = int(stat.S_IFDIR + 511);
6879 if(is_windows and member.external_attr !=0):
6880 fwinattributes = int(member.external_attr);
6881 else:
6882 fwinattributes = int(0);
6883 if(is_unix and member.external_attr !=0):
6884 fmode = int(member.external_attr);
6885 fchmode = int(stat.S_IMODE(member.external_attr));
6886 ftypemod = int(stat.S_IFMT(member.external_attr));
6887 elif(member.is_file()):
6888 fmode = int(stat.S_IFREG + 438);
6889 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
6890 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
6891 elif(member.is_symlink()):
6892 fmode = int(stat.S_IFLNK + 438);
6893 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
6894 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
6895 elif(member.is_dir()):
6896 fmode = int(stat.S_IFDIR + 511);
6897 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6898 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6899 returnval.update({lcfi: member.filename});
6900 if(not verbose):
6901 VerbosePrintOut(member.filename);
6902 if(verbose):
6903 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6904 permissionstr = "";
6905 for fmodval in str(oct(fmode))[-3:]:
6906 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
6907 if(member.is_file()):
6908 ftype = 0;
6909 permissionstr = "-" + permissionstr;
6910 printfname = member.filename;
6911 elif(member.is_symlink()):
6912 ftype = 2;
6913 permissionstr = "l" + permissionstr;
6914 printfname = member.name + " -> " + member.read().decode("UTF-8");
6915 elif(member.is_dir()):
6916 ftype = 5;
6917 permissionstr = "d" + permissionstr;
6918 printfname = member.filename;
6919 try:
6920 fuid = int(os.getuid());
6921 except AttributeError:
6922 fuid = int(0);
6923 except KeyError:
6924 fuid = int(0);
6925 try:
6926 fgid = int(os.getgid());
6927 except AttributeError:
6928 fgid = int(0);
6929 except KeyError:
6930 fgid = int(0);
6931 try:
6932 import pwd;
6933 try:
6934 userinfo = pwd.getpwuid(os.getuid());
6935 funame = userinfo.pw_name;
6936 except KeyError:
6937 funame = "";
6938 except AttributeError:
6939 funame = "";
6940 except ImportError:
6941 funame = "";
6942 fgname = "";
6943 try:
6944 import grp;
6945 try:
6946 groupinfo = grp.getgrgid(os.getgid());
6947 fgname = groupinfo.gr_name;
6948 except KeyError:
6949 fgname = "";
6950 except AttributeError:
6951 fgname = "";
6952 except ImportError:
6953 fgname = "";
6954 fuprint = funame;
6955 if(len(fuprint)<=0):
6956 fuprint = str(fuid);
6957 fgprint = fgname;
6958 if(len(fgprint)<=0):
6959 fgprint = str(fgid);
6960 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + member.mtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
6961 lcfi = lcfi + 1;
6962 if(returnfp):
6963 return listcatfiles['catfp'];
6964 else:
6965 return True;
6966
6967 if(not py7zr_support):
6968 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
6969 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6970 if(not os.path.exists(infile) or not os.path.isfile(infile)):
6971 return False;
6972
6973 if(py7zr_support):
6974 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
6975 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6976 if(not os.path.exists(infile) or not os.path.isfile(infile)):
6977 return False;
6978 lcfi = 0;
6979 returnval = {};
6980 szpfp = py7zr.SevenZipFile(infile, mode="r");
6981 file_content = szpfp.readall();
6982 #sztest = szpfp.testzip();
6983 sztestalt = szpfp.test();
6984 if(sztestalt):
6985 VerbosePrintOut("Bad file found!");
6986 for member in sorted(szpfp.list(), key=lambda x: x.filename):
6987 if(re.findall(r"^[.|/]", member.filename)):
6988 fname = member.filename;
6989 else:
6990 fname = "./"+member.filename;
6991 if(not member.is_directory):
6992 fpremode = int(stat.S_IFREG + 438);
6993 elif(member.is_directory):
6994 fpremode = int(stat.S_IFDIR + 511);
6995 fwinattributes = int(0);
6996 if(member.is_directory):
6997 fmode = int(stat.S_IFDIR + 511);
6998 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6999 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7000 else:
7001 fmode = int(stat.S_IFLNK + 438);
7002 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
7003 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
7004 returnval.update({lcfi: member.filename});
7005 if(not verbose):
7006 VerbosePrintOut(member.filename);
7007 if(verbose):
7008 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7009 permissionstr = "";
7010 for fmodval in str(oct(fmode))[-3:]:
7011 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
7012 fsize = int("0");
7013 if(not member.is_directory):
7014 ftype = 0;
7015 permissionstr = "-" + permissionstr;
7016 printfname = member.filename;
7017 elif(member.is_directory):
7018 ftype = 5;
7019 permissionstr = "d" + permissionstr;
7020 printfname = member.filename;
7021 if(ftype==0):
7022 fsize = len(file_content[member.filename].read());
7023 file_content[member.filename].close();
7024 try:
7025 fuid = int(os.getuid());
7026 except AttributeError:
7027 fuid = int(0);
7028 except KeyError:
7029 fuid = int(0);
7030 try:
7031 fgid = int(os.getgid());
7032 except AttributeError:
7033 fgid = int(0);
7034 except KeyError:
7035 fgid = int(0);
7036 try:
7037 import pwd;
7038 try:
7039 userinfo = pwd.getpwuid(os.getuid());
7040 funame = userinfo.pw_name;
7041 except KeyError:
7042 funame = "";
7043 except AttributeError:
7044 funame = "";
7045 except ImportError:
7046 funame = "";
7047 fgname = "";
7048 try:
7049 import grp;
7050 try:
7051 groupinfo = grp.getgrgid(os.getgid());
7052 fgname = groupinfo.gr_name;
7053 except KeyError:
7054 fgname = "";
7055 except AttributeError:
7056 fgname = "";
7057 except ImportError:
7058 fgname = "";
7059 fuprint = funame;
7060 if(len(fuprint)<=0):
7061 fuprint = str(fuid);
7062 fgprint = fgname;
7063 if(len(fgprint)<=0):
7064 fgprint = str(fgid);
7065 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(fsize).rjust(15) + " " + member.creationtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
7066 lcfi = lcfi + 1;
7067 if(returnfp):
7068 return listcatfiles['catfp'];
7069 else:
7070 return True;
7071
7072 def InFileListFiles(infile, verbose=False, formatspecs=__file_format_dict__, returnfp=False):
7073 formatspecs = FormatSpecsListToDict(formatspecs);
7074 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7075 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
7076 if(checkcompressfile=="tarfile" and TarFileCheck(infile)):
7077 return TarFileListFiles(infile, verbose, returnfp);
7078 elif(checkcompressfile=="zipfile" and zipfile.is_zipfile(infile)):
7079 return ZipFileListFiles(infile, verbose, returnfp);
7080 elif(rarfile_support and checkcompressfile=="rarfile" and (rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile))):
7081 return RarFileListFiles(infile, verbose, returnfp);
7082 elif(py7zr_support and checkcompressfile=="7zipfile" and py7zr.is_7zfile(infile)):
7083 return SevenZipFileListFiles(infile, verbose, returnfp);
7084 elif(checkcompressfile=="catfile"):
7085 return ArchiveFileListFiles(infile, 0, 0, False, formatspecs, verbose, returnfp);
7086 else:
7087 return False;
7088 return False;
7089
7090 def ListDirListFiles(infiles, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7091 formatspecs = FormatSpecsListToDict(formatspecs);
7092 outarray = BytesIO();
7093 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, False, True);
7094 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
7095 return listcatfiles;
7096
7097 def ListDirListFilesAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7098 formatspecs = FormatSpecsListToDict(formatspecs);
7099 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, listonly, checksumtype, formatspecs, verbose);
7100 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
7101 return listcatfiles;
7102
7103 def PackArchiveFileFromListDirAlt(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7104 formatspecs = FormatSpecsListToDict(formatspecs);
7105 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, False, checksumtype, extradata, formatspecs, False);
7106 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, followlink, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
7107 return listcatfiles;
7108
7109 create_alias_function("Pack", __file_format_name__, "FromListDirAlt", PackArchiveFileFromListDirAlt);
7110
7111 def PackArchiveFileFromTarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7112 formatspecs = FormatSpecsListToDict(formatspecs);
7113 outarray = TarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7114 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7115 return listcatfiles;
7116
7117 create_alias_function("Pack", __file_format_name__, "FromTarFileAlt", PackArchiveFileFromTarFileAlt);
7118
7119 def PackArchiveFileFromZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7120 formatspecs = FormatSpecsListToDict(formatspecs);
7121 outarray = ZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7122 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7123 return listcatfiles;
7124
7125 create_alias_function("Pack", __file_format_name__, "FromZipFileAlt", PackArchiveFileFromZipFileAlt);
7126
7127 if(not rarfile_support):
7128 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7129 return False;
7130
7131 if(rarfile_support):
7132 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7133 formatspecs = FormatSpecsListToDict(formatspecs);
7134 outarray = RarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7135 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7136 return listcatfiles;
7137
7138 create_alias_function("Pack", __file_format_name__, "FromRarFileAlt", PackArchiveFileFromRarFileAlt);
7139
7140 if(not py7zr_support):
7141 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7142 return False;
7143
7144 if(py7zr_support):
7145 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7146 formatspecs = FormatSpecsListToDict(formatspecs);
7147 outarray = SevenZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7148 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7149 return listcatfiles;
7150
7151 create_alias_function("Pack", __file_format_name__, "FromSevenZipFileAlt", PackArchiveFileFromSevenZipFileAlt);
7152
7153 def download_file_from_ftp_file(url):
7154 urlparts = urlparse(url);
7155 file_name = os.path.basename(urlparts.path);
7156 file_dir = os.path.dirname(urlparts.path);
7157 if(urlparts.username is not None):
7158 ftp_username = urlparts.username;
7159 else:
7160 ftp_username = "anonymous";
7161 if(urlparts.password is not None):
7162 ftp_password = urlparts.password;
7163 elif(urlparts.password is None and urlparts.username=="anonymous"):
7164 ftp_password = "anonymous";
7165 else:
7166 ftp_password = "";
7167 if(urlparts.scheme=="ftp"):
7168 ftp = FTP();
7169 elif(urlparts.scheme=="ftps" and ftpssl):
7170 ftp = FTP_TLS();
7171 else:
7172 return False;
7173 if(urlparts.scheme=="sftp"):
7174 if(__use_pysftp__):
7175 return download_file_from_pysftp_file(url);
7176 else:
7177 return download_file_from_sftp_file(url);
7178 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7179 return download_file_from_http_file(url);
7180 ftp_port = urlparts.port;
7181 if(urlparts.port is None):
7182 ftp_port = 21;
7183 try:
7184 ftp.connect(urlparts.hostname, ftp_port);
7185 except socket.gaierror:
7186 log.info("Error With URL "+url);
7187 return False;
7188 except socket.timeout:
7189 log.info("Error With URL "+url);
7190 return False;
7191 ftp.login(urlparts.username, urlparts.password);
7192 if(urlparts.scheme=="ftps"):
7193 ftp.prot_p();
7194 ftpfile = BytesIO();
7195 ftp.retrbinary("RETR "+urlparts.path, ftpfile.write);
7196 #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
7197 ftp.close();
7198 ftpfile.seek(0, 0);
7199 return ftpfile;
7200
7201 def download_file_from_ftp_string(url):
7202 ftpfile = download_file_from_ftp_file(url);
7203 return ftpfile.read();
7204
7205 def upload_file_to_ftp_file(ftpfile, url):
7206 urlparts = urlparse(url);
7207 file_name = os.path.basename(urlparts.path);
7208 file_dir = os.path.dirname(urlparts.path);
7209 if(urlparts.username is not None):
7210 ftp_username = urlparts.username;
7211 else:
7212 ftp_username = "anonymous";
7213 if(urlparts.password is not None):
7214 ftp_password = urlparts.password;
7215 elif(urlparts.password is None and urlparts.username=="anonymous"):
7216 ftp_password = "anonymous";
7217 else:
7218 ftp_password = "";
7219 if(urlparts.scheme=="ftp"):
7220 ftp = FTP();
7221 elif(urlparts.scheme=="ftps" and ftpssl):
7222 ftp = FTP_TLS();
7223 else:
7224 return False;
7225 if(urlparts.scheme=="sftp"):
7226 if(__use_pysftp__):
7227 return upload_file_to_pysftp_file(url);
7228 else:
7229 return upload_file_to_sftp_file(url);
7230 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7231 return False;
7232 ftp_port = urlparts.port;
7233 if(urlparts.port is None):
7234 ftp_port = 21;
7235 try:
7236 ftp.connect(urlparts.hostname, ftp_port);
7237 except socket.gaierror:
7238 log.info("Error With URL "+url);
7239 return False;
7240 except socket.timeout:
7241 log.info("Error With URL "+url);
7242 return False;
7243 ftp.login(urlparts.username, urlparts.password);
7244 if(urlparts.scheme=="ftps"):
7245 ftp.prot_p();
7246 ftp.storbinary("STOR "+urlparts.path, ftpfile);
7247 ftp.close();
7248 ftpfile.seek(0, 0);
7249 return ftpfile;
7250
7251 def upload_file_to_ftp_string(ftpstring, url):
7252 ftpfileo = BytesIO(ftpstring);
7253 ftpfile = upload_file_to_ftp_file(ftpfileo, url);
7254 ftpfileo.close();
7255 return ftpfile;
7256
7257 def download_file_from_http_file(url, headers=geturls_headers_pycatfile_python_alt):
7258 # Parse the URL to extract username and password if present
7259 urlparts = urlparse(url);
7260 username = urlparts.username;
7261 password = urlparts.password;
7262 # Rebuild the URL without the username and password
7263 netloc = urlparts.hostname;
7264 if(urlparts.scheme=="sftp"):
7265 if(__use_pysftp__):
7266 return download_file_from_pysftp_file(url);
7267 else:
7268 return download_file_from_sftp_file(url);
7269 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7270 return download_file_from_ftp_file(url);
7271 if urlparts.port:
7272 netloc += ':' + str(urlparts.port);
7273 rebuilt_url = urlunparse((urlparts.scheme, netloc, urlparts.path, urlparts.params, urlparts.query, urlparts.fragment));
7274 # Create a temporary file object
7275 httpfile = BytesIO();
7276 if haverequests:
7277 # Use the requests library if available
7278 if username and password:
7279 response = requests.get(rebuilt_url, headers=headers, auth=(username, password), stream=True);
7280 else:
7281 response = requests.get(rebuilt_url, headers=headers, stream=True);
7282 response.raw.decode_content = True
7283 shutil.copyfileobj(response.raw, httpfile);
7284 else:
7285 # Build a Request object for urllib
7286 request = Request(rebuilt_url, headers=headers);
7287 # Create an opener object for handling URLs
7288 if username and password:
7289 # Create a password manager
7290 password_mgr = HTTPPasswordMgrWithDefaultRealm();
7291 # Add the username and password
7292 password_mgr.add_password(None, rebuilt_url, username, password);
7293 # Create an authentication handler using the password manager
7294 auth_handler = HTTPBasicAuthHandler(password_mgr);
7295 # Build the opener with the authentication handler
7296 opener = build_opener(auth_handler);
7297 else:
7298 opener = build_opener();
7299 with opener.open(request) as response:
7300 shutil.copyfileobj(response, httpfile);
7301 # Reset file pointer to the start
7302 httpfile.seek(0, 0);
7303 # Return the temporary file object
7304 return httpfile;
7305
7306 def download_file_from_http_string(url, headers=geturls_headers_pycatfile_python_alt):
7307 httpfile = download_file_from_http_file(url, headers);
7308 return ftpfile.read();
7309
7310 if(haveparamiko):
7311 def download_file_from_sftp_file(url):
7312 urlparts = urlparse(url);
7313 file_name = os.path.basename(urlparts.path);
7314 file_dir = os.path.dirname(urlparts.path);
7315 sftp_port = urlparts.port;
7316 if(urlparts.port is None):
7317 sftp_port = 22;
7318 else:
7319 sftp_port = urlparts.port;
7320 if(urlparts.username is not None):
7321 sftp_username = urlparts.username;
7322 else:
7323 sftp_username = "anonymous";
7324 if(urlparts.password is not None):
7325 sftp_password = urlparts.password;
7326 elif(urlparts.password is None and urlparts.username=="anonymous"):
7327 sftp_password = "anonymous";
7328 else:
7329 sftp_password = "";
7330 if(urlparts.scheme=="ftp"):
7331 return download_file_from_ftp_file(url);
7332 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7333 return download_file_from_http_file(url);
7334 if(urlparts.scheme!="sftp"):
7335 return False;
7336 ssh = paramiko.SSHClient();
7337 ssh.load_system_host_keys();
7338 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
7339 try:
7340 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7341 except paramiko.ssh_exception.SSHException:
7342 return False;
7343 except socket.gaierror:
7344 log.info("Error With URL "+url);
7345 return False;
7346 except socket.timeout:
7347 log.info("Error With URL "+url);
7348 return False;
7349 sftp = ssh.open_sftp();
7350 sftpfile = BytesIO();
7351 sftp.getfo(urlparts.path, sftpfile);
7352 sftp.close();
7353 ssh.close();
7354 sftpfile.seek(0, 0);
7355 return sftpfile;
7356 else:
7357 def download_file_from_sftp_file(url):
7358 return False;
7359
7360 if(haveparamiko):
7361 def download_file_from_sftp_string(url):
7362 sftpfile = download_file_from_sftp_file(url);
7363 return sftpfile.read();
7364 else:
7365 def download_file_from_ftp_string(url):
7366 return False;
7367
7368 if(haveparamiko):
7369 def upload_file_to_sftp_file(sftpfile, url):
7370 urlparts = urlparse(url);
7371 file_name = os.path.basename(urlparts.path);
7372 file_dir = os.path.dirname(urlparts.path);
7373 sftp_port = urlparts.port;
7374 if(urlparts.port is None):
7375 sftp_port = 22;
7376 else:
7377 sftp_port = urlparts.port;
7378 if(urlparts.username is not None):
7379 sftp_username = urlparts.username;
7380 else:
7381 sftp_username = "anonymous";
7382 if(urlparts.password is not None):
7383 sftp_password = urlparts.password;
7384 elif(urlparts.password is None and urlparts.username=="anonymous"):
7385 sftp_password = "anonymous";
7386 else:
7387 sftp_password = "";
7388 if(urlparts.scheme=="ftp"):
7389 return upload_file_to_ftp_file(url);
7390 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7391 return False;
7392 if(urlparts.scheme!="sftp"):
7393 return False;
7394 ssh = paramiko.SSHClient();
7395 ssh.load_system_host_keys();
7396 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
7397 try:
7398 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7399 except paramiko.ssh_exception.SSHException:
7400 return False;
7401 except socket.gaierror:
7402 log.info("Error With URL "+url);
7403 return False;
7404 except socket.timeout:
7405 log.info("Error With URL "+url);
7406 return False;
7407 sftp = ssh.open_sftp();
7408 sftp.putfo(sftpfile, urlparts.path);
7409 sftp.close();
7410 ssh.close();
7411 sftpfile.seek(0, 0);
7412 return sftpfile;
7413 else:
7414 def upload_file_to_sftp_file(sftpfile, url):
7415 return False;
7416
7417 if(haveparamiko):
7418 def upload_file_to_sftp_string(sftpstring, url):
7419 sftpfileo = BytesIO(sftpstring);
7420 sftpfile = upload_file_to_sftp_files(ftpfileo, url);
7421 sftpfileo.close();
7422 return sftpfile;
7423 else:
7424 def upload_file_to_sftp_string(url):
7425 return False;
7426
7427 if(havepysftp):
7428 def download_file_from_pysftp_file(url):
7429 urlparts = urlparse(url);
7430 file_name = os.path.basename(urlparts.path);
7431 file_dir = os.path.dirname(urlparts.path);
7432 sftp_port = urlparts.port;
7433 if(urlparts.port is None):
7434 sftp_port = 22;
7435 else:
7436 sftp_port = urlparts.port;
7437 if(urlparts.username is not None):
7438 sftp_username = urlparts.username;
7439 else:
7440 sftp_username = "anonymous";
7441 if(urlparts.password is not None):
7442 sftp_password = urlparts.password;
7443 elif(urlparts.password is None and urlparts.username=="anonymous"):
7444 sftp_password = "anonymous";
7445 else:
7446 sftp_password = "";
7447 if(urlparts.scheme=="ftp"):
7448 return download_file_from_ftp_file(url);
7449 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7450 return download_file_from_http_file(url);
7451 if(urlparts.scheme!="sftp"):
7452 return False;
7453 try:
7454 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7455 except paramiko.ssh_exception.SSHException:
7456 return False;
7457 except socket.gaierror:
7458 log.info("Error With URL "+url);
7459 return False;
7460 except socket.timeout:
7461 log.info("Error With URL "+url);
7462 return False;
7463 sftp = ssh.open_sftp();
7464 sftpfile = BytesIO();
7465 sftp.getfo(urlparts.path, sftpfile);
7466 sftp.close();
7467 ssh.close();
7468 sftpfile.seek(0, 0);
7469 return sftpfile;
7470 else:
7471 def download_file_from_pysftp_file(url):
7472 return False;
7473
7474 if(havepysftp):
7475 def download_file_from_pysftp_string(url):
7476 sftpfile = download_file_from_pysftp_file(url);
7477 return sftpfile.read();
7478 else:
7479 def download_file_from_ftp_string(url):
7480 return False;
7481
7482 if(havepysftp):
7483 def upload_file_to_pysftp_file(sftpfile, url):
7484 urlparts = urlparse(url);
7485 file_name = os.path.basename(urlparts.path);
7486 file_dir = os.path.dirname(urlparts.path);
7487 sftp_port = urlparts.port;
7488 if(urlparts.port is None):
7489 sftp_port = 22;
7490 else:
7491 sftp_port = urlparts.port;
7492 if(urlparts.username is not None):
7493 sftp_username = urlparts.username;
7494 else:
7495 sftp_username = "anonymous";
7496 if(urlparts.password is not None):
7497 sftp_password = urlparts.password;
7498 elif(urlparts.password is None and urlparts.username=="anonymous"):
7499 sftp_password = "anonymous";
7500 else:
7501 sftp_password = "";
7502 if(urlparts.scheme=="ftp"):
7503 return upload_file_to_ftp_file(url);
7504 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7505 return False;
7506 if(urlparts.scheme!="sftp"):
7507 return False;
7508 try:
7509 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7510 except paramiko.ssh_exception.SSHException:
7511 return False;
7512 except socket.gaierror:
7513 log.info("Error With URL "+url);
7514 return False;
7515 except socket.timeout:
7516 log.info("Error With URL "+url);
7517 return False;
7518 sftp = ssh.open_sftp();
7519 sftp.putfo(sftpfile, urlparts.path);
7520 sftp.close();
7521 ssh.close();
7522 sftpfile.seek(0, 0);
7523 return sftpfile;
7524 else:
7525 def upload_file_to_pysftp_file(sftpfile, url):
7526 return False;
7527
7528 if(havepysftp):
7529 def upload_file_to_pysftp_string(sftpstring, url):
7530 sftpfileo = BytesIO(sftpstring);
7531 sftpfile = upload_file_to_pysftp_files(ftpfileo, url);
7532 sftpfileo.close();
7533 return sftpfile;
7534 else:
7535 def upload_file_to_pysftp_string(url):
7536 return False;
7537
7538 def download_file_from_internet_file(url, headers=geturls_headers_pycatfile_python_alt):
7539 urlparts = urlparse(url);
7540 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7541 return download_file_from_http_file(url, headers);
7542 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7543 return download_file_from_ftp_file(url);
7544 elif(urlparts.scheme=="sftp"):
7545 if(__use_pysftp__ and havepysftp):
7546 return download_file_from_pysftp_file(url);
7547 else:
7548 return download_file_from_sftp_file(url);
7549 else:
7550 return False;
7551 return False;
7552
7553 def download_file_from_internet_uncompress_file(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_dict__):
7554 formatspecs = FormatSpecsListToDict(formatspecs);
7555 fp = download_file_from_internet_file(url);
7556 fp = UncompressArchiveFile(fp, formatspecs);
7557 fp.seek(0, 0);
7558 if(not fp):
7559 return False;
7560 return fp;
7561
7562 def download_file_from_internet_string(url, headers=geturls_headers_pycatfile_python_alt):
7563 urlparts = urlparse(url);
7564 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7565 return download_file_from_http_string(url, headers);
7566 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7567 return download_file_from_ftp_string(url);
7568 elif(urlparts.scheme=="sftp"):
7569 if(__use_pysftp__ and havepysftp):
7570 return download_file_from_pysftp_string(url);
7571 else:
7572 return download_file_from_sftp_string(url);
7573 else:
7574 return False;
7575 return False;
7576
7577 def download_file_from_internet_uncompress_string(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_dict__):
7578 formatspecs = FormatSpecsListToDict(formatspecs);
7579 fp = download_file_from_internet_string(url);
7580 fp = UncompressArchiveFile(fp, formatspecs);
7581 fp.seek(0, 0);
7582 if(not fp):
7583 return False;
7584 return fp;
7585
7586 def upload_file_to_internet_file(ifp, url):
7587 urlparts = urlparse(url);
7588 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7589 return False;
7590 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7591 return upload_file_to_ftp_file(ifp, url);
7592 elif(urlparts.scheme=="sftp"):
7593 if(__use_pysftp__ and havepysftp):
7594 return upload_file_to_pysftp_file(ifp, url);
7595 else:
7596 return upload_file_to_sftp_file(ifp, url);
7597 else:
7598 return False;
7599 return False;
7600
7601 def upload_file_to_internet_compress_file(ifp, url, formatspecs=__file_format_dict__):
7602 formatspecs = FormatSpecsListToDict(formatspecs);
7603 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
7604 if(not catfileout):
7605 return False;
7606 catfp.seek(0, 0);
7607 upload_file_to_internet_file(catfp, outfile);
7608 return True;
7609
7610 def upload_file_to_internet_string(ifp, url):
7611 urlparts = urlparse(url);
7612 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7613 return False;
7614 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7615 return upload_file_to_ftp_string(ifp, url);
7616 elif(urlparts.scheme=="sftp"):
7617 if(__use_pysftp__ and havepysftp):
7618 return upload_file_to_pysftp_string(ifp, url);
7619 else:
7620 return upload_file_to_sftp_string(ifp, url);
7621 else:
7622 return False;
7623 return False;
7624
7625 def upload_file_to_internet_compress_string(ifp, url, formatspecs=__file_format_dict__):
7626 formatspecs = FormatSpecsListToDict(formatspecs);
7627 catfp = CompressArchiveFile(BytesIO(ifp), compression, compressionlevel, formatspecs);
7628 if(not catfileout):
7629 return False;
7630 catfp.seek(0, 0);
7631 upload_file_to_internet_file(catfp, outfile);
7632 return True;
7633
7634 try:
7635 if(hasattr(shutil, "register_archive_format")):
7636 # Register the packing format
7637 shutil.register_archive_format(__file_format_name__, PackArchiveFileFunc, description='Pack concatenated files');
7638 except shutil.RegistryError:
7639 pass;
7640
7641 try:
7642 if(hasattr(shutil, "register_unpack_format")):
7643 # Register the unpacking format
7644 shutil.register_unpack_format(__file_format_name__, archivefile_extensions, UnPackArchiveFileFunc, description='UnPack concatenated files');
7645 except shutil.RegistryError:
7646 pass;
A tar like file format name catfile after unix cat command (concatenate files).