Add files via upload
[PyCatFile.git] / pycatfile.py
blob080058078cfdbd1685145b24a06ad40a41a005d7
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
4 '''
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the Revised BSD License.
8 This program is distributed in the hope that it will be useful,
9 but WITHOUT ANY WARRANTY; without even the implied warranty of
10 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 Revised BSD License for more details.
13 Copyright 2018-2024 Cool Dude 2k - http://idb.berlios.de/
14 Copyright 2018-2024 Game Maker 2k - http://intdb.sourceforge.net/
15 Copyright 2018-2024 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
17 $FileInfo: pycatfile.py - Last Update: 5/11/2024 Ver. 0.10.8 RC 1 - Author: cooldude2k $
18 '''
20 from __future__ import absolute_import, division, print_function, unicode_literals;
21 import io, os, re, sys, time, stat, zlib, base64, shutil, socket, hashlib, datetime, logging, binascii, tempfile, zipfile, platform;
22 from ftplib import FTP, FTP_TLS;
23 if(sys.version[0]=="2"):
24 from urlparse import urlparse, urlunparse;
25 elif(sys.version[0]>="3"):
26 from urllib.parse import urlunparse;
27 from urllib.parse import urlparse;
29 if os.name == 'nt': # Only modify if on Windows
30 if sys.version[0] == "2":
31 import codecs;
32 sys.stdout = codecs.getwriter('utf-8')(sys.stdout);
33 sys.stderr = codecs.getwriter('utf-8')(sys.stderr);
34 else:
35 import io;
36 sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace', line_buffering=True);
37 sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', errors='replace', line_buffering=True);
39 hashlib_guaranteed = False;
40 os.environ["PYTHONIOENCODING"] = "UTF-8";
41 os.environ["LC_CTYPE"] = "UTF-8";
42 try:
43 reload(sys);
44 try:
45 sys.setdefaultencoding('UTF-8');
46 except NameError:
47 pass;
48 except AttributeError:
49 pass;
50 except NameError:
51 pass;
52 except AttributeError:
53 pass;
55 try:
56 from zlib import crc32;
57 except ImportError:
58 from binascii import crc32;
60 if(sys.version_info[0]==2):
61 FileNotFoundError = IOError;
63 rarfile_support = False;
64 try:
65 import rarfile;
66 rarfile_support = True;
67 except ImportError:
68 rarfile_support = False;
70 py7zr_support = False;
71 try:
72 import py7zr;
73 py7zr_support = True;
74 except ImportError:
75 py7zr_support = False;
77 try:
78 from xtarfile import is_tarfile;
79 except ImportError:
80 try:
81 from safetar import is_tarfile;
82 except ImportError:
83 from tarfile import is_tarfile;
85 try:
86 import xtarfile as tarfile;
87 except ImportError:
88 try:
89 import safetar as tarfile;
90 except ImportError:
91 import tarfile;
93 haveparamiko = False;
94 try:
95 import paramiko;
96 haveparamiko = True;
97 except ImportError:
98 haveparamiko = False;
100 havepysftp = False;
101 try:
102 import pysftp;
103 havepysftp = True;
104 except ImportError:
105 havepysftp = False;
107 haverequests = False;
108 try:
109 import requests;
110 haverequests = True;
111 except ImportError:
112 haverequests = False;
114 if(haverequests):
115 import urllib3;
116 logging.getLogger("urllib3").setLevel(logging.WARNING);
118 try:
119 # Python 3 imports
120 from urllib.request import Request, build_opener, HTTPBasicAuthHandler;
121 from urllib.parse import urlparse;
122 except ImportError:
123 # Python 2 imports
124 from urllib2 import Request, build_opener, HTTPBasicAuthHandler;
125 from urlparse import urlparse;
127 if(sys.version[0]=="2"):
128 try:
129 from io import StringIO, BytesIO;
130 except ImportError:
131 try:
132 from cStringIO import StringIO;
133 from cStringIO import StringIO as BytesIO;
134 except ImportError:
135 from StringIO import StringIO;
136 from StringIO import StringIO as BytesIO;
137 elif(sys.version[0]>="3"):
138 from io import StringIO, BytesIO;
139 else:
140 teststringio = 0;
141 if(teststringio<=0):
142 try:
143 from cStringIO import StringIO as BytesIO;
144 teststringio = 1;
145 except ImportError:
146 teststringio = 0;
147 if(teststringio<=0):
148 try:
149 from StringIO import StringIO as BytesIO;
150 teststringio = 2;
151 except ImportError:
152 teststringio = 0;
153 if(teststringio<=0):
154 try:
155 from io import BytesIO;
156 teststringio = 3;
157 except ImportError:
158 teststringio = 0;
160 __use_pysftp__ = False;
161 if(not havepysftp):
162 __use_pysftp__ = False;
163 __file_format_name__ = "CatFile";
164 __program_name__ = "Py"+__file_format_name__;
165 __file_format_lower__ = __file_format_name__.lower();
166 __file_format_magic__ = __file_format_name__;
167 __file_format_len__ = len(__file_format_magic__);
168 __file_format_hex__ = binascii.hexlify(__file_format_magic__.encode("UTF-8")).decode("UTF-8");
169 __file_format_delimiter__ = "\x00";
170 __file_format_ver__ = "001";
171 __use_new_style__ = True;
172 __use_advanced_list__ = True;
173 __use_alt_inode__ = False;
174 __file_format_list__ = [__file_format_name__, __file_format_magic__, __file_format_lower__, __file_format_len__, __file_format_hex__, __file_format_delimiter__, __file_format_ver__, __use_new_style__, __use_advanced_list__, __use_alt_inode__];
175 __file_format_dict__ = {'format_name': __file_format_name__, 'format_magic': __file_format_magic__, 'format_lower': __file_format_lower__, 'format_len': __file_format_len__, 'format_hex': __file_format_hex__, 'format_delimiter': __file_format_delimiter__, 'format_ver': __file_format_ver__, 'new_style': __use_new_style__, 'use_advanced_list': __use_advanced_list__, 'use_alt_inode': __use_alt_inode__};
176 __project__ = __program_name__;
177 __project_url__ = "https://github.com/GameMaker2k/PyCatFile";
178 __version_info__ = (0, 10, 8, "RC 1", 1);
179 __version_date_info__ = (2024, 5, 11, "RC 1", 1);
180 __version_date__ = str(__version_date_info__[0]) + "." + str(__version_date_info__[1]).zfill(2) + "." + str(__version_date_info__[2]).zfill(2);
181 __revision__ = __version_info__[3];
182 __revision_id__ = "$Id$";
183 if(__version_info__[4] is not None):
184 __version_date_plusrc__ = __version_date__ + "-" + str(__version_date_info__[4]);
185 if(__version_info__[4] is None):
186 __version_date_plusrc__ = __version_date__;
187 if(__version_info__[3] is not None):
188 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]) + " " + str(__version_info__[3]);
189 if(__version_info__[3] is None):
190 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]);
192 PyBitness = platform.architecture();
193 if(PyBitness=="32bit" or PyBitness=="32"):
194 PyBitness = "32";
195 elif(PyBitness=="64bit" or PyBitness=="64"):
196 PyBitness = "64";
197 else:
198 PyBitness = "32";
200 geturls_ua_pycatfile_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
201 if(platform.python_implementation()!=""):
202 py_implementation = platform.python_implementation();
203 if(platform.python_implementation()==""):
204 py_implementation = "Python";
205 geturls_ua_pycatfile_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=py_implementation, pyver=platform.python_version(), proname=__project__, prover=__version__);
206 geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
207 geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
208 geturls_headers_pycatfile_python = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
209 geturls_headers_pycatfile_python_alt = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python_alt, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
210 geturls_headers_googlebot_google = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
211 geturls_headers_googlebot_google_old = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
213 compressionlist = ['auto', 'gzip', 'bzip2', 'zstd', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
214 compressionlistalt = ['gzip', 'bzip2', 'zstd', 'lz4', 'lzo', 'lzma', 'xz'];
215 outextlist = ['gz', 'bz2', 'zst', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
216 outextlistwd = ['.gz', '.bz2', '.zst', '.lz4', '.lzo', '.lzop', '.lzma', '.xz'];
218 tarfile_mimetype = "application/tar";
219 tarfile_tar_mimetype = tarfile_mimetype;
220 zipfile_mimetype = "application/zip";
221 zipfile_zip_mimetype = zipfile_mimetype;
222 rarfile_mimetype = "application/rar";
223 rarfile_rar_mimetype = rarfile_mimetype;
224 archivefile_mimetype = "application/x-"+__file_format_dict__['format_lower']+"";
225 archivefile_cat_mimetype = archivefile_mimetype;
226 archivefile_gzip_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+gzip";
227 archivefile_gz_mimetype = archivefile_gzip_mimetype;
228 archivefile_bzip2_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+bzip2";
229 archivefile_bz2_mimetype = archivefile_bzip2_mimetype;
230 archivefile_lz4_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lz4";
231 archivefile_lzop_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lzop";
232 archivefile_lzo_mimetype = archivefile_lzop_mimetype;
233 archivefile_zstandard_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+zstandard";
234 archivefile_zstd_mimetype = archivefile_zstandard_mimetype;
235 archivefile_lzma_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+lzma";
236 archivefile_xz_mimetype = "application/x-"+__file_format_dict__['format_lower']+"+xz";
237 archivefile_extensions = ['.cat', '.cat.gz', '.cat.bz2', '.cat.zst', '.cat.lz4', '.cat.lzo', '.cat.lzop', '.cat.lzma', '.cat.xz'];
239 if __name__ == "__main__":
240 import subprocess;
241 curscrpath = os.path.dirname(sys.argv[0]);
242 if(curscrpath==""):
243 curscrpath = ".";
244 if(os.sep=="\\"):
245 curscrpath = curscrpath.replace(os.sep, "/");
246 curscrpath = curscrpath + "/";
247 scrfile = curscrpath + "catfile.py";
248 if(os.path.exists(scrfile) and os.path.isfile(scrfile)):
249 scrcmd = subprocess.Popen([sys.executable, scrfile] + sys.argv[1:]);
250 scrcmd.wait();
252 def VerbosePrintOut(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
253 if(not dbgenable):
254 return True;
255 log_functions = {
256 "print": print,
257 "log": logging.info,
258 "warning": logging.warning,
259 "error": logging.error,
260 "critical": logging.critical,
261 "exception": logging.exception,
262 "logalt": lambda x: logging.log(dgblevel, x),
263 "debug": logging.debug
265 log_function = log_functions.get(outtype);
266 if(log_function):
267 log_function(dbgtxt);
268 return True;
269 return False;
271 def VerbosePrintOutReturn(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
272 VerbosePrintOut(dbgtxt, outtype, dbgenable, dgblevel);
273 return dbgtxt;
275 def RemoveWindowsPath(dpath):
276 if(dpath is None):
277 dpath = "";
278 if(os.sep!="/"):
279 dpath = dpath.replace(os.path.sep, "/");
280 dpath = dpath.rstrip("/");
281 if(dpath=="." or dpath==".."):
282 dpath = dpath + "/";
283 return dpath;
285 def NormalizeRelativePath(inpath):
286 inpath = RemoveWindowsPath(inpath);
287 if(os.path.isabs(inpath)):
288 outpath = inpath;
289 else:
290 if(inpath.startswith("./") or inpath.startswith("../")):
291 outpath = inpath;
292 else:
293 outpath = "./" + inpath;
294 return outpath;
296 def PrependPath(base_dir, child_path):
297 # Check if base_dir is None or empty, if so, return child_path as is
298 if not base_dir:
299 return child_path;
300 # Ensure base_dir ends with exactly one slash
301 if not base_dir.endswith('/'):
302 base_dir += '/';
303 # Check if child_path starts with ./ or ../ (indicating a relative path)
304 if child_path.startswith('./') or child_path.startswith('../'):
305 # For relative paths, we don't alter the child_path
306 return base_dir + child_path;
307 else:
308 # For non-relative paths, ensure there's no starting slash on child_path to avoid double slashes
309 return base_dir + child_path.lstrip('/');
311 def ListDir(dirpath, followlink=False, duplicates=False):
312 if(isinstance(dirpath, (list, tuple, ))):
313 dirpath = list(filter(None, dirpath));
314 elif(isinstance(dirpath, (str, ))):
315 dirpath = list(filter(None, [dirpath]));
316 retlist = [];
317 for mydirfile in dirpath:
318 if(not os.path.exists(mydirfile)):
319 return False;
320 mydirfile = NormalizeRelativePath(mydirfile);
321 if(os.path.exists(mydirfile) and os.path.islink(mydirfile)):
322 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile));
323 if(os.path.exists(mydirfile) and os.path.isdir(mydirfile)):
324 for root, dirs, filenames in os.walk(mydirfile):
325 dpath = root;
326 dpath = RemoveWindowsPath(dpath);
327 if(dpath not in retlist and not duplicates):
328 retlist.append(dpath);
329 if(duplicates):
330 retlist.append(dpath);
331 for file in filenames:
332 fpath = os.path.join(root, file);
333 fpath = RemoveWindowsPath(fpath);
334 if(fpath not in retlist and not duplicates):
335 retlist.append(fpath);
336 if(duplicates):
337 retlist.append(fpath);
338 else:
339 retlist.append(RemoveWindowsPath(mydirfile));
340 return retlist;
342 def ListDirAdvanced(dirpath, followlink=False, duplicates=False):
343 if isinstance(dirpath, (list, tuple)):
344 dirpath = list(filter(None, dirpath));
345 elif isinstance(dirpath, str):
346 dirpath = list(filter(None, [dirpath]));
347 retlist = []
348 for mydirfile in dirpath:
349 if not os.path.exists(mydirfile):
350 return False;
351 mydirfile = NormalizeRelativePath(mydirfile);
352 if os.path.exists(mydirfile) and os.path.islink(mydirfile) and followlink:
353 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile))
354 if os.path.exists(mydirfile) and os.path.isdir(mydirfile):
355 for root, dirs, filenames in os.walk(mydirfile):
356 # Sort dirs and filenames alphabetically in place
357 dirs.sort(key=lambda x: x.lower());
358 filenames.sort(key=lambda x: x.lower());
359 dpath = RemoveWindowsPath(root);
360 if not duplicates and dpath not in retlist:
361 retlist.append(dpath);
362 elif duplicates:
363 retlist.append(dpath);
364 for file in filenames:
365 fpath = os.path.join(root, file);
366 fpath = RemoveWindowsPath(fpath);
367 if not duplicates and fpath not in retlist:
368 retlist.append(fpath);
369 elif duplicates:
370 retlist.append(fpath);
371 else:
372 retlist.append(RemoveWindowsPath(mydirfile));
373 return retlist;
375 def create_alias_function(prefix, base_name, suffix, target_function):
376 # Define a new function that wraps the target function
377 def alias_function(*args, **kwargs):
378 return target_function(*args, **kwargs);
380 # Create the function name by combining the prefix, base name, and the suffix
381 function_name = "{}{}{}".format(prefix, base_name, suffix);
383 # Add the new function to the global namespace
384 globals()[function_name] = alias_function;
386 def create_alias_function_alt(prefix, base_name, suffix, target_function):
387 # Create the function name by combining the prefix, base name, and the suffix
388 # Use the format method for string formatting, compatible with Python 2 and 3
389 function_name = "{}{}{}".format(prefix, base_name, suffix)
390 # Add the new function (alias of the target_function) to the global namespace
391 # This line is compatible as-is with both Python 2 and 3
392 globals()[function_name] = target_function
394 # initial_value can be 0xFFFF or 0x0000
395 def crc16_ansi(msg, initial_value=0xFFFF):
396 # CRC-16-IBM / CRC-16-ANSI polynomial and initial value
397 poly = 0x8005; # Polynomial for CRC-16-IBM / CRC-16-ANSI
398 crc = initial_value; # Initial value
399 for b in msg:
400 crc ^= b << 8; # XOR byte into CRC top byte
401 for _ in range(8): # Process each bit
402 if crc & 0x8000: # If the top bit is set
403 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
404 else:
405 crc = crc << 1; # Just shift left
406 crc &= 0xFFFF; # Ensure CRC remains 16-bit
407 return crc;
409 # initial_value can be 0xFFFF or 0x0000
410 def crc16_ibm(msg, initial_value=0xFFFF):
411 return crc16_ansi(msg, initial_value);
413 # initial_value is 0xFFFF
414 def crc16(msg):
415 return crc16_ansi(msg, 0xFFFF);
417 # initial_value can be 0xFFFF, 0x1D0F or 0x0000
418 def crc16_ccitt(msg, initial_value=0xFFFF):
419 # CRC-16-CCITT polynomial
420 poly = 0x1021; # Polynomial for CRC-16-CCITT
421 # Use the specified initial value
422 crc = initial_value;
423 for b in msg:
424 crc ^= b << 8; # XOR byte into CRC top byte
425 for _ in range(8): # Process each bit
426 if crc & 0x8000: # If the top bit is set
427 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
428 else:
429 crc = crc << 1; # Just shift left
430 crc &= 0xFFFF; # Ensure CRC remains 16-bit
431 return crc;
433 # initial_value can be 0x42F0E1EBA9EA3693 or 0x0000000000000000
434 def crc64_ecma(msg, initial_value=0x0000000000000000):
435 # CRC-64-ECMA polynomial and initial value
436 poly = 0x42F0E1EBA9EA3693;
437 crc = initial_value; # Initial value for CRC-64-ECMA
438 for b in msg:
439 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
440 for _ in range(8): # Process each bit
441 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
442 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
443 else:
444 crc <<= 1; # Just shift left if the MSB is 0
445 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
446 return crc;
448 # initial_value can be 0x000000000000001B or 0xFFFFFFFFFFFFFFFF
449 def crc64_iso(msg, initial_value=0xFFFFFFFFFFFFFFFF):
450 # CRC-64-ISO polynomial and initial value
451 poly = 0x000000000000001B;
452 crc = initial_value; # Common initial value for CRC-64-ISO
453 for b in msg:
454 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
455 for _ in range(8): # Process each bit
456 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
457 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
458 else:
459 crc <<= 1; # Just shift left if the MSB is 0
460 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
461 return crc;
463 def GetDataFromArray(data, path, default=None):
464 element = data;
465 try:
466 for key in path:
467 element = element[key];
468 return element;
469 except (KeyError, TypeError, IndexError):
470 return default;
472 def GetDataFromArrayAlt(structure, path, default=None):
473 element = structure;
474 for key in path:
475 if isinstance(element, dict) and key in element:
476 element = element[key];
477 elif isinstance(element, list) and isinstance(key, int) and -len(element) <= key < len(element):
478 element = element[key];
479 else:
480 return default;
481 return element;
483 def GetHeaderChecksum(inlist=[], checksumtype="crc32", encodedata=True, formatspecs=__file_format_dict__):
484 if isinstance(inlist, list):
485 fileheader = AppendNullBytes(inlist, formatspecs['format_delimiter']);
486 else:
487 fileheader = AppendNullByte(inlist, formatspecs['format_delimiter']);
488 if(encodedata):
489 fileheader = fileheader.encode('UTF-8');
490 if(checksumtype=="none" or checksumtype==""):
491 catfileheadercshex = format(0, 'x').lower();
492 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
493 catfileheadercshex = format(crc16(fileheader) & 0xffff, '04x').lower();
494 elif(checksumtype=="crc16_ccitt"):
495 catfileheadercshex = format(crc16_ccitt(fileheader) & 0xffff, '04x').lower();
496 elif(checksumtype=="adler32"):
497 catfileheadercshex = format(zlib.adler32(fileheader) & 0xffffffff, '08x').lower();
498 elif(checksumtype=="crc32"):
499 catfileheadercshex = format(crc32(fileheader) & 0xffffffff, '08x').lower();
500 elif(checksumtype=="crc64_ecma"):
501 catfileheadercshex = format(crc64_ecma(fileheader) & 0xffffffffffffffff, '016x').lower();
502 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
503 catfileheadercshex = format(crc64_iso(fileheader) & 0xffffffffffffffff, '016x').lower();
504 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
505 checksumoutstr = hashlib.new(checksumtype);
506 checksumoutstr.update(fileheader);
507 catfileheadercshex = checksumoutstr.hexdigest().lower();
508 else:
509 catfileheadercshex = format(0, 'x').lower();
510 return catfileheadercshex;
512 def GetFileChecksum(instr, checksumtype="crc32", encodedata=True, formatspecs=__file_format_dict__):
513 if(encodedata):
514 instr = instr.encode('UTF-8');
515 if(checksumtype=="none" or checksumtype==""):
516 catinstrcshex = format(0, 'x').lower();
517 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
518 catinstrcshex = format(crc16(instr) & 0xffff, '04x').lower();
519 elif(checksumtype=="crc16_ccitt"):
520 catinstrcshex = format(crc16_ccitt(instr) & 0xffff, '04x').lower();
521 elif(checksumtype=="adler32"):
522 catinstrcshex = format(zlib.adler32(instr) & 0xffffffff, '08x').lower();
523 elif(checksumtype=="crc32"):
524 catinstrcshex = format(crc32(instr) & 0xffffffff, '08x').lower();
525 elif(checksumtype=="crc64_ecma"):
526 catinstrcshex = format(crc64_ecma(instr) & 0xffffffffffffffff, '016x').lower();
527 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
528 catinstrcshex = format(crc64_iso(instr) & 0xffffffffffffffff, '016x').lower();
529 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
530 checksumoutstr = hashlib.new(checksumtype);
531 checksumoutstr.update(instr);
532 catinstrcshex = checksumoutstr.hexdigest().lower();
533 else:
534 catinstrcshex = format(0, 'x').lower();
535 return catinstrcshex;
537 def ValidateHeaderChecksum(inlist=[], checksumtype="crc32", inchecksum="0", formatspecs=__file_format_dict__):
538 catfileheadercshex = GetHeaderChecksum(inlist, checksumtype, True, formatspecs);
539 inchecksum = inchecksum.lower();
540 catfileheadercshex = catfileheadercshex.lower();
541 if(inchecksum==catfileheadercshex):
542 return True;
543 else:
544 return False;
546 def ValidateFileChecksum(infile, checksumtype="crc32", inchecksum="0", formatspecs=__file_format_dict__):
547 catinfilecshex = GetFileChecksum(infile, checksumtype, True, formatspecs);
548 inchecksum = inchecksum.lower();
549 catinfilecshex = catinfilecshex.lower();
550 if(inchecksum==catinfilecshex):
551 return True;
552 else:
553 return False;
555 def ReadTillNullByteOld(fp, delimiter=__file_format_dict__['format_delimiter']):
556 curbyte = b"";
557 curfullbyte = b"";
558 nullbyte = delimiter.encode("UTF-8");
559 while(True):
560 curbyte = fp.read(1);
561 if(curbyte==nullbyte or not curbyte):
562 break;
563 curfullbyte = curfullbyte + curbyte;
564 return curfullbyte.decode('UTF-8');
566 def ReadUntilNullByteOld(fp, delimiter=__file_format_dict__['format_delimiter']):
567 return ReadTillNullByteOld(fp, delimiter);
569 def ReadTillNullByte(fp, delimiter=__file_format_dict__['format_delimiter'], max_read=1024000):
570 curfullbyte = bytearray();
571 nullbyte = delimiter.encode("UTF-8");
572 total_read = 0; # Track the total number of bytes read
573 while True:
574 curbyte = fp.read(1);
575 if curbyte == nullbyte or not curbyte:
576 break;
577 curfullbyte.extend(curbyte);
578 total_read += 1;
579 if total_read >= max_read:
580 raise MemoryError("Maximum read limit reached without finding the delimiter.");
581 # Decode the full byte array to string once out of the loop
582 try:
583 return curfullbyte.decode('UTF-8');
584 except UnicodeDecodeError:
585 # Handle potential partial UTF-8 characters
586 for i in range(1, 4):
587 try:
588 return curfullbyte[:-i].decode('UTF-8');
589 except UnicodeDecodeError:
590 continue;
591 raise; # Re-raise if decoding fails even after trimming
593 def ReadUntilNullByte(fp, delimiter=__file_format_dict__['format_delimiter'], max_read=1024000):
594 return ReadTillNullByte(fp, delimiter, max_read);
596 def SeekToEndOfFile(fp):
597 lasttell = 0;
598 while(True):
599 fp.seek(1, 1);
600 if(lasttell==fp.tell()):
601 break;
602 lasttell = fp.tell();
603 return True;
605 def ReadFileHeaderData(fp, rounds=0, delimiter=__file_format_dict__['format_delimiter']):
606 rocount = 0;
607 roend = int(rounds);
608 HeaderOut = [];
609 while(rocount<roend):
610 HeaderOut.append(ReadTillNullByte(fp, delimiter));
611 rocount = rocount + 1;
612 return HeaderOut;
614 def ReadFileHeaderDataBySize(fp, delimiter=__file_format_dict__['format_delimiter']):
615 headerpresize = ReadTillNullByte(fp, delimiter);
616 headersize = int(headerpresize, 16);
617 if(headersize<=0):
618 return [];
619 headercontent = str(fp.read(headersize).decode('UTF-8')).split(delimiter);
620 fp.seek(1, 1);
621 rocount = 0;
622 roend = int(len(headercontent));
623 HeaderOut = [headerpresize];
624 while(rocount<roend):
625 HeaderOut.append(headercontent[rocount]);
626 rocount = rocount + 1;
627 return HeaderOut;
629 def ReadFileHeaderDataWoSize(fp, delimiter=__file_format_dict__['format_delimiter']):
630 preheaderdata = ReadFileHeaderData(fp, 2, delimiter);
631 headersize = int(preheaderdata[0], 16);
632 headernumfields = int(preheaderdata[1], 16);
633 if(headersize<=0 or headernumfields<=0):
634 return [];
635 headerdata = ReadFileHeaderData(fp, headernumfields, delimiter);
636 HeaderOut = preheaderdata + headerdata;
637 return HeaderOut;
639 def ReadFileHeaderDataBySizeWithContent(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
640 delimiter = formatspecs['format_delimiter'];
641 fheaderstart = fp.tell();
642 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
643 if(len(HeaderOut)==0):
644 return False;
645 if(re.findall("^[.|/]", HeaderOut[3])):
646 fname = HeaderOut[3];
647 else:
648 fname = "./"+HeaderOut[3];
649 fchecksumtype = HeaderOut[-3].lower();
650 fcs = HeaderOut[-2].lower();
651 fccs = HeaderOut[-1].lower();
652 fsize = int(HeaderOut[5], 16);
653 fcompression = HeaderOut[12];
654 fcsize = int(HeaderOut[13], 16);
655 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
656 if(fcs!=newfcs and not skipchecksum):
657 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
658 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
659 return False;
660 fhend = fp.tell() - 1;
661 fcontentstart = fp.tell();
662 fcontents = BytesIO();
663 if(fsize>0 and not listonly):
664 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
665 fcontents.write(fp.read(fsize));
666 else:
667 fcontents.write(fp.read(fcsize));
668 elif(fsize>0 and listonly):
669 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
670 fp.seek(fsize, 1);
671 else:
672 fp.seek(fcsize, 1);
673 fcontents.seek(0, 0);
674 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
675 if(fccs!=newfccs and not skipchecksum and not listonly):
676 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
677 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
678 return False;
679 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
680 pass;
681 else:
682 fcontents.seek(0, 0);
683 fcontents = UncompressArchiveFile(fcontents, formatspecs);
684 fp.seek(1, 1);
685 fcontentend = fp.tell() - 1;
686 HeaderOut.append(fcontents);
687 return HeaderOut;
689 def ReadFileHeaderDataBySizeWithContentToArray(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
690 delimiter = formatspecs['format_delimiter'];
691 fheaderstart = fp.tell();
692 if(formatspecs['new_style']):
693 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
694 else:
695 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
696 if(len(HeaderOut)==0):
697 return False;
698 fheadsize = int(HeaderOut[0], 16);
699 fnumfields = int(HeaderOut[1], 16);
700 ftype = int(HeaderOut[2], 16);
701 if(re.findall("^[.|/]", HeaderOut[3])):
702 fname = HeaderOut[3];
703 else:
704 fname = "./"+HeaderOut[3];
705 fbasedir = os.path.dirname(fname);
706 flinkname = HeaderOut[4];
707 fsize = int(HeaderOut[5], 16);
708 fatime = int(HeaderOut[6], 16);
709 fmtime = int(HeaderOut[7], 16);
710 fctime = int(HeaderOut[8], 16);
711 fbtime = int(HeaderOut[9], 16);
712 fmode = int(HeaderOut[10], 16);
713 fchmode = stat.S_IMODE(fmode);
714 ftypemod = stat.S_IFMT(fmode);
715 fwinattributes = int(HeaderOut[11], 16);
716 fcompression = HeaderOut[12];
717 fcsize = int(HeaderOut[13], 16);
718 fuid = int(HeaderOut[14], 16);
719 funame = HeaderOut[15];
720 fgid = int(HeaderOut[16], 16);
721 fgname = HeaderOut[17];
722 fid = int(HeaderOut[18], 16);
723 finode = int(HeaderOut[19], 16);
724 flinkcount = int(HeaderOut[20], 16);
725 fdev_minor = int(HeaderOut[21], 16);
726 fdev_major = int(HeaderOut[22], 16);
727 frdev_minor = int(HeaderOut[23], 16);
728 frdev_major = int(HeaderOut[24], 16);
729 fextrasize = int(HeaderOut[25], 16);
730 fextrafields = int(HeaderOut[26], 16);
731 extrafieldslist = [];
732 extrastart = 27;
733 extraend = extrastart + fextrafields;
734 extrafieldslist = [];
735 if(extrastart<extraend):
736 extrafieldslist.append(HeaderOut[extrastart]);
737 extrastart = extrastart + 1;
738 fchecksumtype = HeaderOut[extrastart].lower();
739 fcs = HeaderOut[extrastart + 1].lower();
740 fccs = HeaderOut[extrastart + 2].lower();
741 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
742 if(fcs!=newfcs and not skipchecksum):
743 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
744 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
745 return False;
746 fhend = fp.tell() - 1;
747 fcontentstart = fp.tell();
748 fcontents = BytesIO();
749 pyhascontents = False;
750 if(fsize>0 and not listonly):
751 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
752 fcontents.write(fp.read(fsize));
753 else:
754 fcontents.write(fp.read(fcsize));
755 pyhascontents = True;
756 elif(fsize>0 and listonly):
757 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
758 fp.seek(fsize, 1);
759 else:
760 fp.seek(fcsize, 1);
761 pyhascontents = False;
762 fcontents.seek(0, 0);
763 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
764 if(fccs!=newfccs and not skipchecksum and not listonly):
765 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
766 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
767 return False;
768 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
769 pass;
770 else:
771 fcontents.seek(0, 0);
772 fcontents = UncompressArchiveFile(fcontents, formatspecs);
773 fp.seek(1, 1);
774 fcontentend = fp.tell() - 1;
775 fcontents.seek(0, 0);
776 catlist = {'fheadersize': fheadsize, 'fhstart': fheaderstart, 'fhend': fhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': fchecksumtype, 'fnumfields': fnumfields + 2, 'frawheader': HeaderOut, 'fextrafields': fextrafields, 'fextrafieldsize': fextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': fcs, 'fcontentchecksum': fccs, 'fhascontents': pyhascontents, 'fcontentstart': fcontentstart, 'fcontentend': fcontentend, 'fcontents': fcontents};
777 return catlist;
779 def ReadFileHeaderDataBySizeWithContentToList(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
780 delimiter = formatspecs['format_delimiter'];
781 fheaderstart = fp.tell();
782 if(formatspecs['new_style']):
783 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
784 else:
785 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
786 if(len(HeaderOut)==0):
787 return False;
788 fheadsize = int(HeaderOut[0], 16);
789 fnumfields = int(HeaderOut[1], 16);
790 ftype = int(HeaderOut[2], 16);
791 if(re.findall("^[.|/]", HeaderOut[3])):
792 fname = HeaderOut[3];
793 else:
794 fname = "./"+HeaderOut[3];
795 fbasedir = os.path.dirname(fname);
796 flinkname = HeaderOut[4];
797 fsize = int(HeaderOut[5], 16);
798 fatime = int(HeaderOut[6], 16);
799 fmtime = int(HeaderOut[7], 16);
800 fctime = int(HeaderOut[8], 16);
801 fbtime = int(HeaderOut[9], 16);
802 fmode = int(HeaderOut[10], 16);
803 fchmode = stat.S_IMODE(fmode);
804 ftypemod = stat.S_IFMT(fmode);
805 fwinattributes = int(HeaderOut[11], 16);
806 fcompression = HeaderOut[12];
807 fcsize = int(HeaderOut[13], 16);
808 fuid = int(HeaderOut[14], 16);
809 funame = HeaderOut[15];
810 fgid = int(HeaderOut[16], 16);
811 fgname = HeaderOut[17];
812 fid = int(HeaderOut[18], 16);
813 finode = int(HeaderOut[19], 16);
814 flinkcount = int(HeaderOut[20], 16);
815 fdev_minor = int(HeaderOut[21], 16);
816 fdev_major = int(HeaderOut[22], 16);
817 frdev_minor = int(HeaderOut[23], 16);
818 frdev_major = int(HeaderOut[24], 16);
819 fextrasize = int(HeaderOut[25], 16);
820 fextrafields = int(HeaderOut[26], 16);
821 extrafieldslist = [];
822 extrastart = 27;
823 extraend = extrastart + fextrafields;
824 extrafieldslist = [];
825 if(extrastart<extraend):
826 extrafieldslist.append(HeaderOut[extrastart]);
827 extrastart = extrastart + 1;
828 fchecksumtype = HeaderOut[extrastart].lower();
829 fcs = HeaderOut[extrastart + 1].lower();
830 fccs = HeaderOut[extrastart + 2].lower();
831 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
832 if(fcs!=newfcs and not skipchecksum):
833 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
834 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
835 return False;
836 fhend = fp.tell() - 1;
837 fcontentstart = fp.tell();
838 fcontents = BytesIO();
839 pyhascontents = False;
840 if(fsize>0 and not listonly):
841 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
842 fcontents.write(fp.read(fsize));
843 else:
844 fcontents.write(fp.read(fcsize));
845 pyhascontents = True;
846 elif(fsize>0 and listonly):
847 if(fcompression=="none" or fcompression=="" or fcompression=="atuo"):
848 fp.seek(fsize, 1);
849 else:
850 fp.seek(fcsize, 1);
851 pyhascontents = False;
852 fcontents.seek(0, 0);
853 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
854 if(fccs!=newfccs and not skipchecksum and not listonly):
855 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
856 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
857 return False;
858 if(fcompression=="none" or fcompression=="" or fcompression=="auto"):
859 pass;
860 else:
861 fcontents.seek(0, 0);
862 fcontents = UncompressArchiveFile(fcontents, formatspecs);
863 fp.seek(1, 1);
864 fcontentend = fp.tell() - 1;
865 catlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, extrafieldslist, fchecksumtype, fcontents];
866 return catlist;
868 def ReadFileDataBySizeWithContent(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
869 delimiter = formatspecs['format_delimiter'];
870 curloc = fp.tell();
871 if(curloc>0):
872 fp.seek(0, 0);
873 catheader = ReadFileHeaderData(fp, 4, delimiter);
874 if(curloc>0):
875 fp.seek(curloc, 0);
876 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
877 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
878 if(not headercheck and not skipchecksum):
879 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
880 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
881 return False;
882 fnumfiles = int(catheader[1], 16);
883 countnum = 0;
884 flist = [];
885 while(countnum < fnumfiles):
886 HeaderOut = ReadFileHeaderDataBySizeWithContent(fp, listonly, skipchecksum, formatspecs)
887 if(len(HeaderOut)==0):
888 break;
889 flist.append(HeaderOut);
890 countnum = countnum + 1;
891 return flist;
893 def ReadFileDataBySizeWithContentToArray(fp, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
894 delimiter = formatspecs['format_delimiter'];
895 curloc = fp.tell();
896 if(curloc>0):
897 fp.seek(0, 0);
898 catheader = ReadFileHeaderData(fp, 4, delimiter);
899 if(curloc>0):
900 fp.seek(curloc, 0);
901 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
902 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
903 if(not headercheck and not skipchecksum):
904 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
905 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
906 return False;
907 catstring = catheader[0];
908 catversion = re.findall(r"([\d]+)$", catstring);
909 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
910 fprenumfiles = catheader[1];
911 fnumfiles = int(fprenumfiles, 16);
912 fprechecksumtype = catheader[2];
913 fprechecksum = catheader[3];
914 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
915 if(seekstart<0 and seekstart>fnumfiles):
916 seekstart = 0;
917 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
918 seekend = fnumfiles;
919 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
920 seekend = fnumfiles - abs(seekend);
921 if(seekstart>0):
922 il = 0;
923 while(il < seekstart):
924 prefhstart = fp.tell();
925 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs['format_delimiter']);
926 if(len(preheaderdata)==0):
927 break;
928 prefsize = int(preheaderdata[5], 16);
929 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-3].lower(), True, formatspecs);
930 prefcs = preheaderdata[-2];
931 if(prefcs!=prenewfcs and not skipchecksum):
932 VVerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
933 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
934 return False;
935 valid_archive = False;
936 invalid_archive = True;
937 prefhend = fp.tell() - 1;
938 prefcontentstart = fp.tell();
939 prefcontents = BytesIO();
940 pyhascontents = False;
941 if(prefsize>0):
942 prefcontents.write(fp.read(prefsize));
943 prefcontents.seek(0, 0);
944 prenewfccs = GetFileChecksum(prefcontents.read(), preheaderdata[-3].lower(), False, formatspecs);
945 prefccs = preheaderdata[-1];
946 pyhascontents = True;
947 if(prefccs!=prenewfccs and not skipchecksum):
948 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
949 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
950 return False;
951 fp.seek(1, 1);
952 il = il + 1;
953 realidnum = 0;
954 countnum = seekstart;
955 while(countnum < seekend):
956 catlist['ffilelist'].update({realidnum: {'fid': realidnum, 'fidalt': realidnum}});
957 HeaderOut = ReadFileHeaderDataBySizeWithContentToArray(fp, listonly, skipchecksum, formatspecs);
958 if(len(HeaderOut)==0):
959 break;
960 catlist['ffilelist'][realidnum].update(HeaderOut);
961 countnum = countnum + 1;
962 realidnum = realidnum + 1;
963 return catlist;
965 def ReadFileDataBySizeWithContentToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
966 if(isinstance(infile, dict)):
967 listcatfiles = infile;
968 else:
969 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
970 return False;
971 listcatfiles = ReadFileDataBySizeWithContentToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs);
972 if(not listcatfiles):
973 return False;
974 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
975 lenlist = len(listcatfiles['ffilelist']);
976 lcfi = 0;
977 lcfx = int(listcatfiles['fnumfiles']);
978 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
979 lcfx = int(lenlist);
980 else:
981 lcfx = int(listcatfiles['fnumfiles']);
982 while(lcfi < lcfx):
983 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
984 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
985 catarray['filetoid'].update(filetoidarray);
986 catarray['idtofile'].update(idtofilearray);
987 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
988 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
989 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
990 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
991 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
992 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
993 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
994 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
995 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
996 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
997 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
998 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
999 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
1000 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
1001 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
1002 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
1003 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1004 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1005 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
1006 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
1007 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
1008 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1009 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1010 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
1011 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
1012 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
1013 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
1014 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1015 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1016 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1017 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1018 lcfi = lcfi + 1;
1019 return catarray;
1021 def ReadFileDataBySizeWithContentToList(fp, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
1022 delimiter = formatspecs['format_delimiter'];
1023 curloc = fp.tell();
1024 if(curloc>0):
1025 fp.seek(0, 0);
1026 catheader = ReadFileHeaderData(fp, 4, delimiter);
1027 if(curloc>0):
1028 fp.seek(curloc, 0);
1029 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
1030 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
1031 if(not headercheck and not skipchecksum):
1032 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
1033 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
1034 return False;
1035 catstring = catheader[0];
1036 catversion = re.findall(r"([\d]+)$", catstring);
1037 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
1038 fprenumfiles = catheader[1];
1039 fnumfiles = int(fprenumfiles, 16);
1040 fprechecksumtype = catheader[2];
1041 fprechecksum = catheader[3];
1042 catlist = [];
1043 if(seekstart<0 and seekstart>fnumfiles):
1044 seekstart = 0;
1045 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
1046 seekend = fnumfiles;
1047 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
1048 seekend = fnumfiles - abs(seekend);
1049 if(seekstart>0):
1050 il = 0;
1051 while(il < seekstart):
1052 prefhstart = fp.tell();
1053 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs['format_delimiter']);
1054 if(len(preheaderdata)==0):
1055 break;
1056 prefsize = int(preheaderdata[5], 16);
1057 prefcompression = preheaderdata[12];
1058 prefcsize = int(preheaderdata[13], 16);
1059 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-3].lower(), True, formatspecs);
1060 prefcs = preheaderdata[-2];
1061 if(prefcs!=prenewfcs and not skipchecksum):
1062 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
1063 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
1064 return False;
1065 valid_archive = False;
1066 invalid_archive = True;
1067 prefhend = fp.tell() - 1;
1068 prefcontentstart = fp.tell();
1069 prefcontents = "";
1070 pyhascontents = False;
1071 if(prefsize>0):
1072 if(prefcompression=="none" or prefcompression=="" or prefcompression=="auto"):
1073 prefcontents = catfp.read(prefsize);
1074 else:
1075 prefcontents = catfp.read(prefcsize);
1076 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
1077 prefccs = preheaderdata[-1];
1078 pyhascontents = True;
1079 if(prefccs!=prenewfccs and not skipchecksum):
1080 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
1081 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
1082 return False;
1083 fp.seek(1, 1);
1084 il = il + 1;
1085 realidnum = 0;
1086 countnum = seekstart;
1087 while(countnum < seekend):
1088 HeaderOut = ReadFileHeaderDataBySizeWithContentToList(fp, listonly, skipchecksum, formatspecs);
1089 if(len(HeaderOut)==0):
1090 break;
1091 catlist.append(HeaderOut);
1092 countnum = countnum + 1;
1093 realidnum = realidnum + 1;
1094 return catlist;
1096 def ReadInFileBySizeWithContentToArray(infile, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
1097 delimiter = formatspecs['format_delimiter'];
1098 if(hasattr(infile, "read") or hasattr(infile, "write")):
1099 fp = infile;
1100 fp.seek(0, 0);
1101 fp = UncompressArchiveFile(fp, formatspecs);
1102 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1103 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1104 return False;
1105 if(not fp):
1106 return False;
1107 fp.seek(0, 0);
1108 elif(infile=="-"):
1109 fp = BytesIO();
1110 if(hasattr(sys.stdin, "buffer")):
1111 shutil.copyfileobj(sys.stdin.buffer, fp);
1112 else:
1113 shutil.copyfileobj(sys.stdin, fp);
1114 fp.seek(0, 0);
1115 fp = UncompressArchiveFile(fp, formatspecs);
1116 if(not fp):
1117 return False;
1118 fp.seek(0, 0);
1119 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
1120 fp = download_file_from_internet_file(infile);
1121 fp = UncompressArchiveFile(fp, formatspecs);
1122 fp.seek(0, 0);
1123 if(not fp):
1124 return False;
1125 fp.seek(0, 0);
1126 else:
1127 infile = RemoveWindowsPath(infile);
1128 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
1129 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1130 return False;
1131 compresscheck = CheckCompressionType(infile, formatspecs, True);
1132 if(not compresscheck):
1133 fextname = os.path.splitext(infile)[1];
1134 if(fextname==".gz"):
1135 compresscheck = "gzip";
1136 elif(fextname==".bz2"):
1137 compresscheck = "bzip2";
1138 elif(fextname==".zst"):
1139 compresscheck = "zstd";
1140 elif(fextname==".lz4" or fextname==".clz4"):
1141 compresscheck = "lz4";
1142 elif(fextname==".lzo" or fextname==".lzop"):
1143 compresscheck = "lzo";
1144 elif(fextname==".lzma" or fextname==".xz"):
1145 compresscheck = "lzma";
1146 else:
1147 return False;
1148 if(not compresscheck):
1149 return False;
1150 fp = UncompressFile(infile, formatspecs, "rb");
1151 return ReadFileDataBySizeWithContentToArray(fp, listonly, skipchecksum, formatspecs);
1153 def ReadInFileBySizeWithContentToList(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
1154 delimiter = formatspecs['format_delimiter'];
1155 if(hasattr(infile, "read") or hasattr(infile, "write")):
1156 fp = infile;
1157 fp.seek(0, 0);
1158 fp = UncompressArchiveFile(fp, formatspecs);
1159 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1160 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1161 return False;
1162 if(not fp):
1163 return False;
1164 fp.seek(0, 0);
1165 elif(infile=="-"):
1166 fp = BytesIO();
1167 if(hasattr(sys.stdin, "buffer")):
1168 shutil.copyfileobj(sys.stdin.buffer, fp);
1169 else:
1170 shutil.copyfileobj(sys.stdin, fp);
1171 fp.seek(0, 0);
1172 fp = UncompressArchiveFile(fp, formatspecs);
1173 if(not fp):
1174 return False;
1175 fp.seek(0, 0);
1176 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
1177 fp = download_file_from_internet_file(infile);
1178 fp = UncompressArchiveFile(fp, formatspecs);
1179 fp.seek(0, 0);
1180 if(not fp):
1181 return False;
1182 fp.seek(0, 0);
1183 else:
1184 infile = RemoveWindowsPath(infile);
1185 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
1186 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
1187 return False;
1188 compresscheck = CheckCompressionType(infile, formatspecs, True);
1189 if(not compresscheck):
1190 fextname = os.path.splitext(infile)[1];
1191 if(fextname==".gz"):
1192 compresscheck = "gzip";
1193 elif(fextname==".bz2"):
1194 compresscheck = "bzip2";
1195 elif(fextname==".zst"):
1196 compresscheck = "zstd";
1197 elif(fextname==".lz4" or fextname==".clz4"):
1198 compresscheck = "lz4";
1199 elif(fextname==".lzo" or fextname==".lzop"):
1200 compresscheck = "lzo";
1201 elif(fextname==".lzma" or fextname==".xz"):
1202 compresscheck = "lzma";
1203 else:
1204 return False;
1205 if(not compresscheck):
1206 return False;
1207 fp = UncompressFile(infile, formatspecs, "rb");
1208 return ReadFileDataBySizeWithContentToList(fp, seekstart, seekend, listonly, skipchecksum, formatspecs);
1210 def ReadInFileBySizeWithContentToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__):
1211 if(isinstance(infile, dict)):
1212 listcatfiles = infile;
1213 else:
1214 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
1215 infile = RemoveWindowsPath(infile);
1216 listcatfiles = ReadInFileBySizeWithContentToArray(infile, listonly, skipchecksum, formatspecs);
1217 if(not listcatfiles):
1218 return False;
1219 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
1220 lenlist = len(listcatfiles['ffilelist']);
1221 lcfi = 0;
1222 lcfx = int(listcatfiles['fnumfiles']);
1223 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
1224 lcfx = int(lenlist);
1225 else:
1226 lcfx = int(listcatfiles['fnumfiles']);
1227 while(lcfi < lcfx):
1228 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
1229 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
1230 catarray['filetoid'].update(filetoidarray);
1231 catarray['idtofile'].update(idtofilearray);
1232 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
1233 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
1234 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
1235 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
1236 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
1237 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
1238 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
1239 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
1240 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
1241 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1242 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1243 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
1244 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
1245 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
1246 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
1247 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
1248 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1249 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1250 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
1251 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
1252 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
1253 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1254 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1255 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
1256 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
1257 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
1258 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
1259 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1260 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1261 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1262 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1263 lcfi = lcfi + 1;
1264 return catarray;
1266 def AppendNullByte(indata, delimiter=__file_format_dict__['format_delimiter']):
1267 outdata = str(indata) + delimiter;
1268 return outdata;
1270 def AppendNullBytes(indata=[], delimiter=__file_format_dict__['format_delimiter']):
1271 outdata = "";
1272 inum = 0;
1273 il = len(indata);
1274 while(inum < il):
1275 outdata = outdata + AppendNullByte(indata[inum], delimiter);
1276 inum = inum + 1;
1277 return outdata;
1279 def AppendFileHeader(fp, numfiles, checksumtype="crc32", formatspecs=__file_format_dict__):
1280 delimiter = formatspecs['format_delimiter'];
1281 catver = formatspecs['format_ver'];
1282 fileheaderver = str(int(catver.replace(".", "")));
1283 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
1284 fp.write(fileheader.encode('UTF-8'));
1285 fnumfiles = format(int(numfiles), 'x').lower();
1286 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs['format_delimiter']);
1287 catfileheadercshex = GetFileChecksum(fileheader + fnumfilesa, checksumtype, True, formatspecs);
1288 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
1289 fp.write(fnumfilesa.encode('UTF-8'));
1290 try:
1291 fp.flush();
1292 os.fsync(fp.fileno());
1293 except io.UnsupportedOperation:
1294 pass;
1295 except AttributeError:
1296 pass;
1297 except OSError as e:
1298 pass;
1299 return fp;
1301 def MakeEmptyFilePointer(fp, checksumtype="crc32", formatspecs=__file_format_dict__):
1302 AppendFileHeader(fp, 0, checksumtype, formatspecs);
1303 return fp;
1305 def MakeEmptyFile(outfile, compression="auto", compressionlevel=None, checksumtype="crc32", formatspecs=__file_format_dict__, returnfp=False):
1306 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1307 if(os.path.exists(outfile)):
1308 try:
1309 os.unlink(outfile);
1310 except OSError as e:
1311 pass;
1312 if(outfile=="-"):
1313 verbose = False;
1314 catfpfp = BytesIO();
1315 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1316 catfp = outfile;
1317 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1318 catfp = BytesIO();
1319 else:
1320 fbasename = os.path.splitext(outfile)[0];
1321 fextname = os.path.splitext(outfile)[1];
1322 catfp = CompressOpenFile(outfile, True, compressionlevel);
1323 catfp = AppendFileHeader(catfp, 0, checksumtype, formatspecs);
1324 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1325 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1326 try:
1327 catfp.flush();
1328 os.fsync(catfp.fileno());
1329 except io.UnsupportedOperation:
1330 pass;
1331 except AttributeError:
1332 pass;
1333 except OSError as e:
1334 pass;
1335 if(outfile=="-"):
1336 catfp.seek(0, 0);
1337 if(hasattr(sys.stdout, "buffer")):
1338 shutil.copyfileobj(catfp, sys.stdout.buffer);
1339 else:
1340 shutil.copyfileobj(catfp, sys.stdout);
1341 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1342 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1343 catfp.seek(0, 0);
1344 upload_file_to_internet_file(catfp, outfile);
1345 if(returnfp):
1346 catfp.seek(0, 0);
1347 return catfp;
1348 else:
1349 catfp.close();
1350 return True;
1352 def AppendFileHeaderWithContent(fp, filevalues=[], extradata=[], filecontent="", checksumtype="crc32", formatspecs=__file_format_dict__):
1353 extrafields = format(len(extradata), 'x').lower();
1354 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
1355 if(len(extradata)>0):
1356 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
1357 extrasizelen = format(len(extrasizestr), 'x').lower();
1358 catoutlen = len(filevalues) + len(extradata) + 5;
1359 catoutlenhex = format(catoutlen, 'x').lower();
1360 catoutlist = filevalues;
1361 catoutlist.insert(0, catoutlenhex);
1362 catoutlist.append(extrasizelen);
1363 catoutlist.append(extrafields);
1364 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
1365 if(len(extradata)>0):
1366 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
1367 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
1368 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
1369 catfilecontentcshex = GetFileChecksum(filecontent, checksumtype, False, formatspecs);
1370 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
1371 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
1372 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
1373 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
1374 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
1375 catfileoutstrecd = catfileoutstr.encode('UTF-8');
1376 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
1377 catfileout = catfileoutstrecd + filecontent + nullstrecd;
1378 fp.write(catfileout);
1379 try:
1380 fp.flush();
1381 os.fsync(fp.fileno());
1382 except io.UnsupportedOperation:
1383 pass;
1384 except AttributeError:
1385 pass;
1386 except OSError as e:
1387 pass;
1388 return fp;
1390 def AppendFilesWithContent(infiles, fp, dirlistfromtxt=False, filevalues=[], extradata=[], compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
1391 advancedlist = formatspecs['use_advanced_list'];
1392 altinode = formatspecs['use_alt_inode'];
1393 if(verbose):
1394 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
1395 if(infiles=="-"):
1396 for line in sys.stdin:
1397 infilelist.append(line.strip());
1398 infilelist = list(filter(None, infilelist));
1399 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
1400 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
1401 return False;
1402 with UncompressFile(infiles, formatspecs, "r") as finfile:
1403 for line in finfile:
1404 infilelist.append(line.strip());
1405 infilelist = list(filter(None, infilelist));
1406 else:
1407 if(isinstance(infiles, (list, tuple, ))):
1408 infilelist = list(filter(None, infiles));
1409 elif(isinstance(infiles, (str, ))):
1410 infilelist = list(filter(None, [infiles]));
1411 if(advancedlist):
1412 GetDirList = ListDirAdvanced(infilelist, followlink, False);
1413 else:
1414 GetDirList = ListDir(infilelist, followlink, False);
1415 if(not GetDirList):
1416 return False;
1417 curinode = 0;
1418 curfid = 0;
1419 inodelist = [];
1420 inodetofile = {};
1421 filetoinode = {};
1422 inodetocatinode = {};
1423 numfiles = int(len(GetDirList));
1424 fnumfiles = format(numfiles, 'x').lower();
1425 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
1426 for curfname in GetDirList:
1427 if(re.findall("^[.|/]", curfname)):
1428 fname = curfname;
1429 else:
1430 fname = "./"+curfname;
1431 if(verbose):
1432 VerbosePrintOut(fname);
1433 if(not followlink or followlink is None):
1434 fstatinfo = os.lstat(fname);
1435 else:
1436 fstatinfo = os.stat(fname);
1437 fpremode = fstatinfo.st_mode;
1438 finode = fstatinfo.st_ino;
1439 flinkcount = fstatinfo.st_nlink;
1440 ftype = 0;
1441 if(stat.S_ISREG(fpremode)):
1442 ftype = 0;
1443 elif(stat.S_ISLNK(fpremode)):
1444 ftype = 2;
1445 elif(stat.S_ISCHR(fpremode)):
1446 ftype = 3;
1447 elif(stat.S_ISBLK(fpremode)):
1448 ftype = 4;
1449 elif(stat.S_ISDIR(fpremode)):
1450 ftype = 5;
1451 elif(stat.S_ISFIFO(fpremode)):
1452 ftype = 6;
1453 elif(stat.S_ISSOCK(fpremode)):
1454 ftype = 8;
1455 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
1456 ftype = 9;
1457 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
1458 ftype = 10;
1459 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
1460 ftype = 111;
1461 else:
1462 ftype = 0;
1463 flinkname = "";
1464 fcurfid = format(int(curfid), 'x').lower();
1465 if(not followlink and finode!=0):
1466 if(ftype!=1):
1467 if(finode in inodelist):
1468 ftype = 1;
1469 flinkname = inodetofile[finode];
1470 if(altinode):
1471 fcurinode = format(int(finode), 'x').lower();
1472 else:
1473 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
1474 if(finode not in inodelist):
1475 inodelist.append(finode);
1476 inodetofile.update({finode: fname});
1477 inodetocatinode.update({finode: curinode});
1478 if(altinode):
1479 fcurinode = format(int(finode), 'x').lower();
1480 else:
1481 fcurinode = format(int(curinode), 'x').lower();
1482 curinode = curinode + 1;
1483 else:
1484 fcurinode = format(int(curinode), 'x').lower();
1485 curinode = curinode + 1;
1486 curfid = curfid + 1;
1487 if(ftype==2):
1488 flinkname = os.readlink(fname);
1489 fdev = fstatinfo.st_dev;
1490 getfdev = GetDevMajorMinor(fdev);
1491 fdev_minor = getfdev[0];
1492 fdev_major = getfdev[1];
1493 frdev = fstatinfo.st_dev;
1494 if(hasattr(fstatinfo, "st_rdev")):
1495 frdev = fstatinfo.st_rdev;
1496 else:
1497 frdev = fstatinfo.st_dev;
1498 getfrdev = GetDevMajorMinor(frdev);
1499 frdev_minor = getfrdev[0];
1500 frdev_major = getfrdev[1];
1501 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
1502 fsize = format(int("0"), 'x').lower();
1503 elif(ftype==0 or ftype==7):
1504 fsize = format(int(fstatinfo.st_size), 'x').lower();
1505 else:
1506 fsize = format(int(fstatinfo.st_size)).lower();
1507 fatime = format(int(fstatinfo.st_atime), 'x').lower();
1508 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
1509 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
1510 if(hasattr(fstatinfo, "st_birthtime")):
1511 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
1512 else:
1513 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
1514 fmode = format(int(fstatinfo.st_mode), 'x').lower();
1515 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
1516 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
1517 fuid = format(int(fstatinfo.st_uid), 'x').lower();
1518 fgid = format(int(fstatinfo.st_gid), 'x').lower();
1519 funame = "";
1520 try:
1521 import pwd;
1522 try:
1523 userinfo = pwd.getpwuid(fstatinfo.st_uid);
1524 funame = userinfo.pw_name;
1525 except KeyError:
1526 funame = "";
1527 except ImportError:
1528 funame = "";
1529 fgname = "";
1530 try:
1531 import grp;
1532 try:
1533 groupinfo = grp.getgrgid(fstatinfo.st_gid);
1534 fgname = groupinfo.gr_name;
1535 except KeyError:
1536 fgname = "";
1537 except ImportError:
1538 fgname = "";
1539 fdev_minor = format(int(fdev_minor), 'x').lower();
1540 fdev_major = format(int(fdev_major), 'x').lower();
1541 frdev_minor = format(int(frdev_minor), 'x').lower();
1542 frdev_major = format(int(frdev_major), 'x').lower();
1543 finode = format(int(finode), 'x').lower();
1544 flinkcount = format(int(flinkcount), 'x').lower();
1545 if(hasattr(fstatinfo, "st_file_attributes")):
1546 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
1547 else:
1548 fwinattributes = format(int(0), 'x').lower();
1549 fcompression = "";
1550 fcsize = format(int(0), 'x').lower();
1551 fcontents = BytesIO();
1552 chunk_size = 1024;
1553 if(ftype==0 or ftype==7):
1554 with open(fname, "rb") as fpc:
1555 shutil.copyfileobj(fpc, fcontents);
1556 if(not compresswholefile):
1557 fcontents.seek(0, 2);
1558 ucfsize = fcontents.tell();
1559 fcontents.seek(0, 0);
1560 if(compression=="auto"):
1561 ilsize = len(compressionlistalt);
1562 ilmin = 0;
1563 ilcsize = [];
1564 while(ilmin < ilsize):
1565 cfcontents = BytesIO();
1566 shutil.copyfileobj(fcontents, cfcontents);
1567 fcontents.seek(0, 0);
1568 cfcontents.seek(0, 0);
1569 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
1570 if(cfcontents):
1571 cfcontents.seek(0, 2);
1572 ilcsize.append(cfcontents.tell());
1573 cfcontents.close();
1574 else:
1575 try:
1576 ilcsize.append(sys.maxint);
1577 except AttributeError:
1578 ilcsize.append(sys.maxsize);
1579 ilmin = ilmin + 1;
1580 ilcmin = ilcsize.index(min(ilcsize));
1581 compression = compressionlistalt[ilcmin];
1582 fcontents.seek(0, 0);
1583 cfcontents = BytesIO();
1584 shutil.copyfileobj(fcontents, cfcontents);
1585 cfcontents.seek(0, 0);
1586 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
1587 cfcontents.seek(0, 2);
1588 cfsize = cfcontents.tell();
1589 if(ucfsize > cfsize):
1590 fcsize = format(int(cfsize), 'x').lower();
1591 fcompression = compression;
1592 fcontents.close();
1593 fcontents = cfcontents;
1594 if(followlink and (ftype==1 or ftype==2)):
1595 flstatinfo = os.stat(flinkname);
1596 with open(flinkname, "rb") as fpc:
1597 shutil.copyfileobj(fpc, fcontents);
1598 if(not compresswholefile):
1599 fcontents.seek(0, 2);
1600 ucfsize = fcontents.tell();
1601 fcontents.seek(0, 0);
1602 if(compression=="auto"):
1603 ilsize = len(compressionlistalt);
1604 ilmin = 0;
1605 ilcsize = [];
1606 while(ilmin < ilsize):
1607 cfcontents = BytesIO();
1608 shutil.copyfileobj(fcontents, cfcontents);
1609 fcontents.seek(0, 0);
1610 cfcontents.seek(0, 0);
1611 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
1612 if(cfcontents):
1613 cfcontents.seek(0, 2);
1614 ilcsize.append(cfcontents.tell());
1615 cfcontents.close();
1616 else:
1617 try:
1618 ilcsize.append(sys.maxint);
1619 except AttributeError:
1620 ilcsize.append(sys.maxsize);
1621 ilmin = ilmin + 1;
1622 ilcmin = ilcsize.index(min(ilcsize));
1623 compression = compressionlistalt[ilcmin];
1624 fcontents.seek(0, 0);
1625 cfcontents = BytesIO();
1626 shutil.copyfileobj(fcontents, cfcontents);
1627 cfcontents.seek(0, 0);
1628 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
1629 cfcontents.seek(0, 2);
1630 cfsize = cfcontents.tell();
1631 if(ucfsize > cfsize):
1632 fcsize = format(int(cfsize), 'x').lower();
1633 fcompression = compression;
1634 fcontents.close();
1635 fcontents = cfcontents;
1636 if(fcompression=="none"):
1637 fcompression = "";
1638 fcontents.seek(0, 0);
1639 ftypehex = format(ftype, 'x').lower();
1640 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
1641 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
1642 if(numfiles>0):
1643 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
1644 fp.seek(0, 0);
1645 return fp;
1647 def AppendListsWithContent(inlist, fp, dirlistfromtxt=False, filevalues=[], extradata=[], compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
1648 if(verbose):
1649 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
1650 GetDirList = inlist;
1651 if(not GetDirList):
1652 return False;
1653 curinode = 0;
1654 curfid = 0;
1655 inodelist = [];
1656 inodetofile = {};
1657 filetoinode = {};
1658 inodetocatinode = {};
1659 numfiles = int(len(GetDirList));
1660 fnumfiles = format(numfiles, 'x').lower();
1661 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
1662 for curfname in GetDirList:
1663 ftype = format(curfname[0], 'x').lower();
1664 if(re.findall("^[.|/]", curfname[1])):
1665 fname = curfname[1];
1666 else:
1667 fname = "./"+curfname[1];
1668 fbasedir = os.path.dirname(fname);
1669 flinkname = curfname[2];
1670 fsize = format(curfname[3], 'x').lower();
1671 fatime = format(curfname[4], 'x').lower();
1672 fmtime = format(curfname[5], 'x').lower();
1673 fctime = format(curfname[6], 'x').lower();
1674 fbtime = format(curfname[7], 'x').lower();
1675 fmode = format(curfname[8], 'x').lower();
1676 fwinattributes = format(curfname[9], 'x').lower();
1677 fcompression = curfname[10];
1678 fcsize = format(curfname[11], 'x').lower();
1679 fuid = format(curfname[12], 'x').lower();
1680 funame = curfname[13];
1681 fgid = format(curfname[14], 'x').lower();
1682 fgname = curfname[15];
1683 fid = format(curfname[16], 'x').lower();
1684 finode = format(curfname[17], 'x').lower();
1685 flinkcount = format(curfname[18], 'x').lower();
1686 fdev_minor = format(curfname[19], 'x').lower();
1687 fdev_major = format(curfname[20], 'x').lower();
1688 frdev_minor = format(curfname[21], 'x').lower();
1689 frdev_major = format(curfname[22], 'x').lower();
1690 extradata = curfname[23];
1691 fchecksumtype = curfname[24];
1692 fcontents = curfname[25];
1693 catoutlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
1694 fcontents.seek(0, 0);
1695 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
1696 if(numfiles>0):
1697 fp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
1698 return fp;
1700 def AppendInFileWithContent(infile, fp, dirlistfromtxt=False, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False):
1701 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, False, formatspecs);
1702 return AppendListsWithContent(inlist, fp, dirlistfromtxt, filevalues, extradata, followlink, checksumtype, formatspecs, verbose);
1704 def AppendFilesWithContentToOutFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
1705 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1706 if(os.path.exists(outfile)):
1707 try:
1708 os.unlink(outfile);
1709 except OSError as e:
1710 pass;
1711 if(outfile=="-"):
1712 verbose = False;
1713 catfpfp = BytesIO();
1714 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1715 catfp = outfile;
1716 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1717 catfp = BytesIO();
1718 else:
1719 fbasename = os.path.splitext(outfile)[0];
1720 fextname = os.path.splitext(outfile)[1];
1721 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
1722 catfp = AppendFilesWithContent(infiles, catfp, dirlistfromtxt, filevalues, extradata, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose);
1723 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1724 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1725 try:
1726 catfp.flush();
1727 os.fsync(catfp.fileno());
1728 except io.UnsupportedOperation:
1729 pass;
1730 except AttributeError:
1731 pass;
1732 except OSError as e:
1733 pass;
1734 if(outfile=="-"):
1735 catfp.seek(0, 0);
1736 if(hasattr(sys.stdout, "buffer")):
1737 shutil.copyfileobj(catfp, sys.stdout.buffer);
1738 else:
1739 shutil.copyfileobj(catfp, sys.stdout);
1740 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1741 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1742 catfp.seek(0, 0);
1743 upload_file_to_internet_file(catfp, outfile);
1744 if(returnfp):
1745 catfp.seek(0, 0);
1746 return catfp;
1747 else:
1748 catfp.close();
1749 return True;
1751 def AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
1752 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1753 if(os.path.exists(outfile)):
1754 try:
1755 os.unlink(outfile);
1756 except OSError as e:
1757 pass;
1758 if(outfile=="-"):
1759 verbose = False;
1760 catfpfp = BytesIO();
1761 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1762 catfp = outfile;
1763 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1764 catfp = BytesIO();
1765 else:
1766 fbasename = os.path.splitext(outfile)[0];
1767 fextname = os.path.splitext(outfile)[1];
1768 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
1769 catfp = AppendListsWithContent(inlist, catfp, dirlistfromtxt, filevalues, extradata, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose);
1770 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1771 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1772 try:
1773 catfp.flush();
1774 os.fsync(catfp.fileno());
1775 except io.UnsupportedOperation:
1776 pass;
1777 except AttributeError:
1778 pass;
1779 except OSError as e:
1780 pass;
1781 if(outfile=="-"):
1782 catfp.seek(0, 0);
1783 if(hasattr(sys.stdout, "buffer")):
1784 shutil.copyfileobj(catfp, sys.stdout.buffer);
1785 else:
1786 shutil.copyfileobj(catfp, sys.stdout);
1787 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1788 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1789 catfp.seek(0, 0);
1790 upload_file_to_internet_file(catfp, outfile);
1791 if(returnfp):
1792 catfp.seek(0, 0);
1793 return catfp;
1794 else:
1795 catfp.close();
1796 return True;
1798 def AppendInFileWithContentToOutFile(infile, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
1799 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, False, formatspecs);
1800 return AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt, compression, compresswholefile, compressionlevel, filevalues, extradata, followlink, checksumtype, formatspecs, verbose, returnfp);
1802 def PrintPermissionString(fchmode, ftype):
1803 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
1804 permissionstr = "";
1805 for fmodval in str(oct(fchmode))[-3:]:
1806 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
1807 if(ftype==0 or ftype==7):
1808 permissionstr = "-" + permissionstr;
1809 if(ftype==1):
1810 permissionstr = "h" + permissionstr;
1811 if(ftype==2):
1812 permissionstr = "l" + permissionstr;
1813 if(ftype==3):
1814 permissionstr = "c" + permissionstr;
1815 if(ftype==4):
1816 permissionstr = "b" + permissionstr;
1817 if(ftype==5):
1818 permissionstr = "d" + permissionstr;
1819 if(ftype==6):
1820 permissionstr = "f" + permissionstr;
1821 if(ftype==8):
1822 permissionstr = "D" + permissionstr;
1823 if(ftype==9):
1824 permissionstr = "p" + permissionstr;
1825 if(ftype==10):
1826 permissionstr = "w" + permissionstr;
1827 try:
1828 permissionoutstr = stat.filemode(fchmode);
1829 except AttributeError:
1830 permissionoutstr = permissionstr;
1831 except KeyError:
1832 permissionoutstr = permissionstr;
1833 return permissionoutstr;
1835 def PrintPermissionStringAlt(fchmode, ftype):
1836 permissions = {
1837 '0': '---', '1': '--x', '2': '-w-', '3': '-wx',
1838 '4': 'r--', '5': 'r-x', '6': 'rw-', '7': 'rwx'
1840 # Translate file mode into permission string
1841 permissionstr = ''.join([permissions[i] for i in str(oct(fchmode))[-3:]]);
1842 # Append file type indicator
1843 type_indicators = {
1844 0: '-', 1: 'h', 2: 'l', 3: 'c', 4: 'b',
1845 5: 'd', 6: 'f', 8: 'D', 9: 'p', 10: 'w'
1847 file_type = type_indicators.get(ftype, '-');
1848 permissionstr = file_type + permissionstr;
1849 try:
1850 permissionoutstr = stat.filemode(fchmode);
1851 except AttributeError:
1852 permissionoutstr = permissionstr;
1853 return permissionoutstr;
1855 def CompressionSupport():
1856 compression_list = [];
1857 try:
1858 import gzip;
1859 compression_list.append("gz");
1860 compression_list.append("gzip");
1861 except ImportError:
1862 '''return False;'''
1863 try:
1864 import bz2;
1865 compression_list.append("bz2");
1866 compression_list.append("bzip2");
1867 except ImportError:
1868 '''return False;'''
1869 try:
1870 import lz4;
1871 compression_list.append("lz4");
1872 except ImportError:
1873 '''return False;'''
1874 try:
1875 import lzo;
1876 compression_list.append("lzo");
1877 compression_list.append("lzop");
1878 except ImportError:
1879 '''return False;'''
1880 try:
1881 import zstandard;
1882 compression_list.append("zstd");
1883 compression_list.append("zstandard");
1884 except ImportError:
1885 '''return False;'''
1886 try:
1887 import lzma;
1888 compression_list.append("lzma");
1889 compression_list.append("xz");
1890 except ImportError:
1891 '''return False;'''
1892 return compression_list;
1894 def CheckCompressionType(infile, formatspecs=__file_format_dict__, closefp=True):
1895 if(hasattr(infile, "read") or hasattr(infile, "write")):
1896 catfp = infile;
1897 else:
1898 try:
1899 catfp = open(infile, "rb");
1900 except FileNotFoundError:
1901 return False;
1902 catfp.seek(0, 0);
1903 prefp = catfp.read(2);
1904 filetype = False;
1905 if(prefp==binascii.unhexlify("1f8b")):
1906 filetype = "gzip";
1907 catfp.seek(0, 0);
1908 prefp = catfp.read(3);
1909 if(prefp==binascii.unhexlify("425a68")):
1910 filetype = "bzip2";
1911 if(prefp==binascii.unhexlify("5d0000")):
1912 filetype = "lzma";
1913 catfp.seek(0, 0);
1914 prefp = catfp.read(4);
1915 if(prefp==binascii.unhexlify("28b52ffd")):
1916 filetype = "zstd";
1917 if(prefp==binascii.unhexlify("04224d18")):
1918 filetype = "lz4";
1919 if(prefp==binascii.unhexlify("504B0304")):
1920 filetype = "zipfile";
1921 catfp.seek(0, 0);
1922 prefp = catfp.read(5);
1923 if(prefp==binascii.unhexlify("7573746172")):
1924 filetype = "tarfile";
1925 catfp.seek(0, 0);
1926 prefp = catfp.read(6);
1927 if(prefp==binascii.unhexlify("fd377a585a00")):
1928 filetype = "lzma";
1929 if(prefp==binascii.unhexlify("377abcaf271c")):
1930 filetype = "7zipfile";
1931 catfp.seek(0, 0);
1932 prefp = catfp.read(7);
1933 if(prefp==binascii.unhexlify("526172211a0700")):
1934 filetype = "rarfile";
1935 if(prefp==binascii.unhexlify("43617446696c65")):
1936 filetype = "catfile";
1937 catfp.seek(0, 0);
1938 prefp = catfp.read(8);
1939 if(prefp==binascii.unhexlify("526172211a070100")):
1940 filetype = "rarfile";
1941 catfp.seek(0, 0);
1942 prefp = catfp.read(formatspecs['format_len']);
1943 if(prefp==binascii.unhexlify(formatspecs['format_hex'])):
1944 filetype = formatspecs['format_lower'];
1945 catfp.seek(0, 0);
1946 prefp = catfp.read(9);
1947 if(prefp==binascii.unhexlify("894c5a4f000d0a1a0a")):
1948 filetype = "lzo";
1949 catfp.seek(0, 0);
1950 prefp = catfp.read(10);
1951 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
1952 filetype = "tarfile";
1953 catfp.seek(0, 0);
1954 if(filetype=="gzip" or filetype=="bzip2" or filetype=="lzma" or filetype=="zstd" or filetype=="lz4"):
1955 if(is_tarfile(catfp)):
1956 filetype = "tarfile";
1957 if(not filetype):
1958 if(is_tarfile(catfp)):
1959 filetype = "tarfile";
1960 elif(zipfile.is_zipfile(catfp)):
1961 filetype = "zipfile";
1962 elif(rarfile.is_rarfile(catfp) or rarfile.is_rarfile_sfx(catfp)):
1963 filetype = "rarile";
1964 else:
1965 filetype = False;
1966 catfp.seek(0, 0);
1967 if(closefp):
1968 catfp.close();
1969 return filetype;
1971 def CheckCompressionTypeFromString(instring, formatspecs=__file_format_dict__, closefp=True):
1972 try:
1973 instringsfile = BytesIO(instring);
1974 except TypeError:
1975 instringsfile = BytesIO(instring.encode("UTF-8"));
1976 return CheckCompressionType(instringsfile, formatspecs, closefp);
1978 def GetCompressionMimeType(infile, formatspecs=__file_format_dict__):
1979 compresscheck = CheckCompressionType(fp, formatspecs, False);
1980 if(compresscheck=="gzip" or compresscheck=="gz"):
1981 return archivefile_gzip_mimetype;
1982 if(compresscheck=="bzip2" or compresscheck=="bz2"):
1983 return archivefile_bzip2_mimetype;
1984 if(compresscheck=="zstd" or compresscheck=="zstandard"):
1985 return archivefile_zstandard_mimetype;
1986 if(compresscheck=="lz4"):
1987 return archivefile_lz4_mimetype;
1988 if(compresscheck=="lzo" or compresscheck=="lzop"):
1989 return archivefile_lzop_mimetype;
1990 if(compresscheck=="lzma"):
1991 return archivefile_lzma_mimetype;
1992 if(compresscheck=="xz"):
1993 return archivefile_xz_mimetype;
1994 if(compresscheck=="catfile" or compresscheck=="cat" or compresscheck==formatspecs['format_lower']):
1995 return archivefile_cat_mimetype;
1996 if(not compresscheck):
1997 return False;
1998 return False;
2000 def UncompressArchiveFile(fp, formatspecs=__file_format_dict__):
2001 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
2002 return False;
2003 compresscheck = CheckCompressionType(fp, formatspecs, False);
2004 if(compresscheck=="gzip"):
2005 try:
2006 import gzip;
2007 except ImportError:
2008 return False;
2009 catfp = gzip.GzipFile(fileobj=fp, mode="rb");
2010 if(compresscheck=="bzip2"):
2011 try:
2012 import bz2;
2013 except ImportError:
2014 return False;
2015 catfp = BytesIO();
2016 catfp.write(bz2.decompress(fp.read()));
2017 if(compresscheck=="zstd"):
2018 try:
2019 import zstandard;
2020 except ImportError:
2021 return False;
2022 catfp = BytesIO();
2023 catfp.write(zstandard.decompress(fp.read()));
2024 if(compresscheck=="lz4"):
2025 try:
2026 import lz4.frame;
2027 except ImportError:
2028 return False;
2029 catfp = BytesIO();
2030 catfp.write(lz4.frame.decompress(fp.read()));
2031 if(compresscheck=="lzo" or compresscheck=="lzop"):
2032 try:
2033 import lzo;
2034 except ImportError:
2035 return False;
2036 catfp = BytesIO();
2037 catfp.write(lzo.decompress(fp.read()));
2038 if(compresscheck=="lzma" or compresscheck=="xz"):
2039 try:
2040 import lzma;
2041 except ImportError:
2042 return False;
2043 catfp = BytesIO();
2044 catfp.write(lzma.decompress(fp.read()));
2045 if(compresscheck=="catfile" or compresscheck==formatspecs['format_lower']):
2046 catfp = fp;
2047 if(not compresscheck):
2048 try:
2049 import lzma;
2050 except ImportError:
2051 return False;
2052 catfp = BytesIO();
2053 with fp as fpcontent:
2054 try:
2055 catfp.write(lzma.decompress(fp.read()));
2056 except lzma.LZMAError:
2057 return False;
2058 if(compresscheck!="catfile" or compresscheck!=formatspecs['format_lower']):
2059 fp.close();
2060 return catfp;
2062 create_alias_function("Uncompress", __file_format_name__, "", UncompressArchiveFile);
2064 def UncompressFile(infile, formatspecs=__file_format_dict__, mode="rb"):
2065 compresscheck = CheckCompressionType(infile, formatspecs, False);
2066 if(sys.version_info[0]==2 and compresscheck):
2067 if(mode=="rt"):
2068 mode = "r";
2069 if(mode=="wt"):
2070 mode = "w";
2071 try:
2072 if(compresscheck=="gzip"):
2073 try:
2074 import gzip;
2075 except ImportError:
2076 return False;
2077 try:
2078 filefp = gzip.open(infile, mode, encoding="UTF-8");
2079 except (ValueError, TypeError) as e:
2080 filefp = gzip.open(infile, mode);
2081 if(compresscheck=="bzip2"):
2082 try:
2083 import bz2;
2084 except ImportError:
2085 return False;
2086 try:
2087 filefp = bz2.open(infile, mode, encoding="UTF-8");
2088 except (ValueError, TypeError) as e:
2089 filefp = bz2.open(infile, mode);
2090 if(compresscheck=="zstd"):
2091 try:
2092 import zstandard;
2093 except ImportError:
2094 return False;
2095 try:
2096 filefp = zstandard.open(infile, mode, encoding="UTF-8");
2097 except (ValueError, TypeError) as e:
2098 filefp = zstandard.open(infile, mode);
2099 if(compresscheck=="lz4"):
2100 try:
2101 import lz4.frame;
2102 except ImportError:
2103 return False;
2104 try:
2105 filefp = lz4.frame.open(infile, mode, encoding="UTF-8");
2106 except (ValueError, TypeError) as e:
2107 filefp = lz4.frame.open(infile, mode);
2108 if(compresscheck=="lzo"):
2109 try:
2110 import lzo;
2111 except ImportError:
2112 return False;
2113 try:
2114 filefp = lzo.open(infile, mode, encoding="UTF-8");
2115 except (ValueError, TypeError) as e:
2116 filefp = lzo.open(infile, mode);
2117 if(compresscheck=="lzma"):
2118 try:
2119 import lzma;
2120 except ImportError:
2121 return False;
2122 try:
2123 filefp = lzma.open(infile, mode, encoding="UTF-8");
2124 except (ValueError, TypeError) as e:
2125 filefp = lzma.open(infile, mode);
2126 if(compresscheck=="catfile" or compresscheck==formatspecs['format_lower']):
2127 try:
2128 filefp = open(infile, mode, encoding="UTF-8");
2129 except (ValueError, TypeError) as e:
2130 filefp = open(infile, mode);
2131 if(not compresscheck):
2132 try:
2133 filefp = open(infile, mode, encoding="UTF-8");
2134 except (ValueError, TypeError) as e:
2135 filefp = open(infile, mode);
2136 except FileNotFoundError:
2137 return False;
2138 return filefp;
2140 def UncompressString(infile):
2141 compresscheck = CheckCompressionTypeFromString(infile, formatspecs, False);
2142 if(compresscheck=="gzip"):
2143 try:
2144 import gzip;
2145 except ImportError:
2146 return False;
2147 fileuz = gzip.decompress(infile);
2148 if(compresscheck=="bzip2"):
2149 try:
2150 import bz2;
2151 except ImportError:
2152 return False;
2153 fileuz = bz2.decompress(infile);
2154 if(compresscheck=="zstd"):
2155 try:
2156 import zstandard;
2157 except ImportError:
2158 return False;
2159 fileuz = zstandard.decompress(infile);
2160 if(compresscheck=="lz4"):
2161 try:
2162 import lz4.frame;
2163 except ImportError:
2164 return False;
2165 fileuz = lz4.frame.decompress(infile);
2166 if(compresscheck=="lzo"):
2167 try:
2168 import lzo;
2169 except ImportError:
2170 return False;
2171 fileuz = lzo.decompress(infile);
2172 if(compresscheck=="lzma"):
2173 try:
2174 import lzma;
2175 except ImportError:
2176 return False;
2177 fileuz = lzma.decompress(infile);
2178 if(not compresscheck):
2179 fileuz = infile;
2180 if(hasattr(fileuz, 'decode')):
2181 fileuz = fileuz.decode("UTF-8");
2182 return fileuz;
2184 def UncompressStringAlt(infile):
2185 filefp = StringIO();
2186 outstring = UncompressString(infile);
2187 filefp.write(outstring);
2188 filefp.seek(0, 0);
2189 return filefp;
2191 def CheckCompressionSubType(infile, formatspecs=__file_format_dict__, closefp=True):
2192 compresscheck = CheckCompressionType(infile, formatspecs, False);
2193 if(not compresscheck):
2194 fextname = os.path.splitext(infile)[1];
2195 if(fextname==".gz"):
2196 compresscheck = "gzip";
2197 elif(fextname==".bz2"):
2198 compresscheck = "bzip2";
2199 elif(fextname==".zst"):
2200 compresscheck = "zstd";
2201 elif(fextname==".lz4"):
2202 compresscheck = "lz4";
2203 elif(fextname==".lzo" or fextname==".lzop"):
2204 compresscheck = "lzo";
2205 elif(fextname==".lzma" or fextname==".xz"):
2206 compresscheck = "lzma";
2207 else:
2208 return False;
2209 if(compresscheck=="gzip" or compresscheck=="bzip2" or compresscheck=="lzma" or compresscheck=="zstd" or compresscheck=="lz4"):
2210 if(is_tarfile(infile)):
2211 filetype = "tarfile";
2212 if(not compresscheck):
2213 if(is_tarfile(infile)):
2214 return "tarfile";
2215 elif(zipfile.is_zipfile(infile)):
2216 return "zipfile";
2217 elif(rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile)):
2218 return "rarile";
2219 else:
2220 return False;
2221 return False;
2222 if(compresscheck=="catfile"):
2223 return "catfile";
2224 if(compresscheck==formatspecs['format_lower']):
2225 return formatspecs['format_lower'];
2226 if(compresscheck=="tarfile"):
2227 return "tarfile";
2228 if(compresscheck=="zipfile"):
2229 return "zipfile";
2230 if(rarfile_support and compresscheck=="rarfile"):
2231 return "rarfile";
2232 if(py7zr_support and compresscheck=="7zipfile"):
2233 return "7zipfile";
2234 if(hasattr(infile, "read") or hasattr(infile, "write")):
2235 catfp = UncompressArchiveFile(infile, formatspecs['format_lower']);
2236 else:
2237 try:
2238 if(compresscheck=="gzip"):
2239 try:
2240 import gzip;
2241 except ImportError:
2242 return False;
2243 catfp = gzip.GzipFile(infile, "rb");
2244 if(compresscheck=="bzip2"):
2245 try:
2246 import bz2;
2247 except ImportError:
2248 return False;
2249 catfp = bz2.BZ2File(infile, "rb");
2250 if(compresscheck=="lz4"):
2251 try:
2252 import lz4.frame;
2253 except ImportError:
2254 return False;
2255 catfp = lz4.frame.open(infile, "rb");
2256 if(compresscheck=="zstd"):
2257 try:
2258 import zstandard;
2259 except ImportError:
2260 return False;
2261 catfp = zstandard.open(infile, "rb");
2262 if(compresscheck=="lzma" or compresscheck=="xz"):
2263 try:
2264 import lzma;
2265 except ImportError:
2266 return False;
2267 catfp = lzma.open(infile, "rb");
2268 except FileNotFoundError:
2269 return False;
2270 filetype = False;
2271 prefp = catfp.read(5);
2272 if(prefp==binascii.unhexlify("7573746172")):
2273 filetype = "tarfile";
2274 catfp.seek(0, 0);
2275 prefp = catfp.read(7);
2276 if(prefp==binascii.unhexlify("43617446696c65")):
2277 filetype = "catfile";
2278 catfp.seek(0, 0);
2279 prefp = catfp.read(formatspecs['format_len']);
2280 if(prefp==binascii.unhexlify(formatspecs['format_hex'])):
2281 filetype = formatspecs['format_lower'];
2282 catfp.seek(0, 0);
2283 prefp = catfp.read(10);
2284 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
2285 filetype = "tarfile";
2286 catfp.seek(0, 0);
2287 if(closefp):
2288 catfp.close();
2289 return filetype;
2291 def GZipCompress(data, compresslevel=9):
2292 try:
2293 import gzip;
2294 except ImportError:
2295 return False;
2296 tmpfp = tempfile.NamedTemporaryFile("wb", delete=False);
2297 tmpfp.close();
2298 tmpfp = gzip.GzipFile(tmpfp.name, mode="wb", compresslevel=compresslevel);
2299 tmpfp.write(data);
2300 tmpfp.close();
2301 try:
2302 catfp = open(tmpfp.name, "rb");
2303 except FileNotFoundError:
2304 return False;
2305 catdata = catfp.read();
2306 catfp.close();
2307 return catdata;
2309 def CompressArchiveFile(fp, compression="auto", compressionlevel=None, formatspecs=__file_format_dict__):
2310 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
2311 return False;
2312 fp.seek(0, 0);
2313 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
2314 compression = "auto";
2315 if(compression not in compressionlist and compression is None):
2316 compression = "auto";
2317 if(compression=="gzip"):
2318 try:
2319 import gzip;
2320 except ImportError:
2321 return False;
2322 catfp = BytesIO();
2323 if(compressionlevel is None):
2324 compressionlevel = 9;
2325 else:
2326 compressionlevel = int(compressionlevel);
2327 catfp.write(gzip.compress(fp.read(), compresslevel=compressionlevel));
2328 if(compression=="bzip2"):
2329 try:
2330 import bz2;
2331 except ImportError:
2332 return False;
2333 catfp = BytesIO();
2334 if(compressionlevel is None):
2335 compressionlevel = 9;
2336 else:
2337 compressionlevel = int(compressionlevel);
2338 catfp.write(bz2.compress(fp.read(), compresslevel=compressionlevel));
2339 if(compression=="lz4"):
2340 try:
2341 import lz4.frame;
2342 except ImportError:
2343 return False;
2344 catfp = BytesIO();
2345 if(compressionlevel is None):
2346 compressionlevel = 9;
2347 else:
2348 compressionlevel = int(compressionlevel);
2349 catfp.write(lz4.frame.compress(fp.read(), compression_level=compressionlevel));
2350 if(compression=="lzo" or compression=="lzop"):
2351 try:
2352 import lzo;
2353 except ImportError:
2354 return False;
2355 catfp = BytesIO();
2356 if(compressionlevel is None):
2357 compressionlevel = 9;
2358 else:
2359 compressionlevel = int(compressionlevel);
2360 catfp.write(lzo.compress(fp.read(), compresslevel=compressionlevel));
2361 if(compression=="zstd"):
2362 try:
2363 import zstandard;
2364 except ImportError:
2365 return False;
2366 catfp = BytesIO();
2367 if(compressionlevel is None):
2368 compressionlevel = 10;
2369 else:
2370 compressionlevel = int(compressionlevel);
2371 catfp.write(zstandard.compress(fp.read(), level=compressionlevel));
2372 if(compression=="lzma"):
2373 try:
2374 import lzma;
2375 except ImportError:
2376 return False;
2377 catfp = BytesIO();
2378 if(compressionlevel is None):
2379 compressionlevel = 9;
2380 else:
2381 compressionlevel = int(compressionlevel);
2382 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]));
2383 if(compression=="xz"):
2384 try:
2385 import lzma;
2386 except ImportError:
2387 return False;
2388 catfp = BytesIO();
2389 if(compressionlevel is None):
2390 compressionlevel = 9;
2391 else:
2392 compressionlevel = int(compressionlevel);
2393 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]));
2394 if(compression=="auto" or compression is None):
2395 catfp = fp;
2396 catfp.seek(0, 0);
2397 return catfp;
2399 create_alias_function("Compress", __file_format_name__, "", CompressArchiveFile);
2401 def CompressOpenFile(outfile, compressionenable=True, compressionlevel=None):
2402 if(outfile is None):
2403 return False;
2404 fbasename = os.path.splitext(outfile)[0];
2405 fextname = os.path.splitext(outfile)[1];
2406 if(compressionlevel is None and fextname!=".zst"):
2407 compressionlevel = 9;
2408 elif(compressionlevel is None and fextname==".zst"):
2409 compressionlevel = 10;
2410 else:
2411 compressionlevel = int(compressionlevel);
2412 if(sys.version_info[0]==2):
2413 mode = "w";
2414 else:
2415 mode = "wb";
2416 try:
2417 if(fextname not in outextlistwd or not compressionenable):
2418 try:
2419 outfp = open(outfile, "wb", encoding="UTF-8");
2420 except (ValueError, TypeError) as e:
2421 outfp = open(outfile, "wb");
2422 elif(fextname==".gz"):
2423 try:
2424 import gzip;
2425 except ImportError:
2426 return False;
2427 try:
2428 outfp = gzip.open(outfile, mode, compressionlevel, encoding="UTF-8");
2429 except (ValueError, TypeError) as e:
2430 outfp = gzip.open(outfile, mode, compressionlevel);
2431 elif(fextname==".bz2"):
2432 try:
2433 import bz2;
2434 except ImportError:
2435 return False;
2436 try:
2437 outfp = bz2.open(outfile, mode, compressionlevel, encoding="UTF-8");
2438 except (ValueError, TypeError) as e:
2439 outfp = bz2.open(outfile, mode, compressionlevel);
2440 elif(fextname==".zst"):
2441 try:
2442 import zstandard;
2443 except ImportError:
2444 return False;
2445 try:
2446 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel), encoding="UTF-8");
2447 except (ValueError, TypeError) as e:
2448 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel));
2449 elif(fextname==".xz"):
2450 try:
2451 import lzma;
2452 except ImportError:
2453 return False;
2454 try:
2455 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}], encoding="UTF-8");
2456 except (ValueError, TypeError) as e:
2457 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]);
2458 elif(fextname==".lz4"):
2459 try:
2460 import lz4.frame;
2461 except ImportError:
2462 return False;
2463 try:
2464 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel, encoding="UTF-8");
2465 except (ValueError, TypeError) as e:
2466 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel);
2467 elif(fextname==".lzo"):
2468 try:
2469 import lzo;
2470 except ImportError:
2471 return False;
2472 try:
2473 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel, encoding="UTF-8");
2474 except (ValueError, TypeError) as e:
2475 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel);
2476 elif(fextname==".lzma"):
2477 try:
2478 import lzma;
2479 except ImportError:
2480 return False;
2481 try:
2482 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}], encoding="UTF-8");
2483 except (ValueError, TypeError) as e:
2484 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]);
2485 except FileNotFoundError:
2486 return False;
2487 return outfp;
2489 def GetDevMajorMinor(fdev):
2490 retdev = [];
2491 if(hasattr(os, "minor")):
2492 retdev.append(os.minor(fdev));
2493 else:
2494 retdev.append(0);
2495 if(hasattr(os, "major")):
2496 retdev.append(os.major(fdev));
2497 else:
2498 retdev.append(0);
2499 return retdev;
2501 def CheckSumSupport(checkfor, guaranteed=True):
2502 if(guaranteed):
2503 hash_list = sorted(list(hashlib.algorithms_guaranteed));
2504 else:
2505 hash_list = sorted(list(hashlib.algorithms_available));
2506 checklistout = sorted(hash_list + ['adler32', 'crc16', 'crc16_ansi', 'crc16_ibm', 'crc16_ccitt', 'crc32', 'crc64', 'crc64_ecma', 'crc64_iso', 'none']);
2507 if(checkfor in checklistout):
2508 return True;
2509 else:
2510 return False;
2512 def CheckSumSupportAlt(checkfor, guaranteed=True):
2513 if(guaranteed):
2514 hash_list = sorted(list(hashlib.algorithms_guaranteed));
2515 else:
2516 hash_list = sorted(list(hashlib.algorithms_available));
2517 checklistout = hash_list;
2518 if(checkfor in checklistout):
2519 return True;
2520 else:
2521 return False;
2523 def PackArchiveFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2524 advancedlist = formatspecs['use_advanced_list'];
2525 altinode = formatspecs['use_alt_inode'];
2526 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2527 outfile = RemoveWindowsPath(outfile);
2528 checksumtype = checksumtype.lower();
2529 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
2530 checksumtype="crc32";
2531 if(checksumtype=="none"):
2532 checksumtype = "";
2533 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
2534 compression = "auto";
2535 if(compression not in compressionlist and compression is None):
2536 compression = "auto";
2537 if(verbose):
2538 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2539 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2540 if(os.path.exists(outfile)):
2541 try:
2542 os.unlink(outfile);
2543 except OSError as e:
2544 pass;
2545 if(outfile=="-"):
2546 verbose = False;
2547 catfp = BytesIO();
2548 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2549 catfp = outfile;
2550 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2551 catfp = BytesIO();
2552 else:
2553 fbasename = os.path.splitext(outfile)[0];
2554 fextname = os.path.splitext(outfile)[1];
2555 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
2556 catver = formatspecs['format_ver'];
2557 fileheaderver = str(int(catver.replace(".", "")));
2558 infilelist = [];
2559 if(infiles=="-"):
2560 for line in sys.stdin:
2561 infilelist.append(line.strip());
2562 infilelist = list(filter(None, infilelist));
2563 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
2564 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
2565 return False;
2566 with UncompressFile(infiles, formatspecs, "r") as finfile:
2567 for line in finfile:
2568 infilelist.append(line.strip());
2569 infilelist = list(filter(None, infilelist));
2570 else:
2571 if(isinstance(infiles, (list, tuple, ))):
2572 infilelist = list(filter(None, infiles));
2573 elif(isinstance(infiles, (str, ))):
2574 infilelist = list(filter(None, [infiles]));
2575 if(advancedlist):
2576 GetDirList = ListDirAdvanced(infilelist, followlink, False);
2577 else:
2578 GetDirList = ListDir(infilelist, followlink, False);
2579 if(not GetDirList):
2580 return False;
2581 curinode = 0;
2582 curfid = 0;
2583 inodelist = [];
2584 inodetofile = {};
2585 filetoinode = {};
2586 inodetocatinode = {};
2587 numfiles = int(len(GetDirList));
2588 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
2589 for curfname in GetDirList:
2590 if(re.findall("^[.|/]", curfname)):
2591 fname = curfname;
2592 else:
2593 fname = "./"+curfname;
2594 if(verbose):
2595 VerbosePrintOut(fname);
2596 if(not followlink or followlink is None):
2597 fstatinfo = os.lstat(fname);
2598 else:
2599 fstatinfo = os.stat(fname);
2600 fpremode = fstatinfo.st_mode;
2601 finode = fstatinfo.st_ino;
2602 flinkcount = fstatinfo.st_nlink;
2603 ftype = 0;
2604 if(stat.S_ISREG(fpremode)):
2605 ftype = 0;
2606 elif(stat.S_ISLNK(fpremode)):
2607 ftype = 2;
2608 elif(stat.S_ISCHR(fpremode)):
2609 ftype = 3;
2610 elif(stat.S_ISBLK(fpremode)):
2611 ftype = 4;
2612 elif(stat.S_ISDIR(fpremode)):
2613 ftype = 5;
2614 elif(stat.S_ISFIFO(fpremode)):
2615 ftype = 6;
2616 elif(stat.S_ISSOCK(fpremode)):
2617 ftype = 8;
2618 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
2619 ftype = 9;
2620 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
2621 ftype = 10;
2622 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
2623 ftype = 111;
2624 else:
2625 ftype = 0;
2626 flinkname = "";
2627 fcurfid = format(int(curfid), 'x').lower();
2628 if(not followlink and finode!=0):
2629 if(ftype!=1):
2630 if(finode in inodelist):
2631 ftype = 1;
2632 flinkname = inodetofile[finode];
2633 if(altinode):
2634 fcurinode = format(int(finode), 'x').lower();
2635 else:
2636 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
2637 if(finode not in inodelist):
2638 inodelist.append(finode);
2639 inodetofile.update({finode: fname});
2640 inodetocatinode.update({finode: curinode});
2641 if(altinode):
2642 fcurinode = format(int(finode), 'x').lower();
2643 else:
2644 fcurinode = format(int(curinode), 'x').lower();
2645 curinode = curinode + 1;
2646 else:
2647 fcurinode = format(int(curinode), 'x').lower();
2648 curinode = curinode + 1;
2649 curfid = curfid + 1;
2650 if(ftype==2):
2651 flinkname = os.readlink(fname);
2652 fdev = fstatinfo.st_dev;
2653 getfdev = GetDevMajorMinor(fdev);
2654 fdev_minor = getfdev[0];
2655 fdev_major = getfdev[1];
2656 frdev = fstatinfo.st_dev;
2657 if(hasattr(fstatinfo, "st_rdev")):
2658 frdev = fstatinfo.st_rdev;
2659 else:
2660 frdev = fstatinfo.st_dev;
2661 getfrdev = GetDevMajorMinor(frdev);
2662 frdev_minor = getfrdev[0];
2663 frdev_major = getfrdev[1];
2664 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
2665 fsize = format(int("0"), 'x').lower();
2666 elif(ftype==0 or ftype==7):
2667 fsize = format(int(fstatinfo.st_size), 'x').lower();
2668 else:
2669 fsize = format(int(fstatinfo.st_size)).lower();
2670 fatime = format(int(fstatinfo.st_atime), 'x').lower();
2671 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
2672 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
2673 if(hasattr(fstatinfo, "st_birthtime")):
2674 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
2675 else:
2676 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
2677 fmode = format(int(fstatinfo.st_mode), 'x').lower();
2678 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
2679 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
2680 fuid = format(int(fstatinfo.st_uid), 'x').lower();
2681 fgid = format(int(fstatinfo.st_gid), 'x').lower();
2682 funame = "";
2683 try:
2684 import pwd;
2685 try:
2686 userinfo = pwd.getpwuid(fstatinfo.st_uid);
2687 funame = userinfo.pw_name;
2688 except KeyError:
2689 funame = "";
2690 except ImportError:
2691 funame = "";
2692 fgname = "";
2693 try:
2694 import grp;
2695 try:
2696 groupinfo = grp.getgrgid(fstatinfo.st_gid);
2697 fgname = groupinfo.gr_name;
2698 except KeyError:
2699 fgname = "";
2700 except ImportError:
2701 fgname = "";
2702 fdev_minor = format(int(fdev_minor), 'x').lower();
2703 fdev_major = format(int(fdev_major), 'x').lower();
2704 frdev_minor = format(int(frdev_minor), 'x').lower();
2705 frdev_major = format(int(frdev_major), 'x').lower();
2706 finode = format(int(finode), 'x').lower();
2707 flinkcount = format(int(flinkcount), 'x').lower();
2708 if(hasattr(fstatinfo, "st_file_attributes")):
2709 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
2710 else:
2711 fwinattributes = format(int(0), 'x').lower();
2712 fcompression = "";
2713 fcsize = format(int(0), 'x').lower();
2714 fcontents = BytesIO();
2715 if(ftype==0 or ftype==7):
2716 with open(fname, "rb") as fpc:
2717 shutil.copyfileobj(fpc, fcontents);
2718 if(not compresswholefile):
2719 fcontents.seek(0, 2);
2720 ucfsize = fcontents.tell();
2721 fcontents.seek(0, 0);
2722 if(compression=="auto"):
2723 ilsize = len(compressionlistalt);
2724 ilmin = 0;
2725 ilcsize = [];
2726 while(ilmin < ilsize):
2727 cfcontents = BytesIO();
2728 shutil.copyfileobj(fcontents, cfcontents);
2729 fcontents.seek(0, 0);
2730 cfcontents.seek(0, 0);
2731 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
2732 if(cfcontents):
2733 cfcontents.seek(0, 2);
2734 ilcsize.append(cfcontents.tell());
2735 cfcontents.close();
2736 else:
2737 try:
2738 ilcsize.append(sys.maxint);
2739 except AttributeError:
2740 ilcsize.append(sys.maxsize);
2741 ilmin = ilmin + 1;
2742 ilcmin = ilcsize.index(min(ilcsize));
2743 compression = compressionlistalt[ilcmin];
2744 fcontents.seek(0, 0);
2745 cfcontents = BytesIO();
2746 shutil.copyfileobj(fcontents, cfcontents);
2747 cfcontents.seek(0, 0);
2748 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
2749 cfcontents.seek(0, 2);
2750 cfsize = cfcontents.tell();
2751 if(ucfsize > cfsize):
2752 fcsize = format(int(cfsize), 'x').lower();
2753 fcompression = compression;
2754 fcontents.close();
2755 fcontents = cfcontents;
2756 if(fcompression=="none"):
2757 fcompression = "";
2758 if(followlink and (ftype==1 or ftype==2)):
2759 flstatinfo = os.stat(flinkname);
2760 with open(flinkname, "rb") as fpc:
2761 shutil.copyfileobj(fpc, fcontents);
2762 if(not compresswholefile):
2763 fcontents.seek(0, 2);
2764 ucfsize = fcontents.tell();
2765 fcontents.seek(0, 0);
2766 if(compression=="auto"):
2767 ilsize = len(compressionlistalt);
2768 ilmin = 0;
2769 ilcsize = [];
2770 while(ilmin < ilsize):
2771 cfcontents = BytesIO();
2772 shutil.copyfileobj(fcontents, cfcontents);
2773 fcontents.seek(0, 0);
2774 cfcontents.seek(0, 0);
2775 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
2776 if(cfcontents):
2777 cfcontents.seek(0, 2);
2778 ilcsize.append(cfcontents.tell());
2779 cfcontents.close();
2780 else:
2781 try:
2782 ilcsize.append(sys.maxint);
2783 except AttributeError:
2784 ilcsize.append(sys.maxsize);
2785 ilmin = ilmin + 1;
2786 ilcmin = ilcsize.index(min(ilcsize));
2787 compression = compressionlistalt[ilcmin];
2788 fcontents.seek(0, 0);
2789 cfcontents = BytesIO();
2790 shutil.copyfileobj(fcontents, cfcontents);
2791 cfcontents.seek(0, 0);
2792 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
2793 cfcontents.seek(0, 2);
2794 cfsize = cfcontents.tell();
2795 if(ucfsize > cfsize):
2796 fcsize = format(int(cfsize), 'x').lower();
2797 fcompression = compression;
2798 fcontents.close();
2799 fcontents = cfcontents;
2800 fcontents.seek(0, 0);
2801 ftypehex = format(ftype, 'x').lower();
2802 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
2803 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
2804 fcontents.close();
2805 if(numfiles>0):
2806 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
2807 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
2808 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2809 try:
2810 catfp.flush();
2811 os.fsync(catfp.fileno());
2812 except io.UnsupportedOperation:
2813 pass;
2814 except AttributeError:
2815 pass;
2816 except OSError as e:
2817 pass;
2818 if(outfile=="-"):
2819 catfp.seek(0, 0);
2820 if(hasattr(sys.stdout, "buffer")):
2821 shutil.copyfileobj(catfp, sys.stdout.buffer);
2822 else:
2823 shutil.copyfileobj(catfp, sys.stdout);
2824 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2825 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2826 catfp.seek(0, 0);
2827 upload_file_to_internet_file(catfp, outfile);
2828 if(returnfp):
2829 catfp.seek(0, 0);
2830 return catfp;
2831 else:
2832 catfp.close();
2833 return True;
2835 create_alias_function("Pack", __file_format_name__, "", PackArchiveFile);
2837 if(hasattr(shutil, "register_archive_format")):
2838 def PackArchiveFileFunc(archive_name, source_dir, **kwargs):
2839 return PackArchiveFile(source_dir, archive_name, False, "auto", True, None, False, "crc32", [], __file_format_dict__['format_delimiter'], False, False);
2840 create_alias_function("Pack", __file_format_name__, "Func", PackArchiveFileFunc);
2842 def PackArchiveFileFromDirList(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2843 return PackArchiveFile(infiles, outfile, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, returnfp);
2845 create_alias_function("Pack", __file_format_name__, "FromDirList", PackArchiveFileFromDirList);
2847 def PackArchiveFileFromTarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
2848 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2849 outfile = RemoveWindowsPath(outfile);
2850 checksumtype = checksumtype.lower();
2851 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
2852 checksumtype="crc32";
2853 if(checksumtype=="none"):
2854 checksumtype = "";
2855 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
2856 compression = "auto";
2857 if(compression not in compressionlist and compression is None):
2858 compression = "auto";
2859 if(verbose):
2860 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2861 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2862 if(os.path.exists(outfile)):
2863 try:
2864 os.unlink(outfile);
2865 except OSError as e:
2866 pass;
2867 if(outfile=="-"):
2868 verbose = False;
2869 catfp = BytesIO();
2870 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2871 catfp = outfile;
2872 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2873 catfp = BytesIO();
2874 else:
2875 fbasename = os.path.splitext(outfile)[0];
2876 fextname = os.path.splitext(outfile)[1];
2877 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
2878 catver = formatspecs['format_ver'];
2879 fileheaderver = str(int(catver.replace(".", "")));
2880 curinode = 0;
2881 curfid = 0;
2882 inodelist = [];
2883 inodetofile = {};
2884 filetoinode = {};
2885 inodetocatinode = {};
2886 if(infile=="-"):
2887 infile = BytesIO();
2888 if(hasattr(sys.stdin, "buffer")):
2889 shutil.copyfileobj(sys.stdin.buffer, infile);
2890 else:
2891 shutil.copyfileobj(sys.stdin, infile);
2892 infile.seek(0, 0);
2893 if(not infile):
2894 return False;
2895 infile.seek(0, 0);
2896 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
2897 infile = download_file_from_internet_file(infile);
2898 infile.seek(0, 0);
2899 if(not infile):
2900 return False;
2901 infile.seek(0, 0);
2902 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
2903 return False;
2904 elif(os.path.exists(infile) and os.path.isfile(infile)):
2905 try:
2906 if(not tarfile.is_tarfile(infile)):
2907 return False;
2908 except AttributeError:
2909 if(not is_tarfile(infile)):
2910 return False;
2911 else:
2912 return False;
2913 try:
2914 if(hasattr(infile, "read") or hasattr(infile, "write")):
2915 tarfp = tarfile.open(fileobj=infile, mode="r");
2916 else:
2917 tarfp = tarfile.open(infile, "r");
2918 except FileNotFoundError:
2919 return False;
2920 numfiles = int(len(tarfp.getmembers()));
2921 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
2922 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
2923 if(re.findall("^[.|/]", member.name)):
2924 fname = member.name;
2925 else:
2926 fname = "./"+member.name;
2927 if(verbose):
2928 VerbosePrintOut(fname);
2929 fpremode = member.mode;
2930 ffullmode = member.mode;
2931 flinkcount = 0;
2932 ftype = 0;
2933 if(member.isreg()):
2934 ffullmode = member.mode + stat.S_IFREG;
2935 ftype = 0;
2936 elif(member.isdev()):
2937 ffullmode = member.mode;
2938 ftype = 7;
2939 elif(member.islnk()):
2940 ffullmode = member.mode + stat.S_IFREG;
2941 ftype = 1;
2942 elif(member.issym()):
2943 ffullmode = member.mode + stat.S_IFLNK;
2944 ftype = 2;
2945 elif(member.ischr()):
2946 ffullmode = member.mode + stat.S_IFCHR;
2947 ftype = 3;
2948 elif(member.isblk()):
2949 ffullmode = member.mode + stat.S_IFBLK;
2950 ftype = 4;
2951 elif(member.isdir()):
2952 ffullmode = member.mode + stat.S_IFDIR;
2953 ftype = 5;
2954 elif(member.isfifo()):
2955 ffullmode = member.mode + stat.S_IFIFO;
2956 ftype = 6;
2957 elif(member.issparse()):
2958 ffullmode = member.mode;
2959 ftype = 12;
2960 else:
2961 ffullmode = member.mode;
2962 ftype = 0;
2963 flinkname = "";
2964 fcurfid = format(int(curfid), 'x').lower();
2965 fcurinode = format(int(curfid), 'x').lower();
2966 curfid = curfid + 1;
2967 if(ftype==2):
2968 flinkname = member.linkname;
2969 fdev_minor = format(int(member.devminor), 'x').lower();
2970 fdev_major = format(int(member.devmajor), 'x').lower();
2971 frdev_minor = format(int(member.devminor), 'x').lower();
2972 frdev_major = format(int(member.devmajor), 'x').lower();
2973 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
2974 fsize = format(int("0"), 'x').lower();
2975 elif(ftype==0 or ftype==7):
2976 fsize = format(int(member.size), 'x').lower();
2977 else:
2978 fsize = format(int(member.size), 'x').lower();
2979 fatime = format(int(member.mtime), 'x').lower();
2980 fmtime = format(int(member.mtime), 'x').lower();
2981 fctime = format(int(member.mtime), 'x').lower();
2982 fbtime = format(int(member.mtime), 'x').lower();
2983 fmode = format(int(ffullmode), 'x').lower();
2984 fchmode = format(int(stat.S_IMODE(ffullmode)), 'x').lower();
2985 ftypemod = format(int(stat.S_IFMT(ffullmode)), 'x').lower();
2986 fuid = format(int(member.uid), 'x').lower();
2987 fgid = format(int(member.gid), 'x').lower();
2988 funame = member.uname;
2989 fgname = member.gname;
2990 flinkcount = format(int(flinkcount), 'x').lower();
2991 fwinattributes = format(int(0), 'x').lower();
2992 fcompression = "";
2993 fcsize = format(int(0), 'x').lower();
2994 fcontents = BytesIO();
2995 if(ftype==0 or ftype==7):
2996 with tarfp.extractfile(member) as fpc:
2997 shutil.copyfileobj(fpc, fcontents);
2998 if(not compresswholefile):
2999 fcontents.seek(0, 2);
3000 ucfsize = fcontents.tell();
3001 fcontents.seek(0, 0);
3002 if(compression=="auto"):
3003 ilsize = len(compressionlistalt);
3004 ilmin = 0;
3005 ilcsize = [];
3006 while(ilmin < ilsize):
3007 cfcontents = BytesIO();
3008 shutil.copyfileobj(fcontents, cfcontents);
3009 fcontents.seek(0, 0);
3010 cfcontents.seek(0, 0);
3011 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3012 if(cfcontents):
3013 cfcontents.seek(0, 2);
3014 ilcsize.append(cfcontents.tell());
3015 cfcontents.close();
3016 else:
3017 try:
3018 ilcsize.append(sys.maxint);
3019 except AttributeError:
3020 ilcsize.append(sys.maxsize);
3021 ilmin = ilmin + 1;
3022 ilcmin = ilcsize.index(min(ilcsize));
3023 compression = compressionlistalt[ilcmin];
3024 fcontents.seek(0, 0);
3025 cfcontents = BytesIO();
3026 shutil.copyfileobj(fcontents, cfcontents);
3027 cfcontents.seek(0, 0);
3028 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3029 cfcontents.seek(0, 2);
3030 cfsize = cfcontents.tell();
3031 if(ucfsize > cfsize):
3032 fcsize = format(int(cfsize), 'x').lower();
3033 fcompression = compression;
3034 fcontents.close();
3035 fcontents = cfcontents;
3036 if(fcompression=="none"):
3037 fcompression = "";
3038 fcontents.seek(0, 0);
3039 ftypehex = format(ftype, 'x').lower();
3040 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3041 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3042 fcontents.close();
3043 if(numfiles>0):
3044 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3045 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3046 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3047 try:
3048 catfp.flush();
3049 os.fsync(catfp.fileno());
3050 except io.UnsupportedOperation:
3051 pass;
3052 except AttributeError:
3053 pass;
3054 except OSError as e:
3055 pass;
3056 if(outfile=="-"):
3057 catfp.seek(0, 0);
3058 if(hasattr(sys.stdout, "buffer")):
3059 shutil.copyfileobj(catfp, sys.stdout.buffer);
3060 else:
3061 shutil.copyfileobj(catfp, sys.stdout);
3062 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3063 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3064 catfp.seek(0, 0);
3065 upload_file_to_internet_file(catfp, outfile);
3066 if(returnfp):
3067 catfp.seek(0, 0);
3068 return catfp;
3069 else:
3070 catfp.close();
3071 return True;
3073 create_alias_function("Pack", __file_format_name__, "FromTarFile", PackArchiveFileFromTarFile);
3075 def PackArchiveFileFromZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3076 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3077 outfile = RemoveWindowsPath(outfile);
3078 checksumtype = checksumtype.lower();
3079 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3080 checksumtype="crc32";
3081 if(checksumtype=="none"):
3082 checksumtype = "";
3083 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3084 compression = "auto";
3085 if(compression not in compressionlist and compression is None):
3086 compression = "auto";
3087 if(verbose):
3088 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3089 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3090 if(os.path.exists(outfile)):
3091 try:
3092 os.unlink(outfile);
3093 except OSError as e:
3094 pass;
3095 if(outfile=="-"):
3096 verbose = False;
3097 catfp = BytesIO();
3098 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3099 catfp = outfile;
3100 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3101 catfp = BytesIO();
3102 else:
3103 fbasename = os.path.splitext(outfile)[0];
3104 fextname = os.path.splitext(outfile)[1];
3105 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3106 catver = formatspecs['format_ver'];
3107 fileheaderver = str(int(catver.replace(".", "")));
3108 curinode = 0;
3109 curfid = 0;
3110 inodelist = [];
3111 inodetofile = {};
3112 filetoinode = {};
3113 inodetocatinode = {};
3114 if(infile=="-"):
3115 infile = BytesIO();
3116 if(hasattr(sys.stdin, "buffer")):
3117 shutil.copyfileobj(sys.stdin.buffer, infile);
3118 else:
3119 shutil.copyfileobj(sys.stdin, infile);
3120 infile.seek(0, 0);
3121 if(not infile):
3122 return False;
3123 infile.seek(0, 0);
3124 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3125 infile = download_file_from_internet_file(infile);
3126 infile.seek(0, 0);
3127 if(not infile):
3128 return False;
3129 infile.seek(0, 0);
3130 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
3131 return False;
3132 else:
3133 return False;
3134 if(not zipfile.is_zipfile(infile)):
3135 return False;
3136 try:
3137 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
3138 except FileNotFoundError:
3139 return False;
3140 ziptest = zipfp.testzip();
3141 if(ziptest):
3142 VerbosePrintOut("Bad file found!");
3143 numfiles = int(len(zipfp.infolist()));
3144 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3145 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
3146 if(re.findall("^[.|/]", member.filename)):
3147 fname = member.filename;
3148 else:
3149 fname = "./"+member.filename;
3150 zipinfo = zipfp.getinfo(member.filename);
3151 if(verbose):
3152 VerbosePrintOut(fname);
3153 if(not member.is_dir()):
3154 fpremode = int(stat.S_IFREG + 438);
3155 elif(member.is_dir()):
3156 fpremode = int(stat.S_IFDIR + 511);
3157 flinkcount = 0;
3158 ftype = 0;
3159 if(not member.is_dir()):
3160 ftype = 0;
3161 elif(member.is_dir()):
3162 ftype = 5;
3163 flinkname = "";
3164 fcurfid = format(int(curfid), 'x').lower();
3165 fcurinode = format(int(curfid), 'x').lower();
3166 curfid = curfid + 1;
3167 fdev_minor = format(int(0), 'x').lower();
3168 fdev_major = format(int(0), 'x').lower();
3169 frdev_minor = format(int(0), 'x').lower();
3170 frdev_major = format(int(0), 'x').lower();
3171 if(ftype==5):
3172 fsize = format(int("0"), 'x').lower();
3173 elif(ftype==0):
3174 fsize = format(int(member.file_size), 'x').lower();
3175 else:
3176 fsize = format(int(member.file_size), 'x').lower();
3177 fatime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3178 fmtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3179 fctime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3180 fbtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3181 if(zipinfo.create_system==0 or zipinfo.create_system==10):
3182 fwinattributes = format(int(zipinfo.external_attr), 'x').lower();
3183 if(not member.is_dir()):
3184 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3185 fchmode = stat.S_IMODE(fmode);
3186 ftypemod = stat.S_IFMT(fmode);
3187 elif(member.is_dir()):
3188 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3189 fchmode = stat.S_IMODE(fmode);
3190 ftypemod = stat.S_IFMT(fmode);
3191 elif(zipinfo.create_system==3):
3192 fwinattributes = format(int(0), 'x').lower();
3193 fmode = format(int(zipinfo.external_attr), 'x').lower();
3194 fchmode = stat.S_IMODE(fmode);
3195 ftypemod = stat.S_IFMT(fmode);
3196 else:
3197 fwinattributes = format(int(0), 'x').lower();
3198 if(not member.is_dir()):
3199 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3200 fchmode = stat.S_IMODE(fmode);
3201 ftypemod = stat.S_IFMT(fmode);
3202 elif(member.is_dir()):
3203 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3204 fchmode = stat.S_IMODE(fmode);
3205 ftypemod = stat.S_IFMT(fmode);
3206 fcompression = "";
3207 fcsize = format(int(0), 'x').lower();
3208 try:
3209 fuid = format(int(os.getuid()), 'x').lower();
3210 except AttributeError:
3211 fuid = format(int(0), 'x').lower();
3212 except KeyError:
3213 fuid = format(int(0), 'x').lower();
3214 try:
3215 fgid = format(int(os.getgid()), 'x').lower();
3216 except AttributeError:
3217 fgid = format(int(0), 'x').lower();
3218 except KeyError:
3219 fgid = format(int(0), 'x').lower();
3220 try:
3221 import pwd;
3222 try:
3223 userinfo = pwd.getpwuid(os.getuid());
3224 funame = userinfo.pw_name;
3225 except KeyError:
3226 funame = "";
3227 except AttributeError:
3228 funame = "";
3229 except ImportError:
3230 funame = "";
3231 fgname = "";
3232 try:
3233 import grp;
3234 try:
3235 groupinfo = grp.getgrgid(os.getgid());
3236 fgname = groupinfo.gr_name;
3237 except KeyError:
3238 fgname = "";
3239 except AttributeError:
3240 fgname = "";
3241 except ImportError:
3242 fgname = "";
3243 fcontents = BytesIO();
3244 if(ftype==0):
3245 fcontents.write(zipfp.read(member.filename));
3246 if(not compresswholefile):
3247 fcontents.seek(0, 2);
3248 ucfsize = fcontents.tell();
3249 fcontents.seek(0, 0);
3250 if(compression=="auto"):
3251 ilsize = len(compressionlistalt);
3252 ilmin = 0;
3253 ilcsize = [];
3254 while(ilmin < ilsize):
3255 cfcontents = BytesIO();
3256 shutil.copyfileobj(fcontents, cfcontents);
3257 fcontents.seek(0, 0);
3258 cfcontents.seek(0, 0);
3259 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3260 cfcontents.seek(0, 2);
3261 ilcsize.append(cfcontents.tell());
3262 cfcontents.close();
3263 ilmin = ilmin + 1;
3264 ilcmin = ilcsize.index(min(ilcsize));
3265 compression = compressionlistalt[ilcmin];
3266 fcontents.seek(0, 0);
3267 cfcontents = BytesIO();
3268 shutil.copyfileobj(fcontents, cfcontents);
3269 cfcontents.seek(0, 0);
3270 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3271 cfcontents.seek(0, 2);
3272 cfsize = cfcontents.tell();
3273 if(ucfsize > cfsize):
3274 fcsize = format(int(cfsize), 'x').lower();
3275 fcompression = compression;
3276 fcontents.close();
3277 fcontents = cfcontents;
3278 if(fcompression=="none"):
3279 fcompression = "";
3280 fcontents.seek(0, 0);
3281 ftypehex = format(ftype, 'x').lower();
3282 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3283 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3284 fcontents.close();
3285 if(numfiles>0):
3286 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3287 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3288 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3289 try:
3290 catfp.flush();
3291 os.fsync(catfp.fileno());
3292 except io.UnsupportedOperation:
3293 pass;
3294 except AttributeError:
3295 pass;
3296 except OSError as e:
3297 pass;
3298 if(outfile=="-"):
3299 catfp.seek(0, 0);
3300 if(hasattr(sys.stdout, "buffer")):
3301 shutil.copyfileobj(catfp, sys.stdout.buffer);
3302 else:
3303 shutil.copyfileobj(catfp, sys.stdout);
3304 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3305 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3306 catfp.seek(0, 0);
3307 upload_file_to_internet_file(catfp, outfile);
3308 if(returnfp):
3309 catfp.seek(0, 0);
3310 return catfp;
3311 else:
3312 catfp.close();
3313 return True;
3315 create_alias_function("Pack", __file_format_name__, "FromZipFile", PackArchiveFileFromZipFile);
3317 if(not rarfile_support):
3318 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3319 return False
3321 if(rarfile_support):
3322 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3323 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3324 outfile = RemoveWindowsPath(outfile);
3325 checksumtype = checksumtype.lower();
3326 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3327 checksumtype="crc32";
3328 if(checksumtype=="none"):
3329 checksumtype = "";
3330 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3331 compression = "auto";
3332 if(compression not in compressionlist and compression is None):
3333 compression = "auto";
3334 if(verbose):
3335 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3336 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3337 if(os.path.exists(outfile)):
3338 try:
3339 os.unlink(outfile);
3340 except OSError as e:
3341 pass;
3342 if(outfile=="-"):
3343 verbose = False;
3344 catfp = BytesIO();
3345 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3346 catfp = outfile;
3347 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3348 catfp = BytesIO();
3349 else:
3350 fbasename = os.path.splitext(outfile)[0];
3351 fextname = os.path.splitext(outfile)[1];
3352 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3353 catver = formatspecs['format_ver'];
3354 fileheaderver = str(int(catver.replace(".", "")));
3355 curinode = 0;
3356 curfid = 0;
3357 inodelist = [];
3358 inodetofile = {};
3359 filetoinode = {};
3360 inodetocatinode = {};
3361 if(not os.path.exists(infile) or not os.path.isfile(infile)):
3362 return False;
3363 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
3364 return False;
3365 rarfp = rarfile.RarFile(infile, "r");
3366 rartest = rarfp.testrar();
3367 if(rartest):
3368 VerbosePrintOut("Bad file found!");
3369 numfiles = int(len(rarfp.infolist()));
3370 catfp = AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3371 try:
3372 catfp.flush();
3373 os.fsync(catfp.fileno());
3374 except io.UnsupportedOperation:
3375 pass;
3376 except AttributeError:
3377 pass;
3378 except OSError as e:
3379 pass;
3380 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
3381 is_unix = False;
3382 is_windows = False;
3383 if(member.host_os==rarfile.RAR_OS_UNIX):
3384 is_windows = False;
3385 try:
3386 member.external_attr
3387 is_unix = True;
3388 except AttributeError:
3389 is_unix = False;
3390 elif(member.host_os==rarfile.RAR_OS_WIN32):
3391 is_unix = False;
3392 try:
3393 member.external_attr
3394 is_windows = True;
3395 except AttributeError:
3396 is_windows = False;
3397 else:
3398 is_unix = False;
3399 is_windows = False;
3400 if(re.findall("^[.|/]", member.filename)):
3401 fname = member.filename;
3402 else:
3403 fname = "./"+member.filename;
3404 rarinfo = rarfp.getinfo(member.filename);
3405 if(verbose):
3406 VerbosePrintOut(fname);
3407 if(is_unix and member.external_attr !=0):
3408 fpremode = int(member.external_attr);
3409 elif(member.is_file()):
3410 fpremode = int(stat.S_IFREG + 438);
3411 elif(member.is_symlink()):
3412 fpremode = int(stat.S_IFLNK + 438);
3413 elif(member.is_dir()):
3414 fpremode = int(stat.S_IFDIR + 511);
3415 if(is_windows and member.external_attr !=0):
3416 fwinattributes = format(int(member.external_attr), 'x').lower();
3417 else:
3418 fwinattributes = format(int(0), 'x').lower();
3419 fcompression = "";
3420 fcsize = format(int(0), 'x').lower();
3421 flinkcount = 0;
3422 ftype = 0;
3423 if(member.is_file()):
3424 ftype = 0;
3425 elif(member.is_symlink()):
3426 ftype = 2;
3427 elif(member.is_dir()):
3428 ftype = 5;
3429 flinkname = "";
3430 if(ftype==2):
3431 flinkname = rarfp.read(member.filename).decode("UTF-8");
3432 fcurfid = format(int(curfid), 'x').lower();
3433 fcurinode = format(int(curfid), 'x').lower();
3434 curfid = curfid + 1;
3435 fdev_minor = format(int(0), 'x').lower();
3436 fdev_major = format(int(0), 'x').lower();
3437 frdev_minor = format(int(0), 'x').lower();
3438 frdev_major = format(int(0), 'x').lower();
3439 if(ftype==5):
3440 fsize = format(int("0"), 'x').lower();
3441 elif(ftype==0):
3442 fsize = format(int(member.file_size), 'x').lower();
3443 else:
3444 fsize = format(int(member.file_size), 'x').lower();
3445 try:
3446 if(member.atime):
3447 fatime = format(int(member.atime.timestamp()), 'x').lower();
3448 else:
3449 fatime = format(int(member.mtime.timestamp()), 'x').lower();
3450 except AttributeError:
3451 fatime = format(int(member.mtime.timestamp()), 'x').lower();
3452 fmtime = format(int(member.mtime.timestamp()), 'x').lower();
3453 try:
3454 if(member.ctime):
3455 fctime = format(int(member.ctime.timestamp()), 'x').lower();
3456 else:
3457 fctime = format(int(member.mtime.timestamp()), 'x').lower();
3458 except AttributeError:
3459 fctime = format(int(member.mtime.timestamp()), 'x').lower();
3460 fbtime = format(int(member.mtime.timestamp()), 'x').lower();
3461 if(is_unix and member.external_attr !=0):
3462 fmode = format(int(member.external_attr), 'x').lower();
3463 fchmode = format(int(stat.S_IMODE(member.external_attr)), 'x').lower();
3464 ftypemod = format(int(stat.S_IFMT(member.external_attr)), 'x').lower();
3465 elif(member.is_file()):
3466 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3467 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3468 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3469 elif(member.is_symlink()):
3470 fmode = format(int(stat.S_IFLNK + 438), 'x').lower();
3471 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3472 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3473 elif(member.is_dir()):
3474 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3475 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
3476 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
3477 try:
3478 fuid = format(int(os.getuid()), 'x').lower();
3479 except AttributeError:
3480 fuid = format(int(0), 'x').lower();
3481 except KeyError:
3482 fuid = format(int(0), 'x').lower();
3483 try:
3484 fgid = format(int(os.getgid()), 'x').lower();
3485 except AttributeError:
3486 fgid = format(int(0), 'x').lower();
3487 except KeyError:
3488 fgid = format(int(0), 'x').lower();
3489 try:
3490 import pwd;
3491 try:
3492 userinfo = pwd.getpwuid(os.getuid());
3493 funame = userinfo.pw_name;
3494 except KeyError:
3495 funame = "";
3496 except AttributeError:
3497 funame = "";
3498 except ImportError:
3499 funame = "";
3500 fgname = "";
3501 try:
3502 import grp;
3503 try:
3504 groupinfo = grp.getgrgid(os.getgid());
3505 fgname = groupinfo.gr_name;
3506 except KeyError:
3507 fgname = "";
3508 except AttributeError:
3509 fgname = "";
3510 except ImportError:
3511 fgname = "";
3512 fcontents = BytesIO();
3513 if(ftype==0):
3514 fcontents.write(rarfp.read(member.filename));
3515 if(not compresswholefile):
3516 fcontents.seek(0, 2);
3517 ucfsize = fcontents.tell();
3518 fcontents.seek(0, 0);
3519 if(compression=="auto"):
3520 ilsize = len(compressionlistalt);
3521 ilmin = 0;
3522 ilcsize = [];
3523 while(ilmin < ilsize):
3524 cfcontents = BytesIO();
3525 shutil.copyfileobj(fcontents, cfcontents);
3526 fcontents.seek(0, 0);
3527 cfcontents.seek(0, 0);
3528 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3529 if(cfcontents):
3530 cfcontents.seek(0, 2);
3531 ilcsize.append(cfcontents.tell());
3532 cfcontents.close();
3533 else:
3534 try:
3535 ilcsize.append(sys.maxint);
3536 except AttributeError:
3537 ilcsize.append(sys.maxsize);
3538 ilmin = ilmin + 1;
3539 ilcmin = ilcsize.index(min(ilcsize));
3540 compression = compressionlistalt[ilcmin];
3541 fcontents.seek(0, 0);
3542 cfcontents = BytesIO();
3543 shutil.copyfileobj(fcontents, cfcontents);
3544 cfcontents.seek(0, 0);
3545 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3546 cfcontents.seek(0, 2);
3547 cfsize = cfcontents.tell();
3548 if(ucfsize > cfsize):
3549 fcsize = format(int(cfsize), 'x').lower();
3550 fcompression = compression;
3551 fcontents.close();
3552 fcontents = cfcontents;
3553 if(fcompression=="none"):
3554 fcompression = "";
3555 fcontents.seek(0, 0);
3556 ftypehex = format(ftype, 'x').lower();
3557 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3558 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3559 fcontents.close();
3560 if(numfiles>0):
3561 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3562 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3563 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
3564 try:
3565 catfp.flush();
3566 os.fsync(catfp.fileno());
3567 except io.UnsupportedOperation:
3568 pass;
3569 except AttributeError:
3570 pass;
3571 except OSError as e:
3572 pass;
3573 if(outfile=="-"):
3574 catfp.seek(0, 0)
3575 if(hasattr(sys.stdout, "buffer")):
3576 shutil.copyfileobj(catfp, sys.stdout.buffer);
3577 else:
3578 shutil.copyfileobj(catfp, sys.stdout);
3579 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3580 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3581 catfp.seek(0, 0);
3582 upload_file_to_internet_file(catfp, outfile);
3583 if(returnfp):
3584 catfp.seek(0, 0)
3585 return catfp
3586 else:
3587 catfp.close()
3588 return True;
3590 create_alias_function("Pack", __file_format_name__, "FromRarFile", PackArchiveFileFromRarFile);
3592 if(not py7zr_support):
3593 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3594 return False
3596 if(py7zr_support):
3597 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3598 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3599 outfile = RemoveWindowsPath(outfile);
3600 checksumtype = checksumtype.lower();
3601 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3602 checksumtype="crc32";
3603 if(checksumtype=="none"):
3604 checksumtype = "";
3605 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
3606 compression = "auto";
3607 if(compression not in compressionlist and compression is None):
3608 compression = "auto";
3609 if(verbose):
3610 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3611 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3612 if(os.path.exists(outfile)):
3613 try:
3614 os.unlink(outfile);
3615 except OSError as e:
3616 pass;
3617 if(outfile=="-"):
3618 verbose = False;
3619 catfp = BytesIO();
3620 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3621 catfp = outfile;
3622 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3623 catfp = BytesIO();
3624 else:
3625 fbasename = os.path.splitext(outfile)[0];
3626 fextname = os.path.splitext(outfile)[1];
3627 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
3628 catver = formatspecs['format_ver'];
3629 fileheaderver = str(int(catver.replace(".", "")));
3630 curinode = 0;
3631 curfid = 0;
3632 inodelist = [];
3633 inodetofile = {};
3634 filetoinode = {};
3635 inodetocatinode = {};
3636 if(not os.path.exists(infile) or not os.path.isfile(infile)):
3637 return False;
3638 szpfp = py7zr.SevenZipFile(infile, mode="r");
3639 file_content = szpfp.readall();
3640 #sztest = szpfp.testzip();
3641 sztestalt = szpfp.test();
3642 if(sztestalt):
3643 VerbosePrintOut("Bad file found!");
3644 numfiles = int(len(szpfp.list()));
3645 AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3646 for member in sorted(szpfp.list(), key=lambda x: x.filename):
3647 if(re.findall("^[.|/]", member.filename)):
3648 fname = member.filename;
3649 else:
3650 fname = "./"+member.filename;
3651 if(verbose):
3652 VerbosePrintOut(fname);
3653 if(not member.is_directory):
3654 fpremode = int(stat.S_IFREG + 438);
3655 elif(member.is_directory):
3656 fpremode = int(stat.S_IFDIR + 511);
3657 fwinattributes = format(int(0), 'x').lower();
3658 fcompression = "";
3659 fcsize = format(int(0), 'x').lower();
3660 flinkcount = 0;
3661 ftype = 0;
3662 if(member.is_directory):
3663 ftype = 5;
3664 else:
3665 ftype = 0;
3666 flinkname = "";
3667 fcurfid = format(int(curfid), 'x').lower();
3668 fcurinode = format(int(curfid), 'x').lower();
3669 curfid = curfid + 1;
3670 fdev_minor = format(int(0), 'x').lower();
3671 fdev_major = format(int(0), 'x').lower();
3672 frdev_minor = format(int(0), 'x').lower();
3673 frdev_major = format(int(0), 'x').lower();
3674 if(ftype==5):
3675 fsize = format(int("0"), 'x').lower();
3676 fatime = format(int(member.creationtime.timestamp()), 'x').lower();
3677 fmtime = format(int(member.creationtime.timestamp()), 'x').lower();
3678 fctime = format(int(member.creationtime.timestamp()), 'x').lower();
3679 fbtime = format(int(member.creationtime.timestamp()), 'x').lower();
3680 if(member.is_directory):
3681 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3682 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
3683 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
3684 else:
3685 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3686 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3687 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3688 try:
3689 fuid = format(int(os.getuid()), 'x').lower();
3690 except AttributeError:
3691 fuid = format(int(0), 'x').lower();
3692 except KeyError:
3693 fuid = format(int(0), 'x').lower();
3694 try:
3695 fgid = format(int(os.getgid()), 'x').lower();
3696 except AttributeError:
3697 fgid = format(int(0), 'x').lower();
3698 except KeyError:
3699 fgid = format(int(0), 'x').lower();
3700 try:
3701 import pwd;
3702 try:
3703 userinfo = pwd.getpwuid(os.getuid());
3704 funame = userinfo.pw_name;
3705 except KeyError:
3706 funame = "";
3707 except AttributeError:
3708 funame = "";
3709 except ImportError:
3710 funame = "";
3711 fgname = "";
3712 try:
3713 import grp;
3714 try:
3715 groupinfo = grp.getgrgid(os.getgid());
3716 fgname = groupinfo.gr_name;
3717 except KeyError:
3718 fgname = "";
3719 except AttributeError:
3720 fgname = "";
3721 except ImportError:
3722 fgname = "";
3723 fcontents = BytesIO();
3724 if(ftype==0):
3725 fcontents.write(file_content[member.filename].read());
3726 fsize = format(fcontents.tell(), 'x').lower();
3727 file_content[member.filename].close();
3728 if(not compresswholefile):
3729 fcontents.seek(0, 2);
3730 ucfsize = fcontents.tell();
3731 fcontents.seek(0, 0);
3732 if(compression=="auto"):
3733 ilsize = len(compressionlistalt);
3734 ilmin = 0;
3735 ilcsize = [];
3736 while(ilmin < ilsize):
3737 cfcontents = BytesIO();
3738 shutil.copyfileobj(fcontents, cfcontents);
3739 fcontents.seek(0, 0);
3740 cfcontents.seek(0, 0);
3741 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
3742 if(cfcontents):
3743 cfcontents.seek(0, 2);
3744 ilcsize.append(cfcontents.tell());
3745 cfcontents.close();
3746 else:
3747 try:
3748 ilcsize.append(sys.maxint);
3749 except AttributeError:
3750 ilcsize.append(sys.maxsize);
3751 ilmin = ilmin + 1;
3752 ilcmin = ilcsize.index(min(ilcsize));
3753 compression = compressionlistalt[ilcmin];
3754 fcontents.seek(0, 0);
3755 cfcontents = BytesIO();
3756 shutil.copyfileobj(fcontents, cfcontents);
3757 cfcontents.seek(0, 0);
3758 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
3759 cfcontents.seek(0, 2);
3760 cfsize = cfcontents.tell();
3761 if(ucfsize > cfsize):
3762 fcsize = format(int(cfsize), 'x').lower();
3763 fcompression = compression;
3764 fcontents.close();
3765 fcontents = cfcontents;
3766 if(fcompression=="none"):
3767 fcompression = "";
3768 fcontents.seek(0, 0);
3769 ftypehex = format(ftype, 'x').lower();
3770 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3771 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3772 fcontents.close();
3773 if(numfiles>0):
3774 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
3775 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3776 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
3777 try:
3778 catfp.flush();
3779 os.fsync(catfp.fileno());
3780 except io.UnsupportedOperation:
3781 pass;
3782 except AttributeError:
3783 pass;
3784 except OSError as e:
3785 pass;
3786 if(outfile=="-"):
3787 catfp.seek(0, 0);
3788 if(hasattr(sys.stdout, "buffer")):
3789 shutil.copyfileobj(catfp, sys.stdout.buffer);
3790 else:
3791 shutil.copyfileobj(catfp, sys.stdout);
3792 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3793 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3794 catfp.seek(0, 0);
3795 upload_file_to_internet_file(catfp, outfile);
3796 if(returnfp):
3797 catfp.seek(0, 0)
3798 return catfp
3799 else:
3800 catfp.close()
3801 return True;
3803 create_alias_function("Pack", __file_format_name__, "FromSevenZipFile", PackArchiveFileFromSevenZipFile);
3805 def PackArchiveFileFromInFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
3806 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
3807 if(verbose):
3808 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3809 if(checkcompressfile=="tarfile"):
3810 return PackArchiveFileFromTarFile(infile, outfile, compression, compresswholefile, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3811 elif(checkcompressfile=="zipfile"):
3812 return PackArchiveFileFromZipFile(infile, outfile, compression, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3813 elif(checkcompressfile=="catfile"):
3814 return RePackArchiveFile(infile, outfile, compression, compressionlevel, False, 0, 0, checksumtype, False, extradata, formatspecs, verbose, returnfp);
3815 elif(rarfile_support and checkcompressfile=="rarfile"):
3816 return PackArchiveFileFromRarFile(infile, outfile, compression, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3817 elif(py7zr_support and checkcompressfile=="7zipfile"):
3818 return PackArchiveFileFromSevenZipFile(infile, outfile, compression, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3819 else:
3820 return False;
3821 return False;
3823 create_alias_function("Pack", __file_format_name__, "FromInFile", PackArchiveFileFromInFile);
3825 def ArchiveFileSeekToFileNum(infile, seekto=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
3826 if(hasattr(infile, "read") or hasattr(infile, "write")):
3827 catfp = infile;
3828 catfp.seek(0, 0);
3829 catfp = UncompressArchiveFile(catfp, formatspecs);
3830 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
3831 if(checkcompressfile=="tarfile"):
3832 return TarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3833 if(checkcompressfile=="zipfile"):
3834 return ZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3835 if(rarfile_support and checkcompressfile=="rarfile"):
3836 return RarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3837 if(py7zr_support and checkcompressfile=="7zipfile"):
3838 return SevenZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3839 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
3840 return False;
3841 if(not catfp):
3842 return False;
3843 catfp.seek(0, 0);
3844 elif(infile=="-"):
3845 catfp = BytesIO();
3846 if(hasattr(sys.stdin, "buffer")):
3847 shutil.copyfileobj(sys.stdin.buffer, catfp);
3848 else:
3849 shutil.copyfileobj(sys.stdin, catfp);
3850 catfp.seek(0, 0);
3851 catfp = UncompressArchiveFile(catfp, formatspecs);
3852 if(not catfp):
3853 return False;
3854 catfp.seek(0, 0);
3855 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3856 catfp = download_file_from_internet_file(infile);
3857 catfp.seek(0, 0);
3858 catfp = UncompressArchiveFile(catfp, formatspecs);
3859 if(not catfp):
3860 return False;
3861 catfp.seek(0, 0);
3862 else:
3863 infile = RemoveWindowsPath(infile);
3864 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
3865 if(checkcompressfile=="tarfile"):
3866 return TarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3867 if(checkcompressfile=="zipfile"):
3868 return ZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3869 if(rarfile_support and checkcompressfile=="rarfile"):
3870 return RarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3871 if(py7zr_support and checkcompressfile=="7zipfile"):
3872 return SevenZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3873 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
3874 return False;
3875 compresscheck = CheckCompressionType(infile, formatspecs, True);
3876 if(not compresscheck):
3877 fextname = os.path.splitext(infile)[1];
3878 if(fextname==".gz"):
3879 compresscheck = "gzip";
3880 elif(fextname==".bz2"):
3881 compresscheck = "bzip2";
3882 elif(fextname==".zst"):
3883 compresscheck = "zstd";
3884 elif(fextname==".lz4" or fextname==".clz4"):
3885 compresscheck = "lz4";
3886 elif(fextname==".lzo" or fextname==".lzop"):
3887 compresscheck = "lzo";
3888 elif(fextname==".lzma" or fextname==".xz"):
3889 compresscheck = "lzma";
3890 else:
3891 return False;
3892 if(not compresscheck):
3893 return False;
3894 catfp = UncompressFile(infile, formatspecs, "rb");
3896 try:
3897 catfp.seek(0, 2);
3898 except OSError:
3899 SeekToEndOfFile(catfp);
3900 except ValueError:
3901 SeekToEndOfFile(catfp);
3902 CatSize = catfp.tell();
3903 CatSizeEnd = CatSize;
3905 try:
3906 catfp.seek(0, 0);
3907 except OSError:
3908 return False;
3909 except ValueError:
3910 return False;
3911 curloc = catfp.tell();
3912 if(curloc>0):
3913 catfp.seek(0, 0);
3914 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
3915 if(curloc>0):
3916 catfp.seek(curloc, 0);
3917 catstring = catheader[0];
3918 catversion = re.findall(r"([\d]+)$", catstring);
3919 fprenumfiles = catheader[1];
3920 fnumfiles = int(fprenumfiles, 16);
3921 fprechecksumtype = catheader[2];
3922 fprechecksum = catheader[3];
3923 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
3924 fnumfileshex = format(int(fnumfiles), 'x').lower();
3925 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
3926 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
3927 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
3928 fheadtell = len(fileheader);
3929 if(fprechecksum!=catfileheadercshex and not skipchecksum):
3930 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
3931 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
3932 return False;
3933 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
3934 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
3935 if(seekto>=fnumfiles):
3936 seekto = fnumfiles - 1;
3937 if(seekto<0):
3938 seekto = 0;
3939 if(seekto>=0):
3940 il = -1;
3941 while(il < seekto):
3942 prefhstart = catfp.tell();
3943 if(formatspecs['new_style']):
3944 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
3945 else:
3946 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
3947 if(len(preheaderdata)==0):
3948 break;
3949 prefheadsize = int(preheaderdata[0], 16);
3950 prefnumfields = int(preheaderdata[1], 16);
3951 preftype = int(preheaderdata[2], 16);
3952 if(re.findall("^[.|/]", preheaderdata[3])):
3953 prefname = preheaderdata[3];
3954 else:
3955 prefname = "./"+preheaderdata[3];
3956 prefbasedir = os.path.dirname(prefname);
3957 preflinkname = preheaderdata[4];
3958 prefsize = int(preheaderdata[5], 16);
3959 prefatime = int(preheaderdata[6], 16);
3960 prefmtime = int(preheaderdata[7], 16);
3961 prefctime = int(preheaderdata[8], 16);
3962 prefbtime = int(preheaderdata[9], 16);
3963 prefmode = int(preheaderdata[10], 16);
3964 prefchmode = stat.S_IMODE(prefmode);
3965 preftypemod = stat.S_IFMT(prefmode);
3966 prefwinattributes = int(preheaderdata[11], 16);
3967 prefcompression = preheaderdata[12];
3968 prefcsize = int(preheaderdata[13], 16);
3969 prefuid = int(preheaderdata[14], 16);
3970 prefuname = preheaderdata[15];
3971 prefgid = int(preheaderdata[16], 16);
3972 prefgname = preheaderdata[17];
3973 fid = int(preheaderdata[18], 16);
3974 finode = int(preheaderdata[19], 16);
3975 flinkcount = int(preheaderdata[20], 16);
3976 prefdev_minor = int(preheaderdata[21], 16);
3977 prefdev_major = int(preheaderdata[22], 16);
3978 prefrdev_minor = int(preheaderdata[23], 16);
3979 prefrdev_major = int(preheaderdata[24], 16);
3980 prefextrasize = int(preheaderdata[25], 16);
3981 prefextrafields = int(preheaderdata[26], 16);
3982 extrafieldslist = [];
3983 extrastart = 27;
3984 extraend = extrastart + prefextrafields;
3985 extrafieldslist = [];
3986 if(extrastart<extraend):
3987 extrafieldslist.append(preheaderdata[extrastart]);
3988 extrastart = extrastart + 1;
3989 prefchecksumtype = preheaderdata[extrastart].lower();
3990 prefcs = preheaderdata[extrastart + 1].lower();
3991 prefccs = preheaderdata[extrastart + 2].lower();
3992 hc = 0;
3993 hcmax = len(preheaderdata) - 2;
3994 hout = "";
3995 while(hc<hcmax):
3996 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs['format_delimiter']);
3997 hc = hc + 1;
3998 prenewfcs = GetFileChecksum(hout, preheaderdata[-3].lower(), True, formatspecs);
3999 if(prefcs!=prenewfcs and not skipchecksum):
4000 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4001 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4002 return False;
4003 valid_archive = False;
4004 invalid_archive = True;
4005 prefhend = catfp.tell() - 1;
4006 prefcontentstart = catfp.tell();
4007 prefcontents = "";
4008 pyhascontents = False;
4009 if(prefsize>0):
4010 if(prefcompression):
4011 prefcontents = catfp.read(prefsize);
4012 else:
4013 prefcontents = catfp.read(prefcsize);
4014 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4015 pyhascontents = True;
4016 if(prefccs!=prenewfccs and not skipchecksum):
4017 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4018 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4019 return False;
4020 catfp.seek(1, 1);
4021 il = il + 1;
4022 catfp.seek(seekstart, 0);
4023 fileidnum = il;
4024 catfheadsize = int(preheaderdata[0], 16);
4025 catfnumfields = int(preheaderdata[1], 16);
4026 catftype = int(preheaderdata[2], 16);
4027 if(re.findall("^[.|/]", preheaderdata[3])):
4028 catfname = preheaderdata[3];
4029 else:
4030 catfname = "./"+preheaderdata[3];
4031 catflinkname = preheaderdata[4];
4032 catfsize = int(preheaderdata[5], 16);
4033 catfbasedir = os.path.dirname(catfname);
4034 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
4035 if(returnfp):
4036 catlist.update({'catfp': catfp});
4037 else:
4038 catfp.close();
4039 return catlist;
4041 create_alias_function("", __file_format_name__, "SeekToFileNum", ArchiveFileSeekToFileNum);
4043 def ArchiveFileSeekToFileName(infile, seekfile=None, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4044 if(hasattr(infile, "read") or hasattr(infile, "write")):
4045 catfp = infile;
4046 catfp.seek(0, 0);
4047 catfp = UncompressArchiveFile(catfp, formatspecs);
4048 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4049 if(checkcompressfile=="tarfile"):
4050 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4051 if(checkcompressfile=="zipfile"):
4052 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4053 if(rarfile_support and checkcompressfile=="rarfile"):
4054 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4055 if(py7zr_support and checkcompressfile=="7zipfile"):
4056 return SevenZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4057 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4058 return False;
4059 if(not catfp):
4060 return False;
4061 catfp.seek(0, 0);
4062 elif(infile=="-"):
4063 catfp = BytesIO();
4064 if(hasattr(sys.stdin, "buffer")):
4065 shutil.copyfileobj(sys.stdin.buffer, catfp);
4066 else:
4067 shutil.copyfileobj(sys.stdin, catfp);
4068 catfp.seek(0, 0);
4069 catfp = UncompressArchiveFile(catfp, formatspecs);
4070 if(not catfp):
4071 return False;
4072 catfp.seek(0, 0);
4073 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4074 catfp = download_file_from_internet_file(infile);
4075 catfp = UncompressArchiveFile(catfp, formatspecs);
4076 catfp.seek(0, 0);
4077 if(not catfp):
4078 return False;
4079 catfp.seek(0, 0);
4080 else:
4081 infile = RemoveWindowsPath(infile);
4082 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4083 if(checkcompressfile=="tarfile"):
4084 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4085 if(checkcompressfile=="zipfile"):
4086 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4087 if(rarfile_support and checkcompressfile=="rarfile"):
4088 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4089 if(py7zr_support and checkcompressfile=="7zipfile"):
4090 return SevenZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
4091 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4092 return False;
4093 compresscheck = CheckCompressionType(infile, formatspecs, True);
4094 if(not compresscheck):
4095 fextname = os.path.splitext(infile)[1];
4096 if(fextname==".gz"):
4097 compresscheck = "gzip";
4098 elif(fextname==".bz2"):
4099 compresscheck = "bzip2";
4100 elif(fextname==".zst"):
4101 compresscheck = "zstd";
4102 elif(fextname==".lz4" or fextname==".clz4"):
4103 compresscheck = "lz4";
4104 elif(fextname==".lzo" or fextname==".lzop"):
4105 compresscheck = "lzo";
4106 elif(fextname==".lzma" or fextname==".xz"):
4107 compresscheck = "lzma";
4108 else:
4109 return False;
4110 if(not compresscheck):
4111 return False;
4112 catfp = UncompressFile(infile, formatspecs, "rb");
4114 try:
4115 catfp.seek(0, 2);
4116 except OSError:
4117 SeekToEndOfFile(catfp);
4118 except ValueError:
4119 SeekToEndOfFile(catfp);
4120 CatSize = catfp.tell();
4121 CatSizeEnd = CatSize;
4123 try:
4124 catfp.seek(0, 0);
4125 except OSError:
4126 return False;
4127 except ValueError:
4128 return False;
4129 curloc = catfp.tell();
4130 if(curloc>0):
4131 catfp.seek(0, 0);
4132 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4133 if(curloc>0):
4134 catfp.seek(curloc, 0);
4135 catstring = catheader[0];
4136 catversion = re.findall(r"([\d]+)$", catstring);
4137 fprenumfiles = catheader[1];
4138 fnumfiles = int(fprenumfiles, 16);
4139 fprechecksumtype = catheader[2];
4140 fprechecksum = catheader[3];
4141 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4142 fnumfileshex = format(int(fnumfiles), 'x').lower();
4143 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4144 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4145 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4146 fheadtell = len(fileheader);
4147 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4148 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
4149 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4150 return False;
4151 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
4152 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4153 seekto = fnumfiles - 1
4154 filefound = False;
4155 if(seekto>=0):
4156 il = -1;
4157 while(il < seekto):
4158 prefhstart = catfp.tell();
4159 if(formatspecs['new_style']):
4160 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4161 else:
4162 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4163 if(len(preheaderdata)==0):
4164 break;
4165 prefheadsize = int(preheaderdata[0], 16);
4166 prefnumfields = int(preheaderdata[1], 16);
4167 preftype = int(preheaderdata[2], 16);
4168 if(re.findall("^[.|/]", preheaderdata[3])):
4169 prefname = preheaderdata[3];
4170 else:
4171 prefname = "./"+preheaderdata[3];
4172 prefbasedir = os.path.dirname(prefname);
4173 preflinkname = preheaderdata[4];
4174 prefsize = int(preheaderdata[5], 16);
4175 prefatime = int(preheaderdata[6], 16);
4176 prefmtime = int(preheaderdata[7], 16);
4177 prefctime = int(preheaderdata[8], 16);
4178 prefbtime = int(preheaderdata[9], 16);
4179 prefmode = int(preheaderdata[10], 16);
4180 prefchmode = stat.S_IMODE(prefmode);
4181 preftypemod = stat.S_IFMT(prefmode);
4182 prefwinattributes = int(preheaderdata[11], 16);
4183 prefcompression = preheaderdata[12];
4184 prefcsize = int(preheaderdata[13], 16);
4185 prefuid = int(preheaderdata[14], 16);
4186 prefuname = preheaderdata[15];
4187 prefgid = int(preheaderdata[16], 16);
4188 prefgname = preheaderdata[17];
4189 fid = int(preheaderdata[18], 16);
4190 finode = int(preheaderdata[19], 16);
4191 flinkcount = int(preheaderdata[20], 16);
4192 prefdev_minor = int(preheaderdata[21], 16);
4193 prefdev_major = int(preheaderdata[22], 16);
4194 prefrdev_minor = int(preheaderdata[23], 16);
4195 prefrdev_major = int(preheaderdata[24], 16);
4196 prefextrasize = int(preheaderdata[25], 16);
4197 prefextrafields = int(preheaderdata[26], 16);
4198 extrafieldslist = [];
4199 extrastart = 27;
4200 extraend = extrastart + prefextrafields;
4201 extrafieldslist = [];
4202 if(extrastart<extraend):
4203 extrafieldslist.append(preheaderdata[extrastart]);
4204 extrastart = extrastart + 1;
4205 prefchecksumtype = preheaderdata[extrastart].lower();
4206 prefcs = preheaderdata[extrastart + 1].lower();
4207 prefccs = preheaderdata[extrastart + 2].lower();
4208 hc = 0;
4209 hcmax = len(preheaderdata) - 2;
4210 hout = "";
4211 while(hc<hcmax):
4212 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs['format_delimiter']);
4213 hc = hc + 1;
4214 prenewfcs = GetFileChecksum(hout, preheaderdata[-3].lower(), True, formatspecs);
4215 if(prefcs!=prenewfcs and not skipchecksum):
4216 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4217 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4218 return False;
4219 valid_archive = False;
4220 invalid_archive = True;
4221 prefhend = catfp.tell() - 1;
4222 prefcontentstart = catfp.tell();
4223 prefcontents = "";
4224 pyhascontents = False;
4225 if(prefsize>0):
4226 if(prefcompression):
4227 prefcontents = catfp.read(prefsize);
4228 else:
4229 prefcontents = catfp.read(prefcsize);
4230 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4231 pyhascontents = True;
4232 if(prefccs!=prenewfccs and not skipchecksum):
4233 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4234 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4235 return False;
4236 catfp.seek(1, 1);
4237 il = il + 1;
4238 filefound = False;
4239 prefname = preheaderdata[2];
4240 if(re.findall("^[.|/]", preheaderdata[2])):
4241 prefname = preheaderdata[2];
4242 else:
4243 prefname = "./"+preheaderdata[2];
4244 if(prefname==seekfile):
4245 filefound = True;
4246 break;
4247 catfp.seek(seekstart, 0);
4248 fileidnum = il;
4249 catfheadsize = int(preheaderdata[0], 16);
4250 catfnumfields = int(preheaderdata[1], 16);
4251 catftype = int(preheaderdata[2], 16);
4252 if(re.findall("^[.|/]", preheaderdata[3])):
4253 catfname = preheaderdata[3];
4254 else:
4255 catfname = "./"+preheaderdata[3];
4256 catflinkname = preheaderdata[4];
4257 catfsize = int(preheaderdata[5], 16);
4258 catfbasedir = os.path.dirname(catfname);
4259 if(filefound):
4260 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
4261 else:
4262 return False;
4263 if(returnfp):
4264 catlist.update({'catfp': catfp});
4265 else:
4266 catfp.close();
4267 return catlist;
4269 create_alias_function("", __file_format_name__, "SeekToFileName", ArchiveFileSeekToFileName);
4271 def ArchiveFileValidate(infile, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
4272 if(verbose):
4273 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
4274 if(hasattr(infile, "read") or hasattr(infile, "write")):
4275 catfp = infile;
4276 catfp.seek(0, 0);
4277 catfp = UncompressArchiveFile(catfp, formatspecs);
4278 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4279 if(checkcompressfile=="tarfile"):
4280 return TarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4281 if(checkcompressfile=="zipfile"):
4282 return ZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4283 if(rarfile_support and checkcompressfile=="rarfile"):
4284 return RarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4285 if(py7zr_support and checkcompressfile=="7zipfile"):
4286 return SevenZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4287 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4288 return False;
4289 if(not catfp):
4290 return False;
4291 catfp.seek(0, 0);
4292 elif(infile=="-"):
4293 catfp = BytesIO();
4294 if(hasattr(sys.stdin, "buffer")):
4295 shutil.copyfileobj(sys.stdin.buffer, catfp);
4296 else:
4297 shutil.copyfileobj(sys.stdin, catfp);
4298 catfp.seek(0, 0);
4299 catfp = UncompressArchiveFile(catfp, formatspecs);
4300 if(not catfp):
4301 return False;
4302 catfp.seek(0, 0);
4303 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4304 catfp = download_file_from_internet_file(infile);
4305 catfp = UncompressArchiveFile(catfp, formatspecs);
4306 catfp.seek(0, 0);
4307 if(not catfp):
4308 return False;
4309 catfp.seek(0, 0);
4310 else:
4311 infile = RemoveWindowsPath(infile);
4312 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4313 if(checkcompressfile=="tarfile"):
4314 return TarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4315 if(checkcompressfile=="zipfile"):
4316 return ZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4317 if(rarfile_support and checkcompressfile=="rarfile"):
4318 return RarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4319 if(py7zr_support and checkcompressfile=="7zipfile"):
4320 return SevenZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4321 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4322 return False;
4323 compresscheck = CheckCompressionType(infile, formatspecs, True);
4324 if(not compresscheck):
4325 fextname = os.path.splitext(infile)[1];
4326 if(fextname==".gz"):
4327 compresscheck = "gzip";
4328 elif(fextname==".bz2"):
4329 compresscheck = "bzip2";
4330 elif(fextname==".zst"):
4331 compresscheck = "zstd";
4332 elif(fextname==".lz4" or fextname==".clz4"):
4333 compresscheck = "lz4";
4334 elif(fextname==".lzo" or fextname==".lzop"):
4335 compresscheck = "lzo";
4336 elif(fextname==".lzma" or fextname==".xz"):
4337 compresscheck = "lzma";
4338 else:
4339 return False;
4340 if(not compresscheck):
4341 return False;
4342 catfp = UncompressFile(infile, formatspecs, "rb");
4344 try:
4345 catfp.seek(0, 2);
4346 except OSError:
4347 SeekToEndOfFile(catfp);
4348 except ValueError:
4349 SeekToEndOfFile(catfp);
4350 CatSize = catfp.tell();
4351 CatSizeEnd = CatSize;
4353 try:
4354 catfp.seek(0, 0);
4355 except OSError:
4356 return False;
4357 except ValueError:
4358 return False;
4359 curloc = catfp.tell();
4360 if(curloc>0):
4361 catfp.seek(0, 0);
4362 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4363 if(curloc>0):
4364 catfp.seek(curloc, 0);
4365 catstring = catheader[0];
4366 catversion = re.findall(r"([\d]+)$", catstring);
4367 fprenumfiles = catheader[1];
4368 fnumfiles = int(fprenumfiles, 16);
4369 fprechecksumtype = catheader[2];
4370 fprechecksum = catheader[3];
4371 il = 0;
4372 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4373 fnumfileshex = format(int(fnumfiles), 'x').lower();
4374 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4375 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4376 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4377 valid_archive = True;
4378 invalid_archive = False;
4379 if(verbose):
4380 VerbosePrintOut(infile);
4381 VerbosePrintOut("Number of Records " + str(fnumfiles));
4382 if(fprechecksum==catfileheadercshex):
4383 if(verbose):
4384 VerbosePrintOut("File Header Checksum Passed at offset " + str(0));
4385 VerbosePrintOut("'" + str(fprechecksum) + "' == " + "'" + str(catfileheadercshex) + "'");
4386 else:
4387 if(verbose):
4388 VerbosePrintOut("File Header Checksum Failed at offset " + str(0));
4389 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4390 valid_archive = False;
4391 invalid_archive = True;
4392 if(verbose):
4393 VerbosePrintOut("");
4394 while(il<fnumfiles):
4395 catfhstart = catfp.tell();
4396 if(formatspecs['new_style']):
4397 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4398 else:
4399 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4400 if(len(catheaderdata)==0):
4401 break;
4402 catfheadsize = int(catheaderdata[0], 16);
4403 catfnumfields = int(catheaderdata[1], 16);
4404 catftype = int(catheaderdata[2], 16);
4405 if(re.findall("^[.|/]", catheaderdata[3])):
4406 catfname = catheaderdata[3];
4407 else:
4408 catfname = "./"+catheaderdata[3];
4409 catfbasedir = os.path.dirname(catfname);
4410 catflinkname = catheaderdata[4];
4411 catfsize = int(catheaderdata[5], 16);
4412 catfatime = int(catheaderdata[6], 16);
4413 catfmtime = int(catheaderdata[7], 16);
4414 catfctime = int(catheaderdata[8], 16);
4415 catfbtime = int(catheaderdata[9], 16);
4416 catfmode = int(catheaderdata[10], 16);
4417 catfchmode = stat.S_IMODE(catfmode);
4418 catftypemod = stat.S_IFMT(catfmode);
4419 catfwinattributes = int(catheaderdata[11], 16);
4420 catfcompression = catheaderdata[12];
4421 catfcsize = int(catheaderdata[13], 16);
4422 catfuid = int(catheaderdata[14], 16);
4423 catfuname = catheaderdata[15];
4424 catfgid = int(catheaderdata[16], 16);
4425 catfgname = catheaderdata[17];
4426 fid = int(catheaderdata[18], 16);
4427 finode = int(catheaderdata[19], 16);
4428 flinkcount = int(catheaderdata[20], 16);
4429 catfdev_minor = int(catheaderdata[21], 16);
4430 catfdev_major = int(catheaderdata[22], 16);
4431 catfrdev_minor = int(catheaderdata[23], 16);
4432 catfrdev_major = int(catheaderdata[24], 16);
4433 catfextrasize = int(catheaderdata[25], 16);
4434 catfextrafields = int(catheaderdata[26], 16);
4435 extrafieldslist = [];
4436 extrastart = 27;
4437 extraend = extrastart + catfextrafields;
4438 extrafieldslist = [];
4439 if(extrastart<extraend):
4440 extrafieldslist.append(catheaderdata[extrastart]);
4441 extrastart = extrastart + 1;
4442 catfchecksumtype = catheaderdata[extrastart].lower();
4443 catfcs = catheaderdata[extrastart + 1].lower();
4444 catfccs = catheaderdata[extrastart + 2].lower();
4445 hc = 0;
4446 hcmax = len(catheaderdata) - 2;
4447 hout = "";
4448 while(hc<hcmax):
4449 hout = hout + AppendNullByte(catheaderdata[hc], formatspecs['format_delimiter']);
4450 hc = hc + 1;
4451 catnewfcs = GetFileChecksum(hout, catheaderdata[-3].lower(), True, formatspecs);
4452 if(verbose):
4453 VerbosePrintOut(catfname);
4454 VerbosePrintOut("Record Number " + str(il) + "; File ID " + str(fid) + "; iNode Number " + str(finode));
4455 if(catfcs==catnewfcs):
4456 if(verbose):
4457 VerbosePrintOut("File Header Checksum Passed at offset " + str(catfhstart));
4458 VerbosePrintOut("'" + str(catfcs) + "' == " + "'" + str(catnewfcs) + "'");
4459 else:
4460 if(verbose):
4461 VerbosePrintOut("File Header Checksum Failed at offset " + str(catfhstart));
4462 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
4463 valid_archive = False;
4464 invalid_archive = True;
4465 catfhend = catfp.tell() - 1;
4466 catfcontentstart = catfp.tell();
4467 catfcontents = "";
4468 pyhascontents = False;
4469 if(catfsize>0):
4470 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4471 catfcontents = catfp.read(catfsize);
4472 else:
4473 catfcontents = catfp.read(catfcsize);
4474 catnewfccs = GetFileChecksum(catfcontents, catheaderdata[-3].lower(), False, formatspecs);
4475 pyhascontents = True;
4476 if(catfccs==catnewfccs):
4477 if(verbose):
4478 VerbosePrintOut("File Content Checksum Passed at offset " + str(catfcontentstart));
4479 VerbosePrintOut("'" + str(catfccs) + "' == " + "'" + str(catnewfccs) + "'");
4480 else:
4481 if(verbose):
4482 VerbosePrintOut("File Content Checksum Failed at offset " + str(catfcontentstart));
4483 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
4484 valid_archive = False;
4485 invalid_archive = True;
4486 if(verbose):
4487 VerbosePrintOut("");
4488 catfp.seek(1, 1);
4489 il = il + 1;
4490 if(valid_archive):
4491 if(returnfp):
4492 return catfp;
4493 else:
4494 catfp.close();
4495 return True;
4496 else:
4497 catfp.close();
4498 return False;
4500 create_alias_function("", __file_format_name__, "Validate", ArchiveFileValidate);
4502 def ArchiveFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4503 if(hasattr(infile, "read") or hasattr(infile, "write")):
4504 catfp = infile;
4505 catfp.seek(0, 0);
4506 catfp = UncompressArchiveFile(catfp, formatspecs);
4507 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4508 if(checkcompressfile=="tarfile"):
4509 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4510 if(checkcompressfile=="zipfile"):
4511 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4512 if(rarfile_support and checkcompressfile=="rarfile"):
4513 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4514 if(py7zr_support and checkcompressfile=="7zipfile"):
4515 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4516 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4517 return False;
4518 if(not catfp):
4519 return False;
4520 catfp.seek(0, 0);
4521 elif(infile=="-"):
4522 catfp = BytesIO();
4523 if(hasattr(sys.stdin, "buffer")):
4524 shutil.copyfileobj(sys.stdin.buffer, catfp);
4525 else:
4526 shutil.copyfileobj(sys.stdin, catfp);
4527 catfp.seek(0, 0);
4528 catfp = UncompressArchiveFile(catfp, formatspecs);
4529 if(not catfp):
4530 return False;
4531 catfp.seek(0, 0);
4532 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4533 catfp = download_file_from_internet_file(infile);
4534 catfp = UncompressArchiveFile(catfp, formatspecs);
4535 catfp.seek(0, 0);
4536 if(not catfp):
4537 return False;
4538 catfp.seek(0, 0);
4539 else:
4540 infile = RemoveWindowsPath(infile);
4541 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4542 if(checkcompressfile=="tarfile"):
4543 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4544 if(checkcompressfile=="zipfile"):
4545 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4546 if(rarfile_support and checkcompressfile=="rarfile"):
4547 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4548 if(py7zr_support and checkcompressfile=="7zipfile"):
4549 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4550 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs['format_lower']):
4551 return False;
4552 compresscheck = CheckCompressionType(infile, formatspecs, True);
4553 if(not compresscheck):
4554 fextname = os.path.splitext(infile)[1];
4555 if(fextname==".gz"):
4556 compresscheck = "gzip";
4557 elif(fextname==".bz2"):
4558 compresscheck = "bzip2";
4559 elif(fextname==".zst"):
4560 compresscheck = "zstd";
4561 elif(fextname==".lz4" or fextname==".clz4"):
4562 compresscheck = "lz4";
4563 elif(fextname==".lzo" or fextname==".lzop"):
4564 compresscheck = "lzo";
4565 elif(fextname==".lzma" or fextname==".xz"):
4566 compresscheck = "lzma";
4567 else:
4568 return False;
4569 if(not compresscheck):
4570 return False;
4571 catfp = UncompressFile(infile, formatspecs, "rb");
4573 try:
4574 catfp.seek(0, 2);
4575 except OSError:
4576 SeekToEndOfFile(catfp);
4577 except ValueError:
4578 SeekToEndOfFile(catfp);
4579 CatSize = catfp.tell();
4580 CatSizeEnd = CatSize;
4582 try:
4583 catfp.seek(0, 0);
4584 except OSError:
4585 return False;
4586 except ValueError:
4587 return False;
4588 curloc = catfp.tell();
4589 if(curloc>0):
4590 catfp.seek(0, 0);
4591 catheader = ReadFileHeaderData(catfp, 4, formatspecs['format_delimiter']);
4592 if(curloc>0):
4593 catfp.seek(curloc, 0);
4594 catstring = catheader[0];
4595 catversion = re.findall(r"([\d]+)$", catstring);
4596 fprenumfiles = catheader[1];
4597 fnumfiles = int(fprenumfiles, 16);
4598 fprechecksumtype = catheader[2];
4599 fprechecksum = catheader[3];
4600 fileheader = AppendNullByte(catstring, formatspecs['format_delimiter']);
4601 fnumfileshex = format(int(fnumfiles), 'x').lower();
4602 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs['format_delimiter']);
4603 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4604 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4605 fheadtell = len(fileheader);
4606 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4607 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
4608 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4609 return False;
4610 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
4611 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4612 if(seekstart<0 and seekstart>fnumfiles):
4613 seekstart = 0;
4614 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
4615 seekend = fnumfiles;
4616 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
4617 seekend = fnumfiles - abs(seekend);
4618 if(seekstart>0):
4619 il = 0;
4620 while(il < seekstart):
4621 prefhstart = catfp.tell();
4622 if(formatspecs['new_style']):
4623 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4624 else:
4625 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4626 if(len(preheaderdata)==0):
4627 break;
4628 prefheadsize = int(preheaderdata[0], 16);
4629 prefnumfields = int(preheaderdata[1], 16);
4630 if(re.findall("^[.|/]", preheaderdata[3])):
4631 prefname = preheaderdata[3];
4632 else:
4633 prefname = "./"+preheaderdata[3];
4634 prefsize = int(preheaderdata[5], 16);
4635 prefcompression = preheaderdata[12];
4636 prefcsize = int(preheaderdata[13], 16);
4637 prefextrasize = int(preheaderdata[24], 16);
4638 prefextrafields = int(preheaderdata[25], 16);
4639 extrafieldslist = [];
4640 extrastart = 26;
4641 extraend = extrastart + prefextrafields;
4642 extrafieldslist = [];
4643 if(extrastart<extraend):
4644 extrafieldslist.append(preheaderdata[extrastart]);
4645 extrastart = extrastart + 1;
4646 prefchecksumtype = preheaderdata[extrastart].lower();
4647 prefcs = preheaderdata[extrastart + 1].lower();
4648 prefccs = preheaderdata[extrastart + 2].lower();
4649 hc = 0;
4650 hcmax = len(preheaderdata) - 2;
4651 hout = "";
4652 while(hc<hcmax):
4653 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs['format_delimiter']);
4654 hc = hc + 1;
4655 prenewfcs = GetFileChecksum(hout, preheaderdata[-3].lower(), True, formatspecs);
4656 if(prefcs!=prenewfcs and not skipchecksum):
4657 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4658 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4659 return False;
4660 valid_archive = False;
4661 invalid_archive = True;
4662 prefhend = catfp.tell() - 1;
4663 prefcontentstart = catfp.tell();
4664 prefcontents = "";
4665 pyhascontents = False;
4666 if(prefsize>0):
4667 if(prefcompression=="none" or prefcompression=="" or prefcompression=="auto"):
4668 prefcontents = catfp.read(prefsize);
4669 else:
4670 prefcontents = catfp.read(prefcsize);
4671 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4672 pyhascontents = True;
4673 if(prefccs!=prenewfccs and not skipchecksum):
4674 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4675 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4676 return False;
4677 catfp.seek(1, 1);
4678 il = il + 1;
4679 fileidnum = seekstart;
4680 realidnum = 0;
4681 while(fileidnum<seekend):
4682 catfhstart = catfp.tell();
4683 if(formatspecs['new_style']):
4684 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs['format_delimiter']);
4685 else:
4686 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs['format_delimiter']);
4687 if(len(catheaderdata)==0):
4688 break;
4689 catfheadsize = int(catheaderdata[0], 16);
4690 catfnumfields = int(catheaderdata[1], 16);
4691 catftype = int(catheaderdata[2], 16);
4692 if(re.findall("^[.|/]", catheaderdata[3])):
4693 catfname = catheaderdata[3];
4694 else:
4695 catfname = "./"+catheaderdata[3];
4696 catfbasedir = os.path.dirname(catfname);
4697 catflinkname = catheaderdata[4];
4698 catfsize = int(catheaderdata[5], 16);
4699 catfatime = int(catheaderdata[6], 16);
4700 catfmtime = int(catheaderdata[7], 16);
4701 catfctime = int(catheaderdata[8], 16);
4702 catfbtime = int(catheaderdata[9], 16);
4703 catfmode = int(catheaderdata[10], 16);
4704 catfchmode = stat.S_IMODE(catfmode);
4705 catftypemod = stat.S_IFMT(catfmode);
4706 catfwinattributes = int(catheaderdata[11], 16);
4707 catfcompression = catheaderdata[12];
4708 catfcsize = int(catheaderdata[13], 16);
4709 catfuid = int(catheaderdata[14], 16);
4710 catfuname = catheaderdata[15];
4711 catfgid = int(catheaderdata[16], 16);
4712 catfgname = catheaderdata[17];
4713 catfid = int(catheaderdata[18], 16);
4714 catfinode = int(catheaderdata[19], 16);
4715 catflinkcount = int(catheaderdata[20], 16);
4716 catfdev_minor = int(catheaderdata[21], 16);
4717 catfdev_major = int(catheaderdata[22], 16);
4718 catfrdev_minor = int(catheaderdata[23], 16);
4719 catfrdev_major = int(catheaderdata[24], 16);
4720 catfextrasize = int(catheaderdata[25], 16);
4721 catfextrafields = int(catheaderdata[26], 16);
4722 extrafieldslist = [];
4723 extrastart = 27;
4724 extraend = extrastart + catfextrafields;
4725 extrafieldslist = [];
4726 if(extrastart<extraend):
4727 extrafieldslist.append(catheaderdata[extrastart]);
4728 extrastart = extrastart + 1;
4729 catfchecksumtype = catheaderdata[extrastart].lower();
4730 catfcs = catheaderdata[extrastart + 1].lower();
4731 catfccs = catheaderdata[extrastart + 2].lower();
4732 hc = 0;
4733 hcmax = len(catheaderdata) - 2;
4734 hout = "";
4735 while(hc<hcmax):
4736 hout = hout + AppendNullByte(catheaderdata[hc], formatspecs['format_delimiter']);
4737 hc = hc + 1;
4738 catnewfcs = GetFileChecksum(hout, catheaderdata[-3].lower(), True, formatspecs);
4739 if(catfcs!=catnewfcs and not skipchecksum):
4740 VerbosePrintOut("File Header Checksum Error with file " + catfname + " at offset " + str(catfhstart));
4741 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
4742 return False;
4743 catfhend = catfp.tell() - 1;
4744 catfcontentstart = catfp.tell();
4745 catfcontents = BytesIO();
4746 pyhascontents = False;
4747 if(catfsize>0 and not listonly):
4748 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4749 catfcontents.write(catfp.read(catfsize));
4750 else:
4751 catfcontents.write(catfp.read(catfcsize));
4752 catfcontents.seek(0, 0);
4753 catnewfccs = GetFileChecksum(catfcontents.read(), catheaderdata[-3].lower(), False, formatspecs);
4754 pyhascontents = True;
4755 if(catfccs!=catnewfccs and skipchecksum):
4756 VerbosePrintOut("File Content Checksum Error with file " + catfname + " at offset " + str(catfcontentstart));
4757 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
4758 return False;
4759 if(catfcompression=="none" or catfcompression=="" or catfcompression=="auto"):
4760 pass;
4761 else:
4762 catfcontents.seek(0, 0);
4763 catfcontents = UncompressArchiveFile(catfcontents, formatspecs);
4764 if(catfsize>0 and listonly):
4765 catfp.seek(catfsize, 1);
4766 pyhascontents = False;
4767 catfp.seek(1, 1);
4768 catfcontentend = catfp.tell() - 1;
4769 catfcontents.seek(0, 0);
4770 catlist['ffilelist'].update({realidnum: {'fid': realidnum, 'fidalt': fileidnum, 'fheadersize': catfheadsize, 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize, 'fatime': catfatime, 'fmtime': catfmtime, 'fctime': catfctime, 'fbtime': catfbtime, 'fmode': catfmode, 'fchmode': catfchmode, 'ftypemod': catftypemod, 'fwinattributes': catfwinattributes, 'fcompression': catfcompression, 'fcsize': catfcsize, 'fuid': catfuid, 'funame': catfuname, 'fgid': catfgid, 'fgname': catfgname, 'finode': catfinode, 'flinkcount': catflinkcount, 'fminor': catfdev_minor, 'fmajor': catfdev_major, 'frminor': catfrdev_minor, 'frmajor': catfrdev_major, 'fchecksumtype': catfchecksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': catfextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': catfcs, 'fcontentchecksum': catfccs, 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': catfcontents} });
4771 fileidnum = fileidnum + 1;
4772 realidnum = realidnum + 1;
4773 if(returnfp):
4774 catlist.update({'catfp': catfp});
4775 else:
4776 catfp.close();
4777 return catlist;
4779 create_alias_function("", __file_format_name__, "ToArray", ArchiveFileToArray);
4781 def ArchiveFileStringToArray(catstr, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4782 catfp = BytesIO(catstr);
4783 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4784 return listcatfiles;
4786 create_alias_function("", __file_format_name__, "StringToArray", ArchiveFileStringToArray);
4788 def TarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4789 catfp = BytesIO();
4790 catfp = PackArchiveFileFromTarFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
4791 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4792 return listcatfiles;
4794 def ZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4795 catfp = BytesIO();
4796 catfp = PackArchiveFileFromZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
4797 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4798 return listcatfiles;
4800 if(not rarfile_support):
4801 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4802 return False;
4804 if(rarfile_support):
4805 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4806 catfp = BytesIO();
4807 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
4808 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4809 return listcatfiles;
4811 if(not py7zr_support):
4812 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4813 return False;
4815 if(py7zr_support):
4816 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4817 catfp = BytesIO();
4818 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
4819 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4820 return listcatfiles;
4822 def InFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
4823 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4824 if(checkcompressfile=="tarfile"):
4825 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4826 elif(checkcompressfile=="zipfile"):
4827 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4828 elif(checkcompressfile=="catfile"):
4829 return ArchiveFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4830 elif(rarfile_support and checkcompressfile=="rarfile"):
4831 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4832 elif(py7zr_support and checkcompressfile=="7zipfile"):
4833 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4834 else:
4835 return False;
4836 return False;
4838 def ListDirToArrayAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
4839 catver = formatspecs['format_ver'];
4840 fileheaderver = str(int(catver.replace(".", "")));
4841 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
4842 advancedlist = formatspecs['use_advanced_list'];
4843 altinode = formatspecs['use_alt_inode'];
4844 infilelist = [];
4845 if(infiles=="-"):
4846 for line in sys.stdin:
4847 infilelist.append(line.strip());
4848 infilelist = list(filter(None, infilelist));
4849 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
4850 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
4851 return False;
4852 with UncompressFile(infiles, formatspecs, "r") as finfile:
4853 for line in finfile:
4854 infilelist.append(line.strip());
4855 infilelist = list(filter(None, infilelist));
4856 else:
4857 if(isinstance(infiles, (list, tuple, ))):
4858 infilelist = list(filter(None, infiles));
4859 elif(isinstance(infiles, (str, ))):
4860 infilelist = list(filter(None, [infiles]));
4861 if(advancedlist):
4862 GetDirList = ListDirAdvanced(infilelist, followlink, False);
4863 else:
4864 GetDirList = ListDir(infilelist, followlink, False);
4865 if(not GetDirList):
4866 return False;
4867 curinode = 0;
4868 curfid = 0;
4869 inodelist = [];
4870 inodetofile = {};
4871 filetoinode = {};
4872 inodetocatinode = {};
4873 fileidnum = 0;
4874 fnumfiles = int(len(GetDirList));
4875 catver = formatspecs['format_ver'];
4876 fileheaderver = str(int(catver.replace(".", "")));
4877 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
4878 fnumfileshex = format(int(fnumfiles), 'x').lower();
4879 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
4880 catversion = re.findall(r"([\d]+)$", fileheader);
4881 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
4882 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4883 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
4884 fheadtell = len(fileheader);
4885 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
4886 for curfname in GetDirList:
4887 if(re.findall("^[.|/]", curfname)):
4888 fname = curfname;
4889 else:
4890 fname = "./"+curfname;
4891 if(verbose):
4892 VerbosePrintOut(fname);
4893 if(not followlink or followlink is None):
4894 fstatinfo = os.lstat(fname);
4895 else:
4896 fstatinfo = os.stat(fname);
4897 fpremode = fstatinfo.st_mode;
4898 finode = fstatinfo.st_ino;
4899 flinkcount = fstatinfo.st_nlink;
4900 ftype = 0;
4901 if(stat.S_ISREG(fpremode)):
4902 ftype = 0;
4903 elif(stat.S_ISLNK(fpremode)):
4904 ftype = 2;
4905 elif(stat.S_ISCHR(fpremode)):
4906 ftype = 3;
4907 elif(stat.S_ISBLK(fpremode)):
4908 ftype = 4;
4909 elif(stat.S_ISDIR(fpremode)):
4910 ftype = 5;
4911 elif(stat.S_ISFIFO(fpremode)):
4912 ftype = 6;
4913 elif(stat.S_ISSOCK(fpremode)):
4914 ftype = 8;
4915 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
4916 ftype = 9;
4917 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
4918 ftype = 10;
4919 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
4920 ftype = 11;
4921 else:
4922 ftype = 0;
4923 flinkname = "";
4924 fbasedir = os.path.dirname(fname);
4925 fcurfid = curfid;
4926 if(not followlink and finode!=0):
4927 if(ftype!=1):
4928 if(finode in inodelist):
4929 ftype = 1;
4930 flinkname = inodetofile[finode];
4931 if(altinode):
4932 fcurinode = finode;
4933 else:
4934 fcurinode = inodetocatinode[finode];
4935 if(finode not in inodelist):
4936 inodelist.append(finode);
4937 inodetofile.update({finode: fname});
4938 inodetocatinode.update({finode: curinode});
4939 if(altinode):
4940 fcurinode = finode;
4941 else:
4942 fcurinode = curinode;
4943 curinode = curinode + 1;
4944 else:
4945 fcurinode = curinode;
4946 curinode = curinode + 1;
4947 curfid = curfid + 1;
4948 if(ftype==2):
4949 flinkname = os.readlink(fname);
4950 fdev = fstatinfo.st_dev;
4951 getfdev = GetDevMajorMinor(fdev);
4952 fdev_minor = getfdev[0];
4953 fdev_major = getfdev[1];
4954 frdev = fstatinfo.st_dev;
4955 if(hasattr(fstatinfo, "st_rdev")):
4956 frdev = fstatinfo.st_rdev;
4957 else:
4958 frdev = fstatinfo.st_dev;
4959 getfrdev = GetDevMajorMinor(frdev);
4960 frdev_minor = getfrdev[0];
4961 frdev_major = getfrdev[1];
4962 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
4963 fsize = "0";
4964 if(ftype==0 or ftype==7):
4965 fsize = fstatinfo.st_size;
4966 fatime = fstatinfo.st_atime;
4967 fmtime = fstatinfo.st_mtime;
4968 fctime = fstatinfo.st_ctime;
4969 if(hasattr(fstatinfo, "st_birthtime")):
4970 fbtime = fstatinfo.st_birthtime;
4971 else:
4972 fbtime = fstatinfo.st_ctime;
4973 fmode = fstatinfo.st_mode;
4974 fchmode = stat.S_IMODE(fstatinfo.st_mode);
4975 ftypemod = stat.S_IFMT(fstatinfo.st_mode);
4976 fuid = fstatinfo.st_uid;
4977 fgid = fstatinfo.st_gid;
4978 funame = "";
4979 try:
4980 import pwd;
4981 try:
4982 userinfo = pwd.getpwuid(fstatinfo.st_uid);
4983 funame = userinfo.pw_name;
4984 except KeyError:
4985 funame = "";
4986 except ImportError:
4987 funame = "";
4988 fgname = "";
4989 try:
4990 import grp;
4991 try:
4992 groupinfo = grp.getgrgid(fstatinfo.st_gid);
4993 fgname = groupinfo.gr_name;
4994 except KeyError:
4995 fgname = "";
4996 except ImportError:
4997 fgname = "";
4998 fdev_minor = fdev_minor;
4999 fdev_major = fdev_major;
5000 frdev_minor = frdev_minor;
5001 frdev_major = frdev_major;
5002 flinkcount = flinkcount;
5003 if(hasattr(fstatinfo, "st_file_attributes")):
5004 fwinattributes = fstatinfo.st_file_attributes;
5005 else:
5006 fwinattributes = 0;
5007 fcompression = "";
5008 fcsize = 0;
5009 fcontents = BytesIO();
5010 if(ftype==0 or ftype==7):
5011 with open(fname, "rb") as fpc:
5012 shutil.copyfileobj(fpc, fcontents);
5013 if(followlink and (ftype==1 or ftype==2)):
5014 flstatinfo = os.stat(flinkname);
5015 with open(flinkname, "rb") as fpc:
5016 shutil.copyfileobj(fpc, fcontents);
5017 fcontents.seek(0, 0);
5018 ftypehex = format(ftype, 'x').lower();
5019 extrafields = len(extradata);
5020 extrafieldslist = extradata;
5021 catfextrafields = extrafields;
5022 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5023 if(len(extradata)>0):
5024 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5025 extrasizelen = len(extrasizestr);
5026 extrasizelenhex = format(extrasizelen, 'x').lower();
5027 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5028 catoutlen = len(catoutlist) + len(extradata) + 3;
5029 catoutlenhex = format(catoutlen, 'x').lower();
5030 catoutlist.insert(0, catoutlenhex);
5031 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5032 if(len(extradata)>0):
5033 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5034 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
5035 catfnumfields = catoutlen;
5036 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5037 fcontents.seek(0, 0);
5038 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5039 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5040 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5041 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5042 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5043 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5044 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5045 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5046 fheadtell += len(catfileoutstr) + 1;
5047 catfcontentend = fheadtell - 1;
5048 fcontents.seek(0, 0);
5049 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5050 pyhascontents = False;
5051 if(int(fsize)>0 and not listonly):
5052 pyhascontents = True;
5053 if(int(fsize)>0 and listonly):
5054 fcontents = BytesIO();
5055 pyhascontents = False;
5056 fcontents.seek(0, 0);
5057 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5058 fileidnum = fileidnum + 1;
5059 return catlist;
5061 def TarFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5062 curinode = 0;
5063 curfid = 0;
5064 inodelist = [];
5065 inodetofile = {};
5066 filetoinode = {};
5067 inodetocatinode = {};
5068 fileidnum = 0;
5069 if(infile=="-"):
5070 infile = BytesIO();
5071 if(hasattr(sys.stdin, "buffer")):
5072 shutil.copyfileobj(sys.stdin.buffer, infile);
5073 else:
5074 shutil.copyfileobj(sys.stdin, infile);
5075 infile.seek(0, 0);
5076 if(not infile):
5077 return False;
5078 infile.seek(0, 0);
5079 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
5080 infile = download_file_from_internet_file(infile);
5081 infile.seek(0, 0);
5082 if(not infile):
5083 return False;
5084 infile.seek(0, 0);
5085 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
5086 return False;
5087 elif(os.path.exists(infile) and os.path.isfile(infile)):
5088 try:
5089 if(not tarfile.is_tarfile(infile)):
5090 return False;
5091 except AttributeError:
5092 if(not is_tarfile(infile)):
5093 return False;
5094 else:
5095 return False;
5096 try:
5097 if(hasattr(infile, "read") or hasattr(infile, "write")):
5098 tarfp = tarfile.open(fileobj=infile, mode="r");
5099 else:
5100 tarfp = tarfile.open(infile, "r");
5101 except FileNotFoundError:
5102 return False;
5103 fnumfiles = int(len(tarfp.getmembers()));
5104 catver = formatspecs['format_ver'];
5105 fileheaderver = str(int(catver.replace(".", "")));
5106 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5107 fnumfileshex = format(int(fnumfiles), 'x').lower();
5108 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5109 catversion = re.findall(r"([\d]+)$", fileheader);
5110 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
5111 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
5112 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5113 fheadtell = len(fileheader);
5114 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5115 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
5116 if(re.findall("^[.|/]", member.name)):
5117 fname = member.name;
5118 else:
5119 fname = "./"+member.name;
5120 if(verbose):
5121 VerbosePrintOut(fname);
5122 fpremode = member.mode;
5123 ffullmode = member.mode;
5124 flinkcount = 0;
5125 ftype = 0;
5126 if(member.isreg()):
5127 ffullmode = member.mode + stat.S_IFREG;
5128 ftype = 0;
5129 elif(member.isdev()):
5130 ffullmode = member.mode;
5131 ftype = 7;
5132 elif(member.islnk()):
5133 ffullmode = member.mode + stat.S_IFREG;
5134 ftype = 1;
5135 elif(member.issym()):
5136 ffullmode = member.mode + stat.S_IFLNK;
5137 ftype = 2;
5138 elif(member.ischr()):
5139 ffullmode = member.mode + stat.S_IFCHR;
5140 ftype = 3;
5141 elif(member.isblk()):
5142 ffullmode = member.mode + stat.S_IFBLK;
5143 ftype = 4;
5144 elif(member.isdir()):
5145 ffullmode = member.mode + stat.S_IFDIR;
5146 ftype = 5;
5147 elif(member.isfifo()):
5148 ffullmode = member.mode + stat.S_IFIFO;
5149 ftype = 6;
5150 elif(member.issparse()):
5151 ffullmode = member.mode;
5152 ftype = 12;
5153 else:
5154 ffullmode = member.mode;
5155 ftype = 0;
5156 flinkname = "";
5157 fbasedir = os.path.dirname(fname);
5158 fcurfid = curfid;
5159 fcurinode = curfid;
5160 finode = fcurinode;
5161 curfid = curfid + 1;
5162 if(ftype==2):
5163 flinkname = member.linkname;
5164 fdev_minor = member.devminor;
5165 fdev_major = member.devmajor;
5166 frdev_minor = member.devminor;
5167 frdev_major = member.devmajor;
5168 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
5169 fsize = "0";
5170 elif(ftype==0 or ftype==7):
5171 fsize = member.size;
5172 else:
5173 fsize = member.size;
5174 fatime = member.mtime;
5175 fmtime = member.mtime;
5176 fctime = member.mtime;
5177 fbtime = member.mtime;
5178 fmode = ffullmode;
5179 fchmode = stat.S_IMODE(ffullmode);
5180 ftypemod = stat.S_IFMT(ffullmode);
5181 fuid = member.uid;
5182 fgid = member.gid;
5183 funame = member.uname;
5184 fgname = member.gname;
5185 flinkcount = flinkcount;
5186 fwinattributes = int(0);
5187 fcompression = "";
5188 fcsize = 0;
5189 fcontents = BytesIO();
5190 if(ftype==0 or ftype==7):
5191 with tarfp.extractfile(member) as fpc:
5192 shutil.copyfileobj(fpc, fcontents);
5193 fcontents.seek(0, 0);
5194 ftypehex = format(ftype, 'x').lower();
5195 extrafields = len(extradata);
5196 extrafieldslist = extradata;
5197 catfextrafields = extrafields;
5198 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5199 if(len(extradata)>0):
5200 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5201 extrasizelen = len(extrasizestr);
5202 extrasizelenhex = format(extrasizelen, 'x').lower();
5203 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5204 catoutlen = len(catoutlist) + len(extradata) + 3;
5205 catoutlenhex = format(catoutlen, 'x').lower();
5206 catoutlist.insert(0, catoutlenhex);
5207 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5208 if(len(extradata)>0):
5209 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5210 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
5211 catfnumfields = catoutlen;
5212 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5213 fcontents.seek(0, 0);
5214 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5215 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5216 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5217 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5218 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5219 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5220 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5221 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5222 fheadtell += len(catfileoutstr) + 1;
5223 catfcontentend = fheadtell - 1;
5224 fcontents.seek(0, 0);
5225 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5226 pyhascontents = False;
5227 if(int(fsize)>0 and not listonly):
5228 pyhascontents = True;
5229 if(int(fsize)>0 and listonly):
5230 fcontents = "";
5231 pyhascontents = False;
5232 fcontents.seek(0, 0);
5233 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5234 fileidnum = fileidnum + 1;
5235 return catlist;
5237 def ZipFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5238 curinode = 0;
5239 curfid = 0;
5240 inodelist = [];
5241 inodetofile = {};
5242 filetoinode = {};
5243 inodetocatinode = {};
5244 fileidnum = 0;
5245 if(infile=="-"):
5246 infile = BytesIO();
5247 if(hasattr(sys.stdin, "buffer")):
5248 shutil.copyfileobj(sys.stdin.buffer, infile);
5249 else:
5250 shutil.copyfileobj(sys.stdin, infile);
5251 infile.seek(0, 0);
5252 if(not infile):
5253 return False;
5254 infile.seek(0, 0);
5255 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
5256 infile = download_file_from_internet_file(infile);
5257 infile.seek(0, 0);
5258 if(not infile):
5259 return False;
5260 infile.seek(0, 0);
5261 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
5262 return False;
5263 else:
5264 return False;
5265 if(not zipfile.is_zipfile(infile)):
5266 return False;
5267 try:
5268 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
5269 except FileNotFoundError:
5270 return False;
5271 ziptest = zipfp.testzip();
5272 if(ziptest):
5273 VerbosePrintOut("Bad file found!");
5274 fnumfiles = int(len(zipfp.infolist()));
5275 catver = formatspecs['format_ver'];
5276 fileheaderver = str(int(catver.replace(".", "")));
5277 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5278 catversion = re.findall(r"([\d]+)$", fileheader);
5279 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
5280 fnumfileshex = format(int(fnumfiles), 'x').lower();
5281 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5282 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
5283 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5284 fheadtell = len(fileheader);
5285 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5286 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
5287 if(re.findall("^[.|/]", member.filename)):
5288 fname = member.filename;
5289 else:
5290 fname = "./"+member.filename;
5291 zipinfo = zipfp.getinfo(member.filename);
5292 if(verbose):
5293 VerbosePrintOut(fname);
5294 if(not member.is_dir()):
5295 fpremode = stat.S_IFREG + 438;
5296 elif(member.is_dir()):
5297 fpremode = stat.S_IFDIR + 511;
5298 flinkcount = 0;
5299 ftype = 0;
5300 if(not member.is_dir()):
5301 ftype = 0;
5302 elif(member.is_dir()):
5303 ftype = 5;
5304 flinkname = "";
5305 fbasedir = os.path.dirname(fname);
5306 fcurfid = curfid;
5307 fcurinode = curfid;
5308 finode = fcurinode;
5309 curfid = curfid + 1;
5310 fdev_minor = 0;
5311 fdev_major = 0;
5312 frdev_minor = 0;
5313 frdev_major = 0;
5314 if(ftype==5):
5315 fsize = "0";
5316 elif(ftype==0):
5317 fsize = member.file_size;
5318 else:
5319 fsize = member.file_size;
5320 fatime = time.mktime(member.date_time + (0, 0, -1));
5321 fmtime = time.mktime(member.date_time + (0, 0, -1));
5322 fctime = time.mktime(member.date_time + (0, 0, -1));
5323 fbtime = time.mktime(member.date_time + (0, 0, -1));
5324 if(zipinfo.create_system==0 or zipinfo.create_system==10):
5325 fwinattributes = int(zipinfo.external_attr);
5326 if(not member.is_dir()):
5327 fmode = int(stat.S_IFREG + 438);
5328 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
5329 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
5330 elif(member.is_dir()):
5331 fmode = int(stat.S_IFDIR + 511);
5332 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
5333 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
5334 elif(zipinfo.create_system==3):
5335 fwinattributes = int(0);
5336 fmode = int(zipinfo.external_attr);
5337 else:
5338 fwinattributes = int(0);
5339 if(not member.is_dir()):
5340 fmode = int(stat.S_IFREG + 438);
5341 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
5342 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
5343 elif(member.is_dir()):
5344 fmode = int(stat.S_IFDIR + 511);
5345 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
5346 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
5347 fcompression = "";
5348 fcsize = 0;
5349 try:
5350 fuid = os.getuid();
5351 except AttributeError:
5352 fuid = 0;
5353 except KeyError:
5354 fuid = 0;
5355 try:
5356 fgid = os.getgid();
5357 except AttributeError:
5358 fgid = 0;
5359 except KeyError:
5360 fgid = 0;
5361 try:
5362 import pwd;
5363 try:
5364 userinfo = pwd.getpwuid(os.getuid());
5365 funame = userinfo.pw_name;
5366 except KeyError:
5367 funame = "";
5368 except AttributeError:
5369 funame = "";
5370 except ImportError:
5371 funame = "";
5372 fgname = "";
5373 try:
5374 import grp;
5375 try:
5376 groupinfo = grp.getgrgid(os.getgid());
5377 fgname = groupinfo.gr_name;
5378 except KeyError:
5379 fgname = "";
5380 except AttributeError:
5381 fgname = "";
5382 except ImportError:
5383 fgname = "";
5384 fcontents = BytesIO();
5385 if(ftype==0):
5386 fcontents.write(zipfp.read(member.filename));
5387 fcontents.seek(0, 0);
5388 ftypehex = format(ftype, 'x').lower();
5389 extrafields = len(extradata);
5390 extrafieldslist = extradata;
5391 catfextrafields = extrafields;
5392 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5393 if(len(extradata)>0):
5394 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5395 extrasizelen = len(extrasizestr);
5396 extrasizelenhex = format(extrasizelen, 'x').lower();
5397 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5398 catoutlen = len(catoutlist) + len(extradata) + 3;
5399 catoutlenhex = format(catoutlen, 'x').lower();
5400 catoutlist.insert(0, catoutlenhex);
5401 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5402 if(len(extradata)>0):
5403 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5404 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
5405 catfnumfields = catoutlen;
5406 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5407 fcontents.seek(0, 0);
5408 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5409 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5410 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5411 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5412 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5413 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5414 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5415 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5416 fheadtell += len(catfileoutstr) + 1;
5417 catfcontentend = fheadtell - 1;
5418 fcontents.seek(0, 0);
5419 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5420 pyhascontents = False;
5421 if(int(fsize)>0 and not listonly):
5422 pyhascontents = True;
5423 if(int(fsize)>0 and listonly):
5424 fcontents = BytesIO();
5425 pyhascontents = False;
5426 fcontents.seek(0, 0);
5427 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5428 fileidnum = fileidnum + 1;
5429 return catlist;
5431 if(not rarfile_support):
5432 def RarFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5433 return False;
5435 if(rarfile_support):
5436 def RarFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5437 curinode = 0;
5438 curfid = 0;
5439 inodelist = [];
5440 inodetofile = {};
5441 filetoinode = {};
5442 inodetocatinode = {};
5443 fileidnum = 0;
5444 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
5445 return False;
5446 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
5447 return False;
5448 rarfp = rarfile.RarFile(infile, "r");
5449 rartest = rarfp.testrar();
5450 if(rartest):
5451 VerbosePrintOut("Bad file found!");
5452 fnumfiles = int(len(rarfp.infolist()));
5453 catver = formatspecs['format_ver'];
5454 fileheaderver = str(int(catver.replace(".", "")));
5455 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5456 catversion = re.findall(r"([\d]+)$", fileheader);
5457 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
5458 fnumfileshex = format(int(fnumfiles), 'x').lower();
5459 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5460 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
5461 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5462 fheadtell = len(fileheader);
5463 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5464 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
5465 is_unix = False;
5466 is_windows = False;
5467 if(member.host_os==rarfile.RAR_OS_UNIX):
5468 is_windows = False;
5469 try:
5470 member.external_attr
5471 is_unix = True;
5472 except AttributeError:
5473 is_unix = False;
5474 elif(member.host_os==rarfile.RAR_OS_WIN32):
5475 is_unix = False;
5476 try:
5477 member.external_attr
5478 is_windows = True;
5479 except AttributeError:
5480 is_windows = False;
5481 else:
5482 is_unix = False;
5483 is_windows = False;
5484 if(re.findall("^[.|/]", member.filename)):
5485 fname = member.filename;
5486 else:
5487 fname = "./"+member.filename;
5488 rarinfo = rarfp.getinfo(member.filename);
5489 if(verbose):
5490 VerbosePrintOut(fname);
5491 if(is_unix and member.external_attr !=0):
5492 fpremode = int(member.external_attr);
5493 elif(member.is_file()):
5494 fpremode = stat.S_IFREG + 438;
5495 elif(member.is_symlink()):
5496 fpremode = stat.S_IFLNK + 438;
5497 elif(member.is_dir()):
5498 fpremode = stat.S_IFDIR + 511;
5499 if(is_windows and member.external_attr !=0):
5500 fwinattributes = int(member.external_attr);
5501 else:
5502 fwinattributes = int(0);
5503 fcompression = "";
5504 fcsize = 0;
5505 flinkcount = 0;
5506 ftype = 0;
5507 if(member.is_file()):
5508 ftype = 0;
5509 elif(member.is_symlink()):
5510 ftype = 2;
5511 elif(member.is_dir()):
5512 ftype = 5;
5513 flinkname = "";
5514 if(ftype==2):
5515 flinkname = rarfp.read(member.filename).decode("UTF-8");
5516 fbasedir = os.path.dirname(fname);
5517 fcurfid = curfid;
5518 fcurinode = curfid;
5519 finode = fcurinode;
5520 curfid = curfid + 1;
5521 fdev_minor = 0;
5522 fdev_major = 0;
5523 frdev_minor = 0;
5524 frdev_major = 0;
5525 if(ftype==5):
5526 fsize = "0";
5527 if(ftype==0):
5528 fsize = member.file_size;
5529 try:
5530 if(member.atime):
5531 fatime = int(member.atime.timestamp());
5532 else:
5533 fatime = int(member.mtime.timestamp());
5534 except AttributeError:
5535 fatime = int(member.mtime.timestamp());
5536 fmtime = int(member.mtime.timestamp());
5537 try:
5538 if(member.ctime):
5539 fctime = int(member.ctime.timestamp());
5540 else:
5541 fctime = int(member.mtime.timestamp());
5542 except AttributeError:
5543 fctime = int(member.mtime.timestamp());
5544 fbtime = int(member.mtime.timestamp());
5545 if(is_unix and member.external_attr !=0):
5546 fmode = int(member.external_attr);
5547 fchmode = int(stat.S_IMODE(member.external_attr));
5548 ftypemod = int(stat.S_IFMT(member.external_attr));
5549 elif(member.is_file()):
5550 fmode = int(stat.S_IFREG + 438)
5551 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5552 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5553 elif(member.is_symlink()):
5554 fmode = int(stat.S_IFLNK + 438)
5555 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5556 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5557 elif(member.is_dir()):
5558 fmode = int(stat.S_IFDIR + 511)
5559 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
5560 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
5561 try:
5562 fuid = os.getuid();
5563 except AttributeError:
5564 fuid = 0;
5565 except KeyError:
5566 fuid = 0;
5567 try:
5568 fgid = os.getgid();
5569 except AttributeError:
5570 fgid = 0;
5571 except KeyError:
5572 fgid = 0;
5573 try:
5574 import pwd;
5575 try:
5576 userinfo = pwd.getpwuid(os.getuid());
5577 funame = userinfo.pw_name;
5578 except KeyError:
5579 funame = "";
5580 except AttributeError:
5581 funame = "";
5582 except ImportError:
5583 funame = "";
5584 fgname = "";
5585 try:
5586 import grp;
5587 try:
5588 groupinfo = grp.getgrgid(os.getgid());
5589 fgname = groupinfo.gr_name;
5590 except KeyError:
5591 fgname = "";
5592 except AttributeError:
5593 fgname = "";
5594 except ImportError:
5595 fgname = "";
5596 fcontents = BytesIO();
5597 if(ftype==0):
5598 fcontents.write(rarfp.read(member.filename));
5599 fcontents.seek(0, 0);
5600 ftypehex = format(ftype, 'x').lower();
5601 extrafields = len(extradata);
5602 extrafieldslist = extradata;
5603 catfextrafields = extrafields;
5604 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5605 if(len(extradata)>0):
5606 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5607 extrasizelen = len(extrasizestr);
5608 extrasizelenhex = format(extrasizelen, 'x').lower();
5609 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5610 catoutlen = len(catoutlist) + len(extradata) + 3;
5611 catoutlenhex = format(catoutlen, 'x').lower();
5612 catoutlist.insert(0, catoutlenhex);
5613 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5614 if(len(extradata)>0):
5615 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5616 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
5617 catfnumfields = 24 + catfextrafields;
5618 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5619 fcontents.seek(0, 0);
5620 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5621 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5622 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5623 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5624 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5625 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5626 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5627 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5628 fheadtell += len(catfileoutstr) + 1;
5629 catfcontentend = fheadtell - 1;
5630 fcontents.seek(0, 0);
5631 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5632 pyhascontents = False;
5633 if(int(fsize)>0 and not listonly):
5634 pyhascontents = True;
5635 if(int(fsize)>0 and listonly):
5636 fcontents = BytesIO();
5637 pyhascontents = False;
5638 fcontents.seek(0, 0);
5639 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5640 fileidnum = fileidnum + 1;
5641 return catlist;
5643 if(not py7zr_support):
5644 def SevenZipFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5645 return False;
5647 if(py7zr_support):
5648 def SevenZipFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5649 curinode = 0;
5650 curfid = 0;
5651 inodelist = [];
5652 inodetofile = {};
5653 filetoinode = {};
5654 inodetocatinode = {};
5655 fileidnum = 0;
5656 szpfp = py7zr.SevenZipFile(infile, mode="r");
5657 file_content = szpfp.readall();
5658 #sztest = szpfp.testzip();
5659 sztestalt = szpfp.test();
5660 if(sztestalt):
5661 VerbosePrintOut("Bad file found!");
5662 numfiles = int(len(szpfp.list()));
5663 catver = formatspecs['format_ver'];
5664 fileheaderver = str(int(catver.replace(".", "")));
5665 fileheader = AppendNullByte(formatspecs['format_magic'] + fileheaderver, formatspecs['format_delimiter']);
5666 catversion = re.findall(r"([\d]+)$", fileheader);
5667 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
5668 fnumfileshex = format(int(fnumfiles), 'x').lower();
5669 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs['format_delimiter']);
5670 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
5671 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs['format_delimiter']);
5672 fheadtell = len(fileheader);
5673 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5674 for member in sorted(szpfp.list(), key=lambda x: x.filename):
5675 if(re.findall("^[.|/]", member.filename)):
5676 fname = member.filename;
5677 else:
5678 fname = "./"+member.filename;
5679 if(not member.is_directory):
5680 fpremode = int(stat.S_IFREG + 438);
5681 elif(member.is_directory):
5682 fpremode = int(stat.S_IFDIR + 511);
5683 fwinattributes = int(0);
5684 fcompression = "";
5685 fcsize = 0;
5686 flinkcount = 0;
5687 ftype = 0;
5688 if(member.is_directory):
5689 ftype = 5;
5690 else:
5691 ftype = 0;
5692 flinkname = "";
5693 fbasedir = os.path.dirname(fname);
5694 fcurfid = curfid;
5695 fcurinode = curfid;
5696 finode = fcurinode;
5697 curfid = curfid + 1;
5698 fdev_minor = 0;
5699 fdev_major = 0;
5700 frdev_minor = 0;
5701 frdev_major = 0;
5702 if(ftype==5):
5703 fsize = "0";
5704 fatime = int(member.creationtime.timestamp());
5705 fmtime = int(member.creationtime.timestamp());
5706 fctime = int(member.creationtime.timestamp());
5707 fbtime = int(member.creationtime.timestamp());
5708 if(member.is_directory):
5709 fmode = int(stat.S_IFDIR + 511)
5710 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
5711 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
5712 else:
5713 fmode = int(stat.S_IFLNK + 438)
5714 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5715 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5716 try:
5717 fuid = os.getuid();
5718 except AttributeError:
5719 fuid = 0;
5720 except KeyError:
5721 fuid = 0;
5722 try:
5723 fgid = os.getgid();
5724 except AttributeError:
5725 fgid = 0;
5726 except KeyError:
5727 fgid = 0;
5728 try:
5729 import pwd;
5730 try:
5731 userinfo = pwd.getpwuid(os.getuid());
5732 funame = userinfo.pw_name;
5733 except KeyError:
5734 funame = "";
5735 except AttributeError:
5736 funame = "";
5737 except ImportError:
5738 funame = "";
5739 fgname = "";
5740 try:
5741 import grp;
5742 try:
5743 groupinfo = grp.getgrgid(os.getgid());
5744 fgname = groupinfo.gr_name;
5745 except KeyError:
5746 fgname = "";
5747 except AttributeError:
5748 fgname = "";
5749 except ImportError:
5750 fgname = "";
5751 fcontents = BytesIO();
5752 if(ftype==0):
5753 fcontents.write(file_content[member.filename].read());
5754 fsize = format(fcontents.tell(), 'x').lower();
5755 fileop.close();
5756 fcontents.seek(0, 0);
5757 ftypehex = format(ftype, 'x').lower();
5758 extrafields = len(extradata);
5759 extrafieldslist = extradata;
5760 catfextrafields = extrafields;
5761 extrasizestr = AppendNullByte(extrafields, formatspecs['format_delimiter']);
5762 if(len(extradata)>0):
5763 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5764 extrasizelen = len(extrasizestr);
5765 extrasizelenhex = format(extrasizelen, 'x').lower();
5766 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5767 catoutlen = len(catoutlist) + len(extradata) + 3;
5768 catoutlenhex = format(catoutlen, 'x').lower();
5769 catoutlist.insert(0, catoutlenhex);
5770 catfileoutstr = AppendNullBytes(catoutlist, formatspecs['format_delimiter']);
5771 if(len(extradata)>0):
5772 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs['format_delimiter']);
5773 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs['format_delimiter']);
5774 catfnumfields = 24 + catfextrafields;
5775 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5776 fcontents.seek(0, 0);
5777 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5778 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5779 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5780 catfileoutstr = AppendNullByte(catheaersize, formatspecs['format_delimiter']) + catfileoutstr;
5781 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5782 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs['format_delimiter']);
5783 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5784 nullstrecd = formatspecs['format_delimiter'].encode('UTF-8');
5785 fheadtell += len(catfileoutstr) + 1;
5786 catfcontentend = fheadtell - 1;
5787 fcontents.seek(0, 0);
5788 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5789 pyhascontents = False;
5790 if(int(fsize)>0 and not listonly):
5791 pyhascontents = True;
5792 if(int(fsize)>0 and listonly):
5793 fcontents = "";
5794 pyhascontents = False;
5795 fcontents.seek(0, 0);
5796 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5797 fileidnum = fileidnum + 1;
5798 return catlist;
5800 def InFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5801 checkcompressfile = CheckCompressionSubType(infiles, formatspecs, True);
5802 if(checkcompressfile=="tarfile"):
5803 return TarFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5804 elif(checkcompressfile=="zipfile"):
5805 return ZipFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5806 elif(checkcompressfile=="catfile"):
5807 return ArchiveFileToArray(infile, 0, 0, listonly, False, formatspecs, False);
5808 elif(rarfile_support and checkcompressfile=="rarfile"):
5809 return RarFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5810 elif(py7zr_support and checkcompressfile=="7zipfile"):
5811 return SevenZipFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5812 else:
5813 return False;
5814 return False;
5816 def ListDirToArray(infiles, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, listonly=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
5817 outarray = BytesIO();
5818 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
5819 listcatfiles = ArchiveFileToArray(outarray, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5820 return listcatfiles;
5822 def ArchiveFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
5823 if(isinstance(infile, dict)):
5824 listcatfiles = infile;
5825 else:
5826 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
5827 infile = RemoveWindowsPath(infile);
5828 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5829 if(not listcatfiles):
5830 return False;
5831 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5832 if(returnfp):
5833 catarray.update({'catfp': listcatfiles['catfp']});
5834 lenlist = len(listcatfiles['ffilelist']);
5835 lcfi = 0;
5836 lcfx = int(listcatfiles['fnumfiles']);
5837 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5838 lcfx = int(lenlist);
5839 else:
5840 lcfx = int(listcatfiles['fnumfiles']);
5841 while(lcfi < lcfx):
5842 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5843 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5844 catarray['filetoid'].update(filetoidarray);
5845 catarray['idtofile'].update(idtofilearray);
5846 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5847 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5848 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5849 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5850 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5851 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5852 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5853 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5854 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5855 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5856 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5857 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5858 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5859 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5860 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5861 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5862 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5863 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5864 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5865 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5866 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5867 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5868 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5869 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5870 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5871 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5872 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
5873 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5874 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5875 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5876 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5877 lcfi = lcfi + 1;
5878 return catarray;
5880 create_alias_function("", __file_format_name__, "ToArrayIndex", ArchiveFileToArrayIndex);
5882 def ListDirToArrayIndexAlt(infiles, dirlistfromtxt=False, followlink=False, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5883 listcatfiles = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, listonly, checksumtype, extradata, formatspecs, verbose);
5884 if(not listcatfiles):
5885 return False;
5886 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5887 lenlist = len(listcatfiles['ffilelist']);
5888 lcfi = 0;
5889 lcfx = int(listcatfiles['fnumfiles']);
5890 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5891 lcfx = int(lenlist);
5892 else:
5893 lcfx = int(listcatfiles['fnumfiles']);
5894 while(lcfi < lcfx):
5895 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5896 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5897 catarray['filetoid'].update(filetoidarray);
5898 catarray['idtofile'].update(idtofilearray);
5899 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5900 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5901 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5902 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5903 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5904 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5905 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5906 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5907 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5908 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5909 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5910 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5911 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5912 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5913 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5914 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5915 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5916 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5917 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5918 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5919 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5920 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5921 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5922 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5923 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5924 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5925 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
5926 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5927 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5928 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5929 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5930 lcfi = lcfi + 1;
5931 return catarray;
5933 def TarFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5934 listcatfiles = TarFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5935 if(not listcatfiles):
5936 return False;
5937 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5938 lenlist = len(listcatfiles['ffilelist']);
5939 lcfi = 0;
5940 lcfx = int(listcatfiles['fnumfiles']);
5941 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5942 lcfx = int(lenlist);
5943 else:
5944 lcfx = int(listcatfiles['fnumfiles']);
5945 while(lcfi < lcfx):
5946 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5947 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5948 catarray['filetoid'].update(filetoidarray);
5949 catarray['idtofile'].update(idtofilearray);
5950 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5951 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5952 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5953 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5954 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5955 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5956 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5957 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5958 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5959 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5960 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5961 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5962 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5963 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5964 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5965 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5966 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5967 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5968 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5969 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5970 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5971 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5972 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5973 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5974 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5975 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5976 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
5977 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5978 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5979 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5980 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5981 lcfi = lcfi + 1;
5982 return catarray;
5984 def ZipFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
5985 listcatfiles = ZipFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5986 if(not listcatfiles):
5987 return False;
5988 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5989 lenlist = len(listcatfiles['ffilelist']);
5990 lcfi = 0;
5991 lcfx = int(listcatfiles['fnumfiles']);
5992 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5993 lcfx = int(lenlist);
5994 else:
5995 lcfx = int(listcatfiles['fnumfiles']);
5996 while(lcfi < lcfx):
5997 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5998 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5999 catarray['filetoid'].update(filetoidarray);
6000 catarray['idtofile'].update(idtofilearray);
6001 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6002 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6003 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6004 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6005 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6006 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6007 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6008 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6009 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6010 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6011 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6012 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6013 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6014 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6015 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6016 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6017 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6018 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6019 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6020 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6021 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6022 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6023 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6024 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6025 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6026 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6027 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6028 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6029 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6030 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6031 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6032 lcfi = lcfi + 1;
6033 return catarray;
6035 if(not rarfile_support):
6036 def RarFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6037 return False;
6039 if(rarfile_support):
6040 def RarFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6041 listcatfiles = RarFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
6042 if(not listcatfiles):
6043 return False;
6044 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6045 lenlist = len(listcatfiles['ffilelist']);
6046 lcfi = 0;
6047 lcfx = int(listcatfiles['fnumfiles']);
6048 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6049 lcfx = int(lenlist);
6050 else:
6051 lcfx = int(listcatfiles['fnumfiles']);
6052 while(lcfi < lcfx):
6053 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6054 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6055 catarray['filetoid'].update(filetoidarray);
6056 catarray['idtofile'].update(idtofilearray);
6057 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6058 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6059 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6060 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6061 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6062 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6063 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6064 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6065 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6066 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6067 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6068 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6069 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6070 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6071 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6072 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6073 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6074 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6075 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6076 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6077 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6078 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6079 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6080 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6081 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6082 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6083 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6084 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6085 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6086 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6087 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6088 lcfi = lcfi + 1;
6089 return catarray;
6091 if(not py7zr_support):
6092 def SevenZipFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6093 return False;
6095 if(py7zr_support):
6096 def SevenZipFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False):
6097 listcatfiles = SevenZipFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
6098 if(not listcatfiles):
6099 return False;
6100 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
6101 lenlist = len(listcatfiles['ffilelist']);
6102 lcfi = 0;
6103 lcfx = int(listcatfiles['fnumfiles']);
6104 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6105 lcfx = int(lenlist);
6106 else:
6107 lcfx = int(listcatfiles['fnumfiles']);
6108 while(lcfi < lcfx):
6109 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
6110 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
6111 catarray['filetoid'].update(filetoidarray);
6112 catarray['idtofile'].update(idtofilearray);
6113 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6114 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
6115 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
6116 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6117 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
6118 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
6119 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6120 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6121 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6122 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6123 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6124 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
6125 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
6126 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
6127 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
6128 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
6129 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6130 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6131 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
6132 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
6133 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
6134 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6135 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6136 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6137 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
6138 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
6139 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
6140 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
6141 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
6142 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
6143 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
6144 lcfi = lcfi + 1;
6145 return catarray;
6147 def ArchiveFileStringToArrayIndex(catstr, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6148 catfp = BytesIO(catstr);
6149 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
6150 return listcatfiles;
6152 create_alias_function("", __file_format_name__, "StringToArrayIndex", ArchiveFileStringToArrayIndex);
6154 def TarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6155 catfp = BytesIO();
6156 catfp = PackArchiveFileFromTarFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
6157 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
6158 return listcatfiles;
6160 def ZipFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6161 catfp = BytesIO();
6162 catfp = PackArchiveFileFromZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
6163 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
6164 return listcatfiles;
6166 if(not rarfile_support):
6167 def RarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6168 return False;
6170 if(rarfile_support):
6171 def RarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6172 catfp = BytesIO();
6173 catfp = PackArchiveFileFromRarFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
6174 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
6175 return listcatfiles;
6177 if(not py7zr_support):
6178 def SevenZipFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6179 return False;
6181 if(py7zr_support):
6182 def SevenZipFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_dict__, returnfp=False):
6183 catfp = BytesIO();
6184 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
6185 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
6186 return listcatfiles;
6188 def ListDirToArrayIndex(infiles, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, listonly=False, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6189 outarray = BytesIO();
6190 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, verbose, True);
6191 listcatfiles = ArchiveFileToArrayIndex(outarray, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp)
6192 return listcatfiles;
6194 def RePackArchiveFile(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6195 if(isinstance(infile, dict)):
6196 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6197 listcatfiles = prelistcatfiles['list'];
6198 else:
6199 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6200 infile = RemoveWindowsPath(infile);
6201 if(followlink):
6202 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6203 listcatfiles = prelistcatfiles['list'];
6204 else:
6205 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6206 if(outfile!="-" and not hasattr(infile, "read") and not hasattr(outfile, "write")):
6207 outfile = RemoveWindowsPath(outfile);
6208 checksumtype = checksumtype.lower();
6209 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
6210 checksumtype="crc32";
6211 if(checksumtype=="none"):
6212 checksumtype = "";
6213 if(not compression or compression=="catfile" or compression==formatspecs['format_lower']):
6214 compression = "auto";
6215 if(compression not in compressionlist and compression is None):
6216 compression = "auto";
6217 if(verbose):
6218 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6219 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
6220 if(os.path.exists(outfile)):
6221 try:
6222 os.unlink(outfile);
6223 except OSError as e:
6224 pass;
6225 if(not listcatfiles):
6226 return False;
6227 if(outfile=="-"):
6228 verbose = False;
6229 catfp = BytesIO();
6230 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
6231 catfp = outfile;
6232 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
6233 catfp = BytesIO();
6234 else:
6235 fbasename = os.path.splitext(outfile)[0];
6236 fextname = os.path.splitext(outfile)[1];
6237 catfp = CompressOpenFile(outfile, compresswholefile, compressionlevel);
6238 catver = formatspecs['format_ver'];
6239 fileheaderver = str(int(catver.replace(".", "")));
6240 lenlist = len(listcatfiles['ffilelist']);
6241 fnumfiles = int(listcatfiles['fnumfiles']);
6242 if(lenlist>fnumfiles or lenlist<fnumfiles):
6243 fnumfiles = lenlist;
6244 AppendFileHeader(catfp, fnumfiles, checksumtype, formatspecs);
6245 lenlist = len(listcatfiles['ffilelist']);
6246 fnumfiles = int(listcatfiles['fnumfiles']);
6247 lcfi = 0;
6248 lcfx = int(listcatfiles['fnumfiles']);
6249 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6250 lcfx = int(lenlist);
6251 else:
6252 lcfx = int(listcatfiles['fnumfiles']);
6253 curinode = 0;
6254 curfid = 0;
6255 inodelist = [];
6256 inodetofile = {};
6257 filetoinode = {};
6258 reallcfi = 0;
6259 while(lcfi < lcfx):
6260 if(re.findall("^[.|/]", listcatfiles['ffilelist'][reallcfi]['fname'])):
6261 fname = listcatfiles['ffilelist'][reallcfi]['fname'];
6262 else:
6263 fname = "./"+listcatfiles['ffilelist'][reallcfi]['fname'];
6264 if(verbose):
6265 VerbosePrintOut(fname);
6266 fheadersize = format(int(listcatfiles['ffilelist'][reallcfi]['fheadersize']), 'x').lower();
6267 fsize = format(int(listcatfiles['ffilelist'][reallcfi]['fsize']), 'x').lower();
6268 flinkname = listcatfiles['ffilelist'][reallcfi]['flinkname'];
6269 fatime = format(int(listcatfiles['ffilelist'][reallcfi]['fatime']), 'x').lower();
6270 fmtime = format(int(listcatfiles['ffilelist'][reallcfi]['fmtime']), 'x').lower();
6271 fctime = format(int(listcatfiles['ffilelist'][reallcfi]['fctime']), 'x').lower();
6272 fbtime = format(int(listcatfiles['ffilelist'][reallcfi]['fbtime']), 'x').lower();
6273 fmode = format(int(listcatfiles['ffilelist'][reallcfi]['fmode']), 'x').lower();
6274 fchmode = format(int(listcatfiles['ffilelist'][reallcfi]['fchmode']), 'x').lower();
6275 fuid = format(int(listcatfiles['ffilelist'][reallcfi]['fuid']), 'x').lower();
6276 funame = listcatfiles['ffilelist'][reallcfi]['funame'];
6277 fgid = format(int(listcatfiles['ffilelist'][reallcfi]['fgid']), 'x').lower();
6278 fgname = listcatfiles['ffilelist'][reallcfi]['fgname'];
6279 finode = format(int(listcatfiles['ffilelist'][reallcfi]['finode']), 'x').lower();
6280 flinkcount = format(int(listcatfiles['ffilelist'][reallcfi]['flinkcount']), 'x').lower();
6281 fwinattributes = format(int(listcatfiles['ffilelist'][reallcfi]['fwinattributes']), 'x').lower();
6282 fcompression = listcatfiles['ffilelist'][reallcfi]['fcompression'];
6283 fcsize = format(int(listcatfiles['ffilelist'][reallcfi]['fcsize']), 'x').lower();
6284 fdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['fminor']), 'x').lower();
6285 fdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['fmajor']), 'x').lower();
6286 frdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['frminor']), 'x').lower();
6287 frdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['frmajor']), 'x').lower();
6288 if(len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>listcatfiles['ffilelist'][reallcfi]['fextrafields'] and len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>0):
6289 listcatfiles['ffilelist'][reallcfi]['fextrafields'] = len(listcatfiles['ffilelist'][reallcfi]['fextralist']);
6290 if(not followlink and len(extradata)<0):
6291 extradata = listcatfiles['ffilelist'][reallcfi]['fextralist'];
6292 fcontents = listcatfiles['ffilelist'][reallcfi]['fcontents'];
6293 fcompression = "";
6294 fcsize = format(int(0), 'x').lower();
6295 if(not compresswholefile):
6296 fcontents.seek(0, 2);
6297 ucfsize = fcontents.tell();
6298 fcontents.seek(0, 0);
6299 if(compression=="auto"):
6300 ilsize = len(compressionlistalt);
6301 ilmin = 0;
6302 ilcsize = [];
6303 while(ilmin < ilsize):
6304 cfcontents = BytesIO();
6305 shutil.copyfileobj(fcontents, cfcontents);
6306 fcontents.seek(0, 0);
6307 cfcontents.seek(0, 0);
6308 cfcontents = CompressArchiveFile(cfcontents, compressionlistalt[ilmin], compressionlevel, formatspecs);
6309 if(cfcontents):
6310 cfcontents.seek(0, 2);
6311 ilcsize.append(cfcontents.tell());
6312 cfcontents.close();
6313 else:
6314 try:
6315 ilcsize.append(sys.maxint);
6316 except AttributeError:
6317 ilcsize.append(sys.maxsize);
6318 ilmin = ilmin + 1;
6319 ilcmin = ilcsize.index(min(ilcsize));
6320 compression = compressionlistalt[ilcmin];
6321 fcontents.seek(0, 0);
6322 cfcontents = BytesIO();
6323 shutil.copyfileobj(fcontents, cfcontents);
6324 cfcontents.seek(0, 0);
6325 cfcontents = CompressArchiveFile(cfcontents, compression, compressionlevel, formatspecs);
6326 cfcontents.seek(0, 2);
6327 cfsize = cfcontents.tell();
6328 if(ucfsize > cfsize):
6329 fcsize = format(int(cfsize), 'x').lower();
6330 fcompression = compression;
6331 fcontents.close();
6332 fcontents = cfcontents;
6333 if(followlink):
6334 if(listcatfiles['ffilelist'][reallcfi]['ftype']==1 or listcatfiles['ffilelist'][reallcfi]['ftype']==2):
6335 getflinkpath = listcatfiles['ffilelist'][reallcfi]['flinkname'];
6336 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6337 flinkinfo = listcatfiles['ffilelist'][flinkid];
6338 fheadersize = format(int(flinkinfo['fheadersize']), 'x').lower();
6339 fsize = format(int(flinkinfo['fsize']), 'x').lower();
6340 flinkname = flinkinfo['flinkname'];
6341 fatime = format(int(flinkinfo['fatime']), 'x').lower();
6342 fmtime = format(int(flinkinfo['fmtime']), 'x').lower();
6343 fctime = format(int(flinkinfo['fctime']), 'x').lower();
6344 fbtime = format(int(flinkinfo['fbtime']), 'x').lower();
6345 fmode = format(int(flinkinfo['fmode']), 'x').lower();
6346 fchmode = format(int(flinkinfo['fchmode']), 'x').lower();
6347 fuid = format(int(flinkinfo['fuid']), 'x').lower();
6348 funame = flinkinfo['funame'];
6349 fgid = format(int(flinkinfo['fgid']), 'x').lower();
6350 fgname = flinkinfo['fgname'];
6351 finode = format(int(flinkinfo['finode']), 'x').lower();
6352 flinkcount = format(int(flinkinfo['flinkcount']), 'x').lower();
6353 fwinattributes = format(int(flinkinfo['fwinattributes']), 'x').lower();
6354 fcompression = flinkinfo['fcompression'];
6355 fcsize = format(int(flinkinfo['fcsize']), 'x').lower();
6356 fdev_minor = format(int(flinkinfo['fminor']), 'x').lower();
6357 fdev_major = format(int(flinkinfo['fmajor']), 'x').lower();
6358 frdev_minor = format(int(flinkinfo['frminor']), 'x').lower();
6359 frdev_major = format(int(flinkinfo['frmajor']), 'x').lower();
6360 if(len(flinkinfo['fextralist'])>flinkinfo['fextrafields'] and len(flinkinfo['fextralist'])>0):
6361 flinkinfo['fextrafields'] = len(flinkinfo['fextralist']);
6362 if(len(extradata)<0):
6363 extradata = flinkinfo['fextralist'];
6364 fcontents = flinkinfo['fcontents'];
6365 ftypehex = format(flinkinfo['ftype'], 'x').lower();
6366 else:
6367 ftypehex = format(listcatfiles['ffilelist'][reallcfi]['ftype'], 'x').lower();
6368 fcurfid = format(curfid, 'x').lower();
6369 if(not followlink and finode!=0):
6370 if(listcatfiles['ffilelist'][reallcfi]['ftype']!=1):
6371 fcurinode = format(int(curinode), 'x').lower();
6372 inodetofile.update({curinode: fname});
6373 filetoinode.update({fname: curinode});
6374 curinode = curinode + 1;
6375 else:
6376 fcurinode = format(int(filetoinode[flinkname]), 'x').lower();
6377 else:
6378 fcurinode = format(int(curinode), 'x').lower();
6379 curinode = curinode + 1;
6380 curfid = curfid + 1;
6381 if(fcompression=="none"):
6382 fcompression = "";
6383 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
6384 catfp = AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
6385 fcontents.close();
6386 lcfi = lcfi + 1;
6387 reallcfi = reallcfi + 1;
6388 if(lcfx>0):
6389 catfp.write(AppendNullBytes([0, 0], formatspecs['format_delimiter']).encode("UTF-8"));
6390 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
6391 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
6392 try:
6393 catfp.flush();
6394 os.fsync(catfp.fileno());
6395 except io.UnsupportedOperation:
6396 pass;
6397 except AttributeError:
6398 pass;
6399 except OSError as e:
6400 pass;
6401 if(outfile=="-"):
6402 catfp.seek(0, 0);
6403 if(hasattr(sys.stdout, "buffer")):
6404 shutil.copyfileobj(catfp, sys.stdout.buffer);
6405 else:
6406 shutil.copyfileobj(catfp, sys.stdout);
6407 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
6408 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
6409 catfp.seek(0, 0);
6410 upload_file_to_internet_file(catfp, outfile);
6411 if(returnfp):
6412 catfp.seek(0, 0);
6413 return catfp;
6414 else:
6415 catfp.close();
6416 return True;
6418 create_alias_function("RePack", __file_format_name__, "", RePackArchiveFile);
6420 def RePackArchiveFileFromString(catstr, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6421 catfp = BytesIO(catstr);
6422 listcatfiles = RePackArchiveFile(catfp, compression, compresswholefile, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6423 return listcatfiles;
6425 create_alias_function("RePack", __file_format_name__, "FromString", RePackArchiveFileFromString);
6427 def PackArchiveFileFromListDir(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6428 outarray = BytesIO();
6429 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
6430 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6431 return listcatfiles;
6433 create_alias_function("Pack", __file_format_name__, "FromListDir", PackArchiveFileFromListDir);
6435 def UnPackArchiveFile(infile, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, preservepermissions=True, preservetime=True, verbose=False, returnfp=False):
6436 if(outdir is not None):
6437 outdir = RemoveWindowsPath(outdir);
6438 if(verbose):
6439 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6440 if(isinstance(infile, dict)):
6441 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6442 listcatfiles = prelistcatfiles['list'];
6443 else:
6444 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6445 infile = RemoveWindowsPath(infile);
6446 if(followlink):
6447 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6448 listcatfiles = prelistcatfiles['list'];
6449 else:
6450 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6451 if(not listcatfiles):
6452 return False;
6453 lenlist = len(listcatfiles['ffilelist']);
6454 fnumfiles = int(listcatfiles['fnumfiles']);
6455 lcfi = 0;
6456 lcfx = int(listcatfiles['fnumfiles']);
6457 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6458 lcfx = int(lenlist);
6459 else:
6460 lcfx = int(listcatfiles['fnumfiles']);
6461 while(lcfi<lcfx):
6462 funame = "";
6463 try:
6464 import pwd;
6465 try:
6466 userinfo = pwd.getpwuid(listcatfiles['ffilelist'][lcfi]['fuid']);
6467 funame = userinfo.pw_name;
6468 except KeyError:
6469 funame = "";
6470 except ImportError:
6471 funame = "";
6472 fgname = "";
6473 try:
6474 import grp;
6475 try:
6476 groupinfo = grp.getgrgid(listcatfiles['ffilelist'][lcfi]['fgid']);
6477 fgname = groupinfo.gr_name;
6478 except KeyError:
6479 fgname = "";
6480 except ImportError:
6481 fgname = "";
6482 if(verbose):
6483 VerbosePrintOut(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6484 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6485 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6486 listcatfiles['ffilelist'][lcfi]['fcontents'].seek(0, 0);
6487 shutil.copyfileobj(listcatfiles['ffilelist'][lcfi]['fcontents'], fpc);
6488 try:
6489 fpc.flush();
6490 os.fsync(fpc.fileno());
6491 except io.UnsupportedOperation:
6492 pass;
6493 except AttributeError:
6494 pass;
6495 except OSError as e:
6496 pass;
6497 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
6498 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
6499 if(preservepermissions):
6500 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6501 if(preservetime):
6502 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
6503 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6504 if(followlink):
6505 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
6506 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6507 flinkinfo = listcatfiles['ffilelist'][flinkid];
6508 funame = "";
6509 try:
6510 import pwd;
6511 try:
6512 userinfo = pwd.getpwuid(flinkinfo['fuid']);
6513 funame = userinfo.pw_name;
6514 except KeyError:
6515 funame = "";
6516 except ImportError:
6517 funame = "";
6518 fgname = "";
6519 try:
6520 import grp;
6521 try:
6522 groupinfo = grp.getgrgid(flinkinfo['fgid']);
6523 fgname = groupinfo.gr_name;
6524 except KeyError:
6525 fgname = "";
6526 except ImportError:
6527 fgname = "";
6528 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
6529 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6530 flinkinfo['fcontents'].seek(0, 0);
6531 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
6532 try:
6533 fpc.flush();
6534 os.fsync(fpc.fileno());
6535 except io.UnsupportedOperation:
6536 pass;
6537 except AttributeError:
6538 pass;
6539 except OSError as e:
6540 pass;
6541 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6542 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6543 if(preservepermissions):
6544 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6545 if(preservetime):
6546 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6547 if(flinkinfo['ftype']==1):
6548 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6549 if(flinkinfo['ftype']==2):
6550 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6551 if(flinkinfo['ftype']==5):
6552 if(preservepermissions):
6553 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6554 else:
6555 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6556 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6557 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6558 if(preservepermissions):
6559 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6560 if(preservetime):
6561 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6562 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
6563 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6564 else:
6565 os.link(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6566 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6567 if(followlink):
6568 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
6569 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6570 flinkinfo = listcatfiles['ffilelist'][flinkid];
6571 funame = "";
6572 try:
6573 import pwd;
6574 try:
6575 userinfo = pwd.getpwuid(flinkinfo['fuid']);
6576 funame = userinfo.pw_name;
6577 except KeyError:
6578 funame = "";
6579 except ImportError:
6580 funame = "";
6581 fgname = "";
6582 try:
6583 import grp;
6584 try:
6585 groupinfo = grp.getgrgid(flinkinfo['fgid']);
6586 fgname = groupinfo.gr_name;
6587 except KeyError:
6588 fgname = "";
6589 except ImportError:
6590 fgname = "";
6591 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
6592 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6593 flinkinfo['fcontents'].seek(0, 0);
6594 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
6595 try:
6596 fpc.flush();
6597 os.fsync(fpc.fileno());
6598 except io.UnsupportedOperation:
6599 pass;
6600 except AttributeError:
6601 pass;
6602 except OSError as e:
6603 pass;
6604 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6605 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6606 if(preservepermissions):
6607 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6608 if(preservetime):
6609 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6610 if(flinkinfo['ftype']==1):
6611 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6612 if(flinkinfo['ftype']==2):
6613 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6614 if(flinkinfo['ftype']==5):
6615 if(preservepermissions):
6616 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6617 else:
6618 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6619 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6620 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6621 if(preservepermissions):
6622 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6623 if(preservetime):
6624 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6625 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
6626 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6627 else:
6628 os.symlink(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6629 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6630 if(preservepermissions):
6631 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6632 else:
6633 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6634 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
6635 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
6636 if(preservepermissions):
6637 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6638 if(preservetime):
6639 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
6640 if(listcatfiles['ffilelist'][lcfi]['ftype']==6 and hasattr(os, "mkfifo")):
6641 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6642 lcfi = lcfi + 1;
6643 if(returnfp):
6644 return listcatfiles['ffilelist']['catfp'];
6645 else:
6646 return True;
6648 create_alias_function("UnPack", __file_format_name__, "", UnPackArchiveFile);
6650 if(hasattr(shutil, "register_unpack_format")):
6651 def UnPackArchiveFileFunc(archive_name, extract_dir=None, **kwargs):
6652 return UnPackArchiveFile(archive_name, extract_dir, False, 0, 0, False, __file_format_dict__['format_delimiter'], False, False);
6653 create_alias_function("UnPack", __file_format_name__, "Func", UnPackArchiveFileFunc);
6655 def UnPackArchiveFileString(catstr, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6656 catfp = BytesIO(catstr);
6657 listcatfiles = UnPackArchiveFile(catfp, outdir, followlink, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6658 return listcatfiles;
6660 create_alias_function("UnPack", __file_format_name__, "String", UnPackArchiveFileString);
6662 def ArchiveFileListFiles(infile, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6663 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6664 if(isinstance(infile, dict)):
6665 listcatfiles = infile;
6666 else:
6667 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6668 infile = RemoveWindowsPath(infile);
6669 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, True, skipchecksum, formatspecs, returnfp);
6670 if(not listcatfiles):
6671 return False;
6672 lenlist = len(listcatfiles['ffilelist']);
6673 fnumfiles = int(listcatfiles['fnumfiles']);
6674 lcfi = 0;
6675 lcfx = int(listcatfiles['fnumfiles']);
6676 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6677 lcfx = int(lenlist);
6678 else:
6679 lcfx = int(listcatfiles['fnumfiles']);
6680 returnval = {};
6681 while(lcfi<lcfx):
6682 returnval.update({lcfi: listcatfiles['ffilelist'][lcfi]['fname']});
6683 if(not verbose):
6684 VerbosePrintOut(listcatfiles['ffilelist'][lcfi]['fname']);
6685 if(verbose):
6686 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6687 printfname = listcatfiles['ffilelist'][lcfi]['fname'];
6688 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6689 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " link to " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6690 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6691 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " -> " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6692 fuprint = listcatfiles['ffilelist'][lcfi]['funame'];
6693 if(len(fuprint)<=0):
6694 fuprint = listcatfiles['ffilelist'][lcfi]['fuid'];
6695 fgprint = listcatfiles['ffilelist'][lcfi]['fgname'];
6696 if(len(fgprint)<=0):
6697 fgprint = listcatfiles['ffilelist'][lcfi]['fgid'];
6698 VerbosePrintOut(PrintPermissionString(listcatfiles['ffilelist'][lcfi]['fmode'], listcatfiles['ffilelist'][lcfi]['ftype']) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(listcatfiles['ffilelist'][lcfi]['fsize']).rjust(15) + " " + datetime.datetime.utcfromtimestamp(listcatfiles['ffilelist'][lcfi]['fmtime']).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6699 lcfi = lcfi + 1;
6700 if(returnfp):
6701 return listcatfiles['catfp'];
6702 else:
6703 return True;
6705 create_alias_function("", __file_format_name__, "ListFiles", ArchiveFileListFiles);
6707 def ArchiveFileStringListFiles(catstr, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_dict__, verbose=False, returnfp=False):
6708 catfp = BytesIO(catstr);
6709 listcatfiles = ArchiveFileListFiles(catstr, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6710 return listcatfiles;
6712 create_alias_function("", __file_format_name__, "StringListFiles", ArchiveFileStringListFiles);
6714 def TarFileListFiles(infile, verbose=False, returnfp=False):
6715 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6716 if(infile=="-"):
6717 infile = BytesIO();
6718 if(hasattr(sys.stdin, "buffer")):
6719 shutil.copyfileobj(sys.stdin.buffer, infile);
6720 else:
6721 shutil.copyfileobj(sys.stdin, infile);
6722 infile.seek(0, 0);
6723 if(not infile):
6724 return False;
6725 infile.seek(0, 0);
6726 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
6727 infile = download_file_from_internet_file(infile);
6728 infile.seek(0, 0);
6729 if(not infile):
6730 return False;
6731 infile.seek(0, 0);
6732 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
6733 return False;
6734 elif(os.path.exists(infile) and os.path.isfile(infile)):
6735 try:
6736 if(not tarfile.is_tarfile(infile)):
6737 return False;
6738 except AttributeError:
6739 if(not is_tarfile(infile)):
6740 return False;
6741 else:
6742 return False;
6743 try:
6744 if(hasattr(infile, "read") or hasattr(infile, "write")):
6745 tarfp = tarfile.open(fileobj=infile, mode="r");
6746 else:
6747 tarfp = tarfile.open(infile, "r");
6748 except FileNotFoundError:
6749 return False;
6750 lcfi = 0
6751 returnval = {};
6752 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
6753 returnval.update({lcfi: member.name});
6754 fpremode = member.mode;
6755 ffullmode = member.mode;
6756 flinkcount = 0;
6757 ftype = 0;
6758 if(member.isreg()):
6759 ffullmode = member.mode + stat.S_IFREG;
6760 ftype = 0;
6761 elif(member.isdev()):
6762 ffullmode = member.mode;
6763 ftype = 7;
6764 elif(member.islnk()):
6765 ffullmode = member.mode + stat.S_IFREG;
6766 ftype = 1;
6767 elif(member.issym()):
6768 ffullmode = member.mode + stat.S_IFLNK;
6769 ftype = 2;
6770 elif(member.ischr()):
6771 ffullmode = member.mode + stat.S_IFCHR;
6772 ftype = 3;
6773 elif(member.isblk()):
6774 ffullmode = member.mode + stat.S_IFBLK;
6775 ftype = 4;
6776 elif(member.isdir()):
6777 ffullmode = member.mode + stat.S_IFDIR;
6778 ftype = 5;
6779 elif(member.isfifo()):
6780 ffullmode = member.mode + stat.S_IFIFO;
6781 ftype = 6;
6782 elif(member.issparse()):
6783 ffullmode = member.mode;
6784 ftype = 12;
6785 if(not verbose):
6786 VerbosePrintOut(member.name);
6787 elif(verbose):
6788 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6789 printfname = member.name;
6790 if(member.islnk()):
6791 printfname = member.name + " link to " + member.linkname;
6792 elif(member.issym()):
6793 printfname = member.name + " -> " + member.linkname;
6794 fuprint = member.uname;
6795 if(len(fuprint)<=0):
6796 fuprint = member.uid;
6797 fgprint = member.gname;
6798 if(len(fgprint)<=0):
6799 fgprint = member.gid;
6800 VerbosePrintOut(PrintPermissionString(ffullmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(member.mtime).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6801 lcfi = lcfi + 1;
6802 if(returnfp):
6803 return listcatfiles['catfp'];
6804 else:
6805 return True;
6807 def ZipFileListFiles(infile, verbose=False, returnfp=False):
6808 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6809 if(infile=="-"):
6810 infile = BytesIO();
6811 if(hasattr(sys.stdin, "buffer")):
6812 shutil.copyfileobj(sys.stdin.buffer, infile);
6813 else:
6814 shutil.copyfileobj(sys.stdin, infile);
6815 infile.seek(0, 0);
6816 if(not infile):
6817 return False;
6818 infile.seek(0, 0);
6819 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
6820 infile = download_file_from_internet_file(infile);
6821 infile.seek(0, 0);
6822 if(not infile):
6823 return False;
6824 infile.seek(0, 0);
6825 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
6826 return False;
6827 else:
6828 return False;
6829 if(not zipfile.is_zipfile(infile)):
6830 return False;
6831 try:
6832 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
6833 except FileNotFoundError:
6834 return False;
6835 lcfi = 0;
6836 returnval = {};
6837 ziptest = zipfp.testzip();
6838 if(ziptest):
6839 VerbosePrintOut("Bad file found!");
6840 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
6841 if(zipinfo.create_system==0 or zipinfo.create_system==10):
6842 fwinattributes = int(zipinfo.external_attr);
6843 if(not member.is_dir()):
6844 fmode = int(stat.S_IFREG + 438);
6845 fchmode = int(stat.S_IMODE(fmode));
6846 ftypemod = int(stat.S_IFMT(fmode));
6847 elif(member.is_dir()):
6848 fmode = int(stat.S_IFDIR + 511);
6849 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6850 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6851 elif(zipinfo.create_system==3):
6852 fwinattributes =int(0);
6853 fmode = int(zipinfo.external_attr);
6854 fchmode = int(stat.S_IMODE(fmode));
6855 ftypemod = int(stat.S_IFMT(fmode));
6856 else:
6857 fwinattributes = int(0);
6858 if(not member.is_dir()):
6859 fmode = int(stat.S_IFREG + 438);
6860 fchmode = int(stat.S_IMODE(fmode));
6861 ftypemod = int(stat.S_IFMT(fmode));
6862 elif(member.is_dir()):
6863 fmode = int(stat.S_IFDIR + 511);
6864 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6865 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6866 returnval.update({lcfi: member.filename});
6867 if(not verbose):
6868 VerbosePrintOut(member.filename);
6869 if(verbose):
6870 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6871 permissionstr = "";
6872 for fmodval in str(oct(fmode))[-3:]:
6873 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
6874 if(not member.is_dir()):
6875 ftype = 0;
6876 permissionstr = "-" + permissionstr;
6877 elif(member.is_dir()):
6878 ftype = 5;
6879 permissionstr = "d" + permissionstr;
6880 printfname = member.filename;
6881 try:
6882 fuid = int(os.getuid());
6883 except AttributeError:
6884 fuid = int(0);
6885 except KeyError:
6886 fuid = int(0);
6887 try:
6888 fgid = int(os.getgid());
6889 except AttributeError:
6890 fgid = int(0);
6891 except KeyError:
6892 fgid = int(0);
6893 try:
6894 import pwd;
6895 try:
6896 userinfo = pwd.getpwuid(os.getuid());
6897 funame = userinfo.pw_name;
6898 except KeyError:
6899 funame = "";
6900 except AttributeError:
6901 funame = "";
6902 except ImportError:
6903 funame = "";
6904 fgname = "";
6905 try:
6906 import grp;
6907 try:
6908 groupinfo = grp.getgrgid(os.getgid());
6909 fgname = groupinfo.gr_name;
6910 except KeyError:
6911 fgname = "";
6912 except AttributeError:
6913 fgname = "";
6914 except ImportError:
6915 fgname = "";
6916 fuprint = funame;
6917 if(len(fuprint)<=0):
6918 fuprint = str(fuid);
6919 fgprint = fgname;
6920 if(len(fgprint)<=0):
6921 fgprint = str(fgid);
6922 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(int(time.mktime(member.date_time + (0, 0, -1)))).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6923 lcfi = lcfi + 1;
6924 if(returnfp):
6925 return listcatfiles['catfp'];
6926 else:
6927 return True;
6929 if(not rarfile_support):
6930 def RarFileListFiles(infile, verbose=False, returnfp=False):
6931 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6932 if(not os.path.exists(infile) or not os.path.isfile(infile)):
6933 return False;
6935 if(rarfile_support):
6936 def RarFileListFiles(infile, verbose=False, returnfp=False):
6937 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6938 if(not os.path.exists(infile) or not os.path.isfile(infile)):
6939 return False;
6940 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
6941 return False;
6942 lcfi = 0;
6943 returnval = {};
6944 rarfp = rarfile.RarFile(infile, "r");
6945 rartest = rarfp.testrar();
6946 if(rartest):
6947 VerbosePrintOut("Bad file found!");
6948 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
6949 is_unix = False;
6950 is_windows = False;
6951 if(member.host_os==rarfile.RAR_OS_UNIX):
6952 is_windows = False;
6953 try:
6954 member.external_attr
6955 is_unix = True;
6956 except AttributeError:
6957 is_unix = False;
6958 elif(member.host_os==rarfile.RAR_OS_WIN32):
6959 is_unix = False;
6960 try:
6961 member.external_attr
6962 is_windows = True;
6963 except AttributeError:
6964 is_windows = False;
6965 else:
6966 is_unix = False;
6967 is_windows = False;
6968 if(is_unix and member.external_attr !=0):
6969 fpremode = int(member.external_attr);
6970 elif(member.is_file()):
6971 fpremode = int(stat.S_IFREG + 438);
6972 elif(member.is_symlink()):
6973 fpremode = int(stat.S_IFLNK + 438);
6974 elif(member.is_dir()):
6975 fpremode = int(stat.S_IFDIR + 511);
6976 if(is_windows and member.external_attr !=0):
6977 fwinattributes = int(member.external_attr);
6978 else:
6979 fwinattributes = int(0);
6980 if(is_unix and member.external_attr !=0):
6981 fmode = int(member.external_attr);
6982 fchmode = int(stat.S_IMODE(member.external_attr));
6983 ftypemod = int(stat.S_IFMT(member.external_attr));
6984 elif(member.is_file()):
6985 fmode = int(stat.S_IFREG + 438);
6986 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
6987 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
6988 elif(member.is_symlink()):
6989 fmode = int(stat.S_IFLNK + 438);
6990 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
6991 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
6992 elif(member.is_dir()):
6993 fmode = int(stat.S_IFDIR + 511);
6994 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6995 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6996 returnval.update({lcfi: member.filename});
6997 if(not verbose):
6998 VerbosePrintOut(member.filename);
6999 if(verbose):
7000 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7001 permissionstr = "";
7002 for fmodval in str(oct(fmode))[-3:]:
7003 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
7004 if(member.is_file()):
7005 ftype = 0;
7006 permissionstr = "-" + permissionstr;
7007 printfname = member.filename;
7008 elif(member.is_symlink()):
7009 ftype = 2;
7010 permissionstr = "l" + permissionstr;
7011 printfname = member.name + " -> " + member.read().decode("UTF-8");
7012 elif(member.is_dir()):
7013 ftype = 5;
7014 permissionstr = "d" + permissionstr;
7015 printfname = member.filename;
7016 try:
7017 fuid = int(os.getuid());
7018 except AttributeError:
7019 fuid = int(0);
7020 except KeyError:
7021 fuid = int(0);
7022 try:
7023 fgid = int(os.getgid());
7024 except AttributeError:
7025 fgid = int(0);
7026 except KeyError:
7027 fgid = int(0);
7028 try:
7029 import pwd;
7030 try:
7031 userinfo = pwd.getpwuid(os.getuid());
7032 funame = userinfo.pw_name;
7033 except KeyError:
7034 funame = "";
7035 except AttributeError:
7036 funame = "";
7037 except ImportError:
7038 funame = "";
7039 fgname = "";
7040 try:
7041 import grp;
7042 try:
7043 groupinfo = grp.getgrgid(os.getgid());
7044 fgname = groupinfo.gr_name;
7045 except KeyError:
7046 fgname = "";
7047 except AttributeError:
7048 fgname = "";
7049 except ImportError:
7050 fgname = "";
7051 fuprint = funame;
7052 if(len(fuprint)<=0):
7053 fuprint = str(fuid);
7054 fgprint = fgname;
7055 if(len(fgprint)<=0):
7056 fgprint = str(fgid);
7057 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + member.mtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
7058 lcfi = lcfi + 1;
7059 if(returnfp):
7060 return listcatfiles['catfp'];
7061 else:
7062 return True;
7064 if(not py7zr_support):
7065 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
7066 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7067 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7068 return False;
7070 if(py7zr_support):
7071 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
7072 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7073 if(not os.path.exists(infile) or not os.path.isfile(infile)):
7074 return False;
7075 lcfi = 0;
7076 returnval = {};
7077 szpfp = py7zr.SevenZipFile(infile, mode="r");
7078 file_content = szpfp.readall();
7079 #sztest = szpfp.testzip();
7080 sztestalt = szpfp.test();
7081 if(sztestalt):
7082 VerbosePrintOut("Bad file found!");
7083 for member in sorted(szpfp.list(), key=lambda x: x.filename):
7084 if(re.findall("^[.|/]", member.filename)):
7085 fname = member.filename;
7086 else:
7087 fname = "./"+member.filename;
7088 if(not member.is_directory):
7089 fpremode = int(stat.S_IFREG + 438);
7090 elif(member.is_directory):
7091 fpremode = int(stat.S_IFDIR + 511);
7092 fwinattributes = int(0);
7093 if(member.is_directory):
7094 fmode = int(stat.S_IFDIR + 511);
7095 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
7096 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
7097 else:
7098 fmode = int(stat.S_IFLNK + 438);
7099 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
7100 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
7101 returnval.update({lcfi: member.filename});
7102 if(not verbose):
7103 VerbosePrintOut(member.filename);
7104 if(verbose):
7105 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
7106 permissionstr = "";
7107 for fmodval in str(oct(fmode))[-3:]:
7108 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
7109 fsize = int("0");
7110 if(not member.is_directory):
7111 ftype = 0;
7112 permissionstr = "-" + permissionstr;
7113 printfname = member.filename;
7114 elif(member.is_directory):
7115 ftype = 5;
7116 permissionstr = "d" + permissionstr;
7117 printfname = member.filename;
7118 if(ftype==0):
7119 fsize = len(file_content[member.filename].read());
7120 file_content[member.filename].close();
7121 try:
7122 fuid = int(os.getuid());
7123 except AttributeError:
7124 fuid = int(0);
7125 except KeyError:
7126 fuid = int(0);
7127 try:
7128 fgid = int(os.getgid());
7129 except AttributeError:
7130 fgid = int(0);
7131 except KeyError:
7132 fgid = int(0);
7133 try:
7134 import pwd;
7135 try:
7136 userinfo = pwd.getpwuid(os.getuid());
7137 funame = userinfo.pw_name;
7138 except KeyError:
7139 funame = "";
7140 except AttributeError:
7141 funame = "";
7142 except ImportError:
7143 funame = "";
7144 fgname = "";
7145 try:
7146 import grp;
7147 try:
7148 groupinfo = grp.getgrgid(os.getgid());
7149 fgname = groupinfo.gr_name;
7150 except KeyError:
7151 fgname = "";
7152 except AttributeError:
7153 fgname = "";
7154 except ImportError:
7155 fgname = "";
7156 fuprint = funame;
7157 if(len(fuprint)<=0):
7158 fuprint = str(fuid);
7159 fgprint = fgname;
7160 if(len(fgprint)<=0):
7161 fgprint = str(fgid);
7162 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(fsize).rjust(15) + " " + member.creationtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
7163 lcfi = lcfi + 1;
7164 if(returnfp):
7165 return listcatfiles['catfp'];
7166 else:
7167 return True;
7169 def InFileListFiles(infile, verbose=False, formatspecs=__file_format_dict__, returnfp=False):
7170 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
7171 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
7172 if(checkcompressfile=="tarfile"):
7173 return TarFileListFiles(infile, verbose, returnfp);
7174 elif(checkcompressfile=="zipfile"):
7175 return ZipFileListFiles(infile, verbose, returnfp);
7176 elif(checkcompressfile=="catfile"):
7177 return ArchiveFileListFiles(infile, 0, 0, False, formatspecs, verbose, returnfp);
7178 elif(rarfile_support and checkcompressfile=="rarfile"):
7179 return RarFileListFiles(infile, verbose, returnfp);
7180 elif(py7zr_support and checkcompressfile=="7zipfile"):
7181 return SevenZipFileListFiles(infile, verbose, returnfp);
7182 else:
7183 return False;
7184 return False;
7186 def ListDirListFiles(infiles, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7187 outarray = BytesIO();
7188 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compresswholefile, compressionlevel, followlink, checksumtype, formatspecs, False, True);
7189 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
7190 return listcatfiles;
7192 def ListDirListFilesAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7193 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, listonly, checksumtype, formatspecs, verbose);
7194 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
7195 return listcatfiles;
7197 def PackArchiveFileFromListDirAlt(infiles, outfile, dirlistfromtxt=False, compression="auto", compresswholefile=True, compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7198 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, False, checksumtype, extradata, formatspecs, False);
7199 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, followlink, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
7200 return listcatfiles;
7202 create_alias_function("Pack", __file_format_name__, "FromListDirAlt", PackArchiveFileFromListDirAlt);
7204 def PackArchiveFileFromTarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7205 outarray = TarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7206 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7207 return listcatfiles;
7209 create_alias_function("Pack", __file_format_name__, "FromTarFileAlt", PackArchiveFileFromTarFileAlt);
7211 def PackArchiveFileFromZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7212 outarray = ZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7213 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7214 return listcatfiles;
7216 create_alias_function("Pack", __file_format_name__, "FromZipFileAlt", PackArchiveFileFromZipFileAlt);
7218 if(not rarfile_support):
7219 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7220 return False;
7222 if(rarfile_support):
7223 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7224 outarray = RarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7225 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7226 return listcatfiles;
7228 create_alias_function("Pack", __file_format_name__, "FromRarFileAlt", PackArchiveFileFromRarFileAlt);
7230 if(not py7zr_support):
7231 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7232 return False;
7234 if(py7zr_support):
7235 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compresswholefile=True, compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_dict__, verbose=False, returnfp=False):
7236 outarray = SevenZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7237 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compresswholefile, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7238 return listcatfiles;
7240 create_alias_function("Pack", __file_format_name__, "FromSevenZipFileAlt", PackArchiveFileFromSevenZipFileAlt);
7242 def download_file_from_ftp_file(url):
7243 urlparts = urlparse(url);
7244 file_name = os.path.basename(urlparts.path);
7245 file_dir = os.path.dirname(urlparts.path);
7246 if(urlparts.username is not None):
7247 ftp_username = urlparts.username;
7248 else:
7249 ftp_username = "anonymous";
7250 if(urlparts.password is not None):
7251 ftp_password = urlparts.password;
7252 elif(urlparts.password is None and urlparts.username=="anonymous"):
7253 ftp_password = "anonymous";
7254 else:
7255 ftp_password = "";
7256 if(urlparts.scheme=="ftp"):
7257 ftp = FTP();
7258 elif(urlparts.scheme=="ftps"):
7259 ftp = FTP_TLS();
7260 else:
7261 return False;
7262 if(urlparts.scheme=="sftp"):
7263 if(__use_pysftp__):
7264 return download_file_from_pysftp_file(url);
7265 else:
7266 return download_file_from_sftp_file(url);
7267 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7268 return download_file_from_http_file(url);
7269 ftp_port = urlparts.port;
7270 if(urlparts.port is None):
7271 ftp_port = 21;
7272 try:
7273 ftp.connect(urlparts.hostname, ftp_port);
7274 except socket.gaierror:
7275 log.info("Error With URL "+url);
7276 return False;
7277 except socket.timeout:
7278 log.info("Error With URL "+url);
7279 return False;
7280 ftp.login(urlparts.username, urlparts.password);
7281 if(urlparts.scheme=="ftps"):
7282 ftp.prot_p();
7283 ftpfile = BytesIO();
7284 ftp.retrbinary("RETR "+urlparts.path, ftpfile.write);
7285 #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
7286 ftp.close();
7287 ftpfile.seek(0, 0);
7288 return ftpfile;
7290 def download_file_from_ftp_string(url):
7291 ftpfile = download_file_from_ftp_file(url);
7292 return ftpfile.read();
7294 def upload_file_to_ftp_file(ftpfile, url):
7295 urlparts = urlparse(url);
7296 file_name = os.path.basename(urlparts.path);
7297 file_dir = os.path.dirname(urlparts.path);
7298 if(urlparts.username is not None):
7299 ftp_username = urlparts.username;
7300 else:
7301 ftp_username = "anonymous";
7302 if(urlparts.password is not None):
7303 ftp_password = urlparts.password;
7304 elif(urlparts.password is None and urlparts.username=="anonymous"):
7305 ftp_password = "anonymous";
7306 else:
7307 ftp_password = "";
7308 if(urlparts.scheme=="ftp"):
7309 ftp = FTP();
7310 elif(urlparts.scheme=="ftps"):
7311 ftp = FTP_TLS();
7312 else:
7313 return False;
7314 if(urlparts.scheme=="sftp"):
7315 if(__use_pysftp__):
7316 return upload_file_to_pysftp_file(url);
7317 else:
7318 return upload_file_to_sftp_file(url);
7319 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7320 return False;
7321 ftp_port = urlparts.port;
7322 if(urlparts.port is None):
7323 ftp_port = 21;
7324 try:
7325 ftp.connect(urlparts.hostname, ftp_port);
7326 except socket.gaierror:
7327 log.info("Error With URL "+url);
7328 return False;
7329 except socket.timeout:
7330 log.info("Error With URL "+url);
7331 return False;
7332 ftp.login(urlparts.username, urlparts.password);
7333 if(urlparts.scheme=="ftps"):
7334 ftp.prot_p();
7335 ftp.storbinary("STOR "+urlparts.path, ftpfile);
7336 ftp.close();
7337 ftpfile.seek(0, 0);
7338 return ftpfile;
7340 def upload_file_to_ftp_string(ftpstring, url):
7341 ftpfileo = BytesIO(ftpstring);
7342 ftpfile = upload_file_to_ftp_file(ftpfileo, url);
7343 ftpfileo.close();
7344 return ftpfile;
7346 def download_file_from_http_file(url, headers=geturls_headers_pycatfile_python_alt):
7347 # Parse the URL to extract username and password if present
7348 urlparts = urlparse(url);
7349 username = urlparts.username;
7350 password = urlparts.password;
7351 # Rebuild the URL without the username and password
7352 netloc = urlparts.hostname;
7353 if(urlparts.scheme=="sftp"):
7354 if(__use_pysftp__):
7355 return download_file_from_pysftp_file(url);
7356 else:
7357 return download_file_from_sftp_file(url);
7358 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7359 return download_file_from_ftp_file(url);
7360 if urlparts.port:
7361 netloc += ':' + str(urlparts.port);
7362 rebuilt_url = urlunparse((urlparts.scheme, netloc, urlparts.path, urlparts.params, urlparts.query, urlparts.fragment));
7363 # Create a temporary file object
7364 httpfile = BytesIO();
7365 if haverequests:
7366 # Use the requests library if available
7367 if username and password:
7368 response = requests.get(rebuilt_url, headers=headers, auth=(username, password), stream=True);
7369 else:
7370 response = requests.get(rebuilt_url, headers=headers, stream=True);
7371 response.raw.decode_content = True
7372 shutil.copyfileobj(response.raw, httpfile);
7373 else:
7374 # Build a Request object for urllib
7375 request = Request(rebuilt_url, headers=headers);
7376 # Create an opener object for handling URLs
7377 if username and password:
7378 # Create a password manager
7379 password_mgr = HTTPPasswordMgrWithDefaultRealm();
7380 # Add the username and password
7381 password_mgr.add_password(None, rebuilt_url, username, password);
7382 # Create an authentication handler using the password manager
7383 auth_handler = HTTPBasicAuthHandler(password_mgr);
7384 # Build the opener with the authentication handler
7385 opener = build_opener(auth_handler);
7386 else:
7387 opener = build_opener();
7388 with opener.open(request) as response:
7389 shutil.copyfileobj(response, httpfile);
7390 # Reset file pointer to the start
7391 httpfile.seek(0, 0);
7392 # Return the temporary file object
7393 return httpfile;
7395 def download_file_from_http_string(url, headers=geturls_headers_pycatfile_python_alt):
7396 httpfile = download_file_from_http_file(url, headers);
7397 return ftpfile.read();
7399 if(haveparamiko):
7400 def download_file_from_sftp_file(url):
7401 urlparts = urlparse(url);
7402 file_name = os.path.basename(urlparts.path);
7403 file_dir = os.path.dirname(urlparts.path);
7404 sftp_port = urlparts.port;
7405 if(urlparts.port is None):
7406 sftp_port = 22;
7407 else:
7408 sftp_port = urlparts.port;
7409 if(urlparts.username is not None):
7410 sftp_username = urlparts.username;
7411 else:
7412 sftp_username = "anonymous";
7413 if(urlparts.password is not None):
7414 sftp_password = urlparts.password;
7415 elif(urlparts.password is None and urlparts.username=="anonymous"):
7416 sftp_password = "anonymous";
7417 else:
7418 sftp_password = "";
7419 if(urlparts.scheme=="ftp"):
7420 return download_file_from_ftp_file(url);
7421 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7422 return download_file_from_http_file(url);
7423 if(urlparts.scheme!="sftp"):
7424 return False;
7425 ssh = paramiko.SSHClient();
7426 ssh.load_system_host_keys();
7427 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
7428 try:
7429 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7430 except paramiko.ssh_exception.SSHException:
7431 return False;
7432 except socket.gaierror:
7433 log.info("Error With URL "+url);
7434 return False;
7435 except socket.timeout:
7436 log.info("Error With URL "+url);
7437 return False;
7438 sftp = ssh.open_sftp();
7439 sftpfile = BytesIO();
7440 sftp.getfo(urlparts.path, sftpfile);
7441 sftp.close();
7442 ssh.close();
7443 sftpfile.seek(0, 0);
7444 return sftpfile;
7445 else:
7446 def download_file_from_sftp_file(url):
7447 return False;
7449 if(haveparamiko):
7450 def download_file_from_sftp_string(url):
7451 sftpfile = download_file_from_sftp_file(url);
7452 return sftpfile.read();
7453 else:
7454 def download_file_from_ftp_string(url):
7455 return False;
7457 if(haveparamiko):
7458 def upload_file_to_sftp_file(sftpfile, url):
7459 urlparts = urlparse(url);
7460 file_name = os.path.basename(urlparts.path);
7461 file_dir = os.path.dirname(urlparts.path);
7462 sftp_port = urlparts.port;
7463 if(urlparts.port is None):
7464 sftp_port = 22;
7465 else:
7466 sftp_port = urlparts.port;
7467 if(urlparts.username is not None):
7468 sftp_username = urlparts.username;
7469 else:
7470 sftp_username = "anonymous";
7471 if(urlparts.password is not None):
7472 sftp_password = urlparts.password;
7473 elif(urlparts.password is None and urlparts.username=="anonymous"):
7474 sftp_password = "anonymous";
7475 else:
7476 sftp_password = "";
7477 if(urlparts.scheme=="ftp"):
7478 return upload_file_to_ftp_file(url);
7479 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7480 return False;
7481 if(urlparts.scheme!="sftp"):
7482 return False;
7483 ssh = paramiko.SSHClient();
7484 ssh.load_system_host_keys();
7485 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
7486 try:
7487 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7488 except paramiko.ssh_exception.SSHException:
7489 return False;
7490 except socket.gaierror:
7491 log.info("Error With URL "+url);
7492 return False;
7493 except socket.timeout:
7494 log.info("Error With URL "+url);
7495 return False;
7496 sftp = ssh.open_sftp();
7497 sftp.putfo(sftpfile, urlparts.path);
7498 sftp.close();
7499 ssh.close();
7500 sftpfile.seek(0, 0);
7501 return sftpfile;
7502 else:
7503 def upload_file_to_sftp_file(sftpfile, url):
7504 return False;
7506 if(haveparamiko):
7507 def upload_file_to_sftp_string(sftpstring, url):
7508 sftpfileo = BytesIO(sftpstring);
7509 sftpfile = upload_file_to_sftp_files(ftpfileo, url);
7510 sftpfileo.close();
7511 return sftpfile;
7512 else:
7513 def upload_file_to_sftp_string(url):
7514 return False;
7516 if(havepysftp):
7517 def download_file_from_pysftp_file(url):
7518 urlparts = urlparse(url);
7519 file_name = os.path.basename(urlparts.path);
7520 file_dir = os.path.dirname(urlparts.path);
7521 sftp_port = urlparts.port;
7522 if(urlparts.port is None):
7523 sftp_port = 22;
7524 else:
7525 sftp_port = urlparts.port;
7526 if(urlparts.username is not None):
7527 sftp_username = urlparts.username;
7528 else:
7529 sftp_username = "anonymous";
7530 if(urlparts.password is not None):
7531 sftp_password = urlparts.password;
7532 elif(urlparts.password is None and urlparts.username=="anonymous"):
7533 sftp_password = "anonymous";
7534 else:
7535 sftp_password = "";
7536 if(urlparts.scheme=="ftp"):
7537 return download_file_from_ftp_file(url);
7538 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7539 return download_file_from_http_file(url);
7540 if(urlparts.scheme!="sftp"):
7541 return False;
7542 try:
7543 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7544 except paramiko.ssh_exception.SSHException:
7545 return False;
7546 except socket.gaierror:
7547 log.info("Error With URL "+url);
7548 return False;
7549 except socket.timeout:
7550 log.info("Error With URL "+url);
7551 return False;
7552 sftp = ssh.open_sftp();
7553 sftpfile = BytesIO();
7554 sftp.getfo(urlparts.path, sftpfile);
7555 sftp.close();
7556 ssh.close();
7557 sftpfile.seek(0, 0);
7558 return sftpfile;
7559 else:
7560 def download_file_from_pysftp_file(url):
7561 return False;
7563 if(havepysftp):
7564 def download_file_from_pysftp_string(url):
7565 sftpfile = download_file_from_pysftp_file(url);
7566 return sftpfile.read();
7567 else:
7568 def download_file_from_ftp_string(url):
7569 return False;
7571 if(havepysftp):
7572 def upload_file_to_pysftp_file(sftpfile, url):
7573 urlparts = urlparse(url);
7574 file_name = os.path.basename(urlparts.path);
7575 file_dir = os.path.dirname(urlparts.path);
7576 sftp_port = urlparts.port;
7577 if(urlparts.port is None):
7578 sftp_port = 22;
7579 else:
7580 sftp_port = urlparts.port;
7581 if(urlparts.username is not None):
7582 sftp_username = urlparts.username;
7583 else:
7584 sftp_username = "anonymous";
7585 if(urlparts.password is not None):
7586 sftp_password = urlparts.password;
7587 elif(urlparts.password is None and urlparts.username=="anonymous"):
7588 sftp_password = "anonymous";
7589 else:
7590 sftp_password = "";
7591 if(urlparts.scheme=="ftp"):
7592 return upload_file_to_ftp_file(url);
7593 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7594 return False;
7595 if(urlparts.scheme!="sftp"):
7596 return False;
7597 try:
7598 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7599 except paramiko.ssh_exception.SSHException:
7600 return False;
7601 except socket.gaierror:
7602 log.info("Error With URL "+url);
7603 return False;
7604 except socket.timeout:
7605 log.info("Error With URL "+url);
7606 return False;
7607 sftp = ssh.open_sftp();
7608 sftp.putfo(sftpfile, urlparts.path);
7609 sftp.close();
7610 ssh.close();
7611 sftpfile.seek(0, 0);
7612 return sftpfile;
7613 else:
7614 def upload_file_to_pysftp_file(sftpfile, url):
7615 return False;
7617 if(havepysftp):
7618 def upload_file_to_pysftp_string(sftpstring, url):
7619 sftpfileo = BytesIO(sftpstring);
7620 sftpfile = upload_file_to_pysftp_files(ftpfileo, url);
7621 sftpfileo.close();
7622 return sftpfile;
7623 else:
7624 def upload_file_to_pysftp_string(url):
7625 return False;
7627 def download_file_from_internet_file(url, headers=geturls_headers_pycatfile_python_alt):
7628 urlparts = urlparse(url);
7629 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7630 return download_file_from_http_file(url, headers);
7631 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7632 return download_file_from_ftp_file(url);
7633 elif(urlparts.scheme=="sftp"):
7634 if(__use_pysftp__ and havepysftp):
7635 return download_file_from_pysftp_file(url);
7636 else:
7637 return download_file_from_sftp_file(url);
7638 else:
7639 return False;
7640 return False;
7642 def download_file_from_internet_uncompress_file(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_dict__):
7643 fp = download_file_from_internet_file(url);
7644 fp = UncompressArchiveFile(fp, formatspecs);
7645 fp.seek(0, 0);
7646 if(not fp):
7647 return False;
7648 return fp;
7650 def download_file_from_internet_string(url, headers=geturls_headers_pycatfile_python_alt):
7651 urlparts = urlparse(url);
7652 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7653 return download_file_from_http_string(url, headers);
7654 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7655 return download_file_from_ftp_string(url);
7656 elif(urlparts.scheme=="sftp"):
7657 if(__use_pysftp__ and havepysftp):
7658 return download_file_from_pysftp_string(url);
7659 else:
7660 return download_file_from_sftp_string(url);
7661 else:
7662 return False;
7663 return False;
7665 def download_file_from_internet_uncompress_string(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_dict__):
7666 fp = download_file_from_internet_string(url);
7667 fp = UncompressArchiveFile(fp, formatspecs);
7668 fp.seek(0, 0);
7669 if(not fp):
7670 return False;
7671 return fp;
7673 def upload_file_to_internet_file(ifp, url):
7674 urlparts = urlparse(url);
7675 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7676 return False;
7677 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7678 return upload_file_to_ftp_file(ifp, url);
7679 elif(urlparts.scheme=="sftp"):
7680 if(__use_pysftp__ and havepysftp):
7681 return upload_file_to_pysftp_file(ifp, url);
7682 else:
7683 return upload_file_to_sftp_file(ifp, url);
7684 else:
7685 return False;
7686 return False;
7688 def upload_file_to_internet_compress_file(ifp, url, formatspecs=__file_format_dict__):
7689 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
7690 if(not catfileout):
7691 return False;
7692 catfp.seek(0, 0);
7693 upload_file_to_internet_file(catfp, outfile);
7694 return True;
7696 def upload_file_to_internet_string(ifp, url):
7697 urlparts = urlparse(url);
7698 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7699 return False;
7700 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7701 return upload_file_to_ftp_string(ifp, url);
7702 elif(urlparts.scheme=="sftp"):
7703 if(__use_pysftp__ and havepysftp):
7704 return upload_file_to_pysftp_string(ifp, url);
7705 else:
7706 return upload_file_to_sftp_string(ifp, url);
7707 else:
7708 return False;
7709 return False;
7711 def upload_file_to_internet_compress_string(ifp, url, formatspecs=__file_format_dict__):
7712 catfp = CompressArchiveFile(BytesIO(ifp), compression, compressionlevel, formatspecs);
7713 if(not catfileout):
7714 return False;
7715 catfp.seek(0, 0);
7716 upload_file_to_internet_file(catfp, outfile);
7717 return True;
7719 try:
7720 if(hasattr(shutil, "register_archive_format")):
7721 # Register the packing format
7722 shutil.register_archive_format(__file_format_name__, PackArchiveFileFunc, description='Pack concatenated files');
7723 except shutil.RegistryError:
7724 pass;
7726 try:
7727 if(hasattr(shutil, "register_unpack_format")):
7728 # Register the unpacking format
7729 shutil.register_unpack_format(__file_format_name__, archivefile_extensions, UnPackArchiveFileFunc, description='UnPack concatenated files');
7730 except shutil.RegistryError:
7731 pass;