?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Add files via upload
[PyCatFile.git] / pycatfile.py
blob5ec6b93815afcd58a6031c9c3aad526f1f489b93
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 '''
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the Revised BSD License.
7
8 This program is distributed in the hope that it will be useful,
9 but WITHOUT ANY WARRANTY; without even the implied warranty of
10 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 Revised BSD License for more details.
12
13 Copyright 2018-2024 Cool Dude 2k - http://idb.berlios.de/
14 Copyright 2018-2024 Game Maker 2k - http://intdb.sourceforge.net/
15 Copyright 2018-2024 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
16
17 $FileInfo: pycatfile.py - Last Update: 5/5/2024 Ver. 0.10.2 RC 1 - Author: cooldude2k $
18 '''
19
20 from __future__ import absolute_import, division, print_function, unicode_literals;
21 import io, os, re, sys, time, stat, zlib, base64, shutil, socket, hashlib, datetime, logging, binascii, tempfile, zipfile, platform;
22 from ftplib import FTP, FTP_TLS;
23 if(sys.version[0]=="2"):
24 from urlparse import urlparse, urlunparse;
25 elif(sys.version[0]>="3"):
26 from urllib.parse import urlunparse;
27 from urllib.parse import urlparse;
28
29 if os.name == 'nt': # Only modify if on Windows
30 if sys.version[0] == "2":
31 import codecs;
32 sys.stdout = codecs.getwriter('utf-8')(sys.stdout);
33 sys.stderr = codecs.getwriter('utf-8')(sys.stderr);
34 else:
35 import io;
36 sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace', line_buffering=True);
37 sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', errors='replace', line_buffering=True);
38
39 hashlib_guaranteed = False;
40 os.environ["PYTHONIOENCODING"] = "UTF-8";
41 os.environ["LC_CTYPE"] = "UTF-8";
42 try:
43 reload(sys);
44 try:
45 sys.setdefaultencoding('UTF-8');
46 except NameError:
47 pass;
48 except AttributeError:
49 pass;
50 except NameError:
51 pass;
52 except AttributeError:
53 pass;
54
55 try:
56 from zlib import crc32;
57 except ImportError:
58 from binascii import crc32;
59
60 if(sys.version_info[0]==2):
61 FileNotFoundError = IOError;
62
63 rarfile_support = False;
64 try:
65 import rarfile;
66 rarfile_support = True;
67 except ImportError:
68 rarfile_support = False;
69
70 py7zr_support = False;
71 try:
72 import py7zr;
73 py7zr_support = True;
74 except ImportError:
75 py7zr_support = False;
76
77 try:
78 from xtarfile import is_tarfile;
79 except ImportError:
80 try:
81 from safetar import is_tarfile;
82 except ImportError:
83 from tarfile import is_tarfile;
84
85 try:
86 import xtarfile as tarfile;
87 except ImportError:
88 try:
89 import safetar as tarfile;
90 except ImportError:
91 import tarfile;
92
93 haveparamiko = False;
94 try:
95 import paramiko;
96 haveparamiko = True;
97 except ImportError:
98 haveparamiko = False;
99
100 havepysftp = False;
101 try:
102 import pysftp;
103 havepysftp = True;
104 except ImportError:
105 havepysftp = False;
106
107 haverequests = False;
108 try:
109 import requests;
110 haverequests = True;
111 except ImportError:
112 haverequests = False;
113
114 if(haverequests):
115 import urllib3;
116 logging.getLogger("urllib3").setLevel(logging.WARNING);
117
118 try:
119 # Python 3 imports
120 from urllib.request import Request, build_opener, HTTPBasicAuthHandler;
121 from urllib.parse import urlparse;
122 except ImportError:
123 # Python 2 imports
124 from urllib2 import Request, build_opener, HTTPBasicAuthHandler;
125 from urlparse import urlparse;
126
127 if(sys.version[0]=="2"):
128 try:
129 from io import StringIO, BytesIO;
130 except ImportError:
131 try:
132 from cStringIO import StringIO;
133 from cStringIO import StringIO as BytesIO;
134 except ImportError:
135 from StringIO import StringIO;
136 from StringIO import StringIO as BytesIO;
137 elif(sys.version[0]>="3"):
138 from io import StringIO, BytesIO;
139 else:
140 teststringio = 0;
141 if(teststringio<=0):
142 try:
143 from cStringIO import StringIO as BytesIO;
144 teststringio = 1;
145 except ImportError:
146 teststringio = 0;
147 if(teststringio<=0):
148 try:
149 from StringIO import StringIO as BytesIO;
150 teststringio = 2;
151 except ImportError:
152 teststringio = 0;
153 if(teststringio<=0):
154 try:
155 from io import BytesIO;
156 teststringio = 3;
157 except ImportError:
158 teststringio = 0;
159
160 __use_pysftp__ = False;
161 if(not havepysftp):
162 __use_pysftp__ = False;
163 __file_format_name__ = "CatFile";
164 __program_name__ = "Py"+__file_format_name__;
165 __file_format_lower__ = __file_format_name__.lower();
166 __file_format_magic__ = __file_format_name__;
167 __file_format_len__ = len(__file_format_magic__);
168 __file_format_hex__ = binascii.hexlify(__file_format_magic__.encode("UTF-8")).decode("UTF-8");
169 __file_format_delimiter__ = "\x00";
170 __file_format_ver__ = "001";
171 __use_new_style__ = True;
172 __use_advanced_list__ = True;
173 __use_alt_inode__ = False;
174 __file_format_list__ = [__file_format_name__, __file_format_magic__, __file_format_lower__, __file_format_len__, __file_format_hex__, __file_format_delimiter__, __file_format_ver__, __use_new_style__, __use_advanced_list__, __use_alt_inode__];
175 __project__ = __program_name__;
176 __project_url__ = "https://github.com/GameMaker2k/PyCatFile";
177 __version_info__ = (0, 10, 2, "RC 1", 1);
178 __version_date_info__ = (2024, 5, 5, "RC 1", 1);
179 __version_date__ = str(__version_date_info__[0]) + "." + str(__version_date_info__[1]).zfill(2) + "." + str(__version_date_info__[2]).zfill(2);
180 __revision__ = __version_info__[3];
181 __revision_id__ = "$Id$";
182 if(__version_info__[4] is not None):
183 __version_date_plusrc__ = __version_date__ + "-" + str(__version_date_info__[4]);
184 if(__version_info__[4] is None):
185 __version_date_plusrc__ = __version_date__;
186 if(__version_info__[3] is not None):
187 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]) + " " + str(__version_info__[3]);
188 if(__version_info__[3] is None):
189 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]);
190
191 PyBitness = platform.architecture();
192 if(PyBitness=="32bit" or PyBitness=="32"):
193 PyBitness = "32";
194 elif(PyBitness=="64bit" or PyBitness=="64"):
195 PyBitness = "64";
196 else:
197 PyBitness = "32";
198
199 geturls_ua_pycatfile_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
200 if(platform.python_implementation()!=""):
201 py_implementation = platform.python_implementation();
202 if(platform.python_implementation()==""):
203 py_implementation = "Python";
204 geturls_ua_pycatfile_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=py_implementation, pyver=platform.python_version(), proname=__project__, prover=__version__);
205 geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
206 geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
207 geturls_headers_pycatfile_python = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
208 geturls_headers_pycatfile_python_alt = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python_alt, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
209 geturls_headers_googlebot_google = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
210 geturls_headers_googlebot_google_old = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
211
212 compressionlist = ['auto', 'gzip', 'bzip2', 'zstd', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
213 outextlist = ['gz', 'bz2', 'zst', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
214 outextlistwd = ['.gz', '.bz2', '.zst', '.lz4', '.lzo', '.lzop', '.lzma', '.xz'];
215
216 tarfile_mimetype = "application/tar";
217 tarfile_tar_mimetype = tarfile_mimetype;
218 zipfile_mimetype = "application/zip";
219 zipfile_zip_mimetype = zipfile_mimetype;
220 rarfile_mimetype = "application/rar";
221 rarfile_rar_mimetype = rarfile_mimetype;
222 archivefile_mimetype = "application/x-"+__file_format_list__[1]+"";
223 archivefile_cat_mimetype = archivefile_mimetype;
224 archivefile_gzip_mimetype = "application/x-"+__file_format_list__[1]+"+gzip";
225 archivefile_gz_mimetype = archivefile_gzip_mimetype;
226 archivefile_bzip2_mimetype = "application/x-"+__file_format_list__[1]+"+bzip2";
227 archivefile_bz2_mimetype = archivefile_bzip2_mimetype;
228 archivefile_lz4_mimetype = "application/x-"+__file_format_list__[1]+"+lz4";
229 archivefile_lzop_mimetype = "application/x-"+__file_format_list__[1]+"+lzop";
230 archivefile_lzo_mimetype = archivefile_lzop_mimetype;
231 archivefile_zstandard_mimetype = "application/x-"+__file_format_list__[1]+"+zstandard";
232 archivefile_zstd_mimetype = archivefile_zstandard_mimetype;
233 archivefile_lzma_mimetype = "application/x-"+__file_format_list__[1]+"+lzma";
234 archivefile_xz_mimetype = "application/x-"+__file_format_list__[1]+"+xz";
235 archivefile_extensions = ['.cat', '.cat.gz', '.cat.bz2', '.cat.zst', '.cat.lz4', '.cat.lzo', '.cat.lzop', '.cat.lzma', '.cat.xz'];
236
237 if __name__ == "__main__":
238 import subprocess;
239 curscrpath = os.path.dirname(sys.argv[0]);
240 if(curscrpath==""):
241 curscrpath = ".";
242 if(os.sep=="\\"):
243 curscrpath = curscrpath.replace(os.sep, "/");
244 curscrpath = curscrpath + "/";
245 scrfile = curscrpath + "catfile.py";
246 if(os.path.exists(scrfile) and os.path.isfile(scrfile)):
247 scrcmd = subprocess.Popen([sys.executable, scrfile] + sys.argv[1:]);
248 scrcmd.wait();
249
250 def VerbosePrintOut(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
251 if(not dbgenable):
252 return True;
253 log_functions = {
254 "print": print,
255 "log": logging.info,
256 "warning": logging.warning,
257 "error": logging.error,
258 "critical": logging.critical,
259 "exception": logging.exception,
260 "logalt": lambda x: logging.log(dgblevel, x),
261 "debug": logging.debug
262 };
263 log_function = log_functions.get(outtype);
264 if(log_function):
265 log_function(dbgtxt);
266 return True;
267 return False;
268
269 def VerbosePrintOutReturn(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
270 VerbosePrintOut(dbgtxt, outtype, dbgenable, dgblevel);
271 return dbgtxt;
272
273 def RemoveWindowsPath(dpath):
274 if(dpath is None):
275 dpath = "";
276 if(os.sep!="/"):
277 dpath = dpath.replace(os.path.sep, "/");
278 dpath = dpath.rstrip("/");
279 if(dpath=="." or dpath==".."):
280 dpath = dpath + "/";
281 return dpath;
282
283 def NormalizeRelativePath(inpath):
284 inpath = RemoveWindowsPath(inpath);
285 if(os.path.isabs(inpath)):
286 outpath = inpath;
287 else:
288 if(inpath.startswith("./") or inpath.startswith("../")):
289 outpath = inpath;
290 else:
291 outpath = "./" + inpath;
292 return outpath;
293
294 def PrependPath(base_dir, child_path):
295 # Check if base_dir is None or empty, if so, return child_path as is
296 if not base_dir:
297 return child_path;
298 # Ensure base_dir ends with exactly one slash
299 if not base_dir.endswith('/'):
300 base_dir += '/';
301 # Check if child_path starts with ./ or ../ (indicating a relative path)
302 if child_path.startswith('./') or child_path.startswith('../'):
303 # For relative paths, we don't alter the child_path
304 return base_dir + child_path;
305 else:
306 # For non-relative paths, ensure there's no starting slash on child_path to avoid double slashes
307 return base_dir + child_path.lstrip('/');
308
309 def ListDir(dirpath, followlink=False, duplicates=False):
310 if(isinstance(dirpath, (list, tuple, ))):
311 dirpath = list(filter(None, dirpath));
312 elif(isinstance(dirpath, (str, ))):
313 dirpath = list(filter(None, [dirpath]));
314 retlist = [];
315 for mydirfile in dirpath:
316 if(not os.path.exists(mydirfile)):
317 return False;
318 mydirfile = NormalizeRelativePath(mydirfile);
319 if(os.path.exists(mydirfile) and os.path.islink(mydirfile)):
320 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile));
321 if(os.path.exists(mydirfile) and os.path.isdir(mydirfile)):
322 for root, dirs, filenames in os.walk(mydirfile):
323 dpath = root;
324 dpath = RemoveWindowsPath(dpath);
325 if(dpath not in retlist and not duplicates):
326 retlist.append(dpath);
327 if(duplicates):
328 retlist.append(dpath);
329 for file in filenames:
330 fpath = os.path.join(root, file);
331 fpath = RemoveWindowsPath(fpath);
332 if(fpath not in retlist and not duplicates):
333 retlist.append(fpath);
334 if(duplicates):
335 retlist.append(fpath);
336 else:
337 retlist.append(RemoveWindowsPath(mydirfile));
338 return retlist;
339
340 def ListDirAdvanced(dirpath, followlink=False, duplicates=False):
341 if isinstance(dirpath, (list, tuple)):
342 dirpath = list(filter(None, dirpath));
343 elif isinstance(dirpath, str):
344 dirpath = list(filter(None, [dirpath]));
345 retlist = []
346 for mydirfile in dirpath:
347 if not os.path.exists(mydirfile):
348 return False;
349 mydirfile = NormalizeRelativePath(mydirfile);
350 if os.path.exists(mydirfile) and os.path.islink(mydirfile) and followlink:
351 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile))
352 if os.path.exists(mydirfile) and os.path.isdir(mydirfile):
353 for root, dirs, filenames in os.walk(mydirfile):
354 # Sort dirs and filenames alphabetically in place
355 dirs.sort(key=lambda x: x.lower());
356 filenames.sort(key=lambda x: x.lower());
357 dpath = RemoveWindowsPath(root);
358 if not duplicates and dpath not in retlist:
359 retlist.append(dpath);
360 elif duplicates:
361 retlist.append(dpath);
362 for file in filenames:
363 fpath = os.path.join(root, file);
364 fpath = RemoveWindowsPath(fpath);
365 if not duplicates and fpath not in retlist:
366 retlist.append(fpath);
367 elif duplicates:
368 retlist.append(fpath);
369 else:
370 retlist.append(RemoveWindowsPath(mydirfile));
371 return retlist;
372
373 def create_alias_function(prefix, base_name, suffix, target_function):
374 # Define a new function that wraps the target function
375 def alias_function(*args, **kwargs):
376 return target_function(*args, **kwargs);
377
378 # Create the function name by combining the prefix, base name, and the suffix
379 function_name = "{}{}{}".format(prefix, base_name, suffix);
380
381 # Add the new function to the global namespace
382 globals()[function_name] = alias_function;
383
384 def create_alias_function_alt(prefix, base_name, suffix, target_function):
385 # Create the function name by combining the prefix, base name, and the suffix
386 # Use the format method for string formatting, compatible with Python 2 and 3
387 function_name = "{}{}{}".format(prefix, base_name, suffix)
388 # Add the new function (alias of the target_function) to the global namespace
389 # This line is compatible as-is with both Python 2 and 3
390 globals()[function_name] = target_function
391
392 # initial_value can be 0xFFFF or 0x0000
393 def crc16_ansi(msg, initial_value=0xFFFF):
394 # CRC-16-IBM / CRC-16-ANSI polynomial and initial value
395 poly = 0x8005; # Polynomial for CRC-16-IBM / CRC-16-ANSI
396 crc = initial_value; # Initial value
397 for b in msg:
398 crc ^= b << 8; # XOR byte into CRC top byte
399 for _ in range(8): # Process each bit
400 if crc & 0x8000: # If the top bit is set
401 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
402 else:
403 crc = crc << 1; # Just shift left
404 crc &= 0xFFFF; # Ensure CRC remains 16-bit
405 return crc;
406
407 # initial_value can be 0xFFFF or 0x0000
408 def crc16_ibm(msg, initial_value=0xFFFF):
409 return crc16_ansi(msg, initial_value);
410
411 # initial_value is 0xFFFF
412 def crc16(msg):
413 return crc16_ansi(msg, 0xFFFF);
414
415 # initial_value can be 0xFFFF, 0x1D0F or 0x0000
416 def crc16_ccitt(msg, initial_value=0xFFFF):
417 # CRC-16-CCITT polynomial
418 poly = 0x1021; # Polynomial for CRC-16-CCITT
419 # Use the specified initial value
420 crc = initial_value;
421 for b in msg:
422 crc ^= b << 8; # XOR byte into CRC top byte
423 for _ in range(8): # Process each bit
424 if crc & 0x8000: # If the top bit is set
425 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
426 else:
427 crc = crc << 1; # Just shift left
428 crc &= 0xFFFF; # Ensure CRC remains 16-bit
429 return crc;
430
431 # initial_value can be 0x42F0E1EBA9EA3693 or 0x0000000000000000
432 def crc64_ecma(msg, initial_value=0x0000000000000000):
433 # CRC-64-ECMA polynomial and initial value
434 poly = 0x42F0E1EBA9EA3693;
435 crc = initial_value; # Initial value for CRC-64-ECMA
436 for b in msg:
437 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
438 for _ in range(8): # Process each bit
439 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
440 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
441 else:
442 crc <<= 1; # Just shift left if the MSB is 0
443 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
444 return crc;
445
446 # initial_value can be 0x000000000000001B or 0xFFFFFFFFFFFFFFFF
447 def crc64_iso(msg, initial_value=0xFFFFFFFFFFFFFFFF):
448 # CRC-64-ISO polynomial and initial value
449 poly = 0x000000000000001B;
450 crc = initial_value; # Common initial value for CRC-64-ISO
451 for b in msg:
452 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
453 for _ in range(8): # Process each bit
454 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
455 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
456 else:
457 crc <<= 1; # Just shift left if the MSB is 0
458 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
459 return crc;
460
461 def GetDataFromArray(data, path, default=None):
462 element = data;
463 try:
464 for key in path:
465 element = element[key];
466 return element;
467 except (KeyError, TypeError, IndexError):
468 return default;
469
470 def GetDataFromArrayAlt(structure, path, default=None):
471 element = structure;
472 for key in path:
473 if isinstance(element, dict) and key in element:
474 element = element[key];
475 elif isinstance(element, list) and isinstance(key, int) and -len(element) <= key < len(element):
476 element = element[key];
477 else:
478 return default;
479 return element;
480
481 def GetHeaderChecksum(inlist=[], checksumtype="crc32", encodedata=True, formatspecs=__file_format_list__):
482 if isinstance(inlist, list):
483 fileheader = AppendNullBytes(inlist, formatspecs[5]);
484 else:
485 fileheader = AppendNullByte(inlist, formatspecs[5]);
486 if(encodedata):
487 fileheader = fileheader.encode('UTF-8');
488 if(checksumtype=="none" or checksumtype==""):
489 catfileheadercshex = format(0, 'x').lower();
490 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
491 catfileheadercshex = format(crc16(fileheader) & 0xffff, '04x').lower();
492 elif(checksumtype=="crc16_ccitt"):
493 catfileheadercshex = format(crc16_ccitt(fileheader) & 0xffff, '04x').lower();
494 elif(checksumtype=="adler32"):
495 catfileheadercshex = format(zlib.adler32(fileheader) & 0xffffffff, '08x').lower();
496 elif(checksumtype=="crc32"):
497 catfileheadercshex = format(crc32(fileheader) & 0xffffffff, '08x').lower();
498 elif(checksumtype=="crc64_ecma"):
499 catfileheadercshex = format(crc64_ecma(fileheader) & 0xffffffffffffffff, '016x').lower();
500 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
501 catfileheadercshex = format(crc64_iso(fileheader) & 0xffffffffffffffff, '016x').lower();
502 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
503 checksumoutstr = hashlib.new(checksumtype);
504 checksumoutstr.update(fileheader);
505 catfileheadercshex = checksumoutstr.hexdigest().lower();
506 else:
507 catfileheadercshex = format(0, 'x').lower();
508 return catfileheadercshex;
509
510 def GetFileChecksum(instr, checksumtype="crc32", encodedata=True, formatspecs=__file_format_list__):
511 if(encodedata):
512 instr = instr.encode('UTF-8');
513 if(checksumtype=="none" or checksumtype==""):
514 catinstrcshex = format(0, 'x').lower();
515 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
516 catinstrcshex = format(crc16(instr) & 0xffff, '04x').lower();
517 elif(checksumtype=="crc16_ccitt"):
518 catinstrcshex = format(crc16_ccitt(instr) & 0xffff, '04x').lower();
519 elif(checksumtype=="adler32"):
520 catinstrcshex = format(zlib.adler32(instr) & 0xffffffff, '08x').lower();
521 elif(checksumtype=="crc32"):
522 catinstrcshex = format(crc32(instr) & 0xffffffff, '08x').lower();
523 elif(checksumtype=="crc64_ecma"):
524 catinstrcshex = format(crc64_ecma(instr) & 0xffffffffffffffff, '016x').lower();
525 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
526 catinstrcshex = format(crc64_iso(instr) & 0xffffffffffffffff, '016x').lower();
527 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
528 checksumoutstr = hashlib.new(checksumtype);
529 checksumoutstr.update(instr);
530 catinstrcshex = checksumoutstr.hexdigest().lower();
531 else:
532 catinstrcshex = format(0, 'x').lower();
533 return catinstrcshex;
534
535 def ValidateHeaderChecksum(inlist=[], checksumtype="crc32", inchecksum="0", formatspecs=__file_format_list__):
536 catfileheadercshex = GetHeaderChecksum(inlist, checksumtype, True, formatspecs);
537 inchecksum = inchecksum.lower();
538 catfileheadercshex = catfileheadercshex.lower();
539 if(inchecksum==catfileheadercshex):
540 return True;
541 else:
542 return False;
543
544 def ValidateFileChecksum(infile, checksumtype="crc32", inchecksum="0", formatspecs=__file_format_list__):
545 catinfilecshex = GetFileChecksum(infile, checksumtype, True, formatspecs);
546 inchecksum = inchecksum.lower();
547 catinfilecshex = catinfilecshex.lower();
548 if(inchecksum==catinfilecshex):
549 return True;
550 else:
551 return False;
552
553 def ReadTillNullByte(fp, delimiter=__file_format_delimiter__):
554 curbyte = b"";
555 curfullbyte = b"";
556 nullbyte = delimiter.encode("UTF-8");
557 while(True):
558 curbyte = fp.read(1);
559 if(curbyte==nullbyte or not curbyte):
560 break;
561 curfullbyte = curfullbyte + curbyte;
562 return curfullbyte.decode('UTF-8');
563
564 def ReadUntilNullByte(fp, delimiter=__file_format_delimiter__):
565 return ReadTillNullByte(fp, delimiter);
566
567 def SeekToEndOfFile(fp):
568 lasttell = 0;
569 while(True):
570 fp.seek(1, 1);
571 if(lasttell==fp.tell()):
572 break;
573 lasttell = fp.tell();
574 return True;
575
576 def ReadFileHeaderData(fp, rounds=0, delimiter=__file_format_delimiter__):
577 rocount = 0;
578 roend = int(rounds);
579 HeaderOut = [];
580 while(rocount<roend):
581 HeaderOut.append(ReadTillNullByte(fp, delimiter));
582 rocount = rocount + 1;
583 return HeaderOut;
584
585 def ReadFileHeaderDataBySize(fp, delimiter=__file_format_delimiter__):
586 headerpresize = ReadTillNullByte(fp, delimiter);
587 headersize = int(headerpresize, 16);
588 if(headersize<=0):
589 return [];
590 headercontent = str(fp.read(headersize).decode('UTF-8')).split(delimiter);
591 fp.seek(1, 1);
592 rocount = 0;
593 roend = int(len(headercontent));
594 HeaderOut = [headerpresize];
595 while(rocount<roend):
596 HeaderOut.append(headercontent[rocount]);
597 rocount = rocount + 1;
598 return HeaderOut;
599
600 def ReadFileHeaderDataWoSize(fp, delimiter=__file_format_delimiter__):
601 preheaderdata = ReadFileHeaderData(fp, 2, delimiter);
602 headersize = int(preheaderdata[0], 16);
603 headernumfields = int(preheaderdata[1], 16);
604 if(headersize<=0 or headernumfields<=0):
605 return [];
606 headerdata = ReadFileHeaderData(fp, headernumfields, delimiter);
607 HeaderOut = preheaderdata + headerdata;
608 return HeaderOut;
609
610 def ReadFileHeaderDataBySizeWithContent(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
611 delimiter = formatspecs[5];
612 fheaderstart = fp.tell();
613 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
614 if(len(HeaderOut)==0):
615 return False;
616 if(re.findall("^[.|/]", HeaderOut[3])):
617 fname = HeaderOut[3];
618 else:
619 fname = "./"+HeaderOut[3];
620 fchecksumtype = HeaderOut[-3].lower();
621 fcs = HeaderOut[-2].lower();
622 fccs = HeaderOut[-1].lower();
623 fsize = int(HeaderOut[5], 16);
624 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
625 if(fcs!=newfcs and not skipchecksum):
626 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
627 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
628 return False;
629 fhend = fp.tell() - 1;
630 fcontentstart = fp.tell();
631 fcontents = BytesIO();
632 if(fsize>0 and not listonly):
633 fcontents.write(fp.read(fsize));
634 elif(fsize>0 and listonly):
635 fp.seek(fsize, 1);
636 fcontents.seek(0, 0);
637 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
638 if(fccs!=newfccs and not skipchecksum and not listonly):
639 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
640 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
641 return False;
642 fp.seek(1, 1);
643 fcontentend = fp.tell() - 1;
644 HeaderOut.append(fcontents);
645 return HeaderOut;
646
647 def ReadFileHeaderDataBySizeWithContentToArray(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
648 delimiter = formatspecs[5];
649 fheaderstart = fp.tell();
650 if(formatspecs[7]):
651 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
652 else:
653 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
654 if(len(HeaderOut)==0):
655 return False;
656 fheadsize = int(HeaderOut[0], 16);
657 fnumfields = int(HeaderOut[1], 16);
658 ftype = int(HeaderOut[2], 16);
659 if(re.findall("^[.|/]", HeaderOut[3])):
660 fname = HeaderOut[3];
661 else:
662 fname = "./"+HeaderOut[3];
663 fbasedir = os.path.dirname(fname);
664 flinkname = HeaderOut[4];
665 fsize = int(HeaderOut[5], 16);
666 fatime = int(HeaderOut[6], 16);
667 fmtime = int(HeaderOut[7], 16);
668 fctime = int(HeaderOut[8], 16);
669 fbtime = int(HeaderOut[9], 16);
670 fmode = int(HeaderOut[10], 16);
671 fchmode = stat.S_IMODE(fmode);
672 ftypemod = stat.S_IFMT(fmode);
673 fwinattributes = int(HeaderOut[11], 16);
674 fcompression = HeaderOut[12];
675 fcsize = int(HeaderOut[13], 16);
676 fuid = int(HeaderOut[14], 16);
677 funame = HeaderOut[15];
678 fgid = int(HeaderOut[16], 16);
679 fgname = HeaderOut[17];
680 fid = int(HeaderOut[18], 16);
681 finode = int(HeaderOut[19], 16);
682 flinkcount = int(HeaderOut[20], 16);
683 fdev_minor = int(HeaderOut[21], 16);
684 fdev_major = int(HeaderOut[22], 16);
685 frdev_minor = int(HeaderOut[23], 16);
686 frdev_major = int(HeaderOut[24], 16);
687 fextrasize = int(HeaderOut[25], 16);
688 fextrafields = int(HeaderOut[26], 16);
689 extrafieldslist = [];
690 extrastart = 27;
691 extraend = extrastart + fextrafields;
692 extrafieldslist = [];
693 if(extrastart<extraend):
694 extrafieldslist.append(HeaderOut[extrastart]);
695 extrastart = extrastart + 1;
696 fchecksumtype = HeaderOut[extrastart].lower();
697 fcs = HeaderOut[extrastart + 1].lower();
698 fccs = HeaderOut[extrastart + 2].lower();
699 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
700 if(fcs!=newfcs and not skipchecksum):
701 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
702 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
703 return False;
704 fhend = fp.tell() - 1;
705 fcontentstart = fp.tell();
706 fcontents = BytesIO();
707 pyhascontents = False;
708 if(fsize>0 and not listonly):
709 fcontents.write(fp.read(catfsize));
710 pyhascontents = True;
711 elif(fsize>0 and listonly):
712 fp.seek(fsize, 1);
713 pyhascontents = False;
714 fcontents.seek(0, 0);
715 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
716 if(fccs!=newfccs and not skipchecksum and not listonly):
717 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
718 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
719 return False;
720 fp.seek(1, 1);
721 fcontentend = fp.tell() - 1;
722 fcontents.seek(0, 0);
723 catlist = {'fheadersize': fheadsize, 'fhstart': fheaderstart, 'fhend': fhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': fchecksumtype, 'fnumfields': fnumfields + 2, 'frawheader': HeaderOut, 'fextrafields': fextrafields, 'fextrafieldsize': fextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': fcs, 'fcontentchecksum': fccs, 'fhascontents': pyhascontents, 'fcontentstart': fcontentstart, 'fcontentend': fcontentend, 'fcontents': fcontents};
724 return catlist;
725
726 def ReadFileHeaderDataBySizeWithContentToList(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
727 delimiter = formatspecs[5];
728 fheaderstart = fp.tell();
729 if(formatspecs[7]):
730 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
731 else:
732 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
733 if(len(HeaderOut)==0):
734 return False;
735 fheadsize = int(HeaderOut[0], 16);
736 fnumfields = int(HeaderOut[1], 16);
737 ftype = int(HeaderOut[2], 16);
738 if(re.findall("^[.|/]", HeaderOut[3])):
739 fname = HeaderOut[3];
740 else:
741 fname = "./"+HeaderOut[3];
742 fbasedir = os.path.dirname(fname);
743 flinkname = HeaderOut[4];
744 fsize = int(HeaderOut[5], 16);
745 fatime = int(HeaderOut[6], 16);
746 fmtime = int(HeaderOut[7], 16);
747 fctime = int(HeaderOut[8], 16);
748 fbtime = int(HeaderOut[9], 16);
749 fmode = int(HeaderOut[10], 16);
750 fchmode = stat.S_IMODE(fmode);
751 ftypemod = stat.S_IFMT(fmode);
752 fwinattributes = int(HeaderOut[11], 16);
753 fcompression = HeaderOut[12];
754 fcsize = int(HeaderOut[13], 16);
755 fuid = int(HeaderOut[14], 16);
756 funame = HeaderOut[15];
757 fgid = int(HeaderOut[16], 16);
758 fgname = HeaderOut[17];
759 fid = int(HeaderOut[18], 16);
760 finode = int(HeaderOut[19], 16);
761 flinkcount = int(HeaderOut[20], 16);
762 fdev_minor = int(HeaderOut[21], 16);
763 fdev_major = int(HeaderOut[22], 16);
764 frdev_minor = int(HeaderOut[23], 16);
765 frdev_major = int(HeaderOut[24], 16);
766 fextrasize = int(HeaderOut[25], 16);
767 fextrafields = int(HeaderOut[26], 16);
768 extrafieldslist = [];
769 extrastart = 27;
770 extraend = extrastart + fextrafields;
771 extrafieldslist = [];
772 if(extrastart<extraend):
773 extrafieldslist.append(HeaderOut[extrastart]);
774 extrastart = extrastart + 1;
775 fchecksumtype = HeaderOut[extrastart].lower();
776 fcs = HeaderOut[extrastart + 1].lower();
777 fccs = HeaderOut[extrastart + 2].lower();
778 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
779 if(fcs!=newfcs and not skipchecksum):
780 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
781 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
782 return False;
783 fhend = fp.tell() - 1;
784 fcontentstart = fp.tell();
785 fcontents = BytesIO();
786 pyhascontents = False;
787 if(fsize>0 and not listonly):
788 fcontents.write(fp.read(fsize));
789 pyhascontents = True;
790 elif(fsize>0 and listonly):
791 fp.seek(fsize, 1);
792 pyhascontents = False;
793 fcontents.seek(0, 0);
794 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
795 if(fccs!=newfccs and not skipchecksum and not listonly):
796 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
797 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
798 return False;
799 fp.seek(1, 1);
800 fcontentend = fp.tell() - 1;
801 catlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, extrafieldslist, fchecksumtype, fcontents];
802 return catlist;
803
804 def ReadFileDataBySizeWithContent(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
805 delimiter = formatspecs[5];
806 curloc = fp.tell();
807 if(curloc>0):
808 fp.seek(0, 0);
809 catheader = ReadFileHeaderData(fp, 4, delimiter);
810 if(curloc>0):
811 fp.seek(curloc, 0);
812 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
813 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
814 if(not headercheck and not skipchecksum):
815 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
816 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
817 return False;
818 fnumfiles = int(catheader[1], 16);
819 countnum = 0;
820 flist = [];
821 while(countnum < fnumfiles):
822 HeaderOut = ReadFileHeaderDataBySizeWithContent(fp, listonly, skipchecksum, formatspecs)
823 if(len(HeaderOut)==0):
824 break;
825 flist.append(HeaderOut);
826 countnum = countnum + 1;
827 return flist;
828
829 def ReadFileDataBySizeWithContentToArray(fp, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
830 delimiter = formatspecs[5];
831 curloc = fp.tell();
832 if(curloc>0):
833 fp.seek(0, 0);
834 catheader = ReadFileHeaderData(fp, 4, delimiter);
835 if(curloc>0):
836 fp.seek(curloc, 0);
837 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
838 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
839 if(not headercheck and not skipchecksum):
840 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
841 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
842 return False;
843 catstring = catheader[0];
844 catversion = re.findall(r"([\d]+)$", catstring);
845 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
846 fprenumfiles = catheader[1];
847 fnumfiles = int(fprenumfiles, 16);
848 fprechecksumtype = catheader[2];
849 fprechecksum = catheader[3];
850 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
851 if(seekstart<0 and seekstart>fnumfiles):
852 seekstart = 0;
853 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
854 seekend = fnumfiles;
855 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
856 seekend = fnumfiles - abs(seekend);
857 if(seekstart>0):
858 il = 0;
859 while(il < seekstart):
860 prefhstart = fp.tell();
861 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs[5]);
862 if(len(preheaderdata)==0):
863 break;
864 prefsize = int(preheaderdata[5], 16);
865 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-3].lower(), True, formatspecs);
866 prefcs = preheaderdata[-2];
867 if(prefcs!=prenewfcs and not skipchecksum):
868 VVerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
869 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
870 return False;
871 valid_archive = False;
872 invalid_archive = True;
873 prefhend = fp.tell() - 1;
874 prefcontentstart = fp.tell();
875 prefcontents = BytesIO();
876 pyhascontents = False;
877 if(prefsize>0):
878 prefcontents.write(fp.read(prefsize));
879 prefcontents.seek(0, 0);
880 prenewfccs = GetFileChecksum(prefcontents.read(), preheaderdata[-3].lower(), False, formatspecs);
881 prefccs = preheaderdata[-1];
882 pyhascontents = True;
883 if(prefccs!=prenewfccs and not skipchecksum):
884 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
885 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
886 return False;
887 fp.seek(1, 1);
888 il = il + 1;
889 realidnum = 0;
890 countnum = seekstart;
891 while(countnum < seekend):
892 catlist['ffilelist'].update({realidnum: {'fid': realidnum, 'fidalt': realidnum}});
893 HeaderOut = ReadFileHeaderDataBySizeWithContentToArray(fp, listonly, skipchecksum, formatspecs);
894 if(len(HeaderOut)):
895 break;
896 catlist['ffilelist'][realidnum].update(HeaderOut);
897 countnum = countnum + 1;
898 realidnum = realidnum + 1;
899 return catlist;
900
901 def ReadFileDataBySizeWithContentToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
902 if(isinstance(infile, dict)):
903 listcatfiles = infile;
904 else:
905 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
906 return False;
907 listcatfiles = ReadFileDataBySizeWithContentToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs);
908 if(not listcatfiles):
909 return False;
910 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
911 lenlist = len(listcatfiles['ffilelist']);
912 lcfi = 0;
913 lcfx = int(listcatfiles['fnumfiles']);
914 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
915 lcfx = int(lenlist);
916 else:
917 lcfx = int(listcatfiles['fnumfiles']);
918 while(lcfi < lcfx):
919 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
920 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
921 catarray['filetoid'].update(filetoidarray);
922 catarray['idtofile'].update(idtofilearray);
923 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
924 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
925 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
926 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
927 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
928 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
929 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
930 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
931 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
932 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
933 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
934 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
935 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
936 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
937 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
938 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
939 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
940 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
941 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
942 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
943 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
944 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
945 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
946 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
947 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
948 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
949 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
950 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
951 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
952 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
953 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
954 lcfi = lcfi + 1;
955 return catarray;
956
957 def ReadFileDataBySizeWithContentToList(fp, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
958 delimiter = formatspecs[5];
959 curloc = fp.tell();
960 if(curloc>0):
961 fp.seek(0, 0);
962 catheader = ReadFileHeaderData(fp, 4, delimiter);
963 if(curloc>0):
964 fp.seek(curloc, 0);
965 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
966 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
967 if(not headercheck and not skipchecksum):
968 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
969 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
970 return False;
971 catstring = catheader[0];
972 catversion = re.findall(r"([\d]+)$", catstring);
973 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
974 fprenumfiles = catheader[1];
975 fnumfiles = int(fprenumfiles, 16);
976 fprechecksumtype = catheader[2];
977 fprechecksum = catheader[3];
978 catlist = [];
979 if(seekstart<0 and seekstart>fnumfiles):
980 seekstart = 0;
981 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
982 seekend = fnumfiles;
983 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
984 seekend = fnumfiles - abs(seekend);
985 if(seekstart>0):
986 il = 0;
987 while(il < seekstart):
988 prefhstart = fp.tell();
989 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs[5]);
990 if(len(preheaderdata)==0):
991 break;
992 prefsize = int(preheaderdata[5], 16);
993 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-3].lower(), True, formatspecs);
994 prefcs = preheaderdata[-2];
995 if(prefcs!=prenewfcs and not skipchecksum):
996 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
997 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
998 return False;
999 valid_archive = False;
1000 invalid_archive = True;
1001 prefhend = fp.tell() - 1;
1002 prefcontentstart = fp.tell();
1003 prefcontents = "";
1004 pyhascontents = False;
1005 if(prefsize>0):
1006 prefcontents = fp.read(prefsize);
1007 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
1008 prefccs = preheaderdata[-1];
1009 pyhascontents = True;
1010 if(prefccs!=prenewfccs and not skipchecksum):
1011 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
1012 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
1013 return False;
1014 fp.seek(1, 1);
1015 il = il + 1;
1016 realidnum = 0;
1017 countnum = seekstart;
1018 while(countnum < seekend):
1019 HeaderOut = ReadFileHeaderDataBySizeWithContentToList(fp, listonly, skipchecksum, formatspecs);
1020 if(len(HeaderOut)):
1021 break;
1022 catlist.append(HeaderOut);
1023 countnum = countnum + 1;
1024 realidnum = realidnum + 1;
1025 return catlist;
1026
1027 def ReadInFileBySizeWithContentToArray(infile, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
1028 delimiter = formatspecs[5];
1029 if(hasattr(infile, "read") or hasattr(infile, "write")):
1030 fp = infile;
1031 fp.seek(0, 0);
1032 fp = UncompressArchiveFile(fp, formatspecs);
1033 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1034 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
1035 return False;
1036 if(not fp):
1037 return False;
1038 fp.seek(0, 0);
1039 elif(infile=="-"):
1040 fp = BytesIO();
1041 if(hasattr(sys.stdin, "buffer")):
1042 shutil.copyfileobj(sys.stdin.buffer, fp);
1043 else:
1044 shutil.copyfileobj(sys.stdin, fp);
1045 fp.seek(0, 0);
1046 fp = UncompressArchiveFile(fp, formatspecs);
1047 if(not fp):
1048 return False;
1049 fp.seek(0, 0);
1050 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
1051 fp = download_file_from_internet_file(infile);
1052 fp = UncompressArchiveFile(fp, formatspecs);
1053 fp.seek(0, 0);
1054 if(not fp):
1055 return False;
1056 fp.seek(0, 0);
1057 else:
1058 infile = RemoveWindowsPath(infile);
1059 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
1060 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
1061 return False;
1062 compresscheck = CheckCompressionType(infile, formatspecs, True);
1063 if(not compresscheck):
1064 fextname = os.path.splitext(infile)[1];
1065 if(fextname==".gz"):
1066 compresscheck = "gzip";
1067 elif(fextname==".bz2"):
1068 compresscheck = "bzip2";
1069 elif(fextname==".zst"):
1070 compresscheck = "zstd";
1071 elif(fextname==".lz4" or fextname==".clz4"):
1072 compresscheck = "lz4";
1073 elif(fextname==".lzo" or fextname==".lzop"):
1074 compresscheck = "lzo";
1075 elif(fextname==".lzma" or fextname==".xz"):
1076 compresscheck = "lzma";
1077 else:
1078 return False;
1079 if(not compresscheck):
1080 return False;
1081 fp = UncompressFile(infile, formatspecs, "rb");
1082 return ReadFileDataBySizeWithContentToArray(fp, listonly, skipchecksum, formatspecs);
1083
1084 def ReadInFileBySizeWithContentToList(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
1085 delimiter = formatspecs[5];
1086 if(hasattr(infile, "read") or hasattr(infile, "write")):
1087 fp = infile;
1088 fp.seek(0, 0);
1089 fp = UncompressArchiveFile(fp, formatspecs);
1090 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1091 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
1092 return False;
1093 if(not fp):
1094 return False;
1095 fp.seek(0, 0);
1096 elif(infile=="-"):
1097 fp = BytesIO();
1098 if(hasattr(sys.stdin, "buffer")):
1099 shutil.copyfileobj(sys.stdin.buffer, fp);
1100 else:
1101 shutil.copyfileobj(sys.stdin, fp);
1102 fp.seek(0, 0);
1103 fp = UncompressArchiveFile(fp, formatspecs);
1104 if(not fp):
1105 return False;
1106 fp.seek(0, 0);
1107 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
1108 fp = download_file_from_internet_file(infile);
1109 fp = UncompressArchiveFile(fp, formatspecs);
1110 fp.seek(0, 0);
1111 if(not fp):
1112 return False;
1113 fp.seek(0, 0);
1114 else:
1115 infile = RemoveWindowsPath(infile);
1116 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
1117 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
1118 return False;
1119 compresscheck = CheckCompressionType(infile, formatspecs, True);
1120 if(not compresscheck):
1121 fextname = os.path.splitext(infile)[1];
1122 if(fextname==".gz"):
1123 compresscheck = "gzip";
1124 elif(fextname==".bz2"):
1125 compresscheck = "bzip2";
1126 elif(fextname==".zst"):
1127 compresscheck = "zstd";
1128 elif(fextname==".lz4" or fextname==".clz4"):
1129 compresscheck = "lz4";
1130 elif(fextname==".lzo" or fextname==".lzop"):
1131 compresscheck = "lzo";
1132 elif(fextname==".lzma" or fextname==".xz"):
1133 compresscheck = "lzma";
1134 else:
1135 return False;
1136 if(not compresscheck):
1137 return False;
1138 fp = UncompressFile(infile, formatspecs, "rb");
1139 return ReadFileDataBySizeWithContentToList(fp, seekstart, seekend, listonly, skipchecksum, formatspecs);
1140
1141 def ReadInFileBySizeWithContentToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
1142 if(isinstance(infile, dict)):
1143 listcatfiles = infile;
1144 else:
1145 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
1146 infile = RemoveWindowsPath(infile);
1147 listcatfiles = ReadInFileBySizeWithContentToArray(infile, listonly, skipchecksum, formatspecs);
1148 if(not listcatfiles):
1149 return False;
1150 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
1151 lenlist = len(listcatfiles['ffilelist']);
1152 lcfi = 0;
1153 lcfx = int(listcatfiles['fnumfiles']);
1154 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
1155 lcfx = int(lenlist);
1156 else:
1157 lcfx = int(listcatfiles['fnumfiles']);
1158 while(lcfi < lcfx):
1159 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
1160 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
1161 catarray['filetoid'].update(filetoidarray);
1162 catarray['idtofile'].update(idtofilearray);
1163 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
1164 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
1165 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
1166 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
1167 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
1168 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
1169 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
1170 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
1171 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
1172 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1173 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1174 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
1175 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
1176 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
1177 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
1178 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
1179 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1180 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1181 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
1182 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
1183 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
1184 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1185 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1186 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
1187 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
1188 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
1189 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
1190 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1191 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1192 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1193 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1194 lcfi = lcfi + 1;
1195 return catarray;
1196
1197 def AppendNullByte(indata, delimiter=__file_format_delimiter__):
1198 outdata = str(indata) + delimiter;
1199 return outdata;
1200
1201 def AppendNullBytes(indata=[], delimiter=__file_format_delimiter__):
1202 outdata = "";
1203 inum = 0;
1204 il = len(indata);
1205 while(inum < il):
1206 outdata = outdata + AppendNullByte(indata[inum], delimiter);
1207 inum = inum + 1;
1208 return outdata;
1209
1210 def AppendFileHeader(fp, numfiles, checksumtype="crc32", formatspecs=__file_format_list__):
1211 delimiter = formatspecs[5];
1212 catver = formatspecs[6];
1213 fileheaderver = str(int(catver.replace(".", "")));
1214 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
1215 fp.write(fileheader.encode('UTF-8'));
1216 fnumfiles = format(int(numfiles), 'x').lower();
1217 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs[5]);
1218 catfileheadercshex = GetFileChecksum(fileheader + fnumfilesa, checksumtype, True, formatspecs);
1219 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs[5]);
1220 fp.write(fnumfilesa.encode('UTF-8'));
1221 try:
1222 fp.flush();
1223 os.fsync(fp.fileno());
1224 except io.UnsupportedOperation:
1225 pass;
1226 except AttributeError:
1227 pass;
1228 except OSError as e:
1229 pass;
1230 return fp;
1231
1232 def MakeEmptyFilePointer(fp, checksumtype="crc32", formatspecs=__file_format_list__):
1233 AppendFileHeader(fp, 0, checksumtype, formatspecs);
1234 return fp;
1235
1236 def MakeEmptyFile(outfile, compression="auto", compressionlevel=None, checksumtype="crc32", formatspecs=__file_format_list__, returnfp=False):
1237 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1238 if(os.path.exists(outfile)):
1239 try:
1240 os.unlink(outfile);
1241 except OSError as e:
1242 pass;
1243 if(outfile=="-"):
1244 verbose = False;
1245 catfpfp = BytesIO();
1246 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1247 catfp = outfile;
1248 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1249 catfp = BytesIO();
1250 else:
1251 fbasename = os.path.splitext(outfile)[0];
1252 fextname = os.path.splitext(outfile)[1];
1253 catfp = CompressOpenFile(outfile, True, compressionlevel);
1254 catfp = AppendFileHeader(catfp, 0, checksumtype, formatspecs);
1255 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1256 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1257 try:
1258 catfp.flush();
1259 os.fsync(catfp.fileno());
1260 except io.UnsupportedOperation:
1261 pass;
1262 except AttributeError:
1263 pass;
1264 except OSError as e:
1265 pass;
1266 if(outfile=="-"):
1267 catfp.seek(0, 0);
1268 if(hasattr(sys.stdout, "buffer")):
1269 shutil.copyfileobj(catfp, sys.stdout.buffer);
1270 else:
1271 shutil.copyfileobj(catfp, sys.stdout);
1272 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1273 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1274 catfp.seek(0, 0);
1275 upload_file_to_internet_file(catfp, outfile);
1276 if(returnfp):
1277 catfp.seek(0, 0);
1278 return catfp;
1279 else:
1280 catfp.close();
1281 return True;
1282
1283 def AppendFileHeaderWithContent(fp, filevalues=[], extradata=[], filecontent="", checksumtype="crc32", formatspecs=__file_format_list__):
1284 extrafields = format(len(extradata), 'x').lower();
1285 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
1286 if(len(extradata)>0):
1287 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
1288 extrasizelen = format(len(extrasizestr), 'x').lower();
1289 catoutlen = len(filevalues) + len(extradata) + 5;
1290 catoutlenhex = format(catoutlen, 'x').lower();
1291 catoutlist = filevalues;
1292 catoutlist.insert(0, catoutlenhex);
1293 catoutlist.append(extrasizelen);
1294 catoutlist.append(extrafields);
1295 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
1296 if(len(extradata)>0):
1297 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
1298 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
1299 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
1300 catfilecontentcshex = GetFileChecksum(filecontent, checksumtype, False, formatspecs);
1301 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
1302 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
1303 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
1304 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
1305 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
1306 catfileoutstrecd = catfileoutstr.encode('UTF-8');
1307 nullstrecd = formatspecs[5].encode('UTF-8');
1308 catfileout = catfileoutstrecd + filecontent + nullstrecd;
1309 fp.write(catfileout);
1310 try:
1311 fp.flush();
1312 os.fsync(fp.fileno());
1313 except io.UnsupportedOperation:
1314 pass;
1315 except AttributeError:
1316 pass;
1317 except OSError as e:
1318 pass;
1319 return fp;
1320
1321 def AppendFilesWithContent(infiles, fp, dirlistfromtxt=False, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False):
1322 advancedlist = formatspecs[8];
1323 altinode = formatspecs[9];
1324 if(verbose):
1325 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
1326 if(infiles=="-"):
1327 for line in sys.stdin:
1328 infilelist.append(line.strip());
1329 infilelist = list(filter(None, infilelist));
1330 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
1331 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
1332 return False;
1333 with UncompressFile(infiles, formatspecs, "r") as finfile:
1334 for line in finfile:
1335 infilelist.append(line.strip());
1336 infilelist = list(filter(None, infilelist));
1337 else:
1338 if(isinstance(infiles, (list, tuple, ))):
1339 infilelist = list(filter(None, infiles));
1340 elif(isinstance(infiles, (str, ))):
1341 infilelist = list(filter(None, [infiles]));
1342 if(advancedlist):
1343 GetDirList = ListDirAdvanced(infilelist, followlink, False);
1344 else:
1345 GetDirList = ListDir(infilelist, followlink, False);
1346 if(not GetDirList):
1347 return False;
1348 curinode = 0;
1349 curfid = 0;
1350 inodelist = [];
1351 inodetofile = {};
1352 filetoinode = {};
1353 inodetocatinode = {};
1354 numfiles = int(len(GetDirList));
1355 fnumfiles = format(numfiles, 'x').lower();
1356 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
1357 for curfname in GetDirList:
1358 if(re.findall("^[.|/]", curfname)):
1359 fname = curfname;
1360 else:
1361 fname = "./"+curfname;
1362 if(verbose):
1363 VerbosePrintOut(fname);
1364 if(not followlink or followlink is None):
1365 fstatinfo = os.lstat(fname);
1366 else:
1367 fstatinfo = os.stat(fname);
1368 fpremode = fstatinfo.st_mode;
1369 finode = fstatinfo.st_ino;
1370 flinkcount = fstatinfo.st_nlink;
1371 ftype = 0;
1372 if(stat.S_ISREG(fpremode)):
1373 ftype = 0;
1374 elif(stat.S_ISLNK(fpremode)):
1375 ftype = 2;
1376 elif(stat.S_ISCHR(fpremode)):
1377 ftype = 3;
1378 elif(stat.S_ISBLK(fpremode)):
1379 ftype = 4;
1380 elif(stat.S_ISDIR(fpremode)):
1381 ftype = 5;
1382 elif(stat.S_ISFIFO(fpremode)):
1383 ftype = 6;
1384 elif(stat.S_ISSOCK(fpremode)):
1385 ftype = 8;
1386 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
1387 ftype = 9;
1388 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
1389 ftype = 10;
1390 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
1391 ftype = 111;
1392 else:
1393 ftype = 0;
1394 flinkname = "";
1395 fcurfid = format(int(curfid), 'x').lower();
1396 if(not followlink and finode!=0):
1397 if(ftype!=1):
1398 if(finode in inodelist):
1399 ftype = 1;
1400 flinkname = inodetofile[finode];
1401 if(altinode):
1402 fcurinode = format(int(finode), 'x').lower();
1403 else:
1404 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
1405 if(finode not in inodelist):
1406 inodelist.append(finode);
1407 inodetofile.update({finode: fname});
1408 inodetocatinode.update({finode: curinode});
1409 if(altinode):
1410 fcurinode = format(int(finode), 'x').lower();
1411 else:
1412 fcurinode = format(int(curinode), 'x').lower();
1413 curinode = curinode + 1;
1414 else:
1415 fcurinode = format(int(curinode), 'x').lower();
1416 curinode = curinode + 1;
1417 curfid = curfid + 1;
1418 if(ftype==2):
1419 flinkname = os.readlink(fname);
1420 fdev = fstatinfo.st_dev;
1421 getfdev = GetDevMajorMinor(fdev);
1422 fdev_minor = getfdev[0];
1423 fdev_major = getfdev[1];
1424 frdev = fstatinfo.st_dev;
1425 if(hasattr(fstatinfo, "st_rdev")):
1426 frdev = fstatinfo.st_rdev;
1427 else:
1428 frdev = fstatinfo.st_dev;
1429 getfrdev = GetDevMajorMinor(frdev);
1430 frdev_minor = getfrdev[0];
1431 frdev_major = getfrdev[1];
1432 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
1433 fsize = format(int("0"), 'x').lower();
1434 elif(ftype==0 or ftype==7):
1435 fsize = format(int(fstatinfo.st_size), 'x').lower();
1436 else:
1437 fsize = format(int(fstatinfo.st_size)).lower();
1438 fatime = format(int(fstatinfo.st_atime), 'x').lower();
1439 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
1440 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
1441 if(hasattr(fstatinfo, "st_birthtime")):
1442 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
1443 else:
1444 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
1445 fmode = format(int(fstatinfo.st_mode), 'x').lower();
1446 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
1447 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
1448 fuid = format(int(fstatinfo.st_uid), 'x').lower();
1449 fgid = format(int(fstatinfo.st_gid), 'x').lower();
1450 funame = "";
1451 try:
1452 import pwd;
1453 try:
1454 userinfo = pwd.getpwuid(fstatinfo.st_uid);
1455 funame = userinfo.pw_name;
1456 except KeyError:
1457 funame = "";
1458 except ImportError:
1459 funame = "";
1460 fgname = "";
1461 try:
1462 import grp;
1463 try:
1464 groupinfo = grp.getgrgid(fstatinfo.st_gid);
1465 fgname = groupinfo.gr_name;
1466 except KeyError:
1467 fgname = "";
1468 except ImportError:
1469 fgname = "";
1470 fdev_minor = format(int(fdev_minor), 'x').lower();
1471 fdev_major = format(int(fdev_major), 'x').lower();
1472 frdev_minor = format(int(frdev_minor), 'x').lower();
1473 frdev_major = format(int(frdev_major), 'x').lower();
1474 finode = format(int(finode), 'x').lower();
1475 flinkcount = format(int(flinkcount), 'x').lower();
1476 if(hasattr(fstatinfo, "st_file_attributes")):
1477 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
1478 else:
1479 fwinattributes = format(int(0), 'x').lower();
1480 fcompression = "none";
1481 fcsize = format(int(0), 'x').lower();
1482 fcontents = BytesIO();
1483 chunk_size = 1024;
1484 if(ftype==0 or ftype==7):
1485 with open(fname, "rb") as fpc:
1486 shutil.copyfileobj(fpc, fcontents);
1487 if(followlink and (ftype==1 or ftype==2)):
1488 flstatinfo = os.stat(flinkname);
1489 with open(flinkname, "rb") as fpc:
1490 shutil.copyfileobj(fpc, fcontents);
1491 fcontents.seek(0, 0);
1492 ftypehex = format(ftype, 'x').lower();
1493 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
1494 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
1495 if(numfiles>0):
1496 catfp.write(AppendNullBytes([0, 0], formatspecs[5]).encode("UTF-8"));
1497 fp.seek(0, 0);
1498 return fp;
1499
1500 def AppendListsWithContent(inlist, fp, dirlistfromtxt=False, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False):
1501 if(verbose):
1502 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
1503 GetDirList = inlist;
1504 if(not GetDirList):
1505 return False;
1506 curinode = 0;
1507 curfid = 0;
1508 inodelist = [];
1509 inodetofile = {};
1510 filetoinode = {};
1511 inodetocatinode = {};
1512 numfiles = int(len(GetDirList));
1513 fnumfiles = format(numfiles, 'x').lower();
1514 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
1515 for curfname in GetDirList:
1516 ftype = format(curfname[0], 'x').lower();
1517 if(re.findall("^[.|/]", curfname[1])):
1518 fname = curfname[1];
1519 else:
1520 fname = "./"+curfname[1];
1521 fbasedir = os.path.dirname(fname);
1522 flinkname = curfname[2];
1523 fsize = format(curfname[3], 'x').lower();
1524 fatime = format(curfname[4], 'x').lower();
1525 fmtime = format(curfname[5], 'x').lower();
1526 fctime = format(curfname[6], 'x').lower();
1527 fbtime = format(curfname[7], 'x').lower();
1528 fmode = format(curfname[8], 'x').lower();
1529 fwinattributes = format(curfname[9], 'x').lower();
1530 fcompression = curfname[10];
1531 fcsize = int(curfname[11], 16);
1532 fuid = format(curfname[12], 'x').lower();
1533 funame = curfname[13];
1534 fgid = format(curfname[14], 'x').lower();
1535 fgname = curfname[15];
1536 fid = format(curfname[16], 'x').lower();
1537 finode = format(curfname[17], 'x').lower();
1538 flinkcount = format(curfname[18], 'x').lower();
1539 fdev_minor = format(curfname[19], 'x').lower();
1540 fdev_major = format(curfname[20], 'x').lower();
1541 frdev_minor = format(curfname[21], 'x').lower();
1542 frdev_major = format(curfname[22], 'x').lower();
1543 extradata = curfname[23];
1544 fchecksumtype = curfname[24];
1545 fcontents = curfname[25];
1546 catoutlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
1547 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents, checksumtype, formatspecs);
1548 if(numfiles>0):
1549 catfp.write(AppendNullBytes([0, 0], formatspecs[5]).encode("UTF-8"));
1550 fp.seek(0, 0);
1551 return fp;
1552
1553 def AppendInFileWithContent(infile, fp, dirlistfromtxt=False, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False):
1554 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, False, formatspecs);
1555 return AppendListsWithContent(inlist, fp, dirlistfromtxt, filevalues, extradata, followlink, checksumtype, formatspecs, verbose);
1556
1557 def AppendFilesWithContentToOutFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False, returnfp=False):
1558 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1559 if(os.path.exists(outfile)):
1560 try:
1561 os.unlink(outfile);
1562 except OSError as e:
1563 pass;
1564 if(outfile=="-"):
1565 verbose = False;
1566 catfpfp = BytesIO();
1567 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1568 catfp = outfile;
1569 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1570 catfp = BytesIO();
1571 else:
1572 fbasename = os.path.splitext(outfile)[0];
1573 fextname = os.path.splitext(outfile)[1];
1574 catfp = CompressOpenFile(outfile, True, compressionlevel);
1575 catfp = AppendFilesWithContent(infiles, catfp, dirlistfromtxt, filevalues, extradata, followlink, checksumtype, formatspecs, verbose);
1576 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1577 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1578 try:
1579 catfp.flush();
1580 os.fsync(catfp.fileno());
1581 except io.UnsupportedOperation:
1582 pass;
1583 except AttributeError:
1584 pass;
1585 except OSError as e:
1586 pass;
1587 if(outfile=="-"):
1588 catfp.seek(0, 0);
1589 if(hasattr(sys.stdout, "buffer")):
1590 shutil.copyfileobj(catfp, sys.stdout.buffer);
1591 else:
1592 shutil.copyfileobj(catfp, sys.stdout);
1593 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1594 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1595 catfp.seek(0, 0);
1596 upload_file_to_internet_file(catfp, outfile);
1597 if(returnfp):
1598 catfp.seek(0, 0);
1599 return catfp;
1600 else:
1601 catfp.close();
1602 return True;
1603
1604 def AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt=False, compression="auto", compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False, returnfp=False):
1605 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1606 if(os.path.exists(outfile)):
1607 try:
1608 os.unlink(outfile);
1609 except OSError as e:
1610 pass;
1611 if(outfile=="-"):
1612 verbose = False;
1613 catfpfp = BytesIO();
1614 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1615 catfp = outfile;
1616 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1617 catfp = BytesIO();
1618 else:
1619 fbasename = os.path.splitext(outfile)[0];
1620 fextname = os.path.splitext(outfile)[1];
1621 catfp = CompressOpenFile(outfile, True, compressionlevel);
1622 catfp = AppendListsWithContent(inlist, catfp, dirlistfromtxt, filevalues, extradata, followlink, checksumtype, formatspecs, verbose);
1623 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1624 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1625 try:
1626 catfp.flush();
1627 os.fsync(catfp.fileno());
1628 except io.UnsupportedOperation:
1629 pass;
1630 except AttributeError:
1631 pass;
1632 except OSError as e:
1633 pass;
1634 if(outfile=="-"):
1635 catfp.seek(0, 0);
1636 if(hasattr(sys.stdout, "buffer")):
1637 shutil.copyfileobj(catfp, sys.stdout.buffer);
1638 else:
1639 shutil.copyfileobj(catfp, sys.stdout);
1640 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1641 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1642 catfp.seek(0, 0);
1643 upload_file_to_internet_file(catfp, outfile);
1644 if(returnfp):
1645 catfp.seek(0, 0);
1646 return catfp;
1647 else:
1648 catfp.close();
1649 return True;
1650
1651 def AppendInFileWithContentToOutFile(infile, outfile, dirlistfromtxt=False, compression="auto", compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False, returnfp=False):
1652 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, False, formatspecs);
1653 return AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt, compression, compressionlevel, filevalues, extradata, followlink, checksumtype, formatspecs, verbose, returnfp);
1654
1655 def PrintPermissionString(fchmode, ftype):
1656 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
1657 permissionstr = "";
1658 for fmodval in str(oct(fchmode))[-3:]:
1659 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
1660 if(ftype==0 or ftype==7):
1661 permissionstr = "-" + permissionstr;
1662 if(ftype==1):
1663 permissionstr = "h" + permissionstr;
1664 if(ftype==2):
1665 permissionstr = "l" + permissionstr;
1666 if(ftype==3):
1667 permissionstr = "c" + permissionstr;
1668 if(ftype==4):
1669 permissionstr = "b" + permissionstr;
1670 if(ftype==5):
1671 permissionstr = "d" + permissionstr;
1672 if(ftype==6):
1673 permissionstr = "f" + permissionstr;
1674 if(ftype==8):
1675 permissionstr = "D" + permissionstr;
1676 if(ftype==9):
1677 permissionstr = "p" + permissionstr;
1678 if(ftype==10):
1679 permissionstr = "w" + permissionstr;
1680 try:
1681 permissionoutstr = stat.filemode(fchmode);
1682 except AttributeError:
1683 permissionoutstr = permissionstr;
1684 except KeyError:
1685 permissionoutstr = permissionstr;
1686 return permissionoutstr;
1687
1688 def PrintPermissionStringAlt(fchmode, ftype):
1689 permissions = {
1690 '0': '---', '1': '--x', '2': '-w-', '3': '-wx',
1691 '4': 'r--', '5': 'r-x', '6': 'rw-', '7': 'rwx'
1692 };
1693 # Translate file mode into permission string
1694 permissionstr = ''.join([permissions[i] for i in str(oct(fchmode))[-3:]]);
1695 # Append file type indicator
1696 type_indicators = {
1697 0: '-', 1: 'h', 2: 'l', 3: 'c', 4: 'b',
1698 5: 'd', 6: 'f', 8: 'D', 9: 'p', 10: 'w'
1699 };
1700 file_type = type_indicators.get(ftype, '-');
1701 permissionstr = file_type + permissionstr;
1702 try:
1703 permissionoutstr = stat.filemode(fchmode);
1704 except AttributeError:
1705 permissionoutstr = permissionstr;
1706 return permissionoutstr;
1707
1708 def CompressionSupport():
1709 compression_list = [];
1710 try:
1711 import gzip;
1712 compression_list.append("gz");
1713 compression_list.append("gzip");
1714 except ImportError:
1715 '''return False;'''
1716 try:
1717 import bz2;
1718 compression_list.append("bz2");
1719 compression_list.append("bzip2");
1720 except ImportError:
1721 '''return False;'''
1722 try:
1723 import lz4;
1724 compression_list.append("lz4");
1725 except ImportError:
1726 '''return False;'''
1727 try:
1728 import lzo;
1729 compression_list.append("lzo");
1730 compression_list.append("lzop");
1731 except ImportError:
1732 '''return False;'''
1733 try:
1734 import zstandard;
1735 compression_list.append("zstd");
1736 compression_list.append("zstandard");
1737 except ImportError:
1738 '''return False;'''
1739 try:
1740 import lzma;
1741 compression_list.append("lzma");
1742 compression_list.append("xz");
1743 except ImportError:
1744 '''return False;'''
1745 return compression_list;
1746
1747 def CheckCompressionType(infile, formatspecs=__file_format_list__, closefp=True):
1748 if(hasattr(infile, "read") or hasattr(infile, "write")):
1749 catfp = infile;
1750 else:
1751 try:
1752 catfp = open(infile, "rb");
1753 except FileNotFoundError:
1754 return False;
1755 catfp.seek(0, 0);
1756 prefp = catfp.read(2);
1757 filetype = False;
1758 if(prefp==binascii.unhexlify("1f8b")):
1759 filetype = "gzip";
1760 catfp.seek(0, 0);
1761 prefp = catfp.read(3);
1762 if(prefp==binascii.unhexlify("425a68")):
1763 filetype = "bzip2";
1764 if(prefp==binascii.unhexlify("5d0000")):
1765 filetype = "lzma";
1766 catfp.seek(0, 0);
1767 prefp = catfp.read(4);
1768 if(prefp==binascii.unhexlify("28b52ffd")):
1769 filetype = "zstd";
1770 if(prefp==binascii.unhexlify("04224d18")):
1771 filetype = "lz4";
1772 if(prefp==binascii.unhexlify("504B0304")):
1773 filetype = "zipfile";
1774 catfp.seek(0, 0);
1775 prefp = catfp.read(5);
1776 if(prefp==binascii.unhexlify("7573746172")):
1777 filetype = "tarfile";
1778 catfp.seek(0, 0);
1779 prefp = catfp.read(6);
1780 if(prefp==binascii.unhexlify("fd377a585a00")):
1781 filetype = "lzma";
1782 if(prefp==binascii.unhexlify("377abcaf271c")):
1783 filetype = "7zipfile";
1784 catfp.seek(0, 0);
1785 prefp = catfp.read(7);
1786 if(prefp==binascii.unhexlify("526172211a0700")):
1787 filetype = "rarfile";
1788 if(prefp==binascii.unhexlify("43617446696c65")):
1789 filetype = "catfile";
1790 catfp.seek(0, 0);
1791 prefp = catfp.read(8);
1792 if(prefp==binascii.unhexlify("526172211a070100")):
1793 filetype = "rarfile";
1794 catfp.seek(0, 0);
1795 prefp = catfp.read(formatspecs[3]);
1796 if(prefp==binascii.unhexlify(formatspecs[4])):
1797 filetype = formatspecs[2];
1798 catfp.seek(0, 0);
1799 prefp = catfp.read(9);
1800 if(prefp==binascii.unhexlify("894c5a4f000d0a1a0a")):
1801 filetype = "lzo";
1802 catfp.seek(0, 0);
1803 prefp = catfp.read(10);
1804 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
1805 filetype = "tarfile";
1806 catfp.seek(0, 0);
1807 if(filetype=="gzip" or filetype=="bzip2" or filetype=="lzma" or filetype=="zstd" or filetype=="lz4"):
1808 if(is_tarfile(catfp)):
1809 filetype = "tarfile";
1810 if(not filetype):
1811 if(is_tarfile(catfp)):
1812 filetype = "tarfile";
1813 elif(zipfile.is_zipfile(catfp)):
1814 filetype = "zipfile";
1815 elif(rarfile.is_rarfile(catfp) or rarfile.is_rarfile_sfx(catfp)):
1816 filetype = "rarile";
1817 else:
1818 filetype = False;
1819 catfp.seek(0, 0);
1820 if(closefp):
1821 catfp.close();
1822 return filetype;
1823
1824 def CheckCompressionTypeFromString(instring, formatspecs=__file_format_list__, closefp=True):
1825 try:
1826 instringsfile = BytesIO(instring);
1827 except TypeError:
1828 instringsfile = BytesIO(instring.encode("UTF-8"));
1829 return CheckCompressionType(instringsfile, formatspecs, closefp);
1830
1831 def GetCompressionMimeType(infile, formatspecs=__file_format_list__):
1832 compresscheck = CheckCompressionType(fp, formatspecs, False);
1833 if(compresscheck=="gzip" or compresscheck=="gz"):
1834 return archivefile_gzip_mimetype;
1835 if(compresscheck=="bzip2" or compresscheck=="bz2"):
1836 return archivefile_bzip2_mimetype;
1837 if(compresscheck=="zstd" or compresscheck=="zstandard"):
1838 return archivefile_zstandard_mimetype;
1839 if(compresscheck=="lz4"):
1840 return archivefile_lz4_mimetype;
1841 if(compresscheck=="lzo" or compresscheck=="lzop"):
1842 return archivefile_lzop_mimetype;
1843 if(compresscheck=="lzma"):
1844 return archivefile_lzma_mimetype;
1845 if(compresscheck=="xz"):
1846 return archivefile_xz_mimetype;
1847 if(compresscheck=="catfile" or compresscheck=="cat" or compresscheck==formatspecs[2]):
1848 return archivefile_cat_mimetype;
1849 if(not compresscheck):
1850 return False;
1851 return False;
1852
1853 def UncompressArchiveFile(fp, formatspecs=__file_format_list__):
1854 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
1855 return False;
1856 compresscheck = CheckCompressionType(fp, formatspecs, False);
1857 if(compresscheck=="gzip"):
1858 try:
1859 import gzip;
1860 except ImportError:
1861 return False;
1862 catfp = gzip.GzipFile(fileobj=fp, mode="rb");
1863 if(compresscheck=="bzip2"):
1864 try:
1865 import bz2;
1866 except ImportError:
1867 return False;
1868 catfp = BytesIO();
1869 catfp.write(bz2.decompress(fp.read()));
1870 if(compresscheck=="zstd"):
1871 try:
1872 import zstandard;
1873 except ImportError:
1874 return False;
1875 catfp = BytesIO();
1876 catfp.write(zstandard.decompress(fp.read()));
1877 if(compresscheck=="lz4"):
1878 try:
1879 import lz4.frame;
1880 except ImportError:
1881 return False;
1882 catfp = BytesIO();
1883 catfp.write(lz4.frame.decompress(fp.read()));
1884 if(compresscheck=="lzo" or compresscheck=="lzop"):
1885 try:
1886 import lzo;
1887 except ImportError:
1888 return False;
1889 catfp = BytesIO();
1890 catfp.write(lzo.decompress(fp.read()));
1891 if(compresscheck=="lzma" or compresscheck=="xz"):
1892 try:
1893 import lzma;
1894 except ImportError:
1895 return False;
1896 catfp = BytesIO();
1897 catfp.write(lzma.decompress(fp.read()));
1898 if(compresscheck=="catfile" or compresscheck==formatspecs[2]):
1899 catfp = fp;
1900 if(not compresscheck):
1901 try:
1902 import lzma;
1903 except ImportError:
1904 return False;
1905 catfp = BytesIO();
1906 with fp as fpcontent:
1907 try:
1908 catfp.write(lzma.decompress(fp.read()));
1909 except lzma.LZMAError:
1910 return False;
1911 if(compresscheck!="catfile" or compresscheck!=formatspecs[2]):
1912 fp.close();
1913 return catfp;
1914
1915 create_alias_function("Uncompress", __file_format_name__, "", UncompressArchiveFile);
1916
1917 def UncompressFile(infile, formatspecs=__file_format_list__, mode="rb"):
1918 compresscheck = CheckCompressionType(infile, formatspecs, False);
1919 if(sys.version_info[0]==2 and compresscheck):
1920 if(mode=="rt"):
1921 mode = "r";
1922 if(mode=="wt"):
1923 mode = "w";
1924 try:
1925 if(compresscheck=="gzip"):
1926 try:
1927 import gzip;
1928 except ImportError:
1929 return False;
1930 try:
1931 filefp = gzip.open(infile, mode, encoding="UTF-8");
1932 except (ValueError, TypeError) as e:
1933 filefp = gzip.open(infile, mode);
1934 if(compresscheck=="bzip2"):
1935 try:
1936 import bz2;
1937 except ImportError:
1938 return False;
1939 try:
1940 filefp = bz2.open(infile, mode, encoding="UTF-8");
1941 except (ValueError, TypeError) as e:
1942 filefp = bz2.open(infile, mode);
1943 if(compresscheck=="zstd"):
1944 try:
1945 import zstandard;
1946 except ImportError:
1947 return False;
1948 try:
1949 filefp = zstandard.open(infile, mode, encoding="UTF-8");
1950 except (ValueError, TypeError) as e:
1951 filefp = zstandard.open(infile, mode);
1952 if(compresscheck=="lz4"):
1953 try:
1954 import lz4.frame;
1955 except ImportError:
1956 return False;
1957 try:
1958 filefp = lz4.frame.open(infile, mode, encoding="UTF-8");
1959 except (ValueError, TypeError) as e:
1960 filefp = lz4.frame.open(infile, mode);
1961 if(compresscheck=="lzo"):
1962 try:
1963 import lzo;
1964 except ImportError:
1965 return False;
1966 try:
1967 filefp = lzo.open(infile, mode, encoding="UTF-8");
1968 except (ValueError, TypeError) as e:
1969 filefp = lzo.open(infile, mode);
1970 if(compresscheck=="lzma"):
1971 try:
1972 import lzma;
1973 except ImportError:
1974 return False;
1975 try:
1976 filefp = lzma.open(infile, mode, encoding="UTF-8");
1977 except (ValueError, TypeError) as e:
1978 filefp = lzma.open(infile, mode);
1979 if(compresscheck=="catfile" or compresscheck==formatspecs[2]):
1980 try:
1981 filefp = open(infile, mode, encoding="UTF-8");
1982 except (ValueError, TypeError) as e:
1983 filefp = open(infile, mode);
1984 if(not compresscheck):
1985 try:
1986 filefp = open(infile, mode, encoding="UTF-8");
1987 except (ValueError, TypeError) as e:
1988 filefp = open(infile, mode);
1989 except FileNotFoundError:
1990 return False;
1991 return filefp;
1992
1993 def UncompressString(infile):
1994 compresscheck = CheckCompressionTypeFromString(infile, formatspecs, False);
1995 if(compresscheck=="gzip"):
1996 try:
1997 import gzip;
1998 except ImportError:
1999 return False;
2000 fileuz = gzip.decompress(infile);
2001 if(compresscheck=="bzip2"):
2002 try:
2003 import bz2;
2004 except ImportError:
2005 return False;
2006 fileuz = bz2.decompress(infile);
2007 if(compresscheck=="zstd"):
2008 try:
2009 import zstandard;
2010 except ImportError:
2011 return False;
2012 fileuz = zstandard.decompress(infile);
2013 if(compresscheck=="lz4"):
2014 try:
2015 import lz4.frame;
2016 except ImportError:
2017 return False;
2018 fileuz = lz4.frame.decompress(infile);
2019 if(compresscheck=="lzo"):
2020 try:
2021 import lzo;
2022 except ImportError:
2023 return False;
2024 fileuz = lzo.decompress(infile);
2025 if(compresscheck=="lzma"):
2026 try:
2027 import lzma;
2028 except ImportError:
2029 return False;
2030 fileuz = lzma.decompress(infile);
2031 if(not compresscheck):
2032 fileuz = infile;
2033 if(hasattr(fileuz, 'decode')):
2034 fileuz = fileuz.decode("UTF-8");
2035 return fileuz;
2036
2037 def UncompressStringAlt(infile):
2038 filefp = StringIO();
2039 outstring = UncompressString(infile);
2040 filefp.write(outstring);
2041 filefp.seek(0, 0);
2042 return filefp;
2043
2044 def CheckCompressionSubType(infile, formatspecs=__file_format_list__, closefp=True):
2045 compresscheck = CheckCompressionType(infile, formatspecs, False);
2046 if(not compresscheck):
2047 fextname = os.path.splitext(infile)[1];
2048 if(fextname==".gz"):
2049 compresscheck = "gzip";
2050 elif(fextname==".bz2"):
2051 compresscheck = "bzip2";
2052 elif(fextname==".zst"):
2053 compresscheck = "zstd";
2054 elif(fextname==".lz4"):
2055 compresscheck = "lz4";
2056 elif(fextname==".lzo" or fextname==".lzop"):
2057 compresscheck = "lzo";
2058 elif(fextname==".lzma" or fextname==".xz"):
2059 compresscheck = "lzma";
2060 else:
2061 return False;
2062 if(compresscheck=="gzip" or compresscheck=="bzip2" or compresscheck=="lzma" or compresscheck=="zstd" or compresscheck=="lz4"):
2063 if(is_tarfile(infile)):
2064 filetype = "tarfile";
2065 if(not compresscheck):
2066 if(is_tarfile(infile)):
2067 return "tarfile";
2068 elif(zipfile.is_zipfile(infile)):
2069 return "zipfile";
2070 elif(rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile)):
2071 return "rarile";
2072 else:
2073 return False;
2074 return False;
2075 if(compresscheck=="catfile"):
2076 return "catfile";
2077 if(compresscheck==formatspecs[2]):
2078 return formatspecs[2];
2079 if(compresscheck=="tarfile"):
2080 return "tarfile";
2081 if(compresscheck=="zipfile"):
2082 return "zipfile";
2083 if(rarfile_support and compresscheck=="rarfile"):
2084 return "rarfile";
2085 if(py7zr_support and compresscheck=="7zipfile"):
2086 return "7zipfile";
2087 if(hasattr(infile, "read") or hasattr(infile, "write")):
2088 catfp = UncompressArchiveFile(infile, formatspecs[2]);
2089 else:
2090 try:
2091 if(compresscheck=="gzip"):
2092 try:
2093 import gzip;
2094 except ImportError:
2095 return False;
2096 catfp = gzip.GzipFile(infile, "rb");
2097 if(compresscheck=="bzip2"):
2098 try:
2099 import bz2;
2100 except ImportError:
2101 return False;
2102 catfp = bz2.BZ2File(infile, "rb");
2103 if(compresscheck=="lz4"):
2104 try:
2105 import lz4.frame;
2106 except ImportError:
2107 return False;
2108 catfp = lz4.frame.open(infile, "rb");
2109 if(compresscheck=="zstd"):
2110 try:
2111 import zstandard;
2112 except ImportError:
2113 return False;
2114 catfp = zstandard.open(infile, "rb");
2115 if(compresscheck=="lzma" or compresscheck=="xz"):
2116 try:
2117 import lzma;
2118 except ImportError:
2119 return False;
2120 catfp = lzma.open(infile, "rb");
2121 except FileNotFoundError:
2122 return False;
2123 filetype = False;
2124 prefp = catfp.read(5);
2125 if(prefp==binascii.unhexlify("7573746172")):
2126 filetype = "tarfile";
2127 catfp.seek(0, 0);
2128 prefp = catfp.read(7);
2129 if(prefp==binascii.unhexlify("43617446696c65")):
2130 filetype = "catfile";
2131 catfp.seek(0, 0);
2132 prefp = catfp.read(formatspecs[3]);
2133 if(prefp==binascii.unhexlify(formatspecs[4])):
2134 filetype = formatspecs[2];
2135 catfp.seek(0, 0);
2136 prefp = catfp.read(10);
2137 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
2138 filetype = "tarfile";
2139 catfp.seek(0, 0);
2140 if(closefp):
2141 catfp.close();
2142 return filetype;
2143
2144 def GZipCompress(data, compresslevel=9):
2145 try:
2146 import gzip;
2147 except ImportError:
2148 return False;
2149 tmpfp = tempfile.NamedTemporaryFile("wb", delete=False);
2150 tmpfp.close();
2151 tmpfp = gzip.GzipFile(tmpfp.name, mode="wb", compresslevel=compresslevel);
2152 tmpfp.write(data);
2153 tmpfp.close();
2154 try:
2155 catfp = open(tmpfp.name, "rb");
2156 except FileNotFoundError:
2157 return False;
2158 catdata = catfp.read();
2159 catfp.close();
2160 return catdata;
2161
2162 def CompressArchiveFile(fp, compression="auto", compressionlevel=None, formatspecs=__file_format_list__):
2163 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
2164 return False;
2165 fp.seek(0, 0);
2166 if(not compression or compression=="catfile" or compression==formatspecs[2]):
2167 compression = "auto";
2168 if(compression not in compressionlist and compression is None):
2169 compression = "auto";
2170 if(compression=="gzip"):
2171 try:
2172 import gzip;
2173 except ImportError:
2174 return False;
2175 catfp = BytesIO();
2176 if(compressionlevel is None):
2177 compressionlevel = 9;
2178 else:
2179 compressionlevel = int(compressionlevel);
2180 catfp.write(gzip.compress(fp.read(), compresslevel=compressionlevel));
2181 if(compression=="bzip2"):
2182 try:
2183 import bz2;
2184 except ImportError:
2185 return False;
2186 catfp = BytesIO();
2187 if(compressionlevel is None):
2188 compressionlevel = 9;
2189 else:
2190 compressionlevel = int(compressionlevel);
2191 catfp.write(bz2.compress(fp.read(), compresslevel=compressionlevel));
2192 if(compression=="lz4"):
2193 try:
2194 import lz4.frame;
2195 except ImportError:
2196 return False;
2197 catfp = BytesIO();
2198 if(compressionlevel is None):
2199 compressionlevel = 9;
2200 else:
2201 compressionlevel = int(compressionlevel);
2202 catfp.write(lz4.frame.compress(fp.read(), compression_level=compressionlevel));
2203 if(compression=="lzo" or compression=="lzop"):
2204 try:
2205 import lzo;
2206 except ImportError:
2207 return False;
2208 catfp = BytesIO();
2209 if(compressionlevel is None):
2210 compressionlevel = 9;
2211 else:
2212 compressionlevel = int(compressionlevel);
2213 catfp.write(lzo.compress(fp.read(), compresslevel=compressionlevel));
2214 if(compression=="zstd"):
2215 try:
2216 import zstandard;
2217 except ImportError:
2218 return False;
2219 catfp = BytesIO();
2220 if(compressionlevel is None):
2221 compressionlevel = 10;
2222 else:
2223 compressionlevel = int(compressionlevel);
2224 catfp.write(zstandard.compress(fp.read(), level=compressionlevel));
2225 if(compression=="lzma"):
2226 try:
2227 import lzma;
2228 except ImportError:
2229 return False;
2230 catfp = BytesIO();
2231 if(compressionlevel is None):
2232 compressionlevel = 9;
2233 else:
2234 compressionlevel = int(compressionlevel);
2235 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]));
2236 if(compression=="xz"):
2237 try:
2238 import lzma;
2239 except ImportError:
2240 return False;
2241 catfp = BytesIO();
2242 if(compressionlevel is None):
2243 compressionlevel = 9;
2244 else:
2245 compressionlevel = int(compressionlevel);
2246 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]));
2247 if(compression=="auto" or compression is None):
2248 catfp = fp;
2249 catfp.seek(0, 0);
2250 return catfp;
2251
2252 create_alias_function("Compress", __file_format_name__, "", CompressArchiveFile);
2253
2254 def CompressOpenFile(outfile, compressionenable=True, compressionlevel=None):
2255 if(outfile is None):
2256 return False;
2257 fbasename = os.path.splitext(outfile)[0];
2258 fextname = os.path.splitext(outfile)[1];
2259 if(compressionlevel is None and fextname!=".zst"):
2260 compressionlevel = 9;
2261 elif(compressionlevel is None and fextname==".zst"):
2262 compressionlevel = 10;
2263 else:
2264 compressionlevel = int(compressionlevel);
2265 if(sys.version_info[0]==2):
2266 mode = "w";
2267 else:
2268 mode = "wb";
2269 try:
2270 if(fextname not in outextlistwd or not compressionenable):
2271 try:
2272 outfp = open(outfile, "wb", encoding="UTF-8");
2273 except (ValueError, TypeError) as e:
2274 outfp = open(outfile, "wb");
2275 elif(fextname==".gz"):
2276 try:
2277 import gzip;
2278 except ImportError:
2279 return False;
2280 try:
2281 outfp = gzip.open(outfile, mode, compressionlevel, encoding="UTF-8");
2282 except (ValueError, TypeError) as e:
2283 outfp = gzip.open(outfile, mode, compressionlevel);
2284 elif(fextname==".bz2"):
2285 try:
2286 import bz2;
2287 except ImportError:
2288 return False;
2289 try:
2290 outfp = bz2.open(outfile, mode, compressionlevel, encoding="UTF-8");
2291 except (ValueError, TypeError) as e:
2292 outfp = bz2.open(outfile, mode, compressionlevel);
2293 elif(fextname==".zst"):
2294 try:
2295 import zstandard;
2296 except ImportError:
2297 return False;
2298 try:
2299 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel), encoding="UTF-8");
2300 except (ValueError, TypeError) as e:
2301 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel));
2302 elif(fextname==".xz"):
2303 try:
2304 import lzma;
2305 except ImportError:
2306 return False;
2307 try:
2308 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}], encoding="UTF-8");
2309 except (ValueError, TypeError) as e:
2310 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]);
2311 elif(fextname==".lz4"):
2312 try:
2313 import lz4.frame;
2314 except ImportError:
2315 return False;
2316 try:
2317 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel, encoding="UTF-8");
2318 except (ValueError, TypeError) as e:
2319 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel);
2320 elif(fextname==".lzo"):
2321 try:
2322 import lzo;
2323 except ImportError:
2324 return False;
2325 try:
2326 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel, encoding="UTF-8");
2327 except (ValueError, TypeError) as e:
2328 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel);
2329 elif(fextname==".lzma"):
2330 try:
2331 import lzma;
2332 except ImportError:
2333 return False;
2334 try:
2335 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}], encoding="UTF-8");
2336 except (ValueError, TypeError) as e:
2337 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]);
2338 except FileNotFoundError:
2339 return False;
2340 return outfp;
2341
2342 def GetDevMajorMinor(fdev):
2343 retdev = [];
2344 if(hasattr(os, "minor")):
2345 retdev.append(os.minor(fdev));
2346 else:
2347 retdev.append(0);
2348 if(hasattr(os, "major")):
2349 retdev.append(os.major(fdev));
2350 else:
2351 retdev.append(0);
2352 return retdev;
2353
2354 def CheckSumSupport(checkfor, guaranteed=True):
2355 if(guaranteed):
2356 hash_list = sorted(list(hashlib.algorithms_guaranteed));
2357 else:
2358 hash_list = sorted(list(hashlib.algorithms_available));
2359 checklistout = sorted(hash_list + ['adler32', 'crc16', 'crc16_ansi', 'crc16_ibm', 'crc16_ccitt', 'crc32', 'crc64', 'crc64_ecma', 'crc64_iso', 'none']);
2360 if(checkfor in checklistout):
2361 return True;
2362 else:
2363 return False;
2364
2365 def CheckSumSupportAlt(checkfor, guaranteed=True):
2366 if(guaranteed):
2367 hash_list = sorted(list(hashlib.algorithms_guaranteed));
2368 else:
2369 hash_list = sorted(list(hashlib.algorithms_available));
2370 checklistout = hash_list;
2371 if(checkfor in checklistout):
2372 return True;
2373 else:
2374 return False;
2375
2376 def PackArchiveFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
2377 advancedlist = formatspecs[8];
2378 altinode = formatspecs[9];
2379 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2380 outfile = RemoveWindowsPath(outfile);
2381 checksumtype = checksumtype.lower();
2382 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
2383 checksumtype="crc32";
2384 if(checksumtype=="none"):
2385 checksumtype = "";
2386 if(not compression or compression=="catfile" or compression==formatspecs[2]):
2387 compression = "auto";
2388 if(compression not in compressionlist and compression is None):
2389 compression = "auto";
2390 if(verbose):
2391 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2392 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2393 if(os.path.exists(outfile)):
2394 try:
2395 os.unlink(outfile);
2396 except OSError as e:
2397 pass;
2398 if(outfile=="-"):
2399 verbose = False;
2400 catfp = BytesIO();
2401 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2402 catfp = outfile;
2403 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2404 catfp = BytesIO();
2405 else:
2406 fbasename = os.path.splitext(outfile)[0];
2407 fextname = os.path.splitext(outfile)[1];
2408 catfp = CompressOpenFile(outfile, True, compressionlevel);
2409 catver = formatspecs[6];
2410 fileheaderver = str(int(catver.replace(".", "")));
2411 infilelist = [];
2412 if(infiles=="-"):
2413 for line in sys.stdin:
2414 infilelist.append(line.strip());
2415 infilelist = list(filter(None, infilelist));
2416 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
2417 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
2418 return False;
2419 with UncompressFile(infiles, formatspecs, "r") as finfile:
2420 for line in finfile:
2421 infilelist.append(line.strip());
2422 infilelist = list(filter(None, infilelist));
2423 else:
2424 if(isinstance(infiles, (list, tuple, ))):
2425 infilelist = list(filter(None, infiles));
2426 elif(isinstance(infiles, (str, ))):
2427 infilelist = list(filter(None, [infiles]));
2428 if(advancedlist):
2429 GetDirList = ListDirAdvanced(infilelist, followlink, False);
2430 else:
2431 GetDirList = ListDir(infilelist, followlink, False);
2432 if(not GetDirList):
2433 return False;
2434 curinode = 0;
2435 curfid = 0;
2436 inodelist = [];
2437 inodetofile = {};
2438 filetoinode = {};
2439 inodetocatinode = {};
2440 numfiles = int(len(GetDirList));
2441 AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
2442 for curfname in GetDirList:
2443 if(re.findall("^[.|/]", curfname)):
2444 fname = curfname;
2445 else:
2446 fname = "./"+curfname;
2447 if(verbose):
2448 VerbosePrintOut(fname);
2449 if(not followlink or followlink is None):
2450 fstatinfo = os.lstat(fname);
2451 else:
2452 fstatinfo = os.stat(fname);
2453 fpremode = fstatinfo.st_mode;
2454 finode = fstatinfo.st_ino;
2455 flinkcount = fstatinfo.st_nlink;
2456 ftype = 0;
2457 if(stat.S_ISREG(fpremode)):
2458 ftype = 0;
2459 elif(stat.S_ISLNK(fpremode)):
2460 ftype = 2;
2461 elif(stat.S_ISCHR(fpremode)):
2462 ftype = 3;
2463 elif(stat.S_ISBLK(fpremode)):
2464 ftype = 4;
2465 elif(stat.S_ISDIR(fpremode)):
2466 ftype = 5;
2467 elif(stat.S_ISFIFO(fpremode)):
2468 ftype = 6;
2469 elif(stat.S_ISSOCK(fpremode)):
2470 ftype = 8;
2471 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
2472 ftype = 9;
2473 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
2474 ftype = 10;
2475 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
2476 ftype = 111;
2477 else:
2478 ftype = 0;
2479 flinkname = "";
2480 fcurfid = format(int(curfid), 'x').lower();
2481 if(not followlink and finode!=0):
2482 if(ftype!=1):
2483 if(finode in inodelist):
2484 ftype = 1;
2485 flinkname = inodetofile[finode];
2486 if(altinode):
2487 fcurinode = format(int(finode), 'x').lower();
2488 else:
2489 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
2490 if(finode not in inodelist):
2491 inodelist.append(finode);
2492 inodetofile.update({finode: fname});
2493 inodetocatinode.update({finode: curinode});
2494 if(altinode):
2495 fcurinode = format(int(finode), 'x').lower();
2496 else:
2497 fcurinode = format(int(curinode), 'x').lower();
2498 curinode = curinode + 1;
2499 else:
2500 fcurinode = format(int(curinode), 'x').lower();
2501 curinode = curinode + 1;
2502 curfid = curfid + 1;
2503 if(ftype==2):
2504 flinkname = os.readlink(fname);
2505 fdev = fstatinfo.st_dev;
2506 getfdev = GetDevMajorMinor(fdev);
2507 fdev_minor = getfdev[0];
2508 fdev_major = getfdev[1];
2509 frdev = fstatinfo.st_dev;
2510 if(hasattr(fstatinfo, "st_rdev")):
2511 frdev = fstatinfo.st_rdev;
2512 else:
2513 frdev = fstatinfo.st_dev;
2514 getfrdev = GetDevMajorMinor(frdev);
2515 frdev_minor = getfrdev[0];
2516 frdev_major = getfrdev[1];
2517 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
2518 fsize = format(int("0"), 'x').lower();
2519 elif(ftype==0 or ftype==7):
2520 fsize = format(int(fstatinfo.st_size), 'x').lower();
2521 else:
2522 fsize = format(int(fstatinfo.st_size)).lower();
2523 fatime = format(int(fstatinfo.st_atime), 'x').lower();
2524 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
2525 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
2526 if(hasattr(fstatinfo, "st_birthtime")):
2527 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
2528 else:
2529 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
2530 fmode = format(int(fstatinfo.st_mode), 'x').lower();
2531 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
2532 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
2533 fuid = format(int(fstatinfo.st_uid), 'x').lower();
2534 fgid = format(int(fstatinfo.st_gid), 'x').lower();
2535 funame = "";
2536 try:
2537 import pwd;
2538 try:
2539 userinfo = pwd.getpwuid(fstatinfo.st_uid);
2540 funame = userinfo.pw_name;
2541 except KeyError:
2542 funame = "";
2543 except ImportError:
2544 funame = "";
2545 fgname = "";
2546 try:
2547 import grp;
2548 try:
2549 groupinfo = grp.getgrgid(fstatinfo.st_gid);
2550 fgname = groupinfo.gr_name;
2551 except KeyError:
2552 fgname = "";
2553 except ImportError:
2554 fgname = "";
2555 fdev_minor = format(int(fdev_minor), 'x').lower();
2556 fdev_major = format(int(fdev_major), 'x').lower();
2557 frdev_minor = format(int(frdev_minor), 'x').lower();
2558 frdev_major = format(int(frdev_major), 'x').lower();
2559 finode = format(int(finode), 'x').lower();
2560 flinkcount = format(int(flinkcount), 'x').lower();
2561 if(hasattr(fstatinfo, "st_file_attributes")):
2562 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
2563 else:
2564 fwinattributes = format(int(0), 'x').lower();
2565 fcompression = "none";
2566 fcsize = format(int(0), 'x').lower();
2567 fcontents = BytesIO();
2568 if(ftype==0 or ftype==7):
2569 with open(fname, "rb") as fpc:
2570 shutil.copyfileobj(fpc, fcontents);
2571 if(followlink and (ftype==1 or ftype==2)):
2572 flstatinfo = os.stat(flinkname);
2573 with open(flinkname, "rb") as fpc:
2574 shutil.copyfileobj(fpc, fcontents);
2575 fcontents.seek(0, 0);
2576 ftypehex = format(ftype, 'x').lower();
2577 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
2578 AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
2579 fcontents.close();
2580 if(numfiles>0):
2581 catfp.write(AppendNullBytes([0, 0], formatspecs[5]).encode("UTF-8"));
2582 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
2583 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2584 try:
2585 catfp.flush();
2586 os.fsync(catfp.fileno());
2587 except io.UnsupportedOperation:
2588 pass;
2589 except AttributeError:
2590 pass;
2591 except OSError as e:
2592 pass;
2593 if(outfile=="-"):
2594 catfp.seek(0, 0);
2595 if(hasattr(sys.stdout, "buffer")):
2596 shutil.copyfileobj(catfp, sys.stdout.buffer);
2597 else:
2598 shutil.copyfileobj(catfp, sys.stdout);
2599 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2600 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2601 catfp.seek(0, 0);
2602 upload_file_to_internet_file(catfp, outfile);
2603 if(returnfp):
2604 catfp.seek(0, 0);
2605 return catfp;
2606 else:
2607 catfp.close();
2608 return True;
2609
2610 create_alias_function("Pack", __file_format_name__, "", PackArchiveFile);
2611
2612 if(hasattr(shutil, "register_archive_format")):
2613 def PackArchiveFileFunc(archive_name, source_dir, **kwargs):
2614 return PackArchiveFile(source_dir, archive_name, False, "auto", None, False, "crc32", [], __file_format_delimiter__, False, False);
2615 create_alias_function("Pack", __file_format_name__, "Func", PackArchiveFileFunc);
2616
2617 def PackArchiveFileFromDirList(infiles, outfile, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
2618 return PackArchiveFile(infiles, outfile, dirlistfromtxt, compression, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, returnfp);
2619
2620 create_alias_function("Pack", __file_format_name__, "FromDirList", PackArchiveFileFromDirList);
2621
2622 def PackArchiveFileFromTarFile(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
2623 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2624 outfile = RemoveWindowsPath(outfile);
2625 checksumtype = checksumtype.lower();
2626 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
2627 checksumtype="crc32";
2628 if(checksumtype=="none"):
2629 checksumtype = "";
2630 if(not compression or compression=="catfile" or compression==formatspecs[2]):
2631 compression = "auto";
2632 if(compression not in compressionlist and compression is None):
2633 compression = "auto";
2634 if(verbose):
2635 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2636 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2637 if(os.path.exists(outfile)):
2638 try:
2639 os.unlink(outfile);
2640 except OSError as e:
2641 pass;
2642 if(outfile=="-"):
2643 verbose = False;
2644 catfp = BytesIO();
2645 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2646 catfp = outfile;
2647 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2648 catfp = BytesIO();
2649 else:
2650 fbasename = os.path.splitext(outfile)[0];
2651 fextname = os.path.splitext(outfile)[1];
2652 catfp = CompressOpenFile(outfile, True, compressionlevel);
2653 catver = formatspecs[6];
2654 fileheaderver = str(int(catver.replace(".", "")));
2655 curinode = 0;
2656 curfid = 0;
2657 inodelist = [];
2658 inodetofile = {};
2659 filetoinode = {};
2660 inodetocatinode = {};
2661 if(infile=="-"):
2662 infile = BytesIO();
2663 if(hasattr(sys.stdin, "buffer")):
2664 shutil.copyfileobj(sys.stdin.buffer, infile);
2665 else:
2666 shutil.copyfileobj(sys.stdin, infile);
2667 infile.seek(0, 0);
2668 if(not infile):
2669 return False;
2670 infile.seek(0, 0);
2671 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
2672 infile = download_file_from_internet_file(infile);
2673 infile.seek(0, 0);
2674 if(not infile):
2675 return False;
2676 infile.seek(0, 0);
2677 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
2678 return False;
2679 elif(os.path.exists(infile) and os.path.isfile(infile)):
2680 try:
2681 if(not tarfile.is_tarfile(infile)):
2682 return False;
2683 except AttributeError:
2684 if(not is_tarfile(infile)):
2685 return False;
2686 else:
2687 return False;
2688 try:
2689 if(hasattr(infile, "read") or hasattr(infile, "write")):
2690 tarfp = tarfile.open(fileobj=infile, mode="r");
2691 else:
2692 tarfp = tarfile.open(infile, "r");
2693 except FileNotFoundError:
2694 return False;
2695 numfiles = int(len(tarfp.getmembers()));
2696 AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
2697 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
2698 if(re.findall("^[.|/]", member.name)):
2699 fname = member.name;
2700 else:
2701 fname = "./"+member.name;
2702 if(verbose):
2703 VerbosePrintOut(fname);
2704 fpremode = member.mode;
2705 ffullmode = member.mode;
2706 flinkcount = 0;
2707 ftype = 0;
2708 if(member.isreg()):
2709 ffullmode = member.mode + stat.S_IFREG;
2710 ftype = 0;
2711 elif(member.isdev()):
2712 ffullmode = member.mode;
2713 ftype = 7;
2714 elif(member.islnk()):
2715 ffullmode = member.mode + stat.S_IFREG;
2716 ftype = 1;
2717 elif(member.issym()):
2718 ffullmode = member.mode + stat.S_IFLNK;
2719 ftype = 2;
2720 elif(member.ischr()):
2721 ffullmode = member.mode + stat.S_IFCHR;
2722 ftype = 3;
2723 elif(member.isblk()):
2724 ffullmode = member.mode + stat.S_IFBLK;
2725 ftype = 4;
2726 elif(member.isdir()):
2727 ffullmode = member.mode + stat.S_IFDIR;
2728 ftype = 5;
2729 elif(member.isfifo()):
2730 ffullmode = member.mode + stat.S_IFIFO;
2731 ftype = 6;
2732 elif(member.issparse()):
2733 ffullmode = member.mode;
2734 ftype = 12;
2735 else:
2736 ffullmode = member.mode;
2737 ftype = 0;
2738 flinkname = "";
2739 fcurfid = format(int(curfid), 'x').lower();
2740 fcurinode = format(int(curfid), 'x').lower();
2741 curfid = curfid + 1;
2742 if(ftype==2):
2743 flinkname = member.linkname;
2744 fdev_minor = format(int(member.devminor), 'x').lower();
2745 fdev_major = format(int(member.devmajor), 'x').lower();
2746 frdev_minor = format(int(member.devminor), 'x').lower();
2747 frdev_major = format(int(member.devmajor), 'x').lower();
2748 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
2749 fsize = format(int("0"), 'x').lower();
2750 elif(ftype==0 or ftype==7):
2751 fsize = format(int(member.size), 'x').lower();
2752 else:
2753 fsize = format(int(member.size), 'x').lower();
2754 fatime = format(int(member.mtime), 'x').lower();
2755 fmtime = format(int(member.mtime), 'x').lower();
2756 fctime = format(int(member.mtime), 'x').lower();
2757 fbtime = format(int(member.mtime), 'x').lower();
2758 fmode = format(int(ffullmode), 'x').lower();
2759 fchmode = format(int(stat.S_IMODE(ffullmode)), 'x').lower();
2760 ftypemod = format(int(stat.S_IFMT(ffullmode)), 'x').lower();
2761 fuid = format(int(member.uid), 'x').lower();
2762 fgid = format(int(member.gid), 'x').lower();
2763 funame = member.uname;
2764 fgname = member.gname;
2765 flinkcount = format(int(flinkcount), 'x').lower();
2766 fwinattributes = format(int(0), 'x').lower();
2767 fcompression = "none";
2768 fcsize = format(int(0), 'x').lower();
2769 fcontents = BytesIO();
2770 if(ftype==0 or ftype==7):
2771 with tarfp.extractfile(member) as fpc:
2772 shutil.copyfileobj(fpc, fcontents);
2773 fcontents.seek(0, 0);
2774 ftypehex = format(ftype, 'x').lower();
2775 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
2776 AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
2777 fcontents.close();
2778 if(numfiles>0):
2779 catfp.write(AppendNullBytes([0, 0], formatspecs[5]).encode("UTF-8"));
2780 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
2781 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2782 try:
2783 catfp.flush();
2784 os.fsync(catfp.fileno());
2785 except io.UnsupportedOperation:
2786 pass;
2787 except AttributeError:
2788 pass;
2789 except OSError as e:
2790 pass;
2791 if(outfile=="-"):
2792 catfp.seek(0, 0);
2793 if(hasattr(sys.stdout, "buffer")):
2794 shutil.copyfileobj(catfp, sys.stdout.buffer);
2795 else:
2796 shutil.copyfileobj(catfp, sys.stdout);
2797 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2798 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2799 catfp.seek(0, 0);
2800 upload_file_to_internet_file(catfp, outfile);
2801 if(returnfp):
2802 catfp.seek(0, 0);
2803 return catfp;
2804 else:
2805 catfp.close();
2806 return True;
2807
2808 create_alias_function("Pack", __file_format_name__, "FromTarFile", PackArchiveFileFromTarFile);
2809
2810 def PackArchiveFileFromZipFile(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
2811 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2812 outfile = RemoveWindowsPath(outfile);
2813 checksumtype = checksumtype.lower();
2814 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
2815 checksumtype="crc32";
2816 if(checksumtype=="none"):
2817 checksumtype = "";
2818 if(not compression or compression=="catfile" or compression==formatspecs[2]):
2819 compression = "auto";
2820 if(compression not in compressionlist and compression is None):
2821 compression = "auto";
2822 if(verbose):
2823 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2824 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2825 if(os.path.exists(outfile)):
2826 try:
2827 os.unlink(outfile);
2828 except OSError as e:
2829 pass;
2830 if(outfile=="-"):
2831 verbose = False;
2832 catfp = BytesIO();
2833 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2834 catfp = outfile;
2835 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2836 catfp = BytesIO();
2837 else:
2838 fbasename = os.path.splitext(outfile)[0];
2839 fextname = os.path.splitext(outfile)[1];
2840 catfp = CompressOpenFile(outfile, True, compressionlevel);
2841 catver = formatspecs[6];
2842 fileheaderver = str(int(catver.replace(".", "")));
2843 curinode = 0;
2844 curfid = 0;
2845 inodelist = [];
2846 inodetofile = {};
2847 filetoinode = {};
2848 inodetocatinode = {};
2849 if(infile=="-"):
2850 infile = BytesIO();
2851 if(hasattr(sys.stdin, "buffer")):
2852 shutil.copyfileobj(sys.stdin.buffer, infile);
2853 else:
2854 shutil.copyfileobj(sys.stdin, infile);
2855 infile.seek(0, 0);
2856 if(not infile):
2857 return False;
2858 infile.seek(0, 0);
2859 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
2860 infile = download_file_from_internet_file(infile);
2861 infile.seek(0, 0);
2862 if(not infile):
2863 return False;
2864 infile.seek(0, 0);
2865 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
2866 return False;
2867 else:
2868 return False;
2869 if(not zipfile.is_zipfile(infile)):
2870 return False;
2871 try:
2872 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
2873 except FileNotFoundError:
2874 return False;
2875 ziptest = zipfp.testzip();
2876 if(ziptest):
2877 VerbosePrintOut("Bad file found!");
2878 numfiles = int(len(zipfp.infolist()));
2879 AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
2880 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
2881 if(re.findall("^[.|/]", member.filename)):
2882 fname = member.filename;
2883 else:
2884 fname = "./"+member.filename;
2885 zipinfo = zipfp.getinfo(member.filename);
2886 if(verbose):
2887 VerbosePrintOut(fname);
2888 if(not member.is_dir()):
2889 fpremode = int(stat.S_IFREG + 438);
2890 elif(member.is_dir()):
2891 fpremode = int(stat.S_IFDIR + 511);
2892 flinkcount = 0;
2893 ftype = 0;
2894 if(not member.is_dir()):
2895 ftype = 0;
2896 elif(member.is_dir()):
2897 ftype = 5;
2898 flinkname = "";
2899 fcurfid = format(int(curfid), 'x').lower();
2900 fcurinode = format(int(curfid), 'x').lower();
2901 curfid = curfid + 1;
2902 fdev_minor = format(int(0), 'x').lower();
2903 fdev_major = format(int(0), 'x').lower();
2904 frdev_minor = format(int(0), 'x').lower();
2905 frdev_major = format(int(0), 'x').lower();
2906 if(ftype==5):
2907 fsize = format(int("0"), 'x').lower();
2908 elif(ftype==0):
2909 fsize = format(int(member.file_size), 'x').lower();
2910 else:
2911 fsize = format(int(member.file_size), 'x').lower();
2912 fatime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
2913 fmtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
2914 fctime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
2915 fbtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
2916 if(zipinfo.create_system==0 or zipinfo.create_system==10):
2917 fwinattributes = format(int(zipinfo.external_attr), 'x').lower();
2918 if(not member.is_dir()):
2919 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
2920 fchmode = stat.S_IMODE(fmode);
2921 ftypemod = stat.S_IFMT(fmode);
2922 elif(member.is_dir()):
2923 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
2924 fchmode = stat.S_IMODE(fmode);
2925 ftypemod = stat.S_IFMT(fmode);
2926 elif(zipinfo.create_system==3):
2927 fwinattributes = format(int(0), 'x').lower();
2928 fmode = format(int(zipinfo.external_attr), 'x').lower();
2929 fchmode = stat.S_IMODE(fmode);
2930 ftypemod = stat.S_IFMT(fmode);
2931 else:
2932 fwinattributes = format(int(0), 'x').lower();
2933 if(not member.is_dir()):
2934 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
2935 fchmode = stat.S_IMODE(fmode);
2936 ftypemod = stat.S_IFMT(fmode);
2937 elif(member.is_dir()):
2938 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
2939 fchmode = stat.S_IMODE(fmode);
2940 ftypemod = stat.S_IFMT(fmode);
2941 fcompression = "none";
2942 fcsize = format(int(0), 'x').lower();
2943 try:
2944 fuid = format(int(os.getuid()), 'x').lower();
2945 except AttributeError:
2946 fuid = format(int(0), 'x').lower();
2947 except KeyError:
2948 fuid = format(int(0), 'x').lower();
2949 try:
2950 fgid = format(int(os.getgid()), 'x').lower();
2951 except AttributeError:
2952 fgid = format(int(0), 'x').lower();
2953 except KeyError:
2954 fgid = format(int(0), 'x').lower();
2955 try:
2956 import pwd;
2957 try:
2958 userinfo = pwd.getpwuid(os.getuid());
2959 funame = userinfo.pw_name;
2960 except KeyError:
2961 funame = "";
2962 except AttributeError:
2963 funame = "";
2964 except ImportError:
2965 funame = "";
2966 fgname = "";
2967 try:
2968 import grp;
2969 try:
2970 groupinfo = grp.getgrgid(os.getgid());
2971 fgname = groupinfo.gr_name;
2972 except KeyError:
2973 fgname = "";
2974 except AttributeError:
2975 fgname = "";
2976 except ImportError:
2977 fgname = "";
2978 fcontents = BytesIO();
2979 if(ftype==0):
2980 fcontents.write(zipfp.read(member.filename));
2981 fcontents.seek(0, 0);
2982 ftypehex = format(ftype, 'x').lower();
2983 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
2984 AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
2985 fcontents.close();
2986 if(numfiles>0):
2987 catfp.write(AppendNullBytes([0, 0], formatspecs[5]).encode("UTF-8"));
2988 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
2989 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2990 try:
2991 catfp.flush();
2992 os.fsync(catfp.fileno());
2993 except io.UnsupportedOperation:
2994 pass;
2995 except AttributeError:
2996 pass;
2997 except OSError as e:
2998 pass;
2999 if(outfile=="-"):
3000 catfp.seek(0, 0);
3001 if(hasattr(sys.stdout, "buffer")):
3002 shutil.copyfileobj(catfp, sys.stdout.buffer);
3003 else:
3004 shutil.copyfileobj(catfp, sys.stdout);
3005 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3006 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3007 catfp.seek(0, 0);
3008 upload_file_to_internet_file(catfp, outfile);
3009 if(returnfp):
3010 catfp.seek(0, 0);
3011 return catfp;
3012 else:
3013 catfp.close();
3014 return True;
3015
3016 create_alias_function("Pack", __file_format_name__, "FromZipFile", PackArchiveFileFromZipFile);
3017
3018 if(not rarfile_support):
3019 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
3020 return False
3021
3022 if(rarfile_support):
3023 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
3024 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3025 outfile = RemoveWindowsPath(outfile);
3026 checksumtype = checksumtype.lower();
3027 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3028 checksumtype="crc32";
3029 if(checksumtype=="none"):
3030 checksumtype = "";
3031 if(not compression or compression=="catfile" or compression==formatspecs[2]):
3032 compression = "auto";
3033 if(compression not in compressionlist and compression is None):
3034 compression = "auto";
3035 if(verbose):
3036 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3037 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3038 if(os.path.exists(outfile)):
3039 try:
3040 os.unlink(outfile);
3041 except OSError as e:
3042 pass;
3043 if(outfile=="-"):
3044 verbose = False;
3045 catfp = BytesIO();
3046 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3047 catfp = outfile;
3048 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3049 catfp = BytesIO();
3050 else:
3051 fbasename = os.path.splitext(outfile)[0];
3052 fextname = os.path.splitext(outfile)[1];
3053 catfp = CompressOpenFile(outfile, True, compressionlevel);
3054 catver = formatspecs[6];
3055 fileheaderver = str(int(catver.replace(".", "")));
3056 curinode = 0;
3057 curfid = 0;
3058 inodelist = [];
3059 inodetofile = {};
3060 filetoinode = {};
3061 inodetocatinode = {};
3062 if(not os.path.exists(infile) or not os.path.isfile(infile)):
3063 return False;
3064 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
3065 return False;
3066 rarfp = rarfile.RarFile(infile, "r");
3067 rartest = rarfp.testrar();
3068 if(rartest):
3069 VerbosePrintOut("Bad file found!");
3070 numfiles = int(len(rarfp.infolist()));
3071 AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3072 try:
3073 catfp.flush();
3074 os.fsync(catfp.fileno());
3075 except io.UnsupportedOperation:
3076 pass;
3077 except AttributeError:
3078 pass;
3079 except OSError as e:
3080 pass;
3081 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
3082 is_unix = False;
3083 is_windows = False;
3084 if(member.host_os==rarfile.RAR_OS_UNIX):
3085 is_windows = False;
3086 try:
3087 member.external_attr
3088 is_unix = True;
3089 except AttributeError:
3090 is_unix = False;
3091 elif(member.host_os==rarfile.RAR_OS_WIN32):
3092 is_unix = False;
3093 try:
3094 member.external_attr
3095 is_windows = True;
3096 except AttributeError:
3097 is_windows = False;
3098 else:
3099 is_unix = False;
3100 is_windows = False;
3101 if(re.findall("^[.|/]", member.filename)):
3102 fname = member.filename;
3103 else:
3104 fname = "./"+member.filename;
3105 rarinfo = rarfp.getinfo(member.filename);
3106 if(verbose):
3107 VerbosePrintOut(fname);
3108 if(is_unix and member.external_attr !=0):
3109 fpremode = int(member.external_attr);
3110 elif(member.is_file()):
3111 fpremode = int(stat.S_IFREG + 438);
3112 elif(member.is_symlink()):
3113 fpremode = int(stat.S_IFLNK + 438);
3114 elif(member.is_dir()):
3115 fpremode = int(stat.S_IFDIR + 511);
3116 if(is_windows and member.external_attr !=0):
3117 fwinattributes = format(int(member.external_attr), 'x').lower();
3118 else:
3119 fwinattributes = format(int(0), 'x').lower();
3120 fcompression = "none";
3121 fcsize = format(int(0), 'x').lower();
3122 flinkcount = 0;
3123 ftype = 0;
3124 if(member.is_file()):
3125 ftype = 0;
3126 elif(member.is_symlink()):
3127 ftype = 2;
3128 elif(member.is_dir()):
3129 ftype = 5;
3130 flinkname = "";
3131 if(ftype==2):
3132 flinkname = rarfp.read(member.filename).decode("UTF-8");
3133 fcurfid = format(int(curfid), 'x').lower();
3134 fcurinode = format(int(curfid), 'x').lower();
3135 curfid = curfid + 1;
3136 fdev_minor = format(int(0), 'x').lower();
3137 fdev_major = format(int(0), 'x').lower();
3138 frdev_minor = format(int(0), 'x').lower();
3139 frdev_major = format(int(0), 'x').lower();
3140 if(ftype==5):
3141 fsize = format(int("0"), 'x').lower();
3142 elif(ftype==0):
3143 fsize = format(int(member.file_size), 'x').lower();
3144 else:
3145 fsize = format(int(member.file_size), 'x').lower();
3146 try:
3147 if(member.atime):
3148 fatime = format(int(member.atime.timestamp()), 'x').lower();
3149 else:
3150 fatime = format(int(member.mtime.timestamp()), 'x').lower();
3151 except AttributeError:
3152 fatime = format(int(member.mtime.timestamp()), 'x').lower();
3153 fmtime = format(int(member.mtime.timestamp()), 'x').lower();
3154 try:
3155 if(member.ctime):
3156 fctime = format(int(member.ctime.timestamp()), 'x').lower();
3157 else:
3158 fctime = format(int(member.mtime.timestamp()), 'x').lower();
3159 except AttributeError:
3160 fctime = format(int(member.mtime.timestamp()), 'x').lower();
3161 fbtime = format(int(member.mtime.timestamp()), 'x').lower();
3162 if(is_unix and member.external_attr !=0):
3163 fmode = format(int(member.external_attr), 'x').lower();
3164 fchmode = format(int(stat.S_IMODE(member.external_attr)), 'x').lower();
3165 ftypemod = format(int(stat.S_IFMT(member.external_attr)), 'x').lower();
3166 elif(member.is_file()):
3167 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3168 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3169 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3170 elif(member.is_symlink()):
3171 fmode = format(int(stat.S_IFLNK + 438), 'x').lower();
3172 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3173 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3174 elif(member.is_dir()):
3175 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3176 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
3177 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
3178 try:
3179 fuid = format(int(os.getuid()), 'x').lower();
3180 except AttributeError:
3181 fuid = format(int(0), 'x').lower();
3182 except KeyError:
3183 fuid = format(int(0), 'x').lower();
3184 try:
3185 fgid = format(int(os.getgid()), 'x').lower();
3186 except AttributeError:
3187 fgid = format(int(0), 'x').lower();
3188 except KeyError:
3189 fgid = format(int(0), 'x').lower();
3190 try:
3191 import pwd;
3192 try:
3193 userinfo = pwd.getpwuid(os.getuid());
3194 funame = userinfo.pw_name;
3195 except KeyError:
3196 funame = "";
3197 except AttributeError:
3198 funame = "";
3199 except ImportError:
3200 funame = "";
3201 fgname = "";
3202 try:
3203 import grp;
3204 try:
3205 groupinfo = grp.getgrgid(os.getgid());
3206 fgname = groupinfo.gr_name;
3207 except KeyError:
3208 fgname = "";
3209 except AttributeError:
3210 fgname = "";
3211 except ImportError:
3212 fgname = "";
3213 fcontents = BytesIO();
3214 if(ftype==0):
3215 fcontents.write(rarfp.read(member.filename));
3216 fcontents.seek(0, 0);
3217 ftypehex = format(ftype, 'x').lower();
3218 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3219 AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3220 fcontents.close();
3221 if(numfiles>0):
3222 catfp.write(AppendNullBytes([0, 0], formatspecs[5]).encode("UTF-8"));
3223 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3224 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
3225 try:
3226 catfp.flush();
3227 os.fsync(catfp.fileno());
3228 except io.UnsupportedOperation:
3229 pass;
3230 except AttributeError:
3231 pass;
3232 except OSError as e:
3233 pass;
3234 if(outfile=="-"):
3235 catfp.seek(0, 0)
3236 if(hasattr(sys.stdout, "buffer")):
3237 shutil.copyfileobj(catfp, sys.stdout.buffer);
3238 else:
3239 shutil.copyfileobj(catfp, sys.stdout);
3240 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3241 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3242 catfp.seek(0, 0);
3243 upload_file_to_internet_file(catfp, outfile);
3244 if(returnfp):
3245 catfp.seek(0, 0)
3246 return catfp
3247 else:
3248 catfp.close()
3249 return True;
3250
3251 create_alias_function("Pack", __file_format_name__, "FromRarFile", PackArchiveFileFromRarFile);
3252
3253 if(not py7zr_support):
3254 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
3255 return False
3256
3257 if(py7zr_support):
3258 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
3259 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3260 outfile = RemoveWindowsPath(outfile);
3261 checksumtype = checksumtype.lower();
3262 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3263 checksumtype="crc32";
3264 if(checksumtype=="none"):
3265 checksumtype = "";
3266 if(not compression or compression=="catfile" or compression==formatspecs[2]):
3267 compression = "auto";
3268 if(compression not in compressionlist and compression is None):
3269 compression = "auto";
3270 if(verbose):
3271 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3272 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3273 if(os.path.exists(outfile)):
3274 try:
3275 os.unlink(outfile);
3276 except OSError as e:
3277 pass;
3278 if(outfile=="-"):
3279 verbose = False;
3280 catfp = BytesIO();
3281 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3282 catfp = outfile;
3283 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3284 catfp = BytesIO();
3285 else:
3286 fbasename = os.path.splitext(outfile)[0];
3287 fextname = os.path.splitext(outfile)[1];
3288 catfp = CompressOpenFile(outfile, True, compressionlevel);
3289 catver = formatspecs[6];
3290 fileheaderver = str(int(catver.replace(".", "")));
3291 curinode = 0;
3292 curfid = 0;
3293 inodelist = [];
3294 inodetofile = {};
3295 filetoinode = {};
3296 inodetocatinode = {};
3297 if(not os.path.exists(infile) or not os.path.isfile(infile)):
3298 return False;
3299 szpfp = py7zr.SevenZipFile(infile, mode="r");
3300 file_content = szpfp.readall();
3301 #sztest = szpfp.testzip();
3302 sztestalt = szpfp.test();
3303 if(sztestalt):
3304 VerbosePrintOut("Bad file found!");
3305 numfiles = int(len(szpfp.list()));
3306 AppendFileHeader(catfp, numfiles, checksumtype, formatspecs);
3307 for member in sorted(szpfp.list(), key=lambda x: x.filename):
3308 if(re.findall("^[.|/]", member.filename)):
3309 fname = member.filename;
3310 else:
3311 fname = "./"+member.filename;
3312 if(verbose):
3313 VerbosePrintOut(fname);
3314 if(not member.is_directory):
3315 fpremode = int(stat.S_IFREG + 438);
3316 elif(member.is_directory):
3317 fpremode = int(stat.S_IFDIR + 511);
3318 fwinattributes = format(int(0), 'x').lower();
3319 fcompression = "none";
3320 fcsize = format(int(0), 'x').lower();
3321 flinkcount = 0;
3322 ftype = 0;
3323 if(member.is_directory):
3324 ftype = 5;
3325 else:
3326 ftype = 0;
3327 flinkname = "";
3328 fcurfid = format(int(curfid), 'x').lower();
3329 fcurinode = format(int(curfid), 'x').lower();
3330 curfid = curfid + 1;
3331 fdev_minor = format(int(0), 'x').lower();
3332 fdev_major = format(int(0), 'x').lower();
3333 frdev_minor = format(int(0), 'x').lower();
3334 frdev_major = format(int(0), 'x').lower();
3335 if(ftype==5):
3336 fsize = format(int("0"), 'x').lower();
3337 fatime = format(int(member.creationtime.timestamp()), 'x').lower();
3338 fmtime = format(int(member.creationtime.timestamp()), 'x').lower();
3339 fctime = format(int(member.creationtime.timestamp()), 'x').lower();
3340 fbtime = format(int(member.creationtime.timestamp()), 'x').lower();
3341 if(member.is_directory):
3342 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3343 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
3344 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
3345 else:
3346 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3347 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3348 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3349 try:
3350 fuid = format(int(os.getuid()), 'x').lower();
3351 except AttributeError:
3352 fuid = format(int(0), 'x').lower();
3353 except KeyError:
3354 fuid = format(int(0), 'x').lower();
3355 try:
3356 fgid = format(int(os.getgid()), 'x').lower();
3357 except AttributeError:
3358 fgid = format(int(0), 'x').lower();
3359 except KeyError:
3360 fgid = format(int(0), 'x').lower();
3361 try:
3362 import pwd;
3363 try:
3364 userinfo = pwd.getpwuid(os.getuid());
3365 funame = userinfo.pw_name;
3366 except KeyError:
3367 funame = "";
3368 except AttributeError:
3369 funame = "";
3370 except ImportError:
3371 funame = "";
3372 fgname = "";
3373 try:
3374 import grp;
3375 try:
3376 groupinfo = grp.getgrgid(os.getgid());
3377 fgname = groupinfo.gr_name;
3378 except KeyError:
3379 fgname = "";
3380 except AttributeError:
3381 fgname = "";
3382 except ImportError:
3383 fgname = "";
3384 fcontents = BytesIO();
3385 if(ftype==0):
3386 fcontents.write(file_content[member.filename].read());
3387 fsize = format(fcontents.tell(), 'x').lower();
3388 file_content[member.filename].close();
3389 fcontents.seek(0, 0);
3390 ftypehex = format(ftype, 'x').lower();
3391 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
3392 AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
3393 fcontents.close();
3394 if(numfiles>0):
3395 catfp.write(AppendNullBytes([0, 0], formatspecs[5]).encode("UTF-8"));
3396 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3397 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
3398 try:
3399 catfp.flush();
3400 os.fsync(catfp.fileno());
3401 except io.UnsupportedOperation:
3402 pass;
3403 except AttributeError:
3404 pass;
3405 except OSError as e:
3406 pass;
3407 if(outfile=="-"):
3408 catfp.seek(0, 0);
3409 if(hasattr(sys.stdout, "buffer")):
3410 shutil.copyfileobj(catfp, sys.stdout.buffer);
3411 else:
3412 shutil.copyfileobj(catfp, sys.stdout);
3413 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3414 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3415 catfp.seek(0, 0);
3416 upload_file_to_internet_file(catfp, outfile);
3417 if(returnfp):
3418 catfp.seek(0, 0)
3419 return catfp
3420 else:
3421 catfp.close()
3422 return True;
3423
3424 create_alias_function("Pack", __file_format_name__, "FromSevenZipFile", PackArchiveFileFromSevenZipFile);
3425
3426 def PackArchiveFileFromInFile(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
3427 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
3428 if(verbose):
3429 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3430 if(checkcompressfile=="tarfile"):
3431 return PackArchiveFileFromTarFile(infile, outfile, compression, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3432 elif(checkcompressfile=="zipfile"):
3433 return PackArchiveFileFromZipFile(infile, outfile, compression, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3434 elif(checkcompressfile=="catfile"):
3435 return RePackArchiveFile(infile, outfile, compression, compressionlevel, False, 0, 0, checksumtype, False, extradata, formatspecs, verbose, returnfp);
3436 elif(rarfile_support and checkcompressfile=="rarfile"):
3437 return PackArchiveFileFromRarFile(infile, outfile, compression, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3438 elif(py7zr_support and checkcompressfile=="7zipfile"):
3439 return PackArchiveFileFromSevenZipFile(infile, outfile, compression, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3440 else:
3441 return False;
3442 return False;
3443
3444 create_alias_function("Pack", __file_format_name__, "FromInFile", PackArchiveFileFromInFile);
3445
3446 def ArchiveFileSeekToFileNum(infile, seekto=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
3447 if(hasattr(infile, "read") or hasattr(infile, "write")):
3448 catfp = infile;
3449 catfp.seek(0, 0);
3450 catfp = UncompressArchiveFile(catfp, formatspecs);
3451 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
3452 if(checkcompressfile=="tarfile"):
3453 return TarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3454 if(checkcompressfile=="zipfile"):
3455 return ZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3456 if(rarfile_support and checkcompressfile=="rarfile"):
3457 return RarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3458 if(py7zr_support and checkcompressfile=="7zipfile"):
3459 return SevenZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3460 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
3461 return False;
3462 if(not catfp):
3463 return False;
3464 catfp.seek(0, 0);
3465 elif(infile=="-"):
3466 catfp = BytesIO();
3467 if(hasattr(sys.stdin, "buffer")):
3468 shutil.copyfileobj(sys.stdin.buffer, catfp);
3469 else:
3470 shutil.copyfileobj(sys.stdin, catfp);
3471 catfp.seek(0, 0);
3472 catfp = UncompressArchiveFile(catfp, formatspecs);
3473 if(not catfp):
3474 return False;
3475 catfp.seek(0, 0);
3476 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3477 catfp = download_file_from_internet_file(infile);
3478 catfp.seek(0, 0);
3479 catfp = UncompressArchiveFile(catfp, formatspecs);
3480 if(not catfp):
3481 return False;
3482 catfp.seek(0, 0);
3483 else:
3484 infile = RemoveWindowsPath(infile);
3485 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
3486 if(checkcompressfile=="tarfile"):
3487 return TarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3488 if(checkcompressfile=="zipfile"):
3489 return ZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3490 if(rarfile_support and checkcompressfile=="rarfile"):
3491 return RarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3492 if(py7zr_support and checkcompressfile=="7zipfile"):
3493 return SevenZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3494 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
3495 return False;
3496 compresscheck = CheckCompressionType(infile, formatspecs, True);
3497 if(not compresscheck):
3498 fextname = os.path.splitext(infile)[1];
3499 if(fextname==".gz"):
3500 compresscheck = "gzip";
3501 elif(fextname==".bz2"):
3502 compresscheck = "bzip2";
3503 elif(fextname==".zst"):
3504 compresscheck = "zstd";
3505 elif(fextname==".lz4" or fextname==".clz4"):
3506 compresscheck = "lz4";
3507 elif(fextname==".lzo" or fextname==".lzop"):
3508 compresscheck = "lzo";
3509 elif(fextname==".lzma" or fextname==".xz"):
3510 compresscheck = "lzma";
3511 else:
3512 return False;
3513 if(not compresscheck):
3514 return False;
3515 catfp = UncompressFile(infile, formatspecs, "rb");
3516 '''
3517 try:
3518 catfp.seek(0, 2);
3519 except OSError:
3520 SeekToEndOfFile(catfp);
3521 except ValueError:
3522 SeekToEndOfFile(catfp);
3523 CatSize = catfp.tell();
3524 CatSizeEnd = CatSize;
3525 '''
3526 try:
3527 catfp.seek(0, 0);
3528 except OSError:
3529 return False;
3530 except ValueError:
3531 return False;
3532 curloc = catfp.tell();
3533 if(curloc>0):
3534 catfp.seek(0, 0);
3535 catheader = ReadFileHeaderData(catfp, 4, formatspecs[5]);
3536 if(curloc>0):
3537 catfp.seek(curloc, 0);
3538 catstring = catheader[0];
3539 catversion = re.findall(r"([\d]+)$", catstring);
3540 fprenumfiles = catheader[1];
3541 fnumfiles = int(fprenumfiles, 16);
3542 fprechecksumtype = catheader[2];
3543 fprechecksum = catheader[3];
3544 fileheader = AppendNullByte(catstring, formatspecs[5]);
3545 fnumfileshex = format(int(fnumfiles), 'x').lower();
3546 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs[5]);
3547 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
3548 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
3549 fheadtell = len(fileheader);
3550 if(fprechecksum!=catfileheadercshex and not skipchecksum):
3551 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
3552 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
3553 return False;
3554 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
3555 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
3556 if(seekto>=fnumfiles):
3557 seekto = fnumfiles - 1;
3558 if(seekto<0):
3559 seekto = 0;
3560 if(seekto>=0):
3561 il = -1;
3562 while(il < seekto):
3563 prefhstart = catfp.tell();
3564 if(formatspecs[7]):
3565 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs[5]);
3566 else:
3567 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs[5]);
3568 if(len(preheaderdata)==0):
3569 break;
3570 prefheadsize = int(preheaderdata[0], 16);
3571 prefnumfields = int(preheaderdata[1], 16);
3572 preftype = int(preheaderdata[2], 16);
3573 if(re.findall("^[.|/]", preheaderdata[3])):
3574 prefname = preheaderdata[3];
3575 else:
3576 prefname = "./"+preheaderdata[3];
3577 prefbasedir = os.path.dirname(prefname);
3578 preflinkname = preheaderdata[4];
3579 prefsize = int(preheaderdata[5], 16);
3580 prefatime = int(preheaderdata[6], 16);
3581 prefmtime = int(preheaderdata[7], 16);
3582 prefctime = int(preheaderdata[8], 16);
3583 prefbtime = int(preheaderdata[9], 16);
3584 prefmode = int(preheaderdata[10], 16);
3585 prefchmode = stat.S_IMODE(prefmode);
3586 preftypemod = stat.S_IFMT(prefmode);
3587 prefwinattributes = int(preheaderdata[11], 16);
3588 prefcompression = preheaderdata[12];
3589 prefcsize = preheaderdata[13];
3590 prefuid = int(preheaderdata[14], 16);
3591 prefuname = preheaderdata[15];
3592 prefgid = int(preheaderdata[16], 16);
3593 prefgname = preheaderdata[17];
3594 fid = int(preheaderdata[18], 16);
3595 finode = int(preheaderdata[19], 16);
3596 flinkcount = int(preheaderdata[20], 16);
3597 prefdev_minor = int(preheaderdata[21], 16);
3598 prefdev_major = int(preheaderdata[22], 16);
3599 prefrdev_minor = int(preheaderdata[23], 16);
3600 prefrdev_major = int(preheaderdata[24], 16);
3601 prefextrasize = int(preheaderdata[25], 16);
3602 prefextrafields = int(preheaderdata[26], 16);
3603 extrafieldslist = [];
3604 extrastart = 27;
3605 extraend = extrastart + prefextrafields;
3606 extrafieldslist = [];
3607 if(extrastart<extraend):
3608 extrafieldslist.append(preheaderdata[extrastart]);
3609 extrastart = extrastart + 1;
3610 prefchecksumtype = preheaderdata[extrastart].lower();
3611 prefcs = preheaderdata[extrastart + 1].lower();
3612 prefccs = preheaderdata[extrastart + 2].lower();
3613 hc = 0;
3614 hcmax = len(preheaderdata) - 2;
3615 hout = "";
3616 while(hc<hcmax):
3617 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs[5]);
3618 hc = hc + 1;
3619 prenewfcs = GetFileChecksum(hout, preheaderdata[-3].lower(), True, formatspecs);
3620 if(prefcs!=prenewfcs and not skipchecksum):
3621 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
3622 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
3623 return False;
3624 valid_archive = False;
3625 invalid_archive = True;
3626 prefhend = catfp.tell() - 1;
3627 prefcontentstart = catfp.tell();
3628 prefcontents = "";
3629 pyhascontents = False;
3630 if(prefsize>0):
3631 prefcontents = catfp.read(prefsize);
3632 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
3633 pyhascontents = True;
3634 if(prefccs!=prenewfccs and not skipchecksum):
3635 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
3636 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
3637 return False;
3638 catfp.seek(1, 1);
3639 il = il + 1;
3640 catfp.seek(seekstart, 0);
3641 fileidnum = il;
3642 catfheadsize = int(preheaderdata[0], 16);
3643 catfnumfields = int(preheaderdata[1], 16);
3644 catftype = int(preheaderdata[2], 16);
3645 if(re.findall("^[.|/]", preheaderdata[3])):
3646 catfname = preheaderdata[3];
3647 else:
3648 catfname = "./"+preheaderdata[3];
3649 catflinkname = preheaderdata[4];
3650 catfsize = int(preheaderdata[5], 16);
3651 catfbasedir = os.path.dirname(catfname);
3652 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
3653 if(returnfp):
3654 catlist.update({'catfp': catfp});
3655 else:
3656 catfp.close();
3657 return catlist;
3658
3659 create_alias_function("", __file_format_name__, "SeekToFileNum", ArchiveFileSeekToFileNum);
3660
3661 def ArchiveFileSeekToFileName(infile, seekfile=None, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
3662 if(hasattr(infile, "read") or hasattr(infile, "write")):
3663 catfp = infile;
3664 catfp.seek(0, 0);
3665 catfp = UncompressArchiveFile(catfp, formatspecs);
3666 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
3667 if(checkcompressfile=="tarfile"):
3668 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3669 if(checkcompressfile=="zipfile"):
3670 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3671 if(rarfile_support and checkcompressfile=="rarfile"):
3672 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3673 if(py7zr_support and checkcompressfile=="7zipfile"):
3674 return SevenZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3675 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
3676 return False;
3677 if(not catfp):
3678 return False;
3679 catfp.seek(0, 0);
3680 elif(infile=="-"):
3681 catfp = BytesIO();
3682 if(hasattr(sys.stdin, "buffer")):
3683 shutil.copyfileobj(sys.stdin.buffer, catfp);
3684 else:
3685 shutil.copyfileobj(sys.stdin, catfp);
3686 catfp.seek(0, 0);
3687 catfp = UncompressArchiveFile(catfp, formatspecs);
3688 if(not catfp):
3689 return False;
3690 catfp.seek(0, 0);
3691 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3692 catfp = download_file_from_internet_file(infile);
3693 catfp = UncompressArchiveFile(catfp, formatspecs);
3694 catfp.seek(0, 0);
3695 if(not catfp):
3696 return False;
3697 catfp.seek(0, 0);
3698 else:
3699 infile = RemoveWindowsPath(infile);
3700 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
3701 if(checkcompressfile=="tarfile"):
3702 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3703 if(checkcompressfile=="zipfile"):
3704 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3705 if(rarfile_support and checkcompressfile=="rarfile"):
3706 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3707 if(py7zr_support and checkcompressfile=="7zipfile"):
3708 return SevenZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3709 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
3710 return False;
3711 compresscheck = CheckCompressionType(infile, formatspecs, True);
3712 if(not compresscheck):
3713 fextname = os.path.splitext(infile)[1];
3714 if(fextname==".gz"):
3715 compresscheck = "gzip";
3716 elif(fextname==".bz2"):
3717 compresscheck = "bzip2";
3718 elif(fextname==".zst"):
3719 compresscheck = "zstd";
3720 elif(fextname==".lz4" or fextname==".clz4"):
3721 compresscheck = "lz4";
3722 elif(fextname==".lzo" or fextname==".lzop"):
3723 compresscheck = "lzo";
3724 elif(fextname==".lzma" or fextname==".xz"):
3725 compresscheck = "lzma";
3726 else:
3727 return False;
3728 if(not compresscheck):
3729 return False;
3730 catfp = UncompressFile(infile, formatspecs, "rb");
3731 '''
3732 try:
3733 catfp.seek(0, 2);
3734 except OSError:
3735 SeekToEndOfFile(catfp);
3736 except ValueError:
3737 SeekToEndOfFile(catfp);
3738 CatSize = catfp.tell();
3739 CatSizeEnd = CatSize;
3740 '''
3741 try:
3742 catfp.seek(0, 0);
3743 except OSError:
3744 return False;
3745 except ValueError:
3746 return False;
3747 curloc = catfp.tell();
3748 if(curloc>0):
3749 catfp.seek(0, 0);
3750 catheader = ReadFileHeaderData(catfp, 4, formatspecs[5]);
3751 if(curloc>0):
3752 catfp.seek(curloc, 0);
3753 catstring = catheader[0];
3754 catversion = re.findall(r"([\d]+)$", catstring);
3755 fprenumfiles = catheader[1];
3756 fnumfiles = int(fprenumfiles, 16);
3757 fprechecksumtype = catheader[2];
3758 fprechecksum = catheader[3];
3759 fileheader = AppendNullByte(catstring, formatspecs[5]);
3760 fnumfileshex = format(int(fnumfiles), 'x').lower();
3761 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs[5]);
3762 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
3763 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
3764 fheadtell = len(fileheader);
3765 if(fprechecksum!=catfileheadercshex and not skipchecksum):
3766 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
3767 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
3768 return False;
3769 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
3770 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
3771 seekto = fnumfiles - 1
3772 filefound = False;
3773 if(seekto>=0):
3774 il = -1;
3775 while(il < seekto):
3776 prefhstart = catfp.tell();
3777 if(formatspecs[7]):
3778 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs[5]);
3779 else:
3780 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs[5]);
3781 if(len(preheaderdata)==0):
3782 break;
3783 prefheadsize = int(preheaderdata[0], 16);
3784 prefnumfields = int(preheaderdata[1], 16);
3785 preftype = int(preheaderdata[2], 16);
3786 if(re.findall("^[.|/]", preheaderdata[3])):
3787 prefname = preheaderdata[3];
3788 else:
3789 prefname = "./"+preheaderdata[3];
3790 prefbasedir = os.path.dirname(prefname);
3791 preflinkname = preheaderdata[4];
3792 prefsize = int(preheaderdata[5], 16);
3793 prefatime = int(preheaderdata[6], 16);
3794 prefmtime = int(preheaderdata[7], 16);
3795 prefctime = int(preheaderdata[8], 16);
3796 prefbtime = int(preheaderdata[9], 16);
3797 prefmode = int(preheaderdata[10], 16);
3798 prefchmode = stat.S_IMODE(prefmode);
3799 preftypemod = stat.S_IFMT(prefmode);
3800 prefwinattributes = int(preheaderdata[11], 16);
3801 prefcompression = preheaderdata[12];
3802 prefcsize = preheaderdata[13];
3803 prefuid = int(preheaderdata[14], 16);
3804 prefuname = preheaderdata[15];
3805 prefgid = int(preheaderdata[16], 16);
3806 prefgname = preheaderdata[17];
3807 fid = int(preheaderdata[18], 16);
3808 finode = int(preheaderdata[19], 16);
3809 flinkcount = int(preheaderdata[20], 16);
3810 prefdev_minor = int(preheaderdata[21], 16);
3811 prefdev_major = int(preheaderdata[22], 16);
3812 prefrdev_minor = int(preheaderdata[23], 16);
3813 prefrdev_major = int(preheaderdata[24], 16);
3814 prefextrasize = int(preheaderdata[25], 16);
3815 prefextrafields = int(preheaderdata[26], 16);
3816 extrafieldslist = [];
3817 extrastart = 27;
3818 extraend = extrastart + prefextrafields;
3819 extrafieldslist = [];
3820 if(extrastart<extraend):
3821 extrafieldslist.append(preheaderdata[extrastart]);
3822 extrastart = extrastart + 1;
3823 prefchecksumtype = preheaderdata[extrastart].lower();
3824 prefcs = preheaderdata[extrastart + 1].lower();
3825 prefccs = preheaderdata[extrastart + 2].lower();
3826 hc = 0;
3827 hcmax = len(preheaderdata) - 2;
3828 hout = "";
3829 while(hc<hcmax):
3830 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs[5]);
3831 hc = hc + 1;
3832 prenewfcs = GetFileChecksum(hout, preheaderdata[-3].lower(), True, formatspecs);
3833 if(prefcs!=prenewfcs and not skipchecksum):
3834 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
3835 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
3836 return False;
3837 valid_archive = False;
3838 invalid_archive = True;
3839 prefhend = catfp.tell() - 1;
3840 prefcontentstart = catfp.tell();
3841 prefcontents = "";
3842 pyhascontents = False;
3843 if(prefsize>0):
3844 prefcontents = catfp.read(prefsize);
3845 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
3846 pyhascontents = True;
3847 if(prefccs!=prenewfccs and not skipchecksum):
3848 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
3849 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
3850 return False;
3851 catfp.seek(1, 1);
3852 il = il + 1;
3853 filefound = False;
3854 prefname = preheaderdata[2];
3855 if(re.findall("^[.|/]", preheaderdata[2])):
3856 prefname = preheaderdata[2];
3857 else:
3858 prefname = "./"+preheaderdata[2];
3859 if(prefname==seekfile):
3860 filefound = True;
3861 break;
3862 catfp.seek(seekstart, 0);
3863 fileidnum = il;
3864 catfheadsize = int(preheaderdata[0], 16);
3865 catfnumfields = int(preheaderdata[1], 16);
3866 catftype = int(preheaderdata[2], 16);
3867 if(re.findall("^[.|/]", preheaderdata[3])):
3868 catfname = preheaderdata[3];
3869 else:
3870 catfname = "./"+preheaderdata[3];
3871 catflinkname = preheaderdata[4];
3872 catfsize = int(preheaderdata[5], 16);
3873 catfbasedir = os.path.dirname(catfname);
3874 if(filefound):
3875 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
3876 else:
3877 return False;
3878 if(returnfp):
3879 catlist.update({'catfp': catfp});
3880 else:
3881 catfp.close();
3882 return catlist;
3883
3884 create_alias_function("", __file_format_name__, "SeekToFileName", ArchiveFileSeekToFileName);
3885
3886 def ArchiveFileValidate(infile, formatspecs=__file_format_list__, verbose=False, returnfp=False):
3887 if(verbose):
3888 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3889 if(hasattr(infile, "read") or hasattr(infile, "write")):
3890 catfp = infile;
3891 catfp.seek(0, 0);
3892 catfp = UncompressArchiveFile(catfp, formatspecs);
3893 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
3894 if(checkcompressfile=="tarfile"):
3895 return TarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
3896 if(checkcompressfile=="zipfile"):
3897 return ZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
3898 if(rarfile_support and checkcompressfile=="rarfile"):
3899 return RarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
3900 if(py7zr_support and checkcompressfile=="7zipfile"):
3901 return SevenZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
3902 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
3903 return False;
3904 if(not catfp):
3905 return False;
3906 catfp.seek(0, 0);
3907 elif(infile=="-"):
3908 catfp = BytesIO();
3909 if(hasattr(sys.stdin, "buffer")):
3910 shutil.copyfileobj(sys.stdin.buffer, catfp);
3911 else:
3912 shutil.copyfileobj(sys.stdin, catfp);
3913 catfp.seek(0, 0);
3914 catfp = UncompressArchiveFile(catfp, formatspecs);
3915 if(not catfp):
3916 return False;
3917 catfp.seek(0, 0);
3918 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3919 catfp = download_file_from_internet_file(infile);
3920 catfp = UncompressArchiveFile(catfp, formatspecs);
3921 catfp.seek(0, 0);
3922 if(not catfp):
3923 return False;
3924 catfp.seek(0, 0);
3925 else:
3926 infile = RemoveWindowsPath(infile);
3927 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
3928 if(checkcompressfile=="tarfile"):
3929 return TarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
3930 if(checkcompressfile=="zipfile"):
3931 return ZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
3932 if(rarfile_support and checkcompressfile=="rarfile"):
3933 return RarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
3934 if(py7zr_support and checkcompressfile=="7zipfile"):
3935 return SevenZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
3936 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
3937 return False;
3938 compresscheck = CheckCompressionType(infile, formatspecs, True);
3939 if(not compresscheck):
3940 fextname = os.path.splitext(infile)[1];
3941 if(fextname==".gz"):
3942 compresscheck = "gzip";
3943 elif(fextname==".bz2"):
3944 compresscheck = "bzip2";
3945 elif(fextname==".zst"):
3946 compresscheck = "zstd";
3947 elif(fextname==".lz4" or fextname==".clz4"):
3948 compresscheck = "lz4";
3949 elif(fextname==".lzo" or fextname==".lzop"):
3950 compresscheck = "lzo";
3951 elif(fextname==".lzma" or fextname==".xz"):
3952 compresscheck = "lzma";
3953 else:
3954 return False;
3955 if(not compresscheck):
3956 return False;
3957 catfp = UncompressFile(infile, formatspecs, "rb");
3958 '''
3959 try:
3960 catfp.seek(0, 2);
3961 except OSError:
3962 SeekToEndOfFile(catfp);
3963 except ValueError:
3964 SeekToEndOfFile(catfp);
3965 CatSize = catfp.tell();
3966 CatSizeEnd = CatSize;
3967 '''
3968 try:
3969 catfp.seek(0, 0);
3970 except OSError:
3971 return False;
3972 except ValueError:
3973 return False;
3974 curloc = catfp.tell();
3975 if(curloc>0):
3976 catfp.seek(0, 0);
3977 catheader = ReadFileHeaderData(catfp, 4, formatspecs[5]);
3978 if(curloc>0):
3979 catfp.seek(curloc, 0);
3980 catstring = catheader[0];
3981 catversion = re.findall(r"([\d]+)$", catstring);
3982 fprenumfiles = catheader[1];
3983 fnumfiles = int(fprenumfiles, 16);
3984 fprechecksumtype = catheader[2];
3985 fprechecksum = catheader[3];
3986 il = 0;
3987 fileheader = AppendNullByte(catstring, formatspecs[5]);
3988 fnumfileshex = format(int(fnumfiles), 'x').lower();
3989 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs[5]);
3990 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
3991 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
3992 valid_archive = True;
3993 invalid_archive = False;
3994 if(verbose):
3995 VerbosePrintOut(infile);
3996 VerbosePrintOut("Number of Records " + str(fnumfiles));
3997 if(fprechecksum==catfileheadercshex):
3998 if(verbose):
3999 VerbosePrintOut("File Header Checksum Passed at offset " + str(0));
4000 VerbosePrintOut("'" + str(fprechecksum) + "' == " + "'" + str(catfileheadercshex) + "'");
4001 else:
4002 if(verbose):
4003 VerbosePrintOut("File Header Checksum Failed at offset " + str(0));
4004 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4005 valid_archive = False;
4006 invalid_archive = True;
4007 if(verbose):
4008 VerbosePrintOut("");
4009 while(il<fnumfiles):
4010 catfhstart = catfp.tell();
4011 if(formatspecs[7]):
4012 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs[5]);
4013 else:
4014 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs[5]);
4015 if(len(catheaderdata)==0):
4016 break;
4017 catfheadsize = int(catheaderdata[0], 16);
4018 catfnumfields = int(catheaderdata[1], 16);
4019 catftype = int(catheaderdata[2], 16);
4020 if(re.findall("^[.|/]", catheaderdata[3])):
4021 catfname = catheaderdata[3];
4022 else:
4023 catfname = "./"+catheaderdata[3];
4024 catfbasedir = os.path.dirname(catfname);
4025 catflinkname = catheaderdata[4];
4026 catfsize = int(catheaderdata[5], 16);
4027 catfatime = int(catheaderdata[6], 16);
4028 catfmtime = int(catheaderdata[7], 16);
4029 catfctime = int(catheaderdata[8], 16);
4030 catfbtime = int(catheaderdata[9], 16);
4031 catfmode = int(catheaderdata[10], 16);
4032 catfchmode = stat.S_IMODE(catfmode);
4033 catftypemod = stat.S_IFMT(catfmode);
4034 catfwinattributes = int(catheaderdata[11], 16);
4035 catfcompression = catheaderdata[12];
4036 catfcsize = catheaderdata[13];
4037 catfuid = int(catheaderdata[14], 16);
4038 catfuname = catheaderdata[15];
4039 catfgid = int(catheaderdata[16], 16);
4040 catfgname = catheaderdata[17];
4041 fid = int(catheaderdata[18], 16);
4042 finode = int(catheaderdata[19], 16);
4043 flinkcount = int(catheaderdata[20], 16);
4044 catfdev_minor = int(catheaderdata[21], 16);
4045 catfdev_major = int(catheaderdata[22], 16);
4046 catfrdev_minor = int(catheaderdata[23], 16);
4047 catfrdev_major = int(catheaderdata[24], 16);
4048 catfextrasize = int(catheaderdata[25], 16);
4049 catfextrafields = int(catheaderdata[26], 16);
4050 extrafieldslist = [];
4051 extrastart = 27;
4052 extraend = extrastart + catfextrafields;
4053 extrafieldslist = [];
4054 if(extrastart<extraend):
4055 extrafieldslist.append(catheaderdata[extrastart]);
4056 extrastart = extrastart + 1;
4057 catfchecksumtype = catheaderdata[extrastart].lower();
4058 catfcs = catheaderdata[extrastart + 1].lower();
4059 catfccs = catheaderdata[extrastart + 2].lower();
4060 hc = 0;
4061 hcmax = len(catheaderdata) - 2;
4062 hout = "";
4063 while(hc<hcmax):
4064 hout = hout + AppendNullByte(catheaderdata[hc], formatspecs[5]);
4065 hc = hc + 1;
4066 catnewfcs = GetFileChecksum(hout, catheaderdata[-3].lower(), True, formatspecs);
4067 if(verbose):
4068 VerbosePrintOut(catfname);
4069 VerbosePrintOut("Record Number " + str(il) + "; File ID " + str(fid) + "; iNode Number " + str(finode));
4070 if(catfcs==catnewfcs):
4071 if(verbose):
4072 VerbosePrintOut("File Header Checksum Passed at offset " + str(catfhstart));
4073 VerbosePrintOut("'" + str(catfcs) + "' == " + "'" + str(catnewfcs) + "'");
4074 else:
4075 if(verbose):
4076 VerbosePrintOut("File Header Checksum Failed at offset " + str(catfhstart));
4077 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
4078 valid_archive = False;
4079 invalid_archive = True;
4080 catfhend = catfp.tell() - 1;
4081 catfcontentstart = catfp.tell();
4082 catfcontents = "";
4083 pyhascontents = False;
4084 if(catfsize>0):
4085 catfcontents = catfp.read(catfsize);
4086 catnewfccs = GetFileChecksum(catfcontents, catheaderdata[-3].lower(), False, formatspecs);
4087 pyhascontents = True;
4088 if(catfccs==catnewfccs):
4089 if(verbose):
4090 VerbosePrintOut("File Content Checksum Passed at offset " + str(catfcontentstart));
4091 VerbosePrintOut("'" + str(catfccs) + "' == " + "'" + str(catnewfccs) + "'");
4092 else:
4093 if(verbose):
4094 VerbosePrintOut("File Content Checksum Failed at offset " + str(catfcontentstart));
4095 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
4096 valid_archive = False;
4097 invalid_archive = True;
4098 if(verbose):
4099 VerbosePrintOut("");
4100 catfp.seek(1, 1);
4101 il = il + 1;
4102 if(valid_archive):
4103 if(returnfp):
4104 return catfp;
4105 else:
4106 catfp.close();
4107 return True;
4108 else:
4109 catfp.close();
4110 return False;
4111
4112 create_alias_function("", __file_format_name__, "Validate", ArchiveFileValidate);
4113
4114 def ArchiveFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4115 if(hasattr(infile, "read") or hasattr(infile, "write")):
4116 catfp = infile;
4117 catfp.seek(0, 0);
4118 catfp = UncompressArchiveFile(catfp, formatspecs);
4119 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4120 if(checkcompressfile=="tarfile"):
4121 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4122 if(checkcompressfile=="zipfile"):
4123 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4124 if(rarfile_support and checkcompressfile=="rarfile"):
4125 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4126 if(py7zr_support and checkcompressfile=="7zipfile"):
4127 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4128 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
4129 return False;
4130 if(not catfp):
4131 return False;
4132 catfp.seek(0, 0);
4133 elif(infile=="-"):
4134 catfp = BytesIO();
4135 if(hasattr(sys.stdin, "buffer")):
4136 shutil.copyfileobj(sys.stdin.buffer, catfp);
4137 else:
4138 shutil.copyfileobj(sys.stdin, catfp);
4139 catfp.seek(0, 0);
4140 catfp = UncompressArchiveFile(catfp, formatspecs);
4141 if(not catfp):
4142 return False;
4143 catfp.seek(0, 0);
4144 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4145 catfp = download_file_from_internet_file(infile);
4146 catfp = UncompressArchiveFile(catfp, formatspecs);
4147 catfp.seek(0, 0);
4148 if(not catfp):
4149 return False;
4150 catfp.seek(0, 0);
4151 else:
4152 infile = RemoveWindowsPath(infile);
4153 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4154 if(checkcompressfile=="tarfile"):
4155 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4156 if(checkcompressfile=="zipfile"):
4157 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4158 if(rarfile_support and checkcompressfile=="rarfile"):
4159 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4160 if(py7zr_support and checkcompressfile=="7zipfile"):
4161 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4162 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
4163 return False;
4164 compresscheck = CheckCompressionType(infile, formatspecs, True);
4165 if(not compresscheck):
4166 fextname = os.path.splitext(infile)[1];
4167 if(fextname==".gz"):
4168 compresscheck = "gzip";
4169 elif(fextname==".bz2"):
4170 compresscheck = "bzip2";
4171 elif(fextname==".zst"):
4172 compresscheck = "zstd";
4173 elif(fextname==".lz4" or fextname==".clz4"):
4174 compresscheck = "lz4";
4175 elif(fextname==".lzo" or fextname==".lzop"):
4176 compresscheck = "lzo";
4177 elif(fextname==".lzma" or fextname==".xz"):
4178 compresscheck = "lzma";
4179 else:
4180 return False;
4181 if(not compresscheck):
4182 return False;
4183 catfp = UncompressFile(infile, formatspecs, "rb");
4184 '''
4185 try:
4186 catfp.seek(0, 2);
4187 except OSError:
4188 SeekToEndOfFile(catfp);
4189 except ValueError:
4190 SeekToEndOfFile(catfp);
4191 CatSize = catfp.tell();
4192 CatSizeEnd = CatSize;
4193 '''
4194 try:
4195 catfp.seek(0, 0);
4196 except OSError:
4197 return False;
4198 except ValueError:
4199 return False;
4200 curloc = catfp.tell();
4201 if(curloc>0):
4202 catfp.seek(0, 0);
4203 catheader = ReadFileHeaderData(catfp, 4, formatspecs[5]);
4204 if(curloc>0):
4205 catfp.seek(curloc, 0);
4206 catstring = catheader[0];
4207 catversion = re.findall(r"([\d]+)$", catstring);
4208 fprenumfiles = catheader[1];
4209 fnumfiles = int(fprenumfiles, 16);
4210 fprechecksumtype = catheader[2];
4211 fprechecksum = catheader[3];
4212 fileheader = AppendNullByte(catstring, formatspecs[5]);
4213 fnumfileshex = format(int(fnumfiles), 'x').lower();
4214 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs[5]);
4215 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4216 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
4217 fheadtell = len(fileheader);
4218 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4219 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
4220 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4221 return False;
4222 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
4223 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4224 if(seekstart<0 and seekstart>fnumfiles):
4225 seekstart = 0;
4226 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
4227 seekend = fnumfiles;
4228 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
4229 seekend = fnumfiles - abs(seekend);
4230 if(seekstart>0):
4231 il = 0;
4232 while(il < seekstart):
4233 prefhstart = catfp.tell();
4234 if(formatspecs[7]):
4235 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs[5]);
4236 else:
4237 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs[5]);
4238 if(len(preheaderdata)==0):
4239 break;
4240 prefheadsize = int(preheaderdata[0], 16);
4241 prefnumfields = int(preheaderdata[1], 16);
4242 if(re.findall("^[.|/]", preheaderdata[3])):
4243 prefname = preheaderdata[3];
4244 else:
4245 prefname = "./"+preheaderdata[3];
4246 prefsize = int(preheaderdata[5], 16);
4247 prefextrasize = int(preheaderdata[24], 16);
4248 prefextrafields = int(preheaderdata[25], 16);
4249 extrafieldslist = [];
4250 extrastart = 26;
4251 extraend = extrastart + prefextrafields;
4252 extrafieldslist = [];
4253 if(extrastart<extraend):
4254 extrafieldslist.append(preheaderdata[extrastart]);
4255 extrastart = extrastart + 1;
4256 prefchecksumtype = preheaderdata[extrastart].lower();
4257 prefcs = preheaderdata[extrastart + 1].lower();
4258 prefccs = preheaderdata[extrastart + 2].lower();
4259 hc = 0;
4260 hcmax = len(preheaderdata) - 2;
4261 hout = "";
4262 while(hc<hcmax):
4263 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs[5]);
4264 hc = hc + 1;
4265 prenewfcs = GetFileChecksum(hout, preheaderdata[-3].lower(), True, formatspecs);
4266 if(prefcs!=prenewfcs and not skipchecksum):
4267 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4268 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4269 return False;
4270 valid_archive = False;
4271 invalid_archive = True;
4272 prefhend = catfp.tell() - 1;
4273 prefcontentstart = catfp.tell();
4274 prefcontents = "";
4275 pyhascontents = False;
4276 if(prefsize>0):
4277 prefcontents = catfp.read(prefsize);
4278 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4279 pyhascontents = True;
4280 if(prefccs!=prenewfccs and not skipchecksum):
4281 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4282 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4283 return False;
4284 catfp.seek(1, 1);
4285 il = il + 1;
4286 fileidnum = seekstart;
4287 realidnum = 0;
4288 while(fileidnum<seekend):
4289 catfhstart = catfp.tell();
4290 if(formatspecs[7]):
4291 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs[5]);
4292 else:
4293 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs[5]);
4294 if(len(catheaderdata)==0):
4295 break;
4296 catfheadsize = int(catheaderdata[0], 16);
4297 catfnumfields = int(catheaderdata[1], 16);
4298 catftype = int(catheaderdata[2], 16);
4299 if(re.findall("^[.|/]", catheaderdata[3])):
4300 catfname = catheaderdata[3];
4301 else:
4302 catfname = "./"+catheaderdata[3];
4303 catfbasedir = os.path.dirname(catfname);
4304 catflinkname = catheaderdata[4];
4305 catfsize = int(catheaderdata[5], 16);
4306 catfatime = int(catheaderdata[6], 16);
4307 catfmtime = int(catheaderdata[7], 16);
4308 catfctime = int(catheaderdata[8], 16);
4309 catfbtime = int(catheaderdata[9], 16);
4310 catfmode = int(catheaderdata[10], 16);
4311 catfchmode = stat.S_IMODE(catfmode);
4312 catftypemod = stat.S_IFMT(catfmode);
4313 catfwinattributes = int(catheaderdata[11], 16);
4314 catfcompression = catheaderdata[12];
4315 catfcsize = catheaderdata[13];
4316 catfuid = int(catheaderdata[14], 16);
4317 catfuname = catheaderdata[15];
4318 catfgid = int(catheaderdata[16], 16);
4319 catfgname = catheaderdata[17];
4320 catfid = int(catheaderdata[18], 16);
4321 catfinode = int(catheaderdata[19], 16);
4322 catflinkcount = int(catheaderdata[20], 16);
4323 catfdev_minor = int(catheaderdata[21], 16);
4324 catfdev_major = int(catheaderdata[22], 16);
4325 catfrdev_minor = int(catheaderdata[23], 16);
4326 catfrdev_major = int(catheaderdata[24], 16);
4327 catfextrasize = int(catheaderdata[25], 16);
4328 catfextrafields = int(catheaderdata[26], 16);
4329 extrafieldslist = [];
4330 extrastart = 27;
4331 extraend = extrastart + catfextrafields;
4332 extrafieldslist = [];
4333 if(extrastart<extraend):
4334 extrafieldslist.append(catheaderdata[extrastart]);
4335 extrastart = extrastart + 1;
4336 catfchecksumtype = catheaderdata[extrastart].lower();
4337 catfcs = catheaderdata[extrastart + 1].lower();
4338 catfccs = catheaderdata[extrastart + 2].lower();
4339 hc = 0;
4340 hcmax = len(catheaderdata) - 2;
4341 hout = "";
4342 while(hc<hcmax):
4343 hout = hout + AppendNullByte(catheaderdata[hc], formatspecs[5]);
4344 hc = hc + 1;
4345 catnewfcs = GetFileChecksum(hout, catheaderdata[-3].lower(), True, formatspecs);
4346 if(catfcs!=catnewfcs and not skipchecksum):
4347 VerbosePrintOut("File Header Checksum Error with file " + catfname + " at offset " + str(catfhstart));
4348 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
4349 return False;
4350 catfhend = catfp.tell() - 1;
4351 catfcontentstart = catfp.tell();
4352 catfcontents = BytesIO();
4353 pyhascontents = False;
4354 catfcontents.seek(0, 0);
4355 if(catfsize>0 and not listonly):
4356 catfcontents.write(catfp.read(catfsize));
4357 catnewfccs = GetFileChecksum(catfcontents.read(), catheaderdata[-3].lower(), False, formatspecs);
4358 pyhascontents = True;
4359 if(catfccs!=catnewfccs and skipchecksum):
4360 VerbosePrintOut("File Content Checksum Error with file " + catfname + " at offset " + str(catfcontentstart));
4361 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
4362 return False;
4363 if(catfsize>0 and listonly):
4364 catfp.seek(catfsize, 1);
4365 pyhascontents = False;
4366 catfp.seek(1, 1);
4367 catfcontentend = catfp.tell() - 1;
4368 catfcontents.seek(0, 0);
4369 catlist['ffilelist'].update({realidnum: {'fid': realidnum, 'fidalt': fileidnum, 'fheadersize': catfheadsize, 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize, 'fatime': catfatime, 'fmtime': catfmtime, 'fctime': catfctime, 'fbtime': catfbtime, 'fmode': catfmode, 'fchmode': catfchmode, 'ftypemod': catftypemod, 'fwinattributes': catfwinattributes, 'fcompression': catfcompression, 'fcsize': catfcsize, 'fuid': catfuid, 'funame': catfuname, 'fgid': catfgid, 'fgname': catfgname, 'finode': catfinode, 'flinkcount': catflinkcount, 'fminor': catfdev_minor, 'fmajor': catfdev_major, 'frminor': catfrdev_minor, 'frmajor': catfrdev_major, 'fchecksumtype': catfchecksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': catfextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': catfcs, 'fcontentchecksum': catfccs, 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': catfcontents} });
4370 fileidnum = fileidnum + 1;
4371 realidnum = realidnum + 1;
4372 if(returnfp):
4373 catlist.update({'catfp': catfp});
4374 else:
4375 catfp.close();
4376 return catlist;
4377
4378 create_alias_function("", __file_format_name__, "ToArray", ArchiveFileToArray);
4379
4380 def ArchiveFileStringToArray(catstr, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4381 catfp = BytesIO(catstr);
4382 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4383 return listcatfiles;
4384
4385 create_alias_function("", __file_format_name__, "StringToArray", ArchiveFileStringToArray);
4386
4387 def TarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4388 catfp = BytesIO();
4389 catfp = PackArchiveFileFromTarFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
4390 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4391 return listcatfiles;
4392
4393 def ZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4394 catfp = BytesIO();
4395 catfp = PackArchiveFileFromZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
4396 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4397 return listcatfiles;
4398
4399 if(not rarfile_support):
4400 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4401 return False;
4402
4403 if(rarfile_support):
4404 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4405 catfp = BytesIO();
4406 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
4407 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4408 return listcatfiles;
4409
4410 if(not py7zr_support):
4411 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4412 return False;
4413
4414 if(py7zr_support):
4415 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4416 catfp = BytesIO();
4417 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
4418 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4419 return listcatfiles;
4420
4421 def InFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4422 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4423 if(checkcompressfile=="tarfile"):
4424 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4425 elif(checkcompressfile=="zipfile"):
4426 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4427 elif(checkcompressfile=="catfile"):
4428 return ArchiveFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4429 elif(rarfile_support and checkcompressfile=="rarfile"):
4430 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4431 elif(py7zr_support and checkcompressfile=="7zipfile"):
4432 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4433 else:
4434 return False;
4435 return False;
4436
4437 def ListDirToArrayAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
4438 catver = formatspecs[6];
4439 fileheaderver = str(int(catver.replace(".", "")));
4440 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
4441 advancedlist = formatspecs[8];
4442 altinode = formatspecs[9];
4443 infilelist = [];
4444 if(infiles=="-"):
4445 for line in sys.stdin:
4446 infilelist.append(line.strip());
4447 infilelist = list(filter(None, infilelist));
4448 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
4449 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
4450 return False;
4451 with UncompressFile(infiles, formatspecs, "r") as finfile:
4452 for line in finfile:
4453 infilelist.append(line.strip());
4454 infilelist = list(filter(None, infilelist));
4455 else:
4456 if(isinstance(infiles, (list, tuple, ))):
4457 infilelist = list(filter(None, infiles));
4458 elif(isinstance(infiles, (str, ))):
4459 infilelist = list(filter(None, [infiles]));
4460 if(advancedlist):
4461 GetDirList = ListDirAdvanced(infilelist, followlink, False);
4462 else:
4463 GetDirList = ListDir(infilelist, followlink, False);
4464 if(not GetDirList):
4465 return False;
4466 curinode = 0;
4467 curfid = 0;
4468 inodelist = [];
4469 inodetofile = {};
4470 filetoinode = {};
4471 inodetocatinode = {};
4472 fileidnum = 0;
4473 fnumfiles = int(len(GetDirList));
4474 catver = formatspecs[6];
4475 fileheaderver = str(int(catver.replace(".", "")));
4476 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
4477 fnumfileshex = format(int(fnumfiles), 'x').lower();
4478 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs[5]);
4479 catversion = re.findall(r"([\d]+)$", fileheader);
4480 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
4481 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4482 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
4483 fheadtell = len(fileheader);
4484 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
4485 for curfname in GetDirList:
4486 if(re.findall("^[.|/]", curfname)):
4487 fname = curfname;
4488 else:
4489 fname = "./"+curfname;
4490 if(verbose):
4491 VerbosePrintOut(fname);
4492 if(not followlink or followlink is None):
4493 fstatinfo = os.lstat(fname);
4494 else:
4495 fstatinfo = os.stat(fname);
4496 fpremode = fstatinfo.st_mode;
4497 finode = fstatinfo.st_ino;
4498 flinkcount = fstatinfo.st_nlink;
4499 ftype = 0;
4500 if(stat.S_ISREG(fpremode)):
4501 ftype = 0;
4502 elif(stat.S_ISLNK(fpremode)):
4503 ftype = 2;
4504 elif(stat.S_ISCHR(fpremode)):
4505 ftype = 3;
4506 elif(stat.S_ISBLK(fpremode)):
4507 ftype = 4;
4508 elif(stat.S_ISDIR(fpremode)):
4509 ftype = 5;
4510 elif(stat.S_ISFIFO(fpremode)):
4511 ftype = 6;
4512 elif(stat.S_ISSOCK(fpremode)):
4513 ftype = 8;
4514 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
4515 ftype = 9;
4516 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
4517 ftype = 10;
4518 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
4519 ftype = 11;
4520 else:
4521 ftype = 0;
4522 flinkname = "";
4523 fbasedir = os.path.dirname(fname);
4524 fcurfid = curfid;
4525 if(not followlink and finode!=0):
4526 if(ftype!=1):
4527 if(finode in inodelist):
4528 ftype = 1;
4529 flinkname = inodetofile[finode];
4530 if(altinode):
4531 fcurinode = finode;
4532 else:
4533 fcurinode = inodetocatinode[finode];
4534 if(finode not in inodelist):
4535 inodelist.append(finode);
4536 inodetofile.update({finode: fname});
4537 inodetocatinode.update({finode: curinode});
4538 if(altinode):
4539 fcurinode = finode;
4540 else:
4541 fcurinode = curinode;
4542 curinode = curinode + 1;
4543 else:
4544 fcurinode = curinode;
4545 curinode = curinode + 1;
4546 curfid = curfid + 1;
4547 if(ftype==2):
4548 flinkname = os.readlink(fname);
4549 fdev = fstatinfo.st_dev;
4550 getfdev = GetDevMajorMinor(fdev);
4551 fdev_minor = getfdev[0];
4552 fdev_major = getfdev[1];
4553 frdev = fstatinfo.st_dev;
4554 if(hasattr(fstatinfo, "st_rdev")):
4555 frdev = fstatinfo.st_rdev;
4556 else:
4557 frdev = fstatinfo.st_dev;
4558 getfrdev = GetDevMajorMinor(frdev);
4559 frdev_minor = getfrdev[0];
4560 frdev_major = getfrdev[1];
4561 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
4562 fsize = "0";
4563 if(ftype==0 or ftype==7):
4564 fsize = fstatinfo.st_size;
4565 fatime = fstatinfo.st_atime;
4566 fmtime = fstatinfo.st_mtime;
4567 fctime = fstatinfo.st_ctime;
4568 if(hasattr(fstatinfo, "st_birthtime")):
4569 fbtime = fstatinfo.st_birthtime;
4570 else:
4571 fbtime = fstatinfo.st_ctime;
4572 fmode = fstatinfo.st_mode;
4573 fchmode = stat.S_IMODE(fstatinfo.st_mode);
4574 ftypemod = stat.S_IFMT(fstatinfo.st_mode);
4575 fuid = fstatinfo.st_uid;
4576 fgid = fstatinfo.st_gid;
4577 funame = "";
4578 try:
4579 import pwd;
4580 try:
4581 userinfo = pwd.getpwuid(fstatinfo.st_uid);
4582 funame = userinfo.pw_name;
4583 except KeyError:
4584 funame = "";
4585 except ImportError:
4586 funame = "";
4587 fgname = "";
4588 try:
4589 import grp;
4590 try:
4591 groupinfo = grp.getgrgid(fstatinfo.st_gid);
4592 fgname = groupinfo.gr_name;
4593 except KeyError:
4594 fgname = "";
4595 except ImportError:
4596 fgname = "";
4597 fdev_minor = fdev_minor;
4598 fdev_major = fdev_major;
4599 frdev_minor = frdev_minor;
4600 frdev_major = frdev_major;
4601 flinkcount = flinkcount;
4602 if(hasattr(fstatinfo, "st_file_attributes")):
4603 fwinattributes = fstatinfo.st_file_attributes;
4604 else:
4605 fwinattributes = 0;
4606 fcompression = "none";
4607 fcsize = 0;
4608 fcontents = BytesIO();
4609 if(ftype==0 or ftype==7):
4610 with open(fname, "rb") as fpc:
4611 shutil.copyfileobj(fpc, fcontents);
4612 if(followlink and (ftype==1 or ftype==2)):
4613 flstatinfo = os.stat(flinkname);
4614 with open(flinkname, "rb") as fpc:
4615 shutil.copyfileobj(fpc, fcontents);
4616 fcontents.seek(0, 0);
4617 ftypehex = format(ftype, 'x').lower();
4618 extrafields = len(extradata);
4619 extrafieldslist = extradata;
4620 catfextrafields = extrafields;
4621 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
4622 if(len(extradata)>0):
4623 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
4624 extrasizelen = len(extrasizestr);
4625 extrasizelenhex = format(extrasizelen, 'x').lower();
4626 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
4627 catoutlen = len(catoutlist) + len(extradata) + 3;
4628 catoutlenhex = format(catoutlen, 'x').lower();
4629 catoutlist.insert(0, catoutlenhex);
4630 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
4631 if(len(extradata)>0):
4632 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
4633 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
4634 catfnumfields = catoutlen;
4635 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
4636 fcontents.seek(0, 0);
4637 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
4638 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
4639 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
4640 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
4641 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
4642 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
4643 catfileoutstrecd = catfileoutstr.encode('UTF-8');
4644 nullstrecd = formatspecs[5].encode('UTF-8');
4645 fheadtell += len(catfileoutstr) + 1;
4646 catfcontentend = fheadtell - 1;
4647 fcontents.seek(0, 0);
4648 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
4649 pyhascontents = False;
4650 if(int(fsize)>0 and not listonly):
4651 pyhascontents = True;
4652 if(int(fsize)>0 and listonly):
4653 fcontents = BytesIO();
4654 pyhascontents = False;
4655 fcontents.seek(0, 0);
4656 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
4657 fileidnum = fileidnum + 1;
4658 return catlist;
4659
4660 def TarFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
4661 curinode = 0;
4662 curfid = 0;
4663 inodelist = [];
4664 inodetofile = {};
4665 filetoinode = {};
4666 inodetocatinode = {};
4667 fileidnum = 0;
4668 if(infile=="-"):
4669 infile = BytesIO();
4670 if(hasattr(sys.stdin, "buffer")):
4671 shutil.copyfileobj(sys.stdin.buffer, infile);
4672 else:
4673 shutil.copyfileobj(sys.stdin, infile);
4674 infile.seek(0, 0);
4675 if(not infile):
4676 return False;
4677 infile.seek(0, 0);
4678 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4679 infile = download_file_from_internet_file(infile);
4680 infile.seek(0, 0);
4681 if(not infile):
4682 return False;
4683 infile.seek(0, 0);
4684 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
4685 return False;
4686 elif(os.path.exists(infile) and os.path.isfile(infile)):
4687 try:
4688 if(not tarfile.is_tarfile(infile)):
4689 return False;
4690 except AttributeError:
4691 if(not is_tarfile(infile)):
4692 return False;
4693 else:
4694 return False;
4695 try:
4696 if(hasattr(infile, "read") or hasattr(infile, "write")):
4697 tarfp = tarfile.open(fileobj=infile, mode="r");
4698 else:
4699 tarfp = tarfile.open(infile, "r");
4700 except FileNotFoundError:
4701 return False;
4702 fnumfiles = int(len(tarfp.getmembers()));
4703 catver = formatspecs[6];
4704 fileheaderver = str(int(catver.replace(".", "")));
4705 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
4706 fnumfileshex = format(int(fnumfiles), 'x').lower();
4707 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs[5]);
4708 catversion = re.findall(r"([\d]+)$", fileheader);
4709 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
4710 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4711 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
4712 fheadtell = len(fileheader);
4713 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
4714 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
4715 if(re.findall("^[.|/]", member.name)):
4716 fname = member.name;
4717 else:
4718 fname = "./"+member.name;
4719 if(verbose):
4720 VerbosePrintOut(fname);
4721 fpremode = member.mode;
4722 ffullmode = member.mode;
4723 flinkcount = 0;
4724 ftype = 0;
4725 if(member.isreg()):
4726 ffullmode = member.mode + stat.S_IFREG;
4727 ftype = 0;
4728 elif(member.isdev()):
4729 ffullmode = member.mode;
4730 ftype = 7;
4731 elif(member.islnk()):
4732 ffullmode = member.mode + stat.S_IFREG;
4733 ftype = 1;
4734 elif(member.issym()):
4735 ffullmode = member.mode + stat.S_IFLNK;
4736 ftype = 2;
4737 elif(member.ischr()):
4738 ffullmode = member.mode + stat.S_IFCHR;
4739 ftype = 3;
4740 elif(member.isblk()):
4741 ffullmode = member.mode + stat.S_IFBLK;
4742 ftype = 4;
4743 elif(member.isdir()):
4744 ffullmode = member.mode + stat.S_IFDIR;
4745 ftype = 5;
4746 elif(member.isfifo()):
4747 ffullmode = member.mode + stat.S_IFIFO;
4748 ftype = 6;
4749 elif(member.issparse()):
4750 ffullmode = member.mode;
4751 ftype = 12;
4752 else:
4753 ffullmode = member.mode;
4754 ftype = 0;
4755 flinkname = "";
4756 fbasedir = os.path.dirname(fname);
4757 fcurfid = curfid;
4758 fcurinode = curfid;
4759 finode = fcurinode;
4760 curfid = curfid + 1;
4761 if(ftype==2):
4762 flinkname = member.linkname;
4763 fdev_minor = member.devminor;
4764 fdev_major = member.devmajor;
4765 frdev_minor = member.devminor;
4766 frdev_major = member.devmajor;
4767 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
4768 fsize = "0";
4769 elif(ftype==0 or ftype==7):
4770 fsize = member.size;
4771 else:
4772 fsize = member.size;
4773 fatime = member.mtime;
4774 fmtime = member.mtime;
4775 fctime = member.mtime;
4776 fbtime = member.mtime;
4777 fmode = ffullmode;
4778 fchmode = stat.S_IMODE(ffullmode);
4779 ftypemod = stat.S_IFMT(ffullmode);
4780 fuid = member.uid;
4781 fgid = member.gid;
4782 funame = member.uname;
4783 fgname = member.gname;
4784 flinkcount = flinkcount;
4785 fwinattributes = int(0);
4786 fcompression = "none";
4787 fcsize = 0;
4788 fcontents = BytesIO();
4789 if(ftype==0 or ftype==7):
4790 with tarfp.extractfile(member) as fpc:
4791 shutil.copyfileobj(fpc, fcontents);
4792 fcontents.seek(0, 0);
4793 ftypehex = format(ftype, 'x').lower();
4794 extrafields = len(extradata);
4795 extrafieldslist = extradata;
4796 catfextrafields = extrafields;
4797 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
4798 if(len(extradata)>0):
4799 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
4800 extrasizelen = len(extrasizestr);
4801 extrasizelenhex = format(extrasizelen, 'x').lower();
4802 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
4803 catoutlen = len(catoutlist) + len(extradata) + 3;
4804 catoutlenhex = format(catoutlen, 'x').lower();
4805 catoutlist.insert(0, catoutlenhex);
4806 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
4807 if(len(extradata)>0):
4808 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
4809 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
4810 catfnumfields = catoutlen;
4811 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
4812 fcontents.seek(0, 0);
4813 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
4814 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
4815 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
4816 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
4817 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
4818 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
4819 catfileoutstrecd = catfileoutstr.encode('UTF-8');
4820 nullstrecd = formatspecs[5].encode('UTF-8');
4821 fheadtell += len(catfileoutstr) + 1;
4822 catfcontentend = fheadtell - 1;
4823 fcontents.seek(0, 0);
4824 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
4825 pyhascontents = False;
4826 if(int(fsize)>0 and not listonly):
4827 pyhascontents = True;
4828 if(int(fsize)>0 and listonly):
4829 fcontents = "";
4830 pyhascontents = False;
4831 fcontents.seek(0, 0);
4832 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
4833 fileidnum = fileidnum + 1;
4834 return catlist;
4835
4836 def ZipFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
4837 curinode = 0;
4838 curfid = 0;
4839 inodelist = [];
4840 inodetofile = {};
4841 filetoinode = {};
4842 inodetocatinode = {};
4843 fileidnum = 0;
4844 if(infile=="-"):
4845 infile = BytesIO();
4846 if(hasattr(sys.stdin, "buffer")):
4847 shutil.copyfileobj(sys.stdin.buffer, infile);
4848 else:
4849 shutil.copyfileobj(sys.stdin, infile);
4850 infile.seek(0, 0);
4851 if(not infile):
4852 return False;
4853 infile.seek(0, 0);
4854 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4855 infile = download_file_from_internet_file(infile);
4856 infile.seek(0, 0);
4857 if(not infile):
4858 return False;
4859 infile.seek(0, 0);
4860 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
4861 return False;
4862 else:
4863 return False;
4864 if(not zipfile.is_zipfile(infile)):
4865 return False;
4866 try:
4867 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
4868 except FileNotFoundError:
4869 return False;
4870 ziptest = zipfp.testzip();
4871 if(ziptest):
4872 VerbosePrintOut("Bad file found!");
4873 fnumfiles = int(len(zipfp.infolist()));
4874 catver = formatspecs[6];
4875 fileheaderver = str(int(catver.replace(".", "")));
4876 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
4877 catversion = re.findall(r"([\d]+)$", fileheader);
4878 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
4879 fnumfileshex = format(int(fnumfiles), 'x').lower();
4880 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs[5]);
4881 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4882 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
4883 fheadtell = len(fileheader);
4884 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
4885 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
4886 if(re.findall("^[.|/]", member.filename)):
4887 fname = member.filename;
4888 else:
4889 fname = "./"+member.filename;
4890 zipinfo = zipfp.getinfo(member.filename);
4891 if(verbose):
4892 VerbosePrintOut(fname);
4893 if(not member.is_dir()):
4894 fpremode = stat.S_IFREG + 438;
4895 elif(member.is_dir()):
4896 fpremode = stat.S_IFDIR + 511;
4897 flinkcount = 0;
4898 ftype = 0;
4899 if(not member.is_dir()):
4900 ftype = 0;
4901 elif(member.is_dir()):
4902 ftype = 5;
4903 flinkname = "";
4904 fbasedir = os.path.dirname(fname);
4905 fcurfid = curfid;
4906 fcurinode = curfid;
4907 finode = fcurinode;
4908 curfid = curfid + 1;
4909 fdev_minor = 0;
4910 fdev_major = 0;
4911 frdev_minor = 0;
4912 frdev_major = 0;
4913 if(ftype==5):
4914 fsize = "0";
4915 elif(ftype==0):
4916 fsize = member.file_size;
4917 else:
4918 fsize = member.file_size;
4919 fatime = time.mktime(member.date_time + (0, 0, -1));
4920 fmtime = time.mktime(member.date_time + (0, 0, -1));
4921 fctime = time.mktime(member.date_time + (0, 0, -1));
4922 fbtime = time.mktime(member.date_time + (0, 0, -1));
4923 if(zipinfo.create_system==0 or zipinfo.create_system==10):
4924 fwinattributes = int(zipinfo.external_attr);
4925 if(not member.is_dir()):
4926 fmode = int(stat.S_IFREG + 438);
4927 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
4928 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
4929 elif(member.is_dir()):
4930 fmode = int(stat.S_IFDIR + 511);
4931 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
4932 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
4933 elif(zipinfo.create_system==3):
4934 fwinattributes = int(0);
4935 fmode = int(zipinfo.external_attr);
4936 else:
4937 fwinattributes = int(0);
4938 if(not member.is_dir()):
4939 fmode = int(stat.S_IFREG + 438);
4940 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
4941 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
4942 elif(member.is_dir()):
4943 fmode = int(stat.S_IFDIR + 511);
4944 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
4945 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
4946 fcompression = "none";
4947 fcsize = 0;
4948 try:
4949 fuid = os.getuid();
4950 except AttributeError:
4951 fuid = 0;
4952 except KeyError:
4953 fuid = 0;
4954 try:
4955 fgid = os.getgid();
4956 except AttributeError:
4957 fgid = 0;
4958 except KeyError:
4959 fgid = 0;
4960 try:
4961 import pwd;
4962 try:
4963 userinfo = pwd.getpwuid(os.getuid());
4964 funame = userinfo.pw_name;
4965 except KeyError:
4966 funame = "";
4967 except AttributeError:
4968 funame = "";
4969 except ImportError:
4970 funame = "";
4971 fgname = "";
4972 try:
4973 import grp;
4974 try:
4975 groupinfo = grp.getgrgid(os.getgid());
4976 fgname = groupinfo.gr_name;
4977 except KeyError:
4978 fgname = "";
4979 except AttributeError:
4980 fgname = "";
4981 except ImportError:
4982 fgname = "";
4983 fcontents = BytesIO();
4984 if(ftype==0):
4985 fcontents.write(zipfp.read(member.filename));
4986 fcontents.seek(0, 0);
4987 ftypehex = format(ftype, 'x').lower();
4988 extrafields = len(extradata);
4989 extrafieldslist = extradata;
4990 catfextrafields = extrafields;
4991 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
4992 if(len(extradata)>0):
4993 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
4994 extrasizelen = len(extrasizestr);
4995 extrasizelenhex = format(extrasizelen, 'x').lower();
4996 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
4997 catoutlen = len(catoutlist) + len(extradata) + 3;
4998 catoutlenhex = format(catoutlen, 'x').lower();
4999 catoutlist.insert(0, catoutlenhex);
5000 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
5001 if(len(extradata)>0):
5002 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
5003 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
5004 catfnumfields = catoutlen;
5005 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5006 fcontents.seek(0, 0);
5007 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5008 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5009 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5010 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
5011 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5012 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5013 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5014 nullstrecd = formatspecs[5].encode('UTF-8');
5015 fheadtell += len(catfileoutstr) + 1;
5016 catfcontentend = fheadtell - 1;
5017 fcontents.seek(0, 0);
5018 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5019 pyhascontents = False;
5020 if(int(fsize)>0 and not listonly):
5021 pyhascontents = True;
5022 if(int(fsize)>0 and listonly):
5023 fcontents = BytesIO();
5024 pyhascontents = False;
5025 fcontents.seek(0, 0);
5026 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5027 fileidnum = fileidnum + 1;
5028 return catlist;
5029
5030 if(not rarfile_support):
5031 def RarFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5032 return False;
5033
5034 if(rarfile_support):
5035 def RarFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5036 curinode = 0;
5037 curfid = 0;
5038 inodelist = [];
5039 inodetofile = {};
5040 filetoinode = {};
5041 inodetocatinode = {};
5042 fileidnum = 0;
5043 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
5044 return False;
5045 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
5046 return False;
5047 rarfp = rarfile.RarFile(infile, "r");
5048 rartest = rarfp.testrar();
5049 if(rartest):
5050 VerbosePrintOut("Bad file found!");
5051 fnumfiles = int(len(rarfp.infolist()));
5052 catver = formatspecs[6];
5053 fileheaderver = str(int(catver.replace(".", "")));
5054 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
5055 catversion = re.findall(r"([\d]+)$", fileheader);
5056 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
5057 fnumfileshex = format(int(fnumfiles), 'x').lower();
5058 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs[5]);
5059 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
5060 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
5061 fheadtell = len(fileheader);
5062 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5063 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
5064 is_unix = False;
5065 is_windows = False;
5066 if(member.host_os==rarfile.RAR_OS_UNIX):
5067 is_windows = False;
5068 try:
5069 member.external_attr
5070 is_unix = True;
5071 except AttributeError:
5072 is_unix = False;
5073 elif(member.host_os==rarfile.RAR_OS_WIN32):
5074 is_unix = False;
5075 try:
5076 member.external_attr
5077 is_windows = True;
5078 except AttributeError:
5079 is_windows = False;
5080 else:
5081 is_unix = False;
5082 is_windows = False;
5083 if(re.findall("^[.|/]", member.filename)):
5084 fname = member.filename;
5085 else:
5086 fname = "./"+member.filename;
5087 rarinfo = rarfp.getinfo(member.filename);
5088 if(verbose):
5089 VerbosePrintOut(fname);
5090 if(is_unix and member.external_attr !=0):
5091 fpremode = int(member.external_attr);
5092 elif(member.is_file()):
5093 fpremode = stat.S_IFREG + 438;
5094 elif(member.is_symlink()):
5095 fpremode = stat.S_IFLNK + 438;
5096 elif(member.is_dir()):
5097 fpremode = stat.S_IFDIR + 511;
5098 if(is_windows and member.external_attr !=0):
5099 fwinattributes = int(member.external_attr);
5100 else:
5101 fwinattributes = int(0);
5102 fcompression = "none";
5103 fcsize = 0;
5104 flinkcount = 0;
5105 ftype = 0;
5106 if(member.is_file()):
5107 ftype = 0;
5108 elif(member.is_symlink()):
5109 ftype = 2;
5110 elif(member.is_dir()):
5111 ftype = 5;
5112 flinkname = "";
5113 if(ftype==2):
5114 flinkname = rarfp.read(member.filename).decode("UTF-8");
5115 fbasedir = os.path.dirname(fname);
5116 fcurfid = curfid;
5117 fcurinode = curfid;
5118 finode = fcurinode;
5119 curfid = curfid + 1;
5120 fdev_minor = 0;
5121 fdev_major = 0;
5122 frdev_minor = 0;
5123 frdev_major = 0;
5124 if(ftype==5):
5125 fsize = "0";
5126 if(ftype==0):
5127 fsize = member.file_size;
5128 try:
5129 if(member.atime):
5130 fatime = int(member.atime.timestamp());
5131 else:
5132 fatime = int(member.mtime.timestamp());
5133 except AttributeError:
5134 fatime = int(member.mtime.timestamp());
5135 fmtime = int(member.mtime.timestamp());
5136 try:
5137 if(member.ctime):
5138 fctime = int(member.ctime.timestamp());
5139 else:
5140 fctime = int(member.mtime.timestamp());
5141 except AttributeError:
5142 fctime = int(member.mtime.timestamp());
5143 fbtime = int(member.mtime.timestamp());
5144 if(is_unix and member.external_attr !=0):
5145 fmode = int(member.external_attr);
5146 fchmode = int(stat.S_IMODE(member.external_attr));
5147 ftypemod = int(stat.S_IFMT(member.external_attr));
5148 elif(member.is_file()):
5149 fmode = int(stat.S_IFREG + 438)
5150 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5151 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5152 elif(member.is_symlink()):
5153 fmode = int(stat.S_IFLNK + 438)
5154 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5155 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5156 elif(member.is_dir()):
5157 fmode = int(stat.S_IFDIR + 511)
5158 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
5159 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
5160 try:
5161 fuid = os.getuid();
5162 except AttributeError:
5163 fuid = 0;
5164 except KeyError:
5165 fuid = 0;
5166 try:
5167 fgid = os.getgid();
5168 except AttributeError:
5169 fgid = 0;
5170 except KeyError:
5171 fgid = 0;
5172 try:
5173 import pwd;
5174 try:
5175 userinfo = pwd.getpwuid(os.getuid());
5176 funame = userinfo.pw_name;
5177 except KeyError:
5178 funame = "";
5179 except AttributeError:
5180 funame = "";
5181 except ImportError:
5182 funame = "";
5183 fgname = "";
5184 try:
5185 import grp;
5186 try:
5187 groupinfo = grp.getgrgid(os.getgid());
5188 fgname = groupinfo.gr_name;
5189 except KeyError:
5190 fgname = "";
5191 except AttributeError:
5192 fgname = "";
5193 except ImportError:
5194 fgname = "";
5195 fcontents = BytesIO();
5196 if(ftype==0):
5197 fcontents.write(rarfp.read(member.filename));
5198 fcontents.seek(0, 0);
5199 ftypehex = format(ftype, 'x').lower();
5200 extrafields = len(extradata);
5201 extrafieldslist = extradata;
5202 catfextrafields = extrafields;
5203 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
5204 if(len(extradata)>0):
5205 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
5206 extrasizelen = len(extrasizestr);
5207 extrasizelenhex = format(extrasizelen, 'x').lower();
5208 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5209 catoutlen = len(catoutlist) + len(extradata) + 3;
5210 catoutlenhex = format(catoutlen, 'x').lower();
5211 catoutlist.insert(0, catoutlenhex);
5212 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
5213 if(len(extradata)>0):
5214 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
5215 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
5216 catfnumfields = 24 + catfextrafields;
5217 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5218 fcontents.seek(0, 0);
5219 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5220 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5221 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5222 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
5223 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5224 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5225 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5226 nullstrecd = formatspecs[5].encode('UTF-8');
5227 fheadtell += len(catfileoutstr) + 1;
5228 catfcontentend = fheadtell - 1;
5229 fcontents.seek(0, 0);
5230 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5231 pyhascontents = False;
5232 if(int(fsize)>0 and not listonly):
5233 pyhascontents = True;
5234 if(int(fsize)>0 and listonly):
5235 fcontents = BytesIO();
5236 pyhascontents = False;
5237 fcontents.seek(0, 0);
5238 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5239 fileidnum = fileidnum + 1;
5240 return catlist;
5241
5242 if(not py7zr_support):
5243 def SevenZipFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5244 return False;
5245
5246 if(py7zr_support):
5247 def SevenZipFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5248 curinode = 0;
5249 curfid = 0;
5250 inodelist = [];
5251 inodetofile = {};
5252 filetoinode = {};
5253 inodetocatinode = {};
5254 fileidnum = 0;
5255 szpfp = py7zr.SevenZipFile(infile, mode="r");
5256 file_content = szpfp.readall();
5257 #sztest = szpfp.testzip();
5258 sztestalt = szpfp.test();
5259 if(sztestalt):
5260 VerbosePrintOut("Bad file found!");
5261 numfiles = int(len(szpfp.list()));
5262 catver = formatspecs[6];
5263 fileheaderver = str(int(catver.replace(".", "")));
5264 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
5265 catversion = re.findall(r"([\d]+)$", fileheader);
5266 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
5267 fnumfileshex = format(int(fnumfiles), 'x').lower();
5268 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs[5]);
5269 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
5270 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
5271 fheadtell = len(fileheader);
5272 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5273 for member in sorted(szpfp.list(), key=lambda x: x.filename):
5274 if(re.findall("^[.|/]", member.filename)):
5275 fname = member.filename;
5276 else:
5277 fname = "./"+member.filename;
5278 if(not member.is_directory):
5279 fpremode = int(stat.S_IFREG + 438);
5280 elif(member.is_directory):
5281 fpremode = int(stat.S_IFDIR + 511);
5282 fwinattributes = int(0);
5283 fcompression = "none";
5284 fcsize = 0;
5285 flinkcount = 0;
5286 ftype = 0;
5287 if(member.is_directory):
5288 ftype = 5;
5289 else:
5290 ftype = 0;
5291 flinkname = "";
5292 fbasedir = os.path.dirname(fname);
5293 fcurfid = curfid;
5294 fcurinode = curfid;
5295 finode = fcurinode;
5296 curfid = curfid + 1;
5297 fdev_minor = 0;
5298 fdev_major = 0;
5299 frdev_minor = 0;
5300 frdev_major = 0;
5301 if(ftype==5):
5302 fsize = "0";
5303 fatime = int(member.creationtime.timestamp());
5304 fmtime = int(member.creationtime.timestamp());
5305 fctime = int(member.creationtime.timestamp());
5306 fbtime = int(member.creationtime.timestamp());
5307 if(member.is_directory):
5308 fmode = int(stat.S_IFDIR + 511)
5309 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
5310 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
5311 else:
5312 fmode = int(stat.S_IFLNK + 438)
5313 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5314 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5315 try:
5316 fuid = os.getuid();
5317 except AttributeError:
5318 fuid = 0;
5319 except KeyError:
5320 fuid = 0;
5321 try:
5322 fgid = os.getgid();
5323 except AttributeError:
5324 fgid = 0;
5325 except KeyError:
5326 fgid = 0;
5327 try:
5328 import pwd;
5329 try:
5330 userinfo = pwd.getpwuid(os.getuid());
5331 funame = userinfo.pw_name;
5332 except KeyError:
5333 funame = "";
5334 except AttributeError:
5335 funame = "";
5336 except ImportError:
5337 funame = "";
5338 fgname = "";
5339 try:
5340 import grp;
5341 try:
5342 groupinfo = grp.getgrgid(os.getgid());
5343 fgname = groupinfo.gr_name;
5344 except KeyError:
5345 fgname = "";
5346 except AttributeError:
5347 fgname = "";
5348 except ImportError:
5349 fgname = "";
5350 fcontents = BytesIO();
5351 if(ftype==0):
5352 fcontents.write(file_content[member.filename].read());
5353 fsize = format(fcontents.tell(), 'x').lower();
5354 fileop.close();
5355 fcontents.seek(0, 0);
5356 ftypehex = format(ftype, 'x').lower();
5357 extrafields = len(extradata);
5358 extrafieldslist = extradata;
5359 catfextrafields = extrafields;
5360 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
5361 if(len(extradata)>0):
5362 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
5363 extrasizelen = len(extrasizestr);
5364 extrasizelenhex = format(extrasizelen, 'x').lower();
5365 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), fcompression, format(int(fcsize), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5366 catoutlen = len(catoutlist) + len(extradata) + 3;
5367 catoutlenhex = format(catoutlen, 'x').lower();
5368 catoutlist.insert(0, catoutlenhex);
5369 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
5370 if(len(extradata)>0):
5371 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
5372 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
5373 catfnumfields = 24 + catfextrafields;
5374 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5375 fcontents.seek(0, 0);
5376 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5377 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5378 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5379 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
5380 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5381 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5382 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5383 nullstrecd = formatspecs[5].encode('UTF-8');
5384 fheadtell += len(catfileoutstr) + 1;
5385 catfcontentend = fheadtell - 1;
5386 fcontents.seek(0, 0);
5387 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5388 pyhascontents = False;
5389 if(int(fsize)>0 and not listonly):
5390 pyhascontents = True;
5391 if(int(fsize)>0 and listonly):
5392 fcontents = "";
5393 pyhascontents = False;
5394 fcontents.seek(0, 0);
5395 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fcompression': fcompression, 'fcsize': fcsize, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5396 fileidnum = fileidnum + 1;
5397 return catlist;
5398
5399 def InFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5400 checkcompressfile = CheckCompressionSubType(infiles, formatspecs, True);
5401 if(checkcompressfile=="tarfile"):
5402 return TarFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5403 elif(checkcompressfile=="zipfile"):
5404 return ZipFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5405 elif(checkcompressfile=="catfile"):
5406 return ArchiveFileToArray(infile, 0, 0, listonly, False, formatspecs, False);
5407 elif(rarfile_support and checkcompressfile=="rarfile"):
5408 return RarFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5409 elif(py7zr_support and checkcompressfile=="7zipfile"):
5410 return SevenZipFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5411 else:
5412 return False;
5413 return False;
5414
5415 def ListDirToArray(infiles, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, seekstart=0, seekend=0, listonly=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
5416 outarray = BytesIO();
5417 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
5418 listcatfiles = ArchiveFileToArray(outarray, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5419 return listcatfiles;
5420
5421 def ArchiveFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5422 if(isinstance(infile, dict)):
5423 listcatfiles = infile;
5424 else:
5425 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
5426 infile = RemoveWindowsPath(infile);
5427 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5428 if(not listcatfiles):
5429 return False;
5430 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5431 if(returnfp):
5432 catarray.update({'catfp': listcatfiles['catfp']});
5433 lenlist = len(listcatfiles['ffilelist']);
5434 lcfi = 0;
5435 lcfx = int(listcatfiles['fnumfiles']);
5436 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5437 lcfx = int(lenlist);
5438 else:
5439 lcfx = int(listcatfiles['fnumfiles']);
5440 while(lcfi < lcfx):
5441 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5442 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5443 catarray['filetoid'].update(filetoidarray);
5444 catarray['idtofile'].update(idtofilearray);
5445 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5446 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5447 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5448 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5449 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5450 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5451 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5452 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5453 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5454 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5455 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5456 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5457 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5458 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5459 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5460 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5461 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5462 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5463 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5464 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5465 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5466 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5467 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5468 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5469 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5470 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5471 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
5472 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5473 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5474 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5475 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5476 lcfi = lcfi + 1;
5477 return catarray;
5478
5479 create_alias_function("", __file_format_name__, "ToArrayIndex", ArchiveFileToArrayIndex);
5480
5481 def ListDirToArrayIndexAlt(infiles, dirlistfromtxt=False, followlink=False, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5482 listcatfiles = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, listonly, checksumtype, extradata, formatspecs, verbose);
5483 if(not listcatfiles):
5484 return False;
5485 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5486 lenlist = len(listcatfiles['ffilelist']);
5487 lcfi = 0;
5488 lcfx = int(listcatfiles['fnumfiles']);
5489 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5490 lcfx = int(lenlist);
5491 else:
5492 lcfx = int(listcatfiles['fnumfiles']);
5493 while(lcfi < lcfx):
5494 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5495 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5496 catarray['filetoid'].update(filetoidarray);
5497 catarray['idtofile'].update(idtofilearray);
5498 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5499 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5500 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5501 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5502 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5503 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5504 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5505 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5506 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5507 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5508 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5509 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5510 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5511 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5512 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5513 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5514 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5515 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5516 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5517 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5518 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5519 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5520 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5521 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5522 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5523 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5524 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
5525 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5526 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5527 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5528 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5529 lcfi = lcfi + 1;
5530 return catarray;
5531
5532 def TarFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5533 listcatfiles = TarFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5534 if(not listcatfiles):
5535 return False;
5536 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5537 lenlist = len(listcatfiles['ffilelist']);
5538 lcfi = 0;
5539 lcfx = int(listcatfiles['fnumfiles']);
5540 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5541 lcfx = int(lenlist);
5542 else:
5543 lcfx = int(listcatfiles['fnumfiles']);
5544 while(lcfi < lcfx):
5545 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5546 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5547 catarray['filetoid'].update(filetoidarray);
5548 catarray['idtofile'].update(idtofilearray);
5549 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5550 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5551 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5552 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5553 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5554 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5555 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5556 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5557 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5558 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5559 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5560 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5561 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5562 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5563 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5564 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5565 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5566 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5567 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5568 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5569 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5570 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5571 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5572 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5573 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5574 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5575 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
5576 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5577 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5578 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5579 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5580 lcfi = lcfi + 1;
5581 return catarray;
5582
5583 def ZipFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5584 listcatfiles = ZipFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5585 if(not listcatfiles):
5586 return False;
5587 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5588 lenlist = len(listcatfiles['ffilelist']);
5589 lcfi = 0;
5590 lcfx = int(listcatfiles['fnumfiles']);
5591 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5592 lcfx = int(lenlist);
5593 else:
5594 lcfx = int(listcatfiles['fnumfiles']);
5595 while(lcfi < lcfx):
5596 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5597 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5598 catarray['filetoid'].update(filetoidarray);
5599 catarray['idtofile'].update(idtofilearray);
5600 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5601 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5602 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5603 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5604 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5605 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5606 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5607 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5608 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5609 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5610 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5611 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5612 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5613 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5614 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5615 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5616 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5617 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5618 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5619 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5620 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5621 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5622 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5623 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5624 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5625 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5626 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
5627 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5628 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5629 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5630 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5631 lcfi = lcfi + 1;
5632 return catarray;
5633
5634 if(not rarfile_support):
5635 def RarFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5636 return False;
5637
5638 if(rarfile_support):
5639 def RarFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5640 listcatfiles = RarFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5641 if(not listcatfiles):
5642 return False;
5643 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5644 lenlist = len(listcatfiles['ffilelist']);
5645 lcfi = 0;
5646 lcfx = int(listcatfiles['fnumfiles']);
5647 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5648 lcfx = int(lenlist);
5649 else:
5650 lcfx = int(listcatfiles['fnumfiles']);
5651 while(lcfi < lcfx):
5652 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5653 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5654 catarray['filetoid'].update(filetoidarray);
5655 catarray['idtofile'].update(idtofilearray);
5656 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5657 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5658 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5659 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5660 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5661 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5662 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5663 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5664 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5665 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5666 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5667 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5668 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5669 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5670 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5671 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5672 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5673 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5674 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5675 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5676 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5677 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5678 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5679 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5680 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5681 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5682 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
5683 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5684 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5685 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5686 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5687 lcfi = lcfi + 1;
5688 return catarray;
5689
5690 if(not py7zr_support):
5691 def SevenZipFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5692 return False;
5693
5694 if(py7zr_support):
5695 def SevenZipFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5696 listcatfiles = SevenZipFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5697 if(not listcatfiles):
5698 return False;
5699 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5700 lenlist = len(listcatfiles['ffilelist']);
5701 lcfi = 0;
5702 lcfx = int(listcatfiles['fnumfiles']);
5703 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5704 lcfx = int(lenlist);
5705 else:
5706 lcfx = int(listcatfiles['fnumfiles']);
5707 while(lcfi < lcfx):
5708 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5709 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5710 catarray['filetoid'].update(filetoidarray);
5711 catarray['idtofile'].update(idtofilearray);
5712 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5713 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5714 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5715 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5716 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5717 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5718 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5719 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5720 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5721 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5722 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5723 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5724 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5725 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5726 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5727 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5728 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5729 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5730 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5731 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5732 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5733 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5734 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5735 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5736 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5737 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5738 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
5739 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5740 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5741 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5742 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5743 lcfi = lcfi + 1;
5744 return catarray;
5745
5746 def ArchiveFileStringToArrayIndex(catstr, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5747 catfp = BytesIO(catstr);
5748 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5749 return listcatfiles;
5750
5751 create_alias_function("", __file_format_name__, "StringToArrayIndex", ArchiveFileStringToArrayIndex);
5752
5753 def TarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5754 catfp = BytesIO();
5755 catfp = PackArchiveFileFromTarFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
5756 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5757 return listcatfiles;
5758
5759 def ZipFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5760 catfp = BytesIO();
5761 catfp = PackArchiveFileFromZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
5762 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5763 return listcatfiles;
5764
5765 if(not rarfile_support):
5766 def RarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5767 return False;
5768
5769 if(rarfile_support):
5770 def RarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5771 catfp = BytesIO();
5772 catfp = PackArchiveFileFromRarFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
5773 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5774 return listcatfiles;
5775
5776 if(not py7zr_support):
5777 def SevenZipFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5778 return False;
5779
5780 if(py7zr_support):
5781 def SevenZipFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5782 catfp = BytesIO();
5783 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
5784 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5785 return listcatfiles;
5786
5787 def ListDirToArrayIndex(infiles, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, seekstart=0, seekend=0, listonly=False, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False, returnfp=False):
5788 outarray = BytesIO();
5789 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compressionlevel, followlink, checksumtype, formatspecs, verbose, True);
5790 listcatfiles = ArchiveFileToArrayIndex(outarray, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp)
5791 return listcatfiles;
5792
5793 def RePackArchiveFile(infile, outfile, compression="auto", compressionlevel=None, followlink=False, seekstart=0, seekend=0, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
5794 if(isinstance(infile, dict)):
5795 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
5796 listcatfiles = prelistcatfiles['list'];
5797 else:
5798 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
5799 infile = RemoveWindowsPath(infile);
5800 if(followlink):
5801 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
5802 listcatfiles = prelistcatfiles['list'];
5803 else:
5804 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
5805 if(outfile!="-" and not hasattr(infile, "read") and not hasattr(outfile, "write")):
5806 outfile = RemoveWindowsPath(outfile);
5807 checksumtype = checksumtype.lower();
5808 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
5809 checksumtype="crc32";
5810 if(checksumtype=="none"):
5811 checksumtype = "";
5812 if(not compression or compression=="catfile" or compression==formatspecs[2]):
5813 compression = "auto";
5814 if(compression not in compressionlist and compression is None):
5815 compression = "auto";
5816 if(verbose):
5817 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
5818 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
5819 if(os.path.exists(outfile)):
5820 try:
5821 os.unlink(outfile);
5822 except OSError as e:
5823 pass;
5824 if(not listcatfiles):
5825 return False;
5826 if(outfile=="-"):
5827 verbose = False;
5828 catfp = BytesIO();
5829 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
5830 catfp = outfile;
5831 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
5832 catfp = BytesIO();
5833 else:
5834 fbasename = os.path.splitext(outfile)[0];
5835 fextname = os.path.splitext(outfile)[1];
5836 catfp = CompressOpenFile(outfile, True, compressionlevel);
5837 catver = formatspecs[6];
5838 fileheaderver = str(int(catver.replace(".", "")));
5839 lenlist = len(listcatfiles['ffilelist']);
5840 fnumfiles = int(listcatfiles['fnumfiles']);
5841 if(lenlist>fnumfiles or lenlist<fnumfiles):
5842 fnumfiles = lenlist;
5843 AppendFileHeader(catfp, fnumfiles, checksumtype, formatspecs);
5844 lenlist = len(listcatfiles['ffilelist']);
5845 fnumfiles = int(listcatfiles['fnumfiles']);
5846 lcfi = 0;
5847 lcfx = int(listcatfiles['fnumfiles']);
5848 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5849 lcfx = int(lenlist);
5850 else:
5851 lcfx = int(listcatfiles['fnumfiles']);
5852 curinode = 0;
5853 curfid = 0;
5854 inodelist = [];
5855 inodetofile = {};
5856 filetoinode = {};
5857 reallcfi = 0;
5858 while(lcfi < lcfx):
5859 if(re.findall("^[.|/]", listcatfiles['ffilelist'][reallcfi]['fname'])):
5860 fname = listcatfiles['ffilelist'][reallcfi]['fname'];
5861 else:
5862 fname = "./"+listcatfiles['ffilelist'][reallcfi]['fname'];
5863 if(verbose):
5864 VerbosePrintOut(fname);
5865 fheadersize = format(int(listcatfiles['ffilelist'][reallcfi]['fheadersize']), 'x').lower();
5866 fsize = format(int(listcatfiles['ffilelist'][reallcfi]['fsize']), 'x').lower();
5867 flinkname = listcatfiles['ffilelist'][reallcfi]['flinkname'];
5868 fatime = format(int(listcatfiles['ffilelist'][reallcfi]['fatime']), 'x').lower();
5869 fmtime = format(int(listcatfiles['ffilelist'][reallcfi]['fmtime']), 'x').lower();
5870 fctime = format(int(listcatfiles['ffilelist'][reallcfi]['fctime']), 'x').lower();
5871 fbtime = format(int(listcatfiles['ffilelist'][reallcfi]['fbtime']), 'x').lower();
5872 fmode = format(int(listcatfiles['ffilelist'][reallcfi]['fmode']), 'x').lower();
5873 fchmode = format(int(listcatfiles['ffilelist'][reallcfi]['fchmode']), 'x').lower();
5874 fuid = format(int(listcatfiles['ffilelist'][reallcfi]['fuid']), 'x').lower();
5875 funame = listcatfiles['ffilelist'][reallcfi]['funame'];
5876 fgid = format(int(listcatfiles['ffilelist'][reallcfi]['fgid']), 'x').lower();
5877 fgname = listcatfiles['ffilelist'][reallcfi]['fgname'];
5878 finode = format(listcatfiles['ffilelist'][reallcfi]['finode'], 'x').lower();
5879 flinkcount = format(listcatfiles['ffilelist'][reallcfi]['flinkcount'], 'x').lower();
5880 fwinattributes = format(listcatfiles['ffilelist'][reallcfi]['fwinattributes'], 'x').lower();
5881 fcompression = listcatfiles['ffilelist'][reallcfi]['fcompression'];
5882 fcsize = listcatfiles['ffilelist'][reallcfi]['fcsize'];
5883 fdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['fminor']), 'x').lower();
5884 fdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['fmajor']), 'x').lower();
5885 frdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['frminor']), 'x').lower();
5886 frdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['frmajor']), 'x').lower();
5887 if(len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>listcatfiles['ffilelist'][reallcfi]['fextrafields'] and len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>0):
5888 listcatfiles['ffilelist'][reallcfi]['fextrafields'] = len(listcatfiles['ffilelist'][reallcfi]['fextralist']);
5889 if(not followlink and len(extradata)<0):
5890 extradata = listcatfiles['ffilelist'][reallcfi]['fextralist'];
5891 fcontents = listcatfiles['ffilelist'][reallcfi]['fcontents'];
5892 if(followlink):
5893 if(listcatfiles['ffilelist'][reallcfi]['ftype']==1 or listcatfiles['ffilelist'][reallcfi]['ftype']==2):
5894 getflinkpath = listcatfiles['ffilelist'][reallcfi]['flinkname'];
5895 flinkid = prelistcatfiles['filetoid'][getflinkpath];
5896 flinkinfo = listcatfiles['ffilelist'][flinkid];
5897 fheadersize = format(int(flinkinfo['fheadersize']), 'x').lower();
5898 fsize = format(int(flinkinfo['fsize']), 'x').lower();
5899 flinkname = flinkinfo['flinkname'];
5900 fatime = format(int(flinkinfo['fatime']), 'x').lower();
5901 fmtime = format(int(flinkinfo['fmtime']), 'x').lower();
5902 fctime = format(int(flinkinfo['fctime']), 'x').lower();
5903 fbtime = format(int(flinkinfo['fbtime']), 'x').lower();
5904 fmode = format(int(flinkinfo['fmode']), 'x').lower();
5905 fchmode = format(int(flinkinfo['fchmode']), 'x').lower();
5906 fuid = format(int(flinkinfo['fuid']), 'x').lower();
5907 funame = flinkinfo['funame'];
5908 fgid = format(int(flinkinfo['fgid']), 'x').lower();
5909 fgname = flinkinfo['fgname'];
5910 finode = format(flinkinfo['finode'], 'x').lower();
5911 flinkcount = format(flinkinfo['flinkcount'], 'x').lower();
5912 fwinattributes = format(flinkinfo['fwinattributes'], 'x').lower();
5913 fcompression = flinkinfo['fcompression'];
5914 fcsize = flinkinfo['fcsize'];
5915 fdev_minor = format(int(flinkinfo['fminor']), 'x').lower();
5916 fdev_major = format(int(flinkinfo['fmajor']), 'x').lower();
5917 frdev_minor = format(int(flinkinfo['frminor']), 'x').lower();
5918 frdev_major = format(int(flinkinfo['frmajor']), 'x').lower();
5919 if(len(flinkinfo['fextralist'])>flinkinfo['fextrafields'] and len(flinkinfo['fextralist'])>0):
5920 flinkinfo['fextrafields'] = len(flinkinfo['fextralist']);
5921 if(len(extradata)<0):
5922 extradata = flinkinfo['fextralist'];
5923 fcontents = flinkinfo['fcontents'];
5924 ftypehex = format(flinkinfo['ftype'], 'x').lower();
5925 else:
5926 ftypehex = format(listcatfiles['ffilelist'][reallcfi]['ftype'], 'x').lower();
5927 fcurfid = format(curfid, 'x').lower();
5928 if(not followlink and finode!=0):
5929 if(listcatfiles['ffilelist'][reallcfi]['ftype']!=1):
5930 fcurinode = format(int(curinode), 'x').lower();
5931 inodetofile.update({curinode: fname});
5932 filetoinode.update({fname: curinode});
5933 curinode = curinode + 1;
5934 else:
5935 fcurinode = format(int(filetoinode[flinkname]), 'x').lower();
5936 else:
5937 fcurinode = format(int(curinode), 'x').lower();
5938 curinode = curinode + 1;
5939 curfid = curfid + 1;
5940 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fcompression, fcsize, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
5941 AppendFileHeaderWithContent(catfp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
5942 fcontents.close();
5943 lcfi = lcfi + 1;
5944 reallcfi = reallcfi + 1;
5945 if(lcfx>0):
5946 catfp.write(AppendNullBytes([0, 0], formatspecs[5]).encode("UTF-8"));
5947 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
5948 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
5949 try:
5950 catfp.flush();
5951 os.fsync(catfp.fileno());
5952 except io.UnsupportedOperation:
5953 pass;
5954 except AttributeError:
5955 pass;
5956 except OSError as e:
5957 pass;
5958 if(outfile=="-"):
5959 catfp.seek(0, 0);
5960 if(hasattr(sys.stdout, "buffer")):
5961 shutil.copyfileobj(catfp, sys.stdout.buffer);
5962 else:
5963 shutil.copyfileobj(catfp, sys.stdout);
5964 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
5965 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
5966 catfp.seek(0, 0);
5967 upload_file_to_internet_file(catfp, outfile);
5968 if(returnfp):
5969 catfp.seek(0, 0);
5970 return catfp;
5971 else:
5972 catfp.close();
5973 return True;
5974
5975 create_alias_function("RePack", __file_format_name__, "", RePackArchiveFile);
5976
5977 def RePackArchiveFileFromString(catstr, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
5978 catfp = BytesIO(catstr);
5979 listcatfiles = RePackArchiveFile(catfp, compression, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
5980 return listcatfiles;
5981
5982 create_alias_function("RePack", __file_format_name__, "FromString", RePackArchiveFileFromString);
5983
5984 def PackArchiveFileFromListDir(infiles, outfile, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
5985 outarray = BytesIO();
5986 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
5987 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
5988 return listcatfiles;
5989
5990 create_alias_function("Pack", __file_format_name__, "FromListDir", PackArchiveFileFromListDir);
5991
5992 def UnPackArchiveFile(infile, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_list__, preservepermissions=True, preservetime=True, verbose=False, returnfp=False):
5993 if(outdir is not None):
5994 outdir = RemoveWindowsPath(outdir);
5995 if(verbose):
5996 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
5997 if(isinstance(infile, dict)):
5998 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
5999 listcatfiles = prelistcatfiles['list'];
6000 else:
6001 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6002 infile = RemoveWindowsPath(infile);
6003 if(followlink):
6004 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6005 listcatfiles = prelistcatfiles['list'];
6006 else:
6007 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6008 if(not listcatfiles):
6009 return False;
6010 lenlist = len(listcatfiles['ffilelist']);
6011 fnumfiles = int(listcatfiles['fnumfiles']);
6012 lcfi = 0;
6013 lcfx = int(listcatfiles['fnumfiles']);
6014 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6015 lcfx = int(lenlist);
6016 else:
6017 lcfx = int(listcatfiles['fnumfiles']);
6018 while(lcfi<lcfx):
6019 funame = "";
6020 try:
6021 import pwd;
6022 try:
6023 userinfo = pwd.getpwuid(listcatfiles['ffilelist'][lcfi]['fuid']);
6024 funame = userinfo.pw_name;
6025 except KeyError:
6026 funame = "";
6027 except ImportError:
6028 funame = "";
6029 fgname = "";
6030 try:
6031 import grp;
6032 try:
6033 groupinfo = grp.getgrgid(listcatfiles['ffilelist'][lcfi]['fgid']);
6034 fgname = groupinfo.gr_name;
6035 except KeyError:
6036 fgname = "";
6037 except ImportError:
6038 fgname = "";
6039 if(verbose):
6040 VerbosePrintOut(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6041 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6042 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6043 listcatfiles['ffilelist'][lcfi]['fcontents'].seek(0, 0);
6044 shutil.copyfileobj(listcatfiles['ffilelist'][lcfi]['fcontents'], fpc);
6045 try:
6046 fpc.flush();
6047 os.fsync(fpc.fileno());
6048 except io.UnsupportedOperation:
6049 pass;
6050 except AttributeError:
6051 pass;
6052 except OSError as e:
6053 pass;
6054 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
6055 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
6056 if(preservepermissions):
6057 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6058 if(preservetime):
6059 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
6060 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6061 if(followlink):
6062 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
6063 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6064 flinkinfo = listcatfiles['ffilelist'][flinkid];
6065 funame = "";
6066 try:
6067 import pwd;
6068 try:
6069 userinfo = pwd.getpwuid(flinkinfo['fuid']);
6070 funame = userinfo.pw_name;
6071 except KeyError:
6072 funame = "";
6073 except ImportError:
6074 funame = "";
6075 fgname = "";
6076 try:
6077 import grp;
6078 try:
6079 groupinfo = grp.getgrgid(flinkinfo['fgid']);
6080 fgname = groupinfo.gr_name;
6081 except KeyError:
6082 fgname = "";
6083 except ImportError:
6084 fgname = "";
6085 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
6086 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6087 flinkinfo['fcontents'].seek(0, 0);
6088 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
6089 try:
6090 fpc.flush();
6091 os.fsync(fpc.fileno());
6092 except io.UnsupportedOperation:
6093 pass;
6094 except AttributeError:
6095 pass;
6096 except OSError as e:
6097 pass;
6098 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6099 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6100 if(preservepermissions):
6101 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6102 if(preservetime):
6103 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6104 if(flinkinfo['ftype']==1):
6105 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6106 if(flinkinfo['ftype']==2):
6107 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6108 if(flinkinfo['ftype']==5):
6109 if(preservepermissions):
6110 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6111 else:
6112 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6113 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6114 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6115 if(preservepermissions):
6116 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6117 if(preservetime):
6118 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6119 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
6120 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6121 else:
6122 os.link(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6123 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6124 if(followlink):
6125 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
6126 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6127 flinkinfo = listcatfiles['ffilelist'][flinkid];
6128 funame = "";
6129 try:
6130 import pwd;
6131 try:
6132 userinfo = pwd.getpwuid(flinkinfo['fuid']);
6133 funame = userinfo.pw_name;
6134 except KeyError:
6135 funame = "";
6136 except ImportError:
6137 funame = "";
6138 fgname = "";
6139 try:
6140 import grp;
6141 try:
6142 groupinfo = grp.getgrgid(flinkinfo['fgid']);
6143 fgname = groupinfo.gr_name;
6144 except KeyError:
6145 fgname = "";
6146 except ImportError:
6147 fgname = "";
6148 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
6149 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6150 flinkinfo['fcontents'].seek(0, 0);
6151 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
6152 try:
6153 fpc.flush();
6154 os.fsync(fpc.fileno());
6155 except io.UnsupportedOperation:
6156 pass;
6157 except AttributeError:
6158 pass;
6159 except OSError as e:
6160 pass;
6161 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6162 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6163 if(preservepermissions):
6164 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6165 if(preservetime):
6166 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6167 if(flinkinfo['ftype']==1):
6168 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6169 if(flinkinfo['ftype']==2):
6170 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6171 if(flinkinfo['ftype']==5):
6172 if(preservepermissions):
6173 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6174 else:
6175 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6176 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6177 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6178 if(preservepermissions):
6179 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6180 if(preservetime):
6181 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6182 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
6183 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6184 else:
6185 os.symlink(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6186 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6187 if(preservepermissions):
6188 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6189 else:
6190 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6191 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
6192 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
6193 if(preservepermissions):
6194 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6195 if(preservetime):
6196 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
6197 if(listcatfiles['ffilelist'][lcfi]['ftype']==6 and hasattr(os, "mkfifo")):
6198 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6199 lcfi = lcfi + 1;
6200 if(returnfp):
6201 return listcatfiles['ffilelist']['catfp'];
6202 else:
6203 return True;
6204
6205 create_alias_function("UnPack", __file_format_name__, "", UnPackArchiveFile);
6206
6207 if(hasattr(shutil, "register_unpack_format")):
6208 def UnPackArchiveFileFunc(archive_name, extract_dir=None, **kwargs):
6209 return UnPackArchiveFile(archive_name, extract_dir, False, 0, 0, False, __file_format_delimiter__, False, False);
6210 create_alias_function("UnPack", __file_format_name__, "Func", UnPackArchiveFileFunc);
6211
6212 def UnPackArchiveFileString(catstr, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_list__, verbose=False, returnfp=False):
6213 catfp = BytesIO(catstr);
6214 listcatfiles = UnPackArchiveFile(catfp, outdir, followlink, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6215 return listcatfiles;
6216
6217 create_alias_function("UnPack", __file_format_name__, "String", UnPackArchiveFileString);
6218
6219 def ArchiveFileListFiles(infile, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_list__, verbose=False, returnfp=False):
6220 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6221 if(isinstance(infile, dict)):
6222 listcatfiles = infile;
6223 else:
6224 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6225 infile = RemoveWindowsPath(infile);
6226 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, True, skipchecksum, formatspecs, returnfp);
6227 if(not listcatfiles):
6228 return False;
6229 lenlist = len(listcatfiles['ffilelist']);
6230 fnumfiles = int(listcatfiles['fnumfiles']);
6231 lcfi = 0;
6232 lcfx = int(listcatfiles['fnumfiles']);
6233 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6234 lcfx = int(lenlist);
6235 else:
6236 lcfx = int(listcatfiles['fnumfiles']);
6237 returnval = {};
6238 while(lcfi<lcfx):
6239 returnval.update({lcfi: listcatfiles['ffilelist'][lcfi]['fname']});
6240 if(not verbose):
6241 VerbosePrintOut(listcatfiles['ffilelist'][lcfi]['fname']);
6242 if(verbose):
6243 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6244 printfname = listcatfiles['ffilelist'][lcfi]['fname'];
6245 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6246 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " link to " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6247 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6248 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " -> " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6249 fuprint = listcatfiles['ffilelist'][lcfi]['funame'];
6250 if(len(fuprint)<=0):
6251 fuprint = listcatfiles['ffilelist'][lcfi]['fuid'];
6252 fgprint = listcatfiles['ffilelist'][lcfi]['fgname'];
6253 if(len(fgprint)<=0):
6254 fgprint = listcatfiles['ffilelist'][lcfi]['fgid'];
6255 VerbosePrintOut(PrintPermissionString(listcatfiles['ffilelist'][lcfi]['fmode'], listcatfiles['ffilelist'][lcfi]['ftype']) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(listcatfiles['ffilelist'][lcfi]['fsize']).rjust(15) + " " + datetime.datetime.utcfromtimestamp(listcatfiles['ffilelist'][lcfi]['fmtime']).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6256 lcfi = lcfi + 1;
6257 if(returnfp):
6258 return listcatfiles['catfp'];
6259 else:
6260 return True;
6261
6262 create_alias_function("", __file_format_name__, "ListFiles", ArchiveFileListFiles);
6263
6264 def ArchiveFileStringListFiles(catstr, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_list__, verbose=False, returnfp=False):
6265 catfp = BytesIO(catstr);
6266 listcatfiles = ArchiveFileListFiles(catstr, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6267 return listcatfiles;
6268
6269 create_alias_function("", __file_format_name__, "StringListFiles", ArchiveFileStringListFiles);
6270
6271 def TarFileListFiles(infile, verbose=False, returnfp=False):
6272 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6273 if(infile=="-"):
6274 infile = BytesIO();
6275 if(hasattr(sys.stdin, "buffer")):
6276 shutil.copyfileobj(sys.stdin.buffer, infile);
6277 else:
6278 shutil.copyfileobj(sys.stdin, infile);
6279 infile.seek(0, 0);
6280 if(not infile):
6281 return False;
6282 infile.seek(0, 0);
6283 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
6284 infile = download_file_from_internet_file(infile);
6285 infile.seek(0, 0);
6286 if(not infile):
6287 return False;
6288 infile.seek(0, 0);
6289 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
6290 return False;
6291 elif(os.path.exists(infile) and os.path.isfile(infile)):
6292 try:
6293 if(not tarfile.is_tarfile(infile)):
6294 return False;
6295 except AttributeError:
6296 if(not is_tarfile(infile)):
6297 return False;
6298 else:
6299 return False;
6300 try:
6301 if(hasattr(infile, "read") or hasattr(infile, "write")):
6302 tarfp = tarfile.open(fileobj=infile, mode="r");
6303 else:
6304 tarfp = tarfile.open(infile, "r");
6305 except FileNotFoundError:
6306 return False;
6307 lcfi = 0
6308 returnval = {};
6309 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
6310 returnval.update({lcfi: member.name});
6311 fpremode = member.mode;
6312 ffullmode = member.mode;
6313 flinkcount = 0;
6314 ftype = 0;
6315 if(member.isreg()):
6316 ffullmode = member.mode + stat.S_IFREG;
6317 ftype = 0;
6318 elif(member.isdev()):
6319 ffullmode = member.mode;
6320 ftype = 7;
6321 elif(member.islnk()):
6322 ffullmode = member.mode + stat.S_IFREG;
6323 ftype = 1;
6324 elif(member.issym()):
6325 ffullmode = member.mode + stat.S_IFLNK;
6326 ftype = 2;
6327 elif(member.ischr()):
6328 ffullmode = member.mode + stat.S_IFCHR;
6329 ftype = 3;
6330 elif(member.isblk()):
6331 ffullmode = member.mode + stat.S_IFBLK;
6332 ftype = 4;
6333 elif(member.isdir()):
6334 ffullmode = member.mode + stat.S_IFDIR;
6335 ftype = 5;
6336 elif(member.isfifo()):
6337 ffullmode = member.mode + stat.S_IFIFO;
6338 ftype = 6;
6339 elif(member.issparse()):
6340 ffullmode = member.mode;
6341 ftype = 12;
6342 if(not verbose):
6343 VerbosePrintOut(member.name);
6344 elif(verbose):
6345 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6346 printfname = member.name;
6347 if(member.islnk()):
6348 printfname = member.name + " link to " + member.linkname;
6349 elif(member.issym()):
6350 printfname = member.name + " -> " + member.linkname;
6351 fuprint = member.uname;
6352 if(len(fuprint)<=0):
6353 fuprint = member.uid;
6354 fgprint = member.gname;
6355 if(len(fgprint)<=0):
6356 fgprint = member.gid;
6357 VerbosePrintOut(PrintPermissionString(ffullmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(member.mtime).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6358 lcfi = lcfi + 1;
6359 if(returnfp):
6360 return listcatfiles['catfp'];
6361 else:
6362 return True;
6363
6364 def ZipFileListFiles(infile, verbose=False, returnfp=False):
6365 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6366 if(infile=="-"):
6367 infile = BytesIO();
6368 if(hasattr(sys.stdin, "buffer")):
6369 shutil.copyfileobj(sys.stdin.buffer, infile);
6370 else:
6371 shutil.copyfileobj(sys.stdin, infile);
6372 infile.seek(0, 0);
6373 if(not infile):
6374 return False;
6375 infile.seek(0, 0);
6376 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
6377 infile = download_file_from_internet_file(infile);
6378 infile.seek(0, 0);
6379 if(not infile):
6380 return False;
6381 infile.seek(0, 0);
6382 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
6383 return False;
6384 else:
6385 return False;
6386 if(not zipfile.is_zipfile(infile)):
6387 return False;
6388 try:
6389 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
6390 except FileNotFoundError:
6391 return False;
6392 lcfi = 0;
6393 returnval = {};
6394 ziptest = zipfp.testzip();
6395 if(ziptest):
6396 VerbosePrintOut("Bad file found!");
6397 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
6398 if(zipinfo.create_system==0 or zipinfo.create_system==10):
6399 fwinattributes = int(zipinfo.external_attr);
6400 if(not member.is_dir()):
6401 fmode = int(stat.S_IFREG + 438);
6402 fchmode = int(stat.S_IMODE(fmode));
6403 ftypemod = int(stat.S_IFMT(fmode));
6404 elif(member.is_dir()):
6405 fmode = int(stat.S_IFDIR + 511);
6406 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6407 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6408 elif(zipinfo.create_system==3):
6409 fwinattributes =int(0);
6410 fmode = int(zipinfo.external_attr);
6411 fchmode = int(stat.S_IMODE(fmode));
6412 ftypemod = int(stat.S_IFMT(fmode));
6413 else:
6414 fwinattributes = int(0);
6415 if(not member.is_dir()):
6416 fmode = int(stat.S_IFREG + 438);
6417 fchmode = int(stat.S_IMODE(fmode));
6418 ftypemod = int(stat.S_IFMT(fmode));
6419 elif(member.is_dir()):
6420 fmode = int(stat.S_IFDIR + 511);
6421 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6422 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6423 returnval.update({lcfi: member.filename});
6424 if(not verbose):
6425 VerbosePrintOut(member.filename);
6426 if(verbose):
6427 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6428 permissionstr = "";
6429 for fmodval in str(oct(fmode))[-3:]:
6430 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
6431 if(not member.is_dir()):
6432 ftype = 0;
6433 permissionstr = "-" + permissionstr;
6434 elif(member.is_dir()):
6435 ftype = 5;
6436 permissionstr = "d" + permissionstr;
6437 printfname = member.filename;
6438 try:
6439 fuid = int(os.getuid());
6440 except AttributeError:
6441 fuid = int(0);
6442 except KeyError:
6443 fuid = int(0);
6444 try:
6445 fgid = int(os.getgid());
6446 except AttributeError:
6447 fgid = int(0);
6448 except KeyError:
6449 fgid = int(0);
6450 try:
6451 import pwd;
6452 try:
6453 userinfo = pwd.getpwuid(os.getuid());
6454 funame = userinfo.pw_name;
6455 except KeyError:
6456 funame = "";
6457 except AttributeError:
6458 funame = "";
6459 except ImportError:
6460 funame = "";
6461 fgname = "";
6462 try:
6463 import grp;
6464 try:
6465 groupinfo = grp.getgrgid(os.getgid());
6466 fgname = groupinfo.gr_name;
6467 except KeyError:
6468 fgname = "";
6469 except AttributeError:
6470 fgname = "";
6471 except ImportError:
6472 fgname = "";
6473 fuprint = funame;
6474 if(len(fuprint)<=0):
6475 fuprint = str(fuid);
6476 fgprint = fgname;
6477 if(len(fgprint)<=0):
6478 fgprint = str(fgid);
6479 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(int(time.mktime(member.date_time + (0, 0, -1)))).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6480 lcfi = lcfi + 1;
6481 if(returnfp):
6482 return listcatfiles['catfp'];
6483 else:
6484 return True;
6485
6486 if(not rarfile_support):
6487 def RarFileListFiles(infile, verbose=False, returnfp=False):
6488 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6489 if(not os.path.exists(infile) or not os.path.isfile(infile)):
6490 return False;
6491
6492 if(rarfile_support):
6493 def RarFileListFiles(infile, verbose=False, returnfp=False):
6494 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6495 if(not os.path.exists(infile) or not os.path.isfile(infile)):
6496 return False;
6497 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
6498 return False;
6499 lcfi = 0;
6500 returnval = {};
6501 rarfp = rarfile.RarFile(infile, "r");
6502 rartest = rarfp.testrar();
6503 if(rartest):
6504 VerbosePrintOut("Bad file found!");
6505 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
6506 is_unix = False;
6507 is_windows = False;
6508 if(member.host_os==rarfile.RAR_OS_UNIX):
6509 is_windows = False;
6510 try:
6511 member.external_attr
6512 is_unix = True;
6513 except AttributeError:
6514 is_unix = False;
6515 elif(member.host_os==rarfile.RAR_OS_WIN32):
6516 is_unix = False;
6517 try:
6518 member.external_attr
6519 is_windows = True;
6520 except AttributeError:
6521 is_windows = False;
6522 else:
6523 is_unix = False;
6524 is_windows = False;
6525 if(is_unix and member.external_attr !=0):
6526 fpremode = int(member.external_attr);
6527 elif(member.is_file()):
6528 fpremode = int(stat.S_IFREG + 438);
6529 elif(member.is_symlink()):
6530 fpremode = int(stat.S_IFLNK + 438);
6531 elif(member.is_dir()):
6532 fpremode = int(stat.S_IFDIR + 511);
6533 if(is_windows and member.external_attr !=0):
6534 fwinattributes = int(member.external_attr);
6535 else:
6536 fwinattributes = int(0);
6537 if(is_unix and member.external_attr !=0):
6538 fmode = int(member.external_attr);
6539 fchmode = int(stat.S_IMODE(member.external_attr));
6540 ftypemod = int(stat.S_IFMT(member.external_attr));
6541 elif(member.is_file()):
6542 fmode = int(stat.S_IFREG + 438);
6543 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
6544 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
6545 elif(member.is_symlink()):
6546 fmode = int(stat.S_IFLNK + 438);
6547 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
6548 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
6549 elif(member.is_dir()):
6550 fmode = int(stat.S_IFDIR + 511);
6551 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6552 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6553 returnval.update({lcfi: member.filename});
6554 if(not verbose):
6555 VerbosePrintOut(member.filename);
6556 if(verbose):
6557 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6558 permissionstr = "";
6559 for fmodval in str(oct(fmode))[-3:]:
6560 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
6561 if(member.is_file()):
6562 ftype = 0;
6563 permissionstr = "-" + permissionstr;
6564 printfname = member.filename;
6565 elif(member.is_symlink()):
6566 ftype = 2;
6567 permissionstr = "l" + permissionstr;
6568 printfname = member.name + " -> " + member.read().decode("UTF-8");
6569 elif(member.is_dir()):
6570 ftype = 5;
6571 permissionstr = "d" + permissionstr;
6572 printfname = member.filename;
6573 try:
6574 fuid = int(os.getuid());
6575 except AttributeError:
6576 fuid = int(0);
6577 except KeyError:
6578 fuid = int(0);
6579 try:
6580 fgid = int(os.getgid());
6581 except AttributeError:
6582 fgid = int(0);
6583 except KeyError:
6584 fgid = int(0);
6585 try:
6586 import pwd;
6587 try:
6588 userinfo = pwd.getpwuid(os.getuid());
6589 funame = userinfo.pw_name;
6590 except KeyError:
6591 funame = "";
6592 except AttributeError:
6593 funame = "";
6594 except ImportError:
6595 funame = "";
6596 fgname = "";
6597 try:
6598 import grp;
6599 try:
6600 groupinfo = grp.getgrgid(os.getgid());
6601 fgname = groupinfo.gr_name;
6602 except KeyError:
6603 fgname = "";
6604 except AttributeError:
6605 fgname = "";
6606 except ImportError:
6607 fgname = "";
6608 fuprint = funame;
6609 if(len(fuprint)<=0):
6610 fuprint = str(fuid);
6611 fgprint = fgname;
6612 if(len(fgprint)<=0):
6613 fgprint = str(fgid);
6614 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + member.mtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
6615 lcfi = lcfi + 1;
6616 if(returnfp):
6617 return listcatfiles['catfp'];
6618 else:
6619 return True;
6620
6621 if(not py7zr_support):
6622 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
6623 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6624 if(not os.path.exists(infile) or not os.path.isfile(infile)):
6625 return False;
6626
6627 if(py7zr_support):
6628 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
6629 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6630 if(not os.path.exists(infile) or not os.path.isfile(infile)):
6631 return False;
6632 lcfi = 0;
6633 returnval = {};
6634 szpfp = py7zr.SevenZipFile(infile, mode="r");
6635 file_content = szpfp.readall();
6636 #sztest = szpfp.testzip();
6637 sztestalt = szpfp.test();
6638 if(sztestalt):
6639 VerbosePrintOut("Bad file found!");
6640 for member in sorted(szpfp.list(), key=lambda x: x.filename):
6641 if(re.findall("^[.|/]", member.filename)):
6642 fname = member.filename;
6643 else:
6644 fname = "./"+member.filename;
6645 if(not member.is_directory):
6646 fpremode = int(stat.S_IFREG + 438);
6647 elif(member.is_directory):
6648 fpremode = int(stat.S_IFDIR + 511);
6649 fwinattributes = int(0);
6650 if(member.is_directory):
6651 fmode = int(stat.S_IFDIR + 511);
6652 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6653 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6654 else:
6655 fmode = int(stat.S_IFLNK + 438);
6656 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
6657 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
6658 returnval.update({lcfi: member.filename});
6659 if(not verbose):
6660 VerbosePrintOut(member.filename);
6661 if(verbose):
6662 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6663 permissionstr = "";
6664 for fmodval in str(oct(fmode))[-3:]:
6665 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
6666 fsize = int("0");
6667 if(not member.is_directory):
6668 ftype = 0;
6669 permissionstr = "-" + permissionstr;
6670 printfname = member.filename;
6671 elif(member.is_directory):
6672 ftype = 5;
6673 permissionstr = "d" + permissionstr;
6674 printfname = member.filename;
6675 if(ftype==0):
6676 fsize = len(file_content[member.filename].read());
6677 file_content[member.filename].close();
6678 try:
6679 fuid = int(os.getuid());
6680 except AttributeError:
6681 fuid = int(0);
6682 except KeyError:
6683 fuid = int(0);
6684 try:
6685 fgid = int(os.getgid());
6686 except AttributeError:
6687 fgid = int(0);
6688 except KeyError:
6689 fgid = int(0);
6690 try:
6691 import pwd;
6692 try:
6693 userinfo = pwd.getpwuid(os.getuid());
6694 funame = userinfo.pw_name;
6695 except KeyError:
6696 funame = "";
6697 except AttributeError:
6698 funame = "";
6699 except ImportError:
6700 funame = "";
6701 fgname = "";
6702 try:
6703 import grp;
6704 try:
6705 groupinfo = grp.getgrgid(os.getgid());
6706 fgname = groupinfo.gr_name;
6707 except KeyError:
6708 fgname = "";
6709 except AttributeError:
6710 fgname = "";
6711 except ImportError:
6712 fgname = "";
6713 fuprint = funame;
6714 if(len(fuprint)<=0):
6715 fuprint = str(fuid);
6716 fgprint = fgname;
6717 if(len(fgprint)<=0):
6718 fgprint = str(fgid);
6719 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(fsize).rjust(15) + " " + member.creationtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
6720 lcfi = lcfi + 1;
6721 if(returnfp):
6722 return listcatfiles['catfp'];
6723 else:
6724 return True;
6725
6726 def InFileListFiles(infile, verbose=False, formatspecs=__file_format_list__, returnfp=False):
6727 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6728 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
6729 if(checkcompressfile=="tarfile"):
6730 return TarFileListFiles(infile, verbose, returnfp);
6731 elif(checkcompressfile=="zipfile"):
6732 return ZipFileListFiles(infile, verbose, returnfp);
6733 elif(checkcompressfile=="catfile"):
6734 return ArchiveFileListFiles(infile, 0, 0, False, formatspecs, verbose, returnfp);
6735 elif(rarfile_support and checkcompressfile=="rarfile"):
6736 return RarFileListFiles(infile, verbose, returnfp);
6737 elif(py7zr_support and checkcompressfile=="7zipfile"):
6738 return SevenZipFileListFiles(infile, verbose, returnfp);
6739 else:
6740 return False;
6741 return False;
6742
6743 def ListDirListFiles(infiles, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False, returnfp=False):
6744 outarray = BytesIO();
6745 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compressionlevel, followlink, checksumtype, formatspecs, False, True);
6746 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6747 return listcatfiles;
6748
6749 def ListDirListFilesAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False, returnfp=False):
6750 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, listonly, checksumtype, formatspecs, verbose);
6751 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6752 return listcatfiles;
6753
6754 def PackArchiveFileFromListDirAlt(infiles, outfile, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
6755 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, False, checksumtype, extradata, formatspecs, False);
6756 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compressionlevel, followlink, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6757 return listcatfiles;
6758
6759 create_alias_function("Pack", __file_format_name__, "FromListDirAlt", PackArchiveFileFromListDirAlt);
6760
6761 def PackArchiveFileFromTarFileAlt(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
6762 outarray = TarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
6763 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
6764 return listcatfiles;
6765
6766 create_alias_function("Pack", __file_format_name__, "FromTarFileAlt", PackArchiveFileFromTarFileAlt);
6767
6768 def PackArchiveFileFromZipFileAlt(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
6769 outarray = ZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
6770 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
6771 return listcatfiles;
6772
6773 create_alias_function("Pack", __file_format_name__, "FromZipFileAlt", PackArchiveFileFromZipFileAlt);
6774
6775 if(not rarfile_support):
6776 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
6777 return False;
6778
6779 if(rarfile_support):
6780 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
6781 outarray = RarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
6782 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
6783 return listcatfiles;
6784
6785 create_alias_function("Pack", __file_format_name__, "FromRarFileAlt", PackArchiveFileFromRarFileAlt);
6786
6787 if(not py7zr_support):
6788 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
6789 return False;
6790
6791 if(py7zr_support):
6792 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
6793 outarray = SevenZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
6794 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
6795 return listcatfiles;
6796
6797 create_alias_function("Pack", __file_format_name__, "FromSevenZipFileAlt", PackArchiveFileFromSevenZipFileAlt);
6798
6799 def download_file_from_ftp_file(url):
6800 urlparts = urlparse(url);
6801 file_name = os.path.basename(urlparts.path);
6802 file_dir = os.path.dirname(urlparts.path);
6803 if(urlparts.username is not None):
6804 ftp_username = urlparts.username;
6805 else:
6806 ftp_username = "anonymous";
6807 if(urlparts.password is not None):
6808 ftp_password = urlparts.password;
6809 elif(urlparts.password is None and urlparts.username=="anonymous"):
6810 ftp_password = "anonymous";
6811 else:
6812 ftp_password = "";
6813 if(urlparts.scheme=="ftp"):
6814 ftp = FTP();
6815 elif(urlparts.scheme=="ftps"):
6816 ftp = FTP_TLS();
6817 else:
6818 return False;
6819 if(urlparts.scheme=="sftp"):
6820 if(__use_pysftp__):
6821 return download_file_from_pysftp_file(url);
6822 else:
6823 return download_file_from_sftp_file(url);
6824 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
6825 return download_file_from_http_file(url);
6826 ftp_port = urlparts.port;
6827 if(urlparts.port is None):
6828 ftp_port = 21;
6829 try:
6830 ftp.connect(urlparts.hostname, ftp_port);
6831 except socket.gaierror:
6832 log.info("Error With URL "+url);
6833 return False;
6834 except socket.timeout:
6835 log.info("Error With URL "+url);
6836 return False;
6837 ftp.login(urlparts.username, urlparts.password);
6838 if(urlparts.scheme=="ftps"):
6839 ftp.prot_p();
6840 ftpfile = BytesIO();
6841 ftp.retrbinary("RETR "+urlparts.path, ftpfile.write);
6842 #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
6843 ftp.close();
6844 ftpfile.seek(0, 0);
6845 return ftpfile;
6846
6847 def download_file_from_ftp_string(url):
6848 ftpfile = download_file_from_ftp_file(url);
6849 return ftpfile.read();
6850
6851 def upload_file_to_ftp_file(ftpfile, url):
6852 urlparts = urlparse(url);
6853 file_name = os.path.basename(urlparts.path);
6854 file_dir = os.path.dirname(urlparts.path);
6855 if(urlparts.username is not None):
6856 ftp_username = urlparts.username;
6857 else:
6858 ftp_username = "anonymous";
6859 if(urlparts.password is not None):
6860 ftp_password = urlparts.password;
6861 elif(urlparts.password is None and urlparts.username=="anonymous"):
6862 ftp_password = "anonymous";
6863 else:
6864 ftp_password = "";
6865 if(urlparts.scheme=="ftp"):
6866 ftp = FTP();
6867 elif(urlparts.scheme=="ftps"):
6868 ftp = FTP_TLS();
6869 else:
6870 return False;
6871 if(urlparts.scheme=="sftp"):
6872 if(__use_pysftp__):
6873 return upload_file_to_pysftp_file(url);
6874 else:
6875 return upload_file_to_sftp_file(url);
6876 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
6877 return False;
6878 ftp_port = urlparts.port;
6879 if(urlparts.port is None):
6880 ftp_port = 21;
6881 try:
6882 ftp.connect(urlparts.hostname, ftp_port);
6883 except socket.gaierror:
6884 log.info("Error With URL "+url);
6885 return False;
6886 except socket.timeout:
6887 log.info("Error With URL "+url);
6888 return False;
6889 ftp.login(urlparts.username, urlparts.password);
6890 if(urlparts.scheme=="ftps"):
6891 ftp.prot_p();
6892 ftp.storbinary("STOR "+urlparts.path, ftpfile);
6893 ftp.close();
6894 ftpfile.seek(0, 0);
6895 return ftpfile;
6896
6897 def upload_file_to_ftp_string(ftpstring, url):
6898 ftpfileo = BytesIO(ftpstring);
6899 ftpfile = upload_file_to_ftp_file(ftpfileo, url);
6900 ftpfileo.close();
6901 return ftpfile;
6902
6903 def download_file_from_http_file(url, headers=geturls_headers_pycatfile_python_alt):
6904 # Parse the URL to extract username and password if present
6905 urlparts = urlparse(url);
6906 username = urlparts.username;
6907 password = urlparts.password;
6908 # Rebuild the URL without the username and password
6909 netloc = urlparts.hostname;
6910 if(urlparts.scheme=="sftp"):
6911 if(__use_pysftp__):
6912 return download_file_from_pysftp_file(url);
6913 else:
6914 return download_file_from_sftp_file(url);
6915 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
6916 return download_file_from_ftp_file(url);
6917 if urlparts.port:
6918 netloc += ':' + str(urlparts.port);
6919 rebuilt_url = urlunparse((urlparts.scheme, netloc, urlparts.path, urlparts.params, urlparts.query, urlparts.fragment));
6920 # Create a temporary file object
6921 httpfile = BytesIO();
6922 if haverequests:
6923 # Use the requests library if available
6924 if username and password:
6925 response = requests.get(rebuilt_url, headers=headers, auth=(username, password), stream=True);
6926 else:
6927 response = requests.get(rebuilt_url, headers=headers, stream=True);
6928 response.raw.decode_content = True
6929 shutil.copyfileobj(response.raw, httpfile);
6930 else:
6931 # Build a Request object for urllib
6932 request = Request(rebuilt_url, headers=headers);
6933 # Create an opener object for handling URLs
6934 if username and password:
6935 # Create a password manager
6936 password_mgr = HTTPPasswordMgrWithDefaultRealm();
6937 # Add the username and password
6938 password_mgr.add_password(None, rebuilt_url, username, password);
6939 # Create an authentication handler using the password manager
6940 auth_handler = HTTPBasicAuthHandler(password_mgr);
6941 # Build the opener with the authentication handler
6942 opener = build_opener(auth_handler);
6943 else:
6944 opener = build_opener();
6945 with opener.open(request) as response:
6946 shutil.copyfileobj(response, httpfile);
6947 # Reset file pointer to the start
6948 httpfile.seek(0, 0);
6949 # Return the temporary file object
6950 return httpfile;
6951
6952 def download_file_from_http_string(url, headers=geturls_headers_pycatfile_python_alt):
6953 httpfile = download_file_from_http_file(url, headers);
6954 return ftpfile.read();
6955
6956 if(haveparamiko):
6957 def download_file_from_sftp_file(url):
6958 urlparts = urlparse(url);
6959 file_name = os.path.basename(urlparts.path);
6960 file_dir = os.path.dirname(urlparts.path);
6961 sftp_port = urlparts.port;
6962 if(urlparts.port is None):
6963 sftp_port = 22;
6964 else:
6965 sftp_port = urlparts.port;
6966 if(urlparts.username is not None):
6967 sftp_username = urlparts.username;
6968 else:
6969 sftp_username = "anonymous";
6970 if(urlparts.password is not None):
6971 sftp_password = urlparts.password;
6972 elif(urlparts.password is None and urlparts.username=="anonymous"):
6973 sftp_password = "anonymous";
6974 else:
6975 sftp_password = "";
6976 if(urlparts.scheme=="ftp"):
6977 return download_file_from_ftp_file(url);
6978 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
6979 return download_file_from_http_file(url);
6980 if(urlparts.scheme!="sftp"):
6981 return False;
6982 ssh = paramiko.SSHClient();
6983 ssh.load_system_host_keys();
6984 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
6985 try:
6986 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
6987 except paramiko.ssh_exception.SSHException:
6988 return False;
6989 except socket.gaierror:
6990 log.info("Error With URL "+url);
6991 return False;
6992 except socket.timeout:
6993 log.info("Error With URL "+url);
6994 return False;
6995 sftp = ssh.open_sftp();
6996 sftpfile = BytesIO();
6997 sftp.getfo(urlparts.path, sftpfile);
6998 sftp.close();
6999 ssh.close();
7000 sftpfile.seek(0, 0);
7001 return sftpfile;
7002 else:
7003 def download_file_from_sftp_file(url):
7004 return False;
7005
7006 if(haveparamiko):
7007 def download_file_from_sftp_string(url):
7008 sftpfile = download_file_from_sftp_file(url);
7009 return sftpfile.read();
7010 else:
7011 def download_file_from_ftp_string(url):
7012 return False;
7013
7014 if(haveparamiko):
7015 def upload_file_to_sftp_file(sftpfile, url):
7016 urlparts = urlparse(url);
7017 file_name = os.path.basename(urlparts.path);
7018 file_dir = os.path.dirname(urlparts.path);
7019 sftp_port = urlparts.port;
7020 if(urlparts.port is None):
7021 sftp_port = 22;
7022 else:
7023 sftp_port = urlparts.port;
7024 if(urlparts.username is not None):
7025 sftp_username = urlparts.username;
7026 else:
7027 sftp_username = "anonymous";
7028 if(urlparts.password is not None):
7029 sftp_password = urlparts.password;
7030 elif(urlparts.password is None and urlparts.username=="anonymous"):
7031 sftp_password = "anonymous";
7032 else:
7033 sftp_password = "";
7034 if(urlparts.scheme=="ftp"):
7035 return upload_file_to_ftp_file(url);
7036 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7037 return False;
7038 if(urlparts.scheme!="sftp"):
7039 return False;
7040 ssh = paramiko.SSHClient();
7041 ssh.load_system_host_keys();
7042 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
7043 try:
7044 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7045 except paramiko.ssh_exception.SSHException:
7046 return False;
7047 except socket.gaierror:
7048 log.info("Error With URL "+url);
7049 return False;
7050 except socket.timeout:
7051 log.info("Error With URL "+url);
7052 return False;
7053 sftp = ssh.open_sftp();
7054 sftp.putfo(sftpfile, urlparts.path);
7055 sftp.close();
7056 ssh.close();
7057 sftpfile.seek(0, 0);
7058 return sftpfile;
7059 else:
7060 def upload_file_to_sftp_file(sftpfile, url):
7061 return False;
7062
7063 if(haveparamiko):
7064 def upload_file_to_sftp_string(sftpstring, url):
7065 sftpfileo = BytesIO(sftpstring);
7066 sftpfile = upload_file_to_sftp_files(ftpfileo, url);
7067 sftpfileo.close();
7068 return sftpfile;
7069 else:
7070 def upload_file_to_sftp_string(url):
7071 return False;
7072
7073 if(havepysftp):
7074 def download_file_from_pysftp_file(url):
7075 urlparts = urlparse(url);
7076 file_name = os.path.basename(urlparts.path);
7077 file_dir = os.path.dirname(urlparts.path);
7078 sftp_port = urlparts.port;
7079 if(urlparts.port is None):
7080 sftp_port = 22;
7081 else:
7082 sftp_port = urlparts.port;
7083 if(urlparts.username is not None):
7084 sftp_username = urlparts.username;
7085 else:
7086 sftp_username = "anonymous";
7087 if(urlparts.password is not None):
7088 sftp_password = urlparts.password;
7089 elif(urlparts.password is None and urlparts.username=="anonymous"):
7090 sftp_password = "anonymous";
7091 else:
7092 sftp_password = "";
7093 if(urlparts.scheme=="ftp"):
7094 return download_file_from_ftp_file(url);
7095 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7096 return download_file_from_http_file(url);
7097 if(urlparts.scheme!="sftp"):
7098 return False;
7099 try:
7100 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7101 except paramiko.ssh_exception.SSHException:
7102 return False;
7103 except socket.gaierror:
7104 log.info("Error With URL "+url);
7105 return False;
7106 except socket.timeout:
7107 log.info("Error With URL "+url);
7108 return False;
7109 sftp = ssh.open_sftp();
7110 sftpfile = BytesIO();
7111 sftp.getfo(urlparts.path, sftpfile);
7112 sftp.close();
7113 ssh.close();
7114 sftpfile.seek(0, 0);
7115 return sftpfile;
7116 else:
7117 def download_file_from_pysftp_file(url):
7118 return False;
7119
7120 if(havepysftp):
7121 def download_file_from_pysftp_string(url):
7122 sftpfile = download_file_from_pysftp_file(url);
7123 return sftpfile.read();
7124 else:
7125 def download_file_from_ftp_string(url):
7126 return False;
7127
7128 if(havepysftp):
7129 def upload_file_to_pysftp_file(sftpfile, url):
7130 urlparts = urlparse(url);
7131 file_name = os.path.basename(urlparts.path);
7132 file_dir = os.path.dirname(urlparts.path);
7133 sftp_port = urlparts.port;
7134 if(urlparts.port is None):
7135 sftp_port = 22;
7136 else:
7137 sftp_port = urlparts.port;
7138 if(urlparts.username is not None):
7139 sftp_username = urlparts.username;
7140 else:
7141 sftp_username = "anonymous";
7142 if(urlparts.password is not None):
7143 sftp_password = urlparts.password;
7144 elif(urlparts.password is None and urlparts.username=="anonymous"):
7145 sftp_password = "anonymous";
7146 else:
7147 sftp_password = "";
7148 if(urlparts.scheme=="ftp"):
7149 return upload_file_to_ftp_file(url);
7150 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7151 return False;
7152 if(urlparts.scheme!="sftp"):
7153 return False;
7154 try:
7155 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7156 except paramiko.ssh_exception.SSHException:
7157 return False;
7158 except socket.gaierror:
7159 log.info("Error With URL "+url);
7160 return False;
7161 except socket.timeout:
7162 log.info("Error With URL "+url);
7163 return False;
7164 sftp = ssh.open_sftp();
7165 sftp.putfo(sftpfile, urlparts.path);
7166 sftp.close();
7167 ssh.close();
7168 sftpfile.seek(0, 0);
7169 return sftpfile;
7170 else:
7171 def upload_file_to_pysftp_file(sftpfile, url):
7172 return False;
7173
7174 if(havepysftp):
7175 def upload_file_to_pysftp_string(sftpstring, url):
7176 sftpfileo = BytesIO(sftpstring);
7177 sftpfile = upload_file_to_pysftp_files(ftpfileo, url);
7178 sftpfileo.close();
7179 return sftpfile;
7180 else:
7181 def upload_file_to_pysftp_string(url):
7182 return False;
7183
7184 def download_file_from_internet_file(url, headers=geturls_headers_pycatfile_python_alt):
7185 urlparts = urlparse(url);
7186 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7187 return download_file_from_http_file(url, headers);
7188 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7189 return download_file_from_ftp_file(url);
7190 elif(urlparts.scheme=="sftp"):
7191 if(__use_pysftp__ and havepysftp):
7192 return download_file_from_pysftp_file(url);
7193 else:
7194 return download_file_from_sftp_file(url);
7195 else:
7196 return False;
7197 return False;
7198
7199 def download_file_from_internet_uncompress_file(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_list__):
7200 fp = download_file_from_internet_file(url);
7201 fp = UncompressArchiveFile(fp, formatspecs);
7202 fp.seek(0, 0);
7203 if(not fp):
7204 return False;
7205 return fp;
7206
7207 def download_file_from_internet_string(url, headers=geturls_headers_pycatfile_python_alt):
7208 urlparts = urlparse(url);
7209 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7210 return download_file_from_http_string(url, headers);
7211 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7212 return download_file_from_ftp_string(url);
7213 elif(urlparts.scheme=="sftp"):
7214 if(__use_pysftp__ and havepysftp):
7215 return download_file_from_pysftp_string(url);
7216 else:
7217 return download_file_from_sftp_string(url);
7218 else:
7219 return False;
7220 return False;
7221
7222 def download_file_from_internet_uncompress_string(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_list__):
7223 fp = download_file_from_internet_string(url);
7224 fp = UncompressArchiveFile(fp, formatspecs);
7225 fp.seek(0, 0);
7226 if(not fp):
7227 return False;
7228 return fp;
7229
7230 def upload_file_to_internet_file(ifp, url):
7231 urlparts = urlparse(url);
7232 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7233 return False;
7234 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7235 return upload_file_to_ftp_file(ifp, url);
7236 elif(urlparts.scheme=="sftp"):
7237 if(__use_pysftp__ and havepysftp):
7238 return upload_file_to_pysftp_file(ifp, url);
7239 else:
7240 return upload_file_to_sftp_file(ifp, url);
7241 else:
7242 return False;
7243 return False;
7244
7245 def upload_file_to_internet_compress_file(ifp, url, formatspecs=__file_format_list__):
7246 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
7247 if(not catfileout):
7248 return False;
7249 catfp.seek(0, 0);
7250 upload_file_to_internet_file(catfp, outfile);
7251 return True;
7252
7253 def upload_file_to_internet_string(ifp, url):
7254 urlparts = urlparse(url);
7255 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7256 return False;
7257 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7258 return upload_file_to_ftp_string(ifp, url);
7259 elif(urlparts.scheme=="sftp"):
7260 if(__use_pysftp__ and havepysftp):
7261 return upload_file_to_pysftp_string(ifp, url);
7262 else:
7263 return upload_file_to_sftp_string(ifp, url);
7264 else:
7265 return False;
7266 return False;
7267
7268 def upload_file_to_internet_compress_string(ifp, url, formatspecs=__file_format_list__):
7269 catfp = CompressArchiveFile(BytesIO(ifp), compression, compressionlevel, formatspecs);
7270 if(not catfileout):
7271 return False;
7272 catfp.seek(0, 0);
7273 upload_file_to_internet_file(catfp, outfile);
7274 return True;
7275
7276 try:
7277 if(hasattr(shutil, "register_archive_format")):
7278 # Register the packing format
7279 shutil.register_archive_format(__file_format_name__, PackArchiveFileFunc, description='Pack concatenated files');
7280 except shutil.RegistryError:
7281 pass;
7282
7283 try:
7284 if(hasattr(shutil, "register_unpack_format")):
7285 # Register the unpacking format
7286 shutil.register_unpack_format(__file_format_name__, archivefile_extensions, UnPackArchiveFileFunc, description='UnPack concatenated files');
7287 except shutil.RegistryError:
7288 pass;
A tar like file format name catfile after unix cat command (concatenate files).