?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Add files via upload
[PyCatFile.git] / pycatfile.py
blob27a0d31ab5f6ba245e4536841bab0c0cf86dabb4
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 '''
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the Revised BSD License.
7
8 This program is distributed in the hope that it will be useful,
9 but WITHOUT ANY WARRANTY; without even the implied warranty of
10 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 Revised BSD License for more details.
12
13 Copyright 2018-2024 Cool Dude 2k - http://idb.berlios.de/
14 Copyright 2018-2024 Game Maker 2k - http://intdb.sourceforge.net/
15 Copyright 2018-2024 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
16
17 $FileInfo: pycatfile.py - Last Update: 3/15/2024 Ver. 0.5.0 RC 1 - Author: cooldude2k $
18 '''
19
20 from __future__ import absolute_import, division, print_function, unicode_literals;
21 import io, os, re, sys, time, stat, zlib, base64, shutil, socket, hashlib, datetime, logging, binascii, tempfile, zipfile, platform;
22 from ftplib import FTP, FTP_TLS;
23 if(sys.version[0]=="2"):
24 from urlparse import urlparse, urlunparse;
25 elif(sys.version[0]>="3"):
26 from urllib.parse import urlunparse;
27 from urllib.parse import urlparse;
28
29 if os.name == 'nt': # Only modify if on Windows
30 if sys.version[0] == "2":
31 import codecs;
32 sys.stdout = codecs.getwriter('utf-8')(sys.stdout);
33 sys.stderr = codecs.getwriter('utf-8')(sys.stderr);
34 else:
35 import io;
36 sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace', line_buffering=True);
37 sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', errors='replace', line_buffering=True);
38
39 hashlib_guaranteed = False;
40 os.environ["PYTHONIOENCODING"] = "UTF-8";
41 os.environ["LC_CTYPE"] = "UTF-8";
42 try:
43 reload(sys);
44 try:
45 sys.setdefaultencoding('UTF-8');
46 except NameError:
47 pass;
48 except AttributeError:
49 pass;
50 except NameError:
51 pass;
52 except AttributeError:
53 pass;
54
55 try:
56 from zlib import crc32;
57 except ImportError:
58 from binascii import crc32;
59
60 rarfile_support = False;
61 try:
62 import rarfile;
63 rarfile_support = True;
64 except ImportError:
65 rarfile_support = False;
66
67 try:
68 from safetar import is_tarfile;
69 except ImportError:
70 try:
71 from xtarfile import is_tarfile;
72 except ImportError:
73 from tarfile import is_tarfile;
74
75 try:
76 import safetar as tarfile;
77 except ImportError:
78 try:
79 import xtarfile as tarfile;
80 except ImportError:
81 import tarfile;
82
83 haveparamiko = False;
84 try:
85 import paramiko;
86 haveparamiko = True;
87 except ImportError:
88 haveparamiko = False;
89
90 havepysftp = False;
91 try:
92 import pysftp;
93 havepysftp = True;
94 except ImportError:
95 havepysftp = False;
96
97 haverequests = False;
98 try:
99 import requests;
100 haverequests = True;
101 except ImportError:
102 haverequests = False;
103
104 if(haverequests):
105 import urllib3;
106 logging.getLogger("urllib3").setLevel(logging.WARNING);
107
108 try:
109 # Python 3 imports
110 from urllib.request import Request, build_opener, HTTPBasicAuthHandler;
111 from urllib.parse import urlparse;
112 except ImportError:
113 # Python 2 imports
114 from urllib2 import Request, build_opener, HTTPBasicAuthHandler;
115 from urlparse import urlparse;
116
117 if(sys.version[0]=="2"):
118 try:
119 from io import StringIO, BytesIO;
120 except ImportError:
121 try:
122 from cStringIO import StringIO;
123 from cStringIO import StringIO as BytesIO;
124 except ImportError:
125 from StringIO import StringIO;
126 from StringIO import StringIO as BytesIO;
127 elif(sys.version[0]>="3"):
128 from io import StringIO, BytesIO;
129 else:
130 teststringio = 0;
131 if(teststringio<=0):
132 try:
133 from cStringIO import StringIO as BytesIO;
134 teststringio = 1;
135 except ImportError:
136 teststringio = 0;
137 if(teststringio<=0):
138 try:
139 from StringIO import StringIO as BytesIO;
140 teststringio = 2;
141 except ImportError:
142 teststringio = 0;
143 if(teststringio<=0):
144 try:
145 from io import BytesIO;
146 teststringio = 3;
147 except ImportError:
148 teststringio = 0;
149
150 __use_pysftp__ = False;
151 if(not havepysftp):
152 __use_pysftp__ = False;
153 __file_format_name__ = "CatFile";
154 __program_name__ = "Py"+__file_format_name__;
155 __file_format_lower__ = __file_format_name__.lower();
156 __file_format_magic__ = __file_format_name__;
157 __file_format_len__ = len(__file_format_magic__);
158 __file_format_hex__ = binascii.hexlify(__file_format_magic__.encode("UTF-8")).decode("UTF-8");
159 __file_format_delimiter__ = "\x00";
160 __file_format_ver__ = "001";
161 __use_new_style__ = True;
162 __file_format_list__ = [__file_format_name__, __file_format_magic__, __file_format_lower__, __file_format_len__, __file_format_hex__, __file_format_delimiter__, __file_format_ver__, __use_new_style__];
163 __project__ = __program_name__;
164 __project_url__ = "https://github.com/GameMaker2k/PyCatFile";
165 __version_info__ = (0, 6, 0, "RC 1", 1);
166 __version_date_info__ = (2024, 3, 21, "RC 1", 1);
167 __version_date__ = str(__version_date_info__[0]) + "." + str(__version_date_info__[1]).zfill(2) + "." + str(__version_date_info__[2]).zfill(2);
168 __revision__ = __version_info__[3];
169 __revision_id__ = "$Id$";
170 if(__version_info__[4] is not None):
171 __version_date_plusrc__ = __version_date__ + "-" + str(__version_date_info__[4]);
172 if(__version_info__[4] is None):
173 __version_date_plusrc__ = __version_date__;
174 if(__version_info__[3] is not None):
175 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]) + " " + str(__version_info__[3]);
176 if(__version_info__[3] is None):
177 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]);
178
179 PyBitness = platform.architecture();
180 if(PyBitness=="32bit" or PyBitness=="32"):
181 PyBitness = "32";
182 elif(PyBitness=="64bit" or PyBitness=="64"):
183 PyBitness = "64";
184 else:
185 PyBitness = "32";
186
187 geturls_ua_pycatfile_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
188 if(platform.python_implementation()!=""):
189 py_implementation = platform.python_implementation();
190 if(platform.python_implementation()==""):
191 py_implementation = "Python";
192 geturls_ua_pycatfile_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=py_implementation, pyver=platform.python_version(), proname=__project__, prover=__version__);
193 geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
194 geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
195 geturls_headers_pycatfile_python = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
196 geturls_headers_pycatfile_python_alt = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python_alt, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
197 geturls_headers_googlebot_google = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
198 geturls_headers_googlebot_google_old = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
199
200 compressionlist = ['auto', 'gzip', 'bzip2', 'zstd', 'xz', 'lz4', 'lzo', 'lzma'];
201 outextlist = ['gz', 'bz2', 'zst', 'xz', 'lz4', 'lzo', 'lzma'];
202 outextlistwd = ['.gz', '.bz2', '.zst', '.xz', '.lz4', '.lzo', '.lzma'];
203
204 tarfile_mimetype = "application/tar";
205 tarfile_tar_mimetype = tarfile_mimetype;
206 zipfile_mimetype = "application/zip";
207 zipfile_zip_mimetype = zipfile_mimetype;
208 rarfile_mimetype = "application/rar";
209 rarfile_rar_mimetype = rarfile_mimetype;
210 archivefile_mimetype = "application/x-"+__file_format_list__[1]+"";
211 archivefile_cat_mimetype = archivefile_mimetype;
212 archivefile_gzip_mimetype = "application/x-"+__file_format_list__[1]+"+gzip";
213 archivefile_gz_mimetype = archivefile_gzip_mimetype;
214 archivefile_bzip2_mimetype = "application/x-"+__file_format_list__[1]+"+bzip2";
215 archivefile_bz2_mimetype = archivefile_bzip2_mimetype;
216 archivefile_lz4_mimetype = "application/x-"+__file_format_list__[1]+"+lz4";
217 archivefile_lzop_mimetype = "application/x-"+__file_format_list__[1]+"+lzop";
218 archivefile_lzo_mimetype = archivefile_lzop_mimetype;
219 archivefile_zstandard_mimetype = "application/x-"+__file_format_list__[1]+"+zstandard";
220 archivefile_zstd_mimetype = archivefile_zstandard_mimetype;
221 archivefile_lzma_mimetype = "application/x-"+__file_format_list__[1]+"+lzma";
222 archivefile_xz_mimetype = "application/x-"+__file_format_list__[1]+"+xz";
223 archivefile_extensions = ['.cat', '.cat.gz', '.cat.bz2', '.cat.zst', '.cat.lz4', '.cat.lzo', '.cat.lzop', '.cat.lzma', '.cat.xz'];
224
225 if __name__ == "__main__":
226 import subprocess;
227 curscrpath = os.path.dirname(sys.argv[0]);
228 if(curscrpath==""):
229 curscrpath = ".";
230 if(os.sep=="\\"):
231 curscrpath = curscrpath.replace(os.sep, "/");
232 curscrpath = curscrpath + "/";
233 scrfile = curscrpath + "catfile.py";
234 if(os.path.exists(scrfile) and os.path.isfile(scrfile)):
235 scrcmd = subprocess.Popen([sys.executable, scrfile] + sys.argv[1:]);
236 scrcmd.wait();
237
238 def VerbosePrintOut(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
239 if(not dbgenable):
240 return True;
241 log_functions = {
242 "print": print,
243 "log": logging.info,
244 "warning": logging.warning,
245 "error": logging.error,
246 "critical": logging.critical,
247 "exception": logging.exception,
248 "logalt": lambda x: logging.log(dgblevel, x),
249 "debug": logging.debug
250 };
251 log_function = log_functions.get(outtype);
252 if(log_function):
253 log_function(dbgtxt);
254 return True;
255 return False;
256
257 def VerbosePrintOutReturn(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
258 VerbosePrintOut(dbgtxt, outtype, dbgenable, dgblevel);
259 return dbgtxt;
260
261 def RemoveWindowsPath(dpath):
262 if(dpath is None):
263 dpath = "";
264 if(os.sep!="/"):
265 dpath = dpath.replace(os.path.sep, "/");
266 dpath = dpath.rstrip("/");
267 if(dpath=="." or dpath==".."):
268 dpath = dpath + "/";
269 return dpath;
270
271 def NormalizeRelativePath(inpath):
272 inpath = RemoveWindowsPath(inpath);
273 if(os.path.isabs(inpath)):
274 outpath = inpath;
275 else:
276 if(inpath.startswith("./") or inpath.startswith("../")):
277 outpath = inpath;
278 else:
279 outpath = "./" + inpath;
280 return outpath;
281
282 def ListDir(dirpath, followlink=False, duplicates=False):
283 if(isinstance(dirpath, (list, tuple, ))):
284 dirpath = list(filter(None, dirpath));
285 elif(isinstance(dirpath, (str, ))):
286 dirpath = list(filter(None, [dirpath]));
287 retlist = [];
288 for mydirfile in dirpath:
289 if(not os.path.exists(mydirfile)):
290 return False;
291 mydirfile = NormalizeRelativePath(mydirfile);
292 if(os.path.exists(mydirfile) and os.path.islink(mydirfile)):
293 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile));
294 if(os.path.exists(mydirfile) and os.path.isdir(mydirfile)):
295 for root, dirs, filenames in os.walk(mydirfile):
296 dpath = root;
297 dpath = RemoveWindowsPath(dpath);
298 if(dpath not in retlist and not duplicates):
299 retlist.append(dpath);
300 if(duplicates):
301 retlist.append(dpath);
302 for file in filenames:
303 fpath = os.path.join(root, file);
304 fpath = RemoveWindowsPath(fpath);
305 if(fpath not in retlist and not duplicates):
306 retlist.append(fpath);
307 if(duplicates):
308 retlist.append(fpath);
309 else:
310 retlist.append(RemoveWindowsPath(mydirfile));
311 return retlist;
312
313 def ListDirAdvanced(dirpath, followlink=False, duplicates=False):
314 if isinstance(dirpath, (list, tuple)):
315 dirpath = list(filter(None, dirpath));
316 elif isinstance(dirpath, str):
317 dirpath = list(filter(None, [dirpath]));
318 retlist = []
319 for mydirfile in dirpath:
320 if not os.path.exists(mydirfile):
321 return False;
322 mydirfile = NormalizeRelativePath(mydirfile);
323 if os.path.exists(mydirfile) and os.path.islink(mydirfile) and followlink:
324 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile))
325 if os.path.exists(mydirfile) and os.path.isdir(mydirfile):
326 for root, dirs, filenames in os.walk(mydirfile):
327 # Sort dirs and filenames alphabetically in place
328 dirs.sort(key=lambda x: x.lower());
329 filenames.sort(key=lambda x: x.lower());
330 dpath = RemoveWindowsPath(root);
331 if not duplicates and dpath not in retlist:
332 retlist.append(dpath);
333 elif duplicates:
334 retlist.append(dpath);
335 for file in filenames:
336 fpath = os.path.join(root, file);
337 fpath = RemoveWindowsPath(fpath);
338 if not duplicates and fpath not in retlist:
339 retlist.append(fpath);
340 elif duplicates:
341 retlist.append(fpath);
342 else:
343 retlist.append(RemoveWindowsPath(mydirfile));
344 return retlist;
345
346 def create_alias_function(prefix, base_name, suffix, target_function):
347 # Define a new function that wraps the target function
348 def alias_function(*args, **kwargs):
349 return target_function(*args, **kwargs);
350
351 # Create the function name by combining the prefix, base name, and the suffix
352 function_name = "{}{}{}".format(prefix, base_name, suffix);
353
354 # Add the new function to the global namespace
355 globals()[function_name] = alias_function;
356
357 def create_alias_function_alt(prefix, base_name, suffix, target_function):
358 # Create the function name by combining the prefix, base name, and the suffix
359 # Use the format method for string formatting, compatible with Python 2 and 3
360 function_name = "{}{}{}".format(prefix, base_name, suffix)
361 # Add the new function (alias of the target_function) to the global namespace
362 # This line is compatible as-is with both Python 2 and 3
363 globals()[function_name] = target_function
364
365 # initial_value can be 0xFFFF or 0x0000
366 def crc16_ansi(msg, initial_value=0xFFFF):
367 # CRC-16-IBM / CRC-16-ANSI polynomial and initial value
368 poly = 0x8005; # Polynomial for CRC-16-IBM / CRC-16-ANSI
369 crc = initial_value; # Initial value
370 for b in msg:
371 crc ^= b << 8; # XOR byte into CRC top byte
372 for _ in range(8): # Process each bit
373 if crc & 0x8000: # If the top bit is set
374 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
375 else:
376 crc = crc << 1; # Just shift left
377 crc &= 0xFFFF; # Ensure CRC remains 16-bit
378 return crc;
379
380 # initial_value can be 0xFFFF or 0x0000
381 def crc16_ibm(msg, initial_value=0xFFFF):
382 return crc16_ansi(msg, initial_value);
383
384 # initial_value is 0xFFFF
385 def crc16(msg):
386 return crc16_ansi(msg, 0xFFFF);
387
388 # initial_value can be 0xFFFF, 0x1D0F or 0x0000
389 def crc16_ccitt(msg, initial_value=0xFFFF):
390 # CRC-16-CCITT polynomial
391 poly = 0x1021; # Polynomial for CRC-16-CCITT
392 # Use the specified initial value
393 crc = initial_value;
394 for b in msg:
395 crc ^= b << 8; # XOR byte into CRC top byte
396 for _ in range(8): # Process each bit
397 if crc & 0x8000: # If the top bit is set
398 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
399 else:
400 crc = crc << 1; # Just shift left
401 crc &= 0xFFFF; # Ensure CRC remains 16-bit
402 return crc;
403
404 # initial_value can be 0x42F0E1EBA9EA3693 or 0x0000000000000000
405 def crc64_ecma(msg, initial_value=0x0000000000000000):
406 # CRC-64-ECMA polynomial and initial value
407 poly = 0x42F0E1EBA9EA3693;
408 crc = initial_value; # Initial value for CRC-64-ECMA
409 for b in msg:
410 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
411 for _ in range(8): # Process each bit
412 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
413 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
414 else:
415 crc <<= 1; # Just shift left if the MSB is 0
416 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
417 return crc;
418
419 # initial_value can be 0x000000000000001B or 0xFFFFFFFFFFFFFFFF
420 def crc64_iso(msg, initial_value=0xFFFFFFFFFFFFFFFF):
421 # CRC-64-ISO polynomial and initial value
422 poly = 0x000000000000001B;
423 crc = initial_value; # Common initial value for CRC-64-ISO
424 for b in msg:
425 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
426 for _ in range(8): # Process each bit
427 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
428 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
429 else:
430 crc <<= 1; # Just shift left if the MSB is 0
431 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
432 return crc;
433
434 def GetDataFromArray(data, path, default=None):
435 element = data;
436 try:
437 for key in path:
438 element = element[key];
439 return element;
440 except (KeyError, TypeError, IndexError):
441 return default;
442
443 def GetDataFromArrayAlt(structure, path, default=None):
444 element = structure;
445 for key in path:
446 if isinstance(element, dict) and key in element:
447 element = element[key];
448 elif isinstance(element, list) and isinstance(key, int) and -len(element) <= key < len(element):
449 element = element[key];
450 else:
451 return default;
452 return element;
453
454 def GetHeaderChecksum(inlist=[], checksumtype="crc32", encodedata=True, formatspecs=__file_format_list__):
455 if isinstance(inlist, list):
456 fileheader = AppendNullBytes(inlist, formatspecs[5]);
457 else:
458 fileheader = AppendNullByte(inlist, formatspecs[5]);
459 if(encodedata):
460 fileheader = fileheader.encode('UTF-8');
461 if(checksumtype=="none" or checksumtype==""):
462 catfileheadercshex = format(0, 'x').lower();
463 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
464 catfileheadercshex = format(crc16(fileheader) & 0xffff, '04x').lower();
465 elif(checksumtype=="crc16_ccitt"):
466 catfileheadercshex = format(crc16_ccitt(fileheader) & 0xffff, '04x').lower();
467 elif(checksumtype=="adler32"):
468 catfileheadercshex = format(zlib.adler32(fileheader) & 0xffffffff, '08x').lower();
469 elif(checksumtype=="crc32"):
470 catfileheadercshex = format(crc32(fileheader) & 0xffffffff, '08x').lower();
471 elif(checksumtype=="crc64_ecma"):
472 catfileheadercshex = format(crc64_ecma(fileheader) & 0xffffffffffffffff, '016x').lower();
473 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
474 catfileheadercshex = format(crc64_iso(fileheader) & 0xffffffffffffffff, '016x').lower();
475 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
476 checksumoutstr = hashlib.new(checksumtype);
477 checksumoutstr.update(fileheader);
478 catfileheadercshex = checksumoutstr.hexdigest().lower();
479 else:
480 catfileheadercshex = format(0, 'x').lower();
481 return catfileheadercshex;
482
483 def GetFileChecksum(infile, checksumtype="crc32", encodedata=True, formatspecs=__file_format_list__):
484 if(encodedata):
485 infile = infile.encode('UTF-8');
486 if(checksumtype=="none" or checksumtype==""):
487 catinfilecshex = format(0, 'x').lower();
488 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
489 catinfilecshex = format(crc16(infile) & 0xffff, '04x').lower();
490 elif(checksumtype=="crc16_ccitt"):
491 catinfilecshex = format(crc16_ccitt(infile) & 0xffff, '04x').lower();
492 elif(checksumtype=="adler32"):
493 catinfilecshex = format(zlib.adler32(infile) & 0xffffffff, '08x').lower();
494 elif(checksumtype=="crc32"):
495 catinfilecshex = format(crc32(infile) & 0xffffffff, '08x').lower();
496 elif(checksumtype=="crc64_ecma"):
497 catinfilecshex = format(crc64_ecma(infile) & 0xffffffffffffffff, '016x').lower();
498 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
499 catinfilecshex = format(crc64_iso(infile) & 0xffffffffffffffff, '016x').lower();
500 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
501 checksumoutstr = hashlib.new(checksumtype);
502 checksumoutstr.update(infile);
503 catinfilecshex = checksumoutstr.hexdigest().lower();
504 else:
505 catinfilecshex = format(0, 'x').lower();
506 return catinfilecshex;
507
508 def ValidateHeaderChecksum(inlist=[], checksumtype="crc32", inchecksum="0", formatspecs=__file_format_list__):
509 catfileheadercshex = GetHeaderChecksum(inlist, checksumtype, True, formatspecs);
510 inchecksum = inchecksum.lower();
511 catfileheadercshex = catfileheadercshex.lower();
512 if(inchecksum==catfileheadercshex):
513 return True;
514 else:
515 return False;
516
517 def ValidateFileChecksum(infile, checksumtype="crc32", inchecksum="0", formatspecs=__file_format_list__):
518 catinfilecshex = GetFileChecksum(infile, checksumtype, True, formatspecs);
519 inchecksum = inchecksum.lower();
520 catinfilecshex = catinfilecshex.lower();
521 if(inchecksum==catinfilecshex):
522 return True;
523 else:
524 return False;
525
526 def ReadTillNullByte(fp, delimiter=__file_format_delimiter__):
527 curbyte = b"";
528 curfullbyte = b"";
529 nullbyte = delimiter.encode("UTF-8");
530 while(True):
531 curbyte = fp.read(1);
532 if(curbyte==nullbyte or not curbyte):
533 break;
534 curfullbyte = curfullbyte + curbyte;
535 return curfullbyte.decode('UTF-8');
536
537 def ReadUntilNullByte(fp, delimiter=__file_format_delimiter__):
538 return ReadTillNullByte(fp, delimiter);
539
540 def SeekToEndOfFile(fp):
541 lasttell = 0;
542 while(True):
543 fp.seek(1, 1);
544 if(lasttell==fp.tell()):
545 break;
546 lasttell = fp.tell();
547 return True;
548
549 def ReadFileHeaderData(fp, rounds=0, delimiter=__file_format_delimiter__):
550 rocount = 0;
551 roend = int(rounds);
552 HeaderOut = [];
553 while(rocount<roend):
554 HeaderOut.append(ReadTillNullByte(fp, delimiter));
555 rocount = rocount + 1;
556 return HeaderOut;
557
558 def ReadFileHeaderDataBySize(fp, delimiter=__file_format_delimiter__):
559 headerpresize = ReadTillNullByte(fp, delimiter);
560 headersize = int(headerpresize, 16);
561 headercontent = str(fp.read(headersize).decode('UTF-8')).split(delimiter);
562 fp.seek(1, 1);
563 rocount = 0;
564 roend = int(len(headercontent));
565 HeaderOut = [headerpresize];
566 while(rocount<roend):
567 HeaderOut.append(headercontent[rocount]);
568 rocount = rocount + 1;
569 return HeaderOut;
570
571 def ReadFileHeaderDataWoSize(fp, delimiter=__file_format_delimiter__):
572 headerdata = ReadFileHeaderData(fp, 24, delimiter);
573 fextrafields = int(headerdata[23], 16);
574 extrafieldslist = ReadFileHeaderData(fp, fextrafields, delimiter);
575 checksumsval = ReadFileHeaderData(fp, 3, delimiter);
576 HeaderOut = headerdata + extrafieldslist + checksumsval;
577 return HeaderOut;
578
579 def ReadFileHeaderDataBySizeWithContent(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
580 delimiter = formatspecs[5];
581 fheaderstart = fp.tell();
582 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
583 if(re.findall("^[.|/]", HeaderOut[2])):
584 fname = HeaderOut[2];
585 else:
586 fname = "./"+HeaderOut[2];
587 fchecksumtype = HeaderOut[-3].lower();
588 fcs = HeaderOut[-2].lower();
589 fccs = HeaderOut[-1].lower();
590 fsize = int(HeaderOut[4], 16);
591 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
592 if(fcs!=newfcs and not skipchecksum):
593 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
594 return False;
595 fhend = fp.tell() - 1;
596 fcontentstart = fp.tell();
597 fcontents = "".encode('UTF-8');
598 if(fsize>0 and not listonly):
599 fcontents = fp.read(fsize);
600 elif(fsize>0 and listonly):
601 fp.seek(fsize, 1);
602 newfccs = GetFileChecksum(fcontents, HeaderOut[-3].lower(), False, formatspecs);
603 if(fccs!=newfccs and not skipchecksum and not listonly):
604 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
605 return False;
606 fp.seek(1, 1);
607 fcontentend = fp.tell() - 1;
608 HeaderOut.append(fcontents);
609 return HeaderOut;
610
611 def ReadFileHeaderDataBySizeWithContentToArray(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
612 delimiter = formatspecs[5];
613 fheaderstart = fp.tell();
614 if(formatspecs[7]):
615 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
616 else:
617 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
618 fheadsize = int(HeaderOut[0], 16);
619 ftype = int(HeaderOut[1], 16);
620 if(re.findall("^[.|/]", HeaderOut[2])):
621 fname = HeaderOut[2];
622 else:
623 fname = "./"+HeaderOut[2];
624 fbasedir = os.path.dirname(fname);
625 flinkname = HeaderOut[3];
626 fsize = int(HeaderOut[4], 16);
627 fatime = int(HeaderOut[5], 16);
628 fmtime = int(HeaderOut[6], 16);
629 fctime = int(HeaderOut[7], 16);
630 fbtime = int(HeaderOut[8], 16);
631 fmode = int(HeaderOut[9], 16);
632 fchmode = stat.S_IMODE(fmode);
633 ftypemod = stat.S_IFMT(fmode);
634 fwinattributes = int(HeaderOut[10], 16);
635 fuid = int(HeaderOut[11], 16);
636 funame = HeaderOut[12];
637 fgid = int(HeaderOut[13], 16);
638 fgname = HeaderOut[14];
639 fid = int(HeaderOut[15], 16);
640 finode = int(HeaderOut[16], 16);
641 flinkcount = int(HeaderOut[17], 16);
642 fdev_minor = int(HeaderOut[18], 16);
643 fdev_major = int(HeaderOut[19], 16);
644 frdev_minor = int(HeaderOut[20], 16);
645 frdev_major = int(HeaderOut[21], 16);
646 fextrasize = int(HeaderOut[22], 16);
647 fextrafields = int(HeaderOut[23], 16);
648 extrafieldslist = [];
649 extrastart = 24;
650 extraend = extrastart + fextrafields;
651 extrafieldslist = [];
652 if(extrastart<extraend):
653 extrafieldslist.append(HeaderOut[extrastart]);
654 extrastart = extrastart + 1;
655 fnumfields = 24 + fextrafields;
656 fchecksumtype = HeaderOut[extrastart].lower();
657 fcs = HeaderOut[extrastart + 1].lower();
658 fccs = HeaderOut[extrastart + 2].lower();
659 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
660 if(fcs!=newfcs and not skipchecksum):
661 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
662 return False;
663 fhend = fp.tell() - 1;
664 fcontentstart = fp.tell();
665 fcontents = "".encode('UTF-8');
666 pyhascontents = False;
667 if(fsize>0 and not listonly):
668 fcontents = fp.read(fsize);
669 pyhascontents = True;
670 elif(fsize>0 and listonly):
671 fp.seek(fsize, 1);
672 pyhascontents = False;
673 newfccs = GetFileChecksum(fcontents, HeaderOut[-3].lower(), False, formatspecs);
674 if(fccs!=newfccs and not skipchecksum and not listonly):
675 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
676 return False;
677 fp.seek(1, 1);
678 fcontentend = fp.tell() - 1;
679 catlist = {'fheadersize': fheadsize, 'fhstart': fheaderstart, 'fhend': fhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': fchecksumtype, 'fnumfields': fnumfields, 'fextrafields': fextrafields, 'fextrafieldsize': fextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': fcs, 'fcontentchecksum': fccs, 'fhascontents': pyhascontents, 'fcontentstart': fcontentstart, 'fcontentend': fcontentend, 'fcontents': fcontents}
680 return catlist;
681
682 def ReadFileDataBySizeWithContent(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
683 delimiter = formatspecs[5];
684 catheader = ReadFileHeaderData(fp, 4, delimiter);
685 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
686 if(not headercheck and not skipchecksum):
687 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
688 return False;
689 fnumfiles = int(catheader[1], 16);
690 countnum = 0;
691 flist = [];
692 while(countnum < fnumfiles):
693 flist.append(ReadFileHeaderDataBySizeWithContent(fp, listonly, skipchecksum, formatspecs));
694 countnum = countnum + 1;
695 return flist;
696
697 def ReadFileDataBySizeWithContentToArray(fp, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
698 delimiter = formatspecs[5];
699 catheader = ReadFileHeaderData(fp, 4, delimiter);
700 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
701 if(not headercheck and not skipchecksum):
702 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
703 return False;
704 catstring = catheader[0];
705 catversion = re.findall(r"([\d]+)$", catstring);
706 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
707 fprenumfiles = catheader[1];
708 fnumfiles = int(fprenumfiles, 16);
709 fprechecksumtype = catheader[2];
710 fprechecksum = catheader[3];
711 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
712 if(seekstart<0 and seekstart>fnumfiles):
713 seekstart = 0;
714 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
715 seekend = fnumfiles;
716 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
717 seekend = fnumfiles - abs(seekend);
718 if(seekstart>0):
719 il = 0;
720 while(il < seekstart):
721 prefhstart = catfp.tell();
722 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs[5]);
723 prefsize = int(preheaderdata[4], 16);
724 hout = AppendNullBytes(preheaderdata, formatspecs[5]);
725 if(prefchecksumtype=="none" or prefchecksumtype==""):
726 prenewfcs = 0;
727 elif(prefchecksumtype=="crc16" or prefchecksumtype=="crc16_ansi" or prefchecksumtype=="crc16_ibm"):
728 prenewfcs = format(crc16(hout.encode('UTF-8')) & 0xffff, '04x').lower();
729 elif(prefchecksumtype=="adler32"):
730 prenewfcs = format(zlib.adler32(hout.encode('UTF-8')) & 0xffffffff, '08x').lower();
731 elif(prefchecksumtype=="crc32"):
732 prenewfcs = format(crc32(hout.encode('UTF-8')) & 0xffffffff, '08x').lower();
733 elif(prefchecksumtype=="crc64_ecma"):
734 prenewfcs = format(crc64_ecma(hout.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
735 elif(prefchecksumtype=="crc64" or prefchecksumtype=="crc64_iso"):
736 prenewfcs = format(crc64_iso(hout.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
737 elif(CheckSumSupportAlt(prefchecksumtype, hashlib_guaranteed)):
738 checksumoutstr = hashlib.new(prefchecksumtype);
739 checksumoutstr.update(hout.encode('UTF-8'));
740 prenewfcs = checksumoutstr.hexdigest().lower();
741 if(prefcs!=prenewfcs and not skipchecksum):
742 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
743 return False;
744 valid_archive = False;
745 invalid_archive = True;
746 prefhend = catfp.tell() - 1;
747 prefcontentstart = catfp.tell();
748 prefcontents = "";
749 pyhascontents = False;
750 if(prefsize>0):
751 prefcontents = catfp.read(prefsize);
752 if(prefchecksumtype=="none" or prefchecksumtype==""):
753 prenewfccs = 0;
754 elif(prefchecksumtype=="crc16" or prefchecksumtype=="crc16_ansi" or prefchecksumtype=="crc16_ibm"):
755 prenewfccs = format(crc16(prefcontents) & 0xffff, '04x').lower();
756 elif(prefchecksumtype=="crc16_ccitt"):
757 prenewfcs = format(crc16_ccitt(prefcontents) & 0xffff, '04x').lower();
758 elif(prefchecksumtype=="adler32"):
759 prenewfccs = format(zlib.adler32(prefcontents) & 0xffffffff, '08x').lower();
760 elif(prefchecksumtype=="crc32"):
761 prenewfccs = format(crc32(prefcontents) & 0xffffffff, '08x').lower();
762 elif(prefchecksumtype=="crc64_ecma"):
763 prenewfcs = format(crc64_ecma(prefcontents) & 0xffffffffffffffff, '016x').lower();
764 elif(prefchecksumtype=="crc64" or prefchecksumtype=="crc64_iso"):
765 prenewfcs = format(crc64_iso(prefcontents) & 0xffffffffffffffff, '016x').lower();
766 elif(CheckSumSupportAlt(prefchecksumtype, hashlib_guaranteed)):
767 checksumoutstr = hashlib.new(prefchecksumtype);
768 checksumoutstr.update(prefcontents);
769 prenewfccs = checksumoutstr.hexdigest().lower();
770 pyhascontents = True;
771 if(prefccs!=prenewfccs and not skipchecksum):
772 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
773 return False;
774 catfp.seek(1, 1);
775 il = il + 1;
776 realidnum = 0;
777 countnum = seekstart;
778 while(countnum < seekend):
779 catlist['ffilelist'].update({realidnum: {'fid': realidnum, 'fidalt': realidnum}});
780 catlist['ffilelist'][realidnum].update(ReadFileHeaderDataBySizeWithContentToArray(fp, listonly, skipchecksum, formatspecs));
781 countnum = countnum + 1;
782 realidnum = realidnum + 1;
783 return catlist;
784
785 def ReadFileDataBySizeWithContentToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
786 if(isinstance(infile, dict)):
787 listcatfiles = infile;
788 else:
789 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
790 return False;
791 listcatfiles = ReadFileDataBySizeWithContentToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs);
792 if(not listcatfiles):
793 return False;
794 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
795 lenlist = len(listcatfiles['ffilelist']);
796 lcfi = 0;
797 lcfx = int(listcatfiles['fnumfiles']);
798 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
799 lcfx = int(lenlist);
800 else:
801 lcfx = int(listcatfiles['fnumfiles']);
802 while(lcfi < lcfx):
803 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
804 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
805 catarray['filetoid'].update(filetoidarray);
806 catarray['idtofile'].update(idtofilearray);
807 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
808 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
809 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
810 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
811 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
812 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
813 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
814 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
815 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
816 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
817 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
818 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
819 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
820 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
821 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
822 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
823 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
824 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
825 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
826 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
827 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
828 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
829 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
830 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
831 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
832 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
833 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
834 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
835 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
836 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
837 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
838 lcfi = lcfi + 1;
839 return catarray;
840
841 def ReadInFileBySizeWithContent(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
842 delimiter = formatspecs[5];
843 if(hasattr(infile, "read") or hasattr(infile, "write")):
844 fp = infile;
845 fp.seek(0, 0);
846 fp = UncompressArchiveFile(fp, formatspecs);
847 checkcompressfile = CheckCompressionSubType(fp, formatspecs);
848 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
849 return False;
850 if(not fp):
851 return False;
852 fp.seek(0, 0);
853 elif(infile=="-"):
854 fp = BytesIO();
855 if(hasattr(sys.stdin, "buffer")):
856 shutil.copyfileobj(sys.stdin.buffer, fp);
857 else:
858 shutil.copyfileobj(sys.stdin, fp);
859 fp.seek(0, 0);
860 fp = UncompressArchiveFile(fp, formatspecs);
861 if(not fp):
862 return False;
863 fp.seek(0, 0);
864 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", infile)):
865 fp = download_file_from_internet_file(infile);
866 fp = UncompressArchiveFile(fp, formatspecs);
867 fp.seek(0, 0);
868 if(not fp):
869 return False;
870 fp.seek(0, 0);
871 else:
872 infile = RemoveWindowsPath(infile);
873 checkcompressfile = CheckCompressionSubType(infile, formatspecs);
874 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
875 return False;
876 compresscheck = CheckCompressionType(infile, formatspecs, True);
877 if(not compresscheck):
878 fextname = os.path.splitext(infile)[1];
879 if(fextname==".gz"):
880 compresscheck = "gzip";
881 elif(fextname==".bz2"):
882 compresscheck = "bzip2";
883 elif(fextname==".zst"):
884 compresscheck = "zstd";
885 elif(fextname==".lz4" or fextname==".clz4"):
886 compresscheck = "lz4";
887 elif(fextname==".lzo" or fextname==".lzop"):
888 compresscheck = "lzo";
889 elif(fextname==".lzma" or fextname==".xz"):
890 compresscheck = "lzma";
891 else:
892 return False;
893 if(not compresscheck):
894 return False;
895 fp = UncompressFile(infile, formatspecs, "rb");
896 return ReadFileDataBySizeWithContent(fp, seekstart, seekend, listonly, skipchecksum, formatspecs);
897
898 def ReadInFileBySizeWithContentToArray(infile, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
899 delimiter = formatspecs[5];
900 if(hasattr(infile, "read") or hasattr(infile, "write")):
901 fp = infile;
902 fp.seek(0, 0);
903 fp = UncompressArchiveFile(fp, formatspecs);
904 checkcompressfile = CheckCompressionSubType(fp, formatspecs);
905 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
906 return False;
907 if(not fp):
908 return False;
909 fp.seek(0, 0);
910 elif(infile=="-"):
911 fp = BytesIO();
912 if(hasattr(sys.stdin, "buffer")):
913 shutil.copyfileobj(sys.stdin.buffer, fp);
914 else:
915 shutil.copyfileobj(sys.stdin, fp);
916 fp.seek(0, 0);
917 fp = UncompressArchiveFile(fp, formatspecs);
918 if(not fp):
919 return False;
920 fp.seek(0, 0);
921 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", infile)):
922 fp = download_file_from_internet_file(infile);
923 fp = UncompressArchiveFile(fp, formatspecs);
924 fp.seek(0, 0);
925 if(not fp):
926 return False;
927 fp.seek(0, 0);
928 else:
929 infile = RemoveWindowsPath(infile);
930 checkcompressfile = CheckCompressionSubType(infile, formatspecs);
931 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
932 return False;
933 compresscheck = CheckCompressionType(infile, formatspecs, True);
934 if(not compresscheck):
935 fextname = os.path.splitext(infile)[1];
936 if(fextname==".gz"):
937 compresscheck = "gzip";
938 elif(fextname==".bz2"):
939 compresscheck = "bzip2";
940 elif(fextname==".zst"):
941 compresscheck = "zstd";
942 elif(fextname==".lz4" or fextname==".clz4"):
943 compresscheck = "lz4";
944 elif(fextname==".lzo" or fextname==".lzop"):
945 compresscheck = "lzo";
946 elif(fextname==".lzma" or fextname==".xz"):
947 compresscheck = "lzma";
948 else:
949 return False;
950 if(not compresscheck):
951 return False;
952 fp = UncompressFile(infile, formatspecs, "rb");
953 return ReadFileDataBySizeWithContentToArray(fp, listonly, skipchecksum, formatspecs);
954
955 def ReadInFileBySizeWithContentToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
956 if(isinstance(infile, dict)):
957 listcatfiles = infile;
958 else:
959 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
960 infile = RemoveWindowsPath(infile);
961 listcatfiles = ReadInFileBySizeWithContentToArray(infile, listonly, skipchecksum, formatspecs);
962 if(not listcatfiles):
963 return False;
964 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
965 lenlist = len(listcatfiles['ffilelist']);
966 lcfi = 0;
967 lcfx = int(listcatfiles['fnumfiles']);
968 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
969 lcfx = int(lenlist);
970 else:
971 lcfx = int(listcatfiles['fnumfiles']);
972 while(lcfi < lcfx):
973 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
974 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
975 catarray['filetoid'].update(filetoidarray);
976 catarray['idtofile'].update(idtofilearray);
977 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
978 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
979 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
980 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
981 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
982 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
983 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
984 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
985 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
986 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
987 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
988 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
989 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
990 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
991 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
992 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
993 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
994 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
995 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
996 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
997 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
998 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
999 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1000 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
1001 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
1002 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
1003 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
1004 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1005 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1006 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1007 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1008 lcfi = lcfi + 1;
1009 return catarray;
1010
1011 def AppendNullByte(indata, delimiter=__file_format_delimiter__):
1012 outdata = str(indata) + delimiter;
1013 return outdata;
1014
1015 def AppendNullBytes(indata=[], delimiter=__file_format_delimiter__):
1016 outdata = "";
1017 inum = 0;
1018 il = len(indata);
1019 while(inum < il):
1020 outdata = outdata + AppendNullByte(indata[inum], delimiter);
1021 inum = inum + 1;
1022 return outdata;
1023
1024 def PrintPermissionString(fchmode, ftype):
1025 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
1026 permissionstr = "";
1027 for fmodval in str(oct(fchmode))[-3:]:
1028 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
1029 if(ftype==0 or ftype==7):
1030 permissionstr = "-" + permissionstr;
1031 if(ftype==1):
1032 permissionstr = "h" + permissionstr;
1033 if(ftype==2):
1034 permissionstr = "l" + permissionstr;
1035 if(ftype==3):
1036 permissionstr = "c" + permissionstr;
1037 if(ftype==4):
1038 permissionstr = "b" + permissionstr;
1039 if(ftype==5):
1040 permissionstr = "d" + permissionstr;
1041 if(ftype==6):
1042 permissionstr = "f" + permissionstr;
1043 if(ftype==8):
1044 permissionstr = "D" + permissionstr;
1045 if(ftype==9):
1046 permissionstr = "p" + permissionstr;
1047 if(ftype==10):
1048 permissionstr = "w" + permissionstr;
1049 try:
1050 permissionoutstr = stat.filemode(fchmode);
1051 except AttributeError:
1052 permissionoutstr = permissionstr;
1053 except KeyError:
1054 permissionoutstr = permissionstr;
1055 return permissionoutstr;
1056
1057 def PrintPermissionStringAlt(fchmode, ftype):
1058 permissions = {
1059 '0': '---', '1': '--x', '2': '-w-', '3': '-wx',
1060 '4': 'r--', '5': 'r-x', '6': 'rw-', '7': 'rwx'
1061 };
1062 # Translate file mode into permission string
1063 permissionstr = ''.join([permissions[i] for i in str(oct(fchmode))[-3:]]);
1064 # Append file type indicator
1065 type_indicators = {
1066 0: '-', 1: 'h', 2: 'l', 3: 'c', 4: 'b',
1067 5: 'd', 6: 'f', 8: 'D', 9: 'p', 10: 'w'
1068 };
1069 file_type = type_indicators.get(ftype, '-');
1070 permissionstr = file_type + permissionstr;
1071 try:
1072 permissionoutstr = stat.filemode(fchmode);
1073 except AttributeError:
1074 permissionoutstr = permissionstr;
1075 return permissionoutstr;
1076
1077 def CompressionSupport():
1078 compression_list = [];
1079 try:
1080 import gzip;
1081 compression_list.append("gz");
1082 compression_list.append("gzip");
1083 except ImportError:
1084 '''return False;'''
1085 try:
1086 import bz2;
1087 compression_list.append("bz2");
1088 compression_list.append("bzip2");
1089 except ImportError:
1090 '''return False;'''
1091 try:
1092 import lz4;
1093 compression_list.append("lz4");
1094 except ImportError:
1095 '''return False;'''
1096 try:
1097 import lzo;
1098 compression_list.append("lzo");
1099 compression_list.append("lzop");
1100 except ImportError:
1101 '''return False;'''
1102 try:
1103 import zstandard;
1104 compression_list.append("zstd");
1105 compression_list.append("zstandard");
1106 except ImportError:
1107 '''return False;'''
1108 try:
1109 import lzma;
1110 compression_list.append("lzma");
1111 compression_list.append("xz");
1112 except ImportError:
1113 '''return False;'''
1114 return compression_list;
1115
1116 def CheckCompressionType(infile, formatspecs=__file_format_list__, closefp=True):
1117 if(hasattr(infile, "read") or hasattr(infile, "write")):
1118 catfp = infile;
1119 else:
1120 try:
1121 catfp = open(infile, "rb");
1122 except FileNotFoundError:
1123 return False;
1124 catfp.seek(0, 0);
1125 prefp = catfp.read(2);
1126 filetype = False;
1127 if(prefp==binascii.unhexlify("1f8b")):
1128 filetype = "gzip";
1129 catfp.seek(0, 0);
1130 prefp = catfp.read(3);
1131 if(prefp==binascii.unhexlify("425a68")):
1132 filetype = "bzip2";
1133 if(prefp==binascii.unhexlify("5d0000")):
1134 filetype = "lzma";
1135 catfp.seek(0, 0);
1136 prefp = catfp.read(4);
1137 if(prefp==binascii.unhexlify("28b52ffd")):
1138 filetype = "zstd";
1139 if(prefp==binascii.unhexlify("04224d18")):
1140 filetype = "lz4";
1141 if(prefp==binascii.unhexlify("504B0304")):
1142 filetype = "zipfile";
1143 catfp.seek(0, 0);
1144 prefp = catfp.read(5);
1145 if(prefp==binascii.unhexlify("7573746172")):
1146 filetype = "tarfile";
1147 catfp.seek(0, 0);
1148 prefp = catfp.read(6);
1149 if(prefp==binascii.unhexlify("fd377a585a00")):
1150 filetype = "lzma";
1151 catfp.seek(0, 0);
1152 prefp = catfp.read(7);
1153 if(prefp==binascii.unhexlify("526172211a0700")):
1154 filetype = "rarfile";
1155 if(prefp==binascii.unhexlify("43617446696c65")):
1156 filetype = "catfile";
1157 catfp.seek(0, 0);
1158 prefp = catfp.read(8);
1159 if(prefp==binascii.unhexlify("526172211a070100")):
1160 filetype = "rarfile";
1161 catfp.seek(0, 0);
1162 prefp = catfp.read(formatspecs[3]);
1163 if(prefp==binascii.unhexlify(formatspecs[4])):
1164 filetype = formatspecs[2];
1165 catfp.seek(0, 0);
1166 prefp = catfp.read(9);
1167 if(prefp==binascii.unhexlify("894c5a4f000d0a1a0a")):
1168 filetype = "lzo";
1169 catfp.seek(0, 0);
1170 prefp = catfp.read(10);
1171 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
1172 filetype = "tarfile";
1173 catfp.seek(0, 0);
1174 if(closefp):
1175 catfp.close();
1176 return filetype;
1177
1178 def CheckCompressionTypeFromString(instring, formatspecs=__file_format_list__, closefp=True):
1179 try:
1180 instringsfile = BytesIO(instring);
1181 except TypeError:
1182 instringsfile = BytesIO(instring.encode("UTF-8"));
1183 return CheckCompressionType(instringsfile, formatspecs, closefp);
1184
1185 def GetCompressionMimeType(infile, formatspecs=__file_format_list__):
1186 compresscheck = CheckCompressionType(fp, formatspecs, False);
1187 if(compresscheck=="gzip" or compresscheck=="gz"):
1188 return archivefile_gzip_mimetype;
1189 if(compresscheck=="bzip2" or compresscheck=="bz2"):
1190 return archivefile_bzip2_mimetype;
1191 if(compresscheck=="zstd" or compresscheck=="zstandard"):
1192 return archivefile_zstandard_mimetype;
1193 if(compresscheck=="lz4"):
1194 return archivefile_lz4_mimetype;
1195 if(compresscheck=="lzo" or compresscheck=="lzop"):
1196 return archivefile_lzop_mimetype;
1197 if(compresscheck=="lzma"):
1198 return archivefile_lzma_mimetype;
1199 if(compresscheck=="xz"):
1200 return archivefile_xz_mimetype;
1201 if(compresscheck=="catfile" or compresscheck=="cat" or compresscheck==formatspecs[2]):
1202 return archivefile_cat_mimetype;
1203 if(not compresscheck):
1204 return False;
1205 return False;
1206
1207 def UncompressArchiveFile(fp, formatspecs=__file_format_list__):
1208 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
1209 return False;
1210 compresscheck = CheckCompressionType(fp, formatspecs, False);
1211 if(compresscheck=="gzip"):
1212 try:
1213 import gzip;
1214 except ImportError:
1215 return False;
1216 catfp = gzip.GzipFile(fileobj=fp, mode="rb");
1217 if(compresscheck=="bzip2"):
1218 try:
1219 import bz2;
1220 except ImportError:
1221 return False;
1222 catfp = BytesIO();
1223 catfp.write(bz2.decompress(fp.read()));
1224 if(compresscheck=="zstd"):
1225 try:
1226 import zstandard;
1227 except ImportError:
1228 return False;
1229 catfp = BytesIO();
1230 catfp.write(zstandard.decompress(fp.read()));
1231 if(compresscheck=="lz4"):
1232 try:
1233 import lz4.frame;
1234 except ImportError:
1235 return False;
1236 catfp = BytesIO();
1237 catfp.write(lz4.frame.decompress(fp.read()));
1238 if(compresscheck=="lzo" or compresscheck=="lzop"):
1239 try:
1240 import lzo;
1241 except ImportError:
1242 return False;
1243 catfp = BytesIO();
1244 catfp.write(lzo.decompress(fp.read()));
1245 if(compresscheck=="lzma" or compresscheck=="xz"):
1246 try:
1247 import lzma;
1248 except ImportError:
1249 return False;
1250 catfp = BytesIO();
1251 catfp.write(lzma.decompress(fp.read()));
1252 if(compresscheck=="catfile" or compresscheck==formatspecs[2]):
1253 catfp = fp;
1254 if(not compresscheck):
1255 try:
1256 import lzma;
1257 except ImportError:
1258 return False;
1259 catfp = BytesIO();
1260 with fp as fpcontent:
1261 try:
1262 catfp.write(lzma.decompress(fp.read()));
1263 except lzma.LZMAError:
1264 return False;
1265 return catfp;
1266
1267 create_alias_function("Uncompress", __file_format_name__, "", UncompressArchiveFile);
1268
1269 def UncompressFile(infile, formatspecs=__file_format_list__, mode="rb"):
1270 compresscheck = CheckCompressionType(infile, formatspecs, False);
1271 if(sys.version_info[0]==2 and compresscheck):
1272 if(mode=="rt"):
1273 mode = "r";
1274 if(mode=="wt"):
1275 mode = "w";
1276 try:
1277 if(compresscheck=="gzip"):
1278 try:
1279 import gzip;
1280 except ImportError:
1281 return False;
1282 try:
1283 filefp = gzip.open(infile, mode, encoding="UTF-8");
1284 except (ValueError, TypeError) as e:
1285 filefp = gzip.open(infile, mode);
1286 if(compresscheck=="bzip2"):
1287 try:
1288 import bz2;
1289 except ImportError:
1290 return False;
1291 try:
1292 filefp = bz2.open(infile, mode, encoding="UTF-8");
1293 except (ValueError, TypeError) as e:
1294 filefp = bz2.open(infile, mode);
1295 if(compresscheck=="zstd"):
1296 try:
1297 import zstandard;
1298 except ImportError:
1299 return False;
1300 try:
1301 filefp = zstandard.open(infile, mode, encoding="UTF-8");
1302 except (ValueError, TypeError) as e:
1303 filefp = zstandard.open(infile, mode);
1304 if(compresscheck=="lz4"):
1305 try:
1306 import lz4.frame;
1307 except ImportError:
1308 return False;
1309 try:
1310 filefp = lz4.frame.open(infile, mode, encoding="UTF-8");
1311 except (ValueError, TypeError) as e:
1312 filefp = lz4.frame.open(infile, mode);
1313 if(compresscheck=="lzo"):
1314 try:
1315 import lzo;
1316 except ImportError:
1317 return False;
1318 try:
1319 filefp = lzo.open(infile, mode, encoding="UTF-8");
1320 except (ValueError, TypeError) as e:
1321 filefp = lzo.open(infile, mode);
1322 if(compresscheck=="lzma"):
1323 try:
1324 import lzma;
1325 except ImportError:
1326 return False;
1327 try:
1328 filefp = lzma.open(infile, mode, encoding="UTF-8");
1329 except (ValueError, TypeError) as e:
1330 filefp = lzma.open(infile, mode);
1331 if(compresscheck=="catfile" or compresscheck==formatspecs[2]):
1332 try:
1333 filefp = open(infile, mode, encoding="UTF-8");
1334 except (ValueError, TypeError) as e:
1335 filefp = open(infile, mode);
1336 if(not compresscheck):
1337 try:
1338 filefp = open(infile, mode, encoding="UTF-8");
1339 except (ValueError, TypeError) as e:
1340 filefp = open(infile, mode);
1341 except FileNotFoundError:
1342 return False;
1343 return filefp;
1344
1345 def UncompressString(infile):
1346 compresscheck = CheckCompressionTypeFromString(infile, formatspecs, False);
1347 if(compresscheck=="gzip"):
1348 try:
1349 import gzip;
1350 except ImportError:
1351 return False;
1352 fileuz = gzip.decompress(infile);
1353 if(compresscheck=="bzip2"):
1354 try:
1355 import bz2;
1356 except ImportError:
1357 return False;
1358 fileuz = bz2.decompress(infile);
1359 if(compresscheck=="zstd"):
1360 try:
1361 import zstandard;
1362 except ImportError:
1363 return False;
1364 fileuz = zstandard.decompress(infile);
1365 if(compresscheck=="lz4"):
1366 try:
1367 import lz4.frame;
1368 except ImportError:
1369 return False;
1370 fileuz = lz4.frame.decompress(infile);
1371 if(compresscheck=="lzo"):
1372 try:
1373 import lzo;
1374 except ImportError:
1375 return False;
1376 fileuz = lzo.decompress(infile);
1377 if(compresscheck=="lzma"):
1378 try:
1379 import lzma;
1380 except ImportError:
1381 return False;
1382 fileuz = lzma.decompress(infile);
1383 if(not compresscheck):
1384 fileuz = infile;
1385 if(hasattr(fileuz, 'decode')):
1386 fileuz = fileuz.decode("UTF-8");
1387 return fileuz;
1388
1389 def UncompressStringAlt(infile):
1390 filefp = StringIO();
1391 outstring = UncompressString(infile);
1392 filefp.write(outstring);
1393 filefp.seek(0, 0);
1394 return filefp;
1395
1396 def CheckCompressionSubType(infile, formatspecs=__file_format_list__):
1397 compresscheck = CheckCompressionType(infile, formatspecs, False);
1398 if(not compresscheck):
1399 fextname = os.path.splitext(infile)[1];
1400 if(fextname==".gz"):
1401 compresscheck = "gzip";
1402 elif(fextname==".bz2"):
1403 compresscheck = "bzip2";
1404 elif(fextname==".zst"):
1405 compresscheck = "zstd";
1406 elif(fextname==".lz4"):
1407 compresscheck = "lz4";
1408 elif(fextname==".lzo" or fextname==".lzop"):
1409 compresscheck = "lzo";
1410 elif(fextname==".lzma" or fextname==".xz"):
1411 compresscheck = "lzma";
1412 else:
1413 return False;
1414 if(not compresscheck):
1415 return False;
1416 if(compresscheck=="catfile"):
1417 return "catfile";
1418 if(compresscheck==formatspecs[2]):
1419 return formatspecs[2];
1420 if(compresscheck=="tarfile"):
1421 return "tarfile";
1422 if(compresscheck=="zipfile"):
1423 return "zipfile";
1424 if(hasattr(infile, "read") or hasattr(infile, "write")):
1425 catfp = UncompressArchiveFile(infile, formatspecs[2]);
1426 else:
1427 try:
1428 if(compresscheck=="gzip"):
1429 try:
1430 import gzip;
1431 except ImportError:
1432 return False;
1433 catfp = gzip.GzipFile(infile, "rb");
1434 if(compresscheck=="bzip2"):
1435 try:
1436 import bz2;
1437 except ImportError:
1438 return False;
1439 catfp = bz2.BZ2File(infile, "rb");
1440 if(compresscheck=="lz4"):
1441 try:
1442 import lz4.frame;
1443 except ImportError:
1444 return False;
1445 catfp = lz4.frame.open(infile, "rb");
1446 if(compresscheck=="zstd"):
1447 try:
1448 import zstandard;
1449 except ImportError:
1450 return False;
1451 catfp = zstandard.open(infile, "rb");
1452 if(compresscheck=="lzma" or compresscheck=="xz"):
1453 try:
1454 import lzma;
1455 except ImportError:
1456 return False;
1457 catfp = lzma.open(infile, "rb");
1458 except FileNotFoundError:
1459 return False;
1460 filetype = False;
1461 prefp = catfp.read(5);
1462 if(prefp==binascii.unhexlify("7573746172")):
1463 filetype = "tarfile";
1464 catfp.seek(0, 0);
1465 prefp = catfp.read(7);
1466 if(prefp==binascii.unhexlify("43617446696c65")):
1467 filetype = "catfile";
1468 catfp.seek(0, 0);
1469 prefp = catfp.read(formatspecs[3]);
1470 if(prefp==binascii.unhexlify(formatspecs[4])):
1471 filetype = formatspecs[2];
1472 catfp.seek(0, 0);
1473 prefp = catfp.read(10);
1474 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
1475 filetype = "tarfile";
1476 catfp.seek(0, 0);
1477 catfp.close();
1478 return filetype;
1479
1480 def GZipCompress(data, compresslevel=9):
1481 try:
1482 import gzip;
1483 except ImportError:
1484 return False;
1485 tmpfp = tempfile.NamedTemporaryFile("wb", delete=False);
1486 tmpfp.close();
1487 tmpfp = gzip.GzipFile(tmpfp.name, mode="wb", compresslevel=compresslevel);
1488 tmpfp.write(data);
1489 tmpfp.close();
1490 try:
1491 catfp = open(tmpfp.name, "rb");
1492 except FileNotFoundError:
1493 return False;
1494 catdata = catfp.read();
1495 catfp.close();
1496 return catdata;
1497
1498 def CompressArchiveFile(fp, compression="auto", compressionlevel=None, formatspecs=__file_format_list__):
1499 compressionlist = ['auto', 'gzip', 'bzip2', 'zstd', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
1500 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
1501 return False;
1502 fp.seek(0, 0);
1503 if(not compression or compression or compression=="catfile" or compression==formatspecs[2]):
1504 compression = None;
1505 if(compression not in compressionlist and compression is None):
1506 compression = "auto";
1507 if(compression=="gzip"):
1508 try:
1509 import gzip;
1510 except ImportError:
1511 return False;
1512 catfp = BytesIO();
1513 if(compressionlevel is None):
1514 compressionlevel = 9;
1515 else:
1516 compressionlevel = int(compressionlevel);
1517 catfp.write(gzip.compress(fp.read(), compresslevel=compressionlevel));
1518 if(compression=="bzip2"):
1519 try:
1520 import bz2;
1521 except ImportError:
1522 return False;
1523 catfp = BytesIO();
1524 if(compressionlevel is None):
1525 compressionlevel = 9;
1526 else:
1527 compressionlevel = int(compressionlevel);
1528 catfp.write(bz2.compress(fp.read(), compresslevel=compressionlevel));
1529 if(compression=="lz4"):
1530 try:
1531 import lz4.frame;
1532 except ImportError:
1533 return False;
1534 catfp = BytesIO();
1535 if(compressionlevel is None):
1536 compressionlevel = 9;
1537 else:
1538 compressionlevel = int(compressionlevel);
1539 catfp.write(lz4.frame.compress(fp.read(), compression_level=compressionlevel));
1540 if(compression=="lzo" or compression=="lzop"):
1541 try:
1542 import lzo;
1543 except ImportError:
1544 return False;
1545 catfp = BytesIO();
1546 if(compressionlevel is None):
1547 compressionlevel = 9;
1548 else:
1549 compressionlevel = int(compressionlevel);
1550 catfp.write(lzo.compress(fp.read(), compresslevel=compressionlevel));
1551 if(compression=="zstd"):
1552 try:
1553 import zstandard;
1554 except ImportError:
1555 return False;
1556 catfp = BytesIO();
1557 if(compressionlevel is None):
1558 compressionlevel = 10;
1559 else:
1560 compressionlevel = int(compressionlevel);
1561 catfp.write(zstandard.compress(fp.read(), level=compressionlevel));
1562 if(compression=="lzma"):
1563 try:
1564 import lzma;
1565 except ImportError:
1566 return False;
1567 catfp = BytesIO();
1568 if(compressionlevel is None):
1569 compressionlevel = 9;
1570 else:
1571 compressionlevel = int(compressionlevel);
1572 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]));
1573 if(compression=="xz"):
1574 try:
1575 import lzma;
1576 except ImportError:
1577 return False;
1578 catfp = BytesIO();
1579 if(compressionlevel is None):
1580 compressionlevel = 9;
1581 else:
1582 compressionlevel = int(compressionlevel);
1583 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]));
1584 if(compression=="auto" or compression is None):
1585 catfp = fp;
1586 catfp.seek(0, 0);
1587 return catfp;
1588
1589 create_alias_function("Compress", __file_format_name__, "", CompressArchiveFile);
1590
1591 def CompressOpenFile(outfile, compressionlevel=None):
1592 if(outfile is None):
1593 return False;
1594 fbasename = os.path.splitext(outfile)[0];
1595 fextname = os.path.splitext(outfile)[1];
1596 if(compressionlevel is None and fextname!=".zst"):
1597 compressionlevel = 9;
1598 elif(compressionlevel is None and fextname==".zst"):
1599 compressionlevel = 10;
1600 else:
1601 compressionlevel = int(compressionlevel);
1602 if(sys.version_info[0]==2):
1603 mode = "w";
1604 else:
1605 mode = "wb";
1606 try:
1607 if(fextname not in outextlistwd):
1608 try:
1609 outfp = open(outfile, "wb", encoding="UTF-8");
1610 except (ValueError, TypeError) as e:
1611 outfp = open(outfile, "wb");
1612 elif(fextname==".gz"):
1613 try:
1614 import gzip;
1615 except ImportError:
1616 return False;
1617 try:
1618 outfp = gzip.open(outfile, mode, compressionlevel, encoding="UTF-8");
1619 except (ValueError, TypeError) as e:
1620 outfp = gzip.open(outfile, mode, compressionlevel);
1621 elif(fextname==".bz2"):
1622 try:
1623 import bz2;
1624 except ImportError:
1625 return False;
1626 try:
1627 outfp = bz2.open(outfile, mode, compressionlevel, encoding="UTF-8");
1628 except (ValueError, TypeError) as e:
1629 outfp = bz2.open(outfile, mode, compressionlevel);
1630 elif(fextname==".zst"):
1631 try:
1632 import zstandard;
1633 except ImportError:
1634 return False;
1635 try:
1636 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel), encoding="UTF-8");
1637 except (ValueError, TypeError) as e:
1638 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel));
1639 elif(fextname==".xz"):
1640 try:
1641 import lzma;
1642 except ImportError:
1643 return False;
1644 try:
1645 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}], encoding="UTF-8");
1646 except (ValueError, TypeError) as e:
1647 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]);
1648 elif(fextname==".lz4"):
1649 try:
1650 import lz4.frame;
1651 except ImportError:
1652 return False;
1653 try:
1654 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel, encoding="UTF-8");
1655 except (ValueError, TypeError) as e:
1656 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel);
1657 elif(fextname==".lzo"):
1658 try:
1659 import lzo;
1660 except ImportError:
1661 return False;
1662 try:
1663 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel, encoding="UTF-8");
1664 except (ValueError, TypeError) as e:
1665 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel);
1666 elif(fextname==".lzma"):
1667 try:
1668 import lzma;
1669 except ImportError:
1670 return False;
1671 try:
1672 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}], encoding="UTF-8");
1673 except (ValueError, TypeError) as e:
1674 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]);
1675 except FileNotFoundError:
1676 return False;
1677 return outfp;
1678
1679 def GetDevMajorMinor(fdev):
1680 retdev = [];
1681 if(hasattr(os, "minor")):
1682 retdev.append(os.minor(fdev));
1683 else:
1684 retdev.append(0);
1685 if(hasattr(os, "major")):
1686 retdev.append(os.major(fdev));
1687 else:
1688 retdev.append(0);
1689 return retdev;
1690
1691 def CheckSumSupport(checkfor, guaranteed=True):
1692 if(guaranteed):
1693 hash_list = sorted(list(hashlib.algorithms_guaranteed));
1694 else:
1695 hash_list = sorted(list(hashlib.algorithms_available));
1696 checklistout = sorted(hash_list + ['adler32', 'crc16', 'crc16_ansi', 'crc16_ibm', 'crc16_ccitt', 'crc32', 'crc64', 'crc64_ecma', 'crc64_iso', 'none']);
1697 if(checkfor in checklistout):
1698 return True;
1699 else:
1700 return False;
1701
1702 def CheckSumSupportAlt(checkfor, guaranteed=True):
1703 if(guaranteed):
1704 hash_list = sorted(list(hashlib.algorithms_guaranteed));
1705 else:
1706 hash_list = sorted(list(hashlib.algorithms_available));
1707 checklistout = hash_list;
1708 if(checkfor in checklistout):
1709 return True;
1710 else:
1711 return False;
1712
1713 def PackArchiveFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
1714 compressionlist = ['auto', 'gzip', 'bzip2', 'zstd', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
1715 outextlist = ['gz', 'bz2', 'zst', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
1716 outextlistwd = ['.gz', '.bz2', '.zst', '.lz4', '.lzo', '.lzop', '.lzma', '.xz'];
1717 advancedlist = True;
1718 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1719 outfile = RemoveWindowsPath(outfile);
1720 checksumtype = checksumtype.lower();
1721 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
1722 checksumtype="crc32";
1723 if(checksumtype=="none"):
1724 checksumtype = "";
1725 if(not compression or compression or compression=="catfile" or compression==formatspecs[2]):
1726 compression = None;
1727 if(compression not in compressionlist and compression is None):
1728 compression = "auto";
1729 if(verbose):
1730 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
1731 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1732 if(os.path.exists(outfile)):
1733 os.unlink(outfile);
1734 if(outfile=="-"):
1735 verbose = False;
1736 catfp = BytesIO();
1737 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1738 catfp = outfile;
1739 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", outfile)):
1740 catfp = BytesIO();
1741 else:
1742 fbasename = os.path.splitext(outfile)[0];
1743 fextname = os.path.splitext(outfile)[1];
1744 catfp = CompressOpenFile(outfile, compressionlevel);
1745 catver = formatspecs[6];
1746 fileheaderver = str(int(catver.replace(".", "")));
1747 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
1748 catfp.write(fileheader.encode('UTF-8'));
1749 infilelist = [];
1750 if(infiles=="-"):
1751 for line in sys.stdin:
1752 infilelist.append(line.strip());
1753 infilelist = list(filter(None, infilelist));
1754 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
1755 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
1756 return False;
1757 with open(infiles, "r") as finfile:
1758 for line in finfile:
1759 infilelist.append(line.strip());
1760 infilelist = list(filter(None, infilelist));
1761 else:
1762 if(isinstance(infiles, (list, tuple, ))):
1763 infilelist = list(filter(None, infiles));
1764 elif(isinstance(infiles, (str, ))):
1765 infilelist = list(filter(None, [infiles]));
1766 if(advancedlist):
1767 GetDirList = ListDirAdvanced(infilelist, followlink, False);
1768 else:
1769 GetDirList = ListDir(infilelist, followlink, False);
1770 if(not GetDirList):
1771 return False;
1772 curinode = 0;
1773 curfid = 0;
1774 inodelist = [];
1775 inodetofile = {};
1776 filetoinode = {};
1777 inodetocatinode = {};
1778 fnumfiles = format(int(len(GetDirList)), 'x').lower();
1779 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs[5]);
1780 if(checksumtype=="none" or checksumtype==""):
1781 catfileheadercshex = format(0, 'x').lower();
1782 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
1783 catfileheadercshex = format(crc16(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffff, '04x').lower();
1784 elif(checksumtype=="crc16_ccitt"):
1785 catfileheadercshex = format(crc16_ccitt(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffff, '04x').lower();
1786 elif(checksumtype=="adler32"):
1787 catfileheadercshex = format(zlib.adler32(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffff, '08x').lower();
1788 elif(checksumtype=="crc32"):
1789 catfileheadercshex = format(crc32(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffff, '08x').lower();
1790 elif(checksumtype=="crc64_ecma"):
1791 catfileheadercshex = format(crc64_ecma(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
1792 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
1793 catfileheadercshex = format(crc64_iso(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
1794 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
1795 checksumoutstr = hashlib.new(checksumtype);
1796 checksumoutstr.update(str(fileheader + fnumfilesa).encode('UTF-8'));
1797 catfileheadercshex = checksumoutstr.hexdigest().lower();
1798 else:
1799 catfileheadercshex = format(0, 'x').lower
1800 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs[5]);
1801 catfp.write(fnumfilesa.encode('UTF-8'));
1802 try:
1803 catfp.flush();
1804 os.fsync(catfp.fileno());
1805 except io.UnsupportedOperation:
1806 pass;
1807 except AttributeError:
1808 pass;
1809 for curfname in GetDirList:
1810 catfhstart = catfp.tell();
1811 if(re.findall("^[.|/]", curfname)):
1812 fname = curfname;
1813 else:
1814 fname = "./"+curfname;
1815 if(verbose):
1816 VerbosePrintOut(fname);
1817 if(not followlink or followlink is None):
1818 fstatinfo = os.lstat(fname);
1819 else:
1820 fstatinfo = os.stat(fname);
1821 fpremode = fstatinfo.st_mode;
1822 finode = fstatinfo.st_ino;
1823 flinkcount = fstatinfo.st_nlink;
1824 ftype = 0;
1825 if(stat.S_ISREG(fpremode)):
1826 ftype = 0;
1827 elif(stat.S_ISLNK(fpremode)):
1828 ftype = 2;
1829 elif(stat.S_ISCHR(fpremode)):
1830 ftype = 3;
1831 elif(stat.S_ISBLK(fpremode)):
1832 ftype = 4;
1833 elif(stat.S_ISDIR(fpremode)):
1834 ftype = 5;
1835 elif(stat.S_ISFIFO(fpremode)):
1836 ftype = 6;
1837 elif(stat.S_ISSOCK(fpremode)):
1838 ftype = 8;
1839 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
1840 ftype = 9;
1841 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
1842 ftype = 10;
1843 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
1844 ftype = 111;
1845 else:
1846 ftype = 0;
1847 flinkname = "";
1848 fcurfid = format(int(curfid), 'x').lower();
1849 if(not followlink and finode!=0):
1850 if(ftype!=1):
1851 if(finode in inodelist):
1852 ftype = 1;
1853 flinkname = inodetofile[finode];
1854 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
1855 if(finode not in inodelist):
1856 inodelist.append(finode);
1857 inodetofile.update({finode: fname});
1858 inodetocatinode.update({finode: curinode});
1859 fcurinode = format(int(curinode), 'x').lower();
1860 curinode = curinode + 1;
1861 else:
1862 fcurinode = format(int(curinode), 'x').lower();
1863 curinode = curinode + 1;
1864 curfid = curfid + 1;
1865 if(ftype==2):
1866 flinkname = os.readlink(fname);
1867 fdev = fstatinfo.st_dev;
1868 getfdev = GetDevMajorMinor(fdev);
1869 fdev_minor = getfdev[0];
1870 fdev_major = getfdev[1];
1871 frdev = fstatinfo.st_dev;
1872 if(hasattr(fstatinfo, "st_rdev")):
1873 frdev = fstatinfo.st_rdev;
1874 else:
1875 frdev = fstatinfo.st_dev;
1876 getfrdev = GetDevMajorMinor(frdev);
1877 frdev_minor = getfrdev[0];
1878 frdev_major = getfrdev[1];
1879 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
1880 fsize = format(int("0"), 'x').lower();
1881 elif(ftype==0 or ftype==7):
1882 fsize = format(int(fstatinfo.st_size), 'x').lower();
1883 else:
1884 fsize = format(int(fstatinfo.st_size)).lower();
1885 fatime = format(int(fstatinfo.st_atime), 'x').lower();
1886 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
1887 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
1888 if(hasattr(fstatinfo, "st_birthtime")):
1889 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
1890 else:
1891 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
1892 fmode = format(int(fstatinfo.st_mode), 'x').lower();
1893 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
1894 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
1895 fuid = format(int(fstatinfo.st_uid), 'x').lower();
1896 fgid = format(int(fstatinfo.st_gid), 'x').lower();
1897 funame = "";
1898 try:
1899 import pwd;
1900 try:
1901 userinfo = pwd.getpwuid(fstatinfo.st_uid);
1902 funame = userinfo.pw_name;
1903 except KeyError:
1904 funame = "";
1905 except ImportError:
1906 funame = "";
1907 fgname = "";
1908 try:
1909 import grp;
1910 try:
1911 groupinfo = grp.getgrgid(fstatinfo.st_gid);
1912 fgname = groupinfo.gr_name;
1913 except KeyError:
1914 fgname = "";
1915 except ImportError:
1916 fgname = "";
1917 fdev_minor = format(int(fdev_minor), 'x').lower();
1918 fdev_major = format(int(fdev_major), 'x').lower();
1919 frdev_minor = format(int(frdev_minor), 'x').lower();
1920 frdev_major = format(int(frdev_major), 'x').lower();
1921 finode = format(int(finode), 'x').lower();
1922 flinkcount = format(int(flinkcount), 'x').lower();
1923 if(hasattr(fstatinfo, "st_file_attributes")):
1924 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
1925 else:
1926 fwinattributes = format(int(0), 'x').lower();
1927 fcontents = "".encode('UTF-8');
1928 chunk_size = 1024;
1929 if(ftype == 0 or ftype == 7):
1930 with open(fname, "rb") as fpc:
1931 while(True):
1932 chunk = fpc.read(chunk_size);
1933 if(not chunk):
1934 break
1935 fcontents += chunk;
1936 if(followlink and (ftype == 1 or ftype == 2)):
1937 flstatinfo = os.stat(flinkname);
1938 with open(flinkname, "rb") as fpc:
1939 while(True):
1940 chunk = fpc.read(chunk_size);
1941 if(not chunk):
1942 break;
1943 fcontents += chunk;
1944 ftypehex = format(ftype, 'x').lower();
1945 extrafields = format(len(extradata), 'x').lower();
1946 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
1947 if(len(extradata)>0):
1948 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
1949 extrasizelen = format(len(extrasizestr), 'x').lower();
1950 catfileoutstr = AppendNullBytes([ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, extrasizelen, extrafields], formatspecs[5]);
1951 if(len(extradata)>0):
1952 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
1953 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
1954 if(checksumtype=="none" or checksumtype==""):
1955 catfileheadercshex = format(0, 'x').lower();
1956 catfilecontentcshex = format(0, 'x').lower();
1957 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
1958 catfileheadercshex = format(crc16("".encode('UTF-8')) & 0xffff, '04x').lower();
1959 catfilecontentcshex = format(crc16(fcontents) & 0xffff, '04x').lower();
1960 elif(checksumtype=="crc16_ccitt"):
1961 catfileheadercshex = format(crc16_ccitt("".encode('UTF-8')) & 0xffff, '04x').lower();
1962 catfilecontentcshex = format(crc16_ccitt(fcontents) & 0xffff, '04x').lower();
1963 elif(checksumtype=="adler32"):
1964 catfileheadercshex = format(zlib.adler32("".encode('UTF-8')) & 0xffffffff, '08x').lower();
1965 catfilecontentcshex = format(zlib.adler32(fcontents) & 0xffffffff, '08x').lower();
1966 elif(checksumtype=="crc32"):
1967 catfileheadercshex = format(crc32("".encode('UTF-8')) & 0xffffffff, '08x').lower();
1968 catfilecontentcshex = format(crc32(fcontents) & 0xffffffff, '08x').lower();
1969 elif(checksumtype=="crc64_ecma"):
1970 catfileheadercshex = format(crc64_ecma("".encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
1971 catfilecontentcshex = format(crc64_ecma(fcontents) & 0xffffffffffffffff, '016x').lower();
1972 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
1973 catfileheadercshex = format(crc64_iso("".encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
1974 catfilecontentcshex = format(crc64_iso(fcontents) & 0xffffffffffffffff, '016x').lower();
1975 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
1976 checksumoutstr = hashlib.new(checksumtype);
1977 checksumoutstr.update("".encode('UTF-8'));
1978 catfileheadercshex = checksumoutstr.hexdigest().lower();
1979 checksumoutstr = hashlib.new(checksumtype);
1980 checksumoutstr.update(fcontents);
1981 catfilecontentcshex = checksumoutstr.hexdigest().lower();
1982 else:
1983 catfileheadercshex = format(0, 'x').lower();
1984 catfilecontentcshex = format(0, 'x').lower();
1985 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
1986 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower()
1987 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
1988 if(checksumtype=="none" or checksumtype==""):
1989 catfileheadercshex = format(0, 'x').lower();
1990 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
1991 catfileheadercshex = format(crc16(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
1992 elif(checksumtype=="crc16_ccitt"):
1993 catfileheadercshex = format(crc16_ccitt(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
1994 elif(checksumtype=="adler32"):
1995 catfileheadercshex = format(zlib.adler32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
1996 elif(checksumtype=="crc32"):
1997 catfileheadercshex = format(crc32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
1998 elif(checksumtype=="crc64_ecma"):
1999 catfileheadercshex = format(crc64_ecma(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2000 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
2001 catfileheadercshex = format(crc64_iso(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2002 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
2003 checksumoutstr = hashlib.new(checksumtype);
2004 checksumoutstr.update(catfileoutstr.encode('UTF-8'));
2005 catfileheadercshex = checksumoutstr.hexdigest().lower();
2006 else:
2007 catfileheadercshex = format(0, 'x').lower();
2008 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
2009 catfhend = (catfp.tell() - 1) + len(catfileoutstr);
2010 catfcontentstart = catfp.tell() + len(catfileoutstr);
2011 catfileoutstrecd = catfileoutstr.encode('UTF-8');
2012 nullstrecd = formatspecs[5].encode('UTF-8');
2013 catfileout = catfileoutstrecd + fcontents + nullstrecd;
2014 catfcontentend = (catfp.tell() - 1) + len(catfileout);
2015 catfp.write(catfileout);
2016 try:
2017 catfp.flush();
2018 os.fsync(catfp.fileno());
2019 except io.UnsupportedOperation:
2020 pass;
2021 except AttributeError:
2022 pass;
2023 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
2024 catfp = CompressArchiveFile(catfp, compression, formatspecs);
2025 try:
2026 catfp.flush();
2027 os.fsync(catfp.fileno());
2028 except io.UnsupportedOperation:
2029 pass;
2030 except AttributeError:
2031 pass;
2032 if(outfile=="-"):
2033 catfp.seek(0, 0);
2034 if(hasattr(sys.stdout, "buffer")):
2035 shutil.copyfileobj(catfp, sys.stdout.buffer);
2036 else:
2037 shutil.copyfileobj(catfp, sys.stdout);
2038 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", outfile)):
2039 catfp = CompressArchiveFile(catfp, compression, formatspecs);
2040 catfp.seek(0, 0);
2041 upload_file_to_internet_file(catfp, outfile);
2042 if(returnfp):
2043 catfp.seek(0, 0);
2044 return catfp;
2045 else:
2046 catfp.close();
2047 return True;
2048
2049 create_alias_function("Pack", __file_format_name__, "", PackArchiveFile);
2050
2051 if(hasattr(shutil, "register_archive_format")):
2052 def PackArchiveFileFunc(archive_name, source_dir, **kwargs):
2053 return PackArchiveFile(source_dir, archive_name, False, "auto", None, False, "crc32", [], __file_format_delimiter__, False, False);
2054 create_alias_function("Pack", __file_format_name__, "Func", PackArchiveFileFunc);
2055
2056 def PackArchiveFileFromDirList(infiles, outfile, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
2057 return PackArchiveFile(infiles, outfile, dirlistfromtxt, compression, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, returnfp);
2058
2059 def PackArchiveFileFromTarFile(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
2060 compressionlist = ['auto', 'gzip', 'bzip2', 'zstd', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
2061 outextlist = ['gz', 'bz2', 'zst', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
2062 outextlistwd = ['.gz', '.bz2', '.zst', '.lz4', '.lzo', '.lzop', '.lzma', '.xz'];
2063 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2064 outfile = RemoveWindowsPath(outfile);
2065 checksumtype = checksumtype.lower();
2066 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
2067 checksumtype="crc32";
2068 if(checksumtype=="none"):
2069 checksumtype = "";
2070 if(not compression or compression or compression=="catfile" or compression==formatspecs[2]):
2071 compression = None;
2072 if(compression not in compressionlist and compression is None):
2073 compression = "auto";
2074 if(verbose):
2075 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2076 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2077 if(os.path.exists(outfile)):
2078 os.unlink(outfile);
2079 if(outfile=="-"):
2080 verbose = False;
2081 catfp = BytesIO();
2082 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2083 catfp = outfile;
2084 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", outfile)):
2085 catfp = BytesIO();
2086 else:
2087 fbasename = os.path.splitext(outfile)[0];
2088 fextname = os.path.splitext(outfile)[1];
2089 catfp = CompressOpenFile(outfile, compressionlevel);
2090 catver = formatspecs[6];
2091 fileheaderver = str(int(catver.replace(".", "")));
2092 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
2093 catfp.write(fileheader.encode('UTF-8'));
2094 curinode = 0;
2095 curfid = 0;
2096 inodelist = [];
2097 inodetofile = {};
2098 filetoinode = {};
2099 inodetocatinode = {};
2100 if(infile=="-"):
2101 infile = BytesIO();
2102 if(hasattr(sys.stdin, "buffer")):
2103 shutil.copyfileobj(sys.stdin.buffer, infile);
2104 else:
2105 shutil.copyfileobj(sys.stdin, infile);
2106 infile.seek(0, 0);
2107 if(not infile):
2108 return False;
2109 infile.seek(0, 0);
2110 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", infile)):
2111 infile = download_file_from_internet_file(infile);
2112 infile.seek(0, 0);
2113 if(not infile):
2114 return False;
2115 infile.seek(0, 0);
2116 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
2117 return False;
2118 elif(os.path.exists(infile) and os.path.isfile(infile)):
2119 try:
2120 if(not tarfile.is_tarfile(infile)):
2121 return False;
2122 except AttributeError:
2123 if(not is_tarfile(infile)):
2124 return False;
2125 else:
2126 return False;
2127 try:
2128 if(hasattr(infile, "read") or hasattr(infile, "write")):
2129 tarfp = tarfile.open(fileobj=infile, mode="r");
2130 else:
2131 tarfp = tarfile.open(infile, "r");
2132 except FileNotFoundError:
2133 return False;
2134 fnumfiles = format(int(len(tarfp.getmembers())), 'x').lower();
2135 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs[5]);
2136 if(checksumtype=="none" or checksumtype==""):
2137 catfileheadercshex = format(0, 'x').lower();
2138 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
2139 catfileheadercshex = format(crc16(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffff, '04x').lower();
2140 elif(checksumtype=="crc16_ccitt"):
2141 catfileheadercshex = format(crc16_ccitt(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffff, '04x').lower();
2142 elif(checksumtype=="adler32"):
2143 catfileheadercshex = format(zlib.adler32(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffff, '08x').lower();
2144 elif(checksumtype=="crc32"):
2145 catfileheadercshex = format(crc32(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffff, '08x').lower();
2146 elif(checksumtype=="crc64_ecma"):
2147 catfileheadercshex = format(crc64_ecma(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2148 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
2149 catfileheadercshex = format(crc64_iso(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2150 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
2151 checksumoutstr = hashlib.new(checksumtype);
2152 checksumoutstr.update(str(fileheader + fnumfilesa).encode('UTF-8'));
2153 catfileheadercshex = checksumoutstr.hexdigest().lower();
2154 else:
2155 catfileheadercshex = format(0, 'x').lower();
2156 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs[5]);
2157 catfp.write(fnumfilesa.encode('UTF-8'));
2158 try:
2159 catfp.flush();
2160 os.fsync(catfp.fileno());
2161 except io.UnsupportedOperation:
2162 pass;
2163 except AttributeError:
2164 pass;
2165 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
2166 catfhstart = catfp.tell();
2167 if(re.findall("^[.|/]", member.name)):
2168 fname = member.name;
2169 else:
2170 fname = "./"+member.name;
2171 if(verbose):
2172 VerbosePrintOut(fname);
2173 fpremode = member.mode;
2174 ffullmode = member.mode;
2175 flinkcount = 0;
2176 ftype = 0;
2177 if(member.isreg()):
2178 ffullmode = member.mode + stat.S_IFREG;
2179 ftype = 0;
2180 elif(member.isdev()):
2181 ffullmode = member.mode;
2182 ftype = 7;
2183 elif(member.islnk()):
2184 ffullmode = member.mode + stat.S_IFREG;
2185 ftype = 1;
2186 elif(member.issym()):
2187 ffullmode = member.mode + stat.S_IFLNK;
2188 ftype = 2;
2189 elif(member.ischr()):
2190 ffullmode = member.mode + stat.S_IFCHR;
2191 ftype = 3;
2192 elif(member.isblk()):
2193 ffullmode = member.mode + stat.S_IFBLK;
2194 ftype = 4;
2195 elif(member.isdir()):
2196 ffullmode = member.mode + stat.S_IFDIR;
2197 ftype = 5;
2198 elif(member.isfifo()):
2199 ffullmode = member.mode + stat.S_IFIFO;
2200 ftype = 6;
2201 elif(member.issparse()):
2202 ffullmode = member.mode;
2203 ftype = 12;
2204 else:
2205 ffullmode = member.mode;
2206 ftype = 0;
2207 flinkname = "";
2208 fcurfid = format(int(curfid), 'x').lower();
2209 fcurinode = format(int(0), 'x').lower();
2210 curfid = curfid + 1;
2211 if(ftype==2):
2212 flinkname = member.linkname;
2213 fdev_minor = format(int(member.devminor), 'x').lower();
2214 fdev_major = format(int(member.devmajor), 'x').lower();
2215 frdev_minor = format(int(member.devminor), 'x').lower();
2216 frdev_major = format(int(member.devmajor), 'x').lower();
2217 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
2218 fsize = format(int("0"), 'x').lower();
2219 elif(ftype==0 or ftype==7):
2220 fsize = format(int(member.size), 'x').lower();
2221 else:
2222 fsize = format(int(member.size), 'x').lower();
2223 fatime = format(int(member.mtime), 'x').lower();
2224 fmtime = format(int(member.mtime), 'x').lower();
2225 fctime = format(int(member.mtime), 'x').lower();
2226 fbtime = format(int(member.mtime), 'x').lower();
2227 fmode = format(int(ffullmode), 'x').lower();
2228 fchmode = format(int(stat.S_IMODE(ffullmode)), 'x').lower();
2229 ftypemod = format(int(stat.S_IFMT(ffullmode)), 'x').lower();
2230 fuid = format(int(member.uid), 'x').lower();
2231 fgid = format(int(member.gid), 'x').lower();
2232 funame = member.uname;
2233 fgname = member.gname;
2234 flinkcount = format(int(flinkcount), 'x').lower();
2235 fwinattributes = format(int(0), 'x').lower();
2236 fcontents = "".encode('UTF-8');
2237 chunk_size = 1024;
2238 if(ftype == 0 or ftype == 7):
2239 with tarfp.extractfile(member) as fpc:
2240 while(True):
2241 chunk = fpc.read(chunk_size);
2242 if(not chunk):
2243 break
2244 fcontents += chunk;
2245 ftypehex = format(ftype, 'x').lower();
2246 extrafields = format(len(extradata), 'x').lower();
2247 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
2248 if(len(extradata)>0):
2249 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
2250 extrasizelen = format(len(extrasizestr), 'x').lower();
2251 catfileoutstr = AppendNullBytes([ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, extrasizelen, extrafields], formatspecs[5]);
2252 if(len(extradata)>0):
2253 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
2254 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
2255 catfhend = (catfp.tell() - 1) + len(catfileoutstr);
2256 catfcontentstart = catfp.tell() + len(catfileoutstr);
2257 if(checksumtype=="none" or checksumtype==""):
2258 catfileheadercshex = format(0, 'x').lower();
2259 catfilecontentcshex = format(0, 'x').lower();
2260 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
2261 catfileheadercshex = format(crc16("".encode('UTF-8')) & 0xffff, '04x').lower();
2262 catfilecontentcshex = format(crc16(fcontents) & 0xffff, '04x').lower();
2263 elif(checksumtype=="crc16_ccitt"):
2264 catfileheadercshex = format(crc16_ccitt("".encode('UTF-8')) & 0xffff, '04x').lower();
2265 catfilecontentcshex = format(crc16_ccitt(fcontents) & 0xffff, '04x').lower();
2266 elif(checksumtype=="adler32"):
2267 catfileheadercshex = format(zlib.adler32("".encode('UTF-8')) & 0xffffffff, '08x').lower();
2268 catfilecontentcshex = format(zlib.adler32(fcontents) & 0xffffffff, '08x').lower();
2269 elif(checksumtype=="crc32"):
2270 catfileheadercshex = format(crc32("".encode('UTF-8')) & 0xffffffff, '08x').lower();
2271 catfilecontentcshex = format(crc32(fcontents) & 0xffffffff, '08x').lower();
2272 elif(checksumtype=="crc64_ecma"):
2273 catfileheadercshex = format(crc64_ecma("".encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2274 catfilecontentcshex = format(crc64_ecma(fcontents) & 0xffffffffffffffff, '016x').lower();
2275 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
2276 catfileheadercshex = format(crc64_iso("".encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2277 catfilecontentcshex = format(crc64_iso(fcontents) & 0xffffffffffffffff, '016x').lower();
2278 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
2279 checksumoutstr = hashlib.new(checksumtype);
2280 checksumoutstr.update("".encode('UTF-8'));
2281 catfileheadercshex = checksumoutstr.hexdigest().lower();
2282 checksumoutstr = hashlib.new(checksumtype);
2283 checksumoutstr.update(fcontents);
2284 catfilecontentcshex = checksumoutstr.hexdigest().lower();
2285 else:
2286 catfileheadercshex = format(0, 'x').lower();
2287 catfilecontentcshex = format(0, 'x').lower();
2288 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
2289 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower()
2290 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
2291 if(checksumtype=="none" or checksumtype==""):
2292 catfileheadercshex = format(0, 'x').lower();
2293 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
2294 catfileheadercshex = format(crc16(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
2295 elif(checksumtype=="crc16_ccitt"):
2296 catfileheadercshex = format(crc16_ccitt(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
2297 elif(checksumtype=="adler32"):
2298 catfileheadercshex = format(zlib.adler32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
2299 elif(checksumtype=="crc32"):
2300 catfileheadercshex = format(crc32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
2301 elif(checksumtype=="crc64_ecma"):
2302 catfileheadercshex = format(crc64_ecma(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2303 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
2304 catfileheadercshex = format(crc64_iso(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2305 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
2306 checksumoutstr = hashlib.new(checksumtype);
2307 checksumoutstr.update(catfileoutstr.encode('UTF-8'));
2308 catfileheadercshex = checksumoutstr.hexdigest().lower();
2309 else:
2310 catfileheadercshex = format(0, 'x').lower();
2311 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
2312 catfileoutstrecd = catfileoutstr.encode('UTF-8');
2313 nullstrecd = formatspecs[5].encode('UTF-8');
2314 catfileout = catfileoutstrecd + fcontents + nullstrecd;
2315 catfcontentend = (catfp.tell() - 1) + len(catfileout);
2316 catfp.write(catfileout);
2317 try:
2318 catfp.flush();
2319 os.fsync(catfp.fileno());
2320 except io.UnsupportedOperation:
2321 pass;
2322 except AttributeError:
2323 pass;
2324 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
2325 catfp = CompressArchiveFile(catfp, compression, formatspecs);
2326 try:
2327 catfp.flush();
2328 os.fsync(catfp.fileno());
2329 except io.UnsupportedOperation:
2330 pass;
2331 except AttributeError:
2332 pass;
2333 if(outfile=="-"):
2334 catfp.seek(0, 0);
2335 if(hasattr(sys.stdout, "buffer")):
2336 shutil.copyfileobj(catfp, sys.stdout.buffer);
2337 else:
2338 shutil.copyfileobj(catfp, sys.stdout);
2339 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", outfile)):
2340 catfp = CompressArchiveFile(catfp, compression, formatspecs);
2341 catfp.seek(0, 0);
2342 upload_file_from_internet_file(catfp, outfile);
2343 if(returnfp):
2344 catfp.seek(0, 0);
2345 return catfp;
2346 else:
2347 catfp.close();
2348 return True;
2349
2350 create_alias_function("Pack", __file_format_name__, "FromTarFile", PackArchiveFileFromTarFile);
2351
2352 def PackArchiveFileFromZipFile(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
2353 compressionlist = ['auto', 'gzip', 'bzip2', 'zstd', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
2354 outextlist = ['gz', 'bz2', 'zst', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
2355 outextlistwd = ['.gz', '.bz2', '.zst', '.lz4', '.lzo', '.lzop', '.lzma', '.xz'];
2356 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2357 outfile = RemoveWindowsPath(outfile);
2358 checksumtype = checksumtype.lower();
2359 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
2360 checksumtype="crc32";
2361 if(checksumtype=="none"):
2362 checksumtype = "";
2363 if(not compression or compression or compression=="catfile" or compression==formatspecs[2]):
2364 compression = None;
2365 if(compression not in compressionlist and compression is None):
2366 compression = "auto";
2367 if(verbose):
2368 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2369 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2370 if(os.path.exists(outfile)):
2371 os.unlink(outfile);
2372 if(outfile=="-"):
2373 verbose = False;
2374 catfp = BytesIO();
2375 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2376 catfp = outfile;
2377 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", outfile)):
2378 catfp = BytesIO();
2379 else:
2380 fbasename = os.path.splitext(outfile)[0];
2381 fextname = os.path.splitext(outfile)[1];
2382 catfp = CompressOpenFile(outfile, compressionlevel);
2383 catver = formatspecs[6];
2384 fileheaderver = str(int(catver.replace(".", "")));
2385 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
2386 catfp.write(fileheader.encode('UTF-8'));
2387 curinode = 0;
2388 curfid = 0;
2389 inodelist = [];
2390 inodetofile = {};
2391 filetoinode = {};
2392 inodetocatinode = {};
2393 if(infile=="-"):
2394 infile = BytesIO();
2395 if(hasattr(sys.stdin, "buffer")):
2396 shutil.copyfileobj(sys.stdin.buffer, infile);
2397 else:
2398 shutil.copyfileobj(sys.stdin, infile);
2399 infile.seek(0, 0);
2400 if(not infile):
2401 return False;
2402 infile.seek(0, 0);
2403 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", infile)):
2404 infile = download_file_from_internet_file(infile);
2405 infile.seek(0, 0);
2406 if(not infile):
2407 return False;
2408 infile.seek(0, 0);
2409 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
2410 return False;
2411 else:
2412 return False;
2413 if(not zipfile.is_zipfile(infile)):
2414 return False;
2415 try:
2416 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
2417 except FileNotFoundError:
2418 return False;
2419 ziptest = zipfp.testzip();
2420 if(ziptest):
2421 VerbosePrintOut("Bad file found!");
2422 fnumfiles = format(int(len(zipfp.infolist())), 'x').lower();
2423 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs[5]);
2424 if(checksumtype=="none" or checksumtype==""):
2425 catfileheadercshex = format(0, 'x').lower();
2426 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
2427 catfileheadercshex = format(crc16(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffff, '04x').lower();
2428 elif(checksumtype=="crc16_ccitt"):
2429 catfileheadercshex = format(crc16_ccitt(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffff, '04x').lower();
2430 elif(checksumtype=="adler32"):
2431 catfileheadercshex = format(zlib.adler32(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffff, '08x').lower();
2432 elif(checksumtype=="crc32"):
2433 catfileheadercshex = format(crc32(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffff, '08x').lower();
2434 elif(checksumtype=="crc64_ecma"):
2435 catfileheadercshex = format(crc64_ecma(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2436 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
2437 catfileheadercshex = format(crc64_iso(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2438 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
2439 checksumoutstr = hashlib.new(checksumtype);
2440 checksumoutstr.update(str(fileheader + fnumfilesa).encode('UTF-8'));
2441 catfileheadercshex = checksumoutstr.hexdigest().lower();
2442 else:
2443 catfileheadercshex = format(0, 'x').lower();
2444 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs[5]);
2445 catfp.write(fnumfilesa.encode('UTF-8'));
2446 try:
2447 catfp.flush();
2448 os.fsync(catfp.fileno());
2449 except io.UnsupportedOperation:
2450 pass;
2451 except AttributeError:
2452 pass;
2453 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
2454 catfhstart = catfp.tell();
2455 if(re.findall("^[.|/]", member.filename)):
2456 fname = member.filename;
2457 else:
2458 fname = "./"+member.filename;
2459 zipinfo = zipfp.getinfo(member.filename);
2460 if(verbose):
2461 VerbosePrintOut(fname);
2462 if(not member.is_dir()):
2463 fpremode = int(stat.S_IFREG + 438);
2464 elif(member.is_dir()):
2465 fpremode = int(stat.S_IFDIR + 511);
2466 flinkcount = 0;
2467 ftype = 0;
2468 if(not member.is_dir()):
2469 ftype = 0;
2470 elif(member.is_dir()):
2471 ftype = 5;
2472 flinkname = "";
2473 fcurfid = format(int(curfid), 'x').lower();
2474 fcurinode = format(int(0), 'x').lower();
2475 curfid = curfid + 1;
2476 fdev_minor = format(int(0), 'x').lower();
2477 fdev_major = format(int(0), 'x').lower();
2478 frdev_minor = format(int(0), 'x').lower();
2479 frdev_major = format(int(0), 'x').lower();
2480 if(ftype==5):
2481 fsize = format(int("0"), 'x').lower();
2482 elif(ftype==0):
2483 fsize = format(int(member.file_size), 'x').lower();
2484 else:
2485 fsize = format(int(member.file_size), 'x').lower();
2486 fatime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
2487 fmtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
2488 fctime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
2489 fbtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
2490 if(zipinfo.create_system==0 or zipinfo.create_system==10):
2491 fwinattributes = format(int(zipinfo.external_attr), 'x').lower();
2492 if(not member.is_dir()):
2493 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
2494 fchmode = stat.S_IMODE(fmode);
2495 ftypemod = stat.S_IFMT(fmode);
2496 if(member.is_dir()):
2497 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
2498 fchmode = stat.S_IMODE(fmode);
2499 ftypemod = stat.S_IFMT(fmode);
2500 elif(zipinfo.create_system==3):
2501 fwinattributes = format(int(0), 'x').lower();
2502 fmode = format(int(zipinfo.external_attr), 'x').lower();
2503 fchmode = stat.S_IMODE(fmode);
2504 ftypemod = stat.S_IFMT(fmode);
2505 else:
2506 fwinattributes = format(int(0), 'x').lower();
2507 if(not member.is_dir()):
2508 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
2509 fchmode = stat.S_IMODE(fmode);
2510 ftypemod = stat.S_IFMT(fmode);
2511 if(member.is_dir()):
2512 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
2513 fchmode = stat.S_IMODE(fmode);
2514 ftypemod = stat.S_IFMT(fmode);
2515 try:
2516 fuid = format(int(os.getuid()), 'x').lower();
2517 except AttributeError:
2518 fuid = format(int(0), 'x').lower();
2519 except KeyError:
2520 fuid = format(int(0), 'x').lower();
2521 try:
2522 fgid = format(int(os.getgid()), 'x').lower();
2523 except AttributeError:
2524 fgid = format(int(0), 'x').lower();
2525 except KeyError:
2526 fgid = format(int(0), 'x').lower();
2527 try:
2528 import pwd;
2529 try:
2530 userinfo = pwd.getpwuid(os.getuid());
2531 funame = userinfo.pw_name;
2532 except KeyError:
2533 funame = "";
2534 except AttributeError:
2535 funame = "";
2536 except ImportError:
2537 funame = "";
2538 fgname = "";
2539 try:
2540 import grp;
2541 try:
2542 groupinfo = grp.getgrgid(os.getgid());
2543 fgname = groupinfo.gr_name;
2544 except KeyError:
2545 fgname = "";
2546 except AttributeError:
2547 fgname = "";
2548 except ImportError:
2549 fgname = "";
2550 fcontents = "".encode('UTF-8');
2551 if(ftype==0):
2552 fcontents = zipfp.read(member.filename);
2553 ftypehex = format(ftype, 'x').lower();
2554 extrafields = format(len(extradata), 'x').lower();
2555 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
2556 if(len(extradata)>0):
2557 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
2558 extrasizelen = format(len(extrasizestr), 'x').lower();
2559 catfileoutstr = AppendNullBytes([ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, extrasizelen, extrafields], formatspecs[5]);
2560 if(len(extradata)>0):
2561 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
2562 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
2563 catfhend = (catfp.tell() - 1) + len(catfileoutstr);
2564 catfcontentstart = catfp.tell() + len(catfileoutstr);
2565 if(checksumtype=="none" or checksumtype==""):
2566 catfileheadercshex = format(0, 'x').lower();
2567 catfilecontentcshex = format(0, 'x').lower();
2568 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
2569 catfileheadercshex = format(crc16("".encode('UTF-8')) & 0xffff, '04x').lower();
2570 catfilecontentcshex = format(crc16(fcontents) & 0xffff, '04x').lower();
2571 elif(checksumtype=="crc16_ccitt"):
2572 catfileheadercshex = format(crc16_ccitt("".encode('UTF-8')) & 0xffff, '04x').lower();
2573 catfilecontentcshex = format(crc16_ccitt(fcontents) & 0xffff, '04x').lower();
2574 elif(checksumtype=="adler32"):
2575 catfileheadercshex = format(zlib.adler32("".encode('UTF-8')) & 0xffffffff, '08x').lower();
2576 catfilecontentcshex = format(zlib.adler32(fcontents) & 0xffffffff, '08x').lower();
2577 elif(checksumtype=="crc32"):
2578 catfileheadercshex = format(crc32("".encode('UTF-8')) & 0xffffffff, '08x').lower();
2579 catfilecontentcshex = format(crc32(fcontents) & 0xffffffff, '08x').lower();
2580 elif(checksumtype=="crc64_ecma"):
2581 catfileheadercshex = format(crc64_ecma("".encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2582 catfilecontentcshex = format(crc64_ecma(fcontents) & 0xffffffffffffffff, '016x').lower();
2583 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
2584 catfileheadercshex = format(crc64_iso("".encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2585 catfilecontentcshex = format(crc64_iso(fcontents) & 0xffffffffffffffff, '016x').lower();
2586 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
2587 checksumoutstr = hashlib.new(checksumtype);
2588 checksumoutstr.update("".encode('UTF-8'));
2589 catfileheadercshex = checksumoutstr.hexdigest().lower();
2590 checksumoutstr = hashlib.new(checksumtype);
2591 checksumoutstr.update(fcontents);
2592 catfilecontentcshex = checksumoutstr.hexdigest().lower();
2593 else:
2594 catfileheadercshex = format(0, 'x').lower();
2595 catfilecontentcshex = format(0, 'x').lower();
2596 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
2597 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower()
2598 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
2599 if(checksumtype=="none" or checksumtype==""):
2600 catfileheadercshex = format(0, 'x').lower();
2601 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
2602 catfileheadercshex = format(crc16(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
2603 elif(checksumtype=="crc16_ccitt"):
2604 catfileheadercshex = format(crc16_ccitt(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
2605 elif(checksumtype=="adler32"):
2606 catfileheadercshex = format(zlib.adler32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
2607 elif(checksumtype=="crc32"):
2608 catfileheadercshex = format(crc32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
2609 elif(checksumtype=="crc64_ecma"):
2610 catfileheadercshex = format(crc64_ecma(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2611 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
2612 catfileheadercshex = format(crc64_iso(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2613 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
2614 checksumoutstr = hashlib.new(checksumtype);
2615 checksumoutstr.update(catfileoutstr.encode('UTF-8'));
2616 catfileheadercshex = checksumoutstr.hexdigest().lower();
2617 else:
2618 catfileheadercshex = format(0, 'x').lower();
2619 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
2620 catfileoutstrecd = catfileoutstr.encode('UTF-8');
2621 nullstrecd = formatspecs[5].encode('UTF-8');
2622 catfileout = catfileoutstrecd + fcontents + nullstrecd;
2623 catfcontentend = (catfp.tell() - 1) + len(catfileout);
2624 catfp.write(catfileout);
2625 try:
2626 catfp.flush();
2627 os.fsync(catfp.fileno());
2628 except io.UnsupportedOperation:
2629 pass;
2630 except AttributeError:
2631 pass;
2632 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
2633 catfp = CompressArchiveFile(catfp, compression, formatspecs);
2634 try:
2635 catfp.flush();
2636 os.fsync(catfp.fileno());
2637 except io.UnsupportedOperation:
2638 pass;
2639 except AttributeError:
2640 pass;
2641 if(outfile=="-"):
2642 catfp.seek(0, 0);
2643 if(hasattr(sys.stdout, "buffer")):
2644 shutil.copyfileobj(catfp, sys.stdout.buffer);
2645 else:
2646 shutil.copyfileobj(catfp, sys.stdout);
2647 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", outfile)):
2648 catfp = CompressArchiveFile(catfp, compression, formatspecs);
2649 catfp.seek(0, 0);
2650 upload_file_to_internet_file(catfp, outfile);
2651 if(returnfp):
2652 catfp.seek(0, 0);
2653 return catfp;
2654 else:
2655 catfp.close();
2656 return True;
2657
2658 create_alias_function("Pack", __file_format_name__, "FromZipFile", PackArchiveFileFromZipFile);
2659
2660 if(not rarfile_support):
2661 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
2662 return False
2663
2664 if(rarfile_support):
2665 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
2666 compressionlist = ['auto', 'gzip', 'bzip2', 'zstd', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
2667 outextlist = ['gz', 'bz2', 'zst', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
2668 outextlistwd = ['.gz', '.bz2', '.zst', '.lz4', '.lzo', '.lzop', '.lzma', '.xz'];
2669 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2670 outfile = RemoveWindowsPath(outfile);
2671 checksumtype = checksumtype.lower();
2672 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
2673 checksumtype="crc32";
2674 if(checksumtype=="none"):
2675 checksumtype = "";
2676 if(not compression or compression or compression=="catfile" or compression==formatspecs[2]):
2677 compression = None;
2678 if(compression not in compressionlist and compression is None):
2679 compression = "auto";
2680 if(verbose):
2681 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2682 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2683 if(os.path.exists(outfile)):
2684 os.unlink(outfile);
2685 if(outfile=="-"):
2686 verbose = False;
2687 catfp = BytesIO();
2688 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2689 catfp = outfile;
2690 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", outfile)):
2691 catfp = BytesIO();
2692 else:
2693 fbasename = os.path.splitext(outfile)[0];
2694 fextname = os.path.splitext(outfile)[1];
2695 catfp = CompressOpenFile(outfile, compressionlevel);
2696 catver = formatspecs[6];
2697 fileheaderver = str(int(catver.replace(".", "")));
2698 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
2699 catfp.write(fileheader.encode('UTF-8'));
2700 curinode = 0;
2701 curfid = 0;
2702 inodelist = [];
2703 inodetofile = {};
2704 filetoinode = {};
2705 inodetocatinode = {};
2706 if(not os.path.exists(infile) or not os.path.isfile(infile)):
2707 return False;
2708 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
2709 return False;
2710 rarfp = rarfile.RarFile(infile, "r");
2711 rartest = rarfp.testrar();
2712 if(rartest):
2713 VerbosePrintOut("Bad file found!");
2714 fnumfiles = format(int(len(rarfp.infolist())), 'x').lower();
2715 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs[5]);
2716 if(checksumtype=="none" or checksumtype==""):
2717 catfileheadercshex = format(0, 'x').lower();
2718 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
2719 catfileheadercshex = format(crc16(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffff, '04x').lower();
2720 elif(checksumtype=="crc16_ccitt"):
2721 catfileheadercshex = format(crc16_ccitt(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffff, '04x').lower();
2722 elif(checksumtype=="adler32"):
2723 catfileheadercshex = format(zlib.adler32(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffff, '08x').lower();
2724 elif(checksumtype=="crc32"):
2725 catfileheadercshex = format(crc32(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffff, '08x').lower();
2726 elif(checksumtype=="crc64_ecma"):
2727 catfileheadercshex = format(crc64_ecma(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2728 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
2729 catfileheadercshex = format(crc64_iso(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2730 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
2731 checksumoutstr = hashlib.new(checksumtype);
2732 checksumoutstr.update(str(fileheader + fnumfilesa).encode('UTF-8'));
2733 catfileheadercshex = checksumoutstr.hexdigest().lower();
2734 else:
2735 catfileheadercshex = format(0, 'x').lower();
2736 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs[5]);
2737 catfp.write(fnumfilesa.encode('UTF-8'));
2738 try:
2739 catfp.flush();
2740 os.fsync(catfp.fileno());
2741 except io.UnsupportedOperation:
2742 pass;
2743 except AttributeError:
2744 pass;
2745 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
2746 is_unix = False;
2747 is_windows = False;
2748 if(member.host_os == rarfile.RAR_OS_UNIX):
2749 is_windows = False;
2750 try:
2751 member.external_attr
2752 is_unix = True;
2753 except AttributeError:
2754 is_unix = False;
2755 elif(member.host_os == rarfile.RAR_OS_WIN32):
2756 is_unix = False;
2757 try:
2758 member.external_attr
2759 is_windows = True;
2760 except AttributeError:
2761 is_windows = False;
2762 else:
2763 is_unix = False;
2764 is_windows = False;
2765 catfhstart = catfp.tell();
2766 if(re.findall("^[.|/]", member.filename)):
2767 fname = member.filename;
2768 else:
2769 fname = "./"+member.filename;
2770 rarinfo = rarfp.getinfo(member.filename);
2771 if(verbose):
2772 VerbosePrintOut(fname);
2773 if(is_unix and member.external_attr !=0):
2774 fpremode = int(member.external_attr);
2775 elif(member.is_file()):
2776 fpremode = int(stat.S_IFREG + 438);
2777 elif(member.is_symlink()):
2778 fpremode = int(stat.S_IFLNK + 438);
2779 elif(member.is_dir()):
2780 fpremode = int(stat.S_IFDIR + 511);
2781 if(is_windows and member.external_attr !=0):
2782 fwinattributes = format(int(member.external_attr), 'x').lower();
2783 else:
2784 fwinattributes = format(int(0), 'x').lower();
2785 flinkcount = 0;
2786 ftype = 0;
2787 if(member.is_file()):
2788 ftype = 0;
2789 elif(member.is_symlink()):
2790 ftype = 2;
2791 elif(member.is_dir()):
2792 ftype = 5;
2793 flinkname = "";
2794 if(ftype==2):
2795 flinkname = rarfp.read(member.filename).decode("UTF-8");
2796 fcurfid = format(int(curfid), 'x').lower();
2797 fcurinode = format(int(0), 'x').lower();
2798 curfid = curfid + 1;
2799 fdev_minor = format(int(0), 'x').lower();
2800 fdev_major = format(int(0), 'x').lower();
2801 frdev_minor = format(int(0), 'x').lower();
2802 frdev_major = format(int(0), 'x').lower();
2803 if(ftype==5):
2804 fsize = format(int("0"), 'x').lower();
2805 elif(ftype==0):
2806 fsize = format(int(member.file_size), 'x').lower();
2807 else:
2808 fsize = format(int(member.file_size), 'x').lower();
2809 try:
2810 if(member.atime):
2811 fatime = format(int(member.atime.timestamp()), 'x').lower();
2812 else:
2813 fatime = format(int(member.mtime.timestamp()), 'x').lower();
2814 except AttributeError:
2815 fatime = format(int(member.mtime.timestamp()), 'x').lower();
2816 fmtime = format(int(member.mtime.timestamp()), 'x').lower();
2817 try:
2818 if(member.ctime):
2819 fctime = format(int(member.ctime.timestamp()), 'x').lower();
2820 else:
2821 fctime = format(int(member.mtime.timestamp()), 'x').lower();
2822 except AttributeError:
2823 fctime = format(int(member.mtime.timestamp()), 'x').lower();
2824 fbtime = format(int(member.mtime.timestamp()), 'x').lower();
2825 if(is_unix and member.external_attr !=0):
2826 fmode = format(int(member.external_attr), 'x').lower();
2827 fchmode = format(int(stat.S_IMODE(member.external_attr)), 'x').lower();
2828 ftypemod = format(int(stat.S_IFMT(member.external_attr)), 'x').lower();
2829 elif(member.is_file()):
2830 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
2831 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
2832 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
2833 elif(member.is_symlink()):
2834 fmode = format(int(stat.S_IFLNK + 438), 'x').lower();
2835 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
2836 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
2837 elif(member.is_dir()):
2838 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
2839 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
2840 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
2841 try:
2842 fuid = format(int(os.getuid()), 'x').lower();
2843 except AttributeError:
2844 fuid = format(int(0), 'x').lower();
2845 except KeyError:
2846 fuid = format(int(0), 'x').lower();
2847 try:
2848 fgid = format(int(os.getgid()), 'x').lower();
2849 except AttributeError:
2850 fgid = format(int(0), 'x').lower();
2851 except KeyError:
2852 fgid = format(int(0), 'x').lower();
2853 try:
2854 import pwd;
2855 try:
2856 userinfo = pwd.getpwuid(os.getuid());
2857 funame = userinfo.pw_name;
2858 except KeyError:
2859 funame = "";
2860 except AttributeError:
2861 funame = "";
2862 except ImportError:
2863 funame = "";
2864 fgname = "";
2865 try:
2866 import grp;
2867 try:
2868 groupinfo = grp.getgrgid(os.getgid());
2869 fgname = groupinfo.gr_name;
2870 except KeyError:
2871 fgname = "";
2872 except AttributeError:
2873 fgname = "";
2874 except ImportError:
2875 fgname = "";
2876 fcontents = "".encode('UTF-8');
2877 if(ftype==0):
2878 fcontents = rarfp.read(member.filename);
2879 ftypehex = format(ftype, 'x').lower();
2880 extrafields = format(len(extradata), 'x').lower();
2881 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
2882 if(len(extradata)>0):
2883 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
2884 extrasizelen = format(len(extrasizestr), 'x').lower();
2885 catfileoutstr = AppendNullBytes([ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, extrasizelen, extrafields], formatspecs[5]);
2886 if(len(extradata)>0):
2887 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
2888 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
2889 catfhend = (catfp.tell() - 1) + len(catfileoutstr);
2890 catfcontentstart = catfp.tell() + len(catfileoutstr);
2891 if(checksumtype=="none" or checksumtype==""):
2892 catfileheadercshex = format(0, 'x').lower();
2893 catfilecontentcshex = format(0, 'x').lower();
2894 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
2895 catfileheadercshex = format(crc16("".encode('UTF-8')) & 0xffff, '04x').lower();
2896 catfilecontentcshex = format(crc16(fcontents) & 0xffff, '04x').lower();
2897 elif(checksumtype=="crc16_ccitt"):
2898 catfileheadercshex = format(crc16_ccitt("".encode('UTF-8')) & 0xffff, '04x').lower();
2899 catfilecontentcshex = format(crc16_ccitt(fcontents) & 0xffff, '04x').lower();
2900 elif(checksumtype=="adler32"):
2901 catfileheadercshex = format(zlib.adler32("".encode('UTF-8')) & 0xffffffff, '08x').lower();
2902 catfilecontentcshex = format(zlib.adler32(fcontents) & 0xffffffff, '08x').lower();
2903 elif(checksumtype=="crc32"):
2904 catfileheadercshex = format(crc32("".encode('UTF-8')) & 0xffffffff, '08x').lower();
2905 catfilecontentcshex = format(crc32(fcontents) & 0xffffffff, '08x').lower();
2906 elif(checksumtype=="crc64_ecma"):
2907 catfileheadercshex = format(crc64_ecma("".encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2908 catfilecontentcshex = format(crc64_ecma(fcontents) & 0xffffffffffffffff, '016x').lower();
2909 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
2910 catfileheadercshex = format(crc64_iso("".encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2911 catfilecontentcshex = format(crc64_iso(fcontents) & 0xffffffffffffffff, '016x').lower();
2912 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
2913 checksumoutstr = hashlib.new(checksumtype);
2914 checksumoutstr.update("".encode('UTF-8'));
2915 catfileheadercshex = checksumoutstr.hexdigest().lower();
2916 checksumoutstr = hashlib.new(checksumtype);
2917 checksumoutstr.update(fcontents);
2918 catfilecontentcshex = checksumoutstr.hexdigest().lower();
2919 else:
2920 catfileheadercshex = format(0, 'x').lower();
2921 catfilecontentcshex = format(0, 'x').lower();
2922 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
2923 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower()
2924 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr
2925 if(checksumtype=="none" or checksumtype==""):
2926 catfileheadercshex = format(0, 'x').lower()
2927 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
2928 catfileheadercshex = format(crc16(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower()
2929 elif(checksumtype=="crc16_ccitt"):
2930 catfileheadercshex = format(crc16_ccitt(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower()
2931 elif(checksumtype=="adler32"):
2932 catfileheadercshex = format(zlib.adler32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower()
2933 elif(checksumtype=="crc32"):
2934 catfileheadercshex = format(crc32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower()
2935 elif(checksumtype=="crc64_ecma"):
2936 catfileheadercshex = format(crc64_ecma(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower()
2937 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
2938 catfileheadercshex = format(crc64_iso(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower()
2939 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
2940 checksumoutstr = hashlib.new(checksumtype)
2941 checksumoutstr.update(catfileoutstr.encode('UTF-8'))
2942 catfileheadercshex = checksumoutstr.hexdigest().lower()
2943 else:
2944 catfileheadercshex = format(0, 'x').lower()
2945 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5])
2946 catfileoutstrecd = catfileoutstr.encode('UTF-8')
2947 nullstrecd = formatspecs[5].encode('UTF-8')
2948 catfileout = catfileoutstrecd + fcontents + nullstrecd
2949 catfcontentend = (catfp.tell() - 1) + len(catfileout)
2950 catfp.write(catfileout)
2951 try:
2952 catfp.flush()
2953 os.fsync(catfp.fileno())
2954 except io.UnsupportedOperation:
2955 pass
2956 except AttributeError:
2957 pass
2958 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
2959 catfp = CompressArchiveFile(catfp, compression, formatspecs)
2960 try:
2961 catfp.flush()
2962 os.fsync(catfp.fileno())
2963 except io.UnsupportedOperation:
2964 pass
2965 except AttributeError:
2966 pass
2967 if(outfile=="-"):
2968 catfp.seek(0, 0)
2969 if(hasattr(sys.stdout, "buffer")):
2970 shutil.copyfileobj(catfp, sys.stdout.buffer);
2971 else:
2972 shutil.copyfileobj(catfp, sys.stdout);
2973 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", outfile)):
2974 catfp = CompressArchiveFile(catfp, compression, formatspecs);
2975 catfp.seek(0, 0);
2976 upload_file_to_internet_file(catfp, outfile);
2977 if(returnfp):
2978 catfp.seek(0, 0)
2979 return catfp
2980 else:
2981 catfp.close()
2982 return True;
2983
2984 create_alias_function("Pack", __file_format_name__, "FromRarFile", PackArchiveFileFromRarFile);
2985
2986 def ArchiveFileSeekToFileNum(infile, seekto=0, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
2987 if(hasattr(infile, "read") or hasattr(infile, "write")):
2988 catfp = infile;
2989 catfp.seek(0, 0);
2990 catfp = UncompressArchiveFile(catfp, formatspecs);
2991 checkcompressfile = CheckCompressionSubType(catfp, formatspecs);
2992 if(checkcompressfile=="tarfile"):
2993 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
2994 if(checkcompressfile=="zipfile"):
2995 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
2996 if(rarfile_support and checkcompressfile=="rarfile"):
2997 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
2998 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
2999 return False;
3000 if(not catfp):
3001 return False;
3002 catfp.seek(0, 0);
3003 elif(infile=="-"):
3004 catfp = BytesIO();
3005 if(hasattr(sys.stdin, "buffer")):
3006 shutil.copyfileobj(sys.stdin.buffer, catfp);
3007 else:
3008 shutil.copyfileobj(sys.stdin, catfp);
3009 catfp.seek(0, 0);
3010 catfp = UncompressArchiveFile(catfp, formatspecs);
3011 if(not catfp):
3012 return False;
3013 catfp.seek(0, 0);
3014 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", infile)):
3015 catfp = download_file_from_internet_file(infile);
3016 catfp.seek(0, 0);
3017 catfp = UncompressArchiveFile(catfp, formatspecs);
3018 if(not catfp):
3019 return False;
3020 catfp.seek(0, 0);
3021 else:
3022 infile = RemoveWindowsPath(infile);
3023 checkcompressfile = CheckCompressionSubType(infile, formatspecs);
3024 if(checkcompressfile=="tarfile"):
3025 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3026 if(checkcompressfile=="zipfile"):
3027 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3028 if(rarfile_support and checkcompressfile=="rarfile"):
3029 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3030 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
3031 return False;
3032 compresscheck = CheckCompressionType(infile, formatspecs, True);
3033 if(not compresscheck):
3034 fextname = os.path.splitext(infile)[1];
3035 if(fextname==".gz"):
3036 compresscheck = "gzip";
3037 elif(fextname==".bz2"):
3038 compresscheck = "bzip2";
3039 elif(fextname==".zst"):
3040 compresscheck = "zstd";
3041 elif(fextname==".lz4" or fextname==".clz4"):
3042 compresscheck = "lz4";
3043 elif(fextname==".lzo" or fextname==".lzop"):
3044 compresscheck = "lzo";
3045 elif(fextname==".lzma" or fextname==".xz"):
3046 compresscheck = "lzma";
3047 else:
3048 return False;
3049 if(not compresscheck):
3050 return False;
3051 catfp = UncompressFile(infile, formatspecs, "rb");
3052 '''
3053 try:
3054 catfp.seek(0, 2);
3055 except OSError:
3056 SeekToEndOfFile(catfp);
3057 except ValueError:
3058 SeekToEndOfFile(catfp);
3059 CatSize = catfp.tell();
3060 CatSizeEnd = CatSize;
3061 '''
3062 try:
3063 catfp.seek(0, 0);
3064 except OSError:
3065 return False;
3066 except ValueError:
3067 return False;
3068 catheader = ReadFileHeaderData(catfp, 4, formatspecs[5]);
3069 catstring = catheader[0];
3070 catversion = re.findall(r"([\d]+)$", catstring);
3071 fprenumfiles = catheader[1];
3072 fnumfiles = int(fprenumfiles, 16);
3073 fprechecksumtype = catheader[2];
3074 fprechecksum = catheader[3];
3075 fileheader = AppendNullByte(catstring, formatspecs[5]);
3076 fnumfileshex = format(int(fnumfiles), 'x').lower();
3077 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs[5]);
3078 if(fprechecksumtype=="none" or fprechecksumtype==""):
3079 catfileheadercshex = format(0, 'x').lower();
3080 elif(fprechecksumtype=="crc16" or fprechecksumtype=="crc16_ansi" or fprechecksumtype=="crc16_ibm"):
3081 catfileheadercshex = format(crc16(fileheader.encode('UTF-8')) & 0xffff, '04x').lower();
3082 elif(fprechecksumtype=="crc16_ccitt"):
3083 catfileheadercshex = format(crc16_ccitt(fileheader.encode('UTF-8')) & 0xffff, '04x').lower();
3084 elif(fprechecksumtype=="adler32"):
3085 catfileheadercshex = format(zlib.adler32(fileheader.encode('UTF-8')) & 0xffffffff, '08x').lower();
3086 elif(fprechecksumtype=="crc32"):
3087 catfileheadercshex = format(crc32(fileheader.encode('UTF-8')) & 0xffffffff, '08x').lower();
3088 elif(fprechecksumtype=="crc64_ecma"):
3089 catfileheadercshex = format(crc64_ecma(fileheader.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
3090 elif(fprechecksumtype=="crc64" or fprechecksumtype=="crc64_iso"):
3091 catfileheadercshex = format(crc64_iso(fileheader.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
3092 elif(CheckSumSupportAlt(fprechecksumtype, hashlib_guaranteed)):
3093 checksumoutstr = hashlib.new(fprechecksumtype);
3094 checksumoutstr.update(fileheader.encode('UTF-8'));
3095 catfileheadercshex = checksumoutstr.hexdigest().lower();
3096 else:
3097 catfileheadercshex = format(0, 'x').lower();
3098 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
3099 fheadtell = len(fileheader);
3100 if(fprechecksum!=catfileheadercshex and not skipchecksum):
3101 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
3102 return False;
3103 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
3104 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
3105 if(seekto>=fnumfiles):
3106 seekto = fnumfiles - 1;
3107 if(seekto<0):
3108 seekto = 0;
3109 if(seekto>=0):
3110 il = -1;
3111 while(il < seekto):
3112 prefhstart = catfp.tell();
3113 if(formatspecs[7]):
3114 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs[5]);
3115 else:
3116 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs[5]);
3117 prefheadsize = int(preheaderdata[0], 16);
3118 preftype = int(preheaderdata[1], 16);
3119 if(re.findall("^[.|/]", preheaderdata[2])):
3120 prefname = preheaderdata[2];
3121 else:
3122 prefname = "./"+preheaderdata[2];
3123 prefbasedir = os.path.dirname(prefname);
3124 preflinkname = preheaderdata[3];
3125 prefsize = int(preheaderdata[4], 16);
3126 prefatime = int(preheaderdata[5], 16);
3127 prefmtime = int(preheaderdata[6], 16);
3128 prefctime = int(preheaderdata[7], 16);
3129 prefbtime = int(preheaderdata[8], 16);
3130 prefmode = int(preheaderdata[9], 16);
3131 prefchmode = stat.S_IMODE(prefmode);
3132 preftypemod = stat.S_IFMT(prefmode);
3133 prefwinattributes = int(preheaderdata[10], 16);
3134 prefuid = int(preheaderdata[11], 16);
3135 prefuname = preheaderdata[12];
3136 prefgid = int(preheaderdata[13], 16);
3137 prefgname = preheaderdata[14];
3138 fid = int(preheaderdata[15], 16);
3139 finode = int(preheaderdata[16], 16);
3140 flinkcount = int(preheaderdata[17], 16);
3141 prefdev_minor = int(preheaderdata[18], 16);
3142 prefdev_major = int(preheaderdata[19], 16);
3143 prefrdev_minor = int(preheaderdata[20], 16);
3144 prefrdev_major = int(preheaderdata[21], 16);
3145 prefextrasize = int(preheaderdata[22], 16);
3146 prefextrafields = int(preheaderdata[23], 16);
3147 extrafieldslist = [];
3148 extrastart = 24;
3149 extraend = extrastart + prefextrafields;
3150 extrafieldslist = [];
3151 if(extrastart<extraend):
3152 extrafieldslist.append(preheaderdata[extrastart]);
3153 extrastart = extrastart + 1;
3154 prefchecksumtype = preheaderdata[extrastart].lower();
3155 prefcs = preheaderdata[extrastart + 1].lower();
3156 prefccs = preheaderdata[extrastart + 2].lower();
3157 hc = 0;
3158 hcmax = len(preheaderdata) - 2;
3159 hout = "";
3160 while(hc<hcmax):
3161 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs[5]);
3162 hc = hc + 1;
3163 prefnumfields = 24 + prefextrafields;
3164 if(prefchecksumtype=="none" or prefchecksumtype==""):
3165 prenewfcs = 0;
3166 elif(prefchecksumtype=="crc16" or prefchecksumtype=="crc16_ansi" or prefchecksumtype=="crc16_ibm"):
3167 prenewfcs = format(crc16(hout.encode('UTF-8')) & 0xffff, '04x').lower();
3168 elif(prefchecksumtype=="adler32"):
3169 prenewfcs = format(zlib.adler32(hout.encode('UTF-8')) & 0xffffffff, '08x').lower();
3170 elif(prefchecksumtype=="crc32"):
3171 prenewfcs = format(crc32(hout.encode('UTF-8')) & 0xffffffff, '08x').lower();
3172 elif(prefchecksumtype=="crc64_ecma"):
3173 prenewfcs = format(crc64_ecma(hout.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
3174 elif(prefchecksumtype=="crc64" or prefchecksumtype=="crc64_iso"):
3175 prenewfcs = format(crc64_iso(hout.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
3176 elif(CheckSumSupportAlt(prefchecksumtype, hashlib_guaranteed)):
3177 checksumoutstr = hashlib.new(prefchecksumtype);
3178 checksumoutstr.update(hout.encode('UTF-8'));
3179 prenewfcs = checksumoutstr.hexdigest().lower();
3180 if(prefcs!=prenewfcs and not skipchecksum):
3181 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
3182 return False;
3183 valid_archive = False;
3184 invalid_archive = True;
3185 prefhend = catfp.tell() - 1;
3186 prefcontentstart = catfp.tell();
3187 prefcontents = "";
3188 pyhascontents = False;
3189 if(prefsize>0):
3190 prefcontents = catfp.read(prefsize);
3191 if(prefchecksumtype=="none" or prefchecksumtype==""):
3192 prenewfccs = 0;
3193 elif(prefchecksumtype=="crc16" or prefchecksumtype=="crc16_ansi" or prefchecksumtype=="crc16_ibm"):
3194 prenewfccs = format(crc16(prefcontents) & 0xffff, '04x').lower();
3195 elif(prefchecksumtype=="crc16_ccitt"):
3196 prenewfcs = format(crc16_ccitt(prefcontents) & 0xffff, '04x').lower();
3197 elif(prefchecksumtype=="adler32"):
3198 prenewfccs = format(zlib.adler32(prefcontents) & 0xffffffff, '08x').lower();
3199 elif(prefchecksumtype=="crc32"):
3200 prenewfccs = format(crc32(prefcontents) & 0xffffffff, '08x').lower();
3201 elif(prefchecksumtype=="crc64_ecma"):
3202 prenewfcs = format(crc64_ecma(prefcontents) & 0xffffffffffffffff, '016x').lower();
3203 elif(prefchecksumtype=="crc64" or prefchecksumtype=="crc64_iso"):
3204 prenewfcs = format(crc64_iso(prefcontents) & 0xffffffffffffffff, '016x').lower();
3205 elif(CheckSumSupportAlt(prefchecksumtype, hashlib_guaranteed)):
3206 checksumoutstr = hashlib.new(prefchecksumtype);
3207 checksumoutstr.update(prefcontents);
3208 prenewfccs = checksumoutstr.hexdigest().lower();
3209 pyhascontents = True;
3210 if(prefccs!=prenewfccs and not skipchecksum):
3211 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
3212 return False;
3213 catfp.seek(1, 1);
3214 il = il + 1;
3215 catfp.seek(seekstart, 0);
3216 fileidnum = il;
3217 catfheadsize = int(preheaderdata[0], 16);
3218 catftype = int(preheaderdata[1], 16);
3219 if(re.findall("^[.|/]", preheaderdata[2])):
3220 catfname = preheaderdata[2];
3221 else:
3222 catfname = "./"+preheaderdata[2];
3223 catflinkname = preheaderdata[3];
3224 catfsize = int(preheaderdata[4], 16);
3225 catfbasedir = os.path.dirname(catfname);
3226 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
3227 if(returnfp):
3228 catlist.update({'catfp': catfp});
3229 else:
3230 catfp.close();
3231 return catlist;
3232
3233 create_alias_function("", __file_format_name__, "SeekToFileNum", ArchiveFileSeekToFileNum);
3234
3235 def ArchiveFileSeekToFileName(infile, seekfile=None, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
3236 if(hasattr(infile, "read") or hasattr(infile, "write")):
3237 catfp = infile;
3238 catfp.seek(0, 0);
3239 catfp = UncompressArchiveFile(catfp, formatspecs);
3240 checkcompressfile = CheckCompressionSubType(catfp, formatspecs);
3241 if(checkcompressfile=="tarfile"):
3242 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3243 if(checkcompressfile=="zipfile"):
3244 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3245 if(rarfile_support and checkcompressfile=="rarfile"):
3246 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3247 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
3248 return False;
3249 if(not catfp):
3250 return False;
3251 catfp.seek(0, 0);
3252 elif(infile=="-"):
3253 catfp = BytesIO();
3254 if(hasattr(sys.stdin, "buffer")):
3255 shutil.copyfileobj(sys.stdin.buffer, catfp);
3256 else:
3257 shutil.copyfileobj(sys.stdin, catfp);
3258 catfp.seek(0, 0);
3259 catfp = UncompressArchiveFile(catfp, formatspecs);
3260 if(not catfp):
3261 return False;
3262 catfp.seek(0, 0);
3263 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", infile)):
3264 catfp = download_file_from_internet_file(infile);
3265 catfp = UncompressArchiveFile(catfp, formatspecs);
3266 catfp.seek(0, 0);
3267 if(not catfp):
3268 return False;
3269 catfp.seek(0, 0);
3270 else:
3271 infile = RemoveWindowsPath(infile);
3272 checkcompressfile = CheckCompressionSubType(infile, formatspecs);
3273 if(checkcompressfile=="tarfile"):
3274 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3275 if(checkcompressfile=="zipfile"):
3276 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3277 if(rarfile_support and checkcompressfile=="rarfile"):
3278 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3279 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
3280 return False;
3281 compresscheck = CheckCompressionType(infile, formatspecs, True);
3282 if(not compresscheck):
3283 fextname = os.path.splitext(infile)[1];
3284 if(fextname==".gz"):
3285 compresscheck = "gzip";
3286 elif(fextname==".bz2"):
3287 compresscheck = "bzip2";
3288 elif(fextname==".zst"):
3289 compresscheck = "zstd";
3290 elif(fextname==".lz4" or fextname==".clz4"):
3291 compresscheck = "lz4";
3292 elif(fextname==".lzo" or fextname==".lzop"):
3293 compresscheck = "lzo";
3294 elif(fextname==".lzma" or fextname==".xz"):
3295 compresscheck = "lzma";
3296 else:
3297 return False;
3298 if(not compresscheck):
3299 return False;
3300 catfp = UncompressFile(infile, formatspecs, "rb");
3301 '''
3302 try:
3303 catfp.seek(0, 2);
3304 except OSError:
3305 SeekToEndOfFile(catfp);
3306 except ValueError:
3307 SeekToEndOfFile(catfp);
3308 CatSize = catfp.tell();
3309 CatSizeEnd = CatSize;
3310 '''
3311 try:
3312 catfp.seek(0, 0);
3313 except OSError:
3314 return False;
3315 except ValueError:
3316 return False;
3317 catheader = ReadFileHeaderData(catfp, 4, formatspecs[5]);
3318 catstring = catheader[0];
3319 catversion = re.findall(r"([\d]+)$", catstring);
3320 fprenumfiles = catheader[1];
3321 fnumfiles = int(fprenumfiles, 16);
3322 fprechecksumtype = catheader[2];
3323 fprechecksum = catheader[3];
3324 fileheader = AppendNullByte(catstring, formatspecs[5]);
3325 fnumfileshex = format(int(fnumfiles), 'x').lower();
3326 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs[5]);
3327 if(fprechecksumtype=="none" or fprechecksumtype==""):
3328 catfileheadercshex = format(0, 'x').lower();
3329 elif(fprechecksumtype=="crc16" or fprechecksumtype=="crc16_ansi" or fprechecksumtype=="crc16_ibm"):
3330 catfileheadercshex = format(crc16(fileheader.encode('UTF-8')) & 0xffff, '04x').lower();
3331 elif(fprechecksumtype=="crc16_ccitt"):
3332 catfileheadercshex = format(crc16_ccitt(fileheader.encode('UTF-8')) & 0xffff, '04x').lower();
3333 elif(fprechecksumtype=="adler32"):
3334 catfileheadercshex = format(zlib.adler32(fileheader.encode('UTF-8')) & 0xffffffff, '08x').lower();
3335 elif(fprechecksumtype=="crc32"):
3336 catfileheadercshex = format(crc32(fileheader.encode('UTF-8')) & 0xffffffff, '08x').lower();
3337 elif(fprechecksumtype=="crc64_ecma"):
3338 catfileheadercshex = format(crc64_ecma(fileheader.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
3339 elif(fprechecksumtype=="crc64" or fprechecksumtype=="crc64_iso"):
3340 catfileheadercshex = format(crc64_iso(fileheader.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
3341 elif(CheckSumSupportAlt(fprechecksumtype, hashlib_guaranteed)):
3342 checksumoutstr = hashlib.new(fprechecksumtype);
3343 checksumoutstr.update(fileheader.encode('UTF-8'));
3344 catfileheadercshex = checksumoutstr.hexdigest().lower();
3345 else:
3346 catfileheadercshex = format(0, 'x').lower();
3347 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
3348 fheadtell = len(fileheader);
3349 if(fprechecksum!=catfileheadercshex and not skipchecksum):
3350 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
3351 return False;
3352 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
3353 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
3354 seekto = fnumfiles - 1
3355 filefound = False;
3356 if(seekto>=0):
3357 il = -1;
3358 while(il < seekto):
3359 prefhstart = catfp.tell();
3360 if(formatspecs[7]):
3361 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs[5]);
3362 else:
3363 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs[5]);
3364 prefheadsize = int(preheaderdata[0], 16);
3365 preftype = int(preheaderdata[1], 16);
3366 if(re.findall("^[.|/]", preheaderdata[2])):
3367 prefname = preheaderdata[2];
3368 else:
3369 prefname = "./"+preheaderdata[2];
3370 prefbasedir = os.path.dirname(prefname);
3371 preflinkname = preheaderdata[3];
3372 prefsize = int(preheaderdata[4], 16);
3373 prefatime = int(preheaderdata[5], 16);
3374 prefmtime = int(preheaderdata[6], 16);
3375 prefctime = int(preheaderdata[7], 16);
3376 prefbtime = int(preheaderdata[8], 16);
3377 prefmode = int(preheaderdata[9], 16);
3378 prefchmode = stat.S_IMODE(prefmode);
3379 preftypemod = stat.S_IFMT(prefmode);
3380 prefwinattributes = int(preheaderdata[10], 16);
3381 prefuid = int(preheaderdata[10], 16);
3382 prefuname = preheaderdata[11];
3383 prefgid = int(preheaderdata[12], 16);
3384 prefgname = preheaderdata[13];
3385 fid = int(preheaderdata[14], 16);
3386 finode = int(preheaderdata[15], 16);
3387 flinkcount = int(preheaderdata[16], 16);
3388 prefdev_minor = int(preheaderdata[17], 16);
3389 prefdev_major = int(preheaderdata[18], 16);
3390 prefrdev_minor = int(preheaderdata[19], 16);
3391 prefrdev_major = int(preheaderdata[20], 16);
3392 prefextrasize = int(preheaderdata[21], 16);
3393 prefextrafields = int(preheaderdata[22], 16);
3394 extrafieldslist = [];
3395 extrastart = 24;
3396 extraend = extrastart + prefextrafields;
3397 extrafieldslist = [];
3398 if(extrastart<extraend):
3399 extrafieldslist.append(preheaderdata[extrastart]);
3400 extrastart = extrastart + 1;
3401 prefchecksumtype = preheaderdata[extrastart].lower();
3402 prefcs = preheaderdata[extrastart + 1].lower();
3403 prefccs = preheaderdata[extrastart + 2].lower();
3404 hc = 0;
3405 hcmax = len(preheaderdata) - 2;
3406 hout = "";
3407 while(hc<hcmax):
3408 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs[5]);
3409 hc = hc + 1;
3410 prefnumfields = 24 + prefextrafields;
3411 if(prefchecksumtype=="none" or prefchecksumtype==""):
3412 prenewfcs = 0;
3413 elif(prefchecksumtype=="crc16" or prefchecksumtype=="crc16_ansi" or prefchecksumtype=="crc16_ibm"):
3414 prenewfcs = format(crc16(hout.encode('UTF-8')) & 0xffff, '04x').lower();
3415 elif(prefchecksumtype=="adler32"):
3416 prenewfcs = format(zlib.adler32(hout.encode('UTF-8')) & 0xffffffff, '08x').lower();
3417 elif(prefchecksumtype=="crc32"):
3418 prenewfcs = format(crc32(hout.encode('UTF-8')) & 0xffffffff, '08x').lower();
3419 elif(prefchecksumtype=="crc64_ecma"):
3420 prenewfcs = format(crc64_ecma(hout.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
3421 elif(prefchecksumtype=="crc64" or prefchecksumtype=="crc64_iso"):
3422 prenewfcs = format(crc64_iso(hout.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
3423 elif(CheckSumSupportAlt(prefchecksumtype, hashlib_guaranteed)):
3424 checksumoutstr = hashlib.new(prefchecksumtype);
3425 checksumoutstr.update(hout.encode('UTF-8'));
3426 prenewfcs = checksumoutstr.hexdigest().lower();
3427 if(prefcs!=prenewfcs and not skipchecksum):
3428 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
3429 return False;
3430 valid_archive = False;
3431 invalid_archive = True;
3432 prefhend = catfp.tell() - 1;
3433 prefcontentstart = catfp.tell();
3434 prefcontents = "";
3435 pyhascontents = False;
3436 if(prefsize>0):
3437 prefcontents = catfp.read(prefsize);
3438 if(prefchecksumtype=="none" or prefchecksumtype==""):
3439 prenewfccs = 0;
3440 elif(prefchecksumtype=="crc16" or prefchecksumtype=="crc16_ansi" or prefchecksumtype=="crc16_ibm"):
3441 prenewfccs = format(crc16(prefcontents) & 0xffff, '04x').lower();
3442 elif(prefchecksumtype=="crc16_ccitt"):
3443 prenewfcs = format(crc16_ccitt(prefcontents) & 0xffff, '04x').lower();
3444 elif(prefchecksumtype=="adler32"):
3445 prenewfccs = format(zlib.adler32(prefcontents) & 0xffffffff, '08x').lower();
3446 elif(prefchecksumtype=="crc32"):
3447 prenewfccs = format(crc32(prefcontents) & 0xffffffff, '08x').lower();
3448 elif(prefchecksumtype=="crc64_ecma"):
3449 prenewfcs = format(crc64_ecma(prefcontents) & 0xffffffffffffffff, '016x').lower();
3450 elif(prefchecksumtype=="crc64" or prefchecksumtype=="crc64_iso"):
3451 prenewfcs = format(crc64_iso(prefcontents) & 0xffffffffffffffff, '016x').lower();
3452 elif(CheckSumSupportAlt(prefchecksumtype, hashlib_guaranteed)):
3453 checksumoutstr = hashlib.new(prefchecksumtype);
3454 checksumoutstr.update(prefcontents);
3455 prenewfccs = checksumoutstr.hexdigest().lower();
3456 pyhascontents = True;
3457 if(prefccs!=prenewfccs and not skipchecksum):
3458 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
3459 return False;
3460 catfp.seek(1, 1);
3461 il = il + 1;
3462 filefound = False;
3463 prefname = preheaderdata[2];
3464 if(re.findall("^[.|/]", preheaderdata[2])):
3465 prefname = preheaderdata[2];
3466 else:
3467 prefname = "./"+preheaderdata[2];
3468 if(prefname==seekfile):
3469 filefound = True;
3470 break;
3471 catfp.seek(seekstart, 0);
3472 fileidnum = il;
3473 catfheadsize = int(preheaderdata[0], 16);
3474 catftype = int(preheaderdata[1], 16);
3475 if(re.findall("^[.|/]", preheaderdata[2])):
3476 catfname = preheaderdata[2];
3477 else:
3478 catfname = "./"+preheaderdata[2];
3479 catflinkname = preheaderdata[3];
3480 catfsize = int(preheaderdata[4], 16);
3481 catfbasedir = os.path.dirname(catfname);
3482 if(filefound):
3483 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
3484 else:
3485 return False;
3486 if(returnfp):
3487 catlist.update({'catfp': catfp});
3488 else:
3489 catfp.close();
3490 return catlist;
3491
3492 create_alias_function("", __file_format_name__, "SeekToFileName", ArchiveFileSeekToFileName);
3493
3494 def ArchiveFileValidate(infile, formatspecs=__file_format_list__, verbose=False, returnfp=False):
3495 if(verbose):
3496 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3497 if(hasattr(infile, "read") or hasattr(infile, "write")):
3498 catfp = infile;
3499 catfp.seek(0, 0);
3500 catfp = UncompressArchiveFile(catfp, formatspecs);
3501 checkcompressfile = CheckCompressionSubType(catfp, formatspecs);
3502 if(checkcompressfile=="tarfile"):
3503 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
3504 if(checkcompressfile=="zipfile"):
3505 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
3506 if(rarfile_support and checkcompressfile=="rarfile"):
3507 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
3508 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
3509 return False;
3510 if(not catfp):
3511 return False;
3512 catfp.seek(0, 0);
3513 elif(infile=="-"):
3514 catfp = BytesIO();
3515 if(hasattr(sys.stdin, "buffer")):
3516 shutil.copyfileobj(sys.stdin.buffer, catfp);
3517 else:
3518 shutil.copyfileobj(sys.stdin, catfp);
3519 catfp.seek(0, 0);
3520 catfp = UncompressArchiveFile(catfp, formatspecs);
3521 if(not catfp):
3522 return False;
3523 catfp.seek(0, 0);
3524 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", infile)):
3525 catfp = download_file_from_internet_file(infile);
3526 catfp = UncompressArchiveFile(catfp, formatspecs);
3527 catfp.seek(0, 0);
3528 if(not catfp):
3529 return False;
3530 catfp.seek(0, 0);
3531 else:
3532 infile = RemoveWindowsPath(infile);
3533 checkcompressfile = CheckCompressionSubType(infile, formatspecs);
3534 if(checkcompressfile=="tarfile"):
3535 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
3536 if(checkcompressfile=="zipfile"):
3537 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
3538 if(rarfile_support and checkcompressfile=="rarfile"):
3539 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
3540 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
3541 return False;
3542 compresscheck = CheckCompressionType(infile, formatspecs, True);
3543 if(not compresscheck):
3544 fextname = os.path.splitext(infile)[1];
3545 if(fextname==".gz"):
3546 compresscheck = "gzip";
3547 elif(fextname==".bz2"):
3548 compresscheck = "bzip2";
3549 elif(fextname==".zst"):
3550 compresscheck = "zstd";
3551 elif(fextname==".lz4" or fextname==".clz4"):
3552 compresscheck = "lz4";
3553 elif(fextname==".lzo" or fextname==".lzop"):
3554 compresscheck = "lzo";
3555 elif(fextname==".lzma" or fextname==".xz"):
3556 compresscheck = "lzma";
3557 else:
3558 return False;
3559 if(not compresscheck):
3560 return False;
3561 catfp = UncompressFile(infile, formatspecs, "rb");
3562 '''
3563 try:
3564 catfp.seek(0, 2);
3565 except OSError:
3566 SeekToEndOfFile(catfp);
3567 except ValueError:
3568 SeekToEndOfFile(catfp);
3569 CatSize = catfp.tell();
3570 CatSizeEnd = CatSize;
3571 '''
3572 try:
3573 catfp.seek(0, 0);
3574 except OSError:
3575 return False;
3576 except ValueError:
3577 return False;
3578 catheader = ReadFileHeaderData(catfp, 4, formatspecs[5]);
3579 catstring = catheader[0];
3580 catversion = re.findall(r"([\d]+)$", catstring);
3581 fprenumfiles = catheader[1];
3582 fnumfiles = int(fprenumfiles, 16);
3583 fprechecksumtype = catheader[2];
3584 fprechecksum = catheader[3];
3585 il = 0;
3586 fileheader = AppendNullByte(catstring, formatspecs[5]);
3587 fnumfileshex = format(int(fnumfiles), 'x').lower();
3588 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs[5]);
3589 if(fprechecksumtype=="none" or fprechecksumtype==""):
3590 catfileheadercshex = format(0, 'x').lower();
3591 elif(fprechecksumtype=="crc16" or fprechecksumtype=="crc16_ansi" or fprechecksumtype=="crc16_ibm"):
3592 catfileheadercshex = format(crc16(fileheader.encode('UTF-8')) & 0xffff, '04x').lower();
3593 elif(fprechecksumtype=="crc16_ccitt"):
3594 catfileheadercshex = format(crc16_ccitt(fileheader.encode('UTF-8')) & 0xffff, '04x').lower();
3595 elif(fprechecksumtype=="adler32"):
3596 catfileheadercshex = format(zlib.adler32(fileheader.encode('UTF-8')) & 0xffffffff, '08x').lower();
3597 elif(fprechecksumtype=="crc32"):
3598 catfileheadercshex = format(crc32(fileheader.encode('UTF-8')) & 0xffffffff, '08x').lower();
3599 elif(fprechecksumtype=="crc64_ecma"):
3600 catfileheadercshex = format(crc64_ecma(fileheader.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
3601 elif(fprechecksumtype=="crc64" or fprechecksumtype=="crc64_iso"):
3602 catfileheadercshex = format(crc64_iso(fileheader.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
3603 elif(CheckSumSupportAlt(fprechecksumtype, hashlib_guaranteed)):
3604 checksumoutstr = hashlib.new(fprechecksumtype);
3605 checksumoutstr.update(fileheader.encode('UTF-8'));
3606 catfileheadercshex = checksumoutstr.hexdigest().lower();
3607 else:
3608 catfileheadercshex = format(0, 'x').lower();
3609 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
3610 valid_archive = True;
3611 invalid_archive = False;
3612 if(verbose):
3613 VerbosePrintOut("Checking File Header Checksum of file at offset " + str(0));
3614 VerbosePrintOut(infile);
3615 if(fprechecksum==catfileheadercshex):
3616 if(verbose):
3617 VerbosePrintOut("File Header Checksum Passed");
3618 else:
3619 if(verbose):
3620 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
3621 valid_archive = False;
3622 invalid_archive = True;
3623 while(il<fnumfiles):
3624 catfhstart = catfp.tell();
3625 if(formatspecs[7]):
3626 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs[5]);
3627 else:
3628 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs[5]);
3629 catfheadsize = int(catheaderdata[0], 16);
3630 catftype = int(catheaderdata[1], 16);
3631 if(re.findall("^[.|/]", catheaderdata[2])):
3632 catfname = catheaderdata[2];
3633 else:
3634 catfname = "./"+catheaderdata[2];
3635 catfbasedir = os.path.dirname(catfname);
3636 catflinkname = catheaderdata[3];
3637 catfsize = int(catheaderdata[4], 16);
3638 catfatime = int(catheaderdata[5], 16);
3639 catfmtime = int(catheaderdata[6], 16);
3640 catfctime = int(catheaderdata[7], 16);
3641 catfbtime = int(catheaderdata[8], 16);
3642 catfmode = int(catheaderdata[9], 16);
3643 catfchmode = stat.S_IMODE(catfmode);
3644 catftypemod = stat.S_IFMT(catfmode);
3645 prefwinattributes = int(catheaderdata[10], 16);
3646 catfuid = int(catheaderdata[11], 16);
3647 catfuname = catheaderdata[12];
3648 catfgid = int(catheaderdata[13], 16);
3649 catfgname = catheaderdata[14];
3650 fid = int(catheaderdata[15], 16);
3651 finode = int(catheaderdata[16], 16);
3652 flinkcount = int(catheaderdata[17], 16);
3653 catfdev_minor = int(catheaderdata[18], 16);
3654 catfdev_major = int(catheaderdata[19], 16);
3655 catfrdev_minor = int(catheaderdata[20], 16);
3656 catfrdev_major = int(catheaderdata[21], 16);
3657 catfextrasize = int(catheaderdata[22], 16);
3658 catfextrafields = int(catheaderdata[23], 16);
3659 extrafieldslist = [];
3660 extrastart = 24;
3661 extraend = extrastart + catfextrafields;
3662 extrafieldslist = [];
3663 if(extrastart<extraend):
3664 extrafieldslist.append(catheaderdata[extrastart]);
3665 extrastart = extrastart + 1;
3666 catfchecksumtype = catheaderdata[extrastart].lower();
3667 catfcs = catheaderdata[extrastart + 1].lower();
3668 catfccs = catheaderdata[extrastart + 2].lower();
3669 hc = 0;
3670 hcmax = len(catheaderdata) - 2;
3671 hout = "";
3672 while(hc<hcmax):
3673 hout = hout + AppendNullByte(catheaderdata[hc], formatspecs[5]);
3674 hc = hc + 1;
3675 catfnumfields = 24 + catfextrafields;
3676 if(catfchecksumtype=="none" or catfchecksumtype==""):
3677 catnewfcs = 0;
3678 elif(catfchecksumtype=="crc16" or catfchecksumtype=="crc16_ansi" or catfchecksumtype=="crc16_ibm"):
3679 catnewfcs = format(crc16(hout.encode('UTF-8')) & 0xffff, '04x').lower();
3680 elif(catfchecksumtype=="adler32"):
3681 catnewfcs = format(zlib.adler32(hout.encode('UTF-8')) & 0xffffffff, '08x').lower();
3682 elif(catfchecksumtype=="crc32"):
3683 catnewfcs = format(crc32(hout.encode('UTF-8')) & 0xffffffff, '08x').lower();
3684 elif(catfchecksumtype=="crc64_ecma"):
3685 catnewfcs = format(crc64_ecma(hout.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
3686 elif(catfchecksumtype=="crc64" or catfchecksumtype=="crc64_iso"):
3687 catnewfcs = format(crc64_iso(hout.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
3688 elif(CheckSumSupportAlt(catfchecksumtype, hashlib_guaranteed)):
3689 checksumoutstr = hashlib.new(catfchecksumtype);
3690 checksumoutstr.update(hout.encode('UTF-8'));
3691 catnewfcs = checksumoutstr.hexdigest().lower();
3692 if(verbose):
3693 VerbosePrintOut("Checking File Header Checksum of file at offset " + str(catfhstart));
3694 VerbosePrintOut(catfname);
3695 if(catfcs==catnewfcs):
3696 if(verbose):
3697 VerbosePrintOut("File Header Checksum Passed");
3698 else:
3699 if(verbose):
3700 VerbosePrintOut("File Header Checksum Error with file " + catfname + " at offset " + str(catfhstart));
3701 valid_archive = False;
3702 invalid_archive = True;
3703 catfhend = catfp.tell() - 1;
3704 catfcontentstart = catfp.tell();
3705 catfcontents = "";
3706 pyhascontents = False;
3707 if(catfsize>0):
3708 catfcontents = catfp.read(catfsize);
3709 if(catfchecksumtype=="none" or catfchecksumtype==""):
3710 catnewfccs = 0;
3711 elif(catfchecksumtype=="crc16" or catfchecksumtype=="crc16_ansi" or catfchecksumtype=="crc16_ibm"):
3712 catnewfccs = format(crc16(catfcontents) & 0xffff, '04x').lower();
3713 elif(catfchecksumtype=="crc16_ccitt"):
3714 catnewfcs = format(crc16_ccitt(catfcontents) & 0xffff, '04x').lower();
3715 elif(catfchecksumtype=="adler32"):
3716 catnewfccs = format(zlib.adler32(catfcontents) & 0xffffffff, '08x').lower();
3717 elif(catfchecksumtype=="crc32"):
3718 catnewfccs = format(crc32(catfcontents) & 0xffffffff, '08x').lower();
3719 elif(catfchecksumtype=="crc64_ecma"):
3720 catnewfcs = format(crc64_ecma(catfcontents) & 0xffffffffffffffff, '016x').lower();
3721 elif(catfchecksumtype=="crc64" or catfchecksumtype=="crc64_iso"):
3722 catnewfcs = format(crc64_iso(catfcontents) & 0xffffffffffffffff, '016x').lower();
3723 elif(CheckSumSupportAlt(catfchecksumtype, hashlib_guaranteed)):
3724 checksumoutstr = hashlib.new(catfchecksumtype);
3725 checksumoutstr.update(catfcontents);
3726 catnewfccs = checksumoutstr.hexdigest().lower();
3727 pyhascontents = True;
3728 if(verbose):
3729 VerbosePrintOut("Checking File Content Checksum of file at offset " + str(catfcontentstart));
3730 VerbosePrintOut(catfname);
3731 if(catfccs==catnewfccs):
3732 if(verbose):
3733 VerbosePrintOut("File Content Checksum Passed");
3734 else:
3735 if(verbose):
3736 VerbosePrintOut("File Header Checksum Error with file " + catfname + " at offset " + str(catfcontentstart));
3737 valid_archive = False;
3738 invalid_archive = True;
3739 if(verbose):
3740 VerbosePrintOut("");
3741 catfp.seek(1, 1);
3742 il = il + 1;
3743 if(valid_archive):
3744 if(returnfp):
3745 return catfp;
3746 else:
3747 catfp.close();
3748 return True;
3749 else:
3750 catfp.close();
3751 return False;
3752
3753 create_alias_function("", __file_format_name__, "Validate", ArchiveFileValidate);
3754
3755 def ArchiveFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
3756 if(hasattr(infile, "read") or hasattr(infile, "write")):
3757 catfp = infile;
3758 catfp.seek(0, 0);
3759 catfp = UncompressArchiveFile(catfp, formatspecs);
3760 checkcompressfile = CheckCompressionSubType(catfp, formatspecs);
3761 if(checkcompressfile=="tarfile"):
3762 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
3763 if(checkcompressfile=="zipfile"):
3764 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
3765 if(rarfile_support and checkcompressfile=="rarfile"):
3766 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
3767 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
3768 return False;
3769 if(not catfp):
3770 return False;
3771 catfp.seek(0, 0);
3772 elif(infile=="-"):
3773 catfp = BytesIO();
3774 if(hasattr(sys.stdin, "buffer")):
3775 shutil.copyfileobj(sys.stdin.buffer, catfp);
3776 else:
3777 shutil.copyfileobj(sys.stdin, catfp);
3778 catfp.seek(0, 0);
3779 catfp = UncompressArchiveFile(catfp, formatspecs);
3780 if(not catfp):
3781 return False;
3782 catfp.seek(0, 0);
3783 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", infile)):
3784 catfp = download_file_from_internet_file(infile);
3785 catfp = UncompressArchiveFile(catfp, formatspecs);
3786 catfp.seek(0, 0);
3787 if(not catfp):
3788 return False;
3789 catfp.seek(0, 0);
3790 else:
3791 infile = RemoveWindowsPath(infile);
3792 checkcompressfile = CheckCompressionSubType(infile, formatspecs);
3793 if(checkcompressfile=="tarfile"):
3794 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
3795 if(checkcompressfile=="zipfile"):
3796 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
3797 if(rarfile_support and checkcompressfile=="rarfile"):
3798 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
3799 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
3800 return False;
3801 compresscheck = CheckCompressionType(infile, formatspecs, True);
3802 if(not compresscheck):
3803 fextname = os.path.splitext(infile)[1];
3804 if(fextname==".gz"):
3805 compresscheck = "gzip";
3806 elif(fextname==".bz2"):
3807 compresscheck = "bzip2";
3808 elif(fextname==".zst"):
3809 compresscheck = "zstd";
3810 elif(fextname==".lz4" or fextname==".clz4"):
3811 compresscheck = "lz4";
3812 elif(fextname==".lzo" or fextname==".lzop"):
3813 compresscheck = "lzo";
3814 elif(fextname==".lzma" or fextname==".xz"):
3815 compresscheck = "lzma";
3816 else:
3817 return False;
3818 if(not compresscheck):
3819 return False;
3820 catfp = UncompressFile(infile, formatspecs, "rb");
3821 '''
3822 try:
3823 catfp.seek(0, 2);
3824 except OSError:
3825 SeekToEndOfFile(catfp);
3826 except ValueError:
3827 SeekToEndOfFile(catfp);
3828 CatSize = catfp.tell();
3829 CatSizeEnd = CatSize;
3830 '''
3831 try:
3832 catfp.seek(0, 0);
3833 except OSError:
3834 return False;
3835 except ValueError:
3836 return False;
3837 catheader = ReadFileHeaderData(catfp, 4, formatspecs[5]);
3838 catstring = catheader[0];
3839 catversion = re.findall(r"([\d]+)$", catstring);
3840 fprenumfiles = catheader[1];
3841 fnumfiles = int(fprenumfiles, 16);
3842 fprechecksumtype = catheader[2];
3843 fprechecksum = catheader[3];
3844 fileheader = AppendNullByte(catstring, formatspecs[5]);
3845 fnumfileshex = format(int(fnumfiles), 'x').lower();
3846 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs[5]);
3847 if(fprechecksumtype=="none" or fprechecksumtype==""):
3848 catfileheadercshex = format(0, 'x').lower();
3849 elif(fprechecksumtype=="crc16" or fprechecksumtype=="crc16_ansi" or fprechecksumtype=="crc16_ibm"):
3850 catfileheadercshex = format(crc16(fileheader.encode('UTF-8')) & 0xffff, '04x').lower();
3851 elif(fprechecksumtype=="crc16_ccitt"):
3852 catfileheadercshex = format(crc16_ccitt(fileheader.encode('UTF-8')) & 0xffff, '04x').lower();
3853 elif(fprechecksumtype=="adler32"):
3854 catfileheadercshex = format(zlib.adler32(fileheader.encode('UTF-8')) & 0xffffffff, '08x').lower();
3855 elif(fprechecksumtype=="crc32"):
3856 catfileheadercshex = format(crc32(fileheader.encode('UTF-8')) & 0xffffffff, '08x').lower();
3857 elif(fprechecksumtype=="crc64_ecma"):
3858 catfileheadercshex = format(crc64_ecma(fileheader.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
3859 elif(fprechecksumtype=="crc64" or fprechecksumtype=="crc64_iso"):
3860 catfileheadercshex = format(crc64_iso(fileheader.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
3861 elif(CheckSumSupportAlt(fprechecksumtype, hashlib_guaranteed)):
3862 checksumoutstr = hashlib.new(fprechecksumtype);
3863 checksumoutstr.update(fileheader.encode('UTF-8'));
3864 catfileheadercshex = checksumoutstr.hexdigest().lower();
3865 else:
3866 catfileheadercshex = format(0, 'x').lower();
3867 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
3868 fheadtell = len(fileheader);
3869 if(fprechecksum!=catfileheadercshex and not skipchecksum):
3870 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
3871 return False;
3872 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
3873 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
3874 if(seekstart<0 and seekstart>fnumfiles):
3875 seekstart = 0;
3876 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
3877 seekend = fnumfiles;
3878 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
3879 seekend = fnumfiles - abs(seekend);
3880 if(seekstart>0):
3881 il = 0;
3882 while(il < seekstart):
3883 prefhstart = catfp.tell();
3884 if(formatspecs[7]):
3885 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs[5]);
3886 else:
3887 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs[5]);
3888 prefheadsize = int(preheaderdata[0], 16);
3889 if(re.findall("^[.|/]", preheaderdata[2])):
3890 prefname = preheaderdata[2];
3891 else:
3892 prefname = "./"+preheaderdata[2];
3893 prefsize = int(preheaderdata[4], 16);
3894 prefextrasize = int(preheaderdata[22], 16);
3895 prefextrafields = int(preheaderdata[23], 16);
3896 extrafieldslist = [];
3897 extrastart = 24;
3898 extraend = extrastart + prefextrafields;
3899 extrafieldslist = [];
3900 if(extrastart<extraend):
3901 extrafieldslist.append(preheaderdata[extrastart]);
3902 extrastart = extrastart + 1;
3903 prefchecksumtype = preheaderdata[extrastart].lower();
3904 prefcs = preheaderdata[extrastart + 1].lower();
3905 prefccs = preheaderdata[extrastart + 2].lower();
3906 hc = 0;
3907 hcmax = len(preheaderdata) - 2;
3908 hout = "";
3909 while(hc<hcmax):
3910 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs[5]);
3911 hc = hc + 1;
3912 prefnumfields = 24 + prefextrafields;
3913 if(prefchecksumtype=="none" or prefchecksumtype==""):
3914 prenewfcs = 0;
3915 elif(prefchecksumtype=="crc16" or prefchecksumtype=="crc16_ansi" or prefchecksumtype=="crc16_ibm"):
3916 prenewfcs = format(crc16(hout.encode('UTF-8')) & 0xffff, '04x').lower();
3917 elif(prefchecksumtype=="adler32"):
3918 prenewfcs = format(zlib.adler32(hout.encode('UTF-8')) & 0xffffffff, '08x').lower();
3919 elif(prefchecksumtype=="crc32"):
3920 prenewfcs = format(crc32(hout.encode('UTF-8')) & 0xffffffff, '08x').lower();
3921 elif(prefchecksumtype=="crc64_ecma"):
3922 prenewfcs = format(crc64_ecma(hout.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
3923 elif(prefchecksumtype=="crc64" or prefchecksumtype=="crc64_iso"):
3924 prenewfcs = format(crc64_iso(hout.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
3925 elif(CheckSumSupportAlt(prefchecksumtype, hashlib_guaranteed)):
3926 checksumoutstr = hashlib.new(prefchecksumtype);
3927 checksumoutstr.update(hout.encode('UTF-8'));
3928 prenewfcs = checksumoutstr.hexdigest().lower();
3929 if(prefcs!=prenewfcs and not skipchecksum):
3930 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
3931 return False;
3932 valid_archive = False;
3933 invalid_archive = True;
3934 prefhend = catfp.tell() - 1;
3935 prefcontentstart = catfp.tell();
3936 prefcontents = "";
3937 pyhascontents = False;
3938 if(prefsize>0):
3939 prefcontents = catfp.read(prefsize);
3940 if(prefchecksumtype=="none" or prefchecksumtype==""):
3941 prenewfccs = 0;
3942 elif(prefchecksumtype=="crc16" or prefchecksumtype=="crc16_ansi" or prefchecksumtype=="crc16_ibm"):
3943 prenewfccs = format(crc16(prefcontents) & 0xffff, '04x').lower();
3944 elif(prefchecksumtype=="crc16_ccitt"):
3945 prenewfcs = format(crc16_ccitt(prefcontents) & 0xffff, '04x').lower();
3946 elif(prefchecksumtype=="adler32"):
3947 prenewfccs = format(zlib.adler32(prefcontents) & 0xffffffff, '08x').lower();
3948 elif(prefchecksumtype=="crc32"):
3949 prenewfccs = format(crc32(prefcontents) & 0xffffffff, '08x').lower();
3950 elif(prefchecksumtype=="crc64_ecma"):
3951 prenewfcs = format(crc64_ecma(prefcontents) & 0xffffffffffffffff, '016x').lower();
3952 elif(prefchecksumtype=="crc64" or prefchecksumtype=="crc64_iso"):
3953 prenewfcs = format(crc64_iso(prefcontents) & 0xffffffffffffffff, '016x').lower();
3954 elif(CheckSumSupportAlt(prefchecksumtype, hashlib_guaranteed)):
3955 checksumoutstr = hashlib.new(prefchecksumtype);
3956 checksumoutstr.update(prefcontents);
3957 prenewfccs = checksumoutstr.hexdigest().lower();
3958 pyhascontents = True;
3959 if(prefccs!=prenewfccs and not skipchecksum):
3960 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
3961 return False;
3962 catfp.seek(1, 1);
3963 il = il + 1;
3964 fileidnum = seekstart;
3965 realidnum = 0;
3966 while(fileidnum<seekend):
3967 catfhstart = catfp.tell();
3968 if(formatspecs[7]):
3969 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs[5]);
3970 else:
3971 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs[5]);
3972 catfheadsize = int(catheaderdata[0], 16);
3973 catftype = int(catheaderdata[1], 16);
3974 if(re.findall("^[.|/]", catheaderdata[2])):
3975 catfname = catheaderdata[2];
3976 else:
3977 catfname = "./"+catheaderdata[2];
3978 catfbasedir = os.path.dirname(catfname);
3979 catflinkname = catheaderdata[3];
3980 catfsize = int(catheaderdata[4], 16);
3981 catfatime = int(catheaderdata[5], 16);
3982 catfmtime = int(catheaderdata[6], 16);
3983 catfctime = int(catheaderdata[7], 16);
3984 catfbtime = int(catheaderdata[8], 16);
3985 catfmode = int(catheaderdata[9], 16);
3986 catfchmode = stat.S_IMODE(catfmode);
3987 catftypemod = stat.S_IFMT(catfmode);
3988 catfwinattributes = int(catheaderdata[10], 16);
3989 catfuid = int(catheaderdata[11], 16);
3990 catfuname = catheaderdata[12];
3991 catfgid = int(catheaderdata[13], 16);
3992 catfgname = catheaderdata[14];
3993 fid = int(catheaderdata[15], 16);
3994 finode = int(catheaderdata[16], 16);
3995 flinkcount = int(catheaderdata[17], 16);
3996 catfdev_minor = int(catheaderdata[18], 16);
3997 catfdev_major = int(catheaderdata[19], 16);
3998 catfrdev_minor = int(catheaderdata[20], 16);
3999 catfrdev_major = int(catheaderdata[21], 16);
4000 catfextrasize = int(catheaderdata[22], 16);
4001 catfextrafields = int(catheaderdata[23], 16);
4002 extrafieldslist = [];
4003 extrastart = 24;
4004 extraend = extrastart + catfextrafields;
4005 extrafieldslist = [];
4006 if(extrastart<extraend):
4007 extrafieldslist.append(catheaderdata[extrastart]);
4008 extrastart = extrastart + 1;
4009 catfchecksumtype = catheaderdata[extrastart].lower();
4010 catfcs = catheaderdata[extrastart + 1].lower();
4011 catfccs = catheaderdata[extrastart + 2].lower();
4012 hc = 0;
4013 hcmax = len(catheaderdata) - 2;
4014 hout = "";
4015 while(hc<hcmax):
4016 hout = hout + AppendNullByte(catheaderdata[hc], formatspecs[5]);
4017 hc = hc + 1;
4018 catfnumfields = 24 + catfextrafields;
4019 if(catfchecksumtype=="none" or catfchecksumtype==""):
4020 catnewfcs = 0;
4021 elif(catfchecksumtype=="crc16" or catfchecksumtype=="crc16_ansi" or catfchecksumtype=="crc16_ibm"):
4022 catnewfcs = format(crc16(hout.encode('UTF-8')) & 0xffff, '04x').lower();
4023 elif(catfchecksumtype=="adler32"):
4024 catnewfcs = format(zlib.adler32(hout.encode('UTF-8')) & 0xffffffff, '08x').lower();
4025 elif(catfchecksumtype=="crc32"):
4026 catnewfcs = format(crc32(hout.encode('UTF-8')) & 0xffffffff, '08x').lower();
4027 elif(catfchecksumtype=="crc64_ecma"):
4028 catnewfcs = format(crc64_ecma(hout.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4029 elif(catfchecksumtype=="crc64" or catfchecksumtype=="crc64_iso"):
4030 catnewfcs = format(crc64_iso(hout.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4031 elif(CheckSumSupportAlt(catfchecksumtype, hashlib_guaranteed)):
4032 checksumoutstr = hashlib.new(catfchecksumtype);
4033 checksumoutstr.update(hout.encode('UTF-8'));
4034 catnewfcs = checksumoutstr.hexdigest().lower();
4035 if(catfcs!=catnewfcs and not skipchecksum):
4036 VerbosePrintOut("File Header Checksum Error with file " + catfname + " at offset " + str(catfhstart));
4037 return False;
4038 catfhend = catfp.tell() - 1;
4039 catfcontentstart = catfp.tell();
4040 catfcontents = "";
4041 pyhascontents = False;
4042 if(catfsize>0 and not listonly):
4043 catfcontents = catfp.read(catfsize);
4044 if(catfchecksumtype=="none" or catfchecksumtype==""):
4045 catnewfccs = 0;
4046 elif(catfchecksumtype=="crc16" or catfchecksumtype=="crc16_ansi" or catfchecksumtype=="crc16_ibm"):
4047 catnewfccs = format(crc16(catfcontents) & 0xffff, '04x').lower();
4048 elif(catfchecksumtype=="crc16_ccitt"):
4049 catnewfcs = format(crc16_ccitt(catfcontents) & 0xffff, '04x').lower();
4050 elif(catfchecksumtype=="adler32"):
4051 catnewfccs = format(zlib.adler32(catfcontents) & 0xffffffff, '08x').lower();
4052 elif(catfchecksumtype=="crc32"):
4053 catnewfccs = format(crc32(catfcontents) & 0xffffffff, '08x').lower();
4054 elif(catfchecksumtype=="crc64_ecma"):
4055 catnewfcs = format(crc64_ecma(catfcontents) & 0xffffffffffffffff, '016x').lower();
4056 elif(catfchecksumtype=="crc64" or catfchecksumtype=="crc64_iso"):
4057 catnewfcs = format(crc64_iso(catfcontents) & 0xffffffffffffffff, '016x').lower();
4058 elif(CheckSumSupportAlt(catfchecksumtype, hashlib_guaranteed)):
4059 checksumoutstr = hashlib.new(catfchecksumtype);
4060 checksumoutstr.update(catfcontents);
4061 catnewfccs = checksumoutstr.hexdigest().lower();
4062 pyhascontents = True;
4063 if(catfccs!=catnewfccs and skipchecksum):
4064 VerbosePrintOut("File Content Checksum Error with file " + catfname + " at offset " + str(catfcontentstart));
4065 return False;
4066 if(catfsize>0 and listonly):
4067 catfp.seek(catfsize, 1);
4068 pyhascontents = False;
4069 catfp.seek(1, 1);
4070 catfcontentend = catfp.tell() - 1;
4071 catlist['ffilelist'].update({realidnum: {'fid': realidnum, 'fidalt': fileidnum, 'fheadersize': catfheadsize, 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize, 'fatime': catfatime, 'fmtime': catfmtime, 'fctime': catfctime, 'fbtime': catfbtime, 'fmode': catfmode, 'fchmode': catfchmode, 'ftypemod': catftypemod, 'fwinattributes': catfwinattributes, 'fuid': catfuid, 'funame': catfuname, 'fgid': catfgid, 'fgname': catfgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': catfdev_minor, 'fmajor': catfdev_major, 'frminor': catfrdev_minor, 'frmajor': catfrdev_major, 'fchecksumtype': catfchecksumtype, 'fnumfields': catfnumfields, 'fextrafields': catfextrafields, 'fextrafieldsize': catfextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': catfcs, 'fcontentchecksum': catfccs, 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': catfcontents} });
4072 fileidnum = fileidnum + 1;
4073 realidnum = realidnum + 1;
4074 if(returnfp):
4075 catlist.update({'catfp': catfp});
4076 else:
4077 catfp.close();
4078 return catlist;
4079
4080 create_alias_function("", __file_format_name__, "ToArray", ArchiveFileToArray);
4081
4082 def ArchiveFileStringToArray(catstr, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4083 catfp = BytesIO(catstr);
4084 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4085 return listcatfiles;
4086
4087 create_alias_function("", __file_format_name__, "StringToArray", ArchiveFileStringToArray);
4088
4089 def TarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4090 catfp = BytesIO();
4091 catfp = PackArchiveFileFromTarFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
4092 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4093 return listcatfiles;
4094
4095 def ZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4096 catfp = BytesIO();
4097 catfp = PackArchiveFileFromZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
4098 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4099 return listcatfiles;
4100
4101 if(not rarfile_support):
4102 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4103 return False;
4104
4105 if(rarfile_support):
4106 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4107 catfp = BytesIO();
4108 catfp = PackArchiveFileFromRarFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
4109 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4110 return listcatfiles;
4111
4112 def ListDirToArrayAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
4113 catver = formatspecs[6];
4114 fileheaderver = str(int(catver.replace(".", "")));
4115 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
4116 advancedlist = True;
4117 infilelist = [];
4118 if(infiles=="-"):
4119 for line in sys.stdin:
4120 infilelist.append(line.strip());
4121 infilelist = list(filter(None, infilelist));
4122 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
4123 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
4124 return False;
4125 with open(infiles, "r") as finfile:
4126 for line in finfile:
4127 infilelist.append(line.strip());
4128 infilelist = list(filter(None, infilelist));
4129 else:
4130 if(isinstance(infiles, (list, tuple, ))):
4131 infilelist = list(filter(None, infiles));
4132 elif(isinstance(infiles, (str, ))):
4133 infilelist = list(filter(None, [infiles]));
4134 if(advancedlist):
4135 GetDirList = ListDirAdvanced(infilelist, followlink, False);
4136 else:
4137 GetDirList = ListDir(infilelist, followlink, False);
4138 if(not GetDirList):
4139 return False;
4140 curinode = 0;
4141 curfid = 0;
4142 inodelist = [];
4143 inodetofile = {};
4144 filetoinode = {};
4145 inodetocatinode = {};
4146 fileidnum = 0;
4147 fnumfiles = int(len(GetDirList));
4148 catver = formatspecs[6];
4149 fileheaderver = str(int(catver.replace(".", "")));
4150 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
4151 fnumfileshex = format(int(fnumfiles), 'x').lower();
4152 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs[5]);
4153 catversion = re.findall(r"([\d]+)$", fileheader);
4154 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
4155 if(checksumtype=="none" or checksumtype==""):
4156 catfileheadercshex = format(0, 'x').lower();
4157 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
4158 catfileheadercshex = format(crc16(fileheader.encode('UTF-8')) & 0xffff, '04x').lower();
4159 elif(checksumtype=="crc16_ccitt"):
4160 catfileheadercshex = format(crc16_ccitt(fileheader.encode('UTF-8')) & 0xffff, '04x').lower();
4161 elif(checksumtype=="adler32"):
4162 catfileheadercshex = format(zlib.adler32(fileheader.encode('UTF-8')) & 0xffffffff, '08x').lower();
4163 elif(checksumtype=="crc32"):
4164 catfileheadercshex = format(crc32(fileheader.encode('UTF-8')) & 0xffffffff, '08x').lower();
4165 elif(checksumtype=="crc64_ecma"):
4166 catfileheadercshex = format(crc64_ecma(fileheader.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4167 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
4168 catfileheadercshex = format(crc64_iso(fileheader.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4169 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
4170 checksumoutstr = hashlib.new(checksumtype);
4171 checksumoutstr.update(fileheader.encode('UTF-8'));
4172 catfileheadercshex = checksumoutstr.hexdigest().lower();
4173 else:
4174 catfileheadercshex = format(0, 'x').lower();
4175 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
4176 fheadtell = len(fileheader);
4177 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
4178 for curfname in GetDirList:
4179 if(re.findall("^[.|/]", curfname)):
4180 fname = curfname;
4181 else:
4182 fname = "./"+curfname;
4183 if(verbose):
4184 VerbosePrintOut(fname);
4185 if(not followlink or followlink is None):
4186 fstatinfo = os.lstat(fname);
4187 else:
4188 fstatinfo = os.stat(fname);
4189 fpremode = fstatinfo.st_mode;
4190 finode = fstatinfo.st_ino;
4191 flinkcount = fstatinfo.st_nlink;
4192 ftype = 0;
4193 if(stat.S_ISREG(fpremode)):
4194 ftype = 0;
4195 elif(stat.S_ISLNK(fpremode)):
4196 ftype = 2;
4197 elif(stat.S_ISCHR(fpremode)):
4198 ftype = 3;
4199 elif(stat.S_ISBLK(fpremode)):
4200 ftype = 4;
4201 elif(stat.S_ISDIR(fpremode)):
4202 ftype = 5;
4203 elif(stat.S_ISFIFO(fpremode)):
4204 ftype = 6;
4205 elif(stat.S_ISSOCK(fpremode)):
4206 ftype = 8;
4207 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
4208 ftype = 9;
4209 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
4210 ftype = 10;
4211 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
4212 ftype = 11;
4213 else:
4214 ftype = 0;
4215 flinkname = "";
4216 fbasedir = os.path.dirname(fname);
4217 fcurfid = curfid;
4218 if(not followlink and finode!=0):
4219 if(ftype!=1):
4220 if(finode in inodelist):
4221 ftype = 1;
4222 flinkname = inodetofile[finode];
4223 fcurinode = inodetocatinode[finode];
4224 if(finode not in inodelist):
4225 inodelist.append(finode);
4226 inodetofile.update({finode: fname});
4227 inodetocatinode.update({finode: curinode});
4228 fcurinode = curinode;
4229 curinode = curinode + 1;
4230 else:
4231 fcurinode = curinode;
4232 curinode = curinode + 1;
4233 curfid = curfid + 1;
4234 if(ftype==2):
4235 flinkname = os.readlink(fname);
4236 fdev = fstatinfo.st_dev;
4237 getfdev = GetDevMajorMinor(fdev);
4238 fdev_minor = getfdev[0];
4239 fdev_major = getfdev[1];
4240 frdev = fstatinfo.st_dev;
4241 if(hasattr(fstatinfo, "st_rdev")):
4242 frdev = fstatinfo.st_rdev;
4243 else:
4244 frdev = fstatinfo.st_dev;
4245 getfrdev = GetDevMajorMinor(frdev);
4246 frdev_minor = getfrdev[0];
4247 frdev_major = getfrdev[1];
4248 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
4249 fsize = "0";
4250 if(ftype==0 or ftype==7):
4251 fsize = fstatinfo.st_size;
4252 fatime = fstatinfo.st_atime;
4253 fmtime = fstatinfo.st_mtime;
4254 fctime = fstatinfo.st_ctime;
4255 if(hasattr(fstatinfo, "st_birthtime")):
4256 fbtime = fstatinfo.st_birthtime;
4257 else:
4258 fbtime = fstatinfo.st_ctime;
4259 fmode = fstatinfo.st_mode;
4260 fchmode = stat.S_IMODE(fstatinfo.st_mode);
4261 ftypemod = stat.S_IFMT(fstatinfo.st_mode);
4262 fuid = fstatinfo.st_uid;
4263 fgid = fstatinfo.st_gid;
4264 funame = "";
4265 try:
4266 import pwd;
4267 try:
4268 userinfo = pwd.getpwuid(fstatinfo.st_uid);
4269 funame = userinfo.pw_name;
4270 except KeyError:
4271 funame = "";
4272 except ImportError:
4273 funame = "";
4274 fgname = "";
4275 try:
4276 import grp;
4277 try:
4278 groupinfo = grp.getgrgid(fstatinfo.st_gid);
4279 fgname = groupinfo.gr_name;
4280 except KeyError:
4281 fgname = "";
4282 except ImportError:
4283 fgname = "";
4284 fdev_minor = fdev_minor;
4285 fdev_major = fdev_major;
4286 frdev_minor = frdev_minor;
4287 frdev_major = frdev_major;
4288 finode = finode;
4289 flinkcount = flinkcount;
4290 if(hasattr(fstatinfo, "st_file_attributes")):
4291 fwinattributes = fstatinfo.st_file_attributes;
4292 else:
4293 fwinattributes = 0;
4294 fcontents = "".encode('UTF-8');
4295 chunk_size = 1024;
4296 if(ftype == 0 or ftype == 7):
4297 with open(fname, "rb") as fpc:
4298 while(True):
4299 chunk = fpc.read(chunk_size);
4300 if(not chunk):
4301 break
4302 fcontents += chunk;
4303 if(followlink and (ftype == 1 or ftype == 2)):
4304 flstatinfo = os.stat(flinkname);
4305 with open(flinkname, "rb") as fpc:
4306 while(True):
4307 chunk = fpc.read(chunk_size);
4308 if(not chunk):
4309 break;
4310 fcontents += chunk;
4311 ftypehex = format(ftype, 'x').lower();
4312 extrafields = len(extradata);
4313 extrafieldslist = extradata;
4314 catfextrafields = extrafields;
4315 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
4316 if(len(extradata)>0):
4317 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
4318 extrasizelen = len(extrasizestr);
4319 extrasizelenhex = format(extrasizelen, 'x').lower();
4320 catfileoutstr = AppendNullBytes([ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()], formatspecs[5]);
4321 if(len(extradata)>0):
4322 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
4323 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
4324 catfnumfields = 24 + catfextrafields;
4325 if(checksumtype=="none" or checksumtype==""):
4326 catfileheadercshex = format(0, 'x').lower();
4327 catfilecontentcshex = format(0, 'x').lower();
4328 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
4329 catfileheadercshex = format(crc16("".encode('UTF-8')) & 0xffff, '04x').lower();
4330 catfilecontentcshex = format(crc16(fcontents) & 0xffff, '04x').lower();
4331 elif(checksumtype=="crc16_ccitt"):
4332 catfileheadercshex = format(crc16_ccitt("".encode('UTF-8')) & 0xffff, '04x').lower();
4333 catfilecontentcshex = format(crc16_ccitt(fcontents) & 0xffff, '04x').lower();
4334 elif(checksumtype=="adler32"):
4335 catfileheadercshex = format(zlib.adler32("".encode('UTF-8')) & 0xffffffff, '08x').lower();
4336 catfilecontentcshex = format(zlib.adler32(fcontents) & 0xffffffff, '08x').lower();
4337 elif(checksumtype=="crc32"):
4338 catfileheadercshex = format(crc32("".encode('UTF-8')) & 0xffffffff, '08x').lower();
4339 catfilecontentcshex = format(crc32(fcontents) & 0xffffffff, '08x').lower();
4340 elif(checksumtype=="crc64_ecma"):
4341 catfileheadercshex = format(crc64_ecma("".encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4342 catfilecontentcshex = format(crc64_ecma(fcontents) & 0xffffffffffffffff, '016x').lower();
4343 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
4344 catfileheadercshex = format(crc64_iso("".encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4345 catfilecontentcshex = format(crc64_iso(fcontents) & 0xffffffffffffffff, '016x').lower();
4346 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
4347 checksumoutstr = hashlib.new(checksumtype);
4348 checksumoutstr.update("".encode('UTF-8'));
4349 catfileheadercshex = checksumoutstr.hexdigest().lower();
4350 checksumoutstr = hashlib.new(checksumtype);
4351 checksumoutstr.update(fcontents);
4352 catfilecontentcshex = checksumoutstr.hexdigest().lower();
4353 else:
4354 catfileheadercshex = format(0, 'x').lower();
4355 catfilecontentcshex = format(0, 'x').lower();
4356 catfhstart = fheadtell;
4357 fheadtell += len(catfileoutstr);
4358 catfhend = fheadtell - 1;
4359 catfcontentstart = fheadtell;
4360 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
4361 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower()
4362 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
4363 if(checksumtype=="none" or checksumtype==""):
4364 catfileheadercshex = format(0, 'x').lower();
4365 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
4366 catfileheadercshex = format(crc16(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
4367 elif(checksumtype=="crc16_ccitt"):
4368 catfileheadercshex = format(crc16_ccitt(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
4369 elif(checksumtype=="adler32"):
4370 catfileheadercshex = format(zlib.adler32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
4371 elif(checksumtype=="crc32"):
4372 catfileheadercshex = format(crc32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
4373 elif(checksumtype=="crc64_ecma"):
4374 catfileheadercshex = format(crc64_ecma(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4375 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
4376 catfileheadercshex = format(crc64_iso(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4377 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
4378 checksumoutstr = hashlib.new(checksumtype);
4379 checksumoutstr.update(catfileoutstr.encode('UTF-8'));
4380 catfileheadercshex = checksumoutstr.hexdigest().lower();
4381 else:
4382 catfileheadercshex = format(0, 'x').lower();
4383 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
4384 catfileoutstrecd = catfileoutstr.encode('UTF-8');
4385 nullstrecd = formatspecs[5].encode('UTF-8');
4386 fheadtell += len(catfileoutstr) + 1;
4387 catfcontentend = fheadtell - 1;
4388 catfileout = catfileoutstrecd + fcontents + nullstrecd;
4389 pyhascontents = False;
4390 if(int(fsize)>0 and not listonly):
4391 pyhascontents = True;
4392 if(int(fsize)>0 and listonly):
4393 fcontents = "";
4394 pyhascontents = False;
4395 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
4396 fileidnum = fileidnum + 1;
4397 return catlist;
4398
4399 def TarFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
4400 curinode = 0;
4401 curfid = 0;
4402 inodelist = [];
4403 inodetofile = {};
4404 filetoinode = {};
4405 inodetocatinode = {};
4406 fileidnum = 0;
4407 if(infile=="-"):
4408 infile = BytesIO();
4409 if(hasattr(sys.stdin, "buffer")):
4410 shutil.copyfileobj(sys.stdin.buffer, infile);
4411 else:
4412 shutil.copyfileobj(sys.stdin, infile);
4413 infile.seek(0, 0);
4414 if(not infile):
4415 return False;
4416 infile.seek(0, 0);
4417 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", infile)):
4418 infile = download_file_from_internet_file(infile);
4419 infile.seek(0, 0);
4420 if(not infile):
4421 return False;
4422 infile.seek(0, 0);
4423 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
4424 return False;
4425 elif(os.path.exists(infile) and os.path.isfile(infile)):
4426 try:
4427 if(not tarfile.is_tarfile(infile)):
4428 return False;
4429 except AttributeError:
4430 if(not is_tarfile(infile)):
4431 return False;
4432 else:
4433 return False;
4434 try:
4435 if(hasattr(infile, "read") or hasattr(infile, "write")):
4436 tarfp = tarfile.open(fileobj=infile, mode="r");
4437 else:
4438 tarfp = tarfile.open(infile, "r");
4439 except FileNotFoundError:
4440 return False;
4441 fnumfiles = int(len(tarfp.getmembers()));
4442 catver = formatspecs[6];
4443 fileheaderver = str(int(catver.replace(".", "")));
4444 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
4445 fnumfileshex = format(int(fnumfiles), 'x').lower();
4446 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs[5]);
4447 catversion = re.findall(r"([\d]+)$", fileheader);
4448 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
4449 if(checksumtype=="none" or checksumtype==""):
4450 catfileheadercshex = format(0, 'x').lower();
4451 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
4452 catfileheadercshex = format(crc16(fileheader.encode('UTF-8')) & 0xffff, '04x').lower();
4453 elif(checksumtype=="crc16_ccitt"):
4454 catfileheadercshex = format(crc16_ccitt(fileheader.encode('UTF-8')) & 0xffff, '04x').lower();
4455 elif(checksumtype=="adler32"):
4456 catfileheadercshex = format(zlib.adler32(fileheader.encode('UTF-8')) & 0xffffffff, '08x').lower();
4457 elif(checksumtype=="crc32"):
4458 catfileheadercshex = format(crc32(fileheader.encode('UTF-8')) & 0xffffffff, '08x').lower();
4459 elif(checksumtype=="crc64_ecma"):
4460 catfileheadercshex = format(crc64_ecma(fileheader.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4461 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
4462 catfileheadercshex = format(crc64_iso(fileheader.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4463 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
4464 checksumoutstr = hashlib.new(checksumtype);
4465 checksumoutstr.update(fileheader.encode('UTF-8'));
4466 catfileheadercshex = checksumoutstr.hexdigest().lower();
4467 else:
4468 catfileheadercshex = format(0, 'x').lower();
4469 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
4470 fheadtell = len(fileheader);
4471 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
4472 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
4473 if(re.findall("^[.|/]", member.name)):
4474 fname = member.name;
4475 else:
4476 fname = "./"+member.name;
4477 if(verbose):
4478 VerbosePrintOut(fname);
4479 fpremode = member.mode;
4480 ffullmode = member.mode;
4481 flinkcount = 0;
4482 ftype = 0;
4483 if(member.isreg()):
4484 ffullmode = member.mode + stat.S_IFREG;
4485 ftype = 0;
4486 elif(member.isdev()):
4487 ffullmode = member.mode;
4488 ftype = 7;
4489 elif(member.islnk()):
4490 ffullmode = member.mode + stat.S_IFREG;
4491 ftype = 1;
4492 elif(member.issym()):
4493 ffullmode = member.mode + stat.S_IFLNK;
4494 ftype = 2;
4495 elif(member.ischr()):
4496 ffullmode = member.mode + stat.S_IFCHR;
4497 ftype = 3;
4498 elif(member.isblk()):
4499 ffullmode = member.mode + stat.S_IFBLK;
4500 ftype = 4;
4501 elif(member.isdir()):
4502 ffullmode = member.mode + stat.S_IFDIR;
4503 ftype = 5;
4504 elif(member.isfifo()):
4505 ffullmode = member.mode + stat.S_IFIFO;
4506 ftype = 6;
4507 elif(member.issparse()):
4508 ffullmode = member.mode;
4509 ftype = 12;
4510 else:
4511 ffullmode = member.mode;
4512 ftype = 0;
4513 flinkname = "";
4514 fbasedir = os.path.dirname(fname);
4515 fcurfid = curfid;
4516 fcurinode = 0;
4517 finode = fcurinode;
4518 curfid = curfid + 1;
4519 if(ftype==2):
4520 flinkname = member.linkname;
4521 fdev_minor = member.devminor;
4522 fdev_major = member.devmajor;
4523 frdev_minor = member.devminor;
4524 frdev_major = member.devmajor;
4525 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
4526 fsize = "0";
4527 elif(ftype==0 or ftype==7):
4528 fsize = member.size;
4529 else:
4530 fsize = member.size;
4531 fatime = member.mtime;
4532 fmtime = member.mtime;
4533 fctime = member.mtime;
4534 fbtime = member.mtime;
4535 fmode = ffullmode;
4536 fchmode = stat.S_IMODE(ffullmode);
4537 ftypemod = stat.S_IFMT(ffullmode);
4538 fuid = member.uid;
4539 fgid = member.gid;
4540 funame = member.uname;
4541 fgname = member.gname;
4542 flinkcount = flinkcount;
4543 fwinattributes = int(0);
4544 fcontents = "".encode('UTF-8');
4545 chunk_size = 1024;
4546 if(ftype == 0 or ftype == 7):
4547 with tarfp.extractfile(member) as fpc:
4548 while(True):
4549 chunk = fpc.read(chunk_size);
4550 if(not chunk):
4551 break
4552 fcontents += chunk;
4553 ftypehex = format(ftype, 'x').lower();
4554 extrafields = len(extradata);
4555 extrafieldslist = extradata;
4556 catfextrafields = extrafields;
4557 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
4558 if(len(extradata)>0):
4559 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
4560 extrasizelen = len(extrasizestr);
4561 extrasizelenhex = format(extrasizelen, 'x').lower();
4562 catfileoutstr = AppendNullBytes([ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()], formatspecs[5]);
4563 if(len(extradata)>0):
4564 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
4565 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
4566 catfnumfields = 24 + catfextrafields;
4567 if(checksumtype=="none" or checksumtype==""):
4568 catfileheadercshex = format(0, 'x').lower();
4569 catfilecontentcshex = format(0, 'x').lower();
4570 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
4571 catfileheadercshex = format(crc16("".encode('UTF-8')) & 0xffff, '04x').lower();
4572 catfilecontentcshex = format(crc16(fcontents) & 0xffff, '04x').lower();
4573 elif(checksumtype=="crc16_ccitt"):
4574 catfileheadercshex = format(crc16_ccitt("".encode('UTF-8')) & 0xffff, '04x').lower();
4575 catfilecontentcshex = format(crc16_ccitt(fcontents) & 0xffff, '04x').lower();
4576 elif(checksumtype=="adler32"):
4577 catfileheadercshex = format(zlib.adler32("".encode('UTF-8')) & 0xffffffff, '08x').lower();
4578 catfilecontentcshex = format(zlib.adler32(fcontents) & 0xffffffff, '08x').lower();
4579 elif(checksumtype=="crc32"):
4580 catfileheadercshex = format(crc32("".encode('UTF-8')) & 0xffffffff, '08x').lower();
4581 catfilecontentcshex = format(crc32(fcontents) & 0xffffffff, '08x').lower();
4582 elif(checksumtype=="crc64_ecma"):
4583 catfileheadercshex = format(crc64_ecma("".encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4584 catfilecontentcshex = format(crc64_ecma(fcontents) & 0xffffffffffffffff, '016x').lower();
4585 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
4586 catfileheadercshex = format(crc64_iso("".encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4587 catfilecontentcshex = format(crc64_iso(fcontents) & 0xffffffffffffffff, '016x').lower();
4588 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
4589 checksumoutstr = hashlib.new(checksumtype);
4590 checksumoutstr.update("".encode('UTF-8'));
4591 catfileheadercshex = checksumoutstr.hexdigest().lower();
4592 checksumoutstr = hashlib.new(checksumtype);
4593 checksumoutstr.update(fcontents);
4594 catfilecontentcshex = checksumoutstr.hexdigest().lower();
4595 else:
4596 catfileheadercshex = format(0, 'x').lower();
4597 catfilecontentcshex = format(0, 'x').lower();
4598 catfhstart = fheadtell;
4599 fheadtell += len(catfileoutstr);
4600 catfhend = fheadtell - 1;
4601 catfcontentstart = fheadtell;
4602 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
4603 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower()
4604 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
4605 if(checksumtype=="none" or checksumtype==""):
4606 catfileheadercshex = format(0, 'x').lower();
4607 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
4608 catfileheadercshex = format(crc16(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
4609 elif(checksumtype=="crc16_ccitt"):
4610 catfileheadercshex = format(crc16_ccitt(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
4611 elif(checksumtype=="adler32"):
4612 catfileheadercshex = format(zlib.adler32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
4613 elif(checksumtype=="crc32"):
4614 catfileheadercshex = format(crc32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
4615 elif(checksumtype=="crc64_ecma"):
4616 catfileheadercshex = format(crc64_ecma(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4617 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
4618 catfileheadercshex = format(crc64_iso(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4619 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
4620 checksumoutstr = hashlib.new(checksumtype);
4621 checksumoutstr.update(catfileoutstr.encode('UTF-8'));
4622 catfileheadercshex = checksumoutstr.hexdigest().lower();
4623 else:
4624 catfileheadercshex = format(0, 'x').lower();
4625 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
4626 catfileoutstrecd = catfileoutstr.encode('UTF-8');
4627 nullstrecd = formatspecs[5].encode('UTF-8');
4628 fheadtell += len(catfileoutstr) + 1;
4629 catfcontentend = fheadtell - 1;
4630 catfileout = catfileoutstrecd + fcontents + nullstrecd;
4631 pyhascontents = False;
4632 if(int(fsize)>0 and not listonly):
4633 pyhascontents = True;
4634 if(int(fsize)>0 and listonly):
4635 fcontents = "";
4636 pyhascontents = False;
4637 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
4638 fileidnum = fileidnum + 1;
4639 return catlist;
4640
4641 def ZipFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
4642 advancedlist = True;
4643 curinode = 0;
4644 curfid = 0;
4645 inodelist = [];
4646 inodetofile = {};
4647 filetoinode = {};
4648 inodetocatinode = {};
4649 fileidnum = 0;
4650 if(infile=="-"):
4651 infile = BytesIO();
4652 if(hasattr(sys.stdin, "buffer")):
4653 shutil.copyfileobj(sys.stdin.buffer, infile);
4654 else:
4655 shutil.copyfileobj(sys.stdin, infile);
4656 infile.seek(0, 0);
4657 if(not infile):
4658 return False;
4659 infile.seek(0, 0);
4660 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", infile)):
4661 infile = download_file_from_internet_file(infile);
4662 infile.seek(0, 0);
4663 if(not infile):
4664 return False;
4665 infile.seek(0, 0);
4666 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
4667 return False;
4668 else:
4669 return False;
4670 if(not zipfile.is_zipfile(infile)):
4671 return False;
4672 try:
4673 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
4674 except FileNotFoundError:
4675 return False;
4676 ziptest = zipfp.testzip();
4677 if(ziptest):
4678 VerbosePrintOut("Bad file found!");
4679 fnumfiles = int(len(zipfp.infolist()));
4680 catver = formatspecs[6];
4681 fileheaderver = str(int(catver.replace(".", "")));
4682 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
4683 catversion = re.findall(r"([\d]+)$", fileheader);
4684 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
4685 fnumfileshex = format(int(fnumfiles), 'x').lower();
4686 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs[5]);
4687 if(checksumtype=="none" or checksumtype==""):
4688 catfileheadercshex = format(0, 'x').lower();
4689 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
4690 catfileheadercshex = format(crc16(fileheader.encode('UTF-8')) & 0xffff, '04x').lower();
4691 elif(checksumtype=="crc16_ccitt"):
4692 catfileheadercshex = format(crc16_ccitt(fileheader.encode('UTF-8')) & 0xffff, '04x').lower();
4693 elif(checksumtype=="adler32"):
4694 catfileheadercshex = format(zlib.adler32(fileheader.encode('UTF-8')) & 0xffffffff, '08x').lower();
4695 elif(checksumtype=="crc32"):
4696 catfileheadercshex = format(crc32(fileheader.encode('UTF-8')) & 0xffffffff, '08x').lower();
4697 elif(checksumtype=="crc64_ecma"):
4698 catfileheadercshex = format(crc64_ecma(fileheader.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4699 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
4700 catfileheadercshex = format(crc64_iso(fileheader.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4701 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
4702 checksumoutstr = hashlib.new(checksumtype);
4703 checksumoutstr.update(fileheader.encode('UTF-8'));
4704 catfileheadercshex = checksumoutstr.hexdigest().lower();
4705 else:
4706 catfileheadercshex = format(0, 'x').lower();
4707 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
4708 fheadtell = len(fileheader);
4709 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
4710 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
4711 if(re.findall("^[.|/]", member.filename)):
4712 fname = member.filename;
4713 else:
4714 fname = "./"+member.filename;
4715 zipinfo = zipfp.getinfo(member.filename);
4716 if(verbose):
4717 VerbosePrintOut(fname);
4718 if(not member.is_dir()):
4719 fpremode = stat.S_IFREG + 438;
4720 elif(member.is_dir()):
4721 fpremode = stat.S_IFDIR + 511;
4722 flinkcount = 0;
4723 ftype = 0;
4724 if(not member.is_dir()):
4725 ftype = 0;
4726 elif(member.is_dir()):
4727 ftype = 5;
4728 flinkname = "";
4729 fbasedir = os.path.dirname(fname);
4730 fcurfid = curfid;
4731 fcurinode = 0;
4732 finode = fcurinode;
4733 curfid = curfid + 1;
4734 fdev_minor = 0;
4735 fdev_major = 0;
4736 frdev_minor = 0;
4737 frdev_major = 0;
4738 if(ftype==5):
4739 fsize = "0";
4740 elif(ftype==0):
4741 fsize = member.file_size;
4742 else:
4743 fsize = member.file_size;
4744 fatime = time.mktime(member.date_time + (0, 0, -1));
4745 fmtime = time.mktime(member.date_time + (0, 0, -1));
4746 fctime = time.mktime(member.date_time + (0, 0, -1));
4747 fbtime = time.mktime(member.date_time + (0, 0, -1));
4748 if(zipinfo.create_system==0 or zipinfo.create_system==10):
4749 fwinattributes = format(int(zipinfo.external_attr), 'x').lower();
4750 if(not member.is_dir()):
4751 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
4752 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
4753 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
4754 if(member.is_dir()):
4755 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
4756 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
4757 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
4758 elif(zipinfo.create_system==3):
4759 fwinattributes = format(int(0), 'x').lower();
4760 fmode = format(int(zipinfo.external_attr), 'x').lower();
4761 else:
4762 fwinattributes = format(int(0), 'x').lower();
4763 if(not member.is_dir()):
4764 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
4765 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
4766 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
4767 if(member.is_dir()):
4768 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
4769 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
4770 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
4771 try:
4772 fuid = os.getuid();
4773 except AttributeError:
4774 fuid = 0;
4775 except KeyError:
4776 fuid = 0;
4777 try:
4778 fgid = os.getgid();
4779 except AttributeError:
4780 fgid = 0;
4781 except KeyError:
4782 fgid = 0;
4783 try:
4784 import pwd;
4785 try:
4786 userinfo = pwd.getpwuid(os.getuid());
4787 funame = userinfo.pw_name;
4788 except KeyError:
4789 funame = "";
4790 except AttributeError:
4791 funame = "";
4792 except ImportError:
4793 funame = "";
4794 fgname = "";
4795 try:
4796 import grp;
4797 try:
4798 groupinfo = grp.getgrgid(os.getgid());
4799 fgname = groupinfo.gr_name;
4800 except KeyError:
4801 fgname = "";
4802 except AttributeError:
4803 fgname = "";
4804 except ImportError:
4805 fgname = "";
4806 fcontents = "".encode('UTF-8');
4807 if(ftype==0):
4808 fcontents = zipfp.read(member.filename);
4809 ftypehex = format(ftype, 'x').lower();
4810 extrafields = len(extradata);
4811 extrafieldslist = extradata;
4812 catfextrafields = extrafields;
4813 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
4814 if(len(extradata)>0):
4815 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
4816 extrasizelen = len(extrasizestr);
4817 extrasizelenhex = format(extrasizelen, 'x').lower();
4818 catfileoutstr = AppendNullBytes([ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()], formatspecs[5]);
4819 if(len(extradata)>0):
4820 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
4821 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
4822 catfnumfields = 24 + catfextrafields;
4823 if(checksumtype=="none" or checksumtype==""):
4824 catfileheadercshex = format(0, 'x').lower();
4825 catfilecontentcshex = format(0, 'x').lower();
4826 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
4827 catfileheadercshex = format(crc16("".encode('UTF-8')) & 0xffff, '04x').lower();
4828 catfilecontentcshex = format(crc16(fcontents) & 0xffff, '04x').lower();
4829 elif(checksumtype=="crc16_ccitt"):
4830 catfileheadercshex = format(crc16_ccitt("".encode('UTF-8')) & 0xffff, '04x').lower();
4831 catfilecontentcshex = format(crc16_ccitt(fcontents) & 0xffff, '04x').lower();
4832 elif(checksumtype=="adler32"):
4833 catfileheadercshex = format(zlib.adler32("".encode('UTF-8')) & 0xffffffff, '08x').lower();
4834 catfilecontentcshex = format(zlib.adler32(fcontents) & 0xffffffff, '08x').lower();
4835 elif(checksumtype=="crc32"):
4836 catfileheadercshex = format(crc32("".encode('UTF-8')) & 0xffffffff, '08x').lower();
4837 catfilecontentcshex = format(crc32(fcontents) & 0xffffffff, '08x').lower();
4838 elif(checksumtype=="crc64_ecma"):
4839 catfileheadercshex = format(crc64_ecma("".encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4840 catfilecontentcshex = format(crc64_ecma(fcontents) & 0xffffffffffffffff, '016x').lower();
4841 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
4842 catfileheadercshex = format(crc64_iso("".encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4843 catfilecontentcshex = format(crc64_iso(fcontents) & 0xffffffffffffffff, '016x').lower();
4844 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
4845 checksumoutstr = hashlib.new(checksumtype);
4846 checksumoutstr.update("".encode('UTF-8'));
4847 catfileheadercshex = checksumoutstr.hexdigest().lower();
4848 checksumoutstr = hashlib.new(checksumtype);
4849 checksumoutstr.update(fcontents);
4850 catfilecontentcshex = checksumoutstr.hexdigest().lower();
4851 else:
4852 catfileheadercshex = format(0, 'x').lower();
4853 catfilecontentcshex = format(0, 'x').lower();
4854 catfhstart = fheadtell;
4855 fheadtell += len(catfileoutstr);
4856 catfhend = fheadtell - 1;
4857 catfcontentstart = fheadtell;
4858 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
4859 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower()
4860 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
4861 if(checksumtype=="none" or checksumtype==""):
4862 catfileheadercshex = format(0, 'x').lower();
4863 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
4864 catfileheadercshex = format(crc16(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
4865 elif(checksumtype=="crc16_ccitt"):
4866 catfileheadercshex = format(crc16_ccitt(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
4867 elif(checksumtype=="adler32"):
4868 catfileheadercshex = format(zlib.adler32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
4869 elif(checksumtype=="crc32"):
4870 catfileheadercshex = format(crc32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
4871 elif(checksumtype=="crc64_ecma"):
4872 catfileheadercshex = format(crc64_ecma(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4873 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
4874 catfileheadercshex = format(crc64_iso(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4875 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
4876 checksumoutstr = hashlib.new(checksumtype);
4877 checksumoutstr.update(catfileoutstr.encode('UTF-8'));
4878 catfileheadercshex = checksumoutstr.hexdigest().lower();
4879 else:
4880 catfileheadercshex = format(0, 'x').lower();
4881 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
4882 catfileoutstrecd = catfileoutstr.encode('UTF-8');
4883 nullstrecd = formatspecs[5].encode('UTF-8');
4884 fheadtell += len(catfileoutstr) + 1;
4885 catfcontentend = fheadtell - 1;
4886 catfileout = catfileoutstrecd + fcontents + nullstrecd;
4887 pyhascontents = False;
4888 if(int(fsize)>0 and not listonly):
4889 pyhascontents = True;
4890 if(int(fsize)>0 and listonly):
4891 fcontents = "";
4892 pyhascontents = False;
4893 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
4894 fileidnum = fileidnum + 1;
4895 return catlist;
4896
4897 if(not rarfile_support):
4898 def RarFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
4899 return False;
4900
4901 if(rarfile_support):
4902 def RarFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
4903 advancedlist = True;
4904 curinode = 0;
4905 curfid = 0;
4906 inodelist = [];
4907 inodetofile = {};
4908 filetoinode = {};
4909 inodetocatinode = {};
4910 fileidnum = 0;
4911 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
4912 return False;
4913 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
4914 return False;
4915 rarfp = rarfile.RarFile(infile, "r");
4916 rartest = rarfp.testrar();
4917 if(rartest):
4918 VerbosePrintOut("Bad file found!");
4919 fnumfiles = int(len(rarfp.infolist()));
4920 catver = formatspecs[6];
4921 fileheaderver = str(int(catver.replace(".", "")));
4922 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
4923 catversion = re.findall(r"([\d]+)$", fileheader);
4924 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
4925 fnumfileshex = format(int(fnumfiles), 'x').lower();
4926 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs[5]);
4927 if(checksumtype=="none" or checksumtype==""):
4928 catfileheadercshex = format(0, 'x').lower();
4929 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
4930 catfileheadercshex = format(crc16(fileheader.encode('UTF-8')) & 0xffff, '04x').lower();
4931 elif(checksumtype=="crc16_ccitt"):
4932 catfileheadercshex = format(crc16_ccitt(fileheader.encode('UTF-8')) & 0xffff, '04x').lower();
4933 elif(checksumtype=="adler32"):
4934 catfileheadercshex = format(zlib.adler32(fileheader.encode('UTF-8')) & 0xffffffff, '08x').lower();
4935 elif(checksumtype=="crc32"):
4936 catfileheadercshex = format(crc32(fileheader.encode('UTF-8')) & 0xffffffff, '08x').lower();
4937 elif(checksumtype=="crc64_ecma"):
4938 catfileheadercshex = format(crc64_ecma(fileheader.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4939 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
4940 catfileheadercshex = format(crc64_iso(fileheader.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4941 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
4942 checksumoutstr = hashlib.new(checksumtype);
4943 checksumoutstr.update(fileheader.encode('UTF-8'));
4944 catfileheadercshex = checksumoutstr.hexdigest().lower();
4945 else:
4946 catfileheadercshex = format(0, 'x').lower();
4947 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
4948 fheadtell = len(fileheader);
4949 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
4950 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
4951 is_unix = False;
4952 is_windows = False;
4953 if(member.host_os == rarfile.RAR_OS_UNIX):
4954 is_windows = False;
4955 try:
4956 member.external_attr
4957 is_unix = True;
4958 except AttributeError:
4959 is_unix = False;
4960 elif(member.host_os == rarfile.RAR_OS_WIN32):
4961 is_unix = False;
4962 try:
4963 member.external_attr
4964 is_windows = True;
4965 except AttributeError:
4966 is_windows = False;
4967 else:
4968 is_unix = False;
4969 is_windows = False;
4970 if(re.findall("^[.|/]", member.filename)):
4971 fname = member.filename;
4972 else:
4973 fname = "./"+member.filename;
4974 rarinfo = rarfp.getinfo(member.filename);
4975 if(verbose):
4976 VerbosePrintOut(fname);
4977 if(is_unix and member.external_attr !=0):
4978 fpremode = int(member.external_attr);
4979 elif(member.is_file()):
4980 fpremode = stat.S_IFREG + 438;
4981 elif(member.is_symlink()):
4982 fpremode = stat.S_IFLNK + 438;
4983 elif(member.is_dir()):
4984 fpremode = stat.S_IFDIR + 511;
4985 if(is_windows and member.external_attr !=0):
4986 fwinattributes = int(member.external_attr);
4987 else:
4988 fwinattributes = int(0);
4989 flinkcount = 0;
4990 ftype = 0;
4991 if(member.is_file()):
4992 ftype = 0;
4993 elif(member.is_symlink()):
4994 ftype = 2;
4995 elif(member.is_dir()):
4996 ftype = 5;
4997 flinkname = "";
4998 if(ftype==2):
4999 flinkname = rarfp.read(member.filename).decode("UTF-8");
5000 fbasedir = os.path.dirname(fname);
5001 fcurfid = curfid;
5002 fcurinode = 0;
5003 finode = fcurinode;
5004 curfid = curfid + 1;
5005 fdev_minor = 0;
5006 fdev_major = 0;
5007 frdev_minor = 0;
5008 frdev_major = 0;
5009 if(ftype==5):
5010 fsize = "0";
5011 if(ftype==0):
5012 fsize = member.file_size;
5013 try:
5014 if(member.atime):
5015 fatime = int(member.atime.timestamp());
5016 else:
5017 fatime = int(member.mtime.timestamp());
5018 except AttributeError:
5019 fatime = int(member.mtime.timestamp());
5020 fmtime = int(member.mtime.timestamp());
5021 try:
5022 if(member.ctime):
5023 fctime = int(member.ctime.timestamp());
5024 else:
5025 fctime = int(member.mtime.timestamp());
5026 except AttributeError:
5027 fctime = int(member.mtime.timestamp());
5028 fbtime = int(member.mtime.timestamp());
5029 if(is_unix and member.external_attr !=0):
5030 fmode = format(int(member.external_attr), 'x').lower();
5031 fchmode = format(int(stat.S_IMODE(member.external_attr)), 'x').lower();
5032 ftypemod = format(int(stat.S_IFMT(member.external_attr)), 'x').lower();
5033 elif(member.is_file()):
5034 fmode = int(stat.S_IFREG + 438)
5035 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5036 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5037 elif(member.is_symlink()):
5038 fmode = int(stat.S_IFLNK + 438)
5039 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5040 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5041 elif(member.is_dir()):
5042 fmode = int(stat.S_IFDIR + 511)
5043 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
5044 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
5045 try:
5046 fuid = os.getuid();
5047 except AttributeError:
5048 fuid = 0;
5049 except KeyError:
5050 fuid = 0;
5051 try:
5052 fgid = os.getgid();
5053 except AttributeError:
5054 fgid = 0;
5055 except KeyError:
5056 fgid = 0;
5057 try:
5058 import pwd;
5059 try:
5060 userinfo = pwd.getpwuid(os.getuid());
5061 funame = userinfo.pw_name;
5062 except KeyError:
5063 funame = "";
5064 except AttributeError:
5065 funame = "";
5066 except ImportError:
5067 funame = "";
5068 fgname = "";
5069 try:
5070 import grp;
5071 try:
5072 groupinfo = grp.getgrgid(os.getgid());
5073 fgname = groupinfo.gr_name;
5074 except KeyError:
5075 fgname = "";
5076 except AttributeError:
5077 fgname = "";
5078 except ImportError:
5079 fgname = "";
5080 fcontents = "".encode('UTF-8');
5081 if(ftype==0):
5082 fcontents = rarfp.read(member.filename);
5083 ftypehex = format(ftype, 'x').lower();
5084 extrafields = len(extradata);
5085 extrafieldslist = extradata;
5086 catfextrafields = extrafields;
5087 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
5088 if(len(extradata)>0):
5089 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
5090 extrasizelen = len(extrasizestr);
5091 extrasizelenhex = format(extrasizelen, 'x').lower();
5092 catfileoutstr = AppendNullBytes([ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()], formatspecs[5]);
5093 if(len(extradata)>0):
5094 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
5095 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
5096 catfnumfields = 24 + catfextrafields;
5097 if(checksumtype=="none" or checksumtype==""):
5098 catfileheadercshex = format(0, 'x').lower();
5099 catfilecontentcshex = format(0, 'x').lower();
5100 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
5101 catfileheadercshex = format(crc16(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
5102 catfilecontentcshex = format(crc16(fcontents) & 0xffff, '04x').lower();
5103 elif(checksumtype=="crc16_ccitt"):
5104 catfileheadercshex = format(crc16_ccitt(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
5105 catfilecontentcshex = format(crc16_ccitt(fcontents) & 0xffff, '04x').lower();
5106 elif(checksumtype=="adler32"):
5107 catfileheadercshex = format(zlib.adler32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
5108 catfilecontentcshex = format(zlib.adler32(fcontents) & 0xffffffff, '08x').lower();
5109 elif(checksumtype=="crc32"):
5110 catfileheadercshex = format(crc32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
5111 catfilecontentcshex = format(crc32(fcontents) & 0xffffffff, '08x').lower();
5112 elif(checksumtype=="crc64_ecma"):
5113 catfileheadercshex = format(crc64_ecma(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
5114 catfilecontentcshex = format(crc64_ecma(fcontents) & 0xffffffffffffffff, '016x').lower();
5115 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
5116 catfileheadercshex = format(crc64_iso(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
5117 catfilecontentcshex = format(crc64_iso(fcontents) & 0xffffffffffffffff, '016x').lower();
5118 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
5119 checksumoutstr = hashlib.new(checksumtype);
5120 checksumoutstr.update(catfileoutstr.encode('UTF-8'));
5121 catfileheadercshex = checksumoutstr.hexdigest().lower();
5122 else:
5123 catfileheadercshex = format(0, 'x').lower();
5124 catfilecontentcshex = format(0, 'x').lower();
5125 catfhstart = fheadtell;
5126 fheadtell += len(catfileoutstr);
5127 catfhend = fheadtell - 1;
5128 catfcontentstart = fheadtell;
5129 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5130 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower()
5131 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
5132 if(checksumtype=="none" or checksumtype==""):
5133 catfileheadercshex = format(0, 'x').lower();
5134 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
5135 catfileheadercshex = format(crc16(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
5136 elif(checksumtype=="crc16_ccitt"):
5137 catfileheadercshex = format(crc16_ccitt(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
5138 elif(checksumtype=="adler32"):
5139 catfileheadercshex = format(zlib.adler32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
5140 elif(checksumtype=="crc32"):
5141 catfileheadercshex = format(crc32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
5142 elif(checksumtype=="crc64_ecma"):
5143 catfileheadercshex = format(crc64_ecma(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
5144 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
5145 catfileheadercshex = format(crc64_iso(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
5146 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
5147 checksumoutstr = hashlib.new(checksumtype);
5148 checksumoutstr.update(catfileoutstr.encode('UTF-8'));
5149 catfileheadercshex = checksumoutstr.hexdigest().lower();
5150 else:
5151 catfileheadercshex = format(0, 'x').lower();
5152 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5153 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5154 nullstrecd = formatspecs[5].encode('UTF-8');
5155 fheadtell += len(catfileoutstr) + 1;
5156 catfcontentend = fheadtell - 1;
5157 catfileout = catfileoutstrecd + fcontents + nullstrecd;
5158 pyhascontents = False;
5159 if(int(fsize)>0 and not listonly):
5160 pyhascontents = True;
5161 if(int(fsize)>0 and listonly):
5162 fcontents = "";
5163 pyhascontents = False;
5164 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5165 fileidnum = fileidnum + 1;
5166 return catlist;
5167
5168 def ListDirToArray(infiles, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, seekstart=0, seekend=0, listonly=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
5169 outarray = BytesIO();
5170 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
5171 listcatfiles = ArchiveFileToArray(outarray, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5172 return listcatfiles;
5173
5174 def ArchiveFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5175 if(isinstance(infile, dict)):
5176 listcatfiles = infile;
5177 else:
5178 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
5179 infile = RemoveWindowsPath(infile);
5180 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5181 if(not listcatfiles):
5182 return False;
5183 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5184 if(returnfp):
5185 catarray.update({'catfp': listcatfiles['catfp']});
5186 lenlist = len(listcatfiles['ffilelist']);
5187 lcfi = 0;
5188 lcfx = int(listcatfiles['fnumfiles']);
5189 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5190 lcfx = int(lenlist);
5191 else:
5192 lcfx = int(listcatfiles['fnumfiles']);
5193 while(lcfi < lcfx):
5194 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5195 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5196 catarray['filetoid'].update(filetoidarray);
5197 catarray['idtofile'].update(idtofilearray);
5198 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5199 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5200 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5201 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5202 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5203 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5204 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5205 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5206 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5207 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5208 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5209 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5210 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5211 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5212 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5213 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5214 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5215 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5216 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5217 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5218 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5219 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5220 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5221 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5222 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5223 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5224 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
5225 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5226 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5227 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5228 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5229 lcfi = lcfi + 1;
5230 return catarray;
5231
5232 create_alias_function("", __file_format_name__, "ToArrayIndex", ArchiveFileToArrayIndex);
5233
5234 def ListDirToArrayIndexAlt(infiles, dirlistfromtxt=False, followlink=False, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5235 listcatfiles = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, listonly, checksumtype, extradata, formatspecs, verbose);
5236 if(not listcatfiles):
5237 return False;
5238 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5239 lenlist = len(listcatfiles['ffilelist']);
5240 lcfi = 0;
5241 lcfx = int(listcatfiles['fnumfiles']);
5242 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5243 lcfx = int(lenlist);
5244 else:
5245 lcfx = int(listcatfiles['fnumfiles']);
5246 while(lcfi < lcfx):
5247 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5248 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5249 catarray['filetoid'].update(filetoidarray);
5250 catarray['idtofile'].update(idtofilearray);
5251 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5252 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5253 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5254 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5255 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5256 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5257 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5258 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5259 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5260 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5261 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5262 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5263 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5264 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5265 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5266 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5267 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5268 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5269 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5270 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5271 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5272 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5273 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5274 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5275 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5276 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5277 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
5278 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5279 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5280 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5281 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5282 lcfi = lcfi + 1;
5283 return catarray;
5284
5285 def TarFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5286 listcatfiles = TarFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5287 if(not listcatfiles):
5288 return False;
5289 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5290 lenlist = len(listcatfiles['ffilelist']);
5291 lcfi = 0;
5292 lcfx = int(listcatfiles['fnumfiles']);
5293 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5294 lcfx = int(lenlist);
5295 else:
5296 lcfx = int(listcatfiles['fnumfiles']);
5297 while(lcfi < lcfx):
5298 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5299 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5300 catarray['filetoid'].update(filetoidarray);
5301 catarray['idtofile'].update(idtofilearray);
5302 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5303 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5304 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5305 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5306 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5307 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5308 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5309 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5310 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5311 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5312 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5313 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5314 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5315 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5316 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5317 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5318 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5319 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5320 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5321 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5322 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5323 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5324 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5325 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5326 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5327 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5328 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
5329 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5330 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5331 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5332 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5333 lcfi = lcfi + 1;
5334 return catarray;
5335
5336 def ZipFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5337 listcatfiles = ZipFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5338 if(not listcatfiles):
5339 return False;
5340 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5341 lenlist = len(listcatfiles['ffilelist']);
5342 lcfi = 0;
5343 lcfx = int(listcatfiles['fnumfiles']);
5344 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5345 lcfx = int(lenlist);
5346 else:
5347 lcfx = int(listcatfiles['fnumfiles']);
5348 while(lcfi < lcfx):
5349 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5350 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5351 catarray['filetoid'].update(filetoidarray);
5352 catarray['idtofile'].update(idtofilearray);
5353 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5354 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5355 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5356 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5357 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5358 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5359 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5360 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5361 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5362 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5363 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5364 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5365 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5366 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5367 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5368 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5369 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5370 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5371 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5372 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5373 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5374 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5375 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5376 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5377 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5378 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5379 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
5380 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5381 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5382 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5383 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5384 lcfi = lcfi + 1;
5385 return catarray;
5386
5387 if(not rarfile_support):
5388 def RarFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5389 return False;
5390
5391 if(rarfile_support):
5392 def RarFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5393 listcatfiles = RarFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5394 if(not listcatfiles):
5395 return False;
5396 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5397 lenlist = len(listcatfiles['ffilelist']);
5398 lcfi = 0;
5399 lcfx = int(listcatfiles['fnumfiles']);
5400 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5401 lcfx = int(lenlist);
5402 else:
5403 lcfx = int(listcatfiles['fnumfiles']);
5404 while(lcfi < lcfx):
5405 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5406 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5407 catarray['filetoid'].update(filetoidarray);
5408 catarray['idtofile'].update(idtofilearray);
5409 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5410 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5411 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5412 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5413 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5414 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5415 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5416 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5417 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5418 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5419 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5420 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5421 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5422 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5423 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5424 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5425 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5426 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5427 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5428 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5429 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5430 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5431 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5432 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5433 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5434 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5435 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
5436 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5437 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5438 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5439 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5440 lcfi = lcfi + 1;
5441 return catarray;
5442
5443 def ArchiveFileStringToArrayIndex(catstr, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5444 catfp = BytesIO(catstr);
5445 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5446 return listcatfiles;
5447
5448 create_alias_function("", __file_format_name__, "StringToArrayIndex", ArchiveFileStringToArrayIndex);
5449
5450 def TarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5451 catfp = BytesIO();
5452 catfp = PackArchiveFileFromTarFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
5453 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5454 return listcatfiles;
5455
5456 def ZipFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5457 catfp = BytesIO();
5458 catfp = PackArchiveFileFromZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
5459 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5460 return listcatfiles;
5461
5462 if(not rarfile_support):
5463 def RarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5464 return False;
5465
5466 if(rarfile_support):
5467 def RarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5468 catfp = BytesIO();
5469 catfp = PackArchiveFileFromRarFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
5470 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5471 return listcatfiles;
5472
5473 def ListDirToArrayIndex(infiles, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, seekstart=0, seekend=0, listonly=False, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False, returnfp=False):
5474 outarray = BytesIO();
5475 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compressionlevel, followlink, checksumtype, formatspecs, verbose, True);
5476 listcatfiles = ArchiveFileToArrayIndex(outarray, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp)
5477 return listcatfiles;
5478
5479 def RePackArchiveFile(infile, outfile, compression="auto", compressionlevel=None, followlink=False, seekstart=0, seekend=0, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
5480 compressionlist = ['auto', 'gzip', 'bzip2', 'zstd', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
5481 outextlist = ['gz', 'bz2', 'zst', 'lz4', 'lzop', 'lzo', 'lzma', 'xz'];
5482 outextlistwd = ['.gz', '.bz2', '.zst', '.lz4', 'lzop', '.lzo', '.lzma', '.xz'];
5483 if(isinstance(infile, dict)):
5484 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
5485 listcatfiles = prelistcatfiles['list'];
5486 else:
5487 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
5488 infile = RemoveWindowsPath(infile);
5489 if(followlink):
5490 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
5491 listcatfiles = prelistcatfiles['list'];
5492 else:
5493 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
5494 if(outfile!="-" and not hasattr(infile, "read") and not hasattr(outfile, "write")):
5495 outfile = RemoveWindowsPath(outfile);
5496 checksumtype = checksumtype.lower();
5497 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
5498 checksumtype="crc32";
5499 if(checksumtype=="none"):
5500 checksumtype = "";
5501 if(not compression or compression or compression=="catfile" or compression==formatspecs[2]):
5502 compression = None;
5503 if(compression not in compressionlist and compression is None):
5504 compression = "auto";
5505 if(verbose):
5506 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
5507 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
5508 if(os.path.exists(outfile)):
5509 os.unlink(outfile);
5510 if(not listcatfiles):
5511 return False;
5512 if(outfile=="-"):
5513 verbose = False;
5514 catfp = BytesIO();
5515 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
5516 catfp = outfile;
5517 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", outfile)):
5518 catfp = BytesIO();
5519 else:
5520 fbasename = os.path.splitext(outfile)[0];
5521 fextname = os.path.splitext(outfile)[1];
5522 catfp = CompressOpenFile(outfile, compressionlevel);
5523 catver = formatspecs[6];
5524 fileheaderver = str(int(catver.replace(".", "")));
5525 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
5526 catfp.write(fileheader.encode('UTF-8'));
5527 lenlist = len(listcatfiles['ffilelist']);
5528 fnumfiles = int(listcatfiles['fnumfiles']);
5529 if(lenlist>fnumfiles or lenlist<fnumfiles):
5530 fnumfiles = lenlist;
5531 fnumfileshex = format(int(fnumfiles), 'x').lower();
5532 fnumfilesa = AppendNullBytes([fnumfileshex, checksumtype], formatspecs[5]);
5533 if(checksumtype=="none" or checksumtype==""):
5534 catfileheadercshex = format(0, 'x').lower();
5535 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
5536 catfileheadercshex = format(crc16(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffff, '04x').lower();
5537 elif(checksumtype=="crc16_ccitt"):
5538 catfileheadercshex = format(crc16_ccitt(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffff, '04x').lower();
5539 elif(checksumtype=="adler32"):
5540 catfileheadercshex = format(zlib.adler32(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffff, '08x').lower();
5541 elif(checksumtype=="crc32"):
5542 catfileheadercshex = format(crc32(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffff, '08x').lower();
5543 elif(checksumtype=="crc64_ecma"):
5544 catfileheadercshex = format(crc64_ecma(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
5545 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
5546 catfileheadercshex = format(crc64_iso(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
5547 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
5548 checksumoutstr = hashlib.new(checksumtype);
5549 checksumoutstr.update(str(fileheader + fnumfilesa).encode('UTF-8'));
5550 catfileheadercshex = checksumoutstr.hexdigest().lower();
5551 else:
5552 catfileheadercshex = format(0, 'x').lower();
5553 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs[5]);
5554 catfp.write(fnumfilesa.encode('UTF-8'));
5555 try:
5556 catfp.flush();
5557 os.fsync(catfp.fileno());
5558 except io.UnsupportedOperation:
5559 pass;
5560 except AttributeError:
5561 pass;
5562 lenlist = len(listcatfiles['ffilelist']);
5563 fnumfiles = int(listcatfiles['fnumfiles']);
5564 lcfi = 0;
5565 lcfx = int(listcatfiles['fnumfiles']);
5566 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5567 lcfx = int(lenlist);
5568 else:
5569 lcfx = int(listcatfiles['fnumfiles']);
5570 curinode = 0;
5571 curfid = 0;
5572 inodelist = [];
5573 inodetofile = {};
5574 filetoinode = {};
5575 reallcfi = 0;
5576 while(lcfi < lcfx):
5577 catfhstart = catfp.tell();
5578 if(re.findall("^[.|/]", listcatfiles['ffilelist'][reallcfi]['fname'])):
5579 fname = listcatfiles['ffilelist'][reallcfi]['fname'];
5580 else:
5581 fname = "./"+listcatfiles['ffilelist'][reallcfi]['fname'];
5582 if(verbose):
5583 VerbosePrintOut(fname);
5584 fheadersize = format(int(listcatfiles['ffilelist'][reallcfi]['fheadersize']), 'x').lower();
5585 fsize = format(int(listcatfiles['ffilelist'][reallcfi]['fsize']), 'x').lower();
5586 flinkname = listcatfiles['ffilelist'][reallcfi]['flinkname'];
5587 fatime = format(int(listcatfiles['ffilelist'][reallcfi]['fatime']), 'x').lower();
5588 fmtime = format(int(listcatfiles['ffilelist'][reallcfi]['fmtime']), 'x').lower();
5589 fctime = format(int(listcatfiles['ffilelist'][reallcfi]['fctime']), 'x').lower();
5590 fbtime = format(int(listcatfiles['ffilelist'][reallcfi]['fbtime']), 'x').lower();
5591 fmode = format(int(listcatfiles['ffilelist'][reallcfi]['fmode']), 'x').lower();
5592 fchmode = format(int(listcatfiles['ffilelist'][reallcfi]['fchmode']), 'x').lower();
5593 fuid = format(int(listcatfiles['ffilelist'][reallcfi]['fuid']), 'x').lower();
5594 funame = listcatfiles['ffilelist'][reallcfi]['funame'];
5595 fgid = format(int(listcatfiles['ffilelist'][reallcfi]['fgid']), 'x').lower();
5596 fgname = listcatfiles['ffilelist'][reallcfi]['fgname'];
5597 finode = listcatfiles['ffilelist'][reallcfi]['finode'];
5598 flinkcount = listcatfiles['ffilelist'][reallcfi]['flinkcount'];
5599 fwinattributes = flinkinfo['ffilelist'][reallcfi]['fwinattributes'];
5600 fdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['fminor']), 'x').lower();
5601 fdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['fmajor']), 'x').lower();
5602 frdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['frminor']), 'x').lower();
5603 frdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['frmajor']), 'x').lower();
5604 if(len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>listcatfiles['ffilelist'][reallcfi]['fextrafields'] and len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>0):
5605 listcatfiles['ffilelist'][reallcfi]['fextrafields'] = len(listcatfiles['ffilelist'][reallcfi]['fextralist']);
5606 if(len(extradata) > 0):
5607 listcatfiles['ffilelist'][reallcfi]['fextrafields'] = len(extradata);
5608 listcatfiles['ffilelist'][reallcfi]['fextralist'] = extradata;
5609 extrafields = format(int(listcatfiles['ffilelist'][reallcfi]['fextrafields']), 'x').lower();
5610 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
5611 if(len(extradata)>0):
5612 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
5613 extrasizelen = format(len(extrasizestr), 'x').lower();
5614 fcontents = listcatfiles['ffilelist'][reallcfi]['fcontents'];
5615 if(followlink):
5616 if(listcatfiles['ffilelist'][reallcfi]['ftype']==1 or listcatfiles['ffilelist'][reallcfi]['ftype']==2):
5617 getflinkpath = listcatfiles['ffilelist'][reallcfi]['flinkname'];
5618 flinkid = prelistcatfiles['filetoid'][getflinkpath];
5619 flinkinfo = listcatfiles['ffilelist'][flinkid];
5620 fheadersize = format(int(flinkinfo['fheadersize']), 'x').lower();
5621 fsize = format(int(flinkinfo['fsize']), 'x').lower();
5622 flinkname = flinkinfo['flinkname'];
5623 fatime = format(int(flinkinfo['fatime']), 'x').lower();
5624 fmtime = format(int(flinkinfo['fmtime']), 'x').lower();
5625 fctime = format(int(flinkinfo['fctime']), 'x').lower();
5626 fbtime = format(int(flinkinfo['fbtime']), 'x').lower();
5627 fmode = format(int(flinkinfo['fmode']), 'x').lower();
5628 fchmode = format(int(flinkinfo['fchmode']), 'x').lower();
5629 fuid = format(int(flinkinfo['fuid']), 'x').lower();
5630 funame = flinkinfo['funame'];
5631 fgid = format(int(flinkinfo['fgid']), 'x').lower();
5632 fgname = flinkinfo['fgname'];
5633 finode = flinkinfo['finode'];
5634 flinkcount = flinkinfo['flinkcount'];
5635 fwinattributes = flinkinfo['fwinattributes'];
5636 fdev_minor = format(int(flinkinfo['fminor']), 'x').lower();
5637 fdev_major = format(int(flinkinfo['fmajor']), 'x').lower();
5638 frdev_minor = format(int(flinkinfo['frminor']), 'x').lower();
5639 frdev_major = format(int(flinkinfo['frmajor']), 'x').lower();
5640 if(len(flinkinfo['fextralist'])>flinkinfo['fextrafields'] and len(flinkinfo['fextralist'])>0):
5641 flinkinfo['fextrafields'] = len(flinkinfo['fextralist']);
5642 if(len(extradata) > 0):
5643 flinkinfo['fextrafields'] = len(extradata);
5644 flinkinfo['fextralist'] = extradata;
5645 extrafields = format(int(flinkinfo['fextrafields']), 'x').lower();
5646 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
5647 if(len(extradata)>0):
5648 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
5649 extrasizelen = format(len(extrasizestr), 'x').lower();
5650 fcontents = flinkinfo['fcontents'];
5651 if(flinkinfo['ftype']!=0 and flinkinfo['ftype']!=7):
5652 try:
5653 fcontents = fcontents.encode('UTF-8');
5654 except AttributeError:
5655 pass;
5656 ftypehex = format(flinkinfo['ftype'], 'x').lower();
5657 else:
5658 if(listcatfiles['ffilelist'][reallcfi]['ftype']!=0 and listcatfiles['ffilelist'][reallcfi]['ftype']!=7):
5659 try:
5660 fcontents = fcontents.encode('UTF-8');
5661 except AttributeError:
5662 pass;
5663 ftypehex = format(listcatfiles['ffilelist'][reallcfi]['ftype'], 'x').lower();
5664 fcurfid = format(curfid, 'x').lower();
5665 if(not followlink and finode!=0):
5666 if(listcatfiles['ffilelist'][reallcfi]['ftype']!=1):
5667 fcurinode = format(int(curinode), 'x').lower();
5668 inodetofile.update({curinode: fname});
5669 filetoinode.update({fname: curinode});
5670 curinode = curinode + 1;
5671 else:
5672 fcurinode = format(int(filetoinode[flinkname]), 'x').lower();
5673 else:
5674 fcurinode = format(int(curinode), 'x').lower();
5675 curinode = curinode + 1;
5676 curfid = curfid + 1;
5677 catfileoutstr = AppendNullBytes([ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, extrasizelen, extrafields], formatspecs[5]);
5678 if(listcatfiles['ffilelist'][reallcfi]['fextrafields']>0):
5679 extrafieldslist = [];
5680 exi = 0;
5681 exil = listcatfiles['ffilelist'][reallcfi]['fextrafields'];
5682 while(exi < exil):
5683 extrafieldslist.append(listcatfiles['ffilelist'][reallcfi]['fextralist']);
5684 exi = exi + 1;
5685 catfileoutstr += AppendNullBytes([extrafieldslist], formatspecs[5]);
5686 catfileoutstr += AppendNullBytes([checksumtype], formatspecs[5]);
5687 catfhend = (catfp.tell() - 1) + len(catfileoutstr);
5688 catfcontentstart = catfp.tell() + len(catfileoutstr);
5689 if(checksumtype=="none" or checksumtype==""):
5690 catfileheadercshex = format(0, 'x').lower();
5691 catfilecontentcshex = format(0, 'x').lower();
5692 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
5693 catfileheadercshex = format(crc16("".encode('UTF-8')) & 0xffff, '04x').lower();
5694 catfilecontentcshex = format(crc16(fcontents) & 0xffff, '04x').lower();
5695 elif(checksumtype=="crc16_ccitt"):
5696 catfileheadercshex = format(crc16_ccitt("".encode('UTF-8')) & 0xffff, '04x').lower();
5697 catfilecontentcshex = format(crc16_ccitt(fcontents) & 0xffff, '04x').lower();
5698 elif(checksumtype=="adler32"):
5699 catfileheadercshex = format(zlib.adler32("".encode('UTF-8')) & 0xffffffff, '08x').lower();
5700 catfilecontentcshex = format(zlib.adler32(fcontents) & 0xffffffff, '08x').lower();
5701 elif(checksumtype=="crc32"):
5702 catfileheadercshex = format(crc32("".encode('UTF-8')) & 0xffffffff, '08x').lower();
5703 catfilecontentcshex = format(crc32(fcontents) & 0xffffffff, '08x').lower();
5704 elif(checksumtype=="crc64_ecma"):
5705 catfileheadercshex = format(crc64_ecma("".encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
5706 catfilecontentcshex = format(crc64_ecma(fcontents) & 0xffffffffffffffff, '016x').lower();
5707 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
5708 catfileheadercshex = format(crc64_iso("".encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
5709 catfilecontentcshex = format(crc64_iso(fcontents) & 0xffffffffffffffff, '016x').lower();
5710 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
5711 checksumoutstr = hashlib.new(checksumtype);
5712 checksumoutstr.update("".encode('UTF-8'));
5713 catfileheadercshex = checksumoutstr.hexdigest().lower();
5714 checksumoutstr = hashlib.new(checksumtype);
5715 checksumoutstr.update(fcontents);
5716 catfilecontentcshex = checksumoutstr.hexdigest().lower();
5717 else:
5718 catfileheadercshex = format(0, 'x').lower();
5719 catfilecontentcshex = format(0, 'x').lower();
5720 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5721 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower()
5722 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
5723 if(checksumtype=="none" or checksumtype==""):
5724 catfileheadercshex = format(0, 'x').lower();
5725 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
5726 catfileheadercshex = format(crc16(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
5727 elif(checksumtype=="crc16_ccitt"):
5728 catfileheadercshex = format(crc16_ccitt(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
5729 elif(checksumtype=="adler32"):
5730 catfileheadercshex = format(zlib.adler32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
5731 elif(checksumtype=="crc32"):
5732 catfileheadercshex = format(crc32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
5733 elif(checksumtype=="crc64_ecma"):
5734 catfileheadercshex = format(crc64_ecma(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
5735 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
5736 catfileheadercshex = format(crc64_iso(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
5737 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
5738 checksumoutstr = hashlib.new(checksumtype);
5739 checksumoutstr.update(catfileoutstr.encode('UTF-8'));
5740 catfileheadercshex = checksumoutstr.hexdigest().lower();
5741 else:
5742 catfileheadercshex = format(0, 'x').lower();
5743 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5744 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5745 nullstrecd = formatspecs[5].encode('UTF-8');
5746 catfileout = catfileoutstrecd + fcontents + nullstrecd;
5747 catfcontentend = (catfp.tell() - 1) + len(catfileout);
5748 catfp.write(catfileout);
5749 try:
5750 catfp.flush();
5751 os.fsync(catfp.fileno());
5752 except io.UnsupportedOperation:
5753 pass;
5754 except AttributeError:
5755 pass;
5756 lcfi = lcfi + 1;
5757 reallcfi = reallcfi + 1;
5758 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
5759 catfp = CompressArchiveFile(catfp, compression, formatspecs);
5760 try:
5761 catfp.flush();
5762 os.fsync(catfp.fileno());
5763 except io.UnsupportedOperation:
5764 pass;
5765 except AttributeError:
5766 pass;
5767 if(outfile=="-"):
5768 catfp.seek(0, 0);
5769 if(hasattr(sys.stdout, "buffer")):
5770 shutil.copyfileobj(catfp, sys.stdout.buffer);
5771 else:
5772 shutil.copyfileobj(catfp, sys.stdout);
5773 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", outfile)):
5774 catfp = CompressArchiveFile(catfp, compression, formatspecs);
5775 catfp.seek(0, 0);
5776 upload_file_to_internet_file(catfp, outfile);
5777 if(returnfp):
5778 catfp.seek(0, 0);
5779 return catfp;
5780 else:
5781 catfp.close();
5782 return True;
5783
5784 create_alias_function("RePack", __file_format_name__, "", RePackArchiveFile);
5785
5786 def RePackArchiveFileFromString(catstr, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
5787 catfp = BytesIO(catstr);
5788 listcatfiles = RePackArchiveFile(catfp, compression, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
5789 return listcatfiles;
5790
5791 create_alias_function("RePack", __file_format_name__, "FromString", RePackArchiveFileFromString);
5792
5793 def PackArchiveFileFromListDir(infiles, outfile, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
5794 outarray = BytesIO();
5795 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
5796 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
5797 return listcatfiles;
5798
5799 create_alias_function("Pack", __file_format_name__, "FromListDir", PackArchiveFileFromListDir);
5800
5801 def ArchiveFileArrayBase64Encode(infile, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_list__, verbose=False, returnfp=False):
5802 if(verbose):
5803 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
5804 if(isinstance(infile, dict)):
5805 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
5806 listcatfiles = prelistcatfiles['list'];
5807 else:
5808 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
5809 infile = RemoveWindowsPath(infile);
5810 if(followlink):
5811 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
5812 listcatfiles = prelistcatfiles['list'];
5813 else:
5814 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
5815 if(not listcatfiles):
5816 return False;
5817 lenlist = len(listcatfiles['ffilelist']);
5818 fnumfiles = int(listcatfiles['fnumfiles']);
5819 lcfi = 0;
5820 lcfx = int(listcatfiles['fnumfiles']);
5821 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5822 lcfx = int(lenlist);
5823 else:
5824 lcfx = int(listcatfiles['fnumfiles']);
5825 while(lcfi < lcfx):
5826 if(listcatfiles['ffilelist'][lcfi]['fhascontents']):
5827 listcatfiles['ffilelist'][lcfi]['fcontents'] = base64.b64encode(listcatfiles['ffilelist'][lcfi]['fcontents']).decode("UTF-8");
5828 lcfi = lcfi + 1;
5829 return listcatfiles;
5830
5831 create_alias_function("", __file_format_name__, "ArrayBase64Encode", ArchiveFileArrayBase64Encode);
5832
5833 def ArchiveFileArrayBase64Decode(infile, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_list__, verbose=False, returnfp=False):
5834 if(verbose):
5835 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
5836 if(isinstance(infile, dict)):
5837 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
5838 listcatfiles = prelistcatfiles['list'];
5839 else:
5840 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
5841 infile = RemoveWindowsPath(infile);
5842 if(followlink):
5843 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
5844 listcatfiles = prelistcatfiles['list'];
5845 else:
5846 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
5847 if(not listcatfiles):
5848 return False;
5849 lenlist = len(listcatfiles['ffilelist']);
5850 fnumfiles = int(listcatfiles['fnumfiles']);
5851 lcfi = 0;
5852 lcfx = int(listcatfiles['fnumfiles']);
5853 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5854 lcfx = int(lenlist);
5855 else:
5856 lcfx = int(listcatfiles['fnumfiles']);
5857 while(lcfi < lcfx):
5858 if(listcatfiles['ffilelist'][lcfi]['fhascontents']):
5859 listcatfiles['ffilelist'][lcfi]['fcontents'] = base64.b64decode(listcatfiles['ffilelist'][lcfi]['fcontents'].encode("UTF-8"));
5860 lcfi = lcfi + 1;
5861 return listcatfiles;
5862
5863 create_alias_function("", __file_format_name__, "ArrayBase64Decode", ArchiveFileArrayBase64Decode);
5864
5865 def UnPackArchiveFile(infile, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_list__, preservepermissions=True, preservetime=True, verbose=False, returnfp=False):
5866 if(outdir is not None):
5867 outdir = RemoveWindowsPath(outdir);
5868 if(verbose):
5869 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
5870 if(isinstance(infile, dict)):
5871 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
5872 listcatfiles = prelistcatfiles['list'];
5873 else:
5874 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
5875 infile = RemoveWindowsPath(infile);
5876 if(followlink):
5877 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
5878 listcatfiles = prelistcatfiles['list'];
5879 else:
5880 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
5881 if(not listcatfiles):
5882 return False;
5883 lenlist = len(listcatfiles['ffilelist']);
5884 fnumfiles = int(listcatfiles['fnumfiles']);
5885 lcfi = 0;
5886 lcfx = int(listcatfiles['fnumfiles']);
5887 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5888 lcfx = int(lenlist);
5889 else:
5890 lcfx = int(listcatfiles['fnumfiles']);
5891 while(lcfi < lcfx):
5892 funame = "";
5893 try:
5894 import pwd;
5895 try:
5896 userinfo = pwd.getpwuid(listcatfiles['ffilelist'][lcfi]['fuid']);
5897 funame = userinfo.pw_name;
5898 except KeyError:
5899 funame = "";
5900 except ImportError:
5901 funame = "";
5902 fgname = "";
5903 try:
5904 import grp;
5905 try:
5906 groupinfo = grp.getgrgid(listcatfiles['ffilelist'][lcfi]['fgid']);
5907 fgname = groupinfo.gr_name;
5908 except KeyError:
5909 fgname = "";
5910 except ImportError:
5911 fgname = "";
5912 if(verbose):
5913 VerbosePrintOut(listcatfiles['ffilelist'][lcfi]['fname']);
5914 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5915 with open(listcatfiles['ffilelist'][lcfi]['fname'], "wb") as fpc:
5916 fpc.write(listcatfiles['ffilelist'][lcfi]['fcontents'])
5917 try:
5918 fpc.flush()
5919 os.fsync(fpc.fileno())
5920 except io.UnsupportedOperation:
5921 pass
5922 except AttributeError:
5923 pass
5924 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
5925 os.chown(listcatfiles['ffilelist'][lcfi]['fname'], listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
5926 if(preservepermissions):
5927 os.chmod(listcatfiles['ffilelist'][lcfi]['fname'], listcatfiles['ffilelist'][lcfi]['fchmode']);
5928 if(preservetime):
5929 os.utime(listcatfiles['ffilelist'][lcfi]['fname'], (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
5930 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5931 if(followlink):
5932 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
5933 flinkid = prelistcatfiles['filetoid'][getflinkpath];
5934 flinkinfo = listcatfiles['ffilelist'][flinkid];
5935 funame = "";
5936 try:
5937 import pwd;
5938 try:
5939 userinfo = pwd.getpwuid(flinkinfo['fuid']);
5940 funame = userinfo.pw_name;
5941 except KeyError:
5942 funame = "";
5943 except ImportError:
5944 funame = "";
5945 fgname = "";
5946 try:
5947 import grp;
5948 try:
5949 groupinfo = grp.getgrgid(flinkinfo['fgid']);
5950 fgname = groupinfo.gr_name;
5951 except KeyError:
5952 fgname = "";
5953 except ImportError:
5954 fgname = "";
5955 if(flinkinfo['ftype'] == 0 or flinkinfo['ftype'] == 7):
5956 with open(listcatfiles['ffilelist'][lcfi]['fname'], "wb") as fpc:
5957 fpc.write(flinkinfo['fcontents'])
5958 try:
5959 fpc.flush()
5960 os.fsync(fpc.fileno())
5961 except io.UnsupportedOperation:
5962 pass
5963 except AttributeError:
5964 pass
5965 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
5966 os.chown(listcatfiles['ffilelist'][lcfi]['fname'], flinkinfo['fuid'], flinkinfo['fgid']);
5967 if(preservepermissions):
5968 os.chmod(listcatfiles['ffilelist'][lcfi]['fname'], flinkinfo['fchmode']);
5969 if(preservetime):
5970 os.utime(listcatfiles['ffilelist'][lcfi]['fname'], (flinkinfo['fatime'], flinkinfo['fmtime']));
5971 if(flinkinfo['ftype']==1):
5972 os.link(flinkinfo['flinkname'], listcatfiles['ffilelist'][lcfi]['fname']);
5973 if(flinkinfo['ftype']==2):
5974 os.symlink(flinkinfo['flinkname'], listcatfiles['ffilelist'][lcfi]['fname']);
5975 if(flinkinfo['ftype']==5):
5976 if(preservepermissions):
5977 os.mkdir(listcatfiles['ffilelist'][lcfi]['fname'], flinkinfo['fchmode']);
5978 else:
5979 os.mkdir(listcatfiles['ffilelist'][lcfi]['fname']);
5980 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
5981 os.chown(listcatfiles['ffilelist'][lcfi]['fname'], flinkinfo['fuid'], flinkinfo['fgid']);
5982 if(preservepermissions):
5983 os.chmod(listcatfiles['ffilelist'][lcfi]['fname'], flinkinfo['fchmode']);
5984 if(preservetime):
5985 os.utime(listcatfiles['ffilelist'][lcfi]['fname'], (flinkinfo['fatime'], flinkinfo['fmtime']));
5986 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
5987 os.mkfifo(listcatfiles['ffilelist'][lcfi]['fname'], flinkinfo['fchmode']);
5988 else:
5989 os.link(listcatfiles['ffilelist'][lcfi]['flinkname'], listcatfiles['ffilelist'][lcfi]['fname']);
5990 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5991 if(followlink):
5992 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
5993 flinkid = prelistcatfiles['filetoid'][getflinkpath];
5994 flinkinfo = listcatfiles['ffilelist'][flinkid];
5995 funame = "";
5996 try:
5997 import pwd;
5998 try:
5999 userinfo = pwd.getpwuid(flinkinfo['fuid']);
6000 funame = userinfo.pw_name;
6001 except KeyError:
6002 funame = "";
6003 except ImportError:
6004 funame = "";
6005 fgname = "";
6006 try:
6007 import grp;
6008 try:
6009 groupinfo = grp.getgrgid(flinkinfo['fgid']);
6010 fgname = groupinfo.gr_name;
6011 except KeyError:
6012 fgname = "";
6013 except ImportError:
6014 fgname = "";
6015 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
6016 with open(listcatfiles['ffilelist'][lcfi]['fname'], "wb") as fpc:
6017 fpc.write(flinkinfo['fcontents'])
6018 try:
6019 fpc.flush()
6020 os.fsync(fpc.fileno())
6021 except io.UnsupportedOperation:
6022 pass
6023 except AttributeError:
6024 pass
6025 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6026 os.chown(listcatfiles['ffilelist'][lcfi]['fname'], flinkinfo['fuid'], flinkinfo['fgid']);
6027 if(preservepermissions):
6028 os.chmod(listcatfiles['ffilelist'][lcfi]['fname'], flinkinfo['fchmode']);
6029 if(preservetime):
6030 os.utime(listcatfiles['ffilelist'][lcfi]['fname'], (flinkinfo['fatime'], flinkinfo['fmtime']));
6031 if(flinkinfo['ftype']==1):
6032 os.link(flinkinfo['flinkname'], listcatfiles['ffilelist'][lcfi]['fname']);
6033 if(flinkinfo['ftype']==2):
6034 os.symlink(flinkinfo['flinkname'], listcatfiles['ffilelist'][lcfi]['fname']);
6035 if(flinkinfo['ftype']==5):
6036 if(preservepermissions):
6037 os.mkdir(listcatfiles['ffilelist'][lcfi]['fname'], flinkinfo['fchmode']);
6038 else:
6039 os.mkdir(listcatfiles['ffilelist'][lcfi]['fname']);
6040 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6041 os.chown(listcatfiles['ffilelist'][lcfi]['fname'], flinkinfo['fuid'], flinkinfo['fgid']);
6042 if(preservepermissions):
6043 os.chmod(listcatfiles['ffilelist'][lcfi]['fname'], flinkinfo['fchmode']);
6044 if(preservetime):
6045 os.utime(listcatfiles['ffilelist'][lcfi]['fname'], (flinkinfo['fatime'], flinkinfo['fmtime']));
6046 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
6047 os.mkfifo(listcatfiles['ffilelist'][lcfi]['fname'], flinkinfo['fchmode']);
6048 else:
6049 os.symlink(listcatfiles['ffilelist'][lcfi]['flinkname'], listcatfiles['ffilelist'][lcfi]['fname']);
6050 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6051 if(preservepermissions):
6052 os.mkdir(listcatfiles['ffilelist'][lcfi]['fname'], listcatfiles['ffilelist'][lcfi]['fchmode']);
6053 else:
6054 os.mkdir(listcatfiles['ffilelist'][lcfi]['fname']);
6055 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
6056 os.chown(listcatfiles['ffilelist'][lcfi]['fname'], listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
6057 if(preservepermissions):
6058 os.chmod(listcatfiles['ffilelist'][lcfi]['fname'], listcatfiles['ffilelist'][lcfi]['fchmode']);
6059 if(preservetime):
6060 os.utime(listcatfiles['ffilelist'][lcfi]['fname'], (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
6061 if(listcatfiles['ffilelist'][lcfi]['ftype']==6 and hasattr(os, "mkfifo")):
6062 os.mkfifo(listcatfiles['ffilelist'][lcfi]['fname'], listcatfiles['ffilelist'][lcfi]['fchmode']);
6063 lcfi = lcfi + 1;
6064 if(returnfp):
6065 return listcatfiles['ffilelist']['catfp'];
6066 else:
6067 return True;
6068
6069 create_alias_function("UnPack", __file_format_name__, "", UnPackArchiveFile);
6070
6071 if(hasattr(shutil, "register_unpack_format")):
6072 def UnPackArchiveFileFunc(archive_name, extract_dir=None, **kwargs):
6073 return UnPackArchiveFile(archive_name, extract_dir, False, 0, 0, False, __file_format_delimiter__, False, False);
6074 create_alias_function("UnPack", __file_format_name__, "Func", UnPackArchiveFileFunc);
6075
6076 def UnPackArchiveFileString(catstr, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_list__, verbose=False, returnfp=False):
6077 catfp = BytesIO(catstr);
6078 listcatfiles = UnPackArchiveFile(catfp, outdir, followlink, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6079 return listcatfiles;
6080
6081 create_alias_function("UnPack", __file_format_name__, "String", UnPackArchiveFileString);
6082
6083 def ArchiveFileListFiles(infile, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_list__, verbose=False, returnfp=False):
6084 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6085 if(isinstance(infile, dict)):
6086 listcatfiles = infile;
6087 else:
6088 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6089 infile = RemoveWindowsPath(infile);
6090 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, True, skipchecksum, formatspecs, returnfp);
6091 if(not listcatfiles):
6092 return False;
6093 lenlist = len(listcatfiles['ffilelist']);
6094 fnumfiles = int(listcatfiles['fnumfiles']);
6095 lcfi = 0;
6096 lcfx = int(listcatfiles['fnumfiles']);
6097 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6098 lcfx = int(lenlist);
6099 else:
6100 lcfx = int(listcatfiles['fnumfiles']);
6101 returnval = {};
6102 while(lcfi < lcfx):
6103 returnval.update({lcfi: listcatfiles['ffilelist'][lcfi]['fname']});
6104 if(not verbose):
6105 VerbosePrintOut(listcatfiles['ffilelist'][lcfi]['fname']);
6106 if(verbose):
6107 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6108 printfname = listcatfiles['ffilelist'][lcfi]['fname'];
6109 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6110 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " link to " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6111 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6112 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " -> " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6113 fuprint = listcatfiles['ffilelist'][lcfi]['funame'];
6114 if(len(fuprint)<=0):
6115 fuprint = listcatfiles['ffilelist'][lcfi]['fuid'];
6116 fgprint = listcatfiles['ffilelist'][lcfi]['fgname'];
6117 if(len(fgprint)<=0):
6118 fgprint = listcatfiles['ffilelist'][lcfi]['fgid'];
6119 VerbosePrintOut(PrintPermissionString(listcatfiles['ffilelist'][lcfi]['fmode'], listcatfiles['ffilelist'][lcfi]['ftype']) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(listcatfiles['ffilelist'][lcfi]['fsize']).rjust(15) + " " + datetime.datetime.utcfromtimestamp(listcatfiles['ffilelist'][lcfi]['fmtime']).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6120 lcfi = lcfi + 1;
6121 if(returnfp):
6122 return listcatfiles['catfp'];
6123 else:
6124 return True;
6125
6126 create_alias_function("", __file_format_name__, "ListFiles", ArchiveFileListFiles);
6127
6128 def ArchiveFileStringListFiles(catstr, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_list__, verbose=False, returnfp=False):
6129 catfp = BytesIO(catstr);
6130 listcatfiles = ArchiveFileListFiles(catstr, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6131 return listcatfiles;
6132
6133 create_alias_function("", __file_format_name__, "StringListFiles", ArchiveFileListFiles);
6134
6135 def ArchiveFileListFilesAlt(infile, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_list__, verbose=False, returnfp=False):
6136 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6137 if(isinstance(infile, dict)):
6138 listcatfiles = infile;
6139 else:
6140 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6141 infile = RemoveWindowsPath(infile);
6142 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, True, skipchecksum, formatspecs, returnfp);
6143 if(not listcatfiles):
6144 return False;
6145 lenlist = len(listcatfiles['ffilelist']);
6146 fnumfiles = int(listcatfiles['fnumfiles']);
6147 lcfi = 0;
6148 lcfx = int(listcatfiles['fnumfiles']);
6149 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6150 lcfx = int(lenlist);
6151 else:
6152 lcfx = int(listcatfiles['fnumfiles']);
6153 returnval = {};
6154 while(lcfi < lcfx):
6155 returnval.update({lcfi: listcatfiles['ffilelist'][lcfi]['fname']});
6156 if(not verbose):
6157 VerbosePrintOut(listcatfiles['ffilelist'][lcfi]['fname']);
6158 if(verbose):
6159 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6160 printfname = listcatfiles['ffilelist'][lcfi]['fname'];
6161 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6162 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " link to " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6163 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6164 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " -> " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6165 fuprint = listcatfiles['ffilelist'][lcfi]['funame'];
6166 if(len(fuprint)<=0):
6167 fuprint = listcatfiles['ffilelist'][lcfi]['fuid'];
6168 fgprint = listcatfiles['ffilelist'][lcfi]['fgname'];
6169 if(len(fgprint)<=0):
6170 fgprint = listcatfiles['ffilelist'][lcfi]['fgid'];
6171 VerbosePrintOut(PrintPermissionString(listcatfiles['ffilelist'][lcfi]['fmode'], listcatfiles['ffilelist'][lcfi]['ftype']) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(listcatfiles['ffilelist'][lcfi]['fsize']).rjust(15) + " " + datetime.datetime.utcfromtimestamp(listcatfiles['ffilelist'][lcfi]['fmtime']).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6172 lcfi = lcfi + 1;
6173 if(returnfp):
6174 return listcatfiles['catfp'];
6175 else:
6176 return True;
6177
6178 create_alias_function("", __file_format_name__, "ListFilesAlt", ArchiveFileListFilesAlt);
6179
6180 def ArchiveFileStringListFilesAlt(catstr, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_list__, verbose=False, returnfp=False):
6181 catfp = BytesIO(catstr);
6182 listcatfiles = ArchiveFileListFilesAlt(catstr, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6183 return listcatfiles;
6184
6185 create_alias_function("", __file_format_name__, "StringListFilesAlt", ArchiveFileListFilesAlt);
6186
6187 def TarFileListFiles(infile, verbose=False, returnfp=False):
6188 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6189 if(infile=="-"):
6190 infile = BytesIO();
6191 if(hasattr(sys.stdin, "buffer")):
6192 shutil.copyfileobj(sys.stdin.buffer, infile);
6193 else:
6194 shutil.copyfileobj(sys.stdin, infile);
6195 infile.seek(0, 0);
6196 if(not infile):
6197 return False;
6198 infile.seek(0, 0);
6199 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", infile)):
6200 infile = download_file_from_internet_file(infile);
6201 infile.seek(0, 0);
6202 if(not infile):
6203 return False;
6204 infile.seek(0, 0);
6205 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
6206 return False;
6207 elif(os.path.exists(infile) and os.path.isfile(infile)):
6208 try:
6209 if(not tarfile.is_tarfile(infile)):
6210 return False;
6211 except AttributeError:
6212 if(not is_tarfile(infile)):
6213 return False;
6214 else:
6215 return False;
6216 try:
6217 if(hasattr(infile, "read") or hasattr(infile, "write")):
6218 tarfp = tarfile.open(fileobj=infile, mode="r");
6219 else:
6220 tarfp = tarfile.open(infile, "r");
6221 except FileNotFoundError:
6222 return False;
6223 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
6224 returnval.update({lcfi: member.name});
6225 fpremode = member.mode;
6226 ffullmode = member.mode;
6227 flinkcount = 0;
6228 ftype = 0;
6229 if(member.isreg()):
6230 ffullmode = member.mode + stat.S_IFREG;
6231 ftype = 0;
6232 elif(member.isdev()):
6233 ffullmode = member.mode;
6234 ftype = 7;
6235 elif(member.islnk()):
6236 ffullmode = member.mode + stat.S_IFREG;
6237 ftype = 1;
6238 elif(member.issym()):
6239 ffullmode = member.mode + stat.S_IFLNK;
6240 ftype = 2;
6241 elif(member.ischr()):
6242 ffullmode = member.mode + stat.S_IFCHR;
6243 ftype = 3;
6244 elif(member.isblk()):
6245 ffullmode = member.mode + stat.S_IFBLK;
6246 ftype = 4;
6247 elif(member.isdir()):
6248 ffullmode = member.mode + stat.S_IFDIR;
6249 ftype = 5;
6250 elif(member.isfifo()):
6251 ffullmode = member.mode + stat.S_IFIFO;
6252 ftype = 6;
6253 elif(member.issparse()):
6254 ffullmode = member.mode;
6255 ftype = 12;
6256 if(not verbose):
6257 VerbosePrintOut(member.name);
6258 elif(verbose):
6259 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6260 printfname = member.name;
6261 if(member.islnk()):
6262 printfname = member.name + " link to " + member.linkname;
6263 elif(member.issym()):
6264 printfname = member.name + " -> " + member.linkname;
6265 fuprint = member.uname;
6266 if(len(fuprint)<=0):
6267 fuprint = member.uid;
6268 fgprint = member.gname;
6269 if(len(fgprint)<=0):
6270 fgprint = member.gid;
6271 VerbosePrintOut(PrintPermissionString(ffullmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(member.mtime).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6272 lcfi = lcfi + 1;
6273 if(returnfp):
6274 return listcatfiles['catfp'];
6275 else:
6276 return True;
6277
6278 def ZipFileListFiles(infile, verbose=False, returnfp=False):
6279 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6280 if(infile=="-"):
6281 infile = BytesIO();
6282 if(hasattr(sys.stdin, "buffer")):
6283 shutil.copyfileobj(sys.stdin.buffer, infile);
6284 else:
6285 shutil.copyfileobj(sys.stdin, infile);
6286 infile.seek(0, 0);
6287 if(not infile):
6288 return False;
6289 infile.seek(0, 0);
6290 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", infile)):
6291 infile = download_file_from_internet_file(infile);
6292 infile.seek(0, 0);
6293 if(not infile):
6294 return False;
6295 infile.seek(0, 0);
6296 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
6297 return False;
6298 else:
6299 return False;
6300 if(not zipfile.is_zipfile(infile)):
6301 return False;
6302 try:
6303 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
6304 except FileNotFoundError:
6305 return False;
6306 lcfi = 0;
6307 returnval = {};
6308 ziptest = zipfp.testzip();
6309 if(ziptest):
6310 VerbosePrintOut("Bad file found!");
6311 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
6312 if(zipinfo.create_system==0 or zipinfo.create_system==10):
6313 fwinattributes = format(int(zipinfo.external_attr);
6314 if(not member.is_dir()):
6315 fmode = int(stat.S_IFREG + 438);
6316 fchmode = int(stat.S_IMODE(fmode));
6317 ftypemod = int(stat.S_IFMT(fmode));
6318 if(member.is_dir()):
6319 fmode = int(stat.S_IFDIR + 511), 'x');
6320 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6321 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6322 elif(zipinfo.create_system==3):
6323 fwinattributes =int(0);
6324 fmode = int(zipinfo.external_attr);
6325 fchmode = int(stat.S_IMODE(fmode));
6326 ftypemod = int(stat.S_IFMT(fmode));
6327 else:
6328 fwinattributes = int(0);
6329 if(not member.is_dir()):
6330 fmode = int(stat.S_IFREG + 438);
6331 fchmode = int(stat.S_IMODE(fmode));
6332 ftypemod = int(stat.S_IFMT(fmode));
6333 if(member.is_dir()):
6334 fmode = format(int(stat.S_IFDIR + 511);
6335 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6336 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6337 returnval.update({lcfi: member.filename});
6338 if(not verbose):
6339 VerbosePrintOut(member.filename);
6340 if(verbose):
6341 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6342 permissionstr = "";
6343 for fmodval in str(oct(fmode))[-3:]:
6344 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
6345 if(not member.is_dir()):
6346 ftype = 0;
6347 permissionstr = "-" + permissionstr;
6348 elif(member.is_dir()):
6349 ftype = 5;
6350 permissionstr = "d" + permissionstr;
6351 printfname = member.filename;
6352 try:
6353 fuid = int(os.getuid());
6354 except AttributeError:
6355 fuid = int(0);
6356 except KeyError:
6357 fuid = int(0);
6358 try:
6359 fgid = int(os.getgid());
6360 except AttributeError:
6361 fgid = int(0);
6362 except KeyError:
6363 fgid = int(0);
6364 try:
6365 import pwd;
6366 try:
6367 userinfo = pwd.getpwuid(os.getuid());
6368 funame = userinfo.pw_name;
6369 except KeyError:
6370 funame = "";
6371 except AttributeError:
6372 funame = "";
6373 except ImportError:
6374 funame = "";
6375 fgname = "";
6376 try:
6377 import grp;
6378 try:
6379 groupinfo = grp.getgrgid(os.getgid());
6380 fgname = groupinfo.gr_name;
6381 except KeyError:
6382 fgname = "";
6383 except AttributeError:
6384 fgname = "";
6385 except ImportError:
6386 fgname = "";
6387 fuprint = funame;
6388 if(len(fuprint)<=0):
6389 fuprint = str(fuid);
6390 fgprint = fgname;
6391 if(len(fgprint)<=0):
6392 fgprint = str(fgid);
6393 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(int(time.mktime(member.date_time + (0, 0, -1)))).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6394 lcfi = lcfi + 1;
6395 if(returnfp):
6396 return listcatfiles['catfp'];
6397 else:
6398 return True;
6399
6400 if(not rarfile_support):
6401 def RarFileListFiles(infile, verbose=False, returnfp=False):
6402 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6403 if(not os.path.exists(infile) or not os.path.isfile(infile)):
6404 return False;
6405
6406 if(rarfile_support):
6407 def RarFileListFiles(infile, verbose=False, returnfp=False):
6408 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6409 if(not os.path.exists(infile) or not os.path.isfile(infile)):
6410 return False;
6411 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
6412 return False;
6413 lcfi = 0;
6414 returnval = {};
6415 rarfp = rarfile.RarFile(infile, "r");
6416 rartest = rarfp.testrar();
6417 if(rartest):
6418 VerbosePrintOut("Bad file found!");
6419 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
6420 is_unix = False;
6421 is_windows = False;
6422 if(member.host_os == rarfile.RAR_OS_UNIX):
6423 is_windows = False;
6424 try:
6425 member.external_attr
6426 is_unix = True;
6427 except AttributeError:
6428 is_unix = False;
6429 elif(member.host_os == rarfile.RAR_OS_WIN32):
6430 is_unix = False;
6431 try:
6432 member.external_attr
6433 is_windows = True;
6434 except AttributeError:
6435 is_windows = False;
6436 else:
6437 is_unix = False;
6438 is_windows = False;
6439 if(is_unix and member.external_attr !=0):
6440 fpremode = int(member.external_attr);
6441 elif(member.is_file()):
6442 fpremode = int(stat.S_IFREG + 438);
6443 elif(member.is_symlink()):
6444 fpremode = int(stat.S_IFLNK + 438);
6445 elif(member.is_dir()):
6446 fpremode = int(stat.S_IFDIR + 511);
6447 if(is_windows and member.external_attr !=0):
6448 fwinattributes = int(member.external_attr);
6449 else:
6450 fwinattributes = int(0);
6451 if(is_unix and member.external_attr !=0):
6452 fmode = int(member.external_attr);
6453 fchmode = int(stat.S_IMODE(member.external_attr));
6454 ftypemod = int(stat.S_IFMT(member.external_attr));
6455 elif(member.is_file()):
6456 fmode = int(stat.S_IFREG + 438);
6457 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
6458 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
6459 elif(member.is_symlink()):
6460 fmode = int(stat.S_IFLNK + 438);
6461 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
6462 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
6463 elif(member.is_dir()):
6464 fmode = int(stat.S_IFDIR + 511);
6465 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6466 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6467 returnval.update({lcfi: member.filename});
6468 if(not verbose):
6469 VerbosePrintOut(member.filename);
6470 if(verbose):
6471 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6472 permissionstr = "";
6473 for fmodval in str(oct(fmode))[-3:]:
6474 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
6475 if(member.is_file()):
6476 ftype = 0;
6477 permissionstr = "-" + permissionstr;
6478 printfname = member.filename;
6479 elif(member.is_symlink()):
6480 ftype = 2;
6481 permissionstr = "l" + permissionstr;
6482 printfname = member.name + " -> " + member.read().decode("UTF-8");
6483 elif(member.is_dir()):
6484 ftype = 5;
6485 permissionstr = "d" + permissionstr;
6486 printfname = member.filename;
6487 try:
6488 fuid = int(os.getuid());
6489 except AttributeError:
6490 fuid = int(0);
6491 except KeyError:
6492 fuid = int(0);
6493 try:
6494 fgid = int(os.getgid());
6495 except AttributeError:
6496 fgid = int(0);
6497 except KeyError:
6498 fgid = int(0);
6499 try:
6500 import pwd;
6501 try:
6502 userinfo = pwd.getpwuid(os.getuid());
6503 funame = userinfo.pw_name;
6504 except KeyError:
6505 funame = "";
6506 except AttributeError:
6507 funame = "";
6508 except ImportError:
6509 funame = "";
6510 fgname = "";
6511 try:
6512 import grp;
6513 try:
6514 groupinfo = grp.getgrgid(os.getgid());
6515 fgname = groupinfo.gr_name;
6516 except KeyError:
6517 fgname = "";
6518 except AttributeError:
6519 fgname = "";
6520 except ImportError:
6521 fgname = "";
6522 fuprint = funame;
6523 if(len(fuprint)<=0):
6524 fuprint = str(fuid);
6525 fgprint = fgname;
6526 if(len(fgprint)<=0):
6527 fgprint = str(fgid);
6528 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + member.mtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
6529 lcfi = lcfi + 1;
6530 if(returnfp):
6531 return listcatfiles['catfp'];
6532 else:
6533 return True;
6534
6535 def ListDirListFiles(infiles, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False, returnfp=False):
6536 outarray = BytesIO();
6537 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compressionlevel, followlink, checksumtype, formatspecs, False, True);
6538 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6539 return listcatfiles;
6540
6541 def ListDirListFilesAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=True, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False, returnfp=False):
6542 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, listonly, checksumtype, formatspecs, verbose);
6543 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6544 return listcatfiles;
6545
6546 def PackArchiveFileFromListDirAlt(infiles, outfile, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
6547 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, False, checksumtype, extradata, formatspecs, False);
6548 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compressionlevel, followlink, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6549 return listcatfiles;
6550
6551 create_alias_function("Pack", __file_format_name__, "FromListDirAlt", PackArchiveFileFromListDirAlt);
6552
6553 def PackArchiveFileFromTarFileAlt(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
6554 outarray = TarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
6555 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
6556 return listcatfiles;
6557
6558 create_alias_function("Pack", __file_format_name__, "FromTarFileAlt", PackArchiveFileFromTarFileAlt);
6559
6560 def PackArchiveFileFromZipFileAlt(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
6561 outarray = ZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
6562 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
6563 return listcatfiles;
6564
6565 create_alias_function("Pack", __file_format_name__, "FromZipFileAlt", PackArchiveFileFromZipFileAlt);
6566
6567 if(not rarfile_support):
6568 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
6569 return False;
6570
6571 if(rarfile_support):
6572 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
6573 outarray = RarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
6574 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
6575 return listcatfiles;
6576
6577 create_alias_function("Pack", __file_format_name__, "FromRarFileAlt", PackArchiveFileFromRarFileAlt);
6578
6579 def download_file_from_ftp_file(url):
6580 urlparts = urlparse(url);
6581 file_name = os.path.basename(urlparts.path);
6582 file_dir = os.path.dirname(urlparts.path);
6583 if(urlparts.username is not None):
6584 ftp_username = urlparts.username;
6585 else:
6586 ftp_username = "anonymous";
6587 if(urlparts.password is not None):
6588 ftp_password = urlparts.password;
6589 elif(urlparts.password is None and urlparts.username=="anonymous"):
6590 ftp_password = "anonymous";
6591 else:
6592 ftp_password = "";
6593 if(urlparts.scheme=="ftp"):
6594 ftp = FTP();
6595 elif(urlparts.scheme=="ftps"):
6596 ftp = FTP_TLS();
6597 else:
6598 return False;
6599 if(urlparts.scheme=="sftp"):
6600 if(__use_pysftp__):
6601 return download_file_from_pysftp_file(url);
6602 else:
6603 return download_file_from_sftp_file(url);
6604 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
6605 return download_file_from_http_file(url);
6606 ftp_port = urlparts.port;
6607 if(urlparts.port is None):
6608 ftp_port = 21;
6609 try:
6610 ftp.connect(urlparts.hostname, ftp_port);
6611 except socket.gaierror:
6612 log.info("Error With URL "+url);
6613 return False;
6614 except socket.timeout:
6615 log.info("Error With URL "+url);
6616 return False;
6617 ftp.login(urlparts.username, urlparts.password);
6618 if(urlparts.scheme=="ftps"):
6619 ftp.prot_p();
6620 ftpfile = BytesIO();
6621 ftp.retrbinary("RETR "+urlparts.path, ftpfile.write);
6622 #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
6623 ftp.close();
6624 ftpfile.seek(0, 0);
6625 return ftpfile;
6626
6627 def download_file_from_ftp_string(url):
6628 ftpfile = download_file_from_ftp_file(url);
6629 return ftpfile.read();
6630
6631 def upload_file_to_ftp_file(ftpfile, url):
6632 urlparts = urlparse(url);
6633 file_name = os.path.basename(urlparts.path);
6634 file_dir = os.path.dirname(urlparts.path);
6635 if(urlparts.username is not None):
6636 ftp_username = urlparts.username;
6637 else:
6638 ftp_username = "anonymous";
6639 if(urlparts.password is not None):
6640 ftp_password = urlparts.password;
6641 elif(urlparts.password is None and urlparts.username=="anonymous"):
6642 ftp_password = "anonymous";
6643 else:
6644 ftp_password = "";
6645 if(urlparts.scheme=="ftp"):
6646 ftp = FTP();
6647 elif(urlparts.scheme=="ftps"):
6648 ftp = FTP_TLS();
6649 else:
6650 return False;
6651 if(urlparts.scheme=="sftp"):
6652 if(__use_pysftp__):
6653 return upload_file_from_pysftp_file(url);
6654 else:
6655 return upload_file_from_sftp_file(url);
6656 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
6657 return False;
6658 ftp_port = urlparts.port;
6659 if(urlparts.port is None):
6660 ftp_port = 21;
6661 try:
6662 ftp.connect(urlparts.hostname, ftp_port);
6663 except socket.gaierror:
6664 log.info("Error With URL "+url);
6665 return False;
6666 except socket.timeout:
6667 log.info("Error With URL "+url);
6668 return False;
6669 ftp.login(urlparts.username, urlparts.password);
6670 if(urlparts.scheme=="ftps"):
6671 ftp.prot_p();
6672 ftp.storbinary("STOR "+urlparts.path, ftpfile);
6673 ftp.close();
6674 ftpfile.seek(0, 0);
6675 return ftpfile;
6676
6677 def upload_file_to_ftp_string(ftpstring, url):
6678 ftpfileo = BytesIO(ftpstring);
6679 ftpfile = upload_file_to_ftp_file(ftpfileo, url);
6680 ftpfileo.close();
6681 return ftpfile;
6682
6683 def download_file_from_http_file(url, headers=geturls_headers_pycatfile_python_alt):
6684 # Parse the URL to extract username and password if present
6685 urlparts = urlparse(url);
6686 username = urlparts.username;
6687 password = urlparts.password;
6688 # Rebuild the URL without the username and password
6689 netloc = urlparts.hostname;
6690 if(urlparts.scheme=="sftp"):
6691 if(__use_pysftp__):
6692 return download_file_from_pysftp_file(url);
6693 else:
6694 return download_file_from_sftp_file(url);
6695 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
6696 return download_file_from_ftp_file(url);
6697 if urlparts.port:
6698 netloc += ':' + str(urlparts.port);
6699 rebuilt_url = urlunparse((urlparts.scheme, netloc, urlparts.path, urlparts.params, urlparts.query, urlparts.fragment));
6700 # Create a temporary file object
6701 httpfile = BytesIO();
6702 if haverequests:
6703 # Use the requests library if available
6704 if username and password:
6705 response = requests.get(rebuilt_url, headers=headers, auth=(username, password), stream=True);
6706 else:
6707 response = requests.get(rebuilt_url, headers=headers, stream=True);
6708 response.raw.decode_content = True
6709 shutil.copyfileobj(response.raw, httpfile);
6710 else:
6711 # Build a Request object for urllib
6712 request = Request(rebuilt_url, headers=headers);
6713 # Create an opener object for handling URLs
6714 if username and password:
6715 # Create a password manager
6716 password_mgr = HTTPPasswordMgrWithDefaultRealm();
6717 # Add the username and password
6718 password_mgr.add_password(None, rebuilt_url, username, password);
6719 # Create an authentication handler using the password manager
6720 auth_handler = HTTPBasicAuthHandler(password_mgr);
6721 # Build the opener with the authentication handler
6722 opener = build_opener(auth_handler);
6723 else:
6724 opener = build_opener();
6725 with opener.open(request) as response:
6726 shutil.copyfileobj(response, httpfile);
6727 # Reset file pointer to the start
6728 httpfile.seek(0, 0);
6729 # Return the temporary file object
6730 return httpfile;
6731
6732 def download_file_from_http_string(url, headers=geturls_headers_pycatfile_python_alt):
6733 httpfile = download_file_from_http_file(url, headers);
6734 return ftpfile.read();
6735
6736 if(haveparamiko):
6737 def download_file_from_sftp_file(url):
6738 urlparts = urlparse(url);
6739 file_name = os.path.basename(urlparts.path);
6740 file_dir = os.path.dirname(urlparts.path);
6741 sftp_port = urlparts.port;
6742 if(urlparts.port is None):
6743 sftp_port = 22;
6744 else:
6745 sftp_port = urlparts.port;
6746 if(urlparts.username is not None):
6747 sftp_username = urlparts.username;
6748 else:
6749 sftp_username = "anonymous";
6750 if(urlparts.password is not None):
6751 sftp_password = urlparts.password;
6752 elif(urlparts.password is None and urlparts.username=="anonymous"):
6753 sftp_password = "anonymous";
6754 else:
6755 sftp_password = "";
6756 if(urlparts.scheme=="ftp"):
6757 return download_file_from_ftp_file(url);
6758 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
6759 return download_file_from_http_file(url);
6760 if(urlparts.scheme!="sftp"):
6761 return False;
6762 ssh = paramiko.SSHClient();
6763 ssh.load_system_host_keys();
6764 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
6765 try:
6766 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
6767 except paramiko.ssh_exception.SSHException:
6768 return False;
6769 except socket.gaierror:
6770 log.info("Error With URL "+url);
6771 return False;
6772 except socket.timeout:
6773 log.info("Error With URL "+url);
6774 return False;
6775 sftp = ssh.open_sftp();
6776 sftpfile = BytesIO();
6777 sftp.getfo(urlparts.path, sftpfile);
6778 sftp.close();
6779 ssh.close();
6780 sftpfile.seek(0, 0);
6781 return sftpfile;
6782 else:
6783 def download_file_from_sftp_file(url):
6784 return False;
6785
6786 if(haveparamiko):
6787 def download_file_from_sftp_string(url):
6788 sftpfile = download_file_from_sftp_file(url);
6789 return sftpfile.read();
6790 else:
6791 def download_file_from_ftp_string(url):
6792 return False;
6793
6794 if(haveparamiko):
6795 def upload_file_to_sftp_file(sftpfile, url):
6796 urlparts = urlparse(url);
6797 file_name = os.path.basename(urlparts.path);
6798 file_dir = os.path.dirname(urlparts.path);
6799 sftp_port = urlparts.port;
6800 if(urlparts.port is None):
6801 sftp_port = 22;
6802 else:
6803 sftp_port = urlparts.port;
6804 if(urlparts.username is not None):
6805 sftp_username = urlparts.username;
6806 else:
6807 sftp_username = "anonymous";
6808 if(urlparts.password is not None):
6809 sftp_password = urlparts.password;
6810 elif(urlparts.password is None and urlparts.username=="anonymous"):
6811 sftp_password = "anonymous";
6812 else:
6813 sftp_password = "";
6814 if(urlparts.scheme=="ftp"):
6815 return upload_file_from_ftp_file(url);
6816 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
6817 return False;
6818 if(urlparts.scheme!="sftp"):
6819 return False;
6820 ssh = paramiko.SSHClient();
6821 ssh.load_system_host_keys();
6822 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
6823 try:
6824 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
6825 except paramiko.ssh_exception.SSHException:
6826 return False;
6827 except socket.gaierror:
6828 log.info("Error With URL "+url);
6829 return False;
6830 except socket.timeout:
6831 log.info("Error With URL "+url);
6832 return False;
6833 sftp = ssh.open_sftp();
6834 sftp.putfo(sftpfile, urlparts.path);
6835 sftp.close();
6836 ssh.close();
6837 sftpfile.seek(0, 0);
6838 return sftpfile;
6839 else:
6840 def upload_file_to_sftp_file(sftpfile, url):
6841 return False;
6842
6843 if(haveparamiko):
6844 def upload_file_to_sftp_string(sftpstring, url):
6845 sftpfileo = BytesIO(sftpstring);
6846 sftpfile = upload_file_to_sftp_files(ftpfileo, url);
6847 sftpfileo.close();
6848 return sftpfile;
6849 else:
6850 def upload_file_to_sftp_string(url):
6851 return False;
6852
6853 if(havepysftp):
6854 def download_file_from_pysftp_file(url):
6855 urlparts = urlparse(url);
6856 file_name = os.path.basename(urlparts.path);
6857 file_dir = os.path.dirname(urlparts.path);
6858 sftp_port = urlparts.port;
6859 if(urlparts.port is None):
6860 sftp_port = 22;
6861 else:
6862 sftp_port = urlparts.port;
6863 if(urlparts.username is not None):
6864 sftp_username = urlparts.username;
6865 else:
6866 sftp_username = "anonymous";
6867 if(urlparts.password is not None):
6868 sftp_password = urlparts.password;
6869 elif(urlparts.password is None and urlparts.username=="anonymous"):
6870 sftp_password = "anonymous";
6871 else:
6872 sftp_password = "";
6873 if(urlparts.scheme=="ftp"):
6874 return download_file_from_ftp_file(url);
6875 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
6876 return download_file_from_http_file(url);
6877 if(urlparts.scheme!="sftp"):
6878 return False;
6879 try:
6880 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
6881 except paramiko.ssh_exception.SSHException:
6882 return False;
6883 except socket.gaierror:
6884 log.info("Error With URL "+url);
6885 return False;
6886 except socket.timeout:
6887 log.info("Error With URL "+url);
6888 return False;
6889 sftp = ssh.open_sftp();
6890 sftpfile = BytesIO();
6891 sftp.getfo(urlparts.path, sftpfile);
6892 sftp.close();
6893 ssh.close();
6894 sftpfile.seek(0, 0);
6895 return sftpfile;
6896 else:
6897 def download_file_from_pysftp_file(url):
6898 return False;
6899
6900 if(havepysftp):
6901 def download_file_from_pysftp_string(url):
6902 sftpfile = download_file_from_pysftp_file(url);
6903 return sftpfile.read();
6904 else:
6905 def download_file_from_ftp_string(url):
6906 return False;
6907
6908 if(havepysftp):
6909 def upload_file_to_pysftp_file(sftpfile, url):
6910 urlparts = urlparse(url);
6911 file_name = os.path.basename(urlparts.path);
6912 file_dir = os.path.dirname(urlparts.path);
6913 sftp_port = urlparts.port;
6914 if(urlparts.port is None):
6915 sftp_port = 22;
6916 else:
6917 sftp_port = urlparts.port;
6918 if(urlparts.username is not None):
6919 sftp_username = urlparts.username;
6920 else:
6921 sftp_username = "anonymous";
6922 if(urlparts.password is not None):
6923 sftp_password = urlparts.password;
6924 elif(urlparts.password is None and urlparts.username=="anonymous"):
6925 sftp_password = "anonymous";
6926 else:
6927 sftp_password = "";
6928 if(urlparts.scheme=="ftp"):
6929 return upload_file_from_ftp_file(url);
6930 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
6931 return False;
6932 if(urlparts.scheme!="sftp"):
6933 return False;
6934 try:
6935 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
6936 except paramiko.ssh_exception.SSHException:
6937 return False;
6938 except socket.gaierror:
6939 log.info("Error With URL "+url);
6940 return False;
6941 except socket.timeout:
6942 log.info("Error With URL "+url);
6943 return False;
6944 sftp = ssh.open_sftp();
6945 sftp.putfo(sftpfile, urlparts.path);
6946 sftp.close();
6947 ssh.close();
6948 sftpfile.seek(0, 0);
6949 return sftpfile;
6950 else:
6951 def upload_file_to_pysftp_file(sftpfile, url):
6952 return False;
6953
6954 if(havepysftp):
6955 def upload_file_to_pysftp_string(sftpstring, url):
6956 sftpfileo = BytesIO(sftpstring);
6957 sftpfile = upload_file_to_pysftp_files(ftpfileo, url);
6958 sftpfileo.close();
6959 return sftpfile;
6960 else:
6961 def upload_file_to_pysftp_string(url):
6962 return False;
6963
6964 def download_file_from_internet_file(url, headers=geturls_headers_pycatfile_python_alt):
6965 urlparts = urlparse(url);
6966 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
6967 return download_file_from_http_file(url, headers);
6968 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
6969 return download_file_from_ftp_file(url);
6970 elif(urlparts.scheme=="sftp"):
6971 if(__use_pysftp__ and havepysftp):
6972 return download_file_from_pysftp_file(url);
6973 else:
6974 return download_file_from_sftp_file(url);
6975 else:
6976 return False;
6977 return False;
6978
6979 def download_file_from_internet_string(url, headers=geturls_headers_pycatfile_python_alt):
6980 urlparts = urlparse(url);
6981 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
6982 return download_file_from_http_string(url, headers);
6983 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
6984 return download_file_from_ftp_string(url);
6985 elif(urlparts.scheme=="sftp"):
6986 if(__use_pysftp__ and havepysftp):
6987 return download_file_from_pysftp_string(url);
6988 else:
6989 return download_file_from_sftp_string(url);
6990 else:
6991 return False;
6992 return False;
6993
6994 def upload_file_to_internet_file(ifp, url):
6995 urlparts = urlparse(url);
6996 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
6997 return False;
6998 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
6999 return upload_file_from_ftp_file(ifp, url);
7000 elif(urlparts.scheme=="sftp"):
7001 if(__use_pysftp__ and havepysftp):
7002 return upload_file_from_pysftp_file(ifp, url);
7003 else:
7004 return download_file_from_sftp_file(ifp, url);
7005 else:
7006 return False;
7007 return False;
7008
7009 def upload_file_to_internet_string(ifp, url):
7010 urlparts = urlparse(url);
7011 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7012 return False;
7013 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7014 return upload_file_from_ftp_string(ifp, url);
7015 elif(urlparts.scheme=="sftp"):
7016 if(__use_pysftp__ and havepysftp):
7017 return upload_file_from_pysftp_string(ifp, url);
7018 else:
7019 return download_file_from_sftp_string(ifp, url);
7020 else:
7021 return False;
7022 return False;
7023
7024 try:
7025 if(hasattr(shutil, "register_archive_format")):
7026 # Register the packing format
7027 shutil.register_archive_format(__file_format_name__, PackArchiveFileFunc, description='Pack concatenated files');
7028 except shutil.RegistryError:
7029 pass;
7030
7031 try:
7032 if(hasattr(shutil, "register_unpack_format")):
7033 # Register the unpacking format
7034 shutil.register_unpack_format(__file_format_name__, archivefile_extensions, UnPackArchiveFileFunc, description='UnPack concatenated files');
7035 except shutil.RegistryError:
7036 pass;
A tar like file format name catfile after unix cat command (concatenate files).