?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Add files via upload
[PyCatFile.git] / pycatfile.py
blobf1f48d194be55d9146bf9c4f886a8276ff22762f
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 '''
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the Revised BSD License.
7
8 This program is distributed in the hope that it will be useful,
9 but WITHOUT ANY WARRANTY; without even the implied warranty of
10 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 Revised BSD License for more details.
12
13 Copyright 2018-2024 Cool Dude 2k - http://idb.berlios.de/
14 Copyright 2018-2024 Game Maker 2k - http://intdb.sourceforge.net/
15 Copyright 2018-2024 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
16
17 $FileInfo: pycatfile.py - Last Update: 3/15/2024 Ver. 0.5.0 RC 1 - Author: cooldude2k $
18 '''
19
20 from __future__ import absolute_import, division, print_function, unicode_literals;
21 import io, os, re, sys, time, stat, zlib, base64, shutil, socket, hashlib, datetime, logging, binascii, tempfile, zipfile, platform;
22 from ftplib import FTP, FTP_TLS;
23 if(sys.version[0]=="2"):
24 from urlparse import urlparse, urlunparse;
25 elif(sys.version[0]>="3"):
26 from urllib.parse import urlunparse;
27 from urllib.parse import urlparse;
28
29 if os.name == 'nt': # Only modify if on Windows
30 if sys.version[0] == "2":
31 import codecs;
32 sys.stdout = codecs.getwriter('utf-8')(sys.stdout);
33 sys.stderr = codecs.getwriter('utf-8')(sys.stderr);
34 else:
35 import io;
36 sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace', line_buffering=True);
37 sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', errors='replace', line_buffering=True);
38
39 hashlib_guaranteed = False;
40 os.environ["PYTHONIOENCODING"] = "UTF-8";
41 os.environ["LC_CTYPE"] = "UTF-8";
42 try:
43 reload(sys);
44 try:
45 sys.setdefaultencoding('UTF-8');
46 except NameError:
47 pass;
48 except AttributeError:
49 pass;
50 except NameError:
51 pass;
52 except AttributeError:
53 pass;
54
55 try:
56 from zlib import crc32;
57 except ImportError:
58 from binascii import crc32;
59
60 rarfile_support = False;
61 try:
62 import rarfile;
63 rarfile_support = True;
64 except ImportError:
65 rarfile_support = False;
66
67 try:
68 from safetar import is_tarfile;
69 except ImportError:
70 try:
71 from xtarfile import is_tarfile;
72 except ImportError:
73 from tarfile import is_tarfile;
74
75 try:
76 import safetar as tarfile;
77 except ImportError:
78 try:
79 import xtarfile as tarfile;
80 except ImportError:
81 import tarfile;
82
83 haveparamiko = False;
84 try:
85 import paramiko;
86 haveparamiko = True;
87 except ImportError:
88 haveparamiko = False;
89
90 havepysftp = False;
91 try:
92 import pysftp;
93 havepysftp = True;
94 except ImportError:
95 havepysftp = False;
96
97 haverequests = False;
98 try:
99 import requests;
100 haverequests = True;
101 except ImportError:
102 haverequests = False;
103
104 if(haverequests):
105 import urllib3;
106 logging.getLogger("urllib3").setLevel(logging.WARNING);
107
108 try:
109 # Python 3 imports
110 from urllib.request import Request, build_opener, HTTPBasicAuthHandler;
111 from urllib.parse import urlparse;
112 except ImportError:
113 # Python 2 imports
114 from urllib2 import Request, build_opener, HTTPBasicAuthHandler;
115 from urlparse import urlparse;
116
117 if(sys.version[0]=="2"):
118 try:
119 from io import StringIO, BytesIO;
120 except ImportError:
121 try:
122 from cStringIO import StringIO;
123 from cStringIO import StringIO as BytesIO;
124 except ImportError:
125 from StringIO import StringIO;
126 from StringIO import StringIO as BytesIO;
127 elif(sys.version[0]>="3"):
128 from io import StringIO, BytesIO;
129 else:
130 teststringio = 0;
131 if(teststringio<=0):
132 try:
133 from cStringIO import StringIO as BytesIO;
134 teststringio = 1;
135 except ImportError:
136 teststringio = 0;
137 if(teststringio<=0):
138 try:
139 from StringIO import StringIO as BytesIO;
140 teststringio = 2;
141 except ImportError:
142 teststringio = 0;
143 if(teststringio<=0):
144 try:
145 from io import BytesIO;
146 teststringio = 3;
147 except ImportError:
148 teststringio = 0;
149
150 __use_pysftp__ = False;
151 if(not havepysftp):
152 __use_pysftp__ = False;
153 __file_format_name__ = "CatFile";
154 __program_name__ = "Py"+__file_format_name__;
155 __file_format_lower__ = __file_format_name__.lower();
156 __file_format_magic__ = __file_format_name__;
157 __file_format_len__ = len(__file_format_magic__);
158 __file_format_hex__ = binascii.hexlify(__file_format_magic__.encode("UTF-8")).decode("UTF-8");
159 __file_format_delimiter__ = "\x00";
160 __file_format_ver__ = "001";
161 __use_new_style__ = True;
162 __file_format_list__ = [__file_format_name__, __file_format_magic__, __file_format_lower__, __file_format_len__, __file_format_hex__, __file_format_delimiter__, __file_format_ver__, __use_new_style__];
163 __project__ = __program_name__;
164 __project_url__ = "https://github.com/GameMaker2k/PyCatFile";
165 __version_info__ = (0, 6, 0, "RC 1", 1);
166 __version_date_info__ = (2024, 3, 21, "RC 1", 1);
167 __version_date__ = str(__version_date_info__[0]) + "." + str(__version_date_info__[1]).zfill(2) + "." + str(__version_date_info__[2]).zfill(2);
168 __revision__ = __version_info__[3];
169 __revision_id__ = "$Id$";
170 if(__version_info__[4] is not None):
171 __version_date_plusrc__ = __version_date__ + "-" + str(__version_date_info__[4]);
172 if(__version_info__[4] is None):
173 __version_date_plusrc__ = __version_date__;
174 if(__version_info__[3] is not None):
175 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]) + " " + str(__version_info__[3]);
176 if(__version_info__[3] is None):
177 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]);
178
179 PyBitness = platform.architecture();
180 if(PyBitness=="32bit" or PyBitness=="32"):
181 PyBitness = "32";
182 elif(PyBitness=="64bit" or PyBitness=="64"):
183 PyBitness = "64";
184 else:
185 PyBitness = "32";
186
187 geturls_ua_pycatfile_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
188 if(platform.python_implementation()!=""):
189 py_implementation = platform.python_implementation();
190 if(platform.python_implementation()==""):
191 py_implementation = "Python";
192 geturls_ua_pycatfile_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=py_implementation, pyver=platform.python_version(), proname=__project__, prover=__version__);
193 geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
194 geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
195 geturls_headers_pycatfile_python = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
196 geturls_headers_pycatfile_python_alt = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python_alt, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
197 geturls_headers_googlebot_google = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
198 geturls_headers_googlebot_google_old = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
199
200 compressionlist = ['auto', 'gzip', 'bzip2', 'zstd', 'xz', 'lz4', 'lzo', 'lzma'];
201 outextlist = ['gz', 'bz2', 'zst', 'xz', 'lz4', 'lzo', 'lzma'];
202 outextlistwd = ['.gz', '.bz2', '.zst', '.xz', '.lz4', '.lzo', '.lzma'];
203
204 tarfile_mimetype = "application/tar";
205 tarfile_tar_mimetype = tarfile_mimetype;
206 zipfile_mimetype = "application/zip";
207 zipfile_zip_mimetype = zipfile_mimetype;
208 rarfile_mimetype = "application/rar";
209 rarfile_rar_mimetype = rarfile_mimetype;
210 archivefile_mimetype = "application/x-"+__file_format_list__[1]+"";
211 archivefile_cat_mimetype = archivefile_mimetype;
212 archivefile_gzip_mimetype = "application/x-"+__file_format_list__[1]+"+gzip";
213 archivefile_gz_mimetype = archivefile_gzip_mimetype;
214 archivefile_bzip2_mimetype = "application/x-"+__file_format_list__[1]+"+bzip2";
215 archivefile_bz2_mimetype = archivefile_bzip2_mimetype;
216 archivefile_lz4_mimetype = "application/x-"+__file_format_list__[1]+"+lz4";
217 archivefile_lzop_mimetype = "application/x-"+__file_format_list__[1]+"+lzop";
218 archivefile_lzo_mimetype = archivefile_lzop_mimetype;
219 archivefile_zstandard_mimetype = "application/x-"+__file_format_list__[1]+"+zstandard";
220 archivefile_zstd_mimetype = archivefile_zstandard_mimetype;
221 archivefile_lzma_mimetype = "application/x-"+__file_format_list__[1]+"+lzma";
222 archivefile_xz_mimetype = "application/x-"+__file_format_list__[1]+"+xz";
223 archivefile_extensions = ['.cat', '.cat.gz', '.cat.bz2', '.cat.zst', '.cat.lz4', '.cat.lzo', '.cat.lzop', '.cat.lzma', '.cat.xz'];
224
225 if __name__ == "__main__":
226 import subprocess;
227 curscrpath = os.path.dirname(sys.argv[0]);
228 if(curscrpath==""):
229 curscrpath = ".";
230 if(os.sep=="\\"):
231 curscrpath = curscrpath.replace(os.sep, "/");
232 curscrpath = curscrpath + "/";
233 scrfile = curscrpath + "catfile.py";
234 if(os.path.exists(scrfile) and os.path.isfile(scrfile)):
235 scrcmd = subprocess.Popen([sys.executable, scrfile] + sys.argv[1:]);
236 scrcmd.wait();
237
238 def VerbosePrintOut(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
239 if(not dbgenable):
240 return True;
241 log_functions = {
242 "print": print,
243 "log": logging.info,
244 "warning": logging.warning,
245 "error": logging.error,
246 "critical": logging.critical,
247 "exception": logging.exception,
248 "logalt": lambda x: logging.log(dgblevel, x),
249 "debug": logging.debug
250 };
251 log_function = log_functions.get(outtype);
252 if(log_function):
253 log_function(dbgtxt);
254 return True;
255 return False;
256
257 def VerbosePrintOutReturn(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
258 VerbosePrintOut(dbgtxt, outtype, dbgenable, dgblevel);
259 return dbgtxt;
260
261 def RemoveWindowsPath(dpath):
262 if(dpath is None):
263 dpath = "";
264 if(os.sep!="/"):
265 dpath = dpath.replace(os.path.sep, "/");
266 dpath = dpath.rstrip("/");
267 if(dpath=="." or dpath==".."):
268 dpath = dpath + "/";
269 return dpath;
270
271 def NormalizeRelativePath(inpath):
272 inpath = RemoveWindowsPath(inpath);
273 if(os.path.isabs(inpath)):
274 outpath = inpath;
275 else:
276 if(inpath.startswith("./") or inpath.startswith("../")):
277 outpath = inpath;
278 else:
279 outpath = "./" + inpath;
280 return outpath;
281
282 def ListDir(dirpath, followlink=False, duplicates=False):
283 if(isinstance(dirpath, (list, tuple, ))):
284 dirpath = list(filter(None, dirpath));
285 elif(isinstance(dirpath, (str, ))):
286 dirpath = list(filter(None, [dirpath]));
287 retlist = [];
288 for mydirfile in dirpath:
289 if(not os.path.exists(mydirfile)):
290 return False;
291 mydirfile = NormalizeRelativePath(mydirfile);
292 if(os.path.exists(mydirfile) and os.path.islink(mydirfile)):
293 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile));
294 if(os.path.exists(mydirfile) and os.path.isdir(mydirfile)):
295 for root, dirs, filenames in os.walk(mydirfile):
296 dpath = root;
297 dpath = RemoveWindowsPath(dpath);
298 if(dpath not in retlist and not duplicates):
299 retlist.append(dpath);
300 if(duplicates):
301 retlist.append(dpath);
302 for file in filenames:
303 fpath = os.path.join(root, file);
304 fpath = RemoveWindowsPath(fpath);
305 if(fpath not in retlist and not duplicates):
306 retlist.append(fpath);
307 if(duplicates):
308 retlist.append(fpath);
309 else:
310 retlist.append(RemoveWindowsPath(mydirfile));
311 return retlist;
312
313 def ListDirAdvanced(dirpath, followlink=False, duplicates=False):
314 if isinstance(dirpath, (list, tuple)):
315 dirpath = list(filter(None, dirpath));
316 elif isinstance(dirpath, str):
317 dirpath = list(filter(None, [dirpath]));
318 retlist = []
319 for mydirfile in dirpath:
320 if not os.path.exists(mydirfile):
321 return False;
322 mydirfile = NormalizeRelativePath(mydirfile);
323 if os.path.exists(mydirfile) and os.path.islink(mydirfile) and followlink:
324 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile))
325 if os.path.exists(mydirfile) and os.path.isdir(mydirfile):
326 for root, dirs, filenames in os.walk(mydirfile):
327 # Sort dirs and filenames alphabetically in place
328 dirs.sort(key=lambda x: x.lower());
329 filenames.sort(key=lambda x: x.lower());
330 dpath = RemoveWindowsPath(root);
331 if not duplicates and dpath not in retlist:
332 retlist.append(dpath);
333 elif duplicates:
334 retlist.append(dpath);
335 for file in filenames:
336 fpath = os.path.join(root, file);
337 fpath = RemoveWindowsPath(fpath);
338 if not duplicates and fpath not in retlist:
339 retlist.append(fpath);
340 elif duplicates:
341 retlist.append(fpath);
342 else:
343 retlist.append(RemoveWindowsPath(mydirfile));
344 return retlist;
345
346 def create_alias_function(prefix, base_name, suffix, target_function):
347 # Define a new function that wraps the target function
348 def alias_function(*args, **kwargs):
349 return target_function(*args, **kwargs);
350
351 # Create the function name by combining the prefix, base name, and the suffix
352 function_name = "{}{}{}".format(prefix, base_name, suffix);
353
354 # Add the new function to the global namespace
355 globals()[function_name] = alias_function;
356
357 def create_alias_function_alt(prefix, base_name, suffix, target_function):
358 # Create the function name by combining the prefix, base name, and the suffix
359 # Use the format method for string formatting, compatible with Python 2 and 3
360 function_name = "{}{}{}".format(prefix, base_name, suffix)
361 # Add the new function (alias of the target_function) to the global namespace
362 # This line is compatible as-is with both Python 2 and 3
363 globals()[function_name] = target_function
364
365 # initial_value can be 0xFFFF or 0x0000
366 def crc16_ansi(msg, initial_value=0xFFFF):
367 # CRC-16-IBM / CRC-16-ANSI polynomial and initial value
368 poly = 0x8005; # Polynomial for CRC-16-IBM / CRC-16-ANSI
369 crc = initial_value; # Initial value
370 for b in msg:
371 crc ^= b << 8; # XOR byte into CRC top byte
372 for _ in range(8): # Process each bit
373 if crc & 0x8000: # If the top bit is set
374 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
375 else:
376 crc = crc << 1; # Just shift left
377 crc &= 0xFFFF; # Ensure CRC remains 16-bit
378 return crc;
379
380 # initial_value can be 0xFFFF or 0x0000
381 def crc16_ibm(msg, initial_value=0xFFFF):
382 return crc16_ansi(msg, initial_value);
383
384 # initial_value is 0xFFFF
385 def crc16(msg):
386 return crc16_ansi(msg, 0xFFFF);
387
388 # initial_value can be 0xFFFF, 0x1D0F or 0x0000
389 def crc16_ccitt(msg, initial_value=0xFFFF):
390 # CRC-16-CCITT polynomial
391 poly = 0x1021; # Polynomial for CRC-16-CCITT
392 # Use the specified initial value
393 crc = initial_value;
394 for b in msg:
395 crc ^= b << 8; # XOR byte into CRC top byte
396 for _ in range(8): # Process each bit
397 if crc & 0x8000: # If the top bit is set
398 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
399 else:
400 crc = crc << 1; # Just shift left
401 crc &= 0xFFFF; # Ensure CRC remains 16-bit
402 return crc;
403
404 # initial_value can be 0x42F0E1EBA9EA3693 or 0x0000000000000000
405 def crc64_ecma(msg, initial_value=0x0000000000000000):
406 # CRC-64-ECMA polynomial and initial value
407 poly = 0x42F0E1EBA9EA3693;
408 crc = initial_value; # Initial value for CRC-64-ECMA
409 for b in msg:
410 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
411 for _ in range(8): # Process each bit
412 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
413 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
414 else:
415 crc <<= 1; # Just shift left if the MSB is 0
416 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
417 return crc;
418
419 # initial_value can be 0x000000000000001B or 0xFFFFFFFFFFFFFFFF
420 def crc64_iso(msg, initial_value=0xFFFFFFFFFFFFFFFF):
421 # CRC-64-ISO polynomial and initial value
422 poly = 0x000000000000001B;
423 crc = initial_value; # Common initial value for CRC-64-ISO
424 for b in msg:
425 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
426 for _ in range(8): # Process each bit
427 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
428 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
429 else:
430 crc <<= 1; # Just shift left if the MSB is 0
431 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
432 return crc;
433
434 def GetDataFromArray(data, path, default=None):
435 element = data;
436 try:
437 for key in path:
438 element = element[key];
439 return element;
440 except (KeyError, TypeError, IndexError):
441 return default;
442
443 def GetDataFromArrayAlt(structure, path, default=None):
444 element = structure;
445 for key in path:
446 if isinstance(element, dict) and key in element:
447 element = element[key];
448 elif isinstance(element, list) and isinstance(key, int) and -len(element) <= key < len(element):
449 element = element[key];
450 else:
451 return default;
452 return element;
453
454 def GetHeaderChecksum(inlist=[], checksumtype="crc32", encodedata=True, formatspecs=__file_format_list__):
455 if isinstance(inlist, list):
456 fileheader = AppendNullBytes(inlist, formatspecs[5]);
457 else:
458 fileheader = AppendNullByte(inlist, formatspecs[5]);
459 if(encodedata):
460 fileheader = fileheader.encode('UTF-8');
461 if(checksumtype=="none" or checksumtype==""):
462 catfileheadercshex = format(0, 'x').lower();
463 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
464 catfileheadercshex = format(crc16(fileheader) & 0xffff, '04x').lower();
465 elif(checksumtype=="crc16_ccitt"):
466 catfileheadercshex = format(crc16_ccitt(fileheader) & 0xffff, '04x').lower();
467 elif(checksumtype=="adler32"):
468 catfileheadercshex = format(zlib.adler32(fileheader) & 0xffffffff, '08x').lower();
469 elif(checksumtype=="crc32"):
470 catfileheadercshex = format(crc32(fileheader) & 0xffffffff, '08x').lower();
471 elif(checksumtype=="crc64_ecma"):
472 catfileheadercshex = format(crc64_ecma(fileheader) & 0xffffffffffffffff, '016x').lower();
473 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
474 catfileheadercshex = format(crc64_iso(fileheader) & 0xffffffffffffffff, '016x').lower();
475 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
476 checksumoutstr = hashlib.new(checksumtype);
477 checksumoutstr.update(fileheader);
478 catfileheadercshex = checksumoutstr.hexdigest().lower();
479 else:
480 catfileheadercshex = format(0, 'x').lower();
481 return catfileheadercshex;
482
483 def GetFileChecksum(infile, checksumtype="crc32", encodedata=True, formatspecs=__file_format_list__):
484 if(encodedata):
485 infile = infile.encode('UTF-8');
486 if(checksumtype=="none" or checksumtype==""):
487 catinfilecshex = format(0, 'x').lower();
488 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
489 catinfilecshex = format(crc16(infile) & 0xffff, '04x').lower();
490 elif(checksumtype=="crc16_ccitt"):
491 catinfilecshex = format(crc16_ccitt(infile) & 0xffff, '04x').lower();
492 elif(checksumtype=="adler32"):
493 catinfilecshex = format(zlib.adler32(infile) & 0xffffffff, '08x').lower();
494 elif(checksumtype=="crc32"):
495 catinfilecshex = format(crc32(infile) & 0xffffffff, '08x').lower();
496 elif(checksumtype=="crc64_ecma"):
497 catinfilecshex = format(crc64_ecma(infile) & 0xffffffffffffffff, '016x').lower();
498 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
499 catinfilecshex = format(crc64_iso(infile) & 0xffffffffffffffff, '016x').lower();
500 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
501 checksumoutstr = hashlib.new(checksumtype);
502 checksumoutstr.update(infile);
503 catinfilecshex = checksumoutstr.hexdigest().lower();
504 else:
505 catinfilecshex = format(0, 'x').lower();
506 return catinfilecshex;
507
508 def ValidateHeaderChecksum(inlist=[], checksumtype="crc32", inchecksum="0", formatspecs=__file_format_list__):
509 catfileheadercshex = GetHeaderChecksum(inlist, checksumtype, True, formatspecs);
510 inchecksum = inchecksum.lower();
511 catfileheadercshex = catfileheadercshex.lower();
512 if(inchecksum==catfileheadercshex):
513 return True;
514 else:
515 return False;
516
517 def ValidateFileChecksum(infile, checksumtype="crc32", inchecksum="0", formatspecs=__file_format_list__):
518 catinfilecshex = GetFileChecksum(infile, checksumtype, True, formatspecs);
519 inchecksum = inchecksum.lower();
520 catinfilecshex = catinfilecshex.lower();
521 if(inchecksum==catinfilecshex):
522 return True;
523 else:
524 return False;
525
526 def ReadTillNullByte(fp, delimiter=__file_format_delimiter__):
527 curbyte = b"";
528 curfullbyte = b"";
529 nullbyte = delimiter.encode("UTF-8");
530 while(True):
531 curbyte = fp.read(1);
532 if(curbyte==nullbyte or not curbyte):
533 break;
534 curfullbyte = curfullbyte + curbyte;
535 return curfullbyte.decode('UTF-8');
536
537 def ReadUntilNullByte(fp, delimiter=__file_format_delimiter__):
538 return ReadTillNullByte(fp, delimiter);
539
540 def SeekToEndOfFile(fp):
541 lasttell = 0;
542 while(True):
543 fp.seek(1, 1);
544 if(lasttell==fp.tell()):
545 break;
546 lasttell = fp.tell();
547 return True;
548
549 def ReadFileHeaderData(fp, rounds=0, delimiter=__file_format_delimiter__):
550 rocount = 0;
551 roend = int(rounds);
552 HeaderOut = [];
553 while(rocount<roend):
554 HeaderOut.append(ReadTillNullByte(fp, delimiter));
555 rocount = rocount + 1;
556 return HeaderOut;
557
558 def ReadFileHeaderDataBySize(fp, delimiter=__file_format_delimiter__):
559 headerpresize = ReadTillNullByte(fp, delimiter);
560 headersize = int(headerpresize, 16);
561 headercontent = str(fp.read(headersize).decode('UTF-8')).split(delimiter);
562 fp.seek(1, 1);
563 rocount = 0;
564 roend = int(len(headercontent));
565 HeaderOut = [headerpresize];
566 while(rocount<roend):
567 HeaderOut.append(headercontent[rocount]);
568 rocount = rocount + 1;
569 return HeaderOut;
570
571 def ReadFileHeaderDataWoSize(fp, delimiter=__file_format_delimiter__):
572 preheaderdata = ReadFileHeaderData(fp, 2, delimiter);
573 headersize = int(preheaderdata[0], 16);
574 headernumfields = int(preheaderdata[1], 16);
575 headerdata = ReadFileHeaderData(fp, headernumfields, delimiter);
576 HeaderOut = preheaderdata + headerdata;
577 return HeaderOut;
578
579 def ReadFileHeaderDataBySizeWithContent(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
580 delimiter = formatspecs[5];
581 fheaderstart = fp.tell();
582 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
583 if(re.findall("^[.|/]", HeaderOut[2])):
584 fname = HeaderOut[2];
585 else:
586 fname = "./"+HeaderOut[2];
587 fchecksumtype = HeaderOut[-3].lower();
588 fcs = HeaderOut[-2].lower();
589 fccs = HeaderOut[-1].lower();
590 fsize = int(HeaderOut[4], 16);
591 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
592 if(fcs!=newfcs and not skipchecksum):
593 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
594 return False;
595 fhend = fp.tell() - 1;
596 fcontentstart = fp.tell();
597 fcontents = "".encode('UTF-8');
598 if(fsize>0 and not listonly):
599 fcontents = fp.read(fsize);
600 elif(fsize>0 and listonly):
601 fp.seek(fsize, 1);
602 newfccs = GetFileChecksum(fcontents, HeaderOut[-3].lower(), False, formatspecs);
603 if(fccs!=newfccs and not skipchecksum and not listonly):
604 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
605 return False;
606 fp.seek(1, 1);
607 fcontentend = fp.tell() - 1;
608 HeaderOut.append(fcontents);
609 return HeaderOut;
610
611 def ReadFileHeaderDataBySizeWithContentToArray(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
612 delimiter = formatspecs[5];
613 fheaderstart = fp.tell();
614 if(formatspecs[7]):
615 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
616 else:
617 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
618 fheadsize = int(HeaderOut[0], 16);
619 fnumfields = int(HeaderOut[1], 16);
620 ftype = int(HeaderOut[2], 16);
621 if(re.findall("^[.|/]", HeaderOut[3])):
622 fname = HeaderOut[3];
623 else:
624 fname = "./"+HeaderOut[3];
625 fbasedir = os.path.dirname(fname);
626 flinkname = HeaderOut[4];
627 fsize = int(HeaderOut[5], 16);
628 fatime = int(HeaderOut[6], 16);
629 fmtime = int(HeaderOut[7], 16);
630 fctime = int(HeaderOut[8], 16);
631 fbtime = int(HeaderOut[9], 16);
632 fmode = int(HeaderOut[10], 16);
633 fchmode = stat.S_IMODE(fmode);
634 ftypemod = stat.S_IFMT(fmode);
635 fwinattributes = int(HeaderOut[11], 16);
636 fuid = int(HeaderOut[12], 16);
637 funame = HeaderOut[13];
638 fgid = int(HeaderOut[14], 16);
639 fgname = HeaderOut[15];
640 fid = int(HeaderOut[16], 16);
641 finode = int(HeaderOut[17], 16);
642 flinkcount = int(HeaderOut[18], 16);
643 fdev_minor = int(HeaderOut[19], 16);
644 fdev_major = int(HeaderOut[20], 16);
645 frdev_minor = int(HeaderOut[21], 16);
646 frdev_major = int(HeaderOut[22], 16);
647 fextrasize = int(HeaderOut[23], 16);
648 fextrafields = int(HeaderOut[24], 16);
649 extrafieldslist = [];
650 extrastart = 25;
651 extraend = extrastart + fextrafields;
652 extrafieldslist = [];
653 if(extrastart<extraend):
654 extrafieldslist.append(HeaderOut[extrastart]);
655 extrastart = extrastart + 1;
656 fchecksumtype = HeaderOut[extrastart].lower();
657 fcs = HeaderOut[extrastart + 1].lower();
658 fccs = HeaderOut[extrastart + 2].lower();
659 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
660 if(fcs!=newfcs and not skipchecksum):
661 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
662 return False;
663 fhend = fp.tell() - 1;
664 fcontentstart = fp.tell();
665 fcontents = "".encode('UTF-8');
666 pyhascontents = False;
667 if(fsize>0 and not listonly):
668 fcontents = fp.read(fsize);
669 pyhascontents = True;
670 elif(fsize>0 and listonly):
671 fp.seek(fsize, 1);
672 pyhascontents = False;
673 newfccs = GetFileChecksum(fcontents, HeaderOut[-3].lower(), False, formatspecs);
674 if(fccs!=newfccs and not skipchecksum and not listonly):
675 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
676 return False;
677 fp.seek(1, 1);
678 fcontentend = fp.tell() - 1;
679 catlist = {'fheadersize': fheadsize, 'fhstart': fheaderstart, 'fhend': fhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': fchecksumtype, 'fnumfields': fnumfields + 2, 'fextrafields': fextrafields, 'fextrafieldsize': fextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': fcs, 'fcontentchecksum': fccs, 'fhascontents': pyhascontents, 'fcontentstart': fcontentstart, 'fcontentend': fcontentend, 'fcontents': fcontents}
680 return catlist;
681
682 def ReadFileDataBySizeWithContent(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
683 delimiter = formatspecs[5];
684 catheader = ReadFileHeaderData(fp, 4, delimiter);
685 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
686 if(not headercheck and not skipchecksum):
687 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
688 return False;
689 fnumfiles = int(catheader[1], 16);
690 countnum = 0;
691 flist = [];
692 while(countnum < fnumfiles):
693 flist.append(ReadFileHeaderDataBySizeWithContent(fp, listonly, skipchecksum, formatspecs));
694 countnum = countnum + 1;
695 return flist;
696
697 def ReadFileDataBySizeWithContentToArray(fp, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
698 delimiter = formatspecs[5];
699 catheader = ReadFileHeaderData(fp, 4, delimiter);
700 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
701 if(not headercheck and not skipchecksum):
702 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
703 return False;
704 catstring = catheader[0];
705 catversion = re.findall(r"([\d]+)$", catstring);
706 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
707 fprenumfiles = catheader[1];
708 fnumfiles = int(fprenumfiles, 16);
709 fprechecksumtype = catheader[2];
710 fprechecksum = catheader[3];
711 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
712 if(seekstart<0 and seekstart>fnumfiles):
713 seekstart = 0;
714 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
715 seekend = fnumfiles;
716 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
717 seekend = fnumfiles - abs(seekend);
718 if(seekstart>0):
719 il = 0;
720 while(il < seekstart):
721 prefhstart = catfp.tell();
722 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs[5]);
723 prefsize = int(preheaderdata[4], 16);
724 hout = AppendNullBytes(preheaderdata, formatspecs[5]);
725 if(prefchecksumtype=="none" or prefchecksumtype==""):
726 prenewfcs = 0;
727 elif(prefchecksumtype=="crc16" or prefchecksumtype=="crc16_ansi" or prefchecksumtype=="crc16_ibm"):
728 prenewfcs = format(crc16(hout.encode('UTF-8')) & 0xffff, '04x').lower();
729 elif(prefchecksumtype=="adler32"):
730 prenewfcs = format(zlib.adler32(hout.encode('UTF-8')) & 0xffffffff, '08x').lower();
731 elif(prefchecksumtype=="crc32"):
732 prenewfcs = format(crc32(hout.encode('UTF-8')) & 0xffffffff, '08x').lower();
733 elif(prefchecksumtype=="crc64_ecma"):
734 prenewfcs = format(crc64_ecma(hout.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
735 elif(prefchecksumtype=="crc64" or prefchecksumtype=="crc64_iso"):
736 prenewfcs = format(crc64_iso(hout.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
737 elif(CheckSumSupportAlt(prefchecksumtype, hashlib_guaranteed)):
738 checksumoutstr = hashlib.new(prefchecksumtype);
739 checksumoutstr.update(hout.encode('UTF-8'));
740 prenewfcs = checksumoutstr.hexdigest().lower();
741 if(prefcs!=prenewfcs and not skipchecksum):
742 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
743 return False;
744 valid_archive = False;
745 invalid_archive = True;
746 prefhend = catfp.tell() - 1;
747 prefcontentstart = catfp.tell();
748 prefcontents = "";
749 pyhascontents = False;
750 if(prefsize>0):
751 prefcontents = catfp.read(prefsize);
752 if(prefchecksumtype=="none" or prefchecksumtype==""):
753 prenewfccs = 0;
754 elif(prefchecksumtype=="crc16" or prefchecksumtype=="crc16_ansi" or prefchecksumtype=="crc16_ibm"):
755 prenewfccs = format(crc16(prefcontents) & 0xffff, '04x').lower();
756 elif(prefchecksumtype=="crc16_ccitt"):
757 prenewfcs = format(crc16_ccitt(prefcontents) & 0xffff, '04x').lower();
758 elif(prefchecksumtype=="adler32"):
759 prenewfccs = format(zlib.adler32(prefcontents) & 0xffffffff, '08x').lower();
760 elif(prefchecksumtype=="crc32"):
761 prenewfccs = format(crc32(prefcontents) & 0xffffffff, '08x').lower();
762 elif(prefchecksumtype=="crc64_ecma"):
763 prenewfcs = format(crc64_ecma(prefcontents) & 0xffffffffffffffff, '016x').lower();
764 elif(prefchecksumtype=="crc64" or prefchecksumtype=="crc64_iso"):
765 prenewfcs = format(crc64_iso(prefcontents) & 0xffffffffffffffff, '016x').lower();
766 elif(CheckSumSupportAlt(prefchecksumtype, hashlib_guaranteed)):
767 checksumoutstr = hashlib.new(prefchecksumtype);
768 checksumoutstr.update(prefcontents);
769 prenewfccs = checksumoutstr.hexdigest().lower();
770 pyhascontents = True;
771 if(prefccs!=prenewfccs and not skipchecksum):
772 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
773 return False;
774 catfp.seek(1, 1);
775 il = il + 1;
776 realidnum = 0;
777 countnum = seekstart;
778 while(countnum < seekend):
779 catlist['ffilelist'].update({realidnum: {'fid': realidnum, 'fidalt': realidnum}});
780 catlist['ffilelist'][realidnum].update(ReadFileHeaderDataBySizeWithContentToArray(fp, listonly, skipchecksum, formatspecs));
781 countnum = countnum + 1;
782 realidnum = realidnum + 1;
783 return catlist;
784
785 def ReadFileDataBySizeWithContentToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
786 if(isinstance(infile, dict)):
787 listcatfiles = infile;
788 else:
789 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
790 return False;
791 listcatfiles = ReadFileDataBySizeWithContentToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs);
792 if(not listcatfiles):
793 return False;
794 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
795 lenlist = len(listcatfiles['ffilelist']);
796 lcfi = 0;
797 lcfx = int(listcatfiles['fnumfiles']);
798 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
799 lcfx = int(lenlist);
800 else:
801 lcfx = int(listcatfiles['fnumfiles']);
802 while(lcfi < lcfx):
803 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
804 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
805 catarray['filetoid'].update(filetoidarray);
806 catarray['idtofile'].update(idtofilearray);
807 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
808 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
809 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
810 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
811 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
812 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
813 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
814 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
815 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
816 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
817 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
818 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
819 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
820 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
821 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
822 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
823 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
824 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
825 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
826 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
827 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
828 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
829 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
830 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
831 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
832 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
833 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
834 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
835 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
836 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
837 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
838 lcfi = lcfi + 1;
839 return catarray;
840
841 def ReadInFileBySizeWithContent(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
842 delimiter = formatspecs[5];
843 if(hasattr(infile, "read") or hasattr(infile, "write")):
844 fp = infile;
845 fp.seek(0, 0);
846 fp = UncompressArchiveFile(fp, formatspecs);
847 checkcompressfile = CheckCompressionSubType(fp, formatspecs);
848 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
849 return False;
850 if(not fp):
851 return False;
852 fp.seek(0, 0);
853 elif(infile=="-"):
854 fp = BytesIO();
855 if(hasattr(sys.stdin, "buffer")):
856 shutil.copyfileobj(sys.stdin.buffer, fp);
857 else:
858 shutil.copyfileobj(sys.stdin, fp);
859 fp.seek(0, 0);
860 fp = UncompressArchiveFile(fp, formatspecs);
861 if(not fp):
862 return False;
863 fp.seek(0, 0);
864 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", infile)):
865 fp = download_file_from_internet_file(infile);
866 fp = UncompressArchiveFile(fp, formatspecs);
867 fp.seek(0, 0);
868 if(not fp):
869 return False;
870 fp.seek(0, 0);
871 else:
872 infile = RemoveWindowsPath(infile);
873 checkcompressfile = CheckCompressionSubType(infile, formatspecs);
874 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
875 return False;
876 compresscheck = CheckCompressionType(infile, formatspecs, True);
877 if(not compresscheck):
878 fextname = os.path.splitext(infile)[1];
879 if(fextname==".gz"):
880 compresscheck = "gzip";
881 elif(fextname==".bz2"):
882 compresscheck = "bzip2";
883 elif(fextname==".zst"):
884 compresscheck = "zstd";
885 elif(fextname==".lz4" or fextname==".clz4"):
886 compresscheck = "lz4";
887 elif(fextname==".lzo" or fextname==".lzop"):
888 compresscheck = "lzo";
889 elif(fextname==".lzma" or fextname==".xz"):
890 compresscheck = "lzma";
891 else:
892 return False;
893 if(not compresscheck):
894 return False;
895 fp = UncompressFile(infile, formatspecs, "rb");
896 return ReadFileDataBySizeWithContent(fp, seekstart, seekend, listonly, skipchecksum, formatspecs);
897
898 def ReadInFileBySizeWithContentToArray(infile, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
899 delimiter = formatspecs[5];
900 if(hasattr(infile, "read") or hasattr(infile, "write")):
901 fp = infile;
902 fp.seek(0, 0);
903 fp = UncompressArchiveFile(fp, formatspecs);
904 checkcompressfile = CheckCompressionSubType(fp, formatspecs);
905 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
906 return False;
907 if(not fp):
908 return False;
909 fp.seek(0, 0);
910 elif(infile=="-"):
911 fp = BytesIO();
912 if(hasattr(sys.stdin, "buffer")):
913 shutil.copyfileobj(sys.stdin.buffer, fp);
914 else:
915 shutil.copyfileobj(sys.stdin, fp);
916 fp.seek(0, 0);
917 fp = UncompressArchiveFile(fp, formatspecs);
918 if(not fp):
919 return False;
920 fp.seek(0, 0);
921 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", infile)):
922 fp = download_file_from_internet_file(infile);
923 fp = UncompressArchiveFile(fp, formatspecs);
924 fp.seek(0, 0);
925 if(not fp):
926 return False;
927 fp.seek(0, 0);
928 else:
929 infile = RemoveWindowsPath(infile);
930 checkcompressfile = CheckCompressionSubType(infile, formatspecs);
931 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
932 return False;
933 compresscheck = CheckCompressionType(infile, formatspecs, True);
934 if(not compresscheck):
935 fextname = os.path.splitext(infile)[1];
936 if(fextname==".gz"):
937 compresscheck = "gzip";
938 elif(fextname==".bz2"):
939 compresscheck = "bzip2";
940 elif(fextname==".zst"):
941 compresscheck = "zstd";
942 elif(fextname==".lz4" or fextname==".clz4"):
943 compresscheck = "lz4";
944 elif(fextname==".lzo" or fextname==".lzop"):
945 compresscheck = "lzo";
946 elif(fextname==".lzma" or fextname==".xz"):
947 compresscheck = "lzma";
948 else:
949 return False;
950 if(not compresscheck):
951 return False;
952 fp = UncompressFile(infile, formatspecs, "rb");
953 return ReadFileDataBySizeWithContentToArray(fp, listonly, skipchecksum, formatspecs);
954
955 def ReadInFileBySizeWithContentToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
956 if(isinstance(infile, dict)):
957 listcatfiles = infile;
958 else:
959 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
960 infile = RemoveWindowsPath(infile);
961 listcatfiles = ReadInFileBySizeWithContentToArray(infile, listonly, skipchecksum, formatspecs);
962 if(not listcatfiles):
963 return False;
964 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
965 lenlist = len(listcatfiles['ffilelist']);
966 lcfi = 0;
967 lcfx = int(listcatfiles['fnumfiles']);
968 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
969 lcfx = int(lenlist);
970 else:
971 lcfx = int(listcatfiles['fnumfiles']);
972 while(lcfi < lcfx):
973 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
974 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
975 catarray['filetoid'].update(filetoidarray);
976 catarray['idtofile'].update(idtofilearray);
977 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
978 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
979 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
980 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
981 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
982 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
983 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
984 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
985 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
986 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
987 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
988 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
989 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
990 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
991 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
992 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
993 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
994 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
995 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
996 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
997 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
998 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
999 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1000 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
1001 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
1002 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
1003 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
1004 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1005 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1006 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1007 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1008 lcfi = lcfi + 1;
1009 return catarray;
1010
1011 def AppendNullByte(indata, delimiter=__file_format_delimiter__):
1012 outdata = str(indata) + delimiter;
1013 return outdata;
1014
1015 def AppendNullBytes(indata=[], delimiter=__file_format_delimiter__):
1016 outdata = "";
1017 inum = 0;
1018 il = len(indata);
1019 while(inum < il):
1020 outdata = outdata + AppendNullByte(indata[inum], delimiter);
1021 inum = inum + 1;
1022 return outdata;
1023
1024 def PrintPermissionString(fchmode, ftype):
1025 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
1026 permissionstr = "";
1027 for fmodval in str(oct(fchmode))[-3:]:
1028 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
1029 if(ftype==0 or ftype==7):
1030 permissionstr = "-" + permissionstr;
1031 if(ftype==1):
1032 permissionstr = "h" + permissionstr;
1033 if(ftype==2):
1034 permissionstr = "l" + permissionstr;
1035 if(ftype==3):
1036 permissionstr = "c" + permissionstr;
1037 if(ftype==4):
1038 permissionstr = "b" + permissionstr;
1039 if(ftype==5):
1040 permissionstr = "d" + permissionstr;
1041 if(ftype==6):
1042 permissionstr = "f" + permissionstr;
1043 if(ftype==8):
1044 permissionstr = "D" + permissionstr;
1045 if(ftype==9):
1046 permissionstr = "p" + permissionstr;
1047 if(ftype==10):
1048 permissionstr = "w" + permissionstr;
1049 try:
1050 permissionoutstr = stat.filemode(fchmode);
1051 except AttributeError:
1052 permissionoutstr = permissionstr;
1053 except KeyError:
1054 permissionoutstr = permissionstr;
1055 return permissionoutstr;
1056
1057 def PrintPermissionStringAlt(fchmode, ftype):
1058 permissions = {
1059 '0': '---', '1': '--x', '2': '-w-', '3': '-wx',
1060 '4': 'r--', '5': 'r-x', '6': 'rw-', '7': 'rwx'
1061 };
1062 # Translate file mode into permission string
1063 permissionstr = ''.join([permissions[i] for i in str(oct(fchmode))[-3:]]);
1064 # Append file type indicator
1065 type_indicators = {
1066 0: '-', 1: 'h', 2: 'l', 3: 'c', 4: 'b',
1067 5: 'd', 6: 'f', 8: 'D', 9: 'p', 10: 'w'
1068 };
1069 file_type = type_indicators.get(ftype, '-');
1070 permissionstr = file_type + permissionstr;
1071 try:
1072 permissionoutstr = stat.filemode(fchmode);
1073 except AttributeError:
1074 permissionoutstr = permissionstr;
1075 return permissionoutstr;
1076
1077 def CompressionSupport():
1078 compression_list = [];
1079 try:
1080 import gzip;
1081 compression_list.append("gz");
1082 compression_list.append("gzip");
1083 except ImportError:
1084 '''return False;'''
1085 try:
1086 import bz2;
1087 compression_list.append("bz2");
1088 compression_list.append("bzip2");
1089 except ImportError:
1090 '''return False;'''
1091 try:
1092 import lz4;
1093 compression_list.append("lz4");
1094 except ImportError:
1095 '''return False;'''
1096 try:
1097 import lzo;
1098 compression_list.append("lzo");
1099 compression_list.append("lzop");
1100 except ImportError:
1101 '''return False;'''
1102 try:
1103 import zstandard;
1104 compression_list.append("zstd");
1105 compression_list.append("zstandard");
1106 except ImportError:
1107 '''return False;'''
1108 try:
1109 import lzma;
1110 compression_list.append("lzma");
1111 compression_list.append("xz");
1112 except ImportError:
1113 '''return False;'''
1114 return compression_list;
1115
1116 def CheckCompressionType(infile, formatspecs=__file_format_list__, closefp=True):
1117 if(hasattr(infile, "read") or hasattr(infile, "write")):
1118 catfp = infile;
1119 else:
1120 try:
1121 catfp = open(infile, "rb");
1122 except FileNotFoundError:
1123 return False;
1124 catfp.seek(0, 0);
1125 prefp = catfp.read(2);
1126 filetype = False;
1127 if(prefp==binascii.unhexlify("1f8b")):
1128 filetype = "gzip";
1129 catfp.seek(0, 0);
1130 prefp = catfp.read(3);
1131 if(prefp==binascii.unhexlify("425a68")):
1132 filetype = "bzip2";
1133 if(prefp==binascii.unhexlify("5d0000")):
1134 filetype = "lzma";
1135 catfp.seek(0, 0);
1136 prefp = catfp.read(4);
1137 if(prefp==binascii.unhexlify("28b52ffd")):
1138 filetype = "zstd";
1139 if(prefp==binascii.unhexlify("04224d18")):
1140 filetype = "lz4";
1141 if(prefp==binascii.unhexlify("504B0304")):
1142 filetype = "zipfile";
1143 catfp.seek(0, 0);
1144 prefp = catfp.read(5);
1145 if(prefp==binascii.unhexlify("7573746172")):
1146 filetype = "tarfile";
1147 catfp.seek(0, 0);
1148 prefp = catfp.read(6);
1149 if(prefp==binascii.unhexlify("fd377a585a00")):
1150 filetype = "lzma";
1151 catfp.seek(0, 0);
1152 prefp = catfp.read(7);
1153 if(prefp==binascii.unhexlify("526172211a0700")):
1154 filetype = "rarfile";
1155 if(prefp==binascii.unhexlify("43617446696c65")):
1156 filetype = "catfile";
1157 catfp.seek(0, 0);
1158 prefp = catfp.read(8);
1159 if(prefp==binascii.unhexlify("526172211a070100")):
1160 filetype = "rarfile";
1161 catfp.seek(0, 0);
1162 prefp = catfp.read(formatspecs[3]);
1163 if(prefp==binascii.unhexlify(formatspecs[4])):
1164 filetype = formatspecs[2];
1165 catfp.seek(0, 0);
1166 prefp = catfp.read(9);
1167 if(prefp==binascii.unhexlify("894c5a4f000d0a1a0a")):
1168 filetype = "lzo";
1169 catfp.seek(0, 0);
1170 prefp = catfp.read(10);
1171 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
1172 filetype = "tarfile";
1173 catfp.seek(0, 0);
1174 if(closefp):
1175 catfp.close();
1176 return filetype;
1177
1178 def CheckCompressionTypeFromString(instring, formatspecs=__file_format_list__, closefp=True):
1179 try:
1180 instringsfile = BytesIO(instring);
1181 except TypeError:
1182 instringsfile = BytesIO(instring.encode("UTF-8"));
1183 return CheckCompressionType(instringsfile, formatspecs, closefp);
1184
1185 def GetCompressionMimeType(infile, formatspecs=__file_format_list__):
1186 compresscheck = CheckCompressionType(fp, formatspecs, False);
1187 if(compresscheck=="gzip" or compresscheck=="gz"):
1188 return archivefile_gzip_mimetype;
1189 if(compresscheck=="bzip2" or compresscheck=="bz2"):
1190 return archivefile_bzip2_mimetype;
1191 if(compresscheck=="zstd" or compresscheck=="zstandard"):
1192 return archivefile_zstandard_mimetype;
1193 if(compresscheck=="lz4"):
1194 return archivefile_lz4_mimetype;
1195 if(compresscheck=="lzo" or compresscheck=="lzop"):
1196 return archivefile_lzop_mimetype;
1197 if(compresscheck=="lzma"):
1198 return archivefile_lzma_mimetype;
1199 if(compresscheck=="xz"):
1200 return archivefile_xz_mimetype;
1201 if(compresscheck=="catfile" or compresscheck=="cat" or compresscheck==formatspecs[2]):
1202 return archivefile_cat_mimetype;
1203 if(not compresscheck):
1204 return False;
1205 return False;
1206
1207 def UncompressArchiveFile(fp, formatspecs=__file_format_list__):
1208 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
1209 return False;
1210 compresscheck = CheckCompressionType(fp, formatspecs, False);
1211 if(compresscheck=="gzip"):
1212 try:
1213 import gzip;
1214 except ImportError:
1215 return False;
1216 catfp = gzip.GzipFile(fileobj=fp, mode="rb");
1217 if(compresscheck=="bzip2"):
1218 try:
1219 import bz2;
1220 except ImportError:
1221 return False;
1222 catfp = BytesIO();
1223 catfp.write(bz2.decompress(fp.read()));
1224 if(compresscheck=="zstd"):
1225 try:
1226 import zstandard;
1227 except ImportError:
1228 return False;
1229 catfp = BytesIO();
1230 catfp.write(zstandard.decompress(fp.read()));
1231 if(compresscheck=="lz4"):
1232 try:
1233 import lz4.frame;
1234 except ImportError:
1235 return False;
1236 catfp = BytesIO();
1237 catfp.write(lz4.frame.decompress(fp.read()));
1238 if(compresscheck=="lzo" or compresscheck=="lzop"):
1239 try:
1240 import lzo;
1241 except ImportError:
1242 return False;
1243 catfp = BytesIO();
1244 catfp.write(lzo.decompress(fp.read()));
1245 if(compresscheck=="lzma" or compresscheck=="xz"):
1246 try:
1247 import lzma;
1248 except ImportError:
1249 return False;
1250 catfp = BytesIO();
1251 catfp.write(lzma.decompress(fp.read()));
1252 if(compresscheck=="catfile" or compresscheck==formatspecs[2]):
1253 catfp = fp;
1254 if(not compresscheck):
1255 try:
1256 import lzma;
1257 except ImportError:
1258 return False;
1259 catfp = BytesIO();
1260 with fp as fpcontent:
1261 try:
1262 catfp.write(lzma.decompress(fp.read()));
1263 except lzma.LZMAError:
1264 return False;
1265 return catfp;
1266
1267 create_alias_function("Uncompress", __file_format_name__, "", UncompressArchiveFile);
1268
1269 def UncompressFile(infile, formatspecs=__file_format_list__, mode="rb"):
1270 compresscheck = CheckCompressionType(infile, formatspecs, False);
1271 if(sys.version_info[0]==2 and compresscheck):
1272 if(mode=="rt"):
1273 mode = "r";
1274 if(mode=="wt"):
1275 mode = "w";
1276 try:
1277 if(compresscheck=="gzip"):
1278 try:
1279 import gzip;
1280 except ImportError:
1281 return False;
1282 try:
1283 filefp = gzip.open(infile, mode, encoding="UTF-8");
1284 except (ValueError, TypeError) as e:
1285 filefp = gzip.open(infile, mode);
1286 if(compresscheck=="bzip2"):
1287 try:
1288 import bz2;
1289 except ImportError:
1290 return False;
1291 try:
1292 filefp = bz2.open(infile, mode, encoding="UTF-8");
1293 except (ValueError, TypeError) as e:
1294 filefp = bz2.open(infile, mode);
1295 if(compresscheck=="zstd"):
1296 try:
1297 import zstandard;
1298 except ImportError:
1299 return False;
1300 try:
1301 filefp = zstandard.open(infile, mode, encoding="UTF-8");
1302 except (ValueError, TypeError) as e:
1303 filefp = zstandard.open(infile, mode);
1304 if(compresscheck=="lz4"):
1305 try:
1306 import lz4.frame;
1307 except ImportError:
1308 return False;
1309 try:
1310 filefp = lz4.frame.open(infile, mode, encoding="UTF-8");
1311 except (ValueError, TypeError) as e:
1312 filefp = lz4.frame.open(infile, mode);
1313 if(compresscheck=="lzo"):
1314 try:
1315 import lzo;
1316 except ImportError:
1317 return False;
1318 try:
1319 filefp = lzo.open(infile, mode, encoding="UTF-8");
1320 except (ValueError, TypeError) as e:
1321 filefp = lzo.open(infile, mode);
1322 if(compresscheck=="lzma"):
1323 try:
1324 import lzma;
1325 except ImportError:
1326 return False;
1327 try:
1328 filefp = lzma.open(infile, mode, encoding="UTF-8");
1329 except (ValueError, TypeError) as e:
1330 filefp = lzma.open(infile, mode);
1331 if(compresscheck=="catfile" or compresscheck==formatspecs[2]):
1332 try:
1333 filefp = open(infile, mode, encoding="UTF-8");
1334 except (ValueError, TypeError) as e:
1335 filefp = open(infile, mode);
1336 if(not compresscheck):
1337 try:
1338 filefp = open(infile, mode, encoding="UTF-8");
1339 except (ValueError, TypeError) as e:
1340 filefp = open(infile, mode);
1341 except FileNotFoundError:
1342 return False;
1343 return filefp;
1344
1345 def UncompressString(infile):
1346 compresscheck = CheckCompressionTypeFromString(infile, formatspecs, False);
1347 if(compresscheck=="gzip"):
1348 try:
1349 import gzip;
1350 except ImportError:
1351 return False;
1352 fileuz = gzip.decompress(infile);
1353 if(compresscheck=="bzip2"):
1354 try:
1355 import bz2;
1356 except ImportError:
1357 return False;
1358 fileuz = bz2.decompress(infile);
1359 if(compresscheck=="zstd"):
1360 try:
1361 import zstandard;
1362 except ImportError:
1363 return False;
1364 fileuz = zstandard.decompress(infile);
1365 if(compresscheck=="lz4"):
1366 try:
1367 import lz4.frame;
1368 except ImportError:
1369 return False;
1370 fileuz = lz4.frame.decompress(infile);
1371 if(compresscheck=="lzo"):
1372 try:
1373 import lzo;
1374 except ImportError:
1375 return False;
1376 fileuz = lzo.decompress(infile);
1377 if(compresscheck=="lzma"):
1378 try:
1379 import lzma;
1380 except ImportError:
1381 return False;
1382 fileuz = lzma.decompress(infile);
1383 if(not compresscheck):
1384 fileuz = infile;
1385 if(hasattr(fileuz, 'decode')):
1386 fileuz = fileuz.decode("UTF-8");
1387 return fileuz;
1388
1389 def UncompressStringAlt(infile):
1390 filefp = StringIO();
1391 outstring = UncompressString(infile);
1392 filefp.write(outstring);
1393 filefp.seek(0, 0);
1394 return filefp;
1395
1396 def CheckCompressionSubType(infile, formatspecs=__file_format_list__):
1397 compresscheck = CheckCompressionType(infile, formatspecs, False);
1398 if(not compresscheck):
1399 fextname = os.path.splitext(infile)[1];
1400 if(fextname==".gz"):
1401 compresscheck = "gzip";
1402 elif(fextname==".bz2"):
1403 compresscheck = "bzip2";
1404 elif(fextname==".zst"):
1405 compresscheck = "zstd";
1406 elif(fextname==".lz4"):
1407 compresscheck = "lz4";
1408 elif(fextname==".lzo" or fextname==".lzop"):
1409 compresscheck = "lzo";
1410 elif(fextname==".lzma" or fextname==".xz"):
1411 compresscheck = "lzma";
1412 else:
1413 return False;
1414 if(not compresscheck):
1415 return False;
1416 if(compresscheck=="catfile"):
1417 return "catfile";
1418 if(compresscheck==formatspecs[2]):
1419 return formatspecs[2];
1420 if(compresscheck=="tarfile"):
1421 return "tarfile";
1422 if(compresscheck=="zipfile"):
1423 return "zipfile";
1424 if(hasattr(infile, "read") or hasattr(infile, "write")):
1425 catfp = UncompressArchiveFile(infile, formatspecs[2]);
1426 else:
1427 try:
1428 if(compresscheck=="gzip"):
1429 try:
1430 import gzip;
1431 except ImportError:
1432 return False;
1433 catfp = gzip.GzipFile(infile, "rb");
1434 if(compresscheck=="bzip2"):
1435 try:
1436 import bz2;
1437 except ImportError:
1438 return False;
1439 catfp = bz2.BZ2File(infile, "rb");
1440 if(compresscheck=="lz4"):
1441 try:
1442 import lz4.frame;
1443 except ImportError:
1444 return False;
1445 catfp = lz4.frame.open(infile, "rb");
1446 if(compresscheck=="zstd"):
1447 try:
1448 import zstandard;
1449 except ImportError:
1450 return False;
1451 catfp = zstandard.open(infile, "rb");
1452 if(compresscheck=="lzma" or compresscheck=="xz"):
1453 try:
1454 import lzma;
1455 except ImportError:
1456 return False;
1457 catfp = lzma.open(infile, "rb");
1458 except FileNotFoundError:
1459 return False;
1460 filetype = False;
1461 prefp = catfp.read(5);
1462 if(prefp==binascii.unhexlify("7573746172")):
1463 filetype = "tarfile";
1464 catfp.seek(0, 0);
1465 prefp = catfp.read(7);
1466 if(prefp==binascii.unhexlify("43617446696c65")):
1467 filetype = "catfile";
1468 catfp.seek(0, 0);
1469 prefp = catfp.read(formatspecs[3]);
1470 if(prefp==binascii.unhexlify(formatspecs[4])):
1471 filetype = formatspecs[2];
1472 catfp.seek(0, 0);
1473 prefp = catfp.read(10);
1474 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
1475 filetype = "tarfile";
1476 catfp.seek(0, 0);
1477 catfp.close();
1478 return filetype;
1479
1480 def GZipCompress(data, compresslevel=9):
1481 try:
1482 import gzip;
1483 except ImportError:
1484 return False;
1485 tmpfp = tempfile.NamedTemporaryFile("wb", delete=False);
1486 tmpfp.close();
1487 tmpfp = gzip.GzipFile(tmpfp.name, mode="wb", compresslevel=compresslevel);
1488 tmpfp.write(data);
1489 tmpfp.close();
1490 try:
1491 catfp = open(tmpfp.name, "rb");
1492 except FileNotFoundError:
1493 return False;
1494 catdata = catfp.read();
1495 catfp.close();
1496 return catdata;
1497
1498 def CompressArchiveFile(fp, compression="auto", compressionlevel=None, formatspecs=__file_format_list__):
1499 compressionlist = ['auto', 'gzip', 'bzip2', 'zstd', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
1500 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
1501 return False;
1502 fp.seek(0, 0);
1503 if(not compression or compression or compression=="catfile" or compression==formatspecs[2]):
1504 compression = None;
1505 if(compression not in compressionlist and compression is None):
1506 compression = "auto";
1507 if(compression=="gzip"):
1508 try:
1509 import gzip;
1510 except ImportError:
1511 return False;
1512 catfp = BytesIO();
1513 if(compressionlevel is None):
1514 compressionlevel = 9;
1515 else:
1516 compressionlevel = int(compressionlevel);
1517 catfp.write(gzip.compress(fp.read(), compresslevel=compressionlevel));
1518 if(compression=="bzip2"):
1519 try:
1520 import bz2;
1521 except ImportError:
1522 return False;
1523 catfp = BytesIO();
1524 if(compressionlevel is None):
1525 compressionlevel = 9;
1526 else:
1527 compressionlevel = int(compressionlevel);
1528 catfp.write(bz2.compress(fp.read(), compresslevel=compressionlevel));
1529 if(compression=="lz4"):
1530 try:
1531 import lz4.frame;
1532 except ImportError:
1533 return False;
1534 catfp = BytesIO();
1535 if(compressionlevel is None):
1536 compressionlevel = 9;
1537 else:
1538 compressionlevel = int(compressionlevel);
1539 catfp.write(lz4.frame.compress(fp.read(), compression_level=compressionlevel));
1540 if(compression=="lzo" or compression=="lzop"):
1541 try:
1542 import lzo;
1543 except ImportError:
1544 return False;
1545 catfp = BytesIO();
1546 if(compressionlevel is None):
1547 compressionlevel = 9;
1548 else:
1549 compressionlevel = int(compressionlevel);
1550 catfp.write(lzo.compress(fp.read(), compresslevel=compressionlevel));
1551 if(compression=="zstd"):
1552 try:
1553 import zstandard;
1554 except ImportError:
1555 return False;
1556 catfp = BytesIO();
1557 if(compressionlevel is None):
1558 compressionlevel = 10;
1559 else:
1560 compressionlevel = int(compressionlevel);
1561 catfp.write(zstandard.compress(fp.read(), level=compressionlevel));
1562 if(compression=="lzma"):
1563 try:
1564 import lzma;
1565 except ImportError:
1566 return False;
1567 catfp = BytesIO();
1568 if(compressionlevel is None):
1569 compressionlevel = 9;
1570 else:
1571 compressionlevel = int(compressionlevel);
1572 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]));
1573 if(compression=="xz"):
1574 try:
1575 import lzma;
1576 except ImportError:
1577 return False;
1578 catfp = BytesIO();
1579 if(compressionlevel is None):
1580 compressionlevel = 9;
1581 else:
1582 compressionlevel = int(compressionlevel);
1583 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]));
1584 if(compression=="auto" or compression is None):
1585 catfp = fp;
1586 catfp.seek(0, 0);
1587 return catfp;
1588
1589 create_alias_function("Compress", __file_format_name__, "", CompressArchiveFile);
1590
1591 def CompressOpenFile(outfile, compressionlevel=None):
1592 if(outfile is None):
1593 return False;
1594 fbasename = os.path.splitext(outfile)[0];
1595 fextname = os.path.splitext(outfile)[1];
1596 if(compressionlevel is None and fextname!=".zst"):
1597 compressionlevel = 9;
1598 elif(compressionlevel is None and fextname==".zst"):
1599 compressionlevel = 10;
1600 else:
1601 compressionlevel = int(compressionlevel);
1602 if(sys.version_info[0]==2):
1603 mode = "w";
1604 else:
1605 mode = "wb";
1606 try:
1607 if(fextname not in outextlistwd):
1608 try:
1609 outfp = open(outfile, "wb", encoding="UTF-8");
1610 except (ValueError, TypeError) as e:
1611 outfp = open(outfile, "wb");
1612 elif(fextname==".gz"):
1613 try:
1614 import gzip;
1615 except ImportError:
1616 return False;
1617 try:
1618 outfp = gzip.open(outfile, mode, compressionlevel, encoding="UTF-8");
1619 except (ValueError, TypeError) as e:
1620 outfp = gzip.open(outfile, mode, compressionlevel);
1621 elif(fextname==".bz2"):
1622 try:
1623 import bz2;
1624 except ImportError:
1625 return False;
1626 try:
1627 outfp = bz2.open(outfile, mode, compressionlevel, encoding="UTF-8");
1628 except (ValueError, TypeError) as e:
1629 outfp = bz2.open(outfile, mode, compressionlevel);
1630 elif(fextname==".zst"):
1631 try:
1632 import zstandard;
1633 except ImportError:
1634 return False;
1635 try:
1636 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel), encoding="UTF-8");
1637 except (ValueError, TypeError) as e:
1638 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel));
1639 elif(fextname==".xz"):
1640 try:
1641 import lzma;
1642 except ImportError:
1643 return False;
1644 try:
1645 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}], encoding="UTF-8");
1646 except (ValueError, TypeError) as e:
1647 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]);
1648 elif(fextname==".lz4"):
1649 try:
1650 import lz4.frame;
1651 except ImportError:
1652 return False;
1653 try:
1654 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel, encoding="UTF-8");
1655 except (ValueError, TypeError) as e:
1656 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel);
1657 elif(fextname==".lzo"):
1658 try:
1659 import lzo;
1660 except ImportError:
1661 return False;
1662 try:
1663 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel, encoding="UTF-8");
1664 except (ValueError, TypeError) as e:
1665 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel);
1666 elif(fextname==".lzma"):
1667 try:
1668 import lzma;
1669 except ImportError:
1670 return False;
1671 try:
1672 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}], encoding="UTF-8");
1673 except (ValueError, TypeError) as e:
1674 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]);
1675 except FileNotFoundError:
1676 return False;
1677 return outfp;
1678
1679 def GetDevMajorMinor(fdev):
1680 retdev = [];
1681 if(hasattr(os, "minor")):
1682 retdev.append(os.minor(fdev));
1683 else:
1684 retdev.append(0);
1685 if(hasattr(os, "major")):
1686 retdev.append(os.major(fdev));
1687 else:
1688 retdev.append(0);
1689 return retdev;
1690
1691 def CheckSumSupport(checkfor, guaranteed=True):
1692 if(guaranteed):
1693 hash_list = sorted(list(hashlib.algorithms_guaranteed));
1694 else:
1695 hash_list = sorted(list(hashlib.algorithms_available));
1696 checklistout = sorted(hash_list + ['adler32', 'crc16', 'crc16_ansi', 'crc16_ibm', 'crc16_ccitt', 'crc32', 'crc64', 'crc64_ecma', 'crc64_iso', 'none']);
1697 if(checkfor in checklistout):
1698 return True;
1699 else:
1700 return False;
1701
1702 def CheckSumSupportAlt(checkfor, guaranteed=True):
1703 if(guaranteed):
1704 hash_list = sorted(list(hashlib.algorithms_guaranteed));
1705 else:
1706 hash_list = sorted(list(hashlib.algorithms_available));
1707 checklistout = hash_list;
1708 if(checkfor in checklistout):
1709 return True;
1710 else:
1711 return False;
1712
1713 def PackArchiveFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
1714 compressionlist = ['auto', 'gzip', 'bzip2', 'zstd', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
1715 outextlist = ['gz', 'bz2', 'zst', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
1716 outextlistwd = ['.gz', '.bz2', '.zst', '.lz4', '.lzo', '.lzop', '.lzma', '.xz'];
1717 advancedlist = True;
1718 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1719 outfile = RemoveWindowsPath(outfile);
1720 checksumtype = checksumtype.lower();
1721 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
1722 checksumtype="crc32";
1723 if(checksumtype=="none"):
1724 checksumtype = "";
1725 if(not compression or compression or compression=="catfile" or compression==formatspecs[2]):
1726 compression = None;
1727 if(compression not in compressionlist and compression is None):
1728 compression = "auto";
1729 if(verbose):
1730 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
1731 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1732 if(os.path.exists(outfile)):
1733 os.unlink(outfile);
1734 if(outfile=="-"):
1735 verbose = False;
1736 catfp = BytesIO();
1737 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1738 catfp = outfile;
1739 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", outfile)):
1740 catfp = BytesIO();
1741 else:
1742 fbasename = os.path.splitext(outfile)[0];
1743 fextname = os.path.splitext(outfile)[1];
1744 catfp = CompressOpenFile(outfile, compressionlevel);
1745 catver = formatspecs[6];
1746 fileheaderver = str(int(catver.replace(".", "")));
1747 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
1748 catfp.write(fileheader.encode('UTF-8'));
1749 infilelist = [];
1750 if(infiles=="-"):
1751 for line in sys.stdin:
1752 infilelist.append(line.strip());
1753 infilelist = list(filter(None, infilelist));
1754 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
1755 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
1756 return False;
1757 with open(infiles, "r") as finfile:
1758 for line in finfile:
1759 infilelist.append(line.strip());
1760 infilelist = list(filter(None, infilelist));
1761 else:
1762 if(isinstance(infiles, (list, tuple, ))):
1763 infilelist = list(filter(None, infiles));
1764 elif(isinstance(infiles, (str, ))):
1765 infilelist = list(filter(None, [infiles]));
1766 if(advancedlist):
1767 GetDirList = ListDirAdvanced(infilelist, followlink, False);
1768 else:
1769 GetDirList = ListDir(infilelist, followlink, False);
1770 if(not GetDirList):
1771 return False;
1772 curinode = 0;
1773 curfid = 0;
1774 inodelist = [];
1775 inodetofile = {};
1776 filetoinode = {};
1777 inodetocatinode = {};
1778 fnumfiles = format(int(len(GetDirList)), 'x').lower();
1779 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs[5]);
1780 if(checksumtype=="none" or checksumtype==""):
1781 catfileheadercshex = format(0, 'x').lower();
1782 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
1783 catfileheadercshex = format(crc16(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffff, '04x').lower();
1784 elif(checksumtype=="crc16_ccitt"):
1785 catfileheadercshex = format(crc16_ccitt(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffff, '04x').lower();
1786 elif(checksumtype=="adler32"):
1787 catfileheadercshex = format(zlib.adler32(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffff, '08x').lower();
1788 elif(checksumtype=="crc32"):
1789 catfileheadercshex = format(crc32(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffff, '08x').lower();
1790 elif(checksumtype=="crc64_ecma"):
1791 catfileheadercshex = format(crc64_ecma(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
1792 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
1793 catfileheadercshex = format(crc64_iso(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
1794 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
1795 checksumoutstr = hashlib.new(checksumtype);
1796 checksumoutstr.update(str(fileheader + fnumfilesa).encode('UTF-8'));
1797 catfileheadercshex = checksumoutstr.hexdigest().lower();
1798 else:
1799 catfileheadercshex = format(0, 'x').lower
1800 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs[5]);
1801 catfp.write(fnumfilesa.encode('UTF-8'));
1802 try:
1803 catfp.flush();
1804 os.fsync(catfp.fileno());
1805 except io.UnsupportedOperation:
1806 pass;
1807 except AttributeError:
1808 pass;
1809 for curfname in GetDirList:
1810 catfhstart = catfp.tell();
1811 if(re.findall("^[.|/]", curfname)):
1812 fname = curfname;
1813 else:
1814 fname = "./"+curfname;
1815 if(verbose):
1816 VerbosePrintOut(fname);
1817 if(not followlink or followlink is None):
1818 fstatinfo = os.lstat(fname);
1819 else:
1820 fstatinfo = os.stat(fname);
1821 fpremode = fstatinfo.st_mode;
1822 finode = fstatinfo.st_ino;
1823 flinkcount = fstatinfo.st_nlink;
1824 ftype = 0;
1825 if(stat.S_ISREG(fpremode)):
1826 ftype = 0;
1827 elif(stat.S_ISLNK(fpremode)):
1828 ftype = 2;
1829 elif(stat.S_ISCHR(fpremode)):
1830 ftype = 3;
1831 elif(stat.S_ISBLK(fpremode)):
1832 ftype = 4;
1833 elif(stat.S_ISDIR(fpremode)):
1834 ftype = 5;
1835 elif(stat.S_ISFIFO(fpremode)):
1836 ftype = 6;
1837 elif(stat.S_ISSOCK(fpremode)):
1838 ftype = 8;
1839 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
1840 ftype = 9;
1841 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
1842 ftype = 10;
1843 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
1844 ftype = 111;
1845 else:
1846 ftype = 0;
1847 flinkname = "";
1848 fcurfid = format(int(curfid), 'x').lower();
1849 if(not followlink and finode!=0):
1850 if(ftype!=1):
1851 if(finode in inodelist):
1852 ftype = 1;
1853 flinkname = inodetofile[finode];
1854 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
1855 if(finode not in inodelist):
1856 inodelist.append(finode);
1857 inodetofile.update({finode: fname});
1858 inodetocatinode.update({finode: curinode});
1859 fcurinode = format(int(curinode), 'x').lower();
1860 curinode = curinode + 1;
1861 else:
1862 fcurinode = format(int(curinode), 'x').lower();
1863 curinode = curinode + 1;
1864 curfid = curfid + 1;
1865 if(ftype==2):
1866 flinkname = os.readlink(fname);
1867 fdev = fstatinfo.st_dev;
1868 getfdev = GetDevMajorMinor(fdev);
1869 fdev_minor = getfdev[0];
1870 fdev_major = getfdev[1];
1871 frdev = fstatinfo.st_dev;
1872 if(hasattr(fstatinfo, "st_rdev")):
1873 frdev = fstatinfo.st_rdev;
1874 else:
1875 frdev = fstatinfo.st_dev;
1876 getfrdev = GetDevMajorMinor(frdev);
1877 frdev_minor = getfrdev[0];
1878 frdev_major = getfrdev[1];
1879 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
1880 fsize = format(int("0"), 'x').lower();
1881 elif(ftype==0 or ftype==7):
1882 fsize = format(int(fstatinfo.st_size), 'x').lower();
1883 else:
1884 fsize = format(int(fstatinfo.st_size)).lower();
1885 fatime = format(int(fstatinfo.st_atime), 'x').lower();
1886 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
1887 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
1888 if(hasattr(fstatinfo, "st_birthtime")):
1889 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
1890 else:
1891 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
1892 fmode = format(int(fstatinfo.st_mode), 'x').lower();
1893 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
1894 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
1895 fuid = format(int(fstatinfo.st_uid), 'x').lower();
1896 fgid = format(int(fstatinfo.st_gid), 'x').lower();
1897 funame = "";
1898 try:
1899 import pwd;
1900 try:
1901 userinfo = pwd.getpwuid(fstatinfo.st_uid);
1902 funame = userinfo.pw_name;
1903 except KeyError:
1904 funame = "";
1905 except ImportError:
1906 funame = "";
1907 fgname = "";
1908 try:
1909 import grp;
1910 try:
1911 groupinfo = grp.getgrgid(fstatinfo.st_gid);
1912 fgname = groupinfo.gr_name;
1913 except KeyError:
1914 fgname = "";
1915 except ImportError:
1916 fgname = "";
1917 fdev_minor = format(int(fdev_minor), 'x').lower();
1918 fdev_major = format(int(fdev_major), 'x').lower();
1919 frdev_minor = format(int(frdev_minor), 'x').lower();
1920 frdev_major = format(int(frdev_major), 'x').lower();
1921 finode = format(int(finode), 'x').lower();
1922 flinkcount = format(int(flinkcount), 'x').lower();
1923 if(hasattr(fstatinfo, "st_file_attributes")):
1924 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
1925 else:
1926 fwinattributes = format(int(0), 'x').lower();
1927 fcontents = "".encode('UTF-8');
1928 chunk_size = 1024;
1929 if(ftype == 0 or ftype == 7):
1930 with open(fname, "rb") as fpc:
1931 while(True):
1932 chunk = fpc.read(chunk_size);
1933 if(not chunk):
1934 break
1935 fcontents += chunk;
1936 if(followlink and (ftype == 1 or ftype == 2)):
1937 flstatinfo = os.stat(flinkname);
1938 with open(flinkname, "rb") as fpc:
1939 while(True):
1940 chunk = fpc.read(chunk_size);
1941 if(not chunk):
1942 break;
1943 fcontents += chunk;
1944 ftypehex = format(ftype, 'x').lower();
1945 extrafields = format(len(extradata), 'x').lower();
1946 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
1947 if(len(extradata)>0):
1948 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
1949 extrasizelen = format(len(extrasizestr), 'x').lower();
1950 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, extrasizelen, extrafields];
1951 catoutlen = len(catoutlist) + len(extradata) + 3;
1952 catoutlenhex = format(catoutlen, 'x').lower();
1953 catoutlist.insert(0, catoutlenhex);
1954 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
1955 if(len(extradata)>0):
1956 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
1957 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
1958 if(checksumtype=="none" or checksumtype==""):
1959 catfileheadercshex = format(0, 'x').lower();
1960 catfilecontentcshex = format(0, 'x').lower();
1961 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
1962 catfileheadercshex = format(crc16("".encode('UTF-8')) & 0xffff, '04x').lower();
1963 catfilecontentcshex = format(crc16(fcontents) & 0xffff, '04x').lower();
1964 elif(checksumtype=="crc16_ccitt"):
1965 catfileheadercshex = format(crc16_ccitt("".encode('UTF-8')) & 0xffff, '04x').lower();
1966 catfilecontentcshex = format(crc16_ccitt(fcontents) & 0xffff, '04x').lower();
1967 elif(checksumtype=="adler32"):
1968 catfileheadercshex = format(zlib.adler32("".encode('UTF-8')) & 0xffffffff, '08x').lower();
1969 catfilecontentcshex = format(zlib.adler32(fcontents) & 0xffffffff, '08x').lower();
1970 elif(checksumtype=="crc32"):
1971 catfileheadercshex = format(crc32("".encode('UTF-8')) & 0xffffffff, '08x').lower();
1972 catfilecontentcshex = format(crc32(fcontents) & 0xffffffff, '08x').lower();
1973 elif(checksumtype=="crc64_ecma"):
1974 catfileheadercshex = format(crc64_ecma("".encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
1975 catfilecontentcshex = format(crc64_ecma(fcontents) & 0xffffffffffffffff, '016x').lower();
1976 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
1977 catfileheadercshex = format(crc64_iso("".encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
1978 catfilecontentcshex = format(crc64_iso(fcontents) & 0xffffffffffffffff, '016x').lower();
1979 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
1980 checksumoutstr = hashlib.new(checksumtype);
1981 checksumoutstr.update("".encode('UTF-8'));
1982 catfileheadercshex = checksumoutstr.hexdigest().lower();
1983 checksumoutstr = hashlib.new(checksumtype);
1984 checksumoutstr.update(fcontents);
1985 catfilecontentcshex = checksumoutstr.hexdigest().lower();
1986 else:
1987 catfileheadercshex = format(0, 'x').lower();
1988 catfilecontentcshex = format(0, 'x').lower();
1989 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
1990 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower()
1991 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
1992 if(checksumtype=="none" or checksumtype==""):
1993 catfileheadercshex = format(0, 'x').lower();
1994 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
1995 catfileheadercshex = format(crc16(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
1996 elif(checksumtype=="crc16_ccitt"):
1997 catfileheadercshex = format(crc16_ccitt(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
1998 elif(checksumtype=="adler32"):
1999 catfileheadercshex = format(zlib.adler32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
2000 elif(checksumtype=="crc32"):
2001 catfileheadercshex = format(crc32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
2002 elif(checksumtype=="crc64_ecma"):
2003 catfileheadercshex = format(crc64_ecma(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2004 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
2005 catfileheadercshex = format(crc64_iso(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2006 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
2007 checksumoutstr = hashlib.new(checksumtype);
2008 checksumoutstr.update(catfileoutstr.encode('UTF-8'));
2009 catfileheadercshex = checksumoutstr.hexdigest().lower();
2010 else:
2011 catfileheadercshex = format(0, 'x').lower();
2012 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
2013 catfhend = (catfp.tell() - 1) + len(catfileoutstr);
2014 catfcontentstart = catfp.tell() + len(catfileoutstr);
2015 catfileoutstrecd = catfileoutstr.encode('UTF-8');
2016 nullstrecd = formatspecs[5].encode('UTF-8');
2017 catfileout = catfileoutstrecd + fcontents + nullstrecd;
2018 catfcontentend = (catfp.tell() - 1) + len(catfileout);
2019 catfp.write(catfileout);
2020 try:
2021 catfp.flush();
2022 os.fsync(catfp.fileno());
2023 except io.UnsupportedOperation:
2024 pass;
2025 except AttributeError:
2026 pass;
2027 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
2028 catfp = CompressArchiveFile(catfp, compression, formatspecs);
2029 try:
2030 catfp.flush();
2031 os.fsync(catfp.fileno());
2032 except io.UnsupportedOperation:
2033 pass;
2034 except AttributeError:
2035 pass;
2036 if(outfile=="-"):
2037 catfp.seek(0, 0);
2038 if(hasattr(sys.stdout, "buffer")):
2039 shutil.copyfileobj(catfp, sys.stdout.buffer);
2040 else:
2041 shutil.copyfileobj(catfp, sys.stdout);
2042 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", outfile)):
2043 catfp = CompressArchiveFile(catfp, compression, formatspecs);
2044 catfp.seek(0, 0);
2045 upload_file_to_internet_file(catfp, outfile);
2046 if(returnfp):
2047 catfp.seek(0, 0);
2048 return catfp;
2049 else:
2050 catfp.close();
2051 return True;
2052
2053 create_alias_function("Pack", __file_format_name__, "", PackArchiveFile);
2054
2055 if(hasattr(shutil, "register_archive_format")):
2056 def PackArchiveFileFunc(archive_name, source_dir, **kwargs):
2057 return PackArchiveFile(source_dir, archive_name, False, "auto", None, False, "crc32", [], __file_format_delimiter__, False, False);
2058 create_alias_function("Pack", __file_format_name__, "Func", PackArchiveFileFunc);
2059
2060 def PackArchiveFileFromDirList(infiles, outfile, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
2061 return PackArchiveFile(infiles, outfile, dirlistfromtxt, compression, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, returnfp);
2062
2063 def PackArchiveFileFromTarFile(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
2064 compressionlist = ['auto', 'gzip', 'bzip2', 'zstd', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
2065 outextlist = ['gz', 'bz2', 'zst', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
2066 outextlistwd = ['.gz', '.bz2', '.zst', '.lz4', '.lzo', '.lzop', '.lzma', '.xz'];
2067 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2068 outfile = RemoveWindowsPath(outfile);
2069 checksumtype = checksumtype.lower();
2070 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
2071 checksumtype="crc32";
2072 if(checksumtype=="none"):
2073 checksumtype = "";
2074 if(not compression or compression or compression=="catfile" or compression==formatspecs[2]):
2075 compression = None;
2076 if(compression not in compressionlist and compression is None):
2077 compression = "auto";
2078 if(verbose):
2079 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2080 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2081 if(os.path.exists(outfile)):
2082 os.unlink(outfile);
2083 if(outfile=="-"):
2084 verbose = False;
2085 catfp = BytesIO();
2086 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2087 catfp = outfile;
2088 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", outfile)):
2089 catfp = BytesIO();
2090 else:
2091 fbasename = os.path.splitext(outfile)[0];
2092 fextname = os.path.splitext(outfile)[1];
2093 catfp = CompressOpenFile(outfile, compressionlevel);
2094 catver = formatspecs[6];
2095 fileheaderver = str(int(catver.replace(".", "")));
2096 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
2097 catfp.write(fileheader.encode('UTF-8'));
2098 curinode = 0;
2099 curfid = 0;
2100 inodelist = [];
2101 inodetofile = {};
2102 filetoinode = {};
2103 inodetocatinode = {};
2104 if(infile=="-"):
2105 infile = BytesIO();
2106 if(hasattr(sys.stdin, "buffer")):
2107 shutil.copyfileobj(sys.stdin.buffer, infile);
2108 else:
2109 shutil.copyfileobj(sys.stdin, infile);
2110 infile.seek(0, 0);
2111 if(not infile):
2112 return False;
2113 infile.seek(0, 0);
2114 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", infile)):
2115 infile = download_file_from_internet_file(infile);
2116 infile.seek(0, 0);
2117 if(not infile):
2118 return False;
2119 infile.seek(0, 0);
2120 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
2121 return False;
2122 elif(os.path.exists(infile) and os.path.isfile(infile)):
2123 try:
2124 if(not tarfile.is_tarfile(infile)):
2125 return False;
2126 except AttributeError:
2127 if(not is_tarfile(infile)):
2128 return False;
2129 else:
2130 return False;
2131 try:
2132 if(hasattr(infile, "read") or hasattr(infile, "write")):
2133 tarfp = tarfile.open(fileobj=infile, mode="r");
2134 else:
2135 tarfp = tarfile.open(infile, "r");
2136 except FileNotFoundError:
2137 return False;
2138 fnumfiles = format(int(len(tarfp.getmembers())), 'x').lower();
2139 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs[5]);
2140 if(checksumtype=="none" or checksumtype==""):
2141 catfileheadercshex = format(0, 'x').lower();
2142 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
2143 catfileheadercshex = format(crc16(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffff, '04x').lower();
2144 elif(checksumtype=="crc16_ccitt"):
2145 catfileheadercshex = format(crc16_ccitt(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffff, '04x').lower();
2146 elif(checksumtype=="adler32"):
2147 catfileheadercshex = format(zlib.adler32(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffff, '08x').lower();
2148 elif(checksumtype=="crc32"):
2149 catfileheadercshex = format(crc32(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffff, '08x').lower();
2150 elif(checksumtype=="crc64_ecma"):
2151 catfileheadercshex = format(crc64_ecma(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2152 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
2153 catfileheadercshex = format(crc64_iso(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2154 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
2155 checksumoutstr = hashlib.new(checksumtype);
2156 checksumoutstr.update(str(fileheader + fnumfilesa).encode('UTF-8'));
2157 catfileheadercshex = checksumoutstr.hexdigest().lower();
2158 else:
2159 catfileheadercshex = format(0, 'x').lower();
2160 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs[5]);
2161 catfp.write(fnumfilesa.encode('UTF-8'));
2162 try:
2163 catfp.flush();
2164 os.fsync(catfp.fileno());
2165 except io.UnsupportedOperation:
2166 pass;
2167 except AttributeError:
2168 pass;
2169 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
2170 catfhstart = catfp.tell();
2171 if(re.findall("^[.|/]", member.name)):
2172 fname = member.name;
2173 else:
2174 fname = "./"+member.name;
2175 if(verbose):
2176 VerbosePrintOut(fname);
2177 fpremode = member.mode;
2178 ffullmode = member.mode;
2179 flinkcount = 0;
2180 ftype = 0;
2181 if(member.isreg()):
2182 ffullmode = member.mode + stat.S_IFREG;
2183 ftype = 0;
2184 elif(member.isdev()):
2185 ffullmode = member.mode;
2186 ftype = 7;
2187 elif(member.islnk()):
2188 ffullmode = member.mode + stat.S_IFREG;
2189 ftype = 1;
2190 elif(member.issym()):
2191 ffullmode = member.mode + stat.S_IFLNK;
2192 ftype = 2;
2193 elif(member.ischr()):
2194 ffullmode = member.mode + stat.S_IFCHR;
2195 ftype = 3;
2196 elif(member.isblk()):
2197 ffullmode = member.mode + stat.S_IFBLK;
2198 ftype = 4;
2199 elif(member.isdir()):
2200 ffullmode = member.mode + stat.S_IFDIR;
2201 ftype = 5;
2202 elif(member.isfifo()):
2203 ffullmode = member.mode + stat.S_IFIFO;
2204 ftype = 6;
2205 elif(member.issparse()):
2206 ffullmode = member.mode;
2207 ftype = 12;
2208 else:
2209 ffullmode = member.mode;
2210 ftype = 0;
2211 flinkname = "";
2212 fcurfid = format(int(curfid), 'x').lower();
2213 fcurinode = format(int(0), 'x').lower();
2214 curfid = curfid + 1;
2215 if(ftype==2):
2216 flinkname = member.linkname;
2217 fdev_minor = format(int(member.devminor), 'x').lower();
2218 fdev_major = format(int(member.devmajor), 'x').lower();
2219 frdev_minor = format(int(member.devminor), 'x').lower();
2220 frdev_major = format(int(member.devmajor), 'x').lower();
2221 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
2222 fsize = format(int("0"), 'x').lower();
2223 elif(ftype==0 or ftype==7):
2224 fsize = format(int(member.size), 'x').lower();
2225 else:
2226 fsize = format(int(member.size), 'x').lower();
2227 fatime = format(int(member.mtime), 'x').lower();
2228 fmtime = format(int(member.mtime), 'x').lower();
2229 fctime = format(int(member.mtime), 'x').lower();
2230 fbtime = format(int(member.mtime), 'x').lower();
2231 fmode = format(int(ffullmode), 'x').lower();
2232 fchmode = format(int(stat.S_IMODE(ffullmode)), 'x').lower();
2233 ftypemod = format(int(stat.S_IFMT(ffullmode)), 'x').lower();
2234 fuid = format(int(member.uid), 'x').lower();
2235 fgid = format(int(member.gid), 'x').lower();
2236 funame = member.uname;
2237 fgname = member.gname;
2238 flinkcount = format(int(flinkcount), 'x').lower();
2239 fwinattributes = format(int(0), 'x').lower();
2240 fcontents = "".encode('UTF-8');
2241 chunk_size = 1024;
2242 if(ftype == 0 or ftype == 7):
2243 with tarfp.extractfile(member) as fpc:
2244 while(True):
2245 chunk = fpc.read(chunk_size);
2246 if(not chunk):
2247 break
2248 fcontents += chunk;
2249 ftypehex = format(ftype, 'x').lower();
2250 extrafields = format(len(extradata), 'x').lower();
2251 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
2252 if(len(extradata)>0):
2253 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
2254 extrasizelen = format(len(extrasizestr), 'x').lower();
2255 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, extrasizelen, extrafields];
2256 catoutlen = len(catoutlist) + len(extradata) + 3;
2257 catoutlenhex = format(catoutlen, 'x').lower();
2258 catoutlist.insert(0, catoutlenhex);
2259 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
2260 if(len(extradata)>0):
2261 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
2262 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
2263 catfhend = (catfp.tell() - 1) + len(catfileoutstr);
2264 catfcontentstart = catfp.tell() + len(catfileoutstr);
2265 if(checksumtype=="none" or checksumtype==""):
2266 catfileheadercshex = format(0, 'x').lower();
2267 catfilecontentcshex = format(0, 'x').lower();
2268 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
2269 catfileheadercshex = format(crc16("".encode('UTF-8')) & 0xffff, '04x').lower();
2270 catfilecontentcshex = format(crc16(fcontents) & 0xffff, '04x').lower();
2271 elif(checksumtype=="crc16_ccitt"):
2272 catfileheadercshex = format(crc16_ccitt("".encode('UTF-8')) & 0xffff, '04x').lower();
2273 catfilecontentcshex = format(crc16_ccitt(fcontents) & 0xffff, '04x').lower();
2274 elif(checksumtype=="adler32"):
2275 catfileheadercshex = format(zlib.adler32("".encode('UTF-8')) & 0xffffffff, '08x').lower();
2276 catfilecontentcshex = format(zlib.adler32(fcontents) & 0xffffffff, '08x').lower();
2277 elif(checksumtype=="crc32"):
2278 catfileheadercshex = format(crc32("".encode('UTF-8')) & 0xffffffff, '08x').lower();
2279 catfilecontentcshex = format(crc32(fcontents) & 0xffffffff, '08x').lower();
2280 elif(checksumtype=="crc64_ecma"):
2281 catfileheadercshex = format(crc64_ecma("".encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2282 catfilecontentcshex = format(crc64_ecma(fcontents) & 0xffffffffffffffff, '016x').lower();
2283 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
2284 catfileheadercshex = format(crc64_iso("".encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2285 catfilecontentcshex = format(crc64_iso(fcontents) & 0xffffffffffffffff, '016x').lower();
2286 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
2287 checksumoutstr = hashlib.new(checksumtype);
2288 checksumoutstr.update("".encode('UTF-8'));
2289 catfileheadercshex = checksumoutstr.hexdigest().lower();
2290 checksumoutstr = hashlib.new(checksumtype);
2291 checksumoutstr.update(fcontents);
2292 catfilecontentcshex = checksumoutstr.hexdigest().lower();
2293 else:
2294 catfileheadercshex = format(0, 'x').lower();
2295 catfilecontentcshex = format(0, 'x').lower();
2296 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
2297 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower()
2298 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
2299 if(checksumtype=="none" or checksumtype==""):
2300 catfileheadercshex = format(0, 'x').lower();
2301 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
2302 catfileheadercshex = format(crc16(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
2303 elif(checksumtype=="crc16_ccitt"):
2304 catfileheadercshex = format(crc16_ccitt(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
2305 elif(checksumtype=="adler32"):
2306 catfileheadercshex = format(zlib.adler32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
2307 elif(checksumtype=="crc32"):
2308 catfileheadercshex = format(crc32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
2309 elif(checksumtype=="crc64_ecma"):
2310 catfileheadercshex = format(crc64_ecma(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2311 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
2312 catfileheadercshex = format(crc64_iso(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2313 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
2314 checksumoutstr = hashlib.new(checksumtype);
2315 checksumoutstr.update(catfileoutstr.encode('UTF-8'));
2316 catfileheadercshex = checksumoutstr.hexdigest().lower();
2317 else:
2318 catfileheadercshex = format(0, 'x').lower();
2319 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
2320 catfileoutstrecd = catfileoutstr.encode('UTF-8');
2321 nullstrecd = formatspecs[5].encode('UTF-8');
2322 catfileout = catfileoutstrecd + fcontents + nullstrecd;
2323 catfcontentend = (catfp.tell() - 1) + len(catfileout);
2324 catfp.write(catfileout);
2325 try:
2326 catfp.flush();
2327 os.fsync(catfp.fileno());
2328 except io.UnsupportedOperation:
2329 pass;
2330 except AttributeError:
2331 pass;
2332 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
2333 catfp = CompressArchiveFile(catfp, compression, formatspecs);
2334 try:
2335 catfp.flush();
2336 os.fsync(catfp.fileno());
2337 except io.UnsupportedOperation:
2338 pass;
2339 except AttributeError:
2340 pass;
2341 if(outfile=="-"):
2342 catfp.seek(0, 0);
2343 if(hasattr(sys.stdout, "buffer")):
2344 shutil.copyfileobj(catfp, sys.stdout.buffer);
2345 else:
2346 shutil.copyfileobj(catfp, sys.stdout);
2347 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", outfile)):
2348 catfp = CompressArchiveFile(catfp, compression, formatspecs);
2349 catfp.seek(0, 0);
2350 upload_file_from_internet_file(catfp, outfile);
2351 if(returnfp):
2352 catfp.seek(0, 0);
2353 return catfp;
2354 else:
2355 catfp.close();
2356 return True;
2357
2358 create_alias_function("Pack", __file_format_name__, "FromTarFile", PackArchiveFileFromTarFile);
2359
2360 def PackArchiveFileFromZipFile(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
2361 compressionlist = ['auto', 'gzip', 'bzip2', 'zstd', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
2362 outextlist = ['gz', 'bz2', 'zst', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
2363 outextlistwd = ['.gz', '.bz2', '.zst', '.lz4', '.lzo', '.lzop', '.lzma', '.xz'];
2364 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2365 outfile = RemoveWindowsPath(outfile);
2366 checksumtype = checksumtype.lower();
2367 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
2368 checksumtype="crc32";
2369 if(checksumtype=="none"):
2370 checksumtype = "";
2371 if(not compression or compression or compression=="catfile" or compression==formatspecs[2]):
2372 compression = None;
2373 if(compression not in compressionlist and compression is None):
2374 compression = "auto";
2375 if(verbose):
2376 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2377 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2378 if(os.path.exists(outfile)):
2379 os.unlink(outfile);
2380 if(outfile=="-"):
2381 verbose = False;
2382 catfp = BytesIO();
2383 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2384 catfp = outfile;
2385 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", outfile)):
2386 catfp = BytesIO();
2387 else:
2388 fbasename = os.path.splitext(outfile)[0];
2389 fextname = os.path.splitext(outfile)[1];
2390 catfp = CompressOpenFile(outfile, compressionlevel);
2391 catver = formatspecs[6];
2392 fileheaderver = str(int(catver.replace(".", "")));
2393 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
2394 catfp.write(fileheader.encode('UTF-8'));
2395 curinode = 0;
2396 curfid = 0;
2397 inodelist = [];
2398 inodetofile = {};
2399 filetoinode = {};
2400 inodetocatinode = {};
2401 if(infile=="-"):
2402 infile = BytesIO();
2403 if(hasattr(sys.stdin, "buffer")):
2404 shutil.copyfileobj(sys.stdin.buffer, infile);
2405 else:
2406 shutil.copyfileobj(sys.stdin, infile);
2407 infile.seek(0, 0);
2408 if(not infile):
2409 return False;
2410 infile.seek(0, 0);
2411 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", infile)):
2412 infile = download_file_from_internet_file(infile);
2413 infile.seek(0, 0);
2414 if(not infile):
2415 return False;
2416 infile.seek(0, 0);
2417 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
2418 return False;
2419 else:
2420 return False;
2421 if(not zipfile.is_zipfile(infile)):
2422 return False;
2423 try:
2424 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
2425 except FileNotFoundError:
2426 return False;
2427 ziptest = zipfp.testzip();
2428 if(ziptest):
2429 VerbosePrintOut("Bad file found!");
2430 fnumfiles = format(int(len(zipfp.infolist())), 'x').lower();
2431 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs[5]);
2432 if(checksumtype=="none" or checksumtype==""):
2433 catfileheadercshex = format(0, 'x').lower();
2434 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
2435 catfileheadercshex = format(crc16(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffff, '04x').lower();
2436 elif(checksumtype=="crc16_ccitt"):
2437 catfileheadercshex = format(crc16_ccitt(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffff, '04x').lower();
2438 elif(checksumtype=="adler32"):
2439 catfileheadercshex = format(zlib.adler32(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffff, '08x').lower();
2440 elif(checksumtype=="crc32"):
2441 catfileheadercshex = format(crc32(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffff, '08x').lower();
2442 elif(checksumtype=="crc64_ecma"):
2443 catfileheadercshex = format(crc64_ecma(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2444 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
2445 catfileheadercshex = format(crc64_iso(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2446 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
2447 checksumoutstr = hashlib.new(checksumtype);
2448 checksumoutstr.update(str(fileheader + fnumfilesa).encode('UTF-8'));
2449 catfileheadercshex = checksumoutstr.hexdigest().lower();
2450 else:
2451 catfileheadercshex = format(0, 'x').lower();
2452 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs[5]);
2453 catfp.write(fnumfilesa.encode('UTF-8'));
2454 try:
2455 catfp.flush();
2456 os.fsync(catfp.fileno());
2457 except io.UnsupportedOperation:
2458 pass;
2459 except AttributeError:
2460 pass;
2461 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
2462 catfhstart = catfp.tell();
2463 if(re.findall("^[.|/]", member.filename)):
2464 fname = member.filename;
2465 else:
2466 fname = "./"+member.filename;
2467 zipinfo = zipfp.getinfo(member.filename);
2468 if(verbose):
2469 VerbosePrintOut(fname);
2470 if(not member.is_dir()):
2471 fpremode = int(stat.S_IFREG + 438);
2472 elif(member.is_dir()):
2473 fpremode = int(stat.S_IFDIR + 511);
2474 flinkcount = 0;
2475 ftype = 0;
2476 if(not member.is_dir()):
2477 ftype = 0;
2478 elif(member.is_dir()):
2479 ftype = 5;
2480 flinkname = "";
2481 fcurfid = format(int(curfid), 'x').lower();
2482 fcurinode = format(int(0), 'x').lower();
2483 curfid = curfid + 1;
2484 fdev_minor = format(int(0), 'x').lower();
2485 fdev_major = format(int(0), 'x').lower();
2486 frdev_minor = format(int(0), 'x').lower();
2487 frdev_major = format(int(0), 'x').lower();
2488 if(ftype==5):
2489 fsize = format(int("0"), 'x').lower();
2490 elif(ftype==0):
2491 fsize = format(int(member.file_size), 'x').lower();
2492 else:
2493 fsize = format(int(member.file_size), 'x').lower();
2494 fatime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
2495 fmtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
2496 fctime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
2497 fbtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
2498 if(zipinfo.create_system==0 or zipinfo.create_system==10):
2499 fwinattributes = format(int(zipinfo.external_attr), 'x').lower();
2500 if(not member.is_dir()):
2501 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
2502 fchmode = stat.S_IMODE(fmode);
2503 ftypemod = stat.S_IFMT(fmode);
2504 elif(member.is_dir()):
2505 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
2506 fchmode = stat.S_IMODE(fmode);
2507 ftypemod = stat.S_IFMT(fmode);
2508 elif(zipinfo.create_system==3):
2509 fwinattributes = format(int(0), 'x').lower();
2510 fmode = format(int(zipinfo.external_attr), 'x').lower();
2511 fchmode = stat.S_IMODE(fmode);
2512 ftypemod = stat.S_IFMT(fmode);
2513 else:
2514 fwinattributes = format(int(0), 'x').lower();
2515 if(not member.is_dir()):
2516 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
2517 fchmode = stat.S_IMODE(fmode);
2518 ftypemod = stat.S_IFMT(fmode);
2519 elif(member.is_dir()):
2520 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
2521 fchmode = stat.S_IMODE(fmode);
2522 ftypemod = stat.S_IFMT(fmode);
2523 try:
2524 fuid = format(int(os.getuid()), 'x').lower();
2525 except AttributeError:
2526 fuid = format(int(0), 'x').lower();
2527 except KeyError:
2528 fuid = format(int(0), 'x').lower();
2529 try:
2530 fgid = format(int(os.getgid()), 'x').lower();
2531 except AttributeError:
2532 fgid = format(int(0), 'x').lower();
2533 except KeyError:
2534 fgid = format(int(0), 'x').lower();
2535 try:
2536 import pwd;
2537 try:
2538 userinfo = pwd.getpwuid(os.getuid());
2539 funame = userinfo.pw_name;
2540 except KeyError:
2541 funame = "";
2542 except AttributeError:
2543 funame = "";
2544 except ImportError:
2545 funame = "";
2546 fgname = "";
2547 try:
2548 import grp;
2549 try:
2550 groupinfo = grp.getgrgid(os.getgid());
2551 fgname = groupinfo.gr_name;
2552 except KeyError:
2553 fgname = "";
2554 except AttributeError:
2555 fgname = "";
2556 except ImportError:
2557 fgname = "";
2558 fcontents = "".encode('UTF-8');
2559 if(ftype==0):
2560 fcontents = zipfp.read(member.filename);
2561 ftypehex = format(ftype, 'x').lower();
2562 extrafields = format(len(extradata), 'x').lower();
2563 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
2564 if(len(extradata)>0):
2565 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
2566 extrasizelen = format(len(extrasizestr), 'x').lower();
2567 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, extrasizelen, extrafields];
2568 catoutlen = len(catoutlist) + len(extradata) + 3;
2569 catoutlenhex = format(catoutlen, 'x').lower();
2570 catoutlist.insert(0, catoutlenhex);
2571 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
2572 if(len(extradata)>0):
2573 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
2574 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
2575 catfhend = (catfp.tell() - 1) + len(catfileoutstr);
2576 catfcontentstart = catfp.tell() + len(catfileoutstr);
2577 if(checksumtype=="none" or checksumtype==""):
2578 catfileheadercshex = format(0, 'x').lower();
2579 catfilecontentcshex = format(0, 'x').lower();
2580 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
2581 catfileheadercshex = format(crc16("".encode('UTF-8')) & 0xffff, '04x').lower();
2582 catfilecontentcshex = format(crc16(fcontents) & 0xffff, '04x').lower();
2583 elif(checksumtype=="crc16_ccitt"):
2584 catfileheadercshex = format(crc16_ccitt("".encode('UTF-8')) & 0xffff, '04x').lower();
2585 catfilecontentcshex = format(crc16_ccitt(fcontents) & 0xffff, '04x').lower();
2586 elif(checksumtype=="adler32"):
2587 catfileheadercshex = format(zlib.adler32("".encode('UTF-8')) & 0xffffffff, '08x').lower();
2588 catfilecontentcshex = format(zlib.adler32(fcontents) & 0xffffffff, '08x').lower();
2589 elif(checksumtype=="crc32"):
2590 catfileheadercshex = format(crc32("".encode('UTF-8')) & 0xffffffff, '08x').lower();
2591 catfilecontentcshex = format(crc32(fcontents) & 0xffffffff, '08x').lower();
2592 elif(checksumtype=="crc64_ecma"):
2593 catfileheadercshex = format(crc64_ecma("".encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2594 catfilecontentcshex = format(crc64_ecma(fcontents) & 0xffffffffffffffff, '016x').lower();
2595 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
2596 catfileheadercshex = format(crc64_iso("".encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2597 catfilecontentcshex = format(crc64_iso(fcontents) & 0xffffffffffffffff, '016x').lower();
2598 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
2599 checksumoutstr = hashlib.new(checksumtype);
2600 checksumoutstr.update("".encode('UTF-8'));
2601 catfileheadercshex = checksumoutstr.hexdigest().lower();
2602 checksumoutstr = hashlib.new(checksumtype);
2603 checksumoutstr.update(fcontents);
2604 catfilecontentcshex = checksumoutstr.hexdigest().lower();
2605 else:
2606 catfileheadercshex = format(0, 'x').lower();
2607 catfilecontentcshex = format(0, 'x').lower();
2608 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
2609 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower()
2610 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
2611 if(checksumtype=="none" or checksumtype==""):
2612 catfileheadercshex = format(0, 'x').lower();
2613 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
2614 catfileheadercshex = format(crc16(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
2615 elif(checksumtype=="crc16_ccitt"):
2616 catfileheadercshex = format(crc16_ccitt(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
2617 elif(checksumtype=="adler32"):
2618 catfileheadercshex = format(zlib.adler32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
2619 elif(checksumtype=="crc32"):
2620 catfileheadercshex = format(crc32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
2621 elif(checksumtype=="crc64_ecma"):
2622 catfileheadercshex = format(crc64_ecma(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2623 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
2624 catfileheadercshex = format(crc64_iso(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2625 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
2626 checksumoutstr = hashlib.new(checksumtype);
2627 checksumoutstr.update(catfileoutstr.encode('UTF-8'));
2628 catfileheadercshex = checksumoutstr.hexdigest().lower();
2629 else:
2630 catfileheadercshex = format(0, 'x').lower();
2631 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
2632 catfileoutstrecd = catfileoutstr.encode('UTF-8');
2633 nullstrecd = formatspecs[5].encode('UTF-8');
2634 catfileout = catfileoutstrecd + fcontents + nullstrecd;
2635 catfcontentend = (catfp.tell() - 1) + len(catfileout);
2636 catfp.write(catfileout);
2637 try:
2638 catfp.flush();
2639 os.fsync(catfp.fileno());
2640 except io.UnsupportedOperation:
2641 pass;
2642 except AttributeError:
2643 pass;
2644 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
2645 catfp = CompressArchiveFile(catfp, compression, formatspecs);
2646 try:
2647 catfp.flush();
2648 os.fsync(catfp.fileno());
2649 except io.UnsupportedOperation:
2650 pass;
2651 except AttributeError:
2652 pass;
2653 if(outfile=="-"):
2654 catfp.seek(0, 0);
2655 if(hasattr(sys.stdout, "buffer")):
2656 shutil.copyfileobj(catfp, sys.stdout.buffer);
2657 else:
2658 shutil.copyfileobj(catfp, sys.stdout);
2659 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", outfile)):
2660 catfp = CompressArchiveFile(catfp, compression, formatspecs);
2661 catfp.seek(0, 0);
2662 upload_file_to_internet_file(catfp, outfile);
2663 if(returnfp):
2664 catfp.seek(0, 0);
2665 return catfp;
2666 else:
2667 catfp.close();
2668 return True;
2669
2670 create_alias_function("Pack", __file_format_name__, "FromZipFile", PackArchiveFileFromZipFile);
2671
2672 if(not rarfile_support):
2673 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
2674 return False
2675
2676 if(rarfile_support):
2677 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
2678 compressionlist = ['auto', 'gzip', 'bzip2', 'zstd', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
2679 outextlist = ['gz', 'bz2', 'zst', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
2680 outextlistwd = ['.gz', '.bz2', '.zst', '.lz4', '.lzo', '.lzop', '.lzma', '.xz'];
2681 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2682 outfile = RemoveWindowsPath(outfile);
2683 checksumtype = checksumtype.lower();
2684 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
2685 checksumtype="crc32";
2686 if(checksumtype=="none"):
2687 checksumtype = "";
2688 if(not compression or compression or compression=="catfile" or compression==formatspecs[2]):
2689 compression = None;
2690 if(compression not in compressionlist and compression is None):
2691 compression = "auto";
2692 if(verbose):
2693 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2694 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2695 if(os.path.exists(outfile)):
2696 os.unlink(outfile);
2697 if(outfile=="-"):
2698 verbose = False;
2699 catfp = BytesIO();
2700 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2701 catfp = outfile;
2702 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", outfile)):
2703 catfp = BytesIO();
2704 else:
2705 fbasename = os.path.splitext(outfile)[0];
2706 fextname = os.path.splitext(outfile)[1];
2707 catfp = CompressOpenFile(outfile, compressionlevel);
2708 catver = formatspecs[6];
2709 fileheaderver = str(int(catver.replace(".", "")));
2710 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
2711 catfp.write(fileheader.encode('UTF-8'));
2712 curinode = 0;
2713 curfid = 0;
2714 inodelist = [];
2715 inodetofile = {};
2716 filetoinode = {};
2717 inodetocatinode = {};
2718 if(not os.path.exists(infile) or not os.path.isfile(infile)):
2719 return False;
2720 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
2721 return False;
2722 rarfp = rarfile.RarFile(infile, "r");
2723 rartest = rarfp.testrar();
2724 if(rartest):
2725 VerbosePrintOut("Bad file found!");
2726 fnumfiles = format(int(len(rarfp.infolist())), 'x').lower();
2727 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs[5]);
2728 if(checksumtype=="none" or checksumtype==""):
2729 catfileheadercshex = format(0, 'x').lower();
2730 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
2731 catfileheadercshex = format(crc16(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffff, '04x').lower();
2732 elif(checksumtype=="crc16_ccitt"):
2733 catfileheadercshex = format(crc16_ccitt(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffff, '04x').lower();
2734 elif(checksumtype=="adler32"):
2735 catfileheadercshex = format(zlib.adler32(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffff, '08x').lower();
2736 elif(checksumtype=="crc32"):
2737 catfileheadercshex = format(crc32(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffff, '08x').lower();
2738 elif(checksumtype=="crc64_ecma"):
2739 catfileheadercshex = format(crc64_ecma(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2740 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
2741 catfileheadercshex = format(crc64_iso(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2742 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
2743 checksumoutstr = hashlib.new(checksumtype);
2744 checksumoutstr.update(str(fileheader + fnumfilesa).encode('UTF-8'));
2745 catfileheadercshex = checksumoutstr.hexdigest().lower();
2746 else:
2747 catfileheadercshex = format(0, 'x').lower();
2748 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs[5]);
2749 catfp.write(fnumfilesa.encode('UTF-8'));
2750 try:
2751 catfp.flush();
2752 os.fsync(catfp.fileno());
2753 except io.UnsupportedOperation:
2754 pass;
2755 except AttributeError:
2756 pass;
2757 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
2758 is_unix = False;
2759 is_windows = False;
2760 if(member.host_os == rarfile.RAR_OS_UNIX):
2761 is_windows = False;
2762 try:
2763 member.external_attr
2764 is_unix = True;
2765 except AttributeError:
2766 is_unix = False;
2767 elif(member.host_os == rarfile.RAR_OS_WIN32):
2768 is_unix = False;
2769 try:
2770 member.external_attr
2771 is_windows = True;
2772 except AttributeError:
2773 is_windows = False;
2774 else:
2775 is_unix = False;
2776 is_windows = False;
2777 catfhstart = catfp.tell();
2778 if(re.findall("^[.|/]", member.filename)):
2779 fname = member.filename;
2780 else:
2781 fname = "./"+member.filename;
2782 rarinfo = rarfp.getinfo(member.filename);
2783 if(verbose):
2784 VerbosePrintOut(fname);
2785 if(is_unix and member.external_attr !=0):
2786 fpremode = int(member.external_attr);
2787 elif(member.is_file()):
2788 fpremode = int(stat.S_IFREG + 438);
2789 elif(member.is_symlink()):
2790 fpremode = int(stat.S_IFLNK + 438);
2791 elif(member.is_dir()):
2792 fpremode = int(stat.S_IFDIR + 511);
2793 if(is_windows and member.external_attr !=0):
2794 fwinattributes = format(int(member.external_attr), 'x').lower();
2795 else:
2796 fwinattributes = format(int(0), 'x').lower();
2797 flinkcount = 0;
2798 ftype = 0;
2799 if(member.is_file()):
2800 ftype = 0;
2801 elif(member.is_symlink()):
2802 ftype = 2;
2803 elif(member.is_dir()):
2804 ftype = 5;
2805 flinkname = "";
2806 if(ftype==2):
2807 flinkname = rarfp.read(member.filename).decode("UTF-8");
2808 fcurfid = format(int(curfid), 'x').lower();
2809 fcurinode = format(int(0), 'x').lower();
2810 curfid = curfid + 1;
2811 fdev_minor = format(int(0), 'x').lower();
2812 fdev_major = format(int(0), 'x').lower();
2813 frdev_minor = format(int(0), 'x').lower();
2814 frdev_major = format(int(0), 'x').lower();
2815 if(ftype==5):
2816 fsize = format(int("0"), 'x').lower();
2817 elif(ftype==0):
2818 fsize = format(int(member.file_size), 'x').lower();
2819 else:
2820 fsize = format(int(member.file_size), 'x').lower();
2821 try:
2822 if(member.atime):
2823 fatime = format(int(member.atime.timestamp()), 'x').lower();
2824 else:
2825 fatime = format(int(member.mtime.timestamp()), 'x').lower();
2826 except AttributeError:
2827 fatime = format(int(member.mtime.timestamp()), 'x').lower();
2828 fmtime = format(int(member.mtime.timestamp()), 'x').lower();
2829 try:
2830 if(member.ctime):
2831 fctime = format(int(member.ctime.timestamp()), 'x').lower();
2832 else:
2833 fctime = format(int(member.mtime.timestamp()), 'x').lower();
2834 except AttributeError:
2835 fctime = format(int(member.mtime.timestamp()), 'x').lower();
2836 fbtime = format(int(member.mtime.timestamp()), 'x').lower();
2837 if(is_unix and member.external_attr !=0):
2838 fmode = format(int(member.external_attr), 'x').lower();
2839 fchmode = format(int(stat.S_IMODE(member.external_attr)), 'x').lower();
2840 ftypemod = format(int(stat.S_IFMT(member.external_attr)), 'x').lower();
2841 elif(member.is_file()):
2842 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
2843 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
2844 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
2845 elif(member.is_symlink()):
2846 fmode = format(int(stat.S_IFLNK + 438), 'x').lower();
2847 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
2848 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
2849 elif(member.is_dir()):
2850 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
2851 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
2852 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
2853 try:
2854 fuid = format(int(os.getuid()), 'x').lower();
2855 except AttributeError:
2856 fuid = format(int(0), 'x').lower();
2857 except KeyError:
2858 fuid = format(int(0), 'x').lower();
2859 try:
2860 fgid = format(int(os.getgid()), 'x').lower();
2861 except AttributeError:
2862 fgid = format(int(0), 'x').lower();
2863 except KeyError:
2864 fgid = format(int(0), 'x').lower();
2865 try:
2866 import pwd;
2867 try:
2868 userinfo = pwd.getpwuid(os.getuid());
2869 funame = userinfo.pw_name;
2870 except KeyError:
2871 funame = "";
2872 except AttributeError:
2873 funame = "";
2874 except ImportError:
2875 funame = "";
2876 fgname = "";
2877 try:
2878 import grp;
2879 try:
2880 groupinfo = grp.getgrgid(os.getgid());
2881 fgname = groupinfo.gr_name;
2882 except KeyError:
2883 fgname = "";
2884 except AttributeError:
2885 fgname = "";
2886 except ImportError:
2887 fgname = "";
2888 fcontents = "".encode('UTF-8');
2889 if(ftype==0):
2890 fcontents = rarfp.read(member.filename);
2891 ftypehex = format(ftype, 'x').lower();
2892 extrafields = format(len(extradata), 'x').lower();
2893 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
2894 if(len(extradata)>0):
2895 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
2896 extrasizelen = format(len(extrasizestr), 'x').lower();
2897 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, extrasizelen, extrafields];
2898 catoutlen = len(catoutlist) + len(extradata) + 3;
2899 catoutlenhex = format(catoutlen, 'x').lower();
2900 catoutlist.insert(0, catoutlenhex);
2901 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
2902 if(len(extradata)>0):
2903 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
2904 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
2905 catfhend = (catfp.tell() - 1) + len(catfileoutstr);
2906 catfcontentstart = catfp.tell() + len(catfileoutstr);
2907 if(checksumtype=="none" or checksumtype==""):
2908 catfileheadercshex = format(0, 'x').lower();
2909 catfilecontentcshex = format(0, 'x').lower();
2910 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
2911 catfileheadercshex = format(crc16("".encode('UTF-8')) & 0xffff, '04x').lower();
2912 catfilecontentcshex = format(crc16(fcontents) & 0xffff, '04x').lower();
2913 elif(checksumtype=="crc16_ccitt"):
2914 catfileheadercshex = format(crc16_ccitt("".encode('UTF-8')) & 0xffff, '04x').lower();
2915 catfilecontentcshex = format(crc16_ccitt(fcontents) & 0xffff, '04x').lower();
2916 elif(checksumtype=="adler32"):
2917 catfileheadercshex = format(zlib.adler32("".encode('UTF-8')) & 0xffffffff, '08x').lower();
2918 catfilecontentcshex = format(zlib.adler32(fcontents) & 0xffffffff, '08x').lower();
2919 elif(checksumtype=="crc32"):
2920 catfileheadercshex = format(crc32("".encode('UTF-8')) & 0xffffffff, '08x').lower();
2921 catfilecontentcshex = format(crc32(fcontents) & 0xffffffff, '08x').lower();
2922 elif(checksumtype=="crc64_ecma"):
2923 catfileheadercshex = format(crc64_ecma("".encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2924 catfilecontentcshex = format(crc64_ecma(fcontents) & 0xffffffffffffffff, '016x').lower();
2925 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
2926 catfileheadercshex = format(crc64_iso("".encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
2927 catfilecontentcshex = format(crc64_iso(fcontents) & 0xffffffffffffffff, '016x').lower();
2928 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
2929 checksumoutstr = hashlib.new(checksumtype);
2930 checksumoutstr.update("".encode('UTF-8'));
2931 catfileheadercshex = checksumoutstr.hexdigest().lower();
2932 checksumoutstr = hashlib.new(checksumtype);
2933 checksumoutstr.update(fcontents);
2934 catfilecontentcshex = checksumoutstr.hexdigest().lower();
2935 else:
2936 catfileheadercshex = format(0, 'x').lower();
2937 catfilecontentcshex = format(0, 'x').lower();
2938 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
2939 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower()
2940 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr
2941 if(checksumtype=="none" or checksumtype==""):
2942 catfileheadercshex = format(0, 'x').lower()
2943 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
2944 catfileheadercshex = format(crc16(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower()
2945 elif(checksumtype=="crc16_ccitt"):
2946 catfileheadercshex = format(crc16_ccitt(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower()
2947 elif(checksumtype=="adler32"):
2948 catfileheadercshex = format(zlib.adler32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower()
2949 elif(checksumtype=="crc32"):
2950 catfileheadercshex = format(crc32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower()
2951 elif(checksumtype=="crc64_ecma"):
2952 catfileheadercshex = format(crc64_ecma(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower()
2953 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
2954 catfileheadercshex = format(crc64_iso(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower()
2955 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
2956 checksumoutstr = hashlib.new(checksumtype)
2957 checksumoutstr.update(catfileoutstr.encode('UTF-8'))
2958 catfileheadercshex = checksumoutstr.hexdigest().lower()
2959 else:
2960 catfileheadercshex = format(0, 'x').lower()
2961 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5])
2962 catfileoutstrecd = catfileoutstr.encode('UTF-8')
2963 nullstrecd = formatspecs[5].encode('UTF-8')
2964 catfileout = catfileoutstrecd + fcontents + nullstrecd
2965 catfcontentend = (catfp.tell() - 1) + len(catfileout)
2966 catfp.write(catfileout)
2967 try:
2968 catfp.flush()
2969 os.fsync(catfp.fileno())
2970 except io.UnsupportedOperation:
2971 pass
2972 except AttributeError:
2973 pass
2974 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
2975 catfp = CompressArchiveFile(catfp, compression, formatspecs)
2976 try:
2977 catfp.flush()
2978 os.fsync(catfp.fileno())
2979 except io.UnsupportedOperation:
2980 pass
2981 except AttributeError:
2982 pass
2983 if(outfile=="-"):
2984 catfp.seek(0, 0)
2985 if(hasattr(sys.stdout, "buffer")):
2986 shutil.copyfileobj(catfp, sys.stdout.buffer);
2987 else:
2988 shutil.copyfileobj(catfp, sys.stdout);
2989 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", outfile)):
2990 catfp = CompressArchiveFile(catfp, compression, formatspecs);
2991 catfp.seek(0, 0);
2992 upload_file_to_internet_file(catfp, outfile);
2993 if(returnfp):
2994 catfp.seek(0, 0)
2995 return catfp
2996 else:
2997 catfp.close()
2998 return True;
2999
3000 create_alias_function("Pack", __file_format_name__, "FromRarFile", PackArchiveFileFromRarFile);
3001
3002 def ArchiveFileSeekToFileNum(infile, seekto=0, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
3003 if(hasattr(infile, "read") or hasattr(infile, "write")):
3004 catfp = infile;
3005 catfp.seek(0, 0);
3006 catfp = UncompressArchiveFile(catfp, formatspecs);
3007 checkcompressfile = CheckCompressionSubType(catfp, formatspecs);
3008 if(checkcompressfile=="tarfile"):
3009 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3010 if(checkcompressfile=="zipfile"):
3011 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3012 if(rarfile_support and checkcompressfile=="rarfile"):
3013 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3014 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
3015 return False;
3016 if(not catfp):
3017 return False;
3018 catfp.seek(0, 0);
3019 elif(infile=="-"):
3020 catfp = BytesIO();
3021 if(hasattr(sys.stdin, "buffer")):
3022 shutil.copyfileobj(sys.stdin.buffer, catfp);
3023 else:
3024 shutil.copyfileobj(sys.stdin, catfp);
3025 catfp.seek(0, 0);
3026 catfp = UncompressArchiveFile(catfp, formatspecs);
3027 if(not catfp):
3028 return False;
3029 catfp.seek(0, 0);
3030 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", infile)):
3031 catfp = download_file_from_internet_file(infile);
3032 catfp.seek(0, 0);
3033 catfp = UncompressArchiveFile(catfp, formatspecs);
3034 if(not catfp):
3035 return False;
3036 catfp.seek(0, 0);
3037 else:
3038 infile = RemoveWindowsPath(infile);
3039 checkcompressfile = CheckCompressionSubType(infile, formatspecs);
3040 if(checkcompressfile=="tarfile"):
3041 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3042 if(checkcompressfile=="zipfile"):
3043 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3044 if(rarfile_support and checkcompressfile=="rarfile"):
3045 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3046 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
3047 return False;
3048 compresscheck = CheckCompressionType(infile, formatspecs, True);
3049 if(not compresscheck):
3050 fextname = os.path.splitext(infile)[1];
3051 if(fextname==".gz"):
3052 compresscheck = "gzip";
3053 elif(fextname==".bz2"):
3054 compresscheck = "bzip2";
3055 elif(fextname==".zst"):
3056 compresscheck = "zstd";
3057 elif(fextname==".lz4" or fextname==".clz4"):
3058 compresscheck = "lz4";
3059 elif(fextname==".lzo" or fextname==".lzop"):
3060 compresscheck = "lzo";
3061 elif(fextname==".lzma" or fextname==".xz"):
3062 compresscheck = "lzma";
3063 else:
3064 return False;
3065 if(not compresscheck):
3066 return False;
3067 catfp = UncompressFile(infile, formatspecs, "rb");
3068 '''
3069 try:
3070 catfp.seek(0, 2);
3071 except OSError:
3072 SeekToEndOfFile(catfp);
3073 except ValueError:
3074 SeekToEndOfFile(catfp);
3075 CatSize = catfp.tell();
3076 CatSizeEnd = CatSize;
3077 '''
3078 try:
3079 catfp.seek(0, 0);
3080 except OSError:
3081 return False;
3082 except ValueError:
3083 return False;
3084 catheader = ReadFileHeaderData(catfp, 4, formatspecs[5]);
3085 catstring = catheader[0];
3086 catversion = re.findall(r"([\d]+)$", catstring);
3087 fprenumfiles = catheader[1];
3088 fnumfiles = int(fprenumfiles, 16);
3089 fprechecksumtype = catheader[2];
3090 fprechecksum = catheader[3];
3091 fileheader = AppendNullByte(catstring, formatspecs[5]);
3092 fnumfileshex = format(int(fnumfiles), 'x').lower();
3093 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs[5]);
3094 if(fprechecksumtype=="none" or fprechecksumtype==""):
3095 catfileheadercshex = format(0, 'x').lower();
3096 elif(fprechecksumtype=="crc16" or fprechecksumtype=="crc16_ansi" or fprechecksumtype=="crc16_ibm"):
3097 catfileheadercshex = format(crc16(fileheader.encode('UTF-8')) & 0xffff, '04x').lower();
3098 elif(fprechecksumtype=="crc16_ccitt"):
3099 catfileheadercshex = format(crc16_ccitt(fileheader.encode('UTF-8')) & 0xffff, '04x').lower();
3100 elif(fprechecksumtype=="adler32"):
3101 catfileheadercshex = format(zlib.adler32(fileheader.encode('UTF-8')) & 0xffffffff, '08x').lower();
3102 elif(fprechecksumtype=="crc32"):
3103 catfileheadercshex = format(crc32(fileheader.encode('UTF-8')) & 0xffffffff, '08x').lower();
3104 elif(fprechecksumtype=="crc64_ecma"):
3105 catfileheadercshex = format(crc64_ecma(fileheader.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
3106 elif(fprechecksumtype=="crc64" or fprechecksumtype=="crc64_iso"):
3107 catfileheadercshex = format(crc64_iso(fileheader.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
3108 elif(CheckSumSupportAlt(fprechecksumtype, hashlib_guaranteed)):
3109 checksumoutstr = hashlib.new(fprechecksumtype);
3110 checksumoutstr.update(fileheader.encode('UTF-8'));
3111 catfileheadercshex = checksumoutstr.hexdigest().lower();
3112 else:
3113 catfileheadercshex = format(0, 'x').lower();
3114 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
3115 fheadtell = len(fileheader);
3116 if(fprechecksum!=catfileheadercshex and not skipchecksum):
3117 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
3118 return False;
3119 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
3120 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
3121 if(seekto>=fnumfiles):
3122 seekto = fnumfiles - 1;
3123 if(seekto<0):
3124 seekto = 0;
3125 if(seekto>=0):
3126 il = -1;
3127 while(il < seekto):
3128 prefhstart = catfp.tell();
3129 if(formatspecs[7]):
3130 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs[5]);
3131 else:
3132 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs[5]);
3133 prefheadsize = int(preheaderdata[0], 16);
3134 prefnumfields = int(preheaderdata[1], 16);
3135 preftype = int(preheaderdata[2], 16);
3136 if(re.findall("^[.|/]", preheaderdata[3])):
3137 prefname = preheaderdata[3];
3138 else:
3139 prefname = "./"+preheaderdata[3];
3140 prefbasedir = os.path.dirname(prefname);
3141 preflinkname = preheaderdata[4];
3142 prefsize = int(preheaderdata[5], 16);
3143 prefatime = int(preheaderdata[6], 16);
3144 prefmtime = int(preheaderdata[7], 16);
3145 prefctime = int(preheaderdata[8], 16);
3146 prefbtime = int(preheaderdata[9], 16);
3147 prefmode = int(preheaderdata[10], 16);
3148 prefchmode = stat.S_IMODE(prefmode);
3149 preftypemod = stat.S_IFMT(prefmode);
3150 prefwinattributes = int(preheaderdata[11], 16);
3151 prefuid = int(preheaderdata[12], 16);
3152 prefuname = preheaderdata[13];
3153 prefgid = int(preheaderdata[14], 16);
3154 prefgname = preheaderdata[15];
3155 fid = int(preheaderdata[16], 16);
3156 finode = int(preheaderdata[17], 16);
3157 flinkcount = int(preheaderdata[18], 16);
3158 prefdev_minor = int(preheaderdata[19], 16);
3159 prefdev_major = int(preheaderdata[20], 16);
3160 prefrdev_minor = int(preheaderdata[22], 16);
3161 prefrdev_major = int(preheaderdata[23], 16);
3162 prefextrasize = int(preheaderdata[24], 16);
3163 prefextrafields = int(preheaderdata[25], 16);
3164 extrafieldslist = [];
3165 extrastart = 25;
3166 extraend = extrastart + prefextrafields;
3167 extrafieldslist = [];
3168 if(extrastart<extraend):
3169 extrafieldslist.append(preheaderdata[extrastart]);
3170 extrastart = extrastart + 1;
3171 prefchecksumtype = preheaderdata[extrastart].lower();
3172 prefcs = preheaderdata[extrastart + 1].lower();
3173 prefccs = preheaderdata[extrastart + 2].lower();
3174 hc = 0;
3175 hcmax = len(preheaderdata) - 2;
3176 hout = "";
3177 while(hc<hcmax):
3178 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs[5]);
3179 hc = hc + 1;
3180 if(prefchecksumtype=="none" or prefchecksumtype==""):
3181 prenewfcs = 0;
3182 elif(prefchecksumtype=="crc16" or prefchecksumtype=="crc16_ansi" or prefchecksumtype=="crc16_ibm"):
3183 prenewfcs = format(crc16(hout.encode('UTF-8')) & 0xffff, '04x').lower();
3184 elif(prefchecksumtype=="adler32"):
3185 prenewfcs = format(zlib.adler32(hout.encode('UTF-8')) & 0xffffffff, '08x').lower();
3186 elif(prefchecksumtype=="crc32"):
3187 prenewfcs = format(crc32(hout.encode('UTF-8')) & 0xffffffff, '08x').lower();
3188 elif(prefchecksumtype=="crc64_ecma"):
3189 prenewfcs = format(crc64_ecma(hout.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
3190 elif(prefchecksumtype=="crc64" or prefchecksumtype=="crc64_iso"):
3191 prenewfcs = format(crc64_iso(hout.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
3192 elif(CheckSumSupportAlt(prefchecksumtype, hashlib_guaranteed)):
3193 checksumoutstr = hashlib.new(prefchecksumtype);
3194 checksumoutstr.update(hout.encode('UTF-8'));
3195 prenewfcs = checksumoutstr.hexdigest().lower();
3196 if(prefcs!=prenewfcs and not skipchecksum):
3197 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
3198 return False;
3199 valid_archive = False;
3200 invalid_archive = True;
3201 prefhend = catfp.tell() - 1;
3202 prefcontentstart = catfp.tell();
3203 prefcontents = "";
3204 pyhascontents = False;
3205 if(prefsize>0):
3206 prefcontents = catfp.read(prefsize);
3207 if(prefchecksumtype=="none" or prefchecksumtype==""):
3208 prenewfccs = 0;
3209 elif(prefchecksumtype=="crc16" or prefchecksumtype=="crc16_ansi" or prefchecksumtype=="crc16_ibm"):
3210 prenewfccs = format(crc16(prefcontents) & 0xffff, '04x').lower();
3211 elif(prefchecksumtype=="crc16_ccitt"):
3212 prenewfcs = format(crc16_ccitt(prefcontents) & 0xffff, '04x').lower();
3213 elif(prefchecksumtype=="adler32"):
3214 prenewfccs = format(zlib.adler32(prefcontents) & 0xffffffff, '08x').lower();
3215 elif(prefchecksumtype=="crc32"):
3216 prenewfccs = format(crc32(prefcontents) & 0xffffffff, '08x').lower();
3217 elif(prefchecksumtype=="crc64_ecma"):
3218 prenewfcs = format(crc64_ecma(prefcontents) & 0xffffffffffffffff, '016x').lower();
3219 elif(prefchecksumtype=="crc64" or prefchecksumtype=="crc64_iso"):
3220 prenewfcs = format(crc64_iso(prefcontents) & 0xffffffffffffffff, '016x').lower();
3221 elif(CheckSumSupportAlt(prefchecksumtype, hashlib_guaranteed)):
3222 checksumoutstr = hashlib.new(prefchecksumtype);
3223 checksumoutstr.update(prefcontents);
3224 prenewfccs = checksumoutstr.hexdigest().lower();
3225 pyhascontents = True;
3226 if(prefccs!=prenewfccs and not skipchecksum):
3227 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
3228 return False;
3229 catfp.seek(1, 1);
3230 il = il + 1;
3231 catfp.seek(seekstart, 0);
3232 fileidnum = il;
3233 catfheadsize = int(preheaderdata[0], 16);
3234 catftype = int(preheaderdata[1], 16);
3235 if(re.findall("^[.|/]", preheaderdata[2])):
3236 catfname = preheaderdata[2];
3237 else:
3238 catfname = "./"+preheaderdata[2];
3239 catflinkname = preheaderdata[3];
3240 catfsize = int(preheaderdata[4], 16);
3241 catfbasedir = os.path.dirname(catfname);
3242 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
3243 if(returnfp):
3244 catlist.update({'catfp': catfp});
3245 else:
3246 catfp.close();
3247 return catlist;
3248
3249 create_alias_function("", __file_format_name__, "SeekToFileNum", ArchiveFileSeekToFileNum);
3250
3251 def ArchiveFileSeekToFileName(infile, seekfile=None, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
3252 if(hasattr(infile, "read") or hasattr(infile, "write")):
3253 catfp = infile;
3254 catfp.seek(0, 0);
3255 catfp = UncompressArchiveFile(catfp, formatspecs);
3256 checkcompressfile = CheckCompressionSubType(catfp, formatspecs);
3257 if(checkcompressfile=="tarfile"):
3258 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3259 if(checkcompressfile=="zipfile"):
3260 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3261 if(rarfile_support and checkcompressfile=="rarfile"):
3262 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3263 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
3264 return False;
3265 if(not catfp):
3266 return False;
3267 catfp.seek(0, 0);
3268 elif(infile=="-"):
3269 catfp = BytesIO();
3270 if(hasattr(sys.stdin, "buffer")):
3271 shutil.copyfileobj(sys.stdin.buffer, catfp);
3272 else:
3273 shutil.copyfileobj(sys.stdin, catfp);
3274 catfp.seek(0, 0);
3275 catfp = UncompressArchiveFile(catfp, formatspecs);
3276 if(not catfp):
3277 return False;
3278 catfp.seek(0, 0);
3279 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", infile)):
3280 catfp = download_file_from_internet_file(infile);
3281 catfp = UncompressArchiveFile(catfp, formatspecs);
3282 catfp.seek(0, 0);
3283 if(not catfp):
3284 return False;
3285 catfp.seek(0, 0);
3286 else:
3287 infile = RemoveWindowsPath(infile);
3288 checkcompressfile = CheckCompressionSubType(infile, formatspecs);
3289 if(checkcompressfile=="tarfile"):
3290 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3291 if(checkcompressfile=="zipfile"):
3292 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3293 if(rarfile_support and checkcompressfile=="rarfile"):
3294 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3295 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
3296 return False;
3297 compresscheck = CheckCompressionType(infile, formatspecs, True);
3298 if(not compresscheck):
3299 fextname = os.path.splitext(infile)[1];
3300 if(fextname==".gz"):
3301 compresscheck = "gzip";
3302 elif(fextname==".bz2"):
3303 compresscheck = "bzip2";
3304 elif(fextname==".zst"):
3305 compresscheck = "zstd";
3306 elif(fextname==".lz4" or fextname==".clz4"):
3307 compresscheck = "lz4";
3308 elif(fextname==".lzo" or fextname==".lzop"):
3309 compresscheck = "lzo";
3310 elif(fextname==".lzma" or fextname==".xz"):
3311 compresscheck = "lzma";
3312 else:
3313 return False;
3314 if(not compresscheck):
3315 return False;
3316 catfp = UncompressFile(infile, formatspecs, "rb");
3317 '''
3318 try:
3319 catfp.seek(0, 2);
3320 except OSError:
3321 SeekToEndOfFile(catfp);
3322 except ValueError:
3323 SeekToEndOfFile(catfp);
3324 CatSize = catfp.tell();
3325 CatSizeEnd = CatSize;
3326 '''
3327 try:
3328 catfp.seek(0, 0);
3329 except OSError:
3330 return False;
3331 except ValueError:
3332 return False;
3333 catheader = ReadFileHeaderData(catfp, 4, formatspecs[5]);
3334 catstring = catheader[0];
3335 catversion = re.findall(r"([\d]+)$", catstring);
3336 fprenumfiles = catheader[1];
3337 fnumfiles = int(fprenumfiles, 16);
3338 fprechecksumtype = catheader[2];
3339 fprechecksum = catheader[3];
3340 fileheader = AppendNullByte(catstring, formatspecs[5]);
3341 fnumfileshex = format(int(fnumfiles), 'x').lower();
3342 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs[5]);
3343 if(fprechecksumtype=="none" or fprechecksumtype==""):
3344 catfileheadercshex = format(0, 'x').lower();
3345 elif(fprechecksumtype=="crc16" or fprechecksumtype=="crc16_ansi" or fprechecksumtype=="crc16_ibm"):
3346 catfileheadercshex = format(crc16(fileheader.encode('UTF-8')) & 0xffff, '04x').lower();
3347 elif(fprechecksumtype=="crc16_ccitt"):
3348 catfileheadercshex = format(crc16_ccitt(fileheader.encode('UTF-8')) & 0xffff, '04x').lower();
3349 elif(fprechecksumtype=="adler32"):
3350 catfileheadercshex = format(zlib.adler32(fileheader.encode('UTF-8')) & 0xffffffff, '08x').lower();
3351 elif(fprechecksumtype=="crc32"):
3352 catfileheadercshex = format(crc32(fileheader.encode('UTF-8')) & 0xffffffff, '08x').lower();
3353 elif(fprechecksumtype=="crc64_ecma"):
3354 catfileheadercshex = format(crc64_ecma(fileheader.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
3355 elif(fprechecksumtype=="crc64" or fprechecksumtype=="crc64_iso"):
3356 catfileheadercshex = format(crc64_iso(fileheader.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
3357 elif(CheckSumSupportAlt(fprechecksumtype, hashlib_guaranteed)):
3358 checksumoutstr = hashlib.new(fprechecksumtype);
3359 checksumoutstr.update(fileheader.encode('UTF-8'));
3360 catfileheadercshex = checksumoutstr.hexdigest().lower();
3361 else:
3362 catfileheadercshex = format(0, 'x').lower();
3363 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
3364 fheadtell = len(fileheader);
3365 if(fprechecksum!=catfileheadercshex and not skipchecksum):
3366 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
3367 return False;
3368 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
3369 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
3370 seekto = fnumfiles - 1
3371 filefound = False;
3372 if(seekto>=0):
3373 il = -1;
3374 while(il < seekto):
3375 prefhstart = catfp.tell();
3376 if(formatspecs[7]):
3377 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs[5]);
3378 else:
3379 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs[5]);
3380 prefheadsize = int(preheaderdata[0], 16);
3381 prefnumfields = int(preheaderdata[1], 16);
3382 preftype = int(preheaderdata[2], 16);
3383 if(re.findall("^[.|/]", preheaderdata[3])):
3384 prefname = preheaderdata[3];
3385 else:
3386 prefname = "./"+preheaderdata[3];
3387 prefbasedir = os.path.dirname(prefname);
3388 preflinkname = preheaderdata[4];
3389 prefsize = int(preheaderdata[5], 16);
3390 prefatime = int(preheaderdata[6], 16);
3391 prefmtime = int(preheaderdata[7], 16);
3392 prefctime = int(preheaderdata[8], 16);
3393 prefbtime = int(preheaderdata[9], 16);
3394 prefmode = int(preheaderdata[10], 16);
3395 prefchmode = stat.S_IMODE(prefmode);
3396 preftypemod = stat.S_IFMT(prefmode);
3397 prefwinattributes = int(preheaderdata[11], 16);
3398 prefuid = int(preheaderdata[12], 16);
3399 prefuname = preheaderdata[13];
3400 prefgid = int(preheaderdata[14], 16);
3401 prefgname = preheaderdata[15];
3402 fid = int(preheaderdata[16], 16);
3403 finode = int(preheaderdata[17], 16);
3404 flinkcount = int(preheaderdata[18], 16);
3405 prefdev_minor = int(preheaderdata[19], 16);
3406 prefdev_major = int(preheaderdata[20], 16);
3407 prefrdev_minor = int(preheaderdata[22], 16);
3408 prefrdev_major = int(preheaderdata[23], 16);
3409 prefextrasize = int(preheaderdata[24], 16);
3410 prefextrafields = int(preheaderdata[25], 16);
3411 extrafieldslist = [];
3412 extrastart = 25;
3413 extraend = extrastart + prefextrafields;
3414 extrafieldslist = [];
3415 if(extrastart<extraend):
3416 extrafieldslist.append(preheaderdata[extrastart]);
3417 extrastart = extrastart + 1;
3418 prefchecksumtype = preheaderdata[extrastart].lower();
3419 prefcs = preheaderdata[extrastart + 1].lower();
3420 prefccs = preheaderdata[extrastart + 2].lower();
3421 hc = 0;
3422 hcmax = len(preheaderdata) - 2;
3423 hout = "";
3424 while(hc<hcmax):
3425 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs[5]);
3426 hc = hc + 1;
3427 if(prefchecksumtype=="none" or prefchecksumtype==""):
3428 prenewfcs = 0;
3429 elif(prefchecksumtype=="crc16" or prefchecksumtype=="crc16_ansi" or prefchecksumtype=="crc16_ibm"):
3430 prenewfcs = format(crc16(hout.encode('UTF-8')) & 0xffff, '04x').lower();
3431 elif(prefchecksumtype=="adler32"):
3432 prenewfcs = format(zlib.adler32(hout.encode('UTF-8')) & 0xffffffff, '08x').lower();
3433 elif(prefchecksumtype=="crc32"):
3434 prenewfcs = format(crc32(hout.encode('UTF-8')) & 0xffffffff, '08x').lower();
3435 elif(prefchecksumtype=="crc64_ecma"):
3436 prenewfcs = format(crc64_ecma(hout.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
3437 elif(prefchecksumtype=="crc64" or prefchecksumtype=="crc64_iso"):
3438 prenewfcs = format(crc64_iso(hout.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
3439 elif(CheckSumSupportAlt(prefchecksumtype, hashlib_guaranteed)):
3440 checksumoutstr = hashlib.new(prefchecksumtype);
3441 checksumoutstr.update(hout.encode('UTF-8'));
3442 prenewfcs = checksumoutstr.hexdigest().lower();
3443 if(prefcs!=prenewfcs and not skipchecksum):
3444 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
3445 return False;
3446 valid_archive = False;
3447 invalid_archive = True;
3448 prefhend = catfp.tell() - 1;
3449 prefcontentstart = catfp.tell();
3450 prefcontents = "";
3451 pyhascontents = False;
3452 if(prefsize>0):
3453 prefcontents = catfp.read(prefsize);
3454 if(prefchecksumtype=="none" or prefchecksumtype==""):
3455 prenewfccs = 0;
3456 elif(prefchecksumtype=="crc16" or prefchecksumtype=="crc16_ansi" or prefchecksumtype=="crc16_ibm"):
3457 prenewfccs = format(crc16(prefcontents) & 0xffff, '04x').lower();
3458 elif(prefchecksumtype=="crc16_ccitt"):
3459 prenewfcs = format(crc16_ccitt(prefcontents) & 0xffff, '04x').lower();
3460 elif(prefchecksumtype=="adler32"):
3461 prenewfccs = format(zlib.adler32(prefcontents) & 0xffffffff, '08x').lower();
3462 elif(prefchecksumtype=="crc32"):
3463 prenewfccs = format(crc32(prefcontents) & 0xffffffff, '08x').lower();
3464 elif(prefchecksumtype=="crc64_ecma"):
3465 prenewfcs = format(crc64_ecma(prefcontents) & 0xffffffffffffffff, '016x').lower();
3466 elif(prefchecksumtype=="crc64" or prefchecksumtype=="crc64_iso"):
3467 prenewfcs = format(crc64_iso(prefcontents) & 0xffffffffffffffff, '016x').lower();
3468 elif(CheckSumSupportAlt(prefchecksumtype, hashlib_guaranteed)):
3469 checksumoutstr = hashlib.new(prefchecksumtype);
3470 checksumoutstr.update(prefcontents);
3471 prenewfccs = checksumoutstr.hexdigest().lower();
3472 pyhascontents = True;
3473 if(prefccs!=prenewfccs and not skipchecksum):
3474 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
3475 return False;
3476 catfp.seek(1, 1);
3477 il = il + 1;
3478 filefound = False;
3479 prefname = preheaderdata[2];
3480 if(re.findall("^[.|/]", preheaderdata[2])):
3481 prefname = preheaderdata[2];
3482 else:
3483 prefname = "./"+preheaderdata[2];
3484 if(prefname==seekfile):
3485 filefound = True;
3486 break;
3487 catfp.seek(seekstart, 0);
3488 fileidnum = il;
3489 catfheadsize = int(preheaderdata[0], 16);
3490 catftype = int(preheaderdata[1], 16);
3491 if(re.findall("^[.|/]", preheaderdata[2])):
3492 catfname = preheaderdata[2];
3493 else:
3494 catfname = "./"+preheaderdata[2];
3495 catflinkname = preheaderdata[3];
3496 catfsize = int(preheaderdata[4], 16);
3497 catfbasedir = os.path.dirname(catfname);
3498 if(filefound):
3499 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
3500 else:
3501 return False;
3502 if(returnfp):
3503 catlist.update({'catfp': catfp});
3504 else:
3505 catfp.close();
3506 return catlist;
3507
3508 create_alias_function("", __file_format_name__, "SeekToFileName", ArchiveFileSeekToFileName);
3509
3510 def ArchiveFileValidate(infile, formatspecs=__file_format_list__, verbose=False, returnfp=False):
3511 if(verbose):
3512 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3513 if(hasattr(infile, "read") or hasattr(infile, "write")):
3514 catfp = infile;
3515 catfp.seek(0, 0);
3516 catfp = UncompressArchiveFile(catfp, formatspecs);
3517 checkcompressfile = CheckCompressionSubType(catfp, formatspecs);
3518 if(checkcompressfile=="tarfile"):
3519 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
3520 if(checkcompressfile=="zipfile"):
3521 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
3522 if(rarfile_support and checkcompressfile=="rarfile"):
3523 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
3524 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
3525 return False;
3526 if(not catfp):
3527 return False;
3528 catfp.seek(0, 0);
3529 elif(infile=="-"):
3530 catfp = BytesIO();
3531 if(hasattr(sys.stdin, "buffer")):
3532 shutil.copyfileobj(sys.stdin.buffer, catfp);
3533 else:
3534 shutil.copyfileobj(sys.stdin, catfp);
3535 catfp.seek(0, 0);
3536 catfp = UncompressArchiveFile(catfp, formatspecs);
3537 if(not catfp):
3538 return False;
3539 catfp.seek(0, 0);
3540 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", infile)):
3541 catfp = download_file_from_internet_file(infile);
3542 catfp = UncompressArchiveFile(catfp, formatspecs);
3543 catfp.seek(0, 0);
3544 if(not catfp):
3545 return False;
3546 catfp.seek(0, 0);
3547 else:
3548 infile = RemoveWindowsPath(infile);
3549 checkcompressfile = CheckCompressionSubType(infile, formatspecs);
3550 if(checkcompressfile=="tarfile"):
3551 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
3552 if(checkcompressfile=="zipfile"):
3553 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
3554 if(rarfile_support and checkcompressfile=="rarfile"):
3555 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
3556 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
3557 return False;
3558 compresscheck = CheckCompressionType(infile, formatspecs, True);
3559 if(not compresscheck):
3560 fextname = os.path.splitext(infile)[1];
3561 if(fextname==".gz"):
3562 compresscheck = "gzip";
3563 elif(fextname==".bz2"):
3564 compresscheck = "bzip2";
3565 elif(fextname==".zst"):
3566 compresscheck = "zstd";
3567 elif(fextname==".lz4" or fextname==".clz4"):
3568 compresscheck = "lz4";
3569 elif(fextname==".lzo" or fextname==".lzop"):
3570 compresscheck = "lzo";
3571 elif(fextname==".lzma" or fextname==".xz"):
3572 compresscheck = "lzma";
3573 else:
3574 return False;
3575 if(not compresscheck):
3576 return False;
3577 catfp = UncompressFile(infile, formatspecs, "rb");
3578 '''
3579 try:
3580 catfp.seek(0, 2);
3581 except OSError:
3582 SeekToEndOfFile(catfp);
3583 except ValueError:
3584 SeekToEndOfFile(catfp);
3585 CatSize = catfp.tell();
3586 CatSizeEnd = CatSize;
3587 '''
3588 try:
3589 catfp.seek(0, 0);
3590 except OSError:
3591 return False;
3592 except ValueError:
3593 return False;
3594 catheader = ReadFileHeaderData(catfp, 4, formatspecs[5]);
3595 catstring = catheader[0];
3596 catversion = re.findall(r"([\d]+)$", catstring);
3597 fprenumfiles = catheader[1];
3598 fnumfiles = int(fprenumfiles, 16);
3599 fprechecksumtype = catheader[2];
3600 fprechecksum = catheader[3];
3601 il = 0;
3602 fileheader = AppendNullByte(catstring, formatspecs[5]);
3603 fnumfileshex = format(int(fnumfiles), 'x').lower();
3604 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs[5]);
3605 if(fprechecksumtype=="none" or fprechecksumtype==""):
3606 catfileheadercshex = format(0, 'x').lower();
3607 elif(fprechecksumtype=="crc16" or fprechecksumtype=="crc16_ansi" or fprechecksumtype=="crc16_ibm"):
3608 catfileheadercshex = format(crc16(fileheader.encode('UTF-8')) & 0xffff, '04x').lower();
3609 elif(fprechecksumtype=="crc16_ccitt"):
3610 catfileheadercshex = format(crc16_ccitt(fileheader.encode('UTF-8')) & 0xffff, '04x').lower();
3611 elif(fprechecksumtype=="adler32"):
3612 catfileheadercshex = format(zlib.adler32(fileheader.encode('UTF-8')) & 0xffffffff, '08x').lower();
3613 elif(fprechecksumtype=="crc32"):
3614 catfileheadercshex = format(crc32(fileheader.encode('UTF-8')) & 0xffffffff, '08x').lower();
3615 elif(fprechecksumtype=="crc64_ecma"):
3616 catfileheadercshex = format(crc64_ecma(fileheader.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
3617 elif(fprechecksumtype=="crc64" or fprechecksumtype=="crc64_iso"):
3618 catfileheadercshex = format(crc64_iso(fileheader.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
3619 elif(CheckSumSupportAlt(fprechecksumtype, hashlib_guaranteed)):
3620 checksumoutstr = hashlib.new(fprechecksumtype);
3621 checksumoutstr.update(fileheader.encode('UTF-8'));
3622 catfileheadercshex = checksumoutstr.hexdigest().lower();
3623 else:
3624 catfileheadercshex = format(0, 'x').lower();
3625 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
3626 valid_archive = True;
3627 invalid_archive = False;
3628 if(verbose):
3629 VerbosePrintOut("Checking File Header Checksum of file at offset " + str(0));
3630 VerbosePrintOut(infile);
3631 if(fprechecksum==catfileheadercshex):
3632 if(verbose):
3633 VerbosePrintOut("File Header Checksum Passed");
3634 else:
3635 if(verbose):
3636 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
3637 valid_archive = False;
3638 invalid_archive = True;
3639 while(il<fnumfiles):
3640 catfhstart = catfp.tell();
3641 if(formatspecs[7]):
3642 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs[5]);
3643 else:
3644 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs[5]);
3645 catfheadsize = int(catheaderdata[0], 16);
3646 catfnumfields = int(catheaderdata[1], 16);
3647 catftype = int(catheaderdata[2], 16);
3648 if(re.findall("^[.|/]", catheaderdata[3])):
3649 catfname = catheaderdata[3];
3650 else:
3651 catfname = "./"+catheaderdata[3];
3652 catfbasedir = os.path.dirname(catfname);
3653 catflinkname = catheaderdata[4];
3654 catfsize = int(catheaderdata[5], 16);
3655 catfatime = int(catheaderdata[6], 16);
3656 catfmtime = int(catheaderdata[7], 16);
3657 catfctime = int(catheaderdata[8], 16);
3658 catfbtime = int(catheaderdata[9], 16);
3659 catfmode = int(catheaderdata[10], 16);
3660 catfchmode = stat.S_IMODE(catfmode);
3661 catftypemod = stat.S_IFMT(catfmode);
3662 prefwinattributes = int(catheaderdata[11], 16);
3663 catfuid = int(catheaderdata[12], 16);
3664 catfuname = catheaderdata[13];
3665 catfgid = int(catheaderdata[14], 16);
3666 catfgname = catheaderdata[15];
3667 fid = int(catheaderdata[16], 16);
3668 finode = int(catheaderdata[17], 16);
3669 flinkcount = int(catheaderdata[18], 16);
3670 catfdev_minor = int(catheaderdata[19], 16);
3671 catfdev_major = int(catheaderdata[20], 16);
3672 catfrdev_minor = int(catheaderdata[21], 16);
3673 catfrdev_major = int(catheaderdata[22], 16);
3674 catfextrasize = int(catheaderdata[23], 16);
3675 catfextrafields = int(catheaderdata[24], 16);
3676 extrafieldslist = [];
3677 extrastart = 25;
3678 extraend = extrastart + catfextrafields;
3679 extrafieldslist = [];
3680 if(extrastart<extraend):
3681 extrafieldslist.append(catheaderdata[extrastart]);
3682 extrastart = extrastart + 1;
3683 catfchecksumtype = catheaderdata[extrastart].lower();
3684 catfcs = catheaderdata[extrastart + 1].lower();
3685 catfccs = catheaderdata[extrastart + 2].lower();
3686 hc = 0;
3687 hcmax = len(catheaderdata) - 2;
3688 hout = "";
3689 while(hc<hcmax):
3690 hout = hout + AppendNullByte(catheaderdata[hc], formatspecs[5]);
3691 hc = hc + 1;
3692 if(catfchecksumtype=="none" or catfchecksumtype==""):
3693 catnewfcs = 0;
3694 elif(catfchecksumtype=="crc16" or catfchecksumtype=="crc16_ansi" or catfchecksumtype=="crc16_ibm"):
3695 catnewfcs = format(crc16(hout.encode('UTF-8')) & 0xffff, '04x').lower();
3696 elif(catfchecksumtype=="adler32"):
3697 catnewfcs = format(zlib.adler32(hout.encode('UTF-8')) & 0xffffffff, '08x').lower();
3698 elif(catfchecksumtype=="crc32"):
3699 catnewfcs = format(crc32(hout.encode('UTF-8')) & 0xffffffff, '08x').lower();
3700 elif(catfchecksumtype=="crc64_ecma"):
3701 catnewfcs = format(crc64_ecma(hout.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
3702 elif(catfchecksumtype=="crc64" or catfchecksumtype=="crc64_iso"):
3703 catnewfcs = format(crc64_iso(hout.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
3704 elif(CheckSumSupportAlt(catfchecksumtype, hashlib_guaranteed)):
3705 checksumoutstr = hashlib.new(catfchecksumtype);
3706 checksumoutstr.update(hout.encode('UTF-8'));
3707 catnewfcs = checksumoutstr.hexdigest().lower();
3708 if(verbose):
3709 VerbosePrintOut("Checking File Header Checksum of file at offset " + str(catfhstart));
3710 VerbosePrintOut(catfname);
3711 if(catfcs==catnewfcs):
3712 if(verbose):
3713 VerbosePrintOut("File Header Checksum Passed");
3714 else:
3715 if(verbose):
3716 VerbosePrintOut("File Header Checksum Error with file " + catfname + " at offset " + str(catfhstart));
3717 valid_archive = False;
3718 invalid_archive = True;
3719 catfhend = catfp.tell() - 1;
3720 catfcontentstart = catfp.tell();
3721 catfcontents = "";
3722 pyhascontents = False;
3723 if(catfsize>0):
3724 catfcontents = catfp.read(catfsize);
3725 if(catfchecksumtype=="none" or catfchecksumtype==""):
3726 catnewfccs = 0;
3727 elif(catfchecksumtype=="crc16" or catfchecksumtype=="crc16_ansi" or catfchecksumtype=="crc16_ibm"):
3728 catnewfccs = format(crc16(catfcontents) & 0xffff, '04x').lower();
3729 elif(catfchecksumtype=="crc16_ccitt"):
3730 catnewfcs = format(crc16_ccitt(catfcontents) & 0xffff, '04x').lower();
3731 elif(catfchecksumtype=="adler32"):
3732 catnewfccs = format(zlib.adler32(catfcontents) & 0xffffffff, '08x').lower();
3733 elif(catfchecksumtype=="crc32"):
3734 catnewfccs = format(crc32(catfcontents) & 0xffffffff, '08x').lower();
3735 elif(catfchecksumtype=="crc64_ecma"):
3736 catnewfcs = format(crc64_ecma(catfcontents) & 0xffffffffffffffff, '016x').lower();
3737 elif(catfchecksumtype=="crc64" or catfchecksumtype=="crc64_iso"):
3738 catnewfcs = format(crc64_iso(catfcontents) & 0xffffffffffffffff, '016x').lower();
3739 elif(CheckSumSupportAlt(catfchecksumtype, hashlib_guaranteed)):
3740 checksumoutstr = hashlib.new(catfchecksumtype);
3741 checksumoutstr.update(catfcontents);
3742 catnewfccs = checksumoutstr.hexdigest().lower();
3743 pyhascontents = True;
3744 if(verbose):
3745 VerbosePrintOut("Checking File Content Checksum of file at offset " + str(catfcontentstart));
3746 VerbosePrintOut(catfname);
3747 if(catfccs==catnewfccs):
3748 if(verbose):
3749 VerbosePrintOut("File Content Checksum Passed");
3750 else:
3751 if(verbose):
3752 VerbosePrintOut("File Header Checksum Error with file " + catfname + " at offset " + str(catfcontentstart));
3753 valid_archive = False;
3754 invalid_archive = True;
3755 if(verbose):
3756 VerbosePrintOut("");
3757 catfp.seek(1, 1);
3758 il = il + 1;
3759 if(valid_archive):
3760 if(returnfp):
3761 return catfp;
3762 else:
3763 catfp.close();
3764 return True;
3765 else:
3766 catfp.close();
3767 return False;
3768
3769 create_alias_function("", __file_format_name__, "Validate", ArchiveFileValidate);
3770
3771 def ArchiveFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
3772 if(hasattr(infile, "read") or hasattr(infile, "write")):
3773 catfp = infile;
3774 catfp.seek(0, 0);
3775 catfp = UncompressArchiveFile(catfp, formatspecs);
3776 checkcompressfile = CheckCompressionSubType(catfp, formatspecs);
3777 if(checkcompressfile=="tarfile"):
3778 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
3779 if(checkcompressfile=="zipfile"):
3780 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
3781 if(rarfile_support and checkcompressfile=="rarfile"):
3782 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
3783 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
3784 return False;
3785 if(not catfp):
3786 return False;
3787 catfp.seek(0, 0);
3788 elif(infile=="-"):
3789 catfp = BytesIO();
3790 if(hasattr(sys.stdin, "buffer")):
3791 shutil.copyfileobj(sys.stdin.buffer, catfp);
3792 else:
3793 shutil.copyfileobj(sys.stdin, catfp);
3794 catfp.seek(0, 0);
3795 catfp = UncompressArchiveFile(catfp, formatspecs);
3796 if(not catfp):
3797 return False;
3798 catfp.seek(0, 0);
3799 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", infile)):
3800 catfp = download_file_from_internet_file(infile);
3801 catfp = UncompressArchiveFile(catfp, formatspecs);
3802 catfp.seek(0, 0);
3803 if(not catfp):
3804 return False;
3805 catfp.seek(0, 0);
3806 else:
3807 infile = RemoveWindowsPath(infile);
3808 checkcompressfile = CheckCompressionSubType(infile, formatspecs);
3809 if(checkcompressfile=="tarfile"):
3810 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
3811 if(checkcompressfile=="zipfile"):
3812 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
3813 if(rarfile_support and checkcompressfile=="rarfile"):
3814 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
3815 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
3816 return False;
3817 compresscheck = CheckCompressionType(infile, formatspecs, True);
3818 if(not compresscheck):
3819 fextname = os.path.splitext(infile)[1];
3820 if(fextname==".gz"):
3821 compresscheck = "gzip";
3822 elif(fextname==".bz2"):
3823 compresscheck = "bzip2";
3824 elif(fextname==".zst"):
3825 compresscheck = "zstd";
3826 elif(fextname==".lz4" or fextname==".clz4"):
3827 compresscheck = "lz4";
3828 elif(fextname==".lzo" or fextname==".lzop"):
3829 compresscheck = "lzo";
3830 elif(fextname==".lzma" or fextname==".xz"):
3831 compresscheck = "lzma";
3832 else:
3833 return False;
3834 if(not compresscheck):
3835 return False;
3836 catfp = UncompressFile(infile, formatspecs, "rb");
3837 '''
3838 try:
3839 catfp.seek(0, 2);
3840 except OSError:
3841 SeekToEndOfFile(catfp);
3842 except ValueError:
3843 SeekToEndOfFile(catfp);
3844 CatSize = catfp.tell();
3845 CatSizeEnd = CatSize;
3846 '''
3847 try:
3848 catfp.seek(0, 0);
3849 except OSError:
3850 return False;
3851 except ValueError:
3852 return False;
3853 catheader = ReadFileHeaderData(catfp, 4, formatspecs[5]);
3854 catstring = catheader[0];
3855 catversion = re.findall(r"([\d]+)$", catstring);
3856 fprenumfiles = catheader[1];
3857 fnumfiles = int(fprenumfiles, 16);
3858 fprechecksumtype = catheader[2];
3859 fprechecksum = catheader[3];
3860 fileheader = AppendNullByte(catstring, formatspecs[5]);
3861 fnumfileshex = format(int(fnumfiles), 'x').lower();
3862 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs[5]);
3863 if(fprechecksumtype=="none" or fprechecksumtype==""):
3864 catfileheadercshex = format(0, 'x').lower();
3865 elif(fprechecksumtype=="crc16" or fprechecksumtype=="crc16_ansi" or fprechecksumtype=="crc16_ibm"):
3866 catfileheadercshex = format(crc16(fileheader.encode('UTF-8')) & 0xffff, '04x').lower();
3867 elif(fprechecksumtype=="crc16_ccitt"):
3868 catfileheadercshex = format(crc16_ccitt(fileheader.encode('UTF-8')) & 0xffff, '04x').lower();
3869 elif(fprechecksumtype=="adler32"):
3870 catfileheadercshex = format(zlib.adler32(fileheader.encode('UTF-8')) & 0xffffffff, '08x').lower();
3871 elif(fprechecksumtype=="crc32"):
3872 catfileheadercshex = format(crc32(fileheader.encode('UTF-8')) & 0xffffffff, '08x').lower();
3873 elif(fprechecksumtype=="crc64_ecma"):
3874 catfileheadercshex = format(crc64_ecma(fileheader.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
3875 elif(fprechecksumtype=="crc64" or fprechecksumtype=="crc64_iso"):
3876 catfileheadercshex = format(crc64_iso(fileheader.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
3877 elif(CheckSumSupportAlt(fprechecksumtype, hashlib_guaranteed)):
3878 checksumoutstr = hashlib.new(fprechecksumtype);
3879 checksumoutstr.update(fileheader.encode('UTF-8'));
3880 catfileheadercshex = checksumoutstr.hexdigest().lower();
3881 else:
3882 catfileheadercshex = format(0, 'x').lower();
3883 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
3884 fheadtell = len(fileheader);
3885 if(fprechecksum!=catfileheadercshex and not skipchecksum):
3886 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
3887 return False;
3888 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
3889 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
3890 if(seekstart<0 and seekstart>fnumfiles):
3891 seekstart = 0;
3892 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
3893 seekend = fnumfiles;
3894 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
3895 seekend = fnumfiles - abs(seekend);
3896 if(seekstart>0):
3897 il = 0;
3898 while(il < seekstart):
3899 prefhstart = catfp.tell();
3900 if(formatspecs[7]):
3901 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs[5]);
3902 else:
3903 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs[5]);
3904 prefheadsize = int(preheaderdata[0], 16);
3905 prefnumfields = int(preheaderdata[1], 16);
3906 if(re.findall("^[.|/]", preheaderdata[3])):
3907 prefname = preheaderdata[3];
3908 else:
3909 prefname = "./"+preheaderdata[4];
3910 prefsize = int(preheaderdata[5], 16);
3911 prefextrasize = int(preheaderdata[23], 16);
3912 prefextrafields = int(preheaderdata[24], 16);
3913 extrafieldslist = [];
3914 extrastart = 25;
3915 extraend = extrastart + prefextrafields;
3916 extrafieldslist = [];
3917 if(extrastart<extraend):
3918 extrafieldslist.append(preheaderdata[extrastart]);
3919 extrastart = extrastart + 1;
3920 prefchecksumtype = preheaderdata[extrastart].lower();
3921 prefcs = preheaderdata[extrastart + 1].lower();
3922 prefccs = preheaderdata[extrastart + 2].lower();
3923 hc = 0;
3924 hcmax = len(preheaderdata) - 2;
3925 hout = "";
3926 while(hc<hcmax):
3927 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs[5]);
3928 hc = hc + 1;
3929 if(prefchecksumtype=="none" or prefchecksumtype==""):
3930 prenewfcs = 0;
3931 elif(prefchecksumtype=="crc16" or prefchecksumtype=="crc16_ansi" or prefchecksumtype=="crc16_ibm"):
3932 prenewfcs = format(crc16(hout.encode('UTF-8')) & 0xffff, '04x').lower();
3933 elif(prefchecksumtype=="adler32"):
3934 prenewfcs = format(zlib.adler32(hout.encode('UTF-8')) & 0xffffffff, '08x').lower();
3935 elif(prefchecksumtype=="crc32"):
3936 prenewfcs = format(crc32(hout.encode('UTF-8')) & 0xffffffff, '08x').lower();
3937 elif(prefchecksumtype=="crc64_ecma"):
3938 prenewfcs = format(crc64_ecma(hout.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
3939 elif(prefchecksumtype=="crc64" or prefchecksumtype=="crc64_iso"):
3940 prenewfcs = format(crc64_iso(hout.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
3941 elif(CheckSumSupportAlt(prefchecksumtype, hashlib_guaranteed)):
3942 checksumoutstr = hashlib.new(prefchecksumtype);
3943 checksumoutstr.update(hout.encode('UTF-8'));
3944 prenewfcs = checksumoutstr.hexdigest().lower();
3945 if(prefcs!=prenewfcs and not skipchecksum):
3946 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
3947 return False;
3948 valid_archive = False;
3949 invalid_archive = True;
3950 prefhend = catfp.tell() - 1;
3951 prefcontentstart = catfp.tell();
3952 prefcontents = "";
3953 pyhascontents = False;
3954 if(prefsize>0):
3955 prefcontents = catfp.read(prefsize);
3956 if(prefchecksumtype=="none" or prefchecksumtype==""):
3957 prenewfccs = 0;
3958 elif(prefchecksumtype=="crc16" or prefchecksumtype=="crc16_ansi" or prefchecksumtype=="crc16_ibm"):
3959 prenewfccs = format(crc16(prefcontents) & 0xffff, '04x').lower();
3960 elif(prefchecksumtype=="crc16_ccitt"):
3961 prenewfcs = format(crc16_ccitt(prefcontents) & 0xffff, '04x').lower();
3962 elif(prefchecksumtype=="adler32"):
3963 prenewfccs = format(zlib.adler32(prefcontents) & 0xffffffff, '08x').lower();
3964 elif(prefchecksumtype=="crc32"):
3965 prenewfccs = format(crc32(prefcontents) & 0xffffffff, '08x').lower();
3966 elif(prefchecksumtype=="crc64_ecma"):
3967 prenewfcs = format(crc64_ecma(prefcontents) & 0xffffffffffffffff, '016x').lower();
3968 elif(prefchecksumtype=="crc64" or prefchecksumtype=="crc64_iso"):
3969 prenewfcs = format(crc64_iso(prefcontents) & 0xffffffffffffffff, '016x').lower();
3970 elif(CheckSumSupportAlt(prefchecksumtype, hashlib_guaranteed)):
3971 checksumoutstr = hashlib.new(prefchecksumtype);
3972 checksumoutstr.update(prefcontents);
3973 prenewfccs = checksumoutstr.hexdigest().lower();
3974 pyhascontents = True;
3975 if(prefccs!=prenewfccs and not skipchecksum):
3976 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
3977 return False;
3978 catfp.seek(1, 1);
3979 il = il + 1;
3980 fileidnum = seekstart;
3981 realidnum = 0;
3982 while(fileidnum<seekend):
3983 catfhstart = catfp.tell();
3984 if(formatspecs[7]):
3985 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs[5]);
3986 else:
3987 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs[5]);
3988 catfheadsize = int(catheaderdata[0], 16);
3989 catfnumfields = int(catheaderdata[1], 16);
3990 catftype = int(catheaderdata[2], 16);
3991 if(re.findall("^[.|/]", catheaderdata[3])):
3992 catfname = catheaderdata[3];
3993 else:
3994 catfname = "./"+catheaderdata[3];
3995 catfbasedir = os.path.dirname(catfname);
3996 catflinkname = catheaderdata[4];
3997 catfsize = int(catheaderdata[5], 16);
3998 catfatime = int(catheaderdata[6], 16);
3999 catfmtime = int(catheaderdata[7], 16);
4000 catfctime = int(catheaderdata[8], 16);
4001 catfbtime = int(catheaderdata[9], 16);
4002 catfmode = int(catheaderdata[10], 16);
4003 catfchmode = stat.S_IMODE(catfmode);
4004 catftypemod = stat.S_IFMT(catfmode);
4005 catfwinattributes = int(catheaderdata[11], 16);
4006 catfuid = int(catheaderdata[12], 16);
4007 catfuname = catheaderdata[13];
4008 catfgid = int(catheaderdata[14], 16);
4009 catfgname = catheaderdata[15];
4010 fid = int(catheaderdata[16], 16);
4011 finode = int(catheaderdata[17], 16);
4012 flinkcount = int(catheaderdata[18], 16);
4013 catfdev_minor = int(catheaderdata[19], 16);
4014 catfdev_major = int(catheaderdata[20], 16);
4015 catfrdev_minor = int(catheaderdata[21], 16);
4016 catfrdev_major = int(catheaderdata[22], 16);
4017 catfextrasize = int(catheaderdata[23], 16);
4018 catfextrafields = int(catheaderdata[24], 16);
4019 extrafieldslist = [];
4020 extrastart = 25;
4021 extraend = extrastart + catfextrafields;
4022 extrafieldslist = [];
4023 if(extrastart<extraend):
4024 extrafieldslist.append(catheaderdata[extrastart]);
4025 extrastart = extrastart + 1;
4026 catfchecksumtype = catheaderdata[extrastart].lower();
4027 catfcs = catheaderdata[extrastart + 1].lower();
4028 catfccs = catheaderdata[extrastart + 2].lower();
4029 hc = 0;
4030 hcmax = len(catheaderdata) - 2;
4031 hout = "";
4032 while(hc<hcmax):
4033 hout = hout + AppendNullByte(catheaderdata[hc], formatspecs[5]);
4034 hc = hc + 1;
4035 if(catfchecksumtype=="none" or catfchecksumtype==""):
4036 catnewfcs = 0;
4037 elif(catfchecksumtype=="crc16" or catfchecksumtype=="crc16_ansi" or catfchecksumtype=="crc16_ibm"):
4038 catnewfcs = format(crc16(hout.encode('UTF-8')) & 0xffff, '04x').lower();
4039 elif(catfchecksumtype=="adler32"):
4040 catnewfcs = format(zlib.adler32(hout.encode('UTF-8')) & 0xffffffff, '08x').lower();
4041 elif(catfchecksumtype=="crc32"):
4042 catnewfcs = format(crc32(hout.encode('UTF-8')) & 0xffffffff, '08x').lower();
4043 elif(catfchecksumtype=="crc64_ecma"):
4044 catnewfcs = format(crc64_ecma(hout.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4045 elif(catfchecksumtype=="crc64" or catfchecksumtype=="crc64_iso"):
4046 catnewfcs = format(crc64_iso(hout.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4047 elif(CheckSumSupportAlt(catfchecksumtype, hashlib_guaranteed)):
4048 checksumoutstr = hashlib.new(catfchecksumtype);
4049 checksumoutstr.update(hout.encode('UTF-8'));
4050 catnewfcs = checksumoutstr.hexdigest().lower();
4051 if(catfcs!=catnewfcs and not skipchecksum):
4052 VerbosePrintOut("File Header Checksum Error with file " + catfname + " at offset " + str(catfhstart));
4053 return False;
4054 catfhend = catfp.tell() - 1;
4055 catfcontentstart = catfp.tell();
4056 catfcontents = "";
4057 pyhascontents = False;
4058 if(catfsize>0 and not listonly):
4059 catfcontents = catfp.read(catfsize);
4060 if(catfchecksumtype=="none" or catfchecksumtype==""):
4061 catnewfccs = 0;
4062 elif(catfchecksumtype=="crc16" or catfchecksumtype=="crc16_ansi" or catfchecksumtype=="crc16_ibm"):
4063 catnewfccs = format(crc16(catfcontents) & 0xffff, '04x').lower();
4064 elif(catfchecksumtype=="crc16_ccitt"):
4065 catnewfcs = format(crc16_ccitt(catfcontents) & 0xffff, '04x').lower();
4066 elif(catfchecksumtype=="adler32"):
4067 catnewfccs = format(zlib.adler32(catfcontents) & 0xffffffff, '08x').lower();
4068 elif(catfchecksumtype=="crc32"):
4069 catnewfccs = format(crc32(catfcontents) & 0xffffffff, '08x').lower();
4070 elif(catfchecksumtype=="crc64_ecma"):
4071 catnewfcs = format(crc64_ecma(catfcontents) & 0xffffffffffffffff, '016x').lower();
4072 elif(catfchecksumtype=="crc64" or catfchecksumtype=="crc64_iso"):
4073 catnewfcs = format(crc64_iso(catfcontents) & 0xffffffffffffffff, '016x').lower();
4074 elif(CheckSumSupportAlt(catfchecksumtype, hashlib_guaranteed)):
4075 checksumoutstr = hashlib.new(catfchecksumtype);
4076 checksumoutstr.update(catfcontents);
4077 catnewfccs = checksumoutstr.hexdigest().lower();
4078 pyhascontents = True;
4079 if(catfccs!=catnewfccs and skipchecksum):
4080 VerbosePrintOut("File Content Checksum Error with file " + catfname + " at offset " + str(catfcontentstart));
4081 return False;
4082 if(catfsize>0 and listonly):
4083 catfp.seek(catfsize, 1);
4084 pyhascontents = False;
4085 catfp.seek(1, 1);
4086 catfcontentend = catfp.tell() - 1;
4087 catlist['ffilelist'].update({realidnum: {'fid': realidnum, 'fidalt': fileidnum, 'fheadersize': catfheadsize, 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize, 'fatime': catfatime, 'fmtime': catfmtime, 'fctime': catfctime, 'fbtime': catfbtime, 'fmode': catfmode, 'fchmode': catfchmode, 'ftypemod': catftypemod, 'fwinattributes': catfwinattributes, 'fuid': catfuid, 'funame': catfuname, 'fgid': catfgid, 'fgname': catfgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': catfdev_minor, 'fmajor': catfdev_major, 'frminor': catfrdev_minor, 'frmajor': catfrdev_major, 'fchecksumtype': catfchecksumtype, 'fnumfields': catfnumfields + 2, 'fextrafields': catfextrafields, 'fextrafieldsize': catfextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': catfcs, 'fcontentchecksum': catfccs, 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': catfcontents} });
4088 fileidnum = fileidnum + 1;
4089 realidnum = realidnum + 1;
4090 if(returnfp):
4091 catlist.update({'catfp': catfp});
4092 else:
4093 catfp.close();
4094 return catlist;
4095
4096 create_alias_function("", __file_format_name__, "ToArray", ArchiveFileToArray);
4097
4098 def ArchiveFileStringToArray(catstr, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4099 catfp = BytesIO(catstr);
4100 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4101 return listcatfiles;
4102
4103 create_alias_function("", __file_format_name__, "StringToArray", ArchiveFileStringToArray);
4104
4105 def TarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4106 catfp = BytesIO();
4107 catfp = PackArchiveFileFromTarFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
4108 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4109 return listcatfiles;
4110
4111 def ZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4112 catfp = BytesIO();
4113 catfp = PackArchiveFileFromZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
4114 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4115 return listcatfiles;
4116
4117 if(not rarfile_support):
4118 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4119 return False;
4120
4121 if(rarfile_support):
4122 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4123 catfp = BytesIO();
4124 catfp = PackArchiveFileFromRarFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
4125 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4126 return listcatfiles;
4127
4128 def ListDirToArrayAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
4129 catver = formatspecs[6];
4130 fileheaderver = str(int(catver.replace(".", "")));
4131 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
4132 advancedlist = True;
4133 infilelist = [];
4134 if(infiles=="-"):
4135 for line in sys.stdin:
4136 infilelist.append(line.strip());
4137 infilelist = list(filter(None, infilelist));
4138 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
4139 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
4140 return False;
4141 with open(infiles, "r") as finfile:
4142 for line in finfile:
4143 infilelist.append(line.strip());
4144 infilelist = list(filter(None, infilelist));
4145 else:
4146 if(isinstance(infiles, (list, tuple, ))):
4147 infilelist = list(filter(None, infiles));
4148 elif(isinstance(infiles, (str, ))):
4149 infilelist = list(filter(None, [infiles]));
4150 if(advancedlist):
4151 GetDirList = ListDirAdvanced(infilelist, followlink, False);
4152 else:
4153 GetDirList = ListDir(infilelist, followlink, False);
4154 if(not GetDirList):
4155 return False;
4156 curinode = 0;
4157 curfid = 0;
4158 inodelist = [];
4159 inodetofile = {};
4160 filetoinode = {};
4161 inodetocatinode = {};
4162 fileidnum = 0;
4163 fnumfiles = int(len(GetDirList));
4164 catver = formatspecs[6];
4165 fileheaderver = str(int(catver.replace(".", "")));
4166 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
4167 fnumfileshex = format(int(fnumfiles), 'x').lower();
4168 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs[5]);
4169 catversion = re.findall(r"([\d]+)$", fileheader);
4170 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
4171 if(checksumtype=="none" or checksumtype==""):
4172 catfileheadercshex = format(0, 'x').lower();
4173 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
4174 catfileheadercshex = format(crc16(fileheader.encode('UTF-8')) & 0xffff, '04x').lower();
4175 elif(checksumtype=="crc16_ccitt"):
4176 catfileheadercshex = format(crc16_ccitt(fileheader.encode('UTF-8')) & 0xffff, '04x').lower();
4177 elif(checksumtype=="adler32"):
4178 catfileheadercshex = format(zlib.adler32(fileheader.encode('UTF-8')) & 0xffffffff, '08x').lower();
4179 elif(checksumtype=="crc32"):
4180 catfileheadercshex = format(crc32(fileheader.encode('UTF-8')) & 0xffffffff, '08x').lower();
4181 elif(checksumtype=="crc64_ecma"):
4182 catfileheadercshex = format(crc64_ecma(fileheader.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4183 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
4184 catfileheadercshex = format(crc64_iso(fileheader.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4185 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
4186 checksumoutstr = hashlib.new(checksumtype);
4187 checksumoutstr.update(fileheader.encode('UTF-8'));
4188 catfileheadercshex = checksumoutstr.hexdigest().lower();
4189 else:
4190 catfileheadercshex = format(0, 'x').lower();
4191 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
4192 fheadtell = len(fileheader);
4193 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
4194 for curfname in GetDirList:
4195 if(re.findall("^[.|/]", curfname)):
4196 fname = curfname;
4197 else:
4198 fname = "./"+curfname;
4199 if(verbose):
4200 VerbosePrintOut(fname);
4201 if(not followlink or followlink is None):
4202 fstatinfo = os.lstat(fname);
4203 else:
4204 fstatinfo = os.stat(fname);
4205 fpremode = fstatinfo.st_mode;
4206 finode = fstatinfo.st_ino;
4207 flinkcount = fstatinfo.st_nlink;
4208 ftype = 0;
4209 if(stat.S_ISREG(fpremode)):
4210 ftype = 0;
4211 elif(stat.S_ISLNK(fpremode)):
4212 ftype = 2;
4213 elif(stat.S_ISCHR(fpremode)):
4214 ftype = 3;
4215 elif(stat.S_ISBLK(fpremode)):
4216 ftype = 4;
4217 elif(stat.S_ISDIR(fpremode)):
4218 ftype = 5;
4219 elif(stat.S_ISFIFO(fpremode)):
4220 ftype = 6;
4221 elif(stat.S_ISSOCK(fpremode)):
4222 ftype = 8;
4223 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
4224 ftype = 9;
4225 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
4226 ftype = 10;
4227 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
4228 ftype = 11;
4229 else:
4230 ftype = 0;
4231 flinkname = "";
4232 fbasedir = os.path.dirname(fname);
4233 fcurfid = curfid;
4234 if(not followlink and finode!=0):
4235 if(ftype!=1):
4236 if(finode in inodelist):
4237 ftype = 1;
4238 flinkname = inodetofile[finode];
4239 fcurinode = inodetocatinode[finode];
4240 if(finode not in inodelist):
4241 inodelist.append(finode);
4242 inodetofile.update({finode: fname});
4243 inodetocatinode.update({finode: curinode});
4244 fcurinode = curinode;
4245 curinode = curinode + 1;
4246 else:
4247 fcurinode = curinode;
4248 curinode = curinode + 1;
4249 curfid = curfid + 1;
4250 if(ftype==2):
4251 flinkname = os.readlink(fname);
4252 fdev = fstatinfo.st_dev;
4253 getfdev = GetDevMajorMinor(fdev);
4254 fdev_minor = getfdev[0];
4255 fdev_major = getfdev[1];
4256 frdev = fstatinfo.st_dev;
4257 if(hasattr(fstatinfo, "st_rdev")):
4258 frdev = fstatinfo.st_rdev;
4259 else:
4260 frdev = fstatinfo.st_dev;
4261 getfrdev = GetDevMajorMinor(frdev);
4262 frdev_minor = getfrdev[0];
4263 frdev_major = getfrdev[1];
4264 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
4265 fsize = "0";
4266 if(ftype==0 or ftype==7):
4267 fsize = fstatinfo.st_size;
4268 fatime = fstatinfo.st_atime;
4269 fmtime = fstatinfo.st_mtime;
4270 fctime = fstatinfo.st_ctime;
4271 if(hasattr(fstatinfo, "st_birthtime")):
4272 fbtime = fstatinfo.st_birthtime;
4273 else:
4274 fbtime = fstatinfo.st_ctime;
4275 fmode = fstatinfo.st_mode;
4276 fchmode = stat.S_IMODE(fstatinfo.st_mode);
4277 ftypemod = stat.S_IFMT(fstatinfo.st_mode);
4278 fuid = fstatinfo.st_uid;
4279 fgid = fstatinfo.st_gid;
4280 funame = "";
4281 try:
4282 import pwd;
4283 try:
4284 userinfo = pwd.getpwuid(fstatinfo.st_uid);
4285 funame = userinfo.pw_name;
4286 except KeyError:
4287 funame = "";
4288 except ImportError:
4289 funame = "";
4290 fgname = "";
4291 try:
4292 import grp;
4293 try:
4294 groupinfo = grp.getgrgid(fstatinfo.st_gid);
4295 fgname = groupinfo.gr_name;
4296 except KeyError:
4297 fgname = "";
4298 except ImportError:
4299 fgname = "";
4300 fdev_minor = fdev_minor;
4301 fdev_major = fdev_major;
4302 frdev_minor = frdev_minor;
4303 frdev_major = frdev_major;
4304 finode = finode;
4305 flinkcount = flinkcount;
4306 if(hasattr(fstatinfo, "st_file_attributes")):
4307 fwinattributes = fstatinfo.st_file_attributes;
4308 else:
4309 fwinattributes = 0;
4310 fcontents = "".encode('UTF-8');
4311 chunk_size = 1024;
4312 if(ftype == 0 or ftype == 7):
4313 with open(fname, "rb") as fpc:
4314 while(True):
4315 chunk = fpc.read(chunk_size);
4316 if(not chunk):
4317 break
4318 fcontents += chunk;
4319 if(followlink and (ftype == 1 or ftype == 2)):
4320 flstatinfo = os.stat(flinkname);
4321 with open(flinkname, "rb") as fpc:
4322 while(True):
4323 chunk = fpc.read(chunk_size);
4324 if(not chunk):
4325 break;
4326 fcontents += chunk;
4327 ftypehex = format(ftype, 'x').lower();
4328 extrafields = len(extradata);
4329 extrafieldslist = extradata;
4330 catfextrafields = extrafields;
4331 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
4332 if(len(extradata)>0):
4333 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
4334 extrasizelen = len(extrasizestr);
4335 extrasizelenhex = format(extrasizelen, 'x').lower();
4336 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
4337 catoutlen = len(catoutlist) + len(extradata) + 3;
4338 catoutlenhex = format(catoutlen, 'x').lower();
4339 catoutlist.insert(0, catoutlenhex);
4340 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
4341 if(len(extradata)>0):
4342 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
4343 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
4344 catfnumfields = catoutlen;
4345 if(checksumtype=="none" or checksumtype==""):
4346 catfileheadercshex = format(0, 'x').lower();
4347 catfilecontentcshex = format(0, 'x').lower();
4348 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
4349 catfileheadercshex = format(crc16("".encode('UTF-8')) & 0xffff, '04x').lower();
4350 catfilecontentcshex = format(crc16(fcontents) & 0xffff, '04x').lower();
4351 elif(checksumtype=="crc16_ccitt"):
4352 catfileheadercshex = format(crc16_ccitt("".encode('UTF-8')) & 0xffff, '04x').lower();
4353 catfilecontentcshex = format(crc16_ccitt(fcontents) & 0xffff, '04x').lower();
4354 elif(checksumtype=="adler32"):
4355 catfileheadercshex = format(zlib.adler32("".encode('UTF-8')) & 0xffffffff, '08x').lower();
4356 catfilecontentcshex = format(zlib.adler32(fcontents) & 0xffffffff, '08x').lower();
4357 elif(checksumtype=="crc32"):
4358 catfileheadercshex = format(crc32("".encode('UTF-8')) & 0xffffffff, '08x').lower();
4359 catfilecontentcshex = format(crc32(fcontents) & 0xffffffff, '08x').lower();
4360 elif(checksumtype=="crc64_ecma"):
4361 catfileheadercshex = format(crc64_ecma("".encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4362 catfilecontentcshex = format(crc64_ecma(fcontents) & 0xffffffffffffffff, '016x').lower();
4363 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
4364 catfileheadercshex = format(crc64_iso("".encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4365 catfilecontentcshex = format(crc64_iso(fcontents) & 0xffffffffffffffff, '016x').lower();
4366 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
4367 checksumoutstr = hashlib.new(checksumtype);
4368 checksumoutstr.update("".encode('UTF-8'));
4369 catfileheadercshex = checksumoutstr.hexdigest().lower();
4370 checksumoutstr = hashlib.new(checksumtype);
4371 checksumoutstr.update(fcontents);
4372 catfilecontentcshex = checksumoutstr.hexdigest().lower();
4373 else:
4374 catfileheadercshex = format(0, 'x').lower();
4375 catfilecontentcshex = format(0, 'x').lower();
4376 catfhstart = fheadtell;
4377 fheadtell += len(catfileoutstr);
4378 catfhend = fheadtell - 1;
4379 catfcontentstart = fheadtell;
4380 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
4381 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower()
4382 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
4383 if(checksumtype=="none" or checksumtype==""):
4384 catfileheadercshex = format(0, 'x').lower();
4385 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
4386 catfileheadercshex = format(crc16(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
4387 elif(checksumtype=="crc16_ccitt"):
4388 catfileheadercshex = format(crc16_ccitt(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
4389 elif(checksumtype=="adler32"):
4390 catfileheadercshex = format(zlib.adler32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
4391 elif(checksumtype=="crc32"):
4392 catfileheadercshex = format(crc32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
4393 elif(checksumtype=="crc64_ecma"):
4394 catfileheadercshex = format(crc64_ecma(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4395 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
4396 catfileheadercshex = format(crc64_iso(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4397 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
4398 checksumoutstr = hashlib.new(checksumtype);
4399 checksumoutstr.update(catfileoutstr.encode('UTF-8'));
4400 catfileheadercshex = checksumoutstr.hexdigest().lower();
4401 else:
4402 catfileheadercshex = format(0, 'x').lower();
4403 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
4404 catfileoutstrecd = catfileoutstr.encode('UTF-8');
4405 nullstrecd = formatspecs[5].encode('UTF-8');
4406 fheadtell += len(catfileoutstr) + 1;
4407 catfcontentend = fheadtell - 1;
4408 catfileout = catfileoutstrecd + fcontents + nullstrecd;
4409 pyhascontents = False;
4410 if(int(fsize)>0 and not listonly):
4411 pyhascontents = True;
4412 if(int(fsize)>0 and listonly):
4413 fcontents = "";
4414 pyhascontents = False;
4415 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
4416 fileidnum = fileidnum + 1;
4417 return catlist;
4418
4419 def TarFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
4420 curinode = 0;
4421 curfid = 0;
4422 inodelist = [];
4423 inodetofile = {};
4424 filetoinode = {};
4425 inodetocatinode = {};
4426 fileidnum = 0;
4427 if(infile=="-"):
4428 infile = BytesIO();
4429 if(hasattr(sys.stdin, "buffer")):
4430 shutil.copyfileobj(sys.stdin.buffer, infile);
4431 else:
4432 shutil.copyfileobj(sys.stdin, infile);
4433 infile.seek(0, 0);
4434 if(not infile):
4435 return False;
4436 infile.seek(0, 0);
4437 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", infile)):
4438 infile = download_file_from_internet_file(infile);
4439 infile.seek(0, 0);
4440 if(not infile):
4441 return False;
4442 infile.seek(0, 0);
4443 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
4444 return False;
4445 elif(os.path.exists(infile) and os.path.isfile(infile)):
4446 try:
4447 if(not tarfile.is_tarfile(infile)):
4448 return False;
4449 except AttributeError:
4450 if(not is_tarfile(infile)):
4451 return False;
4452 else:
4453 return False;
4454 try:
4455 if(hasattr(infile, "read") or hasattr(infile, "write")):
4456 tarfp = tarfile.open(fileobj=infile, mode="r");
4457 else:
4458 tarfp = tarfile.open(infile, "r");
4459 except FileNotFoundError:
4460 return False;
4461 fnumfiles = int(len(tarfp.getmembers()));
4462 catver = formatspecs[6];
4463 fileheaderver = str(int(catver.replace(".", "")));
4464 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
4465 fnumfileshex = format(int(fnumfiles), 'x').lower();
4466 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs[5]);
4467 catversion = re.findall(r"([\d]+)$", fileheader);
4468 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
4469 if(checksumtype=="none" or checksumtype==""):
4470 catfileheadercshex = format(0, 'x').lower();
4471 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
4472 catfileheadercshex = format(crc16(fileheader.encode('UTF-8')) & 0xffff, '04x').lower();
4473 elif(checksumtype=="crc16_ccitt"):
4474 catfileheadercshex = format(crc16_ccitt(fileheader.encode('UTF-8')) & 0xffff, '04x').lower();
4475 elif(checksumtype=="adler32"):
4476 catfileheadercshex = format(zlib.adler32(fileheader.encode('UTF-8')) & 0xffffffff, '08x').lower();
4477 elif(checksumtype=="crc32"):
4478 catfileheadercshex = format(crc32(fileheader.encode('UTF-8')) & 0xffffffff, '08x').lower();
4479 elif(checksumtype=="crc64_ecma"):
4480 catfileheadercshex = format(crc64_ecma(fileheader.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4481 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
4482 catfileheadercshex = format(crc64_iso(fileheader.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4483 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
4484 checksumoutstr = hashlib.new(checksumtype);
4485 checksumoutstr.update(fileheader.encode('UTF-8'));
4486 catfileheadercshex = checksumoutstr.hexdigest().lower();
4487 else:
4488 catfileheadercshex = format(0, 'x').lower();
4489 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
4490 fheadtell = len(fileheader);
4491 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
4492 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
4493 if(re.findall("^[.|/]", member.name)):
4494 fname = member.name;
4495 else:
4496 fname = "./"+member.name;
4497 if(verbose):
4498 VerbosePrintOut(fname);
4499 fpremode = member.mode;
4500 ffullmode = member.mode;
4501 flinkcount = 0;
4502 ftype = 0;
4503 if(member.isreg()):
4504 ffullmode = member.mode + stat.S_IFREG;
4505 ftype = 0;
4506 elif(member.isdev()):
4507 ffullmode = member.mode;
4508 ftype = 7;
4509 elif(member.islnk()):
4510 ffullmode = member.mode + stat.S_IFREG;
4511 ftype = 1;
4512 elif(member.issym()):
4513 ffullmode = member.mode + stat.S_IFLNK;
4514 ftype = 2;
4515 elif(member.ischr()):
4516 ffullmode = member.mode + stat.S_IFCHR;
4517 ftype = 3;
4518 elif(member.isblk()):
4519 ffullmode = member.mode + stat.S_IFBLK;
4520 ftype = 4;
4521 elif(member.isdir()):
4522 ffullmode = member.mode + stat.S_IFDIR;
4523 ftype = 5;
4524 elif(member.isfifo()):
4525 ffullmode = member.mode + stat.S_IFIFO;
4526 ftype = 6;
4527 elif(member.issparse()):
4528 ffullmode = member.mode;
4529 ftype = 12;
4530 else:
4531 ffullmode = member.mode;
4532 ftype = 0;
4533 flinkname = "";
4534 fbasedir = os.path.dirname(fname);
4535 fcurfid = curfid;
4536 fcurinode = 0;
4537 finode = fcurinode;
4538 curfid = curfid + 1;
4539 if(ftype==2):
4540 flinkname = member.linkname;
4541 fdev_minor = member.devminor;
4542 fdev_major = member.devmajor;
4543 frdev_minor = member.devminor;
4544 frdev_major = member.devmajor;
4545 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
4546 fsize = "0";
4547 elif(ftype==0 or ftype==7):
4548 fsize = member.size;
4549 else:
4550 fsize = member.size;
4551 fatime = member.mtime;
4552 fmtime = member.mtime;
4553 fctime = member.mtime;
4554 fbtime = member.mtime;
4555 fmode = ffullmode;
4556 fchmode = stat.S_IMODE(ffullmode);
4557 ftypemod = stat.S_IFMT(ffullmode);
4558 fuid = member.uid;
4559 fgid = member.gid;
4560 funame = member.uname;
4561 fgname = member.gname;
4562 flinkcount = flinkcount;
4563 fwinattributes = int(0);
4564 fcontents = "".encode('UTF-8');
4565 chunk_size = 1024;
4566 if(ftype == 0 or ftype == 7):
4567 with tarfp.extractfile(member) as fpc:
4568 while(True):
4569 chunk = fpc.read(chunk_size);
4570 if(not chunk):
4571 break
4572 fcontents += chunk;
4573 ftypehex = format(ftype, 'x').lower();
4574 extrafields = len(extradata);
4575 extrafieldslist = extradata;
4576 catfextrafields = extrafields;
4577 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
4578 if(len(extradata)>0):
4579 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
4580 extrasizelen = len(extrasizestr);
4581 extrasizelenhex = format(extrasizelen, 'x').lower();
4582 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
4583 catoutlen = len(catoutlist) + len(extradata) + 3;
4584 catoutlenhex = format(catoutlen, 'x').lower();
4585 catoutlist.insert(0, catoutlenhex);
4586 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
4587 if(len(extradata)>0):
4588 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
4589 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
4590 catfnumfields = catoutlen;
4591 if(checksumtype=="none" or checksumtype==""):
4592 catfileheadercshex = format(0, 'x').lower();
4593 catfilecontentcshex = format(0, 'x').lower();
4594 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
4595 catfileheadercshex = format(crc16("".encode('UTF-8')) & 0xffff, '04x').lower();
4596 catfilecontentcshex = format(crc16(fcontents) & 0xffff, '04x').lower();
4597 elif(checksumtype=="crc16_ccitt"):
4598 catfileheadercshex = format(crc16_ccitt("".encode('UTF-8')) & 0xffff, '04x').lower();
4599 catfilecontentcshex = format(crc16_ccitt(fcontents) & 0xffff, '04x').lower();
4600 elif(checksumtype=="adler32"):
4601 catfileheadercshex = format(zlib.adler32("".encode('UTF-8')) & 0xffffffff, '08x').lower();
4602 catfilecontentcshex = format(zlib.adler32(fcontents) & 0xffffffff, '08x').lower();
4603 elif(checksumtype=="crc32"):
4604 catfileheadercshex = format(crc32("".encode('UTF-8')) & 0xffffffff, '08x').lower();
4605 catfilecontentcshex = format(crc32(fcontents) & 0xffffffff, '08x').lower();
4606 elif(checksumtype=="crc64_ecma"):
4607 catfileheadercshex = format(crc64_ecma("".encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4608 catfilecontentcshex = format(crc64_ecma(fcontents) & 0xffffffffffffffff, '016x').lower();
4609 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
4610 catfileheadercshex = format(crc64_iso("".encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4611 catfilecontentcshex = format(crc64_iso(fcontents) & 0xffffffffffffffff, '016x').lower();
4612 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
4613 checksumoutstr = hashlib.new(checksumtype);
4614 checksumoutstr.update("".encode('UTF-8'));
4615 catfileheadercshex = checksumoutstr.hexdigest().lower();
4616 checksumoutstr = hashlib.new(checksumtype);
4617 checksumoutstr.update(fcontents);
4618 catfilecontentcshex = checksumoutstr.hexdigest().lower();
4619 else:
4620 catfileheadercshex = format(0, 'x').lower();
4621 catfilecontentcshex = format(0, 'x').lower();
4622 catfhstart = fheadtell;
4623 fheadtell += len(catfileoutstr);
4624 catfhend = fheadtell - 1;
4625 catfcontentstart = fheadtell;
4626 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
4627 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower()
4628 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
4629 if(checksumtype=="none" or checksumtype==""):
4630 catfileheadercshex = format(0, 'x').lower();
4631 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
4632 catfileheadercshex = format(crc16(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
4633 elif(checksumtype=="crc16_ccitt"):
4634 catfileheadercshex = format(crc16_ccitt(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
4635 elif(checksumtype=="adler32"):
4636 catfileheadercshex = format(zlib.adler32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
4637 elif(checksumtype=="crc32"):
4638 catfileheadercshex = format(crc32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
4639 elif(checksumtype=="crc64_ecma"):
4640 catfileheadercshex = format(crc64_ecma(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4641 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
4642 catfileheadercshex = format(crc64_iso(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4643 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
4644 checksumoutstr = hashlib.new(checksumtype);
4645 checksumoutstr.update(catfileoutstr.encode('UTF-8'));
4646 catfileheadercshex = checksumoutstr.hexdigest().lower();
4647 else:
4648 catfileheadercshex = format(0, 'x').lower();
4649 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
4650 catfileoutstrecd = catfileoutstr.encode('UTF-8');
4651 nullstrecd = formatspecs[5].encode('UTF-8');
4652 fheadtell += len(catfileoutstr) + 1;
4653 catfcontentend = fheadtell - 1;
4654 catfileout = catfileoutstrecd + fcontents + nullstrecd;
4655 pyhascontents = False;
4656 if(int(fsize)>0 and not listonly):
4657 pyhascontents = True;
4658 if(int(fsize)>0 and listonly):
4659 fcontents = "";
4660 pyhascontents = False;
4661 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
4662 fileidnum = fileidnum + 1;
4663 return catlist;
4664
4665 def ZipFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
4666 advancedlist = True;
4667 curinode = 0;
4668 curfid = 0;
4669 inodelist = [];
4670 inodetofile = {};
4671 filetoinode = {};
4672 inodetocatinode = {};
4673 fileidnum = 0;
4674 if(infile=="-"):
4675 infile = BytesIO();
4676 if(hasattr(sys.stdin, "buffer")):
4677 shutil.copyfileobj(sys.stdin.buffer, infile);
4678 else:
4679 shutil.copyfileobj(sys.stdin, infile);
4680 infile.seek(0, 0);
4681 if(not infile):
4682 return False;
4683 infile.seek(0, 0);
4684 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", infile)):
4685 infile = download_file_from_internet_file(infile);
4686 infile.seek(0, 0);
4687 if(not infile):
4688 return False;
4689 infile.seek(0, 0);
4690 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
4691 return False;
4692 else:
4693 return False;
4694 if(not zipfile.is_zipfile(infile)):
4695 return False;
4696 try:
4697 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
4698 except FileNotFoundError:
4699 return False;
4700 ziptest = zipfp.testzip();
4701 if(ziptest):
4702 VerbosePrintOut("Bad file found!");
4703 fnumfiles = int(len(zipfp.infolist()));
4704 catver = formatspecs[6];
4705 fileheaderver = str(int(catver.replace(".", "")));
4706 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
4707 catversion = re.findall(r"([\d]+)$", fileheader);
4708 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
4709 fnumfileshex = format(int(fnumfiles), 'x').lower();
4710 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs[5]);
4711 if(checksumtype=="none" or checksumtype==""):
4712 catfileheadercshex = format(0, 'x').lower();
4713 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
4714 catfileheadercshex = format(crc16(fileheader.encode('UTF-8')) & 0xffff, '04x').lower();
4715 elif(checksumtype=="crc16_ccitt"):
4716 catfileheadercshex = format(crc16_ccitt(fileheader.encode('UTF-8')) & 0xffff, '04x').lower();
4717 elif(checksumtype=="adler32"):
4718 catfileheadercshex = format(zlib.adler32(fileheader.encode('UTF-8')) & 0xffffffff, '08x').lower();
4719 elif(checksumtype=="crc32"):
4720 catfileheadercshex = format(crc32(fileheader.encode('UTF-8')) & 0xffffffff, '08x').lower();
4721 elif(checksumtype=="crc64_ecma"):
4722 catfileheadercshex = format(crc64_ecma(fileheader.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4723 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
4724 catfileheadercshex = format(crc64_iso(fileheader.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4725 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
4726 checksumoutstr = hashlib.new(checksumtype);
4727 checksumoutstr.update(fileheader.encode('UTF-8'));
4728 catfileheadercshex = checksumoutstr.hexdigest().lower();
4729 else:
4730 catfileheadercshex = format(0, 'x').lower();
4731 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
4732 fheadtell = len(fileheader);
4733 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
4734 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
4735 if(re.findall("^[.|/]", member.filename)):
4736 fname = member.filename;
4737 else:
4738 fname = "./"+member.filename;
4739 zipinfo = zipfp.getinfo(member.filename);
4740 if(verbose):
4741 VerbosePrintOut(fname);
4742 if(not member.is_dir()):
4743 fpremode = stat.S_IFREG + 438;
4744 elif(member.is_dir()):
4745 fpremode = stat.S_IFDIR + 511;
4746 flinkcount = 0;
4747 ftype = 0;
4748 if(not member.is_dir()):
4749 ftype = 0;
4750 elif(member.is_dir()):
4751 ftype = 5;
4752 flinkname = "";
4753 fbasedir = os.path.dirname(fname);
4754 fcurfid = curfid;
4755 fcurinode = 0;
4756 finode = fcurinode;
4757 curfid = curfid + 1;
4758 fdev_minor = 0;
4759 fdev_major = 0;
4760 frdev_minor = 0;
4761 frdev_major = 0;
4762 if(ftype==5):
4763 fsize = "0";
4764 elif(ftype==0):
4765 fsize = member.file_size;
4766 else:
4767 fsize = member.file_size;
4768 fatime = time.mktime(member.date_time + (0, 0, -1));
4769 fmtime = time.mktime(member.date_time + (0, 0, -1));
4770 fctime = time.mktime(member.date_time + (0, 0, -1));
4771 fbtime = time.mktime(member.date_time + (0, 0, -1));
4772 if(zipinfo.create_system==0 or zipinfo.create_system==10):
4773 fwinattributes = format(int(zipinfo.external_attr), 'x').lower();
4774 if(not member.is_dir()):
4775 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
4776 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
4777 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
4778 elif(member.is_dir()):
4779 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
4780 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
4781 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
4782 elif(zipinfo.create_system==3):
4783 fwinattributes = format(int(0), 'x').lower();
4784 fmode = format(int(zipinfo.external_attr), 'x').lower();
4785 else:
4786 fwinattributes = format(int(0), 'x').lower();
4787 if(not member.is_dir()):
4788 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
4789 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
4790 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
4791 elif(member.is_dir()):
4792 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
4793 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
4794 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
4795 try:
4796 fuid = os.getuid();
4797 except AttributeError:
4798 fuid = 0;
4799 except KeyError:
4800 fuid = 0;
4801 try:
4802 fgid = os.getgid();
4803 except AttributeError:
4804 fgid = 0;
4805 except KeyError:
4806 fgid = 0;
4807 try:
4808 import pwd;
4809 try:
4810 userinfo = pwd.getpwuid(os.getuid());
4811 funame = userinfo.pw_name;
4812 except KeyError:
4813 funame = "";
4814 except AttributeError:
4815 funame = "";
4816 except ImportError:
4817 funame = "";
4818 fgname = "";
4819 try:
4820 import grp;
4821 try:
4822 groupinfo = grp.getgrgid(os.getgid());
4823 fgname = groupinfo.gr_name;
4824 except KeyError:
4825 fgname = "";
4826 except AttributeError:
4827 fgname = "";
4828 except ImportError:
4829 fgname = "";
4830 fcontents = "".encode('UTF-8');
4831 if(ftype==0):
4832 fcontents = zipfp.read(member.filename);
4833 ftypehex = format(ftype, 'x').lower();
4834 extrafields = len(extradata);
4835 extrafieldslist = extradata;
4836 catfextrafields = extrafields;
4837 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
4838 if(len(extradata)>0):
4839 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
4840 extrasizelen = len(extrasizestr);
4841 extrasizelenhex = format(extrasizelen, 'x').lower();
4842 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
4843 catoutlen = len(catoutlist) + len(extradata) + 3;
4844 catoutlenhex = format(catoutlen, 'x').lower();
4845 catoutlist.insert(0, catoutlenhex);
4846 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
4847 if(len(extradata)>0):
4848 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
4849 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
4850 catfnumfields = catoutlen;
4851 if(checksumtype=="none" or checksumtype==""):
4852 catfileheadercshex = format(0, 'x').lower();
4853 catfilecontentcshex = format(0, 'x').lower();
4854 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
4855 catfileheadercshex = format(crc16("".encode('UTF-8')) & 0xffff, '04x').lower();
4856 catfilecontentcshex = format(crc16(fcontents) & 0xffff, '04x').lower();
4857 elif(checksumtype=="crc16_ccitt"):
4858 catfileheadercshex = format(crc16_ccitt("".encode('UTF-8')) & 0xffff, '04x').lower();
4859 catfilecontentcshex = format(crc16_ccitt(fcontents) & 0xffff, '04x').lower();
4860 elif(checksumtype=="adler32"):
4861 catfileheadercshex = format(zlib.adler32("".encode('UTF-8')) & 0xffffffff, '08x').lower();
4862 catfilecontentcshex = format(zlib.adler32(fcontents) & 0xffffffff, '08x').lower();
4863 elif(checksumtype=="crc32"):
4864 catfileheadercshex = format(crc32("".encode('UTF-8')) & 0xffffffff, '08x').lower();
4865 catfilecontentcshex = format(crc32(fcontents) & 0xffffffff, '08x').lower();
4866 elif(checksumtype=="crc64_ecma"):
4867 catfileheadercshex = format(crc64_ecma("".encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4868 catfilecontentcshex = format(crc64_ecma(fcontents) & 0xffffffffffffffff, '016x').lower();
4869 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
4870 catfileheadercshex = format(crc64_iso("".encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4871 catfilecontentcshex = format(crc64_iso(fcontents) & 0xffffffffffffffff, '016x').lower();
4872 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
4873 checksumoutstr = hashlib.new(checksumtype);
4874 checksumoutstr.update("".encode('UTF-8'));
4875 catfileheadercshex = checksumoutstr.hexdigest().lower();
4876 checksumoutstr = hashlib.new(checksumtype);
4877 checksumoutstr.update(fcontents);
4878 catfilecontentcshex = checksumoutstr.hexdigest().lower();
4879 else:
4880 catfileheadercshex = format(0, 'x').lower();
4881 catfilecontentcshex = format(0, 'x').lower();
4882 catfhstart = fheadtell;
4883 fheadtell += len(catfileoutstr);
4884 catfhend = fheadtell - 1;
4885 catfcontentstart = fheadtell;
4886 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
4887 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower()
4888 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
4889 if(checksumtype=="none" or checksumtype==""):
4890 catfileheadercshex = format(0, 'x').lower();
4891 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
4892 catfileheadercshex = format(crc16(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
4893 elif(checksumtype=="crc16_ccitt"):
4894 catfileheadercshex = format(crc16_ccitt(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
4895 elif(checksumtype=="adler32"):
4896 catfileheadercshex = format(zlib.adler32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
4897 elif(checksumtype=="crc32"):
4898 catfileheadercshex = format(crc32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
4899 elif(checksumtype=="crc64_ecma"):
4900 catfileheadercshex = format(crc64_ecma(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4901 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
4902 catfileheadercshex = format(crc64_iso(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4903 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
4904 checksumoutstr = hashlib.new(checksumtype);
4905 checksumoutstr.update(catfileoutstr.encode('UTF-8'));
4906 catfileheadercshex = checksumoutstr.hexdigest().lower();
4907 else:
4908 catfileheadercshex = format(0, 'x').lower();
4909 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
4910 catfileoutstrecd = catfileoutstr.encode('UTF-8');
4911 nullstrecd = formatspecs[5].encode('UTF-8');
4912 fheadtell += len(catfileoutstr) + 1;
4913 catfcontentend = fheadtell - 1;
4914 catfileout = catfileoutstrecd + fcontents + nullstrecd;
4915 pyhascontents = False;
4916 if(int(fsize)>0 and not listonly):
4917 pyhascontents = True;
4918 if(int(fsize)>0 and listonly):
4919 fcontents = "";
4920 pyhascontents = False;
4921 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
4922 fileidnum = fileidnum + 1;
4923 return catlist;
4924
4925 if(not rarfile_support):
4926 def RarFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
4927 return False;
4928
4929 if(rarfile_support):
4930 def RarFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
4931 advancedlist = True;
4932 curinode = 0;
4933 curfid = 0;
4934 inodelist = [];
4935 inodetofile = {};
4936 filetoinode = {};
4937 inodetocatinode = {};
4938 fileidnum = 0;
4939 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
4940 return False;
4941 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
4942 return False;
4943 rarfp = rarfile.RarFile(infile, "r");
4944 rartest = rarfp.testrar();
4945 if(rartest):
4946 VerbosePrintOut("Bad file found!");
4947 fnumfiles = int(len(rarfp.infolist()));
4948 catver = formatspecs[6];
4949 fileheaderver = str(int(catver.replace(".", "")));
4950 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
4951 catversion = re.findall(r"([\d]+)$", fileheader);
4952 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
4953 fnumfileshex = format(int(fnumfiles), 'x').lower();
4954 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs[5]);
4955 if(checksumtype=="none" or checksumtype==""):
4956 catfileheadercshex = format(0, 'x').lower();
4957 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
4958 catfileheadercshex = format(crc16(fileheader.encode('UTF-8')) & 0xffff, '04x').lower();
4959 elif(checksumtype=="crc16_ccitt"):
4960 catfileheadercshex = format(crc16_ccitt(fileheader.encode('UTF-8')) & 0xffff, '04x').lower();
4961 elif(checksumtype=="adler32"):
4962 catfileheadercshex = format(zlib.adler32(fileheader.encode('UTF-8')) & 0xffffffff, '08x').lower();
4963 elif(checksumtype=="crc32"):
4964 catfileheadercshex = format(crc32(fileheader.encode('UTF-8')) & 0xffffffff, '08x').lower();
4965 elif(checksumtype=="crc64_ecma"):
4966 catfileheadercshex = format(crc64_ecma(fileheader.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4967 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
4968 catfileheadercshex = format(crc64_iso(fileheader.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
4969 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
4970 checksumoutstr = hashlib.new(checksumtype);
4971 checksumoutstr.update(fileheader.encode('UTF-8'));
4972 catfileheadercshex = checksumoutstr.hexdigest().lower();
4973 else:
4974 catfileheadercshex = format(0, 'x').lower();
4975 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
4976 fheadtell = len(fileheader);
4977 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
4978 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
4979 is_unix = False;
4980 is_windows = False;
4981 if(member.host_os == rarfile.RAR_OS_UNIX):
4982 is_windows = False;
4983 try:
4984 member.external_attr
4985 is_unix = True;
4986 except AttributeError:
4987 is_unix = False;
4988 elif(member.host_os == rarfile.RAR_OS_WIN32):
4989 is_unix = False;
4990 try:
4991 member.external_attr
4992 is_windows = True;
4993 except AttributeError:
4994 is_windows = False;
4995 else:
4996 is_unix = False;
4997 is_windows = False;
4998 if(re.findall("^[.|/]", member.filename)):
4999 fname = member.filename;
5000 else:
5001 fname = "./"+member.filename;
5002 rarinfo = rarfp.getinfo(member.filename);
5003 if(verbose):
5004 VerbosePrintOut(fname);
5005 if(is_unix and member.external_attr !=0):
5006 fpremode = int(member.external_attr);
5007 elif(member.is_file()):
5008 fpremode = stat.S_IFREG + 438;
5009 elif(member.is_symlink()):
5010 fpremode = stat.S_IFLNK + 438;
5011 elif(member.is_dir()):
5012 fpremode = stat.S_IFDIR + 511;
5013 if(is_windows and member.external_attr !=0):
5014 fwinattributes = int(member.external_attr);
5015 else:
5016 fwinattributes = int(0);
5017 flinkcount = 0;
5018 ftype = 0;
5019 if(member.is_file()):
5020 ftype = 0;
5021 elif(member.is_symlink()):
5022 ftype = 2;
5023 elif(member.is_dir()):
5024 ftype = 5;
5025 flinkname = "";
5026 if(ftype==2):
5027 flinkname = rarfp.read(member.filename).decode("UTF-8");
5028 fbasedir = os.path.dirname(fname);
5029 fcurfid = curfid;
5030 fcurinode = 0;
5031 finode = fcurinode;
5032 curfid = curfid + 1;
5033 fdev_minor = 0;
5034 fdev_major = 0;
5035 frdev_minor = 0;
5036 frdev_major = 0;
5037 if(ftype==5):
5038 fsize = "0";
5039 if(ftype==0):
5040 fsize = member.file_size;
5041 try:
5042 if(member.atime):
5043 fatime = int(member.atime.timestamp());
5044 else:
5045 fatime = int(member.mtime.timestamp());
5046 except AttributeError:
5047 fatime = int(member.mtime.timestamp());
5048 fmtime = int(member.mtime.timestamp());
5049 try:
5050 if(member.ctime):
5051 fctime = int(member.ctime.timestamp());
5052 else:
5053 fctime = int(member.mtime.timestamp());
5054 except AttributeError:
5055 fctime = int(member.mtime.timestamp());
5056 fbtime = int(member.mtime.timestamp());
5057 if(is_unix and member.external_attr !=0):
5058 fmode = format(int(member.external_attr), 'x').lower();
5059 fchmode = format(int(stat.S_IMODE(member.external_attr)), 'x').lower();
5060 ftypemod = format(int(stat.S_IFMT(member.external_attr)), 'x').lower();
5061 elif(member.is_file()):
5062 fmode = int(stat.S_IFREG + 438)
5063 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5064 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5065 elif(member.is_symlink()):
5066 fmode = int(stat.S_IFLNK + 438)
5067 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5068 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5069 elif(member.is_dir()):
5070 fmode = int(stat.S_IFDIR + 511)
5071 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
5072 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
5073 try:
5074 fuid = os.getuid();
5075 except AttributeError:
5076 fuid = 0;
5077 except KeyError:
5078 fuid = 0;
5079 try:
5080 fgid = os.getgid();
5081 except AttributeError:
5082 fgid = 0;
5083 except KeyError:
5084 fgid = 0;
5085 try:
5086 import pwd;
5087 try:
5088 userinfo = pwd.getpwuid(os.getuid());
5089 funame = userinfo.pw_name;
5090 except KeyError:
5091 funame = "";
5092 except AttributeError:
5093 funame = "";
5094 except ImportError:
5095 funame = "";
5096 fgname = "";
5097 try:
5098 import grp;
5099 try:
5100 groupinfo = grp.getgrgid(os.getgid());
5101 fgname = groupinfo.gr_name;
5102 except KeyError:
5103 fgname = "";
5104 except AttributeError:
5105 fgname = "";
5106 except ImportError:
5107 fgname = "";
5108 fcontents = "".encode('UTF-8');
5109 if(ftype==0):
5110 fcontents = rarfp.read(member.filename);
5111 ftypehex = format(ftype, 'x').lower();
5112 extrafields = len(extradata);
5113 extrafieldslist = extradata;
5114 catfextrafields = extrafields;
5115 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
5116 if(len(extradata)>0):
5117 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
5118 extrasizelen = len(extrasizestr);
5119 extrasizelenhex = format(extrasizelen, 'x').lower();
5120 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5121 catoutlen = len(catoutlist) + len(extradata) + 3;
5122 catoutlenhex = format(catoutlen, 'x').lower();
5123 catoutlist.insert(0, catoutlenhex);
5124 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
5125 if(len(extradata)>0):
5126 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
5127 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
5128 catfnumfields = 24 + catfextrafields;
5129 if(checksumtype=="none" or checksumtype==""):
5130 catfileheadercshex = format(0, 'x').lower();
5131 catfilecontentcshex = format(0, 'x').lower();
5132 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
5133 catfileheadercshex = format(crc16(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
5134 catfilecontentcshex = format(crc16(fcontents) & 0xffff, '04x').lower();
5135 elif(checksumtype=="crc16_ccitt"):
5136 catfileheadercshex = format(crc16_ccitt(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
5137 catfilecontentcshex = format(crc16_ccitt(fcontents) & 0xffff, '04x').lower();
5138 elif(checksumtype=="adler32"):
5139 catfileheadercshex = format(zlib.adler32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
5140 catfilecontentcshex = format(zlib.adler32(fcontents) & 0xffffffff, '08x').lower();
5141 elif(checksumtype=="crc32"):
5142 catfileheadercshex = format(crc32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
5143 catfilecontentcshex = format(crc32(fcontents) & 0xffffffff, '08x').lower();
5144 elif(checksumtype=="crc64_ecma"):
5145 catfileheadercshex = format(crc64_ecma(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
5146 catfilecontentcshex = format(crc64_ecma(fcontents) & 0xffffffffffffffff, '016x').lower();
5147 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
5148 catfileheadercshex = format(crc64_iso(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
5149 catfilecontentcshex = format(crc64_iso(fcontents) & 0xffffffffffffffff, '016x').lower();
5150 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
5151 checksumoutstr = hashlib.new(checksumtype);
5152 checksumoutstr.update(catfileoutstr.encode('UTF-8'));
5153 catfileheadercshex = checksumoutstr.hexdigest().lower();
5154 else:
5155 catfileheadercshex = format(0, 'x').lower();
5156 catfilecontentcshex = format(0, 'x').lower();
5157 catfhstart = fheadtell;
5158 fheadtell += len(catfileoutstr);
5159 catfhend = fheadtell - 1;
5160 catfcontentstart = fheadtell;
5161 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5162 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower()
5163 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
5164 if(checksumtype=="none" or checksumtype==""):
5165 catfileheadercshex = format(0, 'x').lower();
5166 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
5167 catfileheadercshex = format(crc16(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
5168 elif(checksumtype=="crc16_ccitt"):
5169 catfileheadercshex = format(crc16_ccitt(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
5170 elif(checksumtype=="adler32"):
5171 catfileheadercshex = format(zlib.adler32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
5172 elif(checksumtype=="crc32"):
5173 catfileheadercshex = format(crc32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
5174 elif(checksumtype=="crc64_ecma"):
5175 catfileheadercshex = format(crc64_ecma(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
5176 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
5177 catfileheadercshex = format(crc64_iso(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
5178 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
5179 checksumoutstr = hashlib.new(checksumtype);
5180 checksumoutstr.update(catfileoutstr.encode('UTF-8'));
5181 catfileheadercshex = checksumoutstr.hexdigest().lower();
5182 else:
5183 catfileheadercshex = format(0, 'x').lower();
5184 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5185 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5186 nullstrecd = formatspecs[5].encode('UTF-8');
5187 fheadtell += len(catfileoutstr) + 1;
5188 catfcontentend = fheadtell - 1;
5189 catfileout = catfileoutstrecd + fcontents + nullstrecd;
5190 pyhascontents = False;
5191 if(int(fsize)>0 and not listonly):
5192 pyhascontents = True;
5193 if(int(fsize)>0 and listonly):
5194 fcontents = "";
5195 pyhascontents = False;
5196 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5197 fileidnum = fileidnum + 1;
5198 return catlist;
5199
5200 def ListDirToArray(infiles, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, seekstart=0, seekend=0, listonly=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
5201 outarray = BytesIO();
5202 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
5203 listcatfiles = ArchiveFileToArray(outarray, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5204 return listcatfiles;
5205
5206 def ArchiveFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5207 if(isinstance(infile, dict)):
5208 listcatfiles = infile;
5209 else:
5210 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
5211 infile = RemoveWindowsPath(infile);
5212 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5213 if(not listcatfiles):
5214 return False;
5215 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5216 if(returnfp):
5217 catarray.update({'catfp': listcatfiles['catfp']});
5218 lenlist = len(listcatfiles['ffilelist']);
5219 lcfi = 0;
5220 lcfx = int(listcatfiles['fnumfiles']);
5221 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5222 lcfx = int(lenlist);
5223 else:
5224 lcfx = int(listcatfiles['fnumfiles']);
5225 while(lcfi < lcfx):
5226 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5227 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5228 catarray['filetoid'].update(filetoidarray);
5229 catarray['idtofile'].update(idtofilearray);
5230 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5231 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5232 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5233 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5234 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5235 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5236 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5237 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5238 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5239 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5240 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5241 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5242 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5243 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5244 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5245 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5246 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5247 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5248 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5249 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5250 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5251 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5252 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5253 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5254 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5255 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5256 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
5257 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5258 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5259 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5260 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5261 lcfi = lcfi + 1;
5262 return catarray;
5263
5264 create_alias_function("", __file_format_name__, "ToArrayIndex", ArchiveFileToArrayIndex);
5265
5266 def ListDirToArrayIndexAlt(infiles, dirlistfromtxt=False, followlink=False, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5267 listcatfiles = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, listonly, checksumtype, extradata, formatspecs, verbose);
5268 if(not listcatfiles):
5269 return False;
5270 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5271 lenlist = len(listcatfiles['ffilelist']);
5272 lcfi = 0;
5273 lcfx = int(listcatfiles['fnumfiles']);
5274 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5275 lcfx = int(lenlist);
5276 else:
5277 lcfx = int(listcatfiles['fnumfiles']);
5278 while(lcfi < lcfx):
5279 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5280 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5281 catarray['filetoid'].update(filetoidarray);
5282 catarray['idtofile'].update(idtofilearray);
5283 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5284 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5285 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5286 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5287 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5288 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5289 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5290 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5291 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5292 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5293 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5294 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5295 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5296 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5297 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5298 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5299 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5300 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5301 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5302 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5303 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5304 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5305 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5306 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5307 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5308 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5309 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
5310 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5311 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5312 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5313 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5314 lcfi = lcfi + 1;
5315 return catarray;
5316
5317 def TarFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5318 listcatfiles = TarFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5319 if(not listcatfiles):
5320 return False;
5321 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5322 lenlist = len(listcatfiles['ffilelist']);
5323 lcfi = 0;
5324 lcfx = int(listcatfiles['fnumfiles']);
5325 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5326 lcfx = int(lenlist);
5327 else:
5328 lcfx = int(listcatfiles['fnumfiles']);
5329 while(lcfi < lcfx):
5330 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5331 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5332 catarray['filetoid'].update(filetoidarray);
5333 catarray['idtofile'].update(idtofilearray);
5334 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5335 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5336 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5337 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5338 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5339 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5340 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5341 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5342 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5343 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5344 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5345 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5346 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5347 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5348 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5349 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5350 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5351 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5352 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5353 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5354 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5355 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5356 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5357 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5358 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5359 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5360 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
5361 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5362 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5363 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5364 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5365 lcfi = lcfi + 1;
5366 return catarray;
5367
5368 def ZipFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5369 listcatfiles = ZipFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5370 if(not listcatfiles):
5371 return False;
5372 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5373 lenlist = len(listcatfiles['ffilelist']);
5374 lcfi = 0;
5375 lcfx = int(listcatfiles['fnumfiles']);
5376 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5377 lcfx = int(lenlist);
5378 else:
5379 lcfx = int(listcatfiles['fnumfiles']);
5380 while(lcfi < lcfx):
5381 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5382 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5383 catarray['filetoid'].update(filetoidarray);
5384 catarray['idtofile'].update(idtofilearray);
5385 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5386 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5387 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5388 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5389 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5390 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5391 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5392 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5393 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5394 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5395 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5396 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5397 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5398 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5399 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5400 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5401 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5402 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5403 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5404 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5405 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5406 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5407 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5408 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5409 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5410 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5411 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
5412 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5413 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5414 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5415 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5416 lcfi = lcfi + 1;
5417 return catarray;
5418
5419 if(not rarfile_support):
5420 def RarFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5421 return False;
5422
5423 if(rarfile_support):
5424 def RarFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5425 listcatfiles = RarFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5426 if(not listcatfiles):
5427 return False;
5428 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5429 lenlist = len(listcatfiles['ffilelist']);
5430 lcfi = 0;
5431 lcfx = int(listcatfiles['fnumfiles']);
5432 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5433 lcfx = int(lenlist);
5434 else:
5435 lcfx = int(listcatfiles['fnumfiles']);
5436 while(lcfi < lcfx):
5437 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5438 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5439 catarray['filetoid'].update(filetoidarray);
5440 catarray['idtofile'].update(idtofilearray);
5441 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5442 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5443 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5444 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5445 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5446 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5447 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5448 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5449 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5450 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5451 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5452 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5453 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5454 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5455 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5456 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5457 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5458 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5459 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5460 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5461 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5462 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5463 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5464 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5465 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5466 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5467 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
5468 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5469 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5470 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5471 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5472 lcfi = lcfi + 1;
5473 return catarray;
5474
5475 def ArchiveFileStringToArrayIndex(catstr, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5476 catfp = BytesIO(catstr);
5477 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5478 return listcatfiles;
5479
5480 create_alias_function("", __file_format_name__, "StringToArrayIndex", ArchiveFileStringToArrayIndex);
5481
5482 def TarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5483 catfp = BytesIO();
5484 catfp = PackArchiveFileFromTarFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
5485 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5486 return listcatfiles;
5487
5488 def ZipFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5489 catfp = BytesIO();
5490 catfp = PackArchiveFileFromZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
5491 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5492 return listcatfiles;
5493
5494 if(not rarfile_support):
5495 def RarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5496 return False;
5497
5498 if(rarfile_support):
5499 def RarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5500 catfp = BytesIO();
5501 catfp = PackArchiveFileFromRarFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
5502 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5503 return listcatfiles;
5504
5505 def ListDirToArrayIndex(infiles, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, seekstart=0, seekend=0, listonly=False, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False, returnfp=False):
5506 outarray = BytesIO();
5507 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compressionlevel, followlink, checksumtype, formatspecs, verbose, True);
5508 listcatfiles = ArchiveFileToArrayIndex(outarray, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp)
5509 return listcatfiles;
5510
5511 def RePackArchiveFile(infile, outfile, compression="auto", compressionlevel=None, followlink=False, seekstart=0, seekend=0, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
5512 compressionlist = ['auto', 'gzip', 'bzip2', 'zstd', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
5513 outextlist = ['gz', 'bz2', 'zst', 'lz4', 'lzop', 'lzo', 'lzma', 'xz'];
5514 outextlistwd = ['.gz', '.bz2', '.zst', '.lz4', 'lzop', '.lzo', '.lzma', '.xz'];
5515 if(isinstance(infile, dict)):
5516 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
5517 listcatfiles = prelistcatfiles['list'];
5518 else:
5519 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
5520 infile = RemoveWindowsPath(infile);
5521 if(followlink):
5522 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
5523 listcatfiles = prelistcatfiles['list'];
5524 else:
5525 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
5526 if(outfile!="-" and not hasattr(infile, "read") and not hasattr(outfile, "write")):
5527 outfile = RemoveWindowsPath(outfile);
5528 checksumtype = checksumtype.lower();
5529 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
5530 checksumtype="crc32";
5531 if(checksumtype=="none"):
5532 checksumtype = "";
5533 if(not compression or compression or compression=="catfile" or compression==formatspecs[2]):
5534 compression = None;
5535 if(compression not in compressionlist and compression is None):
5536 compression = "auto";
5537 if(verbose):
5538 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
5539 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
5540 if(os.path.exists(outfile)):
5541 os.unlink(outfile);
5542 if(not listcatfiles):
5543 return False;
5544 if(outfile=="-"):
5545 verbose = False;
5546 catfp = BytesIO();
5547 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
5548 catfp = outfile;
5549 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", outfile)):
5550 catfp = BytesIO();
5551 else:
5552 fbasename = os.path.splitext(outfile)[0];
5553 fextname = os.path.splitext(outfile)[1];
5554 catfp = CompressOpenFile(outfile, compressionlevel);
5555 catver = formatspecs[6];
5556 fileheaderver = str(int(catver.replace(".", "")));
5557 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
5558 catfp.write(fileheader.encode('UTF-8'));
5559 lenlist = len(listcatfiles['ffilelist']);
5560 fnumfiles = int(listcatfiles['fnumfiles']);
5561 if(lenlist>fnumfiles or lenlist<fnumfiles):
5562 fnumfiles = lenlist;
5563 fnumfileshex = format(int(fnumfiles), 'x').lower();
5564 fnumfilesa = AppendNullBytes([fnumfileshex, checksumtype], formatspecs[5]);
5565 if(checksumtype=="none" or checksumtype==""):
5566 catfileheadercshex = format(0, 'x').lower();
5567 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
5568 catfileheadercshex = format(crc16(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffff, '04x').lower();
5569 elif(checksumtype=="crc16_ccitt"):
5570 catfileheadercshex = format(crc16_ccitt(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffff, '04x').lower();
5571 elif(checksumtype=="adler32"):
5572 catfileheadercshex = format(zlib.adler32(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffff, '08x').lower();
5573 elif(checksumtype=="crc32"):
5574 catfileheadercshex = format(crc32(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffff, '08x').lower();
5575 elif(checksumtype=="crc64_ecma"):
5576 catfileheadercshex = format(crc64_ecma(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
5577 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
5578 catfileheadercshex = format(crc64_iso(str(fileheader + fnumfilesa).encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
5579 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
5580 checksumoutstr = hashlib.new(checksumtype);
5581 checksumoutstr.update(str(fileheader + fnumfilesa).encode('UTF-8'));
5582 catfileheadercshex = checksumoutstr.hexdigest().lower();
5583 else:
5584 catfileheadercshex = format(0, 'x').lower();
5585 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs[5]);
5586 catfp.write(fnumfilesa.encode('UTF-8'));
5587 try:
5588 catfp.flush();
5589 os.fsync(catfp.fileno());
5590 except io.UnsupportedOperation:
5591 pass;
5592 except AttributeError:
5593 pass;
5594 lenlist = len(listcatfiles['ffilelist']);
5595 fnumfiles = int(listcatfiles['fnumfiles']);
5596 lcfi = 0;
5597 lcfx = int(listcatfiles['fnumfiles']);
5598 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5599 lcfx = int(lenlist);
5600 else:
5601 lcfx = int(listcatfiles['fnumfiles']);
5602 curinode = 0;
5603 curfid = 0;
5604 inodelist = [];
5605 inodetofile = {};
5606 filetoinode = {};
5607 reallcfi = 0;
5608 while(lcfi < lcfx):
5609 catfhstart = catfp.tell();
5610 if(re.findall("^[.|/]", listcatfiles['ffilelist'][reallcfi]['fname'])):
5611 fname = listcatfiles['ffilelist'][reallcfi]['fname'];
5612 else:
5613 fname = "./"+listcatfiles['ffilelist'][reallcfi]['fname'];
5614 if(verbose):
5615 VerbosePrintOut(fname);
5616 fheadersize = format(int(listcatfiles['ffilelist'][reallcfi]['fheadersize']), 'x').lower();
5617 fsize = format(int(listcatfiles['ffilelist'][reallcfi]['fsize']), 'x').lower();
5618 flinkname = listcatfiles['ffilelist'][reallcfi]['flinkname'];
5619 fatime = format(int(listcatfiles['ffilelist'][reallcfi]['fatime']), 'x').lower();
5620 fmtime = format(int(listcatfiles['ffilelist'][reallcfi]['fmtime']), 'x').lower();
5621 fctime = format(int(listcatfiles['ffilelist'][reallcfi]['fctime']), 'x').lower();
5622 fbtime = format(int(listcatfiles['ffilelist'][reallcfi]['fbtime']), 'x').lower();
5623 fmode = format(int(listcatfiles['ffilelist'][reallcfi]['fmode']), 'x').lower();
5624 fchmode = format(int(listcatfiles['ffilelist'][reallcfi]['fchmode']), 'x').lower();
5625 fuid = format(int(listcatfiles['ffilelist'][reallcfi]['fuid']), 'x').lower();
5626 funame = listcatfiles['ffilelist'][reallcfi]['funame'];
5627 fgid = format(int(listcatfiles['ffilelist'][reallcfi]['fgid']), 'x').lower();
5628 fgname = listcatfiles['ffilelist'][reallcfi]['fgname'];
5629 finode = listcatfiles['ffilelist'][reallcfi]['finode'];
5630 flinkcount = listcatfiles['ffilelist'][reallcfi]['flinkcount'];
5631 fwinattributes = flinkinfo['ffilelist'][reallcfi]['fwinattributes'];
5632 fdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['fminor']), 'x').lower();
5633 fdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['fmajor']), 'x').lower();
5634 frdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['frminor']), 'x').lower();
5635 frdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['frmajor']), 'x').lower();
5636 if(len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>listcatfiles['ffilelist'][reallcfi]['fextrafields'] and len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>0):
5637 listcatfiles['ffilelist'][reallcfi]['fextrafields'] = len(listcatfiles['ffilelist'][reallcfi]['fextralist']);
5638 if(len(extradata) > 0):
5639 listcatfiles['ffilelist'][reallcfi]['fextrafields'] = len(extradata);
5640 listcatfiles['ffilelist'][reallcfi]['fextralist'] = extradata;
5641 extrafields = format(int(listcatfiles['ffilelist'][reallcfi]['fextrafields']), 'x').lower();
5642 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
5643 if(len(extradata)>0):
5644 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
5645 extrasizelen = format(len(extrasizestr), 'x').lower();
5646 fcontents = listcatfiles['ffilelist'][reallcfi]['fcontents'];
5647 if(followlink):
5648 if(listcatfiles['ffilelist'][reallcfi]['ftype']==1 or listcatfiles['ffilelist'][reallcfi]['ftype']==2):
5649 getflinkpath = listcatfiles['ffilelist'][reallcfi]['flinkname'];
5650 flinkid = prelistcatfiles['filetoid'][getflinkpath];
5651 flinkinfo = listcatfiles['ffilelist'][flinkid];
5652 fheadersize = format(int(flinkinfo['fheadersize']), 'x').lower();
5653 fsize = format(int(flinkinfo['fsize']), 'x').lower();
5654 flinkname = flinkinfo['flinkname'];
5655 fatime = format(int(flinkinfo['fatime']), 'x').lower();
5656 fmtime = format(int(flinkinfo['fmtime']), 'x').lower();
5657 fctime = format(int(flinkinfo['fctime']), 'x').lower();
5658 fbtime = format(int(flinkinfo['fbtime']), 'x').lower();
5659 fmode = format(int(flinkinfo['fmode']), 'x').lower();
5660 fchmode = format(int(flinkinfo['fchmode']), 'x').lower();
5661 fuid = format(int(flinkinfo['fuid']), 'x').lower();
5662 funame = flinkinfo['funame'];
5663 fgid = format(int(flinkinfo['fgid']), 'x').lower();
5664 fgname = flinkinfo['fgname'];
5665 finode = flinkinfo['finode'];
5666 flinkcount = flinkinfo['flinkcount'];
5667 fwinattributes = flinkinfo['fwinattributes'];
5668 fdev_minor = format(int(flinkinfo['fminor']), 'x').lower();
5669 fdev_major = format(int(flinkinfo['fmajor']), 'x').lower();
5670 frdev_minor = format(int(flinkinfo['frminor']), 'x').lower();
5671 frdev_major = format(int(flinkinfo['frmajor']), 'x').lower();
5672 if(len(flinkinfo['fextralist'])>flinkinfo['fextrafields'] and len(flinkinfo['fextralist'])>0):
5673 flinkinfo['fextrafields'] = len(flinkinfo['fextralist']);
5674 if(len(extradata) > 0):
5675 flinkinfo['fextrafields'] = len(extradata);
5676 flinkinfo['fextralist'] = extradata;
5677 extrafields = format(int(flinkinfo['fextrafields']), 'x').lower();
5678 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
5679 if(len(extradata)>0):
5680 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
5681 extrasizelen = format(len(extrasizestr), 'x').lower();
5682 fcontents = flinkinfo['fcontents'];
5683 if(flinkinfo['ftype']!=0 and flinkinfo['ftype']!=7):
5684 try:
5685 fcontents = fcontents.encode('UTF-8');
5686 except AttributeError:
5687 pass;
5688 ftypehex = format(flinkinfo['ftype'], 'x').lower();
5689 else:
5690 if(listcatfiles['ffilelist'][reallcfi]['ftype']!=0 and listcatfiles['ffilelist'][reallcfi]['ftype']!=7):
5691 try:
5692 fcontents = fcontents.encode('UTF-8');
5693 except AttributeError:
5694 pass;
5695 ftypehex = format(listcatfiles['ffilelist'][reallcfi]['ftype'], 'x').lower();
5696 fcurfid = format(curfid, 'x').lower();
5697 if(not followlink and finode!=0):
5698 if(listcatfiles['ffilelist'][reallcfi]['ftype']!=1):
5699 fcurinode = format(int(curinode), 'x').lower();
5700 inodetofile.update({curinode: fname});
5701 filetoinode.update({fname: curinode});
5702 curinode = curinode + 1;
5703 else:
5704 fcurinode = format(int(filetoinode[flinkname]), 'x').lower();
5705 else:
5706 fcurinode = format(int(curinode), 'x').lower();
5707 curinode = curinode + 1;
5708 curfid = curfid + 1;
5709 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, extrasizelen, extrafields];
5710 catoutlen = len(catoutlist) + len(extradata) + 3;
5711 catoutlenhex = format(catoutlen, 'x').lower();
5712 catoutlist.insert(0, catoutlenhex);
5713 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
5714 if(listcatfiles['ffilelist'][reallcfi]['fextrafields']>0):
5715 extrafieldslist = [];
5716 exi = 0;
5717 exil = listcatfiles['ffilelist'][reallcfi]['fextrafields'];
5718 while(exi < exil):
5719 extrafieldslist.append(listcatfiles['ffilelist'][reallcfi]['fextralist']);
5720 exi = exi + 1;
5721 catfileoutstr += AppendNullBytes([extrafieldslist], formatspecs[5]);
5722 catfileoutstr += AppendNullBytes([checksumtype], formatspecs[5]);
5723 catfhend = (catfp.tell() - 1) + len(catfileoutstr);
5724 catfcontentstart = catfp.tell() + len(catfileoutstr);
5725 if(checksumtype=="none" or checksumtype==""):
5726 catfileheadercshex = format(0, 'x').lower();
5727 catfilecontentcshex = format(0, 'x').lower();
5728 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
5729 catfileheadercshex = format(crc16("".encode('UTF-8')) & 0xffff, '04x').lower();
5730 catfilecontentcshex = format(crc16(fcontents) & 0xffff, '04x').lower();
5731 elif(checksumtype=="crc16_ccitt"):
5732 catfileheadercshex = format(crc16_ccitt("".encode('UTF-8')) & 0xffff, '04x').lower();
5733 catfilecontentcshex = format(crc16_ccitt(fcontents) & 0xffff, '04x').lower();
5734 elif(checksumtype=="adler32"):
5735 catfileheadercshex = format(zlib.adler32("".encode('UTF-8')) & 0xffffffff, '08x').lower();
5736 catfilecontentcshex = format(zlib.adler32(fcontents) & 0xffffffff, '08x').lower();
5737 elif(checksumtype=="crc32"):
5738 catfileheadercshex = format(crc32("".encode('UTF-8')) & 0xffffffff, '08x').lower();
5739 catfilecontentcshex = format(crc32(fcontents) & 0xffffffff, '08x').lower();
5740 elif(checksumtype=="crc64_ecma"):
5741 catfileheadercshex = format(crc64_ecma("".encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
5742 catfilecontentcshex = format(crc64_ecma(fcontents) & 0xffffffffffffffff, '016x').lower();
5743 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
5744 catfileheadercshex = format(crc64_iso("".encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
5745 catfilecontentcshex = format(crc64_iso(fcontents) & 0xffffffffffffffff, '016x').lower();
5746 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
5747 checksumoutstr = hashlib.new(checksumtype);
5748 checksumoutstr.update("".encode('UTF-8'));
5749 catfileheadercshex = checksumoutstr.hexdigest().lower();
5750 checksumoutstr = hashlib.new(checksumtype);
5751 checksumoutstr.update(fcontents);
5752 catfilecontentcshex = checksumoutstr.hexdigest().lower();
5753 else:
5754 catfileheadercshex = format(0, 'x').lower();
5755 catfilecontentcshex = format(0, 'x').lower();
5756 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5757 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower()
5758 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
5759 if(checksumtype=="none" or checksumtype==""):
5760 catfileheadercshex = format(0, 'x').lower();
5761 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
5762 catfileheadercshex = format(crc16(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
5763 elif(checksumtype=="crc16_ccitt"):
5764 catfileheadercshex = format(crc16_ccitt(catfileoutstr.encode('UTF-8')) & 0xffff, '04x').lower();
5765 elif(checksumtype=="adler32"):
5766 catfileheadercshex = format(zlib.adler32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
5767 elif(checksumtype=="crc32"):
5768 catfileheadercshex = format(crc32(catfileoutstr.encode('UTF-8')) & 0xffffffff, '08x').lower();
5769 elif(checksumtype=="crc64_ecma"):
5770 catfileheadercshex = format(crc64_ecma(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
5771 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
5772 catfileheadercshex = format(crc64_iso(catfileoutstr.encode('UTF-8')) & 0xffffffffffffffff, '016x').lower();
5773 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
5774 checksumoutstr = hashlib.new(checksumtype);
5775 checksumoutstr.update(catfileoutstr.encode('UTF-8'));
5776 catfileheadercshex = checksumoutstr.hexdigest().lower();
5777 else:
5778 catfileheadercshex = format(0, 'x').lower();
5779 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5780 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5781 nullstrecd = formatspecs[5].encode('UTF-8');
5782 catfileout = catfileoutstrecd + fcontents + nullstrecd;
5783 catfcontentend = (catfp.tell() - 1) + len(catfileout);
5784 catfp.write(catfileout);
5785 try:
5786 catfp.flush();
5787 os.fsync(catfp.fileno());
5788 except io.UnsupportedOperation:
5789 pass;
5790 except AttributeError:
5791 pass;
5792 lcfi = lcfi + 1;
5793 reallcfi = reallcfi + 1;
5794 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
5795 catfp = CompressArchiveFile(catfp, compression, formatspecs);
5796 try:
5797 catfp.flush();
5798 os.fsync(catfp.fileno());
5799 except io.UnsupportedOperation:
5800 pass;
5801 except AttributeError:
5802 pass;
5803 if(outfile=="-"):
5804 catfp.seek(0, 0);
5805 if(hasattr(sys.stdout, "buffer")):
5806 shutil.copyfileobj(catfp, sys.stdout.buffer);
5807 else:
5808 shutil.copyfileobj(catfp, sys.stdout);
5809 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", outfile)):
5810 catfp = CompressArchiveFile(catfp, compression, formatspecs);
5811 catfp.seek(0, 0);
5812 upload_file_to_internet_file(catfp, outfile);
5813 if(returnfp):
5814 catfp.seek(0, 0);
5815 return catfp;
5816 else:
5817 catfp.close();
5818 return True;
5819
5820 create_alias_function("RePack", __file_format_name__, "", RePackArchiveFile);
5821
5822 def RePackArchiveFileFromString(catstr, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
5823 catfp = BytesIO(catstr);
5824 listcatfiles = RePackArchiveFile(catfp, compression, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
5825 return listcatfiles;
5826
5827 create_alias_function("RePack", __file_format_name__, "FromString", RePackArchiveFileFromString);
5828
5829 def PackArchiveFileFromListDir(infiles, outfile, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
5830 outarray = BytesIO();
5831 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
5832 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
5833 return listcatfiles;
5834
5835 create_alias_function("Pack", __file_format_name__, "FromListDir", PackArchiveFileFromListDir);
5836
5837 def ArchiveFileArrayBase64Encode(infile, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_list__, verbose=False, returnfp=False):
5838 if(verbose):
5839 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
5840 if(isinstance(infile, dict)):
5841 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
5842 listcatfiles = prelistcatfiles['list'];
5843 else:
5844 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
5845 infile = RemoveWindowsPath(infile);
5846 if(followlink):
5847 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
5848 listcatfiles = prelistcatfiles['list'];
5849 else:
5850 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
5851 if(not listcatfiles):
5852 return False;
5853 lenlist = len(listcatfiles['ffilelist']);
5854 fnumfiles = int(listcatfiles['fnumfiles']);
5855 lcfi = 0;
5856 lcfx = int(listcatfiles['fnumfiles']);
5857 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5858 lcfx = int(lenlist);
5859 else:
5860 lcfx = int(listcatfiles['fnumfiles']);
5861 while(lcfi < lcfx):
5862 if(listcatfiles['ffilelist'][lcfi]['fhascontents']):
5863 listcatfiles['ffilelist'][lcfi]['fcontents'] = base64.b64encode(listcatfiles['ffilelist'][lcfi]['fcontents']).decode("UTF-8");
5864 lcfi = lcfi + 1;
5865 return listcatfiles;
5866
5867 create_alias_function("", __file_format_name__, "ArrayBase64Encode", ArchiveFileArrayBase64Encode);
5868
5869 def ArchiveFileArrayBase64Decode(infile, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_list__, verbose=False, returnfp=False):
5870 if(verbose):
5871 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
5872 if(isinstance(infile, dict)):
5873 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
5874 listcatfiles = prelistcatfiles['list'];
5875 else:
5876 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
5877 infile = RemoveWindowsPath(infile);
5878 if(followlink):
5879 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
5880 listcatfiles = prelistcatfiles['list'];
5881 else:
5882 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
5883 if(not listcatfiles):
5884 return False;
5885 lenlist = len(listcatfiles['ffilelist']);
5886 fnumfiles = int(listcatfiles['fnumfiles']);
5887 lcfi = 0;
5888 lcfx = int(listcatfiles['fnumfiles']);
5889 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5890 lcfx = int(lenlist);
5891 else:
5892 lcfx = int(listcatfiles['fnumfiles']);
5893 while(lcfi < lcfx):
5894 if(listcatfiles['ffilelist'][lcfi]['fhascontents']):
5895 listcatfiles['ffilelist'][lcfi]['fcontents'] = base64.b64decode(listcatfiles['ffilelist'][lcfi]['fcontents'].encode("UTF-8"));
5896 lcfi = lcfi + 1;
5897 return listcatfiles;
5898
5899 create_alias_function("", __file_format_name__, "ArrayBase64Decode", ArchiveFileArrayBase64Decode);
5900
5901 def UnPackArchiveFile(infile, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_list__, preservepermissions=True, preservetime=True, verbose=False, returnfp=False):
5902 if(outdir is not None):
5903 outdir = RemoveWindowsPath(outdir);
5904 if(verbose):
5905 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
5906 if(isinstance(infile, dict)):
5907 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
5908 listcatfiles = prelistcatfiles['list'];
5909 else:
5910 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
5911 infile = RemoveWindowsPath(infile);
5912 if(followlink):
5913 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
5914 listcatfiles = prelistcatfiles['list'];
5915 else:
5916 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
5917 if(not listcatfiles):
5918 return False;
5919 lenlist = len(listcatfiles['ffilelist']);
5920 fnumfiles = int(listcatfiles['fnumfiles']);
5921 lcfi = 0;
5922 lcfx = int(listcatfiles['fnumfiles']);
5923 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5924 lcfx = int(lenlist);
5925 else:
5926 lcfx = int(listcatfiles['fnumfiles']);
5927 while(lcfi < lcfx):
5928 funame = "";
5929 try:
5930 import pwd;
5931 try:
5932 userinfo = pwd.getpwuid(listcatfiles['ffilelist'][lcfi]['fuid']);
5933 funame = userinfo.pw_name;
5934 except KeyError:
5935 funame = "";
5936 except ImportError:
5937 funame = "";
5938 fgname = "";
5939 try:
5940 import grp;
5941 try:
5942 groupinfo = grp.getgrgid(listcatfiles['ffilelist'][lcfi]['fgid']);
5943 fgname = groupinfo.gr_name;
5944 except KeyError:
5945 fgname = "";
5946 except ImportError:
5947 fgname = "";
5948 if(verbose):
5949 VerbosePrintOut(listcatfiles['ffilelist'][lcfi]['fname']);
5950 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5951 with open(listcatfiles['ffilelist'][lcfi]['fname'], "wb") as fpc:
5952 fpc.write(listcatfiles['ffilelist'][lcfi]['fcontents'])
5953 try:
5954 fpc.flush()
5955 os.fsync(fpc.fileno())
5956 except io.UnsupportedOperation:
5957 pass
5958 except AttributeError:
5959 pass
5960 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
5961 os.chown(listcatfiles['ffilelist'][lcfi]['fname'], listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
5962 if(preservepermissions):
5963 os.chmod(listcatfiles['ffilelist'][lcfi]['fname'], listcatfiles['ffilelist'][lcfi]['fchmode']);
5964 if(preservetime):
5965 os.utime(listcatfiles['ffilelist'][lcfi]['fname'], (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
5966 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5967 if(followlink):
5968 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
5969 flinkid = prelistcatfiles['filetoid'][getflinkpath];
5970 flinkinfo = listcatfiles['ffilelist'][flinkid];
5971 funame = "";
5972 try:
5973 import pwd;
5974 try:
5975 userinfo = pwd.getpwuid(flinkinfo['fuid']);
5976 funame = userinfo.pw_name;
5977 except KeyError:
5978 funame = "";
5979 except ImportError:
5980 funame = "";
5981 fgname = "";
5982 try:
5983 import grp;
5984 try:
5985 groupinfo = grp.getgrgid(flinkinfo['fgid']);
5986 fgname = groupinfo.gr_name;
5987 except KeyError:
5988 fgname = "";
5989 except ImportError:
5990 fgname = "";
5991 if(flinkinfo['ftype'] == 0 or flinkinfo['ftype'] == 7):
5992 with open(listcatfiles['ffilelist'][lcfi]['fname'], "wb") as fpc:
5993 fpc.write(flinkinfo['fcontents'])
5994 try:
5995 fpc.flush()
5996 os.fsync(fpc.fileno())
5997 except io.UnsupportedOperation:
5998 pass
5999 except AttributeError:
6000 pass
6001 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6002 os.chown(listcatfiles['ffilelist'][lcfi]['fname'], flinkinfo['fuid'], flinkinfo['fgid']);
6003 if(preservepermissions):
6004 os.chmod(listcatfiles['ffilelist'][lcfi]['fname'], flinkinfo['fchmode']);
6005 if(preservetime):
6006 os.utime(listcatfiles['ffilelist'][lcfi]['fname'], (flinkinfo['fatime'], flinkinfo['fmtime']));
6007 if(flinkinfo['ftype']==1):
6008 os.link(flinkinfo['flinkname'], listcatfiles['ffilelist'][lcfi]['fname']);
6009 if(flinkinfo['ftype']==2):
6010 os.symlink(flinkinfo['flinkname'], listcatfiles['ffilelist'][lcfi]['fname']);
6011 if(flinkinfo['ftype']==5):
6012 if(preservepermissions):
6013 os.mkdir(listcatfiles['ffilelist'][lcfi]['fname'], flinkinfo['fchmode']);
6014 else:
6015 os.mkdir(listcatfiles['ffilelist'][lcfi]['fname']);
6016 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6017 os.chown(listcatfiles['ffilelist'][lcfi]['fname'], flinkinfo['fuid'], flinkinfo['fgid']);
6018 if(preservepermissions):
6019 os.chmod(listcatfiles['ffilelist'][lcfi]['fname'], flinkinfo['fchmode']);
6020 if(preservetime):
6021 os.utime(listcatfiles['ffilelist'][lcfi]['fname'], (flinkinfo['fatime'], flinkinfo['fmtime']));
6022 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
6023 os.mkfifo(listcatfiles['ffilelist'][lcfi]['fname'], flinkinfo['fchmode']);
6024 else:
6025 os.link(listcatfiles['ffilelist'][lcfi]['flinkname'], listcatfiles['ffilelist'][lcfi]['fname']);
6026 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6027 if(followlink):
6028 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
6029 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6030 flinkinfo = listcatfiles['ffilelist'][flinkid];
6031 funame = "";
6032 try:
6033 import pwd;
6034 try:
6035 userinfo = pwd.getpwuid(flinkinfo['fuid']);
6036 funame = userinfo.pw_name;
6037 except KeyError:
6038 funame = "";
6039 except ImportError:
6040 funame = "";
6041 fgname = "";
6042 try:
6043 import grp;
6044 try:
6045 groupinfo = grp.getgrgid(flinkinfo['fgid']);
6046 fgname = groupinfo.gr_name;
6047 except KeyError:
6048 fgname = "";
6049 except ImportError:
6050 fgname = "";
6051 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
6052 with open(listcatfiles['ffilelist'][lcfi]['fname'], "wb") as fpc:
6053 fpc.write(flinkinfo['fcontents'])
6054 try:
6055 fpc.flush()
6056 os.fsync(fpc.fileno())
6057 except io.UnsupportedOperation:
6058 pass
6059 except AttributeError:
6060 pass
6061 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6062 os.chown(listcatfiles['ffilelist'][lcfi]['fname'], flinkinfo['fuid'], flinkinfo['fgid']);
6063 if(preservepermissions):
6064 os.chmod(listcatfiles['ffilelist'][lcfi]['fname'], flinkinfo['fchmode']);
6065 if(preservetime):
6066 os.utime(listcatfiles['ffilelist'][lcfi]['fname'], (flinkinfo['fatime'], flinkinfo['fmtime']));
6067 if(flinkinfo['ftype']==1):
6068 os.link(flinkinfo['flinkname'], listcatfiles['ffilelist'][lcfi]['fname']);
6069 if(flinkinfo['ftype']==2):
6070 os.symlink(flinkinfo['flinkname'], listcatfiles['ffilelist'][lcfi]['fname']);
6071 if(flinkinfo['ftype']==5):
6072 if(preservepermissions):
6073 os.mkdir(listcatfiles['ffilelist'][lcfi]['fname'], flinkinfo['fchmode']);
6074 else:
6075 os.mkdir(listcatfiles['ffilelist'][lcfi]['fname']);
6076 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6077 os.chown(listcatfiles['ffilelist'][lcfi]['fname'], flinkinfo['fuid'], flinkinfo['fgid']);
6078 if(preservepermissions):
6079 os.chmod(listcatfiles['ffilelist'][lcfi]['fname'], flinkinfo['fchmode']);
6080 if(preservetime):
6081 os.utime(listcatfiles['ffilelist'][lcfi]['fname'], (flinkinfo['fatime'], flinkinfo['fmtime']));
6082 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
6083 os.mkfifo(listcatfiles['ffilelist'][lcfi]['fname'], flinkinfo['fchmode']);
6084 else:
6085 os.symlink(listcatfiles['ffilelist'][lcfi]['flinkname'], listcatfiles['ffilelist'][lcfi]['fname']);
6086 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6087 if(preservepermissions):
6088 os.mkdir(listcatfiles['ffilelist'][lcfi]['fname'], listcatfiles['ffilelist'][lcfi]['fchmode']);
6089 else:
6090 os.mkdir(listcatfiles['ffilelist'][lcfi]['fname']);
6091 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
6092 os.chown(listcatfiles['ffilelist'][lcfi]['fname'], listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
6093 if(preservepermissions):
6094 os.chmod(listcatfiles['ffilelist'][lcfi]['fname'], listcatfiles['ffilelist'][lcfi]['fchmode']);
6095 if(preservetime):
6096 os.utime(listcatfiles['ffilelist'][lcfi]['fname'], (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
6097 if(listcatfiles['ffilelist'][lcfi]['ftype']==6 and hasattr(os, "mkfifo")):
6098 os.mkfifo(listcatfiles['ffilelist'][lcfi]['fname'], listcatfiles['ffilelist'][lcfi]['fchmode']);
6099 lcfi = lcfi + 1;
6100 if(returnfp):
6101 return listcatfiles['ffilelist']['catfp'];
6102 else:
6103 return True;
6104
6105 create_alias_function("UnPack", __file_format_name__, "", UnPackArchiveFile);
6106
6107 if(hasattr(shutil, "register_unpack_format")):
6108 def UnPackArchiveFileFunc(archive_name, extract_dir=None, **kwargs):
6109 return UnPackArchiveFile(archive_name, extract_dir, False, 0, 0, False, __file_format_delimiter__, False, False);
6110 create_alias_function("UnPack", __file_format_name__, "Func", UnPackArchiveFileFunc);
6111
6112 def UnPackArchiveFileString(catstr, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_list__, verbose=False, returnfp=False):
6113 catfp = BytesIO(catstr);
6114 listcatfiles = UnPackArchiveFile(catfp, outdir, followlink, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6115 return listcatfiles;
6116
6117 create_alias_function("UnPack", __file_format_name__, "String", UnPackArchiveFileString);
6118
6119 def ArchiveFileListFiles(infile, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_list__, verbose=False, returnfp=False):
6120 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6121 if(isinstance(infile, dict)):
6122 listcatfiles = infile;
6123 else:
6124 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6125 infile = RemoveWindowsPath(infile);
6126 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, True, skipchecksum, formatspecs, returnfp);
6127 if(not listcatfiles):
6128 return False;
6129 lenlist = len(listcatfiles['ffilelist']);
6130 fnumfiles = int(listcatfiles['fnumfiles']);
6131 lcfi = 0;
6132 lcfx = int(listcatfiles['fnumfiles']);
6133 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6134 lcfx = int(lenlist);
6135 else:
6136 lcfx = int(listcatfiles['fnumfiles']);
6137 returnval = {};
6138 while(lcfi < lcfx):
6139 returnval.update({lcfi: listcatfiles['ffilelist'][lcfi]['fname']});
6140 if(not verbose):
6141 VerbosePrintOut(listcatfiles['ffilelist'][lcfi]['fname']);
6142 if(verbose):
6143 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6144 printfname = listcatfiles['ffilelist'][lcfi]['fname'];
6145 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6146 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " link to " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6147 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6148 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " -> " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6149 fuprint = listcatfiles['ffilelist'][lcfi]['funame'];
6150 if(len(fuprint)<=0):
6151 fuprint = listcatfiles['ffilelist'][lcfi]['fuid'];
6152 fgprint = listcatfiles['ffilelist'][lcfi]['fgname'];
6153 if(len(fgprint)<=0):
6154 fgprint = listcatfiles['ffilelist'][lcfi]['fgid'];
6155 VerbosePrintOut(PrintPermissionString(listcatfiles['ffilelist'][lcfi]['fmode'], listcatfiles['ffilelist'][lcfi]['ftype']) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(listcatfiles['ffilelist'][lcfi]['fsize']).rjust(15) + " " + datetime.datetime.utcfromtimestamp(listcatfiles['ffilelist'][lcfi]['fmtime']).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6156 lcfi = lcfi + 1;
6157 if(returnfp):
6158 return listcatfiles['catfp'];
6159 else:
6160 return True;
6161
6162 create_alias_function("", __file_format_name__, "ListFiles", ArchiveFileListFiles);
6163
6164 def ArchiveFileStringListFiles(catstr, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_list__, verbose=False, returnfp=False):
6165 catfp = BytesIO(catstr);
6166 listcatfiles = ArchiveFileListFiles(catstr, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6167 return listcatfiles;
6168
6169 create_alias_function("", __file_format_name__, "StringListFiles", ArchiveFileListFiles);
6170
6171 def ArchiveFileListFilesAlt(infile, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_list__, verbose=False, returnfp=False):
6172 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6173 if(isinstance(infile, dict)):
6174 listcatfiles = infile;
6175 else:
6176 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6177 infile = RemoveWindowsPath(infile);
6178 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, True, skipchecksum, formatspecs, returnfp);
6179 if(not listcatfiles):
6180 return False;
6181 lenlist = len(listcatfiles['ffilelist']);
6182 fnumfiles = int(listcatfiles['fnumfiles']);
6183 lcfi = 0;
6184 lcfx = int(listcatfiles['fnumfiles']);
6185 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6186 lcfx = int(lenlist);
6187 else:
6188 lcfx = int(listcatfiles['fnumfiles']);
6189 returnval = {};
6190 while(lcfi < lcfx):
6191 returnval.update({lcfi: listcatfiles['ffilelist'][lcfi]['fname']});
6192 if(not verbose):
6193 VerbosePrintOut(listcatfiles['ffilelist'][lcfi]['fname']);
6194 if(verbose):
6195 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6196 printfname = listcatfiles['ffilelist'][lcfi]['fname'];
6197 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6198 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " link to " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6199 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6200 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " -> " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6201 fuprint = listcatfiles['ffilelist'][lcfi]['funame'];
6202 if(len(fuprint)<=0):
6203 fuprint = listcatfiles['ffilelist'][lcfi]['fuid'];
6204 fgprint = listcatfiles['ffilelist'][lcfi]['fgname'];
6205 if(len(fgprint)<=0):
6206 fgprint = listcatfiles['ffilelist'][lcfi]['fgid'];
6207 VerbosePrintOut(PrintPermissionString(listcatfiles['ffilelist'][lcfi]['fmode'], listcatfiles['ffilelist'][lcfi]['ftype']) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(listcatfiles['ffilelist'][lcfi]['fsize']).rjust(15) + " " + datetime.datetime.utcfromtimestamp(listcatfiles['ffilelist'][lcfi]['fmtime']).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6208 lcfi = lcfi + 1;
6209 if(returnfp):
6210 return listcatfiles['catfp'];
6211 else:
6212 return True;
6213
6214 create_alias_function("", __file_format_name__, "ListFilesAlt", ArchiveFileListFilesAlt);
6215
6216 def ArchiveFileStringListFilesAlt(catstr, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_list__, verbose=False, returnfp=False):
6217 catfp = BytesIO(catstr);
6218 listcatfiles = ArchiveFileListFilesAlt(catstr, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6219 return listcatfiles;
6220
6221 create_alias_function("", __file_format_name__, "StringListFilesAlt", ArchiveFileListFilesAlt);
6222
6223 def TarFileListFiles(infile, verbose=False, returnfp=False):
6224 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6225 if(infile=="-"):
6226 infile = BytesIO();
6227 if(hasattr(sys.stdin, "buffer")):
6228 shutil.copyfileobj(sys.stdin.buffer, infile);
6229 else:
6230 shutil.copyfileobj(sys.stdin, infile);
6231 infile.seek(0, 0);
6232 if(not infile):
6233 return False;
6234 infile.seek(0, 0);
6235 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", infile)):
6236 infile = download_file_from_internet_file(infile);
6237 infile.seek(0, 0);
6238 if(not infile):
6239 return False;
6240 infile.seek(0, 0);
6241 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
6242 return False;
6243 elif(os.path.exists(infile) and os.path.isfile(infile)):
6244 try:
6245 if(not tarfile.is_tarfile(infile)):
6246 return False;
6247 except AttributeError:
6248 if(not is_tarfile(infile)):
6249 return False;
6250 else:
6251 return False;
6252 try:
6253 if(hasattr(infile, "read") or hasattr(infile, "write")):
6254 tarfp = tarfile.open(fileobj=infile, mode="r");
6255 else:
6256 tarfp = tarfile.open(infile, "r");
6257 except FileNotFoundError:
6258 return False;
6259 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
6260 returnval.update({lcfi: member.name});
6261 fpremode = member.mode;
6262 ffullmode = member.mode;
6263 flinkcount = 0;
6264 ftype = 0;
6265 if(member.isreg()):
6266 ffullmode = member.mode + stat.S_IFREG;
6267 ftype = 0;
6268 elif(member.isdev()):
6269 ffullmode = member.mode;
6270 ftype = 7;
6271 elif(member.islnk()):
6272 ffullmode = member.mode + stat.S_IFREG;
6273 ftype = 1;
6274 elif(member.issym()):
6275 ffullmode = member.mode + stat.S_IFLNK;
6276 ftype = 2;
6277 elif(member.ischr()):
6278 ffullmode = member.mode + stat.S_IFCHR;
6279 ftype = 3;
6280 elif(member.isblk()):
6281 ffullmode = member.mode + stat.S_IFBLK;
6282 ftype = 4;
6283 elif(member.isdir()):
6284 ffullmode = member.mode + stat.S_IFDIR;
6285 ftype = 5;
6286 elif(member.isfifo()):
6287 ffullmode = member.mode + stat.S_IFIFO;
6288 ftype = 6;
6289 elif(member.issparse()):
6290 ffullmode = member.mode;
6291 ftype = 12;
6292 if(not verbose):
6293 VerbosePrintOut(member.name);
6294 elif(verbose):
6295 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6296 printfname = member.name;
6297 if(member.islnk()):
6298 printfname = member.name + " link to " + member.linkname;
6299 elif(member.issym()):
6300 printfname = member.name + " -> " + member.linkname;
6301 fuprint = member.uname;
6302 if(len(fuprint)<=0):
6303 fuprint = member.uid;
6304 fgprint = member.gname;
6305 if(len(fgprint)<=0):
6306 fgprint = member.gid;
6307 VerbosePrintOut(PrintPermissionString(ffullmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(member.mtime).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6308 lcfi = lcfi + 1;
6309 if(returnfp):
6310 return listcatfiles['catfp'];
6311 else:
6312 return True;
6313
6314 def ZipFileListFiles(infile, verbose=False, returnfp=False):
6315 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6316 if(infile=="-"):
6317 infile = BytesIO();
6318 if(hasattr(sys.stdin, "buffer")):
6319 shutil.copyfileobj(sys.stdin.buffer, infile);
6320 else:
6321 shutil.copyfileobj(sys.stdin, infile);
6322 infile.seek(0, 0);
6323 if(not infile):
6324 return False;
6325 infile.seek(0, 0);
6326 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", infile)):
6327 infile = download_file_from_internet_file(infile);
6328 infile.seek(0, 0);
6329 if(not infile):
6330 return False;
6331 infile.seek(0, 0);
6332 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
6333 return False;
6334 else:
6335 return False;
6336 if(not zipfile.is_zipfile(infile)):
6337 return False;
6338 try:
6339 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
6340 except FileNotFoundError:
6341 return False;
6342 lcfi = 0;
6343 returnval = {};
6344 ziptest = zipfp.testzip();
6345 if(ziptest):
6346 VerbosePrintOut("Bad file found!");
6347 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
6348 if(zipinfo.create_system==0 or zipinfo.create_system==10):
6349 fwinattributes = int(zipinfo.external_attr);
6350 if(not member.is_dir()):
6351 fmode = int(stat.S_IFREG + 438);
6352 fchmode = int(stat.S_IMODE(fmode));
6353 ftypemod = int(stat.S_IFMT(fmode));
6354 elif(member.is_dir()):
6355 fmode = int(stat.S_IFDIR + 511);
6356 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6357 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6358 elif(zipinfo.create_system==3):
6359 fwinattributes =int(0);
6360 fmode = int(zipinfo.external_attr);
6361 fchmode = int(stat.S_IMODE(fmode));
6362 ftypemod = int(stat.S_IFMT(fmode));
6363 else:
6364 fwinattributes = int(0);
6365 if(not member.is_dir()):
6366 fmode = int(stat.S_IFREG + 438);
6367 fchmode = int(stat.S_IMODE(fmode));
6368 ftypemod = int(stat.S_IFMT(fmode));
6369 elif(member.is_dir()):
6370 fmode = int(stat.S_IFDIR + 511);
6371 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6372 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6373 returnval.update({lcfi: member.filename});
6374 if(not verbose):
6375 VerbosePrintOut(member.filename);
6376 if(verbose):
6377 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6378 permissionstr = "";
6379 for fmodval in str(oct(fmode))[-3:]:
6380 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
6381 if(not member.is_dir()):
6382 ftype = 0;
6383 permissionstr = "-" + permissionstr;
6384 elif(member.is_dir()):
6385 ftype = 5;
6386 permissionstr = "d" + permissionstr;
6387 printfname = member.filename;
6388 try:
6389 fuid = int(os.getuid());
6390 except AttributeError:
6391 fuid = int(0);
6392 except KeyError:
6393 fuid = int(0);
6394 try:
6395 fgid = int(os.getgid());
6396 except AttributeError:
6397 fgid = int(0);
6398 except KeyError:
6399 fgid = int(0);
6400 try:
6401 import pwd;
6402 try:
6403 userinfo = pwd.getpwuid(os.getuid());
6404 funame = userinfo.pw_name;
6405 except KeyError:
6406 funame = "";
6407 except AttributeError:
6408 funame = "";
6409 except ImportError:
6410 funame = "";
6411 fgname = "";
6412 try:
6413 import grp;
6414 try:
6415 groupinfo = grp.getgrgid(os.getgid());
6416 fgname = groupinfo.gr_name;
6417 except KeyError:
6418 fgname = "";
6419 except AttributeError:
6420 fgname = "";
6421 except ImportError:
6422 fgname = "";
6423 fuprint = funame;
6424 if(len(fuprint)<=0):
6425 fuprint = str(fuid);
6426 fgprint = fgname;
6427 if(len(fgprint)<=0):
6428 fgprint = str(fgid);
6429 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(int(time.mktime(member.date_time + (0, 0, -1)))).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6430 lcfi = lcfi + 1;
6431 if(returnfp):
6432 return listcatfiles['catfp'];
6433 else:
6434 return True;
6435
6436 if(not rarfile_support):
6437 def RarFileListFiles(infile, verbose=False, returnfp=False):
6438 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6439 if(not os.path.exists(infile) or not os.path.isfile(infile)):
6440 return False;
6441
6442 if(rarfile_support):
6443 def RarFileListFiles(infile, verbose=False, returnfp=False):
6444 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6445 if(not os.path.exists(infile) or not os.path.isfile(infile)):
6446 return False;
6447 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
6448 return False;
6449 lcfi = 0;
6450 returnval = {};
6451 rarfp = rarfile.RarFile(infile, "r");
6452 rartest = rarfp.testrar();
6453 if(rartest):
6454 VerbosePrintOut("Bad file found!");
6455 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
6456 is_unix = False;
6457 is_windows = False;
6458 if(member.host_os == rarfile.RAR_OS_UNIX):
6459 is_windows = False;
6460 try:
6461 member.external_attr
6462 is_unix = True;
6463 except AttributeError:
6464 is_unix = False;
6465 elif(member.host_os == rarfile.RAR_OS_WIN32):
6466 is_unix = False;
6467 try:
6468 member.external_attr
6469 is_windows = True;
6470 except AttributeError:
6471 is_windows = False;
6472 else:
6473 is_unix = False;
6474 is_windows = False;
6475 if(is_unix and member.external_attr !=0):
6476 fpremode = int(member.external_attr);
6477 elif(member.is_file()):
6478 fpremode = int(stat.S_IFREG + 438);
6479 elif(member.is_symlink()):
6480 fpremode = int(stat.S_IFLNK + 438);
6481 elif(member.is_dir()):
6482 fpremode = int(stat.S_IFDIR + 511);
6483 if(is_windows and member.external_attr !=0):
6484 fwinattributes = int(member.external_attr);
6485 else:
6486 fwinattributes = int(0);
6487 if(is_unix and member.external_attr !=0):
6488 fmode = int(member.external_attr);
6489 fchmode = int(stat.S_IMODE(member.external_attr));
6490 ftypemod = int(stat.S_IFMT(member.external_attr));
6491 elif(member.is_file()):
6492 fmode = int(stat.S_IFREG + 438);
6493 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
6494 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
6495 elif(member.is_symlink()):
6496 fmode = int(stat.S_IFLNK + 438);
6497 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
6498 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
6499 elif(member.is_dir()):
6500 fmode = int(stat.S_IFDIR + 511);
6501 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6502 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6503 returnval.update({lcfi: member.filename});
6504 if(not verbose):
6505 VerbosePrintOut(member.filename);
6506 if(verbose):
6507 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6508 permissionstr = "";
6509 for fmodval in str(oct(fmode))[-3:]:
6510 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
6511 if(member.is_file()):
6512 ftype = 0;
6513 permissionstr = "-" + permissionstr;
6514 printfname = member.filename;
6515 elif(member.is_symlink()):
6516 ftype = 2;
6517 permissionstr = "l" + permissionstr;
6518 printfname = member.name + " -> " + member.read().decode("UTF-8");
6519 elif(member.is_dir()):
6520 ftype = 5;
6521 permissionstr = "d" + permissionstr;
6522 printfname = member.filename;
6523 try:
6524 fuid = int(os.getuid());
6525 except AttributeError:
6526 fuid = int(0);
6527 except KeyError:
6528 fuid = int(0);
6529 try:
6530 fgid = int(os.getgid());
6531 except AttributeError:
6532 fgid = int(0);
6533 except KeyError:
6534 fgid = int(0);
6535 try:
6536 import pwd;
6537 try:
6538 userinfo = pwd.getpwuid(os.getuid());
6539 funame = userinfo.pw_name;
6540 except KeyError:
6541 funame = "";
6542 except AttributeError:
6543 funame = "";
6544 except ImportError:
6545 funame = "";
6546 fgname = "";
6547 try:
6548 import grp;
6549 try:
6550 groupinfo = grp.getgrgid(os.getgid());
6551 fgname = groupinfo.gr_name;
6552 except KeyError:
6553 fgname = "";
6554 except AttributeError:
6555 fgname = "";
6556 except ImportError:
6557 fgname = "";
6558 fuprint = funame;
6559 if(len(fuprint)<=0):
6560 fuprint = str(fuid);
6561 fgprint = fgname;
6562 if(len(fgprint)<=0):
6563 fgprint = str(fgid);
6564 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + member.mtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
6565 lcfi = lcfi + 1;
6566 if(returnfp):
6567 return listcatfiles['catfp'];
6568 else:
6569 return True;
6570
6571 def ListDirListFiles(infiles, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False, returnfp=False):
6572 outarray = BytesIO();
6573 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compressionlevel, followlink, checksumtype, formatspecs, False, True);
6574 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6575 return listcatfiles;
6576
6577 def ListDirListFilesAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=True, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False, returnfp=False):
6578 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, listonly, checksumtype, formatspecs, verbose);
6579 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6580 return listcatfiles;
6581
6582 def PackArchiveFileFromListDirAlt(infiles, outfile, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
6583 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, False, checksumtype, extradata, formatspecs, False);
6584 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compressionlevel, followlink, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6585 return listcatfiles;
6586
6587 create_alias_function("Pack", __file_format_name__, "FromListDirAlt", PackArchiveFileFromListDirAlt);
6588
6589 def PackArchiveFileFromTarFileAlt(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
6590 outarray = TarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
6591 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
6592 return listcatfiles;
6593
6594 create_alias_function("Pack", __file_format_name__, "FromTarFileAlt", PackArchiveFileFromTarFileAlt);
6595
6596 def PackArchiveFileFromZipFileAlt(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
6597 outarray = ZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
6598 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
6599 return listcatfiles;
6600
6601 create_alias_function("Pack", __file_format_name__, "FromZipFileAlt", PackArchiveFileFromZipFileAlt);
6602
6603 if(not rarfile_support):
6604 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
6605 return False;
6606
6607 if(rarfile_support):
6608 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
6609 outarray = RarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
6610 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
6611 return listcatfiles;
6612
6613 create_alias_function("Pack", __file_format_name__, "FromRarFileAlt", PackArchiveFileFromRarFileAlt);
6614
6615 def download_file_from_ftp_file(url):
6616 urlparts = urlparse(url);
6617 file_name = os.path.basename(urlparts.path);
6618 file_dir = os.path.dirname(urlparts.path);
6619 if(urlparts.username is not None):
6620 ftp_username = urlparts.username;
6621 else:
6622 ftp_username = "anonymous";
6623 if(urlparts.password is not None):
6624 ftp_password = urlparts.password;
6625 elif(urlparts.password is None and urlparts.username=="anonymous"):
6626 ftp_password = "anonymous";
6627 else:
6628 ftp_password = "";
6629 if(urlparts.scheme=="ftp"):
6630 ftp = FTP();
6631 elif(urlparts.scheme=="ftps"):
6632 ftp = FTP_TLS();
6633 else:
6634 return False;
6635 if(urlparts.scheme=="sftp"):
6636 if(__use_pysftp__):
6637 return download_file_from_pysftp_file(url);
6638 else:
6639 return download_file_from_sftp_file(url);
6640 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
6641 return download_file_from_http_file(url);
6642 ftp_port = urlparts.port;
6643 if(urlparts.port is None):
6644 ftp_port = 21;
6645 try:
6646 ftp.connect(urlparts.hostname, ftp_port);
6647 except socket.gaierror:
6648 log.info("Error With URL "+url);
6649 return False;
6650 except socket.timeout:
6651 log.info("Error With URL "+url);
6652 return False;
6653 ftp.login(urlparts.username, urlparts.password);
6654 if(urlparts.scheme=="ftps"):
6655 ftp.prot_p();
6656 ftpfile = BytesIO();
6657 ftp.retrbinary("RETR "+urlparts.path, ftpfile.write);
6658 #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
6659 ftp.close();
6660 ftpfile.seek(0, 0);
6661 return ftpfile;
6662
6663 def download_file_from_ftp_string(url):
6664 ftpfile = download_file_from_ftp_file(url);
6665 return ftpfile.read();
6666
6667 def upload_file_to_ftp_file(ftpfile, url):
6668 urlparts = urlparse(url);
6669 file_name = os.path.basename(urlparts.path);
6670 file_dir = os.path.dirname(urlparts.path);
6671 if(urlparts.username is not None):
6672 ftp_username = urlparts.username;
6673 else:
6674 ftp_username = "anonymous";
6675 if(urlparts.password is not None):
6676 ftp_password = urlparts.password;
6677 elif(urlparts.password is None and urlparts.username=="anonymous"):
6678 ftp_password = "anonymous";
6679 else:
6680 ftp_password = "";
6681 if(urlparts.scheme=="ftp"):
6682 ftp = FTP();
6683 elif(urlparts.scheme=="ftps"):
6684 ftp = FTP_TLS();
6685 else:
6686 return False;
6687 if(urlparts.scheme=="sftp"):
6688 if(__use_pysftp__):
6689 return upload_file_from_pysftp_file(url);
6690 else:
6691 return upload_file_from_sftp_file(url);
6692 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
6693 return False;
6694 ftp_port = urlparts.port;
6695 if(urlparts.port is None):
6696 ftp_port = 21;
6697 try:
6698 ftp.connect(urlparts.hostname, ftp_port);
6699 except socket.gaierror:
6700 log.info("Error With URL "+url);
6701 return False;
6702 except socket.timeout:
6703 log.info("Error With URL "+url);
6704 return False;
6705 ftp.login(urlparts.username, urlparts.password);
6706 if(urlparts.scheme=="ftps"):
6707 ftp.prot_p();
6708 ftp.storbinary("STOR "+urlparts.path, ftpfile);
6709 ftp.close();
6710 ftpfile.seek(0, 0);
6711 return ftpfile;
6712
6713 def upload_file_to_ftp_string(ftpstring, url):
6714 ftpfileo = BytesIO(ftpstring);
6715 ftpfile = upload_file_to_ftp_file(ftpfileo, url);
6716 ftpfileo.close();
6717 return ftpfile;
6718
6719 def download_file_from_http_file(url, headers=geturls_headers_pycatfile_python_alt):
6720 # Parse the URL to extract username and password if present
6721 urlparts = urlparse(url);
6722 username = urlparts.username;
6723 password = urlparts.password;
6724 # Rebuild the URL without the username and password
6725 netloc = urlparts.hostname;
6726 if(urlparts.scheme=="sftp"):
6727 if(__use_pysftp__):
6728 return download_file_from_pysftp_file(url);
6729 else:
6730 return download_file_from_sftp_file(url);
6731 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
6732 return download_file_from_ftp_file(url);
6733 if urlparts.port:
6734 netloc += ':' + str(urlparts.port);
6735 rebuilt_url = urlunparse((urlparts.scheme, netloc, urlparts.path, urlparts.params, urlparts.query, urlparts.fragment));
6736 # Create a temporary file object
6737 httpfile = BytesIO();
6738 if haverequests:
6739 # Use the requests library if available
6740 if username and password:
6741 response = requests.get(rebuilt_url, headers=headers, auth=(username, password), stream=True);
6742 else:
6743 response = requests.get(rebuilt_url, headers=headers, stream=True);
6744 response.raw.decode_content = True
6745 shutil.copyfileobj(response.raw, httpfile);
6746 else:
6747 # Build a Request object for urllib
6748 request = Request(rebuilt_url, headers=headers);
6749 # Create an opener object for handling URLs
6750 if username and password:
6751 # Create a password manager
6752 password_mgr = HTTPPasswordMgrWithDefaultRealm();
6753 # Add the username and password
6754 password_mgr.add_password(None, rebuilt_url, username, password);
6755 # Create an authentication handler using the password manager
6756 auth_handler = HTTPBasicAuthHandler(password_mgr);
6757 # Build the opener with the authentication handler
6758 opener = build_opener(auth_handler);
6759 else:
6760 opener = build_opener();
6761 with opener.open(request) as response:
6762 shutil.copyfileobj(response, httpfile);
6763 # Reset file pointer to the start
6764 httpfile.seek(0, 0);
6765 # Return the temporary file object
6766 return httpfile;
6767
6768 def download_file_from_http_string(url, headers=geturls_headers_pycatfile_python_alt):
6769 httpfile = download_file_from_http_file(url, headers);
6770 return ftpfile.read();
6771
6772 if(haveparamiko):
6773 def download_file_from_sftp_file(url):
6774 urlparts = urlparse(url);
6775 file_name = os.path.basename(urlparts.path);
6776 file_dir = os.path.dirname(urlparts.path);
6777 sftp_port = urlparts.port;
6778 if(urlparts.port is None):
6779 sftp_port = 22;
6780 else:
6781 sftp_port = urlparts.port;
6782 if(urlparts.username is not None):
6783 sftp_username = urlparts.username;
6784 else:
6785 sftp_username = "anonymous";
6786 if(urlparts.password is not None):
6787 sftp_password = urlparts.password;
6788 elif(urlparts.password is None and urlparts.username=="anonymous"):
6789 sftp_password = "anonymous";
6790 else:
6791 sftp_password = "";
6792 if(urlparts.scheme=="ftp"):
6793 return download_file_from_ftp_file(url);
6794 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
6795 return download_file_from_http_file(url);
6796 if(urlparts.scheme!="sftp"):
6797 return False;
6798 ssh = paramiko.SSHClient();
6799 ssh.load_system_host_keys();
6800 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
6801 try:
6802 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
6803 except paramiko.ssh_exception.SSHException:
6804 return False;
6805 except socket.gaierror:
6806 log.info("Error With URL "+url);
6807 return False;
6808 except socket.timeout:
6809 log.info("Error With URL "+url);
6810 return False;
6811 sftp = ssh.open_sftp();
6812 sftpfile = BytesIO();
6813 sftp.getfo(urlparts.path, sftpfile);
6814 sftp.close();
6815 ssh.close();
6816 sftpfile.seek(0, 0);
6817 return sftpfile;
6818 else:
6819 def download_file_from_sftp_file(url):
6820 return False;
6821
6822 if(haveparamiko):
6823 def download_file_from_sftp_string(url):
6824 sftpfile = download_file_from_sftp_file(url);
6825 return sftpfile.read();
6826 else:
6827 def download_file_from_ftp_string(url):
6828 return False;
6829
6830 if(haveparamiko):
6831 def upload_file_to_sftp_file(sftpfile, url):
6832 urlparts = urlparse(url);
6833 file_name = os.path.basename(urlparts.path);
6834 file_dir = os.path.dirname(urlparts.path);
6835 sftp_port = urlparts.port;
6836 if(urlparts.port is None):
6837 sftp_port = 22;
6838 else:
6839 sftp_port = urlparts.port;
6840 if(urlparts.username is not None):
6841 sftp_username = urlparts.username;
6842 else:
6843 sftp_username = "anonymous";
6844 if(urlparts.password is not None):
6845 sftp_password = urlparts.password;
6846 elif(urlparts.password is None and urlparts.username=="anonymous"):
6847 sftp_password = "anonymous";
6848 else:
6849 sftp_password = "";
6850 if(urlparts.scheme=="ftp"):
6851 return upload_file_from_ftp_file(url);
6852 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
6853 return False;
6854 if(urlparts.scheme!="sftp"):
6855 return False;
6856 ssh = paramiko.SSHClient();
6857 ssh.load_system_host_keys();
6858 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
6859 try:
6860 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
6861 except paramiko.ssh_exception.SSHException:
6862 return False;
6863 except socket.gaierror:
6864 log.info("Error With URL "+url);
6865 return False;
6866 except socket.timeout:
6867 log.info("Error With URL "+url);
6868 return False;
6869 sftp = ssh.open_sftp();
6870 sftp.putfo(sftpfile, urlparts.path);
6871 sftp.close();
6872 ssh.close();
6873 sftpfile.seek(0, 0);
6874 return sftpfile;
6875 else:
6876 def upload_file_to_sftp_file(sftpfile, url):
6877 return False;
6878
6879 if(haveparamiko):
6880 def upload_file_to_sftp_string(sftpstring, url):
6881 sftpfileo = BytesIO(sftpstring);
6882 sftpfile = upload_file_to_sftp_files(ftpfileo, url);
6883 sftpfileo.close();
6884 return sftpfile;
6885 else:
6886 def upload_file_to_sftp_string(url):
6887 return False;
6888
6889 if(havepysftp):
6890 def download_file_from_pysftp_file(url):
6891 urlparts = urlparse(url);
6892 file_name = os.path.basename(urlparts.path);
6893 file_dir = os.path.dirname(urlparts.path);
6894 sftp_port = urlparts.port;
6895 if(urlparts.port is None):
6896 sftp_port = 22;
6897 else:
6898 sftp_port = urlparts.port;
6899 if(urlparts.username is not None):
6900 sftp_username = urlparts.username;
6901 else:
6902 sftp_username = "anonymous";
6903 if(urlparts.password is not None):
6904 sftp_password = urlparts.password;
6905 elif(urlparts.password is None and urlparts.username=="anonymous"):
6906 sftp_password = "anonymous";
6907 else:
6908 sftp_password = "";
6909 if(urlparts.scheme=="ftp"):
6910 return download_file_from_ftp_file(url);
6911 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
6912 return download_file_from_http_file(url);
6913 if(urlparts.scheme!="sftp"):
6914 return False;
6915 try:
6916 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
6917 except paramiko.ssh_exception.SSHException:
6918 return False;
6919 except socket.gaierror:
6920 log.info("Error With URL "+url);
6921 return False;
6922 except socket.timeout:
6923 log.info("Error With URL "+url);
6924 return False;
6925 sftp = ssh.open_sftp();
6926 sftpfile = BytesIO();
6927 sftp.getfo(urlparts.path, sftpfile);
6928 sftp.close();
6929 ssh.close();
6930 sftpfile.seek(0, 0);
6931 return sftpfile;
6932 else:
6933 def download_file_from_pysftp_file(url):
6934 return False;
6935
6936 if(havepysftp):
6937 def download_file_from_pysftp_string(url):
6938 sftpfile = download_file_from_pysftp_file(url);
6939 return sftpfile.read();
6940 else:
6941 def download_file_from_ftp_string(url):
6942 return False;
6943
6944 if(havepysftp):
6945 def upload_file_to_pysftp_file(sftpfile, url):
6946 urlparts = urlparse(url);
6947 file_name = os.path.basename(urlparts.path);
6948 file_dir = os.path.dirname(urlparts.path);
6949 sftp_port = urlparts.port;
6950 if(urlparts.port is None):
6951 sftp_port = 22;
6952 else:
6953 sftp_port = urlparts.port;
6954 if(urlparts.username is not None):
6955 sftp_username = urlparts.username;
6956 else:
6957 sftp_username = "anonymous";
6958 if(urlparts.password is not None):
6959 sftp_password = urlparts.password;
6960 elif(urlparts.password is None and urlparts.username=="anonymous"):
6961 sftp_password = "anonymous";
6962 else:
6963 sftp_password = "";
6964 if(urlparts.scheme=="ftp"):
6965 return upload_file_from_ftp_file(url);
6966 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
6967 return False;
6968 if(urlparts.scheme!="sftp"):
6969 return False;
6970 try:
6971 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
6972 except paramiko.ssh_exception.SSHException:
6973 return False;
6974 except socket.gaierror:
6975 log.info("Error With URL "+url);
6976 return False;
6977 except socket.timeout:
6978 log.info("Error With URL "+url);
6979 return False;
6980 sftp = ssh.open_sftp();
6981 sftp.putfo(sftpfile, urlparts.path);
6982 sftp.close();
6983 ssh.close();
6984 sftpfile.seek(0, 0);
6985 return sftpfile;
6986 else:
6987 def upload_file_to_pysftp_file(sftpfile, url):
6988 return False;
6989
6990 if(havepysftp):
6991 def upload_file_to_pysftp_string(sftpstring, url):
6992 sftpfileo = BytesIO(sftpstring);
6993 sftpfile = upload_file_to_pysftp_files(ftpfileo, url);
6994 sftpfileo.close();
6995 return sftpfile;
6996 else:
6997 def upload_file_to_pysftp_string(url):
6998 return False;
6999
7000 def download_file_from_internet_file(url, headers=geturls_headers_pycatfile_python_alt):
7001 urlparts = urlparse(url);
7002 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7003 return download_file_from_http_file(url, headers);
7004 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7005 return download_file_from_ftp_file(url);
7006 elif(urlparts.scheme=="sftp"):
7007 if(__use_pysftp__ and havepysftp):
7008 return download_file_from_pysftp_file(url);
7009 else:
7010 return download_file_from_sftp_file(url);
7011 else:
7012 return False;
7013 return False;
7014
7015 def download_file_from_internet_string(url, headers=geturls_headers_pycatfile_python_alt):
7016 urlparts = urlparse(url);
7017 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7018 return download_file_from_http_string(url, headers);
7019 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7020 return download_file_from_ftp_string(url);
7021 elif(urlparts.scheme=="sftp"):
7022 if(__use_pysftp__ and havepysftp):
7023 return download_file_from_pysftp_string(url);
7024 else:
7025 return download_file_from_sftp_string(url);
7026 else:
7027 return False;
7028 return False;
7029
7030 def upload_file_to_internet_file(ifp, url):
7031 urlparts = urlparse(url);
7032 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7033 return False;
7034 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7035 return upload_file_from_ftp_file(ifp, url);
7036 elif(urlparts.scheme=="sftp"):
7037 if(__use_pysftp__ and havepysftp):
7038 return upload_file_from_pysftp_file(ifp, url);
7039 else:
7040 return download_file_from_sftp_file(ifp, url);
7041 else:
7042 return False;
7043 return False;
7044
7045 def upload_file_to_internet_string(ifp, url):
7046 urlparts = urlparse(url);
7047 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7048 return False;
7049 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7050 return upload_file_from_ftp_string(ifp, url);
7051 elif(urlparts.scheme=="sftp"):
7052 if(__use_pysftp__ and havepysftp):
7053 return upload_file_from_pysftp_string(ifp, url);
7054 else:
7055 return download_file_from_sftp_string(ifp, url);
7056 else:
7057 return False;
7058 return False;
7059
7060 try:
7061 if(hasattr(shutil, "register_archive_format")):
7062 # Register the packing format
7063 shutil.register_archive_format(__file_format_name__, PackArchiveFileFunc, description='Pack concatenated files');
7064 except shutil.RegistryError:
7065 pass;
7066
7067 try:
7068 if(hasattr(shutil, "register_unpack_format")):
7069 # Register the unpacking format
7070 shutil.register_unpack_format(__file_format_name__, archivefile_extensions, UnPackArchiveFileFunc, description='UnPack concatenated files');
7071 except shutil.RegistryError:
7072 pass;
A tar like file format name catfile after unix cat command (concatenate files).