?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
Add files via upload
[PyCatFile.git] / pycatfile.py
blob39c939d29eee653e7b3e69f146eba47e34283ce9
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 '''
5 This program is free software; you can redistribute it and/or modify
6 it under the terms of the Revised BSD License.
7
8 This program is distributed in the hope that it will be useful,
9 but WITHOUT ANY WARRANTY; without even the implied warranty of
10 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
11 Revised BSD License for more details.
12
13 Copyright 2018-2024 Cool Dude 2k - http://idb.berlios.de/
14 Copyright 2018-2024 Game Maker 2k - http://intdb.sourceforge.net/
15 Copyright 2018-2024 Kazuki Przyborowski - https://github.com/KazukiPrzyborowski
16
17 $FileInfo: pycatfile.py - Last Update: 5/3/2024 Ver. 0.10.0 RC 1 - Author: cooldude2k $
18 '''
19
20 from __future__ import absolute_import, division, print_function, unicode_literals;
21 import io, os, re, sys, time, stat, zlib, base64, shutil, socket, hashlib, datetime, logging, binascii, tempfile, zipfile, platform;
22 from ftplib import FTP, FTP_TLS;
23 if(sys.version[0]=="2"):
24 from urlparse import urlparse, urlunparse;
25 elif(sys.version[0]>="3"):
26 from urllib.parse import urlunparse;
27 from urllib.parse import urlparse;
28
29 if os.name == 'nt': # Only modify if on Windows
30 if sys.version[0] == "2":
31 import codecs;
32 sys.stdout = codecs.getwriter('utf-8')(sys.stdout);
33 sys.stderr = codecs.getwriter('utf-8')(sys.stderr);
34 else:
35 import io;
36 sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace', line_buffering=True);
37 sys.stderr = io.TextIOWrapper(sys.stderr.buffer, encoding='utf-8', errors='replace', line_buffering=True);
38
39 hashlib_guaranteed = False;
40 os.environ["PYTHONIOENCODING"] = "UTF-8";
41 os.environ["LC_CTYPE"] = "UTF-8";
42 try:
43 reload(sys);
44 try:
45 sys.setdefaultencoding('UTF-8');
46 except NameError:
47 pass;
48 except AttributeError:
49 pass;
50 except NameError:
51 pass;
52 except AttributeError:
53 pass;
54
55 try:
56 from zlib import crc32;
57 except ImportError:
58 from binascii import crc32;
59
60 if(sys.version_info[0]==2):
61 FileNotFoundError = IOError;
62
63 rarfile_support = False;
64 try:
65 import rarfile;
66 rarfile_support = True;
67 except ImportError:
68 rarfile_support = False;
69
70 py7zr_support = False;
71 try:
72 import py7zr;
73 py7zr_support = True;
74 except ImportError:
75 py7zr_support = False;
76
77 try:
78 from xtarfile import is_tarfile;
79 except ImportError:
80 try:
81 from safetar import is_tarfile;
82 except ImportError:
83 from tarfile import is_tarfile;
84
85 try:
86 import xtarfile as tarfile;
87 except ImportError:
88 try:
89 import safetar as tarfile;
90 except ImportError:
91 import tarfile;
92
93 haveparamiko = False;
94 try:
95 import paramiko;
96 haveparamiko = True;
97 except ImportError:
98 haveparamiko = False;
99
100 havepysftp = False;
101 try:
102 import pysftp;
103 havepysftp = True;
104 except ImportError:
105 havepysftp = False;
106
107 haverequests = False;
108 try:
109 import requests;
110 haverequests = True;
111 except ImportError:
112 haverequests = False;
113
114 if(haverequests):
115 import urllib3;
116 logging.getLogger("urllib3").setLevel(logging.WARNING);
117
118 try:
119 # Python 3 imports
120 from urllib.request import Request, build_opener, HTTPBasicAuthHandler;
121 from urllib.parse import urlparse;
122 except ImportError:
123 # Python 2 imports
124 from urllib2 import Request, build_opener, HTTPBasicAuthHandler;
125 from urlparse import urlparse;
126
127 if(sys.version[0]=="2"):
128 try:
129 from io import StringIO, BytesIO;
130 except ImportError:
131 try:
132 from cStringIO import StringIO;
133 from cStringIO import StringIO as BytesIO;
134 except ImportError:
135 from StringIO import StringIO;
136 from StringIO import StringIO as BytesIO;
137 elif(sys.version[0]>="3"):
138 from io import StringIO, BytesIO;
139 else:
140 teststringio = 0;
141 if(teststringio<=0):
142 try:
143 from cStringIO import StringIO as BytesIO;
144 teststringio = 1;
145 except ImportError:
146 teststringio = 0;
147 if(teststringio<=0):
148 try:
149 from StringIO import StringIO as BytesIO;
150 teststringio = 2;
151 except ImportError:
152 teststringio = 0;
153 if(teststringio<=0):
154 try:
155 from io import BytesIO;
156 teststringio = 3;
157 except ImportError:
158 teststringio = 0;
159
160 __use_pysftp__ = False;
161 if(not havepysftp):
162 __use_pysftp__ = False;
163 __file_format_name__ = "CatFile";
164 __program_name__ = "Py"+__file_format_name__;
165 __file_format_lower__ = __file_format_name__.lower();
166 __file_format_magic__ = __file_format_name__;
167 __file_format_len__ = len(__file_format_magic__);
168 __file_format_hex__ = binascii.hexlify(__file_format_magic__.encode("UTF-8")).decode("UTF-8");
169 __file_format_delimiter__ = "\x00";
170 __file_format_ver__ = "001";
171 __use_new_style__ = True;
172 __use_advanced_list__ = True;
173 __use_alt_inode__ = False;
174 __file_format_list__ = [__file_format_name__, __file_format_magic__, __file_format_lower__, __file_format_len__, __file_format_hex__, __file_format_delimiter__, __file_format_ver__, __use_new_style__, __use_advanced_list__, __use_alt_inode__];
175 __project__ = __program_name__;
176 __project_url__ = "https://github.com/GameMaker2k/PyCatFile";
177 __version_info__ = (0, 10, 0, "RC 1", 1);
178 __version_date_info__ = (2024, 5, 3, "RC 1", 1);
179 __version_date__ = str(__version_date_info__[0]) + "." + str(__version_date_info__[1]).zfill(2) + "." + str(__version_date_info__[2]).zfill(2);
180 __revision__ = __version_info__[3];
181 __revision_id__ = "$Id$";
182 if(__version_info__[4] is not None):
183 __version_date_plusrc__ = __version_date__ + "-" + str(__version_date_info__[4]);
184 if(__version_info__[4] is None):
185 __version_date_plusrc__ = __version_date__;
186 if(__version_info__[3] is not None):
187 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]) + " " + str(__version_info__[3]);
188 if(__version_info__[3] is None):
189 __version__ = str(__version_info__[0]) + "." + str(__version_info__[1]) + "." + str(__version_info__[2]);
190
191 PyBitness = platform.architecture();
192 if(PyBitness=="32bit" or PyBitness=="32"):
193 PyBitness = "32";
194 elif(PyBitness=="64bit" or PyBitness=="64"):
195 PyBitness = "64";
196 else:
197 PyBitness = "32";
198
199 geturls_ua_pycatfile_python = "Mozilla/5.0 (compatible; {proname}/{prover}; +{prourl})".format(proname=__project__, prover=__version__, prourl=__project_url__);
200 if(platform.python_implementation()!=""):
201 py_implementation = platform.python_implementation();
202 if(platform.python_implementation()==""):
203 py_implementation = "Python";
204 geturls_ua_pycatfile_python_alt = "Mozilla/5.0 ({osver}; {archtype}; +{prourl}) {pyimp}/{pyver} (KHTML, like Gecko) {proname}/{prover}".format(osver=platform.system()+" "+platform.release(), archtype=platform.machine(), prourl=__project_url__, pyimp=py_implementation, pyver=platform.python_version(), proname=__project__, prover=__version__);
205 geturls_ua_googlebot_google = "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)";
206 geturls_ua_googlebot_google_old = "Googlebot/2.1 (+http://www.google.com/bot.html)";
207 geturls_headers_pycatfile_python = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
208 geturls_headers_pycatfile_python_alt = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_pycatfile_python_alt, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close", 'SEC-CH-UA': "\""+__project__+"\";v=\""+str(__version__)+"\", \"Not;A=Brand\";v=\"8\", \""+py_implementation+"\";v=\""+str(platform.release())+"\"", 'SEC-CH-UA-FULL-VERSION': str(__version__), 'SEC-CH-UA-PLATFORM': ""+py_implementation+"", 'SEC-CH-UA-ARCH': ""+platform.machine()+"", 'SEC-CH-UA-PLATFORM': str(__version__), 'SEC-CH-UA-BITNESS': str(PyBitness)};
209 geturls_headers_googlebot_google = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
210 geturls_headers_googlebot_google_old = {'Referer': "http://google.com/", 'User-Agent': geturls_ua_googlebot_google_old, 'Accept-Encoding': "none", 'Accept-Language': "en-US,en;q=0.8,en-CA,en-GB;q=0.6", 'Accept-Charset': "ISO-8859-1,ISO-8859-15,utf-8;q=0.7,*;q=0.7", 'Accept': "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 'Connection': "close"};
211
212 compressionlist = ['auto', 'gzip', 'bzip2', 'zstd', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
213 outextlist = ['gz', 'bz2', 'zst', 'lz4', 'lzo', 'lzop', 'lzma', 'xz'];
214 outextlistwd = ['.gz', '.bz2', '.zst', '.lz4', '.lzo', '.lzop', '.lzma', '.xz'];
215
216 tarfile_mimetype = "application/tar";
217 tarfile_tar_mimetype = tarfile_mimetype;
218 zipfile_mimetype = "application/zip";
219 zipfile_zip_mimetype = zipfile_mimetype;
220 rarfile_mimetype = "application/rar";
221 rarfile_rar_mimetype = rarfile_mimetype;
222 archivefile_mimetype = "application/x-"+__file_format_list__[1]+"";
223 archivefile_cat_mimetype = archivefile_mimetype;
224 archivefile_gzip_mimetype = "application/x-"+__file_format_list__[1]+"+gzip";
225 archivefile_gz_mimetype = archivefile_gzip_mimetype;
226 archivefile_bzip2_mimetype = "application/x-"+__file_format_list__[1]+"+bzip2";
227 archivefile_bz2_mimetype = archivefile_bzip2_mimetype;
228 archivefile_lz4_mimetype = "application/x-"+__file_format_list__[1]+"+lz4";
229 archivefile_lzop_mimetype = "application/x-"+__file_format_list__[1]+"+lzop";
230 archivefile_lzo_mimetype = archivefile_lzop_mimetype;
231 archivefile_zstandard_mimetype = "application/x-"+__file_format_list__[1]+"+zstandard";
232 archivefile_zstd_mimetype = archivefile_zstandard_mimetype;
233 archivefile_lzma_mimetype = "application/x-"+__file_format_list__[1]+"+lzma";
234 archivefile_xz_mimetype = "application/x-"+__file_format_list__[1]+"+xz";
235 archivefile_extensions = ['.cat', '.cat.gz', '.cat.bz2', '.cat.zst', '.cat.lz4', '.cat.lzo', '.cat.lzop', '.cat.lzma', '.cat.xz'];
236
237 if __name__ == "__main__":
238 import subprocess;
239 curscrpath = os.path.dirname(sys.argv[0]);
240 if(curscrpath==""):
241 curscrpath = ".";
242 if(os.sep=="\\"):
243 curscrpath = curscrpath.replace(os.sep, "/");
244 curscrpath = curscrpath + "/";
245 scrfile = curscrpath + "catfile.py";
246 if(os.path.exists(scrfile) and os.path.isfile(scrfile)):
247 scrcmd = subprocess.Popen([sys.executable, scrfile] + sys.argv[1:]);
248 scrcmd.wait();
249
250 def VerbosePrintOut(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
251 if(not dbgenable):
252 return True;
253 log_functions = {
254 "print": print,
255 "log": logging.info,
256 "warning": logging.warning,
257 "error": logging.error,
258 "critical": logging.critical,
259 "exception": logging.exception,
260 "logalt": lambda x: logging.log(dgblevel, x),
261 "debug": logging.debug
262 };
263 log_function = log_functions.get(outtype);
264 if(log_function):
265 log_function(dbgtxt);
266 return True;
267 return False;
268
269 def VerbosePrintOutReturn(dbgtxt, outtype="log", dbgenable=True, dgblevel=20):
270 VerbosePrintOut(dbgtxt, outtype, dbgenable, dgblevel);
271 return dbgtxt;
272
273 def RemoveWindowsPath(dpath):
274 if(dpath is None):
275 dpath = "";
276 if(os.sep!="/"):
277 dpath = dpath.replace(os.path.sep, "/");
278 dpath = dpath.rstrip("/");
279 if(dpath=="." or dpath==".."):
280 dpath = dpath + "/";
281 return dpath;
282
283 def NormalizeRelativePath(inpath):
284 inpath = RemoveWindowsPath(inpath);
285 if(os.path.isabs(inpath)):
286 outpath = inpath;
287 else:
288 if(inpath.startswith("./") or inpath.startswith("../")):
289 outpath = inpath;
290 else:
291 outpath = "./" + inpath;
292 return outpath;
293
294 def PrependPath(base_dir, child_path):
295 # Check if base_dir is None or empty, if so, return child_path as is
296 if not base_dir:
297 return child_path;
298 # Ensure base_dir ends with exactly one slash
299 if not base_dir.endswith('/'):
300 base_dir += '/';
301 # Check if child_path starts with ./ or ../ (indicating a relative path)
302 if child_path.startswith('./') or child_path.startswith('../'):
303 # For relative paths, we don't alter the child_path
304 return base_dir + child_path;
305 else:
306 # For non-relative paths, ensure there's no starting slash on child_path to avoid double slashes
307 return base_dir + child_path.lstrip('/');
308
309 def ListDir(dirpath, followlink=False, duplicates=False):
310 if(isinstance(dirpath, (list, tuple, ))):
311 dirpath = list(filter(None, dirpath));
312 elif(isinstance(dirpath, (str, ))):
313 dirpath = list(filter(None, [dirpath]));
314 retlist = [];
315 for mydirfile in dirpath:
316 if(not os.path.exists(mydirfile)):
317 return False;
318 mydirfile = NormalizeRelativePath(mydirfile);
319 if(os.path.exists(mydirfile) and os.path.islink(mydirfile)):
320 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile));
321 if(os.path.exists(mydirfile) and os.path.isdir(mydirfile)):
322 for root, dirs, filenames in os.walk(mydirfile):
323 dpath = root;
324 dpath = RemoveWindowsPath(dpath);
325 if(dpath not in retlist and not duplicates):
326 retlist.append(dpath);
327 if(duplicates):
328 retlist.append(dpath);
329 for file in filenames:
330 fpath = os.path.join(root, file);
331 fpath = RemoveWindowsPath(fpath);
332 if(fpath not in retlist and not duplicates):
333 retlist.append(fpath);
334 if(duplicates):
335 retlist.append(fpath);
336 else:
337 retlist.append(RemoveWindowsPath(mydirfile));
338 return retlist;
339
340 def ListDirAdvanced(dirpath, followlink=False, duplicates=False):
341 if isinstance(dirpath, (list, tuple)):
342 dirpath = list(filter(None, dirpath));
343 elif isinstance(dirpath, str):
344 dirpath = list(filter(None, [dirpath]));
345 retlist = []
346 for mydirfile in dirpath:
347 if not os.path.exists(mydirfile):
348 return False;
349 mydirfile = NormalizeRelativePath(mydirfile);
350 if os.path.exists(mydirfile) and os.path.islink(mydirfile) and followlink:
351 mydirfile = RemoveWindowsPath(os.path.realpath(mydirfile))
352 if os.path.exists(mydirfile) and os.path.isdir(mydirfile):
353 for root, dirs, filenames in os.walk(mydirfile):
354 # Sort dirs and filenames alphabetically in place
355 dirs.sort(key=lambda x: x.lower());
356 filenames.sort(key=lambda x: x.lower());
357 dpath = RemoveWindowsPath(root);
358 if not duplicates and dpath not in retlist:
359 retlist.append(dpath);
360 elif duplicates:
361 retlist.append(dpath);
362 for file in filenames:
363 fpath = os.path.join(root, file);
364 fpath = RemoveWindowsPath(fpath);
365 if not duplicates and fpath not in retlist:
366 retlist.append(fpath);
367 elif duplicates:
368 retlist.append(fpath);
369 else:
370 retlist.append(RemoveWindowsPath(mydirfile));
371 return retlist;
372
373 def create_alias_function(prefix, base_name, suffix, target_function):
374 # Define a new function that wraps the target function
375 def alias_function(*args, **kwargs):
376 return target_function(*args, **kwargs);
377
378 # Create the function name by combining the prefix, base name, and the suffix
379 function_name = "{}{}{}".format(prefix, base_name, suffix);
380
381 # Add the new function to the global namespace
382 globals()[function_name] = alias_function;
383
384 def create_alias_function_alt(prefix, base_name, suffix, target_function):
385 # Create the function name by combining the prefix, base name, and the suffix
386 # Use the format method for string formatting, compatible with Python 2 and 3
387 function_name = "{}{}{}".format(prefix, base_name, suffix)
388 # Add the new function (alias of the target_function) to the global namespace
389 # This line is compatible as-is with both Python 2 and 3
390 globals()[function_name] = target_function
391
392 # initial_value can be 0xFFFF or 0x0000
393 def crc16_ansi(msg, initial_value=0xFFFF):
394 # CRC-16-IBM / CRC-16-ANSI polynomial and initial value
395 poly = 0x8005; # Polynomial for CRC-16-IBM / CRC-16-ANSI
396 crc = initial_value; # Initial value
397 for b in msg:
398 crc ^= b << 8; # XOR byte into CRC top byte
399 for _ in range(8): # Process each bit
400 if crc & 0x8000: # If the top bit is set
401 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
402 else:
403 crc = crc << 1; # Just shift left
404 crc &= 0xFFFF; # Ensure CRC remains 16-bit
405 return crc;
406
407 # initial_value can be 0xFFFF or 0x0000
408 def crc16_ibm(msg, initial_value=0xFFFF):
409 return crc16_ansi(msg, initial_value);
410
411 # initial_value is 0xFFFF
412 def crc16(msg):
413 return crc16_ansi(msg, 0xFFFF);
414
415 # initial_value can be 0xFFFF, 0x1D0F or 0x0000
416 def crc16_ccitt(msg, initial_value=0xFFFF):
417 # CRC-16-CCITT polynomial
418 poly = 0x1021; # Polynomial for CRC-16-CCITT
419 # Use the specified initial value
420 crc = initial_value;
421 for b in msg:
422 crc ^= b << 8; # XOR byte into CRC top byte
423 for _ in range(8): # Process each bit
424 if crc & 0x8000: # If the top bit is set
425 crc = (crc << 1) ^ poly; # Shift left and XOR with the polynomial
426 else:
427 crc = crc << 1; # Just shift left
428 crc &= 0xFFFF; # Ensure CRC remains 16-bit
429 return crc;
430
431 # initial_value can be 0x42F0E1EBA9EA3693 or 0x0000000000000000
432 def crc64_ecma(msg, initial_value=0x0000000000000000):
433 # CRC-64-ECMA polynomial and initial value
434 poly = 0x42F0E1EBA9EA3693;
435 crc = initial_value; # Initial value for CRC-64-ECMA
436 for b in msg:
437 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
438 for _ in range(8): # Process each bit
439 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
440 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
441 else:
442 crc <<= 1; # Just shift left if the MSB is 0
443 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
444 return crc;
445
446 # initial_value can be 0x000000000000001B or 0xFFFFFFFFFFFFFFFF
447 def crc64_iso(msg, initial_value=0xFFFFFFFFFFFFFFFF):
448 # CRC-64-ISO polynomial and initial value
449 poly = 0x000000000000001B;
450 crc = initial_value; # Common initial value for CRC-64-ISO
451 for b in msg:
452 crc ^= b << 56; # XOR byte into the most significant byte of the CRC
453 for _ in range(8): # Process each bit
454 if crc & (1 << 63): # Check if the leftmost (most significant) bit is set
455 crc = (crc << 1) ^ poly; # Shift left and XOR with poly if the MSB is 1
456 else:
457 crc <<= 1; # Just shift left if the MSB is 0
458 crc &= 0xFFFFFFFFFFFFFFFF; # Ensure CRC remains 64-bit
459 return crc;
460
461 def GetDataFromArray(data, path, default=None):
462 element = data;
463 try:
464 for key in path:
465 element = element[key];
466 return element;
467 except (KeyError, TypeError, IndexError):
468 return default;
469
470 def GetDataFromArrayAlt(structure, path, default=None):
471 element = structure;
472 for key in path:
473 if isinstance(element, dict) and key in element:
474 element = element[key];
475 elif isinstance(element, list) and isinstance(key, int) and -len(element) <= key < len(element):
476 element = element[key];
477 else:
478 return default;
479 return element;
480
481 def GetHeaderChecksum(inlist=[], checksumtype="crc32", encodedata=True, formatspecs=__file_format_list__):
482 if isinstance(inlist, list):
483 fileheader = AppendNullBytes(inlist, formatspecs[5]);
484 else:
485 fileheader = AppendNullByte(inlist, formatspecs[5]);
486 if(encodedata):
487 fileheader = fileheader.encode('UTF-8');
488 if(checksumtype=="none" or checksumtype==""):
489 catfileheadercshex = format(0, 'x').lower();
490 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
491 catfileheadercshex = format(crc16(fileheader) & 0xffff, '04x').lower();
492 elif(checksumtype=="crc16_ccitt"):
493 catfileheadercshex = format(crc16_ccitt(fileheader) & 0xffff, '04x').lower();
494 elif(checksumtype=="adler32"):
495 catfileheadercshex = format(zlib.adler32(fileheader) & 0xffffffff, '08x').lower();
496 elif(checksumtype=="crc32"):
497 catfileheadercshex = format(crc32(fileheader) & 0xffffffff, '08x').lower();
498 elif(checksumtype=="crc64_ecma"):
499 catfileheadercshex = format(crc64_ecma(fileheader) & 0xffffffffffffffff, '016x').lower();
500 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
501 catfileheadercshex = format(crc64_iso(fileheader) & 0xffffffffffffffff, '016x').lower();
502 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
503 checksumoutstr = hashlib.new(checksumtype);
504 checksumoutstr.update(fileheader);
505 catfileheadercshex = checksumoutstr.hexdigest().lower();
506 else:
507 catfileheadercshex = format(0, 'x').lower();
508 return catfileheadercshex;
509
510 def GetFileChecksum(instr, checksumtype="crc32", encodedata=True, formatspecs=__file_format_list__):
511 if(encodedata):
512 instr = instr.encode('UTF-8');
513 if(checksumtype=="none" or checksumtype==""):
514 catinstrcshex = format(0, 'x').lower();
515 elif(checksumtype=="crc16" or checksumtype=="crc16_ansi" or checksumtype=="crc16_ibm"):
516 catinstrcshex = format(crc16(instr) & 0xffff, '04x').lower();
517 elif(checksumtype=="crc16_ccitt"):
518 catinstrcshex = format(crc16_ccitt(instr) & 0xffff, '04x').lower();
519 elif(checksumtype=="adler32"):
520 catinstrcshex = format(zlib.adler32(instr) & 0xffffffff, '08x').lower();
521 elif(checksumtype=="crc32"):
522 catinstrcshex = format(crc32(instr) & 0xffffffff, '08x').lower();
523 elif(checksumtype=="crc64_ecma"):
524 catinstrcshex = format(crc64_ecma(instr) & 0xffffffffffffffff, '016x').lower();
525 elif(checksumtype=="crc64" or checksumtype=="crc64_iso"):
526 catinstrcshex = format(crc64_iso(instr) & 0xffffffffffffffff, '016x').lower();
527 elif(CheckSumSupportAlt(checksumtype, hashlib_guaranteed)):
528 checksumoutstr = hashlib.new(checksumtype);
529 checksumoutstr.update(instr);
530 catinstrcshex = checksumoutstr.hexdigest().lower();
531 else:
532 catinstrcshex = format(0, 'x').lower();
533 return catinstrcshex;
534
535 def ValidateHeaderChecksum(inlist=[], checksumtype="crc32", inchecksum="0", formatspecs=__file_format_list__):
536 catfileheadercshex = GetHeaderChecksum(inlist, checksumtype, True, formatspecs);
537 inchecksum = inchecksum.lower();
538 catfileheadercshex = catfileheadercshex.lower();
539 if(inchecksum==catfileheadercshex):
540 return True;
541 else:
542 return False;
543
544 def ValidateFileChecksum(infile, checksumtype="crc32", inchecksum="0", formatspecs=__file_format_list__):
545 catinfilecshex = GetFileChecksum(infile, checksumtype, True, formatspecs);
546 inchecksum = inchecksum.lower();
547 catinfilecshex = catinfilecshex.lower();
548 if(inchecksum==catinfilecshex):
549 return True;
550 else:
551 return False;
552
553 def ReadTillNullByte(fp, delimiter=__file_format_delimiter__):
554 curbyte = b"";
555 curfullbyte = b"";
556 nullbyte = delimiter.encode("UTF-8");
557 while(True):
558 curbyte = fp.read(1);
559 if(curbyte==nullbyte or not curbyte):
560 break;
561 curfullbyte = curfullbyte + curbyte;
562 return curfullbyte.decode('UTF-8');
563
564 def ReadUntilNullByte(fp, delimiter=__file_format_delimiter__):
565 return ReadTillNullByte(fp, delimiter);
566
567 def SeekToEndOfFile(fp):
568 lasttell = 0;
569 while(True):
570 fp.seek(1, 1);
571 if(lasttell==fp.tell()):
572 break;
573 lasttell = fp.tell();
574 return True;
575
576 def ReadFileHeaderData(fp, rounds=0, delimiter=__file_format_delimiter__):
577 rocount = 0;
578 roend = int(rounds);
579 HeaderOut = [];
580 while(rocount<roend):
581 HeaderOut.append(ReadTillNullByte(fp, delimiter));
582 rocount = rocount + 1;
583 return HeaderOut;
584
585 def ReadFileHeaderDataBySize(fp, delimiter=__file_format_delimiter__):
586 headerpresize = ReadTillNullByte(fp, delimiter);
587 headersize = int(headerpresize, 16);
588 if(headersize<=0):
589 return [];
590 headercontent = str(fp.read(headersize).decode('UTF-8')).split(delimiter);
591 fp.seek(1, 1);
592 rocount = 0;
593 roend = int(len(headercontent));
594 HeaderOut = [headerpresize];
595 while(rocount<roend):
596 HeaderOut.append(headercontent[rocount]);
597 rocount = rocount + 1;
598 return HeaderOut;
599
600 def ReadFileHeaderDataWoSize(fp, delimiter=__file_format_delimiter__):
601 preheaderdata = ReadFileHeaderData(fp, 2, delimiter);
602 headersize = int(preheaderdata[0], 16);
603 headernumfields = int(preheaderdata[1], 16);
604 if(headersize<=0 or headernumfields<=0):
605 return [];
606 headerdata = ReadFileHeaderData(fp, headernumfields, delimiter);
607 HeaderOut = preheaderdata + headerdata;
608 return HeaderOut;
609
610 def ReadFileHeaderDataBySizeWithContent(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
611 delimiter = formatspecs[5];
612 fheaderstart = fp.tell();
613 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
614 if(len(HeaderOut)==0):
615 return False;
616 if(re.findall("^[.|/]", HeaderOut[3])):
617 fname = HeaderOut[3];
618 else:
619 fname = "./"+HeaderOut[3];
620 fchecksumtype = HeaderOut[-3].lower();
621 fcs = HeaderOut[-2].lower();
622 fccs = HeaderOut[-1].lower();
623 fsize = int(HeaderOut[5], 16);
624 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
625 if(fcs!=newfcs and not skipchecksum):
626 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
627 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
628 return False;
629 fhend = fp.tell() - 1;
630 fcontentstart = fp.tell();
631 fcontents = BytesIO();
632 if(fsize>0 and not listonly):
633 fcontents.write(fp.read(fsize));
634 elif(fsize>0 and listonly):
635 fp.seek(fsize, 1);
636 fcontents.seek(0, 0);
637 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
638 if(fccs!=newfccs and not skipchecksum and not listonly):
639 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
640 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
641 return False;
642 fp.seek(1, 1);
643 fcontentend = fp.tell() - 1;
644 HeaderOut.append(fcontents);
645 return HeaderOut;
646
647 def ReadFileHeaderDataBySizeWithContentToArray(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
648 delimiter = formatspecs[5];
649 fheaderstart = fp.tell();
650 if(formatspecs[7]):
651 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
652 else:
653 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
654 if(len(HeaderOut)==0):
655 return False;
656 fheadsize = int(HeaderOut[0], 16);
657 fnumfields = int(HeaderOut[1], 16);
658 ftype = int(HeaderOut[2], 16);
659 if(re.findall("^[.|/]", HeaderOut[3])):
660 fname = HeaderOut[3];
661 else:
662 fname = "./"+HeaderOut[3];
663 fbasedir = os.path.dirname(fname);
664 flinkname = HeaderOut[4];
665 fsize = int(HeaderOut[5], 16);
666 fatime = int(HeaderOut[6], 16);
667 fmtime = int(HeaderOut[7], 16);
668 fctime = int(HeaderOut[8], 16);
669 fbtime = int(HeaderOut[9], 16);
670 fmode = int(HeaderOut[10], 16);
671 fchmode = stat.S_IMODE(fmode);
672 ftypemod = stat.S_IFMT(fmode);
673 fwinattributes = int(HeaderOut[11], 16);
674 fuid = int(HeaderOut[12], 16);
675 funame = HeaderOut[13];
676 fgid = int(HeaderOut[14], 16);
677 fgname = HeaderOut[15];
678 fid = int(HeaderOut[16], 16);
679 finode = int(HeaderOut[17], 16);
680 flinkcount = int(HeaderOut[18], 16);
681 fdev_minor = int(HeaderOut[19], 16);
682 fdev_major = int(HeaderOut[20], 16);
683 frdev_minor = int(HeaderOut[21], 16);
684 frdev_major = int(HeaderOut[22], 16);
685 fextrasize = int(HeaderOut[23], 16);
686 fextrafields = int(HeaderOut[24], 16);
687 extrafieldslist = [];
688 extrastart = 25;
689 extraend = extrastart + fextrafields;
690 extrafieldslist = [];
691 if(extrastart<extraend):
692 extrafieldslist.append(HeaderOut[extrastart]);
693 extrastart = extrastart + 1;
694 fchecksumtype = HeaderOut[extrastart].lower();
695 fcs = HeaderOut[extrastart + 1].lower();
696 fccs = HeaderOut[extrastart + 2].lower();
697 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
698 if(fcs!=newfcs and not skipchecksum):
699 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
700 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
701 return False;
702 fhend = fp.tell() - 1;
703 fcontentstart = fp.tell();
704 fcontents = BytesIO();
705 pyhascontents = False;
706 if(fsize>0 and not listonly):
707 fcontents.write(fp.read(catfsize));
708 pyhascontents = True;
709 elif(fsize>0 and listonly):
710 fp.seek(fsize, 1);
711 pyhascontents = False;
712 fcontents.seek(0, 0);
713 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
714 if(fccs!=newfccs and not skipchecksum and not listonly):
715 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
716 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
717 return False;
718 fp.seek(1, 1);
719 fcontentend = fp.tell() - 1;
720 fcontents.seek(0, 0);
721 catlist = {'fheadersize': fheadsize, 'fhstart': fheaderstart, 'fhend': fhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': fchecksumtype, 'fnumfields': fnumfields + 2, 'frawheader': HeaderOut, 'fextrafields': fextrafields, 'fextrafieldsize': fextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': fcs, 'fcontentchecksum': fccs, 'fhascontents': pyhascontents, 'fcontentstart': fcontentstart, 'fcontentend': fcontentend, 'fcontents': fcontents};
722 return catlist;
723
724 def ReadFileHeaderDataBySizeWithContentToList(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
725 delimiter = formatspecs[5];
726 fheaderstart = fp.tell();
727 if(formatspecs[7]):
728 HeaderOut = ReadFileHeaderDataBySize(fp, delimiter);
729 else:
730 HeaderOut = ReadFileHeaderDataWoSize(fp, delimiter);
731 if(len(HeaderOut)==0):
732 return False;
733 fheadsize = int(HeaderOut[0], 16);
734 fnumfields = int(HeaderOut[1], 16);
735 ftype = int(HeaderOut[2], 16);
736 if(re.findall("^[.|/]", HeaderOut[3])):
737 fname = HeaderOut[3];
738 else:
739 fname = "./"+HeaderOut[3];
740 fbasedir = os.path.dirname(fname);
741 flinkname = HeaderOut[4];
742 fsize = int(HeaderOut[5], 16);
743 fatime = int(HeaderOut[6], 16);
744 fmtime = int(HeaderOut[7], 16);
745 fctime = int(HeaderOut[8], 16);
746 fbtime = int(HeaderOut[9], 16);
747 fmode = int(HeaderOut[10], 16);
748 fchmode = stat.S_IMODE(fmode);
749 ftypemod = stat.S_IFMT(fmode);
750 fwinattributes = int(HeaderOut[11], 16);
751 fuid = int(HeaderOut[12], 16);
752 funame = HeaderOut[13];
753 fgid = int(HeaderOut[14], 16);
754 fgname = HeaderOut[15];
755 fid = int(HeaderOut[16], 16);
756 finode = int(HeaderOut[17], 16);
757 flinkcount = int(HeaderOut[18], 16);
758 fdev_minor = int(HeaderOut[19], 16);
759 fdev_major = int(HeaderOut[20], 16);
760 frdev_minor = int(HeaderOut[21], 16);
761 frdev_major = int(HeaderOut[22], 16);
762 fextrasize = int(HeaderOut[23], 16);
763 fextrafields = int(HeaderOut[24], 16);
764 extrafieldslist = [];
765 extrastart = 25;
766 extraend = extrastart + fextrafields;
767 extrafieldslist = [];
768 if(extrastart<extraend):
769 extrafieldslist.append(HeaderOut[extrastart]);
770 extrastart = extrastart + 1;
771 fchecksumtype = HeaderOut[extrastart].lower();
772 fcs = HeaderOut[extrastart + 1].lower();
773 fccs = HeaderOut[extrastart + 2].lower();
774 newfcs = GetHeaderChecksum(HeaderOut[:-2], HeaderOut[-3].lower(), True, formatspecs);
775 if(fcs!=newfcs and not skipchecksum):
776 VerbosePrintOut("File Header Checksum Error with file " + fname + " at offset " + str(fheaderstart));
777 VerbosePrintOut("'" + str(fcs) + "' != " + "'" + str(newfcs) + "'");
778 return False;
779 fhend = fp.tell() - 1;
780 fcontentstart = fp.tell();
781 fcontents = BytesIO();
782 pyhascontents = False;
783 if(fsize>0 and not listonly):
784 fcontents.write(fp.read(fsize));
785 pyhascontents = True;
786 elif(fsize>0 and listonly):
787 fp.seek(fsize, 1);
788 pyhascontents = False;
789 fcontents.seek(0, 0);
790 newfccs = GetFileChecksum(fcontents.read(), HeaderOut[-3].lower(), False, formatspecs);
791 if(fccs!=newfccs and not skipchecksum and not listonly):
792 VerbosePrintOut("File Content Checksum Error with file " + fname + " at offset " + str(fcontentstart));
793 VerbosePrintOut("'" + str(fccs) + "' != " + "'" + str(newfccs) + "'");
794 return False;
795 fp.seek(1, 1);
796 fcontentend = fp.tell() - 1;
797 catlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, extrafieldslist, fchecksumtype, fcontents];
798 return catlist;
799
800 def ReadFileDataBySizeWithContent(fp, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
801 delimiter = formatspecs[5];
802 curloc = fp.tell();
803 if(curloc>0):
804 fp.seek(0, 0);
805 catheader = ReadFileHeaderData(fp, 4, delimiter);
806 if(curloc>0):
807 fp.seek(curloc, 0);
808 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
809 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
810 if(not headercheck and not skipchecksum):
811 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
812 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
813 return False;
814 fnumfiles = int(catheader[1], 16);
815 countnum = 0;
816 flist = [];
817 while(countnum < fnumfiles):
818 HeaderOut = ReadFileHeaderDataBySizeWithContent(fp, listonly, skipchecksum, formatspecs)
819 if(len(HeaderOut)==0):
820 break;
821 flist.append(HeaderOut);
822 countnum = countnum + 1;
823 return flist;
824
825 def ReadFileDataBySizeWithContentToArray(fp, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
826 delimiter = formatspecs[5];
827 curloc = fp.tell();
828 if(curloc>0):
829 fp.seek(0, 0);
830 catheader = ReadFileHeaderData(fp, 4, delimiter);
831 if(curloc>0):
832 fp.seek(curloc, 0);
833 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
834 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
835 if(not headercheck and not skipchecksum):
836 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
837 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
838 return False;
839 catstring = catheader[0];
840 catversion = re.findall(r"([\d]+)$", catstring);
841 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
842 fprenumfiles = catheader[1];
843 fnumfiles = int(fprenumfiles, 16);
844 fprechecksumtype = catheader[2];
845 fprechecksum = catheader[3];
846 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
847 if(seekstart<0 and seekstart>fnumfiles):
848 seekstart = 0;
849 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
850 seekend = fnumfiles;
851 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
852 seekend = fnumfiles - abs(seekend);
853 if(seekstart>0):
854 il = 0;
855 while(il < seekstart):
856 prefhstart = fp.tell();
857 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs[5]);
858 if(len(preheaderdata)==0):
859 break;
860 prefsize = int(preheaderdata[5], 16);
861 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-3].lower(), True, formatspecs);
862 prefcs = preheaderdata[-2];
863 if(prefcs!=prenewfcs and not skipchecksum):
864 VVerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
865 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
866 return False;
867 valid_archive = False;
868 invalid_archive = True;
869 prefhend = fp.tell() - 1;
870 prefcontentstart = fp.tell();
871 prefcontents = BytesIO();
872 pyhascontents = False;
873 if(prefsize>0):
874 prefcontents.write(fp.read(prefsize));
875 prefcontents.seek(0, 0);
876 prenewfccs = GetFileChecksum(prefcontents.read(), preheaderdata[-3].lower(), False, formatspecs);
877 prefccs = preheaderdata[-1];
878 pyhascontents = True;
879 if(prefccs!=prenewfccs and not skipchecksum):
880 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
881 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
882 return False;
883 fp.seek(1, 1);
884 il = il + 1;
885 realidnum = 0;
886 countnum = seekstart;
887 while(countnum < seekend):
888 catlist['ffilelist'].update({realidnum: {'fid': realidnum, 'fidalt': realidnum}});
889 HeaderOut = ReadFileHeaderDataBySizeWithContentToArray(fp, listonly, skipchecksum, formatspecs);
890 if(len(HeaderOut)):
891 break;
892 catlist['ffilelist'][realidnum].update(HeaderOut);
893 countnum = countnum + 1;
894 realidnum = realidnum + 1;
895 return catlist;
896
897 def ReadFileDataBySizeWithContentToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
898 if(isinstance(infile, dict)):
899 listcatfiles = infile;
900 else:
901 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
902 return False;
903 listcatfiles = ReadFileDataBySizeWithContentToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs);
904 if(not listcatfiles):
905 return False;
906 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
907 lenlist = len(listcatfiles['ffilelist']);
908 lcfi = 0;
909 lcfx = int(listcatfiles['fnumfiles']);
910 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
911 lcfx = int(lenlist);
912 else:
913 lcfx = int(listcatfiles['fnumfiles']);
914 while(lcfi < lcfx):
915 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
916 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
917 catarray['filetoid'].update(filetoidarray);
918 catarray['idtofile'].update(idtofilearray);
919 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
920 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
921 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
922 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
923 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
924 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
925 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
926 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
927 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
928 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
929 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
930 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
931 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
932 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
933 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
934 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
935 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
936 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
937 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
938 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
939 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
940 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
941 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
942 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
943 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
944 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
945 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
946 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
947 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
948 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
949 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
950 lcfi = lcfi + 1;
951 return catarray;
952
953 def ReadFileDataBySizeWithContentToList(fp, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
954 delimiter = formatspecs[5];
955 curloc = fp.tell();
956 if(curloc>0):
957 fp.seek(0, 0);
958 catheader = ReadFileHeaderData(fp, 4, delimiter);
959 if(curloc>0):
960 fp.seek(curloc, 0);
961 headercheck = ValidateHeaderChecksum(catheader[:-1], catheader[2], catheader[3], formatspecs);
962 newfcs = GetHeaderChecksum(catheader[:-2], catheader[2], True, formatspecs);
963 if(not headercheck and not skipchecksum):
964 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
965 VerbosePrintOut("'" + str(newfcs) + "' != " + "'" + str(catheader[3]) + "'");
966 return False;
967 catstring = catheader[0];
968 catversion = re.findall(r"([\d]+)$", catstring);
969 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
970 fprenumfiles = catheader[1];
971 fnumfiles = int(fprenumfiles, 16);
972 fprechecksumtype = catheader[2];
973 fprechecksum = catheader[3];
974 catlist = [];
975 if(seekstart<0 and seekstart>fnumfiles):
976 seekstart = 0;
977 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
978 seekend = fnumfiles;
979 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
980 seekend = fnumfiles - abs(seekend);
981 if(seekstart>0):
982 il = 0;
983 while(il < seekstart):
984 prefhstart = fp.tell();
985 preheaderdata = ReadFileHeaderDataBySize(fp, formatspecs[5]);
986 if(len(preheaderdata)==0):
987 break;
988 prefsize = int(preheaderdata[5], 16);
989 prenewfcs = GetHeaderChecksum(preheaderdata[:-2], preheaderdata[-3].lower(), True, formatspecs);
990 prefcs = preheaderdata[-2];
991 if(prefcs!=prenewfcs and not skipchecksum):
992 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
993 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
994 return False;
995 valid_archive = False;
996 invalid_archive = True;
997 prefhend = fp.tell() - 1;
998 prefcontentstart = fp.tell();
999 prefcontents = "";
1000 pyhascontents = False;
1001 if(prefsize>0):
1002 prefcontents = fp.read(prefsize);
1003 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
1004 prefccs = preheaderdata[-1];
1005 pyhascontents = True;
1006 if(prefccs!=prenewfccs and not skipchecksum):
1007 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
1008 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
1009 return False;
1010 fp.seek(1, 1);
1011 il = il + 1;
1012 realidnum = 0;
1013 countnum = seekstart;
1014 while(countnum < seekend):
1015 HeaderOut = ReadFileHeaderDataBySizeWithContentToList(fp, listonly, skipchecksum, formatspecs);
1016 if(len(HeaderOut)):
1017 break;
1018 catlist.append(HeaderOut);
1019 countnum = countnum + 1;
1020 realidnum = realidnum + 1;
1021 return catlist;
1022
1023 def ReadInFileBySizeWithContentToArray(infile, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
1024 delimiter = formatspecs[5];
1025 if(hasattr(infile, "read") or hasattr(infile, "write")):
1026 fp = infile;
1027 fp.seek(0, 0);
1028 fp = UncompressArchiveFile(fp, formatspecs);
1029 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1030 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
1031 return False;
1032 if(not fp):
1033 return False;
1034 fp.seek(0, 0);
1035 elif(infile=="-"):
1036 fp = BytesIO();
1037 if(hasattr(sys.stdin, "buffer")):
1038 shutil.copyfileobj(sys.stdin.buffer, fp);
1039 else:
1040 shutil.copyfileobj(sys.stdin, fp);
1041 fp.seek(0, 0);
1042 fp = UncompressArchiveFile(fp, formatspecs);
1043 if(not fp):
1044 return False;
1045 fp.seek(0, 0);
1046 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
1047 fp = download_file_from_internet_file(infile);
1048 fp = UncompressArchiveFile(fp, formatspecs);
1049 fp.seek(0, 0);
1050 if(not fp):
1051 return False;
1052 fp.seek(0, 0);
1053 else:
1054 infile = RemoveWindowsPath(infile);
1055 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
1056 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
1057 return False;
1058 compresscheck = CheckCompressionType(infile, formatspecs, True);
1059 if(not compresscheck):
1060 fextname = os.path.splitext(infile)[1];
1061 if(fextname==".gz"):
1062 compresscheck = "gzip";
1063 elif(fextname==".bz2"):
1064 compresscheck = "bzip2";
1065 elif(fextname==".zst"):
1066 compresscheck = "zstd";
1067 elif(fextname==".lz4" or fextname==".clz4"):
1068 compresscheck = "lz4";
1069 elif(fextname==".lzo" or fextname==".lzop"):
1070 compresscheck = "lzo";
1071 elif(fextname==".lzma" or fextname==".xz"):
1072 compresscheck = "lzma";
1073 else:
1074 return False;
1075 if(not compresscheck):
1076 return False;
1077 fp = UncompressFile(infile, formatspecs, "rb");
1078 return ReadFileDataBySizeWithContentToArray(fp, listonly, skipchecksum, formatspecs);
1079
1080 def ReadInFileBySizeWithContentToList(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
1081 delimiter = formatspecs[5];
1082 if(hasattr(infile, "read") or hasattr(infile, "write")):
1083 fp = infile;
1084 fp.seek(0, 0);
1085 fp = UncompressArchiveFile(fp, formatspecs);
1086 checkcompressfile = CheckCompressionSubType(fp, formatspecs, True);
1087 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
1088 return False;
1089 if(not fp):
1090 return False;
1091 fp.seek(0, 0);
1092 elif(infile=="-"):
1093 fp = BytesIO();
1094 if(hasattr(sys.stdin, "buffer")):
1095 shutil.copyfileobj(sys.stdin.buffer, fp);
1096 else:
1097 shutil.copyfileobj(sys.stdin, fp);
1098 fp.seek(0, 0);
1099 fp = UncompressArchiveFile(fp, formatspecs);
1100 if(not fp):
1101 return False;
1102 fp.seek(0, 0);
1103 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
1104 fp = download_file_from_internet_file(infile);
1105 fp = UncompressArchiveFile(fp, formatspecs);
1106 fp.seek(0, 0);
1107 if(not fp):
1108 return False;
1109 fp.seek(0, 0);
1110 else:
1111 infile = RemoveWindowsPath(infile);
1112 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
1113 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
1114 return False;
1115 compresscheck = CheckCompressionType(infile, formatspecs, True);
1116 if(not compresscheck):
1117 fextname = os.path.splitext(infile)[1];
1118 if(fextname==".gz"):
1119 compresscheck = "gzip";
1120 elif(fextname==".bz2"):
1121 compresscheck = "bzip2";
1122 elif(fextname==".zst"):
1123 compresscheck = "zstd";
1124 elif(fextname==".lz4" or fextname==".clz4"):
1125 compresscheck = "lz4";
1126 elif(fextname==".lzo" or fextname==".lzop"):
1127 compresscheck = "lzo";
1128 elif(fextname==".lzma" or fextname==".xz"):
1129 compresscheck = "lzma";
1130 else:
1131 return False;
1132 if(not compresscheck):
1133 return False;
1134 fp = UncompressFile(infile, formatspecs, "rb");
1135 return ReadFileDataBySizeWithContentToList(fp, seekstart, seekend, listonly, skipchecksum, formatspecs);
1136
1137 def ReadInFileBySizeWithContentToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__):
1138 if(isinstance(infile, dict)):
1139 listcatfiles = infile;
1140 else:
1141 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
1142 infile = RemoveWindowsPath(infile);
1143 listcatfiles = ReadInFileBySizeWithContentToArray(infile, listonly, skipchecksum, formatspecs);
1144 if(not listcatfiles):
1145 return False;
1146 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
1147 lenlist = len(listcatfiles['ffilelist']);
1148 lcfi = 0;
1149 lcfx = int(listcatfiles['fnumfiles']);
1150 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
1151 lcfx = int(lenlist);
1152 else:
1153 lcfx = int(listcatfiles['fnumfiles']);
1154 while(lcfi < lcfx):
1155 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
1156 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
1157 catarray['filetoid'].update(filetoidarray);
1158 catarray['idtofile'].update(idtofilearray);
1159 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
1160 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
1161 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
1162 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
1163 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
1164 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
1165 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
1166 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
1167 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
1168 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1169 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1170 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
1171 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
1172 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
1173 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
1174 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
1175 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1176 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1177 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
1178 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
1179 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
1180 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1181 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1182 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
1183 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
1184 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
1185 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
1186 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
1187 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
1188 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
1189 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
1190 lcfi = lcfi + 1;
1191 return catarray;
1192
1193 def AppendNullByte(indata, delimiter=__file_format_delimiter__):
1194 outdata = str(indata) + delimiter;
1195 return outdata;
1196
1197 def AppendNullBytes(indata=[], delimiter=__file_format_delimiter__):
1198 outdata = "";
1199 inum = 0;
1200 il = len(indata);
1201 while(inum < il):
1202 outdata = outdata + AppendNullByte(indata[inum], delimiter);
1203 inum = inum + 1;
1204 return outdata;
1205
1206 def AppendFileHeader(fp, numfiles, checksumtype="crc32", formatspecs=__file_format_list__):
1207 delimiter = formatspecs[5];
1208 catver = formatspecs[6];
1209 fileheaderver = str(int(catver.replace(".", "")));
1210 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
1211 fp.write(fileheader.encode('UTF-8'));
1212 fnumfiles = format(int(numfiles), 'x').lower();
1213 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs[5]);
1214 catfileheadercshex = GetFileChecksum(fileheader + fnumfilesa, checksumtype, True, formatspecs);
1215 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs[5]);
1216 fp.write(fnumfilesa.encode('UTF-8'));
1217 try:
1218 fp.flush();
1219 os.fsync(fp.fileno());
1220 except io.UnsupportedOperation:
1221 pass;
1222 except AttributeError:
1223 pass;
1224 except OSError as e:
1225 pass;
1226 return fp;
1227
1228 def MakeEmptyFilePointer(fp, checksumtype="crc32", formatspecs=__file_format_list__):
1229 AppendFileHeader(fp, 0, checksumtype, formatspecs);
1230 return fp;
1231
1232 def MakeEmptyFile(outfile, compression="auto", compressionlevel=None, checksumtype="crc32", formatspecs=__file_format_list__, returnfp=False):
1233 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1234 if(os.path.exists(outfile)):
1235 try:
1236 os.unlink(outfile);
1237 except OSError as e:
1238 pass;
1239 if(outfile=="-"):
1240 verbose = False;
1241 catfpfp = BytesIO();
1242 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1243 catfp = outfile;
1244 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1245 catfp = BytesIO();
1246 else:
1247 fbasename = os.path.splitext(outfile)[0];
1248 fextname = os.path.splitext(outfile)[1];
1249 catfp = CompressOpenFile(outfile, compressionlevel);
1250 catfp = AppendFileHeader(catfp, 0, checksumtype, formatspecs);
1251 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1252 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1253 try:
1254 catfp.flush();
1255 os.fsync(catfp.fileno());
1256 except io.UnsupportedOperation:
1257 pass;
1258 except AttributeError:
1259 pass;
1260 except OSError as e:
1261 pass;
1262 if(outfile=="-"):
1263 catfp.seek(0, 0);
1264 if(hasattr(sys.stdout, "buffer")):
1265 shutil.copyfileobj(catfp, sys.stdout.buffer);
1266 else:
1267 shutil.copyfileobj(catfp, sys.stdout);
1268 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1269 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1270 catfp.seek(0, 0);
1271 upload_file_to_internet_file(catfp, outfile);
1272 if(returnfp):
1273 catfp.seek(0, 0);
1274 return catfp;
1275 else:
1276 catfp.close();
1277 return True;
1278
1279 def AppendFileHeaderWithContent(fp, filevalues=[], extradata=[], filecontent="", checksumtype="crc32", formatspecs=__file_format_list__):
1280 extrafields = format(len(extradata), 'x').lower();
1281 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
1282 if(len(extradata)>0):
1283 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
1284 extrasizelen = format(len(extrasizestr), 'x').lower();
1285 catoutlen = len(filevalues) + len(extradata) + 5;
1286 catoutlenhex = format(catoutlen, 'x').lower();
1287 catoutlist = filevalues;
1288 catoutlist.insert(0, catoutlenhex);
1289 catoutlist.append(extrasizelen);
1290 catoutlist.append(extrafields);
1291 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
1292 if(len(extradata)>0):
1293 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
1294 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
1295 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
1296 catfilecontentcshex = GetFileChecksum(filecontent, checksumtype, False, formatspecs);
1297 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
1298 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
1299 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
1300 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
1301 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
1302 catfileoutstrecd = catfileoutstr.encode('UTF-8');
1303 nullstrecd = formatspecs[5].encode('UTF-8');
1304 catfileout = catfileoutstrecd + filecontent + nullstrecd;
1305 fp.write(catfileout);
1306 try:
1307 fp.flush();
1308 os.fsync(fp.fileno());
1309 except io.UnsupportedOperation:
1310 pass;
1311 except AttributeError:
1312 pass;
1313 except OSError as e:
1314 pass;
1315 return fp;
1316
1317 def AppendFilesWithContent(infiles, fp, dirlistfromtxt=False, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False):
1318 advancedlist = formatspecs[8];
1319 altinode = formatspecs[9];
1320 if(verbose):
1321 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
1322 if(infiles=="-"):
1323 for line in sys.stdin:
1324 infilelist.append(line.strip());
1325 infilelist = list(filter(None, infilelist));
1326 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
1327 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
1328 return False;
1329 with UncompressFile(infiles, formatspecs, "r") as finfile:
1330 for line in finfile:
1331 infilelist.append(line.strip());
1332 infilelist = list(filter(None, infilelist));
1333 else:
1334 if(isinstance(infiles, (list, tuple, ))):
1335 infilelist = list(filter(None, infiles));
1336 elif(isinstance(infiles, (str, ))):
1337 infilelist = list(filter(None, [infiles]));
1338 if(advancedlist):
1339 GetDirList = ListDirAdvanced(infilelist, followlink, False);
1340 else:
1341 GetDirList = ListDir(infilelist, followlink, False);
1342 if(not GetDirList):
1343 return False;
1344 curinode = 0;
1345 curfid = 0;
1346 inodelist = [];
1347 inodetofile = {};
1348 filetoinode = {};
1349 inodetocatinode = {};
1350 numfiles = int(len(GetDirList));
1351 fnumfiles = format(numfiles, 'x').lower();
1352 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
1353 for curfname in GetDirList:
1354 if(re.findall("^[.|/]", curfname)):
1355 fname = curfname;
1356 else:
1357 fname = "./"+curfname;
1358 if(verbose):
1359 VerbosePrintOut(fname);
1360 if(not followlink or followlink is None):
1361 fstatinfo = os.lstat(fname);
1362 else:
1363 fstatinfo = os.stat(fname);
1364 fpremode = fstatinfo.st_mode;
1365 finode = fstatinfo.st_ino;
1366 flinkcount = fstatinfo.st_nlink;
1367 ftype = 0;
1368 if(stat.S_ISREG(fpremode)):
1369 ftype = 0;
1370 elif(stat.S_ISLNK(fpremode)):
1371 ftype = 2;
1372 elif(stat.S_ISCHR(fpremode)):
1373 ftype = 3;
1374 elif(stat.S_ISBLK(fpremode)):
1375 ftype = 4;
1376 elif(stat.S_ISDIR(fpremode)):
1377 ftype = 5;
1378 elif(stat.S_ISFIFO(fpremode)):
1379 ftype = 6;
1380 elif(stat.S_ISSOCK(fpremode)):
1381 ftype = 8;
1382 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
1383 ftype = 9;
1384 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
1385 ftype = 10;
1386 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
1387 ftype = 111;
1388 else:
1389 ftype = 0;
1390 flinkname = "";
1391 fcurfid = format(int(curfid), 'x').lower();
1392 if(not followlink and finode!=0):
1393 if(ftype!=1):
1394 if(finode in inodelist):
1395 ftype = 1;
1396 flinkname = inodetofile[finode];
1397 if(altinode):
1398 fcurinode = format(int(finode), 'x').lower();
1399 else:
1400 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
1401 if(finode not in inodelist):
1402 inodelist.append(finode);
1403 inodetofile.update({finode: fname});
1404 inodetocatinode.update({finode: curinode});
1405 if(altinode):
1406 fcurinode = format(int(finode), 'x').lower();
1407 else:
1408 fcurinode = format(int(curinode), 'x').lower();
1409 curinode = curinode + 1;
1410 else:
1411 fcurinode = format(int(curinode), 'x').lower();
1412 curinode = curinode + 1;
1413 curfid = curfid + 1;
1414 if(ftype==2):
1415 flinkname = os.readlink(fname);
1416 fdev = fstatinfo.st_dev;
1417 getfdev = GetDevMajorMinor(fdev);
1418 fdev_minor = getfdev[0];
1419 fdev_major = getfdev[1];
1420 frdev = fstatinfo.st_dev;
1421 if(hasattr(fstatinfo, "st_rdev")):
1422 frdev = fstatinfo.st_rdev;
1423 else:
1424 frdev = fstatinfo.st_dev;
1425 getfrdev = GetDevMajorMinor(frdev);
1426 frdev_minor = getfrdev[0];
1427 frdev_major = getfrdev[1];
1428 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
1429 fsize = format(int("0"), 'x').lower();
1430 elif(ftype==0 or ftype==7):
1431 fsize = format(int(fstatinfo.st_size), 'x').lower();
1432 else:
1433 fsize = format(int(fstatinfo.st_size)).lower();
1434 fatime = format(int(fstatinfo.st_atime), 'x').lower();
1435 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
1436 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
1437 if(hasattr(fstatinfo, "st_birthtime")):
1438 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
1439 else:
1440 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
1441 fmode = format(int(fstatinfo.st_mode), 'x').lower();
1442 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
1443 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
1444 fuid = format(int(fstatinfo.st_uid), 'x').lower();
1445 fgid = format(int(fstatinfo.st_gid), 'x').lower();
1446 funame = "";
1447 try:
1448 import pwd;
1449 try:
1450 userinfo = pwd.getpwuid(fstatinfo.st_uid);
1451 funame = userinfo.pw_name;
1452 except KeyError:
1453 funame = "";
1454 except ImportError:
1455 funame = "";
1456 fgname = "";
1457 try:
1458 import grp;
1459 try:
1460 groupinfo = grp.getgrgid(fstatinfo.st_gid);
1461 fgname = groupinfo.gr_name;
1462 except KeyError:
1463 fgname = "";
1464 except ImportError:
1465 fgname = "";
1466 fdev_minor = format(int(fdev_minor), 'x').lower();
1467 fdev_major = format(int(fdev_major), 'x').lower();
1468 frdev_minor = format(int(frdev_minor), 'x').lower();
1469 frdev_major = format(int(frdev_major), 'x').lower();
1470 finode = format(int(finode), 'x').lower();
1471 flinkcount = format(int(flinkcount), 'x').lower();
1472 if(hasattr(fstatinfo, "st_file_attributes")):
1473 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
1474 else:
1475 fwinattributes = format(int(0), 'x').lower();
1476 fcontents = BytesIO();
1477 chunk_size = 1024;
1478 if(ftype==0 or ftype==7):
1479 with open(fname, "rb") as fpc:
1480 shutil.copyfileobj(fpc, fcontents);
1481 if(followlink and (ftype==1 or ftype==2)):
1482 flstatinfo = os.stat(flinkname);
1483 with open(flinkname, "rb") as fpc:
1484 shutil.copyfileobj(fpc, fcontents);
1485 fcontents.seek(0, 0);
1486 ftypehex = format(ftype, 'x').lower();
1487 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
1488 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents.read(), checksumtype, formatspecs);
1489 if(numfiles>0):
1490 catfp.write(AppendNullBytes([0, 0], formatspecs[5]).encode("UTF-8"));
1491 fp.seek(0, 0);
1492 return fp;
1493
1494 def AppendListsWithContent(inlist, fp, dirlistfromtxt=False, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False):
1495 if(verbose):
1496 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
1497 GetDirList = inlist;
1498 if(not GetDirList):
1499 return False;
1500 curinode = 0;
1501 curfid = 0;
1502 inodelist = [];
1503 inodetofile = {};
1504 filetoinode = {};
1505 inodetocatinode = {};
1506 numfiles = int(len(GetDirList));
1507 fnumfiles = format(numfiles, 'x').lower();
1508 AppendFileHeader(fp, fnumfiles, checksumtype, formatspecs);
1509 for curfname in GetDirList:
1510 ftype = format(curfname[0], 'x').lower();
1511 if(re.findall("^[.|/]", curfname[1])):
1512 fname = curfname[1];
1513 else:
1514 fname = "./"+curfname[1];
1515 fbasedir = os.path.dirname(fname);
1516 flinkname = curfname[2];
1517 fsize = format(curfname[3], 'x').lower();
1518 fatime = format(curfname[4], 'x').lower();
1519 fmtime = format(curfname[5], 'x').lower();
1520 fctime = format(curfname[6], 'x').lower();
1521 fbtime = format(curfname[7], 'x').lower();
1522 fmode = format(curfname[8], 'x').lower();
1523 fwinattributes = format(curfname[9], 'x').lower();
1524 fuid = format(curfname[10], 'x').lower();
1525 funame = curfname[11];
1526 fgid = format(curfname[12], 'x').lower();
1527 fgname = curfname[13];
1528 fid = format(curfname[14], 'x').lower();
1529 finode = format(curfname[15], 'x').lower();
1530 flinkcount = format(curfname[16], 'x').lower();
1531 fdev_minor = format(curfname[17], 'x').lower();
1532 fdev_major = format(curfname[18], 'x').lower();
1533 frdev_minor = format(curfname[19], 'x').lower();
1534 frdev_major = format(curfname[20], 'x').lower();
1535 extradata = curfname[21];
1536 fchecksumtype = curfname[22];
1537 fcontents = curfname[23];
1538 catoutlist = [ftype, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fuid, funame, fgid, fgname, fid, finode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major];
1539 fp = AppendFileHeaderWithContent(fp, catoutlist, extradata, fcontents, checksumtype, formatspecs);
1540 if(numfiles>0):
1541 catfp.write(AppendNullBytes([0, 0], formatspecs[5]).encode("UTF-8"));
1542 fp.seek(0, 0);
1543 return fp;
1544
1545 def AppendInFileWithContent(infile, fp, dirlistfromtxt=False, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False):
1546 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, False, formatspecs);
1547 return AppendListsWithContent(inlist, fp, dirlistfromtxt, filevalues, extradata, followlink, checksumtype, formatspecs, verbose);
1548
1549 def AppendFilesWithContentToOutFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False, returnfp=False):
1550 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1551 if(os.path.exists(outfile)):
1552 try:
1553 os.unlink(outfile);
1554 except OSError as e:
1555 pass;
1556 if(outfile=="-"):
1557 verbose = False;
1558 catfpfp = BytesIO();
1559 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1560 catfp = outfile;
1561 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1562 catfp = BytesIO();
1563 else:
1564 fbasename = os.path.splitext(outfile)[0];
1565 fextname = os.path.splitext(outfile)[1];
1566 catfp = CompressOpenFile(outfile, compressionlevel);
1567 catfp = AppendFilesWithContent(infiles, catfp, dirlistfromtxt, filevalues, extradata, followlink, checksumtype, formatspecs, verbose);
1568 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1569 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1570 try:
1571 catfp.flush();
1572 os.fsync(catfp.fileno());
1573 except io.UnsupportedOperation:
1574 pass;
1575 except AttributeError:
1576 pass;
1577 except OSError as e:
1578 pass;
1579 if(outfile=="-"):
1580 catfp.seek(0, 0);
1581 if(hasattr(sys.stdout, "buffer")):
1582 shutil.copyfileobj(catfp, sys.stdout.buffer);
1583 else:
1584 shutil.copyfileobj(catfp, sys.stdout);
1585 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1586 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1587 catfp.seek(0, 0);
1588 upload_file_to_internet_file(catfp, outfile);
1589 if(returnfp):
1590 catfp.seek(0, 0);
1591 return catfp;
1592 else:
1593 catfp.close();
1594 return True;
1595
1596 def AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt=False, compression="auto", compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False, returnfp=False):
1597 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
1598 if(os.path.exists(outfile)):
1599 try:
1600 os.unlink(outfile);
1601 except OSError as e:
1602 pass;
1603 if(outfile=="-"):
1604 verbose = False;
1605 catfpfp = BytesIO();
1606 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
1607 catfp = outfile;
1608 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1609 catfp = BytesIO();
1610 else:
1611 fbasename = os.path.splitext(outfile)[0];
1612 fextname = os.path.splitext(outfile)[1];
1613 catfp = CompressOpenFile(outfile, compressionlevel);
1614 catfp = AppendListsWithContent(inlist, catfp, dirlistfromtxt, filevalues, extradata, followlink, checksumtype, formatspecs, verbose);
1615 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
1616 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1617 try:
1618 catfp.flush();
1619 os.fsync(catfp.fileno());
1620 except io.UnsupportedOperation:
1621 pass;
1622 except AttributeError:
1623 pass;
1624 except OSError as e:
1625 pass;
1626 if(outfile=="-"):
1627 catfp.seek(0, 0);
1628 if(hasattr(sys.stdout, "buffer")):
1629 shutil.copyfileobj(catfp, sys.stdout.buffer);
1630 else:
1631 shutil.copyfileobj(catfp, sys.stdout);
1632 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
1633 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
1634 catfp.seek(0, 0);
1635 upload_file_to_internet_file(catfp, outfile);
1636 if(returnfp):
1637 catfp.seek(0, 0);
1638 return catfp;
1639 else:
1640 catfp.close();
1641 return True;
1642
1643 def AppendInFileWithContentToOutFile(infile, outfile, dirlistfromtxt=False, compression="auto", compressionlevel=None, filevalues=[], extradata=[], followlink=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False, returnfp=False):
1644 inlist = ReadInFileBySizeWithContentToList(infile, 0, 0, False, False, formatspecs);
1645 return AppendListsWithContentToOutFile(inlist, outfile, dirlistfromtxt, compression, compressionlevel, filevalues, extradata, followlink, checksumtype, formatspecs, verbose, returnfp);
1646
1647 def PrintPermissionString(fchmode, ftype):
1648 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
1649 permissionstr = "";
1650 for fmodval in str(oct(fchmode))[-3:]:
1651 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
1652 if(ftype==0 or ftype==7):
1653 permissionstr = "-" + permissionstr;
1654 if(ftype==1):
1655 permissionstr = "h" + permissionstr;
1656 if(ftype==2):
1657 permissionstr = "l" + permissionstr;
1658 if(ftype==3):
1659 permissionstr = "c" + permissionstr;
1660 if(ftype==4):
1661 permissionstr = "b" + permissionstr;
1662 if(ftype==5):
1663 permissionstr = "d" + permissionstr;
1664 if(ftype==6):
1665 permissionstr = "f" + permissionstr;
1666 if(ftype==8):
1667 permissionstr = "D" + permissionstr;
1668 if(ftype==9):
1669 permissionstr = "p" + permissionstr;
1670 if(ftype==10):
1671 permissionstr = "w" + permissionstr;
1672 try:
1673 permissionoutstr = stat.filemode(fchmode);
1674 except AttributeError:
1675 permissionoutstr = permissionstr;
1676 except KeyError:
1677 permissionoutstr = permissionstr;
1678 return permissionoutstr;
1679
1680 def PrintPermissionStringAlt(fchmode, ftype):
1681 permissions = {
1682 '0': '---', '1': '--x', '2': '-w-', '3': '-wx',
1683 '4': 'r--', '5': 'r-x', '6': 'rw-', '7': 'rwx'
1684 };
1685 # Translate file mode into permission string
1686 permissionstr = ''.join([permissions[i] for i in str(oct(fchmode))[-3:]]);
1687 # Append file type indicator
1688 type_indicators = {
1689 0: '-', 1: 'h', 2: 'l', 3: 'c', 4: 'b',
1690 5: 'd', 6: 'f', 8: 'D', 9: 'p', 10: 'w'
1691 };
1692 file_type = type_indicators.get(ftype, '-');
1693 permissionstr = file_type + permissionstr;
1694 try:
1695 permissionoutstr = stat.filemode(fchmode);
1696 except AttributeError:
1697 permissionoutstr = permissionstr;
1698 return permissionoutstr;
1699
1700 def CompressionSupport():
1701 compression_list = [];
1702 try:
1703 import gzip;
1704 compression_list.append("gz");
1705 compression_list.append("gzip");
1706 except ImportError:
1707 '''return False;'''
1708 try:
1709 import bz2;
1710 compression_list.append("bz2");
1711 compression_list.append("bzip2");
1712 except ImportError:
1713 '''return False;'''
1714 try:
1715 import lz4;
1716 compression_list.append("lz4");
1717 except ImportError:
1718 '''return False;'''
1719 try:
1720 import lzo;
1721 compression_list.append("lzo");
1722 compression_list.append("lzop");
1723 except ImportError:
1724 '''return False;'''
1725 try:
1726 import zstandard;
1727 compression_list.append("zstd");
1728 compression_list.append("zstandard");
1729 except ImportError:
1730 '''return False;'''
1731 try:
1732 import lzma;
1733 compression_list.append("lzma");
1734 compression_list.append("xz");
1735 except ImportError:
1736 '''return False;'''
1737 return compression_list;
1738
1739 def CheckCompressionType(infile, formatspecs=__file_format_list__, closefp=True):
1740 if(hasattr(infile, "read") or hasattr(infile, "write")):
1741 catfp = infile;
1742 else:
1743 try:
1744 catfp = open(infile, "rb");
1745 except FileNotFoundError:
1746 return False;
1747 catfp.seek(0, 0);
1748 prefp = catfp.read(2);
1749 filetype = False;
1750 if(prefp==binascii.unhexlify("1f8b")):
1751 filetype = "gzip";
1752 catfp.seek(0, 0);
1753 prefp = catfp.read(3);
1754 if(prefp==binascii.unhexlify("425a68")):
1755 filetype = "bzip2";
1756 if(prefp==binascii.unhexlify("5d0000")):
1757 filetype = "lzma";
1758 catfp.seek(0, 0);
1759 prefp = catfp.read(4);
1760 if(prefp==binascii.unhexlify("28b52ffd")):
1761 filetype = "zstd";
1762 if(prefp==binascii.unhexlify("04224d18")):
1763 filetype = "lz4";
1764 if(prefp==binascii.unhexlify("504B0304")):
1765 filetype = "zipfile";
1766 catfp.seek(0, 0);
1767 prefp = catfp.read(5);
1768 if(prefp==binascii.unhexlify("7573746172")):
1769 filetype = "tarfile";
1770 catfp.seek(0, 0);
1771 prefp = catfp.read(6);
1772 if(prefp==binascii.unhexlify("fd377a585a00")):
1773 filetype = "lzma";
1774 if(prefp==binascii.unhexlify("377abcaf271c")):
1775 filetype = "7zipfile";
1776 catfp.seek(0, 0);
1777 prefp = catfp.read(7);
1778 if(prefp==binascii.unhexlify("526172211a0700")):
1779 filetype = "rarfile";
1780 if(prefp==binascii.unhexlify("43617446696c65")):
1781 filetype = "catfile";
1782 catfp.seek(0, 0);
1783 prefp = catfp.read(8);
1784 if(prefp==binascii.unhexlify("526172211a070100")):
1785 filetype = "rarfile";
1786 catfp.seek(0, 0);
1787 prefp = catfp.read(formatspecs[3]);
1788 if(prefp==binascii.unhexlify(formatspecs[4])):
1789 filetype = formatspecs[2];
1790 catfp.seek(0, 0);
1791 prefp = catfp.read(9);
1792 if(prefp==binascii.unhexlify("894c5a4f000d0a1a0a")):
1793 filetype = "lzo";
1794 catfp.seek(0, 0);
1795 prefp = catfp.read(10);
1796 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
1797 filetype = "tarfile";
1798 catfp.seek(0, 0);
1799 if(filetype=="gzip" or filetype=="bzip2" or filetype=="lzma" or filetype=="zstd" or filetype=="lz4"):
1800 if(is_tarfile(catfp)):
1801 filetype = "tarfile";
1802 if(not filetype):
1803 if(is_tarfile(catfp)):
1804 filetype = "tarfile";
1805 elif(zipfile.is_zipfile(catfp)):
1806 filetype = "zipfile";
1807 elif(rarfile.is_rarfile(catfp) or rarfile.is_rarfile_sfx(catfp)):
1808 filetype = "rarile";
1809 else:
1810 filetype = False;
1811 catfp.seek(0, 0);
1812 if(closefp):
1813 catfp.close();
1814 return filetype;
1815
1816 def CheckCompressionTypeFromString(instring, formatspecs=__file_format_list__, closefp=True):
1817 try:
1818 instringsfile = BytesIO(instring);
1819 except TypeError:
1820 instringsfile = BytesIO(instring.encode("UTF-8"));
1821 return CheckCompressionType(instringsfile, formatspecs, closefp);
1822
1823 def GetCompressionMimeType(infile, formatspecs=__file_format_list__):
1824 compresscheck = CheckCompressionType(fp, formatspecs, False);
1825 if(compresscheck=="gzip" or compresscheck=="gz"):
1826 return archivefile_gzip_mimetype;
1827 if(compresscheck=="bzip2" or compresscheck=="bz2"):
1828 return archivefile_bzip2_mimetype;
1829 if(compresscheck=="zstd" or compresscheck=="zstandard"):
1830 return archivefile_zstandard_mimetype;
1831 if(compresscheck=="lz4"):
1832 return archivefile_lz4_mimetype;
1833 if(compresscheck=="lzo" or compresscheck=="lzop"):
1834 return archivefile_lzop_mimetype;
1835 if(compresscheck=="lzma"):
1836 return archivefile_lzma_mimetype;
1837 if(compresscheck=="xz"):
1838 return archivefile_xz_mimetype;
1839 if(compresscheck=="catfile" or compresscheck=="cat" or compresscheck==formatspecs[2]):
1840 return archivefile_cat_mimetype;
1841 if(not compresscheck):
1842 return False;
1843 return False;
1844
1845 def UncompressArchiveFile(fp, formatspecs=__file_format_list__):
1846 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
1847 return False;
1848 compresscheck = CheckCompressionType(fp, formatspecs, False);
1849 if(compresscheck=="gzip"):
1850 try:
1851 import gzip;
1852 except ImportError:
1853 return False;
1854 catfp = gzip.GzipFile(fileobj=fp, mode="rb");
1855 if(compresscheck=="bzip2"):
1856 try:
1857 import bz2;
1858 except ImportError:
1859 return False;
1860 catfp = BytesIO();
1861 catfp.write(bz2.decompress(fp.read()));
1862 if(compresscheck=="zstd"):
1863 try:
1864 import zstandard;
1865 except ImportError:
1866 return False;
1867 catfp = BytesIO();
1868 catfp.write(zstandard.decompress(fp.read()));
1869 if(compresscheck=="lz4"):
1870 try:
1871 import lz4.frame;
1872 except ImportError:
1873 return False;
1874 catfp = BytesIO();
1875 catfp.write(lz4.frame.decompress(fp.read()));
1876 if(compresscheck=="lzo" or compresscheck=="lzop"):
1877 try:
1878 import lzo;
1879 except ImportError:
1880 return False;
1881 catfp = BytesIO();
1882 catfp.write(lzo.decompress(fp.read()));
1883 if(compresscheck=="lzma" or compresscheck=="xz"):
1884 try:
1885 import lzma;
1886 except ImportError:
1887 return False;
1888 catfp = BytesIO();
1889 catfp.write(lzma.decompress(fp.read()));
1890 if(compresscheck=="catfile" or compresscheck==formatspecs[2]):
1891 catfp = fp;
1892 if(not compresscheck):
1893 try:
1894 import lzma;
1895 except ImportError:
1896 return False;
1897 catfp = BytesIO();
1898 with fp as fpcontent:
1899 try:
1900 catfp.write(lzma.decompress(fp.read()));
1901 except lzma.LZMAError:
1902 return False;
1903 return catfp;
1904
1905 create_alias_function("Uncompress", __file_format_name__, "", UncompressArchiveFile);
1906
1907 def UncompressFile(infile, formatspecs=__file_format_list__, mode="rb"):
1908 compresscheck = CheckCompressionType(infile, formatspecs, False);
1909 if(sys.version_info[0]==2 and compresscheck):
1910 if(mode=="rt"):
1911 mode = "r";
1912 if(mode=="wt"):
1913 mode = "w";
1914 try:
1915 if(compresscheck=="gzip"):
1916 try:
1917 import gzip;
1918 except ImportError:
1919 return False;
1920 try:
1921 filefp = gzip.open(infile, mode, encoding="UTF-8");
1922 except (ValueError, TypeError) as e:
1923 filefp = gzip.open(infile, mode);
1924 if(compresscheck=="bzip2"):
1925 try:
1926 import bz2;
1927 except ImportError:
1928 return False;
1929 try:
1930 filefp = bz2.open(infile, mode, encoding="UTF-8");
1931 except (ValueError, TypeError) as e:
1932 filefp = bz2.open(infile, mode);
1933 if(compresscheck=="zstd"):
1934 try:
1935 import zstandard;
1936 except ImportError:
1937 return False;
1938 try:
1939 filefp = zstandard.open(infile, mode, encoding="UTF-8");
1940 except (ValueError, TypeError) as e:
1941 filefp = zstandard.open(infile, mode);
1942 if(compresscheck=="lz4"):
1943 try:
1944 import lz4.frame;
1945 except ImportError:
1946 return False;
1947 try:
1948 filefp = lz4.frame.open(infile, mode, encoding="UTF-8");
1949 except (ValueError, TypeError) as e:
1950 filefp = lz4.frame.open(infile, mode);
1951 if(compresscheck=="lzo"):
1952 try:
1953 import lzo;
1954 except ImportError:
1955 return False;
1956 try:
1957 filefp = lzo.open(infile, mode, encoding="UTF-8");
1958 except (ValueError, TypeError) as e:
1959 filefp = lzo.open(infile, mode);
1960 if(compresscheck=="lzma"):
1961 try:
1962 import lzma;
1963 except ImportError:
1964 return False;
1965 try:
1966 filefp = lzma.open(infile, mode, encoding="UTF-8");
1967 except (ValueError, TypeError) as e:
1968 filefp = lzma.open(infile, mode);
1969 if(compresscheck=="catfile" or compresscheck==formatspecs[2]):
1970 try:
1971 filefp = open(infile, mode, encoding="UTF-8");
1972 except (ValueError, TypeError) as e:
1973 filefp = open(infile, mode);
1974 if(not compresscheck):
1975 try:
1976 filefp = open(infile, mode, encoding="UTF-8");
1977 except (ValueError, TypeError) as e:
1978 filefp = open(infile, mode);
1979 except FileNotFoundError:
1980 return False;
1981 return filefp;
1982
1983 def UncompressString(infile):
1984 compresscheck = CheckCompressionTypeFromString(infile, formatspecs, False);
1985 if(compresscheck=="gzip"):
1986 try:
1987 import gzip;
1988 except ImportError:
1989 return False;
1990 fileuz = gzip.decompress(infile);
1991 if(compresscheck=="bzip2"):
1992 try:
1993 import bz2;
1994 except ImportError:
1995 return False;
1996 fileuz = bz2.decompress(infile);
1997 if(compresscheck=="zstd"):
1998 try:
1999 import zstandard;
2000 except ImportError:
2001 return False;
2002 fileuz = zstandard.decompress(infile);
2003 if(compresscheck=="lz4"):
2004 try:
2005 import lz4.frame;
2006 except ImportError:
2007 return False;
2008 fileuz = lz4.frame.decompress(infile);
2009 if(compresscheck=="lzo"):
2010 try:
2011 import lzo;
2012 except ImportError:
2013 return False;
2014 fileuz = lzo.decompress(infile);
2015 if(compresscheck=="lzma"):
2016 try:
2017 import lzma;
2018 except ImportError:
2019 return False;
2020 fileuz = lzma.decompress(infile);
2021 if(not compresscheck):
2022 fileuz = infile;
2023 if(hasattr(fileuz, 'decode')):
2024 fileuz = fileuz.decode("UTF-8");
2025 return fileuz;
2026
2027 def UncompressStringAlt(infile):
2028 filefp = StringIO();
2029 outstring = UncompressString(infile);
2030 filefp.write(outstring);
2031 filefp.seek(0, 0);
2032 return filefp;
2033
2034 def CheckCompressionSubType(infile, formatspecs=__file_format_list__, closefp=True):
2035 compresscheck = CheckCompressionType(infile, formatspecs, False);
2036 if(not compresscheck):
2037 fextname = os.path.splitext(infile)[1];
2038 if(fextname==".gz"):
2039 compresscheck = "gzip";
2040 elif(fextname==".bz2"):
2041 compresscheck = "bzip2";
2042 elif(fextname==".zst"):
2043 compresscheck = "zstd";
2044 elif(fextname==".lz4"):
2045 compresscheck = "lz4";
2046 elif(fextname==".lzo" or fextname==".lzop"):
2047 compresscheck = "lzo";
2048 elif(fextname==".lzma" or fextname==".xz"):
2049 compresscheck = "lzma";
2050 else:
2051 return False;
2052 if(compresscheck=="gzip" or compresscheck=="bzip2" or compresscheck=="lzma" or compresscheck=="zstd" or compresscheck=="lz4"):
2053 if(is_tarfile(infile)):
2054 filetype = "tarfile";
2055 if(not compresscheck):
2056 if(is_tarfile(infile)):
2057 return "tarfile";
2058 elif(zipfile.is_zipfile(infile)):
2059 return "zipfile";
2060 elif(rarfile.is_rarfile(infile) or rarfile.is_rarfile_sfx(infile)):
2061 return "rarile";
2062 else:
2063 return False;
2064 return False;
2065 if(compresscheck=="catfile"):
2066 return "catfile";
2067 if(compresscheck==formatspecs[2]):
2068 return formatspecs[2];
2069 if(compresscheck=="tarfile"):
2070 return "tarfile";
2071 if(compresscheck=="zipfile"):
2072 return "zipfile";
2073 if(rarfile_support and compresscheck=="rarfile"):
2074 return "rarfile";
2075 if(py7zr_support and compresscheck=="7zipfile"):
2076 return "7zipfile";
2077 if(hasattr(infile, "read") or hasattr(infile, "write")):
2078 catfp = UncompressArchiveFile(infile, formatspecs[2]);
2079 else:
2080 try:
2081 if(compresscheck=="gzip"):
2082 try:
2083 import gzip;
2084 except ImportError:
2085 return False;
2086 catfp = gzip.GzipFile(infile, "rb");
2087 if(compresscheck=="bzip2"):
2088 try:
2089 import bz2;
2090 except ImportError:
2091 return False;
2092 catfp = bz2.BZ2File(infile, "rb");
2093 if(compresscheck=="lz4"):
2094 try:
2095 import lz4.frame;
2096 except ImportError:
2097 return False;
2098 catfp = lz4.frame.open(infile, "rb");
2099 if(compresscheck=="zstd"):
2100 try:
2101 import zstandard;
2102 except ImportError:
2103 return False;
2104 catfp = zstandard.open(infile, "rb");
2105 if(compresscheck=="lzma" or compresscheck=="xz"):
2106 try:
2107 import lzma;
2108 except ImportError:
2109 return False;
2110 catfp = lzma.open(infile, "rb");
2111 except FileNotFoundError:
2112 return False;
2113 filetype = False;
2114 prefp = catfp.read(5);
2115 if(prefp==binascii.unhexlify("7573746172")):
2116 filetype = "tarfile";
2117 catfp.seek(0, 0);
2118 prefp = catfp.read(7);
2119 if(prefp==binascii.unhexlify("43617446696c65")):
2120 filetype = "catfile";
2121 catfp.seek(0, 0);
2122 prefp = catfp.read(formatspecs[3]);
2123 if(prefp==binascii.unhexlify(formatspecs[4])):
2124 filetype = formatspecs[2];
2125 catfp.seek(0, 0);
2126 prefp = catfp.read(10);
2127 if(prefp==binascii.unhexlify("7061785f676c6f62616c")):
2128 filetype = "tarfile";
2129 catfp.seek(0, 0);
2130 if(closefp):
2131 catfp.close();
2132 return filetype;
2133
2134 def GZipCompress(data, compresslevel=9):
2135 try:
2136 import gzip;
2137 except ImportError:
2138 return False;
2139 tmpfp = tempfile.NamedTemporaryFile("wb", delete=False);
2140 tmpfp.close();
2141 tmpfp = gzip.GzipFile(tmpfp.name, mode="wb", compresslevel=compresslevel);
2142 tmpfp.write(data);
2143 tmpfp.close();
2144 try:
2145 catfp = open(tmpfp.name, "rb");
2146 except FileNotFoundError:
2147 return False;
2148 catdata = catfp.read();
2149 catfp.close();
2150 return catdata;
2151
2152 def CompressArchiveFile(fp, compression="auto", compressionlevel=None, formatspecs=__file_format_list__):
2153 if(not hasattr(fp, "read") and not hasattr(fp, "write")):
2154 return False;
2155 fp.seek(0, 0);
2156 if(not compression or compression=="catfile" or compression==formatspecs[2]):
2157 compression = "auto";
2158 if(compression not in compressionlist and compression is None):
2159 compression = "auto";
2160 if(compression=="gzip"):
2161 try:
2162 import gzip;
2163 except ImportError:
2164 return False;
2165 catfp = BytesIO();
2166 if(compressionlevel is None):
2167 compressionlevel = 9;
2168 else:
2169 compressionlevel = int(compressionlevel);
2170 catfp.write(gzip.compress(fp.read(), compresslevel=compressionlevel));
2171 if(compression=="bzip2"):
2172 try:
2173 import bz2;
2174 except ImportError:
2175 return False;
2176 catfp = BytesIO();
2177 if(compressionlevel is None):
2178 compressionlevel = 9;
2179 else:
2180 compressionlevel = int(compressionlevel);
2181 catfp.write(bz2.compress(fp.read(), compresslevel=compressionlevel));
2182 if(compression=="lz4"):
2183 try:
2184 import lz4.frame;
2185 except ImportError:
2186 return False;
2187 catfp = BytesIO();
2188 if(compressionlevel is None):
2189 compressionlevel = 9;
2190 else:
2191 compressionlevel = int(compressionlevel);
2192 catfp.write(lz4.frame.compress(fp.read(), compression_level=compressionlevel));
2193 if(compression=="lzo" or compression=="lzop"):
2194 try:
2195 import lzo;
2196 except ImportError:
2197 return False;
2198 catfp = BytesIO();
2199 if(compressionlevel is None):
2200 compressionlevel = 9;
2201 else:
2202 compressionlevel = int(compressionlevel);
2203 catfp.write(lzo.compress(fp.read(), compresslevel=compressionlevel));
2204 if(compression=="zstd"):
2205 try:
2206 import zstandard;
2207 except ImportError:
2208 return False;
2209 catfp = BytesIO();
2210 if(compressionlevel is None):
2211 compressionlevel = 10;
2212 else:
2213 compressionlevel = int(compressionlevel);
2214 catfp.write(zstandard.compress(fp.read(), level=compressionlevel));
2215 if(compression=="lzma"):
2216 try:
2217 import lzma;
2218 except ImportError:
2219 return False;
2220 catfp = BytesIO();
2221 if(compressionlevel is None):
2222 compressionlevel = 9;
2223 else:
2224 compressionlevel = int(compressionlevel);
2225 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]));
2226 if(compression=="xz"):
2227 try:
2228 import lzma;
2229 except ImportError:
2230 return False;
2231 catfp = BytesIO();
2232 if(compressionlevel is None):
2233 compressionlevel = 9;
2234 else:
2235 compressionlevel = int(compressionlevel);
2236 catfp.write(lzma.compress(fp.read(), format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]));
2237 if(compression=="auto" or compression is None):
2238 catfp = fp;
2239 catfp.seek(0, 0);
2240 return catfp;
2241
2242 create_alias_function("Compress", __file_format_name__, "", CompressArchiveFile);
2243
2244 def CompressOpenFile(outfile, compressionlevel=None):
2245 if(outfile is None):
2246 return False;
2247 fbasename = os.path.splitext(outfile)[0];
2248 fextname = os.path.splitext(outfile)[1];
2249 if(compressionlevel is None and fextname!=".zst"):
2250 compressionlevel = 9;
2251 elif(compressionlevel is None and fextname==".zst"):
2252 compressionlevel = 10;
2253 else:
2254 compressionlevel = int(compressionlevel);
2255 if(sys.version_info[0]==2):
2256 mode = "w";
2257 else:
2258 mode = "wb";
2259 try:
2260 if(fextname not in outextlistwd):
2261 try:
2262 outfp = open(outfile, "wb", encoding="UTF-8");
2263 except (ValueError, TypeError) as e:
2264 outfp = open(outfile, "wb");
2265 elif(fextname==".gz"):
2266 try:
2267 import gzip;
2268 except ImportError:
2269 return False;
2270 try:
2271 outfp = gzip.open(outfile, mode, compressionlevel, encoding="UTF-8");
2272 except (ValueError, TypeError) as e:
2273 outfp = gzip.open(outfile, mode, compressionlevel);
2274 elif(fextname==".bz2"):
2275 try:
2276 import bz2;
2277 except ImportError:
2278 return False;
2279 try:
2280 outfp = bz2.open(outfile, mode, compressionlevel, encoding="UTF-8");
2281 except (ValueError, TypeError) as e:
2282 outfp = bz2.open(outfile, mode, compressionlevel);
2283 elif(fextname==".zst"):
2284 try:
2285 import zstandard;
2286 except ImportError:
2287 return False;
2288 try:
2289 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel), encoding="UTF-8");
2290 except (ValueError, TypeError) as e:
2291 outfp = zstandard.open(outfile, mode, zstandard.ZstdCompressor(level=compressionlevel));
2292 elif(fextname==".xz"):
2293 try:
2294 import lzma;
2295 except ImportError:
2296 return False;
2297 try:
2298 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}], encoding="UTF-8");
2299 except (ValueError, TypeError) as e:
2300 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_XZ, filters=[{"id": lzma.FILTER_LZMA2, "preset": compressionlevel}]);
2301 elif(fextname==".lz4"):
2302 try:
2303 import lz4.frame;
2304 except ImportError:
2305 return False;
2306 try:
2307 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel, encoding="UTF-8");
2308 except (ValueError, TypeError) as e:
2309 outfp = lz4.frame.open(outfile, mode, compression_level=compressionlevel);
2310 elif(fextname==".lzo"):
2311 try:
2312 import lzo;
2313 except ImportError:
2314 return False;
2315 try:
2316 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel, encoding="UTF-8");
2317 except (ValueError, TypeError) as e:
2318 outfp = lzo.open(outfile, mode, compresslevel=compressionlevel);
2319 elif(fextname==".lzma"):
2320 try:
2321 import lzma;
2322 except ImportError:
2323 return False;
2324 try:
2325 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}], encoding="UTF-8");
2326 except (ValueError, TypeError) as e:
2327 outfp = lzma.open(outfile, mode, format=lzma.FORMAT_ALONE, filters=[{"id": lzma.FILTER_LZMA1, "preset": compressionlevel}]);
2328 except FileNotFoundError:
2329 return False;
2330 return outfp;
2331
2332 def GetDevMajorMinor(fdev):
2333 retdev = [];
2334 if(hasattr(os, "minor")):
2335 retdev.append(os.minor(fdev));
2336 else:
2337 retdev.append(0);
2338 if(hasattr(os, "major")):
2339 retdev.append(os.major(fdev));
2340 else:
2341 retdev.append(0);
2342 return retdev;
2343
2344 def CheckSumSupport(checkfor, guaranteed=True):
2345 if(guaranteed):
2346 hash_list = sorted(list(hashlib.algorithms_guaranteed));
2347 else:
2348 hash_list = sorted(list(hashlib.algorithms_available));
2349 checklistout = sorted(hash_list + ['adler32', 'crc16', 'crc16_ansi', 'crc16_ibm', 'crc16_ccitt', 'crc32', 'crc64', 'crc64_ecma', 'crc64_iso', 'none']);
2350 if(checkfor in checklistout):
2351 return True;
2352 else:
2353 return False;
2354
2355 def CheckSumSupportAlt(checkfor, guaranteed=True):
2356 if(guaranteed):
2357 hash_list = sorted(list(hashlib.algorithms_guaranteed));
2358 else:
2359 hash_list = sorted(list(hashlib.algorithms_available));
2360 checklistout = hash_list;
2361 if(checkfor in checklistout):
2362 return True;
2363 else:
2364 return False;
2365
2366 def PackArchiveFile(infiles, outfile, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
2367 advancedlist = formatspecs[8];
2368 altinode = formatspecs[9];
2369 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2370 outfile = RemoveWindowsPath(outfile);
2371 checksumtype = checksumtype.lower();
2372 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
2373 checksumtype="crc32";
2374 if(checksumtype=="none"):
2375 checksumtype = "";
2376 if(not compression or compression or compression=="catfile" or compression==formatspecs[2]):
2377 compression = None;
2378 if(compression not in compressionlist and compression is None):
2379 compression = "auto";
2380 if(verbose):
2381 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2382 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2383 if(os.path.exists(outfile)):
2384 try:
2385 os.unlink(outfile);
2386 except OSError as e:
2387 pass;
2388 if(outfile=="-"):
2389 verbose = False;
2390 catfp = BytesIO();
2391 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2392 catfp = outfile;
2393 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2394 catfp = BytesIO();
2395 else:
2396 fbasename = os.path.splitext(outfile)[0];
2397 fextname = os.path.splitext(outfile)[1];
2398 catfp = CompressOpenFile(outfile, compressionlevel);
2399 catver = formatspecs[6];
2400 fileheaderver = str(int(catver.replace(".", "")));
2401 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
2402 catfp.write(fileheader.encode('UTF-8'));
2403 infilelist = [];
2404 if(infiles=="-"):
2405 for line in sys.stdin:
2406 infilelist.append(line.strip());
2407 infilelist = list(filter(None, infilelist));
2408 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
2409 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
2410 return False;
2411 with UncompressFile(infiles, formatspecs, "r") as finfile:
2412 for line in finfile:
2413 infilelist.append(line.strip());
2414 infilelist = list(filter(None, infilelist));
2415 else:
2416 if(isinstance(infiles, (list, tuple, ))):
2417 infilelist = list(filter(None, infiles));
2418 elif(isinstance(infiles, (str, ))):
2419 infilelist = list(filter(None, [infiles]));
2420 if(advancedlist):
2421 GetDirList = ListDirAdvanced(infilelist, followlink, False);
2422 else:
2423 GetDirList = ListDir(infilelist, followlink, False);
2424 if(not GetDirList):
2425 return False;
2426 curinode = 0;
2427 curfid = 0;
2428 inodelist = [];
2429 inodetofile = {};
2430 filetoinode = {};
2431 inodetocatinode = {};
2432 numfiles = int(len(GetDirList));
2433 fnumfiles = format(int(len(GetDirList)), 'x').lower();
2434 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs[5]);
2435 catfileheadercshex = GetFileChecksum(fileheader + fnumfilesa, checksumtype, True, formatspecs);
2436 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs[5]);
2437 catfp.write(fnumfilesa.encode('UTF-8'));
2438 try:
2439 catfp.flush();
2440 os.fsync(catfp.fileno());
2441 except io.UnsupportedOperation:
2442 pass;
2443 except AttributeError:
2444 pass;
2445 except OSError as e:
2446 pass;
2447 for curfname in GetDirList:
2448 if(re.findall("^[.|/]", curfname)):
2449 fname = curfname;
2450 else:
2451 fname = "./"+curfname;
2452 if(verbose):
2453 VerbosePrintOut(fname);
2454 if(not followlink or followlink is None):
2455 fstatinfo = os.lstat(fname);
2456 else:
2457 fstatinfo = os.stat(fname);
2458 fpremode = fstatinfo.st_mode;
2459 finode = fstatinfo.st_ino;
2460 flinkcount = fstatinfo.st_nlink;
2461 ftype = 0;
2462 if(stat.S_ISREG(fpremode)):
2463 ftype = 0;
2464 elif(stat.S_ISLNK(fpremode)):
2465 ftype = 2;
2466 elif(stat.S_ISCHR(fpremode)):
2467 ftype = 3;
2468 elif(stat.S_ISBLK(fpremode)):
2469 ftype = 4;
2470 elif(stat.S_ISDIR(fpremode)):
2471 ftype = 5;
2472 elif(stat.S_ISFIFO(fpremode)):
2473 ftype = 6;
2474 elif(stat.S_ISSOCK(fpremode)):
2475 ftype = 8;
2476 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
2477 ftype = 9;
2478 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
2479 ftype = 10;
2480 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
2481 ftype = 111;
2482 else:
2483 ftype = 0;
2484 flinkname = "";
2485 fcurfid = format(int(curfid), 'x').lower();
2486 if(not followlink and finode!=0):
2487 if(ftype!=1):
2488 if(finode in inodelist):
2489 ftype = 1;
2490 flinkname = inodetofile[finode];
2491 if(altinode):
2492 fcurinode = format(int(finode), 'x').lower();
2493 else:
2494 fcurinode = format(int(inodetocatinode[finode]), 'x').lower();
2495 if(finode not in inodelist):
2496 inodelist.append(finode);
2497 inodetofile.update({finode: fname});
2498 inodetocatinode.update({finode: curinode});
2499 if(altinode):
2500 fcurinode = format(int(finode), 'x').lower();
2501 else:
2502 fcurinode = format(int(curinode), 'x').lower();
2503 curinode = curinode + 1;
2504 else:
2505 fcurinode = format(int(curinode), 'x').lower();
2506 curinode = curinode + 1;
2507 curfid = curfid + 1;
2508 if(ftype==2):
2509 flinkname = os.readlink(fname);
2510 fdev = fstatinfo.st_dev;
2511 getfdev = GetDevMajorMinor(fdev);
2512 fdev_minor = getfdev[0];
2513 fdev_major = getfdev[1];
2514 frdev = fstatinfo.st_dev;
2515 if(hasattr(fstatinfo, "st_rdev")):
2516 frdev = fstatinfo.st_rdev;
2517 else:
2518 frdev = fstatinfo.st_dev;
2519 getfrdev = GetDevMajorMinor(frdev);
2520 frdev_minor = getfrdev[0];
2521 frdev_major = getfrdev[1];
2522 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
2523 fsize = format(int("0"), 'x').lower();
2524 elif(ftype==0 or ftype==7):
2525 fsize = format(int(fstatinfo.st_size), 'x').lower();
2526 else:
2527 fsize = format(int(fstatinfo.st_size)).lower();
2528 fatime = format(int(fstatinfo.st_atime), 'x').lower();
2529 fmtime = format(int(fstatinfo.st_mtime), 'x').lower();
2530 fctime = format(int(fstatinfo.st_ctime), 'x').lower();
2531 if(hasattr(fstatinfo, "st_birthtime")):
2532 fbtime = format(int(fstatinfo.st_birthtime), 'x').lower();
2533 else:
2534 fbtime = format(int(fstatinfo.st_ctime), 'x').lower();
2535 fmode = format(int(fstatinfo.st_mode), 'x').lower();
2536 fchmode = format(int(stat.S_IMODE(fstatinfo.st_mode)), 'x').lower();
2537 ftypemod = format(int(stat.S_IFMT(fstatinfo.st_mode)), 'x').lower();
2538 fuid = format(int(fstatinfo.st_uid), 'x').lower();
2539 fgid = format(int(fstatinfo.st_gid), 'x').lower();
2540 funame = "";
2541 try:
2542 import pwd;
2543 try:
2544 userinfo = pwd.getpwuid(fstatinfo.st_uid);
2545 funame = userinfo.pw_name;
2546 except KeyError:
2547 funame = "";
2548 except ImportError:
2549 funame = "";
2550 fgname = "";
2551 try:
2552 import grp;
2553 try:
2554 groupinfo = grp.getgrgid(fstatinfo.st_gid);
2555 fgname = groupinfo.gr_name;
2556 except KeyError:
2557 fgname = "";
2558 except ImportError:
2559 fgname = "";
2560 fdev_minor = format(int(fdev_minor), 'x').lower();
2561 fdev_major = format(int(fdev_major), 'x').lower();
2562 frdev_minor = format(int(frdev_minor), 'x').lower();
2563 frdev_major = format(int(frdev_major), 'x').lower();
2564 finode = format(int(finode), 'x').lower();
2565 flinkcount = format(int(flinkcount), 'x').lower();
2566 if(hasattr(fstatinfo, "st_file_attributes")):
2567 fwinattributes = format(int(fstatinfo.st_file_attributes), 'x').lower();
2568 else:
2569 fwinattributes = format(int(0), 'x').lower();
2570 fcontents = BytesIO();
2571 if(ftype==0 or ftype==7):
2572 with open(fname, "rb") as fpc:
2573 shutil.copyfileobj(fpc, fcontents);
2574 if(followlink and (ftype==1 or ftype==2)):
2575 flstatinfo = os.stat(flinkname);
2576 with open(flinkname, "rb") as fpc:
2577 shutil.copyfileobj(fpc, fcontents);
2578 fcontents.seek(0, 0);
2579 ftypehex = format(ftype, 'x').lower();
2580 extrafields = format(len(extradata), 'x').lower();
2581 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
2582 if(len(extradata)>0):
2583 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
2584 extrasizelen = format(len(extrasizestr), 'x').lower();
2585 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, extrasizelen, extrafields];
2586 catoutlen = len(catoutlist) + len(extradata) + 3;
2587 catoutlenhex = format(catoutlen, 'x').lower();
2588 catoutlist.insert(0, catoutlenhex);
2589 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
2590 if(len(extradata)>0):
2591 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
2592 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
2593 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
2594 fcontents.seek(0, 0);
2595 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
2596 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
2597 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
2598 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
2599 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
2600 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
2601 catfileoutstrecd = catfileoutstr.encode('UTF-8');
2602 nullstrecd = formatspecs[5].encode('UTF-8');
2603 fcontents.seek(0, 0);
2604 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
2605 catfp.write(catfileout);
2606 try:
2607 catfp.flush();
2608 os.fsync(catfp.fileno());
2609 except io.UnsupportedOperation:
2610 pass;
2611 except AttributeError:
2612 pass;
2613 except OSError as e:
2614 pass;
2615 if(numfiles>0):
2616 catfp.write(AppendNullBytes([0, 0], formatspecs[5]).encode("UTF-8"));
2617 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
2618 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2619 try:
2620 catfp.flush();
2621 os.fsync(catfp.fileno());
2622 except io.UnsupportedOperation:
2623 pass;
2624 except AttributeError:
2625 pass;
2626 except OSError as e:
2627 pass;
2628 if(outfile=="-"):
2629 catfp.seek(0, 0);
2630 if(hasattr(sys.stdout, "buffer")):
2631 shutil.copyfileobj(catfp, sys.stdout.buffer);
2632 else:
2633 shutil.copyfileobj(catfp, sys.stdout);
2634 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2635 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2636 catfp.seek(0, 0);
2637 upload_file_to_internet_file(catfp, outfile);
2638 if(returnfp):
2639 catfp.seek(0, 0);
2640 return catfp;
2641 else:
2642 catfp.close();
2643 return True;
2644
2645 create_alias_function("Pack", __file_format_name__, "", PackArchiveFile);
2646
2647 if(hasattr(shutil, "register_archive_format")):
2648 def PackArchiveFileFunc(archive_name, source_dir, **kwargs):
2649 return PackArchiveFile(source_dir, archive_name, False, "auto", None, False, "crc32", [], __file_format_delimiter__, False, False);
2650 create_alias_function("Pack", __file_format_name__, "Func", PackArchiveFileFunc);
2651
2652 def PackArchiveFileFromDirList(infiles, outfile, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
2653 return PackArchiveFile(infiles, outfile, dirlistfromtxt, compression, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, returnfp);
2654
2655 create_alias_function("Pack", __file_format_name__, "FromDirList", PackArchiveFileFromDirList);
2656
2657 def PackArchiveFileFromTarFile(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
2658 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2659 outfile = RemoveWindowsPath(outfile);
2660 checksumtype = checksumtype.lower();
2661 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
2662 checksumtype="crc32";
2663 if(checksumtype=="none"):
2664 checksumtype = "";
2665 if(not compression or compression or compression=="catfile" or compression==formatspecs[2]):
2666 compression = None;
2667 if(compression not in compressionlist and compression is None):
2668 compression = "auto";
2669 if(verbose):
2670 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2671 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2672 if(os.path.exists(outfile)):
2673 try:
2674 os.unlink(outfile);
2675 except OSError as e:
2676 pass;
2677 if(outfile=="-"):
2678 verbose = False;
2679 catfp = BytesIO();
2680 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2681 catfp = outfile;
2682 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2683 catfp = BytesIO();
2684 else:
2685 fbasename = os.path.splitext(outfile)[0];
2686 fextname = os.path.splitext(outfile)[1];
2687 catfp = CompressOpenFile(outfile, compressionlevel);
2688 catver = formatspecs[6];
2689 fileheaderver = str(int(catver.replace(".", "")));
2690 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
2691 catfp.write(fileheader.encode('UTF-8'));
2692 curinode = 0;
2693 curfid = 0;
2694 inodelist = [];
2695 inodetofile = {};
2696 filetoinode = {};
2697 inodetocatinode = {};
2698 if(infile=="-"):
2699 infile = BytesIO();
2700 if(hasattr(sys.stdin, "buffer")):
2701 shutil.copyfileobj(sys.stdin.buffer, infile);
2702 else:
2703 shutil.copyfileobj(sys.stdin, infile);
2704 infile.seek(0, 0);
2705 if(not infile):
2706 return False;
2707 infile.seek(0, 0);
2708 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
2709 infile = download_file_from_internet_file(infile);
2710 infile.seek(0, 0);
2711 if(not infile):
2712 return False;
2713 infile.seek(0, 0);
2714 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
2715 return False;
2716 elif(os.path.exists(infile) and os.path.isfile(infile)):
2717 try:
2718 if(not tarfile.is_tarfile(infile)):
2719 return False;
2720 except AttributeError:
2721 if(not is_tarfile(infile)):
2722 return False;
2723 else:
2724 return False;
2725 try:
2726 if(hasattr(infile, "read") or hasattr(infile, "write")):
2727 tarfp = tarfile.open(fileobj=infile, mode="r");
2728 else:
2729 tarfp = tarfile.open(infile, "r");
2730 except FileNotFoundError:
2731 return False;
2732 numfiles = int(len(tarfp.getmembers()));
2733 fnumfiles = format(int(len(tarfp.getmembers())), 'x').lower();
2734 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs[5]);
2735 catfileheadercshex = GetFileChecksum(fileheader + fnumfilesa, checksumtype, True, formatspecs);
2736 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs[5]);
2737 catfp.write(fnumfilesa.encode('UTF-8'));
2738 try:
2739 catfp.flush();
2740 os.fsync(catfp.fileno());
2741 except io.UnsupportedOperation:
2742 pass;
2743 except AttributeError:
2744 pass;
2745 except OSError as e:
2746 pass;
2747 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
2748 if(re.findall("^[.|/]", member.name)):
2749 fname = member.name;
2750 else:
2751 fname = "./"+member.name;
2752 if(verbose):
2753 VerbosePrintOut(fname);
2754 fpremode = member.mode;
2755 ffullmode = member.mode;
2756 flinkcount = 0;
2757 ftype = 0;
2758 if(member.isreg()):
2759 ffullmode = member.mode + stat.S_IFREG;
2760 ftype = 0;
2761 elif(member.isdev()):
2762 ffullmode = member.mode;
2763 ftype = 7;
2764 elif(member.islnk()):
2765 ffullmode = member.mode + stat.S_IFREG;
2766 ftype = 1;
2767 elif(member.issym()):
2768 ffullmode = member.mode + stat.S_IFLNK;
2769 ftype = 2;
2770 elif(member.ischr()):
2771 ffullmode = member.mode + stat.S_IFCHR;
2772 ftype = 3;
2773 elif(member.isblk()):
2774 ffullmode = member.mode + stat.S_IFBLK;
2775 ftype = 4;
2776 elif(member.isdir()):
2777 ffullmode = member.mode + stat.S_IFDIR;
2778 ftype = 5;
2779 elif(member.isfifo()):
2780 ffullmode = member.mode + stat.S_IFIFO;
2781 ftype = 6;
2782 elif(member.issparse()):
2783 ffullmode = member.mode;
2784 ftype = 12;
2785 else:
2786 ffullmode = member.mode;
2787 ftype = 0;
2788 flinkname = "";
2789 fcurfid = format(int(curfid), 'x').lower();
2790 fcurinode = format(int(curfid), 'x').lower();
2791 curfid = curfid + 1;
2792 if(ftype==2):
2793 flinkname = member.linkname;
2794 fdev_minor = format(int(member.devminor), 'x').lower();
2795 fdev_major = format(int(member.devmajor), 'x').lower();
2796 frdev_minor = format(int(member.devminor), 'x').lower();
2797 frdev_major = format(int(member.devmajor), 'x').lower();
2798 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
2799 fsize = format(int("0"), 'x').lower();
2800 elif(ftype==0 or ftype==7):
2801 fsize = format(int(member.size), 'x').lower();
2802 else:
2803 fsize = format(int(member.size), 'x').lower();
2804 fatime = format(int(member.mtime), 'x').lower();
2805 fmtime = format(int(member.mtime), 'x').lower();
2806 fctime = format(int(member.mtime), 'x').lower();
2807 fbtime = format(int(member.mtime), 'x').lower();
2808 fmode = format(int(ffullmode), 'x').lower();
2809 fchmode = format(int(stat.S_IMODE(ffullmode)), 'x').lower();
2810 ftypemod = format(int(stat.S_IFMT(ffullmode)), 'x').lower();
2811 fuid = format(int(member.uid), 'x').lower();
2812 fgid = format(int(member.gid), 'x').lower();
2813 funame = member.uname;
2814 fgname = member.gname;
2815 flinkcount = format(int(flinkcount), 'x').lower();
2816 fwinattributes = format(int(0), 'x').lower();
2817 fcontents = BytesIO();
2818 if(ftype==0 or ftype==7):
2819 with tarfp.extractfile(member) as fpc:
2820 shutil.copyfileobj(fpc, fcontents);
2821 fcontents.seek(0, 0);
2822 ftypehex = format(ftype, 'x').lower();
2823 extrafields = format(len(extradata), 'x').lower();
2824 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
2825 if(len(extradata)>0):
2826 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
2827 extrasizelen = format(len(extrasizestr), 'x').lower();
2828 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, extrasizelen, extrafields];
2829 catoutlen = len(catoutlist) + len(extradata) + 3;
2830 catoutlenhex = format(catoutlen, 'x').lower();
2831 catoutlist.insert(0, catoutlenhex);
2832 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
2833 if(len(extradata)>0):
2834 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
2835 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
2836 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
2837 fcontents.seek(0, 0);
2838 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
2839 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
2840 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
2841 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
2842 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
2843 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
2844 catfileoutstrecd = catfileoutstr.encode('UTF-8');
2845 nullstrecd = formatspecs[5].encode('UTF-8');
2846 fcontents.seek(0, 0);
2847 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
2848 catfp.write(catfileout);
2849 try:
2850 catfp.flush();
2851 os.fsync(catfp.fileno());
2852 except io.UnsupportedOperation:
2853 pass;
2854 except AttributeError:
2855 pass;
2856 except OSError as e:
2857 pass;
2858 if(numfiles>0):
2859 catfp.write(AppendNullBytes([0, 0], formatspecs[5]).encode("UTF-8"));
2860 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
2861 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2862 try:
2863 catfp.flush();
2864 os.fsync(catfp.fileno());
2865 except io.UnsupportedOperation:
2866 pass;
2867 except AttributeError:
2868 pass;
2869 except OSError as e:
2870 pass;
2871 if(outfile=="-"):
2872 catfp.seek(0, 0);
2873 if(hasattr(sys.stdout, "buffer")):
2874 shutil.copyfileobj(catfp, sys.stdout.buffer);
2875 else:
2876 shutil.copyfileobj(catfp, sys.stdout);
2877 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2878 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
2879 catfp.seek(0, 0);
2880 upload_file_to_internet_file(catfp, outfile);
2881 if(returnfp):
2882 catfp.seek(0, 0);
2883 return catfp;
2884 else:
2885 catfp.close();
2886 return True;
2887
2888 create_alias_function("Pack", __file_format_name__, "FromTarFile", PackArchiveFileFromTarFile);
2889
2890 def PackArchiveFileFromZipFile(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
2891 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2892 outfile = RemoveWindowsPath(outfile);
2893 checksumtype = checksumtype.lower();
2894 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
2895 checksumtype="crc32";
2896 if(checksumtype=="none"):
2897 checksumtype = "";
2898 if(not compression or compression or compression=="catfile" or compression==formatspecs[2]):
2899 compression = None;
2900 if(compression not in compressionlist and compression is None):
2901 compression = "auto";
2902 if(verbose):
2903 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
2904 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
2905 if(os.path.exists(outfile)):
2906 try:
2907 os.unlink(outfile);
2908 except OSError as e:
2909 pass;
2910 if(outfile=="-"):
2911 verbose = False;
2912 catfp = BytesIO();
2913 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
2914 catfp = outfile;
2915 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
2916 catfp = BytesIO();
2917 else:
2918 fbasename = os.path.splitext(outfile)[0];
2919 fextname = os.path.splitext(outfile)[1];
2920 catfp = CompressOpenFile(outfile, compressionlevel);
2921 catver = formatspecs[6];
2922 fileheaderver = str(int(catver.replace(".", "")));
2923 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
2924 catfp.write(fileheader.encode('UTF-8'));
2925 curinode = 0;
2926 curfid = 0;
2927 inodelist = [];
2928 inodetofile = {};
2929 filetoinode = {};
2930 inodetocatinode = {};
2931 if(infile=="-"):
2932 infile = BytesIO();
2933 if(hasattr(sys.stdin, "buffer")):
2934 shutil.copyfileobj(sys.stdin.buffer, infile);
2935 else:
2936 shutil.copyfileobj(sys.stdin, infile);
2937 infile.seek(0, 0);
2938 if(not infile):
2939 return False;
2940 infile.seek(0, 0);
2941 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
2942 infile = download_file_from_internet_file(infile);
2943 infile.seek(0, 0);
2944 if(not infile):
2945 return False;
2946 infile.seek(0, 0);
2947 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
2948 return False;
2949 else:
2950 return False;
2951 if(not zipfile.is_zipfile(infile)):
2952 return False;
2953 try:
2954 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
2955 except FileNotFoundError:
2956 return False;
2957 ziptest = zipfp.testzip();
2958 if(ziptest):
2959 VerbosePrintOut("Bad file found!");
2960 numfiles = int(len(zipfp.infolist()));
2961 fnumfiles = format(int(len(zipfp.infolist())), 'x').lower();
2962 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs[5]);
2963 catfileheadercshex = GetFileChecksum(fileheader + fnumfilesa, checksumtype, True, formatspecs);
2964 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs[5]);
2965 catfp.write(fnumfilesa.encode('UTF-8'));
2966 try:
2967 catfp.flush();
2968 os.fsync(catfp.fileno());
2969 except io.UnsupportedOperation:
2970 pass;
2971 except AttributeError:
2972 pass;
2973 except OSError as e:
2974 pass;
2975 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
2976 if(re.findall("^[.|/]", member.filename)):
2977 fname = member.filename;
2978 else:
2979 fname = "./"+member.filename;
2980 zipinfo = zipfp.getinfo(member.filename);
2981 if(verbose):
2982 VerbosePrintOut(fname);
2983 if(not member.is_dir()):
2984 fpremode = int(stat.S_IFREG + 438);
2985 elif(member.is_dir()):
2986 fpremode = int(stat.S_IFDIR + 511);
2987 flinkcount = 0;
2988 ftype = 0;
2989 if(not member.is_dir()):
2990 ftype = 0;
2991 elif(member.is_dir()):
2992 ftype = 5;
2993 flinkname = "";
2994 fcurfid = format(int(curfid), 'x').lower();
2995 fcurinode = format(int(curfid), 'x').lower();
2996 curfid = curfid + 1;
2997 fdev_minor = format(int(0), 'x').lower();
2998 fdev_major = format(int(0), 'x').lower();
2999 frdev_minor = format(int(0), 'x').lower();
3000 frdev_major = format(int(0), 'x').lower();
3001 if(ftype==5):
3002 fsize = format(int("0"), 'x').lower();
3003 elif(ftype==0):
3004 fsize = format(int(member.file_size), 'x').lower();
3005 else:
3006 fsize = format(int(member.file_size), 'x').lower();
3007 fatime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3008 fmtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3009 fctime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3010 fbtime = format(int(time.mktime(member.date_time + (0, 0, -1))), 'x').lower();
3011 if(zipinfo.create_system==0 or zipinfo.create_system==10):
3012 fwinattributes = format(int(zipinfo.external_attr), 'x').lower();
3013 if(not member.is_dir()):
3014 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3015 fchmode = stat.S_IMODE(fmode);
3016 ftypemod = stat.S_IFMT(fmode);
3017 elif(member.is_dir()):
3018 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3019 fchmode = stat.S_IMODE(fmode);
3020 ftypemod = stat.S_IFMT(fmode);
3021 elif(zipinfo.create_system==3):
3022 fwinattributes = format(int(0), 'x').lower();
3023 fmode = format(int(zipinfo.external_attr), 'x').lower();
3024 fchmode = stat.S_IMODE(fmode);
3025 ftypemod = stat.S_IFMT(fmode);
3026 else:
3027 fwinattributes = format(int(0), 'x').lower();
3028 if(not member.is_dir()):
3029 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3030 fchmode = stat.S_IMODE(fmode);
3031 ftypemod = stat.S_IFMT(fmode);
3032 elif(member.is_dir()):
3033 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3034 fchmode = stat.S_IMODE(fmode);
3035 ftypemod = stat.S_IFMT(fmode);
3036 try:
3037 fuid = format(int(os.getuid()), 'x').lower();
3038 except AttributeError:
3039 fuid = format(int(0), 'x').lower();
3040 except KeyError:
3041 fuid = format(int(0), 'x').lower();
3042 try:
3043 fgid = format(int(os.getgid()), 'x').lower();
3044 except AttributeError:
3045 fgid = format(int(0), 'x').lower();
3046 except KeyError:
3047 fgid = format(int(0), 'x').lower();
3048 try:
3049 import pwd;
3050 try:
3051 userinfo = pwd.getpwuid(os.getuid());
3052 funame = userinfo.pw_name;
3053 except KeyError:
3054 funame = "";
3055 except AttributeError:
3056 funame = "";
3057 except ImportError:
3058 funame = "";
3059 fgname = "";
3060 try:
3061 import grp;
3062 try:
3063 groupinfo = grp.getgrgid(os.getgid());
3064 fgname = groupinfo.gr_name;
3065 except KeyError:
3066 fgname = "";
3067 except AttributeError:
3068 fgname = "";
3069 except ImportError:
3070 fgname = "";
3071 fcontents = BytesIO();
3072 if(ftype==0):
3073 fcontents.write(zipfp.read(member.filename));
3074 fcontents.seek(0, 0);
3075 ftypehex = format(ftype, 'x').lower();
3076 extrafields = format(len(extradata), 'x').lower();
3077 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
3078 if(len(extradata)>0):
3079 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
3080 extrasizelen = format(len(extrasizestr), 'x').lower();
3081 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, extrasizelen, extrafields];
3082 catoutlen = len(catoutlist) + len(extradata) + 3;
3083 catoutlenhex = format(catoutlen, 'x').lower();
3084 catoutlist.insert(0, catoutlenhex);
3085 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
3086 if(len(extradata)>0):
3087 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
3088 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
3089 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
3090 fcontents.seek(0, 0);
3091 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
3092 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
3093 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
3094 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
3095 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
3096 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
3097 catfileoutstrecd = catfileoutstr.encode('UTF-8');
3098 nullstrecd = formatspecs[5].encode('UTF-8');
3099 fcontents.seek(0, 0);
3100 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
3101 catfp.write(catfileout);
3102 try:
3103 catfp.flush();
3104 os.fsync(catfp.fileno());
3105 except io.UnsupportedOperation:
3106 pass;
3107 except AttributeError:
3108 pass;
3109 except OSError as e:
3110 pass;
3111 if(numfiles>0):
3112 catfp.write(AppendNullBytes([0, 0], formatspecs[5]).encode("UTF-8"));
3113 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3114 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3115 try:
3116 catfp.flush();
3117 os.fsync(catfp.fileno());
3118 except io.UnsupportedOperation:
3119 pass;
3120 except AttributeError:
3121 pass;
3122 except OSError as e:
3123 pass;
3124 if(outfile=="-"):
3125 catfp.seek(0, 0);
3126 if(hasattr(sys.stdout, "buffer")):
3127 shutil.copyfileobj(catfp, sys.stdout.buffer);
3128 else:
3129 shutil.copyfileobj(catfp, sys.stdout);
3130 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3131 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3132 catfp.seek(0, 0);
3133 upload_file_to_internet_file(catfp, outfile);
3134 if(returnfp):
3135 catfp.seek(0, 0);
3136 return catfp;
3137 else:
3138 catfp.close();
3139 return True;
3140
3141 create_alias_function("Pack", __file_format_name__, "FromZipFile", PackArchiveFileFromZipFile);
3142
3143 if(not rarfile_support):
3144 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
3145 return False
3146
3147 if(rarfile_support):
3148 def PackArchiveFileFromRarFile(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
3149 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3150 outfile = RemoveWindowsPath(outfile);
3151 checksumtype = checksumtype.lower();
3152 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3153 checksumtype="crc32";
3154 if(checksumtype=="none"):
3155 checksumtype = "";
3156 if(not compression or compression or compression=="catfile" or compression==formatspecs[2]):
3157 compression = None;
3158 if(compression not in compressionlist and compression is None):
3159 compression = "auto";
3160 if(verbose):
3161 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3162 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3163 if(os.path.exists(outfile)):
3164 try:
3165 os.unlink(outfile);
3166 except OSError as e:
3167 pass;
3168 if(outfile=="-"):
3169 verbose = False;
3170 catfp = BytesIO();
3171 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3172 catfp = outfile;
3173 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3174 catfp = BytesIO();
3175 else:
3176 fbasename = os.path.splitext(outfile)[0];
3177 fextname = os.path.splitext(outfile)[1];
3178 catfp = CompressOpenFile(outfile, compressionlevel);
3179 catver = formatspecs[6];
3180 fileheaderver = str(int(catver.replace(".", "")));
3181 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
3182 catfp.write(fileheader.encode('UTF-8'));
3183 curinode = 0;
3184 curfid = 0;
3185 inodelist = [];
3186 inodetofile = {};
3187 filetoinode = {};
3188 inodetocatinode = {};
3189 if(not os.path.exists(infile) or not os.path.isfile(infile)):
3190 return False;
3191 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
3192 return False;
3193 rarfp = rarfile.RarFile(infile, "r");
3194 rartest = rarfp.testrar();
3195 if(rartest):
3196 VerbosePrintOut("Bad file found!");
3197 numfiles = int(len(rarfp.infolist()));
3198 fnumfiles = format(int(len(rarfp.infolist())), 'x').lower();
3199 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs[5]);
3200 catfileheadercshex = GetFileChecksum(fileheader + fnumfilesa, checksumtype, True, formatspecs);
3201 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs[5]);
3202 catfp.write(fnumfilesa.encode('UTF-8'));
3203 try:
3204 catfp.flush();
3205 os.fsync(catfp.fileno());
3206 except io.UnsupportedOperation:
3207 pass;
3208 except AttributeError:
3209 pass;
3210 except OSError as e:
3211 pass;
3212 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
3213 is_unix = False;
3214 is_windows = False;
3215 if(member.host_os==rarfile.RAR_OS_UNIX):
3216 is_windows = False;
3217 try:
3218 member.external_attr
3219 is_unix = True;
3220 except AttributeError:
3221 is_unix = False;
3222 elif(member.host_os==rarfile.RAR_OS_WIN32):
3223 is_unix = False;
3224 try:
3225 member.external_attr
3226 is_windows = True;
3227 except AttributeError:
3228 is_windows = False;
3229 else:
3230 is_unix = False;
3231 is_windows = False;
3232 if(re.findall("^[.|/]", member.filename)):
3233 fname = member.filename;
3234 else:
3235 fname = "./"+member.filename;
3236 rarinfo = rarfp.getinfo(member.filename);
3237 if(verbose):
3238 VerbosePrintOut(fname);
3239 if(is_unix and member.external_attr !=0):
3240 fpremode = int(member.external_attr);
3241 elif(member.is_file()):
3242 fpremode = int(stat.S_IFREG + 438);
3243 elif(member.is_symlink()):
3244 fpremode = int(stat.S_IFLNK + 438);
3245 elif(member.is_dir()):
3246 fpremode = int(stat.S_IFDIR + 511);
3247 if(is_windows and member.external_attr !=0):
3248 fwinattributes = format(int(member.external_attr), 'x').lower();
3249 else:
3250 fwinattributes = format(int(0), 'x').lower();
3251 flinkcount = 0;
3252 ftype = 0;
3253 if(member.is_file()):
3254 ftype = 0;
3255 elif(member.is_symlink()):
3256 ftype = 2;
3257 elif(member.is_dir()):
3258 ftype = 5;
3259 flinkname = "";
3260 if(ftype==2):
3261 flinkname = rarfp.read(member.filename).decode("UTF-8");
3262 fcurfid = format(int(curfid), 'x').lower();
3263 fcurinode = format(int(curfid), 'x').lower();
3264 curfid = curfid + 1;
3265 fdev_minor = format(int(0), 'x').lower();
3266 fdev_major = format(int(0), 'x').lower();
3267 frdev_minor = format(int(0), 'x').lower();
3268 frdev_major = format(int(0), 'x').lower();
3269 if(ftype==5):
3270 fsize = format(int("0"), 'x').lower();
3271 elif(ftype==0):
3272 fsize = format(int(member.file_size), 'x').lower();
3273 else:
3274 fsize = format(int(member.file_size), 'x').lower();
3275 try:
3276 if(member.atime):
3277 fatime = format(int(member.atime.timestamp()), 'x').lower();
3278 else:
3279 fatime = format(int(member.mtime.timestamp()), 'x').lower();
3280 except AttributeError:
3281 fatime = format(int(member.mtime.timestamp()), 'x').lower();
3282 fmtime = format(int(member.mtime.timestamp()), 'x').lower();
3283 try:
3284 if(member.ctime):
3285 fctime = format(int(member.ctime.timestamp()), 'x').lower();
3286 else:
3287 fctime = format(int(member.mtime.timestamp()), 'x').lower();
3288 except AttributeError:
3289 fctime = format(int(member.mtime.timestamp()), 'x').lower();
3290 fbtime = format(int(member.mtime.timestamp()), 'x').lower();
3291 if(is_unix and member.external_attr !=0):
3292 fmode = format(int(member.external_attr), 'x').lower();
3293 fchmode = format(int(stat.S_IMODE(member.external_attr)), 'x').lower();
3294 ftypemod = format(int(stat.S_IFMT(member.external_attr)), 'x').lower();
3295 elif(member.is_file()):
3296 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3297 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3298 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3299 elif(member.is_symlink()):
3300 fmode = format(int(stat.S_IFLNK + 438), 'x').lower();
3301 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3302 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3303 elif(member.is_dir()):
3304 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3305 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
3306 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
3307 try:
3308 fuid = format(int(os.getuid()), 'x').lower();
3309 except AttributeError:
3310 fuid = format(int(0), 'x').lower();
3311 except KeyError:
3312 fuid = format(int(0), 'x').lower();
3313 try:
3314 fgid = format(int(os.getgid()), 'x').lower();
3315 except AttributeError:
3316 fgid = format(int(0), 'x').lower();
3317 except KeyError:
3318 fgid = format(int(0), 'x').lower();
3319 try:
3320 import pwd;
3321 try:
3322 userinfo = pwd.getpwuid(os.getuid());
3323 funame = userinfo.pw_name;
3324 except KeyError:
3325 funame = "";
3326 except AttributeError:
3327 funame = "";
3328 except ImportError:
3329 funame = "";
3330 fgname = "";
3331 try:
3332 import grp;
3333 try:
3334 groupinfo = grp.getgrgid(os.getgid());
3335 fgname = groupinfo.gr_name;
3336 except KeyError:
3337 fgname = "";
3338 except AttributeError:
3339 fgname = "";
3340 except ImportError:
3341 fgname = "";
3342 fcontents = BytesIO();
3343 if(ftype==0):
3344 fcontents.write(rarfp.read(member.filename));
3345 fcontents.seek(0, 0);
3346 ftypehex = format(ftype, 'x').lower();
3347 extrafields = format(len(extradata), 'x').lower();
3348 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
3349 if(len(extradata)>0):
3350 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
3351 extrasizelen = format(len(extrasizestr), 'x').lower();
3352 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, extrasizelen, extrafields];
3353 catoutlen = len(catoutlist) + len(extradata) + 3;
3354 catoutlenhex = format(catoutlen, 'x').lower();
3355 catoutlist.insert(0, catoutlenhex);
3356 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
3357 if(len(extradata)>0):
3358 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
3359 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
3360 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
3361 fcontents.seek(0, 0);
3362 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
3363 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
3364 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
3365 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
3366 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
3367 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5])
3368 catfileoutstrecd = catfileoutstr.encode('UTF-8');
3369 nullstrecd = formatspecs[5].encode('UTF-8');
3370 fcontents.seek(0, 0);
3371 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
3372 catfp.write(catfileout);
3373 try:
3374 catfp.flush();
3375 os.fsync(catfp.fileno());
3376 except io.UnsupportedOperation:
3377 pass;
3378 except AttributeError:
3379 pass;
3380 except OSError as e:
3381 pass;
3382 if(numfiles>0):
3383 catfp.write(AppendNullBytes([0, 0], formatspecs[5]).encode("UTF-8"));
3384 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3385 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
3386 try:
3387 catfp.flush();
3388 os.fsync(catfp.fileno());
3389 except io.UnsupportedOperation:
3390 pass;
3391 except AttributeError:
3392 pass;
3393 except OSError as e:
3394 pass;
3395 if(outfile=="-"):
3396 catfp.seek(0, 0)
3397 if(hasattr(sys.stdout, "buffer")):
3398 shutil.copyfileobj(catfp, sys.stdout.buffer);
3399 else:
3400 shutil.copyfileobj(catfp, sys.stdout);
3401 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3402 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3403 catfp.seek(0, 0);
3404 upload_file_to_internet_file(catfp, outfile);
3405 if(returnfp):
3406 catfp.seek(0, 0)
3407 return catfp
3408 else:
3409 catfp.close()
3410 return True;
3411
3412 create_alias_function("Pack", __file_format_name__, "FromRarFile", PackArchiveFileFromRarFile);
3413
3414 if(not py7zr_support):
3415 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
3416 return False
3417
3418 if(py7zr_support):
3419 def PackArchiveFileFromSevenZipFile(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
3420 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3421 outfile = RemoveWindowsPath(outfile);
3422 checksumtype = checksumtype.lower();
3423 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
3424 checksumtype="crc32";
3425 if(checksumtype=="none"):
3426 checksumtype = "";
3427 if(not compression or compression or compression=="catfile" or compression==formatspecs[2]):
3428 compression = None;
3429 if(compression not in compressionlist and compression is None):
3430 compression = "auto";
3431 if(verbose):
3432 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3433 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
3434 if(os.path.exists(outfile)):
3435 try:
3436 os.unlink(outfile);
3437 except OSError as e:
3438 pass;
3439 if(outfile=="-"):
3440 verbose = False;
3441 catfp = BytesIO();
3442 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
3443 catfp = outfile;
3444 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3445 catfp = BytesIO();
3446 else:
3447 fbasename = os.path.splitext(outfile)[0];
3448 fextname = os.path.splitext(outfile)[1];
3449 catfp = CompressOpenFile(outfile, compressionlevel);
3450 catver = formatspecs[6];
3451 fileheaderver = str(int(catver.replace(".", "")));
3452 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
3453 catfp.write(fileheader.encode('UTF-8'));
3454 curinode = 0;
3455 curfid = 0;
3456 inodelist = [];
3457 inodetofile = {};
3458 filetoinode = {};
3459 inodetocatinode = {};
3460 if(not os.path.exists(infile) or not os.path.isfile(infile)):
3461 return False;
3462 szpfp = py7zr.SevenZipFile(infile, mode="r");
3463 file_content = szpfp.readall();
3464 #sztest = szpfp.testzip();
3465 sztestalt = szpfp.test();
3466 if(sztestalt):
3467 VerbosePrintOut("Bad file found!");
3468 numfiles = int(len(szpfp.list()));
3469 fnumfiles = format(int(len(szpfp.list())), 'x').lower();
3470 fnumfilesa = AppendNullBytes([fnumfiles, checksumtype], formatspecs[5]);
3471 catfileheadercshex = GetFileChecksum(fileheader + fnumfilesa, checksumtype, True, formatspecs);
3472 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs[5]);
3473 catfp.write(fnumfilesa.encode('UTF-8'));
3474 try:
3475 catfp.flush();
3476 os.fsync(catfp.fileno());
3477 except io.UnsupportedOperation:
3478 pass;
3479 except AttributeError:
3480 pass;
3481 except OSError as e:
3482 pass;
3483 for member in sorted(szpfp.list(), key=lambda x: x.filename):
3484 if(re.findall("^[.|/]", member.filename)):
3485 fname = member.filename;
3486 else:
3487 fname = "./"+member.filename;
3488 if(verbose):
3489 VerbosePrintOut(fname);
3490 if(not member.is_directory):
3491 fpremode = int(stat.S_IFREG + 438);
3492 elif(member.is_directory):
3493 fpremode = int(stat.S_IFDIR + 511);
3494 fwinattributes = format(int(0), 'x').lower();
3495 flinkcount = 0;
3496 ftype = 0;
3497 if(member.is_directory):
3498 ftype = 5;
3499 else:
3500 ftype = 0;
3501 flinkname = "";
3502 fcurfid = format(int(curfid), 'x').lower();
3503 fcurinode = format(int(curfid), 'x').lower();
3504 curfid = curfid + 1;
3505 fdev_minor = format(int(0), 'x').lower();
3506 fdev_major = format(int(0), 'x').lower();
3507 frdev_minor = format(int(0), 'x').lower();
3508 frdev_major = format(int(0), 'x').lower();
3509 if(ftype==5):
3510 fsize = format(int("0"), 'x').lower();
3511 fatime = format(int(member.creationtime.timestamp()), 'x').lower();
3512 fmtime = format(int(member.creationtime.timestamp()), 'x').lower();
3513 fctime = format(int(member.creationtime.timestamp()), 'x').lower();
3514 fbtime = format(int(member.creationtime.timestamp()), 'x').lower();
3515 if(member.is_directory):
3516 fmode = format(int(stat.S_IFDIR + 511), 'x').lower();
3517 fchmode = format(int(stat.S_IMODE(int(stat.S_IFDIR + 511))), 'x').lower();
3518 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFDIR + 511))), 'x').lower();
3519 else:
3520 fmode = format(int(stat.S_IFREG + 438), 'x').lower();
3521 fchmode = format(int(stat.S_IMODE(int(stat.S_IFREG + 438))), 'x').lower();
3522 ftypemod = format(int(stat.S_IFMT(int(stat.S_IFREG + 438))), 'x').lower();
3523 try:
3524 fuid = format(int(os.getuid()), 'x').lower();
3525 except AttributeError:
3526 fuid = format(int(0), 'x').lower();
3527 except KeyError:
3528 fuid = format(int(0), 'x').lower();
3529 try:
3530 fgid = format(int(os.getgid()), 'x').lower();
3531 except AttributeError:
3532 fgid = format(int(0), 'x').lower();
3533 except KeyError:
3534 fgid = format(int(0), 'x').lower();
3535 try:
3536 import pwd;
3537 try:
3538 userinfo = pwd.getpwuid(os.getuid());
3539 funame = userinfo.pw_name;
3540 except KeyError:
3541 funame = "";
3542 except AttributeError:
3543 funame = "";
3544 except ImportError:
3545 funame = "";
3546 fgname = "";
3547 try:
3548 import grp;
3549 try:
3550 groupinfo = grp.getgrgid(os.getgid());
3551 fgname = groupinfo.gr_name;
3552 except KeyError:
3553 fgname = "";
3554 except AttributeError:
3555 fgname = "";
3556 except ImportError:
3557 fgname = "";
3558 fcontents = BytesIO();
3559 if(ftype==0):
3560 fcontents.write(file_content[member.filename].read());
3561 fsize = format(fcontents.tell(), 'x').lower();
3562 file_content[member.filename].close();
3563 fcontents.seek(0, 0);
3564 ftypehex = format(ftype, 'x').lower();
3565 extrafields = format(len(extradata), 'x').lower();
3566 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
3567 if(len(extradata)>0):
3568 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
3569 extrasizelen = format(len(extrasizestr), 'x').lower();
3570 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, extrasizelen, extrafields];
3571 catoutlen = len(catoutlist) + len(extradata) + 3;
3572 catoutlenhex = format(catoutlen, 'x').lower();
3573 catoutlist.insert(0, catoutlenhex);
3574 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
3575 if(len(extradata)>0):
3576 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
3577 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
3578 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
3579 fcontents.seek(0, 0);
3580 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
3581 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
3582 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
3583 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
3584 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
3585 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5])
3586 catfileoutstrecd = catfileoutstr.encode('UTF-8');
3587 nullstrecd = formatspecs[5].encode('UTF-8');
3588 fcontents.seek(0, 0);
3589 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
3590 catfp.write(catfileout);
3591 try:
3592 catfp.flush();
3593 os.fsync(catfp.fileno());
3594 except io.UnsupportedOperation:
3595 pass;
3596 except AttributeError:
3597 pass;
3598 except OSError as e:
3599 pass;
3600 if(numfiles>0):
3601 catfp.write(AppendNullBytes([0, 0], formatspecs[5]).encode("UTF-8"));
3602 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
3603 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs)
3604 try:
3605 catfp.flush();
3606 os.fsync(catfp.fileno());
3607 except io.UnsupportedOperation:
3608 pass;
3609 except AttributeError:
3610 pass;
3611 except OSError as e:
3612 pass;
3613 if(outfile=="-"):
3614 catfp.seek(0, 0);
3615 if(hasattr(sys.stdout, "buffer")):
3616 shutil.copyfileobj(catfp, sys.stdout.buffer);
3617 else:
3618 shutil.copyfileobj(catfp, sys.stdout);
3619 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
3620 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
3621 catfp.seek(0, 0);
3622 upload_file_to_internet_file(catfp, outfile);
3623 if(returnfp):
3624 catfp.seek(0, 0)
3625 return catfp
3626 else:
3627 catfp.close()
3628 return True;
3629
3630 create_alias_function("Pack", __file_format_name__, "FromSevenZipFile", PackArchiveFileFromSevenZipFile);
3631
3632 def PackArchiveFileFromInFile(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
3633 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
3634 if(verbose):
3635 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
3636 if(checkcompressfile=="tarfile"):
3637 return PackArchiveFileFromTarFile(infile, outfile, compression, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3638 elif(checkcompressfile=="zipfile"):
3639 return PackArchiveFileFromZipFile(infile, outfile, compression, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3640 elif(checkcompressfile=="catfile"):
3641 return RePackArchiveFile(infile, outfile, compression, compressionlevel, False, 0, 0, checksumtype, False, extradata, formatspecs, verbose, returnfp);
3642 elif(rarfile_support and checkcompressfile=="rarfile"):
3643 return PackArchiveFileFromRarFile(infile, outfile, compression, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3644 elif(py7zr_support and checkcompressfile=="7zipfile"):
3645 return PackArchiveFileFromSevenZipFile(infile, outfile, compression, compressionlevel, checksumtype, extradata, formatspecs, verbose, returnfp);
3646 else:
3647 return False;
3648 return False;
3649
3650 create_alias_function("Pack", __file_format_name__, "FromInFile", PackArchiveFileFromInFile);
3651
3652 def ArchiveFileSeekToFileNum(infile, seekto=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
3653 if(hasattr(infile, "read") or hasattr(infile, "write")):
3654 catfp = infile;
3655 catfp.seek(0, 0);
3656 catfp = UncompressArchiveFile(catfp, formatspecs);
3657 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
3658 if(checkcompressfile=="tarfile"):
3659 return TarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3660 if(checkcompressfile=="zipfile"):
3661 return ZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3662 if(rarfile_support and checkcompressfile=="rarfile"):
3663 return RarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3664 if(py7zr_support and checkcompressfile=="7zipfile"):
3665 return SevenZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3666 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
3667 return False;
3668 if(not catfp):
3669 return False;
3670 catfp.seek(0, 0);
3671 elif(infile=="-"):
3672 catfp = BytesIO();
3673 if(hasattr(sys.stdin, "buffer")):
3674 shutil.copyfileobj(sys.stdin.buffer, catfp);
3675 else:
3676 shutil.copyfileobj(sys.stdin, catfp);
3677 catfp.seek(0, 0);
3678 catfp = UncompressArchiveFile(catfp, formatspecs);
3679 if(not catfp):
3680 return False;
3681 catfp.seek(0, 0);
3682 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3683 catfp = download_file_from_internet_file(infile);
3684 catfp.seek(0, 0);
3685 catfp = UncompressArchiveFile(catfp, formatspecs);
3686 if(not catfp):
3687 return False;
3688 catfp.seek(0, 0);
3689 else:
3690 infile = RemoveWindowsPath(infile);
3691 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
3692 if(checkcompressfile=="tarfile"):
3693 return TarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3694 if(checkcompressfile=="zipfile"):
3695 return ZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3696 if(rarfile_support and checkcompressfile=="rarfile"):
3697 return RarFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3698 if(py7zr_support and checkcompressfile=="7zipfile"):
3699 return SevenZipFileToArray(infile, seekto, 0, listonly, skipchecksum, formatspecs, returnfp);
3700 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
3701 return False;
3702 compresscheck = CheckCompressionType(infile, formatspecs, True);
3703 if(not compresscheck):
3704 fextname = os.path.splitext(infile)[1];
3705 if(fextname==".gz"):
3706 compresscheck = "gzip";
3707 elif(fextname==".bz2"):
3708 compresscheck = "bzip2";
3709 elif(fextname==".zst"):
3710 compresscheck = "zstd";
3711 elif(fextname==".lz4" or fextname==".clz4"):
3712 compresscheck = "lz4";
3713 elif(fextname==".lzo" or fextname==".lzop"):
3714 compresscheck = "lzo";
3715 elif(fextname==".lzma" or fextname==".xz"):
3716 compresscheck = "lzma";
3717 else:
3718 return False;
3719 if(not compresscheck):
3720 return False;
3721 catfp = UncompressFile(infile, formatspecs, "rb");
3722 '''
3723 try:
3724 catfp.seek(0, 2);
3725 except OSError:
3726 SeekToEndOfFile(catfp);
3727 except ValueError:
3728 SeekToEndOfFile(catfp);
3729 CatSize = catfp.tell();
3730 CatSizeEnd = CatSize;
3731 '''
3732 try:
3733 catfp.seek(0, 0);
3734 except OSError:
3735 return False;
3736 except ValueError:
3737 return False;
3738 curloc = catfp.tell();
3739 if(curloc>0):
3740 catfp.seek(0, 0);
3741 catheader = ReadFileHeaderData(catfp, 4, formatspecs[5]);
3742 if(curloc>0):
3743 catfp.seek(curloc, 0);
3744 catstring = catheader[0];
3745 catversion = re.findall(r"([\d]+)$", catstring);
3746 fprenumfiles = catheader[1];
3747 fnumfiles = int(fprenumfiles, 16);
3748 fprechecksumtype = catheader[2];
3749 fprechecksum = catheader[3];
3750 fileheader = AppendNullByte(catstring, formatspecs[5]);
3751 fnumfileshex = format(int(fnumfiles), 'x').lower();
3752 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs[5]);
3753 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
3754 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
3755 fheadtell = len(fileheader);
3756 if(fprechecksum!=catfileheadercshex and not skipchecksum):
3757 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
3758 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
3759 return False;
3760 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
3761 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
3762 if(seekto>=fnumfiles):
3763 seekto = fnumfiles - 1;
3764 if(seekto<0):
3765 seekto = 0;
3766 if(seekto>=0):
3767 il = -1;
3768 while(il < seekto):
3769 prefhstart = catfp.tell();
3770 if(formatspecs[7]):
3771 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs[5]);
3772 else:
3773 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs[5]);
3774 if(len(preheaderdata)==0):
3775 break;
3776 prefheadsize = int(preheaderdata[0], 16);
3777 prefnumfields = int(preheaderdata[1], 16);
3778 preftype = int(preheaderdata[2], 16);
3779 if(re.findall("^[.|/]", preheaderdata[3])):
3780 prefname = preheaderdata[3];
3781 else:
3782 prefname = "./"+preheaderdata[3];
3783 prefbasedir = os.path.dirname(prefname);
3784 preflinkname = preheaderdata[4];
3785 prefsize = int(preheaderdata[5], 16);
3786 prefatime = int(preheaderdata[6], 16);
3787 prefmtime = int(preheaderdata[7], 16);
3788 prefctime = int(preheaderdata[8], 16);
3789 prefbtime = int(preheaderdata[9], 16);
3790 prefmode = int(preheaderdata[10], 16);
3791 prefchmode = stat.S_IMODE(prefmode);
3792 preftypemod = stat.S_IFMT(prefmode);
3793 prefwinattributes = int(preheaderdata[11], 16);
3794 prefuid = int(preheaderdata[12], 16);
3795 prefuname = preheaderdata[13];
3796 prefgid = int(preheaderdata[14], 16);
3797 prefgname = preheaderdata[15];
3798 fid = int(preheaderdata[16], 16);
3799 finode = int(preheaderdata[17], 16);
3800 flinkcount = int(preheaderdata[18], 16);
3801 prefdev_minor = int(preheaderdata[19], 16);
3802 prefdev_major = int(preheaderdata[20], 16);
3803 prefrdev_minor = int(preheaderdata[22], 16);
3804 prefrdev_major = int(preheaderdata[23], 16);
3805 prefextrasize = int(preheaderdata[24], 16);
3806 prefextrafields = int(preheaderdata[25], 16);
3807 extrafieldslist = [];
3808 extrastart = 25;
3809 extraend = extrastart + prefextrafields;
3810 extrafieldslist = [];
3811 if(extrastart<extraend):
3812 extrafieldslist.append(preheaderdata[extrastart]);
3813 extrastart = extrastart + 1;
3814 prefchecksumtype = preheaderdata[extrastart].lower();
3815 prefcs = preheaderdata[extrastart + 1].lower();
3816 prefccs = preheaderdata[extrastart + 2].lower();
3817 hc = 0;
3818 hcmax = len(preheaderdata) - 2;
3819 hout = "";
3820 while(hc<hcmax):
3821 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs[5]);
3822 hc = hc + 1;
3823 prenewfcs = GetFileChecksum(hout, preheaderdata[-3].lower(), True, formatspecs);
3824 if(prefcs!=prenewfcs and not skipchecksum):
3825 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
3826 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
3827 return False;
3828 valid_archive = False;
3829 invalid_archive = True;
3830 prefhend = catfp.tell() - 1;
3831 prefcontentstart = catfp.tell();
3832 prefcontents = "";
3833 pyhascontents = False;
3834 if(prefsize>0):
3835 prefcontents = catfp.read(prefsize);
3836 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
3837 pyhascontents = True;
3838 if(prefccs!=prenewfccs and not skipchecksum):
3839 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
3840 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
3841 return False;
3842 catfp.seek(1, 1);
3843 il = il + 1;
3844 catfp.seek(seekstart, 0);
3845 fileidnum = il;
3846 catfheadsize = int(preheaderdata[0], 16);
3847 catfnumfields = int(preheaderdata[1], 16);
3848 catftype = int(preheaderdata[2], 16);
3849 if(re.findall("^[.|/]", preheaderdata[3])):
3850 catfname = preheaderdata[3];
3851 else:
3852 catfname = "./"+preheaderdata[3];
3853 catflinkname = preheaderdata[4];
3854 catfsize = int(preheaderdata[5], 16);
3855 catfbasedir = os.path.dirname(catfname);
3856 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
3857 if(returnfp):
3858 catlist.update({'catfp': catfp});
3859 else:
3860 catfp.close();
3861 return catlist;
3862
3863 create_alias_function("", __file_format_name__, "SeekToFileNum", ArchiveFileSeekToFileNum);
3864
3865 def ArchiveFileSeekToFileName(infile, seekfile=None, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
3866 if(hasattr(infile, "read") or hasattr(infile, "write")):
3867 catfp = infile;
3868 catfp.seek(0, 0);
3869 catfp = UncompressArchiveFile(catfp, formatspecs);
3870 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
3871 if(checkcompressfile=="tarfile"):
3872 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3873 if(checkcompressfile=="zipfile"):
3874 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3875 if(rarfile_support and checkcompressfile=="rarfile"):
3876 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3877 if(py7zr_support and checkcompressfile=="7zipfile"):
3878 return SevenZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3879 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
3880 return False;
3881 if(not catfp):
3882 return False;
3883 catfp.seek(0, 0);
3884 elif(infile=="-"):
3885 catfp = BytesIO();
3886 if(hasattr(sys.stdin, "buffer")):
3887 shutil.copyfileobj(sys.stdin.buffer, catfp);
3888 else:
3889 shutil.copyfileobj(sys.stdin, catfp);
3890 catfp.seek(0, 0);
3891 catfp = UncompressArchiveFile(catfp, formatspecs);
3892 if(not catfp):
3893 return False;
3894 catfp.seek(0, 0);
3895 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
3896 catfp = download_file_from_internet_file(infile);
3897 catfp = UncompressArchiveFile(catfp, formatspecs);
3898 catfp.seek(0, 0);
3899 if(not catfp):
3900 return False;
3901 catfp.seek(0, 0);
3902 else:
3903 infile = RemoveWindowsPath(infile);
3904 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
3905 if(checkcompressfile=="tarfile"):
3906 return TarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3907 if(checkcompressfile=="zipfile"):
3908 return ZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3909 if(rarfile_support and checkcompressfile=="rarfile"):
3910 return RarFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3911 if(py7zr_support and checkcompressfile=="7zipfile"):
3912 return SevenZipFileToArray(infile, 0, 0, listonly, skipchecksum, formatspecs, returnfp);
3913 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
3914 return False;
3915 compresscheck = CheckCompressionType(infile, formatspecs, True);
3916 if(not compresscheck):
3917 fextname = os.path.splitext(infile)[1];
3918 if(fextname==".gz"):
3919 compresscheck = "gzip";
3920 elif(fextname==".bz2"):
3921 compresscheck = "bzip2";
3922 elif(fextname==".zst"):
3923 compresscheck = "zstd";
3924 elif(fextname==".lz4" or fextname==".clz4"):
3925 compresscheck = "lz4";
3926 elif(fextname==".lzo" or fextname==".lzop"):
3927 compresscheck = "lzo";
3928 elif(fextname==".lzma" or fextname==".xz"):
3929 compresscheck = "lzma";
3930 else:
3931 return False;
3932 if(not compresscheck):
3933 return False;
3934 catfp = UncompressFile(infile, formatspecs, "rb");
3935 '''
3936 try:
3937 catfp.seek(0, 2);
3938 except OSError:
3939 SeekToEndOfFile(catfp);
3940 except ValueError:
3941 SeekToEndOfFile(catfp);
3942 CatSize = catfp.tell();
3943 CatSizeEnd = CatSize;
3944 '''
3945 try:
3946 catfp.seek(0, 0);
3947 except OSError:
3948 return False;
3949 except ValueError:
3950 return False;
3951 curloc = catfp.tell();
3952 if(curloc>0):
3953 catfp.seek(0, 0);
3954 catheader = ReadFileHeaderData(catfp, 4, formatspecs[5]);
3955 if(curloc>0):
3956 catfp.seek(curloc, 0);
3957 catstring = catheader[0];
3958 catversion = re.findall(r"([\d]+)$", catstring);
3959 fprenumfiles = catheader[1];
3960 fnumfiles = int(fprenumfiles, 16);
3961 fprechecksumtype = catheader[2];
3962 fprechecksum = catheader[3];
3963 fileheader = AppendNullByte(catstring, formatspecs[5]);
3964 fnumfileshex = format(int(fnumfiles), 'x').lower();
3965 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs[5]);
3966 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
3967 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
3968 fheadtell = len(fileheader);
3969 if(fprechecksum!=catfileheadercshex and not skipchecksum):
3970 VerbosePrintOut("File Header Checksum Error with file " + infile + " at offset " + str(0));
3971 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
3972 return False;
3973 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
3974 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
3975 seekto = fnumfiles - 1
3976 filefound = False;
3977 if(seekto>=0):
3978 il = -1;
3979 while(il < seekto):
3980 prefhstart = catfp.tell();
3981 if(formatspecs[7]):
3982 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs[5]);
3983 else:
3984 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs[5]);
3985 if(len(preheaderdata)==0):
3986 break;
3987 prefheadsize = int(preheaderdata[0], 16);
3988 prefnumfields = int(preheaderdata[1], 16);
3989 preftype = int(preheaderdata[2], 16);
3990 if(re.findall("^[.|/]", preheaderdata[3])):
3991 prefname = preheaderdata[3];
3992 else:
3993 prefname = "./"+preheaderdata[3];
3994 prefbasedir = os.path.dirname(prefname);
3995 preflinkname = preheaderdata[4];
3996 prefsize = int(preheaderdata[5], 16);
3997 prefatime = int(preheaderdata[6], 16);
3998 prefmtime = int(preheaderdata[7], 16);
3999 prefctime = int(preheaderdata[8], 16);
4000 prefbtime = int(preheaderdata[9], 16);
4001 prefmode = int(preheaderdata[10], 16);
4002 prefchmode = stat.S_IMODE(prefmode);
4003 preftypemod = stat.S_IFMT(prefmode);
4004 prefwinattributes = int(preheaderdata[11], 16);
4005 prefuid = int(preheaderdata[12], 16);
4006 prefuname = preheaderdata[13];
4007 prefgid = int(preheaderdata[14], 16);
4008 prefgname = preheaderdata[15];
4009 fid = int(preheaderdata[16], 16);
4010 finode = int(preheaderdata[17], 16);
4011 flinkcount = int(preheaderdata[18], 16);
4012 prefdev_minor = int(preheaderdata[19], 16);
4013 prefdev_major = int(preheaderdata[20], 16);
4014 prefrdev_minor = int(preheaderdata[22], 16);
4015 prefrdev_major = int(preheaderdata[23], 16);
4016 prefextrasize = int(preheaderdata[24], 16);
4017 prefextrafields = int(preheaderdata[25], 16);
4018 extrafieldslist = [];
4019 extrastart = 25;
4020 extraend = extrastart + prefextrafields;
4021 extrafieldslist = [];
4022 if(extrastart<extraend):
4023 extrafieldslist.append(preheaderdata[extrastart]);
4024 extrastart = extrastart + 1;
4025 prefchecksumtype = preheaderdata[extrastart].lower();
4026 prefcs = preheaderdata[extrastart + 1].lower();
4027 prefccs = preheaderdata[extrastart + 2].lower();
4028 hc = 0;
4029 hcmax = len(preheaderdata) - 2;
4030 hout = "";
4031 while(hc<hcmax):
4032 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs[5]);
4033 hc = hc + 1;
4034 prenewfcs = GetFileChecksum(hout, preheaderdata[-3].lower(), True, formatspecs);
4035 if(prefcs!=prenewfcs and not skipchecksum):
4036 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4037 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4038 return False;
4039 valid_archive = False;
4040 invalid_archive = True;
4041 prefhend = catfp.tell() - 1;
4042 prefcontentstart = catfp.tell();
4043 prefcontents = "";
4044 pyhascontents = False;
4045 if(prefsize>0):
4046 prefcontents = catfp.read(prefsize);
4047 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4048 pyhascontents = True;
4049 if(prefccs!=prenewfccs and not skipchecksum):
4050 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4051 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4052 return False;
4053 catfp.seek(1, 1);
4054 il = il + 1;
4055 filefound = False;
4056 prefname = preheaderdata[2];
4057 if(re.findall("^[.|/]", preheaderdata[2])):
4058 prefname = preheaderdata[2];
4059 else:
4060 prefname = "./"+preheaderdata[2];
4061 if(prefname==seekfile):
4062 filefound = True;
4063 break;
4064 catfp.seek(seekstart, 0);
4065 fileidnum = il;
4066 catfheadsize = int(preheaderdata[0], 16);
4067 catfnumfields = int(preheaderdata[1], 16);
4068 catftype = int(preheaderdata[2], 16);
4069 if(re.findall("^[.|/]", preheaderdata[3])):
4070 catfname = preheaderdata[3];
4071 else:
4072 catfname = "./"+preheaderdata[3];
4073 catflinkname = preheaderdata[4];
4074 catfsize = int(preheaderdata[5], 16);
4075 catfbasedir = os.path.dirname(catfname);
4076 if(filefound):
4077 catlist = {'fid': fileidnum, 'foffset': catfp.tell(), 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize};
4078 else:
4079 return False;
4080 if(returnfp):
4081 catlist.update({'catfp': catfp});
4082 else:
4083 catfp.close();
4084 return catlist;
4085
4086 create_alias_function("", __file_format_name__, "SeekToFileName", ArchiveFileSeekToFileName);
4087
4088 def ArchiveFileValidate(infile, formatspecs=__file_format_list__, verbose=False, returnfp=False):
4089 if(verbose):
4090 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
4091 if(hasattr(infile, "read") or hasattr(infile, "write")):
4092 catfp = infile;
4093 catfp.seek(0, 0);
4094 catfp = UncompressArchiveFile(catfp, formatspecs);
4095 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4096 if(checkcompressfile=="tarfile"):
4097 return TarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4098 if(checkcompressfile=="zipfile"):
4099 return ZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4100 if(rarfile_support and checkcompressfile=="rarfile"):
4101 return RarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4102 if(py7zr_support and checkcompressfile=="7zipfile"):
4103 return SevenZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4104 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
4105 return False;
4106 if(not catfp):
4107 return False;
4108 catfp.seek(0, 0);
4109 elif(infile=="-"):
4110 catfp = BytesIO();
4111 if(hasattr(sys.stdin, "buffer")):
4112 shutil.copyfileobj(sys.stdin.buffer, catfp);
4113 else:
4114 shutil.copyfileobj(sys.stdin, catfp);
4115 catfp.seek(0, 0);
4116 catfp = UncompressArchiveFile(catfp, formatspecs);
4117 if(not catfp):
4118 return False;
4119 catfp.seek(0, 0);
4120 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4121 catfp = download_file_from_internet_file(infile);
4122 catfp = UncompressArchiveFile(catfp, formatspecs);
4123 catfp.seek(0, 0);
4124 if(not catfp):
4125 return False;
4126 catfp.seek(0, 0);
4127 else:
4128 infile = RemoveWindowsPath(infile);
4129 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4130 if(checkcompressfile=="tarfile"):
4131 return TarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4132 if(checkcompressfile=="zipfile"):
4133 return ZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4134 if(rarfile_support and checkcompressfile=="rarfile"):
4135 return RarFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4136 if(py7zr_support and checkcompressfile=="7zipfile"):
4137 return SevenZipFileToArray(infile, 0, 0, False, False, formatspecs, returnfp);
4138 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
4139 return False;
4140 compresscheck = CheckCompressionType(infile, formatspecs, True);
4141 if(not compresscheck):
4142 fextname = os.path.splitext(infile)[1];
4143 if(fextname==".gz"):
4144 compresscheck = "gzip";
4145 elif(fextname==".bz2"):
4146 compresscheck = "bzip2";
4147 elif(fextname==".zst"):
4148 compresscheck = "zstd";
4149 elif(fextname==".lz4" or fextname==".clz4"):
4150 compresscheck = "lz4";
4151 elif(fextname==".lzo" or fextname==".lzop"):
4152 compresscheck = "lzo";
4153 elif(fextname==".lzma" or fextname==".xz"):
4154 compresscheck = "lzma";
4155 else:
4156 return False;
4157 if(not compresscheck):
4158 return False;
4159 catfp = UncompressFile(infile, formatspecs, "rb");
4160 '''
4161 try:
4162 catfp.seek(0, 2);
4163 except OSError:
4164 SeekToEndOfFile(catfp);
4165 except ValueError:
4166 SeekToEndOfFile(catfp);
4167 CatSize = catfp.tell();
4168 CatSizeEnd = CatSize;
4169 '''
4170 try:
4171 catfp.seek(0, 0);
4172 except OSError:
4173 return False;
4174 except ValueError:
4175 return False;
4176 curloc = catfp.tell();
4177 if(curloc>0):
4178 catfp.seek(0, 0);
4179 catheader = ReadFileHeaderData(catfp, 4, formatspecs[5]);
4180 if(curloc>0):
4181 catfp.seek(curloc, 0);
4182 catstring = catheader[0];
4183 catversion = re.findall(r"([\d]+)$", catstring);
4184 fprenumfiles = catheader[1];
4185 fnumfiles = int(fprenumfiles, 16);
4186 fprechecksumtype = catheader[2];
4187 fprechecksum = catheader[3];
4188 il = 0;
4189 fileheader = AppendNullByte(catstring, formatspecs[5]);
4190 fnumfileshex = format(int(fnumfiles), 'x').lower();
4191 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs[5]);
4192 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4193 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
4194 valid_archive = True;
4195 invalid_archive = False;
4196 if(verbose):
4197 VerbosePrintOut(infile);
4198 VerbosePrintOut("Number of Records " + str(fnumfiles));
4199 if(fprechecksum==catfileheadercshex):
4200 if(verbose):
4201 VerbosePrintOut("File Header Checksum Passed at offset " + str(0));
4202 VerbosePrintOut("'" + str(fprechecksum) + "' == " + "'" + str(catfileheadercshex) + "'");
4203 else:
4204 if(verbose):
4205 VerbosePrintOut("File Header Checksum Failed at offset " + str(0));
4206 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4207 valid_archive = False;
4208 invalid_archive = True;
4209 if(verbose):
4210 VerbosePrintOut("");
4211 while(il<fnumfiles):
4212 catfhstart = catfp.tell();
4213 if(formatspecs[7]):
4214 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs[5]);
4215 else:
4216 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs[5]);
4217 if(len(catheaderdata)==0):
4218 break;
4219 catfheadsize = int(catheaderdata[0], 16);
4220 catfnumfields = int(catheaderdata[1], 16);
4221 catftype = int(catheaderdata[2], 16);
4222 if(re.findall("^[.|/]", catheaderdata[3])):
4223 catfname = catheaderdata[3];
4224 else:
4225 catfname = "./"+catheaderdata[3];
4226 catfbasedir = os.path.dirname(catfname);
4227 catflinkname = catheaderdata[4];
4228 catfsize = int(catheaderdata[5], 16);
4229 catfatime = int(catheaderdata[6], 16);
4230 catfmtime = int(catheaderdata[7], 16);
4231 catfctime = int(catheaderdata[8], 16);
4232 catfbtime = int(catheaderdata[9], 16);
4233 catfmode = int(catheaderdata[10], 16);
4234 catfchmode = stat.S_IMODE(catfmode);
4235 catftypemod = stat.S_IFMT(catfmode);
4236 prefwinattributes = int(catheaderdata[11], 16);
4237 catfuid = int(catheaderdata[12], 16);
4238 catfuname = catheaderdata[13];
4239 catfgid = int(catheaderdata[14], 16);
4240 catfgname = catheaderdata[15];
4241 fid = int(catheaderdata[16], 16);
4242 finode = int(catheaderdata[17], 16);
4243 flinkcount = int(catheaderdata[18], 16);
4244 catfdev_minor = int(catheaderdata[19], 16);
4245 catfdev_major = int(catheaderdata[20], 16);
4246 catfrdev_minor = int(catheaderdata[21], 16);
4247 catfrdev_major = int(catheaderdata[22], 16);
4248 catfextrasize = int(catheaderdata[23], 16);
4249 catfextrafields = int(catheaderdata[24], 16);
4250 extrafieldslist = [];
4251 extrastart = 25;
4252 extraend = extrastart + catfextrafields;
4253 extrafieldslist = [];
4254 if(extrastart<extraend):
4255 extrafieldslist.append(catheaderdata[extrastart]);
4256 extrastart = extrastart + 1;
4257 catfchecksumtype = catheaderdata[extrastart].lower();
4258 catfcs = catheaderdata[extrastart + 1].lower();
4259 catfccs = catheaderdata[extrastart + 2].lower();
4260 hc = 0;
4261 hcmax = len(catheaderdata) - 2;
4262 hout = "";
4263 while(hc<hcmax):
4264 hout = hout + AppendNullByte(catheaderdata[hc], formatspecs[5]);
4265 hc = hc + 1;
4266 catnewfcs = GetFileChecksum(hout, catheaderdata[-3].lower(), True, formatspecs);
4267 if(verbose):
4268 VerbosePrintOut(catfname);
4269 VerbosePrintOut("Record Number " + str(il) + "; File ID " + str(fid) + "; iNode Number " + str(finode));
4270 if(catfcs==catnewfcs):
4271 if(verbose):
4272 VerbosePrintOut("File Header Checksum Passed at offset " + str(catfhstart));
4273 VerbosePrintOut("'" + str(catfcs) + "' == " + "'" + str(catnewfcs) + "'");
4274 else:
4275 if(verbose):
4276 VerbosePrintOut("File Header Checksum Failed at offset " + str(catfhstart));
4277 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
4278 valid_archive = False;
4279 invalid_archive = True;
4280 catfhend = catfp.tell() - 1;
4281 catfcontentstart = catfp.tell();
4282 catfcontents = "";
4283 pyhascontents = False;
4284 if(catfsize>0):
4285 catfcontents = catfp.read(catfsize);
4286 catnewfccs = GetFileChecksum(catfcontents, catheaderdata[-3].lower(), False, formatspecs);
4287 pyhascontents = True;
4288 if(catfccs==catnewfccs):
4289 if(verbose):
4290 VerbosePrintOut("File Content Checksum Passed at offset " + str(catfcontentstart));
4291 VerbosePrintOut("'" + str(catfccs) + "' == " + "'" + str(catnewfccs) + "'");
4292 else:
4293 if(verbose):
4294 VerbosePrintOut("File Content Checksum Failed at offset " + str(catfcontentstart));
4295 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
4296 valid_archive = False;
4297 invalid_archive = True;
4298 if(verbose):
4299 VerbosePrintOut("");
4300 catfp.seek(1, 1);
4301 il = il + 1;
4302 if(valid_archive):
4303 if(returnfp):
4304 return catfp;
4305 else:
4306 catfp.close();
4307 return True;
4308 else:
4309 catfp.close();
4310 return False;
4311
4312 create_alias_function("", __file_format_name__, "Validate", ArchiveFileValidate);
4313
4314 def ArchiveFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4315 if(hasattr(infile, "read") or hasattr(infile, "write")):
4316 catfp = infile;
4317 catfp.seek(0, 0);
4318 catfp = UncompressArchiveFile(catfp, formatspecs);
4319 checkcompressfile = CheckCompressionSubType(catfp, formatspecs, True);
4320 if(checkcompressfile=="tarfile"):
4321 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4322 if(checkcompressfile=="zipfile"):
4323 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4324 if(rarfile_support and checkcompressfile=="rarfile"):
4325 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4326 if(py7zr_support and checkcompressfile=="7zipfile"):
4327 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4328 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
4329 return False;
4330 if(not catfp):
4331 return False;
4332 catfp.seek(0, 0);
4333 elif(infile=="-"):
4334 catfp = BytesIO();
4335 if(hasattr(sys.stdin, "buffer")):
4336 shutil.copyfileobj(sys.stdin.buffer, catfp);
4337 else:
4338 shutil.copyfileobj(sys.stdin, catfp);
4339 catfp.seek(0, 0);
4340 catfp = UncompressArchiveFile(catfp, formatspecs);
4341 if(not catfp):
4342 return False;
4343 catfp.seek(0, 0);
4344 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4345 catfp = download_file_from_internet_file(infile);
4346 catfp = UncompressArchiveFile(catfp, formatspecs);
4347 catfp.seek(0, 0);
4348 if(not catfp):
4349 return False;
4350 catfp.seek(0, 0);
4351 else:
4352 infile = RemoveWindowsPath(infile);
4353 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4354 if(checkcompressfile=="tarfile"):
4355 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4356 if(checkcompressfile=="zipfile"):
4357 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4358 if(rarfile_support and checkcompressfile=="rarfile"):
4359 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4360 if(py7zr_support and checkcompressfile=="7zipfile"):
4361 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4362 if(checkcompressfile!="catfile" and checkcompressfile!=formatspecs[2]):
4363 return False;
4364 compresscheck = CheckCompressionType(infile, formatspecs, True);
4365 if(not compresscheck):
4366 fextname = os.path.splitext(infile)[1];
4367 if(fextname==".gz"):
4368 compresscheck = "gzip";
4369 elif(fextname==".bz2"):
4370 compresscheck = "bzip2";
4371 elif(fextname==".zst"):
4372 compresscheck = "zstd";
4373 elif(fextname==".lz4" or fextname==".clz4"):
4374 compresscheck = "lz4";
4375 elif(fextname==".lzo" or fextname==".lzop"):
4376 compresscheck = "lzo";
4377 elif(fextname==".lzma" or fextname==".xz"):
4378 compresscheck = "lzma";
4379 else:
4380 return False;
4381 if(not compresscheck):
4382 return False;
4383 catfp = UncompressFile(infile, formatspecs, "rb");
4384 '''
4385 try:
4386 catfp.seek(0, 2);
4387 except OSError:
4388 SeekToEndOfFile(catfp);
4389 except ValueError:
4390 SeekToEndOfFile(catfp);
4391 CatSize = catfp.tell();
4392 CatSizeEnd = CatSize;
4393 '''
4394 try:
4395 catfp.seek(0, 0);
4396 except OSError:
4397 return False;
4398 except ValueError:
4399 return False;
4400 curloc = catfp.tell();
4401 if(curloc>0):
4402 catfp.seek(0, 0);
4403 catheader = ReadFileHeaderData(catfp, 4, formatspecs[5]);
4404 if(curloc>0):
4405 catfp.seek(curloc, 0);
4406 catstring = catheader[0];
4407 catversion = re.findall(r"([\d]+)$", catstring);
4408 fprenumfiles = catheader[1];
4409 fnumfiles = int(fprenumfiles, 16);
4410 fprechecksumtype = catheader[2];
4411 fprechecksum = catheader[3];
4412 fileheader = AppendNullByte(catstring, formatspecs[5]);
4413 fnumfileshex = format(int(fnumfiles), 'x').lower();
4414 fileheader = fileheader + AppendNullBytes([fnumfileshex, fprechecksumtype], formatspecs[5]);
4415 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4416 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
4417 fheadtell = len(fileheader);
4418 if(fprechecksum!=catfileheadercshex and not skipchecksum):
4419 VerbosePrintOut("File Header Checksum Error with file at offset " + str(0));
4420 VerbosePrintOut("'" + str(fprechecksum) + "' != " + "'" + str(catfileheadercshex) + "'");
4421 return False;
4422 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
4423 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': fprechecksumtype, 'fheaderchecksum': fprechecksum, 'ffilelist': {}};
4424 if(seekstart<0 and seekstart>fnumfiles):
4425 seekstart = 0;
4426 if(seekend==0 or seekend>fnumfiles and seekend<seekstart):
4427 seekend = fnumfiles;
4428 elif(seekend<0 and abs(seekend)<=fnumfiles and abs(seekend)>=seekstart):
4429 seekend = fnumfiles - abs(seekend);
4430 if(seekstart>0):
4431 il = 0;
4432 while(il < seekstart):
4433 prefhstart = catfp.tell();
4434 if(formatspecs[7]):
4435 preheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs[5]);
4436 else:
4437 preheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs[5]);
4438 if(len(preheaderdata)==0):
4439 break;
4440 prefheadsize = int(preheaderdata[0], 16);
4441 prefnumfields = int(preheaderdata[1], 16);
4442 if(re.findall("^[.|/]", preheaderdata[3])):
4443 prefname = preheaderdata[3];
4444 else:
4445 prefname = "./"+preheaderdata[3];
4446 prefsize = int(preheaderdata[5], 16);
4447 prefextrasize = int(preheaderdata[23], 16);
4448 prefextrafields = int(preheaderdata[24], 16);
4449 extrafieldslist = [];
4450 extrastart = 25;
4451 extraend = extrastart + prefextrafields;
4452 extrafieldslist = [];
4453 if(extrastart<extraend):
4454 extrafieldslist.append(preheaderdata[extrastart]);
4455 extrastart = extrastart + 1;
4456 prefchecksumtype = preheaderdata[extrastart].lower();
4457 prefcs = preheaderdata[extrastart + 1].lower();
4458 prefccs = preheaderdata[extrastart + 2].lower();
4459 hc = 0;
4460 hcmax = len(preheaderdata) - 2;
4461 hout = "";
4462 while(hc<hcmax):
4463 hout = hout + AppendNullByte(preheaderdata[hc], formatspecs[5]);
4464 hc = hc + 1;
4465 prenewfcs = GetFileChecksum(hout, preheaderdata[-3].lower(), True, formatspecs);
4466 if(prefcs!=prenewfcs and not skipchecksum):
4467 VerbosePrintOut("File Header Checksum Error with file " + prefname + " at offset " + str(prefhstart));
4468 VerbosePrintOut("'" + str(prefcs) + "' != " + "'" + str(prenewfcs) + "'");
4469 return False;
4470 valid_archive = False;
4471 invalid_archive = True;
4472 prefhend = catfp.tell() - 1;
4473 prefcontentstart = catfp.tell();
4474 prefcontents = "";
4475 pyhascontents = False;
4476 if(prefsize>0):
4477 prefcontents = catfp.read(prefsize);
4478 prenewfccs = GetFileChecksum(prefcontents, preheaderdata[-3].lower(), False, formatspecs);
4479 pyhascontents = True;
4480 if(prefccs!=prenewfccs and not skipchecksum):
4481 VerbosePrintOut("File Content Checksum Error with file " + prefname + " at offset " + str(prefcontentstart));
4482 VerbosePrintOut("'" + str(prefccs) + "' != " + "'" + str(prenewfccs) + "'");
4483 return False;
4484 catfp.seek(1, 1);
4485 il = il + 1;
4486 fileidnum = seekstart;
4487 realidnum = 0;
4488 while(fileidnum<seekend):
4489 catfhstart = catfp.tell();
4490 if(formatspecs[7]):
4491 catheaderdata = ReadFileHeaderDataBySize(catfp, formatspecs[5]);
4492 else:
4493 catheaderdata = ReadFileHeaderDataWoSize(catfp, formatspecs[5]);
4494 if(len(catheaderdata)==0):
4495 break;
4496 catfheadsize = int(catheaderdata[0], 16);
4497 catfnumfields = int(catheaderdata[1], 16);
4498 catftype = int(catheaderdata[2], 16);
4499 if(re.findall("^[.|/]", catheaderdata[3])):
4500 catfname = catheaderdata[3];
4501 else:
4502 catfname = "./"+catheaderdata[3];
4503 catfbasedir = os.path.dirname(catfname);
4504 catflinkname = catheaderdata[4];
4505 catfsize = int(catheaderdata[5], 16);
4506 catfatime = int(catheaderdata[6], 16);
4507 catfmtime = int(catheaderdata[7], 16);
4508 catfctime = int(catheaderdata[8], 16);
4509 catfbtime = int(catheaderdata[9], 16);
4510 catfmode = int(catheaderdata[10], 16);
4511 catfchmode = stat.S_IMODE(catfmode);
4512 catftypemod = stat.S_IFMT(catfmode);
4513 catfwinattributes = int(catheaderdata[11], 16);
4514 catfuid = int(catheaderdata[12], 16);
4515 catfuname = catheaderdata[13];
4516 catfgid = int(catheaderdata[14], 16);
4517 catfgname = catheaderdata[15];
4518 fid = int(catheaderdata[16], 16);
4519 finode = int(catheaderdata[17], 16);
4520 flinkcount = int(catheaderdata[18], 16);
4521 catfdev_minor = int(catheaderdata[19], 16);
4522 catfdev_major = int(catheaderdata[20], 16);
4523 catfrdev_minor = int(catheaderdata[21], 16);
4524 catfrdev_major = int(catheaderdata[22], 16);
4525 catfextrasize = int(catheaderdata[23], 16);
4526 catfextrafields = int(catheaderdata[24], 16);
4527 extrafieldslist = [];
4528 extrastart = 25;
4529 extraend = extrastart + catfextrafields;
4530 extrafieldslist = [];
4531 if(extrastart<extraend):
4532 extrafieldslist.append(catheaderdata[extrastart]);
4533 extrastart = extrastart + 1;
4534 catfchecksumtype = catheaderdata[extrastart].lower();
4535 catfcs = catheaderdata[extrastart + 1].lower();
4536 catfccs = catheaderdata[extrastart + 2].lower();
4537 hc = 0;
4538 hcmax = len(catheaderdata) - 2;
4539 hout = "";
4540 while(hc<hcmax):
4541 hout = hout + AppendNullByte(catheaderdata[hc], formatspecs[5]);
4542 hc = hc + 1;
4543 catnewfcs = GetFileChecksum(hout, catheaderdata[-3].lower(), True, formatspecs);
4544 if(catfcs!=catnewfcs and not skipchecksum):
4545 VerbosePrintOut("File Header Checksum Error with file " + catfname + " at offset " + str(catfhstart));
4546 VerbosePrintOut("'" + str(catfcs) + "' != " + "'" + str(catnewfcs) + "'");
4547 return False;
4548 catfhend = catfp.tell() - 1;
4549 catfcontentstart = catfp.tell();
4550 catfcontents = BytesIO();
4551 pyhascontents = False;
4552 catfcontents.seek(0, 0);
4553 if(catfsize>0 and not listonly):
4554 catfcontents.write(catfp.read(catfsize));
4555 catnewfccs = GetFileChecksum(catfcontents.read(), catheaderdata[-3].lower(), False, formatspecs);
4556 pyhascontents = True;
4557 if(catfccs!=catnewfccs and skipchecksum):
4558 VerbosePrintOut("File Content Checksum Error with file " + catfname + " at offset " + str(catfcontentstart));
4559 VerbosePrintOut("'" + str(catfccs) + "' != " + "'" + str(catnewfccs) + "'");
4560 return False;
4561 if(catfsize>0 and listonly):
4562 catfp.seek(catfsize, 1);
4563 pyhascontents = False;
4564 catfp.seek(1, 1);
4565 catfcontentend = catfp.tell() - 1;
4566 catfcontents.seek(0, 0);
4567 catlist['ffilelist'].update({realidnum: {'fid': realidnum, 'fidalt': fileidnum, 'fheadersize': catfheadsize, 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': catftype, 'fname': catfname, 'fbasedir': catfbasedir, 'flinkname': catflinkname, 'fsize': catfsize, 'fatime': catfatime, 'fmtime': catfmtime, 'fctime': catfctime, 'fbtime': catfbtime, 'fmode': catfmode, 'fchmode': catfchmode, 'ftypemod': catftypemod, 'fwinattributes': catfwinattributes, 'fuid': catfuid, 'funame': catfuname, 'fgid': catfgid, 'fgname': catfgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': catfdev_minor, 'fmajor': catfdev_major, 'frminor': catfrdev_minor, 'frmajor': catfrdev_major, 'fchecksumtype': catfchecksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': catfextrasize, 'fextralist': extrafieldslist, 'fheaderchecksum': catfcs, 'fcontentchecksum': catfccs, 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': catfcontents} });
4568 fileidnum = fileidnum + 1;
4569 realidnum = realidnum + 1;
4570 if(returnfp):
4571 catlist.update({'catfp': catfp});
4572 else:
4573 catfp.close();
4574 return catlist;
4575
4576 create_alias_function("", __file_format_name__, "ToArray", ArchiveFileToArray);
4577
4578 def ArchiveFileStringToArray(catstr, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4579 catfp = BytesIO(catstr);
4580 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4581 return listcatfiles;
4582
4583 create_alias_function("", __file_format_name__, "StringToArray", ArchiveFileStringToArray);
4584
4585 def TarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4586 catfp = BytesIO();
4587 catfp = PackArchiveFileFromTarFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
4588 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4589 return listcatfiles;
4590
4591 def ZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4592 catfp = BytesIO();
4593 catfp = PackArchiveFileFromZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
4594 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4595 return listcatfiles;
4596
4597 if(not rarfile_support):
4598 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4599 return False;
4600
4601 if(rarfile_support):
4602 def RarFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4603 catfp = BytesIO();
4604 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
4605 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4606 return listcatfiles;
4607
4608 if(not py7zr_support):
4609 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4610 return False;
4611
4612 if(py7zr_support):
4613 def SevenZipFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4614 catfp = BytesIO();
4615 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
4616 listcatfiles = ArchiveFileToArray(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4617 return listcatfiles;
4618
4619 def InFileToArray(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
4620 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
4621 if(checkcompressfile=="tarfile"):
4622 return TarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4623 elif(checkcompressfile=="zipfile"):
4624 return ZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4625 elif(checkcompressfile=="catfile"):
4626 return ArchiveFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4627 elif(rarfile_support and checkcompressfile=="rarfile"):
4628 return RarFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4629 elif(py7zr_support and checkcompressfile=="7zipfile"):
4630 return SevenZipFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
4631 else:
4632 return False;
4633 return False;
4634
4635 def ListDirToArrayAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
4636 catver = formatspecs[6];
4637 fileheaderver = str(int(catver.replace(".", "")));
4638 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
4639 advancedlist = formatspecs[8];
4640 altinode = formatspecs[9];
4641 infilelist = [];
4642 if(infiles=="-"):
4643 for line in sys.stdin:
4644 infilelist.append(line.strip());
4645 infilelist = list(filter(None, infilelist));
4646 elif(infiles!="-" and dirlistfromtxt and os.path.exists(infiles) and (os.path.isfile(infiles) or infiles=="/dev/null" or infiles=="NUL")):
4647 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
4648 return False;
4649 with UncompressFile(infiles, formatspecs, "r") as finfile:
4650 for line in finfile:
4651 infilelist.append(line.strip());
4652 infilelist = list(filter(None, infilelist));
4653 else:
4654 if(isinstance(infiles, (list, tuple, ))):
4655 infilelist = list(filter(None, infiles));
4656 elif(isinstance(infiles, (str, ))):
4657 infilelist = list(filter(None, [infiles]));
4658 if(advancedlist):
4659 GetDirList = ListDirAdvanced(infilelist, followlink, False);
4660 else:
4661 GetDirList = ListDir(infilelist, followlink, False);
4662 if(not GetDirList):
4663 return False;
4664 curinode = 0;
4665 curfid = 0;
4666 inodelist = [];
4667 inodetofile = {};
4668 filetoinode = {};
4669 inodetocatinode = {};
4670 fileidnum = 0;
4671 fnumfiles = int(len(GetDirList));
4672 catver = formatspecs[6];
4673 fileheaderver = str(int(catver.replace(".", "")));
4674 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
4675 fnumfileshex = format(int(fnumfiles), 'x').lower();
4676 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs[5]);
4677 catversion = re.findall(r"([\d]+)$", fileheader);
4678 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
4679 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4680 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
4681 fheadtell = len(fileheader);
4682 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
4683 for curfname in GetDirList:
4684 if(re.findall("^[.|/]", curfname)):
4685 fname = curfname;
4686 else:
4687 fname = "./"+curfname;
4688 if(verbose):
4689 VerbosePrintOut(fname);
4690 if(not followlink or followlink is None):
4691 fstatinfo = os.lstat(fname);
4692 else:
4693 fstatinfo = os.stat(fname);
4694 fpremode = fstatinfo.st_mode;
4695 finode = fstatinfo.st_ino;
4696 flinkcount = fstatinfo.st_nlink;
4697 ftype = 0;
4698 if(stat.S_ISREG(fpremode)):
4699 ftype = 0;
4700 elif(stat.S_ISLNK(fpremode)):
4701 ftype = 2;
4702 elif(stat.S_ISCHR(fpremode)):
4703 ftype = 3;
4704 elif(stat.S_ISBLK(fpremode)):
4705 ftype = 4;
4706 elif(stat.S_ISDIR(fpremode)):
4707 ftype = 5;
4708 elif(stat.S_ISFIFO(fpremode)):
4709 ftype = 6;
4710 elif(stat.S_ISSOCK(fpremode)):
4711 ftype = 8;
4712 elif(hasattr(stat, "S_ISDOOR") and stat.S_ISDOOR(fpremode)):
4713 ftype = 9;
4714 elif(hasattr(stat, "S_ISPORT") and stat.S_ISPORT(fpremode)):
4715 ftype = 10;
4716 elif(hasattr(stat, "S_ISWHT") and stat.S_ISWHT(fpremode)):
4717 ftype = 11;
4718 else:
4719 ftype = 0;
4720 flinkname = "";
4721 fbasedir = os.path.dirname(fname);
4722 fcurfid = curfid;
4723 if(not followlink and finode!=0):
4724 if(ftype!=1):
4725 if(finode in inodelist):
4726 ftype = 1;
4727 flinkname = inodetofile[finode];
4728 if(altinode):
4729 fcurinode = finode;
4730 else:
4731 fcurinode = inodetocatinode[finode];
4732 if(finode not in inodelist):
4733 inodelist.append(finode);
4734 inodetofile.update({finode: fname});
4735 inodetocatinode.update({finode: curinode});
4736 if(altinode):
4737 fcurinode = finode;
4738 else:
4739 fcurinode = curinode;
4740 curinode = curinode + 1;
4741 else:
4742 fcurinode = curinode;
4743 curinode = curinode + 1;
4744 curfid = curfid + 1;
4745 if(ftype==2):
4746 flinkname = os.readlink(fname);
4747 fdev = fstatinfo.st_dev;
4748 getfdev = GetDevMajorMinor(fdev);
4749 fdev_minor = getfdev[0];
4750 fdev_major = getfdev[1];
4751 frdev = fstatinfo.st_dev;
4752 if(hasattr(fstatinfo, "st_rdev")):
4753 frdev = fstatinfo.st_rdev;
4754 else:
4755 frdev = fstatinfo.st_dev;
4756 getfrdev = GetDevMajorMinor(frdev);
4757 frdev_minor = getfrdev[0];
4758 frdev_major = getfrdev[1];
4759 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
4760 fsize = "0";
4761 if(ftype==0 or ftype==7):
4762 fsize = fstatinfo.st_size;
4763 fatime = fstatinfo.st_atime;
4764 fmtime = fstatinfo.st_mtime;
4765 fctime = fstatinfo.st_ctime;
4766 if(hasattr(fstatinfo, "st_birthtime")):
4767 fbtime = fstatinfo.st_birthtime;
4768 else:
4769 fbtime = fstatinfo.st_ctime;
4770 fmode = fstatinfo.st_mode;
4771 fchmode = stat.S_IMODE(fstatinfo.st_mode);
4772 ftypemod = stat.S_IFMT(fstatinfo.st_mode);
4773 fuid = fstatinfo.st_uid;
4774 fgid = fstatinfo.st_gid;
4775 funame = "";
4776 try:
4777 import pwd;
4778 try:
4779 userinfo = pwd.getpwuid(fstatinfo.st_uid);
4780 funame = userinfo.pw_name;
4781 except KeyError:
4782 funame = "";
4783 except ImportError:
4784 funame = "";
4785 fgname = "";
4786 try:
4787 import grp;
4788 try:
4789 groupinfo = grp.getgrgid(fstatinfo.st_gid);
4790 fgname = groupinfo.gr_name;
4791 except KeyError:
4792 fgname = "";
4793 except ImportError:
4794 fgname = "";
4795 fdev_minor = fdev_minor;
4796 fdev_major = fdev_major;
4797 frdev_minor = frdev_minor;
4798 frdev_major = frdev_major;
4799 finode = finode;
4800 flinkcount = flinkcount;
4801 if(hasattr(fstatinfo, "st_file_attributes")):
4802 fwinattributes = fstatinfo.st_file_attributes;
4803 else:
4804 fwinattributes = 0;
4805 fcontents = BytesIO();
4806 if(ftype==0 or ftype==7):
4807 with open(fname, "rb") as fpc:
4808 shutil.copyfileobj(fpc, fcontents);
4809 if(followlink and (ftype==1 or ftype==2)):
4810 flstatinfo = os.stat(flinkname);
4811 with open(flinkname, "rb") as fpc:
4812 shutil.copyfileobj(fpc, fcontents);
4813 fcontents.seek(0, 0);
4814 ftypehex = format(ftype, 'x').lower();
4815 extrafields = len(extradata);
4816 extrafieldslist = extradata;
4817 catfextrafields = extrafields;
4818 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
4819 if(len(extradata)>0):
4820 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
4821 extrasizelen = len(extrasizestr);
4822 extrasizelenhex = format(extrasizelen, 'x').lower();
4823 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
4824 catoutlen = len(catoutlist) + len(extradata) + 3;
4825 catoutlenhex = format(catoutlen, 'x').lower();
4826 catoutlist.insert(0, catoutlenhex);
4827 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
4828 if(len(extradata)>0):
4829 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
4830 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
4831 catfnumfields = catoutlen;
4832 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
4833 fcontents.seek(0, 0);
4834 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
4835 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
4836 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
4837 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
4838 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
4839 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
4840 catfileoutstrecd = catfileoutstr.encode('UTF-8');
4841 nullstrecd = formatspecs[5].encode('UTF-8');
4842 fheadtell += len(catfileoutstr) + 1;
4843 catfcontentend = fheadtell - 1;
4844 fcontents.seek(0, 0);
4845 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
4846 pyhascontents = False;
4847 if(int(fsize)>0 and not listonly):
4848 pyhascontents = True;
4849 if(int(fsize)>0 and listonly):
4850 fcontents = BytesIO();
4851 pyhascontents = False;
4852 fcontents.seek(0, 0);
4853 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
4854 fileidnum = fileidnum + 1;
4855 return catlist;
4856
4857 def TarFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
4858 curinode = 0;
4859 curfid = 0;
4860 inodelist = [];
4861 inodetofile = {};
4862 filetoinode = {};
4863 inodetocatinode = {};
4864 fileidnum = 0;
4865 if(infile=="-"):
4866 infile = BytesIO();
4867 if(hasattr(sys.stdin, "buffer")):
4868 shutil.copyfileobj(sys.stdin.buffer, infile);
4869 else:
4870 shutil.copyfileobj(sys.stdin, infile);
4871 infile.seek(0, 0);
4872 if(not infile):
4873 return False;
4874 infile.seek(0, 0);
4875 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
4876 infile = download_file_from_internet_file(infile);
4877 infile.seek(0, 0);
4878 if(not infile):
4879 return False;
4880 infile.seek(0, 0);
4881 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
4882 return False;
4883 elif(os.path.exists(infile) and os.path.isfile(infile)):
4884 try:
4885 if(not tarfile.is_tarfile(infile)):
4886 return False;
4887 except AttributeError:
4888 if(not is_tarfile(infile)):
4889 return False;
4890 else:
4891 return False;
4892 try:
4893 if(hasattr(infile, "read") or hasattr(infile, "write")):
4894 tarfp = tarfile.open(fileobj=infile, mode="r");
4895 else:
4896 tarfp = tarfile.open(infile, "r");
4897 except FileNotFoundError:
4898 return False;
4899 fnumfiles = int(len(tarfp.getmembers()));
4900 catver = formatspecs[6];
4901 fileheaderver = str(int(catver.replace(".", "")));
4902 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
4903 fnumfileshex = format(int(fnumfiles), 'x').lower();
4904 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs[5]);
4905 catversion = re.findall(r"([\d]+)$", fileheader);
4906 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
4907 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
4908 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
4909 fheadtell = len(fileheader);
4910 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
4911 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
4912 if(re.findall("^[.|/]", member.name)):
4913 fname = member.name;
4914 else:
4915 fname = "./"+member.name;
4916 if(verbose):
4917 VerbosePrintOut(fname);
4918 fpremode = member.mode;
4919 ffullmode = member.mode;
4920 flinkcount = 0;
4921 ftype = 0;
4922 if(member.isreg()):
4923 ffullmode = member.mode + stat.S_IFREG;
4924 ftype = 0;
4925 elif(member.isdev()):
4926 ffullmode = member.mode;
4927 ftype = 7;
4928 elif(member.islnk()):
4929 ffullmode = member.mode + stat.S_IFREG;
4930 ftype = 1;
4931 elif(member.issym()):
4932 ffullmode = member.mode + stat.S_IFLNK;
4933 ftype = 2;
4934 elif(member.ischr()):
4935 ffullmode = member.mode + stat.S_IFCHR;
4936 ftype = 3;
4937 elif(member.isblk()):
4938 ffullmode = member.mode + stat.S_IFBLK;
4939 ftype = 4;
4940 elif(member.isdir()):
4941 ffullmode = member.mode + stat.S_IFDIR;
4942 ftype = 5;
4943 elif(member.isfifo()):
4944 ffullmode = member.mode + stat.S_IFIFO;
4945 ftype = 6;
4946 elif(member.issparse()):
4947 ffullmode = member.mode;
4948 ftype = 12;
4949 else:
4950 ffullmode = member.mode;
4951 ftype = 0;
4952 flinkname = "";
4953 fbasedir = os.path.dirname(fname);
4954 fcurfid = curfid;
4955 fcurinode = curfid;
4956 finode = fcurinode;
4957 curfid = curfid + 1;
4958 if(ftype==2):
4959 flinkname = member.linkname;
4960 fdev_minor = member.devminor;
4961 fdev_major = member.devmajor;
4962 frdev_minor = member.devminor;
4963 frdev_major = member.devmajor;
4964 if(ftype==1 or ftype==2 or ftype==3 or ftype==4 or ftype==5 or ftype==6):
4965 fsize = "0";
4966 elif(ftype==0 or ftype==7):
4967 fsize = member.size;
4968 else:
4969 fsize = member.size;
4970 fatime = member.mtime;
4971 fmtime = member.mtime;
4972 fctime = member.mtime;
4973 fbtime = member.mtime;
4974 fmode = ffullmode;
4975 fchmode = stat.S_IMODE(ffullmode);
4976 ftypemod = stat.S_IFMT(ffullmode);
4977 fuid = member.uid;
4978 fgid = member.gid;
4979 funame = member.uname;
4980 fgname = member.gname;
4981 flinkcount = flinkcount;
4982 fwinattributes = int(0);
4983 fcontents = BytesIO();
4984 if(ftype==0 or ftype==7):
4985 with tarfp.extractfile(member) as fpc:
4986 shutil.copyfileobj(fpc, fcontents);
4987 fcontents.seek(0, 0);
4988 ftypehex = format(ftype, 'x').lower();
4989 extrafields = len(extradata);
4990 extrafieldslist = extradata;
4991 catfextrafields = extrafields;
4992 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
4993 if(len(extradata)>0):
4994 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
4995 extrasizelen = len(extrasizestr);
4996 extrasizelenhex = format(extrasizelen, 'x').lower();
4997 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
4998 catoutlen = len(catoutlist) + len(extradata) + 3;
4999 catoutlenhex = format(catoutlen, 'x').lower();
5000 catoutlist.insert(0, catoutlenhex);
5001 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
5002 if(len(extradata)>0):
5003 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
5004 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
5005 catfnumfields = catoutlen;
5006 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5007 fcontents.seek(0, 0);
5008 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5009 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5010 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5011 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
5012 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5013 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5014 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5015 nullstrecd = formatspecs[5].encode('UTF-8');
5016 fheadtell += len(catfileoutstr) + 1;
5017 catfcontentend = fheadtell - 1;
5018 fcontents.seek(0, 0);
5019 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5020 pyhascontents = False;
5021 if(int(fsize)>0 and not listonly):
5022 pyhascontents = True;
5023 if(int(fsize)>0 and listonly):
5024 fcontents = "";
5025 pyhascontents = False;
5026 fcontents.seek(0, 0);
5027 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5028 fileidnum = fileidnum + 1;
5029 return catlist;
5030
5031 def ZipFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5032 curinode = 0;
5033 curfid = 0;
5034 inodelist = [];
5035 inodetofile = {};
5036 filetoinode = {};
5037 inodetocatinode = {};
5038 fileidnum = 0;
5039 if(infile=="-"):
5040 infile = BytesIO();
5041 if(hasattr(sys.stdin, "buffer")):
5042 shutil.copyfileobj(sys.stdin.buffer, infile);
5043 else:
5044 shutil.copyfileobj(sys.stdin, infile);
5045 infile.seek(0, 0);
5046 if(not infile):
5047 return False;
5048 infile.seek(0, 0);
5049 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
5050 infile = download_file_from_internet_file(infile);
5051 infile.seek(0, 0);
5052 if(not infile):
5053 return False;
5054 infile.seek(0, 0);
5055 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
5056 return False;
5057 else:
5058 return False;
5059 if(not zipfile.is_zipfile(infile)):
5060 return False;
5061 try:
5062 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
5063 except FileNotFoundError:
5064 return False;
5065 ziptest = zipfp.testzip();
5066 if(ziptest):
5067 VerbosePrintOut("Bad file found!");
5068 fnumfiles = int(len(zipfp.infolist()));
5069 catver = formatspecs[6];
5070 fileheaderver = str(int(catver.replace(".", "")));
5071 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
5072 catversion = re.findall(r"([\d]+)$", fileheader);
5073 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
5074 fnumfileshex = format(int(fnumfiles), 'x').lower();
5075 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs[5]);
5076 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
5077 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
5078 fheadtell = len(fileheader);
5079 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5080 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
5081 if(re.findall("^[.|/]", member.filename)):
5082 fname = member.filename;
5083 else:
5084 fname = "./"+member.filename;
5085 zipinfo = zipfp.getinfo(member.filename);
5086 if(verbose):
5087 VerbosePrintOut(fname);
5088 if(not member.is_dir()):
5089 fpremode = stat.S_IFREG + 438;
5090 elif(member.is_dir()):
5091 fpremode = stat.S_IFDIR + 511;
5092 flinkcount = 0;
5093 ftype = 0;
5094 if(not member.is_dir()):
5095 ftype = 0;
5096 elif(member.is_dir()):
5097 ftype = 5;
5098 flinkname = "";
5099 fbasedir = os.path.dirname(fname);
5100 fcurfid = curfid;
5101 fcurinode = curfid;
5102 finode = fcurinode;
5103 curfid = curfid + 1;
5104 fdev_minor = 0;
5105 fdev_major = 0;
5106 frdev_minor = 0;
5107 frdev_major = 0;
5108 if(ftype==5):
5109 fsize = "0";
5110 elif(ftype==0):
5111 fsize = member.file_size;
5112 else:
5113 fsize = member.file_size;
5114 fatime = time.mktime(member.date_time + (0, 0, -1));
5115 fmtime = time.mktime(member.date_time + (0, 0, -1));
5116 fctime = time.mktime(member.date_time + (0, 0, -1));
5117 fbtime = time.mktime(member.date_time + (0, 0, -1));
5118 if(zipinfo.create_system==0 or zipinfo.create_system==10):
5119 fwinattributes = int(zipinfo.external_attr);
5120 if(not member.is_dir()):
5121 fmode = int(stat.S_IFREG + 438);
5122 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
5123 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
5124 elif(member.is_dir()):
5125 fmode = int(stat.S_IFDIR + 511);
5126 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
5127 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
5128 elif(zipinfo.create_system==3):
5129 fwinattributes = int(0);
5130 fmode = int(zipinfo.external_attr);
5131 else:
5132 fwinattributes = int(0);
5133 if(not member.is_dir()):
5134 fmode = int(stat.S_IFREG + 438);
5135 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
5136 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
5137 elif(member.is_dir()):
5138 fmode = int(stat.S_IFDIR + 511);
5139 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
5140 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
5141 try:
5142 fuid = os.getuid();
5143 except AttributeError:
5144 fuid = 0;
5145 except KeyError:
5146 fuid = 0;
5147 try:
5148 fgid = os.getgid();
5149 except AttributeError:
5150 fgid = 0;
5151 except KeyError:
5152 fgid = 0;
5153 try:
5154 import pwd;
5155 try:
5156 userinfo = pwd.getpwuid(os.getuid());
5157 funame = userinfo.pw_name;
5158 except KeyError:
5159 funame = "";
5160 except AttributeError:
5161 funame = "";
5162 except ImportError:
5163 funame = "";
5164 fgname = "";
5165 try:
5166 import grp;
5167 try:
5168 groupinfo = grp.getgrgid(os.getgid());
5169 fgname = groupinfo.gr_name;
5170 except KeyError:
5171 fgname = "";
5172 except AttributeError:
5173 fgname = "";
5174 except ImportError:
5175 fgname = "";
5176 fcontents = BytesIO();
5177 if(ftype==0):
5178 fcontents.write(zipfp.read(member.filename));
5179 fcontents.seek(0, 0);
5180 ftypehex = format(ftype, 'x').lower();
5181 extrafields = len(extradata);
5182 extrafieldslist = extradata;
5183 catfextrafields = extrafields;
5184 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
5185 if(len(extradata)>0):
5186 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
5187 extrasizelen = len(extrasizestr);
5188 extrasizelenhex = format(extrasizelen, 'x').lower();
5189 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5190 catoutlen = len(catoutlist) + len(extradata) + 3;
5191 catoutlenhex = format(catoutlen, 'x').lower();
5192 catoutlist.insert(0, catoutlenhex);
5193 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
5194 if(len(extradata)>0):
5195 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
5196 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
5197 catfnumfields = catoutlen;
5198 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5199 fcontents.seek(0, 0);
5200 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5201 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5202 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5203 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
5204 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5205 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5206 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5207 nullstrecd = formatspecs[5].encode('UTF-8');
5208 fheadtell += len(catfileoutstr) + 1;
5209 catfcontentend = fheadtell - 1;
5210 fcontents.seek(0, 0);
5211 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5212 pyhascontents = False;
5213 if(int(fsize)>0 and not listonly):
5214 pyhascontents = True;
5215 if(int(fsize)>0 and listonly):
5216 fcontents = BytesIO();
5217 pyhascontents = False;
5218 fcontents.seek(0, 0);
5219 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5220 fileidnum = fileidnum + 1;
5221 return catlist;
5222
5223 if(not rarfile_support):
5224 def RarFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5225 return False;
5226
5227 if(rarfile_support):
5228 def RarFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5229 curinode = 0;
5230 curfid = 0;
5231 inodelist = [];
5232 inodetofile = {};
5233 filetoinode = {};
5234 inodetocatinode = {};
5235 fileidnum = 0;
5236 if(not os.path.exists(infiles) or not os.path.isfile(infiles)):
5237 return False;
5238 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
5239 return False;
5240 rarfp = rarfile.RarFile(infile, "r");
5241 rartest = rarfp.testrar();
5242 if(rartest):
5243 VerbosePrintOut("Bad file found!");
5244 fnumfiles = int(len(rarfp.infolist()));
5245 catver = formatspecs[6];
5246 fileheaderver = str(int(catver.replace(".", "")));
5247 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
5248 catversion = re.findall(r"([\d]+)$", fileheader);
5249 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
5250 fnumfileshex = format(int(fnumfiles), 'x').lower();
5251 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs[5]);
5252 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
5253 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
5254 fheadtell = len(fileheader);
5255 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5256 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
5257 is_unix = False;
5258 is_windows = False;
5259 if(member.host_os==rarfile.RAR_OS_UNIX):
5260 is_windows = False;
5261 try:
5262 member.external_attr
5263 is_unix = True;
5264 except AttributeError:
5265 is_unix = False;
5266 elif(member.host_os==rarfile.RAR_OS_WIN32):
5267 is_unix = False;
5268 try:
5269 member.external_attr
5270 is_windows = True;
5271 except AttributeError:
5272 is_windows = False;
5273 else:
5274 is_unix = False;
5275 is_windows = False;
5276 if(re.findall("^[.|/]", member.filename)):
5277 fname = member.filename;
5278 else:
5279 fname = "./"+member.filename;
5280 rarinfo = rarfp.getinfo(member.filename);
5281 if(verbose):
5282 VerbosePrintOut(fname);
5283 if(is_unix and member.external_attr !=0):
5284 fpremode = int(member.external_attr);
5285 elif(member.is_file()):
5286 fpremode = stat.S_IFREG + 438;
5287 elif(member.is_symlink()):
5288 fpremode = stat.S_IFLNK + 438;
5289 elif(member.is_dir()):
5290 fpremode = stat.S_IFDIR + 511;
5291 if(is_windows and member.external_attr !=0):
5292 fwinattributes = int(member.external_attr);
5293 else:
5294 fwinattributes = int(0);
5295 flinkcount = 0;
5296 ftype = 0;
5297 if(member.is_file()):
5298 ftype = 0;
5299 elif(member.is_symlink()):
5300 ftype = 2;
5301 elif(member.is_dir()):
5302 ftype = 5;
5303 flinkname = "";
5304 if(ftype==2):
5305 flinkname = rarfp.read(member.filename).decode("UTF-8");
5306 fbasedir = os.path.dirname(fname);
5307 fcurfid = curfid;
5308 fcurinode = curfid;
5309 finode = fcurinode;
5310 curfid = curfid + 1;
5311 fdev_minor = 0;
5312 fdev_major = 0;
5313 frdev_minor = 0;
5314 frdev_major = 0;
5315 if(ftype==5):
5316 fsize = "0";
5317 if(ftype==0):
5318 fsize = member.file_size;
5319 try:
5320 if(member.atime):
5321 fatime = int(member.atime.timestamp());
5322 else:
5323 fatime = int(member.mtime.timestamp());
5324 except AttributeError:
5325 fatime = int(member.mtime.timestamp());
5326 fmtime = int(member.mtime.timestamp());
5327 try:
5328 if(member.ctime):
5329 fctime = int(member.ctime.timestamp());
5330 else:
5331 fctime = int(member.mtime.timestamp());
5332 except AttributeError:
5333 fctime = int(member.mtime.timestamp());
5334 fbtime = int(member.mtime.timestamp());
5335 if(is_unix and member.external_attr !=0):
5336 fmode = int(member.external_attr);
5337 fchmode = int(stat.S_IMODE(member.external_attr));
5338 ftypemod = int(stat.S_IFMT(member.external_attr));
5339 elif(member.is_file()):
5340 fmode = int(stat.S_IFREG + 438)
5341 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5342 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5343 elif(member.is_symlink()):
5344 fmode = int(stat.S_IFLNK + 438)
5345 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5346 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5347 elif(member.is_dir()):
5348 fmode = int(stat.S_IFDIR + 511)
5349 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
5350 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
5351 try:
5352 fuid = os.getuid();
5353 except AttributeError:
5354 fuid = 0;
5355 except KeyError:
5356 fuid = 0;
5357 try:
5358 fgid = os.getgid();
5359 except AttributeError:
5360 fgid = 0;
5361 except KeyError:
5362 fgid = 0;
5363 try:
5364 import pwd;
5365 try:
5366 userinfo = pwd.getpwuid(os.getuid());
5367 funame = userinfo.pw_name;
5368 except KeyError:
5369 funame = "";
5370 except AttributeError:
5371 funame = "";
5372 except ImportError:
5373 funame = "";
5374 fgname = "";
5375 try:
5376 import grp;
5377 try:
5378 groupinfo = grp.getgrgid(os.getgid());
5379 fgname = groupinfo.gr_name;
5380 except KeyError:
5381 fgname = "";
5382 except AttributeError:
5383 fgname = "";
5384 except ImportError:
5385 fgname = "";
5386 fcontents = BytesIO();
5387 if(ftype==0):
5388 fcontents.write(rarfp.read(member.filename));
5389 fcontents.seek(0, 0);
5390 ftypehex = format(ftype, 'x').lower();
5391 extrafields = len(extradata);
5392 extrafieldslist = extradata;
5393 catfextrafields = extrafields;
5394 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
5395 if(len(extradata)>0):
5396 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
5397 extrasizelen = len(extrasizestr);
5398 extrasizelenhex = format(extrasizelen, 'x').lower();
5399 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5400 catoutlen = len(catoutlist) + len(extradata) + 3;
5401 catoutlenhex = format(catoutlen, 'x').lower();
5402 catoutlist.insert(0, catoutlenhex);
5403 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
5404 if(len(extradata)>0):
5405 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
5406 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
5407 catfnumfields = 24 + catfextrafields;
5408 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5409 fcontents.seek(0, 0);
5410 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5411 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5412 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5413 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
5414 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5415 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5416 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5417 nullstrecd = formatspecs[5].encode('UTF-8');
5418 fheadtell += len(catfileoutstr) + 1;
5419 catfcontentend = fheadtell - 1;
5420 fcontents.seek(0, 0);
5421 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5422 pyhascontents = False;
5423 if(int(fsize)>0 and not listonly):
5424 pyhascontents = True;
5425 if(int(fsize)>0 and listonly):
5426 fcontents = BytesIO();
5427 pyhascontents = False;
5428 fcontents.seek(0, 0);
5429 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5430 fileidnum = fileidnum + 1;
5431 return catlist;
5432
5433 if(not py7zr_support):
5434 def SevenZipFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5435 return False;
5436
5437 if(py7zr_support):
5438 def SevenZipFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5439 curinode = 0;
5440 curfid = 0;
5441 inodelist = [];
5442 inodetofile = {};
5443 filetoinode = {};
5444 inodetocatinode = {};
5445 fileidnum = 0;
5446 szpfp = py7zr.SevenZipFile(infile, mode="r");
5447 file_content = szpfp.readall();
5448 #sztest = szpfp.testzip();
5449 sztestalt = szpfp.test();
5450 if(sztestalt):
5451 VerbosePrintOut("Bad file found!");
5452 numfiles = int(len(szpfp.list()));
5453 catver = formatspecs[6];
5454 fileheaderver = str(int(catver.replace(".", "")));
5455 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
5456 catversion = re.findall(r"([\d]+)$", fileheader);
5457 catversions = re.search(r'(.*?)(\d+)$', catstring).groups();
5458 fnumfileshex = format(int(fnumfiles), 'x').lower();
5459 fileheader = fileheader + AppendNullBytes([fnumfileshex, checksumtype], formatspecs[5]);
5460 catfileheadercshex = GetFileChecksum(fileheader, fprechecksumtype, True, formatspecs);
5461 fileheader = fileheader + AppendNullByte(catfileheadercshex, formatspecs[5]);
5462 fheadtell = len(fileheader);
5463 catlist = {'fnumfiles': fnumfiles, 'fformat': catversions[0], 'fversion': catversions[1], 'fformatspecs': formatspecs, 'fchecksumtype': checksumtype, 'fheaderchecksum': catfileheadercshex, 'ffilelist': {}};
5464 for member in sorted(szpfp.list(), key=lambda x: x.filename):
5465 if(re.findall("^[.|/]", member.filename)):
5466 fname = member.filename;
5467 else:
5468 fname = "./"+member.filename;
5469 if(not member.is_directory):
5470 fpremode = int(stat.S_IFREG + 438);
5471 elif(member.is_directory):
5472 fpremode = int(stat.S_IFDIR + 511);
5473 fwinattributes = int(0);
5474 flinkcount = 0;
5475 ftype = 0;
5476 if(member.is_directory):
5477 ftype = 5;
5478 else:
5479 ftype = 0;
5480 flinkname = "";
5481 fbasedir = os.path.dirname(fname);
5482 fcurfid = curfid;
5483 fcurinode = curfid;
5484 finode = fcurinode;
5485 curfid = curfid + 1;
5486 fdev_minor = 0;
5487 fdev_major = 0;
5488 frdev_minor = 0;
5489 frdev_major = 0;
5490 if(ftype==5):
5491 fsize = "0";
5492 fatime = int(member.creationtime.timestamp());
5493 fmtime = int(member.creationtime.timestamp());
5494 fctime = int(member.creationtime.timestamp());
5495 fbtime = int(member.creationtime.timestamp());
5496 if(member.is_directory):
5497 fmode = int(stat.S_IFDIR + 511)
5498 fchmode = int(stat.S_IMODE(stat.S_IFDIR + 511));
5499 ftypemod = int(stat.S_IFMT(stat.S_IFDIR + 511));
5500 else:
5501 fmode = int(stat.S_IFLNK + 438)
5502 fchmode = int(stat.S_IMODE(stat.S_IFREG + 438));
5503 ftypemod = int(stat.S_IFMT(stat.S_IFREG + 438));
5504 try:
5505 fuid = os.getuid();
5506 except AttributeError:
5507 fuid = 0;
5508 except KeyError:
5509 fuid = 0;
5510 try:
5511 fgid = os.getgid();
5512 except AttributeError:
5513 fgid = 0;
5514 except KeyError:
5515 fgid = 0;
5516 try:
5517 import pwd;
5518 try:
5519 userinfo = pwd.getpwuid(os.getuid());
5520 funame = userinfo.pw_name;
5521 except KeyError:
5522 funame = "";
5523 except AttributeError:
5524 funame = "";
5525 except ImportError:
5526 funame = "";
5527 fgname = "";
5528 try:
5529 import grp;
5530 try:
5531 groupinfo = grp.getgrgid(os.getgid());
5532 fgname = groupinfo.gr_name;
5533 except KeyError:
5534 fgname = "";
5535 except AttributeError:
5536 fgname = "";
5537 except ImportError:
5538 fgname = "";
5539 fcontents = BytesIO();
5540 if(ftype==0):
5541 fcontents.write(file_content[member.filename].read());
5542 fsize = format(fcontents.tell(), 'x').lower();
5543 fileop.close();
5544 fcontents.seek(0, 0);
5545 ftypehex = format(ftype, 'x').lower();
5546 extrafields = len(extradata);
5547 extrafieldslist = extradata;
5548 catfextrafields = extrafields;
5549 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
5550 if(len(extradata)>0):
5551 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
5552 extrasizelen = len(extrasizestr);
5553 extrasizelenhex = format(extrasizelen, 'x').lower();
5554 catoutlist = [ftypehex, fname, flinkname, format(int(fsize), 'x').lower(), format(int(fatime), 'x').lower(), format(int(fmtime), 'x').lower(), format(int(fctime), 'x').lower(), format(int(fbtime), 'x').lower(), format(int(fmode), 'x').lower(), format(int(fwinattributes), 'x').lower(), format(int(fuid), 'x').lower(), funame, format(int(fgid), 'x').lower(), fgname, format(int(fcurfid), 'x').lower(), format(int(fcurinode), 'x').lower(), format(int(flinkcount), 'x').lower(), format(int(fdev_minor), 'x').lower(), format(int(fdev_major), 'x').lower(), format(int(frdev_minor), 'x').lower(), format(int(frdev_major), 'x').lower(), extrasizelenhex, format(catfextrafields, 'x').lower()];
5555 catoutlen = len(catoutlist) + len(extradata) + 3;
5556 catoutlenhex = format(catoutlen, 'x').lower();
5557 catoutlist.insert(0, catoutlenhex);
5558 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
5559 if(len(extradata)>0):
5560 catfileoutstr = catfileoutstr + AppendNullBytes(extradata, formatspecs[5]);
5561 catfileoutstr = catfileoutstr + AppendNullByte(checksumtype, formatspecs[5]);
5562 catfnumfields = 24 + catfextrafields;
5563 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5564 fcontents.seek(0, 0);
5565 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
5566 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5567 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
5568 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
5569 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
5570 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
5571 catfileoutstrecd = catfileoutstr.encode('UTF-8');
5572 nullstrecd = formatspecs[5].encode('UTF-8');
5573 fheadtell += len(catfileoutstr) + 1;
5574 catfcontentend = fheadtell - 1;
5575 fcontents.seek(0, 0);
5576 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
5577 pyhascontents = False;
5578 if(int(fsize)>0 and not listonly):
5579 pyhascontents = True;
5580 if(int(fsize)>0 and listonly):
5581 fcontents = "";
5582 pyhascontents = False;
5583 fcontents.seek(0, 0);
5584 catlist['ffilelist'].update({fileidnum: {'fid': fileidnum, 'fidalt': fileidnum, 'fheadersize': int(catheaersize, 16), 'fhstart': catfhstart, 'fhend': catfhend, 'ftype': ftype, 'fname': fname, 'fbasedir': fbasedir, 'flinkname': flinkname, 'fsize': fsize, 'fatime': fatime, 'fmtime': fmtime, 'fctime': fctime, 'fbtime': fbtime, 'fmode': fmode, 'fchmode': fchmode, 'ftypemod': ftypemod, 'fwinattributes': fwinattributes, 'fuid': fuid, 'funame': funame, 'fgid': fgid, 'fgname': fgname, 'finode': finode, 'flinkcount': flinkcount, 'fminor': fdev_minor, 'fmajor': fdev_major, 'frminor': frdev_minor, 'frmajor': frdev_major, 'fchecksumtype': checksumtype, 'fnumfields': catfnumfields + 2, 'frawheader': catheaderdata, 'fextrafields': catfextrafields, 'fextrafieldsize': extrasizelen, 'fextralist': extrafieldslist, 'fheaderchecksum': int(catfileheadercshex, 16), 'fcontentchecksum': int(catfilecontentcshex, 16), 'fhascontents': pyhascontents, 'fcontentstart': catfcontentstart, 'fcontentend': catfcontentend, 'fcontents': fcontents} });
5585 fileidnum = fileidnum + 1;
5586 return catlist;
5587
5588 def InFileToArrayAlt(infiles, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5589 checkcompressfile = CheckCompressionSubType(infiles, formatspecs, True);
5590 if(checkcompressfile=="tarfile"):
5591 return TarFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5592 elif(checkcompressfile=="zipfile"):
5593 return ZipFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5594 elif(checkcompressfile=="catfile"):
5595 return ArchiveFileToArray(infile, 0, 0, listonly, False, formatspecs, False);
5596 elif(rarfile_support and checkcompressfile=="rarfile"):
5597 return RarFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5598 elif(py7zr_support and checkcompressfile=="7zipfile"):
5599 return SevenZipFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5600 else:
5601 return False;
5602 return False;
5603
5604 def ListDirToArray(infiles, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, seekstart=0, seekend=0, listonly=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
5605 outarray = BytesIO();
5606 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
5607 listcatfiles = ArchiveFileToArray(outarray, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5608 return listcatfiles;
5609
5610 def ArchiveFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5611 if(isinstance(infile, dict)):
5612 listcatfiles = infile;
5613 else:
5614 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
5615 infile = RemoveWindowsPath(infile);
5616 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5617 if(not listcatfiles):
5618 return False;
5619 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5620 if(returnfp):
5621 catarray.update({'catfp': listcatfiles['catfp']});
5622 lenlist = len(listcatfiles['ffilelist']);
5623 lcfi = 0;
5624 lcfx = int(listcatfiles['fnumfiles']);
5625 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5626 lcfx = int(lenlist);
5627 else:
5628 lcfx = int(listcatfiles['fnumfiles']);
5629 while(lcfi < lcfx):
5630 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5631 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5632 catarray['filetoid'].update(filetoidarray);
5633 catarray['idtofile'].update(idtofilearray);
5634 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5635 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5636 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5637 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5638 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5639 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5640 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5641 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5642 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5643 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5644 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5645 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5646 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5647 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5648 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5649 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5650 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5651 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5652 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5653 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5654 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5655 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5656 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5657 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5658 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5659 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5660 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
5661 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5662 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5663 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5664 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5665 lcfi = lcfi + 1;
5666 return catarray;
5667
5668 create_alias_function("", __file_format_name__, "ToArrayIndex", ArchiveFileToArrayIndex);
5669
5670 def ListDirToArrayIndexAlt(infiles, dirlistfromtxt=False, followlink=False, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5671 listcatfiles = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, listonly, checksumtype, extradata, formatspecs, verbose);
5672 if(not listcatfiles):
5673 return False;
5674 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5675 lenlist = len(listcatfiles['ffilelist']);
5676 lcfi = 0;
5677 lcfx = int(listcatfiles['fnumfiles']);
5678 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5679 lcfx = int(lenlist);
5680 else:
5681 lcfx = int(listcatfiles['fnumfiles']);
5682 while(lcfi < lcfx):
5683 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5684 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5685 catarray['filetoid'].update(filetoidarray);
5686 catarray['idtofile'].update(idtofilearray);
5687 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5688 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5689 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5690 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5691 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5692 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5693 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5694 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5695 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5696 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5697 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5698 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5699 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5700 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5701 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5702 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5703 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5704 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5705 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5706 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5707 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5708 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5709 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5710 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5711 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5712 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5713 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
5714 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5715 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5716 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5717 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5718 lcfi = lcfi + 1;
5719 return catarray;
5720
5721 def TarFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5722 listcatfiles = TarFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5723 if(not listcatfiles):
5724 return False;
5725 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5726 lenlist = len(listcatfiles['ffilelist']);
5727 lcfi = 0;
5728 lcfx = int(listcatfiles['fnumfiles']);
5729 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5730 lcfx = int(lenlist);
5731 else:
5732 lcfx = int(listcatfiles['fnumfiles']);
5733 while(lcfi < lcfx):
5734 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5735 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5736 catarray['filetoid'].update(filetoidarray);
5737 catarray['idtofile'].update(idtofilearray);
5738 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5739 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5740 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5741 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5742 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5743 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5744 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5745 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5746 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5747 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5748 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5749 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5750 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5751 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5752 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5753 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5754 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5755 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5756 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5757 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5758 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5759 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5760 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5761 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5762 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5763 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5764 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
5765 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5766 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5767 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5768 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5769 lcfi = lcfi + 1;
5770 return catarray;
5771
5772 def ZipFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5773 listcatfiles = ZipFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5774 if(not listcatfiles):
5775 return False;
5776 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5777 lenlist = len(listcatfiles['ffilelist']);
5778 lcfi = 0;
5779 lcfx = int(listcatfiles['fnumfiles']);
5780 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5781 lcfx = int(lenlist);
5782 else:
5783 lcfx = int(listcatfiles['fnumfiles']);
5784 while(lcfi < lcfx):
5785 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5786 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5787 catarray['filetoid'].update(filetoidarray);
5788 catarray['idtofile'].update(idtofilearray);
5789 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5790 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5791 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5792 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5793 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5794 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5795 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5796 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5797 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5798 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5799 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5800 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5801 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5802 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5803 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5804 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5805 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5806 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5807 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5808 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5809 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5810 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5811 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5812 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5813 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5814 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5815 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
5816 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5817 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5818 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5819 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5820 lcfi = lcfi + 1;
5821 return catarray;
5822
5823 if(not rarfile_support):
5824 def RarFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5825 return False;
5826
5827 if(rarfile_support):
5828 def RarFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5829 listcatfiles = RarFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5830 if(not listcatfiles):
5831 return False;
5832 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5833 lenlist = len(listcatfiles['ffilelist']);
5834 lcfi = 0;
5835 lcfx = int(listcatfiles['fnumfiles']);
5836 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5837 lcfx = int(lenlist);
5838 else:
5839 lcfx = int(listcatfiles['fnumfiles']);
5840 while(lcfi < lcfx):
5841 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5842 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5843 catarray['filetoid'].update(filetoidarray);
5844 catarray['idtofile'].update(idtofilearray);
5845 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5846 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5847 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5848 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5849 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5850 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5851 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5852 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5853 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5854 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5855 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5856 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5857 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5858 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5859 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5860 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5861 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5862 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5863 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5864 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5865 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5866 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5867 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5868 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5869 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5870 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5871 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
5872 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5873 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5874 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5875 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5876 lcfi = lcfi + 1;
5877 return catarray;
5878
5879 if(not py7zr_support):
5880 def SevenZipFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5881 return False;
5882
5883 if(py7zr_support):
5884 def SevenZipFileToArrayIndexAlt(infiles, seekstart=0, seekend=0, listonly=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False):
5885 listcatfiles = SevenZipFileToArrayAlt(infiles, listonly, checksumtype, extradata, formatspecs, verbose);
5886 if(not listcatfiles):
5887 return False;
5888 catarray = {'list': listcatfiles, 'filetoid': {}, 'idtofile': {}, 'filetypes': {'directories': {'filetoid': {}, 'idtofile': {}}, 'files': {'filetoid': {}, 'idtofile': {}}, 'links': {'filetoid': {}, 'idtofile': {}}, 'symlinks': {'filetoid': {}, 'idtofile': {}}, 'hardlinks': {'filetoid': {}, 'idtofile': {}}, 'character': {'filetoid': {}, 'idtofile': {}}, 'block': {'filetoid': {}, 'idtofile': {}}, 'fifo': {'filetoid': {}, 'idtofile': {}}, 'devices': {'filetoid': {}, 'idtofile': {}}}};
5889 lenlist = len(listcatfiles['ffilelist']);
5890 lcfi = 0;
5891 lcfx = int(listcatfiles['fnumfiles']);
5892 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
5893 lcfx = int(lenlist);
5894 else:
5895 lcfx = int(listcatfiles['fnumfiles']);
5896 while(lcfi < lcfx):
5897 filetoidarray = {listcatfiles['ffilelist'][lcfi]['fname']: listcatfiles['ffilelist'][lcfi]['fid']};
5898 idtofilearray = {listcatfiles['ffilelist'][lcfi]['fid']: listcatfiles['ffilelist'][lcfi]['fname']};
5899 catarray['filetoid'].update(filetoidarray);
5900 catarray['idtofile'].update(idtofilearray);
5901 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
5902 catarray['filetypes']['files']['filetoid'].update(filetoidarray);
5903 catarray['filetypes']['files']['idtofile'].update(idtofilearray);
5904 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
5905 catarray['filetypes']['hardlinks']['filetoid'].update(filetoidarray);
5906 catarray['filetypes']['hardlinks']['idtofile'].update(idtofilearray);
5907 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5908 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5909 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
5910 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5911 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5912 catarray['filetypes']['links']['filetoid'].update(filetoidarray);
5913 catarray['filetypes']['links']['idtofile'].update(idtofilearray);
5914 if(listcatfiles['ffilelist'][lcfi]['ftype']==3):
5915 catarray['filetypes']['character']['filetoid'].update(filetoidarray);
5916 catarray['filetypes']['character']['idtofile'].update(idtofilearray);
5917 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5918 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5919 if(listcatfiles['ffilelist'][lcfi]['ftype']==4):
5920 catarray['filetypes']['block']['filetoid'].update(filetoidarray);
5921 catarray['filetypes']['block']['idtofile'].update(idtofilearray);
5922 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5923 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5924 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
5925 catarray['filetypes']['directories']['filetoid'].update(filetoidarray);
5926 catarray['filetypes']['directories']['idtofile'].update(idtofilearray);
5927 if(listcatfiles['ffilelist'][lcfi]['ftype']==6):
5928 catarray['filetypes']['symlinks']['filetoid'].update(filetoidarray);
5929 catarray['filetypes']['symlinks']['idtofile'].update(idtofilearray);
5930 catarray['filetypes']['devices']['filetoid'].update(filetoidarray);
5931 catarray['filetypes']['devices']['idtofile'].update(idtofilearray);
5932 lcfi = lcfi + 1;
5933 return catarray;
5934
5935 def ArchiveFileStringToArrayIndex(catstr, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5936 catfp = BytesIO(catstr);
5937 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5938 return listcatfiles;
5939
5940 create_alias_function("", __file_format_name__, "StringToArrayIndex", ArchiveFileStringToArrayIndex);
5941
5942 def TarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5943 catfp = BytesIO();
5944 catfp = PackArchiveFileFromTarFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
5945 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5946 return listcatfiles;
5947
5948 def ZipFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5949 catfp = BytesIO();
5950 catfp = PackArchiveFileFromZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
5951 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5952 return listcatfiles;
5953
5954 if(not rarfile_support):
5955 def RarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5956 return False;
5957
5958 if(rarfile_support):
5959 def RarFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5960 catfp = BytesIO();
5961 catfp = PackArchiveFileFromRarFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
5962 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5963 return listcatfiles;
5964
5965 if(not py7zr_support):
5966 def SevenZipFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5967 return False;
5968
5969 if(py7zr_support):
5970 def SevenZipFileToArrayIndex(infile, seekstart=0, seekend=0, listonly=False, skipchecksum=False, formatspecs=__file_format_list__, returnfp=False):
5971 catfp = BytesIO();
5972 catfp = PackArchiveFileFromSevenZipFile(infile, catfp, "auto", None, "crc32", [], formatspecs, False, True);
5973 listcatfiles = ArchiveFileToArrayIndex(catfp, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp);
5974 return listcatfiles;
5975
5976 def ListDirToArrayIndex(infiles, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, seekstart=0, seekend=0, listonly=False, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False, returnfp=False):
5977 outarray = BytesIO();
5978 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compressionlevel, followlink, checksumtype, formatspecs, verbose, True);
5979 listcatfiles = ArchiveFileToArrayIndex(outarray, seekstart, seekend, listonly, skipchecksum, formatspecs, returnfp)
5980 return listcatfiles;
5981
5982 def RePackArchiveFile(infile, outfile, compression="auto", compressionlevel=None, followlink=False, seekstart=0, seekend=0, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
5983 if(isinstance(infile, dict)):
5984 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
5985 listcatfiles = prelistcatfiles['list'];
5986 else:
5987 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
5988 infile = RemoveWindowsPath(infile);
5989 if(followlink):
5990 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
5991 listcatfiles = prelistcatfiles['list'];
5992 else:
5993 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
5994 if(outfile!="-" and not hasattr(infile, "read") and not hasattr(outfile, "write")):
5995 outfile = RemoveWindowsPath(outfile);
5996 checksumtype = checksumtype.lower();
5997 if(not CheckSumSupport(checksumtype, hashlib_guaranteed)):
5998 checksumtype="crc32";
5999 if(checksumtype=="none"):
6000 checksumtype = "";
6001 if(not compression or compression or compression=="catfile" or compression==formatspecs[2]):
6002 compression = None;
6003 if(compression not in compressionlist and compression is None):
6004 compression = "auto";
6005 if(verbose):
6006 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6007 if(outfile!="-" and not hasattr(outfile, "read") and not hasattr(outfile, "write")):
6008 if(os.path.exists(outfile)):
6009 try:
6010 os.unlink(outfile);
6011 except OSError as e:
6012 pass;
6013 if(not listcatfiles):
6014 return False;
6015 if(outfile=="-"):
6016 verbose = False;
6017 catfp = BytesIO();
6018 elif(hasattr(outfile, "read") or hasattr(outfile, "write")):
6019 catfp = outfile;
6020 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
6021 catfp = BytesIO();
6022 else:
6023 fbasename = os.path.splitext(outfile)[0];
6024 fextname = os.path.splitext(outfile)[1];
6025 catfp = CompressOpenFile(outfile, compressionlevel);
6026 catver = formatspecs[6];
6027 fileheaderver = str(int(catver.replace(".", "")));
6028 fileheader = AppendNullByte(formatspecs[1] + fileheaderver, formatspecs[5]);
6029 catfp.write(fileheader.encode('UTF-8'));
6030 lenlist = len(listcatfiles['ffilelist']);
6031 fnumfiles = int(listcatfiles['fnumfiles']);
6032 if(lenlist>fnumfiles or lenlist<fnumfiles):
6033 fnumfiles = lenlist;
6034 fnumfileshex = format(int(fnumfiles), 'x').lower();
6035 fnumfilesa = AppendNullBytes([fnumfileshex, checksumtype], formatspecs[5]);
6036 catfileheadercshex = GetFileChecksum(fileheader + fnumfilesa, checksumtype, True, formatspecs);
6037 fnumfilesa = fnumfilesa + AppendNullByte(catfileheadercshex, formatspecs[5]);
6038 catfp.write(fnumfilesa.encode('UTF-8'));
6039 try:
6040 catfp.flush();
6041 os.fsync(catfp.fileno());
6042 except io.UnsupportedOperation:
6043 pass;
6044 except AttributeError:
6045 pass;
6046 except OSError as e:
6047 pass;
6048 lenlist = len(listcatfiles['ffilelist']);
6049 fnumfiles = int(listcatfiles['fnumfiles']);
6050 lcfi = 0;
6051 lcfx = int(listcatfiles['fnumfiles']);
6052 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6053 lcfx = int(lenlist);
6054 else:
6055 lcfx = int(listcatfiles['fnumfiles']);
6056 curinode = 0;
6057 curfid = 0;
6058 inodelist = [];
6059 inodetofile = {};
6060 filetoinode = {};
6061 reallcfi = 0;
6062 while(lcfi < lcfx):
6063 if(re.findall("^[.|/]", listcatfiles['ffilelist'][reallcfi]['fname'])):
6064 fname = listcatfiles['ffilelist'][reallcfi]['fname'];
6065 else:
6066 fname = "./"+listcatfiles['ffilelist'][reallcfi]['fname'];
6067 if(verbose):
6068 VerbosePrintOut(fname);
6069 fheadersize = format(int(listcatfiles['ffilelist'][reallcfi]['fheadersize']), 'x').lower();
6070 fsize = format(int(listcatfiles['ffilelist'][reallcfi]['fsize']), 'x').lower();
6071 flinkname = listcatfiles['ffilelist'][reallcfi]['flinkname'];
6072 fatime = format(int(listcatfiles['ffilelist'][reallcfi]['fatime']), 'x').lower();
6073 fmtime = format(int(listcatfiles['ffilelist'][reallcfi]['fmtime']), 'x').lower();
6074 fctime = format(int(listcatfiles['ffilelist'][reallcfi]['fctime']), 'x').lower();
6075 fbtime = format(int(listcatfiles['ffilelist'][reallcfi]['fbtime']), 'x').lower();
6076 fmode = format(int(listcatfiles['ffilelist'][reallcfi]['fmode']), 'x').lower();
6077 fchmode = format(int(listcatfiles['ffilelist'][reallcfi]['fchmode']), 'x').lower();
6078 fuid = format(int(listcatfiles['ffilelist'][reallcfi]['fuid']), 'x').lower();
6079 funame = listcatfiles['ffilelist'][reallcfi]['funame'];
6080 fgid = format(int(listcatfiles['ffilelist'][reallcfi]['fgid']), 'x').lower();
6081 fgname = listcatfiles['ffilelist'][reallcfi]['fgname'];
6082 finode = listcatfiles['ffilelist'][reallcfi]['finode'];
6083 flinkcount = listcatfiles['ffilelist'][reallcfi]['flinkcount'];
6084 fwinattributes = flinkinfo['ffilelist'][reallcfi]['fwinattributes'];
6085 fdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['fminor']), 'x').lower();
6086 fdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['fmajor']), 'x').lower();
6087 frdev_minor = format(int(listcatfiles['ffilelist'][reallcfi]['frminor']), 'x').lower();
6088 frdev_major = format(int(listcatfiles['ffilelist'][reallcfi]['frmajor']), 'x').lower();
6089 if(len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>listcatfiles['ffilelist'][reallcfi]['fextrafields'] and len(listcatfiles['ffilelist'][reallcfi]['fextralist'])>0):
6090 listcatfiles['ffilelist'][reallcfi]['fextrafields'] = len(listcatfiles['ffilelist'][reallcfi]['fextralist']);
6091 if(len(extradata)>0):
6092 listcatfiles['ffilelist'][reallcfi]['fextrafields'] = len(extradata);
6093 listcatfiles['ffilelist'][reallcfi]['fextralist'] = extradata;
6094 extrafields = format(int(listcatfiles['ffilelist'][reallcfi]['fextrafields']), 'x').lower();
6095 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
6096 if(len(extradata)>0):
6097 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
6098 extrasizelen = format(len(extrasizestr), 'x').lower();
6099 fcontents = listcatfiles['ffilelist'][reallcfi]['fcontents'];
6100 if(followlink):
6101 if(listcatfiles['ffilelist'][reallcfi]['ftype']==1 or listcatfiles['ffilelist'][reallcfi]['ftype']==2):
6102 getflinkpath = listcatfiles['ffilelist'][reallcfi]['flinkname'];
6103 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6104 flinkinfo = listcatfiles['ffilelist'][flinkid];
6105 fheadersize = format(int(flinkinfo['fheadersize']), 'x').lower();
6106 fsize = format(int(flinkinfo['fsize']), 'x').lower();
6107 flinkname = flinkinfo['flinkname'];
6108 fatime = format(int(flinkinfo['fatime']), 'x').lower();
6109 fmtime = format(int(flinkinfo['fmtime']), 'x').lower();
6110 fctime = format(int(flinkinfo['fctime']), 'x').lower();
6111 fbtime = format(int(flinkinfo['fbtime']), 'x').lower();
6112 fmode = format(int(flinkinfo['fmode']), 'x').lower();
6113 fchmode = format(int(flinkinfo['fchmode']), 'x').lower();
6114 fuid = format(int(flinkinfo['fuid']), 'x').lower();
6115 funame = flinkinfo['funame'];
6116 fgid = format(int(flinkinfo['fgid']), 'x').lower();
6117 fgname = flinkinfo['fgname'];
6118 finode = flinkinfo['finode'];
6119 flinkcount = flinkinfo['flinkcount'];
6120 fwinattributes = flinkinfo['fwinattributes'];
6121 fdev_minor = format(int(flinkinfo['fminor']), 'x').lower();
6122 fdev_major = format(int(flinkinfo['fmajor']), 'x').lower();
6123 frdev_minor = format(int(flinkinfo['frminor']), 'x').lower();
6124 frdev_major = format(int(flinkinfo['frmajor']), 'x').lower();
6125 if(len(flinkinfo['fextralist'])>flinkinfo['fextrafields'] and len(flinkinfo['fextralist'])>0):
6126 flinkinfo['fextrafields'] = len(flinkinfo['fextralist']);
6127 if(len(extradata)>0):
6128 flinkinfo['fextrafields'] = len(extradata);
6129 flinkinfo['fextralist'] = extradata;
6130 extrafields = format(int(flinkinfo['fextrafields']), 'x').lower();
6131 extrasizestr = AppendNullByte(extrafields, formatspecs[5]);
6132 if(len(extradata)>0):
6133 extrasizestr = extrasizestr + AppendNullBytes(extradata, formatspecs[5]);
6134 extrasizelen = format(len(extrasizestr), 'x').lower();
6135 fcontents = flinkinfo['fcontents'];
6136 ftypehex = format(flinkinfo['ftype'], 'x').lower();
6137 else:
6138 ftypehex = format(listcatfiles['ffilelist'][reallcfi]['ftype'], 'x').lower();
6139 fcurfid = format(curfid, 'x').lower();
6140 if(not followlink and finode!=0):
6141 if(listcatfiles['ffilelist'][reallcfi]['ftype']!=1):
6142 fcurinode = format(int(curinode), 'x').lower();
6143 inodetofile.update({curinode: fname});
6144 filetoinode.update({fname: curinode});
6145 curinode = curinode + 1;
6146 else:
6147 fcurinode = format(int(filetoinode[flinkname]), 'x').lower();
6148 else:
6149 fcurinode = format(int(curinode), 'x').lower();
6150 curinode = curinode + 1;
6151 curfid = curfid + 1;
6152 catoutlist = [ftypehex, fname, flinkname, fsize, fatime, fmtime, fctime, fbtime, fmode, fwinattributes, fuid, funame, fgid, fgname, fcurfid, fcurinode, flinkcount, fdev_minor, fdev_major, frdev_minor, frdev_major, extrasizelen, extrafields];
6153 catoutlen = len(catoutlist) + len(extradata) + 3;
6154 catoutlenhex = format(catoutlen, 'x').lower();
6155 catoutlist.insert(0, catoutlenhex);
6156 catfileoutstr = AppendNullBytes(catoutlist, formatspecs[5]);
6157 if(listcatfiles['ffilelist'][reallcfi]['fextrafields']>0):
6158 extrafieldslist = [];
6159 exi = 0;
6160 exil = listcatfiles['ffilelist'][reallcfi]['fextrafields'];
6161 while(exi<exil):
6162 extrafieldslist.append(listcatfiles['ffilelist'][reallcfi]['fextralist']);
6163 exi = exi + 1;
6164 catfileoutstr += AppendNullBytes([extrafieldslist], formatspecs[5]);
6165 catfileoutstr += AppendNullBytes([checksumtype], formatspecs[5]);
6166 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
6167 fcontents.seek(0, 0);
6168 catfilecontentcshex = GetFileChecksum(fcontents.read(), checksumtype, False, formatspecs);
6169 tmpfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
6170 catheaersize = format(int(len(tmpfileoutstr) - 1), 'x').lower();
6171 catfileoutstr = AppendNullByte(catheaersize, formatspecs[5]) + catfileoutstr;
6172 catfileheadercshex = GetFileChecksum(catfileoutstr, checksumtype, True, formatspecs);
6173 catfileoutstr = catfileoutstr + AppendNullBytes([catfileheadercshex, catfilecontentcshex], formatspecs[5]);
6174 catfileoutstrecd = catfileoutstr.encode('UTF-8');
6175 nullstrecd = formatspecs[5].encode('UTF-8');
6176 fcontents.seek(0, 0);
6177 catfileout = catfileoutstrecd + fcontents.read() + nullstrecd;
6178 catfp.write(catfileout);
6179 try:
6180 catfp.flush();
6181 os.fsync(catfp.fileno());
6182 except io.UnsupportedOperation:
6183 pass;
6184 except AttributeError:
6185 pass;
6186 except OSError as e:
6187 pass;
6188 lcfi = lcfi + 1;
6189 reallcfi = reallcfi + 1;
6190 if(lcfx>0):
6191 catfp.write(AppendNullBytes([0, 0], formatspecs[5]).encode("UTF-8"));
6192 if(outfile=="-" or hasattr(outfile, "read") or hasattr(outfile, "write")):
6193 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
6194 try:
6195 catfp.flush();
6196 os.fsync(catfp.fileno());
6197 except io.UnsupportedOperation:
6198 pass;
6199 except AttributeError:
6200 pass;
6201 except OSError as e:
6202 pass;
6203 if(outfile=="-"):
6204 catfp.seek(0, 0);
6205 if(hasattr(sys.stdout, "buffer")):
6206 shutil.copyfileobj(catfp, sys.stdout.buffer);
6207 else:
6208 shutil.copyfileobj(catfp, sys.stdout);
6209 elif(re.findall(r"^(ftp|ftps|sftp)\:\/\/", str(outfile))):
6210 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
6211 catfp.seek(0, 0);
6212 upload_file_to_internet_file(catfp, outfile);
6213 if(returnfp):
6214 catfp.seek(0, 0);
6215 return catfp;
6216 else:
6217 catfp.close();
6218 return True;
6219
6220 create_alias_function("RePack", __file_format_name__, "", RePackArchiveFile);
6221
6222 def RePackArchiveFileFromString(catstr, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", skipchecksum=False, extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
6223 catfp = BytesIO(catstr);
6224 listcatfiles = RePackArchiveFile(catfp, compression, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6225 return listcatfiles;
6226
6227 create_alias_function("RePack", __file_format_name__, "FromString", RePackArchiveFileFromString);
6228
6229 def PackArchiveFileFromListDir(infiles, outfile, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
6230 outarray = BytesIO();
6231 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compressionlevel, followlink, checksumtype, extradata, formatspecs, verbose, True);
6232 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compressionlevel, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
6233 return listcatfiles;
6234
6235 create_alias_function("Pack", __file_format_name__, "FromListDir", PackArchiveFileFromListDir);
6236
6237 def UnPackArchiveFile(infile, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_list__, preservepermissions=True, preservetime=True, verbose=False, returnfp=False):
6238 if(outdir is not None):
6239 outdir = RemoveWindowsPath(outdir);
6240 if(verbose):
6241 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6242 if(isinstance(infile, dict)):
6243 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6244 listcatfiles = prelistcatfiles['list'];
6245 else:
6246 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6247 infile = RemoveWindowsPath(infile);
6248 if(followlink):
6249 prelistcatfiles = ArchiveFileToArrayIndex(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6250 listcatfiles = prelistcatfiles['list'];
6251 else:
6252 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, False, skipchecksum, formatspecs, returnfp);
6253 if(not listcatfiles):
6254 return False;
6255 lenlist = len(listcatfiles['ffilelist']);
6256 fnumfiles = int(listcatfiles['fnumfiles']);
6257 lcfi = 0;
6258 lcfx = int(listcatfiles['fnumfiles']);
6259 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6260 lcfx = int(lenlist);
6261 else:
6262 lcfx = int(listcatfiles['fnumfiles']);
6263 while(lcfi<lcfx):
6264 funame = "";
6265 try:
6266 import pwd;
6267 try:
6268 userinfo = pwd.getpwuid(listcatfiles['ffilelist'][lcfi]['fuid']);
6269 funame = userinfo.pw_name;
6270 except KeyError:
6271 funame = "";
6272 except ImportError:
6273 funame = "";
6274 fgname = "";
6275 try:
6276 import grp;
6277 try:
6278 groupinfo = grp.getgrgid(listcatfiles['ffilelist'][lcfi]['fgid']);
6279 fgname = groupinfo.gr_name;
6280 except KeyError:
6281 fgname = "";
6282 except ImportError:
6283 fgname = "";
6284 if(verbose):
6285 VerbosePrintOut(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6286 if(listcatfiles['ffilelist'][lcfi]['ftype']==0 or listcatfiles['ffilelist'][lcfi]['ftype']==7):
6287 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6288 listcatfiles['ffilelist'][lcfi]['fcontents'].seek(0, 0);
6289 shutil.copyfileobj(listcatfiles['ffilelist'][lcfi]['fcontents'], fpc);
6290 try:
6291 fpc.flush();
6292 os.fsync(fpc.fileno());
6293 except io.UnsupportedOperation:
6294 pass;
6295 except AttributeError:
6296 pass;
6297 except OSError as e:
6298 pass;
6299 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
6300 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
6301 if(preservepermissions):
6302 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6303 if(preservetime):
6304 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
6305 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6306 if(followlink):
6307 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
6308 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6309 flinkinfo = listcatfiles['ffilelist'][flinkid];
6310 funame = "";
6311 try:
6312 import pwd;
6313 try:
6314 userinfo = pwd.getpwuid(flinkinfo['fuid']);
6315 funame = userinfo.pw_name;
6316 except KeyError:
6317 funame = "";
6318 except ImportError:
6319 funame = "";
6320 fgname = "";
6321 try:
6322 import grp;
6323 try:
6324 groupinfo = grp.getgrgid(flinkinfo['fgid']);
6325 fgname = groupinfo.gr_name;
6326 except KeyError:
6327 fgname = "";
6328 except ImportError:
6329 fgname = "";
6330 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
6331 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6332 flinkinfo['fcontents'].seek(0, 0);
6333 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
6334 try:
6335 fpc.flush();
6336 os.fsync(fpc.fileno());
6337 except io.UnsupportedOperation:
6338 pass;
6339 except AttributeError:
6340 pass;
6341 except OSError as e:
6342 pass;
6343 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6344 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6345 if(preservepermissions):
6346 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6347 if(preservetime):
6348 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6349 if(flinkinfo['ftype']==1):
6350 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6351 if(flinkinfo['ftype']==2):
6352 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6353 if(flinkinfo['ftype']==5):
6354 if(preservepermissions):
6355 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6356 else:
6357 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6358 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6359 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6360 if(preservepermissions):
6361 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6362 if(preservetime):
6363 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6364 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
6365 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6366 else:
6367 os.link(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6368 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6369 if(followlink):
6370 getflinkpath = listcatfiles['ffilelist'][lcfi]['flinkname'];
6371 flinkid = prelistcatfiles['filetoid'][getflinkpath];
6372 flinkinfo = listcatfiles['ffilelist'][flinkid];
6373 funame = "";
6374 try:
6375 import pwd;
6376 try:
6377 userinfo = pwd.getpwuid(flinkinfo['fuid']);
6378 funame = userinfo.pw_name;
6379 except KeyError:
6380 funame = "";
6381 except ImportError:
6382 funame = "";
6383 fgname = "";
6384 try:
6385 import grp;
6386 try:
6387 groupinfo = grp.getgrgid(flinkinfo['fgid']);
6388 fgname = groupinfo.gr_name;
6389 except KeyError:
6390 fgname = "";
6391 except ImportError:
6392 fgname = "";
6393 if(flinkinfo['ftype']==0 or flinkinfo['ftype']==7):
6394 with open(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), "wb") as fpc:
6395 flinkinfo['fcontents'].seek(0, 0);
6396 shutil.copyfileobj(flinkinfo['fcontents'], fpc);
6397 try:
6398 fpc.flush();
6399 os.fsync(fpc.fileno());
6400 except io.UnsupportedOperation:
6401 pass;
6402 except AttributeError:
6403 pass;
6404 except OSError as e:
6405 pass;
6406 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6407 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6408 if(preservepermissions):
6409 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6410 if(preservetime):
6411 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6412 if(flinkinfo['ftype']==1):
6413 os.link(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6414 if(flinkinfo['ftype']==2):
6415 os.symlink(flinkinfo['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6416 if(flinkinfo['ftype']==5):
6417 if(preservepermissions):
6418 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6419 else:
6420 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6421 if(hasattr(os, "chown") and funame==flinkinfo['funame'] and fgname==flinkinfo['fgname'] and preservepermissions):
6422 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fuid'], flinkinfo['fgid']);
6423 if(preservepermissions):
6424 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6425 if(preservetime):
6426 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (flinkinfo['fatime'], flinkinfo['fmtime']));
6427 if(flinkinfo['ftype']==6 and hasattr(os, "mkfifo")):
6428 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), flinkinfo['fchmode']);
6429 else:
6430 os.symlink(listcatfiles['ffilelist'][lcfi]['flinkname'], PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6431 if(listcatfiles['ffilelist'][lcfi]['ftype']==5):
6432 if(preservepermissions):
6433 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6434 else:
6435 os.mkdir(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']));
6436 if(hasattr(os, "chown") and funame==listcatfiles['ffilelist'][lcfi]['funame'] and fgname==listcatfiles['ffilelist'][lcfi]['fgname'] and preservepermissions):
6437 os.chown(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fuid'], listcatfiles['ffilelist'][lcfi]['fgid']);
6438 if(preservepermissions):
6439 os.chmod(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6440 if(preservetime):
6441 os.utime(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), (listcatfiles['ffilelist'][lcfi]['fatime'], listcatfiles['ffilelist'][lcfi]['fmtime']));
6442 if(listcatfiles['ffilelist'][lcfi]['ftype']==6 and hasattr(os, "mkfifo")):
6443 os.mkfifo(PrependPath(outdir, listcatfiles['ffilelist'][lcfi]['fname']), listcatfiles['ffilelist'][lcfi]['fchmode']);
6444 lcfi = lcfi + 1;
6445 if(returnfp):
6446 return listcatfiles['ffilelist']['catfp'];
6447 else:
6448 return True;
6449
6450 create_alias_function("UnPack", __file_format_name__, "", UnPackArchiveFile);
6451
6452 if(hasattr(shutil, "register_unpack_format")):
6453 def UnPackArchiveFileFunc(archive_name, extract_dir=None, **kwargs):
6454 return UnPackArchiveFile(archive_name, extract_dir, False, 0, 0, False, __file_format_delimiter__, False, False);
6455 create_alias_function("UnPack", __file_format_name__, "Func", UnPackArchiveFileFunc);
6456
6457 def UnPackArchiveFileString(catstr, outdir=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_list__, verbose=False, returnfp=False):
6458 catfp = BytesIO(catstr);
6459 listcatfiles = UnPackArchiveFile(catfp, outdir, followlink, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6460 return listcatfiles;
6461
6462 create_alias_function("UnPack", __file_format_name__, "String", UnPackArchiveFileString);
6463
6464 def ArchiveFileListFiles(infile, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_list__, verbose=False, returnfp=False):
6465 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6466 if(isinstance(infile, dict)):
6467 listcatfiles = infile;
6468 else:
6469 if(infile!="-" and not hasattr(infile, "read") and not hasattr(infile, "write")):
6470 infile = RemoveWindowsPath(infile);
6471 listcatfiles = ArchiveFileToArray(infile, seekstart, seekend, True, skipchecksum, formatspecs, returnfp);
6472 if(not listcatfiles):
6473 return False;
6474 lenlist = len(listcatfiles['ffilelist']);
6475 fnumfiles = int(listcatfiles['fnumfiles']);
6476 lcfi = 0;
6477 lcfx = int(listcatfiles['fnumfiles']);
6478 if(lenlist>listcatfiles['fnumfiles'] or lenlist<listcatfiles['fnumfiles']):
6479 lcfx = int(lenlist);
6480 else:
6481 lcfx = int(listcatfiles['fnumfiles']);
6482 returnval = {};
6483 while(lcfi<lcfx):
6484 returnval.update({lcfi: listcatfiles['ffilelist'][lcfi]['fname']});
6485 if(not verbose):
6486 VerbosePrintOut(listcatfiles['ffilelist'][lcfi]['fname']);
6487 if(verbose):
6488 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6489 printfname = listcatfiles['ffilelist'][lcfi]['fname'];
6490 if(listcatfiles['ffilelist'][lcfi]['ftype']==1):
6491 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " link to " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6492 if(listcatfiles['ffilelist'][lcfi]['ftype']==2):
6493 printfname = listcatfiles['ffilelist'][lcfi]['fname'] + " -> " + listcatfiles['ffilelist'][lcfi]['flinkname'];
6494 fuprint = listcatfiles['ffilelist'][lcfi]['funame'];
6495 if(len(fuprint)<=0):
6496 fuprint = listcatfiles['ffilelist'][lcfi]['fuid'];
6497 fgprint = listcatfiles['ffilelist'][lcfi]['fgname'];
6498 if(len(fgprint)<=0):
6499 fgprint = listcatfiles['ffilelist'][lcfi]['fgid'];
6500 VerbosePrintOut(PrintPermissionString(listcatfiles['ffilelist'][lcfi]['fmode'], listcatfiles['ffilelist'][lcfi]['ftype']) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(listcatfiles['ffilelist'][lcfi]['fsize']).rjust(15) + " " + datetime.datetime.utcfromtimestamp(listcatfiles['ffilelist'][lcfi]['fmtime']).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6501 lcfi = lcfi + 1;
6502 if(returnfp):
6503 return listcatfiles['catfp'];
6504 else:
6505 return True;
6506
6507 create_alias_function("", __file_format_name__, "ListFiles", ArchiveFileListFiles);
6508
6509 def ArchiveFileStringListFiles(catstr, seekstart=0, seekend=0, skipchecksum=False, formatspecs=__file_format_list__, verbose=False, returnfp=False):
6510 catfp = BytesIO(catstr);
6511 listcatfiles = ArchiveFileListFiles(catstr, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6512 return listcatfiles;
6513
6514 create_alias_function("", __file_format_name__, "StringListFiles", ArchiveFileStringListFiles);
6515
6516 def TarFileListFiles(infile, verbose=False, returnfp=False):
6517 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6518 if(infile=="-"):
6519 infile = BytesIO();
6520 if(hasattr(sys.stdin, "buffer")):
6521 shutil.copyfileobj(sys.stdin.buffer, infile);
6522 else:
6523 shutil.copyfileobj(sys.stdin, infile);
6524 infile.seek(0, 0);
6525 if(not infile):
6526 return False;
6527 infile.seek(0, 0);
6528 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
6529 infile = download_file_from_internet_file(infile);
6530 infile.seek(0, 0);
6531 if(not infile):
6532 return False;
6533 infile.seek(0, 0);
6534 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
6535 return False;
6536 elif(os.path.exists(infile) and os.path.isfile(infile)):
6537 try:
6538 if(not tarfile.is_tarfile(infile)):
6539 return False;
6540 except AttributeError:
6541 if(not is_tarfile(infile)):
6542 return False;
6543 else:
6544 return False;
6545 try:
6546 if(hasattr(infile, "read") or hasattr(infile, "write")):
6547 tarfp = tarfile.open(fileobj=infile, mode="r");
6548 else:
6549 tarfp = tarfile.open(infile, "r");
6550 except FileNotFoundError:
6551 return False;
6552 lcfi = 0
6553 returnval = {};
6554 for member in sorted(tarfp.getmembers(), key=lambda x: x.name):
6555 returnval.update({lcfi: member.name});
6556 fpremode = member.mode;
6557 ffullmode = member.mode;
6558 flinkcount = 0;
6559 ftype = 0;
6560 if(member.isreg()):
6561 ffullmode = member.mode + stat.S_IFREG;
6562 ftype = 0;
6563 elif(member.isdev()):
6564 ffullmode = member.mode;
6565 ftype = 7;
6566 elif(member.islnk()):
6567 ffullmode = member.mode + stat.S_IFREG;
6568 ftype = 1;
6569 elif(member.issym()):
6570 ffullmode = member.mode + stat.S_IFLNK;
6571 ftype = 2;
6572 elif(member.ischr()):
6573 ffullmode = member.mode + stat.S_IFCHR;
6574 ftype = 3;
6575 elif(member.isblk()):
6576 ffullmode = member.mode + stat.S_IFBLK;
6577 ftype = 4;
6578 elif(member.isdir()):
6579 ffullmode = member.mode + stat.S_IFDIR;
6580 ftype = 5;
6581 elif(member.isfifo()):
6582 ffullmode = member.mode + stat.S_IFIFO;
6583 ftype = 6;
6584 elif(member.issparse()):
6585 ffullmode = member.mode;
6586 ftype = 12;
6587 if(not verbose):
6588 VerbosePrintOut(member.name);
6589 elif(verbose):
6590 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6591 printfname = member.name;
6592 if(member.islnk()):
6593 printfname = member.name + " link to " + member.linkname;
6594 elif(member.issym()):
6595 printfname = member.name + " -> " + member.linkname;
6596 fuprint = member.uname;
6597 if(len(fuprint)<=0):
6598 fuprint = member.uid;
6599 fgprint = member.gname;
6600 if(len(fgprint)<=0):
6601 fgprint = member.gid;
6602 VerbosePrintOut(PrintPermissionString(ffullmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(member.mtime).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6603 lcfi = lcfi + 1;
6604 if(returnfp):
6605 return listcatfiles['catfp'];
6606 else:
6607 return True;
6608
6609 def ZipFileListFiles(infile, verbose=False, returnfp=False):
6610 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6611 if(infile=="-"):
6612 infile = BytesIO();
6613 if(hasattr(sys.stdin, "buffer")):
6614 shutil.copyfileobj(sys.stdin.buffer, infile);
6615 else:
6616 shutil.copyfileobj(sys.stdin, infile);
6617 infile.seek(0, 0);
6618 if(not infile):
6619 return False;
6620 infile.seek(0, 0);
6621 elif(re.findall(r"^(http|https|ftp|ftps|sftp)\:\/\/", str(infile))):
6622 infile = download_file_from_internet_file(infile);
6623 infile.seek(0, 0);
6624 if(not infile):
6625 return False;
6626 infile.seek(0, 0);
6627 elif(not os.path.exists(infile) or not os.path.isfile(infile)):
6628 return False;
6629 else:
6630 return False;
6631 if(not zipfile.is_zipfile(infile)):
6632 return False;
6633 try:
6634 zipfp = zipfile.ZipFile(infile, "r", allowZip64=True);
6635 except FileNotFoundError:
6636 return False;
6637 lcfi = 0;
6638 returnval = {};
6639 ziptest = zipfp.testzip();
6640 if(ziptest):
6641 VerbosePrintOut("Bad file found!");
6642 for member in sorted(zipfp.infolist(), key=lambda x: x.filename):
6643 if(zipinfo.create_system==0 or zipinfo.create_system==10):
6644 fwinattributes = int(zipinfo.external_attr);
6645 if(not member.is_dir()):
6646 fmode = int(stat.S_IFREG + 438);
6647 fchmode = int(stat.S_IMODE(fmode));
6648 ftypemod = int(stat.S_IFMT(fmode));
6649 elif(member.is_dir()):
6650 fmode = int(stat.S_IFDIR + 511);
6651 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6652 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6653 elif(zipinfo.create_system==3):
6654 fwinattributes =int(0);
6655 fmode = int(zipinfo.external_attr);
6656 fchmode = int(stat.S_IMODE(fmode));
6657 ftypemod = int(stat.S_IFMT(fmode));
6658 else:
6659 fwinattributes = int(0);
6660 if(not member.is_dir()):
6661 fmode = int(stat.S_IFREG + 438);
6662 fchmode = int(stat.S_IMODE(fmode));
6663 ftypemod = int(stat.S_IFMT(fmode));
6664 elif(member.is_dir()):
6665 fmode = int(stat.S_IFDIR + 511);
6666 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6667 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6668 returnval.update({lcfi: member.filename});
6669 if(not verbose):
6670 VerbosePrintOut(member.filename);
6671 if(verbose):
6672 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6673 permissionstr = "";
6674 for fmodval in str(oct(fmode))[-3:]:
6675 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
6676 if(not member.is_dir()):
6677 ftype = 0;
6678 permissionstr = "-" + permissionstr;
6679 elif(member.is_dir()):
6680 ftype = 5;
6681 permissionstr = "d" + permissionstr;
6682 printfname = member.filename;
6683 try:
6684 fuid = int(os.getuid());
6685 except AttributeError:
6686 fuid = int(0);
6687 except KeyError:
6688 fuid = int(0);
6689 try:
6690 fgid = int(os.getgid());
6691 except AttributeError:
6692 fgid = int(0);
6693 except KeyError:
6694 fgid = int(0);
6695 try:
6696 import pwd;
6697 try:
6698 userinfo = pwd.getpwuid(os.getuid());
6699 funame = userinfo.pw_name;
6700 except KeyError:
6701 funame = "";
6702 except AttributeError:
6703 funame = "";
6704 except ImportError:
6705 funame = "";
6706 fgname = "";
6707 try:
6708 import grp;
6709 try:
6710 groupinfo = grp.getgrgid(os.getgid());
6711 fgname = groupinfo.gr_name;
6712 except KeyError:
6713 fgname = "";
6714 except AttributeError:
6715 fgname = "";
6716 except ImportError:
6717 fgname = "";
6718 fuprint = funame;
6719 if(len(fuprint)<=0):
6720 fuprint = str(fuid);
6721 fgprint = fgname;
6722 if(len(fgprint)<=0):
6723 fgprint = str(fgid);
6724 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + datetime.datetime.utcfromtimestamp(int(time.mktime(member.date_time + (0, 0, -1)))).strftime('%Y-%m-%d %H:%M') + " " + printfname));
6725 lcfi = lcfi + 1;
6726 if(returnfp):
6727 return listcatfiles['catfp'];
6728 else:
6729 return True;
6730
6731 if(not rarfile_support):
6732 def RarFileListFiles(infile, verbose=False, returnfp=False):
6733 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6734 if(not os.path.exists(infile) or not os.path.isfile(infile)):
6735 return False;
6736
6737 if(rarfile_support):
6738 def RarFileListFiles(infile, verbose=False, returnfp=False):
6739 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6740 if(not os.path.exists(infile) or not os.path.isfile(infile)):
6741 return False;
6742 if(not rarfile.is_rarfile(infile) and not rarfile.is_rarfile_sfx(infile)):
6743 return False;
6744 lcfi = 0;
6745 returnval = {};
6746 rarfp = rarfile.RarFile(infile, "r");
6747 rartest = rarfp.testrar();
6748 if(rartest):
6749 VerbosePrintOut("Bad file found!");
6750 for member in sorted(rarfp.infolist(), key=lambda x: x.filename):
6751 is_unix = False;
6752 is_windows = False;
6753 if(member.host_os==rarfile.RAR_OS_UNIX):
6754 is_windows = False;
6755 try:
6756 member.external_attr
6757 is_unix = True;
6758 except AttributeError:
6759 is_unix = False;
6760 elif(member.host_os==rarfile.RAR_OS_WIN32):
6761 is_unix = False;
6762 try:
6763 member.external_attr
6764 is_windows = True;
6765 except AttributeError:
6766 is_windows = False;
6767 else:
6768 is_unix = False;
6769 is_windows = False;
6770 if(is_unix and member.external_attr !=0):
6771 fpremode = int(member.external_attr);
6772 elif(member.is_file()):
6773 fpremode = int(stat.S_IFREG + 438);
6774 elif(member.is_symlink()):
6775 fpremode = int(stat.S_IFLNK + 438);
6776 elif(member.is_dir()):
6777 fpremode = int(stat.S_IFDIR + 511);
6778 if(is_windows and member.external_attr !=0):
6779 fwinattributes = int(member.external_attr);
6780 else:
6781 fwinattributes = int(0);
6782 if(is_unix and member.external_attr !=0):
6783 fmode = int(member.external_attr);
6784 fchmode = int(stat.S_IMODE(member.external_attr));
6785 ftypemod = int(stat.S_IFMT(member.external_attr));
6786 elif(member.is_file()):
6787 fmode = int(stat.S_IFREG + 438);
6788 fchmode = int(stat.S_IMODE(int(stat.S_IFREG + 438)));
6789 ftypemod = int(stat.S_IFMT(int(stat.S_IFREG + 438)));
6790 elif(member.is_symlink()):
6791 fmode = int(stat.S_IFLNK + 438);
6792 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
6793 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
6794 elif(member.is_dir()):
6795 fmode = int(stat.S_IFDIR + 511);
6796 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6797 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6798 returnval.update({lcfi: member.filename});
6799 if(not verbose):
6800 VerbosePrintOut(member.filename);
6801 if(verbose):
6802 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6803 permissionstr = "";
6804 for fmodval in str(oct(fmode))[-3:]:
6805 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
6806 if(member.is_file()):
6807 ftype = 0;
6808 permissionstr = "-" + permissionstr;
6809 printfname = member.filename;
6810 elif(member.is_symlink()):
6811 ftype = 2;
6812 permissionstr = "l" + permissionstr;
6813 printfname = member.name + " -> " + member.read().decode("UTF-8");
6814 elif(member.is_dir()):
6815 ftype = 5;
6816 permissionstr = "d" + permissionstr;
6817 printfname = member.filename;
6818 try:
6819 fuid = int(os.getuid());
6820 except AttributeError:
6821 fuid = int(0);
6822 except KeyError:
6823 fuid = int(0);
6824 try:
6825 fgid = int(os.getgid());
6826 except AttributeError:
6827 fgid = int(0);
6828 except KeyError:
6829 fgid = int(0);
6830 try:
6831 import pwd;
6832 try:
6833 userinfo = pwd.getpwuid(os.getuid());
6834 funame = userinfo.pw_name;
6835 except KeyError:
6836 funame = "";
6837 except AttributeError:
6838 funame = "";
6839 except ImportError:
6840 funame = "";
6841 fgname = "";
6842 try:
6843 import grp;
6844 try:
6845 groupinfo = grp.getgrgid(os.getgid());
6846 fgname = groupinfo.gr_name;
6847 except KeyError:
6848 fgname = "";
6849 except AttributeError:
6850 fgname = "";
6851 except ImportError:
6852 fgname = "";
6853 fuprint = funame;
6854 if(len(fuprint)<=0):
6855 fuprint = str(fuid);
6856 fgprint = fgname;
6857 if(len(fgprint)<=0):
6858 fgprint = str(fgid);
6859 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(member.file_size).rjust(15) + " " + member.mtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
6860 lcfi = lcfi + 1;
6861 if(returnfp):
6862 return listcatfiles['catfp'];
6863 else:
6864 return True;
6865
6866 if(not py7zr_support):
6867 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
6868 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6869 if(not os.path.exists(infile) or not os.path.isfile(infile)):
6870 return False;
6871
6872 if(py7zr_support):
6873 def SevenZipFileListFiles(infile, verbose=False, returnfp=False):
6874 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6875 if(not os.path.exists(infile) or not os.path.isfile(infile)):
6876 return False;
6877 lcfi = 0;
6878 returnval = {};
6879 szpfp = py7zr.SevenZipFile(infile, mode="r");
6880 file_content = szpfp.readall();
6881 #sztest = szpfp.testzip();
6882 sztestalt = szpfp.test();
6883 if(sztestalt):
6884 VerbosePrintOut("Bad file found!");
6885 for member in sorted(szpfp.list(), key=lambda x: x.filename):
6886 if(re.findall("^[.|/]", member.filename)):
6887 fname = member.filename;
6888 else:
6889 fname = "./"+member.filename;
6890 if(not member.is_directory):
6891 fpremode = int(stat.S_IFREG + 438);
6892 elif(member.is_directory):
6893 fpremode = int(stat.S_IFDIR + 511);
6894 fwinattributes = int(0);
6895 if(member.is_directory):
6896 fmode = int(stat.S_IFDIR + 511);
6897 fchmode = int(stat.S_IMODE(int(stat.S_IFDIR + 511)));
6898 ftypemod = int(stat.S_IFMT(int(stat.S_IFDIR + 511)));
6899 else:
6900 fmode = int(stat.S_IFLNK + 438);
6901 fchmode = int(stat.S_IMODE(int(stat.S_IFLNK + 438)));
6902 ftypemod = int(stat.S_IFMT(int(stat.S_IFLNK + 438)));
6903 returnval.update({lcfi: member.filename});
6904 if(not verbose):
6905 VerbosePrintOut(member.filename);
6906 if(verbose):
6907 permissions = { 'access': { '0': ('---'), '1': ('--x'), '2': ('-w-'), '3': ('-wx'), '4': ('r--'), '5': ('r-x'), '6': ('rw-'), '7': ('rwx') }, 'roles': { 0: 'owner', 1: 'group', 2: 'other' } };
6908 permissionstr = "";
6909 for fmodval in str(oct(fmode))[-3:]:
6910 permissionstr = permissionstr + permissions['access'].get(fmodval, '---');
6911 fsize = int("0");
6912 if(not member.is_directory):
6913 ftype = 0;
6914 permissionstr = "-" + permissionstr;
6915 printfname = member.filename;
6916 elif(member.is_directory):
6917 ftype = 5;
6918 permissionstr = "d" + permissionstr;
6919 printfname = member.filename;
6920 if(ftype==0):
6921 fsize = len(file_content[member.filename].read());
6922 file_content[member.filename].close();
6923 try:
6924 fuid = int(os.getuid());
6925 except AttributeError:
6926 fuid = int(0);
6927 except KeyError:
6928 fuid = int(0);
6929 try:
6930 fgid = int(os.getgid());
6931 except AttributeError:
6932 fgid = int(0);
6933 except KeyError:
6934 fgid = int(0);
6935 try:
6936 import pwd;
6937 try:
6938 userinfo = pwd.getpwuid(os.getuid());
6939 funame = userinfo.pw_name;
6940 except KeyError:
6941 funame = "";
6942 except AttributeError:
6943 funame = "";
6944 except ImportError:
6945 funame = "";
6946 fgname = "";
6947 try:
6948 import grp;
6949 try:
6950 groupinfo = grp.getgrgid(os.getgid());
6951 fgname = groupinfo.gr_name;
6952 except KeyError:
6953 fgname = "";
6954 except AttributeError:
6955 fgname = "";
6956 except ImportError:
6957 fgname = "";
6958 fuprint = funame;
6959 if(len(fuprint)<=0):
6960 fuprint = str(fuid);
6961 fgprint = fgname;
6962 if(len(fgprint)<=0):
6963 fgprint = str(fgid);
6964 VerbosePrintOut(PrintPermissionString(fmode, ftype) + " " + str(str(fuprint) + "/" + str(fgprint) + " " + str(fsize).rjust(15) + " " + member.creationtime.strftime('%Y-%m-%d %H:%M') + " " + printfname));
6965 lcfi = lcfi + 1;
6966 if(returnfp):
6967 return listcatfiles['catfp'];
6968 else:
6969 return True;
6970
6971 def InFileListFiles(infile, verbose=False, formatspecs=__file_format_list__, returnfp=False):
6972 logging.basicConfig(format="%(message)s", stream=sys.stdout, level=logging.DEBUG);
6973 checkcompressfile = CheckCompressionSubType(infile, formatspecs, True);
6974 if(checkcompressfile=="tarfile"):
6975 return TarFileListFiles(infile, verbose, returnfp);
6976 elif(checkcompressfile=="zipfile"):
6977 return ZipFileListFiles(infile, verbose, returnfp);
6978 elif(checkcompressfile=="catfile"):
6979 return ArchiveFileListFiles(infile, 0, 0, False, formatspecs, verbose, returnfp);
6980 elif(rarfile_support and checkcompressfile=="rarfile"):
6981 return RarFileListFiles(infile, verbose, returnfp);
6982 elif(py7zr_support and checkcompressfile=="7zipfile"):
6983 return SevenZipFileListFiles(infile, verbose, returnfp);
6984 else:
6985 return False;
6986 return False;
6987
6988 def ListDirListFiles(infiles, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False, returnfp=False):
6989 outarray = BytesIO();
6990 packcat = PackArchiveFile(infiles, outarray, dirlistfromtxt, compression, compressionlevel, followlink, checksumtype, formatspecs, False, True);
6991 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6992 return listcatfiles;
6993
6994 def ListDirListFilesAlt(infiles, dirlistfromtxt=False, followlink=False, listonly=False, seekstart=0, seekend=0, skipchecksum=False, checksumtype="crc32", formatspecs=__file_format_list__, verbose=False, returnfp=False):
6995 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, listonly, checksumtype, formatspecs, verbose);
6996 listcatfiles = ArchiveFileListFiles(outarray, seekstart, seekend, skipchecksum, formatspecs, verbose, returnfp);
6997 return listcatfiles;
6998
6999 def PackArchiveFileFromListDirAlt(infiles, outfile, dirlistfromtxt=False, compression="auto", compressionlevel=None, followlink=False, skipchecksum=False, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
7000 outarray = ListDirToArrayAlt(infiles, dirlistfromtxt, followlink, False, checksumtype, extradata, formatspecs, False);
7001 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compressionlevel, followlink, checksumtype, skipchecksum, extradata, formatspecs, verbose, returnfp);
7002 return listcatfiles;
7003
7004 create_alias_function("Pack", __file_format_name__, "FromListDirAlt", PackArchiveFileFromListDirAlt);
7005
7006 def PackArchiveFileFromTarFileAlt(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
7007 outarray = TarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7008 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7009 return listcatfiles;
7010
7011 create_alias_function("Pack", __file_format_name__, "FromTarFileAlt", PackArchiveFileFromTarFileAlt);
7012
7013 def PackArchiveFileFromZipFileAlt(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
7014 outarray = ZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7015 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7016 return listcatfiles;
7017
7018 create_alias_function("Pack", __file_format_name__, "FromZipFileAlt", PackArchiveFileFromZipFileAlt);
7019
7020 if(not rarfile_support):
7021 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
7022 return False;
7023
7024 if(rarfile_support):
7025 def PackArchiveFileFromRarFileAlt(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
7026 outarray = RarFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7027 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7028 return listcatfiles;
7029
7030 create_alias_function("Pack", __file_format_name__, "FromRarFileAlt", PackArchiveFileFromRarFileAlt);
7031
7032 if(not py7zr_support):
7033 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
7034 return False;
7035
7036 if(py7zr_support):
7037 def PackArchiveFileFromSevenZipFileAlt(infile, outfile, compression="auto", compressionlevel=None, checksumtype="crc32", extradata=[], formatspecs=__file_format_list__, verbose=False, returnfp=False):
7038 outarray = SevenZipFileToArrayAlt(infile, False, checksumtype, extradata, formatspecs, False);
7039 listcatfiles = RePackArchiveFile(outarray, outfile, compression, compressionlevel, False, checksumtype, False, extradata, formatspecs, verbose, returnfp);
7040 return listcatfiles;
7041
7042 create_alias_function("Pack", __file_format_name__, "FromSevenZipFileAlt", PackArchiveFileFromSevenZipFileAlt);
7043
7044 def download_file_from_ftp_file(url):
7045 urlparts = urlparse(url);
7046 file_name = os.path.basename(urlparts.path);
7047 file_dir = os.path.dirname(urlparts.path);
7048 if(urlparts.username is not None):
7049 ftp_username = urlparts.username;
7050 else:
7051 ftp_username = "anonymous";
7052 if(urlparts.password is not None):
7053 ftp_password = urlparts.password;
7054 elif(urlparts.password is None and urlparts.username=="anonymous"):
7055 ftp_password = "anonymous";
7056 else:
7057 ftp_password = "";
7058 if(urlparts.scheme=="ftp"):
7059 ftp = FTP();
7060 elif(urlparts.scheme=="ftps"):
7061 ftp = FTP_TLS();
7062 else:
7063 return False;
7064 if(urlparts.scheme=="sftp"):
7065 if(__use_pysftp__):
7066 return download_file_from_pysftp_file(url);
7067 else:
7068 return download_file_from_sftp_file(url);
7069 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7070 return download_file_from_http_file(url);
7071 ftp_port = urlparts.port;
7072 if(urlparts.port is None):
7073 ftp_port = 21;
7074 try:
7075 ftp.connect(urlparts.hostname, ftp_port);
7076 except socket.gaierror:
7077 log.info("Error With URL "+url);
7078 return False;
7079 except socket.timeout:
7080 log.info("Error With URL "+url);
7081 return False;
7082 ftp.login(urlparts.username, urlparts.password);
7083 if(urlparts.scheme=="ftps"):
7084 ftp.prot_p();
7085 ftpfile = BytesIO();
7086 ftp.retrbinary("RETR "+urlparts.path, ftpfile.write);
7087 #ftp.storbinary("STOR "+urlparts.path, ftpfile.write);
7088 ftp.close();
7089 ftpfile.seek(0, 0);
7090 return ftpfile;
7091
7092 def download_file_from_ftp_string(url):
7093 ftpfile = download_file_from_ftp_file(url);
7094 return ftpfile.read();
7095
7096 def upload_file_to_ftp_file(ftpfile, url):
7097 urlparts = urlparse(url);
7098 file_name = os.path.basename(urlparts.path);
7099 file_dir = os.path.dirname(urlparts.path);
7100 if(urlparts.username is not None):
7101 ftp_username = urlparts.username;
7102 else:
7103 ftp_username = "anonymous";
7104 if(urlparts.password is not None):
7105 ftp_password = urlparts.password;
7106 elif(urlparts.password is None and urlparts.username=="anonymous"):
7107 ftp_password = "anonymous";
7108 else:
7109 ftp_password = "";
7110 if(urlparts.scheme=="ftp"):
7111 ftp = FTP();
7112 elif(urlparts.scheme=="ftps"):
7113 ftp = FTP_TLS();
7114 else:
7115 return False;
7116 if(urlparts.scheme=="sftp"):
7117 if(__use_pysftp__):
7118 return upload_file_to_pysftp_file(url);
7119 else:
7120 return upload_file_to_sftp_file(url);
7121 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7122 return False;
7123 ftp_port = urlparts.port;
7124 if(urlparts.port is None):
7125 ftp_port = 21;
7126 try:
7127 ftp.connect(urlparts.hostname, ftp_port);
7128 except socket.gaierror:
7129 log.info("Error With URL "+url);
7130 return False;
7131 except socket.timeout:
7132 log.info("Error With URL "+url);
7133 return False;
7134 ftp.login(urlparts.username, urlparts.password);
7135 if(urlparts.scheme=="ftps"):
7136 ftp.prot_p();
7137 ftp.storbinary("STOR "+urlparts.path, ftpfile);
7138 ftp.close();
7139 ftpfile.seek(0, 0);
7140 return ftpfile;
7141
7142 def upload_file_to_ftp_string(ftpstring, url):
7143 ftpfileo = BytesIO(ftpstring);
7144 ftpfile = upload_file_to_ftp_file(ftpfileo, url);
7145 ftpfileo.close();
7146 return ftpfile;
7147
7148 def download_file_from_http_file(url, headers=geturls_headers_pycatfile_python_alt):
7149 # Parse the URL to extract username and password if present
7150 urlparts = urlparse(url);
7151 username = urlparts.username;
7152 password = urlparts.password;
7153 # Rebuild the URL without the username and password
7154 netloc = urlparts.hostname;
7155 if(urlparts.scheme=="sftp"):
7156 if(__use_pysftp__):
7157 return download_file_from_pysftp_file(url);
7158 else:
7159 return download_file_from_sftp_file(url);
7160 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7161 return download_file_from_ftp_file(url);
7162 if urlparts.port:
7163 netloc += ':' + str(urlparts.port);
7164 rebuilt_url = urlunparse((urlparts.scheme, netloc, urlparts.path, urlparts.params, urlparts.query, urlparts.fragment));
7165 # Create a temporary file object
7166 httpfile = BytesIO();
7167 if haverequests:
7168 # Use the requests library if available
7169 if username and password:
7170 response = requests.get(rebuilt_url, headers=headers, auth=(username, password), stream=True);
7171 else:
7172 response = requests.get(rebuilt_url, headers=headers, stream=True);
7173 response.raw.decode_content = True
7174 shutil.copyfileobj(response.raw, httpfile);
7175 else:
7176 # Build a Request object for urllib
7177 request = Request(rebuilt_url, headers=headers);
7178 # Create an opener object for handling URLs
7179 if username and password:
7180 # Create a password manager
7181 password_mgr = HTTPPasswordMgrWithDefaultRealm();
7182 # Add the username and password
7183 password_mgr.add_password(None, rebuilt_url, username, password);
7184 # Create an authentication handler using the password manager
7185 auth_handler = HTTPBasicAuthHandler(password_mgr);
7186 # Build the opener with the authentication handler
7187 opener = build_opener(auth_handler);
7188 else:
7189 opener = build_opener();
7190 with opener.open(request) as response:
7191 shutil.copyfileobj(response, httpfile);
7192 # Reset file pointer to the start
7193 httpfile.seek(0, 0);
7194 # Return the temporary file object
7195 return httpfile;
7196
7197 def download_file_from_http_string(url, headers=geturls_headers_pycatfile_python_alt):
7198 httpfile = download_file_from_http_file(url, headers);
7199 return ftpfile.read();
7200
7201 if(haveparamiko):
7202 def download_file_from_sftp_file(url):
7203 urlparts = urlparse(url);
7204 file_name = os.path.basename(urlparts.path);
7205 file_dir = os.path.dirname(urlparts.path);
7206 sftp_port = urlparts.port;
7207 if(urlparts.port is None):
7208 sftp_port = 22;
7209 else:
7210 sftp_port = urlparts.port;
7211 if(urlparts.username is not None):
7212 sftp_username = urlparts.username;
7213 else:
7214 sftp_username = "anonymous";
7215 if(urlparts.password is not None):
7216 sftp_password = urlparts.password;
7217 elif(urlparts.password is None and urlparts.username=="anonymous"):
7218 sftp_password = "anonymous";
7219 else:
7220 sftp_password = "";
7221 if(urlparts.scheme=="ftp"):
7222 return download_file_from_ftp_file(url);
7223 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7224 return download_file_from_http_file(url);
7225 if(urlparts.scheme!="sftp"):
7226 return False;
7227 ssh = paramiko.SSHClient();
7228 ssh.load_system_host_keys();
7229 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
7230 try:
7231 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7232 except paramiko.ssh_exception.SSHException:
7233 return False;
7234 except socket.gaierror:
7235 log.info("Error With URL "+url);
7236 return False;
7237 except socket.timeout:
7238 log.info("Error With URL "+url);
7239 return False;
7240 sftp = ssh.open_sftp();
7241 sftpfile = BytesIO();
7242 sftp.getfo(urlparts.path, sftpfile);
7243 sftp.close();
7244 ssh.close();
7245 sftpfile.seek(0, 0);
7246 return sftpfile;
7247 else:
7248 def download_file_from_sftp_file(url):
7249 return False;
7250
7251 if(haveparamiko):
7252 def download_file_from_sftp_string(url):
7253 sftpfile = download_file_from_sftp_file(url);
7254 return sftpfile.read();
7255 else:
7256 def download_file_from_ftp_string(url):
7257 return False;
7258
7259 if(haveparamiko):
7260 def upload_file_to_sftp_file(sftpfile, url):
7261 urlparts = urlparse(url);
7262 file_name = os.path.basename(urlparts.path);
7263 file_dir = os.path.dirname(urlparts.path);
7264 sftp_port = urlparts.port;
7265 if(urlparts.port is None):
7266 sftp_port = 22;
7267 else:
7268 sftp_port = urlparts.port;
7269 if(urlparts.username is not None):
7270 sftp_username = urlparts.username;
7271 else:
7272 sftp_username = "anonymous";
7273 if(urlparts.password is not None):
7274 sftp_password = urlparts.password;
7275 elif(urlparts.password is None and urlparts.username=="anonymous"):
7276 sftp_password = "anonymous";
7277 else:
7278 sftp_password = "";
7279 if(urlparts.scheme=="ftp"):
7280 return upload_file_to_ftp_file(url);
7281 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7282 return False;
7283 if(urlparts.scheme!="sftp"):
7284 return False;
7285 ssh = paramiko.SSHClient();
7286 ssh.load_system_host_keys();
7287 ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy());
7288 try:
7289 ssh.connect(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7290 except paramiko.ssh_exception.SSHException:
7291 return False;
7292 except socket.gaierror:
7293 log.info("Error With URL "+url);
7294 return False;
7295 except socket.timeout:
7296 log.info("Error With URL "+url);
7297 return False;
7298 sftp = ssh.open_sftp();
7299 sftp.putfo(sftpfile, urlparts.path);
7300 sftp.close();
7301 ssh.close();
7302 sftpfile.seek(0, 0);
7303 return sftpfile;
7304 else:
7305 def upload_file_to_sftp_file(sftpfile, url):
7306 return False;
7307
7308 if(haveparamiko):
7309 def upload_file_to_sftp_string(sftpstring, url):
7310 sftpfileo = BytesIO(sftpstring);
7311 sftpfile = upload_file_to_sftp_files(ftpfileo, url);
7312 sftpfileo.close();
7313 return sftpfile;
7314 else:
7315 def upload_file_to_sftp_string(url):
7316 return False;
7317
7318 if(havepysftp):
7319 def download_file_from_pysftp_file(url):
7320 urlparts = urlparse(url);
7321 file_name = os.path.basename(urlparts.path);
7322 file_dir = os.path.dirname(urlparts.path);
7323 sftp_port = urlparts.port;
7324 if(urlparts.port is None):
7325 sftp_port = 22;
7326 else:
7327 sftp_port = urlparts.port;
7328 if(urlparts.username is not None):
7329 sftp_username = urlparts.username;
7330 else:
7331 sftp_username = "anonymous";
7332 if(urlparts.password is not None):
7333 sftp_password = urlparts.password;
7334 elif(urlparts.password is None and urlparts.username=="anonymous"):
7335 sftp_password = "anonymous";
7336 else:
7337 sftp_password = "";
7338 if(urlparts.scheme=="ftp"):
7339 return download_file_from_ftp_file(url);
7340 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7341 return download_file_from_http_file(url);
7342 if(urlparts.scheme!="sftp"):
7343 return False;
7344 try:
7345 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7346 except paramiko.ssh_exception.SSHException:
7347 return False;
7348 except socket.gaierror:
7349 log.info("Error With URL "+url);
7350 return False;
7351 except socket.timeout:
7352 log.info("Error With URL "+url);
7353 return False;
7354 sftp = ssh.open_sftp();
7355 sftpfile = BytesIO();
7356 sftp.getfo(urlparts.path, sftpfile);
7357 sftp.close();
7358 ssh.close();
7359 sftpfile.seek(0, 0);
7360 return sftpfile;
7361 else:
7362 def download_file_from_pysftp_file(url):
7363 return False;
7364
7365 if(havepysftp):
7366 def download_file_from_pysftp_string(url):
7367 sftpfile = download_file_from_pysftp_file(url);
7368 return sftpfile.read();
7369 else:
7370 def download_file_from_ftp_string(url):
7371 return False;
7372
7373 if(havepysftp):
7374 def upload_file_to_pysftp_file(sftpfile, url):
7375 urlparts = urlparse(url);
7376 file_name = os.path.basename(urlparts.path);
7377 file_dir = os.path.dirname(urlparts.path);
7378 sftp_port = urlparts.port;
7379 if(urlparts.port is None):
7380 sftp_port = 22;
7381 else:
7382 sftp_port = urlparts.port;
7383 if(urlparts.username is not None):
7384 sftp_username = urlparts.username;
7385 else:
7386 sftp_username = "anonymous";
7387 if(urlparts.password is not None):
7388 sftp_password = urlparts.password;
7389 elif(urlparts.password is None and urlparts.username=="anonymous"):
7390 sftp_password = "anonymous";
7391 else:
7392 sftp_password = "";
7393 if(urlparts.scheme=="ftp"):
7394 return upload_file_to_ftp_file(url);
7395 elif(urlparts.scheme=="http" or urlparts.scheme=="https"):
7396 return False;
7397 if(urlparts.scheme!="sftp"):
7398 return False;
7399 try:
7400 pysftp.Connection(urlparts.hostname, port=sftp_port, username=urlparts.username, password=urlparts.password);
7401 except paramiko.ssh_exception.SSHException:
7402 return False;
7403 except socket.gaierror:
7404 log.info("Error With URL "+url);
7405 return False;
7406 except socket.timeout:
7407 log.info("Error With URL "+url);
7408 return False;
7409 sftp = ssh.open_sftp();
7410 sftp.putfo(sftpfile, urlparts.path);
7411 sftp.close();
7412 ssh.close();
7413 sftpfile.seek(0, 0);
7414 return sftpfile;
7415 else:
7416 def upload_file_to_pysftp_file(sftpfile, url):
7417 return False;
7418
7419 if(havepysftp):
7420 def upload_file_to_pysftp_string(sftpstring, url):
7421 sftpfileo = BytesIO(sftpstring);
7422 sftpfile = upload_file_to_pysftp_files(ftpfileo, url);
7423 sftpfileo.close();
7424 return sftpfile;
7425 else:
7426 def upload_file_to_pysftp_string(url):
7427 return False;
7428
7429 def download_file_from_internet_file(url, headers=geturls_headers_pycatfile_python_alt):
7430 urlparts = urlparse(url);
7431 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7432 return download_file_from_http_file(url, headers);
7433 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7434 return download_file_from_ftp_file(url);
7435 elif(urlparts.scheme=="sftp"):
7436 if(__use_pysftp__ and havepysftp):
7437 return download_file_from_pysftp_file(url);
7438 else:
7439 return download_file_from_sftp_file(url);
7440 else:
7441 return False;
7442 return False;
7443
7444 def download_file_from_internet_uncompress_file(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_list__):
7445 fp = download_file_from_internet_file(url);
7446 fp = UncompressArchiveFile(fp, formatspecs);
7447 fp.seek(0, 0);
7448 if(not fp):
7449 return False;
7450 return fp;
7451
7452 def download_file_from_internet_string(url, headers=geturls_headers_pycatfile_python_alt):
7453 urlparts = urlparse(url);
7454 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7455 return download_file_from_http_string(url, headers);
7456 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7457 return download_file_from_ftp_string(url);
7458 elif(urlparts.scheme=="sftp"):
7459 if(__use_pysftp__ and havepysftp):
7460 return download_file_from_pysftp_string(url);
7461 else:
7462 return download_file_from_sftp_string(url);
7463 else:
7464 return False;
7465 return False;
7466
7467 def download_file_from_internet_uncompress_string(url, headers=geturls_headers_pycatfile_python_alt, formatspecs=__file_format_list__):
7468 fp = download_file_from_internet_string(url);
7469 fp = UncompressArchiveFile(fp, formatspecs);
7470 fp.seek(0, 0);
7471 if(not fp):
7472 return False;
7473 return fp;
7474
7475 def upload_file_to_internet_file(ifp, url):
7476 urlparts = urlparse(url);
7477 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7478 return False;
7479 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7480 return upload_file_to_ftp_file(ifp, url);
7481 elif(urlparts.scheme=="sftp"):
7482 if(__use_pysftp__ and havepysftp):
7483 return upload_file_to_pysftp_file(ifp, url);
7484 else:
7485 return upload_file_to_sftp_file(ifp, url);
7486 else:
7487 return False;
7488 return False;
7489
7490 def upload_file_to_internet_compress_file(ifp, url, formatspecs=__file_format_list__):
7491 catfp = CompressArchiveFile(catfp, compression, compressionlevel, formatspecs);
7492 if(not catfileout):
7493 return False;
7494 catfp.seek(0, 0);
7495 upload_file_to_internet_file(catfp, outfile);
7496 return True;
7497
7498 def upload_file_to_internet_string(ifp, url):
7499 urlparts = urlparse(url);
7500 if(urlparts.scheme=="http" or urlparts.scheme=="https"):
7501 return False;
7502 elif(urlparts.scheme=="ftp" or urlparts.scheme=="ftps"):
7503 return upload_file_to_ftp_string(ifp, url);
7504 elif(urlparts.scheme=="sftp"):
7505 if(__use_pysftp__ and havepysftp):
7506 return upload_file_to_pysftp_string(ifp, url);
7507 else:
7508 return upload_file_to_sftp_string(ifp, url);
7509 else:
7510 return False;
7511 return False;
7512
7513 def upload_file_to_internet_compress_string(ifp, url, formatspecs=__file_format_list__):
7514 catfp = CompressArchiveFile(BytesIO(ifp), compression, compressionlevel, formatspecs);
7515 if(not catfileout):
7516 return False;
7517 catfp.seek(0, 0);
7518 upload_file_to_internet_file(catfp, outfile);
7519 return True;
7520
7521 try:
7522 if(hasattr(shutil, "register_archive_format")):
7523 # Register the packing format
7524 shutil.register_archive_format(__file_format_name__, PackArchiveFileFunc, description='Pack concatenated files');
7525 except shutil.RegistryError:
7526 pass;
7527
7528 try:
7529 if(hasattr(shutil, "register_unpack_format")):
7530 # Register the unpacking format
7531 shutil.register_unpack_format(__file_format_name__, archivefile_extensions, UnPackArchiveFileFunc, description='UnPack concatenated files');
7532 except shutil.RegistryError:
7533 pass;
A tar like file format name catfile after unix cat command (concatenate files).